From a0b60a3b92c44121763f73997e2c21111c72bc7e Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 13 Oct 2026 21:29:38 +0200 Subject: [PATCH 001/198] Optimize: optional target_bytes retries with stronger image settings (#1889) Co-Authored-By: Claude Sonnet 5.5 --- crates/automation/src/lib.rs | 54 +++++++++++++++++++++++++-- crates/automation/src/tools.rs | 3 +- crates/automation/tests/automation.rs | 43 +++++++++++++++++++++ parity/acrobat-features.toml | 4 +- 4 files changed, 98 insertions(+), 6 deletions(-) diff --git a/crates/automation/src/lib.rs b/crates/automation/src/lib.rs index 88856305b2..17c1f7293c 100644 --- a/crates/automation/src/lib.rs +++ b/crates/automation/src/lib.rs @@ -92,6 +92,8 @@ const DEFAULT_DPI: f64 = 96.0; const MAX_DPI: f64 = 600.0; /// How far a watermark may be moved from the page centre, in points (a page is at most 14,400). const WATERMARK_MAX_OFFSET: f64 = 20_000.0; +/// `doc_optimize` with `target_bytes`: (max target ppi, max JPEG quality) per extra attempt, mildest first. +const TARGET_LADDER: [(f64, u8); 5] = [(110.0, 45), (96.0, 35), (72.0, 28), (54.0, 20), (36.0, 12)]; /// Bookmarks in one page of the bookmark list, by default and at most. const BOOKMARK_PAGE: usize = 100; @@ -1120,10 +1122,52 @@ impl Automation { }) .collect::>()?, }; - let (bytes, r) = self.session.optimized_bytes(id, &settings, &discard).map_err(failed)?; + let target = match a.opt_int("target_bytes")? { + None => None, + Some(t) if t >= 1 => Some(t as usize), + Some(t) => return Err(ToolError::InvalidArgs(format!("target_bytes must be at least 1, not {t}"))), + }; + let (mut bytes, mut r) = self.session.optimized_bytes(id, &settings, &discard).map_err(failed)?; + let mut attempts = 1; + let mut step_used = 0; + if let Some(t) = target { + // Stronger image settings, one rung per attempt, until the copy fits. Every attempt + // starts from the open document, so a failed one changes nothing; the number of + // attempts is fixed. The smallest copy is kept if none fits. + for (k, &(ppi, quality)) in TARGET_LADDER.iter().enumerate() { + if bytes.len() <= t { + break; + } + let mut s = settings.clone(); + for img in [&mut s.color, &mut s.gray] { + img.downsample = true; + img.target_ppi = img.target_ppi.min(ppi); + img.above_ppi = img.above_ppi.min(ppi * 1.5).max(img.target_ppi); + img.compression = match img.compression { + Compression::Jpeg(q) => Compression::Jpeg(q.min(quality)), + _ => Compression::Jpeg(quality), + }; + } + let (b2, r2) = self.session.optimized_bytes(id, &s, &discard).map_err(failed)?; + attempts += 1; + if b2.len() < bytes.len() { + (bytes, r, step_used) = (b2, r2, k + 1); + } + } + } write_atomic(&path, &bytes)?; let o = &r.optimize; - Ok(json!({ + let mut extra = json!({}); + if let Some(t) = target { + let (ppi, quality) = step_used.checked_sub(1).and_then(|k| TARGET_LADDER.get(k)).copied().unzip(); + extra = json!({ + "target_bytes": t, + "target_reached": bytes.len() <= t, + "attempts": attempts, + "target_settings": { "step": step_used, "max_ppi": ppi, "max_jpeg_quality": quality }, + }); + } + let mut out = json!({ "path": path.to_string_lossy(), "bytes_before": before, "bytes_after": bytes.len(), @@ -1144,7 +1188,11 @@ impl Automation { "unused_xobjects": o.unused_xobjects, "merged_objects": r.merged, "discarded": r.discarded.iter().map(|(h, n)| json!({ "category": h.id(), "count": n })).collect::>(), - })) + }); + if let (Some(o), Some(e)) = (out.as_object_mut(), extra.as_object()) { + o.extend(e.iter().map(|(k, v)| (k.clone(), v.clone()))); + } + Ok(out) } /// A watermark or background picture: `file` (an image or a PDF) and `file_page` (1-based). diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index 5e89aef78a..2365086a33 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -1303,7 +1303,7 @@ pub fn tools() -> Vec { .with(schema(json!({ "doc": doc() }), &["doc"])), t("doc_open_revision", "Open a revision", "Open saved revision `revision` (1 = the oldest) of a document as a new, unsaved document, to see the file as it was then.") .with(schema(json!({ "doc": doc(), "revision": { "type": "integer", "minimum": 1 } }), &["doc", "revision"])), - t("doc_optimize", "Optimize PDF", "Write an optimized copy to `path` (Acrobat's PDF Optimizer). preset: standard (default; Reduce File Size's choices) or mobile (images to 96 ppi above 144, JPEG high quality); the options below change it. merge_fragments (default true) first puts a page picture drawn as many stacked image strips (as some PDF printers write it) back together into one image, so it can be resampled (merged_images, fragments_merged). color / gray: { downsample, ppi, above_ppi, compression: jpeg|zip|retain, quality 1–100 } (defaults: downsample to 150 ppi above 225, JPEG 60). Images are measured where pages draw them and replaced only when smaller. discard_*: thumbnails (default true), alternate_images (true), tags, print_settings; flate_unencoded (true); remove_invalid_links and remove_unreferenced_dests (true); images and forms the pages list but never draw are always dropped (unused_xobjects). discard: Remove Hidden Information categories (metadata, attachments, comments, form-fields, hidden-text, hidden-layers, bookmarks, links-actions-scripts, private-data). Signed documents are refused. The open document is unchanged.") + t("doc_optimize", "Optimize PDF", "Write an optimized copy to `path` (Acrobat's PDF Optimizer). preset: standard (default; Reduce File Size's choices) or mobile (images to 96 ppi above 144, JPEG high quality); the options below change it. merge_fragments (default true) first puts a page picture drawn as many stacked image strips (as some PDF printers write it) back together into one image, so it can be resampled (merged_images, fragments_merged). color / gray: { downsample, ppi, above_ppi, compression: jpeg|zip|retain, quality 1–100 } (defaults: downsample to 150 ppi above 225, JPEG 60). Images are measured where pages draw them and replaced only when smaller. discard_*: thumbnails (default true), alternate_images (true), tags, print_settings; flate_unencoded (true); remove_invalid_links and remove_unreferenced_dests (true); images and forms the pages list but never draw are always dropped (unused_xobjects). discard: Remove Hidden Information categories (metadata, attachments, comments, form-fields, hidden-text, hidden-layers, bookmarks, links-actions-scripts, private-data). Signed documents are refused. target_bytes (optional): if the copy is larger, retry with up to 5 progressively stronger image settings (downsample to at most 110, 96, 72, 54, 36 ppi with JPEG quality at most 45, 35, 28, 20, 12; applied to colour and grey images) and stop at the first copy that fits; reports target_reached, attempts and target_settings, and writes the smallest copy with target_reached false when none fits. Without it nothing changes. The open document is unchanged.") .cmd("optimize.advanced") .with(schema( json!({ @@ -1313,6 +1313,7 @@ pub fn tools() -> Vec { "merge_fragments": { "type": "boolean" }, "color": { "type": "object" }, "gray": { "type": "object" }, + "target_bytes": { "type": "integer", "minimum": 1 }, "discard_thumbnails": { "type": "boolean" }, "discard_alternate_images": { "type": "boolean" }, "discard_tags": { "type": "boolean" }, diff --git a/crates/automation/tests/automation.rs b/crates/automation/tests/automation.rs index 1e662d8f38..8018c83321 100644 --- a/crates/automation/tests/automation.rs +++ b/crates/automation/tests/automation.rs @@ -4752,6 +4752,49 @@ fn a_page_printed_in_strips_gets_small_through_tools() { assert!(matches!(a.call("doc_optimize", &json!({ "doc": doc, "path": "x.pdf", "preset": "tiny" })), Err(ToolError::InvalidArgs(_)))); } +#[test] +fn optimizing_to_a_target_size() { + let dir = workdir("optimize-target"); + let (w, h) = (2400u32, 1600u32); + let px: Vec = (0..w * h) + .flat_map(|i| { + let (x, y) = (i % w, i / w); + [(x * 255 / w) as u8 ^ ((x * y) & 15) as u8, (y * 255 / h) as u8, ((x + y) & 255) as u8] + }) + .collect(); + let mut jpeg = Vec::new(); + let mut enc = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut jpeg, 95); + enc.set_pixel_density(image::codecs::jpeg::PixelDensity::dpi(600)); + enc.encode(&px, w, h, image::ExtendedColorType::Rgb8).unwrap(); + std::fs::write(dir.join("photo.jpg"), &jpeg).unwrap(); + let mut a = auto(&dir); + let doc = ok(&mut a, "doc_create", json!({ "from": "images", "paths": ["photo.jpg"] }))["doc"].as_u64().unwrap(); + // Without a target nothing changes: no target fields. + let base = ok(&mut a, "doc_optimize", json!({ "doc": doc, "path": "base.pdf" })); + assert!(base.get("target_reached").is_none(), "{base}"); + let base_len = base["bytes_after"].as_u64().unwrap(); + // A target below the default result is met by a stronger step. + let r = ok(&mut a, "doc_optimize", json!({ "doc": doc, "path": "fit.pdf", "target_bytes": base_len - 1 })); + assert_eq!(r["target_reached"], true, "{r}"); + assert!(r["bytes_after"].as_u64().unwrap() < base_len, "{r}"); + assert!(r["target_settings"]["step"].as_u64().unwrap() >= 1, "{r}"); + assert_eq!(std::fs::metadata(dir.join("fit.pdf")).unwrap().len(), r["bytes_after"].as_u64().unwrap()); + // A target already met by the first attempt tries nothing more. + let r = ok(&mut a, "doc_optimize", json!({ "doc": doc, "path": "easy.pdf", "target_bytes": base_len })); + assert_eq!((r["target_reached"].as_bool(), r["attempts"].as_u64()), (Some(true), Some(1)), "{r}"); + // An impossible target: bounded attempts, smallest copy written, said plainly. + let r = ok(&mut a, "doc_optimize", json!({ "doc": doc, "path": "tiny.pdf", "target_bytes": 1 })); + assert_eq!((r["target_reached"].as_bool(), r["attempts"].as_u64()), (Some(false), Some(6)), "{r}"); + assert!(r["bytes_after"].as_u64().unwrap() < base_len, "{r}"); + assert!(std::fs::metadata(dir.join("tiny.pdf")).is_ok()); + // The open document is untouched by the failed attempts, and bad targets are refused. + let again = ok(&mut a, "doc_optimize", json!({ "doc": doc, "path": "again.pdf" })); + assert_eq!(again["bytes_after"].as_u64().unwrap(), base_len); + assert!(matches!(a.call("doc_optimize", &json!({ "doc": doc, "path": "x.pdf", "target_bytes": 0 })), Err(ToolError::InvalidArgs(_)))); + // Confinement still applies. + assert!(a.call("doc_optimize", &json!({ "doc": doc, "path": "../out.pdf", "target_bytes": 1000 })).is_err()); +} + #[test] fn initial_view_through_tools() { let dir = workdir("initial-view"); diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 533013b547..aee6fda80f 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -7260,8 +7260,8 @@ milestone = "M11" status = "partial" commands = ["optimize.advanced"] tools = ["doc_optimize"] -evidence = ["crates/ui-egui/tests/create.rs::the_pdf_optimizer_dialog_saves_an_optimized_copy", "crates/automation/tests/automation.rs::optimizing_through_tools", "crates/ui-egui/src/optimize_ui.rs::the_progress_card_shows_the_stage_and_cancels", "crates/engine/src/tests.rs::an_optimize_job_reports_its_stages_and_can_be_cancelled", "crates/optimize/tests/optimize.rs::images_and_forms_nothing_draws_are_dropped", "crates/automation/tests/automation.rs::a_deleted_image_is_not_kept_by_reduce_or_optimize", "crates/ui-egui/tests/create.rs::the_pdf_optimizer_dialog_picks_a_preset", "crates/automation/tests/automation.rs::a_page_printed_in_strips_gets_small_through_tools", "crates/optimize/tests/optimize.rs::the_mobile_preset_makes_a_printed_page_small"] -notes = "Settings menu with the Standard and Mobile presets (Custom once an option changes; doc_optimize preset), Images (with Merge image fragments: a page picture drawn as one-pixel strips, as some PDF printers write it, is put back together so it can be resampled), Discard Objects, Discard User Data, Clean Up (also drops images and forms the pages list but never draw, such as deleted images); runs in the background with a progress bar (per image) and Cancel; no Fonts or Transparency panels, no saved custom presets, no space audit" +evidence = ["crates/ui-egui/tests/create.rs::the_pdf_optimizer_dialog_saves_an_optimized_copy", "crates/automation/tests/automation.rs::optimizing_through_tools", "crates/automation/tests/automation.rs::optimizing_to_a_target_size", "crates/ui-egui/src/optimize_ui.rs::the_progress_card_shows_the_stage_and_cancels", "crates/engine/src/tests.rs::an_optimize_job_reports_its_stages_and_can_be_cancelled", "crates/optimize/tests/optimize.rs::images_and_forms_nothing_draws_are_dropped", "crates/automation/tests/automation.rs::a_deleted_image_is_not_kept_by_reduce_or_optimize", "crates/ui-egui/tests/create.rs::the_pdf_optimizer_dialog_picks_a_preset", "crates/automation/tests/automation.rs::a_page_printed_in_strips_gets_small_through_tools", "crates/optimize/tests/optimize.rs::the_mobile_preset_makes_a_printed_page_small"] +notes = "Settings menu with the Standard and Mobile presets (Custom once an option changes; doc_optimize preset), Images (with Merge image fragments: a page picture drawn as one-pixel strips, as some PDF printers write it, is put back together so it can be resampled), Discard Objects, Discard User Data, Clean Up (also drops images and forms the pages list but never draw, such as deleted images); runs in the background with a progress bar (per image) and Cancel; no Fonts or Transparency panels, no saved custom presets, no space audit; headless-only optional target_bytes retries up to 5 stronger image steps (an automation extension, not an Acrobat feature)" [[feature]] id = "optimize.downsample-images" From 68d7b4cf3b31516672f124e1a8bf65e8df150714 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 13 Oct 2026 21:20:48 +0200 Subject: [PATCH 002/198] Redact: mark text by caller-supplied regex with optional checksum (#1886) Co-Authored-By: Claude Sonnet 5.5 --- Cargo.lock | 33 ++++ Cargo.toml | 2 + ROADMAP.md | 1 + crates/automation/src/redact.rs | 30 +++- crates/automation/src/tools.rs | 4 +- crates/automation/tests/automation.rs | 28 ++++ crates/engine/src/lib.rs | 1 + crates/redact/Cargo.toml | 1 + crates/redact/src/lib.rs | 1 + crates/redact/src/user_regex.rs | 211 ++++++++++++++++++++++++++ parity/acrobat-features.toml | 12 ++ 11 files changed, 317 insertions(+), 7 deletions(-) create mode 100644 crates/redact/src/user_regex.rs diff --git a/Cargo.lock b/Cargo.lock index 91ed3c4904..268f3c745d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -148,6 +148,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + [[package]] name = "aligned-vec" version = "0.6.4" @@ -4284,6 +4293,7 @@ dependencies = [ "pdfcraft-model", "pdfcraft-testkit", "pdfcraft-xfa", + "regex", "thiserror 2.0.21", ] @@ -5136,6 +5146,29 @@ dependencies = [ "bitflags 2.13.2", ] +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + [[package]] name = "regex-syntax" version = "0.8.11" diff --git a/Cargo.toml b/Cargo.toml index 597dd316fa..cc0ceaa4d9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -62,6 +62,8 @@ weezl = "0.2" # foundation thiserror = "2" +# Linear-time matching (no backtracking) for caller-supplied redaction patterns. +regex = { version = "1", default-features = false, features = ["std", "unicode"] } serde = { version = "1", features = ["derive", "rc"] } serde_json = "1" log = "0.4" diff --git a/ROADMAP.md b/ROADMAP.md index 66b9677569..d1e1c6febf 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -157,6 +157,7 @@ Ranked; detail and the full milestone table in [docs/roadmap.md](docs/roadmap.md | 8 | 1.0 polish: performance, right-to-left, missing languages, keyboard-only (M14) | 55–95 | ## Progress log +- **2026-10-13 (M8, redact by regex):** `redact_mark` takes `regex` (caller expression, linear-time engine, length/size/nesting caps, empty-matching refused) with optional `validate` (luhn, mod97, iban). Headless only; issue #1886. - **2026-10-15 (M2, gap 12):** `cargo xtask text-oracle` scores reading order as well as word F1: the longest common subsequence of the two word sequences over the longer one. F1 is a bag of words and cannot see order at all -- measuring #2431, a change that moves text and nothing else, it called 26 of the 29 files whose output changed identical, where the order score separated 18 improvements from 6 regressions. Reports, does not gate; a disagreement with Poppler, not a verdict. Estimates unchanged. - **2026-10-15 (M2, #2305 follow-up):** WinAnsiEncoding 0xA0 and 0xAD and MacRomanEncoding 0xCA extract as the ordinary space and hyphen Annex D.2 names them, in the vendored hayro tables (patch 30) as well as `crates/fonts`, so search, copy and redaction see characters that were hidden behind U+00A0 and U+00AD. Across the pdf.js corpus the soft hyphens we extract fall 117 to 13 and the non-breaking spaces 30 to 26; the rest come from `/ToUnicode`. An explicit `/Differences` name keeps its glyph list value, as pdftotext does. Estimates unchanged. - **2026-10-13 (community, M3, #1569 review):** Pixel-only Wayland wheel, wheel-tilt and continuous input use standalone moved events, so Ctrl zoom does not remain latched after the modifier is released (#991); vendored winit patch (13), with a policy test. Estimates unchanged. diff --git a/crates/automation/src/redact.rs b/crates/automation/src/redact.rs index 1b6283c0ad..1367cb819c 100644 --- a/crates/automation/src/redact.rs +++ b/crates/automation/src/redact.rs @@ -2,8 +2,8 @@ //! (removing what they cover for good) or clear them. use pdfcraft_engine::{ - Edit, Hidden, NewAnnotation, REDACT_MAX_WORDS, REDACTION_CODE_SETS, RedactPattern, RedactionCodeSet, Shape, Style, find_pattern, rect_quad, - redact_word_list, + Edit, Hidden, NewAnnotation, REDACT_MAX_WORDS, REDACT_VALIDATORS, REDACTION_CODE_SETS, RedactPattern, RedactRegex, RedactValidator, + RedactionCodeSet, Shape, Style, find_pattern, rect_quad, redact_word_list, }; use serde_json::{Value, json}; @@ -49,16 +49,20 @@ impl Automation { a.get("find").is_some(), a.get("words").is_some(), a.get("pattern").is_some(), + a.get("regex").is_some(), a.opt_bool("whole_pages")?.unwrap_or(false), ]; if chosen.iter().filter(|c| **c).count() != 1 { - return Err(ToolError::InvalidArgs("pass exactly one of rect (with page), find, words, pattern or whole_pages: true".into())); + return Err(ToolError::InvalidArgs("pass exactly one of rect (with page), find, words, pattern, regex or whole_pages: true".into())); } let case_sensitive = a.opt_bool("case_sensitive")?.unwrap_or(false); let whole_words = a.opt_bool("whole_words")?.unwrap_or(false); if (case_sensitive || whole_words) && a.get("find").is_none() && a.get("words").is_none() { return Err(ToolError::InvalidArgs("case_sensitive and whole_words apply only to find or words".into())); } + if a.get("validate").is_some() && a.get("regex").is_none() { + return Err(ToolError::InvalidArgs("validate applies only to regex".into())); + } // Per searched word or phrase: how many marks it made. let mut matched_words: Vec<(String, usize)> = Vec::new(); if a.get("rect").is_some() { @@ -79,6 +83,19 @@ impl Automation { Some(p) => Some(RedactPattern::from_id(p).ok_or_else(|| ToolError::InvalidArgs(format!("unknown pattern {p:?}")))?), None => None, }; + let user_re = match a.opt_str("regex")? { + Some(src) => { + let validator = match a.opt_str("validate")? { + Some(v) => Some(RedactValidator::from_id(v).ok_or_else(|| { + let ids: Vec<_> = REDACT_VALIDATORS.iter().map(|v| v.id()).collect(); + ToolError::InvalidArgs(format!("unknown validate {v:?}; allowed: {}", ids.join(", "))) + })?), + None => None, + }; + Some(RedactRegex::new(src, validator).map_err(|e| ToolError::InvalidArgs(e.to_string()))?) + } + None => None, + }; let needles: Vec = match (a.opt_str("find")?, a.get("words")) { (Some(f), _) => vec![f.to_owned()], (None, Some(_)) => { @@ -98,9 +115,10 @@ impl Automation { let texts = self.page_texts(id, &pages)?; let info = self.doc(a)?.info.clone(); for (&p, text) in pages.iter().zip(&texts) { - let hits: Vec<(Option, _)> = match pattern { - Some(pat) => text.find_with(|chars| find_pattern(pat, chars)).into_iter().map(|h| (None, h)).collect(), - None => needles + let hits: Vec<(Option, _)> = match (&user_re, pattern) { + (Some(re), _) => text.find_with(|chars| re.find(chars)).into_iter().map(|h| (None, h)).collect(), + (None, Some(pat)) => text.find_with(|chars| find_pattern(pat, chars)).into_iter().map(|h| (None, h)).collect(), + (None, None) => needles .iter() .enumerate() .flat_map(|(i, w)| text.find_opts(w, case_sensitive, whole_words).into_iter().map(move |h| (Some(i), h))) diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index 5e89aef78a..84831b79a4 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -416,7 +416,7 @@ pub fn tools() -> Vec { t( "redact_mark", "Mark for redaction", - "Mark content for redaction (nothing is removed until redact_apply). One of: rect [x0, y0, x1, y1] (points from the top-left of the displayed page) with page; find (text, every match); words (a list of words or phrases, every match of each; the result's matched_words says how many marks each made); pattern (phone, email, credit-card, ssn, date: every match, Acrobat's Search & Redact patterns); whole_pages: true. find and words accept case_sensitive and whole_words (both default false). find, words, pattern and whole_pages work on pages (default all). overlay: text shown on the box once applied, or code_set (foia: U.S. FOIA (b)(1)(A)…(b)(9); privacy-act: U.S. Privacy Act (d)(5), (j)(1)…(k)(7)) with codes, shown as \"(b)(6), (b)(7)(C)\"; fill: box colour (default black). Undoable.", + "Mark content for redaction (nothing is removed until redact_apply). One of: rect [x0, y0, x1, y1] (points from the top-left of the displayed page) with page; find (text, every match); words (a list of words or phrases, every match of each; the result's matched_words says how many marks each made); pattern (phone, email, credit-card, ssn, date: every match, Acrobat's Search & Redact patterns); regex (your own regular expression, every match, on the page text with words and lines separated by one space; Rust regex syntax: linear-time, no look-around or back-references, case-sensitive unless (?i), at most 1024 bytes, matches of the empty string are refused) with optional validate (luhn, mod97, iban: keep a match only if its checksum holds); whole_pages: true. find and words accept case_sensitive and whole_words (both default false). find, words, pattern and whole_pages work on pages (default all). overlay: text shown on the box once applied, or code_set (foia: U.S. FOIA (b)(1)(A)…(b)(9); privacy-act: U.S. Privacy Act (d)(5), (j)(1)…(k)(7)) with codes, shown as \"(b)(6), (b)(7)(C)\"; fill: box colour (default black). Undoable.", ) .with(schema( json!({ @@ -428,6 +428,8 @@ pub fn tools() -> Vec { "case_sensitive": { "type": "boolean", "description": "Match letter case for find or words (default false)." }, "whole_words": { "type": "boolean", "description": "Match whole words for find or words (default false)." }, "pattern": { "type": "string", "enum": ["phone", "email", "credit-card", "ssn", "date"] }, + "regex": { "type": "string", "minLength": 1, "maxLength": 1024, "description": "Regular expression (Rust regex syntax); every match is marked." }, + "validate": { "type": "string", "enum": ["luhn", "mod97", "iban"], "description": "With regex: keep a match only if its checksum is valid (spaces and hyphens ignored)." }, "whole_pages": { "type": "boolean" }, "pages": pages("to search or mark (default: all)"), "overlay": { "type": "string" }, diff --git a/crates/automation/tests/automation.rs b/crates/automation/tests/automation.rs index 1e662d8f38..b1cb1920cb 100644 --- a/crates/automation/tests/automation.rs +++ b/crates/automation/tests/automation.rs @@ -3295,6 +3295,34 @@ fn redacting_through_tools() { assert!(matches!(a.call("redact_mark", &json!({ "doc": doc })), Err(ToolError::InvalidArgs(_)))); } +#[test] +fn redacting_with_a_caller_regex_through_tools() { + let dir = workdir("redact-regex"); + std::fs::write( + dir.join("ids.txt"), + "Ref 1 85 05 78 006 084 36 end\nRef 2 91 12 34 567 890 12 end\nCard 4539 1488 0343 6467 and 4539 1488 0343 6468", + ) + .unwrap(); + let mut a = auto(&dir); + let doc = ok(&mut a, "doc_create", json!({ "from": "text", "path": "ids.txt" }))["doc"].as_u64().unwrap(); + let bad = |a: &mut Automation, args: Value| matches!(a.call("redact_mark", &args), Err(ToolError::InvalidArgs(_))); + assert!(bad(&mut a, json!({ "doc": doc, "regex": "(unclosed" }))); + assert!(bad(&mut a, json!({ "doc": doc, "regex": "a*" }))); + assert!(bad(&mut a, json!({ "doc": doc, "regex": "x".repeat(2000) }))); + assert!(bad(&mut a, json!({ "doc": doc, "regex": "x", "pattern": "ssn" }))); + assert!(bad(&mut a, json!({ "doc": doc, "regex": "x", "validate": "crc" }))); + assert!(bad(&mut a, json!({ "doc": doc, "find": "x", "validate": "luhn" }))); + assert!(matches!(a.call("redact_mark", &json!({ "doc": doc, "regex": "Z{3}" })), Err(ToolError::Failed(_)))); + let r = ok(&mut a, "redact_mark", json!({ "doc": doc, "regex": r"\d \d\d \d\d \d\d \d{3} \d{3} \d\d" })); + assert_eq!(r["marked"].as_u64(), Some(2)); + let r = ok(&mut a, "redact_mark", json!({ "doc": doc, "regex": r"\d{4}( \d{4}){3}", "validate": "luhn" })); + assert_eq!(r["marked"].as_u64(), Some(1), "only the number with a valid checksum"); + ok(&mut a, "redact_apply", json!({ "doc": doc })); + let text = page_text(&mut a, doc)[0].clone(); + assert!(!text.contains("85 05") && !text.contains("91 12") && !text.contains("6467"), "{text}"); + assert!(text.contains("6468"), "{text}"); +} + #[test] fn redacting_with_codes_through_tools() { let dir = workdir("redact-codes"); diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 01f162ae7d..22743fdc65 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -152,6 +152,7 @@ pub use pdfcraft_redact::patterns::{ MAX_WORDS as REDACT_MAX_WORDS, PATTERNS as REDACT_PATTERNS, Pattern as RedactPattern, find as find_pattern, word_list as redact_word_list, }; pub use pdfcraft_redact::sanitize::{HIDDEN, Hidden}; +pub use pdfcraft_redact::user_regex::{UserRegex as RedactRegex, VALIDATORS as REDACT_VALIDATORS, Validator as RedactValidator}; pub use pdfcraft_sign as sign; pub use pdfcraft_sign::{SignOptions, SignatureInfo, Status as SignatureStatus, TrustStore}; pub use pdfcraft_xfdf::Format as DataFormat; diff --git a/crates/redact/Cargo.toml b/crates/redact/Cargo.toml index 49593855e2..f6f99995ac 100644 --- a/crates/redact/Cargo.toml +++ b/crates/redact/Cargo.toml @@ -17,6 +17,7 @@ pdfcraft-forms = { workspace = true } pdfcraft-edit = { workspace = true } pdfcraft-annot = { workspace = true } pdfcraft-xfa = { workspace = true } +regex = { workspace = true } thiserror = { workspace = true } [dev-dependencies] diff --git a/crates/redact/src/lib.rs b/crates/redact/src/lib.rs index 61231bd982..fc44d941d1 100644 --- a/crates/redact/src/lib.rs +++ b/crates/redact/src/lib.rs @@ -27,6 +27,7 @@ pub mod sanitize; mod tags; #[cfg(test)] mod tests; +pub mod user_regex; use interp::{Mode, Scope, process}; diff --git a/crates/redact/src/user_regex.rs b/crates/redact/src/user_regex.rs new file mode 100644 index 0000000000..043b51d0b0 --- /dev/null +++ b/crates/redact/src/user_regex.rs @@ -0,0 +1,211 @@ +//! Caller-supplied regular expressions for Search & Redact (`redact_mark regex`). +//! +//! The expression is untrusted input. It runs on the `regex` crate, which matches in time +//! linear in the text (no backtracking, no look-around or back-references), and is compiled +//! under hard limits on source length, nesting and compiled size, so a hostile expression is +//! refused with an error instead of exhausting time or memory. An optional [`Validator`] +//! keeps a match only when its checksum holds. + +use std::ops::Range; + +use regex::RegexBuilder; +use thiserror::Error; + +/// Longest expression accepted, in bytes. +pub const MAX_REGEX_LEN: usize = 1024; +/// Most matches returned for one page; the rest are ignored. +pub const MAX_MATCHES_PER_TEXT: usize = 10_000; +/// Cap on the compiled program and on the lazy-DFA cache, in bytes. +const SIZE_LIMIT: usize = 1 << 20; +/// Cap on parenthesis nesting. +const NEST_LIMIT: u32 = 32; + +#[derive(Debug, Error, PartialEq, Eq)] +pub enum RegexError { + #[error("regex is empty")] + Empty, + #[error("regex is {0} bytes long; at most {MAX_REGEX_LEN} are allowed")] + TooLong(usize), + #[error("invalid or too complex regex: {0}")] + Invalid(String), + #[error("regex matches the empty string, so it would mark nothing useful; require at least one character")] + MatchesEmpty, +} + +/// Checksum a match must pass to be kept. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Validator { + /// Luhn (ISO/IEC 7812): digits only, spaces and hyphens ignored. + Luhn, + /// ISO 7064 MOD 97-10: letters count as 10..35 and the number mod 97 must be 1. Letters and + /// digits only, spaces and hyphens ignored. + Mod97, + /// IBAN check (ISO 13616): the first four characters move to the end, then MOD 97-10. + Iban, +} + +/// Ids accepted for `validate`. +pub const VALIDATORS: [Validator; 3] = [Validator::Luhn, Validator::Mod97, Validator::Iban]; + +impl Validator { + pub fn id(self) -> &'static str { + match self { + Validator::Luhn => "luhn", + Validator::Mod97 => "mod97", + Validator::Iban => "iban", + } + } + + pub fn from_id(id: &str) -> Option { + VALIDATORS.into_iter().find(|v| v.id() == id) + } + + /// Whether `s` passes the checksum. + pub fn check(self, s: &str) -> bool { + match self { + Validator::Luhn => luhn(s), + Validator::Mod97 => mod97(s.chars()), + Validator::Iban => { + let kept: Vec = s.chars().filter(|c| *c != ' ' && *c != '-').collect(); + let (head, tail) = kept.split_at(kept.len().min(4)); + mod97(tail.iter().chain(head).copied()) + } + } + } +} + +fn luhn(s: &str) -> bool { + let (mut sum, mut digits, mut double) = (0u32, 0usize, false); + for c in s.chars().rev() { + if c == ' ' || c == '-' { + continue; + } + let Some(d) = c.to_digit(10).filter(|_| c.is_ascii_digit()) else { return false }; + let d = if double { if d * 2 > 9 { d * 2 - 9 } else { d * 2 } } else { d }; + sum = sum.saturating_add(d); + digits += 1; + double = !double; + } + digits >= 2 && sum % 10 == 0 +} + +fn mod97(s: impl Iterator) -> bool { + let (mut rem, mut n) = (0u32, 0usize); + for c in s { + if c == ' ' || c == '-' { + continue; + } + if !c.is_ascii_alphanumeric() { + return false; + } + // Digits stay one decimal digit, letters become two (A = 10 .. Z = 35). + let v = c.to_digit(36).unwrap_or(0); + rem = if v >= 10 { (rem * 100 + v) % 97 } else { (rem * 10 + v) % 97 }; + n += 1; + } + n >= 2 && rem == 1 +} + +/// A compiled caller expression with an optional checksum. +#[derive(Debug, Clone)] +pub struct UserRegex { + re: regex::Regex, + validator: Option, +} + +impl UserRegex { + pub fn new(source: &str, validator: Option) -> Result { + if source.is_empty() { + return Err(RegexError::Empty); + } + if source.len() > MAX_REGEX_LEN { + return Err(RegexError::TooLong(source.len())); + } + let re = RegexBuilder::new(source) + .size_limit(SIZE_LIMIT) + .dfa_size_limit(SIZE_LIMIT) + .nest_limit(NEST_LIMIT) + .build() + .map_err(|e| RegexError::Invalid(e.to_string()))?; + if re.is_match("") { + return Err(RegexError::MatchesEmpty); + } + Ok(Self { re, validator }) + } + + /// Character ranges of `text` the expression matches (non-overlapping, left to right). + pub fn find(&self, text: &[char]) -> Vec> { + let s: String = text.iter().collect(); + // Char index of every byte offset that starts a char (and of the end). + let mut byte_to_char = vec![usize::MAX; s.len().saturating_add(1)]; + for (ci, (bi, _)) in s.char_indices().enumerate() { + if let Some(slot) = byte_to_char.get_mut(bi) { + *slot = ci; + } + } + if let Some(slot) = byte_to_char.get_mut(s.len()) { + *slot = text.len(); + } + let mut out = Vec::new(); + for m in self.re.find_iter(&s) { + if out.len() >= MAX_MATCHES_PER_TEXT { + break; + } + if m.is_empty() || self.validator.is_some_and(|v| !v.check(m.as_str())) { + continue; + } + if let (Some(&a), Some(&b)) = (byte_to_char.get(m.start()), byte_to_char.get(m.end())) + && a < b + && b != usize::MAX + { + out.push(a..b); + } + } + out + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn chars(s: &str) -> Vec { + s.chars().collect() + } + + #[test] + fn finds_matches_as_char_ranges_with_multibyte_text() { + let r = UserRegex::new(r"\d{2}( \d{2}){2}", None).unwrap(); + let t = chars("Ré 85 05 78 et 91 12 34 fin"); + let found: Vec = r.find(&t).into_iter().map(|g| t[g].iter().collect()).collect(); + assert_eq!(found, ["85 05 78", "91 12 34"]); + } + + #[test] + fn validators_filter_matches() { + let t = chars("ok 4539 1488 0343 6467 bad 4539 1488 0343 6468"); + let r = UserRegex::new(r"\d{4}( \d{4}){3}", Some(Validator::Luhn)).unwrap(); + assert_eq!(r.find(&t).len(), 1); + let t = chars("GB82 WEST 1234 5698 7654 32 and GB83 WEST 1234 5698 7654 32"); + let r = UserRegex::new(r"GB\d\d WEST( \d{4}){3} \d\d", Some(Validator::Iban)).unwrap(); + assert_eq!(r.find(&t).len(), 1); + assert!(Validator::Mod97.check("98") && !Validator::Mod97.check("99")); + assert!(!Validator::Luhn.check("12a4") && !Validator::Mod97.check("é1") && !Validator::Luhn.check("0")); + } + + #[test] + fn hostile_expressions_are_refused_or_stay_linear() { + assert_eq!(UserRegex::new("", None).unwrap_err(), RegexError::Empty); + assert!(matches!(UserRegex::new(&"a".repeat(MAX_REGEX_LEN + 1), None), Err(RegexError::TooLong(_)))); + assert!(matches!(UserRegex::new("(unclosed", None), Err(RegexError::Invalid(_)))); + assert!(matches!(UserRegex::new(r"(a)\1", None), Err(RegexError::Invalid(_))), "no back-references"); + assert_eq!(UserRegex::new("a*", None).unwrap_err(), RegexError::MatchesEmpty); + assert!(matches!(UserRegex::new(r"\p{L}{1000}{1000}", None), Err(RegexError::Invalid(_))), "compiled size is capped"); + assert!(matches!(UserRegex::new(&format!("{}a{}", "(".repeat(100), ")".repeat(100)), None), Err(RegexError::Invalid(_)))); + // Classic catastrophic pattern finishes at once on a long non-matching input. + let r = UserRegex::new(r"(a+)+$", None).unwrap(); + let mut t = chars(&"a".repeat(50_000)); + t.push('!'); + assert!(r.find(&t).is_empty()); + } +} diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 533013b547..159f580635 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -5611,6 +5611,18 @@ tools = ["redact_mark"] evidence = ["crates/redact/src/patterns.rs::phone_numbers", "crates/redact/src/patterns.rs::credit_cards_need_a_valid_checksum", "crates/automation/tests/automation.rs::redacting_through_tools"] notes = "phone (North American, E.164 with + country code, national with trunk 0), e-mail, credit card (Luhn), SSN, dates (English US forms); text drawn in a font whose built-in encoding we hold no table for (Symbol, ZapfDingbats, MacExpert) has no Unicode mapping, so a pattern search cannot see it -- it is skipped rather than mis-decoded" +[[feature]] +id = "protect.redact-regex" +title = "Redact by caller-supplied regular expression" +area = "G" +tier = "P2" +milestone = "M8" +status = "shipped" +commands = [] +tools = ["redact_mark"] +evidence = ["crates/redact/src/user_regex.rs::hostile_expressions_are_refused_or_stay_linear", "crates/redact/src/user_regex.rs::validators_filter_matches", "crates/automation/tests/automation.rs::redacting_with_a_caller_regex_through_tools"] +notes = "Headless only (redact_mark regex, optional validate luhn, mod97 or iban; issue #1886): no UI entry yet. Rust regex syntax, matched in linear time with no look-around or back-references; expression capped at 1024 bytes, compiled size and nesting capped, empty-matching expressions refused. Matches run on the page text as the pattern matchers see it (words and lines joined by one space), so a match cannot span a page break. Missing: a UI field, a mask for checksums that need more than the whole match (for example the French social security key), and a check against Acrobat's own custom-pattern syntax (not observed)." + [[feature]] id = "protect.redact-pattern-locales" title = "Pattern sets per locale" From f6e335f3fe0c0ddc00f93b28a8c9f7499bcefa05 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 13 Oct 2026 21:49:37 +0200 Subject: [PATCH 003/198] UI: allow a half-screen window, and fold the mode bar when narrow (#1913) The viewport minimum was 820x520 points, wider than half of a 1440 px screen. It is now MIN_WINDOW_SIZE = 480x520 points. At that width the five mode tabs were hidden under the search field, so the mode bar folds them into one menu named after the current mode, shrinks the search field and turns it into an icon button when it gets too narrow. Notice bars stack their buttons under the message in a narrow document area. Co-Authored-By: Claude Sonnet 5.5 --- apps/pdfcraft/src/main.rs | 2 +- crates/ui-egui/src/canvas.rs | 16 +++++++++++ crates/ui-egui/src/chrome.rs | 43 +++++++++++++++++++++++++----- crates/ui-egui/src/lib.rs | 6 +++++ crates/ui-egui/src/widgets.rs | 7 +++++ crates/ui-egui/tests/min_window.rs | 38 ++++++++++++++++++++++++++ 6 files changed, 105 insertions(+), 7 deletions(-) create mode 100644 crates/ui-egui/tests/min_window.rs diff --git a/apps/pdfcraft/src/main.rs b/apps/pdfcraft/src/main.rs index 7a0bb20432..edc0652d0e 100644 --- a/apps/pdfcraft/src/main.rs +++ b/apps/pdfcraft/src/main.rs @@ -309,7 +309,7 @@ fn native_options(integrated: bool, app_frame: bool, renderer: eframe::Renderer) let mut viewport = egui::ViewportBuilder::default() .with_title("PdfCraft") .with_inner_size([1440.0, 920.0]) - .with_min_inner_size([820.0, 520.0]) + .with_min_inner_size(pdfcraft_ui_egui::MIN_WINDOW_SIZE) .with_drag_and_drop(true) // Wayland app id: matches packaging/linux/ai.storyteller.pdfcraft.desktop. .with_app_id(APP_ID); diff --git a/crates/ui-egui/src/canvas.rs b/crates/ui-egui/src/canvas.rs index 989b731b7e..72a8316ab7 100644 --- a/crates/ui-egui/src/canvas.rs +++ b/crates/ui-egui/src/canvas.rs @@ -4248,8 +4248,24 @@ fn notices( fn notice_bar(ui: &mut egui::Ui, t: &Tokens, icon: &str, icon_color: Color32, text: String, buttons: impl FnOnce(&mut egui::Ui)) { const ROW: f32 = 28.0; const ICON: f32 = 16.0; + /// Content width below which the buttons stack under the message. + const STACK_BELOW: f32 = 300.0; egui::Frame::NONE.fill(t.accent_soft).inner_margin(egui::Margin::symmetric(14, 7)).show(ui, |ui| { let width = ui.available_width(); + if width < STACK_BELOW { + // A narrow document area (a half-width window): the message takes the full width and + // the buttons sit on a row of their own below it, rather than squeezing it to nothing. + ui.with_layout(egui::Layout::left_to_right(egui::Align::Min), |ui| { + let (slot, _) = ui.allocate_exact_size(vec2(ICON, ICON), Sense::hover()); + icons::paint(ui, Rect::from_center_size(slot.center(), vec2(ICON, ICON)), icon, ICON, icon_color); + ui.add(egui::Label::new(egui::RichText::new(text).color(t.text)).wrap()); + }); + ui.allocate_ui_with_layout(vec2(width, ROW), egui::Layout::right_to_left(egui::Align::Center), |ui| { + ui.set_min_height(ROW); + buttons(ui); + }); + return; + } ui.allocate_ui_with_layout(vec2(width, ROW), egui::Layout::right_to_left(egui::Align::Center), |ui| { ui.set_min_height(ROW); buttons(ui); diff --git a/crates/ui-egui/src/chrome.rs b/crates/ui-egui/src/chrome.rs index e8bf3b48dd..88165102ad 100644 --- a/crates/ui-egui/src/chrome.rs +++ b/crates/ui-egui/src/chrome.rs @@ -487,6 +487,11 @@ fn tab( }) } +/// Width the print, save and info buttons and their gaps take in the mode bar. +const MODE_BAR_RIGHT: f32 = 3.0 * 32.0 + 2.0 * 4.0 + 8.0; +/// Narrowest search field the mode bar shows; below it the field becomes an icon button. +const MODE_BAR_SEARCH_MIN: f32 = 140.0; + pub fn mode_bar(app: &mut PdfCraftApp, ui: &mut egui::Ui) { let t = Tokens::get(ui.ctx()); egui::Panel::top("mode_bar") @@ -499,13 +504,32 @@ pub fn mode_bar(app: &mut PdfCraftApp, ui: &mut egui::Ui) { ui.add_space(6.0); ui.painter().vline(ui.cursor().left(), ui.max_rect().y_range().shrink(12.0), Stroke::new(1.0, t.divider)); ui.add_space(10.0); - for (mode, label) in - [(Mode::AllTools, "All tools"), (Mode::Read, "Read"), (Mode::Edit, "Edit"), (Mode::Convert, "Convert"), (Mode::Sign, "E-Sign")] - { - if widgets::mode_tab(ui, tl!(label), app.mode == mode).clicked() { - app.switch_mode(mode, true); + let modes = + [(Mode::AllTools, "All tools"), (Mode::Read, "Read"), (Mode::Edit, "Edit"), (Mode::Convert, "Convert"), (Mode::Sign, "E-Sign")]; + // What the right side needs (three 32 px buttons, gaps) and the least search box worth showing. + let avail = ui.available_width(); + let tabs_w: f32 = modes.iter().map(|(_, l)| widgets::mode_tab_width(ui, tl!(*l))).sum::() + 2.0 * (modes.len() as f32 - 1.0); + let inline_tabs = avail >= tabs_w + MODE_BAR_RIGHT + MODE_BAR_SEARCH_MIN; + if inline_tabs { + for (mode, label) in modes { + if widgets::mode_tab(ui, tl!(label), app.mode == mode).clicked() { + app.switch_mode(mode, true); + } } + } else { + // Narrow window: the five modes fold into one menu named after the current one. + let current = modes.iter().find(|(m, _)| *m == app.mode).map_or("All tools", |(_, l)| *l); + let resp = widgets::ghost_button(ui, "chevron-down", tl!(current)); + egui::Popup::menu(&resp).show(|ui| { + ui.set_min_width(160.0); + for (mode, label) in modes { + if ui.add(egui::Button::selectable(app.mode == mode, tl!(label))).clicked() { + app.switch_mode(mode, true); + } + } + }); } + let left = ui.available_width(); ui.with_layout(Layout::right_to_left(Align::Center), |ui| { ui.spacing_mut().item_spacing.x = 4.0; let has_doc = app.active.is_some(); @@ -524,7 +548,14 @@ pub fn mode_bar(app: &mut PdfCraftApp, ui: &mut egui::Ui) { } }); ui.add_space(8.0); - if widgets::search_box(ui, tl!("Find tools and commands"), 260.0).clicked() { + let search_w = (left - MODE_BAR_RIGHT).clamp(0.0, 260.0); + let opened = if search_w >= MODE_BAR_SEARCH_MIN { + widgets::search_box(ui, tl!("Find tools and commands"), search_w).clicked() + } else { + // Too narrow for the field: a button opens the same command search. + icons::button(ui, "search", 32.0, false, tl!("Find tools and commands")).clicked() + }; + if opened { app.palette_open = true; } }); diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index d101d812e0..832ae30cb4 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -149,6 +149,12 @@ mod open_worker_tests; use pdfcraft_engine::{DocId, Session}; pub use canvas::{DocView, RasterMemory}; + +/// Smallest window the shell is laid out for, in egui points (logical pixels, so the OS scale +/// factor is already applied: 480 points are 600 px at 125 % and 720 px at 150 %). 480 is half of a +/// 1440-wide screen at 150 % scaling, the tightest common half-screen split on Windows; the mode +/// bar, tab strip and notice bars adapt below the old 820 minimum (#1913). +pub const MIN_WINDOW_SIZE: [f32; 2] = [480.0, 520.0]; pub use editing::{CloseRequest, SaveTarget}; pub use files::{ExtractDraft, FilePurpose, FileRequest, RotateDraft, SplitDraft, SplitMode, SplitPlan}; pub use recovery::{AUTOSAVE_SECS, RecoveryMeta, RecoveryStore}; diff --git a/crates/ui-egui/src/widgets.rs b/crates/ui-egui/src/widgets.rs index 4f5aaa79b6..cae7405ac5 100644 --- a/crates/ui-egui/src/widgets.rs +++ b/crates/ui-egui/src/widgets.rs @@ -108,6 +108,7 @@ pub fn mode_tab(ui: &mut egui::Ui, label: &str, active: bool) -> Response { let t = Tokens::get(ui.ctx()); let font = if active { theme::semibold(13.5) } else { theme::medium(13.5) }; let w = ui.fonts_mut(|f| f.layout_no_wrap(label.to_owned(), font.clone(), t.text).size().x); + // Keep in step with `mode_tab_width`. let (rect, resp) = ui.allocate_exact_size(vec2(w + 22.0, 48.0), Sense::click()); resp.widget_info(|| egui::WidgetInfo::selected(egui::WidgetType::Button, ui.is_enabled(), active, label)); let press = theme::Press::track(ui, &resp); @@ -128,6 +129,12 @@ pub fn mode_tab(ui: &mut egui::Ui, label: &str, active: bool) -> Response { theme::hand(resp) } +/// Width [`mode_tab`] takes for `label` (the semibold, active, form is the widest). +pub fn mode_tab_width(ui: &egui::Ui, label: &str) -> f32 { + let t = Tokens::get(ui.ctx()); + ui.fonts_mut(|f| f.layout_no_wrap(label.to_owned(), theme::semibold(13.5), t.text).size().x) + 22.0 +} + /// Rounded pill button; `primary` fills with the accent. pub fn pill_button(ui: &mut egui::Ui, label: &str, primary: bool) -> Response { let t = Tokens::get(ui.ctx()); diff --git a/crates/ui-egui/tests/min_window.rs b/crates/ui-egui/tests/min_window.rs new file mode 100644 index 0000000000..b06e81d12d --- /dev/null +++ b/crates/ui-egui/tests/min_window.rs @@ -0,0 +1,38 @@ +//! The smallest window (#1913): at `MIN_WINDOW_SIZE` the mode bar folds the five modes into a menu +//! and keeps the command search reachable, instead of hiding them under the search field. + +use egui::accesskit::Role; +use egui_kittest::Harness; +use egui_kittest::kittest::Queryable; +use pdfcraft_ui_egui::{MIN_WINDOW_SIZE, PdfCraftApp}; + +fn harness(size: egui::Vec2) -> Harness<'static, PdfCraftApp> { + let mut h = Harness::builder().with_size(size).build_eframe(|_cc| { + let mut app = PdfCraftApp::new(); + app.set_option("language", "en").unwrap(); + app + }); + h.run_steps(8); + h +} + +#[test] +fn at_the_minimum_width_the_modes_fold_into_a_menu_and_search_stays() { + let mut h = harness(egui::vec2(MIN_WINDOW_SIZE[0], MIN_WINDOW_SIZE[1])); + assert!(h.query_by_role_and_label(Role::Button, "Read").is_none(), "no room for five tabs"); + assert!(h.query_by_role_and_label(Role::Button, "Find tools and commands").is_some(), "command search is reachable"); + h.get_by_role_and_label(Role::Button, "All tools").click(); + h.run_steps(4); + h.get_by_role_and_label(Role::Button, "E-Sign").click(); + h.run_steps(4); + assert!(h.query_by_role_and_label(Role::Button, "E-Sign").is_some(), "the menu button is named after the mode"); + assert!(h.query_by_role_and_label(Role::Button, "Menu").is_some(), "main menu reachable"); +} + +#[test] +fn a_wide_window_keeps_the_five_mode_tabs() { + let h = harness(egui::vec2(1400.0, 900.0)); + for mode in ["All tools", "Read", "Edit", "Convert", "E-Sign"] { + assert!(h.query_by_role_and_label(Role::Button, mode).is_some(), "{mode} tab"); + } +} From 03a4c21cfe7f73b890ee4d93a204fe38ad2c9e11 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Wed, 14 Oct 2026 22:40:36 +0200 Subject: [PATCH 004/198] UI: stack notice buttons only below 200 pt so the 900-wide notice keeps its layout (#1913) Co-Authored-By: Claude Sonnet 5.5 --- crates/ui-egui/src/canvas.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/ui-egui/src/canvas.rs b/crates/ui-egui/src/canvas.rs index 72a8316ab7..ca1ff12337 100644 --- a/crates/ui-egui/src/canvas.rs +++ b/crates/ui-egui/src/canvas.rs @@ -4249,7 +4249,7 @@ fn notice_bar(ui: &mut egui::Ui, t: &Tokens, icon: &str, icon_color: Color32, te const ROW: f32 = 28.0; const ICON: f32 = 16.0; /// Content width below which the buttons stack under the message. - const STACK_BELOW: f32 = 300.0; + const STACK_BELOW: f32 = 200.0; egui::Frame::NONE.fill(t.accent_soft).inner_margin(egui::Margin::symmetric(14, 7)).show(ui, |ui| { let width = ui.available_width(); if width < STACK_BELOW { From c3cf27558a6d0f6c8c245b5a4bad813e6ae5de30 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Wed, 14 Oct 2026 22:53:33 +0200 Subject: [PATCH 005/198] UI: wrap stacked notice buttons and scroll the rail in short windows (#1913) Co-Authored-By: Claude Sonnet 5.5 --- crates/ui-egui/src/canvas.rs | 3 +- crates/ui-egui/src/chrome.rs | 241 +++++++++++++++-------------- crates/ui-egui/tests/min_window.rs | 82 ++++++++++ 3 files changed, 211 insertions(+), 115 deletions(-) diff --git a/crates/ui-egui/src/canvas.rs b/crates/ui-egui/src/canvas.rs index ca1ff12337..349b4deded 100644 --- a/crates/ui-egui/src/canvas.rs +++ b/crates/ui-egui/src/canvas.rs @@ -4260,7 +4260,8 @@ fn notice_bar(ui: &mut egui::Ui, t: &Tokens, icon: &str, icon_color: Color32, te icons::paint(ui, Rect::from_center_size(slot.center(), vec2(ICON, ICON)), icon, ICON, icon_color); ui.add(egui::Label::new(egui::RichText::new(text).color(t.text)).wrap()); }); - ui.allocate_ui_with_layout(vec2(width, ROW), egui::Layout::right_to_left(egui::Align::Center), |ui| { + // The buttons wrap onto further rows when even the full width is too narrow for them all. + ui.allocate_ui_with_layout(vec2(width, ROW), egui::Layout::right_to_left(egui::Align::Center).with_main_wrap(true), |ui| { ui.set_min_height(ROW); buttons(ui); }); diff --git a/crates/ui-egui/src/chrome.rs b/crates/ui-egui/src/chrome.rs index 88165102ad..d4c6f09503 100644 --- a/crates/ui-egui/src/chrome.rs +++ b/crates/ui-egui/src/chrome.rs @@ -648,6 +648,9 @@ fn main_menu(app: &mut PdfCraftApp, ui: &mut egui::Ui) { }); } +/// Height the rail's page-navigation cluster (zoom, fit, page box) needs; below it the rail scrolls. +const RAIL_BOTTOM_H: f32 = 316.0; + pub fn right_rail(app: &mut PdfCraftApp, ui: &mut egui::Ui) { let t = Tokens::get(ui.ctx()); let Some((index, id)) = app.active_ids() else { return }; @@ -668,122 +671,132 @@ pub fn right_rail(app: &mut PdfCraftApp, ui: &mut egui::Ui) { .exact_size(48.0) .frame(egui::Frame::NONE.fill(t.chrome).inner_margin(egui::Margin::symmetric(6, 8)).stroke(Stroke::new(1.0, t.divider))) .show(ui, |ui| { - ui.spacing_mut().item_spacing.y = 4.0; - let mut rail_button = |ui: &mut egui::Ui, panel: RightPanel, icon: &str, tip: &str, has: bool| { - let selected = app.right == Some(panel) && (app.mode == Mode::Read || app.docking.exposed(crate::docking::Panel::Inspector)); - let r = icons::button(ui, icon, 34.0, selected, tl!(tip)); - if has && !selected { - let c = r.rect.right_top() + vec2(-8.0, 8.0); - ui.painter().circle_filled(c, 3.0, t.accent); - } - if r.clicked() { - app.choose_right_panel(if selected { None } else { Some(panel) }); - } - }; - rail_button(ui, RightPanel::Comments, "message-square-text", "Comments", has_comments); - rail_button(ui, RightPanel::Bookmarks, "bookmark", "Bookmarks", has_outline); - rail_button(ui, RightPanel::Pages, "files", "Page thumbnails", false); - rail_button(ui, RightPanel::Fields, "text-cursor-input", "Form fields", has_fields); - rail_button(ui, RightPanel::Layers, "layers", "Layers", has_layers); - rail_button(ui, RightPanel::Attachments, "paperclip", "Attachments", has_files); - rail_button(ui, RightPanel::Signatures, "signature", "Signatures", has_signatures); - if has_check { - rail_button(ui, RightPanel::Accessibility, "accessibility", "Accessibility Checker", false); - } - rail_button(ui, RightPanel::Tags, "tag", "Tags", false); + let panel_h = ui.available_height(); + // A window too short for both clusters scrolls the rail instead of overlapping them (#1913). + egui::ScrollArea::vertical().auto_shrink(false).scroll_bar_visibility(egui::scroll_area::ScrollBarVisibility::AlwaysHidden).show( + ui, + |ui| { + ui.spacing_mut().item_spacing.y = 4.0; + let mut rail_button = |ui: &mut egui::Ui, panel: RightPanel, icon: &str, tip: &str, has: bool| { + let selected = app.right == Some(panel) && (app.mode == Mode::Read || app.docking.exposed(crate::docking::Panel::Inspector)); + let r = icons::button(ui, icon, 34.0, selected, tl!(tip)); + if has && !selected { + let c = r.rect.right_top() + vec2(-8.0, 8.0); + ui.painter().circle_filled(c, 3.0, t.accent); + } + if r.clicked() { + app.choose_right_panel(if selected { None } else { Some(panel) }); + } + }; + rail_button(ui, RightPanel::Comments, "message-square-text", "Comments", has_comments); + rail_button(ui, RightPanel::Bookmarks, "bookmark", "Bookmarks", has_outline); + rail_button(ui, RightPanel::Pages, "files", "Page thumbnails", false); + rail_button(ui, RightPanel::Fields, "text-cursor-input", "Form fields", has_fields); + rail_button(ui, RightPanel::Layers, "layers", "Layers", has_layers); + rail_button(ui, RightPanel::Attachments, "paperclip", "Attachments", has_files); + rail_button(ui, RightPanel::Signatures, "signature", "Signatures", has_signatures); + if has_check { + rail_button(ui, RightPanel::Accessibility, "accessibility", "Accessibility Checker", false); + } + rail_button(ui, RightPanel::Tags, "tag", "Tags", false); - // Page navigation cluster at the bottom (as in Acrobat's rail). - let view = &mut app.views[index]; - let mut page_display = None; - ui.with_layout(Layout::bottom_up(Align::Center), |ui| { - ui.spacing_mut().item_spacing.y = 2.0; - let tip = crate::commands::key_tip(ui.ctx(), tl!("Zoom out ({key})"), crate::commands::ZOOM_OUT); - if icons::button(ui, "zoom-out", 32.0, false, &tip).clicked() { - view.zoom_step(false); - } - let tip = crate::commands::key_tip(ui.ctx(), tl!("Zoom in ({key})"), crate::commands::ZOOM_IN); - if icons::button(ui, "zoom-in", 32.0, false, &tip).clicked() { - view.zoom_step(true); - } - // Between rotate and zoom, as in Acrobat. Its command runs once `view` is free. - let tip = crate::i18n::fmt(tl!("Page display: {layout}"), &[("layout", tl!(view.layout.label()))]); - let resp = icons::button(ui, view.layout.icon(), 32.0, false, &tip); - page_display = egui::Popup::menu(&resp) - .show(|ui| { - ui.set_min_width(220.0); - rail_view_menu(ui, view) - }) - .and_then(|r| r.inner); - let tip = crate::commands::key_tip(ui.ctx(), tl!("Rotate view clockwise ({key})"), crate::commands::ROTATE_CW); - if icons::button(ui, "rotate-cw", 32.0, false, &tip).clicked() { - view.rotate_view(true); - } - // Not `columns-2` while fitting the page: that is the two-page view's icon. - let fit_icon = if view.fit == Fit::Width { "maximize" } else { "maximize-2" }; - if icons::button(ui, fit_icon, 32.0, false, tl!("Toggle fit page / fit width")).clicked() { - view.fit = if view.fit == Fit::Width { Fit::Page } else { Fit::Width }; - view.goto = Some((view.current, 0.0)); - } - let zoom = format!("{:.0}%", view.zoom * 100.0); - ui.allocate_ui_with_layout(vec2(40.0, 16.0), Layout::top_down(Align::Center), |ui| { - ui.label(egui::RichText::new(zoom).font(theme::medium(11.0)).color(t.text_muted)); - }); - ui.add_space(6.0); - if icons::button(ui, "chevron-down", 30.0, false, tl!("Next page")).clicked() { - view.step_page(true); - } - if icons::button(ui, "chevron-up", 30.0, false, tl!("Previous page")).clicked() { - view.step_page(false); - } - // History is distinct from stepping to the adjacent page, and belongs to this tab. - for (forward, icon, label, shortcut, enabled) in [ - (true, "chevron-right", tl!("Next view"), crate::commands::NEXT_VIEW, !view.forward.is_empty()), - (false, "chevron-left", tl!("Previous view"), crate::commands::PREV_VIEW, !view.back.is_empty()), - ] { - let tip = format!("{label} ({})", crate::commands::shortcut_label(ui.ctx(), shortcut)); - if ui.add_enabled_ui(enabled, |ui| icons::button(ui, icon, 30.0, false, &tip)).inner.clicked() { - view.view_history(forward); + // Page navigation cluster at the bottom (as in Acrobat's rail). + let view = &mut app.views[index]; + let mut page_display = None; + let cluster_w = ui.available_width(); + // Inside the scroll area `available_height` is unbounded: the panel's height was taken outside. + let cluster_h = (panel_h - ui.min_rect().height() - ui.spacing().item_spacing.y).max(RAIL_BOTTOM_H); + ui.allocate_ui_with_layout(vec2(cluster_w, cluster_h), Layout::bottom_up(Align::Center), |ui| { + ui.spacing_mut().item_spacing.y = 2.0; + let tip = crate::commands::key_tip(ui.ctx(), tl!("Zoom out ({key})"), crate::commands::ZOOM_OUT); + if icons::button(ui, "zoom-out", 32.0, false, &tip).clicked() { + view.zoom_step(false); + } + let tip = crate::commands::key_tip(ui.ctx(), tl!("Zoom in ({key})"), crate::commands::ZOOM_IN); + if icons::button(ui, "zoom-in", 32.0, false, &tip).clicked() { + view.zoom_step(true); + } + // Between rotate and zoom, as in Acrobat. Its command runs once `view` is free. + let tip = crate::i18n::fmt(tl!("Page display: {layout}"), &[("layout", tl!(view.layout.label()))]); + let resp = icons::button(ui, view.layout.icon(), 32.0, false, &tip); + page_display = egui::Popup::menu(&resp) + .show(|ui| { + ui.set_min_width(220.0); + rail_view_menu(ui, view) + }) + .and_then(|r| r.inner); + let tip = crate::commands::key_tip(ui.ctx(), tl!("Rotate view clockwise ({key})"), crate::commands::ROTATE_CW); + if icons::button(ui, "rotate-cw", 32.0, false, &tip).clicked() { + view.rotate_view(true); + } + // Not `columns-2` while fitting the page: that is the two-page view's icon. + let fit_icon = if view.fit == Fit::Width { "maximize" } else { "maximize-2" }; + if icons::button(ui, fit_icon, 32.0, false, tl!("Toggle fit page / fit width")).clicked() { + view.fit = if view.fit == Fit::Width { Fit::Page } else { Fit::Width }; + view.goto = Some((view.current, 0.0)); + } + let zoom = format!("{:.0}%", view.zoom * 100.0); + ui.allocate_ui_with_layout(vec2(40.0, 16.0), Layout::top_down(Align::Center), |ui| { + ui.label(egui::RichText::new(zoom).font(theme::medium(11.0)).color(t.text_muted)); + }); + ui.add_space(6.0); + if icons::button(ui, "chevron-down", 30.0, false, tl!("Next page")).clicked() { + view.step_page(true); + } + if icons::button(ui, "chevron-up", 30.0, false, tl!("Previous page")).clicked() { + view.step_page(false); + } + // History is distinct from stepping to the adjacent page, and belongs to this tab. + for (forward, icon, label, shortcut, enabled) in [ + (true, "chevron-right", tl!("Next view"), crate::commands::NEXT_VIEW, !view.forward.is_empty()), + (false, "chevron-left", tl!("Previous view"), crate::commands::PREV_VIEW, !view.back.is_empty()), + ] { + let tip = format!("{label} ({})", crate::commands::shortcut_label(ui.ctx(), shortcut)); + if ui.add_enabled_ui(enabled, |ui| icons::button(ui, icon, 30.0, false, &tip)).inner.clicked() { + view.view_history(forward); + } + } + ui.label(egui::RichText::new(page_count.to_string()).font(theme::regular(11.0)).color(t.text_muted)); + let page_input_id = view.page_input_id(); + // Consume replacement selection once, including when the editor has lost focus. + let reselect = std::mem::take(&mut view.page_input_reselect); + // Navigation updates the idle display, never the text being edited. + if !ui.memory(|m| m.has_focus(page_input_id)) { + view.page_input = view.current_page_label(labels.get(view.current).map(String::as_str)); + } else if reselect { + // The replacement changed the label while egui retained this editor's cursor. + let mut state = egui::TextEdit::load_state(ui.ctx(), page_input_id).unwrap_or_default(); + let end = egui::text::CCursor::new(view.page_input.chars().count()); + state.cursor.set_char_range(Some(egui::text::CCursorRange::two(egui::text::CCursor::new(0), end))); + state.store(ui.ctx(), page_input_id); + } + let edit = egui::TextEdit::singleline(&mut view.page_input) + .id(page_input_id) + .desired_width(34.0) + .horizontal_align(Align::Center) + .font(theme::medium(12.0)) + .margin(vec2(2.0, 4.0)); + let r = egui::Frame::NONE + .fill(t.field) + .stroke(Stroke::new(1.0, t.border)) + .corner_radius(CornerRadius::same(5)) + .show(ui, |ui| ui.add(edit)) + .inner + .on_hover_text(tl!("Current page — type a page number or label (such as iv) and press Enter")); + if r.lost_focus() { + if ui.input(|i| i.key_pressed(egui::Key::Enter)) { + // A page label first (logical page numbers, as Acrobat), then a number. + let typed = view.page_input.clone(); + view.go_to_typed(&typed, &labels); + } + view.page_input = view.current_page_label(labels.get(view.current).map(String::as_str)); + } + }); + if let Some(id) = page_display { + app.execute(id); } - } - ui.label(egui::RichText::new(page_count.to_string()).font(theme::regular(11.0)).color(t.text_muted)); - let page_input_id = view.page_input_id(); - // Consume replacement selection once, including when the editor has lost focus. - let reselect = std::mem::take(&mut view.page_input_reselect); - // Navigation updates the idle display, never the text being edited. - if !ui.memory(|m| m.has_focus(page_input_id)) { - view.page_input = view.current_page_label(labels.get(view.current).map(String::as_str)); - } else if reselect { - // The replacement changed the label while egui retained this editor's cursor. - let mut state = egui::TextEdit::load_state(ui.ctx(), page_input_id).unwrap_or_default(); - let end = egui::text::CCursor::new(view.page_input.chars().count()); - state.cursor.set_char_range(Some(egui::text::CCursorRange::two(egui::text::CCursor::new(0), end))); - state.store(ui.ctx(), page_input_id); - } - let edit = egui::TextEdit::singleline(&mut view.page_input) - .id(page_input_id) - .desired_width(34.0) - .horizontal_align(Align::Center) - .font(theme::medium(12.0)) - .margin(vec2(2.0, 4.0)); - let r = egui::Frame::NONE - .fill(t.field) - .stroke(Stroke::new(1.0, t.border)) - .corner_radius(CornerRadius::same(5)) - .show(ui, |ui| ui.add(edit)) - .inner - .on_hover_text(tl!("Current page — type a page number or label (such as iv) and press Enter")); - if r.lost_focus() { - if ui.input(|i| i.key_pressed(egui::Key::Enter)) { - // A page label first (logical page numbers, as Acrobat), then a number. - let typed = view.page_input.clone(); - view.go_to_typed(&typed, &labels); - } - view.page_input = view.current_page_label(labels.get(view.current).map(String::as_str)); - } - }); - if let Some(id) = page_display { - app.execute(id); - } + }, + ); }); } diff --git a/crates/ui-egui/tests/min_window.rs b/crates/ui-egui/tests/min_window.rs index b06e81d12d..889d952520 100644 --- a/crates/ui-egui/tests/min_window.rs +++ b/crates/ui-egui/tests/min_window.rs @@ -36,3 +36,85 @@ fn a_wide_window_keeps_the_five_mode_tabs() { assert!(h.query_by_role_and_label(Role::Button, mode).is_some(), "{mode} tab"); } } + +fn fixture(n: usize) -> Vec { + let mut objs: Vec = vec!["<< /Type /Catalog /Pages 2 0 R >>".into()]; + let kids: Vec = (0..n).map(|i| format!("{} 0 R", 4 + 2 * i)).collect(); + objs.push(format!("<< /Type /Pages /Kids [{}] /Count {n} /MediaBox [0 0 200 300] >>", kids.join(" "))); + objs.push("<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>".into()); + for i in 0..n { + objs.push(format!("<< /Type /Page /Parent 2 0 R /Contents {} 0 R /Resources << /Font << /F1 3 0 R >> >> >>", 5 + 2 * i)); + let body = format!("BT /F1 24 Tf 20 150 Td (Page {}) Tj ET", i + 1); + objs.push(format!("<< /Length {} >>\nstream\n{body}\nendstream", body.len())); + } + let mut out = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (i, o) in objs.iter().enumerate() { + offsets.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n{o}\nendobj\n", i + 1).as_bytes()); + } + let xref = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes()); + for o in offsets { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n", objs.len() + 1).as_bytes()); + out +} + +fn protected(user: &str, owner: &str, permissions: i32) -> Vec { + protected_with(pdfcraft_cos::Algorithm::Aes256, user, owner, permissions) +} + +fn protected_with(algorithm: pdfcraft_cos::Algorithm, user: &str, owner: &str, permissions: i32) -> Vec { + let mut doc = pdfcraft_cos::Document::open(std::sync::Arc::new(fixture(2))).unwrap(); + doc.set_encryption(&pdfcraft_cos::NewEncryption { + algorithm, + user_password: user, + owner_password: owner, + permissions, + encrypt_metadata: true, + seed: [4; 32], + }) + .unwrap(); + pdfcraft_cos::write_full(&doc, &Default::default()).unwrap() +} + +/// A narrow window: the stacked notice bar keeps both buttons inside the window and apart. +#[test] +fn the_stacked_notice_keeps_its_buttons_in_view() { + let mut h = Harness::builder().with_size(egui::vec2(MIN_WINDOW_SIZE[0], MIN_WINDOW_SIZE[1])).build_eframe(|_cc| { + let mut app = PdfCraftApp::new(); + app.set_option("language", "en").unwrap(); + app.open_bytes("locked.pdf", None, protected("", "owner", 0b0100)).unwrap(); + app + }); + h.run_steps(8); + let window = egui::Rect::from_min_size(egui::Pos2::ZERO, egui::vec2(MIN_WINDOW_SIZE[0], MIN_WINDOW_SIZE[1])); + let msg = h.get_by_label_contains("This document is secured").rect(); + let (unlock, settings) = (h.get_by_label("Unlock…").rect(), h.get_by_label("Security settings").rect()); + for (name, r) in [("unlock", unlock), ("settings", settings)] { + assert!(window.contains_rect(r), "{name} stays inside the window: {r:?}"); + assert!(r.left() >= msg.left() - 30.0, "{name} is not cut off at the bar's left edge: {r:?} vs {msg:?}"); + assert!(r.top() >= msg.bottom() - 0.5 || r.bottom() <= msg.top() + 0.5, "{name} does not cover the message"); + } + assert!(!unlock.intersects(settings), "the buttons do not overlap: {unlock:?} {settings:?}"); +} + +/// #1913 (1182 x 556): the rail's top buttons and its page-navigation cluster must not overlap. +#[test] +fn a_short_window_does_not_overlap_the_rail_clusters() { + let h = { + let mut h = Harness::builder().with_size(egui::vec2(1182.0, 556.0)).build_eframe(|_cc| { + let mut app = PdfCraftApp::new(); + app.set_option("language", "en").unwrap(); + app.open_bytes("a.pdf", None, fixture(2)).unwrap(); + app + }); + h.run_steps(8); + h + }; + let tags = h.get_by_label("Tags").rect(); + let next = h.get_by_label("Previous page").rect(); + assert!(next.top() >= tags.bottom(), "page navigation sits below the rail buttons: {tags:?} vs {next:?}"); +} From fb7f7252dbc470ef5736295da94134f2c3e5146f Mon Sep 17 00:00:00 2001 From: vangel76 Date: Thu, 15 Oct 2026 23:33:25 +0200 Subject: [PATCH 006/198] UI: size the rail's bottom cluster for the history buttons, fix a pixel-fragile flick test (#1913) The rail's page-navigation cluster measures 360 pt since the two view-history buttons landed; RAIL_BOTTOM_H was 316, so in a short window the cluster overflowed its allocation and overlapped. The min-window test now uses the minimum height and checks the page box, history and zoom buttons. a_quick_flick_on_either_edge_rewraps_the_paragraph grabbed the widened edge 2 px outside the box, at the limit of the grab zone; the rail's scroll area moved the canvas width by 3 px and the press missed. It now grabs 1 px outside. Co-Authored-By: Claude Sonnet 5.5 --- crates/ui-egui/src/chrome.rs | 4 ++-- crates/ui-egui/tests/editing.rs | 4 +++- crates/ui-egui/tests/min_window.rs | 12 ++++++++++-- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/crates/ui-egui/src/chrome.rs b/crates/ui-egui/src/chrome.rs index d4c6f09503..51f269dc43 100644 --- a/crates/ui-egui/src/chrome.rs +++ b/crates/ui-egui/src/chrome.rs @@ -648,8 +648,8 @@ fn main_menu(app: &mut PdfCraftApp, ui: &mut egui::Ui) { }); } -/// Height the rail's page-navigation cluster (zoom, fit, page box) needs; below it the rail scrolls. -const RAIL_BOTTOM_H: f32 = 316.0; +/// Height the rail's page-navigation cluster (zoom, fit, page, history buttons, page box) needs (360 pt measured); below it the rail scrolls. +const RAIL_BOTTOM_H: f32 = 364.0; pub fn right_rail(app: &mut PdfCraftApp, ui: &mut egui::Ui) { let t = Tokens::get(ui.ctx()); diff --git a/crates/ui-egui/tests/editing.rs b/crates/ui-egui/tests/editing.rs index f8db525751..cef7214485 100644 --- a/crates/ui-egui/tests/editing.rs +++ b/crates/ui-egui/tests/editing.rs @@ -3487,7 +3487,9 @@ fn a_quick_flick_on_either_edge_rewraps_the_paragraph() { assert_eq!(text, ["Page", "1"], "narrowed from the right edge"); // Flick the right edge outward, past the box and its handle: back onto one line. let b = block(&h); - let edge = screen(b.rect[2], grip_y(&b)) + egui::vec2(2.0, 0.0); + // 1 px outside the box: the grab zone's outer limit sits near 2 px, where a pixel of page + // scale (the canvas width depends on the rail) decides whether the press lands on the handle. + let edge = screen(b.rect[2], grip_y(&b)) + egui::vec2(1.0, 0.0); flick(&mut h, edge, edge + egui::vec2(80.0 * k, 0.0)); let text: Vec = lines(&h).iter().map(|l| l.text.clone()).collect(); assert_eq!(text, ["Page 1"], "widened by a fast drag that left the handle at once"); diff --git a/crates/ui-egui/tests/min_window.rs b/crates/ui-egui/tests/min_window.rs index 889d952520..8d28c6aff6 100644 --- a/crates/ui-egui/tests/min_window.rs +++ b/crates/ui-egui/tests/min_window.rs @@ -101,11 +101,11 @@ fn the_stacked_notice_keeps_its_buttons_in_view() { assert!(!unlock.intersects(settings), "the buttons do not overlap: {unlock:?} {settings:?}"); } -/// #1913 (1182 x 556): the rail's top buttons and its page-navigation cluster must not overlap. +/// #1913 (the minimum height): the rail's top buttons and its page-navigation cluster must not overlap. #[test] fn a_short_window_does_not_overlap_the_rail_clusters() { let h = { - let mut h = Harness::builder().with_size(egui::vec2(1182.0, 556.0)).build_eframe(|_cc| { + let mut h = Harness::builder().with_size(egui::vec2(1182.0, MIN_WINDOW_SIZE[1])).build_eframe(|_cc| { let mut app = PdfCraftApp::new(); app.set_option("language", "en").unwrap(); app.open_bytes("a.pdf", None, fixture(2)).unwrap(); @@ -117,4 +117,12 @@ fn a_short_window_does_not_overlap_the_rail_clusters() { let tags = h.get_by_label("Tags").rect(); let next = h.get_by_label("Previous page").rect(); assert!(next.top() >= tags.bottom(), "page navigation sits below the rail buttons: {tags:?} vs {next:?}"); + // The history buttons are the cluster's topmost buttons (above the page box). + let history = h.get_by_label_contains("Previous view").rect(); + assert!(history.top() >= tags.bottom(), "view history sits below the rail buttons: {tags:?} vs {history:?}"); + // The topmost thing in the cluster is the page box; the lowest is the zoom-out button. + let page_box = h.get_by_role(Role::TextInput).rect(); + assert!(page_box.top() >= tags.bottom(), "the page box sits below the rail buttons: {tags:?} vs {page_box:?}"); + let zoom_out = h.get_by_label_contains("Zoom out").rect(); + assert!(zoom_out.top() >= history.bottom(), "zoom sits below view history: {history:?} vs {zoom_out:?}"); } From d6d4999a355325e701506df85318ddf50f1689dd Mon Sep 17 00:00:00 2001 From: vangel76 Date: Mon, 19 Oct 2026 02:39:14 +0200 Subject: [PATCH 007/198] Test: the rail now has two text fields (zoom and page box) After merging main's editable zoom field, get_by_role(TextInput) in the min-window test found two nodes. Check every text field of the rail sits below the rail buttons. Co-Authored-By: Claude Sonnet 5.5 --- crates/ui-egui/tests/min_window.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/crates/ui-egui/tests/min_window.rs b/crates/ui-egui/tests/min_window.rs index 8d28c6aff6..005c97dc5f 100644 --- a/crates/ui-egui/tests/min_window.rs +++ b/crates/ui-egui/tests/min_window.rs @@ -120,9 +120,11 @@ fn a_short_window_does_not_overlap_the_rail_clusters() { // The history buttons are the cluster's topmost buttons (above the page box). let history = h.get_by_label_contains("Previous view").rect(); assert!(history.top() >= tags.bottom(), "view history sits below the rail buttons: {tags:?} vs {history:?}"); - // The topmost thing in the cluster is the page box; the lowest is the zoom-out button. - let page_box = h.get_by_role(Role::TextInput).rect(); - assert!(page_box.top() >= tags.bottom(), "the page box sits below the rail buttons: {tags:?} vs {page_box:?}"); + // The cluster holds two text fields, the zoom field and the page box; the page box is the topmost thing in it, the lowest is the zoom-out button. + for field in h.get_all_by_role(Role::TextInput) { + let r = field.rect(); + assert!(r.top() >= tags.bottom(), "the rail's text fields sit below the rail buttons: {tags:?} vs {r:?}"); + } let zoom_out = h.get_by_label_contains("Zoom out").rect(); assert!(zoom_out.top() >= history.bottom(), "zoom sits below view history: {history:?} vs {zoom_out:?}"); } From 8239f9c4eda76ed126ddf2f063544cc5a7819289 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 20 Oct 2026 00:55:51 +0200 Subject: [PATCH 008/198] Optimize target_bytes: keep zip/retain image encoding on retries, only tighten ppi Co-Authored-By: Claude Sonnet 5.5 --- crates/automation/src/lib.rs | 11 +++++++---- crates/automation/src/tools.rs | 2 +- crates/automation/tests/automation.rs | 8 ++++++++ 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/crates/automation/src/lib.rs b/crates/automation/src/lib.rs index 17c1f7293c..a279856fb5 100644 --- a/crates/automation/src/lib.rs +++ b/crates/automation/src/lib.rs @@ -93,6 +93,7 @@ const MAX_DPI: f64 = 600.0; /// How far a watermark may be moved from the page centre, in points (a page is at most 14,400). const WATERMARK_MAX_OFFSET: f64 = 20_000.0; /// `doc_optimize` with `target_bytes`: (max target ppi, max JPEG quality) per extra attempt, mildest first. +/// The quality applies only to images the caller left on JPEG; zip and retain keep their encoding. const TARGET_LADDER: [(f64, u8); 5] = [(110.0, 45), (96.0, 35), (72.0, 28), (54.0, 20), (36.0, 12)]; /// Bookmarks in one page of the bookmark list, by default and at most. @@ -1143,10 +1144,10 @@ impl Automation { img.downsample = true; img.target_ppi = img.target_ppi.min(ppi); img.above_ppi = img.above_ppi.min(ppi * 1.5).max(img.target_ppi); - img.compression = match img.compression { - Compression::Jpeg(q) => Compression::Jpeg(q.min(quality)), - _ => Compression::Jpeg(quality), - }; + // Only a caller who chose JPEG gets a lower quality; zip and retain stay as asked. + if let Compression::Jpeg(q) = img.compression { + img.compression = Compression::Jpeg(q.min(quality)); + } } let (b2, r2) = self.session.optimized_bytes(id, &s, &discard).map_err(failed)?; attempts += 1; @@ -1160,6 +1161,8 @@ impl Automation { let mut extra = json!({}); if let Some(t) = target { let (ppi, quality) = step_used.checked_sub(1).and_then(|k| TARGET_LADDER.get(k)).copied().unzip(); + let lossy = matches!(settings.color.compression, Compression::Jpeg(_)) || matches!(settings.gray.compression, Compression::Jpeg(_)); + let quality = quality.filter(|_| lossy); extra = json!({ "target_bytes": t, "target_reached": bytes.len() <= t, diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index 2365086a33..550fa119d8 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -1303,7 +1303,7 @@ pub fn tools() -> Vec { .with(schema(json!({ "doc": doc() }), &["doc"])), t("doc_open_revision", "Open a revision", "Open saved revision `revision` (1 = the oldest) of a document as a new, unsaved document, to see the file as it was then.") .with(schema(json!({ "doc": doc(), "revision": { "type": "integer", "minimum": 1 } }), &["doc", "revision"])), - t("doc_optimize", "Optimize PDF", "Write an optimized copy to `path` (Acrobat's PDF Optimizer). preset: standard (default; Reduce File Size's choices) or mobile (images to 96 ppi above 144, JPEG high quality); the options below change it. merge_fragments (default true) first puts a page picture drawn as many stacked image strips (as some PDF printers write it) back together into one image, so it can be resampled (merged_images, fragments_merged). color / gray: { downsample, ppi, above_ppi, compression: jpeg|zip|retain, quality 1–100 } (defaults: downsample to 150 ppi above 225, JPEG 60). Images are measured where pages draw them and replaced only when smaller. discard_*: thumbnails (default true), alternate_images (true), tags, print_settings; flate_unencoded (true); remove_invalid_links and remove_unreferenced_dests (true); images and forms the pages list but never draw are always dropped (unused_xobjects). discard: Remove Hidden Information categories (metadata, attachments, comments, form-fields, hidden-text, hidden-layers, bookmarks, links-actions-scripts, private-data). Signed documents are refused. target_bytes (optional): if the copy is larger, retry with up to 5 progressively stronger image settings (downsample to at most 110, 96, 72, 54, 36 ppi with JPEG quality at most 45, 35, 28, 20, 12; applied to colour and grey images) and stop at the first copy that fits; reports target_reached, attempts and target_settings, and writes the smallest copy with target_reached false when none fits. Without it nothing changes. The open document is unchanged.") + t("doc_optimize", "Optimize PDF", "Write an optimized copy to `path` (Acrobat's PDF Optimizer). preset: standard (default; Reduce File Size's choices) or mobile (images to 96 ppi above 144, JPEG high quality); the options below change it. merge_fragments (default true) first puts a page picture drawn as many stacked image strips (as some PDF printers write it) back together into one image, so it can be resampled (merged_images, fragments_merged). color / gray: { downsample, ppi, above_ppi, compression: jpeg|zip|retain, quality 1–100 } (defaults: downsample to 150 ppi above 225, JPEG 60). Images are measured where pages draw them and replaced only when smaller. discard_*: thumbnails (default true), alternate_images (true), tags, print_settings; flate_unencoded (true); remove_invalid_links and remove_unreferenced_dests (true); images and forms the pages list but never draw are always dropped (unused_xobjects). discard: Remove Hidden Information categories (metadata, attachments, comments, form-fields, hidden-text, hidden-layers, bookmarks, links-actions-scripts, private-data). Signed documents are refused. target_bytes (optional): if the copy is larger, retry with up to 5 progressively stronger image settings (downsample to at most 110, 96, 72, 54, 36 ppi with applied to colour and grey images; images set to JPEG also get JPEG quality at most 45, 35, 28, 20, 12, while zip and retain images keep their encoding and only get the lower ppi) and stop at the first copy that fits; reports target_reached, attempts and target_settings, and writes the smallest copy with target_reached false when none fits. Without it nothing changes. The open document is unchanged.") .cmd("optimize.advanced") .with(schema( json!({ diff --git a/crates/automation/tests/automation.rs b/crates/automation/tests/automation.rs index 8018c83321..b6e5305a69 100644 --- a/crates/automation/tests/automation.rs +++ b/crates/automation/tests/automation.rs @@ -4791,6 +4791,14 @@ fn optimizing_to_a_target_size() { let again = ok(&mut a, "doc_optimize", json!({ "doc": doc, "path": "again.pdf" })); assert_eq!(again["bytes_after"].as_u64().unwrap(), base_len); assert!(matches!(a.call("doc_optimize", &json!({ "doc": doc, "path": "x.pdf", "target_bytes": 0 })), Err(ToolError::InvalidArgs(_)))); + // A caller who asked for lossless images keeps them: retries only tighten the ppi, no JPEG quality. + let lossless = json!({ "compression": "zip" }); + let r = ok(&mut a, "doc_optimize", json!({ "doc": doc, "path": "zip.pdf", "color": lossless, "gray": lossless, "target_bytes": 1 })); + assert_eq!(r["attempts"].as_u64(), Some(6), "{r}"); + assert!(r["target_settings"]["max_ppi"].is_number() && r["target_settings"]["max_jpeg_quality"].is_null(), "{r}"); + // A JPEG caller gets the quality step. + let r = ok(&mut a, "doc_optimize", json!({ "doc": doc, "path": "jpg.pdf", "target_bytes": 1 })); + assert!(r["target_settings"]["max_jpeg_quality"].is_number(), "{r}"); // Confinement still applies. assert!(a.call("doc_optimize", &json!({ "doc": doc, "path": "../out.pdf", "target_bytes": 1000 })).is_err()); } From 86af84171f004d1e06564a01152406bf4d997ff4 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sat, 24 Oct 2026 21:11:43 +0200 Subject: [PATCH 009/198] doc_optimize: reject a target_bytes too large for usize Co-Authored-By: Claude Sonnet 5.5 --- crates/automation/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/automation/src/lib.rs b/crates/automation/src/lib.rs index 53e99e008d..ffde1dfab3 100644 --- a/crates/automation/src/lib.rs +++ b/crates/automation/src/lib.rs @@ -1308,7 +1308,7 @@ impl Automation { }; let target = match a.opt_int("target_bytes")? { None => None, - Some(t) if t >= 1 => Some(t as usize), + Some(t) if t >= 1 => Some(usize::try_from(t).map_err(|_| ToolError::InvalidArgs(format!("target_bytes {t} is too large")))?), Some(t) => return Err(ToolError::InvalidArgs(format!("target_bytes must be at least 1, not {t}"))), }; let (mut bytes, mut r) = self.session.optimized_bytes(id, &settings, &discard).map_err(failed)?; From 0f64d9589ed5469490bd5d176ce47101f2c1d45a Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sat, 24 Oct 2026 21:45:07 +0200 Subject: [PATCH 010/198] Run rustfmt on the mode bar after the upstream merge Co-Authored-By: Claude Sonnet 5.5 --- crates/ui-egui/src/chrome.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/ui-egui/src/chrome.rs b/crates/ui-egui/src/chrome.rs index b08c48518d..e5b239ecbb 100644 --- a/crates/ui-egui/src/chrome.rs +++ b/crates/ui-egui/src/chrome.rs @@ -507,8 +507,7 @@ pub fn mode_bar(app: &mut PdfCraftApp, ui: &mut egui::Ui) { ui.add_space(10.0); // Acrobat's mode bar has no Read tab: reading is what the viewer does with no tool // in use. Read mode (the panes hidden) stays a View command and a default workspace. - let modes = - [(Mode::AllTools, "All tools"), (Mode::Edit, "Edit"), (Mode::Convert, "Convert"), (Mode::Sign, "E-Sign")]; + let modes = [(Mode::AllTools, "All tools"), (Mode::Edit, "Edit"), (Mode::Convert, "Convert"), (Mode::Sign, "E-Sign")]; // What the right side needs (three 32 px buttons, gaps) and the least search box worth showing. let avail = ui.available_width(); let tabs_w: f32 = modes.iter().map(|(_, l)| widgets::mode_tab_width(ui, tl!(*l))).sum::() + 2.0 * (modes.len() as f32 - 1.0); From 589c4af8e7138aa22eb51322353835d2ab73b772 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 25 Oct 2026 10:19:17 +0100 Subject: [PATCH 011/198] Redact regex: fail when a page exceeds the match cap UserRegex::find now returns an error instead of silently stopping at 10,000 matches, so redact_mark cannot report success while later matches stay readable. Empty-match detection probes several short texts so patterns like \b are refused with the clear message. Schema notes that the 1024 limit is in bytes. Co-Authored-By: Claude Sonnet 5.5 --- crates/automation/src/redact.rs | 19 ++++++++++++- crates/automation/src/tools.rs | 4 +-- crates/redact/src/user_regex.rs | 48 ++++++++++++++++++++++++--------- 3 files changed, 56 insertions(+), 15 deletions(-) diff --git a/crates/automation/src/redact.rs b/crates/automation/src/redact.rs index 840020c570..808b6eabb7 100644 --- a/crates/automation/src/redact.rs +++ b/crates/automation/src/redact.rs @@ -126,7 +126,24 @@ impl Automation { let info = self.doc(a)?.info.clone(); for (&p, text) in pages.iter().zip(&texts) { let hits: Vec<(Option, _)> = match (&user_re, pattern) { - (Some(re), _) => text.find_with(|chars| re.find(chars)).into_iter().map(|h| (None, h)).collect(), + (Some(re), _) => { + // A page with too many matches fails the call: marking only the first ones + // would leave the rest readable after redact_apply. + let too_many = std::cell::Cell::new(false); + let found = text.find_with(|chars| { + re.find(chars).unwrap_or_else(|_| { + too_many.set(true); + Vec::new() + }) + }); + if too_many.get() { + return Err(ToolError::InvalidArgs(format!( + "regex has more than 10000 matches on page {}; nothing was marked. Narrow the expression so that every match can be marked", + p + 1 + ))); + } + found.into_iter().map(|h| (None, h)).collect() + } (None, Some(pat)) => text.find_with(|chars| find_pattern(pat, chars)).into_iter().map(|h| (None, h)).collect(), (None, None) => needles .iter() diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index 4c651c00dd..0a41cacc0a 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -486,7 +486,7 @@ pub fn tools() -> Vec { t( "redact_mark", "Mark for redaction", - "Mark content for redaction (nothing is removed until redact_apply). One of: rect [x0, y0, x1, y1] (points from the top-left of the displayed page) with page; find (text, every match); words (a list of words or phrases, every match of each; the result's matched_words says how many marks each made); pattern (phone, email, credit-card, ssn, date: every match, Acrobat's Search & Redact patterns); regex (your own regular expression, every match, on the page text with words and lines separated by one space; Rust regex syntax: linear-time, no look-around or back-references, case-sensitive unless (?i), at most 1024 bytes, matches of the empty string are refused) with optional validate (luhn, mod97, iban: keep a match only if its checksum holds); whole_pages: true. find and words accept case_sensitive and whole_words (both default false). Select one page with page or multiple with pages (default all); do not pass both. rect requires page. overlay: text shown on the box once applied, or code_set (foia: U.S. FOIA (b)(1)(A)…(b)(9); privacy-act: U.S. Privacy Act (d)(5), (j)(1)…(k)(7)) with codes, shown as \"(b)(6), (b)(7)(C)\"; fill: box colour (default black). Undoable.", + "Mark content for redaction (nothing is removed until redact_apply). One of: rect [x0, y0, x1, y1] (points from the top-left of the displayed page) with page; find (text, every match); words (a list of words or phrases, every match of each; the result's matched_words says how many marks each made); pattern (phone, email, credit-card, ssn, date: every match, Acrobat's Search & Redact patterns); regex (your own regular expression, every match, on the page text with words and lines separated by one space; Rust regex syntax: linear-time, no look-around or back-references, case-sensitive unless (?i), at most 1024 bytes, more than 10000 matches on one page fail the call, matches of the empty string are refused) with optional validate (luhn, mod97, iban: keep a match only if its checksum holds); whole_pages: true. find and words accept case_sensitive and whole_words (both default false). Select one page with page or multiple with pages (default all); do not pass both. rect requires page. overlay: text shown on the box once applied, or code_set (foia: U.S. FOIA (b)(1)(A)…(b)(9); privacy-act: U.S. Privacy Act (d)(5), (j)(1)…(k)(7)) with codes, shown as \"(b)(6), (b)(7)(C)\"; fill: box colour (default black). Undoable.", ) .with(schema( json!({ @@ -498,7 +498,7 @@ pub fn tools() -> Vec { "case_sensitive": { "type": "boolean", "description": "Match letter case for find or words (default false)." }, "whole_words": { "type": "boolean", "description": "Match whole words for find or words (default false)." }, "pattern": { "type": "string", "enum": ["phone", "email", "credit-card", "ssn", "date"] }, - "regex": { "type": "string", "minLength": 1, "maxLength": 1024, "description": "Regular expression (Rust regex syntax); every match is marked." }, + "regex": { "type": "string", "minLength": 1, "maxLength": 1024, "description": "Regular expression (Rust regex syntax), at most 1024 bytes (maxLength counts characters, the limit is bytes); every match is marked." }, "validate": { "type": "string", "enum": ["luhn", "mod97", "iban"], "description": "With regex: keep a match only if its checksum is valid (spaces and hyphens ignored)." }, "whole_pages": { "type": "boolean" }, "pages": pages("to search or mark (default: all); mutually exclusive with page, and not valid with rect"), diff --git a/crates/redact/src/user_regex.rs b/crates/redact/src/user_regex.rs index 043b51d0b0..9d1c1b0004 100644 --- a/crates/redact/src/user_regex.rs +++ b/crates/redact/src/user_regex.rs @@ -13,7 +13,8 @@ use thiserror::Error; /// Longest expression accepted, in bytes. pub const MAX_REGEX_LEN: usize = 1024; -/// Most matches returned for one page; the rest are ignored. +/// Most matches accepted for one page; one more makes [`UserRegex::find`] fail, so a page is never +/// marked only in part. pub const MAX_MATCHES_PER_TEXT: usize = 10_000; /// Cap on the compiled program and on the lazy-DFA cache, in bytes. const SIZE_LIMIT: usize = 1 << 20; @@ -30,6 +31,8 @@ pub enum RegexError { Invalid(String), #[error("regex matches the empty string, so it would mark nothing useful; require at least one character")] MatchesEmpty, + #[error("regex has more than {MAX_MATCHES_PER_TEXT} matches on one page; narrow it so that every match can be marked")] + TooManyMatches, } /// Checksum a match must pass to be kept. @@ -106,6 +109,12 @@ fn mod97(s: impl Iterator) -> bool { n >= 2 && rem == 1 } +/// Whether `re` can match the empty string, which `is_match("")` alone misses for assertions such +/// as `\b`. Tries a few short texts with letters, digits, space, newline and a non-ASCII letter. +fn can_match_empty(re: ®ex::Regex) -> bool { + ["", " ", "a", "1", "a a", "a\n", "\n", "é", "aa 11"].iter().any(|p| re.find_iter(p).any(|m| m.is_empty())) +} + /// A compiled caller expression with an optional checksum. #[derive(Debug, Clone)] pub struct UserRegex { @@ -127,14 +136,17 @@ impl UserRegex { .nest_limit(NEST_LIMIT) .build() .map_err(|e| RegexError::Invalid(e.to_string()))?; - if re.is_match("") { + if can_match_empty(&re) { return Err(RegexError::MatchesEmpty); } Ok(Self { re, validator }) } /// Character ranges of `text` the expression matches (non-overlapping, left to right). - pub fn find(&self, text: &[char]) -> Vec> { + /// + /// Fails with [`RegexError::TooManyMatches`] when more than [`MAX_MATCHES_PER_TEXT`] matches + /// are kept, instead of returning a partial list the caller would take for complete. + pub fn find(&self, text: &[char]) -> Result>, RegexError> { let s: String = text.iter().collect(); // Char index of every byte offset that starts a char (and of the end). let mut byte_to_char = vec![usize::MAX; s.len().saturating_add(1)]; @@ -148,9 +160,6 @@ impl UserRegex { } let mut out = Vec::new(); for m in self.re.find_iter(&s) { - if out.len() >= MAX_MATCHES_PER_TEXT { - break; - } if m.is_empty() || self.validator.is_some_and(|v| !v.check(m.as_str())) { continue; } @@ -158,10 +167,13 @@ impl UserRegex { && a < b && b != usize::MAX { + if out.len() >= MAX_MATCHES_PER_TEXT { + return Err(RegexError::TooManyMatches); + } out.push(a..b); } } - out + Ok(out) } } @@ -177,7 +189,7 @@ mod tests { fn finds_matches_as_char_ranges_with_multibyte_text() { let r = UserRegex::new(r"\d{2}( \d{2}){2}", None).unwrap(); let t = chars("Ré 85 05 78 et 91 12 34 fin"); - let found: Vec = r.find(&t).into_iter().map(|g| t[g].iter().collect()).collect(); + let found: Vec = r.find(&t).unwrap().into_iter().map(|g| t[g].iter().collect()).collect(); assert_eq!(found, ["85 05 78", "91 12 34"]); } @@ -185,10 +197,10 @@ mod tests { fn validators_filter_matches() { let t = chars("ok 4539 1488 0343 6467 bad 4539 1488 0343 6468"); let r = UserRegex::new(r"\d{4}( \d{4}){3}", Some(Validator::Luhn)).unwrap(); - assert_eq!(r.find(&t).len(), 1); + assert_eq!(r.find(&t).unwrap().len(), 1); let t = chars("GB82 WEST 1234 5698 7654 32 and GB83 WEST 1234 5698 7654 32"); let r = UserRegex::new(r"GB\d\d WEST( \d{4}){3} \d\d", Some(Validator::Iban)).unwrap(); - assert_eq!(r.find(&t).len(), 1); + assert_eq!(r.find(&t).unwrap().len(), 1); assert!(Validator::Mod97.check("98") && !Validator::Mod97.check("99")); assert!(!Validator::Luhn.check("12a4") && !Validator::Mod97.check("é1") && !Validator::Luhn.check("0")); } @@ -199,13 +211,25 @@ mod tests { assert!(matches!(UserRegex::new(&"a".repeat(MAX_REGEX_LEN + 1), None), Err(RegexError::TooLong(_)))); assert!(matches!(UserRegex::new("(unclosed", None), Err(RegexError::Invalid(_)))); assert!(matches!(UserRegex::new(r"(a)\1", None), Err(RegexError::Invalid(_))), "no back-references"); - assert_eq!(UserRegex::new("a*", None).unwrap_err(), RegexError::MatchesEmpty); + for empty in ["a*", r"\b", "(?m)^", r"\B", "(?m)$", r"\d*|x"] { + assert_eq!(UserRegex::new(empty, None).unwrap_err(), RegexError::MatchesEmpty, "{empty}"); + } + assert!(UserRegex::new(r"\bfoo\b", None).is_ok()); assert!(matches!(UserRegex::new(r"\p{L}{1000}{1000}", None), Err(RegexError::Invalid(_))), "compiled size is capped"); assert!(matches!(UserRegex::new(&format!("{}a{}", "(".repeat(100), ")".repeat(100)), None), Err(RegexError::Invalid(_)))); // Classic catastrophic pattern finishes at once on a long non-matching input. let r = UserRegex::new(r"(a+)+$", None).unwrap(); let mut t = chars(&"a".repeat(50_000)); t.push('!'); - assert!(r.find(&t).is_empty()); + assert!(r.find(&t).unwrap().is_empty()); + } + + #[test] + fn more_than_the_cap_of_matches_is_an_error_not_a_partial_list() { + let r = UserRegex::new("x", None).unwrap(); + let at_cap = chars(&"x ".repeat(MAX_MATCHES_PER_TEXT)); + assert_eq!(r.find(&at_cap).unwrap().len(), MAX_MATCHES_PER_TEXT); + let over = chars(&"x ".repeat(MAX_MATCHES_PER_TEXT + 1)); + assert_eq!(r.find(&over).unwrap_err(), RegexError::TooManyMatches); } } From 90fb9b8c9e37adb0663bacca887248672f68d8e4 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Mon, 26 Oct 2026 19:18:16 +0100 Subject: [PATCH 012/198] Test the menu bar's File item instead of the removed Menu button Co-Authored-By: Claude Sonnet 5.5 --- crates/ui-egui/tests/it/min_window.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/ui-egui/tests/it/min_window.rs b/crates/ui-egui/tests/it/min_window.rs index fc395e7cdf..f384eef0eb 100644 --- a/crates/ui-egui/tests/it/min_window.rs +++ b/crates/ui-egui/tests/it/min_window.rs @@ -26,7 +26,7 @@ fn at_the_minimum_width_the_modes_fold_into_a_menu_and_search_stays() { h.get_by_role_and_label(Role::Button, "E-Sign").click(); h.run_steps(4); assert!(h.query_by_role_and_label(Role::Button, "E-Sign").is_some(), "the menu button is named after the mode"); - assert!(h.query_by_role_and_label(Role::Button, "Menu").is_some(), "main menu reachable"); + assert!(h.query_by_role_and_label(Role::MenuItem, "File").is_some(), "main menu reachable"); } #[test] From 88649a8955fa674adcaa35af48340de74209e294 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Mon, 26 Oct 2026 20:35:42 +0100 Subject: [PATCH 013/198] Name Read mode in the folded mode menu The narrow-window mode menu fell back to "All tools" while Read was active. Co-Authored-By: Claude Sonnet 5.5 --- crates/ui-egui/src/chrome.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/crates/ui-egui/src/chrome.rs b/crates/ui-egui/src/chrome.rs index f831d26d9c..5e1fef8860 100644 --- a/crates/ui-egui/src/chrome.rs +++ b/crates/ui-egui/src/chrome.rs @@ -505,8 +505,13 @@ pub fn mode_bar(app: &mut PdfCraftApp, ui: &mut egui::Ui) { } } else { // Narrow window: the modes fold into one menu named after the current one. - let current = modes.iter().find(|(m, _)| *m == app.mode).map_or("All tools", |(_, l)| *l); - let resp = widgets::ghost_button(ui, "chevron-down", tl!(current)); + // Read is not one of the four choices but can be active (View command): name it too. + let current = if app.mode == Mode::Read { + tl!("Read") + } else { + tl!(modes.iter().find(|(m, _)| *m == app.mode).map_or("All tools", |(_, l)| *l)) + }; + let resp = widgets::ghost_button(ui, "chevron-down", current); egui::Popup::menu(&resp).show(|ui| { ui.set_min_width(160.0); for (mode, label) in modes { From fe03703a3b71a92c96a1d5318b3ad4dd9d02ab88 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Mon, 26 Oct 2026 21:55:16 +0100 Subject: [PATCH 014/198] perf(cos): keep a running total of the scan reader's cache size ObjectReader::try_get measured every parsed object in its cache (up to 128) after each read to check the byte budget. Keep a running total instead and measure the whole cache only when something other than one freshly parsed object changed it. Eviction timing is unchanged, so output is identical. Fixes #4867 Co-Authored-By: Claude Sonnet 5.5 --- crates/cos/src/document.rs | 43 +++++++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/crates/cos/src/document.rs b/crates/cos/src/document.rs index 3c7f5fbe18..74b773e026 100644 --- a/crates/cos/src/document.rs +++ b/crates/cos/src/document.rs @@ -445,6 +445,10 @@ pub struct Document { pub(crate) struct ObjectReader { pub(crate) document: Document, decoded_count: usize, + /// Objects in the parsed-object cache and the bytes of their own, as last measured. Kept + /// as a running total so a read does not measure the whole cache again. + counted_objects: usize, + counted_bytes: usize, } impl ObjectReader { @@ -488,8 +492,25 @@ impl ObjectReader { // have cached large objects. if let Ok(mut cache) = self.document.cache.lock() { let data = &self.document.data; - if cache.len() >= Self::OBJECTS || cache.values().fold(0usize, |n, o| n.saturating_add(owned_bytes(o, data, 0))) > Self::STREAM_BYTES { + // The common read parsed exactly this object and nothing else (a hit changes + // nothing): add its size to the running total. Anything else, such as a failed + // load, a /Length that cached further objects or a cache changed from outside, + // measures the whole cache again. + let grew_by_this_object = cache.len() == self.counted_objects.saturating_add(1) + && matches!((&object, cache.get(&reference.num)), (Ok(read), Some(cached)) if Arc::ptr_eq(read, cached)); + if object.is_ok() && cache.len() == self.counted_objects { + // Unchanged: a hit, or an object held by the overlay. + } else if let (true, Ok(read)) = (grew_by_this_object, &object) { + self.counted_bytes = self.counted_bytes.saturating_add(owned_bytes(read, data, 0)); + self.counted_objects = cache.len(); + } else { + self.counted_bytes = cache.values().fold(0usize, |n, o| n.saturating_add(owned_bytes(o, data, 0))); + self.counted_objects = cache.len(); + } + if cache.len() >= Self::OBJECTS || self.counted_bytes > Self::STREAM_BYTES { cache.clear(); + self.counted_objects = 0; + self.counted_bytes = 0; } } object @@ -1176,7 +1197,7 @@ impl Document { let mut document = self.clone(); document.cache = Arc::default(); document.objstms = Arc::default(); - ObjectReader { document, decoded_count: 0 } + ObjectReader { document, decoded_count: 0, counted_objects: 0, counted_bytes: 0 } } fn load(&self, num: u32, depth: u32) -> Result { @@ -3004,11 +3025,27 @@ endobj // Data of its own (decrypted streams, strings) counts: past the budget the cache is // cleared after the next read, including a failed one. let owned = Object::Stream(crate::Stream::from_raw(Dict::new(), vec![b'y'; ObjectReader::STREAM_BYTES + 1])); - reader.document.cache.lock().unwrap().insert(3, Arc::new(owned)); + reader.document.cache.lock().unwrap().insert(5, Arc::new(owned)); assert_eq!(*reader.get(ObjRef::new(99, 0)), Object::Null); assert!(reader.document.cache.lock().unwrap().is_empty(), "owned data over budget is evicted"); } + #[test] + fn temporary_reader_running_total_matches_the_cache() { + let mut bodies = vec!["<< /Type /Catalog /Pages 2 0 R >>".to_owned(), "<< /Type /Pages /Kids [] /Count 0 >>".to_owned()]; + bodies.extend((0..300).map(|i| format!("<< /Value {i} /Name ({}) >>", "s".repeat(i % 40)))); + let refs: Vec<_> = bodies.iter().map(String::as_str).collect(); + let doc = Document::open(Arc::new(build(&refs, "/Root 1 0 R"))).unwrap(); + let mut reader = doc.object_reader(); + for num in (1..=302).chain([5, 301, 99_999, 7]) { + let _ = reader.try_get(ObjRef::new(num, 0)); + let cache = reader.document.cache.lock().unwrap(); + let measured = cache.values().fold(0usize, |n, o| n.saturating_add(owned_bytes(o, &reader.document.data, 0))); + assert_eq!((reader.counted_objects, reader.counted_bytes), (cache.len(), measured), "after object {num}"); + assert!(cache.len() < ObjectReader::OBJECTS); + } + } + #[test] fn temporary_reader_and_full_save_do_not_fill_snapshot_caches() { let mut source = Document::new_empty(); From be8345aa3f2d9930201e766a4c8a8f315a48f8d6 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Mon, 26 Oct 2026 21:57:29 +0100 Subject: [PATCH 015/198] perf(cos): rebuild a damaged cross-reference index without parsing every object Reconstruct parsed every object to see whether it is an object stream, and searched each object's next 1,024 bytes and the whole file byte by byte. Only objects whose dictionary holds ObjStm or a # (a name can hide letters as #xx) are now parsed for that check. Every /XRef position is found once, and find() scans for the needle's rarest byte in 32-byte blocks. The index, trailer and repair log are unchanged. Fixes #4868 Co-Authored-By: Claude Sonnet 5.5 --- crates/cos/src/document.rs | 72 ++++++++++++++++++++++++++++-- crates/cos/tests/it/main.rs | 1 + crates/cos/tests/it/reconstruct.rs | 57 +++++++++++++++++++++++ 3 files changed, 126 insertions(+), 4 deletions(-) create mode 100644 crates/cos/tests/it/reconstruct.rs diff --git a/crates/cos/src/document.rs b/crates/cos/src/document.rs index 3c7f5fbe18..4ad24a11b1 100644 --- a/crates/cos/src/document.rs +++ b/crates/cos/src/document.rs @@ -2052,10 +2052,29 @@ impl Document { // Offsets of objects that may be cross-reference streams: their dictionaries are the // trailers of files that have no `trailer` keyword (§7.5.8). let mut xref_streams = Vec::new(); - for (num, offset, generation, body) in object_headers(&data) { + // Objects whose bytes (the last object of a number wins, as in `entries`) could hold an + // `/ObjStm` type; the others are not parsed for the object-stream check below. + let mut candidates: HashSet = HashSet::new(); + let headers = object_headers(&data); + // Where `/XRef` occurs in the file, found in one pass. + let mut xref_marks = Vec::new(); + let mut mark_from = 0; + while let Some(p) = find(&data, b"/XRef", mark_from, data.len()) { + xref_marks.push(p); + mark_from = p + 1; + } + for (i, &(num, offset, generation, body)) in headers.iter().enumerate() { entries.insert(num, XrefEntry::InFile { offset, generation }); objstms.push(num); - if find(&data, b"/XRef", body, body.saturating_add(1024)).is_some() + let end = headers.get(i + 1).and_then(|h| usize::try_from(h.1).ok()).unwrap_or(data.len()); + if may_be_object_stream(&data, body, end) { + candidates.insert(num); + } else { + candidates.remove(&num); + } + // `/XRef` within the 1024 bytes after the header (the first one that is there). + let first = xref_marks.partition_point(|&p| p < body); + if xref_marks.get(first).is_some_and(|&p| p + 5 <= body.saturating_add(1024).min(data.len())) && let Ok(at) = usize::try_from(offset) { xref_streams.push(at); @@ -2068,7 +2087,8 @@ impl Document { self.cache.lock().map_err(|_| CosError::Poisoned)?.clear(); // Objects inside object streams. for num in objstms { - if let Ok(o) = self.try_get(num) + if candidates.contains(&num) + && let Ok(o) = self.try_get(num) && let Object::Stream(s) = &*o && s.dict.name(b"Type") == Some(b"ObjStm") && let Ok(stm) = self.objstm(num) @@ -2165,6 +2185,24 @@ fn shifted_subsection_end(data: &[u8], at: usize) -> Option { } } +/// Whether the object whose body is `data[from..to]` could be an object stream: its dictionary +/// holds `ObjStm`, or a `#` (a name may hide its letters as `#xx`, as in `/Obj#53tm`). A stream's +/// data is not searched: it ends the dictionary, at the `stream` keyword after `>>`. +fn may_be_object_stream(data: &[u8], from: usize, to: usize) -> bool { + let Some(body) = data.get(from..to.min(data.len())) else { return false }; + let mut dict = body; + let mut at = 0; + while let Some(p) = find(body, b"stream", at, body.len()) { + at = p + 6; + let after_dict = body[..p].trim_ascii_end().ends_with(b">>"); + if after_dict && matches!(body.get(at), Some(b'\r' | b'\n')) { + dict = &body[..p]; + break; + } + } + dict.contains(&b'#') || find(dict, b"ObjStm", 0, dict.len()).is_some() +} + /// Every `N G obj` header in `data`, in file order, as (number, offset, generation, position /// after `obj`). fn object_headers(data: &[u8]) -> Vec<(u32, u64, u16, usize)> { @@ -2241,7 +2279,33 @@ pub(crate) fn find(hay: &[u8], needle: &[u8], from: usize, to: usize) -> Option< if from >= to || needle.is_empty() || to - from < needle.len() { return None; } - hay[from..to].windows(needle.len()).position(|w| w == needle).map(|p| p + from) + let hay = &hay[from..to]; + // Look for the needle's rarest byte first, a plain byte scan, and compare the rest only there. + const COMMON: &[u8] = b" etaoinsrhldcumfpgwybvkxjqz"; + let (k, rare) = + needle.iter().copied().enumerate().max_by_key(|&(_, b)| COMMON.iter().position(|&c| c == b).unwrap_or(COMMON.len())).unwrap_or((0, 0)); + let mut i = k; + while let Some(p) = hay.get(i..).and_then(|rest| position_of(rest, rare)) { + let start = i + p - k; + if hay.get(start..start + needle.len()) == Some(needle) { + return Some(start + from); + } + i += p + 1; + } + None +} + +/// The first position of `byte` in `hay`. Whole 32-byte blocks are tested without branching, which +/// the compiler turns into vector compares; only a block that holds the byte is searched. +fn position_of(hay: &[u8], byte: u8) -> Option { + const BLOCK: usize = 32; + let (blocks, rest) = hay.as_chunks::(); + for (i, block) in blocks.iter().enumerate() { + if block.iter().fold(false, |any, &b| any | (b == byte)) { + return block.iter().position(|&b| b == byte).map(|p| i * BLOCK + p); + } + } + rest.iter().position(|&b| b == byte).map(|p| blocks.len() * BLOCK + p) } fn rfind(hay: &[u8], needle: &[u8], from: usize) -> Option { diff --git a/crates/cos/tests/it/main.rs b/crates/cos/tests/it/main.rs index 52cf05d6c2..07883e7b5a 100644 --- a/crates/cos/tests/it/main.rs +++ b/crates/cos/tests/it/main.rs @@ -4,5 +4,6 @@ mod bounded_writer; mod corpus; mod encryption; mod proptests; +mod reconstruct; mod shared_streams; mod writer; diff --git a/crates/cos/tests/it/reconstruct.rs b/crates/cos/tests/it/reconstruct.rs new file mode 100644 index 0000000000..5f61867b1b --- /dev/null +++ b/crates/cos/tests/it/reconstruct.rs @@ -0,0 +1,57 @@ +//! Rebuilding the cross-reference index of a damaged file finds the objects inside object +//! streams however the stream's type is written. + +use std::sync::Arc; + +use pdfcraft_cos::{Document, Object}; + +/// A file whose `startxref` points nowhere, with three object streams (type written plainly, as +/// `/Obj#53tm`, and after a string that contains the word `stream`), and one ordinary stream +/// whose data mentions `ObjStm` and `#`. +fn damaged() -> Vec { + let mut out = b"%PDF-1.5\n".to_vec(); + let mut add = |n: u32, body: &[u8]| { + out.extend_from_slice(format!("{n} 0 obj\n").as_bytes()); + out.extend_from_slice(body); + out.extend_from_slice(b"\nendobj\n"); + }; + let objstm = |dict_prefix: &str, members: &[(u32, &str)]| { + let mut header = String::new(); + let mut body = String::new(); + for (n, text) in members { + header.push_str(&format!("{n} {} ", body.len())); + body.push_str(text); + body.push('\n'); + } + let mut b = + format!("<< {dict_prefix} /N {} /First {} /Length {} >>\nstream\n", members.len(), header.len(), header.len() + body.len()).into_bytes(); + b.extend_from_slice(header.as_bytes()); + b.extend_from_slice(body.as_bytes()); + b.extend_from_slice(b"\nendstream"); + b + }; + add(1, b"<< /Type /Catalog /Pages 2 0 R >>"); + add(2, b"<< /Type /Pages /Kids [] /Count 0 >>"); + add(3, &objstm("/Type /ObjStm", &[(10, "<< /From 3 >>"), (11, "[3]")])); + add(4, &objstm("/Type /Obj#53tm", &[(12, "<< /From 4 >>")])); + add(5, &objstm("/Note (a stream) /Type /ObjStm", &[(13, "<< /From 5 >>")])); + let data = b"/Type /ObjStm # 14 0 << /From 6 >>"; + let mut plain = format!("<< /Length {} >>\nstream\n", data.len()).into_bytes(); + plain.extend_from_slice(data); + plain.extend_from_slice(b"\nendstream"); + add(6, &plain); + out.extend_from_slice(b"trailer\n<< /Size 20 /Root 1 0 R >>\nstartxref\n99999999\n%%EOF\n"); + out +} + +#[test] +fn objects_in_object_streams_are_found_however_the_type_is_written() { + let doc = Document::open(Arc::new(damaged())).expect("opens"); + assert!(doc.reconstructed()); + for n in [10, 11, 12, 13] { + let o = doc.try_get(n).unwrap_or_else(|e| panic!("object {n}: {e}")); + assert!(matches!(&*o, Object::Dict(_) | Object::Array(_)), "object {n} is {o:?}"); + } + // The data of an ordinary stream is not an object stream, whatever it says. + assert!(!doc.object_numbers().contains(&14)); +} From a7baf8a2af97eeed166325a990bb116cdf3e1d9b Mon Sep 17 00:00:00 2001 From: vangel76 Date: Mon, 26 Oct 2026 21:59:12 +0100 Subject: [PATCH 016/198] perf(cos): write_full keeps the objects it parsed while numbering write_full parsed each object once to number it and again, after the 128-object reader cache had dropped it, to write it. Keep the parsed object from the traversal and write that, within a budget of 64 MiB of owned bytes and 2M tree nodes; past it the object is read again as before, which keeps the memory bound and the "read differently while saving" check. Fixes #4869 Co-Authored-By: Claude Sonnet 5.5 --- crates/cos/src/document.rs | 2 +- crates/cos/src/writer.rs | 89 +++++++++++++++++++++++++++++++++++--- 2 files changed, 84 insertions(+), 7 deletions(-) diff --git a/crates/cos/src/document.rs b/crates/cos/src/document.rs index 74b773e026..9dca286d7e 100644 --- a/crates/cos/src/document.rs +++ b/crates/cos/src/document.rs @@ -530,7 +530,7 @@ const OWNED_BYTES_DEPTH: usize = 256; /// Bytes `object` holds of its own: string and stream data that is not a view into `data` /// (the input file). An object nested deeper than [`OWNED_BYTES_DEPTH`] cannot be measured /// and counts as too large to keep. -fn owned_bytes(object: &Object, data: &Arc>, depth: usize) -> usize { +pub(crate) fn owned_bytes(object: &Object, data: &Arc>, depth: usize) -> usize { if depth > OWNED_BYTES_DEPTH { return usize::MAX; } diff --git a/crates/cos/src/writer.rs b/crates/cos/src/writer.rs index 627f716495..1ba4574318 100644 --- a/crates/cos/src/writer.rs +++ b/crates/cos/src/writer.rs @@ -15,7 +15,7 @@ use std::io::Write; use std::sync::Arc; use crate::CosError; -use crate::document::{Document, ObjectReader, XrefEntry}; +use crate::document::{Document, ObjectReader, XrefEntry, owned_bytes}; use crate::object::{Dict, ObjRef, Object, PdfString, Stream}; /// Serialize one object. Deterministic: same object → same bytes. @@ -280,6 +280,10 @@ pub fn write_full(doc: &Document, opts: &SaveOptions) -> Result, CosErro let mut map: HashMap = HashMap::new(); // Record the type while traversing so classifying packed objects needs no second parse. let mut order: Vec<(ObjRef, bool)> = Vec::new(); + // The parsed object of each entry in `order`, kept for the write pass while the budget + // allows it. Past the budget the write pass reads the object again. + let mut kept: Vec>> = Vec::new(); + let mut kept_budget = KeptObjects::default(); let mut queue: VecDeque = VecDeque::new(); let visit_refs = |o: &Object, queue: &mut VecDeque| collect_refs(o, &mut |r| queue.push_back(r)); for key in [&b"Root"[..], b"Info", b"Encrypt"] { @@ -305,6 +309,7 @@ pub fn write_full(doc: &Document, opts: &SaveOptions) -> Result, CosErro map.insert(r, output_number(order.len())?); order.push((r, matches!(*o, Object::Stream(_)))); visit_refs(&o, &mut queue); + kept.push(kept_budget.admit(&o, reader.document.bytes()).then_some(o)); } let version = reader.document.version().chars().take(3).collect::(); let version = if opts.object_streams && version.as_str() < "1.5" { "1.5".to_string() } else { version }; @@ -329,14 +334,14 @@ pub fn write_full(doc: &Document, opts: &SaveOptions) -> Result, CosErro let mut rows: BTreeMap = BTreeMap::new(); // Keep only references, not renamed copies of the entire object graph. Standalone // objects still precede object streams, preserving the established output order. - let mut packed: Vec<(u32, ObjRef)> = Vec::new(); + let mut packed: Vec<(u32, ObjRef, Option>)> = Vec::new(); for (i, (r, is_stream)) in order.iter().enumerate() { let num = output_number(i)?; let in_stream = opts.object_streams && !is_stream && !encryption_objects.contains(r) && root_outside != Some(*r); if in_stream { - packed.push((num, *r)); + packed.push((num, *r, kept[i].take())); } else { - let o = renumber(&*reread(&mut reader, *r, *is_stream)?, &map); + let o = renumber(&*written_object(&mut reader, *r, *is_stream, kept[i].take())?, &map); rows.insert(num, Row::InFile(out.len() as u64, 0)); write_indirect(num, 0, &prepared(&reader.document, r.num, num, 0, &o), &mut out); } @@ -345,9 +350,9 @@ pub fn write_full(doc: &Document, opts: &SaveOptions) -> Result, CosErro // individually: the whole stream is (§7.5.7, §7.6.2). let mut next = output_number(order.len())?; let mut batch = ObjectStreamBatch::default(); - for (num, reference) in packed { + for (num, reference, parsed) in packed { let mut encoded = Vec::new(); - serialize(&renumber(&*reread(&mut reader, reference, false)?, &map), &mut encoded); + serialize(&renumber(&*written_object(&mut reader, reference, false, parsed)?, &map), &mut encoded); if batch.count > 0 && batch.body.len().saturating_add(encoded.len()).saturating_add(1) > OBJSTM_BYTES { batch.flush(&reader.document, &mut next, &mut rows, &mut out)?; } @@ -400,6 +405,56 @@ pub fn write_full(doc: &Document, opts: &SaveOptions) -> Result, CosErro Ok(out) } +/// What `write_full` keeps of the objects it parsed while numbering them: owned bytes (see +/// [`owned_bytes`]; streams read from the file are views and cost only their dictionary) and +/// tree nodes. Beyond either limit objects are read again when written, which bounds the memory +/// of a save of a huge document. +#[derive(Default)] +struct KeptObjects { + bytes: usize, + nodes: usize, +} + +impl KeptObjects { + const BYTES: usize = 64 * 1024 * 1024; + const NODES: usize = 2_000_000; + + /// Whether `object` still fits the budget; if so it is counted against it. + fn admit(&mut self, object: &Object, data: &Arc>) -> bool { + let bytes = self.bytes.saturating_add(owned_bytes(object, data, 0)); + let nodes = self.nodes.saturating_add(node_count(object, 0)); + if bytes > Self::BYTES || nodes > Self::NODES { + return false; + } + (self.bytes, self.nodes) = (bytes, nodes); + true + } +} + +/// Values in `object`'s tree, counting the object itself. Past the parser's nesting limit +/// the count is `usize::MAX`, so nothing that deep is kept. +fn node_count(object: &Object, depth: usize) -> usize { + if depth > 256 { + return usize::MAX; + } + let children = match object { + Object::Stream(s) => s.dict.iter().fold(0usize, |n, (_, v)| n.saturating_add(node_count(v, depth + 1))), + Object::Array(a) => a.iter().fold(0usize, |n, v| n.saturating_add(node_count(v, depth + 1))), + Object::Dict(d) => d.iter().fold(0usize, |n, (_, v)| n.saturating_add(node_count(v, depth + 1))), + _ => 0, + }; + children.saturating_add(1) +} + +/// The object to write for `r`: the one parsed while numbering when it was kept, else `r` read +/// again. +fn written_object(reader: &mut ObjectReader, r: ObjRef, is_stream: bool, kept: Option>) -> Result, CosError> { + match kept { + Some(o) => Ok(o), + None => reread(reader, r, is_stream), + } +} + /// Object `r` read again for writing. The traversal read it once and only kept its number, so /// a second read that fails, finds nothing or finds a different kind of object must fail the /// save rather than write `null` in its place. @@ -674,6 +729,28 @@ mod tests { Lexer::new(&out, 0).object().expect("parses") } + #[test] + fn kept_objects_stop_at_their_byte_and_node_budget() { + let data = Arc::new(Vec::new()); + let string = Object::String(PdfString::literal(vec![b'x'; 1000])); + let array = Object::Array(vec![Object::Int(1), Object::Null]); + let mut kept = KeptObjects::default(); + assert!(kept.admit(&string, &data) && kept.admit(&array, &data)); + assert_eq!((kept.bytes, kept.nodes), (1000, 4)); + let mut full = KeptObjects { bytes: KeptObjects::BYTES - 999, nodes: 0 }; + assert!(!full.admit(&string, &data), "one byte over the byte budget"); + assert_eq!((full.bytes, full.nodes), (KeptObjects::BYTES - 999, 0), "a refused object is not counted"); + assert!(full.admit(&Object::Int(1), &data)); + let mut full = KeptObjects { bytes: 0, nodes: KeptObjects::NODES - 2 }; + assert!(!full.admit(&array, &data), "a node past the node budget"); + assert!(full.admit(&Object::Int(1), &data)); + let mut deep = Object::Null; + for _ in 0..300 { + deep = Object::Array(vec![deep]); + } + assert!(!KeptObjects::default().admit(&deep, &data), "nesting past the limit is never kept"); + } + #[test] fn a_second_read_that_differs_fails_the_save_instead_of_writing_null() { let mut doc = Document::new_empty(); From 929d350f1dfacb6da308569f2a7de8d412a4049a Mon Sep 17 00:00:00 2001 From: vangel76 Date: Mon, 26 Oct 2026 22:00:43 +0100 Subject: [PATCH 017/198] perf(cos): write hex strings, integers and references without the formatter serialize wrote every byte of a hex string, every integer and every object number through write!. A table lookup and a small digit loop give the same bytes in a fraction of the instructions. The octal escape in literal strings and the #xx escape in names use the same helpers. Fixes #4870 Co-Authored-By: Claude Sonnet 5.5 --- crates/cos/src/writer.rs | 53 ++++++++++-- crates/cos/tests/it/main.rs | 1 + crates/cos/tests/it/serialize_format.rs | 110 ++++++++++++++++++++++++ 3 files changed, 155 insertions(+), 9 deletions(-) create mode 100644 crates/cos/tests/it/serialize_format.rs diff --git a/crates/cos/src/writer.rs b/crates/cos/src/writer.rs index 627f716495..b8a9c53e9c 100644 --- a/crates/cos/src/writer.rs +++ b/crates/cos/src/writer.rs @@ -23,9 +23,7 @@ pub fn serialize(o: &Object, out: &mut Vec) { match o { Object::Null => out.extend_from_slice(b"null"), Object::Bool(b) => out.extend_from_slice(if *b { b"true" } else { b"false" }), - Object::Int(i) => { - let _ = write!(out, "{i}"); - } + Object::Int(i) => write_int(*i, out), Object::Real(r) => write_real(*r, out), Object::String(s) => write_string(s, out), Object::Name(n) => write_name(n, out), @@ -42,11 +40,46 @@ pub fn serialize(o: &Object, out: &mut Vec) { Object::Dict(d) => write_dict(d, out), Object::Stream(s) => write_stream(s, out), Object::Ref(r) => { - let _ = write!(out, "{} {} R", r.num, r.generation); + write_digits(u64::from(r.num), out); + out.push(b' '); + write_digits(u64::from(r.generation), out); + out.extend_from_slice(b" R"); } } } +/// Upper-case hexadecimal digits. +const HEX: &[u8; 16] = b"0123456789ABCDEF"; + +/// The decimal digits of `v`, the same bytes `write!(out, "{v}")` gives, without the formatter. +fn write_digits(mut v: u64, out: &mut Vec) { + // `u64::MAX` has 20 digits, so `at` never goes below 0. + let mut buf = [0u8; 20]; + let mut at = buf.len(); + loop { + at -= 1; + buf[at] = b'0' + (v % 10) as u8; + v /= 10; + if v == 0 { + break; + } + } + out.extend_from_slice(&buf[at..]); +} + +/// A signed integer in decimal, as `write!(out, "{i}")` writes it. +fn write_int(i: i64, out: &mut Vec) { + if i < 0 { + out.push(b'-'); + } + write_digits(i.unsigned_abs(), out); +} + +/// `b` as two upper-case hexadecimal digits. +fn hex_pair(b: u8) -> [u8; 2] { + [HEX[usize::from(b >> 4)], HEX[usize::from(b & 15)]] +} + fn write_real(r: f64, out: &mut Vec) { if !r.is_finite() { out.push(b'0'); @@ -55,7 +88,7 @@ fn write_real(r: f64, out: &mut Vec) { // A whole number is written as an integer only within the integer range Annex C gives // (±2³¹−1), so `4e9` or `1e30` stays a real token for readers with 32-bit integers. if r.fract() == 0.0 && r.abs() <= f64::from(i32::MAX) { - let _ = write!(out, "{}", r as i32); + write_int(i64::from(r as i32), out); return; } // Plain decimal notation (§7.3.3 has no exponents). Rust's `Display` for f64 prints the @@ -76,7 +109,8 @@ fn write_name(n: &[u8], out: &mut Vec) { if regular { out.push(b); } else { - let _ = write!(out, "#{b:02X}"); + out.push(b'#'); + out.extend_from_slice(&hex_pair(b)); } } } @@ -84,9 +118,10 @@ fn write_name(n: &[u8], out: &mut Vec) { fn write_string(s: &PdfString, out: &mut Vec) { let binary = s.bytes.iter().filter(|b| !(b' '..=b'~').contains(*b) && !matches!(b, b'\n' | b'\r' | b'\t')).count(); if s.hex || binary * 4 > s.bytes.len().max(1) { + out.reserve(s.bytes.len().saturating_mul(2).saturating_add(2)); out.push(b'<'); - for b in &s.bytes { - let _ = write!(out, "{b:02X}"); + for &b in &s.bytes { + out.extend_from_slice(&hex_pair(b)); } out.push(b'>'); return; @@ -105,7 +140,7 @@ fn write_string(s: &PdfString, out: &mut Vec) { 0x0C => out.extend_from_slice(b"\\f"), b' '..=b'~' => out.push(b), _ => { - let _ = write!(out, "\\{b:03o}"); + out.extend_from_slice(&[b'\\', b'0' + (b >> 6), b'0' + ((b >> 3) & 7), b'0' + (b & 7)]); } } } diff --git a/crates/cos/tests/it/main.rs b/crates/cos/tests/it/main.rs index 52cf05d6c2..4b2fe32560 100644 --- a/crates/cos/tests/it/main.rs +++ b/crates/cos/tests/it/main.rs @@ -4,5 +4,6 @@ mod bounded_writer; mod corpus; mod encryption; mod proptests; +mod serialize_format; mod shared_streams; mod writer; diff --git a/crates/cos/tests/it/serialize_format.rs b/crates/cos/tests/it/serialize_format.rs new file mode 100644 index 0000000000..19781a802f --- /dev/null +++ b/crates/cos/tests/it/serialize_format.rs @@ -0,0 +1,110 @@ +//! `serialize` writes integers, references, hexadecimal strings, octal escapes and `#xx` name +//! escapes with its own digit loops. They must give the bytes the standard formatter gives. + +use pdfcraft_cos::{ObjRef, Object, PdfString, serialize}; +use proptest::prelude::*; + +fn bytes_of(o: &Object) -> Vec { + let mut v = Vec::new(); + serialize(o, &mut v); + v +} + +#[test] +fn integer_extremes_and_digit_boundaries() { + let mut cases = vec![0i64, 1, -1, 9, 10, 99, 100, i64::MAX, i64::MIN, i64::MIN + 1, i64::from(i32::MAX), i64::from(i32::MIN)]; + let mut p = 1i64; + while let Some(next) = p.checked_mul(10) { + cases.extend([next - 1, next, next + 1, -next]); + p = next; + } + for i in cases { + assert_eq!(bytes_of(&Object::Int(i)), i.to_string().into_bytes(), "{i}"); + } +} + +#[test] +fn reference_extremes() { + for (n, g) in [(0, 0), (1, 0), (u32::MAX, u16::MAX), (10, 65535), (4_000_000_000, 7)] { + assert_eq!(bytes_of(&Object::Ref(ObjRef::new(n, g))), format!("{n} {g} R").into_bytes()); + } +} + +#[test] +fn every_byte_value_in_hex_strings_octal_escapes_and_names() { + let all: Vec = (0..=255).collect(); + let hex: String = all.iter().map(|b| format!("{b:02X}")).collect(); + assert_eq!(bytes_of(&Object::String(PdfString { bytes: all.clone(), hex: true })), format!("<{hex}>").into_bytes()); + // A literal string with a few control bytes escapes them in octal. + let lit = bytes_of(&Object::String(PdfString { bytes: b"abcdefghijkl\x01\x7f\xffz".to_vec(), hex: false })); + assert_eq!(lit, b"(abcdefghijkl\\001\\177\\377z)"); + let name = bytes_of(&Object::Name(all.clone())); + let mut want = b"/".to_vec(); + for &b in &all { + let regular = (b'!'..=b'~').contains(&b) && !matches!(b, b'(' | b')' | b'<' | b'>' | b'[' | b']' | b'{' | b'}' | b'/' | b'%' | b'#'); + if regular { + want.push(b); + } else { + want.extend_from_slice(format!("#{b:02X}").as_bytes()); + } + } + assert_eq!(name, want); +} + +proptest! { + #[test] + fn integers_match_the_formatter(i in any::()) { + prop_assert_eq!(bytes_of(&Object::Int(i)), format!("{i}").into_bytes()); + } + + #[test] + fn whole_reals_match_the_formatter(i in any::()) { + prop_assert_eq!(bytes_of(&Object::Real(f64::from(i))), format!("{i}").into_bytes()); + } + + #[test] + fn references_match_the_formatter(n in any::(), g in any::()) { + prop_assert_eq!(bytes_of(&Object::Ref(ObjRef::new(n, g))), format!("{n} {g} R").into_bytes()); + } + + #[test] + fn hex_strings_match_the_formatter(bytes in proptest::collection::vec(any::(), 0..300)) { + let mut want = String::from("<"); + for b in &bytes { + want.push_str(&format!("{b:02X}")); + } + want.push('>'); + prop_assert_eq!(bytes_of(&Object::String(PdfString { bytes, hex: true })), want.into_bytes()); + } + + #[test] + fn binary_literal_strings_match_the_formatter(bytes in proptest::collection::vec(any::(), 0..300)) { + // Mostly-binary input is written as hex; mostly-text input with escapes is a literal. + let got = bytes_of(&Object::String(PdfString { bytes: bytes.clone(), hex: false })); + let binary = bytes.iter().filter(|b| !(b' '..=b'~').contains(*b) && !matches!(b, b'\n' | b'\r' | b'\t')).count(); + let mut want = Vec::new(); + if binary * 4 > bytes.len().max(1) { + want.push(b'<'); + for b in &bytes { + want.extend_from_slice(format!("{b:02X}").as_bytes()); + } + want.push(b'>'); + } else { + want.push(b'('); + for &b in &bytes { + match b { + b'(' | b')' | b'\\' => want.extend_from_slice(&[b'\\', b]), + b'\n' => want.extend_from_slice(b"\\n"), + b'\r' => want.extend_from_slice(b"\\r"), + b'\t' => want.extend_from_slice(b"\\t"), + 0x08 => want.extend_from_slice(b"\\b"), + 0x0C => want.extend_from_slice(b"\\f"), + b' '..=b'~' => want.push(b), + _ => want.extend_from_slice(format!("\\{b:03o}").as_bytes()), + } + } + want.push(b')'); + } + prop_assert_eq!(got, want); + } +} From d04fad3acd79906445a47e46496fb09fbfffdc11 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 27 Oct 2026 19:45:53 +0100 Subject: [PATCH 018/198] fix(cos): recount the scan reader's cache when a read replaces entries The running total trusted cache length and pointer equality to mean that a read only added the requested object. A read whose nested loads made the editor cache evict (it then keeps only the newcomer) leaves the length unchanged, so a large string stayed cached past the 8 MiB budget. Count the inserts a read makes instead; anything but a hit or exactly one insert measures the whole cache again. Co-Authored-By: Claude Sonnet 5.5 --- crates/cos/src/document.rs | 41 ++++++++++++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/crates/cos/src/document.rs b/crates/cos/src/document.rs index 74b773e026..b7fe54bc19 100644 --- a/crates/cos/src/document.rs +++ b/crates/cos/src/document.rs @@ -21,6 +21,13 @@ thread_local! { static LENGTHS: std::cell::RefCell>> = std::cell::RefCell::new(HashMap::new()); } +thread_local! { + /// Objects this thread has put into a document's parsed-object cache. A reader compares + /// it before and after a read: a read that inserted exactly one object cannot have + /// evicted or replaced anything else, whatever the cache length says. + static CACHE_INSERTS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + /// Marks an object as being loaded on this thread. Loading can re-enter (indirect stream /// `/Length`, object streams): an object already being loaded is a cycle, and the nesting is /// bounded as well. @@ -460,7 +467,9 @@ impl ObjectReader { } pub(crate) fn try_get(&mut self, reference: ObjRef) -> Result, CosError> { + let inserts_before = CACHE_INSERTS.with(|c| c.get()); let object = self.document.try_get(reference.num); + let inserted = CACHE_INSERTS.with(|c| c.get()).wrapping_sub(inserts_before); // A missed eviction here only costs a re-parse, so never fail the read on a poisoned // lock. The editing-cache evictors propagate instead; they guard user edits. if let Ok(mut streams) = self.document.objstms.lock() { @@ -492,13 +501,16 @@ impl ObjectReader { // have cached large objects. if let Ok(mut cache) = self.document.cache.lock() { let data = &self.document.data; - // The common read parsed exactly this object and nothing else (a hit changes - // nothing): add its size to the running total. Anything else, such as a failed - // load, a /Length that cached further objects or a cache changed from outside, - // measures the whole cache again. - let grew_by_this_object = cache.len() == self.counted_objects.saturating_add(1) + // A hit inserted nothing; a read that inserted exactly this object (no nested + // /Length or object stream load) added only it. Cache length and pointer equality + // alone cannot tell that from a read that evicted and replaced entries (a large + // nested object makes the editor cache keep only the newcomer), so count the + // inserts. Anything else, such as a failed load, a /Length that cached further + // objects or a cache changed from outside, measures the whole cache again. + let grew_by_this_object = inserted == 1 + && cache.len() == self.counted_objects.saturating_add(1) && matches!((&object, cache.get(&reference.num)), (Ok(read), Some(cached)) if Arc::ptr_eq(read, cached)); - if object.is_ok() && cache.len() == self.counted_objects { + if object.is_ok() && inserted == 0 && cache.len() == self.counted_objects { // Unchanged: a hit, or an object held by the overlay. } else if let (true, Ok(read)) = (grew_by_this_object, &object) { self.counted_bytes = self.counted_bytes.saturating_add(owned_bytes(read, data, 0)); @@ -1075,6 +1087,7 @@ impl Document { // The editing cache is bounded like the scan reader's: re-parsing a file-backed view // is cheap, and returned objects outlive eviction. self.cache.lock().map_err(|_| CosError::Poisoned)?.insert(num, obj.clone()); + CACHE_INSERTS.with(|c| c.set(c.get().wrapping_add(1))); self.evict_editor_cache_if_over_budget(num, &obj)?; Ok(obj) } @@ -3046,6 +3059,22 @@ endobj } } + #[test] + fn temporary_reader_recounts_when_a_read_replaces_cache_entries() { + // One cached object, then a read of a string over the editor cache's byte budget: the + // editor eviction keeps only the newcomer, so the cache length stays at one. The + // string is data of its own and must not stay cached past the reader's budget. + let big = format!("({})", "a".repeat(Document::EDITOR_CACHE_BYTES + 1)); + let bytes = build(&["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", "<< /Small 1 >>", &big], "/Root 1 0 R"); + let doc = Document::open(Arc::new(bytes)).unwrap(); + let mut reader = doc.object_reader(); + let _ = reader.get(ObjRef::new(3, 0)); + assert_eq!((reader.document.cache.lock().unwrap().len(), reader.counted_objects), (1, 1)); + assert!(matches!(&*reader.get(ObjRef::new(4, 0)), Object::String(_))); + assert!(reader.document.cache.lock().unwrap().is_empty(), "a replaced entry must be counted: the string is over budget"); + assert_eq!((reader.counted_objects, reader.counted_bytes), (0, 0)); + } + #[test] fn temporary_reader_and_full_save_do_not_fill_snapshot_caches() { let mut source = Document::new_empty(); From 8bdd2d37504689fbff50eb198740d384a6a2c054 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Tue, 27 Oct 2026 19:46:44 +0100 Subject: [PATCH 019/198] fix(cos): read the dictionary's syntax before the object-stream pre-check The pre-check cut the dictionary at the next object header or at a '>>' + 'stream' byte pattern, so text inside a string could hide a later /Type /ObjStm. It now reads the dictionary as the parser does (strings, hex strings, comments, nesting) and treats an object whose end is not certain as a candidate, so the reconstructed object set stays the same. Co-Authored-By: Claude Sonnet 5.5 --- crates/cos/src/document.rs | 173 ++++++++++++++++++++++++++--- crates/cos/tests/it/reconstruct.rs | 7 +- 2 files changed, 162 insertions(+), 18 deletions(-) diff --git a/crates/cos/src/document.rs b/crates/cos/src/document.rs index 4ad24a11b1..ffc4f6be72 100644 --- a/crates/cos/src/document.rs +++ b/crates/cos/src/document.rs @@ -2063,11 +2063,10 @@ impl Document { xref_marks.push(p); mark_from = p + 1; } - for (i, &(num, offset, generation, body)) in headers.iter().enumerate() { + for &(num, offset, generation, body) in &headers { entries.insert(num, XrefEntry::InFile { offset, generation }); objstms.push(num); - let end = headers.get(i + 1).and_then(|h| usize::try_from(h.1).ok()).unwrap_or(data.len()); - if may_be_object_stream(&data, body, end) { + if may_be_object_stream(&data, body) { candidates.insert(num); } else { candidates.remove(&num); @@ -2185,22 +2184,107 @@ fn shifted_subsection_end(data: &[u8], at: usize) -> Option { } } -/// Whether the object whose body is `data[from..to]` could be an object stream: its dictionary -/// holds `ObjStm`, or a `#` (a name may hide its letters as `#xx`, as in `/Obj#53tm`). A stream's -/// data is not searched: it ends the dictionary, at the `stream` keyword after `>>`. -fn may_be_object_stream(data: &[u8], from: usize, to: usize) -> bool { - let Some(body) = data.get(from..to.min(data.len())) else { return false }; - let mut dict = body; - let mut at = 0; - while let Some(p) = find(body, b"stream", at, body.len()) { - at = p + 6; - let after_dict = body[..p].trim_ascii_end().ends_with(b">>"); - if after_dict && matches!(body.get(at), Some(b'\r' | b'\n')) { - dict = &body[..p]; - break; +/// Longest dictionary [`may_be_object_stream`] reads before it gives up and says "maybe". +const DICTIONARY_SCAN_LIMIT: usize = 1 << 20; + +/// Where the dictionary that starts at `data[from..]` (after `N G obj`) ends, reading it as the +/// parser does: strings, hex strings, comments, names and nested dictionaries. `Some(None)`: the +/// object is not a dictionary, so it is never a stream. `None`: the end is not certain (no end +/// within the limit, a hex string with a bad digit, a `>>` inside an array); the caller must +/// then treat the object as a possible object stream. +fn dictionary_end(data: &[u8], from: usize) -> Option> { + let limit = data.len().min(from.saturating_add(DICTIONARY_SCAN_LIMIT)); + let mut i = from; + while i < limit && (is_whitespace(data[i]) || data[i] == b'%') { + if data[i] == b'%' { + while i < limit && !matches!(data[i], b'\n' | b'\r') { + i += 1; + } + } else { + i += 1; + } + } + if data.get(i..i + 2) != Some(b"<<") { + return if i < limit { Some(None) } else { None }; + } + i += 2; + let (mut dicts, mut arrays) = (1usize, 0usize); + while i < limit { + match data[i] { + b'(' => { + let mut depth = 1usize; + i += 1; + while i < limit && depth > 0 { + match data[i] { + b'\\' => i += 1, + b'(' => depth += 1, + b')' => depth -= 1, + _ => {} + } + i += 1; + } + if depth > 0 { + return None; + } + } + b'<' if data.get(i + 1) == Some(&b'<') => { + dicts += 1; + i += 2; + } + b'<' => { + i += 1; + while i < limit && data[i] != b'>' { + if !data[i].is_ascii_hexdigit() && !is_whitespace(data[i]) { + return None; + } + i += 1; + } + i += 1; + } + b'>' if data.get(i + 1) == Some(&b'>') => { + if arrays > 0 { + return None; + } + dicts -= 1; + i += 2; + if dicts == 0 { + return Some(Some(i)); + } + } + b'%' => { + while i < limit && !matches!(data[i], b'\n' | b'\r') { + i += 1; + } + } + b'[' => { + arrays += 1; + i += 1; + } + b']' => { + arrays = arrays.saturating_sub(1); + i += 1; + } + _ => i += 1, + } + } + None +} + +/// Whether the object whose body starts at `data[from..]` could be an object stream: its +/// dictionary holds `ObjStm`, or a `#` (a name may hide its letters as `#xx`, as in +/// `/Obj#53tm`). Only the dictionary is searched, not the data of a stream. Where the +/// dictionary ends is read by [`dictionary_end`], so text inside a string (an object header, a +/// `stream` keyword) is not taken for syntax; when that is not certain the object counts as a +/// candidate, so a doubtful object is parsed as it always was. +fn may_be_object_stream(data: &[u8], from: usize) -> bool { + match dictionary_end(data, from) { + Some(None) => false, + None => true, + Some(Some(end)) => { + let dict = &data[from..end]; + dict.contains(&b'#') || find(dict, b"ObjStm", 0, dict.len()).is_some() } } - dict.contains(&b'#') || find(dict, b"ObjStm", 0, dict.len()).is_some() } /// Every `N G obj` header in `data`, in file order, as (number, offset, generation, position @@ -3584,6 +3668,61 @@ endobj /// cross-reference data, as in #2365), is rebuilt from its object headers. The rebuild used /// to look only for the `trailer` keyword, so /Info and /ID were lost, and a full save after /// editing one Info field dropped every other. + /// Whether parsing `body` as an object gives a stream whose /Type is /ObjStm. + fn parses_as_object_stream(file: &[u8]) -> bool { + matches!(parse_indirect(file, 0, &|_| None), Ok((_, Object::Stream(s))) if s.dict.name(b"Type") == Some(b"ObjStm")) + } + + #[test] + fn the_object_stream_pre_check_never_rejects_what_parses_as_one() { + // Two inputs from review: header-shaped text, and `>>` + `stream` + newline, inside a + // string that comes before the /Type. + for dict in [ + "<< /Note (9 0 obj\n) /Type /ObjStm /N 0 /First 0 >>", + "<< /Note (>>stream\n) /Type /ObjStm /N 0 /First 0 >>", + "<< /Note (a (nested\n9 0 obj\nstream) \\) )) /Type /ObjStm >>", + "<< /Note <3e3e> /Other << /A (>>\nstream\n) >> /Type /ObjStm >>", + ] { + let file = format!("1 0 obj\n{dict}\nstream\nxx\nendstream\nendobj\n").into_bytes(); + assert!(parses_as_object_stream(&file), "test input must be an object stream: {dict}"); + assert!(may_be_object_stream(&file, 7), "{dict}"); + } + // Ordinary streams stay out; their data is never read. + let plain = b"1 0 obj\n<< /Length 9 >>\nstream\nObjStm # \nendstream\nendobj\n"; + assert!(!may_be_object_stream(plain, 7)); + assert!(!may_be_object_stream(b"1 0 obj\n[ /ObjStm ]\nendobj\n", 7)); + // Unterminated: not certain, so a candidate. + assert!(may_be_object_stream(b"1 0 obj\n<< /Note (open ", 7)); + // Random dictionaries built from tricky pieces: whatever parses as an object stream is + // a candidate. + let pieces: [&str; 22] = [ + "/Type /ObjStm", "/Type/ObjStm", "/Type /Obj#53tm", "/T#79pe /ObjStm", "(", ")", "\\", "(>>stream\n)", "(9 0 obj\n)", "<3e3e>", "<", ">", ">>", + "<<", "[", "]", "% c\n", "% >>\n", "/N 0", " ", "\n", "", + ]; + let mut seed = 0x2545_F491_4F6C_DD1Du64; + let mut next = move || { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + seed + }; + let mut hits = 0; + for _ in 0..200_000 { + let mut dict = String::from("<<"); + for _ in 0..(next() % 9) { + dict.push_str(pieces[(next() % pieces.len() as u64) as usize]); + dict.push(' '); + } + dict.push_str(if next() % 8 == 0 { "" } else { ">>" }); + let file = format!("1 0 obj\n{dict}\nstream\nxx\nendstream\nendobj\n").into_bytes(); + if parses_as_object_stream(&file) { + hits += 1; + assert!(may_be_object_stream(&file, 7), "rejected an object stream: {dict:?}"); + } + } + assert!(hits > 1000, "the generator must produce object streams ({hits})"); + } + #[test] fn reconstruction_takes_the_trailer_from_a_cross_reference_stream() { let table = build( diff --git a/crates/cos/tests/it/reconstruct.rs b/crates/cos/tests/it/reconstruct.rs index 5f61867b1b..4611c0d17e 100644 --- a/crates/cos/tests/it/reconstruct.rs +++ b/crates/cos/tests/it/reconstruct.rs @@ -40,6 +40,11 @@ fn damaged() -> Vec { plain.extend_from_slice(data); plain.extend_from_slice(b"\nendstream"); add(6, &plain); + // Strings before the type that look like an object header, or like the end of the + // dictionary and the `stream` keyword, and a hex string and a comment. + add(7, &objstm("/Note (\n99 0 obj\n) /Type /ObjStm", &[(15, "<< /From 7 >>")])); + add(8, &objstm("/Note (>>stream\n) /Type /ObjStm", &[(16, "<< /From 8 >>")])); + add(9, &objstm("/Note <3e3e> /Other << /A (a (b) \\) >>\nstream\n) >> % >>\nstream\n\n/Type /ObjStm", &[(17, "<< /From 9 >>")])); out.extend_from_slice(b"trailer\n<< /Size 20 /Root 1 0 R >>\nstartxref\n99999999\n%%EOF\n"); out } @@ -48,7 +53,7 @@ fn damaged() -> Vec { fn objects_in_object_streams_are_found_however_the_type_is_written() { let doc = Document::open(Arc::new(damaged())).expect("opens"); assert!(doc.reconstructed()); - for n in [10, 11, 12, 13] { + for n in [10, 11, 12, 13, 15, 16, 17] { let o = doc.try_get(n).unwrap_or_else(|e| panic!("object {n}: {e}")); assert!(matches!(&*o, Object::Dict(_) | Object::Array(_)), "object {n} is {o:?}"); } From 308d22cbadc84e45102c23b97d95205dbe81c7b4 Mon Sep 17 00:00:00 2001 From: huppiflupp Date: Wed, 28 Oct 2026 09:56:22 +0100 Subject: [PATCH 020/198] M10: Enhance scanned file cleans scanned pages in place MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Scan & OCR ▸ Enhance scanned file (Acrobat's Enhance ▸ Scanned document) now cleans up scanned pages: background removal (the camera clean-up's level 0-100), edge shadow removal (book gutters, copier borders) and despeckle, each its own switch. The camera tool renders the page and replaces it with a picture, which would drop a scan's invisible OCR text, comments and links. Instead each scan image (pictures covering at least half the page, each at least a tenth) is decoded, cleaned and written back into its own image object at the same size and colour space: a JPEG stays a JPEG (quality 90), other grey/RGB images are stored losslessly, a 1-bit image stays 1-bit with its decode array, and every other dictionary entry (SMask, Interpolate, ...) is kept. An image several pages share is cleaned once. What can't be cleaned (CMYK, indexed, JPEG 2000, JBIG2, CCITT, colour-key masks, broken data, oversized images) is reported, never skipped silently; pages that are no scan are listed too. The engine works as a job like Optimize (scan_clean_job, run on a worker with progress and cancel, apply_scan_clean refuses a result for an older document state) and applies one undoable step, "Enhance scanned file". pdfcraft-optimize exposes decode_samples/encode_samples for in-place pixel edits. camera_clean gains clean_samples for grey and RGB samples with separate switches and an edge-shadow pass; the camera tool's output is unchanged. page_enhance_scan drives it headlessly. Not yet an option of Recognize text. Refs #4119. Co-Authored-By: Claude Opus 5.5 --- ROADMAP.md | 2 + crates/automation/src/dispatch.rs | 29 ++ crates/automation/src/tools.rs | 12 + crates/automation/tests/it/automation.rs | 56 +++ crates/engine/src/camera_clean.rs | 235 ++++++++++-- crates/engine/src/catalog.rs | 2 +- crates/engine/src/commands.rs | 1 + crates/engine/src/edit.rs | 7 + crates/engine/src/edit/labels.rs | 2 + crates/engine/src/edit/run.rs | 2 + crates/engine/src/history.rs | 1 + crates/engine/src/lib.rs | 1 + crates/engine/src/scan_clean.rs | 276 ++++++++++++++ crates/engine/src/scan_clean/tests.rs | 338 ++++++++++++++++++ crates/engine/src/scan_fields.rs | 23 +- crates/optimize/src/images.rs | 10 +- crates/optimize/src/lib.rs | 2 + crates/optimize/src/samples.rs | 273 ++++++++++++++ crates/ui-egui/src/app.rs | 3 + crates/ui-egui/src/app/control.rs | 1 + crates/ui-egui/src/app/frame.rs | 1 + crates/ui-egui/src/commands.rs | 1 + crates/ui-egui/src/dialogs.rs | 10 + crates/ui-egui/src/i18n/de.tsv | 19 + crates/ui-egui/src/lib.rs | 4 + crates/ui-egui/src/state.rs | 2 + crates/ui-egui/src/workflows.rs | 1 + crates/ui-egui/src/workflows/scan_clean_ui.rs | 253 +++++++++++++ crates/ui-egui/tests/it/main.rs | 1 + crates/ui-egui/tests/it/scan_clean.rs | 77 ++++ docs/gaps.md | 5 +- docs/roadmap.md | 3 +- docs/target-app-parity.md | 3 +- parity/acrobat-features.toml | 24 +- 34 files changed, 1613 insertions(+), 67 deletions(-) create mode 100644 crates/engine/src/scan_clean.rs create mode 100644 crates/engine/src/scan_clean/tests.rs create mode 100644 crates/optimize/src/samples.rs create mode 100644 crates/ui-egui/src/workflows/scan_clean_ui.rs create mode 100644 crates/ui-egui/tests/it/scan_clean.rs diff --git a/ROADMAP.md b/ROADMAP.md index f5de8044e1..91e8bf1ece 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -2,6 +2,7 @@ **Stage: alpha** · next: beta, ~25% (ready for real work 50% → 75%) and ~400–700 h away +> **Last reviewed:** 2026-10-28 | **Last updated:** 2026-10-28 | **Change:** minor (M10, #4119 partial: Enhance scanned file cleans scanned pages in place: background removal, edge shadow removal, despeckle; estimates unchanged) > **Last reviewed:** 2026-10-27 | **Last updated:** 2026-10-27 | **Change:** minor (M10: OCR model downloads use verified ONNX pairs with legacy discovery fallback; estimates unchanged) > **Last reviewed:** 2026-10-25 | **Last updated:** 2026-10-25 | **Change:** minor (M14 localization current state: Arabic logical catalog, explicit RTL label direction and mirrored stock chrome; native/browser RTL workflows unqualified. Native release Hans face via LXGW WenKai Lite; estimates unchanged) > **Last reviewed:** 2026-10-23 · **Last updated:** 2026-10-23 · **Change:** minor (M7, #2636 partial: pasting added text boxes and images in the engine and through `content_copy`; estimates unchanged) @@ -200,6 +201,7 @@ Ranked; detail and the full milestone table in [docs/roadmap.md](docs/roadmap.md ## Progress log +- **2026-10-28 (M10, #4119 partial):** Scan & OCR ▸ Enhance scanned file cleans scanned pages in place: background removal (level 0–100, as the camera clean-up), edge shadow removal (book gutters, copier borders) and despeckle. Each scan image is decoded, cleaned and written back into its own object at the same size and colour space (JPEG stays JPEG, others lossless, 1-bit stays 1-bit), so the OCR text layer, comments and links stay; one undo step, shared images once, unsupported images (CMYK, indexed, JPEG 2000, JBIG2, CCITT, broken data) reported. `page_enhance_scan` drives it headlessly. Not yet an option of Recognize text. Estimates unchanged. - **2026-10-27 (M10, OCR models):** Model staging downloads and verifies the ONNX pair after the upstream RTEN URL brownout; runtime discovery prefers complete ONNX pairs per directory, retains complete RTEN fallback and preserves directory precedence. - **2026-10-24 (M10, #4158 partial):** Word, HTML, RTF and PowerPoint export keep a block that continues another on its line (an italic case name, a link, a superscript) as a run in its own style inside that paragraph, with the source's word space, instead of a paragraph of its own. Style changes that wrap onto the next line still split paragraphs (the reading blocks keep style boundaries). Estimates unchanged. - **2026-10-24 (M3, window move on Wayland, #3970 follow-up):** After a move or resize by PdfCraft's own title bar or edges, the button release given back to egui goes before the pointer leaving the window, so hover no longer sticks at the old place until the pointer moves. Found by CodeRabbit in review of #3970; a unit test covers it. Estimates unchanged. diff --git a/crates/automation/src/dispatch.rs b/crates/automation/src/dispatch.rs index 87f47e8ebe..b8dfd36388 100644 --- a/crates/automation/src/dispatch.rs +++ b/crates/automation/src/dispatch.rs @@ -106,6 +106,35 @@ impl Automation { out["detected"] = json!(given.is_none()); out } + "page_enhance_scan" => { + let pages = match a.opt_ints("pages")? { + Some(_) => self.pages(&a, "pages")?, + None => Vec::new(), + }; + let level = match a.opt_int("level")? { + None => pdfcraft_engine::scan_clean::ScanCleanSettings::default().level, + Some(l @ 0..=100) => l as u8, + Some(l) => return Err(ToolError::InvalidArgs(format!("level must be between 0 and 100, not {l}"))), + }; + let mut settings = pdfcraft_engine::scan_clean::ScanCleanSettings::at(level); + if let Some(on) = a.opt_bool("edge_shadows")? { + settings.edge_shadows = on; + } + if let Some(on) = a.opt_bool("despeckle")? { + settings.despeckle = on; + } + let id = self.doc(&a)?.id; + let report = self.session.enhance_scanned_pages(id, &pages, settings).map_err(failed)?; + let numbered = |p: &[usize]| p.iter().map(|p| p + 1).collect::>(); + let skipped: Vec = + report.skipped.iter().map(|k| json!({ "pages": numbered(&k.pages), "name": k.name, "reason": k.reason })).collect(); + let mut out = summary(self.doc(&a)?); + out["pages_cleaned"] = json!(numbered(&report.pages)); + out["images"] = json!(report.images); + out["not_scanned"] = json!(numbered(&report.not_scanned)); + out["skipped"] = json!(skipped); + out + } "page_delete" => { let pages = self.pages(&a, "pages")?; self.apply(&a, Edit::DeletePages { pages })? diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index 4b4ac8873b..0d34911154 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -225,6 +225,18 @@ pub fn tools() -> Vec { }), &["doc", "page"], )), + t("page_enhance_scan", "Enhance scanned file", "Clean up scanned pages in place (Scan & OCR ▸ Enhance ▸ Scanned document). A page is a scan when its pictures cover at least half of it; the pictures covering at least a tenth of it are cleaned and written back into the same image objects at the same size and colour space (a JPEG stays a JPEG at quality 90, other grey/RGB images are stored losslessly, a 1-bit image stays 1-bit), so the page's invisible OCR text, comments, links and other content stay as they were, and an image drawn on several pages is cleaned once. level (0–100, default 50) is background removal: the paper's illumination is evened out so a grey, yellowed or unevenly lit background turns white, then the contrast is raised so faint strokes darken; colour ink stays colour. edge_shadows (default: on unless level is 0) lightens shadows along the scan's edges (book gutters, a copier's dark border). despeckle (default: from level 60) turns isolated dark specks white; it is the only change a 1-bit scan takes. Returns pages_cleaned (1-based), images (rewritten), not_scanned (pages that are no scan) and skipped ([{pages, name, reason}]: CMYK, indexed, JPEG 2000, JBIG2 or CCITT images, broken data, or nothing to change). One undoable step; none when nothing changed.") + .cmd("ocr.enhance") + .with(schema( + json!({ + "doc": doc(), + "pages": pages("to clean up (default: all)"), + "level": { "type": "integer", "minimum": 0, "maximum": 100 }, + "edge_shadows": { "type": "boolean" }, + "despeckle": { "type": "boolean" }, + }), + &["doc"], + )), t("page_delete", "Delete pages", "Delete pages. Undoable until saved.") .destructive() .cmd("page.delete") diff --git a/crates/automation/tests/it/automation.rs b/crates/automation/tests/it/automation.rs index 78038d8ebd..c6616f11f3 100644 --- a/crates/automation/tests/it/automation.rs +++ b/crates/automation/tests/it/automation.rs @@ -3376,6 +3376,62 @@ fn enhance_camera_image_through_tools() { )); } +#[test] +fn enhance_scanned_file_cleans_the_scan_in_place_through_tools() { + let dir = workdir("enhance-scan"); + // A 300 × 400 grey scan: dull paper, a dark gutter at the left, text bars and a speck. + let img = image::GrayImage::from_fn(300, 400, |x, y| { + let light = 0.55 + 0.45 * (x as f32 / 60.0).min(1.0); + let ink = if ((60..360).contains(&y) && (y - 60) % 40 < 6 && (60..240).contains(&x)) || (x, y) == (150, 25) { 0.12 } else { 0.78 }; + image::Luma([(ink * light * 255.0) as u8]) + }); + img.save(dir.join("scan.jpg")).unwrap(); + let mut a = auto(&dir); + let scan = ok(&mut a, "doc_create", json!({ "from": "images", "paths": ["scan.jpg"] }))["doc"].as_u64().unwrap(); + ok(&mut a, "image_save", json!({ "doc": scan, "page": 1, "image": 1, "path": "before.jpg" })); + + let r = ok(&mut a, "page_enhance_scan", json!({ "doc": scan, "level": 100 })); + assert_eq!(r["undo"], "Enhance scanned file"); + assert_eq!((r["pages_cleaned"].clone(), r["images"].clone(), r["not_scanned"].clone()), (json!([1]), json!(1), json!([]))); + assert_eq!(r["skipped"], json!([])); + // In place: still one picture of the same size, still a JPEG, now with white paper. + let images = ok(&mut a, "page_images", json!({ "doc": scan, "page": 1 })); + assert_eq!((images["count"].clone(), images["images"][0]["pixels"].clone()), (json!(1), json!([300, 400]))); + let saved = ok(&mut a, "image_save", json!({ "doc": scan, "page": 1, "image": 1, "path": "after" })); + assert_eq!(saved["format"], "jpg"); + let (before, after) = (image::open(dir.join("before.jpg")).unwrap().to_luma8(), image::open(dir.join("after.jpg")).unwrap().to_luma8()); + for (x, y) in [(150, 40), (3, 40), (3, 380), (150, 25)] { + assert!( + before.get_pixel(x, y)[0] < 200 && after.get_pixel(x, y)[0] >= 230, + "({x}, {y}): {:?} → {:?}", + before.get_pixel(x, y), + after.get_pixel(x, y) + ); + } + assert!(after.get_pixel(150, 62)[0] <= 60, "a stroke stays dark"); + // Undo puts the scan back as it was. + ok(&mut a, "edit_undo", json!({ "doc": scan })); + ok(&mut a, "image_save", json!({ "doc": scan, "page": 1, "image": 1, "path": "undone.jpg" })); + assert_eq!(std::fs::read(dir.join("undone.jpg")).unwrap(), std::fs::read(dir.join("before.jpg")).unwrap()); + + // Switches: despeckle alone leaves everything but the speck. + let r = ok(&mut a, "page_enhance_scan", json!({ "doc": scan, "pages": [1], "level": 0, "despeckle": true, "edge_shadows": false })); + assert_eq!(r["images"], 1); + // A document of text pages has no scans: nothing changes, and it says so. + let text = ok(&mut a, "doc_open", json!({ "path": "a.pdf" }))["doc"].as_u64().unwrap(); + let r = ok(&mut a, "page_enhance_scan", json!({ "doc": text })); + assert_eq!((r["images"].clone(), r["not_scanned"].clone(), r["undo"].clone()), (json!(0), json!([1, 2, 3]), Value::Null)); + // Bad arguments are refused before anything changes. + for bad in [ + json!({ "doc": scan, "level": 101 }), + json!({ "doc": scan, "pages": [9] }), + json!({ "doc": scan, "despeckle": "yes" }), + json!({ "doc": scan, "pages": [] }), + ] { + assert!(matches!(a.call("page_enhance_scan", &bad), Err(ToolError::InvalidArgs(_))), "{bad}"); + } +} + #[test] fn enhance_camera_image_finds_the_edges_by_itself() { let dir = workdir("enhance-camera-auto"); diff --git a/crates/engine/src/camera_clean.rs b/crates/engine/src/camera_clean.rs index 9c4120be56..d1768a2a83 100644 --- a/crates/engine/src/camera_clean.rs +++ b/crates/engine/src/camera_clean.rs @@ -1,6 +1,8 @@ -//! Clean up a straightened camera photo (Enhance ▸ Camera image ▸ enhancement level): even out -//! the lighting so the paper turns white (room-light colour casts, a hand's shadow, darker -//! edges go with it), then raise the contrast and drop isolated specks. Colour ink stays colour. +//! Clean up a straightened camera photo (Enhance ▸ Camera image ▸ enhancement level) or a +//! scanned page's picture (Enhance ▸ Scanned document, [`crate::scan_clean`]): even out the +//! lighting so the paper turns white (room-light colour casts, a hand's shadow, darker edges and +//! book gutters go with it), then raise the contrast and drop isolated specks. Colour ink stays +//! colour. //! //! The paper's illumination is estimated per channel on a coarse grid of blocks, from each //! block's bright end (the paper, not the ink), smoothed into a continuous map and divided out. @@ -8,6 +10,8 @@ /// The strongest a level can be. pub const MAX_LEVEL: u8 = 100; +/// The level from which [`clean`] also drops isolated specks. +pub const DESPECKLE_FROM: u8 = 60; /// How an enhanced camera page looks. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] @@ -66,34 +70,158 @@ pub fn apply(rgb: &[u8], width: u32, height: u32, look: Look, level: u8) -> Opti let even = clean(rgb, width, height, 50)?; let t = 0.45 + 0.4 * level.min(MAX_LEVEL) as f32 / MAX_LEVEL as f32; let mut grey: Vec = even.as_chunks::<3>().0.iter().map(|p| if (lum(p) as f32 / 255.0) < t { 0 } else { 255 }).collect(); - despeckle_grey(&mut grey, w, h); + despeckle(&mut grey, 1, w, h); Pixels::Grey(grey) } }) } -/// Clean `rgb` (RGB8, `width` × `height`) at `level` (0 = unchanged … 100 = strongest). -/// Returns `None` when the buffer doesn't match the size. +/// Clean `rgb` (RGB8, `width` × `height`) at `level` (0 = unchanged … 100 = strongest), from +/// [`DESPECKLE_FROM`] without isolated specks. Returns `None` when the buffer doesn't match the +/// size. pub fn clean(rgb: &[u8], width: u32, height: u32, level: u8) -> Option> { + clean_samples(rgb, 3, width, height, CleanUp { level, edge_shadows: false, despeckle: level >= DESPECKLE_FROM }) +} + +/// What [`clean_samples`] does. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct CleanUp { + /// 0 (unchanged) … 100: even out the paper's illumination and raise the contrast, as + /// [`clean`]'s level. + pub level: u8, + /// Even out shadows along the picture's edges first (a book's gutter, a copier's dark edge): + /// lightened line by line to the paper beside them, within the outer fifth of each side. + pub edge_shadows: bool, + /// Drop isolated dark specks. + pub despeckle: bool, +} + +/// [`clean`] for samples of `n` components (1: grey, 3: RGB), with each step chosen apart (each +/// works at level 0 too). Returns `None` when `n` is neither or the buffer doesn't match the size. +pub fn clean_samples(px: &[u8], n: usize, width: u32, height: u32, how: CleanUp) -> Option> { let (w, h) = (width as usize, height as usize); - if w == 0 || h == 0 || w.checked_mul(h)?.checked_mul(3)? != rgb.len() { + if !(n == 1 || n == 3) || w == 0 || h == 0 || w.checked_mul(h)?.checked_mul(n)? != px.len() { return None; } - let level = level.min(MAX_LEVEL); - if level == 0 { - return Some(rgb.to_vec()); + let level = how.level.min(MAX_LEVEL); + let mut out = px.to_vec(); + if how.edge_shadows { + even_edges(&mut out, n, w, h); } + if level > 0 { + out = even_out(&out, n, w, h, level); + } + if how.despeckle { + despeckle(&mut out, n, w, h); + } + Some(out) +} + +/// The share of the width (and of the height) at each side where [`even_edges`] works. +const EDGE_BAND: f32 = 0.2; +/// The most [`even_edges`] lightens a line by; a darker line (a scanner's black border, a +/// picture) is not taken for shadowed paper and stays as it is. +const MAX_EDGE_GAIN: f32 = 3.0; + +/// Even out shadows that run along the picture's edges, in place: a book's gutter, a copier's +/// dark border, a page that curved away from the glass. Within the outer [`EDGE_BAND`] of each +/// side, each column (then each row) gets the paper's level along it (per channel, its 90th +/// percentile, so the text across it doesn't count), and is lightened to the brightest paper of +/// that band. The block grid of [`background`] is too coarse for such a narrow shadow: its +/// blocks borrow the brighter paper next to them. +fn even_edges(px: &mut [u8], n: usize, w: usize, h: usize) { + for vertical in [true, false] { + // Columns: lines across the width, each running down the height; rows the other way. + let (lines, along) = if vertical { (w, h) } else { (h, w) }; + let band = ((lines as f32 * EDGE_BAND) as usize).min(lines / 2); + if band == 0 { + continue; + } + let at = |line: usize, i: usize| if vertical { i * w + line } else { line * w + i }; + // The paper's level of every line in the bands and a few beyond them. + let reach = band.saturating_add(3).min(lines); + let measured = |line: usize| line < reach || line >= lines.saturating_sub(reach); + let mut level: Vec> = vec![None; lines]; + let mut hist = [[0u32; 256]; 3]; + for (line, slot) in level.iter_mut().enumerate().filter(|(line, _)| measured(*line)) { + hist.iter_mut().for_each(|hh| hh.fill(0)); + for i in 0..along { + let p = at(line, i) * n; + for (c, hh) in hist.iter_mut().enumerate().take(n) { + if let Some(v) = px.get(p + c) + && let Some(count) = hh.get_mut(*v as usize) + { + *count += 1; + } + } + } + let want = (along as f32 * 0.9) as u32; + *slot = Some(std::array::from_fn(|c| { + let mut seen = 0; + let v = hist.get(c).and_then(|hh| { + hh.iter().position(|&count| { + seen += count; + seen > want + }) + }); + v.unwrap_or(255) as f32 / 255.0 + })); + } + // Smoothed: the median of five neighbouring lines (fewer at the very edge, centred all + // the same, so a gradient isn't shifted), so one odd line (a rule, a fold) doesn't stand + // out. + let smooth: Vec> = (0..lines) + .map(|line| { + level.get(line).copied().flatten()?; + let r = line.min(lines - 1 - line).min(2); + Some(std::array::from_fn(|c| { + let mut near: Vec = (line - r..=line + r).filter_map(|l| level.get(l).copied().flatten()).map(|v| v[c]).collect(); + near.sort_by(f32::total_cmp); + near.get(near.len() / 2).copied().unwrap_or(1.0) + })) + }) + .collect(); + // Each side's band, and the lines its paper is measured on (the band and a few beyond). + for (shadowed, measured) in [(0..band, 0..reach), (lines - band..lines, lines.saturating_sub(reach)..lines)] { + // The brightest paper there: what the shadow is lightened to. + let paper: [f32; 3] = + std::array::from_fn(|c| measured.clone().filter_map(|l| smooth.get(l).copied().flatten()).map(|v| v[c]).fold(0.0, f32::max)); + for line in shadowed { + let Some(own) = smooth.get(line).copied().flatten() else { continue }; + let gain: [f32; 3] = std::array::from_fn(|c| { + let g = paper[c] / own[c].max(1e-3); + if (1.0..=MAX_EDGE_GAIN).contains(&g) { g } else { 1.0 } + }); + if gain.iter().take(n).all(|&g| g == 1.0) { + continue; + } + for i in 0..along { + let p = at(line, i) * n; + for (c, g) in gain.iter().enumerate().take(n) { + if let Some(v) = px.get_mut(p + c) { + *v = (*v as f32 * g).round().clamp(0.0, 255.0) as u8; + } + } + } + } + } + } +} + +/// The lighting evened out and the contrast raised at `level` (1–100); `px` holds `w` × `h` +/// pixels of `n` components. +fn even_out(px: &[u8], n: usize, w: usize, h: usize, level: u8) -> Vec { let k = level as f32 / MAX_LEVEL as f32; - let background = background(rgb, w, h); + let background = background(px, n, w, h); // Lighting is evened out fully from half strength on; contrast keeps growing to the top. let flatten = (2.0 * k).min(1.0); let white = 1.0 - 0.12 * k; // what reaches white let black = 0.30 * k; // what reaches black - let mut out = Vec::with_capacity(rgb.len()); - for (i, px) in rgb.as_chunks::<3>().0.iter().enumerate() { + let mut out = Vec::with_capacity(px.len()); + for (i, p) in px.chunks_exact(n).enumerate() { let (x, y) = (i % w, i / w); let bg = background.at(x, y); - for (c, &v) in px.iter().enumerate() { + for (c, &v) in p.iter().enumerate() { let v = v as f32 / 255.0; let b = bg.get(c).copied().unwrap_or(1.0).max(0.05); let even = (v / b).min(1.0); @@ -102,10 +230,7 @@ pub fn clean(rgb: &[u8], width: u32, height: u32, level: u8) -> Option> out.push((stretched * 255.0).round() as u8); } } - if level >= 60 { - despeckle(&mut out, w, h); - } - Some(out) + out } /// The paper's illumination: a grid of per-channel values, read back with bilinear @@ -140,9 +265,10 @@ impl Background { } } -/// Estimate the paper's illumination: per block, the 90th percentile of each channel (bright -/// paper, not dark ink); blocks that are mostly ink borrow from their neighbours; then smoothed. -fn background(rgb: &[u8], w: usize, h: usize) -> Background { +/// Estimate the paper's illumination: per block, the 90th percentile of each of the `n` channels +/// (bright paper, not dark ink); blocks that are mostly ink borrow from their neighbours; then +/// smoothed. Channels past `n` stay white. +fn background(px: &[u8], n: usize, w: usize, h: usize) -> Background { let block = (w.min(h) / 16).clamp(8, 128); let (cols, rows) = (w.div_ceil(block), h.div_ceil(block)); let mut cells = Vec::with_capacity(cols * rows); @@ -150,20 +276,20 @@ fn background(rgb: &[u8], w: usize, h: usize) -> Background { for by in 0..rows { for bx in 0..cols { hist.iter_mut().for_each(|hh| hh.fill(0)); - let mut n = 0u32; + let mut count = 0u32; for y in by * block..((by + 1) * block).min(h) { for x in bx * block..((bx + 1) * block).min(w) { - if let Some(p) = rgb.get((y * w + x) * 3..(y * w + x) * 3 + 3) { + if let Some(p) = px.get((y * w + x) * n..(y * w + x) * n + n) { for (c, &v) in p.iter().enumerate() { if let Some(slot) = hist.get_mut(c).and_then(|hh| hh.get_mut(v as usize)) { *slot += 1; } } - n += 1; + count += 1; } } } - let want = (n as f32 * 0.9) as u32; + let want = (count as f32 * 0.9) as u32; cells.push(std::array::from_fn(|c| { let mut seen = 0; let v = hist.get(c).and_then(|hh| { @@ -219,19 +345,15 @@ fn background(rgb: &[u8], w: usize, h: usize) -> Background { Background { cols, rows, block: block as f32, cells } } -/// [`despeckle`] for one grey byte per pixel. -fn despeckle_grey(grey: &mut [u8], w: usize, h: usize) { - let mut rgb: Vec = grey.iter().flat_map(|&v| [v, v, v]).collect(); - despeckle(&mut rgb, w, h); - for (g, p) in grey.iter_mut().zip(rgb.as_chunks::<3>().0.iter()) { - *g = p[0]; - } -} - -/// Lighten dark pixels with at most one dark neighbour (specks), in place. -fn despeckle(rgb: &mut [u8], w: usize, h: usize) { - let lum = |p: &[u8; 3]| (p[0] as u32 * 299 + p[1] as u32 * 587 + p[2] as u32 * 114) / 1000; - let dark: Vec = rgb.as_chunks::<3>().0.iter().map(|p| lum(p) < 128).collect(); +/// Lighten dark pixels with at most one dark neighbour (specks), in place; `px` holds `w` × `h` +/// pixels of `n` components (1 or 3). +fn despeckle(px: &mut [u8], n: usize, w: usize, h: usize) { + let lum = |p: &[u8]| match p { + [r, g, b] => (*r as u32 * 299 + *g as u32 * 587 + *b as u32 * 114) / 1000, + [v, ..] => *v as u32, + [] => 255, + }; + let dark: Vec = px.chunks_exact(n.max(1)).map(|p| lum(p) < 128).collect(); let is_dark = |x: isize, y: isize| { x >= 0 && y >= 0 && (x as usize) < w && (y as usize) < h && dark.get(y as usize * w + x as usize).copied().unwrap_or(false) }; @@ -243,7 +365,7 @@ fn despeckle(rgb: &mut [u8], w: usize, h: usize) { let neighbours = (-1..=1).flat_map(|dy| (-1..=1).map(move |dx| (dx, dy))).filter(|&(dx, dy)| (dx, dy) != (0, 0) && is_dark(x + dx, y + dy)).count(); if neighbours <= 1 - && let Some(p) = rgb.get_mut((y as usize * w + x as usize) * 3..(y as usize * w + x as usize) * 3 + 3) + && let Some(p) = px.get_mut((y as usize * w + x as usize) * n..(y as usize * w + x as usize) * n + n) { p.fill(255); } @@ -388,5 +510,40 @@ mod tests { assert_eq!(clean(&[], 0, 0, 50), None); assert_eq!(clean(&[0; 12], 2, 3, 50), None); assert_eq!(clean(&[9; 3], 1, 1, 255).map(|v| v.len()), Some(3)); + let every = CleanUp { level: 100, edge_shadows: true, despeckle: true }; + assert_eq!(clean_samples(&[0; 12], 2, 2, 3, every), None, "two components"); + assert_eq!(clean_samples(&[0; 5], 1, 2, 3, every), None); + for (w, h) in [(1, 1), (1, 9), (9, 1), (2, 2), (5, 3)] { + assert_eq!(clean_samples(&vec![100; w * h], 1, w as u32, h as u32, every).map(|v| v.len()), Some(w * h)); + } + } + + #[test] + fn edge_shadows_lighten_shadowed_paper_but_not_a_black_border_or_the_middle() { + // Grey paper (200) with a shadow darkening the left 20 columns to 120, a black scanner + // border in the right 8 columns, and a dark block in the middle. + let (w, h) = (100usize, 60usize); + let mut grey = vec![200u8; w * h]; + for y in 0..h { + for x in 0..w { + let v = if x >= w - 8 { + 5 + } else if (40..60).contains(&x) && (20..40).contains(&y) { + 30 + } else if x < 20 { + 120 + (80 * x / 20) as u8 + } else { + 200 + }; + grey[y * w + x] = v; + } + } + let out = clean_samples(&grey, 1, w as u32, h as u32, CleanUp { level: 0, edge_shadows: true, despeckle: false }).unwrap(); + for x in [0, 5, 19] { + assert!(out[30 * w + x].abs_diff(200) <= 3, "column {x}: {} → {}", grey[30 * w + x], out[30 * w + x]); + } + assert_eq!(out[30 * w + w - 3], 5, "a black border is no shadow"); + assert_eq!(out[30 * w + 50], 30, "the middle is untouched"); + assert_eq!(out[30 * w + 30], 200); } } diff --git a/crates/engine/src/catalog.rs b/crates/engine/src/catalog.rs index e479232bbb..47394a7bb5 100644 --- a/crates/engine/src/catalog.rs +++ b/crates/engine/src/catalog.rs @@ -265,7 +265,7 @@ pub static TOOL_GROUPS: &[ToolGroup] = &[ items: &[ item("In this file", "scan-text", "ocr.recognize", Ready), item("In multiple files", "files", "ocr.recognize_batch", Ready), - item("Enhance scanned file", "sparkles", "ocr.enhance", Planned("M10")), + item("Enhance scanned file", "sparkles", "ocr.enhance", Ready), item("Enhance camera image", "camera", "ocr.enhance_camera", Ready), item("Correct recognized text", "text-select", "ocr.correct", Planned("M10")), ], diff --git a/crates/engine/src/commands.rs b/crates/engine/src/commands.rs index 1f25c23c98..8c83996c6d 100644 --- a/crates/engine/src/commands.rs +++ b/crates/engine/src/commands.rs @@ -352,6 +352,7 @@ pub const COMMANDS: &[CommandSpec] = &[ c("tools.document_js", "Document JavaScripts…", None, None, Modification, "file-code"), c("ocr.recognize", "Recognize text…", None, None, Modification, "scan-text"), c("ocr.enhance_camera", "Enhance camera image", None, None, Modification, "camera"), + c("ocr.enhance", "Enhance scanned file…", None, None, Modification, "sparkles"), c("ocr.recognize_batch", "Recognize text in multiple files…", None, None, Nothing, "files"), c("a11y.check", "Check for accessibility…", None, None, Document, "accessibility"), c("a11y.find_untagged", "Find untagged content", None, None, Document, "search"), diff --git a/crates/engine/src/edit.rs b/crates/engine/src/edit.rs index c31984ca19..655a1ce558 100644 --- a/crates/engine/src/edit.rs +++ b/crates/engine/src/edit.rs @@ -375,6 +375,13 @@ pub enum Edit { page: usize, words: Vec, }, + /// Scan & OCR ▸ Enhance ▸ Scanned document: put the scan images + /// [`ScanCleanJob::run`](crate::scan_clean::ScanCleanJob::run) cleaned in place of the old + /// ones (each object must still be an image). Built by + /// [`Session::apply_scan_clean`](crate::Session::apply_scan_clean). + EnhanceScan { + images: Vec<(pdfcraft_cos::ObjRef, pdfcraft_cos::Stream)>, + }, /// Edit a PDF ▸ Edit text: replace the text of line `line` (from `Document::text_lines`) on /// `page`, in its own font when it can show it, else in Helvetica. EditTextLine { diff --git a/crates/engine/src/edit/labels.rs b/crates/engine/src/edit/labels.rs index ea25898068..ff86f21a08 100644 --- a/crates/engine/src/edit/labels.rs +++ b/crates/engine/src/edit/labels.rs @@ -98,6 +98,7 @@ pub const EDIT_LABELS: &[&str] = &[ "Set alternate text", "Mark figure as decorative", "Recognize text", + "Enhance scanned file", "Run JavaScript", "Run form script", "Save as {level}", @@ -239,6 +240,7 @@ impl Edit { Edit::MarkDecorative { .. } => Label::of("Mark figure as decorative"), Edit::MarkTagArtifact { .. } => Label::of("Mark content as artifact"), Edit::AddOcrText { .. } => Label::of("Recognize text"), + Edit::EnhanceScan { .. } => Label::of("Enhance scanned file"), Edit::ApplyScriptChanges { .. } => Label::of("Run JavaScript"), Edit::XfaEvent { .. } => Label::of("Run form script"), // The level is the standard's own name ("PDF/A-2b"), so it goes in as it stands. diff --git a/crates/engine/src/edit/run.rs b/crates/engine/src/edit/run.rs index 49f19fa637..e2a99665aa 100644 --- a/crates/engine/src/edit/run.rs +++ b/crates/engine/src/edit/run.rs @@ -121,6 +121,7 @@ pub(crate) fn check_permission(edit: &Edit, p: &pdfcraft_cos::Permissions) -> Re | Edit::MarkDecorative { .. } | Edit::MarkTagArtifact { .. } | Edit::AddOcrText { .. } + | Edit::EnhanceScan { .. } | Edit::SetDocumentScript { .. } | Edit::SetFieldScript { .. } | Edit::RemoveStandardId { .. } @@ -491,6 +492,7 @@ pub(crate) fn run_edit(doc: &mut pdfcraft_cos::Document, edit: &Edit, cx: &mut E Edit::AddOcrText { page, words } => { pdfcraft_edit::stamp(doc, *page, "OCR", pdfcraft_ocr::text_layer(words))?; } + Edit::EnhanceScan { images } => crate::scan_clean::write_back(doc, images)?, Edit::EditPageImage { page, index, change } => { let img = pdfcraft_edit::page_images(doc, *page)? .into_iter() diff --git a/crates/engine/src/history.rs b/crates/engine/src/history.rs index fafa65b4c2..0fe874e711 100644 --- a/crates/engine/src/history.rs +++ b/crates/engine/src/history.rs @@ -77,6 +77,7 @@ pub(crate) fn scope_of(edit: &Edit) -> Scope { | Edit::EditTextLine { .. } | Edit::EditTextBlock { .. } | Edit::EditPageImage { .. } + | Edit::EnhanceScan { .. } | Edit::AddText { .. } | Edit::AddImage { .. } | Edit::UpdateContent { .. } diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 889b839368..9e2eb54389 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -43,6 +43,7 @@ mod pictures; pub mod print_raster; pub mod printcheck; mod protection; +pub mod scan_clean; mod scan_colours; mod scan_fields; mod session; diff --git a/crates/engine/src/scan_clean.rs b/crates/engine/src/scan_clean.rs new file mode 100644 index 0000000000..bfc8188b29 --- /dev/null +++ b/crates/engine/src/scan_clean.rs @@ -0,0 +1,276 @@ +//! Scan & OCR ▸ Enhance ▸ Scanned document: clean up the pictures scanned pages are made of. +//! +//! Unlike Enhance ▸ Camera image, the page is not rendered and replaced: each scan image is +//! decoded, cleaned ([`crate::camera_clean::clean_samples`]) and written back **into the same +//! image object**, at the same size and in the same colour space. So everything else on the +//! page stays as it was: an invisible OCR text layer, comments, links, form fields, other +//! content, and the image's own soft mask, `/Interpolate` and other entries. A JPEG stays a JPEG +//! (at quality 90), any other grey or RGB image is stored losslessly (Flate), and a black and +//! white (1-bit) image stays 1-bit. An image several pages draw is cleaned once. +//! +//! A page is a scan when its pictures cover at least half of it (as for Recognize text); the +//! pictures each covering at least a tenth of the page are its scan. What is not cleaned is +//! reported, never skipped silently: pages that are no scan, and scan images in a form PdfCraft +//! can't rewrite yet (CMYK, indexed, JPEG 2000, JBIG2, CCITT fax, a stencil mask, a remapping +//! decode array) or whose data is broken. +//! +//! Like Optimize, the work is a job: [`Session::scan_clean_job`] captures the document, +//! [`ScanCleanJob::run`] cleans anywhere (a worker thread), and [`Session::apply_scan_clean`] +//! makes the result one undoable step, "Enhance scanned file" (the name the Scan & OCR tool has). + +use pdfcraft_cos::{ObjRef, Object, Stream}; + +use crate::camera_clean::{CleanUp, MAX_LEVEL, clean_samples}; +use crate::edit::check_permission; +use crate::{DocId, Edit, EditError, Session, guard, optimize}; + +pub use crate::camera_clean::DESPECKLE_FROM; + +/// The JPEG quality a cleaned JPEG scan is written back at (as Enhance camera image). +const QUALITY: u8 = 90; +/// On a scanned page, an image covering at least this share of the page is part of the scan; +/// smaller ones (a logo or a signature pasted on) are left alone. +const PART_OF_SCAN: f64 = 0.1; + +/// How scanned pages are cleaned up. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ScanCleanSettings { + /// Background removal, 0 (off) … 100 (strongest), as Enhance camera image's clean-up level: + /// the paper's illumination is evened out (fully from 50), so a grey, yellowed or unevenly lit + /// background turns white; then the contrast is raised so faint strokes darken. Colour ink + /// stays colour. + pub level: u8, + /// Edge shadow removal: shadows along the scan's edges (a book's gutter, a copier's dark + /// border) are lightened to the paper next to them, line by line, before the level applies. + pub edge_shadows: bool, + /// Despeckle: isolated dark specks (dust, scanner noise), dark pixels with at most one dark + /// neighbour, turn white. The only change a black and white (1-bit) scan takes. + pub despeckle: bool, +} + +impl ScanCleanSettings { + /// The level `level` (capped at 100) with edge shadow removal (unless the level is 0), and + /// despeckling from [`DESPECKLE_FROM`] as the camera tool does. + pub fn at(level: u8) -> Self { + let level = level.min(MAX_LEVEL); + Self { level, edge_shadows: level > 0, despeckle: level >= DESPECKLE_FROM } + } + + /// Whether these settings change anything at all. + pub fn changes_something(self) -> bool { + self.level > 0 || self.edge_shadows || self.despeckle + } +} + +impl Default for ScanCleanSettings { + fn default() -> Self { + Self::at(50) + } +} + +/// What a clean-up did. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct ScanCleanReport { + /// Pages (0-based) whose scan was cleaned, in order. + pub pages: Vec, + /// Images rewritten. One drawn on several pages counts once. + pub images: usize, + /// Pages left alone because no picture covers most of them. + pub not_scanned: Vec, + /// Scan images left as they were, and why. + pub skipped: Vec, +} + +/// A scan image that was not cleaned. +#[derive(Clone, Debug, PartialEq)] +pub struct SkippedImage { + /// The pages (0-based) that draw it, among those asked for. + pub pages: Vec, + /// Its resource name on the first of them. + pub name: String, + /// Why, in a sentence that follows "skipped because". + pub reason: String, +} + +/// One image to clean: its object, its name and the pages that draw it. +struct Target { + object: ObjRef, + name: String, + pages: Vec, +} + +/// What cleaning scanned pages needs, independent of the session. +pub struct ScanCleanJob { + cos: pdfcraft_cos::Document, + targets: Vec, + settings: ScanCleanSettings, + /// The document's edit generation it was taken at: a result is for that state only. + generation: u64, + report: ScanCleanReport, +} + +/// The cleaned images, ready for [`Session::apply_scan_clean`], and the report. +pub struct ScanCleaned { + images: Vec<(ObjRef, Stream)>, + generation: u64, + pub report: ScanCleanReport, +} + +impl Session { + /// Capture what cleaning up the scans on `pages` (0-based; empty = all) of document `id` + /// needs. Errors when the document can't be changed (read-only, or its permissions forbid + /// changes) or a page doesn't exist. + pub fn scan_clean_job(&self, id: DocId, pages: &[usize], settings: ScanCleanSettings) -> Result { + let doc = self.get(id).ok_or(EditError::NoDocument)?; + if let Some(why) = &doc.read_only_reason { + return Err(EditError::ReadOnly(why.clone())); + } + let editor = doc.editor.as_ref().ok_or_else(|| EditError::ReadOnly(String::new()))?; + if let Some(p) = editor.cos.permissions() { + check_permission(&Edit::EnhanceScan { images: Vec::new() }, &p)?; + } + let all = doc.info.pages.len(); + let mut wanted: Vec = if pages.is_empty() { (0..all).collect() } else { pages.to_vec() }; + if let Some(p) = wanted.iter().find(|p| **p >= all) { + return Err(EditError::Invalid(format!("there is no page {}: the document has {all}", p.saturating_add(1)))); + } + wanted.sort_unstable(); + wanted.dedup(); + let mut report = ScanCleanReport::default(); + let mut targets: Vec = Vec::new(); + for page in wanted { + let Some(info) = doc.info.pages.get(page) else { continue }; + let Some(images) = crate::scan_fields::scan_images(doc, page, info) else { + report.not_scanned.push(page); + continue; + }; + for (image, _) in images.into_iter().filter(|(_, share)| *share >= PART_OF_SCAN) { + let Some(object) = image.object else { + report.skipped.push(SkippedImage { + pages: vec![page], + name: image.name, + reason: "it is stored inside the page's resources rather than as an object of its own".into(), + }); + continue; + }; + match targets.iter_mut().find(|t| t.object == object) { + Some(t) if !t.pages.contains(&page) => t.pages.push(page), + Some(_) => {} + None => targets.push(Target { object, name: image.name, pages: vec![page] }), + } + } + } + Ok(ScanCleanJob { cos: editor.cos.clone(), targets, settings, generation: doc.edit_generation(), report }) + } + + /// Write the images [`ScanCleanJob::run`] cleaned into document `id` as one undoable step, + /// "Enhance scanned file" (none when nothing was cleaned), and return the report. Refused + /// when the document changed since the job was taken. + pub fn apply_scan_clean(&mut self, id: DocId, cleaned: ScanCleaned) -> Result { + let doc = self.get(id).ok_or(EditError::NoDocument)?; + if doc.edit_generation() != cleaned.generation { + return Err(EditError::Invalid("the document changed while its scans were being cleaned up; nothing was changed, try again".into())); + } + if !cleaned.images.is_empty() { + self.apply(id, Edit::EnhanceScan { images: cleaned.images })?; + } + Ok(cleaned.report) + } + + /// Enhance ▸ Scanned document in one call: [`Self::scan_clean_job`], run, then + /// [`Self::apply_scan_clean`] (the CLI and agents use this). + pub fn enhance_scanned_pages(&mut self, id: DocId, pages: &[usize], settings: ScanCleanSettings) -> Result { + let job = self.scan_clean_job(id, pages, settings)?; + let cleaned = job.run(|_, _| true)?; + self.apply_scan_clean(id, cleaned) + } +} + +impl ScanCleanJob { + /// The images to clean (one drawn on several pages counts once). + pub fn total(&self) -> usize { + self.targets.len() + } + + /// Clean every scan image. `progress(done, total)` is called before each image; returning + /// `false` stops with [`EditError::Cancelled`]. A panic is returned as an error, so a worker + /// thread always reports back. + pub fn run(self, mut progress: impl FnMut(usize, usize) -> bool) -> Result { + guard(move || self.work(&mut progress)).map_err(|m| EditError::Invalid(format!("cleaning up the scans failed unexpectedly ({m})")))? + } + + fn work(self, progress: &mut dyn FnMut(usize, usize) -> bool) -> Result { + let ScanCleanJob { cos, targets, settings, generation, mut report } = self; + let total = targets.len(); + let mut images = Vec::new(); + let mut changed_pages: Vec = Vec::new(); + for (done, target) in targets.into_iter().enumerate() { + if !progress(done, total) { + return Err(EditError::Cancelled); + } + let mut skip = |reason: String| report.skipped.push(SkippedImage { pages: target.pages.clone(), name: target.name.clone(), reason }); + match clean_image(&cos, target.object, settings) { + Ok(Some(stream)) => { + images.push((target.object, stream)); + changed_pages.extend(target.pages.iter().copied()); + } + Ok(None) => skip("it is already clean: nothing changes at these settings".into()), + Err(reason) => skip(reason), + } + } + changed_pages.sort_unstable(); + changed_pages.dedup(); + report.images = images.len(); + report.pages = changed_pages; + report.skipped.sort_by_key(|s| s.pages.first().copied()); + Ok(ScanCleaned { images, generation, report }) + } +} + +/// Image `object` cleaned with `settings`: the new stream, `None` when nothing changes, or why it +/// can't be cleaned. +fn clean_image(cos: &pdfcraft_cos::Document, object: ObjRef, settings: ScanCleanSettings) -> Result, String> { + let obj = cos.get(object); + let Object::Stream(s) = &*obj else { return Err("it is not an image".into()) }; + if s.dict.name(b"Subtype") != Some(b"Image") { + return Err("it is not an image".into()); + } + // A colour-key mask names sample values to leave unpainted: cleaned samples would match others. + if matches!(s.dict.get(b"Mask").map(|m| cos.resolve(m)).as_deref(), Some(Object::Array(_))) { + return Err("it has a colour-key mask, which cleaning would change".into()); + } + let samples = optimize::decode_samples(cos, s).map_err(|e| e.to_string())?; + let cleaned = if samples.bilevel { + if !settings.despeckle { + return Err("it is black and white, which only Despeckle changes, and Despeckle is off".into()); + } + clean_samples(&samples.data, 1, samples.width, samples.height, CleanUp { level: 0, edge_shadows: false, despeckle: true }) + } else { + if !settings.changes_something() { + return Ok(None); + } + let how = CleanUp { level: settings.level, edge_shadows: settings.edge_shadows, despeckle: settings.despeckle }; + clean_samples(&samples.data, samples.components, samples.width, samples.height, how) + } + .ok_or("its samples don't match its size")?; + if cleaned == samples.data { + return Ok(None); + } + let cleaned = optimize::Samples { data: cleaned, ..samples }; + optimize::encode_samples(s, &cleaned, QUALITY).map(Some).ok_or_else(|| "it could not be written back".into()) +} + +/// [`Edit::EnhanceScan`]: put the cleaned images in place. Each object must still be an image. +pub(crate) fn write_back(doc: &mut pdfcraft_cos::Document, images: &[(ObjRef, Stream)]) -> Result<(), EditError> { + for (object, stream) in images { + let is_image = matches!(&*doc.get(*object), Object::Stream(s) if s.dict.name(b"Subtype") == Some(b"Image")); + if !is_image { + return Err(EditError::Invalid(format!("object {} is no longer an image; the scan was not changed", object.num))); + } + doc.set(*object, Object::Stream(stream.clone())); + } + Ok(()) +} + +#[cfg(test)] +mod tests; diff --git a/crates/engine/src/scan_clean/tests.rs b/crates/engine/src/scan_clean/tests.rs new file mode 100644 index 0000000000..561ba9e2e4 --- /dev/null +++ b/crates/engine/src/scan_clean/tests.rs @@ -0,0 +1,338 @@ +use std::sync::Arc; + +use pdfcraft_testkit::Pdf; + +use super::*; + +const W: usize = 300; +const H: usize = 400; +/// Isolated specks on the paper, away from the text rows. +const SPECKS: [(usize, usize); 3] = [(150, 25), (200, 85), (120, 125)]; + +/// The light falling on the paper at `x`: full in the middle, darker towards a shadowed left +/// edge (as a book's gutter) and a little at the right edge. +fn light(x: usize) -> f32 { + let left = (x as f32 / 60.0).min(1.0); + let right = ((W - 1 - x) as f32 / 30.0).min(1.0); + (0.55 + 0.45 * left) * (0.8 + 0.2 * right) +} + +fn is_stroke(x: usize, y: usize) -> bool { + (60..360).contains(&y) && (y - 60) % 40 < 6 && (60..240).contains(&x) +} + +/// A synthetic scan, `n` components (1 grey, 3 RGB): greyish yellowed paper, a shadowed gutter, +/// dark text strokes and a few specks. +fn scan_pixels(n: usize) -> Vec { + let mut out = Vec::with_capacity(W * H * n); + for y in 0..H { + for x in 0..W { + let ink = if is_stroke(x, y) || SPECKS.contains(&(x, y)) { 0.12 } else { 0.8 }; + let v = ink * light(x); + let px = [v, v, v * 0.85]; + for c in px.iter().take(n) { + out.push((c * 255.0).round().clamp(0.0, 255.0) as u8); + } + } + } + out +} + +fn jpeg(px: &[u8], n: usize) -> Vec { + let mut out = Vec::new(); + let ty = if n == 1 { image::ExtendedColorType::L8 } else { image::ExtendedColorType::Rgb8 }; + image::ImageEncoder::write_image(image::codecs::jpeg::JpegEncoder::new_with_quality(&mut out, 95), px, W as u32, H as u32, ty).unwrap(); + out +} + +/// Three pages: 1 a scan (the image `image_dict`/`data` as object 4) with an invisible text +/// layer and a comment, 2 a page of real text, 3 the same scan image drawn again. +fn scan_pdf(image_dict: &str, data: &[u8]) -> Vec { + let resources = "/Resources << /XObject << /Im0 4 0 R >> /Font << /F1 5 0 R >> >>"; + Pdf::new() + .obj("<< /Type /Catalog /Pages 2 0 R >>") + .obj("<< /Type /Pages /Kids [3 0 R 8 0 R 9 0 R] /Count 3 >>") + .obj(format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 {W} {H}] {resources} /Contents 6 0 R /Annots [7 0 R] >>")) + .stream(&format!("/Type /XObject /Subtype /Image {image_dict}"), data) + .obj("<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>") + .stream("", format!("q {W} 0 0 {H} 0 0 cm /Im0 Do Q BT 3 Tr /F1 12 Tf 20 380 Td (Invisible OCR words) Tj ET")) + .obj("<< /Type /Annot /Subtype /Square /Rect [10 10 50 50] /C [1 0 0] /Contents (A comment) >>") + .obj(format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 {W} {H}] {resources} /Contents 10 0 R >>")) + .obj(format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 {W} {H}] {resources} /Contents 11 0 R >>")) + .stream("", "BT /F1 12 Tf 20 380 Td (Real text) Tj ET") + .stream("", format!("q {W} 0 0 {H} 0 0 cm /Im0 Do Q")) + .build() +} + +fn grey_scan() -> Vec { + scan_pdf(&format!("/Width {W} /Height {H} /ColorSpace /DeviceGray /BitsPerComponent 8 /Interpolate true"), &scan_pixels(1)) +} + +fn open(bytes: Vec) -> (Session, DocId) { + let mut s = Session::new().with_clock(|| 1_700_000_000); + let id = s.open("scan.pdf", None, Arc::new(bytes), None).unwrap(); + (s, id) +} + +fn object(s: &Session, id: DocId, num: u32) -> Arc { + s.get(id).unwrap().editor.as_ref().unwrap().cos.get(ObjRef::new(num, 0)) +} + +/// The samples of the image page 1 draws, now. +fn samples(s: &Session, id: DocId) -> optimize::Samples { + let doc = s.get(id).unwrap(); + let object = doc.page_images(0)[0].object.unwrap(); + let cos = &doc.editor.as_ref().unwrap().cos; + let Object::Stream(img) = &*cos.get(object) else { panic!("the image") }; + optimize::decode_samples(cos, img).unwrap() +} + +fn grey_at(px: &optimize::Samples, x: usize, y: usize) -> u8 { + px.data[(y * W + x) * px.components] +} + +#[test] +fn a_scan_gets_white_paper_even_edges_dark_strokes_and_no_specks_in_place() { + let (mut s, id) = open(grey_scan()); + let before = samples(&s, id); + let kept: Vec> = [3, 5, 6, 7, 8, 10, 11].iter().map(|&n| object(&s, id, n)).collect(); + let text_before = s.get(id).unwrap().text_lines(0); + let original = s.get(id).unwrap().bytes.clone(); + + let report = s.enhance_scanned_pages(id, &[], ScanCleanSettings::at(100)).unwrap(); + assert_eq!(report.pages, vec![0, 2], "the shared image counts for both its pages"); + assert_eq!(report.images, 1); + assert_eq!(report.not_scanned, vec![1], "the text page is no scan"); + assert!(report.skipped.is_empty(), "{:?}", report.skipped); + assert_eq!(s.get(id).unwrap().can_undo(), Some("Enhance scanned file")); + + let after = samples(&s, id); + assert_eq!((after.width, after.height, after.components, after.jpeg), (W as u32, H as u32, 1, false)); + // Paper, also in the shadowed gutter and at the right edge, turns (nearly) white. + for (x, y) in [(150, 40), (150, 380), (3, 40), (3, 380), (30, 200), (W - 2, 40)] { + assert!(grey_at(&before, x, y) < 210, "the fixture's paper is grey at ({x}, {y})"); + assert!(grey_at(&after, x, y) >= 235, "paper at ({x}, {y}): {} → {}", grey_at(&before, x, y), grey_at(&after, x, y)); + } + // The edge is evened out: the gutter is as white as the middle. + assert!(grey_at(&after, 150, 40).abs_diff(grey_at(&after, 3, 40)) <= 12); + // Strokes stay dark (darker, with the contrast), also in the gutter's light. + for (x, y) in [(150, 62), (61, 102), (238, 342)] { + assert!(grey_at(&after, x, y) <= 40, "stroke at ({x}, {y}): {}", grey_at(&after, x, y)); + } + for (x, y) in SPECKS { + assert!(grey_at(&before, x, y) < 60); + assert!(grey_at(&after, x, y) >= 200, "speck at ({x}, {y}): {}", grey_at(&after, x, y)); + } + // Everything but the image is as it was: the page, its invisible text layer, the comment, the + // font, the other pages. The image keeps its entries. + let now: Vec> = [3, 5, 6, 7, 8, 10, 11].iter().map(|&n| object(&s, id, n)).collect(); + assert_eq!(kept, now); + assert_eq!(s.get(id).unwrap().info.pages.len(), 3); + let text_after = s.get(id).unwrap().text_lines(0); + assert!(text_after.iter().any(|l| l.text.contains("Invisible OCR words")), "{text_after:?}"); + assert_eq!(text_before.len(), text_after.len()); + let Object::Stream(img) = &*object(&s, id, 4) else { panic!("the image") }; + assert_eq!(img.dict.get(b"Interpolate"), Some(&Object::Bool(true))); + assert_eq!(img.dict.name(b"ColorSpace"), Some(&b"DeviceGray"[..])); + + // Undo brings the original image, and the original file, back. + s.undo(id).unwrap(); + assert_eq!(samples(&s, id), before); + assert_eq!(s.get(id).unwrap().bytes, original); + assert_eq!(s.get(id).unwrap().can_undo(), None); +} + +#[test] +fn a_jpeg_scan_stays_a_jpeg_in_colour() { + let px = scan_pixels(3); + let (mut s, id) = open(scan_pdf(&format!("/Width {W} /Height {H} /ColorSpace /DeviceRGB /BitsPerComponent 8 /Filter /DCTDecode"), &jpeg(&px, 3))); + let report = s.enhance_scanned_pages(id, &[0], ScanCleanSettings::at(80)).unwrap(); + // Only page 1 was asked for; the image it shares with page 3 is changed for both. + assert_eq!((report.pages.as_slice(), report.images), (&[0][..], 1)); + let after = samples(&s, id); + assert!(after.jpeg && after.components == 3); + let Object::Stream(img) = &*object(&s, id, 4) else { panic!("the image") }; + assert_eq!(img.dict.name(b"Filter"), Some(&b"DCTDecode"[..])); + // Yellowed paper becomes white; the strokes stay dark. + let at = |x: usize, y: usize| after.data[(y * W + x) * 3..][..3].to_vec(); + for (x, y) in [(150, 40), (5, 300)] { + assert!(at(x, y).iter().all(|&v| v >= 230), "paper at ({x}, {y}): {:?}", at(x, y)); + } + assert!(at(150, 63).iter().all(|&v| v <= 60), "stroke: {:?}", at(150, 63)); + // The file still renders. + let doc = s.get(id).unwrap(); + let shot = pdfcraft_render::PageRenderer::new(doc.bytes.clone(), doc.config.clone()).render(pdfcraft_render::RenderRequest { + page: 0, + scale: 1.0, + ..Default::default() + }); + assert!(shot.error.is_none(), "{:?}", shot.error); +} + +#[test] +fn despeckle_is_its_own_switch() { + let speck = |settings: ScanCleanSettings| { + let (mut s, id) = open(grey_scan()); + s.enhance_scanned_pages(id, &[0], settings).unwrap(); + let after = samples(&s, id); + SPECKS.iter().map(|&(x, y)| grey_at(&after, x, y)).max().unwrap_or(0) + }; + assert!(speck(ScanCleanSettings { despeckle: false, ..ScanCleanSettings::at(80) }) < 100, "kept without despeckle"); + assert!(speck(ScanCleanSettings::at(80)) >= 200); + // Despeckle alone, at level 0, changes nothing but the specks. + let (mut s, id) = open(grey_scan()); + let before = samples(&s, id); + s.enhance_scanned_pages(id, &[], ScanCleanSettings { level: 0, edge_shadows: false, despeckle: true }).unwrap(); + let after = samples(&s, id); + let changed: Vec = before.data.iter().zip(&after.data).enumerate().filter(|(_, (a, b))| a != b).map(|(i, _)| i).collect(); + let mut specks: Vec = SPECKS.iter().map(|&(x, y)| y * W + x).collect(); + specks.sort_unstable(); + assert_eq!(changed, specks); + // Nothing to do: no undo step, and the report says why. + let (mut s, id) = open(grey_scan()); + let report = s.enhance_scanned_pages(id, &[], ScanCleanSettings { level: 0, edge_shadows: false, despeckle: false }).unwrap(); + assert_eq!(report.images, 0); + assert!(report.pages.is_empty()); + assert_eq!(report.skipped.len(), 1); + assert!(report.skipped[0].reason.contains("already clean"), "{:?}", report.skipped); + assert_eq!(s.get(id).unwrap().can_undo(), None); + assert!(!s.get(id).unwrap().dirty); +} + +#[test] +fn a_black_and_white_scan_stays_one_bit_and_loses_its_specks() { + // 64 × 64, one bit per pixel, 1 = white: a black bar and two specks. + let (w, h) = (64usize, 64usize); + let mut bits = vec![0xFFu8; w / 8 * h]; + let mut black = |x: usize, y: usize| bits[y * w / 8 + x / 8] &= !(0x80 >> (x % 8)); + for y in 20..30 { + for x in 10..50 { + black(x, y); + } + } + black(5, 5); + black(58, 58); + let bytes = Pdf::new() + .obj("<< /Type /Catalog /Pages 2 0 R >>") + .obj("<< /Type /Pages /Kids [3 0 R] /Count 1 >>") + .obj("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 64 64] /Resources << /XObject << /Im0 4 0 R >> >> /Contents 5 0 R >>") + .stream("/Type /XObject /Subtype /Image /Width 64 /Height 64 /ColorSpace /DeviceGray /BitsPerComponent 1", &bits) + .stream("", "q 64 0 0 64 0 0 cm /Im0 Do Q") + .build(); + let (mut s, id) = open(bytes); + // Only Despeckle changes a black and white scan: without it the image is reported. + let report = s.enhance_scanned_pages(id, &[], ScanCleanSettings { despeckle: false, ..ScanCleanSettings::at(100) }).unwrap(); + assert_eq!(report.images, 0); + assert!(report.skipped.first().is_some_and(|k| k.reason.contains("black and white")), "{:?}", report.skipped); + let report = s.enhance_scanned_pages(id, &[], ScanCleanSettings::at(100)).unwrap(); + assert_eq!(report.images, 1); + let Object::Stream(img) = &*object(&s, id, 4) else { panic!("the image") }; + assert_eq!(img.dict.int(b"BitsPerComponent"), Some(1)); + let px = samples(&s, id); + assert!(px.bilevel); + let at = |x: usize, y: usize| px.data[y * w + x]; + assert_eq!((at(5, 5), at(58, 58)), (255, 255), "specks gone"); + assert_eq!((at(10, 20), at(30, 25), at(49, 29)), (0, 0, 0), "the bar stays"); + assert_eq!(px.data.iter().filter(|&&v| v == 0).count(), 40 * 10); +} + +#[test] +fn hostile_images_are_reported_not_a_panic() { + // Each page draws one image over the whole page. + let images: [(&str, Vec); 6] = [ + ("/Width 100000 /Height 100000 /ColorSpace /DeviceGray /BitsPerComponent 8", vec![0; 64]), + ("/Width 300 /Height 400 /ColorSpace /DeviceGray /BitsPerComponent 8 /Filter /DCTDecode", vec![0xFF, 0xD8, 0xFF, 0xE0, 0, 3, 9]), + ("/Width 300 /Height 400 /ColorSpace /DeviceGray /BitsPerComponent 8", vec![200; 1000]), + ("/Width 1 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8", vec![128]), + ("/Width 8 /Height 8 /ColorSpace /DeviceCMYK /BitsPerComponent 8", vec![0; 256]), + ("/Width 300 /Height 400 /ColorSpace /DeviceGray /BitsPerComponent 8 /Mask [250 255]", vec![200; 300 * 400]), + ]; + let n = images.len(); + let kids: String = (0..n).map(|i| format!("{} 0 R ", 3 + 3 * i)).collect(); + let mut pdf = Pdf::new().obj("<< /Type /Catalog /Pages 2 0 R >>").obj(format!("<< /Type /Pages /Kids [{kids}] /Count {n} >>")); + for (i, (dict, data)) in images.iter().enumerate() { + let (img, contents) = (4 + 3 * i, 5 + 3 * i); + pdf = pdf + .obj(format!( + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 400] /Resources << /XObject << /Im0 {img} 0 R >> >> /Contents {contents} 0 R >>" + )) + .stream(&format!("/Type /XObject /Subtype /Image {dict}"), data) + .stream("", "q 300 0 0 400 0 0 cm /Im0 Do Q"); + } + let (mut s, id) = open(pdf.build()); + let report = s.enhance_scanned_pages(id, &[], ScanCleanSettings::at(100)).unwrap(); + assert_eq!(report.images, 0); + assert_eq!(report.not_scanned, vec![3], "a 1 × 1 image is no scan"); + let why: Vec<(usize, &str)> = report.skipped.iter().map(|k| (k.pages[0], k.reason.as_str())).collect(); + assert_eq!(why.len(), 5, "{why:?}"); + assert!(why[0].0 == 0 && why[0].1.contains("80 million"), "{why:?}"); + assert!(why[1].0 == 1 && why[1].1.contains("could not be decoded"), "{why:?}"); + assert!(why[2].0 == 2 && why[2].1.contains("shorter"), "{why:?}"); + assert!(why[3].0 == 4 && why[3].1.contains("colour space"), "{why:?}"); + assert!(why[4].0 == 5 && why[4].1.contains("colour-key mask"), "{why:?}"); + assert_eq!(s.get(id).unwrap().can_undo(), None); + // Pages that don't exist are an error, not a panic. + assert!(matches!(s.enhance_scanned_pages(id, &[99], ScanCleanSettings::default()), Err(EditError::Invalid(_)))); +} + +#[test] +fn protected_documents_keep_their_encryption_and_restrictions() { + let protect = |user: &str, permissions: i32| { + let mut doc = pdfcraft_cos::Document::open(Arc::new(grey_scan())).unwrap(); + doc.set_encryption(&pdfcraft_cos::NewEncryption { + algorithm: pdfcraft_cos::Algorithm::Aes256, + user_password: user, + owner_password: "owner", + permissions, + encrypt_metadata: true, + seed: [7; 32], + }) + .unwrap(); + Arc::new(pdfcraft_cos::write_full(&doc, &Default::default()).unwrap()) + }; + // Opened with the password: cleaned, saved still encrypted, and the clean scan reads back. + let mut s = Session::new(); + let id = s.open("x.pdf", None, protect("pw", -1), Some("pw")).unwrap(); + assert_eq!(s.enhance_scanned_pages(id, &[], ScanCleanSettings::at(100)).unwrap().images, 1); + let saved = s.save_bytes(id).unwrap(); + assert!(pdfcraft_cos::Document::open(saved.clone()).is_err(), "still protected"); + let id2 = s.open("y.pdf", None, saved, Some("pw")).unwrap(); + assert!(grey_at(&samples(&s, id2), 3, 40) >= 235); + // Changes withheld: refused before any work. + let id = s.open("r.pdf", None, protect("", 0b0100), None).unwrap(); + assert_eq!(s.enhance_scanned_pages(id, &[], ScanCleanSettings::default()).unwrap_err(), EditError::NotPermitted("changes to the document")); + assert!(!s.get(id).unwrap().dirty); +} + +#[test] +fn a_result_for_an_older_state_is_refused() { + let (mut s, id) = open(grey_scan()); + let job = s.scan_clean_job(id, &[], ScanCleanSettings::at(70)).unwrap(); + assert_eq!(job.total(), 1); + let cleaned = job.run(|_, _| true).unwrap(); + s.apply(id, Edit::RotatePages { pages: vec![1], degrees: 90 }).unwrap(); + assert!(matches!(s.apply_scan_clean(id, cleaned), Err(EditError::Invalid(_)))); + assert_eq!(s.get(id).unwrap().can_undo(), Some("Rotate page")); + // Cancelled from the progress callback. + let job = s.scan_clean_job(id, &[], ScanCleanSettings::at(70)).unwrap(); + assert!(matches!(job.run(|_, _| false), Err(EditError::Cancelled))); +} + +#[test] +fn edge_shadow_removal_whitens_the_gutter_background_removal_alone_leaves() { + let gutter = |edge_shadows: bool| { + let (mut s, id) = open(grey_scan()); + s.enhance_scanned_pages(id, &[0], ScanCleanSettings { edge_shadows, ..ScanCleanSettings::at(100) }).unwrap(); + let after = samples(&s, id); + [(3, 40), (3, 380), (10, 200)].map(|(x, y)| grey_at(&after, x, y)) + }; + assert!(gutter(false).iter().any(|&v| v < 200), "{:?}", gutter(false)); + assert!(gutter(true).iter().all(|&v| v >= 235), "{:?}", gutter(true)); + // On its own, at level 0, it lightens the gutter's paper to the paper beside it. + let (mut s, id) = open(grey_scan()); + let before = samples(&s, id); + s.enhance_scanned_pages(id, &[0], ScanCleanSettings { level: 0, edge_shadows: true, despeckle: false }).unwrap(); + let after = samples(&s, id); + assert!(grey_at(&after, 3, 40).abs_diff(grey_at(&before, 150, 40)) <= 4, "{} vs {}", grey_at(&after, 3, 40), grey_at(&before, 150, 40)); + assert_eq!(grey_at(&after, 150, 40), grey_at(&before, 150, 40), "the middle is left alone"); +} diff --git a/crates/engine/src/scan_fields.rs b/crates/engine/src/scan_fields.rs index 8287743fc5..769beaefca 100644 --- a/crates/engine/src/scan_fields.rs +++ b/crates/engine/src/scan_fields.rs @@ -25,24 +25,33 @@ const MIN_WORD_HEIGHT: f64 = 4.0; /// Whether `page` is a picture of a page: images cover most of it. pub(crate) fn is_scan(doc: &Document, page: usize, info: &PageInfo) -> bool { + scan_images(doc, page, info).is_some() +} + +/// The raster images (larger than one pixel) `page` draws, each with the share of the page (its +/// crop box) it covers, when together they cover most of it ([`is_scan`]); `None` otherwise. +/// Enhance ▸ Scanned document cleans these up ([`crate::scan_clean`]). +pub(crate) fn scan_images(doc: &Document, page: usize, info: &PageInfo) -> Option> { let [cx0, cy0, cx1, cy1] = info.crop.map(f64::from); let page_area = (cx1 - cx0) * (cy1 - cy0); if !page_area.is_finite() || page_area <= 0.0 { - return false; + return None; } - let covered: f64 = doc + let images: Vec<(pdfcraft_edit::PageImage, f64)> = doc .page_images(page) - .iter() + .into_iter() .filter(|i| !i.is_form && i.width > 1 && i.height > 1) .map(|i| { let r = i.visible_rect.unwrap_or(i.rect); let w = (r[2].min(cx1) - r[0].max(cx0)).max(0.0); let h = (r[3].min(cy1) - r[1].max(cy0)).max(0.0); - w * h + let share = w * h / page_area; + (i, share) }) - .filter(|a| a.is_finite()) - .sum(); - covered >= COVERED * page_area + .filter(|(_, share)| share.is_finite()) + .collect(); + let covered: f64 = images.iter().map(|(_, share)| share).sum(); + (covered >= COVERED).then_some(images) } /// The page as displayed, in points with y up: where detection on a scan works. diff --git a/crates/optimize/src/images.rs b/crates/optimize/src/images.rs index b9761b85fc..f19d7d8fc7 100644 --- a/crates/optimize/src/images.rs +++ b/crates/optimize/src/images.rs @@ -8,7 +8,7 @@ use pdfcraft_cos::{DecodeBudget, Dict, Document, ObjRef, Object, Stream}; use crate::{Compression, ImageSettings, OptimizeError, Report, Settings, Stage}; /// Images with more pixels than this are left alone (memory). -const MAX_PIXELS: u64 = 80_000_000; +pub(crate) const MAX_PIXELS: u64 = 80_000_000; /// The resources of a page, inherited from the page tree if the page has none. pub(crate) fn page_resources(doc: &Document, page: ObjRef) -> Option { @@ -203,7 +203,7 @@ pub(crate) fn components(doc: &Document, cs: &Object) -> Option { } } -fn is_dct(s: &Stream) -> Option { +pub(crate) fn is_dct(s: &Stream) -> Option { match s.dict.get(b"Filter") { None => Some(false), Some(Object::Name(n)) => match n.as_slice() { @@ -288,7 +288,7 @@ pub(crate) fn pixels(doc: &Document, s: &Stream, n: usize) -> Option> { /// /ColorTransform contradict each other (PdfCraft's renderer lets /ColorTransform 0 win where /// the standard lets the marker win), either holds a value other than 0 or 1, or the parameters /// or the JPEG's segments can't be read. -fn ycc_encoded(doc: &Document, s: &Stream) -> Option { +pub(crate) fn ycc_encoded(doc: &Document, s: &Stream) -> Option { let entry = color_transform(doc, s)?; match adobe_transform(&s.raw)? { Some(t @ (0 | 1)) => entry.is_none_or(|c| c == i64::from(t)).then_some(t == 1), @@ -412,7 +412,7 @@ pub(crate) fn resample(px: &[u8], n: usize, w: u32, h: u32, nw: u32, nh: u32) -> }) } -fn jpeg(px: &[u8], n: usize, w: u32, h: u32, quality: u8) -> Option> { +pub(crate) fn jpeg(px: &[u8], n: usize, w: u32, h: u32, quality: u8) -> Option> { let mut out = Vec::new(); let mut enc = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut out, quality.clamp(1, 100)); let ty = if n == 1 { image::ExtendedColorType::L8 } else { image::ExtendedColorType::Rgb8 }; @@ -435,7 +435,7 @@ pub(crate) fn decodable(doc: &Document, s: &Stream) -> Option { /// Whether `decode` is the default decode array for `n` components ([0 1] each), which changes /// nothing. Scanners write it on page images (#490). -fn is_identity_decode(doc: &Document, decode: &Object, n: usize) -> bool { +pub(crate) fn is_identity_decode(doc: &Document, decode: &Object, n: usize) -> bool { let decode = doc.resolve(decode); let Some(a) = decode.as_array() else { return false }; a.len() == 2 * n && a.chunks(2).all(|p| matches!(p, [lo, hi] if doc.resolve(lo).as_f64() == Some(0.0) && doc.resolve(hi).as_f64() == Some(1.0))) diff --git a/crates/optimize/src/lib.rs b/crates/optimize/src/lib.rs index 9f9fcdf2be..cb6b537115 100644 --- a/crates/optimize/src/lib.rs +++ b/crates/optimize/src/lib.rs @@ -27,6 +27,7 @@ mod audit; pub mod fragments; mod images; mod links; +mod samples; mod unused; pub use audit::{SpaceCategory, SpaceUse, audit_space}; @@ -35,6 +36,7 @@ use pdfcraft_cos::{Dict, Document, MAX_PAGE_DECODED, ObjRef, Object, Stream, ser use pdfcraft_filters::{Filter, Params}; pub use images::effective_resolutions; +pub use samples::{NotDecoded, Samples, decode_samples, encode_samples}; #[derive(Debug, thiserror::Error)] pub enum OptimizeError { diff --git a/crates/optimize/src/samples.rs b/crates/optimize/src/samples.rs new file mode 100644 index 0000000000..7f402b51b8 --- /dev/null +++ b/crates/optimize/src/samples.rs @@ -0,0 +1,273 @@ +//! An image's samples read out and written back at the same size, for tools that change the +//! pixels of an image in place (Scan & OCR ▸ Enhance ▸ Scanned document). The image dictionary +//! stays as it was apart from the encoding: its size, colour space, masks, `/Interpolate`, +//! `/Intent`, `/Metadata` and every other entry are kept. + +use pdfcraft_cos::{Document, Object, Stream}; + +use crate::images::{MAX_PIXELS, components, is_dct, is_identity_decode, jpeg, pixels, ycc_encoded}; + +/// An image's samples, one byte each, row-major. +#[derive(Clone, Debug, PartialEq)] +pub struct Samples { + pub width: u32, + pub height: u32, + /// 1 (grey) or 3 (RGB). + pub components: usize, + /// The samples. A bi-level image's are 0 (black) or 255 (white) as drawn, whatever its + /// decode array. + pub data: Vec, + /// Stored as a JPEG (written back as one); otherwise written back losslessly (Flate). + pub jpeg: bool, + /// One bit per pixel (written back as one). + pub bilevel: bool, + /// A bi-level image whose 1 bits are black (decode array `[1 0]`): written back the same way, + /// so its decode array stays true. + pub inverted: bool, +} + +/// Why an image's samples were not read. Each says what the image is, so a report can name it. +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +pub enum NotDecoded { + #[error("it is a stencil mask, not a picture")] + StencilMask, + #[error("its colour space is not grey or RGB (CMYK, indexed and special colours are not supported yet)")] + ColourSpace, + #[error("it has {0} bits per sample (8, and 1 for black and white, are supported)")] + BitDepth(i64), + #[error("its decode array remaps the samples")] + Decode, + #[error("it is compressed in a format PdfCraft cannot read back yet (JPEG 2000, JBIG2 or CCITT fax)")] + Filter, + #[error("its JPEG colour transform is ambiguous, so readers may disagree about its colours")] + Ambiguous, + #[error("it is larger than 80 million pixels")] + TooLarge, + #[error("its data could not be decoded, or is shorter than its size says")] + Corrupt, +} + +/// Read the samples of image stream `s` (a dictionary with `/Subtype /Image`). +pub fn decode_samples(doc: &Document, s: &Stream) -> Result { + let d = &s.dict; + if matches!(d.get(b"ImageMask").map(|v| doc.resolve(v)).as_deref(), Some(Object::Bool(true))) { + return Err(NotDecoded::StencilMask); + } + let n = d.get(b"ColorSpace").and_then(|cs| components(doc, cs)).ok_or(NotDecoded::ColourSpace)?; + let bpc = d.get(b"BitsPerComponent").map(|v| doc.resolve(v)).and_then(|v| v.as_int()).unwrap_or(0); + let bilevel = match bpc { + 8 => false, + 1 if n == 1 => true, + other => return Err(NotDecoded::BitDepth(other)), + }; + let jpeg = is_dct(s).ok_or(NotDecoded::Filter)?; + let width = d.get(b"Width").map(|v| doc.resolve(v)).and_then(|v| v.as_int()).and_then(|v| u32::try_from(v).ok()).unwrap_or(0); + let height = d.get(b"Height").map(|v| doc.resolve(v)).and_then(|v| v.as_int()).and_then(|v| u32::try_from(v).ok()).unwrap_or(0); + if width == 0 || height == 0 { + return Err(NotDecoded::Corrupt); + } + if u64::from(width) * u64::from(height) > MAX_PIXELS { + return Err(NotDecoded::TooLarge); + } + if bilevel { + if jpeg { + // A JPEG has 8 bits per sample: the dictionary contradicts the data. + return Err(NotDecoded::Corrupt); + } + let inverted = match d.get(b"Decode") { + None => false, + Some(a) if is_identity_decode(doc, a, 1) => false, + Some(a) if is_inverted_decode(doc, a) => true, + Some(_) => return Err(NotDecoded::Decode), + }; + let data = unpack_bits(s, width, height, inverted).ok_or(NotDecoded::Corrupt)?; + return Ok(Samples { width, height, components: 1, data, jpeg: false, bilevel: true, inverted }); + } + if d.get(b"Decode").is_some_and(|a| !is_identity_decode(doc, a, n)) { + return Err(NotDecoded::Decode); + } + if jpeg && n == 3 && ycc_encoded(doc, s).is_none() { + return Err(NotDecoded::Ambiguous); + } + let data = pixels(doc, s, n).ok_or(NotDecoded::Corrupt)?; + Ok(Samples { width, height, components: n, data, jpeg, bilevel: false, inverted: false }) +} + +/// `[1 0]`: a bi-level image whose 1 bits are black. +fn is_inverted_decode(doc: &Document, decode: &Object) -> bool { + let decode = doc.resolve(decode); + matches!(decode.as_array().map(Vec::as_slice), Some([a, b]) if doc.resolve(a).as_f64() == Some(1.0) && doc.resolve(b).as_f64() == Some(0.0)) +} + +/// The bytes a `w` × `h` bi-level image takes: each row padded to whole bytes. +fn packed_len(w: u32, h: u32) -> Option { + usize::try_from(w).ok()?.div_ceil(8).checked_mul(usize::try_from(h).ok()?) +} + +/// A bi-level image's pixels, 0 (black) or 255 (white) as drawn. +fn unpack_bits(s: &Stream, w: u32, h: u32, inverted: bool) -> Option> { + let stride = usize::try_from(w).ok()?.div_ceil(8); + let need = packed_len(w, h)?; + // Some writers pad the data; anything far longer than the image needs is not this image. + let bits = s.decoded_within(need.saturating_add(need / 2).saturating_add(4096)).ok()?; + let bits = bits.get(..need)?; + let (w, h) = (usize::try_from(w).ok()?, usize::try_from(h).ok()?); + let mut out = Vec::with_capacity(w.checked_mul(h)?); + for row in bits.chunks(stride).take(h) { + for x in 0..w { + let byte = row.get(x / 8).copied().unwrap_or(0); + let one = byte & (0x80 >> (x % 8)) != 0; + // Without a decode array 0 is black and 1 white; [1 0] swaps them. + out.push(if one != inverted { 255 } else { 0 }); + } + } + Some(out) +} + +/// The pixels (0 black … 255 white, threshold at half) packed back to bits. +fn pack_bits(px: &[u8], w: u32, h: u32, inverted: bool) -> Option> { + let stride = usize::try_from(w).ok()?.div_ceil(8); + let mut out = vec![0u8; packed_len(w, h)?]; + let (w, h) = (usize::try_from(w).ok()?, usize::try_from(h).ok()?); + if px.len() != w.checked_mul(h)? { + return None; + } + for (y, row) in px.chunks(w.max(1)).enumerate() { + for (x, &v) in row.iter().enumerate() { + let white = v >= 128; + if white != inverted + && let Some(b) = out.get_mut(y * stride + x / 8) + { + *b |= 0x80 >> (x % 8); + } + } + } + Some(out) +} + +/// Image stream `old` with its samples replaced by `samples` (same size and components): a JPEG +/// stays a JPEG (at `quality`, 1–100), anything else becomes Flate, and a bi-level image stays +/// one bit per pixel with its decode array. Filter parameters go (the new data needs none); every +/// other entry stays. `None` if the samples don't fit the image. +pub fn encode_samples(old: &Stream, samples: &Samples, quality: u8) -> Option { + let Samples { width, height, components: n, data, jpeg: as_jpeg, bilevel, inverted } = samples; + let (w, h, n) = (*width, *height, *n); + if !(n == 1 || n == 3) || data.len() != usize::try_from(w).ok()?.checked_mul(usize::try_from(h).ok()?)?.checked_mul(n)? { + return None; + } + let mut d = old.dict.clone(); + for k in [&b"Filter"[..], b"DecodeParms", b"Length", b"DL"] { + d.remove(k); + } + if *bilevel { + if n != 1 { + return None; + } + d.set(b"BitsPerComponent".to_vec(), Object::Int(1)); + return Some(Stream::flate(d, &pack_bits(data, w, h, *inverted)?)); + } + d.set(b"BitsPerComponent".to_vec(), Object::Int(8)); + if *as_jpeg { + let bytes = jpeg(data, n, w, h, quality)?; + d.set(b"Filter".to_vec(), Object::name("DCTDecode")); + Some(Stream::from_raw(d, bytes)) + } else { + Some(Stream::flate(d, data)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use pdfcraft_cos::Dict; + + fn image(dict: &[(&str, Object)], raw: Vec) -> Stream { + let mut d = Dict::new(); + d.set(b"Type".to_vec(), Object::name("XObject")); + d.set(b"Subtype".to_vec(), Object::name("Image")); + for (k, v) in dict { + d.set(k.as_bytes().to_vec(), v.clone()); + } + Stream::from_raw(d, raw) + } + + fn doc() -> Document { + let pdf = b"%PDF-1.7\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n2 0 obj\n<< /Type /Pages /Kids [] /Count 0 >>\nendobj\ntrailer\n<< /Root 1 0 R >>\n%%EOF\n"; + Document::open(std::sync::Arc::new(pdf.to_vec())).unwrap() + } + + #[test] + fn grey_samples_round_trip_losslessly_and_keep_the_other_entries() { + let px: Vec = (0..12 * 5).map(|i| (i * 4) as u8).collect(); + let s = image( + &[ + ("Width", Object::Int(12)), + ("Height", Object::Int(5)), + ("ColorSpace", Object::name("DeviceGray")), + ("BitsPerComponent", Object::Int(8)), + ("Interpolate", Object::Bool(true)), + ], + px.clone(), + ); + let got = decode_samples(&doc(), &s).unwrap(); + assert_eq!((got.width, got.height, got.components, got.jpeg, got.bilevel), (12, 5, 1, false, false)); + assert_eq!(got.data, px); + let back = encode_samples(&s, &got, 90).unwrap(); + assert_eq!(back.dict.name(b"Filter"), Some(&b"FlateDecode"[..])); + assert_eq!(back.dict.get(b"Interpolate"), Some(&Object::Bool(true))); + assert_eq!(decode_samples(&doc(), &back).unwrap().data, px); + } + + #[test] + fn bilevel_samples_keep_their_decode_array_and_bits() { + // 10 × 2: first row black on the left half, second row all white; [1 0] makes 1 black. + let raw = vec![0b1111_1000, 0b0000_0000, 0, 0]; + let s = image( + &[ + ("Width", Object::Int(10)), + ("Height", Object::Int(2)), + ("ColorSpace", Object::name("DeviceGray")), + ("BitsPerComponent", Object::Int(1)), + ("Decode", Object::Array(vec![Object::Int(1), Object::Int(0)])), + ], + raw.clone(), + ); + let got = decode_samples(&doc(), &s).unwrap(); + assert!(got.bilevel); + assert_eq!(&got.data[..10], &[0, 0, 0, 0, 0, 255, 255, 255, 255, 255]); + assert!(got.data[10..].iter().all(|&v| v == 255)); + assert!(got.inverted); + let back = encode_samples(&s, &got, 90).unwrap(); + assert_eq!(back.dict.int(b"BitsPerComponent"), Some(1)); + assert!(back.dict.get(b"Decode").is_some()); + assert_eq!(back.decoded().unwrap(), raw); + } + + #[test] + fn what_is_not_supported_says_why_and_never_panics() { + let base = |extra: &[(&str, Object)], raw: Vec| { + let mut d: Vec<(&str, Object)> = vec![ + ("Width", Object::Int(4)), + ("Height", Object::Int(4)), + ("ColorSpace", Object::name("DeviceGray")), + ("BitsPerComponent", Object::Int(8)), + ]; + d.extend(extra.iter().cloned()); + decode_samples(&doc(), &image(&d, raw)) + }; + assert_eq!(base(&[("ColorSpace", Object::name("DeviceCMYK"))], vec![0; 64]), Err(NotDecoded::ColourSpace)); + assert_eq!(base(&[("BitsPerComponent", Object::Int(4))], vec![0; 8]), Err(NotDecoded::BitDepth(4))); + assert_eq!(base(&[("Filter", Object::name("JBIG2Decode"))], vec![0; 8]), Err(NotDecoded::Filter)); + assert_eq!(base(&[("ImageMask", Object::Bool(true))], vec![0; 8]), Err(NotDecoded::StencilMask)); + assert_eq!(base(&[("Decode", Object::Array(vec![Object::Int(1), Object::Int(0)]))], vec![0; 16]), Err(NotDecoded::Decode)); + // Too short, a huge size, a broken JPEG, no size at all. + assert_eq!(base(&[], vec![0; 5]), Err(NotDecoded::Corrupt)); + assert_eq!(base(&[("Width", Object::Int(1 << 30)), ("Height", Object::Int(1 << 30))], vec![0; 16]), Err(NotDecoded::TooLarge)); + assert_eq!(base(&[("Filter", Object::name("DCTDecode"))], vec![0xFF, 0xD8, 1, 2, 3]), Err(NotDecoded::Corrupt)); + assert_eq!(base(&[("Width", Object::Int(-3))], vec![0; 16]), Err(NotDecoded::Corrupt)); + // Samples that don't fit the image are refused when written back. + let s = image(&[("Width", Object::Int(4)), ("Height", Object::Int(4))], vec![0; 16]); + let wrong = Samples { width: 4, height: 4, components: 1, data: vec![0; 15], jpeg: false, bilevel: false, inverted: false }; + assert!(encode_samples(&s, &wrong, 90).is_none()); + } +} diff --git a/crates/ui-egui/src/app.rs b/crates/ui-egui/src/app.rs index 8925a0dca1..1fd46a956f 100644 --- a/crates/ui-egui/src/app.rs +++ b/crates/ui-egui/src/app.rs @@ -179,6 +179,8 @@ impl PdfCraftApp { stamp_draft: Default::default(), optimize_draft: OptimizeDraft::default(), optimize_run: None, + scan_clean_draft: scan_clean_ui::ScanCleanDraft::default(), + scan_clean_run: None, progress_notice: None, page_clipboard: None, grid_drop: None, @@ -508,6 +510,7 @@ impl eframe::App for PdfCraftApp { self.auto_ocr_scanned_page(); self.poll_ocr(); self.poll_optimize(); + self.poll_scan_clean(); self.poll_action(); self.process_file_requests(); #[cfg(not(target_arch = "wasm32"))] diff --git a/crates/ui-egui/src/app/control.rs b/crates/ui-egui/src/app/control.rs index bb5a65cb34..3fe737b40b 100644 --- a/crates/ui-egui/src/app/control.rs +++ b/crates/ui-egui/src/app/control.rs @@ -101,6 +101,7 @@ impl PdfCraftApp { "export-all-images" => Some(Dialog::Export(export_ui::ExportKind::AllImages)), "accessibility-options" => Some(Dialog::AccessibilityOptions), "recognize-text" => Some(Dialog::RecognizeText), + "enhance-scan" => Some(Dialog::EnhanceScan), "js-console" => Some(Dialog::JsConsole), "document-js" => Some(Dialog::DocumentJs), "preferences" => Some(Dialog::Preferences), diff --git a/crates/ui-egui/src/app/frame.rs b/crates/ui-egui/src/app/frame.rs index e0a8370c60..3d6f493e10 100644 --- a/crates/ui-egui/src/app/frame.rs +++ b/crates/ui-egui/src/app/frame.rs @@ -5,6 +5,7 @@ impl PdfCraftApp { pub(super) fn show_progress(&mut self, ctx: &egui::Context) { if widgets::progress_notice(self, ctx) { self.cancel_optimize(); + self.cancel_scan_clean(); } } diff --git a/crates/ui-egui/src/commands.rs b/crates/ui-egui/src/commands.rs index acccbcc55a..dec99e8b58 100644 --- a/crates/ui-egui/src/commands.rs +++ b/crates/ui-egui/src/commands.rs @@ -500,6 +500,7 @@ impl PdfCraftApp { "export.image" => self.dialog = Some(Dialog::Export(crate::export_ui::ExportKind::Image)), "export.text" => self.dialog = Some(Dialog::Export(crate::export_ui::ExportKind::Text)), "ocr.recognize" => self.dialog = Some(Dialog::RecognizeText), + "ocr.enhance" => self.dialog = Some(Dialog::EnhanceScan), "ocr.recognize_batch" => self.ocr_files_dialog(), "edit.edit_text" => { self.quick_tool = crate::QuickTool::EditText; diff --git a/crates/ui-egui/src/dialogs.rs b/crates/ui-egui/src/dialogs.rs index c21604a4da..a77b0e4d19 100644 --- a/crates/ui-egui/src/dialogs.rs +++ b/crates/ui-egui/src/dialogs.rs @@ -56,6 +56,7 @@ struct Pending { open_revision: Option, a11y_now: bool, ocr_now: bool, + scan_clean_now: bool, compare_now: bool, images_now: bool, stamp_now: bool, @@ -372,6 +373,12 @@ fn dialog_body(ui: &mut egui::Ui, app: &mut PdfCraftApp, dialog: Dialog, req: &m req.close = go || cancel; return false; } + Dialog::EnhanceScan => { + let (go, cancel) = crate::scan_clean_ui::body(ui, app, &t); + req.scan_clean_now = go; + req.close = go || cancel; + return false; + } Dialog::AccessibilityOptions => { let (start, cancel) = crate::a11y_ui::options_body(ui, app, &t); req.a11y_now = start; @@ -1818,6 +1825,9 @@ fn finish_close(app: &mut PdfCraftApp, dialog: Dialog, modal_closed: bool, req: if req.ocr_now { app.start_ocr(); } + if req.scan_clean_now { + app.start_scan_clean(); + } if req.compare_now { app.run_compare(); } diff --git a/crates/ui-egui/src/i18n/de.tsv b/crates/ui-egui/src/i18n/de.tsv index 8586a21d43..076fb6bf66 100644 --- a/crates/ui-egui/src/i18n/de.tsv +++ b/crates/ui-egui/src/i18n/de.tsv @@ -2558,3 +2558,22 @@ animation effects Off Aus Rotate the selected pages clockwise Ausgewählte Seiten im Uhrzeigersinn drehen Extract the selected pages… Ausgewählte Seiten extrahieren… Delete the selected pages Ausgewählte Seiten löschen + Enhance scanned file… Gescannte Datei verbessern… + Enhance Scanned File Gescannte Datei verbessern + Cleans up the scanned pictures in place: recognized text, comments and links on the pages stay as they are. Bereinigt die gescannten Bilder an Ort und Stelle: erkannter Text, Kommentare und Links auf den Seiten bleiben erhalten. + Filters Filter + Background removal Hintergrund entfernen + Evens out the paper so a grey or yellowed background turns white, and darkens faint strokes. 0 is off. Gleicht das Papier aus, sodass ein grauer oder vergilbter Hintergrund weiß wird, und dunkelt blasse Striche nach. 0 ist aus. + Edge shadow removal Randschatten entfernen + Lightens shadows along the edges of the scan, such as a book's gutter or a copier's dark border Hellt Schatten an den Rändern des Scans auf, etwa den Bundsteg eines Buchs oder den dunklen Rand eines Kopierers + Despeckle Flecken entfernen + Removes isolated dark specks such as dust and scanner noise. The only filter black and white scans take. Entfernt einzelne dunkle Flecken wie Staub und Scannerrauschen. Der einzige Filter für Schwarzweiß-Scans. + Enhance Verbessern + No scanned pages were cleaned up. Es wurden keine gescannten Seiten bereinigt. + Scanned pages are already being enhanced Gescannte Seiten werden bereits verbessert + Couldn't enhance the scanned pages Die gescannten Seiten konnten nicht verbessert werden + Enhancing scans… image {d} of {t} Scans werden verbessert… Bild {d} von {t} + Enhancing the scanned pages was cancelled Das Verbessern der gescannten Seiten wurde abgebrochen +@plural Cleaned up the scan on {n} page.|Cleaned up the scans on {n} pages. Scan auf {n} Seite bereinigt.|Scans auf {n} Seiten bereinigt. +@plural {n} page is not a scan.|{n} pages are not scans. {n} Seite ist kein Scan.|{n} Seiten sind keine Scans. +@plural {n} image was left as it was (page {p}: {why}).|{n} images were left as they were (page {p}: {why}, …). {n} Bild blieb unverändert (Seite {p}: {why}).|{n} Bilder blieben unverändert (Seite {p}: {why}, …). diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 65bc7e4b1b..e61e3827a5 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -162,6 +162,7 @@ use workflows::js_ui; use workflows::ocr_ui; use workflows::optimize_ui; use workflows::print_ui; +use workflows::scan_clean_ui; use workflows::web_print; /// Files dropped on a document's page grid or its Pages panel. @@ -494,6 +495,9 @@ pub struct PdfCraftApp { pub optimize_draft: OptimizeDraft, /// The running optimization (Optimize PDF ▸ Advanced optimization). pub optimize_run: Option, + /// Scan & OCR ▸ Enhance scanned file: the dialog's choices and the running clean-up. + pub scan_clean_draft: scan_clean_ui::ScanCleanDraft, + pub scan_clean_run: Option, /// A background job's progress card (see [`widgets::progress_notice`]). pub progress_notice: Option, /// Pages copied or cut in Organize Pages, ready to paste (into any document). diff --git a/crates/ui-egui/src/state.rs b/crates/ui-egui/src/state.rs index dd3b3e90aa..72d3b6ddf2 100644 --- a/crates/ui-egui/src/state.rs +++ b/crates/ui-egui/src/state.rs @@ -162,6 +162,8 @@ pub enum Dialog { AccessibilityOptions, /// Scan & OCR ▸ Recognize text. RecognizeText, + /// Scan & OCR ▸ Enhance scanned file. + EnhanceScan, /// The JavaScript console (⌘J). JsConsole, /// Document JavaScripts. diff --git a/crates/ui-egui/src/workflows.rs b/crates/ui-egui/src/workflows.rs index 63686a70cc..9314914ec4 100644 --- a/crates/ui-egui/src/workflows.rs +++ b/crates/ui-egui/src/workflows.rs @@ -11,4 +11,5 @@ pub(super) mod js_ui; pub(super) mod ocr_ui; pub(super) mod optimize_ui; pub(super) mod print_ui; +pub(super) mod scan_clean_ui; pub(super) mod web_print; diff --git a/crates/ui-egui/src/workflows/scan_clean_ui.rs b/crates/ui-egui/src/workflows/scan_clean_ui.rs new file mode 100644 index 0000000000..a275b5af20 --- /dev/null +++ b/crates/ui-egui/src/workflows/scan_clean_ui.rs @@ -0,0 +1,253 @@ +//! Scan & OCR ▸ Enhance scanned file: the dialog (pages, background removal level, edge +//! shadows, despeckle) and the background run with its progress card. The scans are cleaned +//! on a worker thread ([`pdfcraft_engine::scan_clean`]); the result is applied as one undoable +//! step when it is done, unless the document changed meanwhile. + +use std::sync::{Arc, Mutex}; + +use egui::{Align, Layout}; +use pdfcraft_engine::scan_clean::{ScanCleanReport, ScanCleanSettings, ScanCleaned}; +use pdfcraft_engine::{DocId, EditError}; + +use crate::ocr_ui::OcrPages; +use crate::theme::{self, Tokens}; +use crate::{PdfCraftApp, widgets}; + +/// The dialog's choices (kept between runs). +#[derive(Clone, Debug, PartialEq)] +pub struct ScanCleanDraft { + pub pages: OcrPages, + pub from: usize, + pub to: usize, + pub settings: ScanCleanSettings, +} + +impl Default for ScanCleanDraft { + fn default() -> Self { + ScanCleanDraft { pages: OcrPages::All, from: 1, to: 1, settings: ScanCleanSettings::default() } + } +} + +/// Progress of a run: images done, total, the result once finished, and a cancel request. +#[derive(Default)] +pub struct ScanCleanProgress { + pub done: usize, + pub total: usize, + pub result: Option>, + pub cancel: bool, +} + +pub struct ScanCleanRun { + pub doc: DocId, + pub progress: Arc>, +} + +pub(crate) fn body(ui: &mut egui::Ui, app: &mut PdfCraftApp, t: &Tokens) -> (bool, bool) { + let pages = app.active_ids().and_then(|(_, id)| app.session.get(id)).map_or(1, |d| d.info.pages.len().max(1)); + let d = &mut app.scan_clean_draft; + d.to = d.to.clamp(1, pages); + d.from = d.from.clamp(1, d.to); + ui.label(egui::RichText::new(tl!("Enhance Scanned File")).font(theme::semibold(18.0))); + ui.add_space(4.0); + ui.label( + egui::RichText::new(tl!("Cleans up the scanned pictures in place: recognized text, comments and links on the pages stay as they are.")) + .small() + .color(t.text_muted), + ); + ui.add_space(8.0); + let group = |ui: &mut egui::Ui, title: &str, body: &mut dyn FnMut(&mut egui::Ui)| { + ui.label(egui::RichText::new(tl!(title)).font(theme::semibold(13.0))); + egui::Frame::new().fill(t.hover).corner_radius(egui::CornerRadius::same(6)).inner_margin(egui::Margin::same(10)).show(ui, |ui| { + ui.set_width(ui.available_width()); + body(ui); + }); + ui.add_space(8.0); + }; + group(ui, "Pages", &mut |ui| { + widgets::radio_value(ui, &mut d.pages, OcrPages::All, tl!("All pages")); + widgets::radio_value(ui, &mut d.pages, OcrPages::Current, tl!("Current page")); + ui.horizontal(|ui| { + let from = widgets::radio_value(ui, &mut d.pages, OcrPages::Range, tl!("From")); + let on = d.pages == OcrPages::Range; + ui.add_enabled(on, egui::DragValue::new(&mut d.from).range(1..=pages)).labelled_by(from.id); + let to = ui.label(tl!("to")); + ui.add_enabled(on, egui::DragValue::new(&mut d.to).range(1..=pages)).labelled_by(to.id); + }); + }); + group(ui, "Filters", &mut |ui| { + ui.horizontal(|ui| { + let label = ui.label(tl!("Background removal")); + ui.spacing_mut().slider_width = 220.0; + ui.add(egui::Slider::new(&mut d.settings.level, 0..=pdfcraft_engine::camera_clean::MAX_LEVEL).trailing_fill(true)) + .labelled_by(label.id) + .on_hover_text(tl!("Evens out the paper so a grey or yellowed background turns white, and darkens faint strokes. 0 is off.")); + }); + ui.add(widgets::checkbox(&mut d.settings.edge_shadows, tl!("Edge shadow removal"))) + .on_hover_text(tl!("Lightens shadows along the edges of the scan, such as a book's gutter or a copier's dark border")); + ui.add(widgets::checkbox(&mut d.settings.despeckle, tl!("Despeckle"))) + .on_hover_text(tl!("Removes isolated dark specks such as dust and scanner noise. The only filter black and white scans take.")); + }); + ui.add_space(6.0); + let (mut go, mut cancel) = (false, false); + ui.horizontal(|ui| { + ui.with_layout(Layout::right_to_left(Align::Center), |ui| { + let ready = d.settings.changes_something(); + if ui.add_enabled_ui(ready, |ui| widgets::pill_button(ui, tl!("Enhance"), true)).inner.clicked() { + go = true; + } + if widgets::pill_button(ui, tl!("Cancel"), false).clicked() { + cancel = true; + } + }) + }); + (go, cancel) +} + +/// What a finished run says: what was cleaned, what was not a scan, and what was left and why. +fn summary(r: &ScanCleanReport) -> String { + let lang = crate::i18n::current(); + let mut out = if r.pages.is_empty() { + tl!("No scanned pages were cleaned up.").to_string() + } else { + crate::i18n::trn(lang, r.pages.len() as u64, "Cleaned up the scan on {n} page.", "Cleaned up the scans on {n} pages.") + }; + if !r.not_scanned.is_empty() { + out.push(' '); + out.push_str(&crate::i18n::trn(lang, r.not_scanned.len() as u64, "{n} page is not a scan.", "{n} pages are not scans.")); + } + if let Some(first) = r.skipped.first() { + out.push(' '); + let page = first.pages.first().map_or_else(String::new, |p| (p + 1).to_string()); + // The reason is the engine's own sentence, in English. + out.push_str(&crate::i18n::fmt( + &crate::i18n::trn( + lang, + r.skipped.len() as u64, + "{n} image was left as it was (page {p}: {why}).", + "{n} images were left as they were (page {p}: {why}, …).", + ), + &[("p", &page), ("why", &first.reason)], + )); + } + out +} + +impl PdfCraftApp { + /// Clean up the scans on the pages chosen in the dialog, in the background. + pub fn start_scan_clean(&mut self) { + let Some((vi, id)) = self.active_ids() else { return }; + if self.scan_clean_run.is_some() { + self.notify_tr("Scanned pages are already being enhanced"); + return; + } + // What's typed in a form field is part of the document (#166). + if !self.commit_form_typing() { + return; + } + let d = &self.scan_clean_draft; + let pages: Vec = match d.pages { + OcrPages::All => Vec::new(), + OcrPages::Current => self.views.get(vi).map(|v| vec![v.current]).unwrap_or_default(), + OcrPages::Range => (d.from.saturating_sub(1)..d.to).collect(), + }; + let job = match self.session.scan_clean_job(id, &pages, d.settings) { + Ok(job) => job, + Err(e) => return self.notify(e.to_string()), + }; + let progress = Arc::new(Mutex::new(ScanCleanProgress { total: job.total(), ..Default::default() })); + let p = progress.clone(); + let work = move || { + let result = job.run(|done, total| { + let Ok(mut s) = p.lock() else { return false }; + s.done = done; + s.total = total; + !s.cancel + }); + if let Ok(mut s) = p.lock() { + s.result = Some(result); + } + }; + #[cfg(not(target_arch = "wasm32"))] + if self.run_inline { + work(); + } else if let Err(e) = std::thread::Builder::new().name("pdfcraft-scan-clean".into()).spawn(work) { + // No worker: report the failure instead of waiting for a result that never comes. + if let Ok(mut s) = progress.lock() { + s.result = Some(Err(EditError::Invalid(e.to_string()))); + } + } + #[cfg(target_arch = "wasm32")] + work(); + self.scan_clean_run = Some(ScanCleanRun { doc: id, progress }); + self.poll_scan_clean(); + } + + /// Stop a running clean-up (nothing changes). + pub fn cancel_scan_clean(&mut self) { + if let Some(r) = &self.scan_clean_run + && let Ok(mut s) = r.progress.lock() + { + s.cancel = true; + } + } + + /// Show progress; apply the result once the worker is done. + pub(crate) fn poll_scan_clean(&mut self) { + let Some(run) = self.scan_clean_run.as_ref() else { return }; + let (doc, progress) = (run.doc, run.progress.clone()); + let Ok(mut s) = progress.lock() else { + // The worker panicked while holding the lock: it will never report back. + self.scan_clean_run = None; + self.progress_notice = None; + self.notify_tr("Couldn't enhance the scanned pages"); + return; + }; + let Some(result) = s.result.take() else { + let (cancelling, done, total) = (s.cancel, s.done, s.total); + drop(s); + let label = if cancelling { + tl!("Cancelling…").to_string() + } else { + crate::i18n::fmt( + tl!("Enhancing scans… image {d} of {t}"), + &[("d", &(done + 1).min(total.max(1)).to_string()), ("t", &total.max(1).to_string())], + ) + }; + let fraction = if total == 0 { 1.0 } else { done.min(total) as f32 / total as f32 }; + self.progress_notice = Some(crate::widgets::ProgressNotice { label, fraction, cancellable: !cancelling }); + if let Some(ctx) = &self.ctx { + ctx.request_repaint_after(std::time::Duration::from_millis(50)); + } + return; + }; + drop(s); + self.scan_clean_run = None; + self.progress_notice = None; + match result.and_then(|cleaned| self.session.apply_scan_clean(doc, cleaned)) { + Ok(report) => { + let msg = summary(&report); + self.notify(msg); + } + Err(EditError::Cancelled) => self.notify_tr("Enhancing the scanned pages was cancelled"), + Err(e) => self.notify(e.to_string()), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use pdfcraft_engine::scan_clean::SkippedImage; + + #[test] + fn the_summary_says_what_was_done_and_what_was_left() { + let r = ScanCleanReport { + pages: vec![0, 2], + images: 1, + not_scanned: vec![1], + skipped: vec![SkippedImage { pages: vec![3], name: "Im0".into(), reason: "it is CMYK".into() }], + }; + assert_eq!(summary(&r), "Cleaned up the scans on 2 pages. 1 page is not a scan. 1 image was left as it was (page 4: it is CMYK)."); + assert_eq!(summary(&ScanCleanReport::default()), "No scanned pages were cleaned up."); + } +} diff --git a/crates/ui-egui/tests/it/main.rs b/crates/ui-egui/tests/it/main.rs index a5c3192358..2df736c14c 100644 --- a/crates/ui-egui/tests/it/main.rs +++ b/crates/ui-egui/tests/it/main.rs @@ -68,6 +68,7 @@ mod reading_colors; mod recovery; mod redact; mod reload; +mod scan_clean; mod side_panels; mod signing; mod stamps; diff --git a/crates/ui-egui/tests/it/scan_clean.rs b/crates/ui-egui/tests/it/scan_clean.rs new file mode 100644 index 0000000000..afb4a4f75c --- /dev/null +++ b/crates/ui-egui/tests/it/scan_clean.rs @@ -0,0 +1,77 @@ +//! Scan & OCR ▸ Enhance scanned file in the real shell (egui_kittest): the dialog, the run and +//! its report, and that the page keeps its comment while its scan is cleaned in place. + +use egui_kittest::Harness; +use egui_kittest::kittest::Queryable; +use pdfcraft_engine::{Edit, NewAnnotation, NoteIcon, Session, Shape, Style}; +use pdfcraft_ui_egui::PdfCraftApp; + +/// A two-page PDF: page 1 a grey scan with a darker gutter and text bars (and a sticky note), +/// page 2 the same scan again. +fn scan() -> Vec { + let img = image::GrayImage::from_fn(300, 400, |x, y| { + let light = 0.55 + 0.45 * (x as f32 / 60.0).min(1.0); + let ink = if (60..360).contains(&y) && (y - 60) % 40 < 6 && (60..240).contains(&x) { 0.12 } else { 0.78 }; + image::Luma([(ink * light * 255.0) as u8]) + }); + let mut png = Vec::new(); + img.write_to(&mut std::io::Cursor::new(&mut png), image::ImageFormat::Png).unwrap(); + let mut s = Session::new(); + let bytes = s.create_from_images(&[("a.png".into(), png.clone()), ("b.png".into(), png)]).unwrap(); + let id = s.open("scan.pdf", None, bytes, None).unwrap(); + let shape = Shape::Note { at: [20.0, 20.0], icon: NoteIcon::Comment }; + let note = NewAnnotation { page: 0, style: Style::default_for(&shape), shape, contents: "Keep me".into(), author: "Tester".into() }; + s.apply(id, Edit::AddAnnotation(note)).unwrap(); + s.save_bytes(id).unwrap().to_vec() +} + +fn harness() -> Harness<'static, PdfCraftApp> { + let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).build_eframe(|_cc| { + let mut app = PdfCraftApp::new(); + app.set_option("language", "en").unwrap(); + app.open_bytes("scan.pdf", None, scan()).unwrap(); + app.run_inline = true; + app + }); + h.run_steps(4); + h +} + +#[test] +fn enhance_scanned_file_cleans_the_chosen_pages_and_says_what_it_did() { + let mut h = harness(); + let id = h.state().active_ids().unwrap().1; + assert!(h.state().session.get(id).unwrap().comment_props(0, 0).is_some()); + assert!(h.state_mut().execute("ocr.enhance")); + h.run_steps(2); + assert_eq!(h.query_all_by_label("Background removal").count(), 2, "the label and the slider it names"); + h.get_by_label("Edge shadow removal"); + h.get_by_label("Despeckle"); + h.get_by_role_and_label(egui::accesskit::Role::RadioButton, "Current page").click(); + h.run_steps(1); + h.get_by_label("Enhance").click(); + h.run_steps(3); + let app = h.state(); + assert!(app.scan_clean_run.is_none() && app.dialog.is_none(), "finished and closed"); + let toast = app.toast.clone().unwrap().0; + assert_eq!(toast, "Cleaned up the scan on 1 page."); + let doc = app.session.get(id).unwrap(); + assert_eq!(doc.can_undo(), Some("Enhance scanned file")); + assert_eq!(doc.info.pages.len(), 2); + assert_eq!(doc.comment_props(0, 0).map(|p| p.author), Some("Tester".to_string()), "the sticky note stays"); +} + +#[test] +fn nothing_to_enhance_is_said_not_done() { + let mut h = harness(); + let id = h.state().active_ids().unwrap().1; + h.state_mut().scan_clean_draft.settings = pdfcraft_engine::scan_clean::ScanCleanSettings { level: 0, edge_shadows: false, despeckle: false }; + assert!(h.state_mut().execute("ocr.enhance")); + h.run_steps(2); + // With every filter off the button is disabled; starting anyway changes nothing. + h.state_mut().start_scan_clean(); + h.run_steps(2); + let app = h.state(); + assert!(app.toast.clone().unwrap().0.starts_with("No scanned pages were cleaned up."), "{:?}", app.toast); + assert_eq!(app.session.get(id).unwrap().can_undo(), None); +} diff --git a/docs/gaps.md b/docs/gaps.md index 56ec3ae748..ab6f083ab1 100644 --- a/docs/gaps.md +++ b/docs/gaps.md @@ -1,5 +1,6 @@ # Where PdfCraft falls short of Acrobat Pro +> **Last reviewed:** 2026-10-28 | **Last updated:** 2026-10-28 | **Change:** minor (M10, #4119 partial: Enhance scanned file cleans scanned pages in place: background removal, edge shadow removal, despeckle; estimates unchanged) > **Last reviewed:** 2026-10-25 · **Last updated:** 2026-10-25 · **Change:** minor (gaps 5 and 14 current state: native release Hans face via LXGW WenKai Lite; Arabic logical catalog, explicit RTL direction and mirrored stock chrome, with native/browser/IME workflows unqualified; no readiness re-estimate) > **Last reviewed:** 2026-10-15 · **Last updated:** 2026-10-19 · **Change:** minor (gap 2: composite fonts with `/ToUnicode /Identity-H` read as Unicode, hayro-interpret patch 44; estimates unchanged) · **Target:** Adobe Acrobat Pro (Acrobat DC, continuous track 26.002.21931, macOS) @@ -164,7 +165,9 @@ update the numbers. ### 7. OCR reads only unaccented Latin - **Missing:** every other script and accented Latin; editable-text output; deskew, rotation, - despeckle, background and camera clean-up; MRC compression; pluggable engines; OCR in the + despeckle and background clean-up as options of Recognize text (Enhance scanned file does them + on grey, RGB and 1-bit scans, not CMYK, indexed, JBIG2 or CCITT ones); MRC compression; + pluggable engines; OCR in the browser; scanner input. - **Evidence:** `ocr.ocr-languages` partial ("the models read only the Latin alphabet without accents"); 19 of 26 OCR features planned; [#744](https://github.com/storytold/pdfcraft/issues/744) diff --git a/docs/roadmap.md b/docs/roadmap.md index 25205e92c1..6d910fc795 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -1,5 +1,6 @@ # PdfCraft roadmap: milestones and what's next +> **Last reviewed:** 2026-10-28 | **Last updated:** 2026-10-28 | **Change:** minor (M10, #4119 partial: Enhance scanned file cleans scanned pages in place: background removal, edge shadow removal, despeckle; estimates unchanged) > **Last reviewed:** 2026-10-27 | **Last updated:** 2026-10-27 | **Change:** minor (M10: OCR model downloads use verified ONNX pairs with legacy discovery fallback; estimates unchanged) > **Last reviewed:** 2026-10-25 | **Last updated:** 2026-10-25 | **Change:** minor (M14 localization current state: Arabic logical catalog, explicit RTL label direction and mirrored stock chrome; native/browser RTL workflows unqualified. Native release Hans face via LXGW WenKai Lite; estimates unchanged) > **Last reviewed:** 2026-10-23 · **Last updated:** 2026-10-23 · **Change:** minor (M7, #2636 partial: pasting added text boxes and images in the engine and through `content_copy`; estimates unchanged) @@ -194,7 +195,7 @@ total matches the parity estimate. | M7 | Content editing (text, images, header/footer, watermark) | 250–500 | 20% | 100–180 | Done: header & footer, watermarks, backgrounds, Bates; added text and images that stay editable (move, resize, format, rotate, flip, crop, replace); links (Link tool, Link Properties, create from URLs, remove all). Missing: editing existing text and images in place (the longest pole), image/PDF watermarks | | M8 | Security + redaction | 100–180 | 68% | 15–30 | Done: opening protected documents, permissions, Protect Using Password, Remove security; redaction (mark text/areas/pages, Search & Redact with patterns, apply removing glyphs, image pixels, vectors, XObject content, annotations and fields, verification, full rewrite on save); Remove hidden information and Sanitize.
Done since: redaction codes (U.S. FOIA and Privacy Act sets as overlay text, in Redaction properties and `redact_mark`). Redaction cleans indirect and form structure references, preserving surviving form tags. Redaction retires inherited resource dictionaries, so a parent `/Pages` can't keep a removed XObject reachable; unredacted pages are left untouched. Image redaction removes alternate images and unsupported masks only on affected images. Changed marked-content spans lose replacement descriptions; untouched spans keep theirs. Hidden-text cleanup retires unused named descriptions across inherited/shared page and form resources (64 focused tests and scoped Rust 1.96 Clippy/wasm checks pass; publication CI pending). The redaction audit counts a filled cover that hides half of any one word of a long line (#2662), applied or not. Phone-pattern redaction also matches labelled Italian geographic numbers with `+39` and a slash after the area code (#3205 follow-up); unlabelled international slash numbers remain unsupported. Missing: certificate security, custom redaction code sets and pattern locales, DCT re-encoding | | M9 | Signatures (PAdES, validation) | 160–280 | 60% | 40–70 | Done: new `sign` crate (DER, X.509, CMS, PKCS #12 on RustCrypto; aws-lc-rs for RSA private keys); PAdES B-B signing (visible, invisible, existing fields, certification with DocMDP); validation with trust store and changes-after-signing classification; self-signed digital IDs; Signatures panel, message bar, sign dialogs; agent tools. Checked with pdfsig and OpenSSL.
Done since: macOS Keychain and Windows Current User Personal store signing (software-backed CNG keys), certificate viewer, signed documents protected from rewrites, page validation overlays separate from original signature artwork (baked Not Verified captions remain; #79 open), snapshot tabs that reuse instead of multiplying. Missing: timestamps (B-T), LTV (DSS, OCSP, CRL), FieldMDP, smart cards, PKCS #11 | -| M10 | OCR, create, export, print | 200–350 | 33% | 120–215 | Done: create from blank/text/PNG/JPEG/TIFF (multi-page)/GIF/BMP; export PNG/JPEG/TIFF and text; Print (Acrobat's sizing, n-up, booklet, poster, comments & forms, preview, the CUPS and Windows spoolers, print-ready PDF).
Done since: embedded/72/custom DPI choices for image imports; export all images; OCR (searchable image for pages, ranges and multiple files); single-sided cut-and-stack imposition with cut marks, through Print and doc_print; architectural/ANSI/large ISO sheet presets, Match page at 100% and PDF-scale preview (#599 follow-up); Create PDF from multiple files (PDFs, images and text, edited as pages in the grid); OCR can also read pictures on pages that have text, without adding that text twice (#1028). Model staging fetches SHA-256-verified ONNX detection/recognition files after the upstream RTEN URL brownout; runtime discovery prefers complete ONNX pairs per directory, then complete legacy RTEN pairs, without mixing formats or changing directory precedence. Done since: Word invoice export uses borderless inferred tables, floating positioned text/artwork only where text overlaps an image or is rotated, and per-page sizes, covered by synthetic invoice automation regressions (#773); printing on the web through the browser's print dialog, sheets drawn by PdfCraft (gap 4). Lossless image export applies per-component Decode, including Indexed mapping (#1201 partial). Existing 16-bit precision and color-management limits remain. #3532 partial: vector printing retains existing appearances with absent Subtype through a print-owned Form copy; source artwork stays unchanged. #3120: Office export reads form XObjects 50 levels deep, as the renderer does, and up to 50 000 a page; pages past that are listed in `truncated_pages`. Word table cells use the grid columns they span, including cells after a merged cell. Done since: text in a table slot covered by another cell's span remains as paragraph text in Word, HTML, RTF and PowerPoint. Missing: OCR languages beyond Latin, editable-text OCR output, full Office layout fidelity/import. Done since: dotted-leader contents entries stay in separate complete paragraphs (#1757) #2156 candidate: last selected print paper persists across sessions; other print preferences unchanged. Image export applies soft-mask Decode opacity intervals before resizing, including indirect entries and inverted ranges. Vector printing gives annotation appearances unoccupied XObject names so existing page artwork survives. | +| M10 | OCR, create, export, print | 200–350 | 33% | 120–215 | Done: create from blank/text/PNG/JPEG/TIFF (multi-page)/GIF/BMP; export PNG/JPEG/TIFF and text; Print (Acrobat's sizing, n-up, booklet, poster, comments & forms, preview, the CUPS and Windows spoolers, print-ready PDF).
Done since: embedded/72/custom DPI choices for image imports; export all images; OCR (searchable image for pages, ranges and multiple files); single-sided cut-and-stack imposition with cut marks, through Print and doc_print; architectural/ANSI/large ISO sheet presets, Match page at 100% and PDF-scale preview (#599 follow-up); Create PDF from multiple files (PDFs, images and text, edited as pages in the grid); OCR can also read pictures on pages that have text, without adding that text twice (#1028). Model staging fetches SHA-256-verified ONNX detection/recognition files after the upstream RTEN URL brownout; runtime discovery prefers complete ONNX pairs per directory, then complete legacy RTEN pairs, without mixing formats or changing directory precedence. Done since: Word invoice export uses borderless inferred tables, floating positioned text/artwork only where text overlaps an image or is rotated, and per-page sizes, covered by synthetic invoice automation regressions (#773); printing on the web through the browser's print dialog, sheets drawn by PdfCraft (gap 4). Lossless image export applies per-component Decode, including Indexed mapping (#1201 partial). Existing 16-bit precision and color-management limits remain. #3532 partial: vector printing retains existing appearances with absent Subtype through a print-owned Form copy; source artwork stays unchanged. #3120: Office export reads form XObjects 50 levels deep, as the renderer does, and up to 50 000 a page; pages past that are listed in `truncated_pages`. Word table cells use the grid columns they span, including cells after a merged cell. Done since: text in a table slot covered by another cell's span remains as paragraph text in Word, HTML, RTF and PowerPoint. Missing: OCR languages beyond Latin, editable-text OCR output, full Office layout fidelity/import. Done since: dotted-leader contents entries stay in separate complete paragraphs (#1757) #2156 candidate: last selected print paper persists across sessions; other print preferences unchanged. Image export applies soft-mask Decode opacity intervals before resizing, including indirect entries and inverted ranges. Vector printing gives annotation appearances unoccupied XObject names so existing page artwork survives. Done since: Enhance scanned file cleans scanned pages in place (background removal, edge shadow removal, despeckle; the OCR text layer and comments stay; #4119 partial). | | M11 | Optimize, preflight, PDF/A/X/UA, print production | 200–350 | 18% | 80–140 | Done: new `optimize` crate: Reduce File Size and the PDF Optimizer (images measured where drawn, bicubic downsampling, JPEG/ZIP recompression only when smaller, discard objects and user data, Flate clean-up, resource merging, object streams). Target-version compatibility is partial: newer content is not detected or converted. Content stream boundaries preserve comments, split operands and graphics state when image fragments merge. Image downsampling includes page-local `/UserUnit` scaling when measuring physical resolution. Done since: reject invalid image and soft-mask dimensions with checked sizing (#2178). Missing: fonts and transparency panels, space audit, preflight, PDF/A/X/UA | | M12 | Accessibility, compare, measure, search, XFA | 200–380 | 30% | 90–160 | Done: new `a11y` crate with the Accessibility Checker (all 32 rules, report, Fix/Skip/Explain, options dialog and results panel, agent tools). Locatable figure, annotation and form findings highlight their page bounds. Character encoding accepts Identity-H/V with Adobe CJK collections and predefined CJK CMaps independently (#3673); per-glyph validation and embedded-font extraction remain separate.
Done since: 2D distance, perimeter and area measurements with persistent viewport calibration, snapping, live information and CSV export; a bounded, text-only auto-tag proposal tool for one page. Marking a figure decorative rewrites only that stream (#1324). An XFA radio item whose value contains a space is written back unchanged (#1319). Done since: #2569: Growable multiline XFA text reflows after edits, with undo/redo, save/reopen and print coverage; oversized fields still become read-only page text. Structured-data synchronization warnings retain accepted whitespace without relayout from stale data. Done since: #2572: Every repeated page-master button resolves its own field to the shared XFA click script, including print buttons. Done since: #2570: Dynamic XFA rich-text fields carry exData/maxLength into AcroForm MaxLen for reported and enforced Unicode character limits, retained after save/reopen. Done since: #2589: Active XFA skips PDF-container fallback scripts while retaining requested messages and saved scripts; ordinary/static/unlaid-out documents and explicit console/field calls keep their existing behavior. Missing: writing proposed tags and parent tree, field autotag, visual Tags/Order/Content panels, Reading Order tool, compare, geospatial/3D measurement, search index, XFA
Reading colours: pending page backgrounds follow the configured colour (#2059). | | M13 | Automation (MCP, Action Wizard, CLI) + AI providers | 60–120 | 45% | 25–45 | Done: MCP resources (document info, text, page images); headless tool table (123 tools incl. signing, optimizing, initial view, links, stamps, data exchange, comment review, forms authoring and scripts, redaction, sanitize, print, add content), opt-in MCP server over stdio, CLI `run`/`tools` (closed stdout pipes exit cleanly), UI control channel with drag. Missing: Action Wizard, AI providers
Done since: CLI run and MCP reject missing, empty, repeated and option-valued --root arguments before any tool/script execution or MCP initialization. A single-hyphen directory name remains valid; synthetic CLI regressions preserve a would-be overwritten file. `text_find` also caps the per-page match ranges it materializes at the remaining result limit plus one continuation match. CLI robustness scans visit each directory once through overlapping roots and symlink aliases. | diff --git a/docs/target-app-parity.md b/docs/target-app-parity.md index 6c7920e33c..3c77c491cb 100644 --- a/docs/target-app-parity.md +++ b/docs/target-app-parity.md @@ -1,5 +1,6 @@ # PdfCraft parity with Adobe Acrobat Pro +> **Last reviewed:** 2026-10-28 | **Last updated:** 2026-10-28 | **Change:** minor (M10, #4119 partial: Enhance scanned file cleans scanned pages in place: background removal, edge shadow removal, despeckle; estimates unchanged) > **Last reviewed:** 2026-10-25 · **Last updated:** 2026-10-25 · **Change:** minor (localization current state: Arabic logical catalog, explicit RTL direction and mirrored stock chrome; native/browser RTL workflows unqualified. Native release Hans face via LXGW WenKai Lite. Scores and estimates unchanged) > **Last reviewed:** 2026-10-10 · **Last updated:** 2026-10-13 · **Change:** minor (localization ≈59% → ≈68%; weighted readiness 49.25 → 49.7 ≈ 50%) · **Target:** Adobe Acrobat Pro (Acrobat DC, continuous track 26.002.21931, macOS) @@ -206,7 +207,7 @@ full parity in that area. | F Forms and JavaScript | 79 | 58 | 8 | 78.5% | 65% | 40–75 | Fill and author every AcroForm field type, Acrobat's AF functions in Acrobat's event order, sandboxed JavaScript (boa), data exchange, Fill & Sign; XFA static/dynamic layout, FormCalc and XFA data (partial). CLI info, text and render use the laid-out dynamic pages (#1645). Missing: wider JavaScript object model, document actions, submit, debugger, barcode fields, XFA flattening and full dynamic fidelity | | G Protect, redact, sanitize | 41 | 23 | 2 | 58.5% | 55% | 15–30 | Passwords and permissions, Remove security, redaction that removes glyphs, pixels, vectors, annotations and tags, with verification; word lists, patterns, FOIA/Privacy Act codes; sanitize. Missing: certificate security, pattern locales, custom code sets, partial-word redaction, folder redaction, protected view | | H Digital signatures | 45 | 23 | 10 | 62.2% | 45% | 40–70 | PAdES B-B signing and validation (PKCS #12, macOS Keychain, Windows store with PIN prompts), certification, encrypted documents, certificate usage checks; B-T, LTV, OCSP/CRL with caller-supplied evidence. Missing: PKCS #11 and smart cards outside Windows, timestamp-server preferences, online revocation fetching, FieldMDP and field locking, OS trust store, EU Trusted Lists, compare signed version | -| I Scan and OCR | 26 | 5 | 2 | 23.1% | 15% | 40–75 | Searchable image (ocrs, Latin script without accents) on pages, ranges and many files. Missing: every other language (#744 Dutch; accents, CJK, Cyrillic, Arabic), editable-text output, deskew, rotation, despeckle, camera images beyond straightening (found or placed corners): orientation, clean-up, MRC compression, scanners | +| I Scan and OCR | 26 | 5 | 2 | 23.1% | 15% | 40–75 | Searchable image (ocrs, Latin script without accents) on pages, ranges and many files. Missing: every other language (#744 Dutch; accents, CJK, Cyrillic, Arabic), editable-text output, deskew, rotation, camera images beyond straightening (found or placed corners): orientation, clean-up, MRC compression, scanners. Background removal, edge shadow removal and despeckle work on scanned pages (Enhance scanned file) but not yet as options of Recognize text, nor on CMYK, indexed, JBIG2 or CCITT scans | | J Create and export | 34 | 12 | 3 | 39.7% | 25% | 55–100 | Create from images, multipage TIFF, text, clipboard, blank, many files at once; export PNG/JPEG/TIFF, text, RTF; Word and HTML partial. Missing: Office/HTML/web/PostScript to PDF, Excel and PowerPoint export, Word layout fidelity (#773), SVG/XML/EPS/PS/JPEG 2000 export, accessible text | | K Optimize and standards | 58 | 15 | 10 | 34.5% | 20% | 65–115 | Reduce File Size and the Optimizer (images, discards, clean-up, space audit); PDF/A-2b/3b verify and convert (partial). Missing: Preflight (profiles, fixups, reports), PDF/A-1/4 and a/u levels, PDF/X, PDF/UA, PDF/E, PDF/VT, transparency flattening, font unembedding, monochrome JBIG2/CCITT | | L Print | 39 | 15 | 1 | 39.7% | 35% | 30–55 | Acrobat's sizing, n-up, booklet, poster, comments & forms, preview, print-ready PDF, CUPS on macOS/Linux, the Windows spooler (#756) and the browser's print dialog on the web. Missing: print as image, PostScript, all print-production tools (output preview, separations, ink manager, printer marks, colour conversion) | diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 47ac06fdd4..1ea04254aa 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -6674,10 +6674,10 @@ area = "I" tier = "P1" milestone = "M10" status = "partial" -commands = ["ocr.enhance_camera"] -tools = ["page_enhance_camera"] -evidence = ["crates/engine/src/camera_clean.rs::a_shadow_and_a_yellow_cast_become_white_paper", "crates/engine/src/tests.rs::enhance_camera_page_at_a_level_whitens_a_shadowed_sheet", "crates/ui-egui/tests/it/camera.rs::the_clean_up_level_enhances_again_from_the_photo_and_keeps_the_turn"] -notes = "Camera images only: from clean-up level 60, isolated dark pixels are removed. Missing: for scanned pages and as an OCR option." +commands = ["ocr.enhance_camera", "ocr.enhance"] +tools = ["page_enhance_camera", "page_enhance_scan"] +evidence = ["crates/engine/src/camera_clean.rs::a_shadow_and_a_yellow_cast_become_white_paper", "crates/engine/src/tests.rs::enhance_camera_page_at_a_level_whitens_a_shadowed_sheet", "crates/ui-egui/tests/it/camera.rs::the_clean_up_level_enhances_again_from_the_photo_and_keeps_the_turn", "crates/engine/src/scan_clean/tests.rs::despeckle_is_its_own_switch", "crates/automation/tests/it/automation.rs::enhance_scanned_file_cleans_the_scan_in_place_through_tools", "crates/ui-egui/tests/it/scan_clean.rs::enhance_scanned_file_cleans_the_chosen_pages_and_says_what_it_did"] +notes = "Camera images, and scanned pages through Enhance scanned file (ocr.enhance, page_enhance_scan despeckle): isolated dark pixels (at most one dark neighbour) turn white, on grey, RGB and 1-bit scans, in place (the OCR text layer, comments and links stay). Missing: as an option of Recognize text; specks larger than two pixels; CMYK, indexed, JPEG 2000, JBIG2 and CCITT scans (reported, not changed)." [[feature]] id = "ocr.background-removal" @@ -6686,10 +6686,10 @@ area = "I" tier = "P1" milestone = "M10" status = "partial" -commands = ["ocr.enhance_camera"] -tools = ["page_enhance_camera"] -evidence = ["crates/engine/src/camera_clean.rs::a_shadow_and_a_yellow_cast_become_white_paper", "crates/engine/src/tests.rs::enhance_camera_page_at_a_level_whitens_a_shadowed_sheet", "crates/ui-egui/tests/it/camera.rs::the_clean_up_level_enhances_again_from_the_photo_and_keeps_the_turn"] -notes = "Camera images only: Enhance camera image's clean-up level (and page_enhance_camera level) evens out the paper's illumination per channel, so the background turns white and colour casts go; colour ink stays. Missing: the same for scanned pages and as an OCR option." +commands = ["ocr.enhance_camera", "ocr.enhance"] +tools = ["page_enhance_camera", "page_enhance_scan"] +evidence = ["crates/engine/src/camera_clean.rs::a_shadow_and_a_yellow_cast_become_white_paper", "crates/engine/src/tests.rs::enhance_camera_page_at_a_level_whitens_a_shadowed_sheet", "crates/ui-egui/tests/it/camera.rs::the_clean_up_level_enhances_again_from_the_photo_and_keeps_the_turn", "crates/engine/src/scan_clean/tests.rs::a_scan_gets_white_paper_even_edges_dark_strokes_and_no_specks_in_place", "crates/automation/tests/it/automation.rs::enhance_scanned_file_cleans_the_scan_in_place_through_tools", "crates/ui-egui/tests/it/scan_clean.rs::enhance_scanned_file_cleans_the_chosen_pages_and_says_what_it_did"] +notes = "Camera images (Enhance camera image level) and scanned pages (Enhance scanned file ▸ Background removal, page_enhance_scan level): the paper's illumination is evened out per channel so a grey, yellowed or unevenly lit background turns white, then the contrast is raised; colour ink stays. Scans are cleaned in place in their own image objects (JPEG stays JPEG at quality 90, others lossless), so the OCR text layer, comments and links stay; one undo step. Missing: as an option of Recognize text; CMYK, indexed, JPEG 2000, JBIG2 and CCITT scans and scans inside form XObjects (reported, not changed); a preview." [[feature]] id = "ocr.descreen" @@ -6722,10 +6722,10 @@ area = "I" tier = "P2" milestone = "M10" status = "partial" -commands = ["ocr.enhance_camera"] -tools = ["page_enhance_camera"] -evidence = ["crates/engine/src/camera_clean.rs::a_shadow_and_a_yellow_cast_become_white_paper", "crates/engine/src/tests.rs::enhance_camera_page_at_a_level_whitens_a_shadowed_sheet", "crates/ui-egui/tests/it/camera.rs::the_clean_up_level_enhances_again_from_the_photo_and_keeps_the_turn"] -notes = "Camera images only: the clean-up level divides out a smooth illumination estimate, which removes shadows and darker edges on the straightened photo. Missing: for scanned pages (book gutters) and as an OCR option." +commands = ["ocr.enhance_camera", "ocr.enhance"] +tools = ["page_enhance_camera", "page_enhance_scan"] +evidence = ["crates/engine/src/camera_clean.rs::a_shadow_and_a_yellow_cast_become_white_paper", "crates/engine/src/tests.rs::enhance_camera_page_at_a_level_whitens_a_shadowed_sheet", "crates/ui-egui/tests/it/camera.rs::the_clean_up_level_enhances_again_from_the_photo_and_keeps_the_turn", "crates/engine/src/scan_clean/tests.rs::edge_shadow_removal_whitens_the_gutter_background_removal_alone_leaves", "crates/automation/tests/it/automation.rs::enhance_scanned_file_cleans_the_scan_in_place_through_tools", "crates/ui-egui/tests/it/scan_clean.rs::enhance_scanned_file_cleans_the_chosen_pages_and_says_what_it_did"] +notes = "Camera images (the clean-up level divides out a smooth illumination estimate) and scanned pages (Enhance scanned file ▸ Edge shadow removal, page_enhance_scan edge_shadows): within the outer fifth of each side, every column and row is lightened to the brightest paper of that band, which removes book gutters and copier borders the coarse estimate misses; a black scanner border is left alone. Missing: as an option of Recognize text; curved gutters that also bend the text (no dewarping); the scan formats background removal also skips." [[feature]] id = "ocr.camera-perspective" From 622625a178b9bec514b586f12b53fe1c28ab8f33 Mon Sep 17 00:00:00 2001 From: huppiflupp Date: Wed, 28 Oct 2026 13:37:00 +0100 Subject: [PATCH 021/198] M10: Enhance scanned file reports scans inside Form XObjects and names JPEG 2000 - A scan drawn only through grouped Form artwork is reported in skipped (once per form, with every page that draws it) instead of being counted as no scan; a Form of vector artwork holding only a small picture (under 36 dpi over its area) still counts as no scan. - decode_samples checks the filter before the colour space and bit depth, so a JPEG 2000 image without /ColorSpace or /BitsPerComponent is reported as JPEG 2000, not as a colour-space or bit-depth problem. - Optimize and Enhance scanned file no longer start while the other runs: they share the progress card and capture the same document state. - docs/gaps.md: background and edge-shadow removal apply to grey and RGB scans, despeckle also to 1-bit ones. Co-Authored-By: Claude Opus 5.5 --- crates/automation/src/tools.rs | 2 +- crates/engine/src/copy_image.rs | 2 +- crates/engine/src/scan_clean.rs | 25 ++++++- crates/engine/src/scan_clean/tests.rs | 53 ++++++++++++++ crates/engine/src/scan_fields.rs | 70 +++++++++++++++---- crates/optimize/src/samples.rs | 9 ++- crates/ui-egui/src/workflows/optimize_ui.rs | 23 ++++++ crates/ui-egui/src/workflows/scan_clean_ui.rs | 5 ++ docs/gaps.md | 6 +- parity/acrobat-features.toml | 2 +- 10 files changed, 176 insertions(+), 21 deletions(-) diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index 0d34911154..c718e4a47a 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -225,7 +225,7 @@ pub fn tools() -> Vec { }), &["doc", "page"], )), - t("page_enhance_scan", "Enhance scanned file", "Clean up scanned pages in place (Scan & OCR ▸ Enhance ▸ Scanned document). A page is a scan when its pictures cover at least half of it; the pictures covering at least a tenth of it are cleaned and written back into the same image objects at the same size and colour space (a JPEG stays a JPEG at quality 90, other grey/RGB images are stored losslessly, a 1-bit image stays 1-bit), so the page's invisible OCR text, comments, links and other content stay as they were, and an image drawn on several pages is cleaned once. level (0–100, default 50) is background removal: the paper's illumination is evened out so a grey, yellowed or unevenly lit background turns white, then the contrast is raised so faint strokes darken; colour ink stays colour. edge_shadows (default: on unless level is 0) lightens shadows along the scan's edges (book gutters, a copier's dark border). despeckle (default: from level 60) turns isolated dark specks white; it is the only change a 1-bit scan takes. Returns pages_cleaned (1-based), images (rewritten), not_scanned (pages that are no scan) and skipped ([{pages, name, reason}]: CMYK, indexed, JPEG 2000, JBIG2 or CCITT images, broken data, or nothing to change). One undoable step; none when nothing changed.") + t("page_enhance_scan", "Enhance scanned file", "Clean up scanned pages in place (Scan & OCR ▸ Enhance ▸ Scanned document). A page is a scan when its pictures cover at least half of it; the pictures covering at least a tenth of it are cleaned and written back into the same image objects at the same size and colour space (a JPEG stays a JPEG at quality 90, other grey/RGB images are stored losslessly, a 1-bit image stays 1-bit), so the page's invisible OCR text, comments, links and other content stay as they were, and an image drawn on several pages is cleaned once. level (0–100, default 50) is background removal: the paper's illumination is evened out so a grey, yellowed or unevenly lit background turns white, then the contrast is raised so faint strokes darken; colour ink stays colour. edge_shadows (default: on unless level is 0) lightens shadows along the scan's edges (book gutters, a copier's dark border). despeckle (default: from level 60) turns isolated dark specks white; it is the only change a 1-bit scan takes. Returns pages_cleaned (1-based), images (rewritten), not_scanned (pages that are no scan) and skipped ([{pages, name, reason}]: CMYK, indexed, JPEG 2000, JBIG2 or CCITT images, a scan drawn inside grouped Form artwork, broken data, or nothing to change). One undoable step; none when nothing changed.") .cmd("ocr.enhance") .with(schema( json!({ diff --git a/crates/engine/src/copy_image.rs b/crates/engine/src/copy_image.rs index b377903ee4..1407c9f1d2 100644 --- a/crates/engine/src/copy_image.rs +++ b/crates/engine/src/copy_image.rs @@ -113,7 +113,7 @@ pub(crate) fn added_picture(cos: &Cos, file: &[u8]) -> Option { } /// The largest raster image Form `form` draws through its resources (nested Forms included). -fn picture_in_form(cos: &Cos, form: ObjRef) -> Option { +pub(crate) fn picture_in_form(cos: &Cos, form: ObjRef) -> Option { let mut seen = HashSet::from([form]); let mut best: Option<(u64, ObjRef)> = None; let res = resources(cos, form)?; diff --git a/crates/engine/src/scan_clean.rs b/crates/engine/src/scan_clean.rs index bfc8188b29..48ff15093a 100644 --- a/crates/engine/src/scan_clean.rs +++ b/crates/engine/src/scan_clean.rs @@ -12,7 +12,7 @@ //! pictures each covering at least a tenth of the page are its scan. What is not cleaned is //! reported, never skipped silently: pages that are no scan, and scan images in a form PdfCraft //! can't rewrite yet (CMYK, indexed, JPEG 2000, JBIG2, CCITT fax, a stencil mask, a remapping -//! decode array) or whose data is broken. +//! decode array, a scan drawn inside grouped Form artwork) or whose data is broken. //! //! Like Optimize, the work is a job: [`Session::scan_clean_job`] captures the document, //! [`ScanCleanJob::run`] cleans anywhere (a worker thread), and [`Session::apply_scan_clean`] @@ -31,6 +31,8 @@ const QUALITY: u8 = 90; /// On a scanned page, an image covering at least this share of the page is part of the scan; /// smaller ones (a logo or a signature pasted on) are left alone. const PART_OF_SCAN: f64 = 0.1; +/// Why a scan drawn through grouped Form artwork stays as it is. +const IN_FORM: &str = "it is drawn inside grouped artwork (a form XObject), which can't be cleaned up yet"; /// How scanned pages are cleaned up. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -138,10 +140,29 @@ impl Session { wanted.dedup(); let mut report = ScanCleanReport::default(); let mut targets: Vec = Vec::new(); + // Form artwork already reported, and where in `report.skipped`. + let mut forms_reported: Vec<(ObjRef, usize)> = Vec::new(); for page in wanted { let Some(info) = doc.info.pages.get(page) else { continue }; let Some(images) = crate::scan_fields::scan_images(doc, page, info) else { - report.not_scanned.push(page); + // A scan drawn through a Form XObject is a scan all the same: say why it stays. + let forms: Vec<_> = crate::scan_fields::scan_forms(doc, page, info).into_iter().filter(|(_, share)| *share >= PART_OF_SCAN).collect(); + if forms.is_empty() { + report.not_scanned.push(page); + } + for (form, _) in forms { + let known = form.object.and_then(|o| forms_reported.iter().find(|(r, _)| *r == o)).and_then(|(_, k)| report.skipped.get_mut(*k)); + match known { + Some(s) if !s.pages.contains(&page) => s.pages.push(page), + Some(_) => {} + None => { + if let Some(o) = form.object { + forms_reported.push((o, report.skipped.len())); + } + report.skipped.push(SkippedImage { pages: vec![page], name: form.name, reason: IN_FORM.into() }); + } + } + } continue; }; for (image, _) in images.into_iter().filter(|(_, share)| *share >= PART_OF_SCAN) { diff --git a/crates/engine/src/scan_clean/tests.rs b/crates/engine/src/scan_clean/tests.rs index 561ba9e2e4..f39f8c0432 100644 --- a/crates/engine/src/scan_clean/tests.rs +++ b/crates/engine/src/scan_clean/tests.rs @@ -275,6 +275,59 @@ fn hostile_images_are_reported_not_a_panic() { assert!(matches!(s.enhance_scanned_pages(id, &[99], ScanCleanSettings::default()), Err(EditError::Invalid(_)))); } +#[test] +fn a_scan_drawn_inside_a_form_xobject_is_reported_not_called_no_scan() { + // Pages 1 and 3 draw the scan through Form Fm0 (as some scanners write it), page 2 is text. + let page = |contents: usize| { + format!( + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 {W} {H}] /Resources << /XObject << /Fm0 4 0 R >> /Font << /F1 9 0 R >> >> /Contents {contents} 0 R >>" + ) + }; + let bytes = Pdf::new() + .obj("<< /Type /Catalog /Pages 2 0 R >>") + .obj("<< /Type /Pages /Kids [3 0 R 7 0 R 10 0 R] /Count 3 >>") + .obj(page(6)) + .stream( + &format!("/Type /XObject /Subtype /Form /BBox [0 0 {W} {H}] /Resources << /XObject << /Im0 5 0 R >> >>"), + format!("q {W} 0 0 {H} 0 0 cm /Im0 Do Q"), + ) + .stream(&format!("/Type /XObject /Subtype /Image /Width {W} /Height {H} /ColorSpace /DeviceGray /BitsPerComponent 8"), scan_pixels(1)) + .stream("", "/Fm0 Do") + .obj(page(8)) + .stream("", "BT /F1 12 Tf 20 380 Td (Real text) Tj ET") + .obj("<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>") + .obj(page(11)) + .stream("", "q /Fm0 Do Q") + .build(); + let (mut s, id) = open(bytes); + let before = object(&s, id, 5); + let report = s.enhance_scanned_pages(id, &[], ScanCleanSettings::at(100)).unwrap(); + assert!(report.pages.is_empty() && report.images == 0, "{report:?}"); + assert_eq!(report.not_scanned, vec![1], "only the text page is no scan"); + assert_eq!(report.skipped.len(), 1, "the form is reported once: {report:?}"); + let k = &report.skipped[0]; + assert_eq!((k.pages.as_slice(), k.name.as_str()), (&[0, 2][..], "Fm0")); + assert!(k.reason.contains("form XObject"), "{k:?}"); + assert_eq!(object(&s, id, 5), before, "the picture is left as it was"); + assert_eq!(s.get(id).unwrap().can_undo(), None); + + // A page of vector artwork in a Form, holding only a small logo, is no scan. + let logo = Pdf::new() + .obj("<< /Type /Catalog /Pages 2 0 R >>") + .obj("<< /Type /Pages /Kids [3 0 R] /Count 1 >>") + .obj(format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 {W} {H}] /Resources << /XObject << /Fm0 4 0 R >> >> /Contents 6 0 R >>")) + .stream( + &format!("/Type /XObject /Subtype /Form /BBox [0 0 {W} {H}] /Resources << /XObject << /Im0 5 0 R >> >>"), + format!("0 0 {W} {H} re f q 40 0 0 20 10 370 cm /Im0 Do Q"), + ) + .stream("/Type /XObject /Subtype /Image /Width 40 /Height 20 /ColorSpace /DeviceGray /BitsPerComponent 8", [128; 800]) + .stream("", "/Fm0 Do") + .build(); + let (mut s, id) = open(logo); + let report = s.enhance_scanned_pages(id, &[], ScanCleanSettings::at(100)).unwrap(); + assert_eq!((report.not_scanned.as_slice(), report.skipped.len()), (&[0][..], 0), "{report:?}"); +} + #[test] fn protected_documents_keep_their_encryption_and_restrictions() { let protect = |user: &str, permissions: i32| { diff --git a/crates/engine/src/scan_fields.rs b/crates/engine/src/scan_fields.rs index 769beaefca..d2e663f9a5 100644 --- a/crates/engine/src/scan_fields.rs +++ b/crates/engine/src/scan_fields.rs @@ -19,6 +19,9 @@ const SCALE: f32 = 2.0; const MAX_SIDE: f32 = 3000.0; /// A page is a scan when its images cover at least this share of it. const COVERED: f64 = 0.5; +/// A picture inside Form artwork is taken for a scan when it has at least this resolution over +/// the area the artwork covers (a quarter of the pixels a 72 dpi scan would have). +const MIN_FORM_DPI: f64 = 36.0; /// Recognised "words" less tall than this (points) are specks read as letters, and would make /// the box they sit on look filled in. const MIN_WORD_HEIGHT: f64 = 4.0; @@ -32,28 +35,69 @@ pub(crate) fn is_scan(doc: &Document, page: usize, info: &PageInfo) -> bool { /// crop box) it covers, when together they cover most of it ([`is_scan`]); `None` otherwise. /// Enhance ▸ Scanned document cleans these up ([`crate::scan_clean`]). pub(crate) fn scan_images(doc: &Document, page: usize, info: &PageInfo) -> Option> { - let [cx0, cy0, cx1, cy1] = info.crop.map(f64::from); - let page_area = (cx1 - cx0) * (cy1 - cy0); - if !page_area.is_finite() || page_area <= 0.0 { - return None; - } + let area = page_area(info)?; let images: Vec<(pdfcraft_edit::PageImage, f64)> = doc .page_images(page) .into_iter() .filter(|i| !i.is_form && i.width > 1 && i.height > 1) - .map(|i| { - let r = i.visible_rect.unwrap_or(i.rect); - let w = (r[2].min(cx1) - r[0].max(cx0)).max(0.0); - let h = (r[3].min(cy1) - r[1].max(cy0)).max(0.0); - let share = w * h / page_area; - (i, share) - }) - .filter(|(_, share)| share.is_finite()) + .filter_map(|i| Some((share(info, area, &i)?, i))) + .map(|(share, i)| (i, share)) .collect(); let covered: f64 = images.iter().map(|(_, share)| share).sum(); (covered >= COVERED).then_some(images) } +/// The grouped Form artwork on `page` that holds a raster picture, each with the share of the +/// page it covers, when it and the page's own images together cover most of the page: a scan +/// drawn through a Form XObject, which [`scan_images`] does not count and Enhance ▸ Scanned +/// document can't clean yet, so it reports these. Empty otherwise. +pub(crate) fn scan_forms(doc: &Document, page: usize, info: &PageInfo) -> Vec<(pdfcraft_edit::PageImage, f64)> { + let (Some(area), Some(editor)) = (page_area(info), doc.editor.as_ref()) else { return Vec::new() }; + let mut covered = 0.0; + let mut forms = Vec::new(); + for i in doc.page_images(page) { + let Some(share) = share(info, area, &i) else { continue }; + if !i.is_form { + if i.width > 1 && i.height > 1 { + covered += share; + } + } else if i.object.is_some_and(|form| holds_scan(&editor.cos, form, share * area)) { + covered += share; + forms.push((i, share)); + } + } + if covered >= COVERED { forms } else { Vec::new() } +} + +/// Whether Form `form`, covering `points` square points of the page, draws a picture that could +/// be a scan of it: one with at least [`MIN_FORM_DPI`] over that area, not a logo inside a page +/// of vector artwork. +fn holds_scan(cos: &pdfcraft_cos::Document, form: pdfcraft_cos::ObjRef, points: f64) -> bool { + let Some(picture) = crate::copy_image::picture_in_form(cos, form) else { return false }; + let obj = cos.get(picture); + let pdfcraft_cos::Object::Stream(s) = &*obj else { return false }; + let px = |k: &[u8]| s.dict.int(k).unwrap_or(0).max(0) as f64; + let per_point = MIN_FORM_DPI / 72.0; + px(b"Width") * px(b"Height") >= points * per_point * per_point +} + +/// The area of `info`'s crop box, when it is a usable one. +fn page_area(info: &PageInfo) -> Option { + let [cx0, cy0, cx1, cy1] = info.crop.map(f64::from); + let area = (cx1 - cx0) * (cy1 - cy0); + (area.is_finite() && area > 0.0).then_some(area) +} + +/// The share of the crop box (of area `area`) that `image` covers, when it is a number. +fn share(info: &PageInfo, area: f64, image: &pdfcraft_edit::PageImage) -> Option { + let [cx0, cy0, cx1, cy1] = info.crop.map(f64::from); + let r = image.visible_rect.unwrap_or(image.rect); + let w = (r[2].min(cx1) - r[0].max(cx0)).max(0.0); + let h = (r[3].min(cy1) - r[1].max(cy0)).max(0.0); + let share = w * h / area; + share.is_finite().then_some(share) +} + /// The page as displayed, in points with y up: where detection on a scan works. struct Frame<'a>(&'a PageInfo); diff --git a/crates/optimize/src/samples.rs b/crates/optimize/src/samples.rs index 7f402b51b8..1ed9f16b1f 100644 --- a/crates/optimize/src/samples.rs +++ b/crates/optimize/src/samples.rs @@ -53,6 +53,9 @@ pub fn decode_samples(doc: &Document, s: &Stream) -> Result if matches!(d.get(b"ImageMask").map(|v| doc.resolve(v)).as_deref(), Some(Object::Bool(true))) { return Err(NotDecoded::StencilMask); } + // The filter first: a JPEG 2000 image may leave its colour space and bit depth to the data, + // and the report should name the format, not the missing entries. + let jpeg = is_dct(s).ok_or(NotDecoded::Filter)?; let n = d.get(b"ColorSpace").and_then(|cs| components(doc, cs)).ok_or(NotDecoded::ColourSpace)?; let bpc = d.get(b"BitsPerComponent").map(|v| doc.resolve(v)).and_then(|v| v.as_int()).unwrap_or(0); let bilevel = match bpc { @@ -60,7 +63,6 @@ pub fn decode_samples(doc: &Document, s: &Stream) -> Result 1 if n == 1 => true, other => return Err(NotDecoded::BitDepth(other)), }; - let jpeg = is_dct(s).ok_or(NotDecoded::Filter)?; let width = d.get(b"Width").map(|v| doc.resolve(v)).and_then(|v| v.as_int()).and_then(|v| u32::try_from(v).ok()).unwrap_or(0); let height = d.get(b"Height").map(|v| doc.resolve(v)).and_then(|v| v.as_int()).and_then(|v| u32::try_from(v).ok()).unwrap_or(0); if width == 0 || height == 0 { @@ -258,6 +260,11 @@ mod tests { assert_eq!(base(&[("ColorSpace", Object::name("DeviceCMYK"))], vec![0; 64]), Err(NotDecoded::ColourSpace)); assert_eq!(base(&[("BitsPerComponent", Object::Int(4))], vec![0; 8]), Err(NotDecoded::BitDepth(4))); assert_eq!(base(&[("Filter", Object::name("JBIG2Decode"))], vec![0; 8]), Err(NotDecoded::Filter)); + // JPEG 2000 may leave the colour space and bit depth to its data: the format is the reason. + let jpx = |dict: &[(&str, Object)]| decode_samples(&doc(), &image(dict, vec![0; 8])); + let size = [("Width", Object::Int(4)), ("Height", Object::Int(4)), ("Filter", Object::name("JPXDecode"))]; + assert_eq!(jpx(&size), Err(NotDecoded::Filter)); + assert_eq!(jpx(&[size[0].clone(), size[1].clone(), size[2].clone(), ("ColorSpace", Object::name("DeviceRGB"))]), Err(NotDecoded::Filter)); assert_eq!(base(&[("ImageMask", Object::Bool(true))], vec![0; 8]), Err(NotDecoded::StencilMask)); assert_eq!(base(&[("Decode", Object::Array(vec![Object::Int(1), Object::Int(0)]))], vec![0; 16]), Err(NotDecoded::Decode)); // Too short, a huge size, a broken JPEG, no size at all. diff --git a/crates/ui-egui/src/workflows/optimize_ui.rs b/crates/ui-egui/src/workflows/optimize_ui.rs index 3bb80ac439..4e1a7c20bf 100644 --- a/crates/ui-egui/src/workflows/optimize_ui.rs +++ b/crates/ui-egui/src/workflows/optimize_ui.rs @@ -360,6 +360,11 @@ impl PdfCraftApp { self.notify_tr("Optimization is already running"); return; } + // Both jobs share the progress card and capture the same document state. + if self.scan_clean_run.is_some() { + self.notify_tr("Scanned pages are already being enhanced"); + return; + } let job = match self.session.optimize_job(id, settings, discard) { Ok(job) => job, Err(e) => return self.save_optimized(id, kind, Err(e)), @@ -491,4 +496,22 @@ mod tests { app.optimize_with_draft(); assert_eq!(app.toast.as_ref().map(|t| t.0.as_str()), Some("Optimization is already running")); } + + #[test] + fn optimize_and_enhance_scanned_file_do_not_run_at_once() { + let mut app = PdfCraftApp::new(); + app.set_option("language", "en").unwrap(); + app.open_bytes("notes.txt", None, b"hello".to_vec()).unwrap(); + let (_, doc) = app.active_ids().unwrap(); + app.scan_clean_run = Some(crate::scan_clean_ui::ScanCleanRun { doc, progress: Arc::default() }); + app.optimize_with_draft(); + assert_eq!(app.toast.as_ref().map(|t| t.0.as_str()), Some("Scanned pages are already being enhanced")); + assert!(app.optimize_run.is_none()); + + app.scan_clean_run = None; + app.optimize_run = Some(OptimizeRun { doc, kind: OptimizeKind::Reduce, progress: Arc::default() }); + app.start_scan_clean(); + assert_eq!(app.toast.as_ref().map(|t| t.0.as_str()), Some("Optimization is already running")); + assert!(app.scan_clean_run.is_none()); + } } diff --git a/crates/ui-egui/src/workflows/scan_clean_ui.rs b/crates/ui-egui/src/workflows/scan_clean_ui.rs index a275b5af20..64f8ce48eb 100644 --- a/crates/ui-egui/src/workflows/scan_clean_ui.rs +++ b/crates/ui-egui/src/workflows/scan_clean_ui.rs @@ -140,6 +140,11 @@ impl PdfCraftApp { self.notify_tr("Scanned pages are already being enhanced"); return; } + // Both jobs share the progress card and capture the same document state. + if self.optimize_run.is_some() { + self.notify_tr("Optimization is already running"); + return; + } // What's typed in a form field is part of the document (#166). if !self.commit_form_typing() { return; diff --git a/docs/gaps.md b/docs/gaps.md index ab6f083ab1..dacca2515b 100644 --- a/docs/gaps.md +++ b/docs/gaps.md @@ -165,8 +165,10 @@ update the numbers. ### 7. OCR reads only unaccented Latin - **Missing:** every other script and accented Latin; editable-text output; deskew, rotation, - despeckle and background clean-up as options of Recognize text (Enhance scanned file does them - on grey, RGB and 1-bit scans, not CMYK, indexed, JBIG2 or CCITT ones); MRC compression; + despeckle and background clean-up as options of Recognize text (Enhance scanned file removes + the background and edge shadows on grey and RGB scans and despeckles those and 1-bit ones; it + reports CMYK, indexed, JPEG 2000, JBIG2 and CCITT scans and scans drawn inside Form XObjects + and leaves them as they are); MRC compression; pluggable engines; OCR in the browser; scanner input. - **Evidence:** `ocr.ocr-languages` partial ("the models read only the Latin alphabet without diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 1ea04254aa..5f48c815a9 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -6677,7 +6677,7 @@ status = "partial" commands = ["ocr.enhance_camera", "ocr.enhance"] tools = ["page_enhance_camera", "page_enhance_scan"] evidence = ["crates/engine/src/camera_clean.rs::a_shadow_and_a_yellow_cast_become_white_paper", "crates/engine/src/tests.rs::enhance_camera_page_at_a_level_whitens_a_shadowed_sheet", "crates/ui-egui/tests/it/camera.rs::the_clean_up_level_enhances_again_from_the_photo_and_keeps_the_turn", "crates/engine/src/scan_clean/tests.rs::despeckle_is_its_own_switch", "crates/automation/tests/it/automation.rs::enhance_scanned_file_cleans_the_scan_in_place_through_tools", "crates/ui-egui/tests/it/scan_clean.rs::enhance_scanned_file_cleans_the_chosen_pages_and_says_what_it_did"] -notes = "Camera images, and scanned pages through Enhance scanned file (ocr.enhance, page_enhance_scan despeckle): isolated dark pixels (at most one dark neighbour) turn white, on grey, RGB and 1-bit scans, in place (the OCR text layer, comments and links stay). Missing: as an option of Recognize text; specks larger than two pixels; CMYK, indexed, JPEG 2000, JBIG2 and CCITT scans (reported, not changed)." +notes = "Camera images, and scanned pages through Enhance scanned file (ocr.enhance, page_enhance_scan despeckle): isolated dark pixels (at most one dark neighbour) turn white, on grey, RGB and 1-bit scans, in place (the OCR text layer, comments and links stay). Missing: as an option of Recognize text; specks larger than two pixels; CMYK, indexed, JPEG 2000, JBIG2 and CCITT scans and scans inside form XObjects (reported, not changed)." [[feature]] id = "ocr.background-removal" From 7878a4cb20604a57b7ae0cbc964bcf3c29b3a711 Mon Sep 17 00:00:00 2001 From: huppiflupp Date: Thu, 29 Oct 2026 16:37:23 +0100 Subject: [PATCH 022/198] M10: Enhance scanned file is tested to leave unchosen pages' scans alone A new engine test builds two pages that each draw their own scan image, cleans one page, and checks that page's image changed while the other image object, and its stream bytes, stayed identical; then the same with the other page chosen. Taking every page instead of the chosen ones fails it, on the report and, with that assertion removed, on the other image. Co-Authored-By: Claude Opus 5.5 --- crates/engine/src/scan_clean/tests.rs | 34 +++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/crates/engine/src/scan_clean/tests.rs b/crates/engine/src/scan_clean/tests.rs index f39f8c0432..11a1bca9c9 100644 --- a/crates/engine/src/scan_clean/tests.rs +++ b/crates/engine/src/scan_clean/tests.rs @@ -389,3 +389,37 @@ fn edge_shadow_removal_whitens_the_gutter_background_removal_alone_leaves() { assert!(grey_at(&after, 3, 40).abs_diff(grey_at(&before, 150, 40)) <= 4, "{} vs {}", grey_at(&after, 3, 40), grey_at(&before, 150, 40)); assert_eq!(grey_at(&after, 150, 40), grey_at(&before, 150, 40), "the middle is left alone"); } + +/// Two pages, each with its own scan image (objects 4 and 7), so cleaning one page can be told +/// apart from cleaning the other. +fn two_scans_pdf() -> Vec { + let dict = format!("/Type /XObject /Subtype /Image /Width {W} /Height {H} /ColorSpace /DeviceGray /BitsPerComponent 8"); + let draw = format!("q {W} 0 0 {H} 0 0 cm /Im0 Do Q"); + Pdf::new() + .obj("<< /Type /Catalog /Pages 2 0 R >>") + .obj("<< /Type /Pages /Kids [3 0 R 6 0 R] /Count 2 >>") + .obj(format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 {W} {H}] /Resources << /XObject << /Im0 4 0 R >> >> /Contents 5 0 R >>")) + .stream(&dict, scan_pixels(1)) + .stream("", &draw) + .obj(format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 {W} {H}] /Resources << /XObject << /Im0 7 0 R >> >> /Contents 8 0 R >>")) + .stream(&dict, scan_pixels(1)) + .stream("", &draw) + .build() +} + +#[test] +fn only_the_chosen_pages_scan_is_cleaned() { + // Clean page 1, then page 2 of a fresh copy: the other page's image stays byte-identical. + for (chosen, image, other) in [(0usize, 4u32, 7u32), (1, 7, 4)] { + let (mut s, id) = open(two_scans_pdf()); + let (image_before, other_before) = (object(&s, id, image), object(&s, id, other)); + assert_eq!(s.get(id).unwrap().page_images(chosen)[0].object, Some(ObjRef::new(image, 0)), "the fixture"); + let report = s.enhance_scanned_pages(id, &[chosen], ScanCleanSettings::at(100)).unwrap(); + assert_eq!((report.pages.as_slice(), report.images), (&[chosen][..], 1), "page {}", chosen + 1); + assert_ne!(object(&s, id, image), image_before, "page {}'s image was not cleaned", chosen + 1); + assert_eq!(object(&s, id, other), other_before, "cleaning page {} changed the other page's image", chosen + 1); + let (Object::Stream(was), Object::Stream(now)) = (&*other_before, &*object(&s, id, other)) else { panic!("the images") }; + assert_eq!(was.raw, now.raw, "the other image's stream bytes"); + assert_eq!(s.get(id).unwrap().can_undo(), Some("Enhance scanned file")); + } +} From 1a7663cb48d1ba6740a50c3a3a23b89f27e7a5f9 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sat, 31 Oct 2026 13:41:56 +0100 Subject: [PATCH 023/198] fix(ui): select the bookmark title when the rename editor opens (#5529) The inline rename field asked for focus without setting a selection, so the caret landed at the end of the default text and typing appended to it. The field now selects the whole title on the frame it takes focus, so typing replaces the old title at once, for both new bookmarks and rename. Later frames leave the user's own caret and selection alone. Tests: three unit tests on rename_field (first focus, multibyte title, caret kept across frames) and two kittest tests driving the panel like a user (typing replaces the title; the caret survives a collapse). Co-Authored-By: Zoo --- crates/ui-egui/src/panels/bookmarks.rs | 87 ++++++++++++++++++++++++-- crates/ui-egui/tests/it/bookmarks.rs | 34 ++++++++++ parity/acrobat-features.toml | 4 +- 3 files changed, 118 insertions(+), 7 deletions(-) diff --git a/crates/ui-egui/src/panels/bookmarks.rs b/crates/ui-egui/src/panels/bookmarks.rs index ccc45d4eb0..2a92a4c227 100644 --- a/crates/ui-egui/src/panels/bookmarks.rs +++ b/crates/ui-egui/src/panels/bookmarks.rs @@ -88,6 +88,24 @@ pub(crate) fn ellipsized_prefix(text: &str, mut fits: impl FnMut(&str) -> bool) } } +/// The inline rename field (#5529). On the frame it takes focus it selects the whole title, so +/// typing replaces the old text at once; later frames leave the user's own caret and selection +/// alone. Same pattern as the zoom box in `chrome.rs`. +fn rename_field(ui: &mut egui::Ui, text: &mut String, id: egui::Id) -> egui::Response { + let first_frame = !ui.memory(|m| m.has_focus(id)); + let resp = ui.add(crate::widgets::line(text).desired_width(ui.available_width() - 8.0).id(id)); + if !resp.has_focus() && !resp.lost_focus() { + if first_frame { + let mut state = egui::TextEdit::load_state(ui.ctx(), id).unwrap_or_default(); + let all = egui::text::CCursorRange::two(egui::text::CCursor::new(0), egui::text::CCursor::new(text.chars().count())); + state.cursor.set_char_range(Some(all)); + state.store(ui.ctx(), id); + } + resp.request_focus(); + } + resp +} + /// How a bookmark's title is drawn: in its own style and colour (ISO 32000-2 §12.3.3, Tables 151 /// and 152), bold in the heavier face and italic slanted, its colour kept readable on the panel. /// Top-level bookmarks without a style keep the medium face. @@ -138,11 +156,7 @@ pub(super) fn outline_item( ui.horizontal(|ui| { ui.add_space(x_text); crate::widgets::begin_dialog(ui); - let edit = crate::widgets::line(text).desired_width(ui.available_width() - 8.0).id(id.with("rename")); - let resp = ui.add(edit); - if !resp.has_focus() && !resp.lost_focus() { - resp.request_focus(); - } + let resp = rename_field(ui, text, id.with("rename")); if ui.input(|i| i.key_pressed(egui::Key::Escape)) { cancel = true; } else if resp.lost_focus() { @@ -360,3 +374,66 @@ mod bookmark_filter { assert!(outline_matches(&outline, &fold("izmir")).is_empty()); } } + +#[cfg(test)] +mod rename_field { + use super::rename_field as field; + + fn cursor_range(ctx: &egui::Context, id: egui::Id) -> Option<(usize, usize)> { + egui::TextEdit::load_state(ctx, id) + .and_then(|s| s.cursor.char_range()) + .map(|r| (r.primary.index.0.min(r.secondary.index.0), r.primary.index.0.max(r.secondary.index.0))) + } + + /// #5529: the rename field opens with the whole title selected, so typing replaces it. + #[test] + fn first_focus_selects_the_whole_title() { + let ctx = egui::Context::default(); + let id = egui::Id::unique("bm-rename-test"); + let mut text = String::from("Untitled"); + for _ in 0..2 { + let output = ctx.run_ui(egui::RawInput::default(), |ui| { + field(ui, &mut text, id); + }); + output.drop_without_applying_deltas(); + } + assert_eq!(cursor_range(&ctx, id), Some((0, 8))); + assert!(ctx.memory(|m| m.has_focus(id)), "the field takes focus"); + } + + /// A Unicode title is counted in characters, not bytes. + #[test] + fn a_multibyte_title_is_selected_in_characters() { + let ctx = egui::Context::default(); + let id = egui::Id::unique("bm-rename-unicode"); + let mut text = String::from("図面（一階平面図）"); + let output = ctx.run_ui(egui::RawInput::default(), |ui| { + field(ui, &mut text, id); + }); + output.drop_without_applying_deltas(); + assert_eq!(cursor_range(&ctx, id), Some((0, 9))); + } + + /// A later frame must not reset the selection the user made; a caret stays a caret. + #[test] + fn a_later_frame_keeps_the_users_selection() { + let ctx = egui::Context::default(); + let id = egui::Id::unique("bm-rename-keep"); + let mut text = String::from("Chapter"); + for _ in 0..2 { + let output = ctx.run_ui(egui::RawInput::default(), |ui| { + field(ui, &mut text, id); + }); + output.drop_without_applying_deltas(); + } + // The user moves the caret to 3 with no selection. + let mut state = egui::TextEdit::load_state(&ctx, id).unwrap_or_default(); + state.cursor.set_char_range(Some(egui::text::CCursorRange::one(egui::text::CCursor::new(3)))); + state.store(&ctx, id); + let output = ctx.run_ui(egui::RawInput::default(), |ui| { + field(ui, &mut text, id); + }); + output.drop_without_applying_deltas(); + assert_eq!(cursor_range(&ctx, id), Some((3, 3)), "the user's caret is kept"); + } +} diff --git a/crates/ui-egui/tests/it/bookmarks.rs b/crates/ui-egui/tests/it/bookmarks.rs index 1f07102a18..d5cd5efd2f 100644 --- a/crates/ui-egui/tests/it/bookmarks.rs +++ b/crates/ui-egui/tests/it/bookmarks.rs @@ -119,6 +119,40 @@ fn the_panel_keeps_its_width_while_the_pointer_moves() { assert_eq!(panel_width(&h), before, "the Bookmarks panel drifted wider"); } +/// #5529: the title is selected when the rename editor opens, so typing replaces it at once, +/// with no Ctrl+A. New bookmarks and double-click rename both get this. +#[test] +fn typing_replaces_the_title_when_the_editor_opens() { + let mut h = harness(); + h.key_press_modifiers(Modifiers::COMMAND, Key::B); + h.run_steps(3); + field(&h, "Untitled").type_text("Intro"); // the whole default is replaced + h.key_press(Key::Enter); + h.run_steps(3); + assert_eq!(outline(&h), [("Intro".into(), Some(0), 0)]); + + menu(&mut h, "Intro", "Rename"); + field(&h, "Intro").type_text("Introduction"); + h.key_press(Key::Enter); + h.run_steps(3); + assert_eq!(outline(&h)[0].0, "Introduction"); +} + +/// The selection is one-shot: after the user collapses it to a caret, later frames must not +/// reselect, so typing at that caret inserts. +#[test] +fn a_rename_editor_keeps_the_users_caret_across_frames() { + let mut h = harness(); + h.key_press_modifiers(Modifiers::COMMAND, Key::B); + h.run_steps(3); + h.key_press(Key::ArrowLeft); // collapses the opening selection to the caret at 0 + h.run_steps(2); + field(&h, "Untitled").type_text("I"); // inserts at the caret, does not replace + h.key_press(Key::Enter); + h.run_steps(3); + assert_eq!(outline(&h)[0].0, "IUntitled"); +} + #[test] fn escape_cancels_a_rename() { let mut h = harness(); diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 0b6f658dd6..d0377fc883 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -3291,8 +3291,8 @@ milestone = "M4" status = "shipped" commands = [] tools = ["bookmark_rename"] -evidence = ["crates/ui-egui/tests/it/bookmarks.rs::new_rename_reorder_indent_delete_and_undo", "crates/ui-egui/tests/it/bookmarks.rs::escape_cancels_a_rename", "crates/organize/src/tests.rs::bookmarks_add_rename_move_delete", "crates/organize/src/tests.rs::bookmark_edits_keep_unknown_keys_and_touch_few_objects"] -notes = "" +evidence = ["crates/ui-egui/tests/it/bookmarks.rs::new_rename_reorder_indent_delete_and_undo", "crates/ui-egui/tests/it/bookmarks.rs::escape_cancels_a_rename", "crates/ui-egui/tests/it/bookmarks.rs::typing_replaces_the_title_when_the_editor_opens", "crates/ui-egui/tests/it/bookmarks.rs::a_rename_editor_keeps_the_users_caret_across_frames", "crates/organize/src/tests.rs::bookmarks_add_rename_move_delete", "crates/organize/src/tests.rs::bookmark_edits_keep_unknown_keys_and_touch_few_objects"] +notes = "The inline editor opens with the whole title selected, so typing replaces it at once (#5529); a caret the user moves afterwards is kept by later frames." [[feature]] id = "organize.bookmark-delete" From 52080ac8983c616e1916beea4c5ecc4d76dd78eb Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sat, 31 Oct 2026 13:56:51 +0100 Subject: [PATCH 024/198] M3/M4: add Crop pages to the document quick toolbar (#5330) Co-Authored-By: Codex --- ROADMAP.md | 3 ++ crates/ui-egui/src/canvas/bars.rs | 6 +++ crates/ui-egui/tests/it/pageboxes.rs | 64 ++++++++++++++++++++++++++-- docs/roadmap.md | 5 ++- parity/acrobat-features.toml | 4 +- 5 files changed, 76 insertions(+), 6 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 56f6237e8d..9c098748d2 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,5 +1,6 @@ # PdfCraft roadmap +> **Last reviewed:** 2026-10-31 | **Last updated:** 2026-10-31 | **Change:** minor (M3/M4, #5330: Crop pages is directly available in the document quick toolbar; estimates unchanged) > **Last reviewed:** 2026-10-31 | **Last updated:** 2026-10-31 | **Change:** patch (M2, PR #4172: GBK rasters compared with explicit Unicode and independently selected installed faces; wrong-glyph and wrong-family negative controls fail; estimates unchanged) > **Last reviewed:** 2026-10-31 | **Last updated:** 2026-10-31 | **Change:** patch (M2, PR #4172: integrate main fa5f9f8c; retain CJK viewer and upstream raster regressions, restore explicit display_config imports; estimates unchanged) > **Last reviewed:** 2026-10-28 · **Last updated:** 2026-10-28 · **Change:** patch (M2, PR #4172: integrate main 5af10276; retain both roadmap histories and viewer-only CJK font policy; estimates unchanged) @@ -213,6 +214,8 @@ Ranked; detail and the full milestone table in [docs/roadmap.md](docs/roadmap.md ## Progress log +- **2026-10-31 (M3/M4, #5330):** The document quick toolbar offers Crop pages directly through the existing page.crop command, with the command’s assembly-permission check and an active-tool highlight. The existing crop, undo and headless page_set_box paths are unchanged. No toolbar customization or new shortcut. Estimates unchanged. + - **2026-10-31 (M2, PR #4172 review):** Added same-machine pixel oracles for GBK Song, Hei, Kai and FangSong, including all four subset-prefixed Founder aliases. Reference PDFs use explicit Unicode mappings and independently selected installed faces; font data and rasters remain in memory. All eight cases ran; wrong-glyph and wrong-family production mutations each failed the new assertion and were restored. Validation: 422 renderer unit tests, renderer all-target Clippy with -D warnings, formatting, layers/assets/parity and renderer WASM checks passed; existing optional-patch/WASM warnings and parity notes remain. Estimates unchanged. - **2026-10-31 (M2, PR #4172):** Integrated main fa5f9f8c, retaining both roadmap histories and raster regression groups. Restored display_config and thumbnail_config imports after the upstream explicit-import migration. Validation: 422 renderer unit tests, engine/UI all-target checks, scoped all-target Clippy with -D warnings, formatting, layers, assets, parity and renderer/engine WASM checks passed. Existing optional-patch/WASM warnings and parity notes remain; no full engine/UI runtime test rerun. Display-only font policy and estimates unchanged. diff --git a/crates/ui-egui/src/canvas/bars.rs b/crates/ui-egui/src/canvas/bars.rs index 2472366672..7c2ef781d8 100644 --- a/crates/ui-egui/src/canvas/bars.rs +++ b/crates/ui-egui/src/canvas/bars.rs @@ -296,6 +296,12 @@ pub(super) fn quick_bar(app: &mut PdfCraftApp, area: Rect, ui: &mut egui::Ui) { if icons::button(ui, "hand", 32.0, app.quick_tool == QuickTool::Hand, tl!("Hand (H)")).clicked() { app.quick_tool = QuickTool::Hand; } + let may_crop = pdfcraft_engine::commands::command("page.crop").is_some_and(|spec| app.command_enabled(spec)); + ui.add_enabled_ui(may_crop, |ui| { + if icons::button(ui, "crop", 32.0, app.quick_tool == QuickTool::Crop, tl!("Crop pages")).clicked() { + app.execute("page.crop"); + } + }); // Comment ▸, Highlight ▸, Draw ▸ (Acrobat's comment toolbar groups). Clicking a // group selects its last-used tool; clicking it again opens the flyout. for g in 0..comments::GROUPS.len() { diff --git a/crates/ui-egui/tests/it/pageboxes.rs b/crates/ui-egui/tests/it/pageboxes.rs index 2e2829e05b..c54b4f01ef 100644 --- a/crates/ui-egui/tests/it/pageboxes.rs +++ b/crates/ui-egui/tests/it/pageboxes.rs @@ -1,7 +1,7 @@ //! Crop tool, Set Page Boxes and Duplicate pages in the real shell (egui_kittest). use egui_kittest::Harness; -use egui_kittest::kittest::Queryable; +use egui_kittest::kittest::{NodeT, Queryable}; use pdfcraft_ui_egui::{Dialog, PdfCraftApp, QuickTool}; const FIXTURE: &[u8] = b"%PDF-1.7 @@ -37,6 +37,64 @@ fn the_crop_tool_crops_to_the_dragged_rectangle_then_returns_to_select() { assert!(h.state_mut().execute("page.crop")); assert_eq!(h.state().quick_tool, QuickTool::Crop); h.run_steps(2); + crop_middle_half(&mut h); +} + +#[test] +fn quick_toolbar_crop_crops_the_page_and_undo_restores_it() { + let mut h = harness(); + h.get_by_label("Crop pages").click(); + h.run_steps(2); + assert_eq!(h.state().quick_tool, QuickTool::Crop); + assert!(!h.state().views[0].organize); + crop_middle_half(&mut h); + assert!(h.state_mut().execute("edit.undo")); + h.run_steps(3); + assert_eq!(size(&h, 0), (300.0, 400.0)); +} + +#[test] +fn quick_toolbar_crop_is_disabled_when_page_assembly_is_forbidden() { + let mut doc = pdfcraft_cos::Document::open(std::sync::Arc::new(FIXTURE.to_vec())).unwrap(); + doc.set_encryption(&pdfcraft_cos::NewEncryption { + algorithm: pdfcraft_cos::Algorithm::Aes256, + user_password: "", + owner_password: "owner", + permissions: 0b0100, + encrypt_metadata: true, + seed: [5; 32], + }) + .unwrap(); + let bytes = pdfcraft_cos::write_full(&doc, &Default::default()).unwrap(); + let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).build_eframe(move |_| { + let mut app = PdfCraftApp::new(); + app.set_option("language", "en").unwrap(); + app.open_bytes("restricted.pdf", None, bytes).unwrap(); + app.set_option("left", "closed").unwrap(); + app + }); + h.run_steps(6); + assert!(h.get_by_label("Crop pages").accesskit_node().is_disabled()); + assert!(!h.state_mut().execute("page.crop")); + assert_eq!(h.state().quick_tool, QuickTool::Select); + assert_eq!(size(&h, 0), (300.0, 400.0)); +} + +#[test] +fn quick_toolbar_crop_is_hidden_on_home_and_in_read_mode() { + let mut h = harness(); + h.state_mut().execute("view.read_mode"); + h.run_steps(3); + assert!(h.query_by_label("Crop pages").is_none()); + h.state_mut().execute("view.read_mode"); + h.run_steps(3); + h.get_by_label("Crop pages"); + h.state_mut().active = None; + h.run_steps(3); + assert!(h.query_by_label("Crop pages").is_none()); +} + +fn crop_middle_half(h: &mut Harness<'static, PdfCraftApp>) { let r = h.state().views[0].page_screen_rect(0).unwrap(); // The middle half of the page in both directions. let (a, b) = (r.min + r.size() * 0.25, r.min + r.size() * 0.75); @@ -50,9 +108,9 @@ fn the_crop_tool_crops_to_the_dragged_rectangle_then_returns_to_select() { } h.drop_at(b); h.run_steps(4); - let (w, hh) = size(&h, 0); + let (w, hh) = size(h, 0); assert!((w - 150.0).abs() < 2.0 && (hh - 200.0).abs() < 2.0, "{w}×{hh}"); - assert_eq!(size(&h, 1), (300.0, 400.0)); + assert_eq!(size(h, 1), (300.0, 400.0)); assert_eq!(h.state().quick_tool, QuickTool::Select); assert_eq!(h.state().session.get(h.state().views[0].id).unwrap().can_undo(), Some("Crop page")); } diff --git a/docs/roadmap.md b/docs/roadmap.md index 0f1fb6f4d5..a09e597a36 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -1,5 +1,6 @@ # PdfCraft roadmap: milestones and what's next +> **Last reviewed:** 2026-10-31 | **Last updated:** 2026-10-31 | **Change:** minor (M3/M4, #5330: Crop pages is directly available in the document quick toolbar; estimates unchanged) > **Last reviewed:** 2026-10-31 | **Last updated:** 2026-10-31 | **Change:** patch (M2, PR #4172: GBK rasters compared with explicit Unicode and independently selected installed faces; wrong-glyph and wrong-family negative controls fail; estimates unchanged) > **Last reviewed:** 2026-10-31 | **Last updated:** 2026-10-31 | **Change:** patch (M2, PR #4172: integrate main fa5f9f8c; retain CJK viewer and upstream raster regressions, restore explicit display_config imports; estimates unchanged) > **Last reviewed:** 2026-10-28 · **Last updated:** 2026-10-28 · **Change:** patch (M2, PR #4172: integrate main 5af10276; retain both roadmap histories and viewer-only CJK font policy; estimates unchanged) @@ -194,7 +195,7 @@ total matches the parity estimate. | M2 | Model, render, text | 200–350 | 15% | 130–230 | hayro bootstrap renderer (vendored patches); inspector predictor scratch is bounded by available input, and JPX sample rescaling handles 1–32 bits safely. Text extraction reaches word-F1 0.98 against pdftotext. Short same-line redraws split before placement rather than interleaving. Missing: model crate, fonts, DisplayList, renderer independent of hayro. Inspector page-label traversal and output are resource-bounded, with physical-number fallback warnings. Packed TIFF-predictor RGB rows retain later rows in decoding and rendering. Page marks, added content and print output retain small page-unit transform coefficients when serialized. Forms without Resources use page Properties for ActualText and layer visibility; own dictionaries stop fallback. Oversized missing inline-image samples, including header and terminator overhead, no longer pin the content window (#3107 / PR #3438); normal large streamed images retain deterministic sample boundaries. Radial luminosity fades have synthetic coverage for the existing #2132 mask guard, including per-pixel references, antialiased clips, previews, scales and tiles; no additional production fix was needed. Dotted TOC leaders no longer inflate letter tracking or hide word spaces (#3086); phrase search and selection are covered. Short tracked ellipses retain their gap samples and remain searchable. Missing spacing glyphs in non-embedded Japanese CID substitutes render blank without changing PDF advances or Unicode (#3601). The desktop viewer can use installed CJK fonts, including Windows GBK Song/Hei/Kai/FangSong and subset-prefixed Founder names; Combine thumbnails and accessibility figure previews use the same display-only fallback, without changing export or print font policy (#234, continuation of #1567, review #4172). Print previews exclude installed faces and isolate their font cache; GBK regressions cover view/print switching for inline and pooled whole-page/tile rendering. Installed-font helpers and their regression contracts have function documentation. GBK regression rasters match explicit-Unicode references with independently selected installed faces; wrong-glyph and wrong-family negative controls fail. Integration with main fa5f9f8c retains viewer fallback across the split engine session modules, renderer tests and Combine thumbnail workers. | | M2 | Model, render, text | 200–350 | 15% | 130–230 | hayro bootstrap renderer (vendored patches); inspector predictor scratch is bounded by available input, and JPX sample rescaling handles 1–32 bits safely. Text extraction reaches word-F1 0.98 against pdftotext. Short same-line redraws split before placement rather than interleaving. Missing: model crate, fonts, DisplayList, renderer independent of hayro. Inspector page-label traversal and output are resource-bounded, with physical-number fallback warnings. Packed TIFF-predictor RGB rows retain later rows in decoding and rendering. The renderer keeps its one-point crop-size floor in physical points after UserUnit scaling, with regression coverage for crop origins, rotations and f32 reciprocal rounding. Existing unit policy and nearly-zero fallback are retained. | | M2 | Model, render, text | 200–350 | 15% | 130–230 | hayro bootstrap renderer (vendored patches); inspector predictor scratch is bounded by available input, and JPX sample rescaling handles 1–32 bits safely. Text extraction reaches word-F1 0.98 against pdftotext. Short same-line redraws split before placement rather than interleaving. Missing: model crate, fonts, DisplayList, renderer independent of hayro. Inspector page-label traversal and output are resource-bounded, with physical-number fallback warnings | -| M3 | Viewer app (native + web) | 80–150 | 90% | 10–20 | Acrobat-style shell, find, select, panels, tiles, web build, UI control channel for agents (opt-in). #3808: the Pages inspector offers a 50–150% thumbnail slider and buttons, also available through `ui.set`. Missing: 60 fps test on a 500-page document, snapshot tests of every panel
Done since: fit visible, document title in the window; Linux middle-button scrolling; shared three-state theme preferences in the toolbar, View menu and control channel; numeric Linux portal events and idle wakeups completing #881, with native appearance inheritance retained (#446); late browser startup downloads preserve the current document and Save target (#171). Selected-text Copy in Edit and the canvas context menu is covered by focused regressions and native clipboard checks (#1225); document copy restrictions apply to the floating action too. #1364 candidate: absolute local GoToR links ask before target access; headless callers explicitly opt in. Native-opening candidate: chosen tools survive preparation and password retries; startup intake waits for pending preparation. Page-label navigation displays logical labels, retains the current page on unchanged submission and refreshes focused same-document replacements; clean file-watcher reload preserves the chosen scrolling mode, preventing a Single Page reader from jumping to page 1 during replacement. Synthetic control regressions cover these behaviors. #3149: page-list changes discard render demand based on old geometry; actual raster dimensions are checked against the GPU limit before upload. Synthetic crash and continued-rendering regressions cover the replacement path. Save-prompt shortcuts yield to dialogs above them; stacked link-consent UI regression covers Escape and Return. Comment hover follows individual marked lines and original annotation paint order (#3541/#3620), independently of spatial inspection sorting; synthetic rotation, differing-upper-edge overlap and unchanged-document regressions. Optional page gaps join page edges without cropping PDF margins; current-document toggle, persisted default and UI control options are covered by viewer tests. Gap changes in Organize Pages preserve document scroll position, and both comparison panes follow menu and control choices. Repeated window resizes by an edge and moves by the tab strip on GNOME/Wayland work without a click in between. | +| M3 | Viewer app (native + web) | 80–150 | 90% | 10–20 | Acrobat-style shell, find, select, panels, tiles, web build, UI control channel for agents (opt-in). #3808: the Pages inspector offers a 50–150% thumbnail slider and buttons, also available through `ui.set`. Missing: 60 fps test on a 500-page document, snapshot tests of every panel
Done since: fit visible, document title in the window; Linux middle-button scrolling; shared three-state theme preferences in the toolbar, View menu and control channel; numeric Linux portal events and idle wakeups completing #881, with native appearance inheritance retained (#446); late browser startup downloads preserve the current document and Save target (#171). Selected-text Copy in Edit and the canvas context menu is covered by focused regressions and native clipboard checks (#1225); document copy restrictions apply to the floating action too. #1364 candidate: absolute local GoToR links ask before target access; headless callers explicitly opt in. Native-opening candidate: chosen tools survive preparation and password retries; startup intake waits for pending preparation. Page-label navigation displays logical labels, retains the current page on unchanged submission and refreshes focused same-document replacements; clean file-watcher reload preserves the chosen scrolling mode, preventing a Single Page reader from jumping to page 1 during replacement. Synthetic control regressions cover these behaviors. #3149: page-list changes discard render demand based on old geometry; actual raster dimensions are checked against the GPU limit before upload. Synthetic crash and continued-rendering regressions cover the replacement path. Save-prompt shortcuts yield to dialogs above them; stacked link-consent UI regression covers Escape and Return. Comment hover follows individual marked lines and original annotation paint order (#3541/#3620), independently of spatial inspection sorting; synthetic rotation, differing-upper-edge overlap and unchanged-document regressions. Optional page gaps join page edges without cropping PDF margins; current-document toggle, persisted default and UI control options are covered by viewer tests. Gap changes in Organize Pages preserve document scroll position, and both comparison panes follow menu and control choices. Repeated window resizes by an edge and moves by the tab strip on GNOME/Wayland work without a click in between. Crop pages is directly available in the quick toolbar (#5330). | | M3 | Viewer app (native + web) | 80–150 | 90% | 10–20 | Acrobat-style shell, find, select, panels, tiles, web build, UI control channel for agents (opt-in). Missing: 60 fps test on a 500-page document, snapshot tests of every panel
Done since: fit visible, document title in the window; Linux middle-button scrolling; shared three-state theme preferences in the toolbar, View menu and control channel; numeric Linux portal events and idle wakeups completing #881, with native appearance inheritance retained (#446); late browser startup downloads preserve the current document and Save target (#171). Selected-text Copy in Edit and the canvas context menu is covered by focused regressions and native clipboard checks (#1225); document copy restrictions apply to the floating action too. #1364 candidate: absolute local GoToR links ask before target access; headless callers explicitly opt in. Native-opening candidate: chosen tools survive preparation and password retries; startup intake waits for pending preparation. Page-label navigation displays logical labels, retains the current page on unchanged submission and refreshes focused same-document replacements; clean file-watcher reload preserves the chosen scrolling mode, preventing a Single Page reader from jumping to page 1 during replacement. Synthetic control regressions cover these behaviors. Wrapped revision buttons stay above Binding in Document Properties (#2739); a synthetic UI test checks layout and saved revision content. | | M3 | Viewer app (native + web) | 80–150 | 90% | 10–20 | Acrobat-style shell, find, select, panels, tiles, web build, UI control channel for agents (opt-in). Missing: 60 fps test on a 500-page document, snapshot tests of every panel
Done since: fit visible, document title in the window; Linux middle-button scrolling; shared three-state theme preferences in the toolbar, View menu and control channel; numeric Linux portal events and idle wakeups completing #881, with native appearance inheritance retained (#446); late browser startup downloads preserve the current document and Save target (#171). #1364 candidate: absolute local GoToR links ask before target access; headless callers explicitly opt in. Native-opening candidate: chosen tools survive preparation and password retries; startup intake waits for pending preparation. Later native-open replies stay queued while a modal or picker owns the foreground; regressions cover recipient ownership, completion/cancellation and one-shot wakes. | | M3 | Viewer app (native + web) | 80–150 | 90% | 10–20 | Acrobat-style shell, find, select, panels, tiles, web build, UI control channel for agents (opt-in). Missing: 60 fps test on a 500-page document, snapshot tests of every panel
Done since: fit visible, document title in the window; Linux middle-button scrolling; shared three-state theme preferences in the toolbar, View menu and control channel; numeric Linux portal events and idle wakeups completing #881, with native appearance inheritance retained (#446); late browser startup downloads preserve the current document and Save target (#171). #1364 candidate: absolute local GoToR links ask before target access; headless callers explicitly opt in. Native-opening candidate: chosen tools survive preparation and password retries; startup intake waits for pending preparation. #1980: exact quit modifiers preserve Ctrl+Alt+Q email input; Ctrl+Q/Cmd+Q asks before discarding unsaved documents. | @@ -274,6 +275,8 @@ M2 (renderer), M7 (content editing), M10 (OCR and Office), M11 (Preflight and st ## Revision history +**2026-10-31 · minor:** M3/M4 #5330: direct quick-toolbar Crop pages action reuses command permissions and the existing crop tool. Estimates unchanged. + **2026-10-26 · patch:** M2/PR #4172: reconcile current main and preserve upstream Japanese serif/weight selection alongside the installed CJK viewer and preview fallback. Estimates unchanged. **2026-10-25 · minor:** M14 localization current state: Arabic catalogs stay logical, Arabic/Kurdish labels take an explicit RTL direction, and stock chrome mirrors; native/browser RTL input, IME and complex workflows remain unqualified. Native Simplified Chinese uses LXGW WenKai Lite; the web build has no Hans face. Estimates unchanged. diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 0b6f658dd6..bb3f8e7762 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -3195,8 +3195,8 @@ milestone = "M4" status = "shipped" commands = ["page.crop"] tools = ["page_set_box"] -evidence = ["crates/ui-egui/tests/it/pageboxes.rs::the_crop_tool_crops_to_the_dragged_rectangle_then_returns_to_select"] -notes = "drag to crop; double-click opens Set Page Boxes" +evidence = ["crates/ui-egui/tests/it/pageboxes.rs::the_crop_tool_crops_to_the_dragged_rectangle_then_returns_to_select", "crates/ui-egui/tests/it/pageboxes.rs::quick_toolbar_crop_crops_the_page_and_undo_restores_it", "crates/ui-egui/tests/it/pageboxes.rs::quick_toolbar_crop_is_disabled_when_page_assembly_is_forbidden", "crates/ui-egui/tests/it/pageboxes.rs::quick_toolbar_crop_is_hidden_on_home_and_in_read_mode"] +notes = "drag to crop; double-click opens Set Page Boxes; Crop pages is also directly available in the document quick toolbar (#5330), using the existing command availability and assembly permission check. The quick toolbar is hidden on Home and in Read/full-screen modes; arbitrary toolbar customization remains outside this feature." [[feature]] id = "organize.remove-white-margins" From 0e7e6a00205a3e44d1e16a83daaf9f258402c92a Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 1 Nov 2026 13:13:29 +0100 Subject: [PATCH 025/198] Test: pick the camera panel's Crop pages button now that the quick toolbar has one Co-Authored-By: Claude Sonnet 5.5 --- crates/ui-egui/tests/it/camera.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/ui-egui/tests/it/camera.rs b/crates/ui-egui/tests/it/camera.rs index 99d44ec3b7..88d79d29c1 100644 --- a/crates/ui-egui/tests/it/camera.rs +++ b/crates/ui-egui/tests/it/camera.rs @@ -147,7 +147,8 @@ fn after_enhance_page_crop_and_organize_take_over() { h.get_by_label("Enhance page").click(); h.run_steps(4); assert_eq!(h.state().views[0].camera_done.map(|d| d.page), Some(0)); - h.get_by_label(button).click(); + // The quick toolbar also has a "Crop pages" button; the camera panel's pill is the first match. + h.get_all_by_label(button).next().unwrap().click(); h.run_steps(3); assert_eq!(h.state().quick_tool, then, "{button}"); assert_eq!(h.state().views[0].camera_done, None, "{button}"); From 35eaeca321597b44026f5f8a71bd0af9c9076e64 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sun, 1 Nov 2026 13:35:14 +0100 Subject: [PATCH 026/198] ci: re-run checks Co-Authored-By: Claude Sonnet 5.5 From 315e83b88c8e06f308fbbd1826ddc3c864e4ba18 Mon Sep 17 00:00:00 2001 From: KenM76 Date: Mon, 2 Nov 2026 15:22:50 -0500 Subject: [PATCH 027/198] perf(cos): an object that can't be read is searched for once, not on every read Document::try_get kept only objects that loaded. An object whose entry points at something that doesn't parse falls back to a whole-file search (scan_for); that failed too, nothing was kept, and the next read searched the whole file again. A caller that reads such an object repeatedly, such as the window title of a file whose document information dictionary is damaged and that asks for its title to be shown, paid one whole-file search per read, several per frame. The cache now keeps the load result, a failure as well. It is the same map with the same bounds, clears and per-reader privacy, so every place that drops loaded objects (authentication, reconstruction, private scan readers, eviction) drops remembered failures too. Edits are read from the overlay before the cache, as before, so a set or freed number still answers for itself. A failure is kept in nested loads too, as a success already is; only a poisoned lock is never kept. Failures count towards the cache's object caps and hold no bytes of their own. Sharing the map between clones stays sound because entries, data and the stream limit don't change after open. Test: an_unreadable_object_is_searched_for_once counts whole-file searches through a test-only counter (like SECTION_READS): one for eleven reads and a clone's read. With failures not kept it counts 22. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01SqmHYx8FiWM5PcsoiErgp9 --- crates/cos/src/document.rs | 10 +++++---- crates/cos/src/document/objects.rs | 35 +++++++++++++++++------------- crates/cos/src/document/state.rs | 12 +++++++++- crates/cos/src/document/tests.rs | 31 +++++++++++++++++++++++--- 4 files changed, 65 insertions(+), 23 deletions(-) diff --git a/crates/cos/src/document.rs b/crates/cos/src/document.rs index 082d1910d1..6e5891d1a4 100644 --- a/crates/cos/src/document.rs +++ b/crates/cos/src/document.rs @@ -24,10 +24,10 @@ pub(crate) use reader::find; #[cfg(test)] use reader::rfind; pub(crate) use state::ObjectReader; -use state::{ObjStm, Slot}; -pub use state::{ReadState, SourceIdentity}; +use state::{Loaded, ObjStm, Slot}; #[cfg(test)] -use state::{SECTION_READS, objstm_member, owned_bytes}; +use state::{OBJECT_SEARCHES, SECTION_READS, objstm_member, owned_bytes}; +pub use state::{ReadState, SourceIdentity}; use xref::XrefIndex; pub use xref::{Revision, XrefEntry}; @@ -47,7 +47,9 @@ pub struct Document { /// `fill_xref_gaps`. located: Arc>, repair_log: Arc>, - cache: Arc>>>, + /// What loading each file object gave, a failure as well: a damaged object is otherwise + /// searched for across the whole file on every read (`Document::load`). + cache: Arc>>, objstms: Arc>>>, /// What `/JBIG2Globals` streams of the file decode to, by object number: the images of a scan /// share them, and each would otherwise decode them again (see `Document::jbig2_globals`). diff --git a/crates/cos/src/document/objects.rs b/crates/cos/src/document/objects.rs index 2ac18cff03..2ca327de74 100644 --- a/crates/cos/src/document/objects.rs +++ b/crates/cos/src/document/objects.rs @@ -1,5 +1,5 @@ use super::reader::find; -use super::state::{LENGTHS, LoadGuard, ObjStm, ObjectReader, ReadState, Slot, SourceIdentity, objstm_member, owned_bytes}; +use super::state::{LENGTHS, LoadGuard, Loaded, ObjStm, ObjectReader, ReadState, Slot, SourceIdentity, loaded_bytes, objstm_member}; use super::xref::{Revision, XrefEntry}; use super::{ Arc, CosError, Cow, Dict, Document, Lexer, MAX_DECODED, MAX_JBIG2_GLOBALS, ObjRef, Object, Stream, is_whitespace, parse_indirect_shared, @@ -388,20 +388,23 @@ impl Document { Some(Slot::Freed(_)) => return Ok(Arc::new(Object::Null)), None => {} } - if let Some(o) = self.cache.lock().map_err(|_| CosError::Poisoned)?.get(&num) { - return Ok(o.clone()); + if let Some(loaded) = self.cache.lock().map_err(|_| CosError::Poisoned)?.get(&num) { + return loaded.clone(); } // Loading can re-enter (indirect stream `/Length`, object streams); damaged files make // cycles such as `6 0 obj << /Length 6 0 R >>`. Bound the nesting on every path. let _guard = LoadGuard::enter(num).ok_or_else(|| CosError::Syntax { offset: 0, detail: format!("reference cycle while loading object {num}") })?; - let obj = self.load(num, 0)?; - let obj = Arc::new(obj); - // The editing cache is bounded like the scan reader's: re-parsing a file-backed view - // is cheap, and returned objects outlive eviction. - self.cache.lock().map_err(|_| CosError::Poisoned)?.insert(num, obj.clone()); - self.evict_editor_cache_if_over_budget(num, &obj)?; - Ok(obj) + let loaded = self.load(num, 0).map(Arc::new); + // A failure is kept like an object, or every read of a damaged object searches the whole + // file again. A poisoned lock says nothing about the object. + if !matches!(loaded, Err(CosError::Poisoned)) { + // The editing cache is bounded like the scan reader's: re-parsing a file-backed view + // is cheap, and returned objects outlive eviction. + self.cache.lock().map_err(|_| CosError::Poisoned)?.insert(num, loaded.clone()); + self.evict_editor_cache_if_over_budget(num, &loaded)?; + } + loaded } /// Bounds for the editing cache: roomier than the scan reader's (128 objects / 8 MB) @@ -430,17 +433,17 @@ impl Document { /// byte budget can double. const EDITOR_LARGE_OBJECT: usize = 256 * 1024; - fn evict_editor_cache_if_over_budget(&self, fresh_num: u32, fresh: &Object) -> Result<(), CosError> { + fn evict_editor_cache_if_over_budget(&self, fresh_num: u32, fresh: &Loaded) -> Result<(), CosError> { // Fast path: the full fold is O(cache) per insert, so skip it for a small newcomer // below the count cap. `len` is copied out first; nothing here holds the lock. let len = self.cache.lock().map_err(|_| CosError::Poisoned)?.len(); - if len <= Self::EDITOR_CACHE_OBJECTS && owned_bytes(fresh, &self.data, 0) <= Self::EDITOR_LARGE_OBJECT { + if len <= Self::EDITOR_CACHE_OBJECTS && loaded_bytes(fresh, &self.data) <= Self::EDITOR_LARGE_OBJECT { return Ok(()); } // Walk a snapshot off-lock; re-lock only to evict. - let live: Vec<(u32, Arc)> = self.cache.lock().map_err(|_| CosError::Poisoned)?.iter().map(|(n, o)| (*n, o.clone())).collect(); + let live: Vec<(u32, Loaded)> = self.cache.lock().map_err(|_| CosError::Poisoned)?.iter().map(|(n, o)| (*n, o.clone())).collect(); let data = &self.data; - let owned: usize = live.iter().fold(0usize, |n, (_, o)| n.saturating_add(owned_bytes(o, data, 0))); + let owned: usize = live.iter().fold(0usize, |n, (_, o)| n.saturating_add(loaded_bytes(o, data))); if live.len() <= Self::EDITOR_CACHE_OBJECTS && owned <= Self::EDITOR_CACHE_BYTES { return Ok(()); } @@ -450,7 +453,7 @@ impl Document { let mut cache = self.cache.lock().map_err(|_| CosError::Poisoned)?; if cache.len() <= Self::EDITOR_CACHE_OBJECTS { let data = &self.data; - let owned: usize = cache.values().fold(0usize, |n, o| n.saturating_add(owned_bytes(o, data, 0))); + let owned: usize = cache.values().fold(0usize, |n, o| n.saturating_add(loaded_bytes(o, data))); if owned <= Self::EDITOR_CACHE_BYTES { return Ok(()); } @@ -602,6 +605,8 @@ impl Document { /// Last-resort lookup: scan the file for `num G obj`. fn scan_for(&self, num: u32) -> Option<(ObjRef, Object)> { + #[cfg(test)] + super::OBJECT_SEARCHES.with(|c| c.set(c.get() + 1)); let needle = format!("{num} "); let data = &self.data; let mut found = None; diff --git a/crates/cos/src/document/state.rs b/crates/cos/src/document/state.rs index b4323f982e..121a59c2e7 100644 --- a/crates/cos/src/document/state.rs +++ b/crates/cos/src/document/state.rs @@ -30,6 +30,14 @@ impl LoadGuard { } } +/// What loading a file object gave, kept in the document's cache either way. +pub(super) type Loaded = Result, CosError>; + +/// The bytes a load result holds of its own ([`owned_bytes`]); a failure holds none. +pub(super) fn loaded_bytes(loaded: &Loaded, data: &Arc>) -> usize { + loaded.as_ref().map_or(0, |o| owned_bytes(o, data, 0)) +} + impl Drop for LoadGuard { fn drop(&mut self) { let outermost = LOADING.with(|l| { @@ -165,7 +173,7 @@ impl ObjectReader { // have cached large objects. if let Ok(mut cache) = self.document.cache.lock() { let data = &self.document.data; - if cache.len() >= Self::OBJECTS || cache.values().fold(0usize, |n, o| n.saturating_add(owned_bytes(o, data, 0))) > Self::STREAM_BYTES { + if cache.len() >= Self::OBJECTS || cache.values().fold(0usize, |n, o| n.saturating_add(loaded_bytes(o, data))) > Self::STREAM_BYTES { cache.clear(); } } @@ -177,6 +185,8 @@ impl ObjectReader { thread_local! { /// Cross-reference sections read on this thread. pub(super) static SECTION_READS: std::cell::Cell = const { std::cell::Cell::new(0) }; + /// Whole-file searches for an object (`Document::scan_for`) on this thread. + pub(super) static OBJECT_SEARCHES: std::cell::Cell = const { std::cell::Cell::new(0) }; } /// Nesting [`owned_bytes`] walks: past the parser's own nesting limit, so every parsed object diff --git a/crates/cos/src/document/tests.rs b/crates/cos/src/document/tests.rs index cca4b87326..3cebe0136d 100644 --- a/crates/cos/src/document/tests.rs +++ b/crates/cos/src/document/tests.rs @@ -588,6 +588,31 @@ fn an_entry_that_misses_its_object_is_found_by_searching_but_is_not_exact() { assert!(doc.xref_entry_is_exact(1) && doc.xref_entry_is_exact(2)); } +/// An object that can't be read at its entry or anywhere else is searched for in the whole file +/// once: every later read, in a clone too, gives the same error without searching again. An +/// edit of that number still answers for it. +#[test] +fn an_unreadable_object_is_searched_for_once() { + let bodies = ["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", "<< /Odd Key(x) >>"]; + let doc = Document::open(Arc::new(build(&bodies, "/Root 1 0 R /Info 3 0 R"))).unwrap(); + // Counted wherever the whole-file search is done (`scan_for`). + OBJECT_SEARCHES.with(|c| c.set(0)); + let first = doc.try_get(3); + assert!(first.is_err(), "{first:?}"); + for _ in 0..10 { + assert_eq!(doc.try_get(3), first); + assert_eq!(*doc.get(ObjRef::new(3, 0)), Object::Null); + } + let mut edited = doc.clone(); + assert_eq!(edited.try_get(3), first); + assert_eq!(OBJECT_SEARCHES.with(|c| c.get()), 1, "searches for an object that isn't there"); + edited.set(ObjRef::new(3, 0), Object::Int(7)); + assert_eq!(edited.try_get(3).as_deref(), Ok(&Object::Int(7))); + assert_eq!(doc.try_get(3), first, "the edit is the clone's alone"); + edited.free(ObjRef::new(3, 0)); + assert_eq!(edited.try_get(3).as_deref(), Ok(&Object::Null)); +} + /// A classic table with `/Size` rows and object 0's free entry first, its subsection numbered /// `1 n` instead of `0 n` (#2676): every row is one number off. fn misnumbered_table(start: u32) -> Vec { @@ -1109,7 +1134,7 @@ fn temporary_reader_keeps_file_views_and_evicts_data_of_its_own() { // Data of its own (decrypted streams, strings) counts: past the budget the cache is // cleared after the next read, including a failed one. let owned = Object::Stream(crate::Stream::from_raw(Dict::new(), vec![b'y'; ObjectReader::STREAM_BYTES + 1])); - reader.document.cache.lock().unwrap().insert(3, Arc::new(owned)); + reader.document.cache.lock().unwrap().insert(3, Ok(Arc::new(owned))); assert_eq!(*reader.get(ObjRef::new(99, 0)), Object::Null); assert!(reader.document.cache.lock().unwrap().is_empty(), "owned data over budget is evicted"); } @@ -1198,8 +1223,8 @@ fn editor_cache_counts_deeply_nested_strings() { let Object::Ref(reference) = reference else { panic!("indirect object") }; drop(doc.get(*reference)); let cache = doc.cache.lock().unwrap(); - let owned = cache.values().fold(0usize, |n, o| n.saturating_add(owned_bytes(o, &doc.data, 0))); - let strings = cache.values().filter(|o| o.as_array().is_some()).count(); + let owned = cache.values().flatten().fold(0usize, |n, o| n.saturating_add(owned_bytes(o, &doc.data, 0))); + let strings = cache.values().flatten().filter(|o| o.as_array().is_some()).count(); assert!(strings << 20 <= Document::EDITOR_CACHE_BYTES + (1 << 20), "{strings} MiB strings cached, {owned} bytes counted"); } } From 84d4b8cab9b8ee1d6d0c3fbed70ea3291c22a1b4 Mon Sep 17 00:00:00 2001 From: KenM76 Date: Mon, 2 Nov 2026 16:21:45 -0500 Subject: [PATCH 028/198] fix(cos): a load that read one clone's edits is not shared with the others Clones share the object cache and the parsed object streams. A clone that freed or replaced an object stream then read a member of it put the result in those shared maps: a failure (which this branch now caches) or, as on main, the replacement stream and the member parsed from it. Another clone whose stream is intact then read that instead of the file. A load that reads an unsaved edit (an edited object stream, or an edited indirect /Length) is no longer cached, and an edited object stream is parsed for its own document only. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01SqmHYx8FiWM5PcsoiErgp9 --- crates/cos/src/document/objects.rs | 30 ++++++++++++++++++++++-------- crates/cos/src/document/state.rs | 11 +++++++++++ crates/cos/src/document/tests.rs | 21 +++++++++++++++++++++ 3 files changed, 54 insertions(+), 8 deletions(-) diff --git a/crates/cos/src/document/objects.rs b/crates/cos/src/document/objects.rs index 2ca327de74..55a01e117c 100644 --- a/crates/cos/src/document/objects.rs +++ b/crates/cos/src/document/objects.rs @@ -1,5 +1,7 @@ use super::reader::find; -use super::state::{LENGTHS, LoadGuard, Loaded, ObjStm, ObjectReader, ReadState, Slot, SourceIdentity, loaded_bytes, objstm_member}; +use super::state::{ + LENGTHS, LoadGuard, Loaded, ObjStm, ObjectReader, ReadState, Slot, SourceIdentity, loaded_bytes, note_overlay_read, objstm_member, overlay_reads, +}; use super::xref::{Revision, XrefEntry}; use super::{ Arc, CosError, Cow, Dict, Document, Lexer, MAX_DECODED, MAX_JBIG2_GLOBALS, ObjRef, Object, Stream, is_whitespace, parse_indirect_shared, @@ -383,10 +385,12 @@ impl Document { } pub fn try_get(&self, num: u32) -> Result, CosError> { - match self.overlay.get(&num) { - Some(Slot::Set(_, o)) => return Ok(o.clone()), - Some(Slot::Freed(_)) => return Ok(Arc::new(Object::Null)), - None => {} + if let Some(slot) = self.overlay.get(&num) { + note_overlay_read(); + return Ok(match slot { + Slot::Set(_, o) => o.clone(), + Slot::Freed(_) => Arc::new(Object::Null), + }); } if let Some(loaded) = self.cache.lock().map_err(|_| CosError::Poisoned)?.get(&num) { return loaded.clone(); @@ -395,10 +399,12 @@ impl Document { // cycles such as `6 0 obj << /Length 6 0 R >>`. Bound the nesting on every path. let _guard = LoadGuard::enter(num).ok_or_else(|| CosError::Syntax { offset: 0, detail: format!("reference cycle while loading object {num}") })?; + let edits_read = overlay_reads(); let loaded = self.load(num, 0).map(Arc::new); // A failure is kept like an object, or every read of a damaged object searches the whole - // file again. A poisoned lock says nothing about the object. - if !matches!(loaded, Err(CosError::Poisoned)) { + // file again. A poisoned lock says nothing about the object. Clones share the cache, so + // a load that read this document's edits (an edited object stream or `/Length`) is not kept. + if overlay_reads() == edits_read && !matches!(loaded, Err(CosError::Poisoned)) { // The editing cache is bounded like the scan reader's: re-parsing a file-backed view // is cheap, and returned objects outlive eviction. self.cache.lock().map_err(|_| CosError::Poisoned)?.insert(num, loaded.clone()); @@ -540,6 +546,9 @@ impl Document { None | Some(XrefEntry::Free { .. }) => Ok(Object::Null), Some(XrefEntry::InFile { offset, .. }) => { let resolve = |r: ObjRef| { + if self.overlay.contains_key(&r.num) { + note_overlay_read(); + } if let Some(known) = LENGTHS.with(|m| m.borrow().get(&r.num).copied()) { return known; } @@ -570,7 +579,9 @@ impl Document { } pub(super) fn objstm(&self, num: u32) -> Result, CosError> { - if let Some(s) = self.objstms.lock().map_err(|_| CosError::Poisoned)?.get(&num) { + // An edited or freed stream is this document's alone: clones share the parsed streams. + let edited = self.overlay.contains_key(&num); + if !edited && let Some(s) = self.objstms.lock().map_err(|_| CosError::Poisoned)?.get(&num) { return Ok(s.clone()); } // A file entry that cannot place the stream (one that says it is itself compressed @@ -596,6 +607,9 @@ impl Document { index.push(objstm_member(onum, off, first, data.len())); } let stm = Arc::new(ObjStm { index, data: Arc::new(data) }); + if edited { + return Ok(stm); + } // Same bound as the object cache: decoded object streams can be megabytes each. // Insert first so the retained `num` is actually in the map. self.objstms.lock().map_err(|_| CosError::Poisoned)?.insert(num, stm.clone()); diff --git a/crates/cos/src/document/state.rs b/crates/cos/src/document/state.rs index 121a59c2e7..286973943e 100644 --- a/crates/cos/src/document/state.rs +++ b/crates/cos/src/document/state.rs @@ -8,6 +8,17 @@ thread_local! { /// retries a parse several ways, and each retry would resolve the same lengths again /// (exponential in the chain of streams whose lengths point at further streams). pub(super) static LENGTHS: std::cell::RefCell>> = std::cell::RefCell::new(HashMap::new()); + /// Unsaved edits read on this thread (`Document::try_get`, and a `/Length` in `Document::load`). + static OVERLAY_READS: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + +/// Unsaved edits read so far on this thread: a load during which this changes depended on one. +pub(super) fn overlay_reads() -> u64 { + OVERLAY_READS.with(std::cell::Cell::get) +} + +pub(super) fn note_overlay_read() { + OVERLAY_READS.with(|c| c.set(c.get().wrapping_add(1))); } /// Marks an object as being loaded on this thread. Loading can re-enter (indirect stream diff --git a/crates/cos/src/document/tests.rs b/crates/cos/src/document/tests.rs index 3cebe0136d..ee7fd28750 100644 --- a/crates/cos/src/document/tests.rs +++ b/crates/cos/src/document/tests.rs @@ -613,6 +613,27 @@ fn an_unreadable_object_is_searched_for_once() { assert_eq!(edited.try_get(3).as_deref(), Ok(&Object::Null)); } +/// Clones share the cache, and an edit is one clone's alone: a member of an object stream that +/// one clone freed or replaced still reads, in the others, from the file's stream. +#[test] +fn an_edited_object_stream_leaves_other_clones_members_alone() { + let file = Arc::new(object_stream_file("3 0 ", "<< /Hidden (yes) >>", 0)); + let doc = Document::open(file.clone()).unwrap(); + let mut freed = doc.clone(); + freed.free(ObjRef::new(4, 0)); + assert_eq!(freed.try_get(3), Err(CosError::MissingObject(4))); + assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("yes"), "the free reached another clone"); + let doc = Document::open(file).unwrap(); + let mut replaced = doc.clone(); + let mut dict = Dict::new(); + dict.set(b"Type".to_vec(), Object::name("ObjStm")); + dict.set(b"N".to_vec(), Object::Int(1)); + dict.set(b"First".to_vec(), Object::Int(4)); + replaced.set(ObjRef::new(4, 0), crate::Stream::from_raw(dict, b"3 0 << /Hidden (no) >>".to_vec())); + assert_eq!(text_of(&replaced, 3, b"Hidden").as_deref(), Some("no")); + assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("yes"), "the replacement reached another clone"); +} + /// A classic table with `/Size` rows and object 0's free entry first, its subsection numbered /// `1 n` instead of `0 n` (#2676): every row is one number off. fn misnumbered_table(start: u32) -> Vec { From 85df8ee66817a726af56a7101bab81a2a791f823 Mon Sep 17 00:00:00 2001 From: KenM76 Date: Mon, 2 Nov 2026 17:08:05 -0500 Subject: [PATCH 029/198] feat(cos): list the objects that could not be read in doc_info MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An object whose load fails reads as null (ISO 32000-2 §7.3.10) and the file opens, but nothing said so: the repair log records only document-level repairs, so a file with an unparseable /Info showed no repair and no reason for its missing title. pdfcraft-cos Document now records each object whose load fails, with its generation and the reason, in a set shared by clones and scans like the object cache: at most 100, keyed by number, first reason kept, cut to 160 characters, built only for an object not yet listed. It is reset where the caches are (authenticate, reconstruct); edited objects drop off a document's list. Document::unreadable_objects() gives neutral lines ("object 4 0 could not be read: ..."), the engine passes them on, and automation doc_info adds an unreadable_objects array, kept apart from repairs so the "has been repaired" notice and the clean-file checks keep their meaning. Tests: a cos unit test for listing, dedup, clones, scans and edits; a cos test for the 100 cap and the no-readable-copy reason; an end-to-end doc_info test on a synthetic broken file. Each fails with the fix reverted. The core.repair-log parity entry cites them and says what is not listed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01SqmHYx8FiWM5PcsoiErgp9 --- crates/automation/src/json.rs | 3 + crates/automation/src/tools.rs | 2 +- crates/automation/tests/it/main.rs | 1 + .../automation/tests/it/unreadable_objects.rs | 64 +++++++++++++++++++ crates/cos/src/document.rs | 5 +- crates/cos/src/document/objects.rs | 48 +++++++++++++- crates/cos/src/document/open.rs | 3 + crates/cos/src/document/reader.rs | 3 + crates/cos/src/document/state.rs | 48 ++++++++++++++ crates/cos/src/document/tests.rs | 58 +++++++++++++++++ crates/engine/src/document.rs | 6 ++ parity/acrobat-features.toml | 4 +- 12 files changed, 238 insertions(+), 7 deletions(-) create mode 100644 crates/automation/tests/it/unreadable_objects.rs diff --git a/crates/automation/src/json.rs b/crates/automation/src/json.rs index 9e93d4c347..c950e72f05 100644 --- a/crates/automation/src/json.rs +++ b/crates/automation/src/json.rs @@ -375,6 +375,9 @@ pub(crate) fn info(d: &Document) -> Value { })).collect::>(), "warnings": i.warnings, "repairs": d.repair_log(), + // Objects that could not be read (each reads as null): nothing was repaired, what they held is + // lost. Last, so it includes what reading the fields above failed on. + "unreadable_objects": d.unreadable_objects(), }) } diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index 7cafcb4a37..06927dee90 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -142,7 +142,7 @@ pub fn tools() -> Vec { .cmd("file.open") .with(schema(json!({ "path": { "type": "string" }, "password": { "type": "string", "description": "User or owner password for encrypted files." } }), &["path"])), t("doc_list", "List open documents", "List the open documents with their ids, page counts and unsaved state.").ro().with(schema(json!({}), &[])), - t("doc_info", "Inspect a document", "Metadata, page sizes and labels, bookmarks, annotations, form fields, links, layers, page transitions and display times, the destination it opens at, attachments, fonts, security and repair notes.") + t("doc_info", "Inspect a document", "Metadata, page sizes and labels, bookmarks, annotations, form fields, links, layers, page transitions and display times, the destination it opens at, attachments, fonts, security, repair notes and the objects that could not be read.") .ro() .cmd("file.properties") .with(schema(json!({ "doc": doc() }), &["doc"])), diff --git a/crates/automation/tests/it/main.rs b/crates/automation/tests/it/main.rs index d36dbe93a9..14e485c9f3 100644 --- a/crates/automation/tests/it/main.rs +++ b/crates/automation/tests/it/main.rs @@ -12,3 +12,4 @@ mod optimize_content_boundaries; mod redact_after_stray_q_form; mod redact_text_after_q; mod render_warnings; +mod unreadable_objects; diff --git a/crates/automation/tests/it/unreadable_objects.rs b/crates/automation/tests/it/unreadable_objects.rs new file mode 100644 index 0000000000..f19a19078a --- /dev/null +++ b/crates/automation/tests/it/unreadable_objects.rs @@ -0,0 +1,64 @@ +//! `doc_info` lists the objects of a file that could not be read: each reads as null, so the file +//! opens, but what it held is lost and nothing was repaired. + +use pdfcraft_automation::{Automation, Content}; +use serde_json::{Value, json}; + +/// A one-page PDF whose document information dictionary (object 4) is `info`. +fn with_info(info: &str) -> Vec { + let bodies = [ + "<< /Type /Catalog /Pages 2 0 R >>", + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] >>", + info, + ]; + let mut out = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (i, body) in bodies.iter().enumerate() { + offsets.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n{body}\nendobj\n", i + 1).as_bytes()); + } + let xref = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", bodies.len() + 1).as_bytes()); + for o in offsets { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R /Info 4 0 R >>\nstartxref\n{xref}\n%%EOF\n", bodies.len() + 1).as_bytes()); + out +} + +fn info_of(a: &mut Automation, path: &str) -> Value { + let doc = match a.call("doc_open", &json!({ "path": path })).unwrap().remove(0) { + Content::Json(v) => v["doc"].as_u64().unwrap(), + other => panic!("expected JSON: {other:?}"), + }; + match a.call("doc_info", &json!({ "doc": doc })).unwrap().remove(0) { + Content::Json(v) => v, + other => panic!("expected JSON: {other:?}"), + } +} + +/// An unparseable `/Info` is listed with its number and why, the rest of the file reads, and +/// nothing is reported as repaired. +#[test] +fn doc_info_lists_an_object_that_could_not_be_read() { + let dir = std::env::temp_dir().join(format!("pdfcraft-unreadable-objects-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("broken.pdf"), with_info("<< /Title (Report) /Author ) >>")).unwrap(); + std::fs::write(dir.join("sound.pdf"), with_info("<< /Title (Report) >>")).unwrap(); + let mut a = Automation::new().with_root(&dir).unwrap(); + + let info = info_of(&mut a, "broken.pdf"); + let lines = info["unreadable_objects"].as_array().unwrap(); + assert_eq!(lines.len(), 1, "{info}"); + let line = lines[0].as_str().unwrap(); + assert!(line.starts_with("object 4 0 could not be read: syntax error"), "{line}"); + assert_eq!(info["repairs"], json!([]), "nothing was repaired"); + assert_eq!(info["pages"].as_array().unwrap().len(), 1, "the rest of the file reads"); + + let info = info_of(&mut a, "sound.pdf"); + assert_eq!(info["unreadable_objects"], json!([]), "{info}"); + assert_eq!(info["title"], "Report"); + let _ = std::fs::remove_dir_all(dir); +} diff --git a/crates/cos/src/document.rs b/crates/cos/src/document.rs index 082d1910d1..81eb1fd4d9 100644 --- a/crates/cos/src/document.rs +++ b/crates/cos/src/document.rs @@ -24,7 +24,7 @@ pub(crate) use reader::find; #[cfg(test)] use reader::rfind; pub(crate) use state::ObjectReader; -use state::{ObjStm, Slot}; +use state::{ObjStm, Slot, Unreadable}; pub use state::{ReadState, SourceIdentity}; #[cfg(test)] use state::{SECTION_READS, objstm_member, owned_bytes}; @@ -49,6 +49,9 @@ pub struct Document { repair_log: Arc>, cache: Arc>>>, objstms: Arc>>>, + /// See [`Self::unreadable_objects`]. Shared by clones, like the caches: it describes the + /// source, and is reset wherever they are. + unreadable: Arc>, /// What `/JBIG2Globals` streams of the file decode to, by object number: the images of a scan /// share them, and each would otherwise decode them again (see `Document::jbig2_globals`). jbig2_globals: Arc>>>>, diff --git a/crates/cos/src/document/objects.rs b/crates/cos/src/document/objects.rs index 2ac18cff03..fdda5f5cf5 100644 --- a/crates/cos/src/document/objects.rs +++ b/crates/cos/src/document/objects.rs @@ -86,6 +86,38 @@ impl Document { &self.repair_log } + /// Objects of the file that could not be read, lowest number first, as lines such as + /// "object 11 0 could not be read: syntax error at byte 512: ...". [`Self::get`] reads each + /// as null (ISO 32000-2 §7.3.10), so a document with one opens, but what it held is lost: + /// unlike [`Self::repair_log`], nothing was repaired. Lists the objects read so far, by this + /// document, its clones or a scan of it, that have not since been edited; past 100, a last + /// line says there were more. Opening again, authenticating or rebuilding the index starts + /// the list afresh, as it does the object cache. + pub fn unreadable_objects(&self) -> Vec { + let Ok(unreadable) = self.unreadable.lock() else { return Vec::new() }; + unreadable.lines(|num| self.overlay.contains_key(&num)) + } + + /// Record `num` as unreadable. `reason` is built only for an object not yet listed, so a file + /// whose broken object is asked for again and again pays for it once. + fn note_unreadable(&self, num: u32, reason: impl FnOnce() -> String) { + // A poisoned lock only means the object goes unlisted; the read itself still fails. + let Ok(mut unreadable) = self.unreadable.lock() else { return }; + if unreadable.wants(num) { + unreadable.note(num, self.generation(num), &reason()); + } + } + + /// Why an object listed at `offset` could not be read: what parsing it there, without resolving + /// an indirect `/Length`, found. A best account, not the only one: the reader also tried + /// elsewhere. + fn why_unreadable(&self, offset: usize) -> String { + match parse_indirect_shared(&self.data, offset, &|_| None, &self.endstreams) { + Err(e) => e.to_string(), + Ok(_) => "no readable copy was found at its cross-reference offset or elsewhere in the file".into(), + } + } + /// Add a line to [`Self::repair_log`]: a leniency applied while reading that the user should /// know about (fidelity: never silent). pub fn note_repair(&mut self, line: String) { @@ -395,8 +427,15 @@ impl Document { // cycles such as `6 0 obj << /Length 6 0 R >>`. Bound the nesting on every path. let _guard = LoadGuard::enter(num).ok_or_else(|| CosError::Syntax { offset: 0, detail: format!("reference cycle while loading object {num}") })?; - let obj = self.load(num, 0)?; - let obj = Arc::new(obj); + let obj = match self.load(num, 0) { + Ok(obj) => Arc::new(obj), + Err(e) => { + if !matches!(e, CosError::Poisoned) { + self.note_unreadable(num, || e.to_string()); + } + return Err(e); + } + }; // The editing cache is bounded like the scan reader's: re-parsing a file-backed view // is cheap, and returned objects outlive eviction. self.cache.lock().map_err(|_| CosError::Poisoned)?.insert(num, obj.clone()); @@ -554,7 +593,10 @@ impl Document { .or_else(|| if self.header_offset == 0 { None } else { at(off.saturating_add(self.header_offset)) }) .or_else(|| self.located.get(&num).and_then(|(o, _)| usize::try_from(*o).ok()).and_then(at)) .or_else(|| self.scan_for(num).map(|(id, o)| self.decrypted(id, o))) - .ok_or(CosError::MissingObject(num)) + .ok_or_else(|| { + self.note_unreadable(num, || self.why_unreadable(off)); + CosError::MissingObject(num) + }) } Some(XrefEntry::InStream { stream, index }) => { let stm = self.objstm(stream)?; diff --git a/crates/cos/src/document/open.rs b/crates/cos/src/document/open.rs index 395424b771..21c0dd83ba 100644 --- a/crates/cos/src/document/open.rs +++ b/crates/cos/src/document/open.rs @@ -19,6 +19,7 @@ impl Document { repair_log: Arc::new(Vec::new()), cache: Arc::default(), objstms: Arc::default(), + unreadable: Arc::default(), jbig2_globals: Arc::default(), stream_limit: crate::object::MAX_DECODED, overlay: BTreeMap::new(), @@ -103,6 +104,7 @@ impl Document { repair_log: Arc::new(Vec::new()), cache: Arc::default(), objstms: Arc::default(), + unreadable: Arc::default(), jbig2_globals: Arc::default(), stream_limit: stream_limit.min(crate::object::MAX_DECODED), overlay: BTreeMap::new(), @@ -200,6 +202,7 @@ impl Document { // Objects read while locating the catalog were read before decryption was possible. self.cache = Arc::default(); self.objstms = Arc::default(); + self.unreadable = Arc::default(); Ok(()) } diff --git a/crates/cos/src/document/reader.rs b/crates/cos/src/document/reader.rs index d21bf9403f..1f944fd870 100644 --- a/crates/cos/src/document/reader.rs +++ b/crates/cos/src/document/reader.rs @@ -1,5 +1,6 @@ #[cfg(test)] use super::state::SECTION_READS; +use super::state::Unreadable; use super::xref::{ MAX_COMPACT_XREF_BYTES, MAX_COMPACT_XREF_ROWS, Revision, XrefEntry, XrefIndex, XrefRange, XrefRevisionSection, XrefSection, XrefSectionData, }; @@ -633,6 +634,7 @@ impl Document { } self.entries = Arc::new(XrefIndex::Materialized(Arc::new(entries.clone()))); self.cache.lock().map_err(|_| CosError::Poisoned)?.clear(); + *self.unreadable.lock().map_err(|_| CosError::Poisoned)? = Unreadable::default(); // Objects inside object streams. for num in objstms { if let Ok(o) = self.try_get(num) @@ -650,6 +652,7 @@ impl Document { let object_count = entries.len(); self.entries = Arc::new(XrefIndex::Materialized(Arc::new(entries))); self.cache.lock().map_err(|_| CosError::Poisoned)?.clear(); + *self.unreadable.lock().map_err(|_| CosError::Poisoned)? = Unreadable::default(); // Trailer: the last `trailer` dictionary or cross-reference stream dictionary with a // usable /Root (last in the file, whichever kind), else the catalog itself. One whose // /Root is a catalog (as `root_is_catalog` judges it) beats one whose /Root is another diff --git a/crates/cos/src/document/state.rs b/crates/cos/src/document/state.rs index b4323f982e..8dbba4a93a 100644 --- a/crates/cos/src/document/state.rs +++ b/crates/cos/src/document/state.rs @@ -61,6 +61,54 @@ pub(super) fn objstm_member(num: i64, offset: i64, first: Option, len: us (start < len).then_some((num, start)) } +/// Objects the reader could not read, each with the reason (see `Document::unreadable_objects`). +/// Bounded: past [`Unreadable::MAX`] objects only the fact that there were more is kept. +#[derive(Debug, Default)] +pub(super) struct Unreadable { + pub(super) objects: BTreeMap, + pub(super) more: bool, +} + +impl Unreadable { + pub(super) const MAX: usize = 100; + /// Characters of a reason kept: a parser's message can quote the bytes it stopped at. + const REASON: usize = 160; + + /// Whether `num` is to be recorded: not listed yet and there is room. A full list notes that + /// there were more, so a caller asks this before building the reason. + pub(super) fn wants(&mut self, num: u32) -> bool { + if self.objects.contains_key(&num) { + return false; + } + if self.objects.len() >= Self::MAX { + self.more = true; + return false; + } + true + } + + /// Record `num` once; the first reason given is kept. + pub(super) fn note(&mut self, num: u32, generation: u16, reason: &str) { + if self.wants(num) { + self.objects.insert(num, (generation, reason.chars().take(Self::REASON).collect())); + } + } + + /// One line per recorded object not `hidden`, then one saying there were more, if there were. + pub(super) fn lines(&self, hidden: impl Fn(u32) -> bool) -> Vec { + let mut lines: Vec = self + .objects + .iter() + .filter(|(num, _)| !hidden(**num)) + .map(|(num, (generation, reason))| format!("object {num} {generation} could not be read: {reason}")) + .collect(); + if self.more { + lines.push(format!("more objects could not be read; only the first {} are listed", Self::MAX)); + } + lines + } +} + /// Where an overlay slot came from. #[derive(Clone, Debug, PartialEq)] pub(super) enum Slot { diff --git a/crates/cos/src/document/tests.rs b/crates/cos/src/document/tests.rs index cca4b87326..c400e029d2 100644 --- a/crates/cos/src/document/tests.rs +++ b/crates/cos/src/document/tests.rs @@ -1701,3 +1701,61 @@ fn a_stream_whose_length_is_itself_and_never_ends_loads_quickly() { }); rx.recv_timeout(std::time::Duration::from_secs(20)).expect("loading must not retry exponentially"); } + +/// A file whose object 3 (its `/Info`) can't be parsed: the rest reads. +fn with_unreadable_info() -> Vec { + build( + &["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", "<< /Title (Report) /Author ) >>"], + "/Root 1 0 R /Info 3 0 R", + ) +} + +/// An object that can't be parsed reads as null and is listed once, by number and generation, +/// with why. Clones and scans share the list, an edit takes the object off it (in that document +/// only), and a sound file lists nothing. +#[test] +fn unreadable_objects_are_listed_with_the_reason() { + let doc = Document::open(Arc::new(with_unreadable_info())).unwrap(); + assert!(doc.unreadable_objects().is_empty(), "nothing has read object 3 yet"); + assert_eq!(*doc.get(ObjRef::new(3, 0)), Object::Null); + assert!(doc.try_get(3).is_err()); + let lines = doc.unreadable_objects(); + assert_eq!(lines.len(), 1, "read twice, listed once: {lines:?}"); + assert!(lines[0].starts_with("object 3 0 could not be read: syntax error"), "{lines:?}"); + let scanned = doc.clone(); + assert_eq!(scanned.scan_objects_checked().filter(|(_, o)| o.is_err()).count(), 1); + assert_eq!(doc.unreadable_objects(), lines, "the scan found the same object"); + let mut edited = doc.clone(); + edited.set(ObjRef::new(3, 0), Object::Dict(Dict::new())); + assert!(edited.unreadable_objects().is_empty(), "the edit replaced it"); + assert_eq!(doc.unreadable_objects(), lines); + let sound = + build(&["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", "<< /Title (Report) >>"], "/Root 1 0 R /Info 3 0 R"); + let sound = Document::open(Arc::new(sound)).unwrap(); + assert_eq!(sound.scan_objects_checked().filter(|(_, o)| o.is_err()).count(), 0); + assert!(sound.unreadable_objects().is_empty()); +} + +/// The list keeps at most 100 objects and then says there were more; an object whose +/// cross-reference offset holds no object is listed with that as its reason. +#[test] +fn the_unreadable_object_list_is_bounded() { + let mut bodies = vec!["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>"]; + bodies.extend(std::iter::repeat_n("<< /A ) >>", 120)); + let doc = Document::open(Arc::new(build(&bodies, "/Root 1 0 R"))).unwrap(); + assert_eq!(doc.scan_objects_checked().filter(|(_, o)| o.is_err()).count(), 120); + let lines = doc.unreadable_objects(); + assert_eq!(lines.len(), 101, "{:?}", lines.last()); + assert!(lines[0].starts_with("object 3 0 could not be read"), "{}", lines[0]); + assert_eq!(lines[100], "more objects could not be read; only the first 100 are listed"); + + let base = build(&["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", "(three)"], "/Root 1 0 R"); + let text = String::from_utf8_lossy(&base).into_owned(); + let gone = text.replacen("3 0 obj\n(three)", "9 0 obj\n(three)", 1); + let doc = Document::open(Arc::new(gone.into_bytes())).unwrap(); + assert!(doc.try_get(3).is_err()); + assert_eq!( + doc.unreadable_objects(), + ["object 3 0 could not be read: no readable copy was found at its cross-reference offset or elsewhere in the file"] + ); +} diff --git a/crates/engine/src/document.rs b/crates/engine/src/document.rs index fa98a18417..b3e93cab0b 100644 --- a/crates/engine/src/document.rs +++ b/crates/engine/src/document.rs @@ -631,6 +631,12 @@ impl Document { pub fn repair_log(&self) -> Vec { self.editor.as_ref().map(|e| e.cos.repair_log().to_vec()).unwrap_or_default() } + + /// Objects of the file that could not be read and read as null, one line each (see + /// [`pdfcraft_cos::Document::unreadable_objects`]): only those read so far. + pub fn unreadable_objects(&self) -> Vec { + self.editor.as_ref().map(|e| e.cos.unreadable_objects()).unwrap_or_default() + } } /// A title that names no document: what a web browser writes when it saves a blank page or a diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 290a410e14..451e84f4aa 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -70,8 +70,8 @@ milestone = "M1" status = "shipped" commands = [] tools = ["doc_info"] -evidence = ["crates/ui-egui/tests/it/viewer.rs::damaged_files_say_they_were_repaired"] -notes = "a notice says the file was repaired; Document Properties ▸ Advanced lists the repairs; doc_info reports them" +evidence = ["crates/ui-egui/tests/it/viewer.rs::damaged_files_say_they_were_repaired", "crates/cos/src/document/tests.rs::unreadable_objects_are_listed_with_the_reason", "crates/automation/tests/it/unreadable_objects.rs::doc_info_lists_an_object_that_could_not_be_read"] +notes = "a notice says the file was repaired; Document Properties ▸ Advanced lists the repairs; doc_info reports them, and lists apart (unreadable_objects) each object that could not be read and reads as null, with its number and why: only objects read so far, including any object a scan, save or audit read, at most 100; a stream whose own data fails to decode is not listed, but the objects of an object stream that cannot be decoded are" [[feature]] id = "core.xref-streams" From 8cc4d378476ddfa13c8a7540f05e8edfef6728cf Mon Sep 17 00:00:00 2001 From: KenM76 Date: Mon, 2 Nov 2026 17:18:30 -0500 Subject: [PATCH 030/198] fix(cos): an object stream framed by one clone's edited /Length is not shared An unedited object stream whose indirect /Length is edited in one clone was parsed with that clone's length and put in the object-stream map the clones share, so the other clones read its members within the edited length. A container load that read the clone's edits is now kept to that clone, like an edited stream. Test: an_edited_object_stream_length_leaves_other_clones_members_alone fails without the fix (the other clone reads None) and passes with it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01SqmHYx8FiWM5PcsoiErgp9 --- crates/cos/src/document/objects.rs | 5 ++++- crates/cos/src/document/tests.rs | 32 ++++++++++++++++++++++++++---- 2 files changed, 32 insertions(+), 5 deletions(-) diff --git a/crates/cos/src/document/objects.rs b/crates/cos/src/document/objects.rs index 55a01e117c..b8f9a64870 100644 --- a/crates/cos/src/document/objects.rs +++ b/crates/cos/src/document/objects.rs @@ -589,6 +589,7 @@ impl Document { // header, as a plain object with a wrong offset is. An edited or freed stream, in the // overlay or in a saved revision's table, is what that says it is: the file's copy // must not come back. + let edits_read = overlay_reads(); let loaded = match self.try_get(num) { Ok(o) if matches!(*o, Object::Stream(_)) => o, _ if self.overlay.contains_key(&num) || matches!(self.xref_entry(num), Some(XrefEntry::Free { .. })) => { @@ -596,6 +597,8 @@ impl Document { } _ => Arc::new(self.scan_for(num).map(|(id, o)| self.decrypted(id, o)).ok_or(CosError::MissingObject(num))?), }; + // A stream whose load read this document's edits (its `/Length`) is as much its own as an edited one. + let own = edited || overlay_reads() != edits_read; let Object::Stream(s) = &*loaded else { return Err(CosError::MissingObject(num)) }; let n = s.dict.int(b"N").unwrap_or(0).clamp(0, 1_000_000) as usize; let first = usize::try_from(s.dict.int(b"First").unwrap_or(0).max(0)).ok(); @@ -607,7 +610,7 @@ impl Document { index.push(objstm_member(onum, off, first, data.len())); } let stm = Arc::new(ObjStm { index, data: Arc::new(data) }); - if edited { + if own { return Ok(stm); } // Same bound as the object cache: decoded object streams can be megabytes each. diff --git a/crates/cos/src/document/tests.rs b/crates/cos/src/document/tests.rs index ee7fd28750..15c9d8cd61 100644 --- a/crates/cos/src/document/tests.rs +++ b/crates/cos/src/document/tests.rs @@ -634,6 +634,20 @@ fn an_edited_object_stream_leaves_other_clones_members_alone() { assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("yes"), "the replacement reached another clone"); } +/// An object stream framed by one clone's edit of its indirect `/Length` is that clone's alone: +/// the others still read its members within the file's length. +#[test] +fn an_edited_object_stream_length_leaves_other_clones_members_alone() { + let file = Arc::new(object_stream_file_with("3 0 ", "<< /Hidden (cut endstream here) >>", 0, true)); + let doc = Document::open(file).unwrap(); + let mut edited = doc.clone(); + // Ends the stream at the `endstream` inside the member's string. + let cut = "3 0 << /Hidden (cut".len(); + edited.set(ObjRef::new(6, 0), Object::Int(cut as i64)); + assert_ne!(text_of(&edited, 3, b"Hidden").as_deref(), Some("cut endstream here"), "the edited length was not used"); + assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("cut endstream here"), "the edited length reached another clone"); +} + /// A classic table with `/Size` rows and object 0's free entry first, its subsection numbered /// `1 n` instead of `0 n` (#2676): every row is one number off. fn misnumbered_table(start: u32) -> Vec { @@ -1367,8 +1381,14 @@ fn hybrid_reference_files_read_the_xref_stream_too() { /// stream puts it at position `member` of object stream 4, which holds `header` (the number /// and offset pairs) and then `members`. fn object_stream_file(header: &str, members: &str, member: u8) -> Vec { + object_stream_file_with(header, members, member, false) +} + +/// [`object_stream_file`], with the stream's `/Length` in object 6 when `indirect_length`. +fn object_stream_file_with(header: &str, members: &str, member: u8, indirect_length: bool) -> Vec { let mut out = b"%PDF-1.5\n".to_vec(); - let mut offs = [0usize; 6]; + let size = if indirect_length { 7 } else { 6 }; + let mut offs = vec![0usize; size]; let mut obj = |out: &mut Vec, n: usize, body: &[u8]| { offs[n] = out.len(); out.extend_from_slice(format!("{n} 0 obj\n").as_bytes()); @@ -1379,7 +1399,8 @@ fn object_stream_file(header: &str, members: &str, member: u8) -> Vec { obj(&mut out, 2, b"<< /Type /Pages /Kids [] /Count 0 /Extra 3 0 R >>"); let count = header.split_whitespace().count() / 2; let inner = format!("{header}{members}"); - let mut stm = format!("<< /Type /ObjStm /N {count} /First {} /Length {} >>\nstream\n", header.len(), inner.len()).into_bytes(); + let length = if indirect_length { "6 0 R".to_string() } else { inner.len().to_string() }; + let mut stm = format!("<< /Type /ObjStm /N {count} /First {} /Length {length} >>\nstream\n", header.len()).into_bytes(); stm.extend_from_slice(inner.as_bytes()); stm.extend_from_slice(b"\nendstream"); obj(&mut out, 4, &stm); @@ -1388,13 +1409,16 @@ fn object_stream_file(header: &str, members: &str, member: u8) -> Vec { xs.extend_from_slice(&data); xs.extend_from_slice(b"\nendstream"); obj(&mut out, 5, &xs); + if indirect_length { + obj(&mut out, 6, inner.len().to_string().as_bytes()); + } let table = out.len(); - out.extend_from_slice(b"xref\n0 6\n0000000000 65535 f \n"); + out.extend_from_slice(format!("xref\n0 {size}\n0000000000 65535 f \n").as_bytes()); for (n, off) in offs.iter().enumerate().skip(1) { let line = if n == 3 { "0000000000 65535 f \n".to_string() } else { format!("{off:010} 00000 n \n") }; out.extend_from_slice(line.as_bytes()); } - out.extend_from_slice(format!("trailer << /Size 6 /Root 1 0 R /XRefStm {} >>\nstartxref\n{table}\n%%EOF\n", offs[5]).as_bytes()); + out.extend_from_slice(format!("trailer << /Size {size} /Root 1 0 R /XRefStm {} >>\nstartxref\n{table}\n%%EOF\n", offs[5]).as_bytes()); out } From 96d3d1e073b61a00ab4bea312fdb8cdf5e974013 Mon Sep 17 00:00:00 2001 From: KenM76 Date: Mon, 2 Nov 2026 17:56:44 -0500 Subject: [PATCH 031/198] fix(cos): a reused /Length keeps the edits it was read through The indirect /Length memo kept only the value. A length first resolved through one clone's edited object stream and reused later in the same load did not count as an edit read, so an object stream framed by it could still be parsed with the edited length and put in the object-stream map and cache the clones share. The memo now keeps whether resolving the length read edits, and a reuse counts it again. Test: a_reused_edited_length_leaves_other_clones_members_alone fails without the fix (the other clone reads None) and passes with it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01SqmHYx8FiWM5PcsoiErgp9 --- crates/cos/src/document/objects.rs | 10 ++++-- crates/cos/src/document/state.rs | 5 +-- crates/cos/src/document/tests.rs | 58 ++++++++++++++++++++++++++++++ 3 files changed, 69 insertions(+), 4 deletions(-) diff --git a/crates/cos/src/document/objects.rs b/crates/cos/src/document/objects.rs index b8f9a64870..976e5e360c 100644 --- a/crates/cos/src/document/objects.rs +++ b/crates/cos/src/document/objects.rs @@ -549,11 +549,17 @@ impl Document { if self.overlay.contains_key(&r.num) { note_overlay_read(); } - if let Some(known) = LENGTHS.with(|m| m.borrow().get(&r.num).copied()) { + // A length resolved through this document's edits counts as an edit read + // on every reuse. + if let Some((known, edits)) = LENGTHS.with(|m| m.borrow().get(&r.num).copied()) { + if edits { + note_overlay_read(); + } return known; } + let edits_read = overlay_reads(); let length = self.try_get(r.num).ok().and_then(|o| o.as_int()); - LENGTHS.with(|m| m.borrow_mut().insert(r.num, length)); + LENGTHS.with(|m| m.borrow_mut().insert(r.num, (length, overlay_reads() != edits_read))); length }; let off = offset as usize; diff --git a/crates/cos/src/document/state.rs b/crates/cos/src/document/state.rs index 286973943e..29e2ea4879 100644 --- a/crates/cos/src/document/state.rs +++ b/crates/cos/src/document/state.rs @@ -6,8 +6,9 @@ thread_local! { static LOADING: std::cell::RefCell> = const { std::cell::RefCell::new(Vec::new()) }; /// Indirect `/Length` values resolved during the current outermost load: a damaged file /// retries a parse several ways, and each retry would resolve the same lengths again - /// (exponential in the chain of streams whose lengths point at further streams). - pub(super) static LENGTHS: std::cell::RefCell>> = std::cell::RefCell::new(HashMap::new()); + /// (exponential in the chain of streams whose lengths point at further streams). Each is + /// kept with whether resolving it read unsaved edits. + pub(super) static LENGTHS: std::cell::RefCell, bool)>> = std::cell::RefCell::new(HashMap::new()); /// Unsaved edits read on this thread (`Document::try_get`, and a `/Length` in `Document::load`). static OVERLAY_READS: std::cell::Cell = const { std::cell::Cell::new(0) }; } diff --git a/crates/cos/src/document/tests.rs b/crates/cos/src/document/tests.rs index 15c9d8cd61..84dc17cc7f 100644 --- a/crates/cos/src/document/tests.rs +++ b/crates/cos/src/document/tests.rs @@ -648,6 +648,64 @@ fn an_edited_object_stream_length_leaves_other_clones_members_alone() { assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("cut endstream here"), "the edited length reached another clone"); } +/// A `/Length` read through one clone's edits and reused within the same load still keeps that +/// load to the clone. Here the file's offsets are header-relative behind junk, so object 10's +/// offset first lands on object 12, whose length (object 8, in the edited object stream 9) is +/// read; object 10's own length then loads object stream 4, framed by object 8 too. +#[test] +fn a_reused_edited_length_leaves_other_clones_members_alone() { + let mut out = b"%PDF-1.5\n".to_vec(); + let mut offs = [0usize; 13]; + let mut obj = |out: &mut Vec, n: usize, body: &[u8]| { + offs[n] = out.len(); + out.extend_from_slice(format!("{n} 0 obj\n").as_bytes()); + out.extend_from_slice(body); + out.extend_from_slice(b"\nendobj\n"); + }; + let stream = |dict: &str, data: &str| format!("<< {dict} >>\nstream\n{data}\nendstream").into_bytes(); + obj(&mut out, 1, b"<< /Type /Catalog /Pages 2 0 R >>"); + obj(&mut out, 2, b"<< /Type /Pages /Kids [] /Count 0 /Extra 3 0 R >>"); + let inner = "11 0 3 2 5 << /Hidden (cut endstream here) >>"; + obj(&mut out, 4, &stream("/Type /ObjStm /N 2 /First 9 /Length 8 0 R", inner)); + let lengths = format!("8 0 {}", inner.len()); + obj(&mut out, 9, &stream(&format!("/Type /ObjStm /N 1 /First 4 /Length {}", lengths.len()), &lengths)); + obj(&mut out, 12, &stream("/Length 8 0 R", &"x".repeat(40))); + obj(&mut out, 10, &stream("/Length 11 0 R", "PDATA")); + // A cross-reference stream: a hybrid file's /XRefStm is not looked for behind junk. + let compressed = [(3, 4, 1), (8, 9, 0), (11, 4, 0)]; + let mut data = Vec::new(); + for (n, off) in offs.iter().enumerate() { + let row = match compressed.iter().find(|c| c.0 == n) { + Some(&(_, stream, index)) => [2, 0, stream, index], + None if *off == 0 => [0, 0, 0, 0], + None => { + let [.., hi, lo] = u32::try_from(*off).unwrap().to_be_bytes(); + [1, hi, lo, 0] + } + }; + data.extend_from_slice(&row); + } + let table = out.len(); + let dict = format!("<< /Type /XRef /Size 14 /W [1 2 1] /Index [0 13] /Root 1 0 R /Length {} >>", data.len()); + out.extend_from_slice(format!("13 0 obj\n{dict}\nstream\n").as_bytes()); + out.extend_from_slice(&data); + out.extend_from_slice(format!("\nendstream\nendobj\nstartxref\n{table}\n%%EOF\n").as_bytes()); + let mut file = vec![b'J'; offs[10] - offs[12] - 1]; + file.push(b'\n'); + file.extend_from_slice(&out); + let doc = Document::open(Arc::new(file)).unwrap(); + let mut edited = doc.clone(); + let cut = "11 0 3 2 5 << /Hidden (cut".len(); + let mut dict = Dict::new(); + dict.set(b"Type".to_vec(), Object::name("ObjStm")); + dict.set(b"N".to_vec(), Object::Int(1)); + dict.set(b"First".to_vec(), Object::Int(4)); + edited.set(ObjRef::new(9, 0), crate::Stream::from_raw(dict, format!("8 0 {cut}").into_bytes())); + assert!(matches!(&*edited.get(ObjRef::new(10, 0)), Object::Stream(s) if &s.raw[..] == b"PDATA")); + assert_ne!(text_of(&edited, 3, b"Hidden").as_deref(), Some("cut endstream here"), "the edited length was not used"); + assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("cut endstream here"), "the edited length reached another clone"); +} + /// A classic table with `/Size` rows and object 0's free entry first, its subsection numbered /// `1 n` instead of `0 n` (#2676): every row is one number off. fn misnumbered_table(start: u32) -> Vec { From 649e97930f0f516ac684393f9b16e5c3c00120a6 Mon Sep 17 00:00:00 2001 From: KenM76 Date: Mon, 2 Nov 2026 18:03:00 -0500 Subject: [PATCH 032/198] fix(cos): a rebuilt index keeps the read failures its scan found Rebuilding the cross-reference index from object headers reset the unreadable-object list a second time, after adding the object-stream members, so an object the header scan failed to read dropped off unreadable_objects() unless something read it again. Those failures still stand against the full index: the added members leave every scanned object's entry as it was, and a /Length found only then changes how much of a stream is read, never whether it reads. The list is now reset once, before the scan; the object cache is still cleared after it. Test: a_rebuilt_file_lists_the_objects_its_scan_could_not_read fails without the fix (the list is empty) and passes with it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01SqmHYx8FiWM5PcsoiErgp9 --- crates/cos/src/document/reader.rs | 3 ++- crates/cos/src/document/tests.rs | 11 +++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/crates/cos/src/document/reader.rs b/crates/cos/src/document/reader.rs index 1f944fd870..d556c1423b 100644 --- a/crates/cos/src/document/reader.rs +++ b/crates/cos/src/document/reader.rs @@ -651,8 +651,9 @@ impl Document { } let object_count = entries.len(); self.entries = Arc::new(XrefIndex::Materialized(Arc::new(entries))); + // Failures from the scan stand: the members added change no scanned entry, and a + // `/Length` found only now cannot make a stream unreadable. self.cache.lock().map_err(|_| CosError::Poisoned)?.clear(); - *self.unreadable.lock().map_err(|_| CosError::Poisoned)? = Unreadable::default(); // Trailer: the last `trailer` dictionary or cross-reference stream dictionary with a // usable /Root (last in the file, whichever kind), else the catalog itself. One whose // /Root is a catalog (as `root_is_catalog` judges it) beats one whose /Root is another diff --git a/crates/cos/src/document/tests.rs b/crates/cos/src/document/tests.rs index c400e029d2..b587471c62 100644 --- a/crates/cos/src/document/tests.rs +++ b/crates/cos/src/document/tests.rs @@ -1736,6 +1736,17 @@ fn unreadable_objects_are_listed_with_the_reason() { assert!(sound.unreadable_objects().is_empty()); } +/// A file rebuilt from its object headers still lists an object the scan could not read. +#[test] +fn a_rebuilt_file_lists_the_objects_its_scan_could_not_read() { + let text = "%PDF-1.7\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n2 0 obj\n<< /Type /Pages /Kids [] /Count 0 >>\nendobj\n3 0 obj\n<< /A ) >>\nendobj\n%%EOF\n"; + let doc = Document::open(Arc::new(text.as_bytes().to_vec())).unwrap(); + assert!(doc.repair_log().iter().any(|l| l.starts_with("rebuilt cross-reference index")), "{:?}", doc.repair_log()); + let lines = doc.unreadable_objects(); + assert_eq!(lines.len(), 1, "{lines:?}"); + assert!(lines[0].starts_with("object 3 0 could not be read: syntax error"), "{lines:?}"); +} + /// The list keeps at most 100 objects and then says there were more; an object whose /// cross-reference offset holds no object is listed with that as its reason. #[test] From 10172d7a94d93f9c5030478c81bf6fdf6760e54b Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Mon, 2 Nov 2026 20:36:50 +0100 Subject: [PATCH 033/198] M2.6: a Flate stage pays for what it reads The image allowance charged a Flate stage the bytes it made, and its input only as what the stage before it made. Inflating does work for every block it reads, whether or not the block makes anything: a stream of empty fixed-Huffman blocks, ten bits each, took 1,383 ns a byte to make nothing (miniz_oxide 0.9.1, release), one of a literal each 768 ns a byte, empty dynamic blocks 106, against the 3 ns a step budget.rs states. A 250-byte stream of four Flates over empty blocks took 47 s and was charged about 34 million steps, so a review of #5698 worked out about 7 hours before a 240 KB file of them was refused, through clearing, Optimize, export, compare and redaction's images and dictionaries alike. Where decoding is paid for (decode_tolerant_within, decode_within), a Flate stage now pays FLATE_READ_WORK, 128 steps, for each byte it reads, before it runs: a stage the allowance can't pay for doesn't start. Per byte read, the worst of the shapes measured comes to about 11 ns a step at that price, inside what a JBIG2 step takes. The raw retry of a zlib header that only looked valid reads the bytes again, so it pays again, makes no more than is left once it has, and is refused when it can't be paid for. Byte counts for a byte allowance (decode_counted, decode_tolerant_counted, as a page's content uses) are unchanged. With a tiny file's allowance (2^34 steps), the 250-byte stream is decoded three times in 141 s, and the fourth stops before its last stage reads the empty blocks (8.2 ns a step, release). Data an encoder makes took 0.4 to 3.2 ns a byte it made, reading included, and each byte of a file adds 64 times the price to an operation's allowance. Tests: a_flate_stage_pays_for_what_it_reads (empty blocks: 129 steps a byte, refused a step short with nothing inflated) and a_second_reading_is_paid_for; the exact costs in filters, cos and optimize tests add what Flate reads. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/cos/src/budget.rs | 37 ++++--- crates/filters/README.md | 2 +- crates/filters/src/flate.rs | 51 +++++++--- crates/filters/src/lib.rs | 159 ++++++++++++++++++++++++------- crates/optimize/src/fragments.rs | 4 +- crates/optimize/src/images.rs | 4 +- 6 files changed, 191 insertions(+), 66 deletions(-) diff --git a/crates/cos/src/budget.rs b/crates/cos/src/budget.rs index 376b108be3..d3e4a71974 100644 --- a/crates/cos/src/budget.rs +++ b/crates/cos/src/budget.rs @@ -11,12 +11,14 @@ use crate::{CosError, Document, ObjRef, Object, Stream}; /// 16 ns, so this is 2 to 5 minutes, and about as long again for what an operation does with the /// pixels. A page scanned at 300 dpi takes about 17 million, 8.4 for its pixels and that again to /// decode them, so this alone reads about 1,000 of them. A byte into or out of a filter is a step -/// too, and takes less: 3 ns at most, or up to 19 ns through a predictor on samples of a bit. +/// too, and takes less: 3 ns at most, or up to 19 ns through a predictor on samples of a bit. A +/// byte Flate reads is [`pdfcraft_filters::FLATE_READ_WORK`] steps besides, since inflating blocks +/// that make nothing takes up to 1.4 µs a byte: about 11 ns a step. const IMAGE_WORK: u64 = 1 << 34; /// The steps each byte of the file adds to [`IMAGE_WORK`], so that a long scan is read whole: such a /// page pays for itself with about 2 KB of the file, about the smallest a page of text is stored -/// in, and a stream of any filter this reads can be decoded about 8 times at Flate's best, 1,032 to -/// one. So what a hostile file can ask for grows with its length, by about 14 minutes for 10 MB. +/// in, and a stream of any filter this reads can be decoded about 7 times at Flate's best, 1,032 to +/// one, its reading included. So what a hostile file can ask for grows with its length, by about 14 minutes for 10 MB. const IMAGE_WORK_PER_BYTE: u64 = 1 << 13; /// Decoded bytes one page's content walk may produce in all: its `/Contents` streams and every @@ -361,6 +363,11 @@ mod image_tests { Stream::from_raw(dict, pdfcraft_filters::encode_jbig2_generic(&[0xFF; 4], 16, 2).unwrap()) } + /// What a Flate stage pays to read `n` bytes, besides what it makes. + fn read(n: usize) -> u64 { + pdfcraft_filters::FLATE_READ_WORK * n as u64 + } + /// Each image spends what it took from one allowance, and those past it are refused and /// counted, whatever their filters. What can't be decoded here costs nothing, and fails /// without being one of these. @@ -407,7 +414,7 @@ mod image_tests { let mut dict = Dict::new(); dict.set(b"Filter".to_vec(), Object::Array(vec![Object::name("FlateDecode"), Object::name("FlateDecode")])); let bomb = Stream::from_raw(dict, pdfcraft_filters::encode_flate(&inner)); - let cost = (bomb.raw.len() + inner.len() + samples.len()) as u64; + let cost = (bomb.raw.len() + inner.len() + samples.len()) as u64 + read(bomb.raw.len()) + read(inner.len()); let mut budget = ImageBudget::with_work(2 * cost + 1); for left in [cost + 1, 1] { assert_eq!(budget.decode(&doc, &bomb, 1 << 20).unwrap(), samples); @@ -436,17 +443,17 @@ mod image_tests { let image = behind(pdfcraft_filters::encode_flate(&alone.raw)); let mut budget = ImageBudget::with_work(u64::MAX); assert_eq!(budget.decode(&doc, &image, 1 << 20).unwrap(), samples); - assert_eq!(u64::MAX - budget.work_left(), decoding + (image.raw.len() + alone.raw.len()) as u64); + assert_eq!(u64::MAX - budget.work_left(), decoding + (image.raw.len() + alone.raw.len()) as u64 + read(image.raw.len())); // Past this image's limit: an error, with what finding out took paid and the rest left. let bomb = behind(pdfcraft_filters::encode_flate(&vec![0; 1 << 16])); let mut budget = ImageBudget::with_work(u64::MAX); assert!(budget.decode(&doc, &bomb, 1000).is_err()); - assert_eq!(u64::MAX - budget.work_left(), bomb.raw.len() as u64 + 1000); + assert_eq!(u64::MAX - budget.work_left(), bomb.raw.len() as u64 + read(bomb.raw.len()) + 1000); assert_eq!(budget.refused(), 0, "that image's limit isn't the run's"); // A small allowance is a limit too: it stops there, and the image is refused. - let mut budget = ImageBudget::with_work(bomb.raw.len() as u64 + 10); + let mut budget = ImageBudget::with_work(bomb.raw.len() as u64 + read(bomb.raw.len()) + 10); assert!(budget.decode(&doc, &bomb, 1 << 20).is_err()); assert_eq!((budget.work_left(), budget.refused()), (0, 1)); } @@ -486,7 +493,7 @@ mod image_tests { let doc = Document::open(std::sync::Arc::new(file)).unwrap(); let image = doc.get(doc.root().unwrap()).as_dict().unwrap().reference(b"Image").unwrap(); let s = stream(&doc, image); - let filters = (raw.len() + skipped.len()) as u64; + let filters = (raw.len() + skipped.len()) as u64 + read(raw.len()); assert_eq!(spent(&doc, &s), first, "the first read undoes them"); assert_eq!(spent(&doc, &s), first - filters, "kept"); @@ -501,7 +508,7 @@ mod image_tests { let image = jbig2(Some(Object::Dict(parms))); let mut budget = ImageBudget::with_work(u64::MAX); assert!(budget.decode(&doc, &image, 1 << 20).unwrap_err().to_string().contains("globals")); - assert_eq!(u64::MAX - budget.work_left(), (over.len() + crate::MAX_JBIG2_GLOBALS) as u64); + assert_eq!(u64::MAX - budget.work_left(), (over.len() + crate::MAX_JBIG2_GLOBALS) as u64 + read(over.len())); assert_eq!(budget.refused(), 0); } @@ -522,7 +529,7 @@ mod image_tests { dict.set(b"DecodeParms".to_vec(), Object::Array(vec![Object::Null, Object::Dict(parms)])); let image = Stream::from_raw(dict, pdfcraft_filters::encode_flate(&coded)); let pixels = u64::from(w) * u64::from(h); - let cost = (image.raw.len() + coded.len() + 4 * coded.len() + 512 * h as usize) as u64 + pixels; + let cost = (image.raw.len() + coded.len() + 4 * coded.len() + 512 * h as usize) as u64 + read(image.raw.len()) + pixels; let mut budget = ImageBudget::with_work(cost); assert_eq!(budget.decode(&doc, &image, 1 << 20).unwrap().len(), 512 * h as usize); assert_eq!(budget.work_left(), 0, "exactly that"); @@ -547,7 +554,7 @@ mod image_tests { }; let rows = vec![0xFFu8; 512 * 1024]; let flate = blank(Some("FlateDecode"), pdfcraft_filters::encode_flate(&rows)); - let cost = (flate.raw.len() + rows.len()) as u64 + 4096 * 1024; + let cost = (flate.raw.len() + rows.len()) as u64 + read(flate.raw.len()) + 4096 * 1024; let mut budget = ImageBudget::with_work(cost); assert_eq!(budget.decode(&doc, &flate, 1 << 24).unwrap(), rows); assert_eq!((budget.work_left(), budget.refused()), (0, 0), "exactly that"); @@ -603,8 +610,8 @@ mod image_tests { budget.decode(&doc, s, 1 << 24).unwrap(); u64::MAX - budget.work_left() }; - assert_eq!(cost(&claims), (claims.raw.len() + 16 + 16 * 8) as u64, "its bytes, and a pixel a bit of them"); - assert_eq!(cost(&image), (image.raw.len() + 64 * 64) as u64 + 64 * 64, "its bytes, and its pixels"); + assert_eq!(cost(&claims), (claims.raw.len() + 16 + 16 * 8) as u64 + read(claims.raw.len()), "its bytes, and a pixel a bit of them"); + assert_eq!(cost(&image), (image.raw.len() + 64 * 64) as u64 + read(image.raw.len()) + 64 * 64, "its bytes, and its pixels"); for decode in [ImageBudget::decode, ImageBudget::decode_filters] { let mut budget = ImageBudget::with_work(cost(&claims) + cost(&image)); assert_eq!(decode(&mut budget, &doc, &claims, 1 << 24).unwrap(), [0; 16]); @@ -621,7 +628,7 @@ mod image_tests { let samples = vec![7u8; 1000]; let raw = pdfcraft_filters::encode_flate(&samples); let chain = [(Filter::Flate, Params::default())]; - let cost = (raw.len() + samples.len()) as u64; + let cost = (raw.len() + samples.len()) as u64 + read(raw.len()); let mut budget = ImageBudget::with_work(cost + 10); assert_eq!(budget.decode_chain(&chain, &raw, 1 << 20).unwrap(), samples); assert_eq!(budget.work_left(), 10); @@ -658,7 +665,7 @@ mod image_tests { assert_eq!(budget.decode_filters(&doc, &image, 1 << 20).unwrap(), rows); let cost = u64::MAX - budget.work_left(); // The tag byte of each of its 100 rows counts, which the predictor takes off. - assert_eq!(cost, (image.raw.len() + rows.len() + 100) as u64); + assert_eq!(cost, (image.raw.len() + rows.len() + 100) as u64 + read(image.raw.len())); let mut budget = ImageBudget::with_work(cost - 1); assert!(budget.decode_filters(&doc, &image, 1 << 20).is_err()); assert_eq!((budget.work_left(), budget.refused()), (0, 1)); diff --git a/crates/filters/README.md b/crates/filters/README.md index 90e1485177..38666e6369 100644 --- a/crates/filters/README.md +++ b/crates/filters/README.md @@ -37,7 +37,7 @@ encode_jbig2(data, columns, rows) -> Result, FilterError> // 1-bit row - Image codecs (DCT, JPX, JBIG2, CCITT), `Crypt` and unknown filters anywhere in a chain give `Unsupported`: callers keep such data encoded. So does a chain longer than `MAX_CHAIN` (8) filters, before any stage runs. Decryption belongs to `crypt`. - `max_output` bounds every intermediate buffer (including the pre-predictor buffer) and gives `LimitExceeded(max_output)`. This also holds when Flate retries raw deflate after a misleading zlib header, in both strict and tolerant decoding. - `decode_counted` and `decode_tolerant_counted` are for a caller charging a budget (`pdfcraft_cos::DecodeBudget`). `max_output` bounds all stages together (each stage gets what the earlier ones left), and they give the bytes every stage that ran decoded, on success and on failure. A stage's output counts as its decoder produced it, before a predictor or a later stage shrinks it, and a failed decode pays for what it decoded first. -- `decode_tolerant_within` is for a run of images (`pdfcraft_cos::ImageBudget`), as `decode_within` (its strict form), `decode_jbig2_within` and `decode_ccitt_within` are, all spending from one `work`, in the JBIG2 decoder's steps. `max_output` bounds each stage, as in `decode_tolerant`, and no stage decodes past what `work` has left. It pays a step a byte of `data` first and a step a byte each stage decodes, so a chain that fails has paid for what it did: 0.2 to 3 ns a byte in or out, up to 19 ns through a predictor on samples of a bit, against the 6 to 16 ns of a JBIG2 step. What `work` can't pay for spends all of it, and gives `Spent`; with nothing left nothing is read. A chain that can't be decoded at all costs nothing and gives its usual error. +- `decode_tolerant_within` is for a run of images (`pdfcraft_cos::ImageBudget`), as `decode_within` (its strict form), `decode_jbig2_within` and `decode_ccitt_within` are, all spending from one `work`, in the JBIG2 decoder's steps. `max_output` bounds each stage, as in `decode_tolerant`, and no stage decodes past what `work` has left. It pays a step a byte of `data` first and a step a byte each stage decodes, so a chain that fails has paid for what it did: 0.2 to 3 ns a byte in or out, up to 19 ns through a predictor on samples of a bit, against the 6 to 16 ns of a JBIG2 step. A Flate stage also pays `FLATE_READ_WORK` (128) steps a byte it reads, before it runs, and again if it reads them a second time as raw deflate: inflating does work for every block it reads, and a stream of empty blocks took 1.4 µs a byte to make nothing, about 11 ns a step at that price. What `work` can't pay for spends all of it, and gives `Spent`; with nothing left nothing is read. A chain that can't be decoded at all costs nothing and gives its usual error. - `decode_tolerant` turns `Corrupt` into "what was decoded so far" plus `partial = true`, then runs the rest of the chain on it (predictors are applied to partial output too). `LimitExceeded` and `Unsupported` are still errors. ## Behaviour diff --git a/crates/filters/src/flate.rs b/crates/filters/src/flate.rs index 1c38f72a53..ffe46207b7 100644 --- a/crates/filters/src/flate.rs +++ b/crates/filters/src/flate.rs @@ -21,25 +21,48 @@ fn zlib_header(data: &[u8]) -> Option<(u8, u8)> { ok.then_some((cmf, flg)) } -pub(crate) fn decode(data: &[u8], max: usize) -> Step { +/// `data` inflated within `max`, and whether it was read twice: a zlib header that only looked +/// valid is read again as raw deflate, within `again`, what a caller paying for what is read has +/// left for it once it has paid for the second reading. `None`, that reading can't be paid for, +/// and is [`FilterError::Spent`](crate::FilterError::Spent). +pub(crate) fn decode(data: &[u8], max: usize, again: Option) -> (Step, bool) { match zlib_header(data) { - Some((_, flg)) if flg & 0x20 != 0 => Err(Failure::corrupt(NAME, "zlib preset dictionary is not allowed in PDF", Vec::new())), + Some((_, flg)) if flg & 0x20 != 0 => (Err(Failure::corrupt(NAME, "zlib preset dictionary is not allowed in PDF", Vec::new())), false), Some(_) => match inflate_raw(&data[2..], max) { // A header that only looked valid: retry the whole input as raw deflate. - Err(f) if f.partial.is_empty() && matches!(f.error, crate::FilterError::Corrupt { .. }) => match inflate_raw(data, max) { - // Keep the original corruption diagnostic only if the retry is corrupt - // too. A limit error must remain fatal even for tolerant decoding. What the - // retry decoded isn't kept, but a caller charging for work has to hear of it. - Err(retry) if matches!(retry.error, crate::FilterError::Corrupt { .. }) => Err(Failure { discarded: retry.partial.len(), ..f }), - retry => retry, - }, - r => r, + Err(f) if f.partial.is_empty() && matches!(f.error, crate::FilterError::Corrupt { .. }) => { + let Some(again) = again else { + return (Err(Failure { error: crate::FilterError::Spent, partial: Vec::new(), discarded: 0 }), false); + }; + let step = match inflate_raw(data, again) { + // Keep the original corruption diagnostic only if the retry is corrupt + // too. A limit error must remain fatal even for tolerant decoding. What the + // retry decoded isn't kept, but a caller charging for work has to hear of it. + Err(retry) if matches!(retry.error, crate::FilterError::Corrupt { .. }) => Err(Failure { discarded: retry.partial.len(), ..f }), + retry => retry, + }; + (step, true) + } + r => (r, false), }, - None => inflate_raw(data, max), + None => (inflate_raw(data, max), false), } } +#[cfg(test)] +thread_local! { + /// Bytes inflating gave back on this thread, its partial output included. + pub(crate) static INFLATED: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + fn inflate_raw(input: &[u8], max: usize) -> Step { + let step = inflate(input, max); + #[cfg(test)] + INFLATED.with(|c| c.set(c.get() + step.as_ref().map_or_else(|f| f.partial.len(), Vec::len))); + step +} + +fn inflate(input: &[u8], max: usize) -> Step { let mut d = Decompress::new(false); let mut out: Vec = Vec::new(); // One byte beyond the limit lets us tell "exactly max" from "more than max". @@ -87,7 +110,7 @@ mod tests { use crate::FilterError; fn dec(d: &[u8]) -> Result, FilterError> { - decode(d, 1 << 24).map_err(|f| f.error) + decode(d, 1 << 24, Some(1 << 24)).0.map_err(|f| f.error) } #[test] @@ -144,7 +167,7 @@ mod tests { fn corrupt_and_truncated() { let data: Vec = (0..10_000u32).map(|i| (i % 13) as u8).collect(); let z = encode(&data); - match decode(&z[..z.len() / 2], 1 << 24) { + match decode(&z[..z.len() / 2], 1 << 24, Some(1 << 24)).0 { Err(f) => { assert!(matches!(f.error, FilterError::Corrupt { .. })); assert_eq!(&data[..f.partial.len()], &f.partial[..]); @@ -168,6 +191,6 @@ mod tests { } fn dec_limit(d: &[u8], max: usize) -> Result, FilterError> { - decode(d, max).map_err(|f| f.error) + decode(d, max, Some(max)).0.map_err(|f| f.error) } } diff --git a/crates/filters/src/lib.rs b/crates/filters/src/lib.rs index 181f96ab66..cf10d0586b 100644 --- a/crates/filters/src/lib.rs +++ b/crates/filters/src/lib.rs @@ -188,7 +188,7 @@ pub(crate) fn pay(work: &mut u64, steps: u64) -> Result<(), FilterError> { /// [`FilterError::Unsupported`] (callers keep such data encoded), as does a chain longer than /// [`MAX_CHAIN`]. `max_output` bounds every intermediate buffer. pub fn decode(chain: &[(Filter, Params)], data: &[u8], max_output: usize) -> Result, FilterError> { - run(chain, data, max_output, None, true).map(|(v, _, _)| v).map_err(|(e, _)| e) + run(chain, data, max_output, None, true, false).map(|(v, _, _)| v).map_err(|(e, _)| e) } /// Like [`decode`] for a caller charging an allowance: `max_output` bounds the bytes all stages @@ -196,27 +196,28 @@ pub fn decode(chain: &[(Filter, Params)], data: &[u8], max_output: usize) -> Res /// bytes, on success and on failure. A stage's output counts even when a later stage or its /// predictor shrinks it. pub fn decode_counted(chain: &[(Filter, Params)], data: &[u8], max_output: usize) -> Result<(Vec, usize), (FilterError, usize)> { - run(chain, data, max_output, Some(max_output), true).map(|(v, _, spent)| (v, spent)) + run(chain, data, max_output, Some(max_output), true, false).map(|(v, _, spent)| (v, spent)) } /// Like [`decode`], but on corrupt data returns what was decoded before the error and /// reports `partial = true` (the behaviour viewers expect). The remaining filters of the /// chain still run on the partial data. Limit and unsupported errors are still errors. pub fn decode_tolerant(chain: &[(Filter, Params)], data: &[u8], max_output: usize) -> Result<(Vec, bool), FilterError> { - run(chain, data, max_output, None, false).map(|(v, partial, _)| (v, partial)).map_err(|(e, _)| e) + run(chain, data, max_output, None, false, false).map(|(v, partial, _)| (v, partial)).map_err(|(e, _)| e) } /// [`decode_tolerant`] bounded and counted as [`decode_counted`] is. pub fn decode_tolerant_counted(chain: &[(Filter, Params)], data: &[u8], max_output: usize) -> Result<(Vec, bool, usize), (FilterError, usize)> { - run(chain, data, max_output, Some(max_output), false) + run(chain, data, max_output, Some(max_output), false, false) } /// [`decode_tolerant`], spending from `work`, what decoding may still cost across a run of images: -/// a step for each byte of `data`, paid before anything is decoded, and one for each byte a stage -/// decodes, so a chain that fails, or a stage that goes past `max_output`, has paid for what it did. -/// No stage decodes past what is left: a chain that `work` can't pay for spends all of it and stops -/// there, so with nothing left nothing is read. A chain that can't be decoded at all costs nothing, -/// whatever is left, so what an image's filters are is known from the error. +/// a step for each byte of `data`, paid before anything is decoded, [`FLATE_READ_WORK`] for each +/// byte a Flate stage reads, paid before it runs (again if it reads them twice), and one for each +/// byte a stage decodes, so a chain that fails, or a stage that goes past `max_output`, has paid for +/// what it did. No stage decodes past what is left: a chain that `work` can't pay for spends all of +/// it and stops there, so with nothing left nothing is read. A chain that can't be decoded at all +/// costs nothing, whatever is left, so what an image's filters are is known from the error. /// /// Unlike [`decode_tolerant`], no filters at all gives `data` itself, within `max_output`. pub fn decode_tolerant_within(chain: &[(Filter, Params)], data: &[u8], max_output: usize, work: &mut u64) -> Result, FilterError> { @@ -236,14 +237,15 @@ fn spending(chain: &[(Filter, Params)], data: &[u8], max_output: usize, work: &m return if data.len() > max_output { Err(FilterError::LimitExceeded(max_output)) } else { Ok(data.to_vec()) }; } let left = usize::try_from(*work).unwrap_or(usize::MAX); - match run(chain, data, max_output, Some(left), strict) { + match run(chain, data, max_output, Some(left), strict, true) { Ok((out, _, decoded)) => pay(work, decoded as u64).map(|()| out), // The stage stopped at the lower of the two, which took as much to find out. Err((FilterError::LimitExceeded(_), decoded)) if left.saturating_sub(decoded) > max_output => { pay(work, (decoded as u64).saturating_add(max_output as u64))?; Err(FilterError::LimitExceeded(max_output)) } - Err((FilterError::LimitExceeded(_), _)) => { + // Past what is left, a stage stops, or a Flate stage doesn't start. + Err((FilterError::LimitExceeded(_) | FilterError::Spent, _)) => { *work = 0; Err(FilterError::Spent) } @@ -255,6 +257,17 @@ fn spending(chain: &[(Filter, Params)], data: &[u8], max_output: usize, work: &m /// up to the output limit, so the chain length multiplies the work. pub const MAX_CHAIN: usize = 8; +/// The steps each byte Flate reads costs where decoding is paid for ([`decode_tolerant_within`], +/// [`decode_within`]), besides a step for each byte it makes. Inflating does work for every block +/// it reads, whether or not the block makes anything: an empty fixed-Huffman block is ten bits +/// (RFC 1951 §3.2.6), and a stream of them took 1,383 ns a byte to make nothing (miniz_oxide +/// 0.8.9, release build), one of a literal each 768 ns a byte, and empty dynamic blocks 106. At +/// this price the worst of these comes to about 11 ns a step, inside what a step of JBIG2 +/// decoding takes. Data a real encoder makes took 0.4 to 3.2 ns for each byte it made, its reading +/// included, so it pays far more than reading takes; each byte of a file still adds 64 times this +/// price to an operation's allowance (8,192 steps, `IMAGE_WORK_PER_BYTE` in `pdfcraft_cos`). +pub const FLATE_READ_WORK: u64 = 1 << 7; + /// Whether a chain is one [`run`] takes, and if not, the error. fn decodable(chain: &[(Filter, Params)]) -> Result<(), FilterError> { if chain.len() > MAX_CHAIN { @@ -268,22 +281,39 @@ fn decodable(chain: &[(Filter, Params)]) -> Result<(), FilterError> { /// The decoded data, whether it is partial and the bytes all stages decoded, or the error and /// the bytes decoded before it. `max` bounds each stage, and `total` all of them together. +/// `priced`, the count is steps: each Flate stage also pays [`FLATE_READ_WORK`] for each byte it +/// reads, before it runs, so a stage `total` can't pay for doesn't run +/// ([`FilterError::Spent`]). fn run( chain: &[(Filter, Params)], data: &[u8], max: usize, total: Option, strict: bool, + priced: bool, ) -> Result<(Vec, bool, usize), (FilterError, usize)> { decodable(chain).map_err(|e| (e, 0))?; let mut cur: Cow<[u8]> = Cow::Borrowed(data); let mut partial = false; let mut spent = 0usize; + let price = usize::try_from(FLATE_READ_WORK).unwrap_or(usize::MAX); for (filter, params) in chain { + let read = if priced && matches!(filter, Filter::Flate) { price.saturating_mul(cur.len()) } else { 0 }; + let room = total.map_or(usize::MAX, |total| total.saturating_sub(spent)); + if read > room { + return Err((FilterError::Spent, spent)); + } + spent = spent.saturating_add(read); #[cfg(test)] STAGE_BYTES.with(|c| c.set(c.get() + cur.len())); let limit = total.map_or(max, |total| max.min(total.saturating_sub(spent))); - let (step, work) = decode_one(filter, params, &cur, limit); + // A second reading of the same bytes (Flate's raw retry) is paid for as the first was, and + // makes no more than is left once it has been. + let again = room.checked_sub(read.saturating_mul(2)).map(|left| max.min(left)); + let (step, work, again) = decode_one(filter, params, &cur, limit, again); + if again { + spent = spent.saturating_add(read); + } spent = spent.saturating_add(work); cur = Cow::Owned(match step { Ok(v) => v, @@ -298,15 +328,21 @@ fn run( Ok((cur.into_owned(), partial, spent)) } -/// One stage's result and the bytes its decoder produced (before any predictor). -fn decode_one(filter: &Filter, params: &Params, data: &[u8], max: usize) -> (Step, usize) { +/// One stage's result, the bytes its decoder produced (before any predictor), and whether it +/// read its input twice, which it does only within `again` ([`flate::decode`]). +fn decode_one(filter: &Filter, params: &Params, data: &[u8], max: usize, again: Option) -> (Step, usize, bool) { + let once = |(step, n): (Step, usize)| (step, n, false); match filter { - Filter::Flate => with_predictor(flate::decode(data, max), params, "FlateDecode"), - Filter::Lzw => with_predictor(lzw::decode(data, params.early_change != 0, max), params, "LZWDecode"), - Filter::AsciiHex => counted(asciihex::decode(data, max)), - Filter::Ascii85 => counted(ascii85::decode(data, max)), - Filter::RunLength => counted(runlength::decode(data, max)), - other => (Err(Failure { error: FilterError::Unsupported(other.name().to_owned()), partial: Vec::new(), discarded: 0 }), 0), + Filter::Flate => { + let (step, twice) = flate::decode(data, max, again); + let (step, n) = with_predictor(step, params, "FlateDecode"); + (step, n, twice) + } + Filter::Lzw => once(with_predictor(lzw::decode(data, params.early_change != 0, max), params, "LZWDecode")), + Filter::AsciiHex => once(counted(asciihex::decode(data, max))), + Filter::Ascii85 => once(counted(ascii85::decode(data, max))), + Filter::RunLength => once(counted(runlength::decode(data, max))), + other => (Err(Failure { error: FilterError::Unsupported(other.name().to_owned()), partial: Vec::new(), discarded: 0 }), 0, false), } } @@ -552,7 +588,7 @@ mod tests { let data = encode_flate(&hex); let chain = [(Filter::Flate, Params::default()), (Filter::AsciiHex, Params::default())]; let whole = decode(&chain, &data, 1 << 20).unwrap(); - let cost = (data.len() + hex.len() + whole.len()) as u64; + let cost = (data.len() + hex.len() + whole.len()) as u64 + FLATE_READ_WORK * data.len() as u64; let mut work = u64::MAX; assert_eq!(decode_tolerant_within(&chain, &data, 1 << 20, &mut work).unwrap(), whole); assert_eq!(u64::MAX - work, cost); @@ -584,8 +620,11 @@ mod tests { assert_eq!(work, 0); super::STAGE_BYTES.with(|c| c.get()) }; - assert_eq!(read((bomb.len() + inner.len()) as u64 + 10), bomb.len() + inner.len(), "the second stage stops"); - assert_eq!(read(bomb.len() as u64 + 10), bomb.len(), "the first stops, and the second never runs"); + let paid = |n: usize| (1 + FLATE_READ_WORK) * n as u64; + assert_eq!(read(paid(bomb.len()) + paid(inner.len()) + 10), bomb.len() + inner.len(), "the second stage stops"); + assert_eq!(read(paid(bomb.len()) + FLATE_READ_WORK * inner.len() as u64 - 1), bomb.len(), "the second can't pay to read"); + assert_eq!(read(paid(bomb.len()) + 10), bomb.len(), "the first stops, and the second never runs"); + assert_eq!(read(paid(bomb.len()) - 1), 0, "the first can't pay to read"); assert_eq!(read(bomb.len() as u64 - 1), 0, "it can't pay to be read"); } @@ -597,18 +636,19 @@ mod tests { let chain = [(Filter::Flate, Params::default())]; let mut work = u64::MAX; assert!(matches!(decode_tolerant_within(&chain, &data, 1000, &mut work), Err(FilterError::LimitExceeded(1000)))); - assert_eq!(u64::MAX - work, data.len() as u64 + 1000); + let read = |n: usize| FLATE_READ_WORK * n as u64; + assert_eq!(u64::MAX - work, data.len() as u64 + read(data.len()) + 1000); // The later stage of two, which pays for the first as well. let bomb = encode_flate(&data); let two = [(Filter::Flate, Params::default()), (Filter::Flate, Params::default())]; let mut work = u64::MAX; assert!(matches!(decode_tolerant_within(&two, &bomb, data.len(), &mut work), Err(FilterError::LimitExceeded(_)))); - assert_eq!(u64::MAX - work, (bomb.len() + 2 * data.len()) as u64); + assert_eq!(u64::MAX - work, (bomb.len() + 2 * data.len()) as u64 + read(bomb.len()) + read(data.len())); // Both limits at once: a run that can't go on is what a caller needs to hear. - let mut work = data.len() as u64 + 1000; + let mut work = data.len() as u64 + read(data.len()) + 1000; assert!(refused(&decode_tolerant_within(&chain, &data, 1000, &mut work).unwrap_err())); assert_eq!(work, 0); - let mut work = data.len() as u64 + 1001; + let mut work = data.len() as u64 + read(data.len()) + 1001; assert!(matches!(decode_tolerant_within(&chain, &data, 1000, &mut work), Err(FilterError::LimitExceeded(1000)))); assert_eq!(work, 1, "the image's limit, with a step to spare"); } @@ -624,21 +664,22 @@ mod tests { assert!(!partial.is_empty()); let mut work = u64::MAX; assert_eq!(decode_tolerant_within(&chain, &cut, 1 << 20, &mut work).unwrap(), partial); - assert_eq!(u64::MAX - work, (cut.len() + partial.len()) as u64); + let read = |n: usize| FLATE_READ_WORK * n as u64; + assert_eq!(u64::MAX - work, (cut.len() + partial.len()) as u64 + read(cut.len())); // Read strictly it is an error, for the same work. let mut work = u64::MAX; assert!(matches!(decode_within(&chain, &cut, 1 << 20, &mut work), Err(FilterError::Corrupt { .. }))); - assert_eq!(u64::MAX - work, (cut.len() + partial.len()) as u64); + assert_eq!(u64::MAX - work, (cut.len() + partial.len()) as u64 + read(cut.len())); let whole = encode_flate(&data); let mut work = u64::MAX; assert_eq!(decode_within(&chain, &whole, 1 << 20, &mut work).unwrap(), data); - assert_eq!(u64::MAX - work, (whole.len() + data.len()) as u64); + assert_eq!(u64::MAX - work, (whole.len() + data.len()) as u64 + read(whole.len())); // A predictor that comes to nothing, after all of this was inflated for it. let p = Params { predictor: 99, ..Params::default() }; let data = encode_flate(&vec![0; 1 << 16]); let mut work = u64::MAX; assert!(decode_tolerant_within(&[(Filter::Flate, p)], &data, 1 << 20, &mut work).unwrap().is_empty()); - assert_eq!(u64::MAX - work, data.len() as u64 + (1 << 16)); + assert_eq!(u64::MAX - work, data.len() as u64 + read(data.len()) + (1 << 16)); } /// A stream that reads as zlib only until it fails, and as raw deflate until that fails too, @@ -660,13 +701,67 @@ mod tests { assert_eq!(spent, 0x0601); let mut work = u64::MAX; assert!(decode_tolerant_within(&chain, &data, 1 << 20, &mut work).unwrap().is_empty()); - assert_eq!(u64::MAX - work, (data.len() + 0x0601) as u64); + // Read twice: as zlib, then as raw deflate. + assert_eq!(u64::MAX - work, (data.len() + 0x0601) as u64 + 2 * FLATE_READ_WORK * data.len() as u64); // Through a predictor, which would otherwise lose the count. let p = Params { predictor: 12, columns: 4, ..Params::default() }; let (_, _, spent) = decode_tolerant_counted(&[(Filter::Flate, p)], &data, 1 << 20).unwrap(); assert_eq!(spent, 0x0601); } + /// Inflating does work for every block it reads, whether or not the block makes anything: empty + /// fixed-Huffman blocks, ten bits each, make nothing however many there are. Each byte of them + /// is paid for at Flate's price before any is read. + #[test] + fn a_flate_stage_pays_for_what_it_reads() { + // Four empty blocks in five bytes, as raw deflate, then an empty final block. + let mut hollow = [0x02, 0x08, 0x20, 0x80, 0x00].repeat(1 << 12); + hollow.extend([0x03, 0x00]); + let chain = [(Filter::Flate, Params::default())]; + assert_eq!(decode(&chain, &hollow, 1 << 20).unwrap(), b""); + let cost = (1 + FLATE_READ_WORK) * hollow.len() as u64; + let mut work = u64::MAX; + assert_eq!(decode_tolerant_within(&chain, &hollow, 1 << 20, &mut work).unwrap(), b""); + assert_eq!(u64::MAX - work, cost); + let mut work = cost; + assert!(decode_tolerant_within(&chain, &hollow, 1 << 20, &mut work).is_ok(), "exactly enough"); + assert_eq!(work, 0); + super::STAGE_BYTES.with(|c| c.set(0)); + let mut work = cost - 1; + assert!(refused(&decode_tolerant_within(&chain, &hollow, 1 << 20, &mut work).unwrap_err())); + assert_eq!((work, super::STAGE_BYTES.with(|c| c.get())), (0, 0), "nothing was inflated"); + // Counted for a byte allowance, as a page's content is, reading costs nothing. + assert_eq!(decode_counted(&chain, &hollow, 1 << 20).unwrap(), (Vec::new(), 0)); + } + + /// A zlib header that only looked valid is read again as raw deflate, and the second reading + /// is paid for as the first was. One that can't be paid for is refused, not read as damaged. + #[test] + fn a_second_reading_is_paid_for() { + // As zlib: a block of a reserved type at once. As raw deflate: a stored block of this many + // zero bytes, then an empty final block. + let mut data = vec![0x78, 0x01, 0x06, 0xFE, 0xF9]; + data.extend(vec![0u8; 0x0601]); + data.extend([0x03, 0x00]); + let chain = [(Filter::Flate, Params::default())]; + assert_eq!(decode(&chain, &data, 1 << 20).unwrap(), vec![0u8; 0x0601]); + let read = FLATE_READ_WORK * data.len() as u64; + let mut work = (data.len() + 0x0601) as u64 + 2 * read; + assert_eq!(decode_tolerant_within(&chain, &data, 1 << 20, &mut work).unwrap(), vec![0u8; 0x0601]); + assert_eq!(work, 0, "exactly enough"); + // Enough to read it once and not twice: it isn't read again. + super::flate::INFLATED.with(|c| c.set(0)); + let mut work = data.len() as u64 + 2 * read - 1; + assert!(refused(&decode_tolerant_within(&chain, &data, 1 << 20, &mut work).unwrap_err())); + assert_eq!((work, super::flate::INFLATED.with(|c| c.get())), (0, 0)); + // Enough to read it twice, and a byte short of what it makes: the second reading makes no + // more than is left once it has been paid for. + super::flate::INFLATED.with(|c| c.set(0)); + let mut work = (data.len() + 0x0601) as u64 + 2 * read - 1; + assert!(refused(&decode_tolerant_within(&chain, &data, 1 << 20, &mut work).unwrap_err())); + assert!(super::flate::INFLATED.with(|c| c.get()) < 0x0601, "it stopped short"); + } + /// With nothing left nothing is read. What can't be decoded at all never costs anything, and /// says so whatever is left. Without filters, what is read is the data. #[test] diff --git a/crates/optimize/src/fragments.rs b/crates/optimize/src/fragments.rs index f74b091984..b5662a2bb6 100644 --- a/crates/optimize/src/fragments.rs +++ b/crates/optimize/src/fragments.rs @@ -394,8 +394,8 @@ mod tests { let mut budget = ImageBudget::with_work(0); assert_eq!(merge_page(&mut doc.clone(), page, &mut Report::default(), &mut |_| true, &mut budget).unwrap(), 0); assert_eq!(budget.refused(), 1); - // Reading each strip, and building the picture of 8 rows of 8 they make. - let cost = raw + 8 * (8 + 8) + 8 * 8; + // Reading each strip, inflating it, and building the picture of 8 rows of 8 they make. + let cost = (1 + pdfcraft_filters::FLATE_READ_WORK) * raw + 8 * (8 + 8) + 8 * 8; let mut budget = ImageBudget::with_work(u64::MAX); assert_eq!(merge_page(&mut doc.clone(), page, &mut Report::default(), &mut |_| true, &mut budget).unwrap(), 8); assert_eq!(u64::MAX - budget.work_left(), cost); diff --git a/crates/optimize/src/images.rs b/crates/optimize/src/images.rs index 92e68582c2..3ad9409f7d 100644 --- a/crates/optimize/src/images.rs +++ b/crates/optimize/src/images.rs @@ -1515,7 +1515,7 @@ mod dimension_tests { dict.set(b"BitsPerComponent".to_vec(), Object::Int(8)); dict.set(b"ColorSpace".to_vec(), Object::name("DeviceGray")); let image = Stream::flate(dict, &samples); - let read = (image.raw.len() + samples.len()) as u64 + 256 * 256; + let read = (image.raw.len() + samples.len()) as u64 + pdfcraft_filters::FLATE_READ_WORK * image.raw.len() as u64 + 256 * 256; let original = doc.add(Object::Stream(image.clone())); let mut xobjects = Dict::new(); xobjects.set(b"I0".to_vec(), Object::Ref(original)); @@ -1607,7 +1607,7 @@ mod dimension_tests { }; let mut doc = Document::new_empty(); let mask = image("DeviceGray"); - let cost = (mask.raw.len() + noise.len()) as u64 + 256 * 256; + let cost = (mask.raw.len() + noise.len()) as u64 + pdfcraft_filters::FLATE_READ_WORK * mask.raw.len() as u64 + 256 * 256; let masked = doc.add(Object::Stream(mask)); let mut with_mask = image("DeviceGray"); with_mask.dict.set(b"SMask".to_vec(), Object::Ref(masked)); From f16c7db8f4aef7c965f7111d77a184b6605b78d8 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Mon, 2 Nov 2026 20:41:12 +0100 Subject: [PATCH 034/198] M2.6: a preset dictionary is refused whatever follows its header Mutation testing of the commit before this one left the preset-dictionary check in flate.rs alive: the test's bytes after the header fail to inflate either way. A header with FDICT set followed by data that inflates as raw deflate is refused too. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/filters/src/flate.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/filters/src/flate.rs b/crates/filters/src/flate.rs index ffe46207b7..ce57121f4a 100644 --- a/crates/filters/src/flate.rs +++ b/crates/filters/src/flate.rs @@ -178,6 +178,8 @@ mod tests { assert!(matches!(dec(&[0x78, 0x9c, 0x07, 0, 0, 0]), Err(FilterError::Corrupt { .. }))); // Preset dictionary. assert!(matches!(dec(&[0x78, 0x20, 0, 0, 0, 0]), Err(FilterError::Corrupt { .. }))); + // One whose header is followed by data that inflates: refused all the same. + assert!(matches!(dec(&[0x78, 0xBB, 0x4b, 0x04, 0x00]), Err(FilterError::Corrupt { .. }))); } #[test] From 6771ddffe87b902b8d61b1b19d5914020d436fbd Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Mon, 2 Nov 2026 23:07:28 +0100 Subject: [PATCH 035/198] M2.6: inflate with zlib-rs everywhere, so reading Flate costs 2 steps a byte The Resources review of the commits before this one found that pricing what Flate reads at 128 steps a byte refused real redactions. Redaction decodes a picture once for each page that draws it, so 240 pages sharing a 1 MB scan-like Flate picture, a mark on each, went from Ok (240 cleared, 20.3 s) to ImagesPastAllowance, its allowance paying for 188 decodes: each paid about 27 times what it had. The cost being priced is miniz_oxide's: about 1.7 us for every fixed-Huffman block, empty or not. flate2's zlib-rs backend took about 11 to 13 ns a fixed block, and the Windows build already used it (craft-update turns on zip's deflate-flate2-zlib-rs, and with both backends on, flate2 inflates with zlib-rs, measured). Every platform now builds flate2 with zlib-rs, which it prefers to the miniz_oxide other dependencies still build. In a release build (zlib-rs 0.6.8), per byte read: empty fixed blocks 9 ns, fixed blocks of one literal 6, empty dynamic 21, empty stored 1.3, against 1,383, 768, 106 and 2 before. Each backend inflating what its own encoder made, 16 MiB of text took 2.6 ms against 6.3 to 13.7, and a scan-like picture 38 ms against 53. FLATE_READ_WORK is now 2: per byte read, the worst shape comes to about 7 ns a step, and a picture an encoder made pays twice its stored length more than unpriced (an_image_an_encoder_made_pays_little_for_its_reading: at most half again). The same review: export and compare's Office export read an indexed image's lookup table outside the allowance, once per image, even with nothing left (1.4 s each for 1 MiB of empty blocks under miniz_oxide). It is read through ImageBudget::decode_filters now, and refused past it (a_lookup_table_is_read_within_the_exports_allowance). filters, cos, optimize, redact, create, edit, export and engine pass with the new backend. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- Cargo.toml | 4 +++- crates/cos/src/budget.rs | 35 ++++++++++++++++++++++++++++++++--- crates/create/src/extract.rs | 23 +++++++++++++++-------- crates/create/src/tests.rs | 32 ++++++++++++++++++++++++++++++++ crates/filters/README.md | 2 +- crates/filters/src/lib.rs | 19 ++++++++++--------- 6 files changed, 93 insertions(+), 22 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index edd9e682b7..19714f41a6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,7 +64,9 @@ pdfcraft-automation = { path = "crates/automation", version = "0.17.0", default- hayro = "0.7.1" hayro-syntax = "0.7.2" lopdf = "0.45" -flate2 = { version = "1.1", default-features = false, features = ["rust_backend"] } +# zlib-rs, which the Windows build already got through craft-update: miniz_oxide spends about +# 1.7 µs on every deflate block, empty ones included (`pdfcraft_filters::FLATE_READ_WORK`). +flate2 = { version = "1.1", default-features = false, features = ["zlib-rs"] } weezl = "0.2" # CCITT Group 4 encoding (pdfcraft-filters); already in the tree through tiff. fax = "0.2.7" diff --git a/crates/cos/src/budget.rs b/crates/cos/src/budget.rs index d3e4a71974..ef68bfffdf 100644 --- a/crates/cos/src/budget.rs +++ b/crates/cos/src/budget.rs @@ -13,12 +13,12 @@ use crate::{CosError, Document, ObjRef, Object, Stream}; /// decode them, so this alone reads about 1,000 of them. A byte into or out of a filter is a step /// too, and takes less: 3 ns at most, or up to 19 ns through a predictor on samples of a bit. A /// byte Flate reads is [`pdfcraft_filters::FLATE_READ_WORK`] steps besides, since inflating blocks -/// that make nothing takes up to 1.4 µs a byte: about 11 ns a step. +/// that make nothing takes up to 21 ns a byte: about 7 ns a step. const IMAGE_WORK: u64 = 1 << 34; /// The steps each byte of the file adds to [`IMAGE_WORK`], so that a long scan is read whole: such a /// page pays for itself with about 2 KB of the file, about the smallest a page of text is stored -/// in, and a stream of any filter this reads can be decoded about 7 times at Flate's best, 1,032 to -/// one, its reading included. So what a hostile file can ask for grows with its length, by about 14 minutes for 10 MB. +/// in, and a stream of any filter this reads can be decoded about 8 times at Flate's best, 1,032 to +/// one. So what a hostile file can ask for grows with its length, by about 14 minutes for 10 MB. const IMAGE_WORK_PER_BYTE: u64 = 1 << 13; /// Decoded bytes one page's content walk may produce in all: its `/Contents` streams and every @@ -591,6 +591,35 @@ mod image_tests { assert_eq!((budget.work_left(), budget.refused()), (0, 1)); } + /// What Flate reads is priced for blocks that make nothing, and an image an encoder made pays + /// little for it: a scan-like picture stored in under half its size costs no more than half + /// again its bytes, samples and pixels. Redaction decodes a picture once for each page that draws it, so + /// a letterhead on hundreds of pages has to fit as it did unpriced. + #[test] + fn an_image_an_encoder_made_pays_little_for_its_reading() { + let doc = Document::new_empty(); + let mut seed = 0x2545_F491_4F6C_DD1Du64; + let samples: Vec = (0..256 * 256usize) + .map(|i| { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + ((i % 256) as u8 / 16) ^ (seed as u8 & 3) + }) + .collect(); + let mut dict = Dict::new(); + dict.set(b"Width".to_vec(), Object::Int(256)); + dict.set(b"Height".to_vec(), Object::Int(256)); + let image = Stream::flate(dict, &samples); + assert!(image.raw.len() * 2 < samples.len(), "stored smaller: {}", image.raw.len()); + let unpriced = (image.raw.len() + samples.len()) as u64 + 256 * 256; + let mut budget = ImageBudget::with_work(u64::MAX); + budget.decode(&doc, &image, 1 << 20).unwrap(); + let paid = u64::MAX - budget.work_left(); + assert_eq!(paid, unpriced + read(image.raw.len())); + assert!(2 * paid <= 3 * unpriced, "{paid} steps against {unpriced}"); + } + /// An image pays for the pixels its data holds, not those its size declares: one claiming /// 131072 by 131072 pixels with a few bytes of Flate costs what those bytes give, and the /// images after it are decoded as they would be without it. diff --git a/crates/create/src/extract.rs b/crates/create/src/extract.rs index d5103071e6..8fc7fea984 100644 --- a/crates/create/src/extract.rs +++ b/crates/create/src/extract.rs @@ -184,11 +184,11 @@ fn image(doc: &Document, s: &Stream, budget: &mut ImageBudget) -> Result<(&'stat // In a PDF a CMYK JPEG's samples are ink amounts unless /Decode inverts them, while image // viewers read a CMYK JPEG file as inverted (Adobe's convention): as is, it would show as // a negative. - let as_is_negative = jpeg && !inverted && matches!(space(doc, s.dict.get(b"ColorSpace")), Ok(Space::Cmyk)); + let as_is_negative = jpeg && !inverted && matches!(space(doc, s.dict.get(b"ColorSpace"), budget), Ok(Space::Cmyk)); if jpeg && alpha.is_none() && key.is_none() && !as_is_negative { return as_is(); } - let mut space = if mask { Space::Gray } else { space(doc, s.dict.get(b"ColorSpace"))? }; + let mut space = if mask { Space::Gray } else { space(doc, s.dict.get(b"ColorSpace"), budget)? }; let (samples, bpc) = if jpeg { if !budget.pay_jpeg((w as u64).saturating_mul(h as u64).saturating_mul(space.components() as u64), &s.raw) { return Err(FilterError::Spent.to_string()); @@ -436,7 +436,10 @@ impl Space { /// A lookup table holds at most 256 entries of 4 bytes; decoding stops well past that. const MAX_LOOKUP: usize = 64 << 10; -fn space(doc: &Document, cs: Option<&Object>) -> Result { +/// The colour space `cs` names; an indexed one's lookup table is read within `budget`, as the +/// image's samples are, since each image reads it again and its filters can do work that makes +/// nothing. +fn space(doc: &Document, cs: Option<&Object>, budget: &mut ImageBudget) -> Result { let Some(cs) = cs else { return Err("the image has no colour space".into()) }; let cs = doc.resolve(cs); let unsupported = |n: &[u8]| Err(format!("{} images can't be exported yet", String::from_utf8_lossy(n))); @@ -470,7 +473,7 @@ fn space(doc: &Document, cs: Option<&Object>) -> Result { if nested { return Err("nested indexed colour spaces aren't valid".into()); } - let base = space(doc, a.get(1))?; + let base = space(doc, a.get(1), budget)?; let hival = match a.get(2).map(|value| doc.resolve(value)).as_deref() { Some(Object::Int(value)) if (0..=255).contains(value) => *value as u8, _ => return Err("an indexed colour space with an invalid hival".into()), @@ -478,10 +481,14 @@ fn space(doc: &Document, cs: Option<&Object>) -> Result { let table = match a.get(3).map(|t| doc.resolve(t)).as_deref() { Some(Object::String(s)) => s.bytes.clone(), // Through its parameters given by reference too, as the image's samples. - Some(Object::Stream(t)) => doc - .with_direct_parms(t) - .decoded_within(MAX_LOOKUP) - .map_err(|_| "an indexed colour space with an oversized or damaged lookup table".to_string())?, + Some(Object::Stream(t)) => { + let refused = budget.refused(); + match budget.decode_filters(doc, t, MAX_LOOKUP) { + Ok(table) => table, + Err(_) if budget.refused() > refused => return Err(FilterError::Spent.to_string()), + Err(_) => return Err("an indexed colour space with an oversized or damaged lookup table".into()), + } + } _ => return Err("an indexed colour space without a lookup table".into()), }; Ok(Space::Indexed(Box::new(base), hival, table)) diff --git a/crates/create/src/tests.rs b/crates/create/src/tests.rs index a7a5611ffc..644fed4bdb 100644 --- a/crates/create/src/tests.rs +++ b/crates/create/src/tests.rs @@ -2312,6 +2312,38 @@ fn flate_images_past_the_exports_allowance_are_left_out_with_the_reason() { assert_eq!(budget.refused(), 1); } +/// An indexed image's lookup table, given as a stream, is read within the export's allowance as +/// the image's samples are: each image reads it again, and its filters can do work that makes +/// nothing (Flate over empty blocks). Past the allowance the image is left out with the reason. +#[test] +fn a_lookup_table_is_read_within_the_exports_allowance() { + use pdfcraft_cos::{Dict, Document, ImageBudget, Object, PdfString, Stream}; + let table = [255, 0, 0, 0, 0, 255]; + let lookup = Stream::flate(Dict::new(), &table); + let exported = |as_stream: bool, work: u64| { + let mut doc = Document::new_empty(); + let table = if as_stream { Object::Ref(doc.add(Object::Stream(lookup.clone()))) } else { Object::String(PdfString::literal(table.to_vec())) }; + let mut dict = Dict::new(); + dict.set(b"Subtype".to_vec(), Object::name("Image")); + dict.set(b"Width".to_vec(), Object::Int(2)); + dict.set(b"Height".to_vec(), Object::Int(1)); + dict.set(b"BitsPerComponent".to_vec(), Object::Int(8)); + dict.set(b"ColorSpace".to_vec(), Object::Array(vec![Object::name("Indexed"), Object::name("DeviceRGB"), Object::Int(1), table])); + let image = doc.add(Object::Stream(Stream::flate(dict, &[0, 1]))); + let mut budget = ImageBudget::with_work(work); + let file = crate::extract::image_file_within(&doc, image, &mut budget); + (file, work - budget.work_left(), budget.refused()) + }; + let (direct, image_alone, _) = exported(false, u64::MAX); + let (streamed, both, _) = exported(true, u64::MAX); + assert_eq!(streamed.unwrap(), direct.unwrap(), "the same picture"); + let read = (lookup.raw.len() + table.len()) as u64 + pdfcraft_filters::FLATE_READ_WORK * lookup.raw.len() as u64; + assert_eq!(both, image_alone + read, "the table is paid for as it is read"); + let (refused, _, count) = exported(true, image_alone); + assert!(refused.unwrap_err().contains("more image decoding than")); + assert_eq!(count, 1); +} + /// A JPEG is written as it is, at no cost, unless something means decoding it, here a mask: that /// pays for its samples, and past the allowance the image goes, with the reason, as it would come out without /// its transparency otherwise. diff --git a/crates/filters/README.md b/crates/filters/README.md index 38666e6369..6c831ebe54 100644 --- a/crates/filters/README.md +++ b/crates/filters/README.md @@ -37,7 +37,7 @@ encode_jbig2(data, columns, rows) -> Result, FilterError> // 1-bit row - Image codecs (DCT, JPX, JBIG2, CCITT), `Crypt` and unknown filters anywhere in a chain give `Unsupported`: callers keep such data encoded. So does a chain longer than `MAX_CHAIN` (8) filters, before any stage runs. Decryption belongs to `crypt`. - `max_output` bounds every intermediate buffer (including the pre-predictor buffer) and gives `LimitExceeded(max_output)`. This also holds when Flate retries raw deflate after a misleading zlib header, in both strict and tolerant decoding. - `decode_counted` and `decode_tolerant_counted` are for a caller charging a budget (`pdfcraft_cos::DecodeBudget`). `max_output` bounds all stages together (each stage gets what the earlier ones left), and they give the bytes every stage that ran decoded, on success and on failure. A stage's output counts as its decoder produced it, before a predictor or a later stage shrinks it, and a failed decode pays for what it decoded first. -- `decode_tolerant_within` is for a run of images (`pdfcraft_cos::ImageBudget`), as `decode_within` (its strict form), `decode_jbig2_within` and `decode_ccitt_within` are, all spending from one `work`, in the JBIG2 decoder's steps. `max_output` bounds each stage, as in `decode_tolerant`, and no stage decodes past what `work` has left. It pays a step a byte of `data` first and a step a byte each stage decodes, so a chain that fails has paid for what it did: 0.2 to 3 ns a byte in or out, up to 19 ns through a predictor on samples of a bit, against the 6 to 16 ns of a JBIG2 step. A Flate stage also pays `FLATE_READ_WORK` (128) steps a byte it reads, before it runs, and again if it reads them a second time as raw deflate: inflating does work for every block it reads, and a stream of empty blocks took 1.4 µs a byte to make nothing, about 11 ns a step at that price. What `work` can't pay for spends all of it, and gives `Spent`; with nothing left nothing is read. A chain that can't be decoded at all costs nothing and gives its usual error. +- `decode_tolerant_within` is for a run of images (`pdfcraft_cos::ImageBudget`), as `decode_within` (its strict form), `decode_jbig2_within` and `decode_ccitt_within` are, all spending from one `work`, in the JBIG2 decoder's steps. `max_output` bounds each stage, as in `decode_tolerant`, and no stage decodes past what `work` has left. It pays a step a byte of `data` first and a step a byte each stage decodes, so a chain that fails has paid for what it did: 0.2 to 3 ns a byte in or out, up to 19 ns through a predictor on samples of a bit, against the 6 to 16 ns of a JBIG2 step. A Flate stage also pays `FLATE_READ_WORK` (2) steps a byte it reads, before it runs, and again if it reads them a second time as raw deflate: inflating does work for every block it reads, and a stream of empty dynamic blocks took 21 ns a byte to make nothing, about 7 ns a step at that price. Inflating is flate2's zlib-rs backend on every platform; miniz_oxide, before, took 1.4 µs a byte on empty fixed blocks. What `work` can't pay for spends all of it, and gives `Spent`; with nothing left nothing is read. A chain that can't be decoded at all costs nothing and gives its usual error. - `decode_tolerant` turns `Corrupt` into "what was decoded so far" plus `partial = true`, then runs the rest of the chain on it (predictors are applied to partial output too). `LimitExceeded` and `Unsupported` are still errors. ## Behaviour diff --git a/crates/filters/src/lib.rs b/crates/filters/src/lib.rs index cf10d0586b..5a679d5270 100644 --- a/crates/filters/src/lib.rs +++ b/crates/filters/src/lib.rs @@ -258,15 +258,16 @@ fn spending(chain: &[(Filter, Params)], data: &[u8], max_output: usize, work: &m pub const MAX_CHAIN: usize = 8; /// The steps each byte Flate reads costs where decoding is paid for ([`decode_tolerant_within`], -/// [`decode_within`]), besides a step for each byte it makes. Inflating does work for every block -/// it reads, whether or not the block makes anything: an empty fixed-Huffman block is ten bits -/// (RFC 1951 §3.2.6), and a stream of them took 1,383 ns a byte to make nothing (miniz_oxide -/// 0.8.9, release build), one of a literal each 768 ns a byte, and empty dynamic blocks 106. At -/// this price the worst of these comes to about 11 ns a step, inside what a step of JBIG2 -/// decoding takes. Data a real encoder makes took 0.4 to 3.2 ns for each byte it made, its reading -/// included, so it pays far more than reading takes; each byte of a file still adds 64 times this -/// price to an operation's allowance (8,192 steps, `IMAGE_WORK_PER_BYTE` in `pdfcraft_cos`). -pub const FLATE_READ_WORK: u64 = 1 << 7; +/// [`decode_within`]), besides the step it was paid as input and a step for each byte it makes. +/// Inflating does work for every block it reads, whether or not the block makes anything: an empty +/// fixed-Huffman block is ten bits (RFC 1951 §3.2.6). With flate2's zlib-rs backend (zlib-rs +/// 0.6.8, release build) the slowest blocks measured, empty dynamic ones, took 21 ns a byte read +/// to make nothing, empty fixed ones 9 and fixed ones of a literal each 6, so at this price the +/// worst comes to about 7 ns a step, inside what a step of JBIG2 decoding takes. (miniz_oxide +/// 0.8.9, flate2's backend here before, builds a block's tables for every block, and took 1,383 ns +/// a byte read on empty fixed blocks.) Data a real encoder makes took 0.2 to 2.3 ns a byte made, its +/// reading included, so an image pays twice its stored length more than it would unpriced. +pub const FLATE_READ_WORK: u64 = 2; /// Whether a chain is one [`run`] takes, and if not, the error. fn decodable(chain: &[(Filter, Params)]) -> Result<(), FilterError> { From f8788dff118f2e45a5029904d173ec71ff254a96 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Mon, 2 Nov 2026 23:33:59 +0100 Subject: [PATCH 036/198] M2.6: say what zlib-rs reads differently, and test which backend inflates The Consumers review of the commit before this one: zlib-rs refuses a back-reference to before the start of the output, which miniz_oxide filled with zeros and went on from. Read strictly such a stream is now refused, and read tolerantly what came before it is kept, as zlib has it. flate.rs's quirks and the filters README say so, and a_copy_from_before_the_start_is_damage checks it, which also fails under miniz_oxide, so it shows which backend runs: the read price is set for zlib-rs. The same review: other dependencies still build flate2's default miniz_oxide backend, and zlib-rs runs because flate2 prefers it when both are built, not because it is the only one. The Cargo.toml comment says so. The miniz_oxide measured was 0.9.1, flate2's (0.8.9 is png's); FLATE_READ_WORK's doc says so. The filters README named miniz_oxide as the backend. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- Cargo.toml | 6 ++-- crates/filters/README.md | 6 ++-- crates/filters/src/flate.rs | 67 +++++++++++++++++++++++++++++++++++++ crates/filters/src/lib.rs | 2 +- 4 files changed, 75 insertions(+), 6 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 19714f41a6..51a598d53b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,8 +64,10 @@ pdfcraft-automation = { path = "crates/automation", version = "0.17.0", default- hayro = "0.7.1" hayro-syntax = "0.7.2" lopdf = "0.45" -# zlib-rs, which the Windows build already got through craft-update: miniz_oxide spends about -# 1.7 µs on every deflate block, empty ones included (`pdfcraft_filters::FLATE_READ_WORK`). +# zlib-rs: miniz_oxide spends about 1.7 µs on every deflate block, empty ones included +# (`pdfcraft_filters::FLATE_READ_WORK`). Other dependencies still build flate2's default +# miniz_oxide backend; flate2 prefers zlib-rs when both are built, as the Windows build already did +# through craft-update. `flate::tests::a_copy_from_before_the_start_is_damage` shows which runs. flate2 = { version = "1.1", default-features = false, features = ["zlib-rs"] } weezl = "0.2" # CCITT Group 4 encoding (pdfcraft-filters); already in the tree through tiff. diff --git a/crates/filters/README.md b/crates/filters/README.md index 6c831ebe54..6fb820f8d8 100644 --- a/crates/filters/README.md +++ b/crates/filters/README.md @@ -3,7 +3,7 @@ PDF stream filters: the decoders and encoders behind `/Filter` and `/DecodeParms`. - **Layer:** L0, standalone. No dependency on any other PdfCraft crate. `cos` depends on it (architecture §3 rule 3). -- **Licence:** MIT OR Apache-2.0. Clean-room: written from ISO 32000-2 §7.4, RFC 1950/1951 (through `flate2` with the pure-Rust `miniz_oxide` backend), TIFF 6.0 §14 (predictor 2) and RFC 2083 §6 (PNG filters). CCITT Group 4 encoding (ITU-T T.6) is the `fax` crate's (MIT, already in the tree through `tiff`), checked here by round trips through an independent decoder; CCITT decoding (T.4 and T.6) is the `hayro-ccitt` crate's, and JBIG2 decoding the `hayro-jbig2` crate's (both MIT OR Apache-2.0, already in the tree through hayro), bounded here. JBIG2 generic region encoding is written from the public ISO/IEC 14492 Final Committee Draft (WG1 N1359, 1999); Annex H.2 checks the arithmetic coder byte for byte. +- **Licence:** MIT OR Apache-2.0. Clean-room: written from ISO 32000-2 §7.4, RFC 1950/1951 (through `flate2` with the pure-Rust `zlib-rs` backend), TIFF 6.0 §14 (predictor 2) and RFC 2083 §6 (PNG filters). CCITT Group 4 encoding (ITU-T T.6) is the `fax` crate's (MIT, already in the tree through `tiff`), checked here by round trips through an independent decoder; CCITT decoding (T.4 and T.6) is the `hayro-ccitt` crate's, and JBIG2 decoding the `hayro-jbig2` crate's (both MIT OR Apache-2.0, already in the tree through hayro), bounded here. JBIG2 generic region encoding is written from the public ISO/IEC 14492 Final Committee Draft (WG1 N1359, 1999); Annex H.2 checks the arithmetic coder byte for byte. - `#![forbid(unsafe_code)]`. Builds for `wasm32-unknown-unknown`. @@ -37,14 +37,14 @@ encode_jbig2(data, columns, rows) -> Result, FilterError> // 1-bit row - Image codecs (DCT, JPX, JBIG2, CCITT), `Crypt` and unknown filters anywhere in a chain give `Unsupported`: callers keep such data encoded. So does a chain longer than `MAX_CHAIN` (8) filters, before any stage runs. Decryption belongs to `crypt`. - `max_output` bounds every intermediate buffer (including the pre-predictor buffer) and gives `LimitExceeded(max_output)`. This also holds when Flate retries raw deflate after a misleading zlib header, in both strict and tolerant decoding. - `decode_counted` and `decode_tolerant_counted` are for a caller charging a budget (`pdfcraft_cos::DecodeBudget`). `max_output` bounds all stages together (each stage gets what the earlier ones left), and they give the bytes every stage that ran decoded, on success and on failure. A stage's output counts as its decoder produced it, before a predictor or a later stage shrinks it, and a failed decode pays for what it decoded first. -- `decode_tolerant_within` is for a run of images (`pdfcraft_cos::ImageBudget`), as `decode_within` (its strict form), `decode_jbig2_within` and `decode_ccitt_within` are, all spending from one `work`, in the JBIG2 decoder's steps. `max_output` bounds each stage, as in `decode_tolerant`, and no stage decodes past what `work` has left. It pays a step a byte of `data` first and a step a byte each stage decodes, so a chain that fails has paid for what it did: 0.2 to 3 ns a byte in or out, up to 19 ns through a predictor on samples of a bit, against the 6 to 16 ns of a JBIG2 step. A Flate stage also pays `FLATE_READ_WORK` (2) steps a byte it reads, before it runs, and again if it reads them a second time as raw deflate: inflating does work for every block it reads, and a stream of empty dynamic blocks took 21 ns a byte to make nothing, about 7 ns a step at that price. Inflating is flate2's zlib-rs backend on every platform; miniz_oxide, before, took 1.4 µs a byte on empty fixed blocks. What `work` can't pay for spends all of it, and gives `Spent`; with nothing left nothing is read. A chain that can't be decoded at all costs nothing and gives its usual error. +- `decode_tolerant_within` is for a run of images (`pdfcraft_cos::ImageBudget`), as `decode_within` (its strict form), `decode_jbig2_within` and `decode_ccitt_within` are, all spending from one `work`, in the JBIG2 decoder's steps. `max_output` bounds each stage, as in `decode_tolerant`, and no stage decodes past what `work` has left. It pays a step a byte of `data` first and a step a byte each stage decodes, so a chain that fails has paid for what it did: 0.2 to 3 ns a byte in or out, up to 19 ns through a predictor on samples of a bit, against the 6 to 16 ns of a JBIG2 step. A Flate stage also pays `FLATE_READ_WORK` (2) steps a byte it reads, before it runs, and again if it reads them a second time as raw deflate: inflating does work for every block it reads, and a stream of empty dynamic blocks took 21 ns a byte to make nothing, about 7 ns a step at that price. flate2 inflates with its zlib-rs backend, which it prefers to miniz_oxide when other dependencies build that too; miniz_oxide, its backend here before, took 1.4 µs a byte on empty fixed blocks. What `work` can't pay for spends all of it, and gives `Spent`; with nothing left nothing is read. A chain that can't be decoded at all costs nothing and gives its usual error. - `decode_tolerant` turns `Corrupt` into "what was decoded so far" plus `partial = true`, then runs the rest of the chain on it (predictors are applied to partial output too). `LimitExceeded` and `Unsupported` are still errors. ## Behaviour | Filter | Decode | Encode | |---|---|---| -| FlateDecode | zlib header checked, then inflated raw: a missing or wrong Adler-32 is tolerated; no valid header → raw deflate; trailing bytes ignored; preset dictionary → Corrupt; truncated → Corrupt (partial in tolerant mode) | zlib level 6 | +| FlateDecode | zlib header checked, then inflated raw: a missing or wrong Adler-32 is tolerated; no valid header → raw deflate; trailing bytes ignored; preset dictionary → Corrupt; truncated, or a back-reference to before the start of the output → Corrupt (partial in tolerant mode) | zlib level 6 | | LZWDecode | 9–12-bit MSB-first codes, clear (256), EOD (257), `EarlyChange` 0/1; missing EOD/initial clear tolerated; a full table stops growing | greedy, leading clear, clear-table when the table fills, EOD | | Predictors (Flate/LZW) | 2 = TIFF (1/2/4/8/16 bpc), 10–15 = PNG with per-row tags 0–4; partial final rows decoded; bad params or tags → Corrupt | 2, 10 None, 11 Sub, 12 Up, 13 Average, 14 Paeth, 15 per-row optimum (min sum of signed residuals) | | ASCIIHexDecode | white space ignored, odd digit padded with 0, `>` EOD (optional), other characters → Corrupt | upper case, newline every 64 digits, `>` | diff --git a/crates/filters/src/flate.rs b/crates/filters/src/flate.rs index ce57121f4a..c013b709da 100644 --- a/crates/filters/src/flate.rs +++ b/crates/filters/src/flate.rs @@ -5,6 +5,10 @@ //! deflate stream is inflated raw after the 2-byte zlib header; //! - streams without a valid zlib header are inflated as raw deflate; //! - data after the end of the deflate stream is ignored. +//! +//! A back-reference to before the start of the output points at nothing, and is damage, as zlib +//! has it: read strictly the stream is refused, and read tolerantly what came before it is kept. +//! (miniz_oxide, flate2's backend here before zlib-rs, copied zeros and went on.) use std::io::Write; @@ -195,4 +199,67 @@ mod tests { fn dec_limit(d: &[u8], max: usize) -> Result, FilterError> { decode(d, max, Some(max)).0.map_err(|f| f.error) } + + /// Deflate's bits: header fields and extra bits least significant first, Huffman codes most + /// significant first (RFC 1951 §3.1.1). + #[derive(Default)] + struct Bits { + out: Vec, + acc: u32, + n: u32, + } + + impl Bits { + fn put(&mut self, value: u32, len: u32) { + for i in 0..len { + self.acc |= ((value >> i) & 1) << self.n; + self.n += 1; + if self.n == 8 { + self.out.push(self.acc as u8); + (self.acc, self.n) = (0, 0); + } + } + } + + fn code(&mut self, code: u32, len: u32) { + for i in (0..len).rev() { + self.put((code >> i) & 1, 1); + } + } + + fn finish(mut self) -> Vec { + if self.n > 0 { + self.out.push(self.acc as u8); + } + self.out + } + } + + /// A back-reference to before the start of the output points at nothing: read strictly the + /// stream is refused, and what came before it is what a tolerant reading keeps. miniz_oxide, + /// flate2's backend here before zlib-rs, copied zeros and went on, so this also shows which + /// backend inflates (the price of reading Flate, `FLATE_READ_WORK`, is set for zlib-rs). + #[test] + fn a_copy_from_before_the_start_is_damage() { + let literal = |bits: &mut Bits, b: u8| bits.code(0x30 + u32::from(b), 8); + let mut bits = Bits::default(); + bits.put(1, 1); // the final block, + bits.put(1, 2); // fixed Huffman codes (§3.2.6) + literal(&mut bits, b'A'); + literal(&mut bits, b'B'); + bits.code(1, 7); // length 3 + bits.code(4, 5); // distance 5 (code 4 and an extra bit of 0), three bytes before the start + bits.put(0, 1); + literal(&mut bits, b'C'); + bits.code(0, 7); // end of block + let data = bits.finish(); + assert!(zlib_header(&data).is_none(), "read as raw deflate"); + match decode(&data, 1 << 20, Some(1 << 20)).0 { + Err(f) => { + assert!(matches!(f.error, FilterError::Corrupt { .. }), "{}", f.error); + assert_eq!(f.partial, b"AB"); + } + Ok(out) => panic!("read as {out:?}"), + } + } } diff --git a/crates/filters/src/lib.rs b/crates/filters/src/lib.rs index 5a679d5270..f8b073f967 100644 --- a/crates/filters/src/lib.rs +++ b/crates/filters/src/lib.rs @@ -264,7 +264,7 @@ pub const MAX_CHAIN: usize = 8; /// 0.6.8, release build) the slowest blocks measured, empty dynamic ones, took 21 ns a byte read /// to make nothing, empty fixed ones 9 and fixed ones of a literal each 6, so at this price the /// worst comes to about 7 ns a step, inside what a step of JBIG2 decoding takes. (miniz_oxide -/// 0.8.9, flate2's backend here before, builds a block's tables for every block, and took 1,383 ns +/// 0.9.1, flate2's backend here before, builds a block's tables for every block, and took 1,383 ns /// a byte read on empty fixed blocks.) Data a real encoder makes took 0.2 to 2.3 ns a byte made, its /// reading included, so an image pays twice its stored length more than it would unpriced. pub const FLATE_READ_WORK: u64 = 2; From 05300ea5bc389d312c169c6f5327233be40989ed Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Mon, 2 Nov 2026 23:58:17 +0100 Subject: [PATCH 037/198] M2.6: say what was measured, not why, and test the other block zlib-rs refuses The Claims review of the branch: the docs said miniz_oxide "builds a block's tables for every block", which a timing doesn't show. They now say what was timed: about 1.7 us for every fixed-Huffman block, empty or not. "The worst comes to about 7 ns a step" was per byte read; a stage's start, about 0.25 us under zlib-rs (1.6 us for eight over an empty stream, release), isn't priced, since every way to repeat one takes bytes of the file. The letterhead test's doc promised more than it checks. The same review found a second shape zlib-rs refuses where miniz_oxide read on: a dynamic block whose code has no end-of-block symbol. flate.rs's quirks and the README row say so, and a_dynamic_code_without_an_end_is_damage checks that nothing of it is kept. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- Cargo.toml | 2 +- crates/cos/src/budget.rs | 3 ++- crates/filters/README.md | 2 +- crates/filters/src/flate.rs | 41 ++++++++++++++++++++++++++++++++++++- crates/filters/src/lib.rs | 10 +++++---- 5 files changed, 50 insertions(+), 8 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 51a598d53b..b73a85f1de 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,7 +64,7 @@ pdfcraft-automation = { path = "crates/automation", version = "0.17.0", default- hayro = "0.7.1" hayro-syntax = "0.7.2" lopdf = "0.45" -# zlib-rs: miniz_oxide spends about 1.7 µs on every deflate block, empty ones included +# zlib-rs: miniz_oxide took about 1.7 µs for every fixed-Huffman block, empty or not # (`pdfcraft_filters::FLATE_READ_WORK`). Other dependencies still build flate2's default # miniz_oxide backend; flate2 prefers zlib-rs when both are built, as the Windows build already did # through craft-update. `flate::tests::a_copy_from_before_the_start_is_damage` shows which runs. diff --git a/crates/cos/src/budget.rs b/crates/cos/src/budget.rs index ef68bfffdf..7f37a39fb0 100644 --- a/crates/cos/src/budget.rs +++ b/crates/cos/src/budget.rs @@ -594,7 +594,8 @@ mod image_tests { /// What Flate reads is priced for blocks that make nothing, and an image an encoder made pays /// little for it: a scan-like picture stored in under half its size costs no more than half /// again its bytes, samples and pixels. Redaction decodes a picture once for each page that draws it, so - /// a letterhead on hundreds of pages has to fit as it did unpriced. + /// a letterhead on hundreds of pages has to fit about as often as it did unpriced (here, at + /// most half again the cost). #[test] fn an_image_an_encoder_made_pays_little_for_its_reading() { let doc = Document::new_empty(); diff --git a/crates/filters/README.md b/crates/filters/README.md index 6fb820f8d8..2aab578686 100644 --- a/crates/filters/README.md +++ b/crates/filters/README.md @@ -44,7 +44,7 @@ encode_jbig2(data, columns, rows) -> Result, FilterError> // 1-bit row | Filter | Decode | Encode | |---|---|---| -| FlateDecode | zlib header checked, then inflated raw: a missing or wrong Adler-32 is tolerated; no valid header → raw deflate; trailing bytes ignored; preset dictionary → Corrupt; truncated, or a back-reference to before the start of the output → Corrupt (partial in tolerant mode) | zlib level 6 | +| FlateDecode | zlib header checked, then inflated raw: a missing or wrong Adler-32 is tolerated; no valid header → raw deflate; trailing bytes ignored; preset dictionary → Corrupt; truncated, a back-reference to before the start of the output, or a dynamic block without an end-of-block code → Corrupt (partial in tolerant mode) | zlib level 6 | | LZWDecode | 9–12-bit MSB-first codes, clear (256), EOD (257), `EarlyChange` 0/1; missing EOD/initial clear tolerated; a full table stops growing | greedy, leading clear, clear-table when the table fills, EOD | | Predictors (Flate/LZW) | 2 = TIFF (1/2/4/8/16 bpc), 10–15 = PNG with per-row tags 0–4; partial final rows decoded; bad params or tags → Corrupt | 2, 10 None, 11 Sub, 12 Up, 13 Average, 14 Paeth, 15 per-row optimum (min sum of signed residuals) | | ASCIIHexDecode | white space ignored, odd digit padded with 0, `>` EOD (optional), other characters → Corrupt | upper case, newline every 64 digits, `>` | diff --git a/crates/filters/src/flate.rs b/crates/filters/src/flate.rs index c013b709da..ab7c501488 100644 --- a/crates/filters/src/flate.rs +++ b/crates/filters/src/flate.rs @@ -8,7 +8,8 @@ //! //! A back-reference to before the start of the output points at nothing, and is damage, as zlib //! has it: read strictly the stream is refused, and read tolerantly what came before it is kept. -//! (miniz_oxide, flate2's backend here before zlib-rs, copied zeros and went on.) +//! (miniz_oxide, flate2's backend here before zlib-rs, copied zeros and went on.) So is a dynamic +//! block whose code has no end-of-block symbol, and nothing of that block is kept. use std::io::Write; @@ -235,6 +236,44 @@ mod tests { } } + /// A dynamic block whose literal and length code has no end-of-block symbol can't end (RFC 1951 + /// §3.2.5): it is refused, and nothing of it is kept. + #[test] + fn a_dynamic_code_without_an_end_is_damage() { + let mut bits = Bits::default(); + bits.put(1, 1); // the final block, + bits.put(2, 2); // dynamic Huffman codes (§3.2.7) + bits.put(0, 5); // 257 literal and length codes + bits.put(0, 5); // 1 distance code + bits.put(14, 4); // 18 code length codes, in their order up to the one for 1 + for symbol in [16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1] { + bits.put(u32::from(symbol == 18 || symbol == 1), 3); + } + // Code length codes: 1 is 0, 18 is 1. 'A' and 'B' get a bit each, 256 none. + let zeros = |bits: &mut Bits, n: u32| { + bits.code(1, 1); + bits.put(n - 11, 7); + }; + zeros(&mut bits, 65); + bits.code(0, 1); + bits.code(0, 1); + zeros(&mut bits, 138); + zeros(&mut bits, 52); + bits.code(0, 1); // the distance code: a bit + for _ in 0..4 { + bits.code(0, 1); // 'A' + } + let data = bits.finish(); + assert!(zlib_header(&data).is_none(), "read as raw deflate"); + match decode(&data, 1 << 20, Some(1 << 20)).0 { + Err(f) => { + assert!(matches!(f.error, FilterError::Corrupt { .. }), "{}", f.error); + assert_eq!(f.partial, b""); + } + Ok(out) => panic!("read as {out:?}"), + } + } + /// A back-reference to before the start of the output points at nothing: read strictly the /// stream is refused, and what came before it is what a tolerant reading keeps. miniz_oxide, /// flate2's backend here before zlib-rs, copied zeros and went on, so this also shows which diff --git a/crates/filters/src/lib.rs b/crates/filters/src/lib.rs index f8b073f967..4b1e0ce7cb 100644 --- a/crates/filters/src/lib.rs +++ b/crates/filters/src/lib.rs @@ -263,10 +263,12 @@ pub const MAX_CHAIN: usize = 8; /// fixed-Huffman block is ten bits (RFC 1951 §3.2.6). With flate2's zlib-rs backend (zlib-rs /// 0.6.8, release build) the slowest blocks measured, empty dynamic ones, took 21 ns a byte read /// to make nothing, empty fixed ones 9 and fixed ones of a literal each 6, so at this price the -/// worst comes to about 7 ns a step, inside what a step of JBIG2 decoding takes. (miniz_oxide -/// 0.9.1, flate2's backend here before, builds a block's tables for every block, and took 1,383 ns -/// a byte read on empty fixed blocks.) Data a real encoder makes took 0.2 to 2.3 ns a byte made, its -/// reading included, so an image pays twice its stored length more than it would unpriced. +/// worst per byte read comes to about 7 ns a step, inside what a step of JBIG2 decoding takes. +/// (miniz_oxide 0.9.1, flate2's backend here before, took about 1.7 µs for every fixed-Huffman +/// block, empty or not: 1,383 ns a byte read on empty ones.) A stage's start isn't priced, about +/// 0.25 µs: every way to repeat one takes bytes of the file, so what starts cost is bounded by the +/// file's length. Data a real encoder makes took 0.2 to 2.3 ns a byte made, its reading included, so +/// an image pays twice its stored length more than it would unpriced. pub const FLATE_READ_WORK: u64 = 2; /// Whether a chain is one [`run`] takes, and if not, the error. From 0c9cd267f502f8de80f8ffc87648a5b7ac44a05f Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Tue, 3 Nov 2026 00:43:53 +0100 Subject: [PATCH 038/198] M2.6: a JPEG's colour space is read once for an export CodeRabbit on #5825: a JPEG that isn't inverted read its colour space to know whether it is CMYK, and then, with a soft mask, a stencil mask or a colour key keeping it from being written as it is, read it again for its samples. An indexed one (not valid, written as it is in the end) paid for its lookup table twice, and could be refused with an allowance that covers the image and one reading. The first reading is kept and used. a_jpegs_lookup_table_is_paid_for_once: an indexed JPEG with a soft mask pays one reading of its table, and is written as it is. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/create/src/extract.rs | 11 +++++++--- crates/create/src/tests.rs | 40 ++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/crates/create/src/extract.rs b/crates/create/src/extract.rs index 8fc7fea984..863ad2da0a 100644 --- a/crates/create/src/extract.rs +++ b/crates/create/src/extract.rs @@ -183,12 +183,17 @@ fn image(doc: &Document, s: &Stream, budget: &mut ImageBudget) -> Result<(&'stat .unwrap_or(false); // In a PDF a CMYK JPEG's samples are ink amounts unless /Decode inverts them, while image // viewers read a CMYK JPEG file as inverted (Adobe's convention): as is, it would show as - // a negative. - let as_is_negative = jpeg && !inverted && matches!(space(doc, s.dict.get(b"ColorSpace"), budget), Ok(Space::Cmyk)); + // a negative. The space is read once: an indexed one's lookup table is paid for each time. + let read = (jpeg && !inverted).then(|| space(doc, s.dict.get(b"ColorSpace"), budget)); + let as_is_negative = matches!(read, Some(Ok(Space::Cmyk))); if jpeg && alpha.is_none() && key.is_none() && !as_is_negative { return as_is(); } - let mut space = if mask { Space::Gray } else { space(doc, s.dict.get(b"ColorSpace"), budget)? }; + let mut space = match read { + _ if mask => Space::Gray, + Some(read) => read?, + None => space(doc, s.dict.get(b"ColorSpace"), budget)?, + }; let (samples, bpc) = if jpeg { if !budget.pay_jpeg((w as u64).saturating_mul(h as u64).saturating_mul(space.components() as u64), &s.raw) { return Err(FilterError::Spent.to_string()); diff --git a/crates/create/src/tests.rs b/crates/create/src/tests.rs index 644fed4bdb..f701463eae 100644 --- a/crates/create/src/tests.rs +++ b/crates/create/src/tests.rs @@ -2344,6 +2344,46 @@ fn a_lookup_table_is_read_within_the_exports_allowance() { assert_eq!(count, 1); } +/// A JPEG's colour space is read once for an export, its lookup table paid for once: whether it is +/// CMYK (a negative as is) and what its samples are both come from that reading. An indexed JPEG +/// isn't valid, and is written as it is, but its soft mask makes it go on past the first. +#[test] +fn a_jpegs_lookup_table_is_paid_for_once() { + use pdfcraft_cos::{Dict, Document, ImageBudget, Object, PdfString, Stream}; + let table = [255, 0, 0, 0, 0, 255]; + let lookup = Stream::flate(Dict::new(), &table); + let mut jpeg = Vec::new(); + image::GrayImage::from_pixel(2, 1, image::Luma([1])).write_to(&mut std::io::Cursor::new(&mut jpeg), image::ImageFormat::Jpeg).unwrap(); + let spent = |as_stream: bool| { + let mut doc = Document::new_empty(); + let table = if as_stream { Object::Ref(doc.add(Object::Stream(lookup.clone()))) } else { Object::String(PdfString::literal(table.to_vec())) }; + let mut mask = Dict::new(); + for (k, v) in + [("Subtype", Object::name("Image")), ("Width", Object::Int(2)), ("Height", Object::Int(1)), ("BitsPerComponent", Object::Int(8))] + { + mask.set(k.as_bytes().to_vec(), v); + } + mask.set(b"ColorSpace".to_vec(), Object::name("DeviceGray")); + let mask = doc.add(Object::Stream(Stream::flate(mask, &[255, 128]))); + let mut dict = Dict::new(); + for (k, v) in + [("Subtype", Object::name("Image")), ("Width", Object::Int(2)), ("Height", Object::Int(1)), ("BitsPerComponent", Object::Int(8))] + { + dict.set(k.as_bytes().to_vec(), v); + } + dict.set(b"ColorSpace".to_vec(), Object::Array(vec![Object::name("Indexed"), Object::name("DeviceRGB"), Object::Int(1), table])); + dict.set(b"Filter".to_vec(), Object::name("DCTDecode")); + dict.set(b"SMask".to_vec(), Object::Ref(mask)); + let image = doc.add(Object::Stream(Stream::from_raw(dict, jpeg.clone()))); + let mut budget = ImageBudget::with_work(u64::MAX); + let file = crate::extract::image_file_within(&doc, image, &mut budget); + assert_eq!(file, Ok(("jpg", jpeg.clone())), "written as it is"); + u64::MAX - budget.work_left() + }; + let read = (lookup.raw.len() + table.len()) as u64 + pdfcraft_filters::FLATE_READ_WORK * lookup.raw.len() as u64; + assert_eq!(spent(true), spent(false) + read, "one reading of the table"); +} + /// A JPEG is written as it is, at no cost, unless something means decoding it, here a mask: that /// pays for its samples, and past the allowance the image goes, with the reason, as it would come out without /// its transparency otherwise. From a2b2bc5e6da2ce10ccdb07bcf5abcb4103bf0e5a Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Tue, 3 Nov 2026 06:37:59 +0100 Subject: [PATCH 039/198] M2.6: test that a JPEG refused at its lookup table is left out CodeRabbit's follow-up on #5825: the indexed JPEG test ran only with an unlimited allowance, so nothing checked that a refused lookup-table read leaves a JPEG out (`a_lookup_table_is_read_within_the_exports_allowance` checks a Flate image). The test now measures what the soft mask costs alone (no colour space, so nothing is read after it) and allows a step less than the mask and the table: the export is refused with the reason, and only the table is refused, nothing after it. A step short of the mask, the mask alone is refused, so the mask is read first and the case above is refused at the table. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/create/src/tests.rs | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/crates/create/src/tests.rs b/crates/create/src/tests.rs index f701463eae..8c7db432fe 100644 --- a/crates/create/src/tests.rs +++ b/crates/create/src/tests.rs @@ -2346,7 +2346,8 @@ fn a_lookup_table_is_read_within_the_exports_allowance() { /// A JPEG's colour space is read once for an export, its lookup table paid for once: whether it is /// CMYK (a negative as is) and what its samples are both come from that reading. An indexed JPEG -/// isn't valid, and is written as it is, but its soft mask makes it go on past the first. +/// isn't valid, and is written as it is, but its soft mask makes it go on past the first. With the +/// mask, a table the allowance can't pay for leaves the image out, with the reason. #[test] fn a_jpegs_lookup_table_is_paid_for_once() { use pdfcraft_cos::{Dict, Document, ImageBudget, Object, PdfString, Stream}; @@ -2354,9 +2355,12 @@ fn a_jpegs_lookup_table_is_paid_for_once() { let lookup = Stream::flate(Dict::new(), &table); let mut jpeg = Vec::new(); image::GrayImage::from_pixel(2, 1, image::Luma([1])).write_to(&mut std::io::Cursor::new(&mut jpeg), image::ImageFormat::Jpeg).unwrap(); - let spent = |as_stream: bool| { + // `given`: None leaves the colour space out, Some(true) gives the table as a stream. + let exported = |given: Option, work: u64| { let mut doc = Document::new_empty(); - let table = if as_stream { Object::Ref(doc.add(Object::Stream(lookup.clone()))) } else { Object::String(PdfString::literal(table.to_vec())) }; + let table = given.map(|as_stream| { + if as_stream { Object::Ref(doc.add(Object::Stream(lookup.clone()))) } else { Object::String(PdfString::literal(table.to_vec())) } + }); let mut mask = Dict::new(); for (k, v) in [("Subtype", Object::name("Image")), ("Width", Object::Int(2)), ("Height", Object::Int(1)), ("BitsPerComponent", Object::Int(8))] @@ -2371,17 +2375,33 @@ fn a_jpegs_lookup_table_is_paid_for_once() { { dict.set(k.as_bytes().to_vec(), v); } - dict.set(b"ColorSpace".to_vec(), Object::Array(vec![Object::name("Indexed"), Object::name("DeviceRGB"), Object::Int(1), table])); + if let Some(table) = table { + dict.set(b"ColorSpace".to_vec(), Object::Array(vec![Object::name("Indexed"), Object::name("DeviceRGB"), Object::Int(1), table])); + } dict.set(b"Filter".to_vec(), Object::name("DCTDecode")); dict.set(b"SMask".to_vec(), Object::Ref(mask)); let image = doc.add(Object::Stream(Stream::from_raw(dict, jpeg.clone()))); - let mut budget = ImageBudget::with_work(u64::MAX); + let mut budget = ImageBudget::with_work(work); let file = crate::extract::image_file_within(&doc, image, &mut budget); - assert_eq!(file, Ok(("jpg", jpeg.clone())), "written as it is"); - u64::MAX - budget.work_left() + (file, work - budget.work_left(), budget.refused()) }; + let (direct, direct_cost, _) = exported(Some(false), u64::MAX); + let (streamed, streamed_cost, _) = exported(Some(true), u64::MAX); + assert_eq!(direct, Ok(("jpg", jpeg.clone())), "written as it is"); + assert_eq!(streamed, direct); let read = (lookup.raw.len() + table.len()) as u64 + pdfcraft_filters::FLATE_READ_WORK * lookup.raw.len() as u64; - assert_eq!(spent(true), spent(false) + read, "one reading of the table"); + assert_eq!(streamed_cost, direct_cost + read, "one reading of the table"); + + // Without a colour space nothing is read after the soft mask, so this is what the mask costs. + let (none, mask, _) = exported(None, u64::MAX); + assert_eq!(none.unwrap_err(), "the image has no colour space"); + // Read first, the table would be refused a step short of the mask too, and the mask after it. + assert!(read >= mask, "the table costs at least what the mask does"); + assert_eq!(exported(Some(true), mask - 1).2, 1, "the mask is read first, and refused alone"); + let (refused, _, count) = exported(Some(true), mask + read - 1); + let refused = refused.unwrap_err(); + assert!(refused.contains("more image decoding than"), "{refused}"); + assert_eq!(count, 1, "refused at the table, and nothing after it asks to be paid for"); } /// A JPEG is written as it is, at no cost, unless something means decoding it, here a mask: that From fbd2813d7a95be479ab4efb0ee56f29d3136c1b9 Mon Sep 17 00:00:00 2001 From: Andrew Date: Mon, 2 Nov 2026 06:19:05 -0700 Subject: [PATCH 040/198] M5.7: the Fill & Sign text controls use a colour dot and a size flyout The quick bar is too narrow for the "Font size:" label, number box and "Colour" text, which wrapped mid-word. They now work like Draw's controls: a divider, a colour dot that opens the swatches, and an icon that opens a Font size flyout with a slider. Picking a colour also recolours the selected Fill & Sign text, as Draw's colour does for its own marks. --- crates/ui-egui/src/hotbar.rs | 90 +++++++++++-------- crates/ui-egui/src/tools/comments.rs | 2 +- crates/ui-egui/src/tools/comments/menu.rs | 29 +++--- crates/ui-egui/tests/it/fill_sign.rs | 103 +++++++++++++++++++++- 4 files changed, 173 insertions(+), 51 deletions(-) diff --git a/crates/ui-egui/src/hotbar.rs b/crates/ui-egui/src/hotbar.rs index c66b7e30b0..636414502c 100644 --- a/crates/ui-egui/src/hotbar.rs +++ b/crates/ui-egui/src/hotbar.rs @@ -287,6 +287,30 @@ fn core_tool_buttons(ui: &mut egui::Ui, app: &mut PdfCraftApp, cell: f32, popup_ }); } +fn controls_divider(ui: &mut egui::Ui, t: &Tokens, vertical: bool) { + let (r, _) = if vertical { + ui.allocate_exact_size(vec2(crate::canvas::bars::QUICK_BAR_WIDTH, 9.0), Sense::hover()) + } else { + ui.allocate_exact_size(vec2(9.0, crate::canvas::bars::QUICK_BAR_WIDTH), Sense::hover()) + }; + if vertical { + ui.painter().hline(r.x_range().shrink(6.0), r.center().y, Stroke::new(1.0, t.divider)); + } else { + ui.painter().vline(r.center().x, r.y_range().shrink(6.0), Stroke::new(1.0, t.divider)); + } +} + +fn selected_fill_text(app: &PdfCraftApp) -> Option<(usize, usize)> { + let (index, id) = app.active_ids()?; + let (page, annotation) = app.views.get(index)?.comments.selected?; + let doc = app.session.get(id).filter(|doc| doc.allows_annotation())?; + doc.info + .annotations + .iter() + .any(|a| a.page == page && a.index == annotation && a.fill_sign && a.subtype == "FreeText" && !a.locked) + .then_some((page, annotation)) +} + /// Extra controls rendered below/beside the buttons when a comment or fill tool is active. fn active_tool_controls(ui: &mut egui::Ui, app: &mut PdfCraftApp, t: &Tokens, vertical: bool) { let current_fill = match app.quick_tool { @@ -294,47 +318,37 @@ fn active_tool_controls(ui: &mut egui::Ui, app: &mut PdfCraftApp, t: &Tokens, ve _ => None, }; if matches!(current_fill, Some(crate::fill_sign::FillTool::Text | crate::fill_sign::FillTool::Date)) { - if vertical { - ui.label(tl!("Font size:")); - } - let size = ui.add( - egui::DragValue::new(&mut app.fill_text_size) - .range(crate::fill_sign::MIN_TEXT_SIZE..=crate::fill_sign::MAX_TEXT_SIZE) - .speed(0.25) - .suffix(" pt"), - ); - if ((size.changed() && !size.dragged()) || size.drag_stopped()) - && let Some((index, id)) = app.active_ids() - && let Some((page, annotation)) = app.views.get(index).and_then(|view| view.comments.selected) - && app.session.get(id).is_some_and(|doc| { - doc.allows_annotation() - && doc - .info - .annotations - .iter() - .any(|a| a.page == page && a.index == annotation && a.fill_sign && a.subtype == "FreeText" && !a.locked) - }) - { - app.apply_edit(Edit::FontSizeAnnotation { page, index: annotation, size: app.fill_text_size }); - } - ui.menu_button(tl!("Colour"), |ui| { - if let Some(color) = comments::swatch_grid(ui, Some(app.fill_text_color)) { - app.fill_text_color = color; - ui.close(); + controls_divider(ui, t, vertical); + if let Some(color) = comments::quick_bar_colour(ui, app.fill_text_color) { + app.fill_text_color = color; + if let Some((page, index)) = selected_fill_text(app) { + app.apply_edit(Edit::StyleAnnotation { page, index, color: Some(color), opacity: None, width: None, endings: None }); } - }); + } + let resp = icons::button(ui, "sliders-horizontal", 32.0, false, tl!("Font size")); + let resized = egui::Popup::menu(&resp) + .align(egui::RectAlign::RIGHT_START) + .close_behavior(egui::PopupCloseBehavior::CloseOnClickOutside) + .show(|ui| { + ui.set_min_width(180.0); + ui.label(egui::RichText::new(tl!("Font size")).font(theme::semibold(12.0))); + let size = ui.add( + egui::Slider::new(&mut app.fill_text_size, crate::fill_sign::MIN_TEXT_SIZE..=crate::fill_sign::MAX_TEXT_SIZE) + .logarithmic(true) + .suffix(" pt"), + ); + // #1274: the size also applies to the Fill & Sign text that is selected, + // once per change (a typed value, or the end of a drag), so a drag adds + // one undo step and not one per frame. + (size.changed() && !size.dragged()) || size.drag_stopped() + }) + .is_some_and(|popup| popup.inner); + if resized && let Some((page, index)) = selected_fill_text(app) { + app.apply_edit(Edit::FontSizeAnnotation { page, index, size: app.fill_text_size }); + } } if let QuickTool::Comment(tool) = app.quick_tool { - let (r, _) = if vertical { - ui.allocate_exact_size(vec2(crate::canvas::bars::QUICK_BAR_WIDTH, 9.0), Sense::hover()) - } else { - ui.allocate_exact_size(vec2(9.0, crate::canvas::bars::QUICK_BAR_WIDTH), Sense::hover()) - }; - if vertical { - ui.painter().hline(r.x_range().shrink(6.0), r.center().y, Stroke::new(1.0, t.divider)); - } else { - ui.painter().vline(r.center().x, r.y_range().shrink(6.0), Stroke::new(1.0, t.divider)); - } + controls_divider(ui, t, vertical); if let Some(color) = comments::quick_bar_controls(ui, tool, &mut app.comment_prefs) && let Some((index, id)) = app.active_ids() && let Some((page, annotation)) = app.views.get(index).and_then(|view| view.comments.selected) diff --git a/crates/ui-egui/src/tools/comments.rs b/crates/ui-egui/src/tools/comments.rs index cb8bafe642..95053a16fe 100644 --- a/crates/ui-egui/src/tools/comments.rs +++ b/crates/ui-egui/src/tools/comments.rs @@ -41,7 +41,7 @@ pub(crate) use input::{describe, fill_grabs, hit, keys, nudge_keys, page_after_t pub use menu::{CanvasAction, fill_picker, recent_custom_colours, remember_custom_colour, status_badge, swatch_grid}; #[cfg(test)] use menu::{RECENT_CUSTOM, same_colour}; -pub(crate) use menu::{attachment_items, context_menu, quick_bar_controls}; +pub(crate) use menu::{attachment_items, context_menu, quick_bar_colour, quick_bar_controls}; pub(crate) use paint::paint_page; #[cfg(test)] use paint::{changing_points, drawn_shape, edited_in_view}; diff --git a/crates/ui-egui/src/tools/comments/menu.rs b/crates/ui-egui/src/tools/comments/menu.rs index b577fbbd29..0e0da5caf3 100644 --- a/crates/ui-egui/src/tools/comments/menu.rs +++ b/crates/ui-egui/src/tools/comments/menu.rs @@ -286,25 +286,32 @@ pub fn fill_picker(ui: &mut egui::Ui, current: Option) -> Option Option { - let style = prefs.style(tool); - let mut selected_color = None; - if icons::button(ui, "pin", 32.0, prefs.pinned, if prefs.pinned { tl!("Keep tool selected: on") } else { tl!("Keep tool selected") }).clicked() { - prefs.pinned = !prefs.pinned; - } +pub(crate) fn quick_bar_colour(ui: &mut egui::Ui, current: Rgb) -> Option { let (r, resp) = ui.allocate_exact_size(vec2(32.0, 32.0), Sense::click()); - ui.painter().circle_filled(r.center(), 9.0, color32(style.color)); + ui.painter().circle_filled(r.center(), 9.0, color32(current)); ui.painter().circle_stroke(r.center(), 9.0, Stroke::new(1.0, Color32::from_black_alpha(50))); resp.widget_info(|| egui::WidgetInfo::labeled(egui::Role::Button, true, tl!("Colour"))); let resp = resp.on_hover_text(tl!("Colour")); + let mut picked = None; egui::Popup::menu(&resp).align(egui::RectAlign::RIGHT_START).close_behavior(egui::PopupCloseBehavior::CloseOnClickOutside).show(|ui| { - if let Some(c) = swatch_grid(ui, Some(style.color)) { - prefs.set_color(tool, c); - selected_color = Some(c); + if let Some(c) = swatch_grid(ui, Some(current)) { + picked = Some(c); ui.close(); } }); + picked +} + +/// The comment tools' extra quick-bar controls: pin, colour, fill, opacity and thickness. +pub(crate) fn quick_bar_controls(ui: &mut egui::Ui, tool: CommentTool, prefs: &mut CommentPrefs) -> Option { + let style = prefs.style(tool); + if icons::button(ui, "pin", 32.0, prefs.pinned, if prefs.pinned { tl!("Keep tool selected: on") } else { tl!("Keep tool selected") }).clicked() { + prefs.pinned = !prefs.pinned; + } + let selected_color = quick_bar_colour(ui, style.color); + if let Some(c) = selected_color { + prefs.set_color(tool, c); + } if tool.has_fill() { let resp = fill_button(ui, style.fill); egui::Popup::menu(&resp).align(egui::RectAlign::RIGHT_START).close_behavior(egui::PopupCloseBehavior::CloseOnClickOutside).show(|ui| { diff --git a/crates/ui-egui/tests/it/fill_sign.rs b/crates/ui-egui/tests/it/fill_sign.rs index ce21bd9f74..979ba0ff97 100644 --- a/crates/ui-egui/tests/it/fill_sign.rs +++ b/crates/ui-egui/tests/it/fill_sign.rs @@ -164,7 +164,7 @@ fn fill_text_size_controls_new_text_and_survives_settings_restore() { let mut h = harness(); h.state_mut().execute("sign.fill.text"); h.run_steps(2); - h.get_by_label("Font size:"); + h.get_by_label("Font size"); h.state_mut().fill_text_size = 24.0; click(&mut h, 50.0, 250.0); assert_eq!(h.state().views[0].fill_text.as_ref().map(|t| t.size), Some(24.0)); @@ -201,6 +201,8 @@ fn fill_text_size_resizes_the_selected_text() { }; let before = height(&h); let set_size = |h: &mut Harness<'static, PdfCraftApp>, value: &str| { + h.get_by_role_and_label(egui::accesskit::Role::Button, "Font size").click(); + h.run_steps(2); h.query_all_by_role(egui::accesskit::Role::SpinButton) .find(|n| n.value().is_some_and(|v| v.ends_with(" pt"))) .expect("the Font size control") @@ -211,6 +213,8 @@ fn fill_text_size_resizes_the_selected_text() { h.run_steps(1); h.key_press(egui::Key::Enter); h.run_steps(4); + h.get_by_role_and_label(egui::accesskit::Role::Button, "Font size").click(); + h.run_steps(2); }; // Nothing selected: only the size for new text changes. h.state_mut().views[0].comments.selected = None; @@ -1812,3 +1816,100 @@ fn mode_tabs_keep_the_hand_over_a_type_box_scrolled_under_them() { h.run_steps(3); assert_eq!(h.state().mode, pdfcraft_ui_egui::Mode::Convert, "the tab takes the click, not the text box"); } + +fn place_selected_text(h: &mut Harness<'static, PdfCraftApp>, text: &str) { + h.state_mut().execute("sign.fill.text"); + click(h, 50.0, 250.0); + h.event(egui::Event::Text(text.into())); + h.run_steps(1); + h.key_press(egui::Key::Enter); + h.run_steps(3); + let selected = { + let s = h.state(); + s.session + .get(s.views[0].id) + .unwrap() + .info + .annotations + .iter() + .find(|a| a.contents.as_deref() == Some(text)) + .map(|a| (a.page, a.index)) + .unwrap() + }; + h.state_mut().views[0].comments.selected = Some(selected); + h.run_steps(2); +} + +fn placed_height(h: &Harness<'static, PdfCraftApp>, text: &str) -> f32 { + let s = h.state(); + let a = s.session.get(s.views[0].id).unwrap().info.annotations.iter().find(|a| a.contents.as_deref() == Some(text)).unwrap(); + a.rect[3] - a.rect[1] +} + +fn placed_color(h: &Harness<'static, PdfCraftApp>, text: &str) -> Option<[f32; 3]> { + let s = h.state(); + s.session.get(s.views[0].id).unwrap().info.annotations.iter().find(|a| a.contents.as_deref() == Some(text)).unwrap().color +} + +#[test] +fn the_fill_text_size_flyout_opens_from_an_icon_and_leaves_a_stored_size_alone() { + let mut h = harness(); + h.state_mut().fill_text_size = 9.1; + place_selected_text(&mut h, "Keep me"); + let height = placed_height(&h, "Keep me"); + assert_eq!(h.query_all_by_role(egui::accesskit::Role::SpinButton).filter(|n| n.value().is_some_and(|v| v.ends_with(" pt"))).count(), 0); + h.get_by_label("Font size").click(); + h.run_steps(4); + assert!( + h.query_all_by_role(egui::accesskit::Role::SpinButton).any(|n| n.value().is_some_and(|v| v.ends_with(" pt"))), + "the flyout holds the size" + ); + assert_eq!(h.state().fill_text_size, 9.1, "opening the flyout does not round the size"); + assert_eq!(placed_height(&h, "Keep me"), height, "opening the flyout does not resize the selected text"); +} + +#[test] +fn dragging_the_fill_text_size_slider_is_one_undo_step() { + let mut h = harness(); + place_selected_text(&mut h, "Drag me"); + let height = placed_height(&h, "Drag me"); + h.get_by_label("Font size").click(); + h.run_steps(2); + let slider = h.get_by_role(egui::accesskit::Role::Slider).rect(); + let y = (slider.min.y + slider.max.y) / 2.0; + let from = pos2(slider.min.x + 10.0, y); + let to = pos2(slider.max.x - 10.0, y); + h.hover_at(from); + h.run_steps(1); + h.drag_at(from); + h.run_steps(1); + for k in 1..=4 { + h.hover_at(from + (to - from) * (k as f32 / 4.0)); + h.run_steps(1); + } + assert_eq!(placed_height(&h, "Drag me"), height, "nothing is applied before the release"); + h.drop_at(to); + h.run_steps(3); + assert!(placed_height(&h, "Drag me") > height, "the release resizes the selected text"); + h.state_mut().execute("edit.undo"); + h.run_steps(3); + assert_eq!(placed_height(&h, "Drag me"), height, "one undo takes the whole drag back"); +} + +#[test] +fn the_fill_colour_dot_recolours_the_selected_text() { + let mut h = harness(); + place_selected_text(&mut h, "Paint me"); + let before = placed_color(&h, "Paint me"); + h.get_by_label("Colour").click(); + h.run_steps(2); + h.get_by_label("Blue").click(); + h.run_steps(3); + let color = h.state().fill_text_color.map(|v| v as f32); + assert!(color[2] > color[0], "blue was selected: {color:?}"); + assert_ne!(before, Some(color)); + assert_eq!(placed_color(&h, "Paint me"), Some(color)); + h.state_mut().execute("edit.undo"); + h.run_steps(3); + assert_eq!(placed_color(&h, "Paint me"), before, "one undo takes the colour back"); +} From b3047998125db72655e9b9441a7cfc5f29c4dba6 Mon Sep 17 00:00:00 2001 From: Andrew Date: Mon, 2 Nov 2026 06:37:33 -0700 Subject: [PATCH 041/198] M5.7: the Fill & Sign Font size button uses Lucide's a-large-small icon The sliders icon is also Draw's Line thickness and doesn't say "text". Adds one icon to the Lucide set, from lucide-static 1.49.0, with its attribution. --- ATTRIBUTION.md | 3 ++- ATTRIBUTION.toml | 11 +++++++++++ assets/icons/a-large-small.svg | 18 ++++++++++++++++++ crates/ui-egui/src/hotbar.rs | 2 +- crates/ui-egui/src/widgets/icon_data.rs | 1 + crates/ui-egui/src/widgets/icons.rs | 2 +- 6 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 assets/icons/a-large-small.svg diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index fc88e8b664..6327de11ca 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -4,7 +4,7 @@ Every asset PdfCraft includes, bundles or uses to build its published material, with its author, source and licence. The policy is in [AGENTS.md](AGENTS.md) §1. The machine-readable list, with the SHA-256 of every asset but the translation catalogs, is [ATTRIBUTION.toml](ATTRIBUTION.toml). Licence texts are kept beside the assets and summarised in [NOTICE](NOTICE). -## In this repository (289) +## In this repository (290) | Asset | Title | Author | Licence | Source | Used for | |---|---|---|---|---|---| @@ -293,6 +293,7 @@ Every asset PdfCraft includes, bundles or uses to build its published material, | `assets/icons/superscript.svg` | Lucide icon "superscript" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/superscript.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/subscript.svg` | Lucide icon "subscript" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/subscript.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/pilcrow.svg` | Lucide icon "pilcrow" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/pilcrow.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | +| `assets/icons/a-large-small.svg` | Lucide icon "a-large-small" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/a-large-small.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/move-horizontal.svg` | Lucide icon "move-horizontal" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/move-horizontal.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `crates/ui-egui/src/i18n/sq.tsv` | Albanian interface translation catalog | PdfCraft contributors | MIT OR Apache-2.0 | Contributor-original: clean-room translations of PdfCraft's English UI labels, no proprietary localisation resources; 2026-10-22 page-gap labels are contributor-original AI-assisted translations from English, not separately native-reviewed; 2026-10-22 signature-appearance labels are contributor-original AI-assisted translations from English, not separately native-reviewed; 2026-10-22 text-field-table labels are contributor-original AI-assisted translations from English, not separately native-reviewed; 2026-10-22 secured-document notice labels are contributor-original AI-assisted translations from English, not separately native-reviewed; the 2026-10-22 reply-count label is an AI-assisted translation from English, not separately native-reviewed | Albanian UI strings, compiled in by crates/ui-egui/src/i18n.rs | | `crates/ui-egui/src/i18n/th.tsv` | Thai interface translation catalog | PdfCraft contributors | MIT OR Apache-2.0 | Contributor-original: clean-room translations of PdfCraft's English UI labels, no proprietary localisation resources; 2026-10-22 page-gap labels are contributor-original AI-assisted translations from English, not separately native-reviewed; 2026-10-22 secured-document notice labels are contributor-original AI-assisted translations from English, not separately native-reviewed; the 2026-10-22 reply-count label is an AI-assisted translation from English, not separately native-reviewed | Thai UI strings, compiled in by crates/ui-egui/src/i18n.rs | diff --git a/ATTRIBUTION.toml b/ATTRIBUTION.toml index 76f4cf40a8..7c27c8c85d 100644 --- a/ATTRIBUTION.toml +++ b/ATTRIBUTION.toml @@ -3489,6 +3489,17 @@ kind = "icon" usage = "UI icon, embedded via crates/ui-egui/src/icon_data.rs" sha256 = "d344d9f9222f6c1b698c0c82a753a2212d9bd7785f06c00cd6e4910fe2fee8f8" +[[asset]] +path = "assets/icons/a-large-small.svg" +title = "Lucide icon \"a-large-small\"" +author = "Lucide Contributors (portions Cole Bemis, Feather)" +source = "https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/a-large-small.svg)" +licence = "ISC AND MIT" +licence_file = "assets/icons/LICENSE-lucide.txt" +kind = "icon" +usage = "UI icon, embedded via crates/ui-egui/src/icon_data.rs" +sha256 = "d4985fc71174d108e322c3b25cb6abe2837a115fe3f239a1918645db436743ec" + [[asset]] path = "assets/icons/move-horizontal.svg" title = "Lucide icon \"move-horizontal\"" diff --git a/assets/icons/a-large-small.svg b/assets/icons/a-large-small.svg new file mode 100644 index 0000000000..7716db3cf2 --- /dev/null +++ b/assets/icons/a-large-small.svg @@ -0,0 +1,18 @@ + + + + + + + diff --git a/crates/ui-egui/src/hotbar.rs b/crates/ui-egui/src/hotbar.rs index 636414502c..a6bf36ec9f 100644 --- a/crates/ui-egui/src/hotbar.rs +++ b/crates/ui-egui/src/hotbar.rs @@ -325,7 +325,7 @@ fn active_tool_controls(ui: &mut egui::Ui, app: &mut PdfCraftApp, t: &Tokens, ve app.apply_edit(Edit::StyleAnnotation { page, index, color: Some(color), opacity: None, width: None, endings: None }); } } - let resp = icons::button(ui, "sliders-horizontal", 32.0, false, tl!("Font size")); + let resp = icons::button(ui, "a-large-small", 32.0, false, tl!("Font size")); let resized = egui::Popup::menu(&resp) .align(egui::RectAlign::RIGHT_START) .close_behavior(egui::PopupCloseBehavior::CloseOnClickOutside) diff --git a/crates/ui-egui/src/widgets/icon_data.rs b/crates/ui-egui/src/widgets/icon_data.rs index 72829fe4b8..d58ddcbab6 100644 --- a/crates/ui-egui/src/widgets/icon_data.rs +++ b/crates/ui-egui/src/widgets/icon_data.rs @@ -187,6 +187,7 @@ pub static ICONS: &[(&str, &[u8])] = &[ ("wrench", include_bytes!("../../../../assets/icons/wrench.svg")), ("move-horizontal", include_bytes!("../../../../assets/icons/move-horizontal.svg")), ("pilcrow", include_bytes!("../../../../assets/icons/pilcrow.svg")), + ("a-large-small", include_bytes!("../../../../assets/icons/a-large-small.svg")), ("subscript", include_bytes!("../../../../assets/icons/subscript.svg")), ("superscript", include_bytes!("../../../../assets/icons/superscript.svg")), ("x", include_bytes!("../../../../assets/icons/x.svg")), diff --git a/crates/ui-egui/src/widgets/icons.rs b/crates/ui-egui/src/widgets/icons.rs index 5ad5425e8d..13f50a0d6c 100644 --- a/crates/ui-egui/src/widgets/icons.rs +++ b/crates/ui-egui/src/widgets/icons.rs @@ -99,7 +99,7 @@ mod tests { #[test] fn shared_catalog_preserves_existing_icons_and_fallback_uri() { - assert_eq!(ICONS.len(), 190); + assert_eq!(ICONS.len(), 191); assert!(exists("minimize-2")); for (name, original) in ICONS { let expected = From 84aa947c0f97feb51dbef78b66a402d6aab071fa Mon Sep 17 00:00:00 2001 From: Andrew Date: Mon, 2 Nov 2026 07:08:41 -0700 Subject: [PATCH 042/198] M5.7: the quick bar's comments no longer say Fill & Sign widens it The Text and Date controls now fit the 32 pt column, so the bar stays at its narrowest with every tool. The centring still measures the rendered width. --- crates/ui-egui/src/canvas/bars.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/ui-egui/src/canvas/bars.rs b/crates/ui-egui/src/canvas/bars.rs index dd3fe8c498..fbd4200217 100644 --- a/crates/ui-egui/src/canvas/bars.rs +++ b/crates/ui-egui/src/canvas/bars.rs @@ -315,7 +315,7 @@ mod tests { #[test] fn a_wider_quick_bar_centres_by_its_own_width_and_needs_a_wider_margin() { - // Fill & Sign's Text tool widens the bar to 70 pt: it centres by that, and 72 pt no longer holds it. + // A 70 pt bar centres by that, and 72 pt no longer holds it. assert_eq!(quick_bar_offset(120.0, 70.0), 15.0); assert_eq!(quick_bar_offset(72.0, 70.0), 14.0); } From 546a9d90a5572b2216143ac6a0328dcaedd463ff Mon Sep 17 00:00:00 2001 From: Andrew Date: Mon, 2 Nov 2026 07:28:12 -0700 Subject: [PATCH 043/198] M5.7: the a-large-small attribution names where the icon is embedded icon_data.rs moved to crates/ui-egui/src/widgets/. ATTRIBUTION.md is regenerated with cargo xtask assets --write. --- ATTRIBUTION.md | 2 +- ATTRIBUTION.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index 6327de11ca..a3855b5c70 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -293,7 +293,7 @@ Every asset PdfCraft includes, bundles or uses to build its published material, | `assets/icons/superscript.svg` | Lucide icon "superscript" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/superscript.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/subscript.svg` | Lucide icon "subscript" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/subscript.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/pilcrow.svg` | Lucide icon "pilcrow" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/pilcrow.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | -| `assets/icons/a-large-small.svg` | Lucide icon "a-large-small" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/a-large-small.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | +| `assets/icons/a-large-small.svg` | Lucide icon "a-large-small" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/a-large-small.svg) | UI icon, embedded via crates/ui-egui/src/widgets/icon_data.rs | | `assets/icons/move-horizontal.svg` | Lucide icon "move-horizontal" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/move-horizontal.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `crates/ui-egui/src/i18n/sq.tsv` | Albanian interface translation catalog | PdfCraft contributors | MIT OR Apache-2.0 | Contributor-original: clean-room translations of PdfCraft's English UI labels, no proprietary localisation resources; 2026-10-22 page-gap labels are contributor-original AI-assisted translations from English, not separately native-reviewed; 2026-10-22 signature-appearance labels are contributor-original AI-assisted translations from English, not separately native-reviewed; 2026-10-22 text-field-table labels are contributor-original AI-assisted translations from English, not separately native-reviewed; 2026-10-22 secured-document notice labels are contributor-original AI-assisted translations from English, not separately native-reviewed; the 2026-10-22 reply-count label is an AI-assisted translation from English, not separately native-reviewed | Albanian UI strings, compiled in by crates/ui-egui/src/i18n.rs | | `crates/ui-egui/src/i18n/th.tsv` | Thai interface translation catalog | PdfCraft contributors | MIT OR Apache-2.0 | Contributor-original: clean-room translations of PdfCraft's English UI labels, no proprietary localisation resources; 2026-10-22 page-gap labels are contributor-original AI-assisted translations from English, not separately native-reviewed; 2026-10-22 secured-document notice labels are contributor-original AI-assisted translations from English, not separately native-reviewed; the 2026-10-22 reply-count label is an AI-assisted translation from English, not separately native-reviewed | Thai UI strings, compiled in by crates/ui-egui/src/i18n.rs | diff --git a/ATTRIBUTION.toml b/ATTRIBUTION.toml index 7c27c8c85d..f287a4aede 100644 --- a/ATTRIBUTION.toml +++ b/ATTRIBUTION.toml @@ -3497,7 +3497,7 @@ source = "https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icon licence = "ISC AND MIT" licence_file = "assets/icons/LICENSE-lucide.txt" kind = "icon" -usage = "UI icon, embedded via crates/ui-egui/src/icon_data.rs" +usage = "UI icon, embedded via crates/ui-egui/src/widgets/icon_data.rs" sha256 = "d4985fc71174d108e322c3b25cb6abe2837a115fe3f239a1918645db436743ec" [[asset]] From 0a8aa374eb1aba1bc9f87b7ec7c554e52437cbc3 Mon Sep 17 00:00:00 2001 From: Andrew Date: Mon, 2 Nov 2026 10:15:33 -0700 Subject: [PATCH 044/198] M5.7: pressing the Fill & Sign size slider without moving it adds no undo step The slider senses drags only, so a press and release counts as a finished drag even when the value did not move. The edit is now skipped when the selected text already has that size, compared to the three decimals the size is saved with. --- crates/ui-egui/src/hotbar.rs | 11 +++++- crates/ui-egui/tests/it/fill_sign.rs | 56 ++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/crates/ui-egui/src/hotbar.rs b/crates/ui-egui/src/hotbar.rs index a6bf36ec9f..328f85e706 100644 --- a/crates/ui-egui/src/hotbar.rs +++ b/crates/ui-egui/src/hotbar.rs @@ -343,8 +343,15 @@ fn active_tool_controls(ui: &mut egui::Ui, app: &mut PdfCraftApp, t: &Tokens, ve (size.changed() && !size.dragged()) || size.drag_stopped() }) .is_some_and(|popup| popup.inner); - if resized && let Some((page, index)) = selected_fill_text(app) { - app.apply_edit(Edit::FontSizeAnnotation { page, index, size: app.fill_text_size }); + let size = app.fill_text_size; + if resized + && let Some((page, index)) = selected_fill_text(app) + && app + .active_ids() + .and_then(|(_, id)| app.session.get(id)?.comment_props(page, index)?.font_size) + .is_none_or(|stored| format!("{stored:.3}") != format!("{size:.3}")) + { + app.apply_edit(Edit::FontSizeAnnotation { page, index, size }); } } if let QuickTool::Comment(tool) = app.quick_tool { diff --git a/crates/ui-egui/tests/it/fill_sign.rs b/crates/ui-egui/tests/it/fill_sign.rs index 979ba0ff97..931018214a 100644 --- a/crates/ui-egui/tests/it/fill_sign.rs +++ b/crates/ui-egui/tests/it/fill_sign.rs @@ -1896,6 +1896,62 @@ fn dragging_the_fill_text_size_slider_is_one_undo_step() { assert_eq!(placed_height(&h, "Drag me"), height, "one undo takes the whole drag back"); } +#[test] +fn pressing_the_fill_text_size_slider_without_moving_it_adds_no_undo_step() { + let mut h = harness(); + place_selected_text(&mut h, "Hold me"); + let height = placed_height(&h, "Hold me"); + h.get_by_label("Font size").click(); + h.run_steps(2); + let slider = h.get_by_role(egui::accesskit::Role::Slider).rect(); + let at = pos2(slider.min.x + (slider.max.x - slider.min.x) * 0.75, (slider.min.y + slider.max.y) / 2.0); + let press = |h: &mut Harness<'static, PdfCraftApp>| { + h.hover_at(at); + h.run_steps(1); + h.drag_at(at); + h.run_steps(1); + h.drop_at(at); + h.run_steps(3); + }; + press(&mut h); + let resized = placed_height(&h, "Hold me"); + assert!(resized > height, "the first press moves the size: {height} -> {resized}"); + press(&mut h); + assert_eq!(placed_height(&h, "Hold me"), resized); + h.state_mut().execute("edit.undo"); + h.run_steps(3); + assert_eq!(placed_height(&h, "Hold me"), height, "one undo takes back the only change; the second press added no step"); +} + +#[test] +fn typing_a_fill_text_size_that_rounds_to_the_stored_one_adds_no_undo_step() { + let mut h = harness(); + place_selected_text(&mut h, "Round me"); + let height = placed_height(&h, "Round me"); + h.get_by_label("Font size").click(); + h.run_steps(2); + let type_size = |h: &mut Harness<'static, PdfCraftApp>, value: &str| { + h.query_all_by_role(egui::accesskit::Role::SpinButton) + .find(|n| n.value().is_some_and(|v| v.ends_with(" pt"))) + .expect("the Font size control") + .click(); + h.run_steps(2); + h.key_press_modifiers(egui::Modifiers::COMMAND, egui::Key::A); + h.event(egui::Event::Text(value.into())); + h.run_steps(1); + h.key_press(egui::Key::Enter); + h.run_steps(4); + }; + type_size(&mut h, "36.1234"); + let resized = placed_height(&h, "Round me"); + assert!(resized > height, "the first size applies: {height} -> {resized}"); + type_size(&mut h, "36.1231"); + assert_eq!(h.state().fill_text_size, 36.1231); + h.state_mut().execute("edit.undo"); + h.run_steps(3); + assert_eq!(placed_height(&h, "Round me"), height, "one undo takes back the only change; the rounded retype added no step"); +} + #[test] fn the_fill_colour_dot_recolours_the_selected_text() { let mut h = harness(); From e90c92124c1695dc87780f763f70a2556e63b81e Mon Sep 17 00:00:00 2001 From: Andrew Date: Fri, 30 Oct 2026 07:31:31 -0600 Subject: [PATCH 045/198] M5.7: the quick bar's Fill & Sign button shows the tool it picks The button showed a generic pen until clicked, then swapped to Add text's icon. It now works like the Comment, Highlight and Draw groups: it shows the last Fill & Sign tool used (Signature at first) and clicking it picks that tool. The flyout marks the active saved signature or initials card, and the cards get the pointing-hand cursor like the other rows. --- crates/ui-egui/src/app.rs | 1 + crates/ui-egui/src/app/control.rs | 9 ++++-- crates/ui-egui/src/commands.rs | 1 + crates/ui-egui/src/dialogs.rs | 1 + crates/ui-egui/src/hotbar.rs | 16 +++++++++-- crates/ui-egui/src/lib.rs | 1 + crates/ui-egui/src/tools/fill_sign.rs | 13 ++++++--- crates/ui-egui/tests/it/comments.rs | 2 +- crates/ui-egui/tests/it/fill_sign.rs | 40 +++++++++++++++++++++++---- 9 files changed, 69 insertions(+), 15 deletions(-) diff --git a/crates/ui-egui/src/app.rs b/crates/ui-egui/src/app.rs index 6ac8ea8783..bb1748f705 100644 --- a/crates/ui-egui/src/app.rs +++ b/crates/ui-egui/src/app.rs @@ -108,6 +108,7 @@ impl PdfCraftApp { ui_scale_applied: None, flatten_fill_sign_on_save: false, fill_text_size: fill_sign::TEXT_SIZE, + fill_tool: fill_sign::FillTool::Signature, fill_text_color: [0.0; 3], dialog: None, updates: updates::Updates::default(), diff --git a/crates/ui-egui/src/app/control.rs b/crates/ui-egui/src/app/control.rs index 0309042bf1..fee5378b6e 100644 --- a/crates/ui-egui/src/app/control.rs +++ b/crates/ui-egui/src/app/control.rs @@ -333,9 +333,12 @@ impl PdfCraftApp { field if field.starts_with("field-") => QuickTool::Field( prepare::FieldTool::from_command(&format!("form.add.{}", &field[6..])).ok_or_else(|| format!("unknown tool {field}"))?, ), - fill if fill.starts_with("fill-") => QuickTool::Fill( - fill_sign::FillTool::from_command(&format!("sign.fill.{}", &fill[5..])).ok_or_else(|| format!("unknown tool {fill}"))?, - ), + fill if fill.starts_with("fill-") => { + let f = + fill_sign::FillTool::from_command(&format!("sign.fill.{}", &fill[5..])).ok_or_else(|| format!("unknown tool {fill}"))?; + self.fill_tool = f; + QuickTool::Fill(f) + } other => { let t = comments::CommentTool::from_command(&format!("comment.{other}")).ok_or_else(|| format!("unknown tool {other}"))?; self.comment_prefs.group_tool[t.group()] = t; diff --git a/crates/ui-egui/src/commands.rs b/crates/ui-egui/src/commands.rs index f072b7febe..4fdec5af8a 100644 --- a/crates/ui-egui/src/commands.rs +++ b/crates/ui-egui/src/commands.rs @@ -697,6 +697,7 @@ impl PdfCraftApp { } fill if crate::fill_sign::FillTool::from_command(fill).is_some() => { let Some(tool) = crate::fill_sign::FillTool::from_command(fill) else { return Some(false) }; + self.fill_tool = tool; self.quick_tool = crate::QuickTool::Fill(tool); let initials = tool == crate::fill_sign::FillTool::Initials; if (tool == crate::fill_sign::FillTool::Signature && self.signature.is_none()) || (initials && self.initials.is_none()) { diff --git a/crates/ui-egui/src/dialogs.rs b/crates/ui-egui/src/dialogs.rs index 048d519222..8105a22342 100644 --- a/crates/ui-egui/src/dialogs.rs +++ b/crates/ui-egui/src/dialogs.rs @@ -1349,6 +1349,7 @@ fn dialog_signature(ui: &mut egui::Ui, app: &mut PdfCraftApp, req: &mut Pending) app.signature = d.saved(); crate::fill_sign::FillTool::Signature }; + app.fill_tool = tool; app.quick_tool = crate::QuickTool::Fill(tool); app.toast = None; } diff --git a/crates/ui-egui/src/hotbar.rs b/crates/ui-egui/src/hotbar.rs index c66b7e30b0..bbd98be2ef 100644 --- a/crates/ui-egui/src/hotbar.rs +++ b/crates/ui-egui/src/hotbar.rs @@ -239,8 +239,20 @@ fn core_tool_buttons(ui: &mut egui::Ui, app: &mut PdfCraftApp, cell: f32, popup_ QuickTool::Fill(f) => Some(f), _ => None, }; - let shown = current_fill.unwrap_or(crate::fill_sign::FillTool::Text); - let resp = icons::button(ui, if current_fill.is_some() { shown.icon() } else { "pen-line" }, cell, current_fill.is_some(), tl!("Fill & Sign")); + let shown = app.fill_tool; + let tip = match shown { + crate::fill_sign::FillTool::Signature | crate::fill_sign::FillTool::Initials => { + let initials = shown == crate::fill_sign::FillTool::Initials; + let saved = if initials { app.initials.is_some() } else { app.signature.is_some() }; + crate::i18n::fmt( + tl!(if saved { "Place saved {what}" } else { "Add {what}" }), + &[("what", tl!(if initials { "initials" } else { "signature" }))], + ) + } + _ => tl!(shown.label()).to_string(), + }; + let tip = format!("{}: {tip}", tl!("Fill & Sign")); + let resp = icons::button(ui, shown.icon(), cell, current_fill.is_some(), &tip); let r = resp.rect; let tri = [r.right_bottom() + vec2(-4.0, -4.0), r.right_bottom() + vec2(-9.0, -4.0), r.right_bottom() + vec2(-4.0, -9.0)]; ui.painter().add(egui::Shape::convex_polygon(tri.to_vec(), if current_fill.is_some() { t.accent } else { t.text_faint }, Stroke::NONE)); diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 643d5e90ff..ae89f96bcb 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -331,6 +331,7 @@ pub struct PdfCraftApp { pub flatten_fill_sign_on_save: bool, /// Point size for new Fill & Sign text and dates, kept across launches. pub fill_text_size: f64, + pub fill_tool: fill_sign::FillTool, /// Colour for new Fill & Sign text and dates, kept across launches. pub fill_text_color: [f64; 3], pub dialog: Option, diff --git a/crates/ui-egui/src/tools/fill_sign.rs b/crates/ui-egui/src/tools/fill_sign.rs index 045bb64974..9beaa9b834 100644 --- a/crates/ui-egui/src/tools/fill_sign.rs +++ b/crates/ui-egui/src/tools/fill_sign.rs @@ -373,20 +373,22 @@ pub(crate) fn signature_entries(ui: &mut egui::Ui, app: &mut crate::PdfCraftApp, } continue; }; - ui.horizontal(|ui| { + let on = app.quick_tool == crate::QuickTool::Fill(if i == 0 { FillTool::Signature } else { FillTool::Initials }); + let wash = ui.painter().add(egui::Shape::Noop); + let row = ui.horizontal(|ui| { let (rect, response) = ui.allocate_exact_size(vec2((ui.available_width() - 68.0).max(140.0), 52.0), Sense::click()); let use_label = label("Use {what}"); - response.widget_info(|| egui::WidgetInfo::labeled(egui::Role::Button, true, use_label.clone())); + response.widget_info(|| egui::WidgetInfo::selected(egui::Role::Button, true, on, use_label.clone())); let painter = ui.painter_at(rect); painter.rect_filled(rect, CornerRadius::same(4), Color32::WHITE); painter.rect_stroke( rect, CornerRadius::same(4), - Stroke::new(1.0, if response.hovered() { t.accent } else { t.border }), + Stroke::new(1.0, if on || response.hovered() { t.accent } else { t.border }), egui::StrokeKind::Inside, ); paint_saved(ui, rect.shrink(8.0), saved, &mut app.saved_signature_previews[i], &format!("saved-{what}")); - if response.on_hover_text(label("Place saved {what}")).clicked() { + if crate::theme::hand(response).on_hover_text(label("Place saved {what}")).clicked() { command = Some(use_id); } if crate::icons::button(ui, "pencil", 26.0, false, &label("Change {what}")).clicked() { @@ -396,6 +398,9 @@ pub(crate) fn signature_entries(ui: &mut egui::Ui, app: &mut crate::PdfCraftApp, command = Some(remove_id); } }); + if on { + ui.painter().set(wash, egui::Shape::rect_filled(row.response.rect.expand(3.0), CornerRadius::same(6), t.accent_soft)); + } ui.add_space(4.0); } command diff --git a/crates/ui-egui/tests/it/comments.rs b/crates/ui-egui/tests/it/comments.rs index 939de58dff..51b9eabce1 100644 --- a/crates/ui-egui/tests/it/comments.rs +++ b/crates/ui-egui/tests/it/comments.rs @@ -3280,7 +3280,7 @@ fn hotbar_docked_left_does_not_override_quick_tools() { }); h.run_steps(5); // Controls in the docked bar share one layout, so none may overlap (#5890). - let labels = ["Drag hotbar", "Select (V)", "Hand (H)", "Add a sticky note", "Highlight", "Draw", "Fill & Sign", "Hotbar settings"]; + let labels = ["Drag hotbar", "Select (V)", "Hand (H)", "Add a sticky note", "Highlight", "Draw", "Fill & Sign: Add signature", "Hotbar settings"]; let rects: Vec<_> = labels.iter().map(|&label| h.get_by_label(label).rect()).collect(); for (i, a) in rects.iter().enumerate() { for (j, b) in rects.iter().enumerate().skip(i + 1) { diff --git a/crates/ui-egui/tests/it/fill_sign.rs b/crates/ui-egui/tests/it/fill_sign.rs index ce21bd9f74..503080e71c 100644 --- a/crates/ui-egui/tests/it/fill_sign.rs +++ b/crates/ui-egui/tests/it/fill_sign.rs @@ -2,7 +2,7 @@ use egui::{Pos2, pos2}; use egui_kittest::Harness; -use egui_kittest::kittest::Queryable; +use egui_kittest::kittest::{NodeT, Queryable}; use pdfcraft_ui_egui::fill_sign::{FillTool, SavedSig}; use pdfcraft_ui_egui::{Dialog, PdfCraftApp, QuickTool}; @@ -423,9 +423,9 @@ fn changing_saved_signatures_and_initials_preserves_placed_marks() { let original = format!("{:?}", h.state().session.get(h.state().views[0].id).unwrap().info.annotations); // The quick toolbar exposes replacement without changing the normal placement action. - h.get_by_label("Fill & Sign").click(); + h.get_by_label("Fill & Sign: Place saved signature").click(); h.run_steps(2); - h.get_by_label("Fill & Sign").click(); + h.get_by_label("Fill & Sign: Place saved signature").click(); h.run_steps(2); h.get_by_label("Change signature").click(); h.run_steps(2); @@ -445,9 +445,9 @@ fn changing_saved_signatures_and_initials_preserves_placed_marks() { click(&mut h, 40.0, 200.0); assert_eq!(items(&h).len(), 2); - h.get_by_label("Fill & Sign").click(); + h.get_by_label("Fill & Sign: Place saved signature").click(); h.run_steps(2); - h.get_by_label("Fill & Sign").click(); + h.get_by_label("Fill & Sign: Place saved signature").click(); h.run_steps(2); h.get_by_label("Change initials").click(); h.run_steps(2); @@ -1812,3 +1812,33 @@ fn mode_tabs_keep_the_hand_over_a_type_box_scrolled_under_them() { h.run_steps(3); assert_eq!(h.state().mode, pdfcraft_ui_egui::Mode::Convert, "the tab takes the click, not the text box"); } + +#[test] +fn the_quick_bar_fill_button_shows_the_tool_it_picks() { + let mut h = harness(); + h.get_by_label("Fill & Sign: Add signature").click(); + h.run_steps(2); + assert_eq!(h.state().quick_tool, QuickTool::Fill(FillTool::Signature)); + assert_eq!(h.state().dialog, Some(Dialog::Signature)); + + h.state_mut().dialog = None; + h.state_mut().signature = Some(SavedSig::Typed("Ada Lovelace".into())); + h.state_mut().quick_tool = QuickTool::Select; + h.run_steps(2); + h.get_by_label("Fill & Sign: Place saved signature").click(); + h.run_steps(2); + assert_eq!(h.state().quick_tool, QuickTool::Fill(FillTool::Signature)); + assert_eq!(h.state().dialog, None); + h.get_by_label("Fill & Sign: Place saved signature").click(); + h.run_steps(2); + assert_eq!(h.get_by_label("Use signature").accesskit_node().toggled(), Some(egui::accesskit::Toggled::True), "the active tool's card is marked"); + h.key_press(egui::Key::Escape); + h.run_steps(2); + + h.state_mut().execute("sign.fill.check"); + h.state_mut().quick_tool = QuickTool::Select; + h.run_steps(2); + h.get_by_label("Fill & Sign: Checkmark").click(); + h.run_steps(2); + assert_eq!(h.state().quick_tool, QuickTool::Fill(FillTool::Check), "the button keeps the last Fill & Sign tool"); +} From edee301ebf8f719a5574949e17919c1bb8cddd16 Mon Sep 17 00:00:00 2001 From: Andrew Date: Fri, 30 Oct 2026 07:55:28 -0600 Subject: [PATCH 046/198] M5.7: the Fill & Sign button follows the active tool and names it Tools restored without a command (leaving full screen, the temporary tool, the control channel) left the remembered Fill & Sign tool stale, so the button could show Checkmark while a click placed a signature. The bar now takes the remembered tool from the active one, which replaces the dialog's and control channel's own updates. While a Fill & Sign tool is active the button opens the flyout, so its label names the tool instead of describing placing or adding a signature. --- crates/ui-egui/src/app/control.rs | 9 +++------ crates/ui-egui/src/dialogs.rs | 1 - crates/ui-egui/src/hotbar.rs | 5 ++++- crates/ui-egui/tests/it/fill_sign.rs | 14 +++++++++++--- 4 files changed, 18 insertions(+), 11 deletions(-) diff --git a/crates/ui-egui/src/app/control.rs b/crates/ui-egui/src/app/control.rs index fee5378b6e..0309042bf1 100644 --- a/crates/ui-egui/src/app/control.rs +++ b/crates/ui-egui/src/app/control.rs @@ -333,12 +333,9 @@ impl PdfCraftApp { field if field.starts_with("field-") => QuickTool::Field( prepare::FieldTool::from_command(&format!("form.add.{}", &field[6..])).ok_or_else(|| format!("unknown tool {field}"))?, ), - fill if fill.starts_with("fill-") => { - let f = - fill_sign::FillTool::from_command(&format!("sign.fill.{}", &fill[5..])).ok_or_else(|| format!("unknown tool {fill}"))?; - self.fill_tool = f; - QuickTool::Fill(f) - } + fill if fill.starts_with("fill-") => QuickTool::Fill( + fill_sign::FillTool::from_command(&format!("sign.fill.{}", &fill[5..])).ok_or_else(|| format!("unknown tool {fill}"))?, + ), other => { let t = comments::CommentTool::from_command(&format!("comment.{other}")).ok_or_else(|| format!("unknown tool {other}"))?; self.comment_prefs.group_tool[t.group()] = t; diff --git a/crates/ui-egui/src/dialogs.rs b/crates/ui-egui/src/dialogs.rs index 8105a22342..048d519222 100644 --- a/crates/ui-egui/src/dialogs.rs +++ b/crates/ui-egui/src/dialogs.rs @@ -1349,7 +1349,6 @@ fn dialog_signature(ui: &mut egui::Ui, app: &mut PdfCraftApp, req: &mut Pending) app.signature = d.saved(); crate::fill_sign::FillTool::Signature }; - app.fill_tool = tool; app.quick_tool = crate::QuickTool::Fill(tool); app.toast = None; } diff --git a/crates/ui-egui/src/hotbar.rs b/crates/ui-egui/src/hotbar.rs index bbd98be2ef..88c45bbd1e 100644 --- a/crates/ui-egui/src/hotbar.rs +++ b/crates/ui-egui/src/hotbar.rs @@ -239,9 +239,12 @@ fn core_tool_buttons(ui: &mut egui::Ui, app: &mut PdfCraftApp, cell: f32, popup_ QuickTool::Fill(f) => Some(f), _ => None, }; + if let Some(f) = current_fill { + app.fill_tool = f; + } let shown = app.fill_tool; let tip = match shown { - crate::fill_sign::FillTool::Signature | crate::fill_sign::FillTool::Initials => { + crate::fill_sign::FillTool::Signature | crate::fill_sign::FillTool::Initials if current_fill.is_none() => { let initials = shown == crate::fill_sign::FillTool::Initials; let saved = if initials { app.initials.is_some() } else { app.signature.is_some() }; crate::i18n::fmt( diff --git a/crates/ui-egui/tests/it/fill_sign.rs b/crates/ui-egui/tests/it/fill_sign.rs index 503080e71c..bd8bd30939 100644 --- a/crates/ui-egui/tests/it/fill_sign.rs +++ b/crates/ui-egui/tests/it/fill_sign.rs @@ -425,7 +425,7 @@ fn changing_saved_signatures_and_initials_preserves_placed_marks() { // The quick toolbar exposes replacement without changing the normal placement action. h.get_by_label("Fill & Sign: Place saved signature").click(); h.run_steps(2); - h.get_by_label("Fill & Sign: Place saved signature").click(); + h.get_by_label("Fill & Sign: Sign").click(); h.run_steps(2); h.get_by_label("Change signature").click(); h.run_steps(2); @@ -447,7 +447,7 @@ fn changing_saved_signatures_and_initials_preserves_placed_marks() { h.get_by_label("Fill & Sign: Place saved signature").click(); h.run_steps(2); - h.get_by_label("Fill & Sign: Place saved signature").click(); + h.get_by_label("Fill & Sign: Sign").click(); h.run_steps(2); h.get_by_label("Change initials").click(); h.run_steps(2); @@ -1829,7 +1829,7 @@ fn the_quick_bar_fill_button_shows_the_tool_it_picks() { h.run_steps(2); assert_eq!(h.state().quick_tool, QuickTool::Fill(FillTool::Signature)); assert_eq!(h.state().dialog, None); - h.get_by_label("Fill & Sign: Place saved signature").click(); + h.get_by_label("Fill & Sign: Sign").click(); h.run_steps(2); assert_eq!(h.get_by_label("Use signature").accesskit_node().toggled(), Some(egui::accesskit::Toggled::True), "the active tool's card is marked"); h.key_press(egui::Key::Escape); @@ -1841,4 +1841,12 @@ fn the_quick_bar_fill_button_shows_the_tool_it_picks() { h.get_by_label("Fill & Sign: Checkmark").click(); h.run_steps(2); assert_eq!(h.state().quick_tool, QuickTool::Fill(FillTool::Check), "the button keeps the last Fill & Sign tool"); + + h.state_mut().quick_tool = QuickTool::Fill(FillTool::Initials); + h.run_steps(2); + h.state_mut().quick_tool = QuickTool::Select; + h.run_steps(2); + h.get_by_label("Fill & Sign: Add initials").click(); + h.run_steps(2); + assert_eq!(h.state().quick_tool, QuickTool::Fill(FillTool::Initials), "a tool restored without a command still updates the button"); } From 2e699f489a093a4edb8e57cd7f6bd8c10cf455f6 Mon Sep 17 00:00:00 2001 From: Andrew Date: Mon, 2 Nov 2026 05:33:39 -0700 Subject: [PATCH 047/198] M5.7: a test that the Fill & Sign button keeps Date across documents Picks Date from the flyout, deselects it, and checks that the button still shows Date and that a click picks it, in a newly opened document and again after every document has been closed. --- crates/ui-egui/tests/it/fill_sign.rs | 33 ++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/crates/ui-egui/tests/it/fill_sign.rs b/crates/ui-egui/tests/it/fill_sign.rs index bd8bd30939..77e4420ba2 100644 --- a/crates/ui-egui/tests/it/fill_sign.rs +++ b/crates/ui-egui/tests/it/fill_sign.rs @@ -1850,3 +1850,36 @@ fn the_quick_bar_fill_button_shows_the_tool_it_picks() { h.run_steps(2); assert_eq!(h.state().quick_tool, QuickTool::Fill(FillTool::Initials), "a tool restored without a command still updates the button"); } + +#[test] +fn the_quick_bar_fill_button_keeps_date_after_deselecting_and_across_documents() { + let mut h = harness(); + h.get_by_label("Fill & Sign: Add signature").click_secondary(); + h.run_steps(2); + h.get_by_label("Date").click(); + h.run_steps(2); + assert_eq!(h.state().quick_tool, QuickTool::Fill(FillTool::Date)); + h.get_by_label("Select (V)").click(); + h.run_steps(2); + assert_eq!(h.state().quick_tool, QuickTool::Select); + + h.state_mut().open_bytes("second.pdf", None, FIXTURE.to_vec()).unwrap(); + h.state_mut().active = Some(1); + h.run_steps(2); + h.get_by_label("Fill & Sign: Date").click(); + h.run_steps(2); + assert_eq!(h.state().quick_tool, QuickTool::Fill(FillTool::Date), "a click picks the shown tool, not the default"); + assert_eq!(h.state().dialog, None, "the default tool's dialog stays closed"); + h.get_by_label("Select (V)").click(); + h.run_steps(2); + + h.state_mut().close_tab(1); + h.state_mut().close_tab(0); + h.run_steps(2); + h.state_mut().open_bytes("third.pdf", None, FIXTURE.to_vec()).unwrap(); + h.state_mut().active = Some(0); + h.run_steps(2); + h.get_by_label("Fill & Sign: Date").click(); + h.run_steps(2); + assert_eq!(h.state().quick_tool, QuickTool::Fill(FillTool::Date), "the tool outlives closing every document"); +} From 6028bdab7c13b0d11c5495f1eb91b1a10f898f41 Mon Sep 17 00:00:00 2001 From: Andrew Date: Mon, 2 Nov 2026 06:47:43 -0700 Subject: [PATCH 048/198] M5.7: the Fill & Sign button remembers tools where they are selected The quick bar copied the active tool into fill_tool on every draw, so a tool restored on leaving full screen replaced the last selection, and a tool selected while the bar was hidden was forgotten. The signature dialog and the control channel now record the tool when they select it, and the bar only reads: the active tool if there is one, otherwise the remembered one. --- crates/ui-egui/src/app/control.rs | 9 ++++-- crates/ui-egui/src/dialogs.rs | 1 + crates/ui-egui/src/hotbar.rs | 5 +-- crates/ui-egui/tests/it/fill_sign.rs | 46 ++++++++++++++++++++++++++-- 4 files changed, 51 insertions(+), 10 deletions(-) diff --git a/crates/ui-egui/src/app/control.rs b/crates/ui-egui/src/app/control.rs index 0309042bf1..443b970148 100644 --- a/crates/ui-egui/src/app/control.rs +++ b/crates/ui-egui/src/app/control.rs @@ -333,9 +333,12 @@ impl PdfCraftApp { field if field.starts_with("field-") => QuickTool::Field( prepare::FieldTool::from_command(&format!("form.add.{}", &field[6..])).ok_or_else(|| format!("unknown tool {field}"))?, ), - fill if fill.starts_with("fill-") => QuickTool::Fill( - fill_sign::FillTool::from_command(&format!("sign.fill.{}", &fill[5..])).ok_or_else(|| format!("unknown tool {fill}"))?, - ), + fill if fill.starts_with("fill-") => { + let tool = + fill_sign::FillTool::from_command(&format!("sign.fill.{}", &fill[5..])).ok_or_else(|| format!("unknown tool {fill}"))?; + self.fill_tool = tool; + QuickTool::Fill(tool) + } other => { let t = comments::CommentTool::from_command(&format!("comment.{other}")).ok_or_else(|| format!("unknown tool {other}"))?; self.comment_prefs.group_tool[t.group()] = t; diff --git a/crates/ui-egui/src/dialogs.rs b/crates/ui-egui/src/dialogs.rs index 048d519222..8105a22342 100644 --- a/crates/ui-egui/src/dialogs.rs +++ b/crates/ui-egui/src/dialogs.rs @@ -1349,6 +1349,7 @@ fn dialog_signature(ui: &mut egui::Ui, app: &mut PdfCraftApp, req: &mut Pending) app.signature = d.saved(); crate::fill_sign::FillTool::Signature }; + app.fill_tool = tool; app.quick_tool = crate::QuickTool::Fill(tool); app.toast = None; } diff --git a/crates/ui-egui/src/hotbar.rs b/crates/ui-egui/src/hotbar.rs index 88c45bbd1e..2b1ec38a96 100644 --- a/crates/ui-egui/src/hotbar.rs +++ b/crates/ui-egui/src/hotbar.rs @@ -239,10 +239,7 @@ fn core_tool_buttons(ui: &mut egui::Ui, app: &mut PdfCraftApp, cell: f32, popup_ QuickTool::Fill(f) => Some(f), _ => None, }; - if let Some(f) = current_fill { - app.fill_tool = f; - } - let shown = app.fill_tool; + let shown = current_fill.unwrap_or(app.fill_tool); let tip = match shown { crate::fill_sign::FillTool::Signature | crate::fill_sign::FillTool::Initials if current_fill.is_none() => { let initials = shown == crate::fill_sign::FillTool::Initials; diff --git a/crates/ui-egui/tests/it/fill_sign.rs b/crates/ui-egui/tests/it/fill_sign.rs index 77e4420ba2..241726cc9e 100644 --- a/crates/ui-egui/tests/it/fill_sign.rs +++ b/crates/ui-egui/tests/it/fill_sign.rs @@ -1841,14 +1841,54 @@ fn the_quick_bar_fill_button_shows_the_tool_it_picks() { h.get_by_label("Fill & Sign: Checkmark").click(); h.run_steps(2); assert_eq!(h.state().quick_tool, QuickTool::Fill(FillTool::Check), "the button keeps the last Fill & Sign tool"); +} - h.state_mut().quick_tool = QuickTool::Fill(FillTool::Initials); +#[test] +fn a_restored_fill_tool_shows_without_replacing_the_last_selection() { + let mut h = harness(); + h.state_mut().execute("sign.fill.date"); h.run_steps(2); + h.state_mut().apply_full_screen(true); + h.run_steps(2); + h.state_mut().execute("sign.fill.check"); + h.run_steps(2); + h.state_mut().apply_full_screen(false); + h.run_steps(2); + assert_eq!(h.state().quick_tool, QuickTool::Fill(FillTool::Date)); + h.get_by_label("Fill & Sign: Date"); + h.state_mut().quick_tool = QuickTool::Select; h.run_steps(2); - h.get_by_label("Fill & Sign: Add initials").click(); + h.get_by_label("Fill & Sign: Checkmark").click(); h.run_steps(2); - assert_eq!(h.state().quick_tool, QuickTool::Fill(FillTool::Initials), "a tool restored without a command still updates the button"); + assert_eq!(h.state().quick_tool, QuickTool::Fill(FillTool::Check)); +} + +#[test] +fn a_fill_tool_selected_while_the_quick_bar_is_hidden_is_remembered() { + let mut h = harness(); + let mode = h.state().mode; + h.state_mut().set_option("mode", "read").unwrap(); + h.run_steps(2); + h.state_mut().set_option("quick", "fill-date").unwrap(); + h.run_steps(2); + h.state_mut().set_option("quick", "select").unwrap(); + h.state_mut().mode = mode; + h.run_steps(2); + h.get_by_label("Fill & Sign: Date"); + + h.state_mut().initials = Some(SavedSig::Typed("AL".into())); + h.state_mut().set_option("mode", "read").unwrap(); + h.run_steps(2); + h.state_mut().execute("sign.fill.initials.change"); + h.run_steps(2); + h.get_by_label("Apply").click(); + h.run_steps(3); + assert_eq!(h.state().quick_tool, QuickTool::Fill(FillTool::Initials)); + h.state_mut().quick_tool = QuickTool::Select; + h.state_mut().mode = mode; + h.run_steps(2); + h.get_by_label("Fill & Sign: Place saved initials"); } #[test] From bad9a8b44c89cf8384252fcc57a20ad8cf33041b Mon Sep 17 00:00:00 2001 From: KenM76 Date: Tue, 3 Nov 2026 12:47:31 -0500 Subject: [PATCH 049/198] cos: an edited object stream forgets its members' cached failures A member that failed to load from the file's object stream was cached as a failure. Replacing or freeing that stream left the failure in place, so the member kept reading as null instead of from the edit (CodeRabbit on #5802). set and free now drop the cached failures of the stream's members. Cached objects are untouched; a failure holds no bytes, so the cache's total stands. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01SqmHYx8FiWM5PcsoiErgp9 --- crates/cos/src/document/objects.rs | 11 +++++++++++ crates/cos/src/document/state.rs | 5 +++++ crates/cos/src/document/tests.rs | 20 ++++++++++++++++++++ 3 files changed, 36 insertions(+) diff --git a/crates/cos/src/document/objects.rs b/crates/cos/src/document/objects.rs index 96d3a9dee4..619e64eb5f 100644 --- a/crates/cos/src/document/objects.rs +++ b/crates/cos/src/document/objects.rs @@ -642,6 +642,7 @@ impl Document { pub fn set(&mut self, r: ObjRef, obj: impl Into) { let obj = self.adopt(obj.into()); self.overlay.insert(r.num, Slot::Set(r.generation, Arc::new(obj))); + self.forget_member_failures(r.num); if r.num >= self.next_num { self.next_num = r.num.saturating_add(1); } @@ -669,6 +670,16 @@ impl Document { /// Delete an object (its number becomes free with the next generation). pub fn free(&mut self, r: ObjRef) { self.overlay.insert(r.num, Slot::Freed(r.generation.saturating_add(1))); + self.forget_member_failures(r.num); + } + + /// Forget cached failures of the members `stream` held in the file: they were failures of + /// the file's copy, and an edit of `stream` may read. Clones share the cache, so theirs are + /// searched again too, which costs time only. A poisoned lock leaves the cache as it is. + fn forget_member_failures(&self, stream: u32) { + if let Ok(mut cache) = self.cache.lock() { + cache.forget_failures(|n| matches!(self.entries.lookup(*n, &self.data), Some(XrefEntry::InStream { stream: s, .. }) if s == stream)); + } } /// Free what was added since the file was read and nothing refers to any more: the objects diff --git a/crates/cos/src/document/state.rs b/crates/cos/src/document/state.rs index 5f9bb49852..b3e6c1314a 100644 --- a/crates/cos/src/document/state.rs +++ b/crates/cos/src/document/state.rs @@ -250,6 +250,11 @@ impl ObjectCache { self.owned = 0; } + /// Drop the cached failures `stale` picks. A failure holds no bytes, so the total stands. + pub(crate) fn forget_failures(&mut self, mut stale: impl FnMut(&u32) -> bool) { + self.entries.retain(|num, (loaded, _)| loaded.is_ok() || !stale(num)); + } + pub(crate) fn retain(&mut self, mut keep: impl FnMut(&u32) -> bool) { self.entries.retain(|num, _| keep(num)); self.owned = self.entries.values().fold(0usize, |n, (_, owned)| n.saturating_add(*owned)); diff --git a/crates/cos/src/document/tests.rs b/crates/cos/src/document/tests.rs index 3b18521a4e..62481df4bc 100644 --- a/crates/cos/src/document/tests.rs +++ b/crates/cos/src/document/tests.rs @@ -634,6 +634,26 @@ fn an_edited_object_stream_leaves_other_clones_members_alone() { assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("yes"), "the replacement reached another clone"); } +/// A member that failed to load from the file's object stream is kept as a failure, but replacing +/// or freeing that stream forgets it: the member then reads from the edit. +#[test] +fn an_edited_object_stream_forgets_its_members_failures() { + let file = Arc::new(object_stream_file("3 0 ", ") broken", 0)); + let mut doc = Document::open(file.clone()).unwrap(); + assert!(doc.try_get(3).is_err(), "the file's member does not parse"); + let mut dict = Dict::new(); + dict.set(b"Type".to_vec(), Object::name("ObjStm")); + dict.set(b"N".to_vec(), Object::Int(1)); + dict.set(b"First".to_vec(), Object::Int(4)); + doc.set(ObjRef::new(4, 0), crate::Stream::from_raw(dict, b"3 0 << /Hidden (fixed) >>".to_vec())); + assert_eq!(text_of(&doc, 3, b"Hidden").as_deref(), Some("fixed"), "the cached failure outlived the edit"); + assert!(doc.try_get(3).is_ok()); + let mut doc = Document::open(file).unwrap(); + assert!(matches!(doc.try_get(3), Err(CosError::Syntax { .. })), "the file's member does not parse"); + doc.free(ObjRef::new(4, 0)); + assert_eq!(doc.try_get(3), Err(CosError::MissingObject(4)), "the freed stream's member"); +} + /// An object stream framed by one clone's edit of its indirect `/Length` is that clone's alone: /// the others still read its members within the file's length. #[test] From 57ca07dea3b56bfb678f7831bbc11767c03b476f Mon Sep 17 00:00:00 2001 From: KenM76 Date: Tue, 3 Nov 2026 12:51:45 -0500 Subject: [PATCH 050/198] cos: test that a clone's scan reaches an unread original The clone-scan check ran after the original had already listed the object, so it passed whether or not scan findings were shared (CodeRabbit on #5816). A fresh document is now cloned before anything reads it, only the clone is scanned, and the original must list the object. Seen failing with the scan's list unshared. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01SqmHYx8FiWM5PcsoiErgp9 --- crates/cos/src/document/tests.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/cos/src/document/tests.rs b/crates/cos/src/document/tests.rs index 50a69159f1..fda2128ebc 100644 --- a/crates/cos/src/document/tests.rs +++ b/crates/cos/src/document/tests.rs @@ -1727,6 +1727,10 @@ fn unreadable_objects_are_listed_with_the_reason() { let scanned = doc.clone(); assert_eq!(scanned.scan_objects_checked().filter(|(_, o)| o.is_err()).count(), 1); assert_eq!(doc.unreadable_objects(), lines, "the scan found the same object"); + let unread = Document::open(Arc::new(with_unreadable_info())).unwrap(); + let scanned = unread.clone(); + assert_eq!(scanned.scan_objects_checked().filter(|(_, o)| o.is_err()).count(), 1); + assert_eq!(unread.unreadable_objects(), lines, "a clone's scan reaches the original"); let mut edited = doc.clone(); edited.set(ObjRef::new(3, 0), Object::Dict(Dict::new())); assert!(edited.unreadable_objects().is_empty(), "the edit replaced it"); From e0f3afe2e1bb900cb98f2b38a38f589d867b20c9 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Wed, 4 Nov 2026 02:12:24 +0100 Subject: [PATCH 051/198] contributors: set names for vangel76 Co-Authored-By: Claude Sonnet 5.5 --- contributors/people.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/contributors/people.toml b/contributors/people.toml index fecfb68c90..f059791248 100644 --- a/contributors/people.toml +++ b/contributors/people.toml @@ -183,3 +183,8 @@ sync_github_name = true [people.Combiler01] sync_github_name = true + +[people.vangel76] +real_name = "Valentin Klein" +display_name = "vangel76" +sync_github_name = false From 36d24eae6e78b1c7d154eebdf86ae411a98fca4f Mon Sep 17 00:00:00 2001 From: Hasan Date: Mon, 2 Nov 2026 18:04:58 +0500 Subject: [PATCH 052/198] M14: Preferences can turn off the update check at start (#5737, #5460) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The desktop app asks for a newer release on its first frame, and nothing could stop it, though the README says PdfCraft works offline. A preference, on by default so nothing changes for anyone who leaves it, keeps the app off the network until Help ▸ Check for updates is chosen. It has its own settings key: check_updates_at_start was an opt-in, off unless chosen, so settings written then would turn this off unasked. --- ROADMAP.md | 2 + crates/ui-egui/src/app.rs | 1 + crates/ui-egui/src/app/control.rs | 3 + crates/ui-egui/src/app/prefs.rs | 6 ++ crates/ui-egui/src/dialogs/preferences.rs | 2 + crates/ui-egui/src/lib.rs | 3 + crates/ui-egui/src/platform/updates.rs | 6 +- crates/ui-egui/tests/it/updates.rs | 76 +++++++++++++++++++++++ docs/roadmap.md | 1 + parity/acrobat-features.toml | 4 +- 10 files changed, 100 insertions(+), 4 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 5bfe1fc817..0a03866be3 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -231,6 +231,8 @@ Ranked; detail and the full milestone table in [docs/roadmap.md](docs/roadmap.md - **2026-11-02 (M10, #5743):** macOS print jobs name their title and owner (`com.apple.print.JobInfo.PMJobName`, `PMJobOwner`), as the system print dialog does, because a Kyocera filter crashes on a job without them. Checked by a test and through `doc_print` against a stand-in `lp`; not yet on a physical printer. +- **2026-11-02 (M14, #5737, #5460):** Preferences can turn off the update check at start. It stays on by default, and Help ▸ Check for updates still asks. + - **2026-11-01 (M7, #5657):** Header/footer, watermark and background additions and replacements stage the complete call before committing. Synthetic later-page failures preserve exact incremental bytes, object count, dirty state and allocation cursor; success controls retain other marks and shared/inherited resources. Estimates unchanged. - **2026-11-01 (M14, #5572):** Preferences ▸ Documents and view sets how many recent files to keep, from 0 to 50 with 12 by default. At 0 Home has no Recent section and File ▸ Open Recent is disabled. The list is cut when Preferences closes, so the number can pass through 0 on its way to another. diff --git a/crates/ui-egui/src/app.rs b/crates/ui-egui/src/app.rs index 6ac8ea8783..d2cec1a29f 100644 --- a/crates/ui-egui/src/app.rs +++ b/crates/ui-egui/src/app.rs @@ -128,6 +128,7 @@ impl PdfCraftApp { last_session: Default::default(), quit_session: None, reload_changed_files: true, + update_check_at_start: true, show_secured_notice: true, always_default_view: false, #[cfg(not(target_arch = "wasm32"))] diff --git a/crates/ui-egui/src/app/control.rs b/crates/ui-egui/src/app/control.rs index 0309042bf1..403af6f4c6 100644 --- a/crates/ui-egui/src/app/control.rs +++ b/crates/ui-egui/src/app/control.rs @@ -47,6 +47,9 @@ impl PdfCraftApp { ("show_page_edge_shadows", _) => { self.show_page_edge_shadows = value.parse::().map_err(|_| "show_page_edge_shadows must be true or false")?; } + ("update_check_at_start", _) => { + self.update_check_at_start = value.parse::().map_err(|_| "update_check_at_start must be true or false")?; + } ("scroll_speed", _) => { self.scroll_speed = scroll_speed::ScrollSpeed::parse(value).ok_or("scroll_speed must be slow, normal, or fast")?; } diff --git a/crates/ui-egui/src/app/prefs.rs b/crates/ui-egui/src/app/prefs.rs index 08a20afb45..0df7d6f30b 100644 --- a/crates/ui-egui/src/app/prefs.rs +++ b/crates/ui-egui/src/app/prefs.rs @@ -147,6 +147,9 @@ impl PdfCraftApp { // Set outside json! above, which is at serde_json's macro recursion limit. v["hide_empty_highlight_hover"] = serde_json::Value::Bool(self.comment_prefs.hide_empty_highlight_hover); v["show_page_edge_shadows"] = self.show_page_edge_shadows.into(); + // Not `check_updates_at_start`: that key was an opt-in of its own, off unless chosen, and + // settings written then would turn this off unasked. + v["update_check_at_start"] = self.update_check_at_start.into(); v["hotbar"] = serde_json::to_value(&self.hotbar).unwrap_or_default(); v["comment_tool_pinned"] = serde_json::Value::Bool(self.comment_prefs.pinned); v["show_secured_notice"] = serde_json::Value::Bool(self.show_secured_notice); @@ -219,6 +222,9 @@ impl PdfCraftApp { if let Some(on) = v["reload_changed_files"].as_bool() { self.reload_changed_files = on; } + if let Some(on) = v["update_check_at_start"].as_bool() { + self.update_check_at_start = on; + } if let Some(on) = v["show_secured_notice"].as_bool() { self.show_secured_notice = on; } diff --git a/crates/ui-egui/src/dialogs/preferences.rs b/crates/ui-egui/src/dialogs/preferences.rs index 9e3182e416..ea985f3207 100644 --- a/crates/ui-egui/src/dialogs/preferences.rs +++ b/crates/ui-egui/src/dialogs/preferences.rs @@ -659,6 +659,8 @@ fn card_general(ui: &mut egui::Ui, app: &mut PdfCraftApp, t: &Tokens, ctx: &mut ctx.checkbox(ui, &mut app.restore_last_view, tl!("Restore the last view (page, zoom and page display) when a document is opened again"), t); #[cfg(not(target_arch = "wasm32"))] ctx.checkbox(ui, &mut app.reload_changed_files, tl!("Reload documents when their files change on disk"), t); + #[cfg(not(target_arch = "wasm32"))] + ctx.checkbox(ui, &mut app.update_check_at_start, tl!("Check for updates when PdfCraft starts"), t); ctx.checkbox(ui, &mut app.show_secured_notice, tl!("Show a notice when a document is secured"), t); #[cfg(not(target_arch = "wasm32"))] diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 643d5e90ff..cb6c99bbe5 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -370,6 +370,9 @@ pub struct PdfCraftApp { quit_session: Option, /// Preferences: reload a document when its file changes on disk (#431). pub reload_changed_files: bool, + /// Preferences: ask for a newer release when the desktop app starts (#5737). Off, the app + /// asks only from Help ▸ Check for updates. + pub update_check_at_start: bool, /// Preferences: tell the user, above the pages, that a document is secured (its security /// settings restrict changes). Off only hides that notice (#4035); kept in the settings. pub show_secured_notice: bool, diff --git a/crates/ui-egui/src/platform/updates.rs b/crates/ui-egui/src/platform/updates.rs index 6b7ea291fa..622006cd6e 100644 --- a/crates/ui-egui/src/platform/updates.rs +++ b/crates/ui-egui/src/platform/updates.rs @@ -21,7 +21,8 @@ pub(crate) struct Updates { updater: Updater, /// The Updates dialog is showing. pub(crate) open: bool, - /// Ask for a newer release at start; the desktop app turns it on, so tests never reach the network. + /// Ask for a newer release at start, unless Preferences says not to; the desktop app turns + /// it on, so tests never reach the network. #[cfg(not(target_arch = "wasm32"))] startup_check: bool, #[cfg(not(target_arch = "wasm32"))] @@ -73,10 +74,11 @@ impl PdfCraftApp { #[cfg(not(target_arch = "wasm32"))] { let Some(ctx) = self.ctx.clone() else { return }; + let at_start = self.update_check_at_start; let updates = &mut self.updates; if std::mem::take(&mut updates.check_requested) { updates.updater.check_now(&ctx); - } else if updates.startup_check { + } else if updates.startup_check && at_start { updates.updater.check_at_start(&ctx); } updates.updater.tick(&ctx); diff --git a/crates/ui-egui/tests/it/updates.rs b/crates/ui-egui/tests/it/updates.rs index 3ea55e434e..469a42f805 100644 --- a/crates/ui-egui/tests/it/updates.rs +++ b/crates/ui-egui/tests/it/updates.rs @@ -100,6 +100,82 @@ fn nothing_is_asked_until_the_user_checks() { h.get_by_label_contains("PdfCraft 99.0.0 is available"); } +/// The desktop app's start, with `settings` restored, and the count of what the source was asked. +fn started_with(settings: &'static str) -> (Harness<'static, PdfCraftApp>, Arc) { + let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let counted = calls.clone(); + let h = Harness::builder().with_size(egui::vec2(1200.0, 800.0)).build_eframe(move |_cc| { + let mut app = PdfCraftApp::new(); + app.set_option("language", "en").unwrap(); + app.set_startup_update_check(true); + app.restore(settings); + let counted = counted.clone(); + app.set_update_source(Arc::new(move || { + counted.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Ok(Some(release("v99.0.0"))) + })); + app + }); + (h, calls) +} + +/// Run frames for `time`, or until the source has been asked. The check at start shows nothing +/// while it runs, so `settle` can't wait for it. +fn run_until_asked(h: &mut Harness<'static, PdfCraftApp>, calls: &std::sync::atomic::AtomicUsize, time: std::time::Duration) { + let deadline = std::time::Instant::now() + time; + while calls.load(std::sync::atomic::Ordering::SeqCst) == 0 && std::time::Instant::now() < deadline { + h.run_steps(2); + std::thread::sleep(std::time::Duration::from_millis(5)); + } +} + +#[test] +fn the_preference_keeps_the_check_at_start_from_reaching_the_network() { + use std::sync::atomic::Ordering::SeqCst; + assert!(PdfCraftApp::new().update_check_at_start, "on by default"); + + // The same start with the preference left on does ask, so a count of none below means + // the preference held the check back and not that it hadn't run yet. + let (mut h, calls) = started_with("{}"); + run_until_asked(&mut h, &calls, std::time::Duration::from_secs(30)); + assert_eq!(calls.load(SeqCst), 1, "on, the app asks at start"); + + let (mut h, calls) = started_with(r#"{"update_check_at_start": false}"#); + run_until_asked(&mut h, &calls, std::time::Duration::from_millis(500)); + assert_eq!(calls.load(SeqCst), 0, "nothing is asked at start"); + + let mut restarted = PdfCraftApp::new(); + restarted.restore(&h.state().persist()); + assert!(!restarted.update_check_at_start, "the choice is kept"); + + // Help ▸ Check for updates still asks. + h.state_mut().execute("help.check_updates"); + settle(&mut h); + assert_eq!(calls.load(SeqCst), 1); + h.get_by_label_contains("PdfCraft 99.0.0 is available"); +} + +#[test] +fn preferences_turn_the_check_at_start_off() { + let (mut h, _calls) = started_with(r#"{"update_check_at_start": true}"#); + h.state_mut().execute("app.preferences"); + h.run_steps(3); + h.get_by_label("Check for updates when PdfCraft starts").click(); + h.run_steps(3); + assert!(!h.state().update_check_at_start); + // The control channel's option, as an agent sets it. + h.state_mut().set_option("update_check_at_start", "true").unwrap(); + assert!(h.state().update_check_at_start); + assert!(h.state_mut().set_option("update_check_at_start", "maybe").is_err()); +} + +#[test] +fn settings_from_the_old_opt_in_leave_the_check_at_start_on() { + let mut app = PdfCraftApp::new(); + app.restore(r#"{"check_updates_at_start": false}"#); + assert!(app.update_check_at_start, "a key of the removed option decides nothing"); +} + #[test] fn the_check_at_start_offers_the_prompt_and_later_dismisses_it() { let mut h = Harness::builder().with_size(egui::vec2(1200.0, 800.0)).build_eframe(|_cc| { diff --git a/docs/roadmap.md b/docs/roadmap.md index f376f7cb00..1c99dddeb8 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -307,6 +307,7 @@ Estimates unchanged. | 2026-11-03 | patch | M5 #5850 review: indirect typewriter intent and composer edits preserve outer whitespace; movement regression commits by outside click; estimates unchanged. | | 2026-11-03 | patch | M5 #5850: Enter adds Fill & Sign line breaks; outside clicks commit, Escape cancels, and saved multiline appearances retain their text and height; estimates unchanged. | | 2026-11-02 | patch | M10 #5743: macOS print jobs carry the job name and owner some vendors' filters need; not yet checked on a physical printer | +| 2026-11-02 | patch | M14 #5737, #5460: Preferences turns off the update check at start, on by default | | 2026-11-01 | patch | M7 #5657: header/footer, watermark and background operations commit all selected pages together; later-page errors preserve marks, content, shared/inherited resources and pending edits; estimates unchanged. | | 2026-10-31 | patch | #1577: reconcile optional page shadows with current viewer/organizer and settings modules; CI validation pending. | | 2026-10-31 | patch | M3 #1354: reconcile the launch override with split app/file modules and the structured open result; preserve headless help/version exits. CI validation pending. | diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 8b7ac909a0..00b3dae2c2 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -1778,8 +1778,8 @@ milestone = "M3" status = "partial" commands = ["help.check_updates"] tools = [] -evidence = ["crates/ui-egui/tests/it/updates.rs::a_newer_release_is_offered_for_download", "crates/ui-egui/tests/it/updates.rs::an_up_to_date_or_failed_check_says_so", "crates/ui-egui/tests/it/updates.rs::the_check_at_start_offers_the_prompt_and_later_dismisses_it", "crates/ui-egui/tests/it/updates.rs::nothing_is_asked_until_the_user_checks"] -notes = "asks GitHub Releases through craft-update at start (desktop app only) and on request, offers the release page, and updates and restarts in place where the install type allows it; an install waits while documents are unsaved" +evidence = ["crates/ui-egui/tests/it/updates.rs::a_newer_release_is_offered_for_download", "crates/ui-egui/tests/it/updates.rs::an_up_to_date_or_failed_check_says_so", "crates/ui-egui/tests/it/updates.rs::the_check_at_start_offers_the_prompt_and_later_dismisses_it", "crates/ui-egui/tests/it/updates.rs::nothing_is_asked_until_the_user_checks", "crates/ui-egui/tests/it/updates.rs::the_preference_keeps_the_check_at_start_from_reaching_the_network", "crates/ui-egui/tests/it/updates.rs::settings_from_the_old_opt_in_leave_the_check_at_start_on"] +notes = "asks GitHub Releases through craft-update at start (desktop app only) and on request, offers the release page, and updates and restarts in place where the install type allows it; an install waits while documents are unsaved; Preferences ▸ Check for updates when PdfCraft starts (update_check_at_start, on by default) turns the check at start off, leaving only the request from the Help menu" [[feature]] id = "view.single-key-accelerators" From 1c4ef83491d414b09eba4a033836815b066dcd87 Mon Sep 17 00:00:00 2001 From: Hasan Date: Mon, 2 Nov 2026 21:04:08 +0500 Subject: [PATCH 053/198] M14: decide the update check at start once, on the first frame (#5737) Review of #5758: with the preference off, the startup request stayed pending, so turning the preference on later in the session sent a network request at once. The request is now consumed on the first poll whatever the preference says, and a later change waits for the next start. --- crates/ui-egui/src/platform/updates.rs | 5 +++-- crates/ui-egui/tests/it/updates.rs | 11 +++++++++++ parity/acrobat-features.toml | 4 ++-- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/crates/ui-egui/src/platform/updates.rs b/crates/ui-egui/src/platform/updates.rs index 622006cd6e..bcf6783cdb 100644 --- a/crates/ui-egui/src/platform/updates.rs +++ b/crates/ui-egui/src/platform/updates.rs @@ -74,11 +74,12 @@ impl PdfCraftApp { #[cfg(not(target_arch = "wasm32"))] { let Some(ctx) = self.ctx.clone() else { return }; - let at_start = self.update_check_at_start; + // Settled on the first frame: a preference turned on later waits for the next start. + let at_start = std::mem::take(&mut self.updates.startup_check) && self.update_check_at_start; let updates = &mut self.updates; if std::mem::take(&mut updates.check_requested) { updates.updater.check_now(&ctx); - } else if updates.startup_check && at_start { + } else if at_start { updates.updater.check_at_start(&ctx); } updates.updater.tick(&ctx); diff --git a/crates/ui-egui/tests/it/updates.rs b/crates/ui-egui/tests/it/updates.rs index 469a42f805..e6cc09d7c5 100644 --- a/crates/ui-egui/tests/it/updates.rs +++ b/crates/ui-egui/tests/it/updates.rs @@ -155,6 +155,17 @@ fn the_preference_keeps_the_check_at_start_from_reaching_the_network() { h.get_by_label_contains("PdfCraft 99.0.0 is available"); } +/// The setting is about starting: turning it on asks nothing until the next start. +#[test] +fn turning_the_check_at_start_on_waits_for_the_next_start() { + use std::sync::atomic::Ordering::SeqCst; + let (mut h, calls) = started_with(r#"{"update_check_at_start": false}"#); + h.run_steps(3); + h.state_mut().set_option("update_check_at_start", "true").unwrap(); + run_until_asked(&mut h, &calls, std::time::Duration::from_millis(500)); + assert_eq!(calls.load(SeqCst), 0, "nothing is asked this session"); +} + #[test] fn preferences_turn_the_check_at_start_off() { let (mut h, _calls) = started_with(r#"{"update_check_at_start": true}"#); diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 00b3dae2c2..4aef4fdcc6 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -1778,8 +1778,8 @@ milestone = "M3" status = "partial" commands = ["help.check_updates"] tools = [] -evidence = ["crates/ui-egui/tests/it/updates.rs::a_newer_release_is_offered_for_download", "crates/ui-egui/tests/it/updates.rs::an_up_to_date_or_failed_check_says_so", "crates/ui-egui/tests/it/updates.rs::the_check_at_start_offers_the_prompt_and_later_dismisses_it", "crates/ui-egui/tests/it/updates.rs::nothing_is_asked_until_the_user_checks", "crates/ui-egui/tests/it/updates.rs::the_preference_keeps_the_check_at_start_from_reaching_the_network", "crates/ui-egui/tests/it/updates.rs::settings_from_the_old_opt_in_leave_the_check_at_start_on"] -notes = "asks GitHub Releases through craft-update at start (desktop app only) and on request, offers the release page, and updates and restarts in place where the install type allows it; an install waits while documents are unsaved; Preferences ▸ Check for updates when PdfCraft starts (update_check_at_start, on by default) turns the check at start off, leaving only the request from the Help menu" +evidence = ["crates/ui-egui/tests/it/updates.rs::a_newer_release_is_offered_for_download", "crates/ui-egui/tests/it/updates.rs::an_up_to_date_or_failed_check_says_so", "crates/ui-egui/tests/it/updates.rs::the_check_at_start_offers_the_prompt_and_later_dismisses_it", "crates/ui-egui/tests/it/updates.rs::nothing_is_asked_until_the_user_checks", "crates/ui-egui/tests/it/updates.rs::turning_the_check_at_start_on_waits_for_the_next_start", "crates/ui-egui/tests/it/updates.rs::the_preference_keeps_the_check_at_start_from_reaching_the_network", "crates/ui-egui/tests/it/updates.rs::settings_from_the_old_opt_in_leave_the_check_at_start_on"] +notes = "asks GitHub Releases through craft-update at start (desktop app only) and on request, offers the release page, and updates and restarts in place where the install type allows it; an install waits while documents are unsaved; Preferences ▸ Check for updates when PdfCraft starts (update_check_at_start, on by default) turns the check at start off, leaving only the request from the Help menu; turning it on takes effect at the next start" [[feature]] id = "view.single-key-accelerators" From ff227b770ca6e436ee0853a2585739a8be9eb1e9 Mon Sep 17 00:00:00 2001 From: kronbii <145912020+Kronbii@users.noreply.github.com> Date: Thu, 29 Oct 2026 13:15:30 +0200 Subject: [PATCH 054/198] M2: area-average shrunk scans and smooth enlarged ones Scanned pages looked thin and broken at screen size. An image that does not ask for /Interpolate (most scans) was pre-shrunk with nearest-neighbour since about one sample in ten and dropped most strokes of its text, and the nearest sampler then read the shrunk raster again at a fractional offset. - Shrinking (vendored hayro patch 21): non-interpolated images outside Type 3 glyphs are area-averaged (area_shrink: each device pixel the coverage-weighted mean of the samples it covers, 16.16 fixed point whose weights sum to exactly one, premultiplied when masked) to their snapped device rectangle and drawn 1:1. /Interpolate governs smoothing when an image is enlarged (ISO 32000-2 8.9.5.3); averaging adds nothing between samples and drops none. PicScale's Box/Area filters were not usable: shrinking, they average a window four target pixels wide. - Enlarging (patch 28): an image of page-scan size (512+ samples on its shorter side) is drawn bilinearly when enlarged, so a zoomed scan has no staircase edges; icons, QR codes and pixel art keep their hard pixels. - A snapped image whose shrink failed keeps the full transform instead of being drawn at its source size (latent in the interpolated path too). Measured on a 13-page 300 dpi scan at 96, 120 and 192 dpi: every rendered sample is within 1 of the exact area average (it was up to 255 off, 17-23% of samples by more than 1). Bilinear zoom costs nothing measurable (four 1024 px tiles at 400%: 50 ms, nearest 48 ms; bicubic was 267 ms). Review fixes: - a tile snaps a shrunk image's half-pixel edges where the whole page does - a scan shrunk one way is smoothed the way it is stretched - a smaller soft mask doesn't let transparent colour through the area shrink Co-Authored-By: Claude Opus 5.5 --- ROADMAP.md | 3 +- crates/render/src/raster/tests.rs | 214 +++++++++++++--- docs/roadmap.md | 3 +- parity/acrobat-features.toml | 4 +- vendor/README.md | 2 +- vendor/hayro/src/renderer.rs | 411 ++++++++++++++++++++++++++++-- 6 files changed, 571 insertions(+), 66 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index 5bfe1fc817..ae64d4f352 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -23,7 +23,7 @@ **Stage: alpha** · next: beta, ~25% (ready for real work 50% → 75%) and ~400–700 h away -> **Last reviewed:** 2026-10-29 | **Last updated:** 2026-10-29 | **Change:** minor (M7 #2517: a narrowed then widened paragraph keeps the typed spaces the wrap consumed, after save and reopen; estimates unchanged) +> **Last reviewed:** 2026-10-29 | **Last updated:** 2026-10-29 | **Change:** minor (M2: scans keep their strokes at screen size and are smoothed when zoomed; M7 #2517: a narrowed then widened paragraph keeps the typed spaces the wrap consumed, after save and reopen; estimates unchanged) > **Last reviewed:** 2026-10-27 | **Last updated:** 2026-10-27 | **Change:** minor (M10: OCR model downloads use verified ONNX pairs with legacy discovery fallback; estimates unchanged) > **Last reviewed:** 2026-10-25 | **Last updated:** 2026-10-25 | **Change:** minor (M14 localization current state: Arabic logical catalog, explicit RTL label direction and mirrored stock chrome; native/browser RTL workflows unqualified. Native release Hans face via LXGW WenKai Lite; estimates unchanged) > **Last reviewed:** 2026-10-23 · **Last updated:** 2026-10-23 · **Change:** minor (M7, #2636 partial: pasting added text boxes and images in the engine and through `content_copy`; estimates unchanged) @@ -257,6 +257,7 @@ Ranked; detail and the full milestone table in [docs/roadmap.md](docs/roadmap.md - **2026-10-26 (M2, PR #4172):** Reconciled current main, preserving its Japanese serif/weight selection and both branches' progress records. Installed CJK viewer and preview fallback remains display-only; estimates unchanged. - **2026-10-29 (M7, #2517):** Widening a paragraph rejoins a wrap with the number of spaces that wrap consumed, including a typed run and none for a Chinese or Japanese word split. The count survives an incremental save and a full rewrite, and a width-only `text_edit` uses it. Estimates unchanged. +- **2026-10-29 (M2):** Scanned pages look like the paper again. An image that doesn't ask for /Interpolate was shrunk by nearest-neighbour (since #635), which kept one sample in N per axis, so the text of a 300 dpi scan at 100% lost most of its strokes; it is now area-averaged to whole device pixels (on a 13-page 300 dpi scan every sample at 96–192 dpi is within 1 of the exact area average; it was up to 255 off). Enlarged, an image of page-scan size is smoothed bilinearly, so a zoomed scan has no staircases; icons, QR codes and pixel art keep hard pixels. Estimates unchanged. - **2026-10-26 (M11, JBIG2 symbols):** Lossless JBIG2 now stores each distinct mark of a black-and-white image once, as a symbol, and places its exact copies, when that is smaller than the generic region: on six pages of rendered 300 dpi text 28% of the generic region's size and 17% of Group 4's, and the same bits back through two independent decoders (hayro-jbig2 and Poppler). Scans, whose letters are seldom identical, gain little until refinement coding; a dictionary shared between pages (`/JBIG2Globals`) is next. Estimates unchanged. - **2026-10-25 (M5, comment replies):** One reply can be deleted from the Comments panel without its thread. On the selected card each reply has its own "…" menu (Edit, Copy text, Delete); Delete takes that reply with its pop-up and the replies to it, as `comment_delete` already did headlessly, and the comment and its other replies stay. The card stays selected, and the panel's editor stays open, on what they were on though the delete moved it in `/Annots`. Undo and Redo close the editor, since either can move what it was open on. A locked reply can't be deleted. Estimates unchanged. - **2026-10-24 (M10, #4158 partial):** Word, HTML, RTF and PowerPoint export keep a block that continues another on its line (an italic case name, a link, a superscript) as a run in its own style inside that paragraph, with the source's word space, instead of a paragraph of its own. Style changes that wrap onto the next line still split paragraphs (the reading blocks keep style boundaries). Estimates unchanged. diff --git a/crates/render/src/raster/tests.rs b/crates/render/src/raster/tests.rs index 9c5d75bfbc..7312f3ddaf 100644 --- a/crates/render/src/raster/tests.rs +++ b/crates/render/src/raster/tests.rs @@ -6436,8 +6436,8 @@ fn draw_minified_image( /// making scans, screenshots and rasterized text look soft. #[test] fn non_interpolated_images_are_not_smoothed_when_minified() { - // Top band K W R G, bottom band B Y M C. Nearest 8×4 → 4×2 sampling picks each block's - // pixel, so every device pixel is one palette colour; Catmull-Rom blends across blocks. + // Top band K W R G, bottom band B Y M C. Shrinking 8×4 → 4×2 averages each 2×2 block, + // so every device pixel is one palette colour; Catmull-Rom blends across blocks. let palette: [[u8; 3]; 8] = [[0, 0, 0], [255, 255, 255], [255, 0, 0], [0, 255, 0], [0, 0, 255], [255, 255, 0], [255, 0, 255], [0, 255, 255]]; let rgb: Vec = block_source(palette).concat(); let exact: Vec<[u8; 4]> = palette.iter().map(|c| [c[0], c[1], c[2], 255]).collect(); @@ -6455,38 +6455,76 @@ fn non_interpolated_images_are_not_smoothed_when_minified() { assert!(pixels(&enabled).iter().any(|px| px[..3].iter().any(|&c| (32..=223).contains(&c))), "/Interpolate true made no mid-tone"); } -/// A shrunk, interpolated image whose edges fall between device pixels is snapped to whole -/// pixels (shrunk once to that size, drawn 1:1). Before, the shrunk raster was sampled a second -/// time at the fractional offset, so every edge of a scan was a blended, soft pixel. +/// A shrunk image whose edges fall between device pixels is snapped to whole pixels (shrunk +/// once to that size, drawn 1:1), interpolated or not. Before, the shrunk raster was sampled a +/// second time at the fractional offset, so every edge of a scan was a blended, soft pixel. #[test] -fn interpolated_minified_image_edges_snap_to_whole_pixels() { - let black = vec![0u8; 40 * 40]; - let content = "q 10 0 0 10 0.4 0.4 cm /Im0 Do Q"; - let image = hex(&black); - let pdf = format!( - "%PDF-1.4\n1 0 obj << /Type /Catalog /Pages 2 0 R >> endobj\n\ +fn minified_image_edges_snap_to_whole_pixels() { + for interpolate in ["/Interpolate true ", ""] { + let black = vec![0u8; 40 * 40]; + let content = "q 10 0 0 10 0.4 0.4 cm /Im0 Do Q"; + let image = hex(&black); + let pdf = format!( + "%PDF-1.4\n1 0 obj << /Type /Catalog /Pages 2 0 R >> endobj\n\ 2 0 obj << /Type /Pages /Kids [3 0 R] /Count 1 >> endobj\n\ 3 0 obj << /Type /Page /Parent 2 0 R /MediaBox [0 0 11 11] /Contents 4 0 R /Resources << /XObject << /Im0 5 0 R >> >> >> endobj\n\ 4 0 obj << /Length {} >> stream\n{content}\nendstream endobj\n\ - 5 0 obj << /Type /XObject /Subtype /Image /Width 40 /Height 40 /ColorSpace /DeviceGray /BitsPerComponent 8 /Interpolate true /Filter /ASCIIHexDecode /Length {} >> stream\n{image}>\nendstream endobj\n\ + 5 0 obj << /Type /XObject /Subtype /Image /Width 40 /Height 40 /ColorSpace /DeviceGray /BitsPerComponent 8 {interpolate}/Filter /ASCIIHexDecode /Length {} >> stream\n{image}>\nendstream endobj\n\ trailer << /Root 1 0 R >>\n%%EOF\n", - content.len(), - image.len() + 1 - ); - let mut renderer = PageRenderer::new(Arc::new(pdf.into_bytes()), RenderConfig::default()); - let page = renderer.render(RenderRequest { page: 0, kind: RequestKind::Pixels, scale: 1.0, ..Default::default() }); - assert!(page.error.is_none(), "{:?}", page.error); - assert_eq!((page.width, page.height), (11, 11)); - let gray = |x: usize, y: usize| page.rgba[(y * 11 + x) * 4]; - // Device box 0.4..10.4 × 0.6..10.6 snaps to columns 0..10 and rows 1..11. - for y in 1..11 { - for x in 0..10 { - assert_eq!(gray(x, y), 0, "pixel ({x}, {y}) inside the snapped image"); + content.len(), + image.len() + 1 + ); + let mut renderer = PageRenderer::new(Arc::new(pdf.into_bytes()), RenderConfig::default()); + let page = renderer.render(RenderRequest { page: 0, kind: RequestKind::Pixels, scale: 1.0, ..Default::default() }); + assert!(page.error.is_none(), "{:?}", page.error); + assert_eq!((page.width, page.height), (11, 11)); + let gray = |x: usize, y: usize| page.rgba[(y * 11 + x) * 4]; + // Device box 0.4..10.4 × 0.6..10.6 snaps to columns 0..10 and rows 1..11. + for y in 1..11 { + for x in 0..10 { + assert_eq!(gray(x, y), 0, "{interpolate:?}: pixel ({x}, {y}) inside the snapped image"); + } + assert_eq!(gray(10, y), 255, "{interpolate:?}: column 10 is outside the snapped image (row {y})"); + } + for x in 0..11 { + assert_eq!(gray(x, 0), 255, "{interpolate:?}: row 0 is outside the snapped image (column {x})"); } - assert_eq!(gray(10, y), 255, "column 10 is outside the snapped image (row {y})"); } - for x in 0..11 { - assert_eq!(gray(x, 0), 255, "row 0 is outside the snapped image (column {x})"); +} + +/// A tile draws exactly the pixels of the whole page under it, also where a shrunk image's +/// edge falls on a half pixel. The edges were snapped by rounding halves away from zero, so +/// the tile at x = 5, which moves the left edge from 2.5 to -2.5, snapped it one pixel left of +/// where the whole page did and shifted every column of the image. +#[test] +fn a_tile_snaps_half_pixel_image_edges_as_the_whole_page_does() { + for interpolate in ["/Interpolate true ", ""] { + let ramp: Vec = (0..40 * 40).map(|i| (i % 40 * 6) as u8).collect(); + let content = "q 10 0 0 10 2.5 2.5 cm /Im0 Do Q"; + let image = hex(&ramp); + let pdf = format!( + "%PDF-1.4\n1 0 obj << /Type /Catalog /Pages 2 0 R >> endobj\n\ + 2 0 obj << /Type /Pages /Kids [3 0 R] /Count 1 >> endobj\n\ + 3 0 obj << /Type /Page /Parent 2 0 R /MediaBox [0 0 20 20] /Contents 4 0 R /Resources << /XObject << /Im0 5 0 R >> >> >> endobj\n\ + 4 0 obj << /Length {} >> stream\n{content}\nendstream endobj\n\ + 5 0 obj << /Type /XObject /Subtype /Image /Width 40 /Height 40 /ColorSpace /DeviceGray /BitsPerComponent 8 {interpolate}/Filter /ASCIIHexDecode /Length {} >> stream\n{image}>\nendstream endobj\n\ + trailer << /Root 1 0 R >>\n%%EOF\n", + content.len(), + image.len() + 1 + ); + let mut renderer = PageRenderer::new(Arc::new(pdf.into_bytes()), RenderConfig::default()); + let req = RenderRequest { page: 0, kind: RequestKind::Pixels, scale: 1.0, ..Default::default() }; + let full = renderer.render(req); + let tile = Tile { x: 5, y: 5, w: 10, h: 10 }; + let part = renderer.render(RenderRequest { tile: Some(tile), ..req }); + assert!(full.error.is_none() && part.error.is_none(), "{:?} {:?}", full.error, part.error); + assert_eq!((full.width, part.width, part.height), (20, 10, 10)); + for y in 0..10usize { + for x in 0..10usize { + let whole = full.rgba[((y + 5) * 20 + x + 5) * 4]; + assert_eq!(part.rgba[(y * 10 + x) * 4], whole, "{interpolate:?}: pixel ({x}, {y}) of the tile"); + } + } } } @@ -6570,28 +6608,132 @@ fn non_interpolated_gray_images_are_not_smoothed_when_minified() { } #[test] -fn non_integer_minification_samples_the_same_columns() { - // Ten columns drawn six pixels wide: centre sampling reads columns 0, 2, 4, 5, 7 and 9. +fn non_integer_minification_averages_the_columns_each_pixel_covers() { + // Ten columns drawn six pixels wide: each pixel covers 10/6 columns. Pixel 0 is 60% of + // column 0 and 40% of column 1 (0.6 × 10 + 0.4 × 20 = 14), pixel 1 is 20% of column 1, + // all of column 2's 60% and 20% of column 3 (30), and so on. Nearest-neighbour read whole + // columns 0, 2, 4, 5, 7 and 9 instead, dropping the four others. let ramp = [10u8, 20, 30, 40, 50, 60, 70, 80, 90, 100]; let page = draw_minified_image("DeviceGray", (10, 1), None, &ramp, None, (6, 1)); let grays: Vec = page.rgba.as_chunks::<4>().0.iter().map(|px| px[0]).collect(); - assert_eq!(grays, [10, 30, 50, 60, 80, 100]); + assert_eq!(grays, [14, 30, 46, 64, 80, 96]); } #[test] -fn non_integer_minification_samples_colour_and_mask_from_the_same_columns() { - // Ten gray columns with a soft mask, drawn six pixels wide: columns 0, 2, 4, 5, 7 and 9 are - // sampled, so the mask must pick the same columns as the colours. +fn non_integer_minification_averages_colour_and_mask_together() { + // Ten gray columns with a soft mask, drawn six pixels wide. Colours are averaged + // premultiplied, over the same columns as the mask, and composited over the white page. + // Pixel 0: 60% of column 0 (gray 10, opaque) and 40% of column 1 (transparent) is + // premultiplied gray 6 at alpha 153, so 6 + 255 × (1 − 153/255) = 108. Pixels 1 and 4 + // cover only transparent columns and show the page. let ramp = [10u8, 20, 30, 40, 50, 60, 70, 80, 90, 100]; let mask = [255u8, 0, 0, 0, 255, 255, 0, 0, 0, 255]; let page = draw_minified_image("DeviceGray", (10, 1), None, &ramp, Some(&mask), (6, 1)); - // Opaque columns keep their gray; transparent columns show the white page. assert_eq!( page.rgba.as_chunks::<4>().0.to_vec(), - vec![[10, 10, 10, 255], [255, 255, 255, 255], [50, 50, 50, 255], [60, 60, 60, 255], [255, 255, 255, 255], [100, 100, 100, 255]] + vec![[108, 108, 108, 255], [255, 255, 255, 255], [132, 132, 132, 255], [138, 138, 138, 255], [255, 255, 255, 255], [162, 162, 162, 255]] ); } +/// A scan shown at screen size: a stroke one sample wide in an image drawn at a third of its +/// resolution. Nearest-neighbour read columns 1, 4 and 7 and lost the stroke in column 3, which +/// is how the text of 300 dpi scans broke up at 100% zoom. Area averaging keeps it as the +/// third of pixel 1 it covers. +#[test] +fn non_interpolated_minification_keeps_thin_strokes() { + let mut stroke = [255u8; 9]; + stroke[3] = 0; + let page = draw_minified_image("DeviceGray", (9, 1), None, &stroke, None, (3, 1)); + let grays: Vec = page.rgba.as_chunks::<4>().0.iter().map(|px| px[0]).collect(); + assert_eq!(grays, [255, 170, 255]); +} + +/// A zoomed scan: an image of page-scan size drawn three times its resolution without +/// `/Interpolate`. Nearest-neighbour turned each sample into a 3 × 3 block, so the edges of +/// scanned text were staircases; it is smoothed (bilinearly) instead, as browsers draw it. +#[test] +fn large_non_interpolated_images_are_smoothed_when_enlarged() { + let side = 512usize; + let half: Vec = (0..side * side).map(|i| if i % side < side / 2 { 0 } else { 255 }).collect(); + let page = draw_minified_image("DeviceGray", (512, 512), None, &half, None, (1536, 1536)); + let row = 768 * 1536; + let edge: Vec = (760..776).map(|x| page.rgba[(row + x) * 4]).collect(); + assert!(edge.iter().any(|v| (32..=223).contains(v)), "a hard edge, not a smoothed one: {edge:?}"); + assert_eq!((page.rgba[(row + 700) * 4], page.rgba[(row + 836) * 4]), (0, 255), "away from the edge"); +} + +/// Small images (icons, QR codes, pixel art) keep their hard pixels when enlarged. +#[test] +fn small_non_interpolated_images_keep_hard_pixels_when_enlarged() { + let checker: Vec = (0..64).map(|i| if (i / 8 + i % 8) % 2 == 0 { 0 } else { 255 }).collect(); + let page = draw_minified_image("DeviceGray", (8, 8), None, &checker, None, (80, 80)); + assert!(page.rgba.as_chunks::<4>().0.iter().all(|px| px[0] == 0 || px[0] == 255), "a small image was smoothed"); +} + +/// A scan shrunk across and stretched down (a fax shown at screen size, zoomed in): the shrunk +/// side is still averaged, and the stretched side is smoothed as a scan enlarged both ways +/// is. Shrinking one side kept the nearest sampler for the other, so its edges were staircases. +#[test] +fn a_scan_shrunk_one_way_is_smoothed_the_way_it_is_stretched() { + // Top half black; bottom half columns alternating black and white, which averages to gray. + let side = 512usize; + let source: Vec = (0..side * side).map(|i| if i / side < side / 2 || i % 2 == 0 { 0 } else { 255 }).collect(); + let page = draw_minified_image("DeviceGray", (512, 512), None, &source, None, (256, 1536)); + let gray = |x: usize, y: usize| page.rgba[(y * 256 + x) * 4]; + let edge: Vec = (760..776).map(|y| gray(128, y)).collect(); + assert!(edge.iter().any(|v| (16..=112).contains(v)), "a hard edge down the stretched side: {edge:?}"); + assert_eq!(gray(128, 700), 0, "above the edge"); + assert!((0..256).all(|x| gray(x, 836).abs_diff(128) <= 1), "the shrunk side is averaged at 1:1"); +} + +/// An image shrunk across and stretched down (a 204 × 98 dpi fax shown at screen size): the +/// shrunk side is still averaged. The area shrink was asked to enlarge the other side too, +/// refused, and the image fell back to its source resolution and the nearest sampler, which +/// read single columns. +#[test] +fn an_image_shrunk_one_way_and_stretched_the_other_is_still_averaged() { + let ramp: Vec = [10u8, 20, 30, 40, 50, 60, 70, 80].repeat(2); + let page = draw_minified_image("DeviceGray", (8, 2), None, &ramp, None, (4, 4)); + let grays: Vec = page.rgba.as_chunks::<4>().0.iter().map(|px| px[0]).collect(); + assert_eq!(grays, [15, 35, 55, 75].repeat(4), "each pixel is the mean of the two columns it covers"); +} + +/// Averaging straight colours would tint an opaque sample with the colour of a transparent +/// neighbour: white beside transparent black, shrunk to one pixel, came out gray over the white +/// page. Premultiplied, the transparent sample adds nothing but its transparency. +#[test] +fn non_interpolated_minification_does_not_bleed_transparent_colour() { + let page = draw_minified_image("DeviceGray", (2, 1), None, &[255, 0], Some(&[255, 0]), (1, 1)); + assert_eq!(page.rgba.as_chunks::<4>().0.to_vec(), vec![[255, 255, 255, 255]]); +} + +/// The same with a soft mask smaller than its image: red, red, blue, blue under a mask that is +/// opaque, then transparent, shrunk to one pixel, is half-covered red over the white page. +/// Colour and mask were averaged apart and multiplied, so the transparent blue came through +/// as purple (191, 127, 191). +#[test] +fn a_smaller_soft_mask_does_not_bleed_transparent_colour_when_minified() { + let content = "q 1 0 0 1 0 0 cm /Im0 Do Q"; + let (image, mask) = ("FF0000FF00000000FF0000FF", "FF00"); + let pdf = format!( + "%PDF-1.4\n1 0 obj << /Type /Catalog /Pages 2 0 R >> endobj\n\ + 2 0 obj << /Type /Pages /Kids [3 0 R] /Count 1 >> endobj\n\ + 3 0 obj << /Type /Page /Parent 2 0 R /MediaBox [0 0 1 1] /Contents 4 0 R /Resources << /XObject << /Im0 5 0 R >> >> >> endobj\n\ + 4 0 obj << /Length {} >> stream\n{content}\nendstream endobj\n\ + 5 0 obj << /Type /XObject /Subtype /Image /Width 4 /Height 1 /ColorSpace /DeviceRGB /BitsPerComponent 8 /SMask 6 0 R /Filter /ASCIIHexDecode /Length {} >> stream\n{image}>\nendstream endobj\n\ + 6 0 obj << /Type /XObject /Subtype /Image /Width 2 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8 /Filter /ASCIIHexDecode /Length {} >> stream\n{mask}>\nendstream endobj\n\ + trailer << /Root 1 0 R >>\n%%EOF\n", + content.len(), + image.len() + 1, + mask.len() + 1 + ); + let mut renderer = PageRenderer::new(Arc::new(pdf.into_bytes()), RenderConfig::default()); + let page = renderer.render(RenderRequest { page: 0, kind: RequestKind::Pixels, scale: 1.0, ..Default::default() }); + assert!(page.error.is_none(), "{:?}", page.error); + let [r, g, b, _] = page.rgba.as_chunks::<4>().0.first().copied().unwrap_or_default(); + assert!(r >= 254 && g.abs_diff(128) <= 1 && b == g, "half-covered red over white, got ({r}, {g}, {b})"); +} + #[test] fn non_interpolated_alpha_images_keep_their_colours_and_mask_when_minified() { let palette: [[u8; 3]; 8] = [[0, 0, 0], [255, 255, 255], [255, 0, 0], [0, 255, 0], [0, 0, 255], [255, 255, 0], [255, 0, 255], [0, 255, 255]]; @@ -7716,7 +7858,7 @@ fn stitched_render_partitions_match_a_whole_render() { /// and square caps, a sharp miter, a rotated and a stretched matrix, a hairline, dashes, a /// curve, text, and a page shown rotated. /// -/// Images are skipped the same way (vendored patch (10)): an image on a tile, one crossing +/// Images are skipped the same way (vendored patch (11)): an image on a tile, one crossing /// tile lines, and one fully off the page. #[test] fn skipping_paths_off_the_canvas_changes_no_pixel() { diff --git a/docs/roadmap.md b/docs/roadmap.md index f376f7cb00..d83744afcf 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -22,7 +22,7 @@ > **Last reviewed:** 2026-10-26 · **Last updated:** 2026-10-26 · **Change:** patch (M2, PR #4172: document installed-font helpers and regression contracts; behavior and estimates unchanged) > **Last reviewed:** 2026-10-26 · **Last updated:** 2026-10-26 · **Change:** patch (M2, PR #4172: print-preview font policy matches output, with isolated caches and GBK switching regressions; estimates unchanged) > **Last reviewed:** 2026-10-26 · **Last updated:** 2026-10-26 · **Change:** patch (M2, PR #4172: reconcile current main and retain Japanese serif/weight selection alongside display-only CJK fallback; estimates unchanged) -> **Last reviewed:** 2026-10-29 | **Last updated:** 2026-10-29 | **Change:** minor (M7 #2517: a narrowed then widened paragraph keeps the typed spaces the wrap consumed, after save and reopen; estimates unchanged) +> **Last reviewed:** 2026-10-29 | **Last updated:** 2026-10-29 | **Change:** minor (M2: scans keep their strokes at screen size and are smoothed when zoomed; M7 #2517: a narrowed then widened paragraph keeps the typed spaces the wrap consumed, after save and reopen; estimates unchanged) > **Last reviewed:** 2026-10-27 | **Last updated:** 2026-10-27 | **Change:** minor (M10: OCR model downloads use verified ONNX pairs with legacy discovery fallback; estimates unchanged) > **Last reviewed:** 2026-10-25 | **Last updated:** 2026-10-25 | **Change:** minor (M5: one reply in the Comments panel is edited, copied or deleted from its own "…" menu; estimates unchanged) > **Last reviewed:** 2026-10-26 · **Last updated:** 2026-10-26 · **Change:** minor (M11: lossless JBIG2 stores each distinct mark once, as a symbol; estimates unchanged) @@ -321,6 +321,7 @@ Estimates unchanged. | 2026-10-23 | minor | M11: black-and-white images (1-bit, and image masks) are stored losslessly as CCITT Group 4 when that is smaller: PDF Optimizer ▸ Images ▸ Monochrome Images (on by default, so Reduce File Size too), `doc_optimize` `mono`; `pdfcraft_filters::encode_g4`. JBIG2 and monochrome downsampling not yet. Estimates unchanged. | | 2026-10-23 | minor | M6: calculation checks: a formula reading a field the form doesn't have, and a script building the names it reads, are flagged by `form_calculation_order` and on the Calculation order page (a red mark, a card naming the broken formulas, a note for built names); `form_calculation_order` gives each formula and `form_fields` a custom Calculate script. Estimates unchanged. | | 2026-10-29 | minor | M7 #2517: widening a paragraph rejoins a wrap with the spaces that wrap consumed, including a typed run and none for a Chinese or Japanese word split, after an incremental save and a full rewrite. Estimates unchanged. | +| 2026-10-29 | minor | M2: non-interpolated images drawn below their resolution are area-averaged to whole device pixels instead of nearest-neighbour (scanned text kept one sample in N per axis); page-scan-size images are smoothed when enlarged. Estimates unchanged. | | 2026-10-27 | minor | M10: OCR stages SHA-256-verified ONNX models after the upstream RTEN URL brownout; discovery prefers a complete ONNX pair per directory, falls back to a complete RTEN pair, and never mixes formats or directories; language/web limits and estimates unchanged. | | 2026-10-24 | minor | M3, #3970 follow-up: the button release given back after a window move or resize goes before the pointer leaving, so hover doesn't stick. Estimates unchanged. | | 2026-10-23 | minor | M8, #3205 follow-up: phone-pattern redaction includes the +39 country code on labelled Italian geographic numbers with a slash; dates, protocol identifiers and whole-run guards remain. Estimates unchanged. | diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 8b7ac909a0..367253b4a7 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -734,8 +734,8 @@ milestone = "M2" status = "partial" commands = [] tools = ["page_render"] -evidence = ["crates/render/src/raster/tests.rs::renders_a_blue_rectangle", "crates/automation/tests/it/automation.rs::open_inspect_render_and_find", "crates/render/src/raster/tests.rs::user_unit_scales_the_page_and_everything_on_it"] -notes = "hayro bootstrap renderer; own DisplayList devices in M2.6. Pages with /UserUnit are shown at their size in points, as Acrobat reads the value" +evidence = ["crates/render/src/raster/tests.rs::renders_a_blue_rectangle", "crates/automation/tests/it/automation.rs::open_inspect_render_and_find", "crates/render/src/raster/tests.rs::user_unit_scales_the_page_and_everything_on_it", "crates/render/src/raster/tests.rs::non_interpolated_minification_keeps_thin_strokes", "crates/render/src/raster/tests.rs::minified_image_edges_snap_to_whole_pixels", "crates/render/src/raster/tests.rs::large_non_interpolated_images_are_smoothed_when_enlarged", "crates/render/src/raster/tests.rs::small_non_interpolated_images_keep_hard_pixels_when_enlarged"] +notes = "hayro bootstrap renderer; own DisplayList devices in M2.6. Pages with /UserUnit are shown at their size in points, as Acrobat reads the value. An image drawn below its resolution is shrunk once to whole device pixels: by area averaging unless it asks for /Interpolate (then Catmull-Rom), so the text of a 300 dpi scan at 100% keeps its strokes (it was nearest-neighbour, which dropped them). Enlarged, an image of page-scan size (512 samples or more on its shorter side) is smoothed bilinearly whatever its /Interpolate, so a zoomed scan has no staircases; smaller images keep their hard pixels. Not measured against Acrobat" [[feature]] id = "view.render-shadings-patterns" diff --git a/vendor/README.md b/vendor/README.md index 4fda6483af..1a58ad6c1f 100644 --- a/vendor/README.md +++ b/vendor/README.md @@ -40,7 +40,7 @@ Other resource categories retain their existing parent-chain lookup. | Crate | Version | Licence | Patches | Test | |---|---|---|---|---| | hayro-interpret | 0.7.0 | Apache-2.0 OR MIT | (1) select `/AP /N` appearance states via `/AS` (checkboxes, radios, stateful icons) and honour the NoView flag; (2) Type3 Unicode fallback to `/Encoding` glyph names, then printable ASCII; (3) `Type3Glyph::advance_width`; (4) `InterpreterSettings::ocg_overrides` for viewer layer toggles; (5) soft masks whose group lacks `/CS` were dropped (Chrome gradient text); (6) paint nesting cap `MAX_PAINT_NESTING` for tiling patterns and Type 3 glyphs (self-referencing content overflowed the stack); (7) CID width ranges in `/W` and `/W2` clamped to `MAX_CID` (a `0 4294967295 w` range hung); (8) `InterpreterSettings::hide_comments` skips markup annotations but keeps widgets and links (View ▸ Hide all comments); (9) `image_size_ok`: images and masks over 2^28 pixels are not decoded (a fuzzed `/W 4294967295` stencil mask allocated 4 GiB); (10) `MAX_NESTED_PAINTS`: nested paints (forms, Type 3 glyphs and tiling patterns inside one another) share a per-page budget of 50,000 (a glyph showing eight glyphs of its own font, sixteen deep, hung); (11) depends on skrifa 0.47 instead of 0.42 (upstream read-fonts fixed a Type 1 tokenizer that recursed through runs of numbers: a fuzzed font hung or overflowed the stack), with the one API change (`Charset::glyph_id` returns `Option`); (12) annotation appearances whose `/BBox` or `/Matrix` collapses to a line or a point are skipped (mapping them onto `/Rect` divided by zero and handed the device infinite or NaN matrices); (13) a Highlight annotation without a usable normal appearance is drawn from its `/QuadPoints`, `/C` and `/CA` with Multiply blending, as pdfcraft-annot draws highlights (it was not drawn at all); (14) simple-font `/Widths` (`read_widths`): entries only for codes 0–255 (one entry per code below `/FirstChar` was allocated: `4294967295` asked for 34 GB), an entry that isn't a number leaves its code missing instead of ending the array, and a `/LastChar` below `/FirstChar` gives no widths instead of discarding the font for a standard one; (15) the appearance box is mapped onto `/Rect` corner to corner: the translation now accounts for the scale (an appearance whose box didn't start at the origin and differed in size from `/Rect` was drawn shifted); (16) a clip set by `W`/`W*` takes effect after whichever path-painting operator ends the path (`S`, `f`, `B`, … as well as `n`; ISO 32000-2 §8.5.4): it was applied only on `n`, so `re W* S` clipped nothing and the clip stayed pending for a later, unrelated `n`; (17) `InterpreterSettings::cancelled`: once set, interpretation stops before the next content operator (a render pool sets it for every worker when dropped, after an edit or on closing, and for one worker when the watchdog gives up on its render, so renders whose answers nobody can receive stop instead of running to the end); (18) `MAX_PAGE_CONTENT`: the content a page holds at once (its contents, and the forms, tiling patterns and Type 3 glyph procedures being painted) stays within 512 MiB (`decode_content`; once it runs out, `InterpreterWarning::ContentTruncated` is reported (once per page by pdfcraft-render) so a truncated page is not a clean render; each level gives its bytes back when it finishes): forms nest fifty deep and each level kept its content while the next ran, so a form painting itself around a stream inflating to 256 MiB would have held over 12 GB; and Type 0 sampled functions take at most `MAX_SAMPLE_ENTRIES` (2^21) table points and `MAX_SAMPLE_VALUES` (2^23) samples, read no further than `/Size` asks (256 MiB of 1-bit samples asked for over 8 GB); (19) mesh shadings (types 4–7) are sampled only inside the area being drawn (`ShadingPattern::encode_within`, which the vendored hayro renderer calls with that area plus a pixel), within 8 visits per pixel of it plus 4 per triangle (`TriangleSampler`), and patches are triangulated one at a time: points far off the page were sampled over their whole bounding box, up to 65536 × 65536 entries (a fuzzed 147-byte patch stream; a synthetic page passed 6 GB in six seconds), ten thousand overlapping triangles ran for minutes, and every patch became 722 triangles before any limit; meshes hold at most `MAX_MESH_PATCHES` (2^16) patches and `MAX_MESH_TRIANGLES` (2^20) triangles, a lattice needs two to 2^19 vertices per row (`/VerticesPerRow 0` pushed empty rows for ever; one huge row read the whole stream), and a page samples at most `MAX_PAGE_MESH_TRIANGLES` (2^25) mesh triangles in all (a mesh painted a thousand times was triangulated a thousand times); each cut logs a warning; (20) PDF function construction is limited to 64 levels and 10,000 total nodes with cycle detection; calculator parsing and evaluation share procedure depth/work budgets, integer division/bitshift/rotation/index are panic-free, and fixed-size stack errors propagate; (21) a Type 0 font drawn with a substitute that isn't CID-keyed (it isn't embedded, or its program doesn't load) draws `.notdef` for a CID of an Adobe character collection (Japan1, GB1, CNS1, Korea1) that it can't find by Unicode: hayro took the CID for a glyph index of the substitute, which numbers its glyphs its own way, so the character was drawn as an unrelated one (𠮷 as "Ｑ" in BIZ UDPGothic); (22) a Type 0 font drawn with a substitute finds a character given with a variation selector (`/ToUnicode`, or Adobe-Japan1-UCS2, which maps the JIS X 0208 forms of the kanji JIS X 0213:2004 redrew that way: CID 1247 is 噂 U+E0100) through the substitute's variation sequences, else as the character alone: such a CID found no glyph and was drawn as an unrelated one; (23) backport of hayro 0.8's `gradient` module: `EncodedShadingPattern::as_svg_gradient` turns an axial or radial shading into stops of an sRGB gradient (within 0.01 per channel, a stop at every stitching-function bound via `Function::stitching_bounds`) when a native gradient draws it exactly (extended past the box drawn in; a radial one with no background colour); (24) `DecodedImageCache`: decoded image pixels shared across renders through the interpreter cache within a 64 MiB budget (an image-heavy page decoded its thousands of images again per tile and per scale), keyed by every decode input (`DecodedImageKey`: the stream's place and dictionary bytes, its object, mask/stencil/raster output, interpolation, decode target size, preview or final); images under a transfer function and inline images are not cached, an image over the budget is never kept, and a poisoned lock decodes uncached; (25) `Device::actual_text` hands the device a marked content sequence's `/ActualText` (inline or named in `/Properties`), so text extraction keeps the Thai, Indic and Arabic clusters Chrome marks this way (their shaped tone marks and vowels have no Unicode of their own); (26) non-embedded simple fonts drawn with a standard-font stand-in build the Latin Extended-A letters the stand-in lacks (`/Gbreve`, `/Idotaccent`, `/Scedilla`, `/aogonek`, `/ccaron`…) from the base letter and accent it has (`StandardKind::composite_glyph`, `ACCENTED_LETTERS`): PDFium's Foxit stand-ins hold only the Standard/WinAnsi/Mac Roman glyphs, so iText letters with Times New Roman unembedded showed blanks for Ğ, İ, Ş, ğ and ş; the comma-below letters (`/scommaaccent`, `/Tcommaaccent`, `/kcommaaccent`…, found by character, so `/uni0136` and the glyph list's `/Kcedilla` too) are the letter and the font's comma, smaller and under the baseline, since the stand-ins have no `commaaccent` (Romanian ș drew nothing and ț a cedilla in forms laid out from XFA; ģ, whose comma is turned and above, is not built); (27) an embedded TrueType font without a base encoding reads its codes as WinAnsi for text extraction (`EmbeddedKind::code_to_unicode_name`), matching pdftotext's output, instead of Standard: OCR text layers of scans write WinAnsi codes with no `/Encoding`, and Standard has no ü, ö or ß and reads é as Ø (glyph selection unchanged); (28) an image whose pixels hold at most 16 bits of samples (an 8-bit Separation or DeviceN image, a 1-, 2- or 4-bit gray or indexed one) is converted once per possible pixel value through a lookup table instead of once per pixel (`rgb_via_lookup`): a spot-colour tint transform is often a PostScript calculator program, run for every pixel, so a fillable character sheet's 3304 × 4105 DeviceN background took four seconds and 430 MB to draw a page; each entry is converted on its own (`convert_each`), so a tint transform giving the wrong number of components for a value the image never uses cannot shift the colours of the values it does; (29) `Device::begin_annotations`/`end_annotations` bracket the page's annotations, so a device can tell comment paints from document paints (reading colours recolour the document, never a comment, #1548); (30) the Annex D.2 alias codes read as the ink they draw: WinAnsiEncoding 0xA0 and 0xAD, and MacRomanEncoding 0xCA, carry the glyph names `space` and `hyphen` -- the names those tables also give 0x20 and 0x2D -- not `nbspace` and `sfthyphen`, whose glyph list values U+00A0 and U+00AD are invisible formatting characters. Extraction hid drawn ink behind them, so search and redaction matched neither (`XiaoBiaoSong.pdf` in the pdf.js corpus drew 104 soft hyphens; across the 989 files the soft hyphens we extract fall from 117 to 13 and the non-breaking spaces from 30 to 26, what remains coming from `/ToUnicode` maps, which say so explicitly). The base table only, in the text direction only: a `/Differences` name keeps its glyph list value and glyph lookup is untouched, both as pdftotext does; (31) a Type 0 sampled function keeps its samples in one flat array indexed by strides (`Type0::entry`) instead of a hash map keyed by each grid point: the map was built again, one hashed insert per sample, every time a shading using the function was painted (a 2,097,152-sample ramp painted 30 times took 38 s; `bug1721218` in the pdf.js corpus spent most of a 1.5 s page on it); the interpolation is unchanged, so results are bit for bit the same, a sample missing from a short stream still gives no value, and the maximum sample value is computed without overflowing at 32 bits per sample; (32) the decoded-image cache (24) keys a decode by the size the image is drawn at only when a JPX stream decodes it or its soft mask or mask (`decode_reads_target`): JPX is the only filter that decodes a reduced resolution for a size, so a JPEG or Flate scan was decoded again at every zoom step and kept once per zoom level; one decode now serves every size; `InterpreterCache::decoded_image_decodes` counts the cache's decodes; (33) Coons and tensor-product patches copy their vertices' colours less: each point of a patch's 20 × 20 triangle grid is interpolated once (`generate_patch_triangles`; each cell interpolated its four corners itself, 1,444 interpolations a patch for 400 points), a cell's second triangle takes the vertices its first copied, and the mesh sampler moves a patch's triangles into device space instead of copying each vertex (`TriangleSampler::sample_owned`); the same values, 10–16% faster on mesh-gradient pages; (34) `color::tint_transform_runs`: how many colours the calling thread has run through a Separation or DeviceN tint transform (counted once per batch in `convert_f32`), so a test counts patch (28)'s once-per-sample-value runs instead of timing them (its 10 s deadline failed on a busy FreeBSD VM in CI); (35) `EncodedShadingPattern::extent`: where a mesh shading (types 4–7) can paint at all, the pixels its triangles touched when it was encoded (`None` for the types that paint everywhere, and when the shading has a `/Background`), so the renderer samples a texture there alone and skips a mesh with nothing on the canvas; (36) a mesh shading's pixel whose corner no triangle holds strictly (on an edge two triangles share, or on the mesh's side) takes the triangle that covers its centre, edges included (`Triangle::covers`): testing only the corner left a white seam along every shared edge, and the colour is still taken at the corner; (38) the `/DefaultGray`, `/DefaultRGB` and `/DefaultCMYK` colour spaces of a resource dictionary (ISO 32000-2 §8.6.5.6) replace the device space content selects: with `g`/`G`, `rg`/`RG`, `k`/`K` and `cs`/`CS` (by name, or through a `/ColorSpace` resource that is a device space), and as the `/ColorSpace` of an image XObject, an inline image or a shading painted by `sh` or as a shading pattern (`DefaultColorSpaces`, looked up once per content window in the resources and those they inherit, so a form XObject without a default of its own draws in its page's, as Poppler and MuPDF draw it); a default that is a Lab, Indexed, Pattern or device space, has another number of components or is no colour space is ignored and device colour is drawn as before; device spaces inside other spaces (an Indexed base, a Separation or DeviceN alternate, a Pattern's underlying space), shading dictionaries, masks and transparency group spaces are unchanged, and the decoded-image cache (24) keys a decode by the default it was drawn in. hayro drew device colour as it was, so pages that calibrate it this way (common in PDF/A) came out in the wrong colours; (39) knockout groups (#2971): `Device::set_knockout` right after a `/K true` group is pushed; `Device::knockout_image` for an unmasked image before its group is pushed, when `Device::in_knockout_group`: the device clears its placement rectangle only if it will draw the image (on the canvas, not a preview's tiny image, and it decodes), decoded at its draw size so the draw reuses that decode; and, when `Device::knockout_erase_wanted`, a nested group's content drawn once between `begin_knockout_erase` and `end_knockout_erase` before the group itself, which then always gets a layer of its own; (40) an optional content membership dictionary's visibility expression (`/VE`: `/And`, `/Or`, `/Not`, nested up to 32 deep) decides its visibility ahead of `/OCGs` and `/P` (ISO 32000-2 §8.11.2.2); it was ignored, so content under it stayed visible with its layers hidden; a malformed expression, including one naming an object the document doesn't list in `/OCProperties /OCGs`, falls back to `/OCGs` and `/P` (#2986); (41) a transparency group that is neither isolated nor knockout, entered at full opacity with no soft mask and Normal blending, is drawn straight onto the page instead of into a layer of its own (§11.4.7: its elements composite with the backdrop; the layer made a `/BM /Multiply` highlight appearance an opaque block, #1968); (42) text in a Type 3 font is drawn filled whatever the text rendering mode, except mode 3 (invisible): only that mode has any effect on Type 3 fonts (ISO 32000-1 §9.3.6). Stroking ran the glyph procedure a second time, so `2 Tr` made bitmap fonts look bold, `1 Tr` painted an uncoloured glyph in the stroking colour, and `7 Tr` hid it (`7 Tr` since follows 64); (43) Lab colour spaces honour their own `/WhitePoint`: L\*a\*b\* goes to XYZ relative to that white, is Bradford-adapted to D65 and converted to sRGB (ISO 32000-2 §8.6.5.4) instead of through `LAB.icc` as if every white were D50 (a D65 blue came out 45 levels too red; the profile is removed); components normalised to [0, 1] (`manual_scale`, an `/Indexed` palette over Lab) map onto L\* 0..100 and the a\*/b\* `/Range`; (44) a Type 0 font whose `/ToUnicode` is the name `/Identity-H` or `/Identity-V` (not a CMap stream; WSCAD writes it) reads each code as its Unicode value, as Acrobat shows it (`Type0Font::identity_to_unicode`); the identity CMap the name resolved to maps codes to CIDs only, so such pages had no text to select or find; (46) an image XObject's group applies the fill opacity, so the fill alpha is reset to 1 before a stencil mask takes the fill colour as its paint, and a pattern paint drops the opacity it captured at `scn` (`Pattern::without_opacity`) (it was applied twice: a mask at `ca 0.5` was drawn at a quarter, #2975); (47) `path_as_rect`: a five-element path is a rectangle only when its four points are distinct, each step from one to the next runs along a single axis, and the last returns to the first. The length test admits one MoveTo and four LineTo as well as one MoveTo, three LineTo and a ClosePath, and asking only that every corner of the bounding box was visited says nothing about the order or the steps between, so a five-point polyline whose ends reached the corners was painted as that box -- a fourth side the path does not have -- and a clip of that shape was discarded; the coordinates are compared exactly, since the matrix is applied after this check and a `cm` of 10,000 turns `is_nearly_equal`'s 256th of a unit into tens of device units (#3327); (48) an `/ICCBased` colour space is cached by what its profile is read from (`IccProfileKey`): the stream object it names, looked up without reading the stream, and then the stream's dictionary and its own bytes (decrypted, not decoded), so a byte-for-byte copy of a profile in another object is read once, and a profile stream written directly in its array, which reports the object holding the array as every other stream written there does, is told apart by its bytes. hayro keyed it by the stream dictionary's bytes alone, so two different profiles with the same `/N` and `/Length` shared whichever was read first, and colours in the second were drawn through the first. `InterpreterCache::icc_profile_reads` counts the profiles read; (49) `OpenTypeFontBlob::new` also accepts a `post` table version 1.0, 2.0 or 3.0 written with its 16-bit halves swapped (`00 00 00 03`), as ScanSoft PDF Create! embeds every font; such fonts were refused and their text drawn with a substitute (Hebrew came out garbled or missing, #2946); `glyph_names` reads such a table from a copy with the version written correctly, so a 2.0 table's names serve `/Differences`. Any other unknown version is still refused (pdf.js 9462); (50) `CffFontBlob::glyph_index_by_name` looks a glyph name up in a name to glyph map built on first use (shared between clones) instead of searching the whole charset, with a string lookup per glyph, for every character code of a font; the first glyph with a name still wins; (51) at most 4096 graphics states saved at once, holding at most 64 MiB together (each `q` copies the whole state, dash array and clip list included, so short operators multiplied one large state); a `q` past either limit is counted but not copied, and its `Q` still removes the clips added since (from storytold/craft-libs#239); (52) a `/W2` run whose first CID is past `MAX_CID` stops there, as in `/W` (7), checked before the run's metrics are read: a first CID of 4294967295 inserted one entry and then overflowed, discarding the whole `/W2` table (from storytold/craft-libs#254); (55) a Separation or DeviceN tint transform that cannot be read keeps its colour space, and one that cannot be read or evaluated paints the alternate space's initial colour, as evaluation failures already did; that fallback, and a `cs`/`CS` colour space that cannot be read (drawn in DeviceGray as before), are now reported once per content stream as `InterpreterWarning::ColorFallback` (`RenderWarning::ColourFallback` in `pdfcraft-render`) instead of silently (#4260); (56) an annotation whose Invisible flag (bit 1) is set and whose `/Subtype` is none of the standard types of ISO 32000-2 Table 171 (`is_standard_annotation_type`; no `/Subtype` counts as unknown) is not drawn (Table 167), and a `/Popup` annotation's appearance is never painted on the page (§12.5.6.14: a popup has no appearance stream of its own; the viewer shows it as a window for its parent annotation). Both were drawn from their `/AP`; (57) a simple font drawn with a standard-font stand-in because the document names a font it doesn't embed (`StandardKind` with `fallback`) narrows a glyph to a narrower `/Widths` advance but no longer widens it to a wider one: the advance already places the next glyph (ISO 32000-2 §9.2.4; /Widths in Table 109), and widening drew a non-embedded `Arial,Bold` with 600-unit widths as a heavy fixed-pitch face whose `i` and `l` were solid blocks twice their width; an exact standard-14 font still matches its standard metrics both ways; (58) the fallback request for a Type 0 font that is not embedded (`FallbackFontQuery::with_descriptor`) takes its serif, italic and small-cap flags, `/FontWeight`, `/FontFamily` and `/FontStretch` from the descendant CIDFont's font descriptor (ISO 32000-2 §9.7.4.1, §9.7.6.1; §9.8): it read them from the Type 0 dictionary, which has none, so every such request asked for a regular sans-serif face and no resolver could tell a Song or Mincho font from a Hei or Gothic one, or see `/FontWeight 700`; (59) the NoRotate flag (`/F` bit 5) is honoured: the appearance, fitted to `/Rect`, is turned against the page's `/Rotate` about the rectangle's upper-left corner, which stays where it is, so it reads upright as the page is displayed (ISO 32000-2 §12.5.3). Sticky notes, which are saved NoRotate, were drawn turned with the page (#2583). Form fields and links stay in `/Rect`, where PdfCraft takes their click targets from it; (64) text in a Type 3 font follows ISO 32000-2 §9.3.6 in the clipping modes: under `7 Tr` it is not rendered (reported to the device as clipping text, `GlyphDrawMode::Clip`, so extraction keeps it), and in modes 4 to 7 a Type 3 glyph adds nothing to the clipping path (`clip_glyph`), so text shown only in a Type 3 font leaves no clip at `ET`. Since (42) `7 Tr` painted the glyphs in the fill colour (black by default), so a fill meant to follow showed them wherever it didn't cover them; 4 to 6 still render filled; (66) a Separation colour space whose colourant is the special name `All` paints every colourant at once (ISO 32000-2 §8.6.6.4): on screen and in an RGB print raster its tint t is the gray 1 − t, complemented because the device is additive, whatever its alternate space and tint transform say (hayro drew it through the tint transform, so a `/DeviceCMYK` transform giving cyan drew registration marks light cyan); `/None` still paints nothing, any other name still goes through its alternate, and a DeviceN space naming `All` keeps its alternate, since the clause gives the name that meaning in a Separation space; (63) a luminosity soft mask whose group colour space is `/DeviceCMYK` (by name, not an ICC-based or calibrated space) reads a `/DeviceCMYK` fill or stroke colour drawn in it, and its `/BC`, as their gray `1 - min(1, 0.3 c + 0.59 m + 0.11 y + k)` (ISO 32000-2 §10.4.2.3, §11.5.3: device colour converts to gray with no colour calibration; `Context::luminosity_in_device_cmyk`, `color::device_cmyk_to_gray`): it went through the CMYK display profile, where 100% K is a dark grey, so a 100% K mask let about an eighth of the masked paint through. CMYK images, shadings and tiling patterns inside such a mask, and CMYK on the page, still use the profile; (71) a lookup table in an indirect string reads: it was read as a stream first, which uses up a reference (ISO 32000-2 §8.6.6.3); (72) an uncoloured pattern's underlying space given as a name or a reference is used: it was skipped, so a CMYK cyan drew red (ISO 32000-2 §8.7.3.3); (73) `InterpreterSettings::jbig2_work`, set for each page in `interpret_page` and ended with it, and a JBIG2 image or an image with a JBIG2 mask past it left out with `ContentTruncated`; (74) colour spaces nest at most 32 levels, through every way one names another, so a chain of them ends instead of overflowing the stack; an ICC profile's `/Alternate` is read where it's met and isn't kept with the profile, or a page could build a chain past the limit on spaces already read; (75) a soft hyphen (U+00AD) that a CFF substitute for a CID font has no glyph for draws its hyphen, as Annex D.2 reads WinAnsi 0xAD in (30), instead of the CID taken as a glyph index (#5888) | (1) `raster::tests::widget_appearance_states`; (2, 3) `xtask text-oracle` on corpus/pdfjs issue918; (4) `raster::tests::layer_override_hides_content`; (5) `raster::tests::soft_mask_without_group_colour_space_is_applied`; (6) `raster::tests::self_referencing_tiling_pattern_terminates`, `raster::tests::self_referencing_type3_glyph_terminates`; (7) `raster::tests::huge_cid_width_ranges_terminate`; (8) `raster::tests::hiding_comments_keeps_fields`; (9) `raster::tests::absurd_mask_and_fax_sizes_are_refused_before_decoding`; (10) `raster::tests::type3_glyphs_that_fan_out_into_themselves_finish`; (11) `raster::tests::type1_font_programs_with_long_runs_of_numbers_load_quickly`; (12) `raster::tests::degenerate_annotation_appearances_are_skipped`; (13) `raster::tests::highlights_without_an_appearance_are_drawn`; (14) `raster::tests::malformed_simple_font_widths_keep_later_widths_and_the_font`, `raster::tests::huge_simple_font_first_char_terminates`; (15) `raster::tests::annotation_appearance_box_maps_onto_rect`; (16) `raster::tests::clipping_path_applies_after_any_painting_operator`; (17) `raster::tests::dropping_the_pool_cancels_its_renders`, `raster::tests::the_watchdog_stops_the_render_it_gives_up_on`; (18) `raster::tests::page_contents_and_nested_forms_share_one_budget`, `raster::tests::content_painted_again_and_again_keeps_drawing`, `raster::tests::sampled_functions_with_billions_of_samples_are_refused`; (19) `raster::tests::mesh_shadings_far_off_the_page_render`, `raster::tests::many_overlapping_mesh_triangles_finish`, `raster::tests::mesh_shadings_with_many_patches_or_no_rows_finish`; (20) `raster::tests::function_limits_calculator_arithmetic_and_depth`, `raster::tests::function_limits_public_construction_work_and_depth`, `raster::tests::function_limits_stitching_cycles_are_refused_and_shared_children_work`, `raster::tests::function_limits_invalid_calculator_keeps_the_page_renderable`; (21) `raster::tests::cids_a_substitute_lacks_are_not_drawn_as_other_glyphs`; (22) `raster::tests::variation_sequences_draw_their_kanji_in_a_substitute`; (23) `raster::tests::axial_and_radial_shadings_draw_as_native_gradients`; (24) `raster::tests::tiles_match_full_render`, `raster::tests::skipping_paths_off_the_canvas_changes_no_pixel` (both decode through the cache), `raster::tests::decoded_images_with_identical_dictionaries_keep_their_own_pixels`, `raster::tests::decoded_images_under_a_transfer_function_are_not_shared`; (25) `text::tests::actual_text_replaces_the_glyphs_it_marks_in_every_writing_direction`, `text::tests::actual_text_follows_the_pdf_rules_for_marked_content`; (26) `raster::tests::accented_letters_the_standard_stand_ins_lack_are_built_from_their_parts`, `raster::tests::comma_below_letters_in_standard_fonts_draw_a_comma_under_the_letter`; (27) `text::tests::truetype_without_an_encoding_reads_its_codes_as_win_ansi`; (28) `raster::tests::spot_colour_images_run_their_tint_transform_once_per_sample_value`, `raster::tests::images_converted_through_a_table_keep_every_samples_colour`, `raster::tests::a_malformed_tint_for_an_unused_sample_keeps_the_used_colours`; (29) `raster::tests::reading_colors_keep_highlight_annotations_readable`; (30) `text::tests::the_annex_d2_alias_codes_read_as_an_ordinary_hyphen_and_space`; (31) `raster::tests::shadings_with_large_sampled_functions_paint_quickly`, `raster::tests::sampled_functions_interpolate_their_grid_as_before`; (32) `raster::tests::decoded_images_serve_every_size_they_are_drawn_at`; (33) `raster::tests::patch_meshes_keep_their_corner_colours`; (34) `raster::tests::spot_colour_images_run_their_tint_transform_once_per_sample_value`; (35) `mesh_extent_tests::{a_mesh_paints_its_triangles_alone, a_mesh_keeps_the_graphics_state_alpha, a_mesh_off_the_canvas_paints_nothing}` (`cargo test -p hayro`); (36) `raster::tests::mesh_triangles_sharing_an_edge_leave_no_seam`; (38) `raster::tests::default_colour_spaces::device_rgb_operators_draw_in_default_rgb`, `raster::tests::default_colour_spaces::device_cmyk_operators_draw_in_default_cmyk`, `raster::tests::default_colour_spaces::device_gray_operators_draw_in_default_gray`, `raster::tests::default_colour_spaces::device_images_draw_in_the_default_colour_spaces`, `raster::tests::default_colour_spaces::device_shadings_draw_in_the_default_colour_spaces`, `raster::tests::default_colour_spaces::an_image_drawn_with_and_without_a_default_keeps_both_colours`, `raster::tests::default_colour_spaces::forms_inherit_the_default_colour_spaces_they_do_not_override`, `raster::tests::default_colour_spaces::unusable_defaults_leave_device_colour_alone`; (39) `raster::tests::knockout_groups_replace_what_their_elements_cover`; (40) `raster::tests::visibility_expressions_decide_what_a_membership_dictionary_shows`; (41) `raster::tests::a_multiply_highlight_inside_a_transparency_group_keeps_the_text`; (42) `raster::tests::rendering_modes_but_3_and_7_draw_type3_glyphs_as_filled`; (43) `raster::tests::lab_honours_the_space_white_point`, `raster::tests::indexed_palettes_over_lab_keep_their_colours`; (44) `text::tests::identity_to_unicode_names_give_text`; (46) `raster::tests::stencil_masks_are_drawn_at_the_fill_opacity_once`; (47) `raster::tests::a_five_point_polyline_is_not_painted_as_its_bounding_box`, `raster::tests::a_path_that_visits_every_corner_out_of_order_is_not_a_rectangle`, `raster::tests::a_rectangle_is_still_painted_as_a_rectangle`; (48) `raster::tests::icc_profile_cache::icc_profiles_with_identical_dictionaries_draw_their_own_colours`, `raster::tests::icc_profile_cache::icc_profiles_used_again_are_read_once`; (49) `raster::tests::a_font_whose_post_version_has_swapped_halves_is_still_used`; (50) `font::blob::tests::glyph_names_resolve_to_the_first_glyph_with_that_name`; (51) `context::saved_state_tests::repeated_saves_stay_bounded_and_balanced`; (52) `font::cid::tests::w2_run_past_the_largest_cid_keeps_the_rest_of_the_table` (hayro-interpret); (55) `raster::tests::devicen_inks::a_tint_transform_that_cannot_be_evaluated_draws_the_rest_of_the_page_and_warns`, `raster::tests::devicen_inks::a_tint_transform_that_cannot_be_read_falls_back_and_warns`; (56) `raster::tests::invisible_unknown_annotations_and_popups_paint_nothing`; (57) `substitute_widths::a_substitute_keeps_its_glyph_shapes_at_wider_advances`, `substitute_widths::a_substitute_still_narrows_glyphs_to_narrower_advances` (pdfcraft-render `tests/it`); (58) `cid_fallback_query::a_cid_font_fallback_request_carries_its_descendants_style`, `cid_fallback_query::a_cid_font_without_a_descriptor_asks_for_the_default_style` (pdfcraft-render integration tests); (59) `raster::tests::no_rotate_notes_stay_upright_at_their_corner_on_rotated_pages` (hayro-interpret); (64) `type3_text_clip::{clip_only_type3_text_is_not_rendered_and_leaves_no_clip, invisible_type3_text_is_not_rendered, other_modes_render_type3_glyphs_filled_and_never_clip, only_outline_glyphs_shape_a_clip_shared_with_type3_text, self_nesting_type3_text_in_a_clip_mode_stays_bounded}` (`cargo test -p pdfcraft-render --test it`), `raster::tests::rendering_modes_but_3_and_7_draw_type3_glyphs_as_filled`, `text::tests::clipping_mode_type3_text_is_kept_by_the_ocr_filter`; (66) `separation_all::{separation_all_fills_paint_the_complemented_tint_as_gray, separation_all_strokes_paint_the_complemented_tint_as_gray, separation_all_images_paint_each_sample_as_the_complemented_gray, separation_all_shadings_paint_the_complemented_gray, separation_none_paints_nothing, a_named_separation_still_paints_through_its_alternate, a_device_n_space_naming_all_still_paints_through_its_alternate}` (`cargo test -p pdfcraft-render --test it`); (63) `color::device_cmyk_gray_tests::{each_ink_darkens_by_its_iso_weight, ink_past_full_coverage_is_black_and_bad_components_read_as_zero}` (hayro-interpret), `it::luminosity_group_space::{black_ink_in_a_device_cmyk_group_masks_fully_and_no_ink_shows_fully, each_process_ink_in_a_device_cmyk_group_masks_by_its_weight, a_device_cmyk_backdrop_colour_masks_by_its_gray, gray_and_rgb_groups_mask_as_before, device_cmyk_on_the_page_keeps_its_display_colour}` (pdfcraft-render); (71) `raster::tests::an_indexed_lookup_table_is_read_in_every_legal_form` (pdfcraft-render); (72) `raster::tests::an_uncoloured_pattern_uses_its_underlying_space_in_every_form` (pdfcraft-render); (73) `raster::tests::the_jbig2_images_of_a_page_share_one_allowance`, `a_picture_whose_jbig2_mask_is_past_the_pages_allowance_is_not_drawn` and `a_pages_jbig2_allowance_ends_with_the_page` (pdfcraft-render); (74) `raster::tests::{a_chain_of_colour_spaces_ends, colour_spaces_are_read_to_the_limit_of_their_nesting, a_chain_built_on_spaces_already_read_ends, a_profile_read_at_the_limit_keeps_its_alternate_for_a_read_nearer_the_top}` (pdfcraft-render); (75) `raster::tests::a_soft_hyphen_a_substitute_lacks_draws_its_hyphen` (pdfcraft-render) | -| hayro | 0.7.1 | Apache-2.0 OR MIT | (1) `RenderSettings::{x_offset, y_offset}` to render one tile of a large page; (2) images over `MAX_IMAGE_PIXELS` (2^28) are skipped (a fuzzed `/W 4294967295` hung in resampling); (3) `tiling_cell_scale` keeps a tiling pattern's cell pixmap within 3000 px a side whatever its `/XStep`/`/YStep` (a fuzzed step far beyond `/BBox` allocated 2 GB); (4) stroke widths clamped to 4 × (canvas width + height) device pixels (a fuzzed `/LW 9223372036854775807` allocated 10 GB in stroke expansion); (5) `usable_dash_pattern`: a stroke whose dash pattern could cut it into more than `MAX_DASHES_PER_STROKE` (250,000) pieces, counting kurbo's restart of the pattern at every subpath and `ClosePath`, is drawn solid (a fuzzed `/D [[11] 0]` on a line from x = 92234775807 allocated over 5 GB, `[0.000001] 0 d` on a 20 pt line over 10 GB); (6) a dash pattern with a negative or non-finite entry, or a period that isn't positive, is drawn solid (`[-1 -1] 0 d` looped for ever in kurbo); (7) `render_size` and `render_into`: render into a buffer the caller allocates (PdfCraft renders into pixel-aligned memory that the GUI takes over as texture data); `render` is `render_into` on a new pixmap; (8) checked image source/target/RGBA, alpha-premultiplication scratch and intermediate sizes before resampling, finite transforms and exact source/mask buffers; sources wider than u16 may shrink before backend limits, with a 2^20 source-axis cap and conservative 128 MiB filter-planning estimate; an over-limit or failed resize keeps the original image resolution and geometry when the source fits the backend, otherwise skips it, with a logged warning; staging, resampling output/scratch, RGBA conversion and padding buffers use fallible reserves; (9) `may_paint_canvas`: a fill or stroke whose control-point box, grown by how far its paint reaches past the path (half the stroke width times the larger of the miter limit and √2) plus 0.5 user units and 2 pixels, misses the canvas is not drawn, so a tile no longer strokes and fills every path of its page (item 5 of #623); `set_skip_offscreen_paths` turns this off for the regression test; (10) `/Interpolate` governs the pre-resize too (#624): a minified image that does not request interpolation is pre-resized with nearest-neighbour, planned without the Catmull-Rom filter limits (`image_resampling_size` takes the flag); a Type 3 glyph image is sampled bicubically only when interpolated; a minified image with a matching soft mask is nearest-sampled straight into premultiplied RGBA; (11) `may_paint_rect`: an image whose stream-size box, mapped to the device, misses the canvas is skipped before decoding (decoded data can only shrink against that box), so a tile of an image-heavy page no longer decodes, resamples and rasterizes every image on it; (12) preview renders (`InterpreterSettings::preview`: backdrops, thumbnails, grid cells, signature-drag backgrounds) skip images under `PREVIEW_MIN_IMAGE_AREA` device pixels, while final renders always draw them; (13) reading colours (#897): `RenderSettings::color_replacement` takes a `ColorReplacement` (text colour, background, `only_black`, `line_art`) that remaps solid text and line-art paint while rendering, glyph outlines counting as text and stencil-mask ink as line art; images, image alpha masks, soft masks and pattern cells keep their colours; `render_pdf` gains a `color_replacement: Option` argument (pass `None` for true colours); the upstream-only `[dev-dependencies.log]` entry is removed from `Cargo.toml` (log is a normal dependency since the logging patch), so `cargo test -p hayro` runs this crate's unit tests; (14) a shrunk, interpolated image whose axes lie along the device's (upright, mirrored or a quarter turn, as on a page under `/Rotate`) is snapped to whole device pixels: shrunk once to its rounded device rectangle and drawn 1:1 from the rounded corner, instead of a ceil-sized raster resampled again at a fractional scale and offset (`snap_minified_image`; scans looked soft next to Acrobat, and until #5530 a turned one still did, up to 200 grey levels off the same page drawn upright). An image counts as axis-aligned when the terms the snap drops move its far edges by at most a hundredth of a device pixel (`SNAP_SKEW`), not by a tolerance on the coefficients, which let a slight skew of a very wide image through; and the snapped 1:1 transform is used only when the shrink produced the snapped size, so a failed resize keeps the full transform instead of drawing the source raster at its source size; (15) a soft mask of another size than its image is filtered when shrunk, whatever its `/Interpolate` (#1649): such a mask carries the shape on its own (Word saves text as a tiny black image with a glyph-bitmap mask), and nearest-neighbour dropped every stroke thinner than the shrink factor; (16) under a soft mask or a blend mode, axial and radial shadings are sampled instead of drawn as native gradients (#2128): vello_cpu 0.0.8 copies an opaque gradient over fully covered tiles without the mask or blend mode, so PowerPoint's faded gradient fills came out solid; (17) the renderer keeps annotation paints out of the reading-colours mapping between `begin_annotations` and `end_annotations` (#1548); (18) the page renders as one transparency group; under reading colours its background is painted inside it as the initial backdrop, so a Multiply highlight blends with the dark background instead of staying an opaque block, and otherwise the background stays outside, so a blend mode over an empty part of the page keeps its colour instead of blending with the white paper (#2970); (19) `set_paint` samples a mesh shading's texture over its triangles alone (`EncodedShadingPattern::extent`, padded rather than repeated) and skips a mesh with none on the canvas: a BricsCAD plan's 310 one-triangle `sh` each covered the whole tile, 2 s per 1024 px tile at any zoom (the watchdog gave the page up on a busy machine), now about 70 ms; (20) a tiling pattern whose step is too large for a cell pixmap, where what is painted (a fill, or a stroke with its width) can only show the first cell (PowerPoint's `/XStep 2147480100` for a picture fill drawn once), has its `/BBox` drawn at full scale with a transparent border and padded instead of tiled (`shows_one_cell`; a cell whose scale isn't finite and positive, from a `/BBox` too thin to divide by, keeps the old path: `usable_cell_scale`); shrinking the cell to fit the 3000 px limit of (3) left the picture under a pixel, so network diagrams lost their pictures (#3349); (22) knockout groups: while one is innermost, fills, strokes, glyph outlines and shadings are drawn with `Compose::Copy` (weighted by coverage), so each element replaces what earlier elements of the group drew instead of compositing over it; an image or nested group, drawn into a layer of its own, first has what is under its shape cleared (`knockout_image` for an unmasked image's rectangle, only when `draw_image` will draw it, decoded at the size it is drawn; a knockout erase pass that draws a nested group's content with every paint clearing instead of painting, a shading fill's clear clipped to the shading's `/BBox` as its paint is) and then composites over it (#2971); (23) backport of hayro 0.8: shadings that `as_svg_gradient` can represent are painted as a `vello_cpu` linear or two-point radial gradient instead of a texture sampled pixel by pixel through the shading function (InDesign fades are luminosity soft masks of one full-page axial `sh`: 40 on a report page took 15 s at 144 dpi, now under 1 s); without upstream's half-pixel shift, which suits vello_cpu 0.3 but moves 0.0.8's gradients off pixel centres; (25) a tiling cell is drawn at the scale that makes one `/XStep` (`/YStep`) exactly the cell pixmap's whole number of pixels (`tiling_cell_pixels`, at least 1 and at most the cap), so the repeated pixmap puts cells at whole multiples of the step at any device scale; drawing at the unrounded scale repeated the cell up to half a pixel off at every step, and the error grew with distance from the pattern's origin (ISO 32000-2 8.7.3.1) | (1) `raster::tests::tiles_match_full_render`; (2) `raster::tests::absurd_image_dimensions_are_skipped`; (3) `raster::tests::tiling_cells_stay_small_whatever_the_step`; (4) `raster::tests::absurd_predictor_columns_and_line_widths_render`; (5) `raster::tests::dash_patterns_that_cut_a_stroke_into_billions_of_pieces_render`; (6) `raster::tests::invalid_dash_patterns_terminate_and_are_drawn_solid`; (7) `raster::tests::pixels_are_a_plain_renders_bytes_in_a_buffer_a_gui_can_take_over`; (8) `raster::tests::image_resampling_rejects_anisotropic_target_growth`, `raster::tests::image_resampling_bounds_intermediate_before_planning`, `raster::tests::image_resampling_rejects_invalid_sources_and_scales`, `raster::tests::image_resampling_keeps_valid_sizes_and_exact_limits`, `raster::tests::resampling_fallback_preserves_original_geometry_and_pixels`, `raster::tests::image_resampling_wide_sources_shrink_before_backend_side_limits`, `raster::tests::image_resampling_wide_transparent_sources_still_shrink`, `raster::tests::image_resampling_padded_backend_dimensions_stay_checked`, `raster::tests::image_resampling_mismatched_alpha_mask_keeps_pixels_and_placement`; (9) `raster::tests::skipping_paths_off_the_canvas_changes_no_pixel`; (10) `raster::tests::non_interpolated_images_are_not_smoothed_when_minified`, `raster::tests::non_interpolated_gray_images_are_not_smoothed_when_minified`, `raster::tests::non_integer_minification_samples_the_same_columns`, `raster::tests::non_integer_minification_samples_colour_and_mask_from_the_same_columns`, `raster::tests::non_interpolated_alpha_images_keep_their_colours_and_mask_when_minified`, `raster::tests::non_interpolated_type3_glyph_images_are_not_smoothed_when_minified`, `raster::tests::image_resampling_nearest_accepts_wide_sources_the_filter_refuses`, `raster::tests::wide_non_interpolated_sources_draw_after_shrinking`; (11) `raster::tests::skipping_paths_off_the_canvas_changes_no_pixel`; (12) `raster::tests::preview_renders_skip_tiny_images_but_finals_draw_everything`; (13) `color_replacement_tests::{black_maps_to_text_colour, white_maps_to_background, midgrey_interpolates, alpha_copied_verbatim, only_black_threshold, line_art_false_skips_paths_but_not_text}`, `color_replacement_render_tests::{color_replacement_maps_text_and_line_art, color_replacement_maps_stencil_mask_ink}`; (14) `raster::tests::interpolated_minified_image_edges_snap_to_whole_pixels` (`cargo test -p hayro`), `raster::tests::a_turned_or_mirrored_minified_image_snaps_to_whole_pixels`, `rotated_image_snap::a_shrunk_image_on_a_turned_page_is_the_upright_render_turned` (`cargo test -p pdfcraft-render --test it`), `renderer::snap_tests::a_skew_that_moves_a_far_edge_is_not_snapped` (`cargo test -p hayro`); (15) `raster::tests::shrunk_soft_mask_larger_than_its_image_keeps_thin_strokes`; (16) `raster::tests::soft_mask_applies_to_a_shading_pattern_fill`; (17), (18) `raster::tests::reading_colors_keep_highlight_annotations_readable`; (19) `mesh_extent_tests::{a_mesh_paints_its_triangles_alone, a_mesh_keeps_the_graphics_state_alpha, a_mesh_off_the_canvas_paints_nothing}` (`cargo test -p hayro`); (20) `raster::tests::a_tiling_pattern_drawn_once_shows_its_cell`, `raster::tests::a_large_step_still_tiles_where_the_next_cell_shows`, `raster::tests::a_tiling_cell_too_thin_to_scale_keeps_the_old_path`; (22) `raster::tests::knockout_groups_replace_what_their_elements_cover`; (23) `raster::tests::axial_and_radial_shadings_draw_as_native_gradients`; (25) `tiling_cell_placement::{far_cells_of_a_fill_sit_at_whole_steps, far_cells_sit_at_whole_steps_under_a_turned_scaled_matrix, far_cells_of_a_stroke_sit_at_whole_steps, a_cell_pixmap_spans_exactly_one_step}` (`cargo test -p pdfcraft-render --test it`) | +| hayro | 0.7.1 | Apache-2.0 OR MIT | (1) `RenderSettings::{x_offset, y_offset}` to render one tile of a large page; (2) images over `MAX_IMAGE_PIXELS` (2^28) are skipped (a fuzzed `/W 4294967295` hung in resampling); (3) `tiling_cell_scale` keeps a tiling pattern's cell pixmap within 3000 px a side whatever its `/XStep`/`/YStep` (a fuzzed step far beyond `/BBox` allocated 2 GB); (4) stroke widths clamped to 4 × (canvas width + height) device pixels (a fuzzed `/LW 9223372036854775807` allocated 10 GB in stroke expansion); (5) `usable_dash_pattern`: a stroke whose dash pattern could cut it into more than `MAX_DASHES_PER_STROKE` (250,000) pieces, counting kurbo's restart of the pattern at every subpath and `ClosePath`, is drawn solid (a fuzzed `/D [[11] 0]` on a line from x = 92234775807 allocated over 5 GB, `[0.000001] 0 d` on a 20 pt line over 10 GB); (6) a dash pattern with a negative or non-finite entry, or a period that isn't positive, is drawn solid (`[-1 -1] 0 d` looped for ever in kurbo); (7) `render_size` and `render_into`: render into a buffer the caller allocates (PdfCraft renders into pixel-aligned memory that the GUI takes over as texture data); `render` is `render_into` on a new pixmap; (8) checked image source/target/RGBA, alpha-premultiplication scratch and intermediate sizes before resampling, finite transforms and exact source/mask buffers; sources wider than u16 may shrink before backend limits, with a 2^20 source-axis cap and conservative 128 MiB filter-planning estimate; an over-limit or failed resize keeps the original image resolution and geometry when the source fits the backend, otherwise skips it, with a logged warning; staging, resampling output/scratch, RGBA conversion and padding buffers use fallible reserves; (9) `may_paint_canvas`: a fill or stroke whose control-point box, grown by how far its paint reaches past the path (half the stroke width times the larger of the miter limit and √2) plus 0.5 user units and 2 pixels, misses the canvas is not drawn, so a tile no longer strokes and fills every path of its page (item 5 of #623); `set_skip_offscreen_paths` turns this off for the regression test; (10) `/Interpolate` governs the pre-resize too (#624): a minified image that does not request interpolation is pre-resized with nearest-neighbour (since (21), only inside Type 3 glyphs), planned without the Catmull-Rom filter limits (`image_resampling_size` takes the flag); a Type 3 glyph image is sampled bicubically only when interpolated; a minified image with a matching soft mask is nearest-sampled straight into premultiplied RGBA; (11) `may_paint_rect`: an image whose stream-size box, mapped to the device, misses the canvas is skipped before decoding (decoded data can only shrink against that box), so a tile of an image-heavy page no longer decodes, resamples and rasterizes every image on it; (12) preview renders (`InterpreterSettings::preview`: backdrops, thumbnails, grid cells, signature-drag backgrounds) skip images under `PREVIEW_MIN_IMAGE_AREA` device pixels, while final renders always draw them; (13) reading colours (#897): `RenderSettings::color_replacement` takes a `ColorReplacement` (text colour, background, `only_black`, `line_art`) that remaps solid text and line-art paint while rendering, glyph outlines counting as text and stencil-mask ink as line art; images, image alpha masks, soft masks and pattern cells keep their colours; `render_pdf` gains a `color_replacement: Option` argument (pass `None` for true colours); the upstream-only `[dev-dependencies.log]` entry is removed from `Cargo.toml` (log is a normal dependency since the logging patch), so `cargo test -p hayro` runs this crate's unit tests; (14) a shrunk image that is interpolated (or, since (21), area-averaged) and whose axes lie along the device's (upright, mirrored or a quarter turn, as on a page under `/Rotate`) is snapped to whole device pixels: shrunk once to its rounded device rectangle (halves round up, so a tile snaps an edge where the whole page does) and drawn 1:1 from the rounded corner, instead of a ceil-sized raster resampled again at a fractional scale and offset (`snap_minified_image`; scans looked soft next to Acrobat, and until #5530 a turned one still did, up to 200 grey levels off the same page drawn upright). An image counts as axis-aligned when the terms the snap drops move its far edges by at most a hundredth of a device pixel (`SNAP_SKEW`), not by a tolerance on the coefficients, which let a slight skew of a very wide image through; and the snapped 1:1 transform is used only when the shrink produced the snapped size, so a failed resize keeps the full transform instead of drawing the source raster at its source size; (15) a soft mask of another size than its image is filtered when shrunk, whatever its `/Interpolate` (#1649): such a mask carries the shape on its own (Word saves text as a tiny black image with a glyph-bitmap mask), and nearest-neighbour dropped every stroke thinner than the shrink factor; (16) under a soft mask or a blend mode, axial and radial shadings are sampled instead of drawn as native gradients (#2128): vello_cpu 0.0.8 copies an opaque gradient over fully covered tiles without the mask or blend mode, so PowerPoint's faded gradient fills came out solid; (17) the renderer keeps annotation paints out of the reading-colours mapping between `begin_annotations` and `end_annotations` (#1548); (18) the page renders as one transparency group; under reading colours its background is painted inside it as the initial backdrop, so a Multiply highlight blends with the dark background instead of staying an opaque block, and otherwise the background stays outside, so a blend mode over an empty part of the page keeps its colour instead of blending with the white paper (#2970); (19) `set_paint` samples a mesh shading's texture over its triangles alone (`EncodedShadingPattern::extent`, padded rather than repeated) and skips a mesh with none on the canvas: a BricsCAD plan's 310 one-triangle `sh` each covered the whole tile, 2 s per 1024 px tile at any zoom (the watchdog gave the page up on a busy machine), now about 70 ms; (20) a tiling pattern whose step is too large for a cell pixmap, where what is painted (a fill, or a stroke with its width) can only show the first cell (PowerPoint's `/XStep 2147480100` for a picture fill drawn once), has its `/BBox` drawn at full scale with a transparent border and padded instead of tiled (`shows_one_cell`; a cell whose scale isn't finite and positive, from a `/BBox` too thin to divide by, keeps the old path: `usable_cell_scale`); shrinking the cell to fit the 3000 px limit of (3) left the picture under a pixel, so network diagrams lost their pictures (#3349); (21) an image that does not request interpolation and is drawn below its resolution, outside a Type 3 glyph, is shrunk by area averaging (`area_shrink`: each device pixel the coverage-weighted mean of the samples it covers, in 16.16 fixed point whose weights sum to exactly one, premultiplied when the image has a mask; a soft mask no larger than the image is first sampled onto the image's grid by `mask_on_image_grid`, so its colour and mask are averaged together instead of the averaged colour of transparent samples showing under a mask layer at the mask's edges) to its snapped device rectangle, instead of nearest-neighbour: `/Interpolate` asks for smoothing only when an image is enlarged (ISO 32000-2 §8.9.5.3), and nearest-neighbour kept one sample in N per axis, so the text of a 300 dpi scan at 100% lost most of its strokes (sampling them again at a fractional offset dropped more). PicScale's `Box` and `Area` filters were not used: shrinking, both average a window four target pixels wide. On a 13-page 300 dpi scan every rendered sample at 96, 120 and 192 dpi is within 1 of the exact area average (it was up to 255 off, 17-23% of samples by more than 1). A snapped image whose shrink failed keeps the full transform, so it is not drawn at its source size. The shrink never enlarges: a side drawn at or above its resolution (an image shrunk across and stretched down, as a 204 × 98 dpi fax is) keeps its samples for the sampler. Rows are averaged first (contiguous, cheap) and the fewer rows that leaves are then shrunk across with taps worked out once per image, which made the shrink about three times faster (a 300 dpi page at 96 dpi: 25 ms, nearest-neighbour 11 ms). A source with a side over `MAX_RESAMPLING_SOURCE_SIDE` (2^20) keeps (10)'s nearest-neighbour shrink, and `area_shrink` refuses it, so its accumulator row and tap table stay a few times one source row (a 2^27 × 2 image of zeros is a tiny file); (22) knockout groups: while one is innermost, fills, strokes, glyph outlines and shadings are drawn with `Compose::Copy` (weighted by coverage), so each element replaces what earlier elements of the group drew instead of compositing over it; an image or nested group, drawn into a layer of its own, first has what is under its shape cleared (`knockout_image` for an unmasked image's rectangle, only when `draw_image` will draw it, decoded at the size it is drawn; a knockout erase pass that draws a nested group's content with every paint clearing instead of painting, a shading fill's clear clipped to the shading's `/BBox` as its paint is) and then composites over it (#2971); (23) backport of hayro 0.8: shadings that `as_svg_gradient` can represent are painted as a `vello_cpu` linear or two-point radial gradient instead of a texture sampled pixel by pixel through the shading function (InDesign fades are luminosity soft masks of one full-page axial `sh`: 40 on a report page took 15 s at 144 dpi, now under 1 s); without upstream's half-pixel shift, which suits vello_cpu 0.3 but moves 0.0.8's gradients off pixel centres; (25) a tiling cell is drawn at the scale that makes one `/XStep` (`/YStep`) exactly the cell pixmap's whole number of pixels (`tiling_cell_pixels`, at least 1 and at most the cap), so the repeated pixmap puts cells at whole multiples of the step at any device scale; drawing at the unrounded scale repeated the cell up to half a pixel off at every step, and the error grew with distance from the pattern's origin (ISO 32000-2 8.7.3.1); (28) an image of page-scan size (`SMOOTHED_IMAGE_SIDE`: 512 samples or more on its shorter side) that does not request interpolation is drawn with `ImageQuality::Medium` (bilinear, as interpolated images are) when enlarged on either side, also when the other side is shrunk (the area shrink leaves that side at 1:1), outside a Type 3 glyph: scans rarely set `/Interpolate`, and drawn nearest-neighbour every sample of a zoomed scan became a block, so its text had staircase edges; icons, QR codes and pixel art keep their hard pixels. Bicubic (`High`) looked the same on a zoomed 300 dpi scan and drew four 1024 px tiles at 400% in 267 ms against 50 ms | (1) `raster::tests::tiles_match_full_render`; (2) `raster::tests::absurd_image_dimensions_are_skipped`; (3) `raster::tests::tiling_cells_stay_small_whatever_the_step`; (4) `raster::tests::absurd_predictor_columns_and_line_widths_render`; (5) `raster::tests::dash_patterns_that_cut_a_stroke_into_billions_of_pieces_render`; (6) `raster::tests::invalid_dash_patterns_terminate_and_are_drawn_solid`; (7) `raster::tests::pixels_are_a_plain_renders_bytes_in_a_buffer_a_gui_can_take_over`; (8) `raster::tests::image_resampling_rejects_anisotropic_target_growth`, `raster::tests::image_resampling_bounds_intermediate_before_planning`, `raster::tests::image_resampling_rejects_invalid_sources_and_scales`, `raster::tests::image_resampling_keeps_valid_sizes_and_exact_limits`, `raster::tests::resampling_fallback_preserves_original_geometry_and_pixels`, `raster::tests::image_resampling_wide_sources_shrink_before_backend_side_limits`, `raster::tests::image_resampling_wide_transparent_sources_still_shrink`, `raster::tests::image_resampling_padded_backend_dimensions_stay_checked`, `raster::tests::image_resampling_mismatched_alpha_mask_keeps_pixels_and_placement`; (9) `raster::tests::skipping_paths_off_the_canvas_changes_no_pixel`; (10) `raster::tests::non_interpolated_images_are_not_smoothed_when_minified`, `raster::tests::non_interpolated_gray_images_are_not_smoothed_when_minified`, `raster::tests::non_integer_minification_averages_the_columns_each_pixel_covers`, `raster::tests::non_integer_minification_averages_colour_and_mask_together`, `raster::tests::non_interpolated_alpha_images_keep_their_colours_and_mask_when_minified`, `raster::tests::non_interpolated_type3_glyph_images_are_not_smoothed_when_minified`, `raster::tests::image_resampling_nearest_accepts_wide_sources_the_filter_refuses`, `raster::tests::wide_non_interpolated_sources_draw_after_shrinking`; (11) `raster::tests::skipping_paths_off_the_canvas_changes_no_pixel`; (12) `raster::tests::preview_renders_skip_tiny_images_but_finals_draw_everything`; (13) `color_replacement_tests::{black_maps_to_text_colour, white_maps_to_background, midgrey_interpolates, alpha_copied_verbatim, only_black_threshold, line_art_false_skips_paths_but_not_text}`, `color_replacement_render_tests::{color_replacement_maps_text_and_line_art, color_replacement_maps_stencil_mask_ink}`; (14) `raster::tests::minified_image_edges_snap_to_whole_pixels`, `raster::tests::a_tile_snaps_half_pixel_image_edges_as_the_whole_page_does` (`cargo test -p hayro`), `raster::tests::a_turned_or_mirrored_minified_image_snaps_to_whole_pixels`, `rotated_image_snap::a_shrunk_image_on_a_turned_page_is_the_upright_render_turned` (`cargo test -p pdfcraft-render --test it`), `renderer::snap_tests::a_skew_that_moves_a_far_edge_is_not_snapped` (`cargo test -p hayro`); (15) `raster::tests::shrunk_soft_mask_larger_than_its_image_keeps_thin_strokes`; (16) `raster::tests::soft_mask_applies_to_a_shading_pattern_fill`; (17), (18) `raster::tests::reading_colors_keep_highlight_annotations_readable`; (19) `mesh_extent_tests::{a_mesh_paints_its_triangles_alone, a_mesh_keeps_the_graphics_state_alpha, a_mesh_off_the_canvas_paints_nothing}` (`cargo test -p hayro`); (20) `raster::tests::a_tiling_pattern_drawn_once_shows_its_cell`, `raster::tests::a_large_step_still_tiles_where_the_next_cell_shows`, `raster::tests::a_tiling_cell_too_thin_to_scale_keeps_the_old_path`; (21) `raster::tests::non_interpolated_minification_keeps_thin_strokes`, `raster::tests::non_integer_minification_averages_the_columns_each_pixel_covers`, `raster::tests::non_integer_minification_averages_colour_and_mask_together`, `raster::tests::non_interpolated_minification_does_not_bleed_transparent_colour`, `raster::tests::a_smaller_soft_mask_does_not_bleed_transparent_colour_when_minified`, `raster::tests::minified_image_edges_snap_to_whole_pixels`, `raster::tests::an_image_shrunk_one_way_and_stretched_the_other_is_still_averaged`, `raster::tests::wide_non_interpolated_sources_draw_after_shrinking`, `renderer::area_shrink_tests` (`cargo test -p hayro`); (22) `raster::tests::knockout_groups_replace_what_their_elements_cover`; (23) `raster::tests::axial_and_radial_shadings_draw_as_native_gradients`; (25) `tiling_cell_placement::{far_cells_of_a_fill_sit_at_whole_steps, far_cells_sit_at_whole_steps_under_a_turned_scaled_matrix, far_cells_of_a_stroke_sit_at_whole_steps, a_cell_pixmap_spans_exactly_one_step}` (`cargo test -p pdfcraft-render --test it`); (28) `raster::tests::large_non_interpolated_images_are_smoothed_when_enlarged`, `raster::tests::small_non_interpolated_images_keep_hard_pixels_when_enlarged`, `raster::tests::a_scan_shrunk_one_way_is_smoothed_the_way_it_is_stretched` | | hayro-jbig2 | 0.3.0 | Apache-2.0 OR MIT | (1) at most 2^28 pixels per bitmap and page bitmap, and 2^25 cells per halftone grid and grayscale image (each side could be 65535: 4.3 billion pixels decoded one by one, and a 4-byte value per cell); (2) at most 2^26 symbol pixels per symbol dictionary (a fuzzed dictionary decoded for minutes); (3) a symbol dictionary's height classes are capped at one per new symbol and its export runs at `2 × symbols + 1`, each plus `EXTRA_EMPTY_STEPS` (1024): an empty height class or a zero-length export run doesn't advance, and past the end of its data the arithmetic decoder can decode them for ever (one changed byte hung the renderer); (4) a segment header's long-form referred-to segment count is capped at `u16::MAX` and a larger count is refused before its retention or referred-to bytes are read: the 29-bit count went straight to `Vec::with_capacity`, a reservation of about 2 GiB whose failure aborts the process (from storytold/craft-libs#176); (5) what a page's segments hold for later ones (intermediate regions, pattern and symbol dictionaries with their headers and retained contexts, and Huffman tables) is at most 64 MiB, two square bitmaps of 16,384 a side (`MAX_RETAINED_WORDS`), and a dictionary's new and exported symbols are counted against it as they are made: each was bounded on its own, but a few dozen bytes an intermediate region held 33 MB each, and 8 MiB symbols exported again and again killed the process, so a few kilobytes could exhaust memory; (6) a segment may refer to at most 2^20 symbols, checked before any are collected: references may repeat, and 65,535 of them to one dictionary, in a 78 KB stream, asked for 17 GB; (7) a pattern dictionary keeps at most 16 MiB of shifted copies, which only speed up drawing, and none that wouldn't fit what the page has left (`MAX_SHIFTED_WORDS`); (8) a custom Huffman table has at most 2^16 lines, and a stream with its globals at most 2^20 segments and 2^22 references, checked as they are read: both grew before anything counted them, and PDFs of a kilobyte or so reached 5 GB and more behind nested Flate; (9) a page past the bounds of (1) is refused when the stream is opened: a caller sizes its output from the header before the page bitmap is made, and an Indexed image in a 743-byte PDF asked for 137 GB, which aborts; (10) a stream may ask for at most 2^30 units of decoding work, four of the largest pages, about 6 s to 17 s (`MAX_WORK`): decoding a bitmap costs a pixel each and 4 a row, copying one a word each and 1 a row, and each integer an item reads costs 38, what one takes at worst, so one with no pixels pays too, before it is decoded: each bitmap was bounded, but nothing bounded how many there were, and a 795-byte PDF took 265 s; (11) `Image::decode_within` spends from what a run of images may still do, starting with its page's pixels, and `charge_reading` charges reading a stream and its globals before anything is parsed, so an operation over a document can bound what `MAX_WORK` bounds for one image, and images that fail or decode nothing add up too: a file of thousands of images made to reach that limit took as many times as long; (12) adding to a location, a reference offset, a symbol's corner or a Huffman range line near the end of an `i32`, and negating a decoded integer of 2^31, overflowed: a panic wherever overflow is checked. The refinement context, a text region's corners, the integer decoder and the Huffman table wrap, as a release build did, and `Bitmap::combine` saturates, which comes to the same, as a location that overflows is past the end of any page. What is drawn doesn't change | (1) `raster::tests::jbig2_regions_of_billions_of_pixels_are_refused`, `raster::tests::jbig2_halftone_grids_past_their_bound_are_refused`; (2) the nightly `cargo xtask fuzz` finding, which now finishes in about a second; (3) `raster::tests::jbig2_symbol_dictionaries_that_never_advance_terminate`; (4) `segment::tests::test_referred_to_count_is_capped` (`cargo test --manifest-path vendor/hayro-jbig2/Cargo.toml`); (5) `raster::tests::jbig2_regions_held_past_a_pages_bound_are_refused`, `jbig2_symbols_without_pixels_count_towards_a_pages_bound`, `jbig2_retained_contexts_count_towards_a_pages_bound`, `jbig2_huffman_tables_count_towards_a_pages_bound`, `jbig2_new_symbols_count_towards_a_pages_bound`, `jbig2_new_symbols_are_held_to_what_a_page_has_left`, `jbig2_pattern_dictionaries_count_towards_a_pages_bound`, and in the crate (`cargo test --manifest-path vendor/hayro-jbig2/Cargo.toml`) `decode::symbol::tests::exported_symbols_are_counted_before_they_are_copied`, `tests::the_room_left_shrinks_as_segments_are_stored`; (6) `raster::tests::jbig2_segments_referring_to_too_many_symbols_are_refused`; (7) `raster::tests::jbig2_pattern_dictionaries_too_large_to_shift_still_draw`, `jbig2_shifted_patterns_never_refuse_an_image`; (8) `raster::tests::jbig2_huffman_tables_of_too_many_lines_are_refused`, `jbig2_streams_of_too_many_segments_are_refused`, `jbig2_streams_of_too_many_references_are_refused`, and in the crate `tests::the_segments_of_a_stream_and_its_globals_count_together`; (9) `raster::tests::jbig2_pages_past_the_bounds_are_refused_before_anything_is_sized_from_them`, `tests::a_page_past_the_bounds_is_refused_when_opened`; (10) `raster::tests::jbig2_decoding_work_is_counted_item_by_item`; (11) `jbig2::decode::tests::a_run_of_images_spends_its_allowance_down_to_the_step` (pdfcraft-filters); (12) `raster::tests::jbig2_placements_at_the_end_of_an_i32_draw_nothing` | | lopdf | 0.45.0 | MIT | (1) PNG predictor rows longer than the data are refused before their buffers are allocated (a fuzzed `/Columns 4294967295` allocated two 4 GiB rows), and empty frames return without row buffers; (2) predictor row widths use saturating arithmetic or reject overflow; (3) sub-byte TIFF Predictor 2 scratch is bounded by the samples present in the input instead of the declared `/Colors`, aligned with [lopdf#589](https://github.com/J-F-Liu/lopdf/pull/589); (4) the object lookup keeps its cycle history through compressed objects, requires an ordinary in-file container, and caps the chain of indirect lookups at 16 (a chain of new objects, each a `/Length` compressed in a container whose own `/Length` names the next, or a stream whose `/Length` names the next stream, overflowed the stack: a dev build's 2 MiB thread within 800 hops); (5) object-stream members are parsed within their own extent (up to the next distinct offset), an entry whose offset repeats an earlier entry's is skipped (so a repeated object number can resolve to its earlier entry), and offsets are processed in index order; (6) `find_latest_trailer`: a rebuild takes a `/Type /XRef` stream dictionary as the trailer when the file has no `trailer` keyword (the latest in the file wins, as a later `trailer` would), so a concatenated or truncated file keeps its /Info and /ID instead of failing to load; a candidate whose /Root is the catalog beats one whose /Root is another dictionary, and an in-file /Root must match its generation (#2365) | (1) `inspect::tests::png_predictor_rows_longer_than_the_data_are_refused_before_allocating`, `inspect::tests::empty_png_predictor_frames_do_not_allocate_rows`; (2) `inspect::tests::tiff_subbyte_predictor_row_width_overflow_is_refused`; (3) `inspect::tests::tiff_subbyte_predictor_scratch_is_bounded_by_available_samples`, `inspect::tests::tiff_subbyte_predictors_keep_components_rows_and_padding`; (4) `inspect::tests::compressed_object_self_container_does_not_recurse`, `inspect::tests::compressed_object_mutual_containers_do_not_recurse`, `inspect::tests::object_stream_length_cycle_does_not_recurse`, `inspect::tests::compressed_object_container_chain_does_not_recurse`, `inspect::tests::compressed_length_chain_does_not_recurse`, `inspect::tests::plain_stream_length_chain_does_not_recurse`, `inspect::tests::compressed_objects_and_indirect_stream_length_still_load`, `inspect::tests::compressed_length_in_a_container_with_an_indirect_length_still_loads`; (5) `inspect::tests::object_stream_member_extents_bound_owned_strings`, `inspect::tests::object_stream_member_extents_preserve_index_order_and_whitespace`; (6) `inspect::tests::rebuilt_file_keeps_the_info_of_its_cross_reference_stream`, `inspect::tests::rebuilt_file_prefers_a_trailer_whose_root_is_the_catalog` | | hayro-syntax | 0.7.2 | Apache-2.0 OR MIT | (1) Page-tree cycle guard in `resolve_pages` (visited set + depth cap): a `/Kids` loop among compressed objects overflowed the stack and aborted the process; (2) `ccitt_size_ok`: CCITT images wider than 2^20 columns or over 2^28 pixels are not decoded (a fuzzed `/Columns 4294967295` allocated 4 GiB); (3) Flate/LZW predictor row widths are saturating `usize`, a zero width is refused and a row longer than the data is not allocated (a fuzzed `/Columns 9223372036854775807` wrapped to a 2^61-byte allocation and aborted); (4) the inline-image end-of-data check (`EI`) doesn't nest inside its own look-ahead and examines at most 256 candidates and steps (50 images with "EI" in their data ran for over ten minutes); an EI followed by a delimiter (EI%..., EI[...) ends the image like one followed by white space; only a regular character after EI skips the candidate; (5) a stream whose `/Length` is wrong is recovered only when `stream` is the next token — scanning ahead to a later object's `stream` made an indirect appearance-state dictionary (Quartz checkboxes: `/AP /N` → `<< /Yes … /Off … >>`) render as a non-form, so the widget was skipped; (6) JPX rescaling uses an exact integer sample range for 1–32-bit components (32 shifted past u32, and floating-point maxima at 25–31 bits wrapped to zero), and refuses zero-sized rows; (7) `decode_limit`: Flate (through flate2 and the fallback inflater), LZW, RunLength and ASCII85 never write past `MAX_DECODED_STREAM` (256 MiB), or, for the last filter of an image of known size, past twice its pixel bytes (a predictor tag byte per byte), at most 1 GiB; `Stream::decoded_within` for budgeted callers; a page's `/Contents` array shares one `MAX_DECODED_STREAM` (a 2 MB page whose content stream or 8 × 8 image inflated to 2 GB took the renderer past 3 GB in under a second; naming the stream twenty times in `/Contents`, 5 GB); (8) high-precision numbers (jsPDF MediaBox coordinates with over 19 digits) parse without overflowing the u64 mantissa: fractional digits past it are dropped, integer digits past it scale the value (a real, kept finite); (9) large dictionary indexes are built on first use and shared across repeated reads within each PDF (#307); the validated-byte cache is bounded, reader contexts remain separate, repairs and inline/content dictionaries bypass sharing, and small dictionaries stay eager; (10) decoded object-stream slots publish once under contention, cold loading rejects recursive dependencies, and xref repair serializes without holding recursive read guards; this permits one parser per native render pool with private worker caches; (11) `sync.rs`: a poisoned lock is recovered instead of unwrapped (a worker panic no longer cascades into every reader of the same PDF), and xref repair takes a blocking write lock instead of asserting a non-blocking one; (12) `SegmentList` (`util.rs`) returns `None` for an out-of-range index instead of panicking; (13) a page's `/UserUnit` scales `render_dimensions` and `initial_transform`, so the page and everything on it are sized in points, as Acrobat shows them; it is read as Acrobat reads it (the page's own entry, not inherited; a number from 1 up, at most 75,000; anything else counts as 1), and a page without it keeps exactly the transform it had; (14) revision 2–4 passwords are tried as given and then in PDFDocEncoding (`legacy_passwords` in `crypto/mod.rs`, which resolves the upstream `TODO: Convert to PDFDocEncoding`): passwords arrive as UTF-8 text, so a password with characters outside ASCII (`Çok güzel`, `ılık€`) written in PDFDocEncoding, as ISO 32000-2 §7.6.4.3.2 asks and qpdf and Acrobat write it, was always wrong; (15) page content is read a piece at a time (`page::ContentReader`, `Page::content_reader`): plain streams and Flate streams without a predictor are decoded as they are read, with a budget of 64 times their compressed size and never less than `MAX_DECODED_STREAM` (a stream that expands past it is cut and reported); other streams are decoded whole, as before, and share `MAX_DECODED_STREAM`. The content tokenizer reads a window at a time (`UntypedIter::new_window`, `TypedIter::needs_more`, `TypedIter::committed`): an instruction that runs past the window is not decided until more data follows, which `Reader` records by flagging reads and `at_end` past the end of the data, including through the clones the parser makes. The Flate decoder keeps its own 32 KiB history, so it pauses at any output size. `ContentReader::new` reuses raw/decrypted stream bytes for budget accounting and piece creation, and `decoded_raw_within` applies the existing Whole-stream filters and limits to those bytes without decrypting twice; (16) revision 5–6 (AES-256) passwords are tried in their SASLprep form (RFC 4013, then UTF-8, at most 127 bytes, as ISO 32000-2 §7.6.4.3.3 Algorithm 2.A asks and pdfcraft-crypt writes them) and then, if that differs, as given (`aes256_passwords` in `crypto/mod.rs`, which resolves the upstream `TODO: Do the above.`; `stringprep`, already in the tree through pdfcraft-crypt, is an optional dependency enabled by `std`, and without `std` passwords are used as given): the renderer used the password as typed, so one that SASLprep changes (a decomposed `café`, a no-break space, the fullwidth `ａｂ` a Japanese IME types in full-width mode) was wrong, although pdfcraft-cos accepted it: protecting a document with such an open password was rolled back ("the password is incorrect"), and a file with such a password did not open with it.; (17) CCITT `/Rows 0` means the height is not predetermined (ISO 32000-2 Table 11), not zero rows: scanned Group 4 pages written that way rendered blank; (18) The streaming Flate decoder (`lzw_flate::flate::fallback`) copies a back-reference in runs of up to its distance (`extend_from_within`), not byte by byte. The copy reads only bytes already in the output, and a copy whose source does not exist still writes nothing, so the bytes are those a byte-by-byte copy writes. Inflating the A1 sample's content stream (278.6 MB) takes about 14% less time (0.66 s to 0.56 s); the output is identical; (19) CCITT `/Rows` below the image's `/Height` decodes to the height: a plotter driver wrote `/Rows` equal to `/Columns`, so only the first rows were decoded and the rest drawn black (#3308) (20) `stream` followed by spaces or tabs and then the line break (`stream \r\n`, as Microsoft Reporting Services writes it) starts the data after the line break; the content stream was taken to start at the space and the page rendered blank.; (21) a JP2 colour specification naming an enumerated colour space the decoder refuses (CIEJab in a veraPDF fixture, YCbCr, PhotoYCC, YPbPr, bilevel…) no longer fails the image: it is decoded again from a copy whose `colr` names the space the decoder assumes without one, greyscale, sRGB or CMYK by component count, as MuPDF and Poppler draw it (`without_refused_enumerated_colour`; the page drew blank, #3306); (22) an object stream's index of `objnum offset` pairs is parsed once per stream and kept beside its decoded data (`ObjectStreamIndexes`, `XRef::with_object_stream_indexes`), and an offset past `usize` is malformed instead of wrapping: `XRef::get_with` parsed the whole index again for every object it fetched from the stream, so reading all of a stream of N objects took N² steps (a Firefox page whose 65,542 objects share one stream took 37 s to draw and 40 s to extract its text); (23) the inline-image end-of-data look-ahead no longer rejects a candidate `EI` because a byte above 0x7F lies between it and the next one: when the candidate is the end, those bytes are the content stream that follows the image, and its strings carry such bytes (WinAnsi `(Grüße) Tj`, the two-byte codes of any Type 0 font), so the operator check that follows decides alone, bounded as in (4) (a scanned page whose OCR text held an accented word and a later capital "EI" — EINGANG, BEI, KLEIN — drew and extracted nothing after the image: 40 of 275 pages across five real documents); (24) `xref::prev_offset`: a trailer `/Prev` written as a reference `N G R` is read as the offset that object holds when the section's own entries place it (iText's `indirectPrev.pdf`; each section's entries are read before its `Prev` is followed), and otherwise as offset N (tiff2pdf; JHOVE's PDF-HUL-157 files, #2366), as `pdfcraft-cos` and other readers take it, so the earlier sections' pages are seen instead of one; (25) direct arrays and dictionaries share a reader-local nesting cap of 128, counted while skipping and while parsing a dictionary (its lenient repair reads, rather than skips, an object found where a key belongs, so `<< << …` recursed through that read); deeper objects return `None` instead of overflowing the stack (including unused trailer values, trailer entries, content operands and marked-content properties); a chain of 65 direct stitching functions now stops at this cap, before the 64-function guard; (26) an xref stream subsection must be backed by its decoded data: all-zero `/W` widths (an entry reads no bytes) and a declared count longer than the data are refused before any entry is inserted, where a tiny stream with a large `/Size` or `/Index` count inserted that many entries (from storytold/craft-libs#239); a subsection running past object number `i32::MAX` is refused instead of overflowing or turning negative; (27) an xref stream's type field is read big-endian at its full `/W[0]` width, where only its first byte was read and wider fields shifted every later entry, and `/W[0]` over 8 bytes is refused like `/W[1]` (from storytold/craft-libs#241); (28) a content instruction keeps up to 256 operands (`MAX_OPERANDS`) instead of 10, and past that drops the oldest instead of ending the content stream: the eleventh operand of a twelve-ink DeviceN `scn` (ISO 32000-2 §8.6.6.5 allows any number of components) blanked the rest of the page (#4260); (29) a `/JBIG2Globals` stream that is itself JBIG2 isn't decoded: globals are raw segments, and a chain of 16,000 such streams overflowed the stack, which aborts the process; (31) `set_page_jbig2_work`: the JBIG2 images decoded on a thread spend one allowance, set per page, 2^30 steps (`MAX_PAGE_JBIG2_WORK`), so a page of many images made to reach their own limit stops at one, and what they cost to read counts, whether they decode or not; unbounded until a page sets it; (32) a group of array items read together (a tuple: an image's `/Decode`, `/W`, `/W2`, and a function's `/Domain`, `/Range`, `/Encode` and `/Decode`) can start with a reference: it was taken for the whole group and lost, which ended the array, so an image with `/Decode [7 0 R 0]` didn't draw and CID widths fell back to `/DW`; (33) an object that holds only a reference stands for what that names, stored on its own or in an object stream, up to 32 references: read as a number it was the reference's own object number, so a `/Decode` or `/W` entry naming such an object took a wrong value, and as anything else it was lost. Past 32, as any cycle goes, it reads as a missing object. An object stream isn't reached that way, as its checks are of the object named; reading its index counts against the 64 nested loads and reads its dictionary alone, as a chain of them, each with a member of the next for its `/N`, `/First` or `/Length`, ran out of stack; and a comment inside a reference counts as white space; (34) a table rebuilt by scanning the file (`fallback_xref_map`) finds an object by its number when its header names another generation, including one no generation may be (`3 18446744073709551616 obj` in `poppler-85140-0.pdf`, #5580): the header is read leniently only in that scan (`read_object_header`), the object is also filed under generation 0 unless a real `N 0 obj` holds that key (a later header for the number moves the stand-in), `XRef::get_with` matches by object number only in a repaired table, and `XRef::objects` yields each location once | (1) `cos/tests/writer.rs::page_tree_loops_in_object_streams_do_not_crash_readers`; (2) `raster::tests::absurd_mask_and_fax_sizes_are_refused_before_decoding`; (3) `raster::tests::absurd_predictor_columns_and_line_widths_render`; (4) `raster::tests::inline_images_with_ei_in_their_data_parse_in_linear_time`, `content::tests::inline_image_ei_may_be_followed_by_a_delimiter`; (5) `raster::tests::indirect_appearance_state_dict_is_not_another_stream`; (6) `raster::tests::jpx_stream_bit_depth_boundaries_do_not_panic`, `raster::tests::jpx_32_bit_image_does_not_prevent_other_page_content_from_rendering`, `raster::tests::jpx_gray_rgb_and_alpha_keep_eight_bit_samples`, `raster::tests::jpx_streams_reject_out_of_range_bit_depths`; (7) `raster::tests::decompression_bombs_stop_at_the_stream_limit`, `raster::tests::page_contents_and_nested_forms_share_one_budget`; (8) `inspect::tests::jspdf_high_precision_mediabox_does_not_overflow_dimensions`, `object::number::tests::high_precision_float_does_not_overflow_mantissa` (hayro-syntax); (9) `object::dict::lazy_index_tests`, `raster::tests::large_resource_indexes_preserve_pixels_and_text`; (10) `data::tests`, `xref::tests::concurrent_cold_object_stream_reads_all_succeed`, `raster::tests::shared_parser_is_lazy_and_initializes_once_under_contention`; (11) `data::tests::panicking_initializer_wakes_waiters_and_prevents_retries`, `xref::tests::concurrent_cold_object_stream_reads_all_succeed`; (12) `util::tests::segment_list_rejects_out_of_range_indices`; (13) `raster::tests::user_unit_scales_the_page_and_everything_on_it`; (14) `crypto::pdfcraft_password_tests`, `pdfcraft-engine` `tests::legacy_passwords_with_turkish_characters_open`; (15) `content::tests::windows_keep_every_instruction_at_every_split`, `content::tests::windows_keep_every_instruction_in_pieces_of_any_size`, `content::tests::windows_match_the_slice_on_random_content`, `content::tests::windows_drop_blanks_and_keep_a_comment_that_may_go_on`, `page::content_reader_tests::content_past_the_cap_is_read_to_the_end`, `page::content_reader_tests::a_stream_that_expands_past_the_budget_is_cut_there`, `page::content_reader_tests::a_small_expansion_under_the_cap_is_not_cut`, `raster::tests::page_content_past_the_old_cap_renders`, `raster::tests::content_reader_returns_the_same_bytes_in_any_pieces`, `raster::tests::content_bombs_stop_at_the_budget_and_say_so`, `page::content_reader_tests::plain_and_whole_pieces_keep_their_bytes`, `page::content_reader_tests::repeated_content_reference_counts_compressed_bytes_once`, `page::content_reader_tests::whole_cap_still_counts_later_unique_streams`; (16) `crypto::pdfcraft_password_tests::aes256_passwords_are_tried_in_their_saslprep_form_then_as_given`, `crypto::pdfcraft_password_tests::aes256_passwords_open_as_typed_and_in_their_saslprep_form`, `pdfcraft-engine` `tests::aes256_open_passwords_that_saslprep_changes_protect_and_reopen_as_typed`, `tests::aes256_files_whose_passwords_saslprep_changes_open_as_typed`; (17) `raster::tests::group4_fax_with_rows_zero_decodes_to_the_image_height`; (18) `filter::lzw_flate::tests::back_reference_longer_than_its_distance_repeats_its_source`, `back_reference_cut_by_the_limit_resumes_in_the_next_call` and `back_reference_reads_bytes_written_by_an_earlier_call`; each fails when the copy returns the wrong remaining count; (19) `raster::tests::group4_fax_with_rows_below_its_height_decodes_to_the_image_height`; (20) `raster::tests::space_before_the_line_break_after_stream_still_renders`; (21) `raster::tests::a_jp2_image_in_a_refused_enumerated_colour_space_is_drawn`, `filter::jpx::tests::refused_enumerated_colour_spaces_become_the_component_counts_default`, `filter::jpx::tests::what_the_decoder_handles_or_cant_be_read_is_left_alone` (`cargo test --manifest-path vendor/hayro-syntax/Cargo.toml`); (22) `raster::tests::objects_in_one_large_object_stream_are_read_in_linear_time`; (23) `text::tests::text_after_an_inline_image_survives_a_non_ascii_byte_before_a_later_ei`, `content::tests::an_inline_image_ends_at_its_own_ei_when_later_text_holds_a_non_ascii_byte` (hayro-syntax); (24) `xref::tests::prev_as_reference_reaches_the_earlier_section`, `xref::tests::prev_as_reference_to_an_object_holding_the_offset_reads_that_offset`, `pdfcraft-cos` `document::tests::a_prev_written_as_a_reference_still_reaches_the_earlier_sections`, `document::tests::a_prev_reference_to_an_object_holding_the_offset_reads_that_offset`; (25) `raster::tests::deeply_nested_direct_objects_do_not_overflow_the_stack`, `raster::tests::dictionaries_nested_without_keys_in_content_do_not_overflow_the_stack`, `raster::tests::dictionaries_nested_without_keys_in_the_trailer_do_not_overflow_the_stack`, `raster::tests::dictionaries_nested_without_keys_as_marked_content_properties_do_not_overflow_the_stack`, `raster::tests::dictionaries_nested_without_keys_are_read_up_to_the_nesting_cap`, `raster::tests::nesting_at_the_cap_fits_a_512_kib_stack` (native only), `raster::tests::function_limits_depth_guard_refuses_a_chain_of_references`; (26) `xref::tests::xref_stream_entries_need_backing_bytes`; (27) `xref::tests::xref_stream_type_field_uses_the_full_w0_width`, `xref::tests::xref_stream_loader_rejects_oversized_w0` (`cargo test --manifest-path vendor/hayro-syntax/Cargo.toml`); (28) `content::ops::tests::scn_with_twelve_components_keeps_them_all`, `content::ops::tests::operands_past_the_cap_keep_the_latest_in_order` (hayro-syntax), `raster::tests::devicen_inks::a_twelve_ink_devicen_colour_paints`, `raster::tests::devicen_inks::a_four_ink_devicen_colour_paints`, `raster::tests::devicen_inks::an_operator_with_hundreds_of_operands_does_not_end_the_content`; (29) `raster::tests::jbig2_globals_that_are_jbig2_themselves_are_not_decoded`, `raster::tests::jbig2_globals_are_read`, `raster::tests::the_jbig2_images_of_a_page_share_one_allowance`, `raster::tests::an_image_that_fails_spends_the_pages_allowance_too`; (32) `raster::tests::a_decode_array_with_references_in_it_is_read`, `raster::tests::a_functions_arrays_can_start_with_a_reference` (pdfcraft-render), `font::cid::width_tests::a_width_group_can_start_with_a_reference` (hayro-interpret); (33) `raster::tests::a_reference_to_a_reference_is_what_it_names`, `raster::tests::a_decode_array_with_references_in_it_is_read` (pdfcraft-render), `font::cid::width_tests::a_width_group_can_start_with_a_reference` (hayro-interpret), `xref::tests::an_object_stream_isnt_reached_through_a_reference`, `xref::tests::a_chain_of_object_streams_ends_at_the_limit_of_nested_loads`, `xref::tests::a_chain_of_object_streams_takes_no_more_work_for_two_ways_on`, `xref::tests::a_reference_held_in_either_storage_is_followed`, `xref::tests::a_page_a_reference_is_held_to_is_found_once` (hayro-syntax); (34) `raster::tests::a_rebuilt_table_finds_an_object_whose_header_names_another_generation`, `raster::tests::a_rebuilt_tables_generation_0_stand_in_is_not_a_second_object` (pdfcraft-render), `xref::tests::an_object_stream_and_its_first_member_are_both_enumerated` (hayro-syntax) | diff --git a/vendor/hayro/src/renderer.rs b/vendor/hayro/src/renderer.rs index 0b9cfe779e..120a751281 100644 --- a/vendor/hayro/src/renderer.rs +++ b/vendor/hayro/src/renderer.rs @@ -193,6 +193,27 @@ fn image_buffer(len: usize) -> Option> { Some(out) } +/// PdfCraft patch: `mask` sampled onto the grid of a `width` × `height` image, each image sample +/// taking the mask sample under its centre. `None` for an empty side or a buffer over the limits. +fn mask_on_image_grid(mask: &LumaData, width: u32, height: u32) -> Option> { + if width == 0 || height == 0 || mask.width == 0 || mask.height == 0 { + return None; + } + let mut out = image_buffer(source_byte_len(width, height, 1, MAX_IMAGE_PIXELS)?)?; + let mut columns = Vec::new(); + columns.try_reserve_exact(width as usize).ok()?; + columns.extend((0..width).map(|x| nearest_source_index(mask.width, width, x))); + let mask_width = mask.width as usize; + for y in 0..height { + let start = nearest_source_index(mask.height, height, y).checked_mul(mask_width)?; + let row = mask.data.get(start..start.checked_add(mask_width)?)?; + for &x in &columns { + out.push(*row.get(x)?); + } + } + Some(out) +} + /// PdfCraft patch (#624): the source index that PicScale's nearest-neighbour plan copies for /// destination `index` of `dst` samples taken from `src`. It is the same fixed-point centre /// sampling as `ResampleNearestPlan`, so these are the samples a nearest resize would copy. @@ -236,6 +257,177 @@ fn nearest_premultiplied_rgba( Some(out) } +/// PdfCraft patch: the shorter side, in samples, from which an image that did not ask for +/// `/Interpolate` is still smoothed when enlarged. A page scanned at 72 dpi is 612 samples wide; +/// icons, QR codes and pixel art are usually far smaller and keep their hard pixels. +const SMOOTHED_IMAGE_SIDE: u32 = 512; + +/// PdfCraft patch: fixed-point unit of [`area_shrink`]'s weights. +const AREA_ONE: u64 = 1 << 16; + +/// PdfCraft patch: the source samples target sample `t` covers when `src` samples shrink to `dst` +/// (`dst <= src`), with their weights in [`AREA_ONE`] units. Target `t` spans `[t·src, (t+1)·src)` +/// and source `s` spans `[s·dst, (s+1)·dst)`, both in units of 1/(src·dst) of the axis, so every +/// overlap is exact. The weights are differences of one rounded running sum, so they add up to +/// exactly [`AREA_ONE`] and a flat area keeps its exact value. +fn area_taps(src: u32, dst: u32, t: u32) -> impl Iterator { + let (src, dst, t) = (u64::from(src), u64::from(dst), u64::from(t)); + // `t < dst <= src <= 2^28` (MAX_IMAGE_PIXELS), so every product below fits in a u64. + let (lo, hi) = (t * src, (t + 1) * src); + let fixed = move |x: u64| (x * AREA_ONE + src / 2) / src; + (lo / dst..hi.div_ceil(dst)).map(move |s| { + let from = (s * dst).max(lo) - lo; + let to = ((s + 1) * dst).min(hi) - lo; + (s as usize, fixed(to) - fixed(from)) + }) +} + +/// PdfCraft patch: [`area_taps`] of every target sample of one axis, worked out once: for each +/// target, the first source sample it covers and the weights of the samples from there on (the +/// samples a target covers are consecutive). There are at most `src + dst` taps in all. +struct AreaAxis { + first: Vec, + /// Where each target's weights start in `weights`, and where the last one's end. + offsets: Vec, + weights: Vec, +} + +impl AreaAxis { + fn new(src: u32, dst: u32) -> Option { + let n = dst as usize; + let (mut first, mut offsets, mut weights) = (Vec::new(), Vec::new(), Vec::new()); + first.try_reserve_exact(n).ok()?; + offsets.try_reserve_exact(n.checked_add(1)?).ok()?; + weights.try_reserve_exact((src as usize).checked_add(n.checked_mul(2)?)?).ok()?; + offsets.push(0); + for t in 0..dst { + let mut start = None; + for (s, w) in area_taps(src, dst, t) { + start.get_or_insert(s); + // A weight is at most AREA_ONE, so it fits in a u32. + weights.push(w as u32); + } + first.push(start?); + offsets.push(weights.len()); + } + Some(Self { first, offsets, weights }) + } + + /// Target `t`'s first source sample and its weights. + fn get(&self, t: usize) -> Option<(usize, &[u32])> { + let (lo, hi) = (*self.offsets.get(t)?, *self.offsets.get(t.checked_add(1)?)?); + Some((*self.first.get(t)?, self.weights.get(lo..hi)?)) + } +} + +/// PdfCraft patch: shrinks `N`-channel samples from `source` to `target` (no side larger than its +/// source side) by area averaging: each target sample is the mean of the source it covers, +/// weighted by how much of it it covers. Nothing is invented between samples, which is what +/// `/Interpolate` governs (ISO 32000-2 §8.9.5.3, for images drawn above their resolution), and +/// nothing is dropped: nearest-neighbour keeps one sample in N per axis, so a scan shown at screen +/// size lost the strokes of its text. Premultiplied input stays premultiplied. Work memory is one +/// source row of `u32` accumulators, the horizontal taps (at most `sw + 2 × tw`) and a vertically +/// shrunk copy no larger than the source; a side over `MAX_RESAMPLING_SOURCE_SIDE` is refused, so +/// none of it exceeds a few times one source row. `None` if nothing shrinks, the sizes don't fit +/// or a buffer cannot be allocated. +fn area_shrink(data: &[u8], source: (u32, u32), target: (u32, u32)) -> Option> { + let ((sw, sh), (tw, th)) = (source, target); + // A side over MAX_RESAMPLING_SOURCE_SIDE would make the accumulator row and the tap table + // several times larger than the samples (a 2^27 × 2 image of zeros is a tiny file). + if tw == 0 || th == 0 || tw > sw || th > sh || sw > MAX_RESAMPLING_SOURCE_SIDE || sh > MAX_RESAMPLING_SOURCE_SIDE { + return None; + } + let (sw_us, tw_us) = (sw as usize, tw as usize); + if data.len() != source_byte_len(sw, sh, N, MAX_IMAGE_PIXELS)? { + return None; + } + // Weights sum to AREA_ONE (2^16) and samples are at most 255, so every sum fits in a u32. + let round = |acc: u32| ((acc + (AREA_ONE as u32) / 2) >> 16).min(255) as u8; + // Vertical pass first: groups of whole source rows averaged into each target row. Rows are + // contiguous, so this pass is cheap, and it leaves the horizontal pass, which gathers samples + // per target, a third as many rows when a scan shrinks threefold. + let tall = if th == sh { + None + } else { + let row_len = sw_us * N; + let mut rows = image_buffer(source_byte_len(sw, th, N, MAX_IMAGE_PIXELS)?)?; + let mut acc: Vec = Vec::new(); + acc.try_reserve_exact(row_len).ok()?; + acc.resize(row_len, 0); + for t in 0..th { + acc.fill(0); + for (s, w) in area_taps(sh, th, t) { + let row = data.get(s * row_len..(s + 1) * row_len)?; + // A weight is at most AREA_ONE, so it fits in a u32. + let w = w as u32; + for (a, v) in acc.iter_mut().zip(row) { + *a += w * u32::from(*v); + } + } + rows.extend(acc.iter().map(|a| round(*a))); + } + Some(rows) + }; + if tw == sw { + return tall; + } + let rows = tall.as_deref().unwrap_or(data); + // Horizontal pass: each of those rows shrunk to `tw` samples. The taps depend only on the + // widths, so they are worked out once, not again for each row. + let taps = AreaAxis::new(sw, tw)?; + let mut out = image_buffer(image_byte_len(tw, th, N, MAX_IMAGE_PIXELS)?)?; + for row in rows.chunks_exact(sw_us * N) { + for t in 0..tw_us { + let (first, weights) = taps.get(t)?; + let samples = row.get(first * N..(first + weights.len()) * N)?; + let mut acc = [0_u32; N]; + for (w, px) in weights.iter().zip(samples.chunks_exact(N)) { + for (a, v) in acc.iter_mut().zip(px) { + *a += w * u32::from(*v); + } + } + out.extend(acc.map(round)); + } + } + Some(out) +} + +/// PdfCraft patch: [`area_shrink`] for an image's pixel format. If it cannot run, the image keeps +/// its source resolution, as when [`Renderer::resize_image_data`] fails. +fn area_resize( + data: Vec, + source: (u32, u32), + target: (u32, u32), + format: ImagePixelFormat, +) -> (Vec, u32, u32) { + let shrunk = match format { + ImagePixelFormat::Luma => area_shrink::<1>(&data, source, target), + ImagePixelFormat::Rgb => area_shrink::<3>(&data, source, target), + ImagePixelFormat::Rgba => area_shrink::<4>(&data, source, target), + }; + match shrunk { + Some(out) => (out, target.0, target.1), + None => { + log::warn!( + "Image area shrink failed or exceeded its buffer limit; falling back to source data" + ); + (data, source.0, source.1) + } + } +} + +/// PdfCraft patch: premultiplies straight RGBA samples in place. +fn premultiply_rgba(rgba: &mut [u8]) { + let (chunks, _) = rgba.as_chunks_mut::<4>(); + for chunk in chunks { + let [r, g, b, a] = *chunk; + *chunk = AlphaColor::from_rgba8(r, g, b, a) + .premultiply() + .to_rgba8() + .to_u8_array(); + } +} + /// PdfCraft patch: most pieces one dashed stroke may be cut into. Stroke expansion keeps every /// dash, so a pattern that is tiny next to its path asks for billions: a fuzzed `/D [[11] 0]` on /// a line from x = 92234775807 allocated over 5 GB, and `[0.000001] 0 d` on a 20 pt line over @@ -423,9 +615,12 @@ fn snap_minified_image( planned: (u32, u32), ) -> Option<(u32, u32, Affine)> { let [a, b, c, d, e, f] = transform.as_coeffs(); + // Halves round up, not away from zero: a tile's origin moves edges by whole pixels, and only + // then do they snap where the whole page snaps them (-2.5 must go to -2 as 2.5 goes to 3). + let snap = |v: f64| (v + 0.5).floor(); // The device span of a source edge from `origin` across `side` samples of `scale`, rounded. let edge = |origin: f64, scale: f64, side: u32| { - let (p, q) = (origin.round(), (origin + scale * f64::from(side)).round()); + let (p, q) = (snap(origin), snap(origin + scale * f64::from(side))); let (lo, hi) = (p.min(q), p.max(q)); (hi - lo >= 1.0 && lo.abs() < 1e6 && hi.abs() < 1e6).then_some((lo, hi)) }; @@ -935,17 +1130,51 @@ impl Renderer { return; } let interpolate = image_data.interpolate(); - if let Some(a) = &alpha_data { + let requested_resize = x_scale < 1.0 || y_scale < 1.0; + // PdfCraft patch: an image that did not ask for interpolation and is drawn below its + // resolution is shrunk by area averaging (`area_shrink`), not nearest-neighbour, which + // broke up the text of scans shown at screen size. Type 3 glyph images keep the nearest + // shrink of #624: they are drawn with a frame of their own and never snapped. + let area = !interpolate + && requested_resize + && !self.in_type3_glyph + // Wider sources keep #624's bounded nearest-neighbour shrink (`area_shrink` refuses + // them): skipping them, as a refused shrink at source size would, loses the image. + && source_width.max(source_height) <= MAX_RESAMPLING_SOURCE_SIDE; + let mut alpha_data = alpha_data; + if let Some(a) = alpha_data.take() { if source_byte_len(a.width, a.height, 1, MAX_IMAGE_PIXELS) != Some(a.data.len()) { log::warn!("Image has invalid or over-limit alpha dimensions/data and was skipped"); return; } if a.width != source_width || a.height != source_height || a.interpolate != interpolate { - if let Some(alpha) = alpha_data { - self.draw_image_with_alpha_mask(image_data, alpha); + // PdfCraft patch: a soft mask no larger than its area-shrunk image is sampled onto + // the image's grid, so colour and mask are averaged together, premultiplied. Drawn + // as a mask layer over the image instead, the averaged colour (transparent samples + // included) was multiplied by the averaged mask, and the colour of transparent + // samples showed at the mask's edges. A larger mask carries detail the image does + // not (#1649) and keeps the layer. + let on_grid = (area && a.width <= source_width && a.height <= source_height) + .then(|| mask_on_image_grid(&a, source_width, source_height)) + .flatten(); + match on_grid { + Some(data) => { + alpha_data = Some(LumaData { + data, + width: source_width, + height: source_height, + interpolate, + ..a + }); + } + None => { + self.draw_image_with_alpha_mask(image_data, a); + return; + } } - return; + } else { + alpha_data = Some(a); } } let mut quality = if interpolate { @@ -955,7 +1184,6 @@ impl Renderer { }; let has_alpha = alpha_data.is_some(); let mut may_have_transparency = has_alpha || self.force_images_may_have_transparency(); - let requested_resize = x_scale < 1.0 || y_scale < 1.0; let (new_width, new_height) = match image_resampling_size( source_width, source_height, @@ -985,12 +1213,28 @@ impl Renderer { if requested_resize && self.in_type3_glyph && interpolate { quality = ImageQuality::High; } - // PdfCraft patch: an interpolated image that is shrunk, with its axes along the device's - // (upright, mirrored or a quarter turn), lands on whole device pixels. It is shrunk once to - // exactly its snapped device rectangle and drawn 1:1 from the snapped corner. Otherwise the - // shrunk raster (rounded up to a whole size) is sampled a second time at a fractional scale - // and offset, which blurs scans and photos. - let snapped = if interpolate + // PdfCraft patch: an image the size of a page scan is smoothed when drawn above its + // resolution, even without `/Interpolate`: bilinearly, as images that ask for it are. + // Scans rarely ask, and drawn nearest-neighbour each sample of a zoomed scan became a + // block, so the edges of its text were staircases. Bicubic looked the same on a zoomed + // 300 dpi scan and drew its tiles five times slower. Smaller images (icons, QR codes, + // pixel art) keep their hard pixels. A scan shrunk on one side and stretched on the other + // is smoothed too: the area shrink leaves the shrunk side at 1:1, where bilinear adds + // nothing, and the stretched side would otherwise be drawn nearest-neighbour. + if !interpolate + && !self.in_type3_glyph + && (x_scale > 1.0 || y_scale > 1.0) + && source_width.min(source_height) >= SMOOTHED_IMAGE_SIDE + { + quality = ImageQuality::Medium; + } + // PdfCraft patch: an image that is shrunk (filtered or area-averaged), with its axes along + // the device's (upright, mirrored or a quarter turn), lands on whole device pixels. It is + // shrunk once to exactly its snapped device rectangle and drawn 1:1 from the snapped + // corner. Otherwise the shrunk raster (rounded up to a whole size) is sampled a second time + // at a fractional scale and offset, which blurs scans and photos (or, by the nearest + // sampler of a non-interpolated image, drops samples again). + let snapped = if (interpolate || area) && requested_resize && !self.in_type3_glyph && (new_width, new_height) != (source_width, source_height) @@ -1000,13 +1244,22 @@ impl Renderer { None }; let (new_width, new_height) = snapped.map_or((new_width, new_height), |s| (s.0, s.1)); + // PdfCraft patch: an area shrink never enlarges. A side drawn at or above its resolution + // (an image shrunk across and stretched down, as a 204 × 98 dpi fax is, or a snapped side + // rounded past its source) keeps its samples and is enlarged by the sampler; the snap + // then no longer matches, so the full transform places the image. + let (new_width, new_height) = if area { + (new_width.min(source_width), new_height.min(source_height)) + } else { + (new_width, new_height) + }; let needs_resize = (new_width, new_height) != (source_width, source_height); - // PdfCraft patch (#624): a minified image with a soft mask that is not interpolated is - // nearest-sampled straight into premultiplied target RGBA. That skips the full-resolution - // RGBA copy below and the premultiply pass after it. + // PdfCraft patch (#624): a minified Type 3 glyph image with a soft mask that is not + // interpolated is nearest-sampled straight into premultiplied target RGBA. That skips the + // full-resolution RGBA copy below and the premultiply pass after it. let direct = match &alpha_data { - Some(alpha) if needs_resize && !interpolate => nearest_premultiplied_rgba( + Some(alpha) if needs_resize && !interpolate && !area => nearest_premultiplied_rgba( &image_data, &alpha.data, (source_width, source_height), @@ -1014,7 +1267,7 @@ impl Renderer { ), _ => None, }; - let premultiplied = direct.is_some(); + let mut premultiplied = direct.is_some(); // Preserve the single-channel/RGB fast paths. With alpha, expand directly to RGBA; // every source and alpha length was validated before zipping or allocating. let (data, format) = match (direct, image_data, alpha_data) { @@ -1057,6 +1310,15 @@ impl Renderer { }; let (data, mut img_width, mut img_height) = if premultiplied { (data, new_width, new_height) + } else if needs_resize && area { + let mut data = data; + if matches!(format, ImagePixelFormat::Rgba) { + // Averaging straight colours would bleed the colour of transparent samples into + // their neighbours; premultiplied colours count each by its coverage. + premultiply_rgba(&mut data); + premultiplied = true; + } + area_resize(data, (source_width, source_height), (new_width, new_height), format) } else if needs_resize { self.resize_image_data( data, @@ -1103,14 +1365,7 @@ impl Renderer { rgba }; if has_alpha && !premultiplied { - let (chunks, _) = rgba_data.as_chunks_mut::<4>(); - for chunk in chunks { - let [r, g, b, a] = *chunk; - *chunk = AlphaColor::from_rgba8(r, g, b, a) - .premultiply() - .to_rgba8() - .to_u8_array(); - } + premultiply_rgba(&mut rgba_data); } // Type 3 glyphs get a transparent two-pixel frame for interpolation. Check the padded // RGBA allocation and u16 dimensions too, before narrowing or growing either side. @@ -2635,3 +2890,109 @@ mod snap_tests { assert!(snap_minified_image(&turned_noisy, source, (20_000, 200)).is_some()); } } + +/// PdfCraft patch: the area-averaging shrink of non-interpolated images. +#[cfg(test)] +mod area_shrink_tests { + use super::{ + AREA_ONE, AreaAxis, LumaData, MAX_IMAGE_PIXELS, area_shrink, area_taps, mask_on_image_grid, + }; + + fn mask(data: &[u8], width: u32, height: u32) -> LumaData { + LumaData { data: data.to_vec(), width, height, interpolate: false, scale_factors: (1.0, 1.0) } + } + + /// Each image sample takes the mask sample under its centre, at whole and fractional ratios. + #[test] + fn a_mask_is_sampled_onto_the_image_grid() { + assert_eq!(mask_on_image_grid(&mask(&[10, 20], 2, 1), 4, 2), Some(vec![10, 10, 20, 20, 10, 10, 20, 20])); + // Centres of 7 samples over 3: 0.5/7·3 = 0.21 → 0, 1.5/7·3 = 0.64 → 0, 2.5/7·3 = 1.07 → 1, … + assert_eq!(mask_on_image_grid(&mask(&[1, 2, 3], 3, 1), 7, 1), Some(vec![1, 1, 2, 2, 2, 3, 3])); + assert_eq!(mask_on_image_grid(&mask(&[1, 2, 3, 4], 1, 4), 1, 2), Some(vec![2, 4])); + } + + #[test] + fn malformed_masks_are_not_sampled() { + assert_eq!(mask_on_image_grid(&mask(&[], 0, 1), 4, 1), None, "an empty mask"); + assert_eq!(mask_on_image_grid(&mask(&[1, 2], 2, 1), 0, 1), None, "an empty image"); + assert_eq!(mask_on_image_grid(&mask(&[1], 2, 1), 4, 1), None, "mask data shorter than its size"); + } + + /// Every target sample's weights add up to exactly one, and together the targets take every + /// source sample exactly once (each `dst / src` of it, in fixed point within rounding). + #[test] + fn taps_are_exact_partitions_of_unity() { + for src in 1..=40_u32 { + for dst in 1..=src { + let mut taken = vec![0_u64; src as usize]; + for t in 0..dst { + let taps: Vec<_> = area_taps(src, dst, t).collect(); + assert_eq!(taps.iter().map(|(_, w)| w).sum::(), AREA_ONE, "{src} → {dst}, target {t}"); + assert!(taps.iter().all(|(_, w)| *w > 0), "{src} → {dst}, target {t}: an empty tap"); + for (s, w) in taps { + taken[s] += w; + } + } + // Source s is dst/src of one target in all; scaled by src/dst, that is one. + for (s, w) in taken.iter().enumerate() { + let share = (*w * u64::from(src)) as f64 / (AREA_ONE * u64::from(dst)) as f64; + assert!((share - 1.0).abs() < 1e-3, "{src} → {dst}: source {s} weighs {share}"); + } + } + } + } + + /// The table the horizontal pass reads holds exactly the taps of every target, in order. + #[test] + fn the_tap_table_holds_every_targets_taps() { + for src in 1..=40_u32 { + for dst in 1..=src { + let axis = AreaAxis::new(src, dst).expect("table"); + for t in 0..dst { + let (first, weights) = axis.get(t as usize).expect("target"); + let expected: Vec<(usize, u64)> = area_taps(src, dst, t).collect(); + let got: Vec<(usize, u64)> = weights.iter().enumerate().map(|(k, w)| (first + k, u64::from(*w))).collect(); + assert_eq!(got, expected, "{src} → {dst}, target {t}"); + } + assert_eq!(axis.get(dst as usize), None, "{src} → {dst}: no target past the last"); + } + } + } + + /// The largest side an image may have still computes in u64 without overflowing. + #[test] + fn taps_at_the_pixel_limit_do_not_overflow() { + let side = MAX_IMAGE_PIXELS as u32; + for (dst, t) in [(1, 0), (3, 2), (side - 1, side - 2), (side, side - 1)] { + assert_eq!(area_taps(side, dst, t).map(|(_, w)| w).sum::(), AREA_ONE, "{side} → {dst}, target {t}"); + } + } + + #[test] + fn flat_images_keep_their_exact_value() { + for (source, target) in [((7, 5), (3, 2)), ((10, 3), (6, 3)), ((4, 9), (4, 4)), ((13, 11), (1, 1))] { + let data = vec![77_u8; (source.0 * source.1 * 3) as usize]; + let out = area_shrink::<3>(&data, source, target).expect("shrinks"); + assert_eq!(out.len(), (target.0 * target.1 * 3) as usize); + assert!(out.iter().all(|v| *v == 77), "{source:?} → {target:?}: {out:?}"); + } + } + + /// A side over the resampling limit is refused before any work buffer is allocated. + #[test] + fn sides_over_the_resampling_limit_are_refused() { + let wide = (super::MAX_RESAMPLING_SOURCE_SIDE + 1) as usize; + assert_eq!(area_shrink::<1>(&vec![0_u8; wide], (wide as u32, 1), (10, 1)), None, "too wide"); + assert_eq!(area_shrink::<1>(&vec![0_u8; wide], (1, wide as u32), (1, 10)), None, "too tall"); + } + + #[test] + fn malformed_shrinks_are_refused() { + let data = vec![0_u8; 4 * 4]; + assert_eq!(area_shrink::<1>(&data, (4, 4), (4, 4)), None, "nothing shrinks"); + assert_eq!(area_shrink::<1>(&data, (4, 4), (5, 2)), None, "a side grows"); + assert_eq!(area_shrink::<1>(&data, (4, 4), (0, 2)), None, "an empty side"); + assert_eq!(area_shrink::<1>(&data[1..], (4, 4), (2, 2)), None, "data shorter than the source"); + assert_eq!(area_shrink::<3>(&data, (4, 4), (2, 2)), None, "data of another channel count"); + } +} From 5ed2f4ba4f34d5eff689727d468186bc18b53350 Mon Sep 17 00:00:00 2001 From: kronbii <145912020+Kronbii@users.noreply.github.com> Date: Fri, 30 Oct 2026 01:50:48 +0200 Subject: [PATCH 055/198] M2: keep an interpolated soft mask smooth on a shrunk image's grid A soft mask no larger than its area-shrunk image is sampled onto the image's grid, and it was always sampled nearest-neighbour: a mask that asks for /Interpolate became a hard step where main drew a ramp (jpeskov's 64 x 64 image at 32 x 32 under a 2 x 2 mask). An interpolated mask is now blended bilinearly onto the grid, centres aligned; one without /Interpolate is still sampled nearest, which keeps the colour-fringe fix. Co-Authored-By: Claude Opus 5.5 --- crates/render/src/raster/tests.rs | 29 ++++++++++++ vendor/README.md | 2 +- vendor/hayro/src/renderer.rs | 74 ++++++++++++++++++++++++++----- 3 files changed, 93 insertions(+), 12 deletions(-) diff --git a/crates/render/src/raster/tests.rs b/crates/render/src/raster/tests.rs index 7312f3ddaf..f6c0530164 100644 --- a/crates/render/src/raster/tests.rs +++ b/crates/render/src/raster/tests.rs @@ -6734,6 +6734,35 @@ fn a_smaller_soft_mask_does_not_bleed_transparent_colour_when_minified() { assert!(r >= 254 && g.abs_diff(128) <= 1 && b == g, "half-covered red over white, got ({r}, {g}, {b})"); } +/// A smaller soft mask that asks for `/Interpolate` stays smooth when it is sampled onto the grid +/// of an area-shrunk image. Here a 64 × 64 black image is drawn at 32 × 32 under a 2 × 2 mask, +/// opaque on the left and clear on the right: the mask is enlarged while the image shrinks, and +/// sampled nearest-neighbour onto the image's grid the edge became a hard black and white step. +#[test] +fn an_interpolated_smaller_soft_mask_stays_smooth_when_its_image_is_minified() { + let content = "q 32 0 0 32 0 0 cm /Im0 Do Q"; + let (image, mask) = ("00".repeat(64 * 64), "FF00FF00"); + let pdf = format!( + "%PDF-1.4\n1 0 obj << /Type /Catalog /Pages 2 0 R >> endobj\n\ + 2 0 obj << /Type /Pages /Kids [3 0 R] /Count 1 >> endobj\n\ + 3 0 obj << /Type /Page /Parent 2 0 R /MediaBox [0 0 32 32] /Contents 4 0 R /Resources << /XObject << /Im0 5 0 R >> >> >> endobj\n\ + 4 0 obj << /Length {} >> stream\n{content}\nendstream endobj\n\ + 5 0 obj << /Type /XObject /Subtype /Image /Width 64 /Height 64 /ColorSpace /DeviceGray /BitsPerComponent 8 /SMask 6 0 R /Filter /ASCIIHexDecode /Length {} >> stream\n{image}>\nendstream endobj\n\ + 6 0 obj << /Type /XObject /Subtype /Image /Width 2 /Height 2 /ColorSpace /DeviceGray /BitsPerComponent 8 /Interpolate true /Filter /ASCIIHexDecode /Length {} >> stream\n{mask}>\nendstream endobj\n\ + trailer << /Root 1 0 R >>\n%%EOF\n", + content.len(), + image.len() + 1, + mask.len() + 1 + ); + let mut renderer = PageRenderer::new(Arc::new(pdf.into_bytes()), RenderConfig::default()); + let page = renderer.render(RenderRequest { page: 0, kind: RequestKind::Pixels, scale: 1.0, ..Default::default() }); + assert!(page.error.is_none(), "{:?}", page.error); + assert_eq!((page.width, page.height), (32, 32)); + let row: Vec = page.rgba.as_chunks::<4>().0.iter().skip(16 * 32).take(32).map(|[r, ..]| *r).collect(); + let ramp = row.iter().filter(|v| (1..255).contains(*v)).count(); + assert!(ramp >= 8 && row.is_sorted() && row.first() == Some(&0) && row.last() == Some(&255), "a smooth black to white edge, got {row:?}"); +} + #[test] fn non_interpolated_alpha_images_keep_their_colours_and_mask_when_minified() { let palette: [[u8; 3]; 8] = [[0, 0, 0], [255, 255, 255], [255, 0, 0], [0, 255, 0], [0, 0, 255], [255, 255, 0], [255, 0, 255], [0, 255, 255]]; diff --git a/vendor/README.md b/vendor/README.md index 1a58ad6c1f..0d90ad397e 100644 --- a/vendor/README.md +++ b/vendor/README.md @@ -40,7 +40,7 @@ Other resource categories retain their existing parent-chain lookup. | Crate | Version | Licence | Patches | Test | |---|---|---|---|---| | hayro-interpret | 0.7.0 | Apache-2.0 OR MIT | (1) select `/AP /N` appearance states via `/AS` (checkboxes, radios, stateful icons) and honour the NoView flag; (2) Type3 Unicode fallback to `/Encoding` glyph names, then printable ASCII; (3) `Type3Glyph::advance_width`; (4) `InterpreterSettings::ocg_overrides` for viewer layer toggles; (5) soft masks whose group lacks `/CS` were dropped (Chrome gradient text); (6) paint nesting cap `MAX_PAINT_NESTING` for tiling patterns and Type 3 glyphs (self-referencing content overflowed the stack); (7) CID width ranges in `/W` and `/W2` clamped to `MAX_CID` (a `0 4294967295 w` range hung); (8) `InterpreterSettings::hide_comments` skips markup annotations but keeps widgets and links (View ▸ Hide all comments); (9) `image_size_ok`: images and masks over 2^28 pixels are not decoded (a fuzzed `/W 4294967295` stencil mask allocated 4 GiB); (10) `MAX_NESTED_PAINTS`: nested paints (forms, Type 3 glyphs and tiling patterns inside one another) share a per-page budget of 50,000 (a glyph showing eight glyphs of its own font, sixteen deep, hung); (11) depends on skrifa 0.47 instead of 0.42 (upstream read-fonts fixed a Type 1 tokenizer that recursed through runs of numbers: a fuzzed font hung or overflowed the stack), with the one API change (`Charset::glyph_id` returns `Option`); (12) annotation appearances whose `/BBox` or `/Matrix` collapses to a line or a point are skipped (mapping them onto `/Rect` divided by zero and handed the device infinite or NaN matrices); (13) a Highlight annotation without a usable normal appearance is drawn from its `/QuadPoints`, `/C` and `/CA` with Multiply blending, as pdfcraft-annot draws highlights (it was not drawn at all); (14) simple-font `/Widths` (`read_widths`): entries only for codes 0–255 (one entry per code below `/FirstChar` was allocated: `4294967295` asked for 34 GB), an entry that isn't a number leaves its code missing instead of ending the array, and a `/LastChar` below `/FirstChar` gives no widths instead of discarding the font for a standard one; (15) the appearance box is mapped onto `/Rect` corner to corner: the translation now accounts for the scale (an appearance whose box didn't start at the origin and differed in size from `/Rect` was drawn shifted); (16) a clip set by `W`/`W*` takes effect after whichever path-painting operator ends the path (`S`, `f`, `B`, … as well as `n`; ISO 32000-2 §8.5.4): it was applied only on `n`, so `re W* S` clipped nothing and the clip stayed pending for a later, unrelated `n`; (17) `InterpreterSettings::cancelled`: once set, interpretation stops before the next content operator (a render pool sets it for every worker when dropped, after an edit or on closing, and for one worker when the watchdog gives up on its render, so renders whose answers nobody can receive stop instead of running to the end); (18) `MAX_PAGE_CONTENT`: the content a page holds at once (its contents, and the forms, tiling patterns and Type 3 glyph procedures being painted) stays within 512 MiB (`decode_content`; once it runs out, `InterpreterWarning::ContentTruncated` is reported (once per page by pdfcraft-render) so a truncated page is not a clean render; each level gives its bytes back when it finishes): forms nest fifty deep and each level kept its content while the next ran, so a form painting itself around a stream inflating to 256 MiB would have held over 12 GB; and Type 0 sampled functions take at most `MAX_SAMPLE_ENTRIES` (2^21) table points and `MAX_SAMPLE_VALUES` (2^23) samples, read no further than `/Size` asks (256 MiB of 1-bit samples asked for over 8 GB); (19) mesh shadings (types 4–7) are sampled only inside the area being drawn (`ShadingPattern::encode_within`, which the vendored hayro renderer calls with that area plus a pixel), within 8 visits per pixel of it plus 4 per triangle (`TriangleSampler`), and patches are triangulated one at a time: points far off the page were sampled over their whole bounding box, up to 65536 × 65536 entries (a fuzzed 147-byte patch stream; a synthetic page passed 6 GB in six seconds), ten thousand overlapping triangles ran for minutes, and every patch became 722 triangles before any limit; meshes hold at most `MAX_MESH_PATCHES` (2^16) patches and `MAX_MESH_TRIANGLES` (2^20) triangles, a lattice needs two to 2^19 vertices per row (`/VerticesPerRow 0` pushed empty rows for ever; one huge row read the whole stream), and a page samples at most `MAX_PAGE_MESH_TRIANGLES` (2^25) mesh triangles in all (a mesh painted a thousand times was triangulated a thousand times); each cut logs a warning; (20) PDF function construction is limited to 64 levels and 10,000 total nodes with cycle detection; calculator parsing and evaluation share procedure depth/work budgets, integer division/bitshift/rotation/index are panic-free, and fixed-size stack errors propagate; (21) a Type 0 font drawn with a substitute that isn't CID-keyed (it isn't embedded, or its program doesn't load) draws `.notdef` for a CID of an Adobe character collection (Japan1, GB1, CNS1, Korea1) that it can't find by Unicode: hayro took the CID for a glyph index of the substitute, which numbers its glyphs its own way, so the character was drawn as an unrelated one (𠮷 as "Ｑ" in BIZ UDPGothic); (22) a Type 0 font drawn with a substitute finds a character given with a variation selector (`/ToUnicode`, or Adobe-Japan1-UCS2, which maps the JIS X 0208 forms of the kanji JIS X 0213:2004 redrew that way: CID 1247 is 噂 U+E0100) through the substitute's variation sequences, else as the character alone: such a CID found no glyph and was drawn as an unrelated one; (23) backport of hayro 0.8's `gradient` module: `EncodedShadingPattern::as_svg_gradient` turns an axial or radial shading into stops of an sRGB gradient (within 0.01 per channel, a stop at every stitching-function bound via `Function::stitching_bounds`) when a native gradient draws it exactly (extended past the box drawn in; a radial one with no background colour); (24) `DecodedImageCache`: decoded image pixels shared across renders through the interpreter cache within a 64 MiB budget (an image-heavy page decoded its thousands of images again per tile and per scale), keyed by every decode input (`DecodedImageKey`: the stream's place and dictionary bytes, its object, mask/stencil/raster output, interpolation, decode target size, preview or final); images under a transfer function and inline images are not cached, an image over the budget is never kept, and a poisoned lock decodes uncached; (25) `Device::actual_text` hands the device a marked content sequence's `/ActualText` (inline or named in `/Properties`), so text extraction keeps the Thai, Indic and Arabic clusters Chrome marks this way (their shaped tone marks and vowels have no Unicode of their own); (26) non-embedded simple fonts drawn with a standard-font stand-in build the Latin Extended-A letters the stand-in lacks (`/Gbreve`, `/Idotaccent`, `/Scedilla`, `/aogonek`, `/ccaron`…) from the base letter and accent it has (`StandardKind::composite_glyph`, `ACCENTED_LETTERS`): PDFium's Foxit stand-ins hold only the Standard/WinAnsi/Mac Roman glyphs, so iText letters with Times New Roman unembedded showed blanks for Ğ, İ, Ş, ğ and ş; the comma-below letters (`/scommaaccent`, `/Tcommaaccent`, `/kcommaaccent`…, found by character, so `/uni0136` and the glyph list's `/Kcedilla` too) are the letter and the font's comma, smaller and under the baseline, since the stand-ins have no `commaaccent` (Romanian ș drew nothing and ț a cedilla in forms laid out from XFA; ģ, whose comma is turned and above, is not built); (27) an embedded TrueType font without a base encoding reads its codes as WinAnsi for text extraction (`EmbeddedKind::code_to_unicode_name`), matching pdftotext's output, instead of Standard: OCR text layers of scans write WinAnsi codes with no `/Encoding`, and Standard has no ü, ö or ß and reads é as Ø (glyph selection unchanged); (28) an image whose pixels hold at most 16 bits of samples (an 8-bit Separation or DeviceN image, a 1-, 2- or 4-bit gray or indexed one) is converted once per possible pixel value through a lookup table instead of once per pixel (`rgb_via_lookup`): a spot-colour tint transform is often a PostScript calculator program, run for every pixel, so a fillable character sheet's 3304 × 4105 DeviceN background took four seconds and 430 MB to draw a page; each entry is converted on its own (`convert_each`), so a tint transform giving the wrong number of components for a value the image never uses cannot shift the colours of the values it does; (29) `Device::begin_annotations`/`end_annotations` bracket the page's annotations, so a device can tell comment paints from document paints (reading colours recolour the document, never a comment, #1548); (30) the Annex D.2 alias codes read as the ink they draw: WinAnsiEncoding 0xA0 and 0xAD, and MacRomanEncoding 0xCA, carry the glyph names `space` and `hyphen` -- the names those tables also give 0x20 and 0x2D -- not `nbspace` and `sfthyphen`, whose glyph list values U+00A0 and U+00AD are invisible formatting characters. Extraction hid drawn ink behind them, so search and redaction matched neither (`XiaoBiaoSong.pdf` in the pdf.js corpus drew 104 soft hyphens; across the 989 files the soft hyphens we extract fall from 117 to 13 and the non-breaking spaces from 30 to 26, what remains coming from `/ToUnicode` maps, which say so explicitly). The base table only, in the text direction only: a `/Differences` name keeps its glyph list value and glyph lookup is untouched, both as pdftotext does; (31) a Type 0 sampled function keeps its samples in one flat array indexed by strides (`Type0::entry`) instead of a hash map keyed by each grid point: the map was built again, one hashed insert per sample, every time a shading using the function was painted (a 2,097,152-sample ramp painted 30 times took 38 s; `bug1721218` in the pdf.js corpus spent most of a 1.5 s page on it); the interpolation is unchanged, so results are bit for bit the same, a sample missing from a short stream still gives no value, and the maximum sample value is computed without overflowing at 32 bits per sample; (32) the decoded-image cache (24) keys a decode by the size the image is drawn at only when a JPX stream decodes it or its soft mask or mask (`decode_reads_target`): JPX is the only filter that decodes a reduced resolution for a size, so a JPEG or Flate scan was decoded again at every zoom step and kept once per zoom level; one decode now serves every size; `InterpreterCache::decoded_image_decodes` counts the cache's decodes; (33) Coons and tensor-product patches copy their vertices' colours less: each point of a patch's 20 × 20 triangle grid is interpolated once (`generate_patch_triangles`; each cell interpolated its four corners itself, 1,444 interpolations a patch for 400 points), a cell's second triangle takes the vertices its first copied, and the mesh sampler moves a patch's triangles into device space instead of copying each vertex (`TriangleSampler::sample_owned`); the same values, 10–16% faster on mesh-gradient pages; (34) `color::tint_transform_runs`: how many colours the calling thread has run through a Separation or DeviceN tint transform (counted once per batch in `convert_f32`), so a test counts patch (28)'s once-per-sample-value runs instead of timing them (its 10 s deadline failed on a busy FreeBSD VM in CI); (35) `EncodedShadingPattern::extent`: where a mesh shading (types 4–7) can paint at all, the pixels its triangles touched when it was encoded (`None` for the types that paint everywhere, and when the shading has a `/Background`), so the renderer samples a texture there alone and skips a mesh with nothing on the canvas; (36) a mesh shading's pixel whose corner no triangle holds strictly (on an edge two triangles share, or on the mesh's side) takes the triangle that covers its centre, edges included (`Triangle::covers`): testing only the corner left a white seam along every shared edge, and the colour is still taken at the corner; (38) the `/DefaultGray`, `/DefaultRGB` and `/DefaultCMYK` colour spaces of a resource dictionary (ISO 32000-2 §8.6.5.6) replace the device space content selects: with `g`/`G`, `rg`/`RG`, `k`/`K` and `cs`/`CS` (by name, or through a `/ColorSpace` resource that is a device space), and as the `/ColorSpace` of an image XObject, an inline image or a shading painted by `sh` or as a shading pattern (`DefaultColorSpaces`, looked up once per content window in the resources and those they inherit, so a form XObject without a default of its own draws in its page's, as Poppler and MuPDF draw it); a default that is a Lab, Indexed, Pattern or device space, has another number of components or is no colour space is ignored and device colour is drawn as before; device spaces inside other spaces (an Indexed base, a Separation or DeviceN alternate, a Pattern's underlying space), shading dictionaries, masks and transparency group spaces are unchanged, and the decoded-image cache (24) keys a decode by the default it was drawn in. hayro drew device colour as it was, so pages that calibrate it this way (common in PDF/A) came out in the wrong colours; (39) knockout groups (#2971): `Device::set_knockout` right after a `/K true` group is pushed; `Device::knockout_image` for an unmasked image before its group is pushed, when `Device::in_knockout_group`: the device clears its placement rectangle only if it will draw the image (on the canvas, not a preview's tiny image, and it decodes), decoded at its draw size so the draw reuses that decode; and, when `Device::knockout_erase_wanted`, a nested group's content drawn once between `begin_knockout_erase` and `end_knockout_erase` before the group itself, which then always gets a layer of its own; (40) an optional content membership dictionary's visibility expression (`/VE`: `/And`, `/Or`, `/Not`, nested up to 32 deep) decides its visibility ahead of `/OCGs` and `/P` (ISO 32000-2 §8.11.2.2); it was ignored, so content under it stayed visible with its layers hidden; a malformed expression, including one naming an object the document doesn't list in `/OCProperties /OCGs`, falls back to `/OCGs` and `/P` (#2986); (41) a transparency group that is neither isolated nor knockout, entered at full opacity with no soft mask and Normal blending, is drawn straight onto the page instead of into a layer of its own (§11.4.7: its elements composite with the backdrop; the layer made a `/BM /Multiply` highlight appearance an opaque block, #1968); (42) text in a Type 3 font is drawn filled whatever the text rendering mode, except mode 3 (invisible): only that mode has any effect on Type 3 fonts (ISO 32000-1 §9.3.6). Stroking ran the glyph procedure a second time, so `2 Tr` made bitmap fonts look bold, `1 Tr` painted an uncoloured glyph in the stroking colour, and `7 Tr` hid it (`7 Tr` since follows 64); (43) Lab colour spaces honour their own `/WhitePoint`: L\*a\*b\* goes to XYZ relative to that white, is Bradford-adapted to D65 and converted to sRGB (ISO 32000-2 §8.6.5.4) instead of through `LAB.icc` as if every white were D50 (a D65 blue came out 45 levels too red; the profile is removed); components normalised to [0, 1] (`manual_scale`, an `/Indexed` palette over Lab) map onto L\* 0..100 and the a\*/b\* `/Range`; (44) a Type 0 font whose `/ToUnicode` is the name `/Identity-H` or `/Identity-V` (not a CMap stream; WSCAD writes it) reads each code as its Unicode value, as Acrobat shows it (`Type0Font::identity_to_unicode`); the identity CMap the name resolved to maps codes to CIDs only, so such pages had no text to select or find; (46) an image XObject's group applies the fill opacity, so the fill alpha is reset to 1 before a stencil mask takes the fill colour as its paint, and a pattern paint drops the opacity it captured at `scn` (`Pattern::without_opacity`) (it was applied twice: a mask at `ca 0.5` was drawn at a quarter, #2975); (47) `path_as_rect`: a five-element path is a rectangle only when its four points are distinct, each step from one to the next runs along a single axis, and the last returns to the first. The length test admits one MoveTo and four LineTo as well as one MoveTo, three LineTo and a ClosePath, and asking only that every corner of the bounding box was visited says nothing about the order or the steps between, so a five-point polyline whose ends reached the corners was painted as that box -- a fourth side the path does not have -- and a clip of that shape was discarded; the coordinates are compared exactly, since the matrix is applied after this check and a `cm` of 10,000 turns `is_nearly_equal`'s 256th of a unit into tens of device units (#3327); (48) an `/ICCBased` colour space is cached by what its profile is read from (`IccProfileKey`): the stream object it names, looked up without reading the stream, and then the stream's dictionary and its own bytes (decrypted, not decoded), so a byte-for-byte copy of a profile in another object is read once, and a profile stream written directly in its array, which reports the object holding the array as every other stream written there does, is told apart by its bytes. hayro keyed it by the stream dictionary's bytes alone, so two different profiles with the same `/N` and `/Length` shared whichever was read first, and colours in the second were drawn through the first. `InterpreterCache::icc_profile_reads` counts the profiles read; (49) `OpenTypeFontBlob::new` also accepts a `post` table version 1.0, 2.0 or 3.0 written with its 16-bit halves swapped (`00 00 00 03`), as ScanSoft PDF Create! embeds every font; such fonts were refused and their text drawn with a substitute (Hebrew came out garbled or missing, #2946); `glyph_names` reads such a table from a copy with the version written correctly, so a 2.0 table's names serve `/Differences`. Any other unknown version is still refused (pdf.js 9462); (50) `CffFontBlob::glyph_index_by_name` looks a glyph name up in a name to glyph map built on first use (shared between clones) instead of searching the whole charset, with a string lookup per glyph, for every character code of a font; the first glyph with a name still wins; (51) at most 4096 graphics states saved at once, holding at most 64 MiB together (each `q` copies the whole state, dash array and clip list included, so short operators multiplied one large state); a `q` past either limit is counted but not copied, and its `Q` still removes the clips added since (from storytold/craft-libs#239); (52) a `/W2` run whose first CID is past `MAX_CID` stops there, as in `/W` (7), checked before the run's metrics are read: a first CID of 4294967295 inserted one entry and then overflowed, discarding the whole `/W2` table (from storytold/craft-libs#254); (55) a Separation or DeviceN tint transform that cannot be read keeps its colour space, and one that cannot be read or evaluated paints the alternate space's initial colour, as evaluation failures already did; that fallback, and a `cs`/`CS` colour space that cannot be read (drawn in DeviceGray as before), are now reported once per content stream as `InterpreterWarning::ColorFallback` (`RenderWarning::ColourFallback` in `pdfcraft-render`) instead of silently (#4260); (56) an annotation whose Invisible flag (bit 1) is set and whose `/Subtype` is none of the standard types of ISO 32000-2 Table 171 (`is_standard_annotation_type`; no `/Subtype` counts as unknown) is not drawn (Table 167), and a `/Popup` annotation's appearance is never painted on the page (§12.5.6.14: a popup has no appearance stream of its own; the viewer shows it as a window for its parent annotation). Both were drawn from their `/AP`; (57) a simple font drawn with a standard-font stand-in because the document names a font it doesn't embed (`StandardKind` with `fallback`) narrows a glyph to a narrower `/Widths` advance but no longer widens it to a wider one: the advance already places the next glyph (ISO 32000-2 §9.2.4; /Widths in Table 109), and widening drew a non-embedded `Arial,Bold` with 600-unit widths as a heavy fixed-pitch face whose `i` and `l` were solid blocks twice their width; an exact standard-14 font still matches its standard metrics both ways; (58) the fallback request for a Type 0 font that is not embedded (`FallbackFontQuery::with_descriptor`) takes its serif, italic and small-cap flags, `/FontWeight`, `/FontFamily` and `/FontStretch` from the descendant CIDFont's font descriptor (ISO 32000-2 §9.7.4.1, §9.7.6.1; §9.8): it read them from the Type 0 dictionary, which has none, so every such request asked for a regular sans-serif face and no resolver could tell a Song or Mincho font from a Hei or Gothic one, or see `/FontWeight 700`; (59) the NoRotate flag (`/F` bit 5) is honoured: the appearance, fitted to `/Rect`, is turned against the page's `/Rotate` about the rectangle's upper-left corner, which stays where it is, so it reads upright as the page is displayed (ISO 32000-2 §12.5.3). Sticky notes, which are saved NoRotate, were drawn turned with the page (#2583). Form fields and links stay in `/Rect`, where PdfCraft takes their click targets from it; (64) text in a Type 3 font follows ISO 32000-2 §9.3.6 in the clipping modes: under `7 Tr` it is not rendered (reported to the device as clipping text, `GlyphDrawMode::Clip`, so extraction keeps it), and in modes 4 to 7 a Type 3 glyph adds nothing to the clipping path (`clip_glyph`), so text shown only in a Type 3 font leaves no clip at `ET`. Since (42) `7 Tr` painted the glyphs in the fill colour (black by default), so a fill meant to follow showed them wherever it didn't cover them; 4 to 6 still render filled; (66) a Separation colour space whose colourant is the special name `All` paints every colourant at once (ISO 32000-2 §8.6.6.4): on screen and in an RGB print raster its tint t is the gray 1 − t, complemented because the device is additive, whatever its alternate space and tint transform say (hayro drew it through the tint transform, so a `/DeviceCMYK` transform giving cyan drew registration marks light cyan); `/None` still paints nothing, any other name still goes through its alternate, and a DeviceN space naming `All` keeps its alternate, since the clause gives the name that meaning in a Separation space; (63) a luminosity soft mask whose group colour space is `/DeviceCMYK` (by name, not an ICC-based or calibrated space) reads a `/DeviceCMYK` fill or stroke colour drawn in it, and its `/BC`, as their gray `1 - min(1, 0.3 c + 0.59 m + 0.11 y + k)` (ISO 32000-2 §10.4.2.3, §11.5.3: device colour converts to gray with no colour calibration; `Context::luminosity_in_device_cmyk`, `color::device_cmyk_to_gray`): it went through the CMYK display profile, where 100% K is a dark grey, so a 100% K mask let about an eighth of the masked paint through. CMYK images, shadings and tiling patterns inside such a mask, and CMYK on the page, still use the profile; (71) a lookup table in an indirect string reads: it was read as a stream first, which uses up a reference (ISO 32000-2 §8.6.6.3); (72) an uncoloured pattern's underlying space given as a name or a reference is used: it was skipped, so a CMYK cyan drew red (ISO 32000-2 §8.7.3.3); (73) `InterpreterSettings::jbig2_work`, set for each page in `interpret_page` and ended with it, and a JBIG2 image or an image with a JBIG2 mask past it left out with `ContentTruncated`; (74) colour spaces nest at most 32 levels, through every way one names another, so a chain of them ends instead of overflowing the stack; an ICC profile's `/Alternate` is read where it's met and isn't kept with the profile, or a page could build a chain past the limit on spaces already read; (75) a soft hyphen (U+00AD) that a CFF substitute for a CID font has no glyph for draws its hyphen, as Annex D.2 reads WinAnsi 0xAD in (30), instead of the CID taken as a glyph index (#5888) | (1) `raster::tests::widget_appearance_states`; (2, 3) `xtask text-oracle` on corpus/pdfjs issue918; (4) `raster::tests::layer_override_hides_content`; (5) `raster::tests::soft_mask_without_group_colour_space_is_applied`; (6) `raster::tests::self_referencing_tiling_pattern_terminates`, `raster::tests::self_referencing_type3_glyph_terminates`; (7) `raster::tests::huge_cid_width_ranges_terminate`; (8) `raster::tests::hiding_comments_keeps_fields`; (9) `raster::tests::absurd_mask_and_fax_sizes_are_refused_before_decoding`; (10) `raster::tests::type3_glyphs_that_fan_out_into_themselves_finish`; (11) `raster::tests::type1_font_programs_with_long_runs_of_numbers_load_quickly`; (12) `raster::tests::degenerate_annotation_appearances_are_skipped`; (13) `raster::tests::highlights_without_an_appearance_are_drawn`; (14) `raster::tests::malformed_simple_font_widths_keep_later_widths_and_the_font`, `raster::tests::huge_simple_font_first_char_terminates`; (15) `raster::tests::annotation_appearance_box_maps_onto_rect`; (16) `raster::tests::clipping_path_applies_after_any_painting_operator`; (17) `raster::tests::dropping_the_pool_cancels_its_renders`, `raster::tests::the_watchdog_stops_the_render_it_gives_up_on`; (18) `raster::tests::page_contents_and_nested_forms_share_one_budget`, `raster::tests::content_painted_again_and_again_keeps_drawing`, `raster::tests::sampled_functions_with_billions_of_samples_are_refused`; (19) `raster::tests::mesh_shadings_far_off_the_page_render`, `raster::tests::many_overlapping_mesh_triangles_finish`, `raster::tests::mesh_shadings_with_many_patches_or_no_rows_finish`; (20) `raster::tests::function_limits_calculator_arithmetic_and_depth`, `raster::tests::function_limits_public_construction_work_and_depth`, `raster::tests::function_limits_stitching_cycles_are_refused_and_shared_children_work`, `raster::tests::function_limits_invalid_calculator_keeps_the_page_renderable`; (21) `raster::tests::cids_a_substitute_lacks_are_not_drawn_as_other_glyphs`; (22) `raster::tests::variation_sequences_draw_their_kanji_in_a_substitute`; (23) `raster::tests::axial_and_radial_shadings_draw_as_native_gradients`; (24) `raster::tests::tiles_match_full_render`, `raster::tests::skipping_paths_off_the_canvas_changes_no_pixel` (both decode through the cache), `raster::tests::decoded_images_with_identical_dictionaries_keep_their_own_pixels`, `raster::tests::decoded_images_under_a_transfer_function_are_not_shared`; (25) `text::tests::actual_text_replaces_the_glyphs_it_marks_in_every_writing_direction`, `text::tests::actual_text_follows_the_pdf_rules_for_marked_content`; (26) `raster::tests::accented_letters_the_standard_stand_ins_lack_are_built_from_their_parts`, `raster::tests::comma_below_letters_in_standard_fonts_draw_a_comma_under_the_letter`; (27) `text::tests::truetype_without_an_encoding_reads_its_codes_as_win_ansi`; (28) `raster::tests::spot_colour_images_run_their_tint_transform_once_per_sample_value`, `raster::tests::images_converted_through_a_table_keep_every_samples_colour`, `raster::tests::a_malformed_tint_for_an_unused_sample_keeps_the_used_colours`; (29) `raster::tests::reading_colors_keep_highlight_annotations_readable`; (30) `text::tests::the_annex_d2_alias_codes_read_as_an_ordinary_hyphen_and_space`; (31) `raster::tests::shadings_with_large_sampled_functions_paint_quickly`, `raster::tests::sampled_functions_interpolate_their_grid_as_before`; (32) `raster::tests::decoded_images_serve_every_size_they_are_drawn_at`; (33) `raster::tests::patch_meshes_keep_their_corner_colours`; (34) `raster::tests::spot_colour_images_run_their_tint_transform_once_per_sample_value`; (35) `mesh_extent_tests::{a_mesh_paints_its_triangles_alone, a_mesh_keeps_the_graphics_state_alpha, a_mesh_off_the_canvas_paints_nothing}` (`cargo test -p hayro`); (36) `raster::tests::mesh_triangles_sharing_an_edge_leave_no_seam`; (38) `raster::tests::default_colour_spaces::device_rgb_operators_draw_in_default_rgb`, `raster::tests::default_colour_spaces::device_cmyk_operators_draw_in_default_cmyk`, `raster::tests::default_colour_spaces::device_gray_operators_draw_in_default_gray`, `raster::tests::default_colour_spaces::device_images_draw_in_the_default_colour_spaces`, `raster::tests::default_colour_spaces::device_shadings_draw_in_the_default_colour_spaces`, `raster::tests::default_colour_spaces::an_image_drawn_with_and_without_a_default_keeps_both_colours`, `raster::tests::default_colour_spaces::forms_inherit_the_default_colour_spaces_they_do_not_override`, `raster::tests::default_colour_spaces::unusable_defaults_leave_device_colour_alone`; (39) `raster::tests::knockout_groups_replace_what_their_elements_cover`; (40) `raster::tests::visibility_expressions_decide_what_a_membership_dictionary_shows`; (41) `raster::tests::a_multiply_highlight_inside_a_transparency_group_keeps_the_text`; (42) `raster::tests::rendering_modes_but_3_and_7_draw_type3_glyphs_as_filled`; (43) `raster::tests::lab_honours_the_space_white_point`, `raster::tests::indexed_palettes_over_lab_keep_their_colours`; (44) `text::tests::identity_to_unicode_names_give_text`; (46) `raster::tests::stencil_masks_are_drawn_at_the_fill_opacity_once`; (47) `raster::tests::a_five_point_polyline_is_not_painted_as_its_bounding_box`, `raster::tests::a_path_that_visits_every_corner_out_of_order_is_not_a_rectangle`, `raster::tests::a_rectangle_is_still_painted_as_a_rectangle`; (48) `raster::tests::icc_profile_cache::icc_profiles_with_identical_dictionaries_draw_their_own_colours`, `raster::tests::icc_profile_cache::icc_profiles_used_again_are_read_once`; (49) `raster::tests::a_font_whose_post_version_has_swapped_halves_is_still_used`; (50) `font::blob::tests::glyph_names_resolve_to_the_first_glyph_with_that_name`; (51) `context::saved_state_tests::repeated_saves_stay_bounded_and_balanced`; (52) `font::cid::tests::w2_run_past_the_largest_cid_keeps_the_rest_of_the_table` (hayro-interpret); (55) `raster::tests::devicen_inks::a_tint_transform_that_cannot_be_evaluated_draws_the_rest_of_the_page_and_warns`, `raster::tests::devicen_inks::a_tint_transform_that_cannot_be_read_falls_back_and_warns`; (56) `raster::tests::invisible_unknown_annotations_and_popups_paint_nothing`; (57) `substitute_widths::a_substitute_keeps_its_glyph_shapes_at_wider_advances`, `substitute_widths::a_substitute_still_narrows_glyphs_to_narrower_advances` (pdfcraft-render `tests/it`); (58) `cid_fallback_query::a_cid_font_fallback_request_carries_its_descendants_style`, `cid_fallback_query::a_cid_font_without_a_descriptor_asks_for_the_default_style` (pdfcraft-render integration tests); (59) `raster::tests::no_rotate_notes_stay_upright_at_their_corner_on_rotated_pages` (hayro-interpret); (64) `type3_text_clip::{clip_only_type3_text_is_not_rendered_and_leaves_no_clip, invisible_type3_text_is_not_rendered, other_modes_render_type3_glyphs_filled_and_never_clip, only_outline_glyphs_shape_a_clip_shared_with_type3_text, self_nesting_type3_text_in_a_clip_mode_stays_bounded}` (`cargo test -p pdfcraft-render --test it`), `raster::tests::rendering_modes_but_3_and_7_draw_type3_glyphs_as_filled`, `text::tests::clipping_mode_type3_text_is_kept_by_the_ocr_filter`; (66) `separation_all::{separation_all_fills_paint_the_complemented_tint_as_gray, separation_all_strokes_paint_the_complemented_tint_as_gray, separation_all_images_paint_each_sample_as_the_complemented_gray, separation_all_shadings_paint_the_complemented_gray, separation_none_paints_nothing, a_named_separation_still_paints_through_its_alternate, a_device_n_space_naming_all_still_paints_through_its_alternate}` (`cargo test -p pdfcraft-render --test it`); (63) `color::device_cmyk_gray_tests::{each_ink_darkens_by_its_iso_weight, ink_past_full_coverage_is_black_and_bad_components_read_as_zero}` (hayro-interpret), `it::luminosity_group_space::{black_ink_in_a_device_cmyk_group_masks_fully_and_no_ink_shows_fully, each_process_ink_in_a_device_cmyk_group_masks_by_its_weight, a_device_cmyk_backdrop_colour_masks_by_its_gray, gray_and_rgb_groups_mask_as_before, device_cmyk_on_the_page_keeps_its_display_colour}` (pdfcraft-render); (71) `raster::tests::an_indexed_lookup_table_is_read_in_every_legal_form` (pdfcraft-render); (72) `raster::tests::an_uncoloured_pattern_uses_its_underlying_space_in_every_form` (pdfcraft-render); (73) `raster::tests::the_jbig2_images_of_a_page_share_one_allowance`, `a_picture_whose_jbig2_mask_is_past_the_pages_allowance_is_not_drawn` and `a_pages_jbig2_allowance_ends_with_the_page` (pdfcraft-render); (74) `raster::tests::{a_chain_of_colour_spaces_ends, colour_spaces_are_read_to_the_limit_of_their_nesting, a_chain_built_on_spaces_already_read_ends, a_profile_read_at_the_limit_keeps_its_alternate_for_a_read_nearer_the_top}` (pdfcraft-render); (75) `raster::tests::a_soft_hyphen_a_substitute_lacks_draws_its_hyphen` (pdfcraft-render) | -| hayro | 0.7.1 | Apache-2.0 OR MIT | (1) `RenderSettings::{x_offset, y_offset}` to render one tile of a large page; (2) images over `MAX_IMAGE_PIXELS` (2^28) are skipped (a fuzzed `/W 4294967295` hung in resampling); (3) `tiling_cell_scale` keeps a tiling pattern's cell pixmap within 3000 px a side whatever its `/XStep`/`/YStep` (a fuzzed step far beyond `/BBox` allocated 2 GB); (4) stroke widths clamped to 4 × (canvas width + height) device pixels (a fuzzed `/LW 9223372036854775807` allocated 10 GB in stroke expansion); (5) `usable_dash_pattern`: a stroke whose dash pattern could cut it into more than `MAX_DASHES_PER_STROKE` (250,000) pieces, counting kurbo's restart of the pattern at every subpath and `ClosePath`, is drawn solid (a fuzzed `/D [[11] 0]` on a line from x = 92234775807 allocated over 5 GB, `[0.000001] 0 d` on a 20 pt line over 10 GB); (6) a dash pattern with a negative or non-finite entry, or a period that isn't positive, is drawn solid (`[-1 -1] 0 d` looped for ever in kurbo); (7) `render_size` and `render_into`: render into a buffer the caller allocates (PdfCraft renders into pixel-aligned memory that the GUI takes over as texture data); `render` is `render_into` on a new pixmap; (8) checked image source/target/RGBA, alpha-premultiplication scratch and intermediate sizes before resampling, finite transforms and exact source/mask buffers; sources wider than u16 may shrink before backend limits, with a 2^20 source-axis cap and conservative 128 MiB filter-planning estimate; an over-limit or failed resize keeps the original image resolution and geometry when the source fits the backend, otherwise skips it, with a logged warning; staging, resampling output/scratch, RGBA conversion and padding buffers use fallible reserves; (9) `may_paint_canvas`: a fill or stroke whose control-point box, grown by how far its paint reaches past the path (half the stroke width times the larger of the miter limit and √2) plus 0.5 user units and 2 pixels, misses the canvas is not drawn, so a tile no longer strokes and fills every path of its page (item 5 of #623); `set_skip_offscreen_paths` turns this off for the regression test; (10) `/Interpolate` governs the pre-resize too (#624): a minified image that does not request interpolation is pre-resized with nearest-neighbour (since (21), only inside Type 3 glyphs), planned without the Catmull-Rom filter limits (`image_resampling_size` takes the flag); a Type 3 glyph image is sampled bicubically only when interpolated; a minified image with a matching soft mask is nearest-sampled straight into premultiplied RGBA; (11) `may_paint_rect`: an image whose stream-size box, mapped to the device, misses the canvas is skipped before decoding (decoded data can only shrink against that box), so a tile of an image-heavy page no longer decodes, resamples and rasterizes every image on it; (12) preview renders (`InterpreterSettings::preview`: backdrops, thumbnails, grid cells, signature-drag backgrounds) skip images under `PREVIEW_MIN_IMAGE_AREA` device pixels, while final renders always draw them; (13) reading colours (#897): `RenderSettings::color_replacement` takes a `ColorReplacement` (text colour, background, `only_black`, `line_art`) that remaps solid text and line-art paint while rendering, glyph outlines counting as text and stencil-mask ink as line art; images, image alpha masks, soft masks and pattern cells keep their colours; `render_pdf` gains a `color_replacement: Option` argument (pass `None` for true colours); the upstream-only `[dev-dependencies.log]` entry is removed from `Cargo.toml` (log is a normal dependency since the logging patch), so `cargo test -p hayro` runs this crate's unit tests; (14) a shrunk image that is interpolated (or, since (21), area-averaged) and whose axes lie along the device's (upright, mirrored or a quarter turn, as on a page under `/Rotate`) is snapped to whole device pixels: shrunk once to its rounded device rectangle (halves round up, so a tile snaps an edge where the whole page does) and drawn 1:1 from the rounded corner, instead of a ceil-sized raster resampled again at a fractional scale and offset (`snap_minified_image`; scans looked soft next to Acrobat, and until #5530 a turned one still did, up to 200 grey levels off the same page drawn upright). An image counts as axis-aligned when the terms the snap drops move its far edges by at most a hundredth of a device pixel (`SNAP_SKEW`), not by a tolerance on the coefficients, which let a slight skew of a very wide image through; and the snapped 1:1 transform is used only when the shrink produced the snapped size, so a failed resize keeps the full transform instead of drawing the source raster at its source size; (15) a soft mask of another size than its image is filtered when shrunk, whatever its `/Interpolate` (#1649): such a mask carries the shape on its own (Word saves text as a tiny black image with a glyph-bitmap mask), and nearest-neighbour dropped every stroke thinner than the shrink factor; (16) under a soft mask or a blend mode, axial and radial shadings are sampled instead of drawn as native gradients (#2128): vello_cpu 0.0.8 copies an opaque gradient over fully covered tiles without the mask or blend mode, so PowerPoint's faded gradient fills came out solid; (17) the renderer keeps annotation paints out of the reading-colours mapping between `begin_annotations` and `end_annotations` (#1548); (18) the page renders as one transparency group; under reading colours its background is painted inside it as the initial backdrop, so a Multiply highlight blends with the dark background instead of staying an opaque block, and otherwise the background stays outside, so a blend mode over an empty part of the page keeps its colour instead of blending with the white paper (#2970); (19) `set_paint` samples a mesh shading's texture over its triangles alone (`EncodedShadingPattern::extent`, padded rather than repeated) and skips a mesh with none on the canvas: a BricsCAD plan's 310 one-triangle `sh` each covered the whole tile, 2 s per 1024 px tile at any zoom (the watchdog gave the page up on a busy machine), now about 70 ms; (20) a tiling pattern whose step is too large for a cell pixmap, where what is painted (a fill, or a stroke with its width) can only show the first cell (PowerPoint's `/XStep 2147480100` for a picture fill drawn once), has its `/BBox` drawn at full scale with a transparent border and padded instead of tiled (`shows_one_cell`; a cell whose scale isn't finite and positive, from a `/BBox` too thin to divide by, keeps the old path: `usable_cell_scale`); shrinking the cell to fit the 3000 px limit of (3) left the picture under a pixel, so network diagrams lost their pictures (#3349); (21) an image that does not request interpolation and is drawn below its resolution, outside a Type 3 glyph, is shrunk by area averaging (`area_shrink`: each device pixel the coverage-weighted mean of the samples it covers, in 16.16 fixed point whose weights sum to exactly one, premultiplied when the image has a mask; a soft mask no larger than the image is first sampled onto the image's grid by `mask_on_image_grid`, so its colour and mask are averaged together instead of the averaged colour of transparent samples showing under a mask layer at the mask's edges) to its snapped device rectangle, instead of nearest-neighbour: `/Interpolate` asks for smoothing only when an image is enlarged (ISO 32000-2 §8.9.5.3), and nearest-neighbour kept one sample in N per axis, so the text of a 300 dpi scan at 100% lost most of its strokes (sampling them again at a fractional offset dropped more). PicScale's `Box` and `Area` filters were not used: shrinking, both average a window four target pixels wide. On a 13-page 300 dpi scan every rendered sample at 96, 120 and 192 dpi is within 1 of the exact area average (it was up to 255 off, 17-23% of samples by more than 1). A snapped image whose shrink failed keeps the full transform, so it is not drawn at its source size. The shrink never enlarges: a side drawn at or above its resolution (an image shrunk across and stretched down, as a 204 × 98 dpi fax is) keeps its samples for the sampler. Rows are averaged first (contiguous, cheap) and the fewer rows that leaves are then shrunk across with taps worked out once per image, which made the shrink about three times faster (a 300 dpi page at 96 dpi: 25 ms, nearest-neighbour 11 ms). A source with a side over `MAX_RESAMPLING_SOURCE_SIDE` (2^20) keeps (10)'s nearest-neighbour shrink, and `area_shrink` refuses it, so its accumulator row and tap table stay a few times one source row (a 2^27 × 2 image of zeros is a tiny file); (22) knockout groups: while one is innermost, fills, strokes, glyph outlines and shadings are drawn with `Compose::Copy` (weighted by coverage), so each element replaces what earlier elements of the group drew instead of compositing over it; an image or nested group, drawn into a layer of its own, first has what is under its shape cleared (`knockout_image` for an unmasked image's rectangle, only when `draw_image` will draw it, decoded at the size it is drawn; a knockout erase pass that draws a nested group's content with every paint clearing instead of painting, a shading fill's clear clipped to the shading's `/BBox` as its paint is) and then composites over it (#2971); (23) backport of hayro 0.8: shadings that `as_svg_gradient` can represent are painted as a `vello_cpu` linear or two-point radial gradient instead of a texture sampled pixel by pixel through the shading function (InDesign fades are luminosity soft masks of one full-page axial `sh`: 40 on a report page took 15 s at 144 dpi, now under 1 s); without upstream's half-pixel shift, which suits vello_cpu 0.3 but moves 0.0.8's gradients off pixel centres; (25) a tiling cell is drawn at the scale that makes one `/XStep` (`/YStep`) exactly the cell pixmap's whole number of pixels (`tiling_cell_pixels`, at least 1 and at most the cap), so the repeated pixmap puts cells at whole multiples of the step at any device scale; drawing at the unrounded scale repeated the cell up to half a pixel off at every step, and the error grew with distance from the pattern's origin (ISO 32000-2 8.7.3.1); (28) an image of page-scan size (`SMOOTHED_IMAGE_SIDE`: 512 samples or more on its shorter side) that does not request interpolation is drawn with `ImageQuality::Medium` (bilinear, as interpolated images are) when enlarged on either side, also when the other side is shrunk (the area shrink leaves that side at 1:1), outside a Type 3 glyph: scans rarely set `/Interpolate`, and drawn nearest-neighbour every sample of a zoomed scan became a block, so its text had staircase edges; icons, QR codes and pixel art keep their hard pixels. Bicubic (`High`) looked the same on a zoomed 300 dpi scan and drew four 1024 px tiles at 400% in 267 ms against 50 ms | (1) `raster::tests::tiles_match_full_render`; (2) `raster::tests::absurd_image_dimensions_are_skipped`; (3) `raster::tests::tiling_cells_stay_small_whatever_the_step`; (4) `raster::tests::absurd_predictor_columns_and_line_widths_render`; (5) `raster::tests::dash_patterns_that_cut_a_stroke_into_billions_of_pieces_render`; (6) `raster::tests::invalid_dash_patterns_terminate_and_are_drawn_solid`; (7) `raster::tests::pixels_are_a_plain_renders_bytes_in_a_buffer_a_gui_can_take_over`; (8) `raster::tests::image_resampling_rejects_anisotropic_target_growth`, `raster::tests::image_resampling_bounds_intermediate_before_planning`, `raster::tests::image_resampling_rejects_invalid_sources_and_scales`, `raster::tests::image_resampling_keeps_valid_sizes_and_exact_limits`, `raster::tests::resampling_fallback_preserves_original_geometry_and_pixels`, `raster::tests::image_resampling_wide_sources_shrink_before_backend_side_limits`, `raster::tests::image_resampling_wide_transparent_sources_still_shrink`, `raster::tests::image_resampling_padded_backend_dimensions_stay_checked`, `raster::tests::image_resampling_mismatched_alpha_mask_keeps_pixels_and_placement`; (9) `raster::tests::skipping_paths_off_the_canvas_changes_no_pixel`; (10) `raster::tests::non_interpolated_images_are_not_smoothed_when_minified`, `raster::tests::non_interpolated_gray_images_are_not_smoothed_when_minified`, `raster::tests::non_integer_minification_averages_the_columns_each_pixel_covers`, `raster::tests::non_integer_minification_averages_colour_and_mask_together`, `raster::tests::non_interpolated_alpha_images_keep_their_colours_and_mask_when_minified`, `raster::tests::non_interpolated_type3_glyph_images_are_not_smoothed_when_minified`, `raster::tests::image_resampling_nearest_accepts_wide_sources_the_filter_refuses`, `raster::tests::wide_non_interpolated_sources_draw_after_shrinking`; (11) `raster::tests::skipping_paths_off_the_canvas_changes_no_pixel`; (12) `raster::tests::preview_renders_skip_tiny_images_but_finals_draw_everything`; (13) `color_replacement_tests::{black_maps_to_text_colour, white_maps_to_background, midgrey_interpolates, alpha_copied_verbatim, only_black_threshold, line_art_false_skips_paths_but_not_text}`, `color_replacement_render_tests::{color_replacement_maps_text_and_line_art, color_replacement_maps_stencil_mask_ink}`; (14) `raster::tests::minified_image_edges_snap_to_whole_pixels`, `raster::tests::a_tile_snaps_half_pixel_image_edges_as_the_whole_page_does` (`cargo test -p hayro`), `raster::tests::a_turned_or_mirrored_minified_image_snaps_to_whole_pixels`, `rotated_image_snap::a_shrunk_image_on_a_turned_page_is_the_upright_render_turned` (`cargo test -p pdfcraft-render --test it`), `renderer::snap_tests::a_skew_that_moves_a_far_edge_is_not_snapped` (`cargo test -p hayro`); (15) `raster::tests::shrunk_soft_mask_larger_than_its_image_keeps_thin_strokes`; (16) `raster::tests::soft_mask_applies_to_a_shading_pattern_fill`; (17), (18) `raster::tests::reading_colors_keep_highlight_annotations_readable`; (19) `mesh_extent_tests::{a_mesh_paints_its_triangles_alone, a_mesh_keeps_the_graphics_state_alpha, a_mesh_off_the_canvas_paints_nothing}` (`cargo test -p hayro`); (20) `raster::tests::a_tiling_pattern_drawn_once_shows_its_cell`, `raster::tests::a_large_step_still_tiles_where_the_next_cell_shows`, `raster::tests::a_tiling_cell_too_thin_to_scale_keeps_the_old_path`; (21) `raster::tests::non_interpolated_minification_keeps_thin_strokes`, `raster::tests::non_integer_minification_averages_the_columns_each_pixel_covers`, `raster::tests::non_integer_minification_averages_colour_and_mask_together`, `raster::tests::non_interpolated_minification_does_not_bleed_transparent_colour`, `raster::tests::a_smaller_soft_mask_does_not_bleed_transparent_colour_when_minified`, `raster::tests::minified_image_edges_snap_to_whole_pixels`, `raster::tests::an_image_shrunk_one_way_and_stretched_the_other_is_still_averaged`, `raster::tests::wide_non_interpolated_sources_draw_after_shrinking`, `renderer::area_shrink_tests` (`cargo test -p hayro`); (22) `raster::tests::knockout_groups_replace_what_their_elements_cover`; (23) `raster::tests::axial_and_radial_shadings_draw_as_native_gradients`; (25) `tiling_cell_placement::{far_cells_of_a_fill_sit_at_whole_steps, far_cells_sit_at_whole_steps_under_a_turned_scaled_matrix, far_cells_of_a_stroke_sit_at_whole_steps, a_cell_pixmap_spans_exactly_one_step}` (`cargo test -p pdfcraft-render --test it`); (28) `raster::tests::large_non_interpolated_images_are_smoothed_when_enlarged`, `raster::tests::small_non_interpolated_images_keep_hard_pixels_when_enlarged`, `raster::tests::a_scan_shrunk_one_way_is_smoothed_the_way_it_is_stretched` | +| hayro | 0.7.1 | Apache-2.0 OR MIT | (1) `RenderSettings::{x_offset, y_offset}` to render one tile of a large page; (2) images over `MAX_IMAGE_PIXELS` (2^28) are skipped (a fuzzed `/W 4294967295` hung in resampling); (3) `tiling_cell_scale` keeps a tiling pattern's cell pixmap within 3000 px a side whatever its `/XStep`/`/YStep` (a fuzzed step far beyond `/BBox` allocated 2 GB); (4) stroke widths clamped to 4 × (canvas width + height) device pixels (a fuzzed `/LW 9223372036854775807` allocated 10 GB in stroke expansion); (5) `usable_dash_pattern`: a stroke whose dash pattern could cut it into more than `MAX_DASHES_PER_STROKE` (250,000) pieces, counting kurbo's restart of the pattern at every subpath and `ClosePath`, is drawn solid (a fuzzed `/D [[11] 0]` on a line from x = 92234775807 allocated over 5 GB, `[0.000001] 0 d` on a 20 pt line over 10 GB); (6) a dash pattern with a negative or non-finite entry, or a period that isn't positive, is drawn solid (`[-1 -1] 0 d` looped for ever in kurbo); (7) `render_size` and `render_into`: render into a buffer the caller allocates (PdfCraft renders into pixel-aligned memory that the GUI takes over as texture data); `render` is `render_into` on a new pixmap; (8) checked image source/target/RGBA, alpha-premultiplication scratch and intermediate sizes before resampling, finite transforms and exact source/mask buffers; sources wider than u16 may shrink before backend limits, with a 2^20 source-axis cap and conservative 128 MiB filter-planning estimate; an over-limit or failed resize keeps the original image resolution and geometry when the source fits the backend, otherwise skips it, with a logged warning; staging, resampling output/scratch, RGBA conversion and padding buffers use fallible reserves; (9) `may_paint_canvas`: a fill or stroke whose control-point box, grown by how far its paint reaches past the path (half the stroke width times the larger of the miter limit and √2) plus 0.5 user units and 2 pixels, misses the canvas is not drawn, so a tile no longer strokes and fills every path of its page (item 5 of #623); `set_skip_offscreen_paths` turns this off for the regression test; (10) `/Interpolate` governs the pre-resize too (#624): a minified image that does not request interpolation is pre-resized with nearest-neighbour (since (21), only inside Type 3 glyphs), planned without the Catmull-Rom filter limits (`image_resampling_size` takes the flag); a Type 3 glyph image is sampled bicubically only when interpolated; a minified image with a matching soft mask is nearest-sampled straight into premultiplied RGBA; (11) `may_paint_rect`: an image whose stream-size box, mapped to the device, misses the canvas is skipped before decoding (decoded data can only shrink against that box), so a tile of an image-heavy page no longer decodes, resamples and rasterizes every image on it; (12) preview renders (`InterpreterSettings::preview`: backdrops, thumbnails, grid cells, signature-drag backgrounds) skip images under `PREVIEW_MIN_IMAGE_AREA` device pixels, while final renders always draw them; (13) reading colours (#897): `RenderSettings::color_replacement` takes a `ColorReplacement` (text colour, background, `only_black`, `line_art`) that remaps solid text and line-art paint while rendering, glyph outlines counting as text and stencil-mask ink as line art; images, image alpha masks, soft masks and pattern cells keep their colours; `render_pdf` gains a `color_replacement: Option` argument (pass `None` for true colours); the upstream-only `[dev-dependencies.log]` entry is removed from `Cargo.toml` (log is a normal dependency since the logging patch), so `cargo test -p hayro` runs this crate's unit tests; (14) a shrunk image that is interpolated (or, since (21), area-averaged) and whose axes lie along the device's (upright, mirrored or a quarter turn, as on a page under `/Rotate`) is snapped to whole device pixels: shrunk once to its rounded device rectangle (halves round up, so a tile snaps an edge where the whole page does) and drawn 1:1 from the rounded corner, instead of a ceil-sized raster resampled again at a fractional scale and offset (`snap_minified_image`; scans looked soft next to Acrobat, and until #5530 a turned one still did, up to 200 grey levels off the same page drawn upright). An image counts as axis-aligned when the terms the snap drops move its far edges by at most a hundredth of a device pixel (`SNAP_SKEW`), not by a tolerance on the coefficients, which let a slight skew of a very wide image through; and the snapped 1:1 transform is used only when the shrink produced the snapped size, so a failed resize keeps the full transform instead of drawing the source raster at its source size; (15) a soft mask of another size than its image is filtered when shrunk, whatever its `/Interpolate` (#1649): such a mask carries the shape on its own (Word saves text as a tiny black image with a glyph-bitmap mask), and nearest-neighbour dropped every stroke thinner than the shrink factor; (16) under a soft mask or a blend mode, axial and radial shadings are sampled instead of drawn as native gradients (#2128): vello_cpu 0.0.8 copies an opaque gradient over fully covered tiles without the mask or blend mode, so PowerPoint's faded gradient fills came out solid; (17) the renderer keeps annotation paints out of the reading-colours mapping between `begin_annotations` and `end_annotations` (#1548); (18) the page renders as one transparency group; under reading colours its background is painted inside it as the initial backdrop, so a Multiply highlight blends with the dark background instead of staying an opaque block, and otherwise the background stays outside, so a blend mode over an empty part of the page keeps its colour instead of blending with the white paper (#2970); (19) `set_paint` samples a mesh shading's texture over its triangles alone (`EncodedShadingPattern::extent`, padded rather than repeated) and skips a mesh with none on the canvas: a BricsCAD plan's 310 one-triangle `sh` each covered the whole tile, 2 s per 1024 px tile at any zoom (the watchdog gave the page up on a busy machine), now about 70 ms; (20) a tiling pattern whose step is too large for a cell pixmap, where what is painted (a fill, or a stroke with its width) can only show the first cell (PowerPoint's `/XStep 2147480100` for a picture fill drawn once), has its `/BBox` drawn at full scale with a transparent border and padded instead of tiled (`shows_one_cell`; a cell whose scale isn't finite and positive, from a `/BBox` too thin to divide by, keeps the old path: `usable_cell_scale`); shrinking the cell to fit the 3000 px limit of (3) left the picture under a pixel, so network diagrams lost their pictures (#3349); (21) an image that does not request interpolation and is drawn below its resolution, outside a Type 3 glyph, is shrunk by area averaging (`area_shrink`: each device pixel the coverage-weighted mean of the samples it covers, in 16.16 fixed point whose weights sum to exactly one, premultiplied when the image has a mask; a soft mask no larger than the image is first sampled onto the image's grid by `mask_on_image_grid` (nearest-neighbour, or bilinearly when the mask asks for `/Interpolate`, so its edge stays as smooth as an enlarged mask's), so its colour and mask are averaged together instead of the averaged colour of transparent samples showing under a mask layer at the mask's edges) to its snapped device rectangle, instead of nearest-neighbour: `/Interpolate` asks for smoothing only when an image is enlarged (ISO 32000-2 §8.9.5.3), and nearest-neighbour kept one sample in N per axis, so the text of a 300 dpi scan at 100% lost most of its strokes (sampling them again at a fractional offset dropped more). PicScale's `Box` and `Area` filters were not used: shrinking, both average a window four target pixels wide. On a 13-page 300 dpi scan every rendered sample at 96, 120 and 192 dpi is within 1 of the exact area average (it was up to 255 off, 17-23% of samples by more than 1). A snapped image whose shrink failed keeps the full transform, so it is not drawn at its source size. The shrink never enlarges: a side drawn at or above its resolution (an image shrunk across and stretched down, as a 204 × 98 dpi fax is) keeps its samples for the sampler. Rows are averaged first (contiguous, cheap) and the fewer rows that leaves are then shrunk across with taps worked out once per image, which made the shrink about three times faster (a 300 dpi page at 96 dpi: 25 ms, nearest-neighbour 11 ms). A source with a side over `MAX_RESAMPLING_SOURCE_SIDE` (2^20) keeps (10)'s nearest-neighbour shrink, and `area_shrink` refuses it, so its accumulator row and tap table stay a few times one source row (a 2^27 × 2 image of zeros is a tiny file); (22) knockout groups: while one is innermost, fills, strokes, glyph outlines and shadings are drawn with `Compose::Copy` (weighted by coverage), so each element replaces what earlier elements of the group drew instead of compositing over it; an image or nested group, drawn into a layer of its own, first has what is under its shape cleared (`knockout_image` for an unmasked image's rectangle, only when `draw_image` will draw it, decoded at the size it is drawn; a knockout erase pass that draws a nested group's content with every paint clearing instead of painting, a shading fill's clear clipped to the shading's `/BBox` as its paint is) and then composites over it (#2971); (23) backport of hayro 0.8: shadings that `as_svg_gradient` can represent are painted as a `vello_cpu` linear or two-point radial gradient instead of a texture sampled pixel by pixel through the shading function (InDesign fades are luminosity soft masks of one full-page axial `sh`: 40 on a report page took 15 s at 144 dpi, now under 1 s); without upstream's half-pixel shift, which suits vello_cpu 0.3 but moves 0.0.8's gradients off pixel centres; (25) a tiling cell is drawn at the scale that makes one `/XStep` (`/YStep`) exactly the cell pixmap's whole number of pixels (`tiling_cell_pixels`, at least 1 and at most the cap), so the repeated pixmap puts cells at whole multiples of the step at any device scale; drawing at the unrounded scale repeated the cell up to half a pixel off at every step, and the error grew with distance from the pattern's origin (ISO 32000-2 8.7.3.1); (28) an image of page-scan size (`SMOOTHED_IMAGE_SIDE`: 512 samples or more on its shorter side) that does not request interpolation is drawn with `ImageQuality::Medium` (bilinear, as interpolated images are) when enlarged on either side, also when the other side is shrunk (the area shrink leaves that side at 1:1), outside a Type 3 glyph: scans rarely set `/Interpolate`, and drawn nearest-neighbour every sample of a zoomed scan became a block, so its text had staircase edges; icons, QR codes and pixel art keep their hard pixels. Bicubic (`High`) looked the same on a zoomed 300 dpi scan and drew four 1024 px tiles at 400% in 267 ms against 50 ms | (1) `raster::tests::tiles_match_full_render`; (2) `raster::tests::absurd_image_dimensions_are_skipped`; (3) `raster::tests::tiling_cells_stay_small_whatever_the_step`; (4) `raster::tests::absurd_predictor_columns_and_line_widths_render`; (5) `raster::tests::dash_patterns_that_cut_a_stroke_into_billions_of_pieces_render`; (6) `raster::tests::invalid_dash_patterns_terminate_and_are_drawn_solid`; (7) `raster::tests::pixels_are_a_plain_renders_bytes_in_a_buffer_a_gui_can_take_over`; (8) `raster::tests::image_resampling_rejects_anisotropic_target_growth`, `raster::tests::image_resampling_bounds_intermediate_before_planning`, `raster::tests::image_resampling_rejects_invalid_sources_and_scales`, `raster::tests::image_resampling_keeps_valid_sizes_and_exact_limits`, `raster::tests::resampling_fallback_preserves_original_geometry_and_pixels`, `raster::tests::image_resampling_wide_sources_shrink_before_backend_side_limits`, `raster::tests::image_resampling_wide_transparent_sources_still_shrink`, `raster::tests::image_resampling_padded_backend_dimensions_stay_checked`, `raster::tests::image_resampling_mismatched_alpha_mask_keeps_pixels_and_placement`; (9) `raster::tests::skipping_paths_off_the_canvas_changes_no_pixel`; (10) `raster::tests::non_interpolated_images_are_not_smoothed_when_minified`, `raster::tests::non_interpolated_gray_images_are_not_smoothed_when_minified`, `raster::tests::non_integer_minification_averages_the_columns_each_pixel_covers`, `raster::tests::non_integer_minification_averages_colour_and_mask_together`, `raster::tests::non_interpolated_alpha_images_keep_their_colours_and_mask_when_minified`, `raster::tests::non_interpolated_type3_glyph_images_are_not_smoothed_when_minified`, `raster::tests::image_resampling_nearest_accepts_wide_sources_the_filter_refuses`, `raster::tests::wide_non_interpolated_sources_draw_after_shrinking`; (11) `raster::tests::skipping_paths_off_the_canvas_changes_no_pixel`; (12) `raster::tests::preview_renders_skip_tiny_images_but_finals_draw_everything`; (13) `color_replacement_tests::{black_maps_to_text_colour, white_maps_to_background, midgrey_interpolates, alpha_copied_verbatim, only_black_threshold, line_art_false_skips_paths_but_not_text}`, `color_replacement_render_tests::{color_replacement_maps_text_and_line_art, color_replacement_maps_stencil_mask_ink}`; (14) `raster::tests::minified_image_edges_snap_to_whole_pixels`, `raster::tests::a_tile_snaps_half_pixel_image_edges_as_the_whole_page_does` (`cargo test -p hayro`), `raster::tests::a_turned_or_mirrored_minified_image_snaps_to_whole_pixels`, `rotated_image_snap::a_shrunk_image_on_a_turned_page_is_the_upright_render_turned` (`cargo test -p pdfcraft-render --test it`), `renderer::snap_tests::a_skew_that_moves_a_far_edge_is_not_snapped` (`cargo test -p hayro`); (15) `raster::tests::shrunk_soft_mask_larger_than_its_image_keeps_thin_strokes`; (16) `raster::tests::soft_mask_applies_to_a_shading_pattern_fill`; (17), (18) `raster::tests::reading_colors_keep_highlight_annotations_readable`; (19) `mesh_extent_tests::{a_mesh_paints_its_triangles_alone, a_mesh_keeps_the_graphics_state_alpha, a_mesh_off_the_canvas_paints_nothing}` (`cargo test -p hayro`); (20) `raster::tests::a_tiling_pattern_drawn_once_shows_its_cell`, `raster::tests::a_large_step_still_tiles_where_the_next_cell_shows`, `raster::tests::a_tiling_cell_too_thin_to_scale_keeps_the_old_path`; (21) `raster::tests::non_interpolated_minification_keeps_thin_strokes`, `raster::tests::non_integer_minification_averages_the_columns_each_pixel_covers`, `raster::tests::non_integer_minification_averages_colour_and_mask_together`, `raster::tests::non_interpolated_minification_does_not_bleed_transparent_colour`, `raster::tests::a_smaller_soft_mask_does_not_bleed_transparent_colour_when_minified`, `raster::tests::an_interpolated_smaller_soft_mask_stays_smooth_when_its_image_is_minified`, `raster::tests::minified_image_edges_snap_to_whole_pixels`, `raster::tests::an_image_shrunk_one_way_and_stretched_the_other_is_still_averaged`, `raster::tests::wide_non_interpolated_sources_draw_after_shrinking`, `renderer::area_shrink_tests` (`cargo test -p hayro`); (22) `raster::tests::knockout_groups_replace_what_their_elements_cover`; (23) `raster::tests::axial_and_radial_shadings_draw_as_native_gradients`; (25) `tiling_cell_placement::{far_cells_of_a_fill_sit_at_whole_steps, far_cells_sit_at_whole_steps_under_a_turned_scaled_matrix, far_cells_of_a_stroke_sit_at_whole_steps, a_cell_pixmap_spans_exactly_one_step}` (`cargo test -p pdfcraft-render --test it`); (28) `raster::tests::large_non_interpolated_images_are_smoothed_when_enlarged`, `raster::tests::small_non_interpolated_images_keep_hard_pixels_when_enlarged`, `raster::tests::a_scan_shrunk_one_way_is_smoothed_the_way_it_is_stretched` | | hayro-jbig2 | 0.3.0 | Apache-2.0 OR MIT | (1) at most 2^28 pixels per bitmap and page bitmap, and 2^25 cells per halftone grid and grayscale image (each side could be 65535: 4.3 billion pixels decoded one by one, and a 4-byte value per cell); (2) at most 2^26 symbol pixels per symbol dictionary (a fuzzed dictionary decoded for minutes); (3) a symbol dictionary's height classes are capped at one per new symbol and its export runs at `2 × symbols + 1`, each plus `EXTRA_EMPTY_STEPS` (1024): an empty height class or a zero-length export run doesn't advance, and past the end of its data the arithmetic decoder can decode them for ever (one changed byte hung the renderer); (4) a segment header's long-form referred-to segment count is capped at `u16::MAX` and a larger count is refused before its retention or referred-to bytes are read: the 29-bit count went straight to `Vec::with_capacity`, a reservation of about 2 GiB whose failure aborts the process (from storytold/craft-libs#176); (5) what a page's segments hold for later ones (intermediate regions, pattern and symbol dictionaries with their headers and retained contexts, and Huffman tables) is at most 64 MiB, two square bitmaps of 16,384 a side (`MAX_RETAINED_WORDS`), and a dictionary's new and exported symbols are counted against it as they are made: each was bounded on its own, but a few dozen bytes an intermediate region held 33 MB each, and 8 MiB symbols exported again and again killed the process, so a few kilobytes could exhaust memory; (6) a segment may refer to at most 2^20 symbols, checked before any are collected: references may repeat, and 65,535 of them to one dictionary, in a 78 KB stream, asked for 17 GB; (7) a pattern dictionary keeps at most 16 MiB of shifted copies, which only speed up drawing, and none that wouldn't fit what the page has left (`MAX_SHIFTED_WORDS`); (8) a custom Huffman table has at most 2^16 lines, and a stream with its globals at most 2^20 segments and 2^22 references, checked as they are read: both grew before anything counted them, and PDFs of a kilobyte or so reached 5 GB and more behind nested Flate; (9) a page past the bounds of (1) is refused when the stream is opened: a caller sizes its output from the header before the page bitmap is made, and an Indexed image in a 743-byte PDF asked for 137 GB, which aborts; (10) a stream may ask for at most 2^30 units of decoding work, four of the largest pages, about 6 s to 17 s (`MAX_WORK`): decoding a bitmap costs a pixel each and 4 a row, copying one a word each and 1 a row, and each integer an item reads costs 38, what one takes at worst, so one with no pixels pays too, before it is decoded: each bitmap was bounded, but nothing bounded how many there were, and a 795-byte PDF took 265 s; (11) `Image::decode_within` spends from what a run of images may still do, starting with its page's pixels, and `charge_reading` charges reading a stream and its globals before anything is parsed, so an operation over a document can bound what `MAX_WORK` bounds for one image, and images that fail or decode nothing add up too: a file of thousands of images made to reach that limit took as many times as long; (12) adding to a location, a reference offset, a symbol's corner or a Huffman range line near the end of an `i32`, and negating a decoded integer of 2^31, overflowed: a panic wherever overflow is checked. The refinement context, a text region's corners, the integer decoder and the Huffman table wrap, as a release build did, and `Bitmap::combine` saturates, which comes to the same, as a location that overflows is past the end of any page. What is drawn doesn't change | (1) `raster::tests::jbig2_regions_of_billions_of_pixels_are_refused`, `raster::tests::jbig2_halftone_grids_past_their_bound_are_refused`; (2) the nightly `cargo xtask fuzz` finding, which now finishes in about a second; (3) `raster::tests::jbig2_symbol_dictionaries_that_never_advance_terminate`; (4) `segment::tests::test_referred_to_count_is_capped` (`cargo test --manifest-path vendor/hayro-jbig2/Cargo.toml`); (5) `raster::tests::jbig2_regions_held_past_a_pages_bound_are_refused`, `jbig2_symbols_without_pixels_count_towards_a_pages_bound`, `jbig2_retained_contexts_count_towards_a_pages_bound`, `jbig2_huffman_tables_count_towards_a_pages_bound`, `jbig2_new_symbols_count_towards_a_pages_bound`, `jbig2_new_symbols_are_held_to_what_a_page_has_left`, `jbig2_pattern_dictionaries_count_towards_a_pages_bound`, and in the crate (`cargo test --manifest-path vendor/hayro-jbig2/Cargo.toml`) `decode::symbol::tests::exported_symbols_are_counted_before_they_are_copied`, `tests::the_room_left_shrinks_as_segments_are_stored`; (6) `raster::tests::jbig2_segments_referring_to_too_many_symbols_are_refused`; (7) `raster::tests::jbig2_pattern_dictionaries_too_large_to_shift_still_draw`, `jbig2_shifted_patterns_never_refuse_an_image`; (8) `raster::tests::jbig2_huffman_tables_of_too_many_lines_are_refused`, `jbig2_streams_of_too_many_segments_are_refused`, `jbig2_streams_of_too_many_references_are_refused`, and in the crate `tests::the_segments_of_a_stream_and_its_globals_count_together`; (9) `raster::tests::jbig2_pages_past_the_bounds_are_refused_before_anything_is_sized_from_them`, `tests::a_page_past_the_bounds_is_refused_when_opened`; (10) `raster::tests::jbig2_decoding_work_is_counted_item_by_item`; (11) `jbig2::decode::tests::a_run_of_images_spends_its_allowance_down_to_the_step` (pdfcraft-filters); (12) `raster::tests::jbig2_placements_at_the_end_of_an_i32_draw_nothing` | | lopdf | 0.45.0 | MIT | (1) PNG predictor rows longer than the data are refused before their buffers are allocated (a fuzzed `/Columns 4294967295` allocated two 4 GiB rows), and empty frames return without row buffers; (2) predictor row widths use saturating arithmetic or reject overflow; (3) sub-byte TIFF Predictor 2 scratch is bounded by the samples present in the input instead of the declared `/Colors`, aligned with [lopdf#589](https://github.com/J-F-Liu/lopdf/pull/589); (4) the object lookup keeps its cycle history through compressed objects, requires an ordinary in-file container, and caps the chain of indirect lookups at 16 (a chain of new objects, each a `/Length` compressed in a container whose own `/Length` names the next, or a stream whose `/Length` names the next stream, overflowed the stack: a dev build's 2 MiB thread within 800 hops); (5) object-stream members are parsed within their own extent (up to the next distinct offset), an entry whose offset repeats an earlier entry's is skipped (so a repeated object number can resolve to its earlier entry), and offsets are processed in index order; (6) `find_latest_trailer`: a rebuild takes a `/Type /XRef` stream dictionary as the trailer when the file has no `trailer` keyword (the latest in the file wins, as a later `trailer` would), so a concatenated or truncated file keeps its /Info and /ID instead of failing to load; a candidate whose /Root is the catalog beats one whose /Root is another dictionary, and an in-file /Root must match its generation (#2365) | (1) `inspect::tests::png_predictor_rows_longer_than_the_data_are_refused_before_allocating`, `inspect::tests::empty_png_predictor_frames_do_not_allocate_rows`; (2) `inspect::tests::tiff_subbyte_predictor_row_width_overflow_is_refused`; (3) `inspect::tests::tiff_subbyte_predictor_scratch_is_bounded_by_available_samples`, `inspect::tests::tiff_subbyte_predictors_keep_components_rows_and_padding`; (4) `inspect::tests::compressed_object_self_container_does_not_recurse`, `inspect::tests::compressed_object_mutual_containers_do_not_recurse`, `inspect::tests::object_stream_length_cycle_does_not_recurse`, `inspect::tests::compressed_object_container_chain_does_not_recurse`, `inspect::tests::compressed_length_chain_does_not_recurse`, `inspect::tests::plain_stream_length_chain_does_not_recurse`, `inspect::tests::compressed_objects_and_indirect_stream_length_still_load`, `inspect::tests::compressed_length_in_a_container_with_an_indirect_length_still_loads`; (5) `inspect::tests::object_stream_member_extents_bound_owned_strings`, `inspect::tests::object_stream_member_extents_preserve_index_order_and_whitespace`; (6) `inspect::tests::rebuilt_file_keeps_the_info_of_its_cross_reference_stream`, `inspect::tests::rebuilt_file_prefers_a_trailer_whose_root_is_the_catalog` | | hayro-syntax | 0.7.2 | Apache-2.0 OR MIT | (1) Page-tree cycle guard in `resolve_pages` (visited set + depth cap): a `/Kids` loop among compressed objects overflowed the stack and aborted the process; (2) `ccitt_size_ok`: CCITT images wider than 2^20 columns or over 2^28 pixels are not decoded (a fuzzed `/Columns 4294967295` allocated 4 GiB); (3) Flate/LZW predictor row widths are saturating `usize`, a zero width is refused and a row longer than the data is not allocated (a fuzzed `/Columns 9223372036854775807` wrapped to a 2^61-byte allocation and aborted); (4) the inline-image end-of-data check (`EI`) doesn't nest inside its own look-ahead and examines at most 256 candidates and steps (50 images with "EI" in their data ran for over ten minutes); an EI followed by a delimiter (EI%..., EI[...) ends the image like one followed by white space; only a regular character after EI skips the candidate; (5) a stream whose `/Length` is wrong is recovered only when `stream` is the next token — scanning ahead to a later object's `stream` made an indirect appearance-state dictionary (Quartz checkboxes: `/AP /N` → `<< /Yes … /Off … >>`) render as a non-form, so the widget was skipped; (6) JPX rescaling uses an exact integer sample range for 1–32-bit components (32 shifted past u32, and floating-point maxima at 25–31 bits wrapped to zero), and refuses zero-sized rows; (7) `decode_limit`: Flate (through flate2 and the fallback inflater), LZW, RunLength and ASCII85 never write past `MAX_DECODED_STREAM` (256 MiB), or, for the last filter of an image of known size, past twice its pixel bytes (a predictor tag byte per byte), at most 1 GiB; `Stream::decoded_within` for budgeted callers; a page's `/Contents` array shares one `MAX_DECODED_STREAM` (a 2 MB page whose content stream or 8 × 8 image inflated to 2 GB took the renderer past 3 GB in under a second; naming the stream twenty times in `/Contents`, 5 GB); (8) high-precision numbers (jsPDF MediaBox coordinates with over 19 digits) parse without overflowing the u64 mantissa: fractional digits past it are dropped, integer digits past it scale the value (a real, kept finite); (9) large dictionary indexes are built on first use and shared across repeated reads within each PDF (#307); the validated-byte cache is bounded, reader contexts remain separate, repairs and inline/content dictionaries bypass sharing, and small dictionaries stay eager; (10) decoded object-stream slots publish once under contention, cold loading rejects recursive dependencies, and xref repair serializes without holding recursive read guards; this permits one parser per native render pool with private worker caches; (11) `sync.rs`: a poisoned lock is recovered instead of unwrapped (a worker panic no longer cascades into every reader of the same PDF), and xref repair takes a blocking write lock instead of asserting a non-blocking one; (12) `SegmentList` (`util.rs`) returns `None` for an out-of-range index instead of panicking; (13) a page's `/UserUnit` scales `render_dimensions` and `initial_transform`, so the page and everything on it are sized in points, as Acrobat shows them; it is read as Acrobat reads it (the page's own entry, not inherited; a number from 1 up, at most 75,000; anything else counts as 1), and a page without it keeps exactly the transform it had; (14) revision 2–4 passwords are tried as given and then in PDFDocEncoding (`legacy_passwords` in `crypto/mod.rs`, which resolves the upstream `TODO: Convert to PDFDocEncoding`): passwords arrive as UTF-8 text, so a password with characters outside ASCII (`Çok güzel`, `ılık€`) written in PDFDocEncoding, as ISO 32000-2 §7.6.4.3.2 asks and qpdf and Acrobat write it, was always wrong; (15) page content is read a piece at a time (`page::ContentReader`, `Page::content_reader`): plain streams and Flate streams without a predictor are decoded as they are read, with a budget of 64 times their compressed size and never less than `MAX_DECODED_STREAM` (a stream that expands past it is cut and reported); other streams are decoded whole, as before, and share `MAX_DECODED_STREAM`. The content tokenizer reads a window at a time (`UntypedIter::new_window`, `TypedIter::needs_more`, `TypedIter::committed`): an instruction that runs past the window is not decided until more data follows, which `Reader` records by flagging reads and `at_end` past the end of the data, including through the clones the parser makes. The Flate decoder keeps its own 32 KiB history, so it pauses at any output size. `ContentReader::new` reuses raw/decrypted stream bytes for budget accounting and piece creation, and `decoded_raw_within` applies the existing Whole-stream filters and limits to those bytes without decrypting twice; (16) revision 5–6 (AES-256) passwords are tried in their SASLprep form (RFC 4013, then UTF-8, at most 127 bytes, as ISO 32000-2 §7.6.4.3.3 Algorithm 2.A asks and pdfcraft-crypt writes them) and then, if that differs, as given (`aes256_passwords` in `crypto/mod.rs`, which resolves the upstream `TODO: Do the above.`; `stringprep`, already in the tree through pdfcraft-crypt, is an optional dependency enabled by `std`, and without `std` passwords are used as given): the renderer used the password as typed, so one that SASLprep changes (a decomposed `café`, a no-break space, the fullwidth `ａｂ` a Japanese IME types in full-width mode) was wrong, although pdfcraft-cos accepted it: protecting a document with such an open password was rolled back ("the password is incorrect"), and a file with such a password did not open with it.; (17) CCITT `/Rows 0` means the height is not predetermined (ISO 32000-2 Table 11), not zero rows: scanned Group 4 pages written that way rendered blank; (18) The streaming Flate decoder (`lzw_flate::flate::fallback`) copies a back-reference in runs of up to its distance (`extend_from_within`), not byte by byte. The copy reads only bytes already in the output, and a copy whose source does not exist still writes nothing, so the bytes are those a byte-by-byte copy writes. Inflating the A1 sample's content stream (278.6 MB) takes about 14% less time (0.66 s to 0.56 s); the output is identical; (19) CCITT `/Rows` below the image's `/Height` decodes to the height: a plotter driver wrote `/Rows` equal to `/Columns`, so only the first rows were decoded and the rest drawn black (#3308) (20) `stream` followed by spaces or tabs and then the line break (`stream \r\n`, as Microsoft Reporting Services writes it) starts the data after the line break; the content stream was taken to start at the space and the page rendered blank.; (21) a JP2 colour specification naming an enumerated colour space the decoder refuses (CIEJab in a veraPDF fixture, YCbCr, PhotoYCC, YPbPr, bilevel…) no longer fails the image: it is decoded again from a copy whose `colr` names the space the decoder assumes without one, greyscale, sRGB or CMYK by component count, as MuPDF and Poppler draw it (`without_refused_enumerated_colour`; the page drew blank, #3306); (22) an object stream's index of `objnum offset` pairs is parsed once per stream and kept beside its decoded data (`ObjectStreamIndexes`, `XRef::with_object_stream_indexes`), and an offset past `usize` is malformed instead of wrapping: `XRef::get_with` parsed the whole index again for every object it fetched from the stream, so reading all of a stream of N objects took N² steps (a Firefox page whose 65,542 objects share one stream took 37 s to draw and 40 s to extract its text); (23) the inline-image end-of-data look-ahead no longer rejects a candidate `EI` because a byte above 0x7F lies between it and the next one: when the candidate is the end, those bytes are the content stream that follows the image, and its strings carry such bytes (WinAnsi `(Grüße) Tj`, the two-byte codes of any Type 0 font), so the operator check that follows decides alone, bounded as in (4) (a scanned page whose OCR text held an accented word and a later capital "EI" — EINGANG, BEI, KLEIN — drew and extracted nothing after the image: 40 of 275 pages across five real documents); (24) `xref::prev_offset`: a trailer `/Prev` written as a reference `N G R` is read as the offset that object holds when the section's own entries place it (iText's `indirectPrev.pdf`; each section's entries are read before its `Prev` is followed), and otherwise as offset N (tiff2pdf; JHOVE's PDF-HUL-157 files, #2366), as `pdfcraft-cos` and other readers take it, so the earlier sections' pages are seen instead of one; (25) direct arrays and dictionaries share a reader-local nesting cap of 128, counted while skipping and while parsing a dictionary (its lenient repair reads, rather than skips, an object found where a key belongs, so `<< << …` recursed through that read); deeper objects return `None` instead of overflowing the stack (including unused trailer values, trailer entries, content operands and marked-content properties); a chain of 65 direct stitching functions now stops at this cap, before the 64-function guard; (26) an xref stream subsection must be backed by its decoded data: all-zero `/W` widths (an entry reads no bytes) and a declared count longer than the data are refused before any entry is inserted, where a tiny stream with a large `/Size` or `/Index` count inserted that many entries (from storytold/craft-libs#239); a subsection running past object number `i32::MAX` is refused instead of overflowing or turning negative; (27) an xref stream's type field is read big-endian at its full `/W[0]` width, where only its first byte was read and wider fields shifted every later entry, and `/W[0]` over 8 bytes is refused like `/W[1]` (from storytold/craft-libs#241); (28) a content instruction keeps up to 256 operands (`MAX_OPERANDS`) instead of 10, and past that drops the oldest instead of ending the content stream: the eleventh operand of a twelve-ink DeviceN `scn` (ISO 32000-2 §8.6.6.5 allows any number of components) blanked the rest of the page (#4260); (29) a `/JBIG2Globals` stream that is itself JBIG2 isn't decoded: globals are raw segments, and a chain of 16,000 such streams overflowed the stack, which aborts the process; (31) `set_page_jbig2_work`: the JBIG2 images decoded on a thread spend one allowance, set per page, 2^30 steps (`MAX_PAGE_JBIG2_WORK`), so a page of many images made to reach their own limit stops at one, and what they cost to read counts, whether they decode or not; unbounded until a page sets it; (32) a group of array items read together (a tuple: an image's `/Decode`, `/W`, `/W2`, and a function's `/Domain`, `/Range`, `/Encode` and `/Decode`) can start with a reference: it was taken for the whole group and lost, which ended the array, so an image with `/Decode [7 0 R 0]` didn't draw and CID widths fell back to `/DW`; (33) an object that holds only a reference stands for what that names, stored on its own or in an object stream, up to 32 references: read as a number it was the reference's own object number, so a `/Decode` or `/W` entry naming such an object took a wrong value, and as anything else it was lost. Past 32, as any cycle goes, it reads as a missing object. An object stream isn't reached that way, as its checks are of the object named; reading its index counts against the 64 nested loads and reads its dictionary alone, as a chain of them, each with a member of the next for its `/N`, `/First` or `/Length`, ran out of stack; and a comment inside a reference counts as white space; (34) a table rebuilt by scanning the file (`fallback_xref_map`) finds an object by its number when its header names another generation, including one no generation may be (`3 18446744073709551616 obj` in `poppler-85140-0.pdf`, #5580): the header is read leniently only in that scan (`read_object_header`), the object is also filed under generation 0 unless a real `N 0 obj` holds that key (a later header for the number moves the stand-in), `XRef::get_with` matches by object number only in a repaired table, and `XRef::objects` yields each location once | (1) `cos/tests/writer.rs::page_tree_loops_in_object_streams_do_not_crash_readers`; (2) `raster::tests::absurd_mask_and_fax_sizes_are_refused_before_decoding`; (3) `raster::tests::absurd_predictor_columns_and_line_widths_render`; (4) `raster::tests::inline_images_with_ei_in_their_data_parse_in_linear_time`, `content::tests::inline_image_ei_may_be_followed_by_a_delimiter`; (5) `raster::tests::indirect_appearance_state_dict_is_not_another_stream`; (6) `raster::tests::jpx_stream_bit_depth_boundaries_do_not_panic`, `raster::tests::jpx_32_bit_image_does_not_prevent_other_page_content_from_rendering`, `raster::tests::jpx_gray_rgb_and_alpha_keep_eight_bit_samples`, `raster::tests::jpx_streams_reject_out_of_range_bit_depths`; (7) `raster::tests::decompression_bombs_stop_at_the_stream_limit`, `raster::tests::page_contents_and_nested_forms_share_one_budget`; (8) `inspect::tests::jspdf_high_precision_mediabox_does_not_overflow_dimensions`, `object::number::tests::high_precision_float_does_not_overflow_mantissa` (hayro-syntax); (9) `object::dict::lazy_index_tests`, `raster::tests::large_resource_indexes_preserve_pixels_and_text`; (10) `data::tests`, `xref::tests::concurrent_cold_object_stream_reads_all_succeed`, `raster::tests::shared_parser_is_lazy_and_initializes_once_under_contention`; (11) `data::tests::panicking_initializer_wakes_waiters_and_prevents_retries`, `xref::tests::concurrent_cold_object_stream_reads_all_succeed`; (12) `util::tests::segment_list_rejects_out_of_range_indices`; (13) `raster::tests::user_unit_scales_the_page_and_everything_on_it`; (14) `crypto::pdfcraft_password_tests`, `pdfcraft-engine` `tests::legacy_passwords_with_turkish_characters_open`; (15) `content::tests::windows_keep_every_instruction_at_every_split`, `content::tests::windows_keep_every_instruction_in_pieces_of_any_size`, `content::tests::windows_match_the_slice_on_random_content`, `content::tests::windows_drop_blanks_and_keep_a_comment_that_may_go_on`, `page::content_reader_tests::content_past_the_cap_is_read_to_the_end`, `page::content_reader_tests::a_stream_that_expands_past_the_budget_is_cut_there`, `page::content_reader_tests::a_small_expansion_under_the_cap_is_not_cut`, `raster::tests::page_content_past_the_old_cap_renders`, `raster::tests::content_reader_returns_the_same_bytes_in_any_pieces`, `raster::tests::content_bombs_stop_at_the_budget_and_say_so`, `page::content_reader_tests::plain_and_whole_pieces_keep_their_bytes`, `page::content_reader_tests::repeated_content_reference_counts_compressed_bytes_once`, `page::content_reader_tests::whole_cap_still_counts_later_unique_streams`; (16) `crypto::pdfcraft_password_tests::aes256_passwords_are_tried_in_their_saslprep_form_then_as_given`, `crypto::pdfcraft_password_tests::aes256_passwords_open_as_typed_and_in_their_saslprep_form`, `pdfcraft-engine` `tests::aes256_open_passwords_that_saslprep_changes_protect_and_reopen_as_typed`, `tests::aes256_files_whose_passwords_saslprep_changes_open_as_typed`; (17) `raster::tests::group4_fax_with_rows_zero_decodes_to_the_image_height`; (18) `filter::lzw_flate::tests::back_reference_longer_than_its_distance_repeats_its_source`, `back_reference_cut_by_the_limit_resumes_in_the_next_call` and `back_reference_reads_bytes_written_by_an_earlier_call`; each fails when the copy returns the wrong remaining count; (19) `raster::tests::group4_fax_with_rows_below_its_height_decodes_to_the_image_height`; (20) `raster::tests::space_before_the_line_break_after_stream_still_renders`; (21) `raster::tests::a_jp2_image_in_a_refused_enumerated_colour_space_is_drawn`, `filter::jpx::tests::refused_enumerated_colour_spaces_become_the_component_counts_default`, `filter::jpx::tests::what_the_decoder_handles_or_cant_be_read_is_left_alone` (`cargo test --manifest-path vendor/hayro-syntax/Cargo.toml`); (22) `raster::tests::objects_in_one_large_object_stream_are_read_in_linear_time`; (23) `text::tests::text_after_an_inline_image_survives_a_non_ascii_byte_before_a_later_ei`, `content::tests::an_inline_image_ends_at_its_own_ei_when_later_text_holds_a_non_ascii_byte` (hayro-syntax); (24) `xref::tests::prev_as_reference_reaches_the_earlier_section`, `xref::tests::prev_as_reference_to_an_object_holding_the_offset_reads_that_offset`, `pdfcraft-cos` `document::tests::a_prev_written_as_a_reference_still_reaches_the_earlier_sections`, `document::tests::a_prev_reference_to_an_object_holding_the_offset_reads_that_offset`; (25) `raster::tests::deeply_nested_direct_objects_do_not_overflow_the_stack`, `raster::tests::dictionaries_nested_without_keys_in_content_do_not_overflow_the_stack`, `raster::tests::dictionaries_nested_without_keys_in_the_trailer_do_not_overflow_the_stack`, `raster::tests::dictionaries_nested_without_keys_as_marked_content_properties_do_not_overflow_the_stack`, `raster::tests::dictionaries_nested_without_keys_are_read_up_to_the_nesting_cap`, `raster::tests::nesting_at_the_cap_fits_a_512_kib_stack` (native only), `raster::tests::function_limits_depth_guard_refuses_a_chain_of_references`; (26) `xref::tests::xref_stream_entries_need_backing_bytes`; (27) `xref::tests::xref_stream_type_field_uses_the_full_w0_width`, `xref::tests::xref_stream_loader_rejects_oversized_w0` (`cargo test --manifest-path vendor/hayro-syntax/Cargo.toml`); (28) `content::ops::tests::scn_with_twelve_components_keeps_them_all`, `content::ops::tests::operands_past_the_cap_keep_the_latest_in_order` (hayro-syntax), `raster::tests::devicen_inks::a_twelve_ink_devicen_colour_paints`, `raster::tests::devicen_inks::a_four_ink_devicen_colour_paints`, `raster::tests::devicen_inks::an_operator_with_hundreds_of_operands_does_not_end_the_content`; (29) `raster::tests::jbig2_globals_that_are_jbig2_themselves_are_not_decoded`, `raster::tests::jbig2_globals_are_read`, `raster::tests::the_jbig2_images_of_a_page_share_one_allowance`, `raster::tests::an_image_that_fails_spends_the_pages_allowance_too`; (32) `raster::tests::a_decode_array_with_references_in_it_is_read`, `raster::tests::a_functions_arrays_can_start_with_a_reference` (pdfcraft-render), `font::cid::width_tests::a_width_group_can_start_with_a_reference` (hayro-interpret); (33) `raster::tests::a_reference_to_a_reference_is_what_it_names`, `raster::tests::a_decode_array_with_references_in_it_is_read` (pdfcraft-render), `font::cid::width_tests::a_width_group_can_start_with_a_reference` (hayro-interpret), `xref::tests::an_object_stream_isnt_reached_through_a_reference`, `xref::tests::a_chain_of_object_streams_ends_at_the_limit_of_nested_loads`, `xref::tests::a_chain_of_object_streams_takes_no_more_work_for_two_ways_on`, `xref::tests::a_reference_held_in_either_storage_is_followed`, `xref::tests::a_page_a_reference_is_held_to_is_found_once` (hayro-syntax); (34) `raster::tests::a_rebuilt_table_finds_an_object_whose_header_names_another_generation`, `raster::tests::a_rebuilt_tables_generation_0_stand_in_is_not_a_second_object` (pdfcraft-render), `xref::tests::an_object_stream_and_its_first_member_are_both_enumerated` (hayro-syntax) | diff --git a/vendor/hayro/src/renderer.rs b/vendor/hayro/src/renderer.rs index 120a751281..c3d1d18520 100644 --- a/vendor/hayro/src/renderer.rs +++ b/vendor/hayro/src/renderer.rs @@ -193,27 +193,67 @@ fn image_buffer(len: usize) -> Option> { Some(out) } -/// PdfCraft patch: `mask` sampled onto the grid of a `width` × `height` image, each image sample -/// taking the mask sample under its centre. `None` for an empty side or a buffer over the limits. +/// PdfCraft patch: `mask` sampled onto the grid of a `width` × `height` image (no side smaller +/// than the mask's). Each image sample takes the mask sample under its centre or, for a mask that +/// asks for `/Interpolate`, the bilinear blend of the four around it, so the mask's edges stay as +/// smooth as when it was enlarged on its own. `None` for an empty side, an image side smaller than +/// the mask's, or a buffer over the limits. fn mask_on_image_grid(mask: &LumaData, width: u32, height: u32) -> Option> { if width == 0 || height == 0 || mask.width == 0 || mask.height == 0 { return None; } let mut out = image_buffer(source_byte_len(width, height, 1, MAX_IMAGE_PIXELS)?)?; - let mut columns = Vec::new(); - columns.try_reserve_exact(width as usize).ok()?; - columns.extend((0..width).map(|x| nearest_source_index(mask.width, width, x))); + let columns = grid_axis(mask.width, width, mask.interpolate)?; + let rows = grid_axis(mask.height, height, mask.interpolate)?; let mask_width = mask.width as usize; - for y in 0..height { - let start = nearest_source_index(mask.height, height, y).checked_mul(mask_width)?; - let row = mask.data.get(start..start.checked_add(mask_width)?)?; - for &x in &columns { - out.push(*row.get(x)?); + let row = |y: usize| { + let start = y.checked_mul(mask_width)?; + mask.data.get(start..start.checked_add(mask_width)?) + }; + let one = u64::from(GRID_ONE); + for &(y0, y1, fy) in &rows { + let (top, bottom, fy) = (row(y0)?, row(y1)?, u64::from(fy)); + for &(x0, x1, fx) in &columns { + let fx = u64::from(fx); + let blend = |r: &[u8]| Some(u64::from(*r.get(x0)?) * (one - fx) + u64::from(*r.get(x1)?) * fx); + // At most 255 × GRID_ONE², so the sum fits in a u64 and the result in a u8. + let v = (blend(top)? * (one - fy) + blend(bottom)? * fy + one * one / 2) / (one * one); + out.push(v.min(255) as u8); } } Some(out) } +/// PdfCraft patch: fixed-point unit of [`grid_axis`]'s blend weights. +const GRID_ONE: u32 = 1 << 16; + +/// PdfCraft patch: for each of `dst` samples (`dst >= src`) taken from `src` along one axis, the +/// two source samples it lies between and the weight of the second, in [`GRID_ONE`] units. +/// Centres line up: target `t` lies at `(t + ½)·src/dst − ½` source samples, held to the ends. +/// Without `smooth` both are the nearest sample, so the weight does not matter. +fn grid_axis(src: u32, dst: u32, smooth: bool) -> Option> { + if src == 0 || dst < src { + return None; + } + let mut out = Vec::new(); + out.try_reserve_exact(dst as usize).ok()?; + let last = u64::from(src - 1) << 16; + for t in 0..dst { + if !smooth { + let s = nearest_source_index(src, dst, t); + out.push((s, s, 0)); + continue; + } + // In u128, as `(2t + 1)·src·2^16` can pass 2^64 for a long side; the quotient is at + // most `src·2^16`, so it fits back in a u64. + let centre = ((2 * u128::from(t) + 1) * u128::from(src) << 16) / (2 * u128::from(dst)); + let at = u64::try_from(centre).ok()?.saturating_sub(u64::from(GRID_ONE / 2)).min(last); + let s = (at >> 16) as usize; + out.push((s, (s + 1).min(src as usize - 1), (at & 0xFFFF) as u32)); + } + Some(out) +} + /// PdfCraft patch (#624): the source index that PicScale's nearest-neighbour plan copies for /// destination `index` of `dst` samples taken from `src`. It is the same fixed-point centre /// sampling as `ResampleNearestPlan`, so these are the samples a nearest resize would copy. @@ -2908,7 +2948,18 @@ mod area_shrink_tests { assert_eq!(mask_on_image_grid(&mask(&[10, 20], 2, 1), 4, 2), Some(vec![10, 10, 20, 20, 10, 10, 20, 20])); // Centres of 7 samples over 3: 0.5/7·3 = 0.21 → 0, 1.5/7·3 = 0.64 → 0, 2.5/7·3 = 1.07 → 1, … assert_eq!(mask_on_image_grid(&mask(&[1, 2, 3], 3, 1), 7, 1), Some(vec![1, 1, 2, 2, 2, 3, 3])); - assert_eq!(mask_on_image_grid(&mask(&[1, 2, 3, 4], 1, 4), 1, 2), Some(vec![2, 4])); + } + + /// A mask that asks for `/Interpolate` is blended between the samples around each centre, + /// and held at its ends: the centres of 4 samples over 2 lie at -¼, ¼, ¾ and 1¼. + #[test] + fn an_interpolated_mask_is_blended_onto_the_image_grid() { + let smooth = |data: &[u8], width, height| LumaData { interpolate: true, ..mask(data, width, height) }; + assert_eq!(mask_on_image_grid(&smooth(&[0, 255], 2, 1), 4, 2), Some(vec![0, 64, 191, 255, 0, 64, 191, 255])); + // Centres of 7 samples over 3 lie 3/7 apart, from -2/7 to 2 2/7: a ramp, flat at the ends. + assert_eq!(mask_on_image_grid(&smooth(&[0, 70, 140], 3, 1), 7, 1), Some(vec![0, 10, 40, 70, 100, 130, 140])); + assert_eq!(mask_on_image_grid(&smooth(&[0, 255], 1, 2), 1, 4), Some(vec![0, 64, 191, 255])); + assert_eq!(mask_on_image_grid(&smooth(&[7, 9], 2, 1), 2, 1), Some(vec![7, 9]), "same size is unchanged"); } #[test] @@ -2916,6 +2967,7 @@ mod area_shrink_tests { assert_eq!(mask_on_image_grid(&mask(&[], 0, 1), 4, 1), None, "an empty mask"); assert_eq!(mask_on_image_grid(&mask(&[1, 2], 2, 1), 0, 1), None, "an empty image"); assert_eq!(mask_on_image_grid(&mask(&[1], 2, 1), 4, 1), None, "mask data shorter than its size"); + assert_eq!(mask_on_image_grid(&mask(&[1, 2, 3, 4], 1, 4), 1, 2), None, "a mask larger than the image"); } /// Every target sample's weights add up to exactly one, and together the targets take every From c5065554a1946e4ca8895048d70095f68a4ecec5 Mon Sep 17 00:00:00 2001 From: kronbii <145912020+Kronbii@users.noreply.github.com> Date: Fri, 30 Oct 2026 05:07:44 +0200 Subject: [PATCH 056/198] M2: shrink masked images a row at a time; keep near-1:1 scans sharp Review of #3858 (aiongg): - An area-shrunk image with a soft mask, a stencil's ink included, built its full-resolution RGBA before shrinking. A 10718 x 13701 stencil at page size peaked at 1.16 GB (587 MB of RGBA). It is now premultiplied a source row at a time as it is averaged, and the shrink works a target row at a time, so it peaks at 1.03 GB, as the nearest-neighbour path did. Same bytes; about 5% slower per shrink. - An image whose f32 scale passed 1 only by rounding was smoothed bilinearly and blurred by half a pixel at a half-pixel offset. It is now smoothed only above 1.01. - A shrunk /ImageMask stencil keeping its thin strokes (#4800) is now tested. Co-Authored-By: Claude Opus 5.5 --- crates/render/src/raster/tests.rs | 53 +++++++ parity/acrobat-features.toml | 4 +- vendor/README.md | 2 +- vendor/hayro/src/renderer.rs | 255 ++++++++++++++++++++++++------ 4 files changed, 267 insertions(+), 47 deletions(-) diff --git a/crates/render/src/raster/tests.rs b/crates/render/src/raster/tests.rs index f6c0530164..5692634c30 100644 --- a/crates/render/src/raster/tests.rs +++ b/crates/render/src/raster/tests.rs @@ -6814,6 +6814,59 @@ fn shrunk_soft_mask_larger_than_its_image_keeps_thin_strokes() { assert_eq!(grays[3], 255, "{grays:?}"); } +/// Issue #4800: scanners save the text of a page as a 1-bit `/ImageMask` stencil, painted in the +/// fill colour, and a stencil shrunk by nearest-neighbour lost the strokes thinner than the shrink +/// factor. Here a one-column red stroke of an 8-column stencil is drawn 4 pixels wide: nearest +/// reads columns 1, 3, 5 and 7 and misses it, while the area average makes it a half-covered red. +#[test] +fn a_shrunk_stencil_mask_keeps_thin_strokes() { + // Sample 0 paints (the default `/Decode [0 1]`), so 0xDF is a stroke in column 2. + let content = "q 1 0 0 rg 4 0 0 1 0 0 cm /Im0 Do Q"; + let pdf = format!( + "%PDF-1.4\n1 0 obj << /Type /Catalog /Pages 2 0 R >> endobj\n\ + 2 0 obj << /Type /Pages /Kids [3 0 R] /Count 1 >> endobj\n\ + 3 0 obj << /Type /Page /Parent 2 0 R /MediaBox [0 0 4 1] /Contents 4 0 R /Resources << /XObject << /Im0 5 0 R >> >> >> endobj\n\ + 4 0 obj << /Length {} >> stream\n{content}\nendstream endobj\n\ + 5 0 obj << /Type /XObject /Subtype /Image /Width 8 /Height 1 /ImageMask true /Filter /ASCIIHexDecode /Length 3 >> stream\nDF>\nendstream endobj\n\ + trailer << /Root 1 0 R >>\n%%EOF\n", + content.len() + ); + let mut renderer = PageRenderer::new(Arc::new(pdf.into_bytes()), RenderConfig::default()); + let page = renderer.render(RenderRequest { page: 0, kind: RequestKind::Pixels, scale: 1.0, ..Default::default() }); + assert!(page.error.is_none(), "{:?}", page.error); + assert_eq!((page.width, page.height), (4, 1)); + let px = page.rgba.as_chunks::<4>().0.to_vec(); + let [r, g, b, _] = px.get(1).copied().unwrap_or_default(); + assert!(r >= 254 && g.abs_diff(128) <= 1 && b == g, "half-covered red over white, got {px:?}"); + assert_eq!(px.get(3), Some(&[255, 255, 255, 255]), "{px:?}"); +} + +/// A scan drawn at its own resolution keeps its hard pixels when its scale passes 1 only by +/// rounding: `x_scale` is an `f32`, and a 192 dpi scan on a 2× display at 100% can come out a +/// few ulps above 1. Smoothed bilinearly at a half-pixel offset, one-pixel stripes became gray. +/// Here `600.0001` (an `f32` of 600.00012) draws 600 samples 2·10⁻⁷ wider than 1:1. +#[test] +fn a_scan_drawn_a_rounding_error_above_its_resolution_stays_sharp() { + let content = "q 600.0001 0 0 600 0.5 0 cm /Im0 Do Q"; + let image: String = (0..600 * 600).map(|i| if i % 2 == 0 { "00" } else { "FF" }).collect(); + let pdf = format!( + "%PDF-1.4\n1 0 obj << /Type /Catalog /Pages 2 0 R >> endobj\n\ + 2 0 obj << /Type /Pages /Kids [3 0 R] /Count 1 >> endobj\n\ + 3 0 obj << /Type /Page /Parent 2 0 R /MediaBox [0 0 601 600] /Contents 4 0 R /Resources << /XObject << /Im0 5 0 R >> >> >> endobj\n\ + 4 0 obj << /Length {} >> stream\n{content}\nendstream endobj\n\ + 5 0 obj << /Type /XObject /Subtype /Image /Width 600 /Height 600 /ColorSpace /DeviceGray /BitsPerComponent 8 /Filter /ASCIIHexDecode /Length {} >> stream\n{image}>\nendstream endobj\n\ + trailer << /Root 1 0 R >>\n%%EOF\n", + content.len(), + image.len() + 1 + ); + let mut renderer = PageRenderer::new(Arc::new(pdf.into_bytes()), RenderConfig::default()); + let page = renderer.render(RenderRequest { page: 0, kind: RequestKind::Pixels, scale: 1.0, ..Default::default() }); + assert!(page.error.is_none(), "{:?}", page.error); + let row: Vec = page.rgba.as_chunks::<4>().0.iter().skip(300 * page.width as usize).take(page.width as usize).map(|[r, ..]| *r).collect(); + let gray = row.iter().skip(10).take(580).filter(|v| (16..240).contains(*v)).count(); + assert_eq!(gray, 0, "one-pixel stripes were blurred: {:?}", row.get(10..30)); +} + /// Type 3 glyph images used to be sampled bicubically (`ImageQuality::High`) whatever their /// `/Interpolate`, which blends the nearest pre-resize back in. The glyph is 4.6 × 2.3 pixels /// and its image pre-resizes to 5 × 3, so the final draw samples fractionally and bicubic diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 367253b4a7..a0119d06e2 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -734,8 +734,8 @@ milestone = "M2" status = "partial" commands = [] tools = ["page_render"] -evidence = ["crates/render/src/raster/tests.rs::renders_a_blue_rectangle", "crates/automation/tests/it/automation.rs::open_inspect_render_and_find", "crates/render/src/raster/tests.rs::user_unit_scales_the_page_and_everything_on_it", "crates/render/src/raster/tests.rs::non_interpolated_minification_keeps_thin_strokes", "crates/render/src/raster/tests.rs::minified_image_edges_snap_to_whole_pixels", "crates/render/src/raster/tests.rs::large_non_interpolated_images_are_smoothed_when_enlarged", "crates/render/src/raster/tests.rs::small_non_interpolated_images_keep_hard_pixels_when_enlarged"] -notes = "hayro bootstrap renderer; own DisplayList devices in M2.6. Pages with /UserUnit are shown at their size in points, as Acrobat reads the value. An image drawn below its resolution is shrunk once to whole device pixels: by area averaging unless it asks for /Interpolate (then Catmull-Rom), so the text of a 300 dpi scan at 100% keeps its strokes (it was nearest-neighbour, which dropped them). Enlarged, an image of page-scan size (512 samples or more on its shorter side) is smoothed bilinearly whatever its /Interpolate, so a zoomed scan has no staircases; smaller images keep their hard pixels. Not measured against Acrobat" +evidence = ["crates/render/src/raster/tests.rs::renders_a_blue_rectangle", "crates/automation/tests/it/automation.rs::open_inspect_render_and_find", "crates/render/src/raster/tests.rs::user_unit_scales_the_page_and_everything_on_it", "crates/render/src/raster/tests.rs::non_interpolated_minification_keeps_thin_strokes", "crates/render/src/raster/tests.rs::a_shrunk_stencil_mask_keeps_thin_strokes", "crates/render/src/raster/tests.rs::minified_image_edges_snap_to_whole_pixels", "crates/render/src/raster/tests.rs::large_non_interpolated_images_are_smoothed_when_enlarged", "crates/render/src/raster/tests.rs::small_non_interpolated_images_keep_hard_pixels_when_enlarged"] +notes = "hayro bootstrap renderer; own DisplayList devices in M2.6. Pages with /UserUnit are shown at their size in points, as Acrobat reads the value. An image drawn below its resolution is shrunk once to whole device pixels: by area averaging unless it asks for /Interpolate (then Catmull-Rom), so the text of a 300 dpi scan at 100% keeps its strokes, a 1-bit /ImageMask text layer included (it was nearest-neighbour, which dropped them). Enlarged by more than 1%, an image of page-scan size (512 samples or more on its shorter side) is smoothed bilinearly whatever its /Interpolate, so a zoomed scan has no staircases; smaller images keep their hard pixels. Not measured against Acrobat" [[feature]] id = "view.render-shadings-patterns" diff --git a/vendor/README.md b/vendor/README.md index 0d90ad397e..28cc7ce9e7 100644 --- a/vendor/README.md +++ b/vendor/README.md @@ -40,7 +40,7 @@ Other resource categories retain their existing parent-chain lookup. | Crate | Version | Licence | Patches | Test | |---|---|---|---|---| | hayro-interpret | 0.7.0 | Apache-2.0 OR MIT | (1) select `/AP /N` appearance states via `/AS` (checkboxes, radios, stateful icons) and honour the NoView flag; (2) Type3 Unicode fallback to `/Encoding` glyph names, then printable ASCII; (3) `Type3Glyph::advance_width`; (4) `InterpreterSettings::ocg_overrides` for viewer layer toggles; (5) soft masks whose group lacks `/CS` were dropped (Chrome gradient text); (6) paint nesting cap `MAX_PAINT_NESTING` for tiling patterns and Type 3 glyphs (self-referencing content overflowed the stack); (7) CID width ranges in `/W` and `/W2` clamped to `MAX_CID` (a `0 4294967295 w` range hung); (8) `InterpreterSettings::hide_comments` skips markup annotations but keeps widgets and links (View ▸ Hide all comments); (9) `image_size_ok`: images and masks over 2^28 pixels are not decoded (a fuzzed `/W 4294967295` stencil mask allocated 4 GiB); (10) `MAX_NESTED_PAINTS`: nested paints (forms, Type 3 glyphs and tiling patterns inside one another) share a per-page budget of 50,000 (a glyph showing eight glyphs of its own font, sixteen deep, hung); (11) depends on skrifa 0.47 instead of 0.42 (upstream read-fonts fixed a Type 1 tokenizer that recursed through runs of numbers: a fuzzed font hung or overflowed the stack), with the one API change (`Charset::glyph_id` returns `Option`); (12) annotation appearances whose `/BBox` or `/Matrix` collapses to a line or a point are skipped (mapping them onto `/Rect` divided by zero and handed the device infinite or NaN matrices); (13) a Highlight annotation without a usable normal appearance is drawn from its `/QuadPoints`, `/C` and `/CA` with Multiply blending, as pdfcraft-annot draws highlights (it was not drawn at all); (14) simple-font `/Widths` (`read_widths`): entries only for codes 0–255 (one entry per code below `/FirstChar` was allocated: `4294967295` asked for 34 GB), an entry that isn't a number leaves its code missing instead of ending the array, and a `/LastChar` below `/FirstChar` gives no widths instead of discarding the font for a standard one; (15) the appearance box is mapped onto `/Rect` corner to corner: the translation now accounts for the scale (an appearance whose box didn't start at the origin and differed in size from `/Rect` was drawn shifted); (16) a clip set by `W`/`W*` takes effect after whichever path-painting operator ends the path (`S`, `f`, `B`, … as well as `n`; ISO 32000-2 §8.5.4): it was applied only on `n`, so `re W* S` clipped nothing and the clip stayed pending for a later, unrelated `n`; (17) `InterpreterSettings::cancelled`: once set, interpretation stops before the next content operator (a render pool sets it for every worker when dropped, after an edit or on closing, and for one worker when the watchdog gives up on its render, so renders whose answers nobody can receive stop instead of running to the end); (18) `MAX_PAGE_CONTENT`: the content a page holds at once (its contents, and the forms, tiling patterns and Type 3 glyph procedures being painted) stays within 512 MiB (`decode_content`; once it runs out, `InterpreterWarning::ContentTruncated` is reported (once per page by pdfcraft-render) so a truncated page is not a clean render; each level gives its bytes back when it finishes): forms nest fifty deep and each level kept its content while the next ran, so a form painting itself around a stream inflating to 256 MiB would have held over 12 GB; and Type 0 sampled functions take at most `MAX_SAMPLE_ENTRIES` (2^21) table points and `MAX_SAMPLE_VALUES` (2^23) samples, read no further than `/Size` asks (256 MiB of 1-bit samples asked for over 8 GB); (19) mesh shadings (types 4–7) are sampled only inside the area being drawn (`ShadingPattern::encode_within`, which the vendored hayro renderer calls with that area plus a pixel), within 8 visits per pixel of it plus 4 per triangle (`TriangleSampler`), and patches are triangulated one at a time: points far off the page were sampled over their whole bounding box, up to 65536 × 65536 entries (a fuzzed 147-byte patch stream; a synthetic page passed 6 GB in six seconds), ten thousand overlapping triangles ran for minutes, and every patch became 722 triangles before any limit; meshes hold at most `MAX_MESH_PATCHES` (2^16) patches and `MAX_MESH_TRIANGLES` (2^20) triangles, a lattice needs two to 2^19 vertices per row (`/VerticesPerRow 0` pushed empty rows for ever; one huge row read the whole stream), and a page samples at most `MAX_PAGE_MESH_TRIANGLES` (2^25) mesh triangles in all (a mesh painted a thousand times was triangulated a thousand times); each cut logs a warning; (20) PDF function construction is limited to 64 levels and 10,000 total nodes with cycle detection; calculator parsing and evaluation share procedure depth/work budgets, integer division/bitshift/rotation/index are panic-free, and fixed-size stack errors propagate; (21) a Type 0 font drawn with a substitute that isn't CID-keyed (it isn't embedded, or its program doesn't load) draws `.notdef` for a CID of an Adobe character collection (Japan1, GB1, CNS1, Korea1) that it can't find by Unicode: hayro took the CID for a glyph index of the substitute, which numbers its glyphs its own way, so the character was drawn as an unrelated one (𠮷 as "Ｑ" in BIZ UDPGothic); (22) a Type 0 font drawn with a substitute finds a character given with a variation selector (`/ToUnicode`, or Adobe-Japan1-UCS2, which maps the JIS X 0208 forms of the kanji JIS X 0213:2004 redrew that way: CID 1247 is 噂 U+E0100) through the substitute's variation sequences, else as the character alone: such a CID found no glyph and was drawn as an unrelated one; (23) backport of hayro 0.8's `gradient` module: `EncodedShadingPattern::as_svg_gradient` turns an axial or radial shading into stops of an sRGB gradient (within 0.01 per channel, a stop at every stitching-function bound via `Function::stitching_bounds`) when a native gradient draws it exactly (extended past the box drawn in; a radial one with no background colour); (24) `DecodedImageCache`: decoded image pixels shared across renders through the interpreter cache within a 64 MiB budget (an image-heavy page decoded its thousands of images again per tile and per scale), keyed by every decode input (`DecodedImageKey`: the stream's place and dictionary bytes, its object, mask/stencil/raster output, interpolation, decode target size, preview or final); images under a transfer function and inline images are not cached, an image over the budget is never kept, and a poisoned lock decodes uncached; (25) `Device::actual_text` hands the device a marked content sequence's `/ActualText` (inline or named in `/Properties`), so text extraction keeps the Thai, Indic and Arabic clusters Chrome marks this way (their shaped tone marks and vowels have no Unicode of their own); (26) non-embedded simple fonts drawn with a standard-font stand-in build the Latin Extended-A letters the stand-in lacks (`/Gbreve`, `/Idotaccent`, `/Scedilla`, `/aogonek`, `/ccaron`…) from the base letter and accent it has (`StandardKind::composite_glyph`, `ACCENTED_LETTERS`): PDFium's Foxit stand-ins hold only the Standard/WinAnsi/Mac Roman glyphs, so iText letters with Times New Roman unembedded showed blanks for Ğ, İ, Ş, ğ and ş; the comma-below letters (`/scommaaccent`, `/Tcommaaccent`, `/kcommaaccent`…, found by character, so `/uni0136` and the glyph list's `/Kcedilla` too) are the letter and the font's comma, smaller and under the baseline, since the stand-ins have no `commaaccent` (Romanian ș drew nothing and ț a cedilla in forms laid out from XFA; ģ, whose comma is turned and above, is not built); (27) an embedded TrueType font without a base encoding reads its codes as WinAnsi for text extraction (`EmbeddedKind::code_to_unicode_name`), matching pdftotext's output, instead of Standard: OCR text layers of scans write WinAnsi codes with no `/Encoding`, and Standard has no ü, ö or ß and reads é as Ø (glyph selection unchanged); (28) an image whose pixels hold at most 16 bits of samples (an 8-bit Separation or DeviceN image, a 1-, 2- or 4-bit gray or indexed one) is converted once per possible pixel value through a lookup table instead of once per pixel (`rgb_via_lookup`): a spot-colour tint transform is often a PostScript calculator program, run for every pixel, so a fillable character sheet's 3304 × 4105 DeviceN background took four seconds and 430 MB to draw a page; each entry is converted on its own (`convert_each`), so a tint transform giving the wrong number of components for a value the image never uses cannot shift the colours of the values it does; (29) `Device::begin_annotations`/`end_annotations` bracket the page's annotations, so a device can tell comment paints from document paints (reading colours recolour the document, never a comment, #1548); (30) the Annex D.2 alias codes read as the ink they draw: WinAnsiEncoding 0xA0 and 0xAD, and MacRomanEncoding 0xCA, carry the glyph names `space` and `hyphen` -- the names those tables also give 0x20 and 0x2D -- not `nbspace` and `sfthyphen`, whose glyph list values U+00A0 and U+00AD are invisible formatting characters. Extraction hid drawn ink behind them, so search and redaction matched neither (`XiaoBiaoSong.pdf` in the pdf.js corpus drew 104 soft hyphens; across the 989 files the soft hyphens we extract fall from 117 to 13 and the non-breaking spaces from 30 to 26, what remains coming from `/ToUnicode` maps, which say so explicitly). The base table only, in the text direction only: a `/Differences` name keeps its glyph list value and glyph lookup is untouched, both as pdftotext does; (31) a Type 0 sampled function keeps its samples in one flat array indexed by strides (`Type0::entry`) instead of a hash map keyed by each grid point: the map was built again, one hashed insert per sample, every time a shading using the function was painted (a 2,097,152-sample ramp painted 30 times took 38 s; `bug1721218` in the pdf.js corpus spent most of a 1.5 s page on it); the interpolation is unchanged, so results are bit for bit the same, a sample missing from a short stream still gives no value, and the maximum sample value is computed without overflowing at 32 bits per sample; (32) the decoded-image cache (24) keys a decode by the size the image is drawn at only when a JPX stream decodes it or its soft mask or mask (`decode_reads_target`): JPX is the only filter that decodes a reduced resolution for a size, so a JPEG or Flate scan was decoded again at every zoom step and kept once per zoom level; one decode now serves every size; `InterpreterCache::decoded_image_decodes` counts the cache's decodes; (33) Coons and tensor-product patches copy their vertices' colours less: each point of a patch's 20 × 20 triangle grid is interpolated once (`generate_patch_triangles`; each cell interpolated its four corners itself, 1,444 interpolations a patch for 400 points), a cell's second triangle takes the vertices its first copied, and the mesh sampler moves a patch's triangles into device space instead of copying each vertex (`TriangleSampler::sample_owned`); the same values, 10–16% faster on mesh-gradient pages; (34) `color::tint_transform_runs`: how many colours the calling thread has run through a Separation or DeviceN tint transform (counted once per batch in `convert_f32`), so a test counts patch (28)'s once-per-sample-value runs instead of timing them (its 10 s deadline failed on a busy FreeBSD VM in CI); (35) `EncodedShadingPattern::extent`: where a mesh shading (types 4–7) can paint at all, the pixels its triangles touched when it was encoded (`None` for the types that paint everywhere, and when the shading has a `/Background`), so the renderer samples a texture there alone and skips a mesh with nothing on the canvas; (36) a mesh shading's pixel whose corner no triangle holds strictly (on an edge two triangles share, or on the mesh's side) takes the triangle that covers its centre, edges included (`Triangle::covers`): testing only the corner left a white seam along every shared edge, and the colour is still taken at the corner; (38) the `/DefaultGray`, `/DefaultRGB` and `/DefaultCMYK` colour spaces of a resource dictionary (ISO 32000-2 §8.6.5.6) replace the device space content selects: with `g`/`G`, `rg`/`RG`, `k`/`K` and `cs`/`CS` (by name, or through a `/ColorSpace` resource that is a device space), and as the `/ColorSpace` of an image XObject, an inline image or a shading painted by `sh` or as a shading pattern (`DefaultColorSpaces`, looked up once per content window in the resources and those they inherit, so a form XObject without a default of its own draws in its page's, as Poppler and MuPDF draw it); a default that is a Lab, Indexed, Pattern or device space, has another number of components or is no colour space is ignored and device colour is drawn as before; device spaces inside other spaces (an Indexed base, a Separation or DeviceN alternate, a Pattern's underlying space), shading dictionaries, masks and transparency group spaces are unchanged, and the decoded-image cache (24) keys a decode by the default it was drawn in. hayro drew device colour as it was, so pages that calibrate it this way (common in PDF/A) came out in the wrong colours; (39) knockout groups (#2971): `Device::set_knockout` right after a `/K true` group is pushed; `Device::knockout_image` for an unmasked image before its group is pushed, when `Device::in_knockout_group`: the device clears its placement rectangle only if it will draw the image (on the canvas, not a preview's tiny image, and it decodes), decoded at its draw size so the draw reuses that decode; and, when `Device::knockout_erase_wanted`, a nested group's content drawn once between `begin_knockout_erase` and `end_knockout_erase` before the group itself, which then always gets a layer of its own; (40) an optional content membership dictionary's visibility expression (`/VE`: `/And`, `/Or`, `/Not`, nested up to 32 deep) decides its visibility ahead of `/OCGs` and `/P` (ISO 32000-2 §8.11.2.2); it was ignored, so content under it stayed visible with its layers hidden; a malformed expression, including one naming an object the document doesn't list in `/OCProperties /OCGs`, falls back to `/OCGs` and `/P` (#2986); (41) a transparency group that is neither isolated nor knockout, entered at full opacity with no soft mask and Normal blending, is drawn straight onto the page instead of into a layer of its own (§11.4.7: its elements composite with the backdrop; the layer made a `/BM /Multiply` highlight appearance an opaque block, #1968); (42) text in a Type 3 font is drawn filled whatever the text rendering mode, except mode 3 (invisible): only that mode has any effect on Type 3 fonts (ISO 32000-1 §9.3.6). Stroking ran the glyph procedure a second time, so `2 Tr` made bitmap fonts look bold, `1 Tr` painted an uncoloured glyph in the stroking colour, and `7 Tr` hid it (`7 Tr` since follows 64); (43) Lab colour spaces honour their own `/WhitePoint`: L\*a\*b\* goes to XYZ relative to that white, is Bradford-adapted to D65 and converted to sRGB (ISO 32000-2 §8.6.5.4) instead of through `LAB.icc` as if every white were D50 (a D65 blue came out 45 levels too red; the profile is removed); components normalised to [0, 1] (`manual_scale`, an `/Indexed` palette over Lab) map onto L\* 0..100 and the a\*/b\* `/Range`; (44) a Type 0 font whose `/ToUnicode` is the name `/Identity-H` or `/Identity-V` (not a CMap stream; WSCAD writes it) reads each code as its Unicode value, as Acrobat shows it (`Type0Font::identity_to_unicode`); the identity CMap the name resolved to maps codes to CIDs only, so such pages had no text to select or find; (46) an image XObject's group applies the fill opacity, so the fill alpha is reset to 1 before a stencil mask takes the fill colour as its paint, and a pattern paint drops the opacity it captured at `scn` (`Pattern::without_opacity`) (it was applied twice: a mask at `ca 0.5` was drawn at a quarter, #2975); (47) `path_as_rect`: a five-element path is a rectangle only when its four points are distinct, each step from one to the next runs along a single axis, and the last returns to the first. The length test admits one MoveTo and four LineTo as well as one MoveTo, three LineTo and a ClosePath, and asking only that every corner of the bounding box was visited says nothing about the order or the steps between, so a five-point polyline whose ends reached the corners was painted as that box -- a fourth side the path does not have -- and a clip of that shape was discarded; the coordinates are compared exactly, since the matrix is applied after this check and a `cm` of 10,000 turns `is_nearly_equal`'s 256th of a unit into tens of device units (#3327); (48) an `/ICCBased` colour space is cached by what its profile is read from (`IccProfileKey`): the stream object it names, looked up without reading the stream, and then the stream's dictionary and its own bytes (decrypted, not decoded), so a byte-for-byte copy of a profile in another object is read once, and a profile stream written directly in its array, which reports the object holding the array as every other stream written there does, is told apart by its bytes. hayro keyed it by the stream dictionary's bytes alone, so two different profiles with the same `/N` and `/Length` shared whichever was read first, and colours in the second were drawn through the first. `InterpreterCache::icc_profile_reads` counts the profiles read; (49) `OpenTypeFontBlob::new` also accepts a `post` table version 1.0, 2.0 or 3.0 written with its 16-bit halves swapped (`00 00 00 03`), as ScanSoft PDF Create! embeds every font; such fonts were refused and their text drawn with a substitute (Hebrew came out garbled or missing, #2946); `glyph_names` reads such a table from a copy with the version written correctly, so a 2.0 table's names serve `/Differences`. Any other unknown version is still refused (pdf.js 9462); (50) `CffFontBlob::glyph_index_by_name` looks a glyph name up in a name to glyph map built on first use (shared between clones) instead of searching the whole charset, with a string lookup per glyph, for every character code of a font; the first glyph with a name still wins; (51) at most 4096 graphics states saved at once, holding at most 64 MiB together (each `q` copies the whole state, dash array and clip list included, so short operators multiplied one large state); a `q` past either limit is counted but not copied, and its `Q` still removes the clips added since (from storytold/craft-libs#239); (52) a `/W2` run whose first CID is past `MAX_CID` stops there, as in `/W` (7), checked before the run's metrics are read: a first CID of 4294967295 inserted one entry and then overflowed, discarding the whole `/W2` table (from storytold/craft-libs#254); (55) a Separation or DeviceN tint transform that cannot be read keeps its colour space, and one that cannot be read or evaluated paints the alternate space's initial colour, as evaluation failures already did; that fallback, and a `cs`/`CS` colour space that cannot be read (drawn in DeviceGray as before), are now reported once per content stream as `InterpreterWarning::ColorFallback` (`RenderWarning::ColourFallback` in `pdfcraft-render`) instead of silently (#4260); (56) an annotation whose Invisible flag (bit 1) is set and whose `/Subtype` is none of the standard types of ISO 32000-2 Table 171 (`is_standard_annotation_type`; no `/Subtype` counts as unknown) is not drawn (Table 167), and a `/Popup` annotation's appearance is never painted on the page (§12.5.6.14: a popup has no appearance stream of its own; the viewer shows it as a window for its parent annotation). Both were drawn from their `/AP`; (57) a simple font drawn with a standard-font stand-in because the document names a font it doesn't embed (`StandardKind` with `fallback`) narrows a glyph to a narrower `/Widths` advance but no longer widens it to a wider one: the advance already places the next glyph (ISO 32000-2 §9.2.4; /Widths in Table 109), and widening drew a non-embedded `Arial,Bold` with 600-unit widths as a heavy fixed-pitch face whose `i` and `l` were solid blocks twice their width; an exact standard-14 font still matches its standard metrics both ways; (58) the fallback request for a Type 0 font that is not embedded (`FallbackFontQuery::with_descriptor`) takes its serif, italic and small-cap flags, `/FontWeight`, `/FontFamily` and `/FontStretch` from the descendant CIDFont's font descriptor (ISO 32000-2 §9.7.4.1, §9.7.6.1; §9.8): it read them from the Type 0 dictionary, which has none, so every such request asked for a regular sans-serif face and no resolver could tell a Song or Mincho font from a Hei or Gothic one, or see `/FontWeight 700`; (59) the NoRotate flag (`/F` bit 5) is honoured: the appearance, fitted to `/Rect`, is turned against the page's `/Rotate` about the rectangle's upper-left corner, which stays where it is, so it reads upright as the page is displayed (ISO 32000-2 §12.5.3). Sticky notes, which are saved NoRotate, were drawn turned with the page (#2583). Form fields and links stay in `/Rect`, where PdfCraft takes their click targets from it; (64) text in a Type 3 font follows ISO 32000-2 §9.3.6 in the clipping modes: under `7 Tr` it is not rendered (reported to the device as clipping text, `GlyphDrawMode::Clip`, so extraction keeps it), and in modes 4 to 7 a Type 3 glyph adds nothing to the clipping path (`clip_glyph`), so text shown only in a Type 3 font leaves no clip at `ET`. Since (42) `7 Tr` painted the glyphs in the fill colour (black by default), so a fill meant to follow showed them wherever it didn't cover them; 4 to 6 still render filled; (66) a Separation colour space whose colourant is the special name `All` paints every colourant at once (ISO 32000-2 §8.6.6.4): on screen and in an RGB print raster its tint t is the gray 1 − t, complemented because the device is additive, whatever its alternate space and tint transform say (hayro drew it through the tint transform, so a `/DeviceCMYK` transform giving cyan drew registration marks light cyan); `/None` still paints nothing, any other name still goes through its alternate, and a DeviceN space naming `All` keeps its alternate, since the clause gives the name that meaning in a Separation space; (63) a luminosity soft mask whose group colour space is `/DeviceCMYK` (by name, not an ICC-based or calibrated space) reads a `/DeviceCMYK` fill or stroke colour drawn in it, and its `/BC`, as their gray `1 - min(1, 0.3 c + 0.59 m + 0.11 y + k)` (ISO 32000-2 §10.4.2.3, §11.5.3: device colour converts to gray with no colour calibration; `Context::luminosity_in_device_cmyk`, `color::device_cmyk_to_gray`): it went through the CMYK display profile, where 100% K is a dark grey, so a 100% K mask let about an eighth of the masked paint through. CMYK images, shadings and tiling patterns inside such a mask, and CMYK on the page, still use the profile; (71) a lookup table in an indirect string reads: it was read as a stream first, which uses up a reference (ISO 32000-2 §8.6.6.3); (72) an uncoloured pattern's underlying space given as a name or a reference is used: it was skipped, so a CMYK cyan drew red (ISO 32000-2 §8.7.3.3); (73) `InterpreterSettings::jbig2_work`, set for each page in `interpret_page` and ended with it, and a JBIG2 image or an image with a JBIG2 mask past it left out with `ContentTruncated`; (74) colour spaces nest at most 32 levels, through every way one names another, so a chain of them ends instead of overflowing the stack; an ICC profile's `/Alternate` is read where it's met and isn't kept with the profile, or a page could build a chain past the limit on spaces already read; (75) a soft hyphen (U+00AD) that a CFF substitute for a CID font has no glyph for draws its hyphen, as Annex D.2 reads WinAnsi 0xAD in (30), instead of the CID taken as a glyph index (#5888) | (1) `raster::tests::widget_appearance_states`; (2, 3) `xtask text-oracle` on corpus/pdfjs issue918; (4) `raster::tests::layer_override_hides_content`; (5) `raster::tests::soft_mask_without_group_colour_space_is_applied`; (6) `raster::tests::self_referencing_tiling_pattern_terminates`, `raster::tests::self_referencing_type3_glyph_terminates`; (7) `raster::tests::huge_cid_width_ranges_terminate`; (8) `raster::tests::hiding_comments_keeps_fields`; (9) `raster::tests::absurd_mask_and_fax_sizes_are_refused_before_decoding`; (10) `raster::tests::type3_glyphs_that_fan_out_into_themselves_finish`; (11) `raster::tests::type1_font_programs_with_long_runs_of_numbers_load_quickly`; (12) `raster::tests::degenerate_annotation_appearances_are_skipped`; (13) `raster::tests::highlights_without_an_appearance_are_drawn`; (14) `raster::tests::malformed_simple_font_widths_keep_later_widths_and_the_font`, `raster::tests::huge_simple_font_first_char_terminates`; (15) `raster::tests::annotation_appearance_box_maps_onto_rect`; (16) `raster::tests::clipping_path_applies_after_any_painting_operator`; (17) `raster::tests::dropping_the_pool_cancels_its_renders`, `raster::tests::the_watchdog_stops_the_render_it_gives_up_on`; (18) `raster::tests::page_contents_and_nested_forms_share_one_budget`, `raster::tests::content_painted_again_and_again_keeps_drawing`, `raster::tests::sampled_functions_with_billions_of_samples_are_refused`; (19) `raster::tests::mesh_shadings_far_off_the_page_render`, `raster::tests::many_overlapping_mesh_triangles_finish`, `raster::tests::mesh_shadings_with_many_patches_or_no_rows_finish`; (20) `raster::tests::function_limits_calculator_arithmetic_and_depth`, `raster::tests::function_limits_public_construction_work_and_depth`, `raster::tests::function_limits_stitching_cycles_are_refused_and_shared_children_work`, `raster::tests::function_limits_invalid_calculator_keeps_the_page_renderable`; (21) `raster::tests::cids_a_substitute_lacks_are_not_drawn_as_other_glyphs`; (22) `raster::tests::variation_sequences_draw_their_kanji_in_a_substitute`; (23) `raster::tests::axial_and_radial_shadings_draw_as_native_gradients`; (24) `raster::tests::tiles_match_full_render`, `raster::tests::skipping_paths_off_the_canvas_changes_no_pixel` (both decode through the cache), `raster::tests::decoded_images_with_identical_dictionaries_keep_their_own_pixels`, `raster::tests::decoded_images_under_a_transfer_function_are_not_shared`; (25) `text::tests::actual_text_replaces_the_glyphs_it_marks_in_every_writing_direction`, `text::tests::actual_text_follows_the_pdf_rules_for_marked_content`; (26) `raster::tests::accented_letters_the_standard_stand_ins_lack_are_built_from_their_parts`, `raster::tests::comma_below_letters_in_standard_fonts_draw_a_comma_under_the_letter`; (27) `text::tests::truetype_without_an_encoding_reads_its_codes_as_win_ansi`; (28) `raster::tests::spot_colour_images_run_their_tint_transform_once_per_sample_value`, `raster::tests::images_converted_through_a_table_keep_every_samples_colour`, `raster::tests::a_malformed_tint_for_an_unused_sample_keeps_the_used_colours`; (29) `raster::tests::reading_colors_keep_highlight_annotations_readable`; (30) `text::tests::the_annex_d2_alias_codes_read_as_an_ordinary_hyphen_and_space`; (31) `raster::tests::shadings_with_large_sampled_functions_paint_quickly`, `raster::tests::sampled_functions_interpolate_their_grid_as_before`; (32) `raster::tests::decoded_images_serve_every_size_they_are_drawn_at`; (33) `raster::tests::patch_meshes_keep_their_corner_colours`; (34) `raster::tests::spot_colour_images_run_their_tint_transform_once_per_sample_value`; (35) `mesh_extent_tests::{a_mesh_paints_its_triangles_alone, a_mesh_keeps_the_graphics_state_alpha, a_mesh_off_the_canvas_paints_nothing}` (`cargo test -p hayro`); (36) `raster::tests::mesh_triangles_sharing_an_edge_leave_no_seam`; (38) `raster::tests::default_colour_spaces::device_rgb_operators_draw_in_default_rgb`, `raster::tests::default_colour_spaces::device_cmyk_operators_draw_in_default_cmyk`, `raster::tests::default_colour_spaces::device_gray_operators_draw_in_default_gray`, `raster::tests::default_colour_spaces::device_images_draw_in_the_default_colour_spaces`, `raster::tests::default_colour_spaces::device_shadings_draw_in_the_default_colour_spaces`, `raster::tests::default_colour_spaces::an_image_drawn_with_and_without_a_default_keeps_both_colours`, `raster::tests::default_colour_spaces::forms_inherit_the_default_colour_spaces_they_do_not_override`, `raster::tests::default_colour_spaces::unusable_defaults_leave_device_colour_alone`; (39) `raster::tests::knockout_groups_replace_what_their_elements_cover`; (40) `raster::tests::visibility_expressions_decide_what_a_membership_dictionary_shows`; (41) `raster::tests::a_multiply_highlight_inside_a_transparency_group_keeps_the_text`; (42) `raster::tests::rendering_modes_but_3_and_7_draw_type3_glyphs_as_filled`; (43) `raster::tests::lab_honours_the_space_white_point`, `raster::tests::indexed_palettes_over_lab_keep_their_colours`; (44) `text::tests::identity_to_unicode_names_give_text`; (46) `raster::tests::stencil_masks_are_drawn_at_the_fill_opacity_once`; (47) `raster::tests::a_five_point_polyline_is_not_painted_as_its_bounding_box`, `raster::tests::a_path_that_visits_every_corner_out_of_order_is_not_a_rectangle`, `raster::tests::a_rectangle_is_still_painted_as_a_rectangle`; (48) `raster::tests::icc_profile_cache::icc_profiles_with_identical_dictionaries_draw_their_own_colours`, `raster::tests::icc_profile_cache::icc_profiles_used_again_are_read_once`; (49) `raster::tests::a_font_whose_post_version_has_swapped_halves_is_still_used`; (50) `font::blob::tests::glyph_names_resolve_to_the_first_glyph_with_that_name`; (51) `context::saved_state_tests::repeated_saves_stay_bounded_and_balanced`; (52) `font::cid::tests::w2_run_past_the_largest_cid_keeps_the_rest_of_the_table` (hayro-interpret); (55) `raster::tests::devicen_inks::a_tint_transform_that_cannot_be_evaluated_draws_the_rest_of_the_page_and_warns`, `raster::tests::devicen_inks::a_tint_transform_that_cannot_be_read_falls_back_and_warns`; (56) `raster::tests::invisible_unknown_annotations_and_popups_paint_nothing`; (57) `substitute_widths::a_substitute_keeps_its_glyph_shapes_at_wider_advances`, `substitute_widths::a_substitute_still_narrows_glyphs_to_narrower_advances` (pdfcraft-render `tests/it`); (58) `cid_fallback_query::a_cid_font_fallback_request_carries_its_descendants_style`, `cid_fallback_query::a_cid_font_without_a_descriptor_asks_for_the_default_style` (pdfcraft-render integration tests); (59) `raster::tests::no_rotate_notes_stay_upright_at_their_corner_on_rotated_pages` (hayro-interpret); (64) `type3_text_clip::{clip_only_type3_text_is_not_rendered_and_leaves_no_clip, invisible_type3_text_is_not_rendered, other_modes_render_type3_glyphs_filled_and_never_clip, only_outline_glyphs_shape_a_clip_shared_with_type3_text, self_nesting_type3_text_in_a_clip_mode_stays_bounded}` (`cargo test -p pdfcraft-render --test it`), `raster::tests::rendering_modes_but_3_and_7_draw_type3_glyphs_as_filled`, `text::tests::clipping_mode_type3_text_is_kept_by_the_ocr_filter`; (66) `separation_all::{separation_all_fills_paint_the_complemented_tint_as_gray, separation_all_strokes_paint_the_complemented_tint_as_gray, separation_all_images_paint_each_sample_as_the_complemented_gray, separation_all_shadings_paint_the_complemented_gray, separation_none_paints_nothing, a_named_separation_still_paints_through_its_alternate, a_device_n_space_naming_all_still_paints_through_its_alternate}` (`cargo test -p pdfcraft-render --test it`); (63) `color::device_cmyk_gray_tests::{each_ink_darkens_by_its_iso_weight, ink_past_full_coverage_is_black_and_bad_components_read_as_zero}` (hayro-interpret), `it::luminosity_group_space::{black_ink_in_a_device_cmyk_group_masks_fully_and_no_ink_shows_fully, each_process_ink_in_a_device_cmyk_group_masks_by_its_weight, a_device_cmyk_backdrop_colour_masks_by_its_gray, gray_and_rgb_groups_mask_as_before, device_cmyk_on_the_page_keeps_its_display_colour}` (pdfcraft-render); (71) `raster::tests::an_indexed_lookup_table_is_read_in_every_legal_form` (pdfcraft-render); (72) `raster::tests::an_uncoloured_pattern_uses_its_underlying_space_in_every_form` (pdfcraft-render); (73) `raster::tests::the_jbig2_images_of_a_page_share_one_allowance`, `a_picture_whose_jbig2_mask_is_past_the_pages_allowance_is_not_drawn` and `a_pages_jbig2_allowance_ends_with_the_page` (pdfcraft-render); (74) `raster::tests::{a_chain_of_colour_spaces_ends, colour_spaces_are_read_to_the_limit_of_their_nesting, a_chain_built_on_spaces_already_read_ends, a_profile_read_at_the_limit_keeps_its_alternate_for_a_read_nearer_the_top}` (pdfcraft-render); (75) `raster::tests::a_soft_hyphen_a_substitute_lacks_draws_its_hyphen` (pdfcraft-render) | -| hayro | 0.7.1 | Apache-2.0 OR MIT | (1) `RenderSettings::{x_offset, y_offset}` to render one tile of a large page; (2) images over `MAX_IMAGE_PIXELS` (2^28) are skipped (a fuzzed `/W 4294967295` hung in resampling); (3) `tiling_cell_scale` keeps a tiling pattern's cell pixmap within 3000 px a side whatever its `/XStep`/`/YStep` (a fuzzed step far beyond `/BBox` allocated 2 GB); (4) stroke widths clamped to 4 × (canvas width + height) device pixels (a fuzzed `/LW 9223372036854775807` allocated 10 GB in stroke expansion); (5) `usable_dash_pattern`: a stroke whose dash pattern could cut it into more than `MAX_DASHES_PER_STROKE` (250,000) pieces, counting kurbo's restart of the pattern at every subpath and `ClosePath`, is drawn solid (a fuzzed `/D [[11] 0]` on a line from x = 92234775807 allocated over 5 GB, `[0.000001] 0 d` on a 20 pt line over 10 GB); (6) a dash pattern with a negative or non-finite entry, or a period that isn't positive, is drawn solid (`[-1 -1] 0 d` looped for ever in kurbo); (7) `render_size` and `render_into`: render into a buffer the caller allocates (PdfCraft renders into pixel-aligned memory that the GUI takes over as texture data); `render` is `render_into` on a new pixmap; (8) checked image source/target/RGBA, alpha-premultiplication scratch and intermediate sizes before resampling, finite transforms and exact source/mask buffers; sources wider than u16 may shrink before backend limits, with a 2^20 source-axis cap and conservative 128 MiB filter-planning estimate; an over-limit or failed resize keeps the original image resolution and geometry when the source fits the backend, otherwise skips it, with a logged warning; staging, resampling output/scratch, RGBA conversion and padding buffers use fallible reserves; (9) `may_paint_canvas`: a fill or stroke whose control-point box, grown by how far its paint reaches past the path (half the stroke width times the larger of the miter limit and √2) plus 0.5 user units and 2 pixels, misses the canvas is not drawn, so a tile no longer strokes and fills every path of its page (item 5 of #623); `set_skip_offscreen_paths` turns this off for the regression test; (10) `/Interpolate` governs the pre-resize too (#624): a minified image that does not request interpolation is pre-resized with nearest-neighbour (since (21), only inside Type 3 glyphs), planned without the Catmull-Rom filter limits (`image_resampling_size` takes the flag); a Type 3 glyph image is sampled bicubically only when interpolated; a minified image with a matching soft mask is nearest-sampled straight into premultiplied RGBA; (11) `may_paint_rect`: an image whose stream-size box, mapped to the device, misses the canvas is skipped before decoding (decoded data can only shrink against that box), so a tile of an image-heavy page no longer decodes, resamples and rasterizes every image on it; (12) preview renders (`InterpreterSettings::preview`: backdrops, thumbnails, grid cells, signature-drag backgrounds) skip images under `PREVIEW_MIN_IMAGE_AREA` device pixels, while final renders always draw them; (13) reading colours (#897): `RenderSettings::color_replacement` takes a `ColorReplacement` (text colour, background, `only_black`, `line_art`) that remaps solid text and line-art paint while rendering, glyph outlines counting as text and stencil-mask ink as line art; images, image alpha masks, soft masks and pattern cells keep their colours; `render_pdf` gains a `color_replacement: Option` argument (pass `None` for true colours); the upstream-only `[dev-dependencies.log]` entry is removed from `Cargo.toml` (log is a normal dependency since the logging patch), so `cargo test -p hayro` runs this crate's unit tests; (14) a shrunk image that is interpolated (or, since (21), area-averaged) and whose axes lie along the device's (upright, mirrored or a quarter turn, as on a page under `/Rotate`) is snapped to whole device pixels: shrunk once to its rounded device rectangle (halves round up, so a tile snaps an edge where the whole page does) and drawn 1:1 from the rounded corner, instead of a ceil-sized raster resampled again at a fractional scale and offset (`snap_minified_image`; scans looked soft next to Acrobat, and until #5530 a turned one still did, up to 200 grey levels off the same page drawn upright). An image counts as axis-aligned when the terms the snap drops move its far edges by at most a hundredth of a device pixel (`SNAP_SKEW`), not by a tolerance on the coefficients, which let a slight skew of a very wide image through; and the snapped 1:1 transform is used only when the shrink produced the snapped size, so a failed resize keeps the full transform instead of drawing the source raster at its source size; (15) a soft mask of another size than its image is filtered when shrunk, whatever its `/Interpolate` (#1649): such a mask carries the shape on its own (Word saves text as a tiny black image with a glyph-bitmap mask), and nearest-neighbour dropped every stroke thinner than the shrink factor; (16) under a soft mask or a blend mode, axial and radial shadings are sampled instead of drawn as native gradients (#2128): vello_cpu 0.0.8 copies an opaque gradient over fully covered tiles without the mask or blend mode, so PowerPoint's faded gradient fills came out solid; (17) the renderer keeps annotation paints out of the reading-colours mapping between `begin_annotations` and `end_annotations` (#1548); (18) the page renders as one transparency group; under reading colours its background is painted inside it as the initial backdrop, so a Multiply highlight blends with the dark background instead of staying an opaque block, and otherwise the background stays outside, so a blend mode over an empty part of the page keeps its colour instead of blending with the white paper (#2970); (19) `set_paint` samples a mesh shading's texture over its triangles alone (`EncodedShadingPattern::extent`, padded rather than repeated) and skips a mesh with none on the canvas: a BricsCAD plan's 310 one-triangle `sh` each covered the whole tile, 2 s per 1024 px tile at any zoom (the watchdog gave the page up on a busy machine), now about 70 ms; (20) a tiling pattern whose step is too large for a cell pixmap, where what is painted (a fill, or a stroke with its width) can only show the first cell (PowerPoint's `/XStep 2147480100` for a picture fill drawn once), has its `/BBox` drawn at full scale with a transparent border and padded instead of tiled (`shows_one_cell`; a cell whose scale isn't finite and positive, from a `/BBox` too thin to divide by, keeps the old path: `usable_cell_scale`); shrinking the cell to fit the 3000 px limit of (3) left the picture under a pixel, so network diagrams lost their pictures (#3349); (21) an image that does not request interpolation and is drawn below its resolution, outside a Type 3 glyph, is shrunk by area averaging (`area_shrink`: each device pixel the coverage-weighted mean of the samples it covers, in 16.16 fixed point whose weights sum to exactly one, premultiplied when the image has a mask; a soft mask no larger than the image is first sampled onto the image's grid by `mask_on_image_grid` (nearest-neighbour, or bilinearly when the mask asks for `/Interpolate`, so its edge stays as smooth as an enlarged mask's), so its colour and mask are averaged together instead of the averaged colour of transparent samples showing under a mask layer at the mask's edges) to its snapped device rectangle, instead of nearest-neighbour: `/Interpolate` asks for smoothing only when an image is enlarged (ISO 32000-2 §8.9.5.3), and nearest-neighbour kept one sample in N per axis, so the text of a 300 dpi scan at 100% lost most of its strokes (sampling them again at a fractional offset dropped more). PicScale's `Box` and `Area` filters were not used: shrinking, both average a window four target pixels wide. On a 13-page 300 dpi scan every rendered sample at 96, 120 and 192 dpi is within 1 of the exact area average (it was up to 255 off, 17-23% of samples by more than 1). A snapped image whose shrink failed keeps the full transform, so it is not drawn at its source size. The shrink never enlarges: a side drawn at or above its resolution (an image shrunk across and stretched down, as a 204 × 98 dpi fax is) keeps its samples for the sampler. Rows are averaged first (contiguous, cheap) and the fewer rows that leaves are then shrunk across with taps worked out once per image, which made the shrink about three times faster (a 300 dpi page at 96 dpi: 25 ms, nearest-neighbour 11 ms). A source with a side over `MAX_RESAMPLING_SOURCE_SIDE` (2^20) keeps (10)'s nearest-neighbour shrink, and `area_shrink` refuses it, so its accumulator row and tap table stay a few times one source row (a 2^27 × 2 image of zeros is a tiny file); (22) knockout groups: while one is innermost, fills, strokes, glyph outlines and shadings are drawn with `Compose::Copy` (weighted by coverage), so each element replaces what earlier elements of the group drew instead of compositing over it; an image or nested group, drawn into a layer of its own, first has what is under its shape cleared (`knockout_image` for an unmasked image's rectangle, only when `draw_image` will draw it, decoded at the size it is drawn; a knockout erase pass that draws a nested group's content with every paint clearing instead of painting, a shading fill's clear clipped to the shading's `/BBox` as its paint is) and then composites over it (#2971); (23) backport of hayro 0.8: shadings that `as_svg_gradient` can represent are painted as a `vello_cpu` linear or two-point radial gradient instead of a texture sampled pixel by pixel through the shading function (InDesign fades are luminosity soft masks of one full-page axial `sh`: 40 on a report page took 15 s at 144 dpi, now under 1 s); without upstream's half-pixel shift, which suits vello_cpu 0.3 but moves 0.0.8's gradients off pixel centres; (25) a tiling cell is drawn at the scale that makes one `/XStep` (`/YStep`) exactly the cell pixmap's whole number of pixels (`tiling_cell_pixels`, at least 1 and at most the cap), so the repeated pixmap puts cells at whole multiples of the step at any device scale; drawing at the unrounded scale repeated the cell up to half a pixel off at every step, and the error grew with distance from the pattern's origin (ISO 32000-2 8.7.3.1); (28) an image of page-scan size (`SMOOTHED_IMAGE_SIDE`: 512 samples or more on its shorter side) that does not request interpolation is drawn with `ImageQuality::Medium` (bilinear, as interpolated images are) when enlarged on either side, also when the other side is shrunk (the area shrink leaves that side at 1:1), outside a Type 3 glyph: scans rarely set `/Interpolate`, and drawn nearest-neighbour every sample of a zoomed scan became a block, so its text had staircase edges; icons, QR codes and pixel art keep their hard pixels. Bicubic (`High`) looked the same on a zoomed 300 dpi scan and drew four 1024 px tiles at 400% in 267 ms against 50 ms | (1) `raster::tests::tiles_match_full_render`; (2) `raster::tests::absurd_image_dimensions_are_skipped`; (3) `raster::tests::tiling_cells_stay_small_whatever_the_step`; (4) `raster::tests::absurd_predictor_columns_and_line_widths_render`; (5) `raster::tests::dash_patterns_that_cut_a_stroke_into_billions_of_pieces_render`; (6) `raster::tests::invalid_dash_patterns_terminate_and_are_drawn_solid`; (7) `raster::tests::pixels_are_a_plain_renders_bytes_in_a_buffer_a_gui_can_take_over`; (8) `raster::tests::image_resampling_rejects_anisotropic_target_growth`, `raster::tests::image_resampling_bounds_intermediate_before_planning`, `raster::tests::image_resampling_rejects_invalid_sources_and_scales`, `raster::tests::image_resampling_keeps_valid_sizes_and_exact_limits`, `raster::tests::resampling_fallback_preserves_original_geometry_and_pixels`, `raster::tests::image_resampling_wide_sources_shrink_before_backend_side_limits`, `raster::tests::image_resampling_wide_transparent_sources_still_shrink`, `raster::tests::image_resampling_padded_backend_dimensions_stay_checked`, `raster::tests::image_resampling_mismatched_alpha_mask_keeps_pixels_and_placement`; (9) `raster::tests::skipping_paths_off_the_canvas_changes_no_pixel`; (10) `raster::tests::non_interpolated_images_are_not_smoothed_when_minified`, `raster::tests::non_interpolated_gray_images_are_not_smoothed_when_minified`, `raster::tests::non_integer_minification_averages_the_columns_each_pixel_covers`, `raster::tests::non_integer_minification_averages_colour_and_mask_together`, `raster::tests::non_interpolated_alpha_images_keep_their_colours_and_mask_when_minified`, `raster::tests::non_interpolated_type3_glyph_images_are_not_smoothed_when_minified`, `raster::tests::image_resampling_nearest_accepts_wide_sources_the_filter_refuses`, `raster::tests::wide_non_interpolated_sources_draw_after_shrinking`; (11) `raster::tests::skipping_paths_off_the_canvas_changes_no_pixel`; (12) `raster::tests::preview_renders_skip_tiny_images_but_finals_draw_everything`; (13) `color_replacement_tests::{black_maps_to_text_colour, white_maps_to_background, midgrey_interpolates, alpha_copied_verbatim, only_black_threshold, line_art_false_skips_paths_but_not_text}`, `color_replacement_render_tests::{color_replacement_maps_text_and_line_art, color_replacement_maps_stencil_mask_ink}`; (14) `raster::tests::minified_image_edges_snap_to_whole_pixels`, `raster::tests::a_tile_snaps_half_pixel_image_edges_as_the_whole_page_does` (`cargo test -p hayro`), `raster::tests::a_turned_or_mirrored_minified_image_snaps_to_whole_pixels`, `rotated_image_snap::a_shrunk_image_on_a_turned_page_is_the_upright_render_turned` (`cargo test -p pdfcraft-render --test it`), `renderer::snap_tests::a_skew_that_moves_a_far_edge_is_not_snapped` (`cargo test -p hayro`); (15) `raster::tests::shrunk_soft_mask_larger_than_its_image_keeps_thin_strokes`; (16) `raster::tests::soft_mask_applies_to_a_shading_pattern_fill`; (17), (18) `raster::tests::reading_colors_keep_highlight_annotations_readable`; (19) `mesh_extent_tests::{a_mesh_paints_its_triangles_alone, a_mesh_keeps_the_graphics_state_alpha, a_mesh_off_the_canvas_paints_nothing}` (`cargo test -p hayro`); (20) `raster::tests::a_tiling_pattern_drawn_once_shows_its_cell`, `raster::tests::a_large_step_still_tiles_where_the_next_cell_shows`, `raster::tests::a_tiling_cell_too_thin_to_scale_keeps_the_old_path`; (21) `raster::tests::non_interpolated_minification_keeps_thin_strokes`, `raster::tests::non_integer_minification_averages_the_columns_each_pixel_covers`, `raster::tests::non_integer_minification_averages_colour_and_mask_together`, `raster::tests::non_interpolated_minification_does_not_bleed_transparent_colour`, `raster::tests::a_smaller_soft_mask_does_not_bleed_transparent_colour_when_minified`, `raster::tests::an_interpolated_smaller_soft_mask_stays_smooth_when_its_image_is_minified`, `raster::tests::minified_image_edges_snap_to_whole_pixels`, `raster::tests::an_image_shrunk_one_way_and_stretched_the_other_is_still_averaged`, `raster::tests::wide_non_interpolated_sources_draw_after_shrinking`, `renderer::area_shrink_tests` (`cargo test -p hayro`); (22) `raster::tests::knockout_groups_replace_what_their_elements_cover`; (23) `raster::tests::axial_and_radial_shadings_draw_as_native_gradients`; (25) `tiling_cell_placement::{far_cells_of_a_fill_sit_at_whole_steps, far_cells_sit_at_whole_steps_under_a_turned_scaled_matrix, far_cells_of_a_stroke_sit_at_whole_steps, a_cell_pixmap_spans_exactly_one_step}` (`cargo test -p pdfcraft-render --test it`); (28) `raster::tests::large_non_interpolated_images_are_smoothed_when_enlarged`, `raster::tests::small_non_interpolated_images_keep_hard_pixels_when_enlarged`, `raster::tests::a_scan_shrunk_one_way_is_smoothed_the_way_it_is_stretched` | +| hayro | 0.7.1 | Apache-2.0 OR MIT | (1) `RenderSettings::{x_offset, y_offset}` to render one tile of a large page; (2) images over `MAX_IMAGE_PIXELS` (2^28) are skipped (a fuzzed `/W 4294967295` hung in resampling); (3) `tiling_cell_scale` keeps a tiling pattern's cell pixmap within 3000 px a side whatever its `/XStep`/`/YStep` (a fuzzed step far beyond `/BBox` allocated 2 GB); (4) stroke widths clamped to 4 × (canvas width + height) device pixels (a fuzzed `/LW 9223372036854775807` allocated 10 GB in stroke expansion); (5) `usable_dash_pattern`: a stroke whose dash pattern could cut it into more than `MAX_DASHES_PER_STROKE` (250,000) pieces, counting kurbo's restart of the pattern at every subpath and `ClosePath`, is drawn solid (a fuzzed `/D [[11] 0]` on a line from x = 92234775807 allocated over 5 GB, `[0.000001] 0 d` on a 20 pt line over 10 GB); (6) a dash pattern with a negative or non-finite entry, or a period that isn't positive, is drawn solid (`[-1 -1] 0 d` looped for ever in kurbo); (7) `render_size` and `render_into`: render into a buffer the caller allocates (PdfCraft renders into pixel-aligned memory that the GUI takes over as texture data); `render` is `render_into` on a new pixmap; (8) checked image source/target/RGBA, alpha-premultiplication scratch and intermediate sizes before resampling, finite transforms and exact source/mask buffers; sources wider than u16 may shrink before backend limits, with a 2^20 source-axis cap and conservative 128 MiB filter-planning estimate; an over-limit or failed resize keeps the original image resolution and geometry when the source fits the backend, otherwise skips it, with a logged warning; staging, resampling output/scratch, RGBA conversion and padding buffers use fallible reserves; (9) `may_paint_canvas`: a fill or stroke whose control-point box, grown by how far its paint reaches past the path (half the stroke width times the larger of the miter limit and √2) plus 0.5 user units and 2 pixels, misses the canvas is not drawn, so a tile no longer strokes and fills every path of its page (item 5 of #623); `set_skip_offscreen_paths` turns this off for the regression test; (10) `/Interpolate` governs the pre-resize too (#624): a minified image that does not request interpolation is pre-resized with nearest-neighbour (since (21), only inside Type 3 glyphs), planned without the Catmull-Rom filter limits (`image_resampling_size` takes the flag); a Type 3 glyph image is sampled bicubically only when interpolated; a minified image with a matching soft mask is nearest-sampled straight into premultiplied RGBA; (11) `may_paint_rect`: an image whose stream-size box, mapped to the device, misses the canvas is skipped before decoding (decoded data can only shrink against that box), so a tile of an image-heavy page no longer decodes, resamples and rasterizes every image on it; (12) preview renders (`InterpreterSettings::preview`: backdrops, thumbnails, grid cells, signature-drag backgrounds) skip images under `PREVIEW_MIN_IMAGE_AREA` device pixels, while final renders always draw them; (13) reading colours (#897): `RenderSettings::color_replacement` takes a `ColorReplacement` (text colour, background, `only_black`, `line_art`) that remaps solid text and line-art paint while rendering, glyph outlines counting as text and stencil-mask ink as line art; images, image alpha masks, soft masks and pattern cells keep their colours; `render_pdf` gains a `color_replacement: Option` argument (pass `None` for true colours); the upstream-only `[dev-dependencies.log]` entry is removed from `Cargo.toml` (log is a normal dependency since the logging patch), so `cargo test -p hayro` runs this crate's unit tests; (14) a shrunk image that is interpolated (or, since (21), area-averaged) and whose axes lie along the device's (upright, mirrored or a quarter turn, as on a page under `/Rotate`) is snapped to whole device pixels: shrunk once to its rounded device rectangle (halves round up, so a tile snaps an edge where the whole page does) and drawn 1:1 from the rounded corner, instead of a ceil-sized raster resampled again at a fractional scale and offset (`snap_minified_image`; scans looked soft next to Acrobat, and until #5530 a turned one still did, up to 200 grey levels off the same page drawn upright). An image counts as axis-aligned when the terms the snap drops move its far edges by at most a hundredth of a device pixel (`SNAP_SKEW`), not by a tolerance on the coefficients, which let a slight skew of a very wide image through; and the snapped 1:1 transform is used only when the shrink produced the snapped size, so a failed resize keeps the full transform instead of drawing the source raster at its source size; (15) a soft mask of another size than its image is filtered when shrunk, whatever its `/Interpolate` (#1649): such a mask carries the shape on its own (Word saves text as a tiny black image with a glyph-bitmap mask), and nearest-neighbour dropped every stroke thinner than the shrink factor; (16) under a soft mask or a blend mode, axial and radial shadings are sampled instead of drawn as native gradients (#2128): vello_cpu 0.0.8 copies an opaque gradient over fully covered tiles without the mask or blend mode, so PowerPoint's faded gradient fills came out solid; (17) the renderer keeps annotation paints out of the reading-colours mapping between `begin_annotations` and `end_annotations` (#1548); (18) the page renders as one transparency group; under reading colours its background is painted inside it as the initial backdrop, so a Multiply highlight blends with the dark background instead of staying an opaque block, and otherwise the background stays outside, so a blend mode over an empty part of the page keeps its colour instead of blending with the white paper (#2970); (19) `set_paint` samples a mesh shading's texture over its triangles alone (`EncodedShadingPattern::extent`, padded rather than repeated) and skips a mesh with none on the canvas: a BricsCAD plan's 310 one-triangle `sh` each covered the whole tile, 2 s per 1024 px tile at any zoom (the watchdog gave the page up on a busy machine), now about 70 ms; (20) a tiling pattern whose step is too large for a cell pixmap, where what is painted (a fill, or a stroke with its width) can only show the first cell (PowerPoint's `/XStep 2147480100` for a picture fill drawn once), has its `/BBox` drawn at full scale with a transparent border and padded instead of tiled (`shows_one_cell`; a cell whose scale isn't finite and positive, from a `/BBox` too thin to divide by, keeps the old path: `usable_cell_scale`); shrinking the cell to fit the 3000 px limit of (3) left the picture under a pixel, so network diagrams lost their pictures (#3349); (21) an image that does not request interpolation and is drawn below its resolution, outside a Type 3 glyph, is shrunk by area averaging (`area_shrink`: each device pixel the coverage-weighted mean of the samples it covers, in 16.16 fixed point whose weights sum to exactly one, premultiplied when the image has a mask; a soft mask no larger than the image is first sampled onto the image's grid by `mask_on_image_grid` (nearest-neighbour, or bilinearly when the mask asks for `/Interpolate`, so its edge stays as smooth as an enlarged mask's), so its colour and mask are averaged together instead of the averaged colour of transparent samples showing under a mask layer at the mask's edges) to its snapped device rectangle, instead of nearest-neighbour: `/Interpolate` asks for smoothing only when an image is enlarged (ISO 32000-2 §8.9.5.3), and nearest-neighbour kept one sample in N per axis, so the text of a 300 dpi scan at 100% lost most of its strokes (sampling them again at a fractional offset dropped more). PicScale's `Box` and `Area` filters were not used: shrinking, both average a window four target pixels wide. On a 13-page 300 dpi scan every rendered sample at 96, 120 and 192 dpi is within 1 of the exact area average (it was up to 255 off, 17-23% of samples by more than 1). A snapped image whose shrink failed keeps the full transform, so it is not drawn at its source size. The shrink never enlarges: a side drawn at or above its resolution (an image shrunk across and stretched down, as a 204 × 98 dpi fax is) keeps its samples for the sampler. Each target row averages the whole source rows it covers (contiguous, cheap) and is then shrunk across with taps worked out once per image, which made the shrink about three times faster (a 300 dpi page at 96 dpi: 25 ms, nearest-neighbour 11 ms); working a row at a time (`area_shrink_rows`) keeps its work memory to a few rows. An image with a soft mask, a stencil's ink included, is premultiplied a source row at a time as it is averaged (`area_shrink_premultiplied`), never as a full-resolution RGBA copy: a 10718 × 13701 stencil shown at page size peaked at 1.16 GB with that copy and peaks at 1.03 GB without it, as with nearest-neighbour. A source with a side over `MAX_RESAMPLING_SOURCE_SIDE` (2^20) keeps (10)'s nearest-neighbour shrink, and `area_shrink` refuses it, so its accumulator row and tap table stay a few times one source row (a 2^27 × 2 image of zeros is a tiny file); (22) knockout groups: while one is innermost, fills, strokes, glyph outlines and shadings are drawn with `Compose::Copy` (weighted by coverage), so each element replaces what earlier elements of the group drew instead of compositing over it; an image or nested group, drawn into a layer of its own, first has what is under its shape cleared (`knockout_image` for an unmasked image's rectangle, only when `draw_image` will draw it, decoded at the size it is drawn; a knockout erase pass that draws a nested group's content with every paint clearing instead of painting, a shading fill's clear clipped to the shading's `/BBox` as its paint is) and then composites over it (#2971); (23) backport of hayro 0.8: shadings that `as_svg_gradient` can represent are painted as a `vello_cpu` linear or two-point radial gradient instead of a texture sampled pixel by pixel through the shading function (InDesign fades are luminosity soft masks of one full-page axial `sh`: 40 on a report page took 15 s at 144 dpi, now under 1 s); without upstream's half-pixel shift, which suits vello_cpu 0.3 but moves 0.0.8's gradients off pixel centres; (25) a tiling cell is drawn at the scale that makes one `/XStep` (`/YStep`) exactly the cell pixmap's whole number of pixels (`tiling_cell_pixels`, at least 1 and at most the cap), so the repeated pixmap puts cells at whole multiples of the step at any device scale; drawing at the unrounded scale repeated the cell up to half a pixel off at every step, and the error grew with distance from the pattern's origin (ISO 32000-2 8.7.3.1); (28) an image of page-scan size (`SMOOTHED_IMAGE_SIDE`: 512 samples or more on its shorter side) that does not request interpolation is drawn with `ImageQuality::Medium` (bilinear, as interpolated images are) when enlarged by more than 1% on either side (`SMOOTHED_IMAGE_SCALE`: a scan drawn at its resolution can come out a few ulps above 1, and blurred by half a pixel at a half-pixel offset), also when the other side is shrunk (the area shrink leaves that side at 1:1), outside a Type 3 glyph: scans rarely set `/Interpolate`, and drawn nearest-neighbour every sample of a zoomed scan became a block, so its text had staircase edges; icons, QR codes and pixel art keep their hard pixels. Bicubic (`High`) looked the same on a zoomed 300 dpi scan and drew four 1024 px tiles at 400% in 267 ms against 50 ms | (1) `raster::tests::tiles_match_full_render`; (2) `raster::tests::absurd_image_dimensions_are_skipped`; (3) `raster::tests::tiling_cells_stay_small_whatever_the_step`; (4) `raster::tests::absurd_predictor_columns_and_line_widths_render`; (5) `raster::tests::dash_patterns_that_cut_a_stroke_into_billions_of_pieces_render`; (6) `raster::tests::invalid_dash_patterns_terminate_and_are_drawn_solid`; (7) `raster::tests::pixels_are_a_plain_renders_bytes_in_a_buffer_a_gui_can_take_over`; (8) `raster::tests::image_resampling_rejects_anisotropic_target_growth`, `raster::tests::image_resampling_bounds_intermediate_before_planning`, `raster::tests::image_resampling_rejects_invalid_sources_and_scales`, `raster::tests::image_resampling_keeps_valid_sizes_and_exact_limits`, `raster::tests::resampling_fallback_preserves_original_geometry_and_pixels`, `raster::tests::image_resampling_wide_sources_shrink_before_backend_side_limits`, `raster::tests::image_resampling_wide_transparent_sources_still_shrink`, `raster::tests::image_resampling_padded_backend_dimensions_stay_checked`, `raster::tests::image_resampling_mismatched_alpha_mask_keeps_pixels_and_placement`; (9) `raster::tests::skipping_paths_off_the_canvas_changes_no_pixel`; (10) `raster::tests::non_interpolated_images_are_not_smoothed_when_minified`, `raster::tests::non_interpolated_gray_images_are_not_smoothed_when_minified`, `raster::tests::non_integer_minification_averages_the_columns_each_pixel_covers`, `raster::tests::non_integer_minification_averages_colour_and_mask_together`, `raster::tests::non_interpolated_alpha_images_keep_their_colours_and_mask_when_minified`, `raster::tests::non_interpolated_type3_glyph_images_are_not_smoothed_when_minified`, `raster::tests::image_resampling_nearest_accepts_wide_sources_the_filter_refuses`, `raster::tests::wide_non_interpolated_sources_draw_after_shrinking`; (11) `raster::tests::skipping_paths_off_the_canvas_changes_no_pixel`; (12) `raster::tests::preview_renders_skip_tiny_images_but_finals_draw_everything`; (13) `color_replacement_tests::{black_maps_to_text_colour, white_maps_to_background, midgrey_interpolates, alpha_copied_verbatim, only_black_threshold, line_art_false_skips_paths_but_not_text}`, `color_replacement_render_tests::{color_replacement_maps_text_and_line_art, color_replacement_maps_stencil_mask_ink}`; (14) `raster::tests::minified_image_edges_snap_to_whole_pixels`, `raster::tests::a_tile_snaps_half_pixel_image_edges_as_the_whole_page_does` (`cargo test -p hayro`), `raster::tests::a_turned_or_mirrored_minified_image_snaps_to_whole_pixels`, `rotated_image_snap::a_shrunk_image_on_a_turned_page_is_the_upright_render_turned` (`cargo test -p pdfcraft-render --test it`), `renderer::snap_tests::a_skew_that_moves_a_far_edge_is_not_snapped` (`cargo test -p hayro`); (15) `raster::tests::shrunk_soft_mask_larger_than_its_image_keeps_thin_strokes`; (16) `raster::tests::soft_mask_applies_to_a_shading_pattern_fill`; (17), (18) `raster::tests::reading_colors_keep_highlight_annotations_readable`; (19) `mesh_extent_tests::{a_mesh_paints_its_triangles_alone, a_mesh_keeps_the_graphics_state_alpha, a_mesh_off_the_canvas_paints_nothing}` (`cargo test -p hayro`); (20) `raster::tests::a_tiling_pattern_drawn_once_shows_its_cell`, `raster::tests::a_large_step_still_tiles_where_the_next_cell_shows`, `raster::tests::a_tiling_cell_too_thin_to_scale_keeps_the_old_path`; (21) `raster::tests::non_interpolated_minification_keeps_thin_strokes`, `raster::tests::non_integer_minification_averages_the_columns_each_pixel_covers`, `raster::tests::non_integer_minification_averages_colour_and_mask_together`, `raster::tests::non_interpolated_minification_does_not_bleed_transparent_colour`, `raster::tests::a_smaller_soft_mask_does_not_bleed_transparent_colour_when_minified`, `raster::tests::an_interpolated_smaller_soft_mask_stays_smooth_when_its_image_is_minified`, `raster::tests::a_shrunk_stencil_mask_keeps_thin_strokes`, `raster::tests::minified_image_edges_snap_to_whole_pixels`, `raster::tests::an_image_shrunk_one_way_and_stretched_the_other_is_still_averaged`, `raster::tests::wide_non_interpolated_sources_draw_after_shrinking`, `renderer::area_shrink_tests` (`cargo test -p hayro`); (22) `raster::tests::knockout_groups_replace_what_their_elements_cover`; (23) `raster::tests::axial_and_radial_shadings_draw_as_native_gradients`; (25) `tiling_cell_placement::{far_cells_of_a_fill_sit_at_whole_steps, far_cells_sit_at_whole_steps_under_a_turned_scaled_matrix, far_cells_of_a_stroke_sit_at_whole_steps, a_cell_pixmap_spans_exactly_one_step}` (`cargo test -p pdfcraft-render --test it`); (28) `raster::tests::large_non_interpolated_images_are_smoothed_when_enlarged`, `raster::tests::small_non_interpolated_images_keep_hard_pixels_when_enlarged`, `raster::tests::a_scan_shrunk_one_way_is_smoothed_the_way_it_is_stretched`, `raster::tests::a_scan_drawn_a_rounding_error_above_its_resolution_stays_sharp` | | hayro-jbig2 | 0.3.0 | Apache-2.0 OR MIT | (1) at most 2^28 pixels per bitmap and page bitmap, and 2^25 cells per halftone grid and grayscale image (each side could be 65535: 4.3 billion pixels decoded one by one, and a 4-byte value per cell); (2) at most 2^26 symbol pixels per symbol dictionary (a fuzzed dictionary decoded for minutes); (3) a symbol dictionary's height classes are capped at one per new symbol and its export runs at `2 × symbols + 1`, each plus `EXTRA_EMPTY_STEPS` (1024): an empty height class or a zero-length export run doesn't advance, and past the end of its data the arithmetic decoder can decode them for ever (one changed byte hung the renderer); (4) a segment header's long-form referred-to segment count is capped at `u16::MAX` and a larger count is refused before its retention or referred-to bytes are read: the 29-bit count went straight to `Vec::with_capacity`, a reservation of about 2 GiB whose failure aborts the process (from storytold/craft-libs#176); (5) what a page's segments hold for later ones (intermediate regions, pattern and symbol dictionaries with their headers and retained contexts, and Huffman tables) is at most 64 MiB, two square bitmaps of 16,384 a side (`MAX_RETAINED_WORDS`), and a dictionary's new and exported symbols are counted against it as they are made: each was bounded on its own, but a few dozen bytes an intermediate region held 33 MB each, and 8 MiB symbols exported again and again killed the process, so a few kilobytes could exhaust memory; (6) a segment may refer to at most 2^20 symbols, checked before any are collected: references may repeat, and 65,535 of them to one dictionary, in a 78 KB stream, asked for 17 GB; (7) a pattern dictionary keeps at most 16 MiB of shifted copies, which only speed up drawing, and none that wouldn't fit what the page has left (`MAX_SHIFTED_WORDS`); (8) a custom Huffman table has at most 2^16 lines, and a stream with its globals at most 2^20 segments and 2^22 references, checked as they are read: both grew before anything counted them, and PDFs of a kilobyte or so reached 5 GB and more behind nested Flate; (9) a page past the bounds of (1) is refused when the stream is opened: a caller sizes its output from the header before the page bitmap is made, and an Indexed image in a 743-byte PDF asked for 137 GB, which aborts; (10) a stream may ask for at most 2^30 units of decoding work, four of the largest pages, about 6 s to 17 s (`MAX_WORK`): decoding a bitmap costs a pixel each and 4 a row, copying one a word each and 1 a row, and each integer an item reads costs 38, what one takes at worst, so one with no pixels pays too, before it is decoded: each bitmap was bounded, but nothing bounded how many there were, and a 795-byte PDF took 265 s; (11) `Image::decode_within` spends from what a run of images may still do, starting with its page's pixels, and `charge_reading` charges reading a stream and its globals before anything is parsed, so an operation over a document can bound what `MAX_WORK` bounds for one image, and images that fail or decode nothing add up too: a file of thousands of images made to reach that limit took as many times as long; (12) adding to a location, a reference offset, a symbol's corner or a Huffman range line near the end of an `i32`, and negating a decoded integer of 2^31, overflowed: a panic wherever overflow is checked. The refinement context, a text region's corners, the integer decoder and the Huffman table wrap, as a release build did, and `Bitmap::combine` saturates, which comes to the same, as a location that overflows is past the end of any page. What is drawn doesn't change | (1) `raster::tests::jbig2_regions_of_billions_of_pixels_are_refused`, `raster::tests::jbig2_halftone_grids_past_their_bound_are_refused`; (2) the nightly `cargo xtask fuzz` finding, which now finishes in about a second; (3) `raster::tests::jbig2_symbol_dictionaries_that_never_advance_terminate`; (4) `segment::tests::test_referred_to_count_is_capped` (`cargo test --manifest-path vendor/hayro-jbig2/Cargo.toml`); (5) `raster::tests::jbig2_regions_held_past_a_pages_bound_are_refused`, `jbig2_symbols_without_pixels_count_towards_a_pages_bound`, `jbig2_retained_contexts_count_towards_a_pages_bound`, `jbig2_huffman_tables_count_towards_a_pages_bound`, `jbig2_new_symbols_count_towards_a_pages_bound`, `jbig2_new_symbols_are_held_to_what_a_page_has_left`, `jbig2_pattern_dictionaries_count_towards_a_pages_bound`, and in the crate (`cargo test --manifest-path vendor/hayro-jbig2/Cargo.toml`) `decode::symbol::tests::exported_symbols_are_counted_before_they_are_copied`, `tests::the_room_left_shrinks_as_segments_are_stored`; (6) `raster::tests::jbig2_segments_referring_to_too_many_symbols_are_refused`; (7) `raster::tests::jbig2_pattern_dictionaries_too_large_to_shift_still_draw`, `jbig2_shifted_patterns_never_refuse_an_image`; (8) `raster::tests::jbig2_huffman_tables_of_too_many_lines_are_refused`, `jbig2_streams_of_too_many_segments_are_refused`, `jbig2_streams_of_too_many_references_are_refused`, and in the crate `tests::the_segments_of_a_stream_and_its_globals_count_together`; (9) `raster::tests::jbig2_pages_past_the_bounds_are_refused_before_anything_is_sized_from_them`, `tests::a_page_past_the_bounds_is_refused_when_opened`; (10) `raster::tests::jbig2_decoding_work_is_counted_item_by_item`; (11) `jbig2::decode::tests::a_run_of_images_spends_its_allowance_down_to_the_step` (pdfcraft-filters); (12) `raster::tests::jbig2_placements_at_the_end_of_an_i32_draw_nothing` | | lopdf | 0.45.0 | MIT | (1) PNG predictor rows longer than the data are refused before their buffers are allocated (a fuzzed `/Columns 4294967295` allocated two 4 GiB rows), and empty frames return without row buffers; (2) predictor row widths use saturating arithmetic or reject overflow; (3) sub-byte TIFF Predictor 2 scratch is bounded by the samples present in the input instead of the declared `/Colors`, aligned with [lopdf#589](https://github.com/J-F-Liu/lopdf/pull/589); (4) the object lookup keeps its cycle history through compressed objects, requires an ordinary in-file container, and caps the chain of indirect lookups at 16 (a chain of new objects, each a `/Length` compressed in a container whose own `/Length` names the next, or a stream whose `/Length` names the next stream, overflowed the stack: a dev build's 2 MiB thread within 800 hops); (5) object-stream members are parsed within their own extent (up to the next distinct offset), an entry whose offset repeats an earlier entry's is skipped (so a repeated object number can resolve to its earlier entry), and offsets are processed in index order; (6) `find_latest_trailer`: a rebuild takes a `/Type /XRef` stream dictionary as the trailer when the file has no `trailer` keyword (the latest in the file wins, as a later `trailer` would), so a concatenated or truncated file keeps its /Info and /ID instead of failing to load; a candidate whose /Root is the catalog beats one whose /Root is another dictionary, and an in-file /Root must match its generation (#2365) | (1) `inspect::tests::png_predictor_rows_longer_than_the_data_are_refused_before_allocating`, `inspect::tests::empty_png_predictor_frames_do_not_allocate_rows`; (2) `inspect::tests::tiff_subbyte_predictor_row_width_overflow_is_refused`; (3) `inspect::tests::tiff_subbyte_predictor_scratch_is_bounded_by_available_samples`, `inspect::tests::tiff_subbyte_predictors_keep_components_rows_and_padding`; (4) `inspect::tests::compressed_object_self_container_does_not_recurse`, `inspect::tests::compressed_object_mutual_containers_do_not_recurse`, `inspect::tests::object_stream_length_cycle_does_not_recurse`, `inspect::tests::compressed_object_container_chain_does_not_recurse`, `inspect::tests::compressed_length_chain_does_not_recurse`, `inspect::tests::plain_stream_length_chain_does_not_recurse`, `inspect::tests::compressed_objects_and_indirect_stream_length_still_load`, `inspect::tests::compressed_length_in_a_container_with_an_indirect_length_still_loads`; (5) `inspect::tests::object_stream_member_extents_bound_owned_strings`, `inspect::tests::object_stream_member_extents_preserve_index_order_and_whitespace`; (6) `inspect::tests::rebuilt_file_keeps_the_info_of_its_cross_reference_stream`, `inspect::tests::rebuilt_file_prefers_a_trailer_whose_root_is_the_catalog` | | hayro-syntax | 0.7.2 | Apache-2.0 OR MIT | (1) Page-tree cycle guard in `resolve_pages` (visited set + depth cap): a `/Kids` loop among compressed objects overflowed the stack and aborted the process; (2) `ccitt_size_ok`: CCITT images wider than 2^20 columns or over 2^28 pixels are not decoded (a fuzzed `/Columns 4294967295` allocated 4 GiB); (3) Flate/LZW predictor row widths are saturating `usize`, a zero width is refused and a row longer than the data is not allocated (a fuzzed `/Columns 9223372036854775807` wrapped to a 2^61-byte allocation and aborted); (4) the inline-image end-of-data check (`EI`) doesn't nest inside its own look-ahead and examines at most 256 candidates and steps (50 images with "EI" in their data ran for over ten minutes); an EI followed by a delimiter (EI%..., EI[...) ends the image like one followed by white space; only a regular character after EI skips the candidate; (5) a stream whose `/Length` is wrong is recovered only when `stream` is the next token — scanning ahead to a later object's `stream` made an indirect appearance-state dictionary (Quartz checkboxes: `/AP /N` → `<< /Yes … /Off … >>`) render as a non-form, so the widget was skipped; (6) JPX rescaling uses an exact integer sample range for 1–32-bit components (32 shifted past u32, and floating-point maxima at 25–31 bits wrapped to zero), and refuses zero-sized rows; (7) `decode_limit`: Flate (through flate2 and the fallback inflater), LZW, RunLength and ASCII85 never write past `MAX_DECODED_STREAM` (256 MiB), or, for the last filter of an image of known size, past twice its pixel bytes (a predictor tag byte per byte), at most 1 GiB; `Stream::decoded_within` for budgeted callers; a page's `/Contents` array shares one `MAX_DECODED_STREAM` (a 2 MB page whose content stream or 8 × 8 image inflated to 2 GB took the renderer past 3 GB in under a second; naming the stream twenty times in `/Contents`, 5 GB); (8) high-precision numbers (jsPDF MediaBox coordinates with over 19 digits) parse without overflowing the u64 mantissa: fractional digits past it are dropped, integer digits past it scale the value (a real, kept finite); (9) large dictionary indexes are built on first use and shared across repeated reads within each PDF (#307); the validated-byte cache is bounded, reader contexts remain separate, repairs and inline/content dictionaries bypass sharing, and small dictionaries stay eager; (10) decoded object-stream slots publish once under contention, cold loading rejects recursive dependencies, and xref repair serializes without holding recursive read guards; this permits one parser per native render pool with private worker caches; (11) `sync.rs`: a poisoned lock is recovered instead of unwrapped (a worker panic no longer cascades into every reader of the same PDF), and xref repair takes a blocking write lock instead of asserting a non-blocking one; (12) `SegmentList` (`util.rs`) returns `None` for an out-of-range index instead of panicking; (13) a page's `/UserUnit` scales `render_dimensions` and `initial_transform`, so the page and everything on it are sized in points, as Acrobat shows them; it is read as Acrobat reads it (the page's own entry, not inherited; a number from 1 up, at most 75,000; anything else counts as 1), and a page without it keeps exactly the transform it had; (14) revision 2–4 passwords are tried as given and then in PDFDocEncoding (`legacy_passwords` in `crypto/mod.rs`, which resolves the upstream `TODO: Convert to PDFDocEncoding`): passwords arrive as UTF-8 text, so a password with characters outside ASCII (`Çok güzel`, `ılık€`) written in PDFDocEncoding, as ISO 32000-2 §7.6.4.3.2 asks and qpdf and Acrobat write it, was always wrong; (15) page content is read a piece at a time (`page::ContentReader`, `Page::content_reader`): plain streams and Flate streams without a predictor are decoded as they are read, with a budget of 64 times their compressed size and never less than `MAX_DECODED_STREAM` (a stream that expands past it is cut and reported); other streams are decoded whole, as before, and share `MAX_DECODED_STREAM`. The content tokenizer reads a window at a time (`UntypedIter::new_window`, `TypedIter::needs_more`, `TypedIter::committed`): an instruction that runs past the window is not decided until more data follows, which `Reader` records by flagging reads and `at_end` past the end of the data, including through the clones the parser makes. The Flate decoder keeps its own 32 KiB history, so it pauses at any output size. `ContentReader::new` reuses raw/decrypted stream bytes for budget accounting and piece creation, and `decoded_raw_within` applies the existing Whole-stream filters and limits to those bytes without decrypting twice; (16) revision 5–6 (AES-256) passwords are tried in their SASLprep form (RFC 4013, then UTF-8, at most 127 bytes, as ISO 32000-2 §7.6.4.3.3 Algorithm 2.A asks and pdfcraft-crypt writes them) and then, if that differs, as given (`aes256_passwords` in `crypto/mod.rs`, which resolves the upstream `TODO: Do the above.`; `stringprep`, already in the tree through pdfcraft-crypt, is an optional dependency enabled by `std`, and without `std` passwords are used as given): the renderer used the password as typed, so one that SASLprep changes (a decomposed `café`, a no-break space, the fullwidth `ａｂ` a Japanese IME types in full-width mode) was wrong, although pdfcraft-cos accepted it: protecting a document with such an open password was rolled back ("the password is incorrect"), and a file with such a password did not open with it.; (17) CCITT `/Rows 0` means the height is not predetermined (ISO 32000-2 Table 11), not zero rows: scanned Group 4 pages written that way rendered blank; (18) The streaming Flate decoder (`lzw_flate::flate::fallback`) copies a back-reference in runs of up to its distance (`extend_from_within`), not byte by byte. The copy reads only bytes already in the output, and a copy whose source does not exist still writes nothing, so the bytes are those a byte-by-byte copy writes. Inflating the A1 sample's content stream (278.6 MB) takes about 14% less time (0.66 s to 0.56 s); the output is identical; (19) CCITT `/Rows` below the image's `/Height` decodes to the height: a plotter driver wrote `/Rows` equal to `/Columns`, so only the first rows were decoded and the rest drawn black (#3308) (20) `stream` followed by spaces or tabs and then the line break (`stream \r\n`, as Microsoft Reporting Services writes it) starts the data after the line break; the content stream was taken to start at the space and the page rendered blank.; (21) a JP2 colour specification naming an enumerated colour space the decoder refuses (CIEJab in a veraPDF fixture, YCbCr, PhotoYCC, YPbPr, bilevel…) no longer fails the image: it is decoded again from a copy whose `colr` names the space the decoder assumes without one, greyscale, sRGB or CMYK by component count, as MuPDF and Poppler draw it (`without_refused_enumerated_colour`; the page drew blank, #3306); (22) an object stream's index of `objnum offset` pairs is parsed once per stream and kept beside its decoded data (`ObjectStreamIndexes`, `XRef::with_object_stream_indexes`), and an offset past `usize` is malformed instead of wrapping: `XRef::get_with` parsed the whole index again for every object it fetched from the stream, so reading all of a stream of N objects took N² steps (a Firefox page whose 65,542 objects share one stream took 37 s to draw and 40 s to extract its text); (23) the inline-image end-of-data look-ahead no longer rejects a candidate `EI` because a byte above 0x7F lies between it and the next one: when the candidate is the end, those bytes are the content stream that follows the image, and its strings carry such bytes (WinAnsi `(Grüße) Tj`, the two-byte codes of any Type 0 font), so the operator check that follows decides alone, bounded as in (4) (a scanned page whose OCR text held an accented word and a later capital "EI" — EINGANG, BEI, KLEIN — drew and extracted nothing after the image: 40 of 275 pages across five real documents); (24) `xref::prev_offset`: a trailer `/Prev` written as a reference `N G R` is read as the offset that object holds when the section's own entries place it (iText's `indirectPrev.pdf`; each section's entries are read before its `Prev` is followed), and otherwise as offset N (tiff2pdf; JHOVE's PDF-HUL-157 files, #2366), as `pdfcraft-cos` and other readers take it, so the earlier sections' pages are seen instead of one; (25) direct arrays and dictionaries share a reader-local nesting cap of 128, counted while skipping and while parsing a dictionary (its lenient repair reads, rather than skips, an object found where a key belongs, so `<< << …` recursed through that read); deeper objects return `None` instead of overflowing the stack (including unused trailer values, trailer entries, content operands and marked-content properties); a chain of 65 direct stitching functions now stops at this cap, before the 64-function guard; (26) an xref stream subsection must be backed by its decoded data: all-zero `/W` widths (an entry reads no bytes) and a declared count longer than the data are refused before any entry is inserted, where a tiny stream with a large `/Size` or `/Index` count inserted that many entries (from storytold/craft-libs#239); a subsection running past object number `i32::MAX` is refused instead of overflowing or turning negative; (27) an xref stream's type field is read big-endian at its full `/W[0]` width, where only its first byte was read and wider fields shifted every later entry, and `/W[0]` over 8 bytes is refused like `/W[1]` (from storytold/craft-libs#241); (28) a content instruction keeps up to 256 operands (`MAX_OPERANDS`) instead of 10, and past that drops the oldest instead of ending the content stream: the eleventh operand of a twelve-ink DeviceN `scn` (ISO 32000-2 §8.6.6.5 allows any number of components) blanked the rest of the page (#4260); (29) a `/JBIG2Globals` stream that is itself JBIG2 isn't decoded: globals are raw segments, and a chain of 16,000 such streams overflowed the stack, which aborts the process; (31) `set_page_jbig2_work`: the JBIG2 images decoded on a thread spend one allowance, set per page, 2^30 steps (`MAX_PAGE_JBIG2_WORK`), so a page of many images made to reach their own limit stops at one, and what they cost to read counts, whether they decode or not; unbounded until a page sets it; (32) a group of array items read together (a tuple: an image's `/Decode`, `/W`, `/W2`, and a function's `/Domain`, `/Range`, `/Encode` and `/Decode`) can start with a reference: it was taken for the whole group and lost, which ended the array, so an image with `/Decode [7 0 R 0]` didn't draw and CID widths fell back to `/DW`; (33) an object that holds only a reference stands for what that names, stored on its own or in an object stream, up to 32 references: read as a number it was the reference's own object number, so a `/Decode` or `/W` entry naming such an object took a wrong value, and as anything else it was lost. Past 32, as any cycle goes, it reads as a missing object. An object stream isn't reached that way, as its checks are of the object named; reading its index counts against the 64 nested loads and reads its dictionary alone, as a chain of them, each with a member of the next for its `/N`, `/First` or `/Length`, ran out of stack; and a comment inside a reference counts as white space; (34) a table rebuilt by scanning the file (`fallback_xref_map`) finds an object by its number when its header names another generation, including one no generation may be (`3 18446744073709551616 obj` in `poppler-85140-0.pdf`, #5580): the header is read leniently only in that scan (`read_object_header`), the object is also filed under generation 0 unless a real `N 0 obj` holds that key (a later header for the number moves the stand-in), `XRef::get_with` matches by object number only in a repaired table, and `XRef::objects` yields each location once | (1) `cos/tests/writer.rs::page_tree_loops_in_object_streams_do_not_crash_readers`; (2) `raster::tests::absurd_mask_and_fax_sizes_are_refused_before_decoding`; (3) `raster::tests::absurd_predictor_columns_and_line_widths_render`; (4) `raster::tests::inline_images_with_ei_in_their_data_parse_in_linear_time`, `content::tests::inline_image_ei_may_be_followed_by_a_delimiter`; (5) `raster::tests::indirect_appearance_state_dict_is_not_another_stream`; (6) `raster::tests::jpx_stream_bit_depth_boundaries_do_not_panic`, `raster::tests::jpx_32_bit_image_does_not_prevent_other_page_content_from_rendering`, `raster::tests::jpx_gray_rgb_and_alpha_keep_eight_bit_samples`, `raster::tests::jpx_streams_reject_out_of_range_bit_depths`; (7) `raster::tests::decompression_bombs_stop_at_the_stream_limit`, `raster::tests::page_contents_and_nested_forms_share_one_budget`; (8) `inspect::tests::jspdf_high_precision_mediabox_does_not_overflow_dimensions`, `object::number::tests::high_precision_float_does_not_overflow_mantissa` (hayro-syntax); (9) `object::dict::lazy_index_tests`, `raster::tests::large_resource_indexes_preserve_pixels_and_text`; (10) `data::tests`, `xref::tests::concurrent_cold_object_stream_reads_all_succeed`, `raster::tests::shared_parser_is_lazy_and_initializes_once_under_contention`; (11) `data::tests::panicking_initializer_wakes_waiters_and_prevents_retries`, `xref::tests::concurrent_cold_object_stream_reads_all_succeed`; (12) `util::tests::segment_list_rejects_out_of_range_indices`; (13) `raster::tests::user_unit_scales_the_page_and_everything_on_it`; (14) `crypto::pdfcraft_password_tests`, `pdfcraft-engine` `tests::legacy_passwords_with_turkish_characters_open`; (15) `content::tests::windows_keep_every_instruction_at_every_split`, `content::tests::windows_keep_every_instruction_in_pieces_of_any_size`, `content::tests::windows_match_the_slice_on_random_content`, `content::tests::windows_drop_blanks_and_keep_a_comment_that_may_go_on`, `page::content_reader_tests::content_past_the_cap_is_read_to_the_end`, `page::content_reader_tests::a_stream_that_expands_past_the_budget_is_cut_there`, `page::content_reader_tests::a_small_expansion_under_the_cap_is_not_cut`, `raster::tests::page_content_past_the_old_cap_renders`, `raster::tests::content_reader_returns_the_same_bytes_in_any_pieces`, `raster::tests::content_bombs_stop_at_the_budget_and_say_so`, `page::content_reader_tests::plain_and_whole_pieces_keep_their_bytes`, `page::content_reader_tests::repeated_content_reference_counts_compressed_bytes_once`, `page::content_reader_tests::whole_cap_still_counts_later_unique_streams`; (16) `crypto::pdfcraft_password_tests::aes256_passwords_are_tried_in_their_saslprep_form_then_as_given`, `crypto::pdfcraft_password_tests::aes256_passwords_open_as_typed_and_in_their_saslprep_form`, `pdfcraft-engine` `tests::aes256_open_passwords_that_saslprep_changes_protect_and_reopen_as_typed`, `tests::aes256_files_whose_passwords_saslprep_changes_open_as_typed`; (17) `raster::tests::group4_fax_with_rows_zero_decodes_to_the_image_height`; (18) `filter::lzw_flate::tests::back_reference_longer_than_its_distance_repeats_its_source`, `back_reference_cut_by_the_limit_resumes_in_the_next_call` and `back_reference_reads_bytes_written_by_an_earlier_call`; each fails when the copy returns the wrong remaining count; (19) `raster::tests::group4_fax_with_rows_below_its_height_decodes_to_the_image_height`; (20) `raster::tests::space_before_the_line_break_after_stream_still_renders`; (21) `raster::tests::a_jp2_image_in_a_refused_enumerated_colour_space_is_drawn`, `filter::jpx::tests::refused_enumerated_colour_spaces_become_the_component_counts_default`, `filter::jpx::tests::what_the_decoder_handles_or_cant_be_read_is_left_alone` (`cargo test --manifest-path vendor/hayro-syntax/Cargo.toml`); (22) `raster::tests::objects_in_one_large_object_stream_are_read_in_linear_time`; (23) `text::tests::text_after_an_inline_image_survives_a_non_ascii_byte_before_a_later_ei`, `content::tests::an_inline_image_ends_at_its_own_ei_when_later_text_holds_a_non_ascii_byte` (hayro-syntax); (24) `xref::tests::prev_as_reference_reaches_the_earlier_section`, `xref::tests::prev_as_reference_to_an_object_holding_the_offset_reads_that_offset`, `pdfcraft-cos` `document::tests::a_prev_written_as_a_reference_still_reaches_the_earlier_sections`, `document::tests::a_prev_reference_to_an_object_holding_the_offset_reads_that_offset`; (25) `raster::tests::deeply_nested_direct_objects_do_not_overflow_the_stack`, `raster::tests::dictionaries_nested_without_keys_in_content_do_not_overflow_the_stack`, `raster::tests::dictionaries_nested_without_keys_in_the_trailer_do_not_overflow_the_stack`, `raster::tests::dictionaries_nested_without_keys_as_marked_content_properties_do_not_overflow_the_stack`, `raster::tests::dictionaries_nested_without_keys_are_read_up_to_the_nesting_cap`, `raster::tests::nesting_at_the_cap_fits_a_512_kib_stack` (native only), `raster::tests::function_limits_depth_guard_refuses_a_chain_of_references`; (26) `xref::tests::xref_stream_entries_need_backing_bytes`; (27) `xref::tests::xref_stream_type_field_uses_the_full_w0_width`, `xref::tests::xref_stream_loader_rejects_oversized_w0` (`cargo test --manifest-path vendor/hayro-syntax/Cargo.toml`); (28) `content::ops::tests::scn_with_twelve_components_keeps_them_all`, `content::ops::tests::operands_past_the_cap_keep_the_latest_in_order` (hayro-syntax), `raster::tests::devicen_inks::a_twelve_ink_devicen_colour_paints`, `raster::tests::devicen_inks::a_four_ink_devicen_colour_paints`, `raster::tests::devicen_inks::an_operator_with_hundreds_of_operands_does_not_end_the_content`; (29) `raster::tests::jbig2_globals_that_are_jbig2_themselves_are_not_decoded`, `raster::tests::jbig2_globals_are_read`, `raster::tests::the_jbig2_images_of_a_page_share_one_allowance`, `raster::tests::an_image_that_fails_spends_the_pages_allowance_too`; (32) `raster::tests::a_decode_array_with_references_in_it_is_read`, `raster::tests::a_functions_arrays_can_start_with_a_reference` (pdfcraft-render), `font::cid::width_tests::a_width_group_can_start_with_a_reference` (hayro-interpret); (33) `raster::tests::a_reference_to_a_reference_is_what_it_names`, `raster::tests::a_decode_array_with_references_in_it_is_read` (pdfcraft-render), `font::cid::width_tests::a_width_group_can_start_with_a_reference` (hayro-interpret), `xref::tests::an_object_stream_isnt_reached_through_a_reference`, `xref::tests::a_chain_of_object_streams_ends_at_the_limit_of_nested_loads`, `xref::tests::a_chain_of_object_streams_takes_no_more_work_for_two_ways_on`, `xref::tests::a_reference_held_in_either_storage_is_followed`, `xref::tests::a_page_a_reference_is_held_to_is_found_once` (hayro-syntax); (34) `raster::tests::a_rebuilt_table_finds_an_object_whose_header_names_another_generation`, `raster::tests::a_rebuilt_tables_generation_0_stand_in_is_not_a_second_object` (pdfcraft-render), `xref::tests::an_object_stream_and_its_first_member_are_both_enumerated` (hayro-syntax) | diff --git a/vendor/hayro/src/renderer.rs b/vendor/hayro/src/renderer.rs index c3d1d18520..1e46e3b79e 100644 --- a/vendor/hayro/src/renderer.rs +++ b/vendor/hayro/src/renderer.rs @@ -302,6 +302,12 @@ fn nearest_premultiplied_rgba( /// icons, QR codes and pixel art are usually far smaller and keep their hard pixels. const SMOOTHED_IMAGE_SIDE: u32 = 512; +/// PdfCraft patch: the scale above which such an image counts as enlarged. A scan drawn at its own +/// resolution can come out a few ulps above 1 (the scale is an `f32`: a 192 dpi scan on a 2× +/// display at 100%), and sampled bilinearly at a half-pixel offset it blurred by half a pixel. +/// Below 1% nearest-neighbour repeats at most one sample in a hundred. +const SMOOTHED_IMAGE_SCALE: f32 = 1.01; + /// PdfCraft patch: fixed-point unit of [`area_shrink`]'s weights. const AREA_ONE: u64 = 1 << 16; @@ -365,60 +371,132 @@ impl AreaAxis { /// weighted by how much of it it covers. Nothing is invented between samples, which is what /// `/Interpolate` governs (ISO 32000-2 §8.9.5.3, for images drawn above their resolution), and /// nothing is dropped: nearest-neighbour keeps one sample in N per axis, so a scan shown at screen -/// size lost the strokes of its text. Premultiplied input stays premultiplied. Work memory is one -/// source row of `u32` accumulators, the horizontal taps (at most `sw + 2 × tw`) and a vertically -/// shrunk copy no larger than the source; a side over `MAX_RESAMPLING_SOURCE_SIDE` is refused, so -/// none of it exceeds a few times one source row. `None` if nothing shrinks, the sizes don't fit -/// or a buffer cannot be allocated. +/// size lost the strokes of its text. Premultiplied input stays premultiplied. `None` if nothing +/// shrinks, the sizes don't fit or a buffer cannot be allocated. fn area_shrink(data: &[u8], source: (u32, u32), target: (u32, u32)) -> Option> { + if data.len() != source_byte_len(source.0, source.1, N, MAX_IMAGE_PIXELS)? { + return None; + } + let row_len = (source.0 as usize).checked_mul(N)?; + area_shrink_rows::(source, target, |s, row| { + let start = s.checked_mul(row_len)?; + row.copy_from_slice(data.get(start..start.checked_add(row_len)?)?); + Some(()) + }) +} + +/// PdfCraft patch: [`area_shrink`] of a colour image under its same-size soft mask (a stencil's +/// ink under the stencil, or an image under a mask sampled onto its grid), premultiplied a source +/// row at a time as it is averaged. The result equals premultiplying the expanded RGBA and +/// shrinking that, without building it: a 10718 × 13701 stencil shown at page size peaked at +/// 1.16 GB with the full-resolution RGBA, which is 587 MB. +fn area_shrink_premultiplied( + color: &ImageData, + alpha: &[u8], + source: (u32, u32), + target: (u32, u32), +) -> Option> { + let (samples, channels) = match color { + ImageData::Luma(luma) => (luma.data.as_slice(), 1), + ImageData::Rgb(rgb) => (rgb.data.as_slice(), 3), + }; + if samples.len() != source_byte_len(source.0, source.1, channels, MAX_IMAGE_PIXELS)? + || alpha.len() != source_byte_len(source.0, source.1, 1, MAX_IMAGE_PIXELS)? + { + return None; + } + let width = source.0 as usize; + area_shrink_rows::<4>(source, target, |s, row| { + let start = s.checked_mul(width)?; + let end = start.checked_add(width)?; + let colors = samples.get(start.checked_mul(channels)?..end.checked_mul(channels)?)?; + let alphas = alpha.get(start..end)?; + for ((out, px), a) in row.chunks_exact_mut(4).zip(colors.chunks_exact(channels)).zip(alphas) { + let [r, g, b] = match px { + [v] => [*v; 3], + [r, g, b] => [*r, *g, *b], + _ => return None, + }; + out.copy_from_slice(&AlphaColor::from_rgba8(r, g, b, *a).premultiply().to_rgba8().to_u8_array()); + } + Some(()) + }) +} + +/// PdfCraft patch: the area shrink behind [`area_shrink`], reading source row `s` through +/// `read_row(s, row)` into a buffer of one row. Each target row averages the source rows it +/// covers, and that row is shrunk across at once with taps worked out once per image, so work +/// memory is one source row of samples, one of `u32` accumulators, one shrunk row and the +/// horizontal taps (at most `sw + 2 × tw`): never the image. A side over +/// `MAX_RESAMPLING_SOURCE_SIDE` is refused, so none of it exceeds a few times one source row. +fn area_shrink_rows( + source: (u32, u32), + target: (u32, u32), + mut read_row: impl FnMut(usize, &mut [u8]) -> Option<()>, +) -> Option> { let ((sw, sh), (tw, th)) = (source, target); // A side over MAX_RESAMPLING_SOURCE_SIDE would make the accumulator row and the tap table // several times larger than the samples (a 2^27 × 2 image of zeros is a tiny file). - if tw == 0 || th == 0 || tw > sw || th > sh || sw > MAX_RESAMPLING_SOURCE_SIDE || sh > MAX_RESAMPLING_SOURCE_SIDE { - return None; - } - let (sw_us, tw_us) = (sw as usize, tw as usize); - if data.len() != source_byte_len(sw, sh, N, MAX_IMAGE_PIXELS)? { + if tw == 0 + || th == 0 + || tw > sw + || th > sh + || (tw, th) == (sw, sh) + || sw > MAX_RESAMPLING_SOURCE_SIDE + || sh > MAX_RESAMPLING_SOURCE_SIDE + { return None; } + let row_len = (sw as usize).checked_mul(N)?; + let zeroed = |len: usize| { + let mut v = image_buffer(len)?; + v.resize(len, 0); + Some(v) + }; + let (mut source_row, mut shrunk_row) = (zeroed(row_len)?, image_buffer(row_len)?); + let mut acc: Vec = Vec::new(); + acc.try_reserve_exact(row_len).ok()?; + acc.resize(row_len, 0); + // The taps depend only on the widths, so they are worked out once, not again for each row. + let taps = if tw == sw { None } else { Some(AreaAxis::new(sw, tw)?) }; + let out_len = match taps { + None => source_byte_len(tw, th, N, MAX_IMAGE_PIXELS)?, + Some(_) => image_byte_len(tw, th, N, MAX_IMAGE_PIXELS)?, + }; + let mut out = image_buffer(out_len)?; // Weights sum to AREA_ONE (2^16) and samples are at most 255, so every sum fits in a u32. let round = |acc: u32| ((acc + (AREA_ONE as u32) / 2) >> 16).min(255) as u8; - // Vertical pass first: groups of whole source rows averaged into each target row. Rows are - // contiguous, so this pass is cheap, and it leaves the horizontal pass, which gathers samples - // per target, a third as many rows when a scan shrinks threefold. - let tall = if th == sh { - None - } else { - let row_len = sw_us * N; - let mut rows = image_buffer(source_byte_len(sw, th, N, MAX_IMAGE_PIXELS)?)?; - let mut acc: Vec = Vec::new(); - acc.try_reserve_exact(row_len).ok()?; - acc.resize(row_len, 0); - for t in 0..th { + // The last source row read: neighbouring target rows share the row on their boundary. + let mut loaded = None; + for t in 0..th { + // Vertical: the whole source rows target row `t` covers, averaged. + let row: &[u8] = if th == sh { + read_row(t as usize, &mut source_row)?; + &source_row + } else { acc.fill(0); for (s, w) in area_taps(sh, th, t) { - let row = data.get(s * row_len..(s + 1) * row_len)?; + if loaded != Some(s) { + read_row(s, &mut source_row)?; + loaded = Some(s); + } // A weight is at most AREA_ONE, so it fits in a u32. let w = w as u32; - for (a, v) in acc.iter_mut().zip(row) { + for (a, v) in acc.iter_mut().zip(&source_row) { *a += w * u32::from(*v); } } - rows.extend(acc.iter().map(|a| round(*a))); - } - Some(rows) - }; - if tw == sw { - return tall; - } - let rows = tall.as_deref().unwrap_or(data); - // Horizontal pass: each of those rows shrunk to `tw` samples. The taps depend only on the - // widths, so they are worked out once, not again for each row. - let taps = AreaAxis::new(sw, tw)?; - let mut out = image_buffer(image_byte_len(tw, th, N, MAX_IMAGE_PIXELS)?)?; - for row in rows.chunks_exact(sw_us * N) { - for t in 0..tw_us { - let (first, weights) = taps.get(t)?; + shrunk_row.clear(); + shrunk_row.extend(acc.iter().map(|a| round(*a))); + &shrunk_row + }; + // Horizontal: that row shrunk to `tw` samples. + let Some(taps) = &taps else { + out.extend_from_slice(row); + continue; + }; + for x in 0..tw as usize { + let (first, weights) = taps.get(x)?; let samples = row.get(first * N..(first + weights.len()) * N)?; let mut acc = [0_u32; N]; for (w, px) in weights.iter().zip(samples.chunks_exact(N)) { @@ -1263,7 +1341,7 @@ impl Renderer { // nothing, and the stretched side would otherwise be drawn nearest-neighbour. if !interpolate && !self.in_type3_glyph - && (x_scale > 1.0 || y_scale > 1.0) + && (x_scale > SMOOTHED_IMAGE_SCALE || y_scale > SMOOTHED_IMAGE_SCALE) && source_width.min(source_height) >= SMOOTHED_IMAGE_SIDE { quality = ImageQuality::Medium; @@ -1295,9 +1373,12 @@ impl Renderer { }; let needs_resize = (new_width, new_height) != (source_width, source_height); - // PdfCraft patch (#624): a minified Type 3 glyph image with a soft mask that is not - // interpolated is nearest-sampled straight into premultiplied target RGBA. That skips the - // full-resolution RGBA copy below and the premultiply pass after it. + // PdfCraft patch (#624): a minified image with a soft mask that is neither interpolated nor + // area-averaged (a Type 3 glyph image, or a source wider than MAX_RESAMPLING_SOURCE_SIDE) + // is nearest-sampled straight into premultiplied target RGBA. That skips the + // full-resolution RGBA copy below and the premultiply pass after it. PdfCraft patch: an + // area-averaged one (a stencil's ink, or an image under its mask) skips them too, and is + // premultiplied a source row at a time as it is averaged (`area_shrink_premultiplied`). let direct = match &alpha_data { Some(alpha) if needs_resize && !interpolate && !area => nearest_premultiplied_rgba( &image_data, @@ -1305,6 +1386,12 @@ impl Renderer { (source_width, source_height), (new_width, new_height), ), + Some(alpha) if needs_resize && area => area_shrink_premultiplied( + &image_data, + &alpha.data, + (source_width, source_height), + (new_width, new_height), + ), _ => None, }; let mut premultiplied = direct.is_some(); @@ -2935,9 +3022,89 @@ mod snap_tests { #[cfg(test)] mod area_shrink_tests { use super::{ - AREA_ONE, AreaAxis, LumaData, MAX_IMAGE_PIXELS, area_shrink, area_taps, mask_on_image_grid, + AREA_ONE, AreaAxis, ImageData, LumaData, MAX_IMAGE_PIXELS, RgbData, area_shrink, + area_shrink_premultiplied, area_taps, mask_on_image_grid, premultiply_rgba, }; + /// Deterministic samples that vary from one to the next. + fn noise(len: usize, seed: u32) -> Vec { + let mut x = seed.wrapping_mul(2_654_435_761) | 1; + (0..len) + .map(|_| { + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + (x >> 24) as u8 + }) + .collect() + } + + /// The shrink as two whole passes, every row averaged down and then every row across, each + /// rounded to a byte: what `area_shrink` did before it worked a row at a time. + fn two_pass_shrink(data: &[u8], (sw, sh): (u32, u32), (tw, th): (u32, u32)) -> Vec { + let round = |acc: u64| ((acc + AREA_ONE / 2) >> 16).min(255) as u8; + let row_len = sw as usize * N; + let tall: Vec = (0..th) + .flat_map(|t| { + (0..row_len).map(move |i| { + round(area_taps(sh, th, t).map(|(s, w)| w * u64::from(data[s * row_len + i])).sum()) + }) + }) + .collect(); + tall.chunks_exact(row_len) + .flat_map(|row| { + (0..tw).flat_map(move |t| { + (0..N).map(move |c| round(area_taps(sw, tw, t).map(|(s, w)| w * u64::from(row[s * N + c])).sum())) + }) + }) + .collect() + } + + /// Working a row at a time gives the same bytes as two whole passes, in both directions, one + /// direction alone, and with source rows shared by neighbouring targets. + #[test] + fn a_row_at_a_time_shrink_equals_two_whole_passes() { + let sizes = [((7, 5), (3, 2)), ((10, 3), (6, 3)), ((4, 9), (4, 4)), ((13, 11), (1, 1)), ((31, 17), (12, 7))]; + for (i, (source, target)) in sizes.into_iter().enumerate() { + let gray = noise((source.0 * source.1) as usize, i as u32); + assert_eq!(area_shrink::<1>(&gray, source, target), Some(two_pass_shrink::<1>(&gray, source, target)), "{source:?} → {target:?}"); + let rgba = noise((source.0 * source.1 * 4) as usize, i as u32 + 99); + assert_eq!(area_shrink::<4>(&rgba, source, target), Some(two_pass_shrink::<4>(&rgba, source, target)), "{source:?} → {target:?}"); + } + } + + /// Premultiplying a row at a time as it is averaged gives the same bytes as premultiplying the + /// whole expanded RGBA first and shrinking that, for gray and RGB colour. + #[test] + fn premultiplying_row_by_row_equals_premultiplying_the_whole_image() { + let sizes = [((7, 5), (3, 2)), ((10, 3), (6, 3)), ((4, 9), (4, 4)), ((31, 17), (12, 7))]; + for (i, (source, target)) in sizes.into_iter().enumerate() { + let (w, h) = source; + let pixels = (w * h) as usize; + let alpha = noise(pixels, i as u32 + 7); + let gray = noise(pixels, i as u32 + 11); + let rgb = noise(pixels * 3, i as u32 + 13); + let colors = [ + ImageData::Luma(LumaData { data: gray.clone(), width: w, height: h, interpolate: false, scale_factors: (1.0, 1.0) }), + ImageData::Rgb(RgbData { data: rgb.clone(), width: w, height: h, interpolate: false, scale_factors: (1.0, 1.0) }), + ]; + for color in &colors { + let mut whole: Vec = match color { + ImageData::Luma(_) => gray.iter().zip(&alpha).flat_map(|(g, a)| [*g, *g, *g, *a]).collect(), + ImageData::Rgb(_) => rgb.chunks_exact(3).zip(&alpha).flat_map(|(c, a)| [c[0], c[1], c[2], *a]).collect(), + }; + premultiply_rgba(&mut whole); + assert_eq!( + area_shrink_premultiplied(color, &alpha, source, target), + area_shrink::<4>(&whole, source, target), + "{source:?} → {target:?}" + ); + } + let short = alpha.get(1..).unwrap_or_default(); + assert_eq!(area_shrink_premultiplied(&colors[0], short, source, target), None, "alpha shorter than the image"); + } + } + fn mask(data: &[u8], width: u32, height: u32) -> LumaData { LumaData { data: data.to_vec(), width, height, interpolate: false, scale_factors: (1.0, 1.0) } } From 7971fb73f2794cd9089838dd07365cfa9f41ca5a Mon Sep 17 00:00:00 2001 From: KenM76 Date: Wed, 4 Nov 2026 11:04:00 -0500 Subject: [PATCH 057/198] fix(cos): a clone's edited objects don't use up its unreadable-object allowance The unreadable list is shared by clones, and each one hides the entries it has edited. The 100-entry cap counted the whole shared list, so a clone that edited every listed object recorded none of its own failures and listed only "more objects could not be read". The cap now counts the entries the asking document still shows. Each document still lists at most 100 entries, then the "more" line. Test: objects_a_clone_edited_do_not_use_up_its_unreadable_allowance. It was seen failing with each half reverted (left: 1, right: 10; then left: 110, right: 101). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01SqmHYx8FiWM5PcsoiErgp9 --- crates/cos/src/document/objects.rs | 5 +++-- crates/cos/src/document/state.rs | 30 ++++++++++++++++++------------ crates/cos/src/document/tests.rs | 26 ++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 14 deletions(-) diff --git a/crates/cos/src/document/objects.rs b/crates/cos/src/document/objects.rs index f85b7b9191..6cf1df9b1e 100644 --- a/crates/cos/src/document/objects.rs +++ b/crates/cos/src/document/objects.rs @@ -103,8 +103,9 @@ impl Document { fn note_unreadable(&self, num: u32, reason: impl FnOnce() -> String) { // A poisoned lock only means the object goes unlisted; the read itself still fails. let Ok(mut unreadable) = self.unreadable.lock() else { return }; - if unreadable.wants(num) { - unreadable.note(num, self.generation(num), &reason()); + let hidden = |n: u32| self.overlay.contains_key(&n); + if unreadable.wants(num, hidden) { + unreadable.note(num, self.generation(num), &reason(), hidden); } } diff --git a/crates/cos/src/document/state.rs b/crates/cos/src/document/state.rs index a6171479ca..3627280960 100644 --- a/crates/cos/src/document/state.rs +++ b/crates/cos/src/document/state.rs @@ -62,7 +62,9 @@ pub(super) fn objstm_member(num: i64, offset: i64, first: Option, len: us } /// Objects the reader could not read, each with the reason (see `Document::unreadable_objects`). -/// Bounded: past [`Unreadable::MAX`] objects only the fact that there were more is kept. +/// Bounded: a document records past [`Unreadable::MAX`] of the objects it shows only the fact that +/// there were more. Entries a clone has edited are hidden from it and do not count, so the whole +/// list holds at most `MAX` plus the objects edited in clones. #[derive(Debug, Default)] pub(super) struct Unreadable { pub(super) objects: BTreeMap, @@ -74,13 +76,15 @@ impl Unreadable { /// Characters of a reason kept: a parser's message can quote the bytes it stopped at. const REASON: usize = 160; - /// Whether `num` is to be recorded: not listed yet and there is room. A full list notes that - /// there were more, so a caller asks this before building the reason. - pub(super) fn wants(&mut self, num: u32) -> bool { + /// Whether `num` is to be recorded: not listed yet and there is room among the entries the + /// asking document still shows (`hidden` are the ones it has edited, as in [`Self::lines`]), + /// so a clone's edits never use up its own allowance. A full list notes that there were more, + /// so a caller asks this before building the reason. + pub(super) fn wants(&mut self, num: u32, hidden: impl Fn(u32) -> bool) -> bool { if self.objects.contains_key(&num) { return false; } - if self.objects.len() >= Self::MAX { + if self.objects.keys().filter(|n| !hidden(**n)).count() >= Self::MAX { self.more = true; return false; } @@ -88,21 +92,23 @@ impl Unreadable { } /// Record `num` once; the first reason given is kept. - pub(super) fn note(&mut self, num: u32, generation: u16, reason: &str) { - if self.wants(num) { + pub(super) fn note(&mut self, num: u32, generation: u16, reason: &str, hidden: impl Fn(u32) -> bool) { + if self.wants(num, hidden) { self.objects.insert(num, (generation, reason.chars().take(Self::REASON).collect())); } } /// One line per recorded object not `hidden`, then one saying there were more, if there were. pub(super) fn lines(&self, hidden: impl Fn(u32) -> bool) -> Vec { - let mut lines: Vec = self - .objects - .iter() - .filter(|(num, _)| !hidden(**num)) + // Clones share the list and each hides its own edits, so the list may hold more than + // `MAX` entries in all; each document still shows at most `MAX`. + let mut shown = self.objects.iter().filter(|(num, _)| !hidden(**num)); + let mut lines: Vec = shown + .by_ref() + .take(Self::MAX) .map(|(num, (generation, reason))| format!("object {num} {generation} could not be read: {reason}")) .collect(); - if self.more { + if self.more || shown.next().is_some() { lines.push(format!("more objects could not be read; only the first {} are listed", Self::MAX)); } lines diff --git a/crates/cos/src/document/tests.rs b/crates/cos/src/document/tests.rs index 2f40b25b07..4dbc99bb4e 100644 --- a/crates/cos/src/document/tests.rs +++ b/crates/cos/src/document/tests.rs @@ -1919,6 +1919,32 @@ fn the_unreadable_object_list_is_bounded() { ); } +/// The unreadable list is shared by clones, and each hides what it has edited. A clone that edited +/// every listed object still lists its own failures: hidden entries don't use up its allowance. +#[test] +fn objects_a_clone_edited_do_not_use_up_its_unreadable_allowance() { + let mut bodies = vec!["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>"]; + bodies.extend(std::iter::repeat_n("<< /A ) >>", 110)); + let doc = Document::open(Arc::new(build(&bodies, "/Root 1 0 R"))).unwrap(); + for num in 3..103 { + assert!(doc.try_get(num).is_err()); + } + assert_eq!(doc.unreadable_objects().len(), 100); + let mut edited = doc.clone(); + for num in 3..103 { + edited.set(ObjRef::new(num, 0), Object::Null); + } + for num in 103..113 { + assert!(edited.try_get(num).is_err()); + } + let lines = edited.unreadable_objects(); + assert_eq!(lines.len(), 10, "{lines:?}"); + assert!(lines[0].starts_with("object 103 0 could not be read"), "{}", lines[0]); + let original = doc.unreadable_objects(); + assert_eq!(original.len(), 101, "the original still shows at most 100, then says there were more"); + assert_eq!(original[100], "more objects could not be read; only the first 100 are listed"); +} + /// The cache keeps the sum of what its objects hold as they come and go, so neither reader has /// to measure the cache again on every read (#5873). #[test] From 352355ab77163db6c7bfecdac0f0ca96a5f6aacf Mon Sep 17 00:00:00 2001 From: KenM76 Date: Wed, 4 Nov 2026 11:58:06 -0500 Subject: [PATCH 058/198] M10: copy keeps formatting as HTML (Copy with formatting) Copying selected page text now also puts an HTML fragment on the system clipboard, so a paste into a word processor or mail client keeps paragraphs, bold, italic, size, colour and font family. The plain text is unchanged and stays on the clipboard as the alternative. - export: clip.rs, the clipboard model (ClipText of paragraphs of styled spans), its builder with caps (20 000 spans, 2 MiB of text, 16 MiB of HTML), font_family (subset tag and style suffixes off, camel case spaced) and clip_html. Family names are reduced to letters, digits, space, '-', '_' and '.', so a font name cannot leave the CSS string; text goes through esc, which drops control characters. - engine: Document::rich_copy(page, text, glyphs). Glyphs (view space) take the style of the reading block their centre lies in; blocks are style-homogeneous. A line wrap inside a block is a space (none after a hyphen), a change of block a new paragraph. Refused when the document does not allow copying. compare.rs's display unit and weight/slant inference are shared, not duplicated. - automation: text_extract takes format "text" (default) or "html"; html adds each page's html and styles_incomplete, for the whole page or only what rect takes. Other values are InvalidArgs. - ui-egui: Copy of a run or column selection writes HTML plus the plain text in one arboard set (desktop only); the web build and Select All of the whole document stay plain text. last_copied_html records what was produced, for tests. - parity: view.copy-with-layout is partial; notes list what is missing (RTF, underline/strike-through, tables, Select All, web, glyphs outside reading blocks). Formatting inference and the HTML serialiser are adapted from pdfcer (MIT, same author) at 540b00ef1a405d1baa67c92f3858535037c7cde8. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01SqmHYx8FiWM5PcsoiErgp9 --- crates/automation/src/pages.rs | 22 +- crates/automation/src/tools.rs | 5 +- crates/automation/tests/it/automation.rs | 30 +++ crates/engine/src/compare.rs | 24 +- crates/engine/src/lib.rs | 2 + crates/engine/src/rich_copy.rs | 220 ++++++++++++++++ crates/export/src/clip.rs | 322 +++++++++++++++++++++++ crates/export/src/lib.rs | 2 + crates/ui-egui/src/app.rs | 1 + crates/ui-egui/src/app/clipboard.rs | 23 +- crates/ui-egui/src/canvas/selection.rs | 15 ++ crates/ui-egui/src/lib.rs | 3 + crates/ui-egui/tests/it/text_select.rs | 56 +++- parity/acrobat-features.toml | 10 +- 14 files changed, 718 insertions(+), 17 deletions(-) create mode 100644 crates/engine/src/rich_copy.rs create mode 100644 crates/export/src/clip.rs diff --git a/crates/automation/src/pages.rs b/crates/automation/src/pages.rs index 8c793be153..ad6a819027 100644 --- a/crates/automation/src/pages.rs +++ b/crates/automation/src/pages.rs @@ -201,6 +201,11 @@ impl Automation { } pub(crate) fn text_extract(&mut self, a: &Args) -> Result { + let html = match a.opt_str("format")?.unwrap_or("text") { + "text" => false, + "html" => true, + other => return Err(ToolError::InvalidArgs(format!("format must be \"text\" or \"html\", not {other:?}"))), + }; let doc = self.doc_to_copy(a)?; let id = doc.id; let pages = match a.opt_ints("pages")? { @@ -214,7 +219,22 @@ impl Automation { Some(r) => t.column_text(&t.glyphs_in(r)), None => t.plain_text(), }; - let out: Vec = pages.iter().zip(texts).map(|(p, t)| json!({ "page": p + 1, "text": text(&t) })).collect(); + let doc = self.doc(a)?; + let mut out = Vec::with_capacity(pages.len()); + for (&p, t) in pages.iter().zip(texts) { + let mut page = json!({ "page": p + 1, "text": text(&t) }); + if html { + // Copy with formatting: the same glyphs as an HTML fragment. + let glyphs: Vec = match rect { + Some(r) => t.glyphs_in(r), + None => (0..t.glyphs.len()).collect(), + }; + let copy = doc.rich_copy(p, &t, &glyphs).map_err(failed)?; + page["html"] = json!(copy.html); + page["styles_incomplete"] = json!(copy.styles_incomplete); + } + out.push(page); + } Ok(json!({ "pages": out })) } diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index 1b4b6ac4ba..dae824cf5b 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -195,13 +195,14 @@ pub fn tools() -> Vec { .with(schema(json!({ "doc": doc(), "pages": pages("to count (default: all)"), "exclude_tables": { "type": "boolean" }, "exclude_code": { "type": "boolean" }, "exclude_images": { "type": "boolean" } }), &["doc"])), - t("text_extract", "Extract text", "Extract the text of some or all pages, in reading order. With rect, only the text inside that rectangle on each page, as Column select takes it: one row per visual line, side-by-side pieces (table cells) separated by a tab.") + t("text_extract", "Extract text", "Extract the text of some or all pages, in reading order. With rect, only the text inside that rectangle on each page, as Column select takes it: one row per visual line, side-by-side pieces (table cells) separated by a tab. With format \"html\", each page also carries html, the same text as Copy with formatting puts on the clipboard (paragraphs, bold, italic, size, colour and font family), and styles_incomplete when some styles had to be left out.") .ro() .cmd("edit.column_select") .with(schema( json!({ "doc": doc(), "pages": pages("to extract (default: all)"), - "rect": { "type": "array", "items": { "type": "number" }, "minItems": 4, "maxItems": 4, "description": "Column select: [x0, y0, x1, y1] in points from the top-left of the displayed page; a glyph counts when its centre is inside." } + "rect": { "type": "array", "items": { "type": "number" }, "minItems": 4, "maxItems": 4, "description": "Column select: [x0, y0, x1, y1] in points from the top-left of the displayed page; a glyph counts when its centre is inside." }, + "format": { "type": "string", "enum": ["text", "html"], "description": "text (default) or html: also return each page's text as formatted HTML." } }), &["doc"], )), diff --git a/crates/automation/tests/it/automation.rs b/crates/automation/tests/it/automation.rs index 13f9af4371..72abdf71a5 100644 --- a/crates/automation/tests/it/automation.rs +++ b/crates/automation/tests/it/automation.rs @@ -2420,6 +2420,7 @@ fn protecting_through_tools() { // Copying wasn't allowed either (#568): no text or images come out, though search works. for (tool, args) in [ ("text_extract", json!({ "doc": re })), + ("text_extract", json!({ "doc": re, "format": "html" })), ("text_word_count", json!({ "doc": re })), ("text_paragraphs", json!({ "doc": re, "page": 1 })), ("doc_export_text", json!({ "doc": re, "path": "out.txt" })), @@ -8978,6 +8979,35 @@ fn text_extract_with_a_rect_takes_one_column() { } } +/// Copy with formatting, headless: `format: "html"` adds each page's text as the HTML fragment +/// Copy puts on the clipboard, for the whole page or only what `rect` takes. +#[test] +fn text_extract_as_html_keeps_the_formatting() { + let dir = workdir("rich-copy"); + std::fs::write(dir.join("table.pdf"), table_pdf()).unwrap(); + let mut a = auto(&dir); + let doc = ok(&mut a, "doc_open", json!({ "path": "table.pdf" }))["doc"].as_u64().unwrap(); + let page = &ok(&mut a, "text_extract", json!({ "doc": doc, "pages": [1], "format": "html" }))["pages"][0]; + let html = page["html"].as_str().unwrap(); + assert!(html.starts_with("

Name"), "{html}"); + assert!(html.contains("Pear") && html.contains("Qty"), "{html}"); + assert_eq!(page["text"], "Name\nApple\nPear\nQty\n12\n7", "the plain text is unchanged"); + assert_eq!(page["styles_incomplete"], false); + // Plain text by default: no html key. + assert!(ok(&mut a, "text_extract", json!({ "doc": doc, "pages": [1] }))["pages"][0].get("html").is_none()); + // Only what the rect takes; nothing inside gives empty HTML. + let column = ok(&mut a, "text_extract", json!({ "doc": doc, "pages": [1], "rect": [120, 0, 300, 300], "format": "html" })); + let html = column["pages"][0]["html"].as_str().unwrap(); + assert!(html.contains("Qty") && !html.contains("Name"), "{html}"); + let none = ok(&mut a, "text_extract", json!({ "doc": doc, "pages": [1], "rect": [0, 0, 5, 5], "format": "html" })); + assert_eq!(none["pages"][0]["html"], ""); + // An unknown format is refused. + for bad in [json!("rtf"), json!(1)] { + let e = a.call("text_extract", &json!({ "doc": doc, "format": bad })).unwrap_err(); + assert!(matches!(e, ToolError::InvalidArgs(_)), "{bad}: {e}"); + } +} + #[test] fn mixed_visual_region_edit_preview_save_reopen_and_undo_through_tools() { use pdfcraft_cos::{Dict, Document, Object, Stream, write_incremental}; diff --git a/crates/engine/src/compare.rs b/crates/engine/src/compare.rs index 9be1954b09..fe5af55706 100644 --- a/crates/engine/src/compare.rs +++ b/crates/engine/src/compare.rs @@ -187,6 +187,20 @@ pub struct OfficeExport { /// `Err(false)`: it can't be encoded). type PreparedImage = Result<(&'static str, Arc<[u8]>), bool>; +/// Displayed points per user-space unit on `info`'s page (`/UserUnit`), 1 when the page has no +/// usable size. +pub(crate) fn display_unit(info: &pdfcraft_render::PageInfo) -> f64 { + let crop_width = f64::from(info.crop[2] - info.crop[0]); + let display_width = f64::from(if info.rotation.is_multiple_of(180) { info.width } else { info.height }); + if crop_width > 0.0 && crop_width.is_finite() && display_width.is_finite() { display_width / crop_width } else { 1.0 } +} + +/// Bold and italic as a font's name says them (`Arial-BoldItalicMT`). +pub(crate) fn weight_and_slant(base_font: &str) -> (bool, bool) { + let f = base_font.to_ascii_lowercase(); + (f.contains("bold") || f.contains("black") || f.contains("heavy"), f.contains("italic") || f.contains("oblique")) +} + impl crate::Document { /// The pages as paragraphs and images (for Word, HTML and RTF export), including what /// form XObjects draw. @@ -228,9 +242,7 @@ impl crate::Document { let b = info.user_to_view(r[2] as f32, r[3] as f32); [a[0].min(b[0]) as f64, (info.height - a[1].max(b[1])) as f64, a[0].max(b[0]) as f64, (info.height - a[1].min(b[1])) as f64] }; - let crop_width = f64::from(info.crop[2] - info.crop[0]); - let display_width = f64::from(if info.rotation % 180 == 0 { info.width } else { info.height }); - let unit = if crop_width > 0.0 && crop_width.is_finite() && display_width.is_finite() { display_width / crop_width } else { 1.0 }; + let unit = display_unit(info); let reading = pdfcraft_edit::reading_blocks_checked(cos, i).unwrap_or_default(); let forms_unread = reading.truncated; let blocks = reading @@ -238,7 +250,7 @@ impl crate::Document { .into_iter() .filter(|b| !b.text.trim().is_empty()) .map(|b| { - let f = b.base_font.to_ascii_lowercase(); + let (bold, italic) = weight_and_slant(&b.base_font); // `text` is trimmed; the editor text keeps the source's own spaces. let space_around = [b.editor_text.starts_with(char::is_whitespace), b.editor_text.ends_with(char::is_whitespace)]; pdfcraft_export::Block { @@ -246,8 +258,8 @@ impl crate::Document { text: b.text, rect: visible_rect(b.rect), size: b.size * unit, - bold: f.contains("bold") || f.contains("black") || f.contains("heavy"), - italic: f.contains("italic") || f.contains("oblique"), + bold, + italic, color: b.color, rotation: b.rotation - f64::from(info.rotation), } diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index fe139164be..bf142bd037 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -43,6 +43,7 @@ mod pictures; pub mod print_raster; pub mod printcheck; mod protection; +mod rich_copy; mod scan_colours; mod scan_fields; mod session; @@ -66,6 +67,7 @@ pub use links::{ MAX_REMOTE_DOCUMENT_BYTES, RemoteDocumentPathError, read_remote_document, remote_document_path_is_absolute, resolve_remote_document_path, }; pub use pdfcraft_edit::strip_subset_tag; +pub use rich_copy::RichCopy; pub use signature_image::{ImageSignaturePreview, SignatureImage}; pub use signature_scan::{PhotoRect, ScanError, SignaturePhoto, Turn}; pub use stamp_preview::{CommentLayer, StampPreview}; diff --git a/crates/engine/src/rich_copy.rs b/crates/engine/src/rich_copy.rs new file mode 100644 index 0000000000..99c2938350 --- /dev/null +++ b/crates/engine/src/rich_copy.rs @@ -0,0 +1,220 @@ +//! Copy with formatting: the selected glyphs of a page as paragraphs of styled spans and an +//! HTML fragment ([`pdfcraft_export::clip_html`]) for the clipboard, next to the plain text. +//! +//! Glyphs come from the text layer (view space); style comes from the page's reading blocks, +//! which never mix styles, mapped into view space. A glyph takes the style of the block its +//! centre lies in; a glyph in no block is copied unstyled. + +use pdfcraft_export::{ClipBuilder, ClipStyle, ClipText, clip_html}; +use pdfcraft_render::{PageInfo, PageText}; + +use crate::EditError; +use crate::compare::{display_unit, weight_and_slant}; + +/// Block tests spent matching glyphs to blocks for one copy; past it the rest is unstyled. +const MAX_MATCH_STEPS: usize = 20_000_000; +/// How far outside a block's box (points) a glyph centre still belongs to it. +const SLACK: f32 = 0.5; + +/// Selected text with its formatting. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct RichCopy { + /// The paragraphs and spans. + pub text: ClipText, + /// The HTML fragment for the clipboard (empty when nothing was selected). + pub html: String, + /// Styles were left out: reading stopped at its limits on this page, or matching glyphs to + /// blocks ran out of steps. + pub styles_incomplete: bool, +} + +struct StyledBlock { + rect: [f32; 4], + style: ClipStyle, +} + +impl crate::Document { + /// Selected `glyphs` of `page` (indices into `text`, in copy order) with their formatting. + /// Refused when the document does not allow copying. + pub fn rich_copy(&self, page: usize, text: &PageText, glyphs: &[usize]) -> Result { + self.check_copying()?; + let (blocks, mut incomplete) = self.styled_blocks(page); + let mut b = ClipBuilder::new(); + let mut steps = MAX_MATCH_STEPS; + // The previous glyph's line and block, and whether it was a hyphen. + let mut prev: Option<(usize, Option, bool)> = None; + for &i in glyphs { + let (Some(g), Some(&line)) = (text.glyphs.get(i), text.line_of.get(i)) else { continue }; + let found = block_at(&blocks, g.rect, prev.and_then(|(_, b, _)| b), &mut steps); + incomplete |= steps == 0; + let line = usize::try_from(line).unwrap_or(usize::MAX); + if let Some((prev_line, prev_block, hyphen)) = prev { + if line != prev_line { + if found == prev_block && found.is_some() { + // A paragraph wrapping: one space, none after a hyphen. + if !hyphen { + b.space(); + } + } else if found.is_none() && prev_block.is_none() { + b.line_break(); + } else { + b.paragraph(); + } + } else if text.space_before.get(i).copied().unwrap_or(false) { + b.space(); + } + } + let style = found.and_then(|k| blocks.get(k)).map(|s| &s.style); + if !b.push(&g.text, style.unwrap_or(&ClipStyle::default())) { + break; + } + prev = Some((line, found, g.text.ends_with('-'))); + } + let text = b.finish(); + Ok(RichCopy { html: clip_html(&text), text, styles_incomplete: incomplete }) + } + + /// `page`'s reading blocks in view space with their style, and whether reading stopped at + /// its limits. + fn styled_blocks(&self, page: usize) -> (Vec, bool) { + let (Some(editor), Some(info)) = (self.editor.as_ref(), self.info.pages.get(page)) else { return (Vec::new(), false) }; + let Ok(reading) = pdfcraft_edit::reading_blocks_checked(&editor.cos, page) else { return (Vec::new(), true) }; + let unit = display_unit(info); + let blocks = reading + .blocks + .into_iter() + .filter(|b| !b.text.trim().is_empty()) + .map(|b| { + let (bold, italic) = weight_and_slant(&b.base_font); + let style = ClipStyle { + family: pdfcraft_export::font_family(&b.base_font), + size: b.size * unit, + bold: b.bold || bold, + italic: b.italic || italic, + color: b.color, + }; + StyledBlock { rect: view_rect(info, b.rect), style } + }) + .collect(); + (blocks, reading.truncated) + } +} + +/// A user-space box as a view-space box (points from the displayed page's top-left, y down). +fn view_rect(info: &PageInfo, r: [f64; 4]) -> [f32; 4] { + let a = info.user_to_view(r[0] as f32, r[1] as f32); + let b = info.user_to_view(r[2] as f32, r[3] as f32); + [a[0].min(b[0]), a[1].min(b[1]), a[0].max(b[0]), a[1].max(b[1])] +} + +/// The block whose box holds the centre of `glyph` (view space): `hint` first, then the +/// smallest that holds it. Each block tested costs a step; with none left, no block. +fn block_at(blocks: &[StyledBlock], glyph: [f32; 4], hint: Option, steps: &mut usize) -> Option { + let (x, y) = ((glyph[0] + glyph[2]) / 2.0, (glyph[1] + glyph[3]) / 2.0); + let holds = |r: &[f32; 4]| x >= r[0] - SLACK && x <= r[2] + SLACK && y >= r[1] - SLACK && y <= r[3] + SLACK; + let area = |r: &[f32; 4]| (r[2] - r[0]) * (r[3] - r[1]); + if let Some(k) = hint + && *steps > 0 + { + *steps -= 1; + if blocks.get(k).is_some_and(|b| holds(&b.rect)) { + return Some(k); + } + } + let mut best: Option<(usize, f32)> = None; + for (k, b) in blocks.iter().enumerate() { + if *steps == 0 { + return None; + } + *steps -= 1; + if holds(&b.rect) && best.is_none_or(|(_, a)| area(&b.rect) < a) { + best = Some((k, area(&b.rect))); + } + } + best.map(|(k, _)| k) +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use crate::Session; + + /// A heading in bold, a body paragraph of two lines with HTML-special characters, and a red + /// word on the body's first line. + fn styled_page() -> Vec { + let content = b"BT /F1 24 Tf 20 250 Td (Title) Tj ET \ + BT /F2 12 Tf 20 200 Td (Body text) Tj ET \ + BT 1 0 0 rg /F2 12 Tf 90 200 Td (red) Tj ET \ + BT /F2 12 Tf 20 186 Td (a>") + .obj("<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 300 300] >>") + .obj("<< /Type /Page /Parent 2 0 R /Contents 4 0 R /Resources << /Font << /F1 5 0 R /F2 6 0 R >> >> >>") + .stream("", content) + .obj("<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica-Bold >>") + .obj("<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>") + .build() + } + + fn page_text(s: &Session, id: crate::DocId) -> Arc { + let doc = s.get(id).unwrap(); + let mut r = pdfcraft_render::PageRenderer::new(doc.bytes.clone(), Default::default()); + let request = pdfcraft_render::RenderRequest { page: 0, kind: pdfcraft_render::RequestKind::Text, scale: 1.0, ..Default::default() }; + r.render(request).text.unwrap() + } + + #[test] + fn copy_keeps_headings_body_and_colour_as_html() { + let mut s = Session::new(); + let id = s.open("styled.pdf", None, Arc::new(styled_page()), None).unwrap(); + let text = page_text(&s, id); + let all: Vec = (0..text.glyphs.len()).collect(); + let copy = s.get(id).unwrap().rich_copy(0, &text, &all).unwrap(); + let html = ©.html; + assert!(!copy.styles_incomplete); + assert!( + html.starts_with("

Title

\n"), + "{html}" + ); + assert!(html.contains("font-size:12pt\">Body text "), "{html}"); + assert!(html.contains("font-size:12pt;color:#FF0000\">red"), "{html}"); + assert!(html.contains("a<b & c"), "{html}"); + assert!(!html.contains("= 2, "{html}"); + // The heading and the body are separate paragraphs; the red word shares the body's. + let body = copy.text.paragraphs.iter().find(|p| p.iter().any(|s| s.text.contains("Body"))).unwrap(); + assert!(body.iter().any(|s| s.text == "red" && s.style.color == [1.0, 0.0, 0.0]), "{body:?}"); + + // A part of the selection: only those glyphs. + let part = s.get(id).unwrap().rich_copy(0, &text, &all[..5]).unwrap(); + assert_eq!(part.text.paragraphs.len(), 1); + assert!(part.html.contains("Title"), "{}", part.html); + // Nothing selected, and indices past the text: an empty copy, not a panic. + let none = s.get(id).unwrap().rich_copy(0, &text, &[]).unwrap(); + assert!(none.html.is_empty() && none.text.paragraphs.is_empty()); + let past = s.get(id).unwrap().rich_copy(7, &text, &[usize::MAX, text.glyphs.len()]).unwrap(); + assert!(past.html.is_empty()); + } + + #[test] + fn copy_with_formatting_is_refused_when_copying_is_withheld() { + let mut cos = pdfcraft_cos::Document::open(Arc::new(styled_page())).unwrap(); + cos.set_encryption(&pdfcraft_cos::NewEncryption { + algorithm: pdfcraft_cos::Algorithm::Aes256, + user_password: "", + owner_password: "owner", + permissions: 0b0100, // print only + encrypt_metadata: true, + seed: [7; 32], + }) + .unwrap(); + let bytes = Arc::new(pdfcraft_cos::write_full(&cos, &Default::default()).unwrap()); + let mut s = Session::new(); + let id = s.open("locked.pdf", None, bytes, None).unwrap(); + let text = pdfcraft_render::PageText { glyphs: Vec::new(), line_of: Vec::new(), space_before: Vec::new() }; + let err = s.get(id).unwrap().rich_copy(0, &text, &[]).unwrap_err(); + assert!(matches!(err, crate::EditError::NotPermitted(_)), "{err:?}"); + } +} diff --git a/crates/export/src/clip.rs b/crates/export/src/clip.rs new file mode 100644 index 0000000000..0b214c2bcd --- /dev/null +++ b/crates/export/src/clip.rs @@ -0,0 +1,322 @@ +//! Copy with formatting: selected text as paragraphs of styled spans, written as an HTML +//! fragment for the clipboard (inline styles, so a word processor keeps font, size, weight, +//! slant and colour when it pastes). + +use crate::{esc, hex}; + +/// Spans kept for one copy; text past it is left out and [`ClipText::truncated`] is set. +pub const MAX_CLIP_SPANS: usize = 20_000; +/// Text bytes kept for one copy (before escaping). +pub const MAX_CLIP_TEXT_BYTES: usize = 2 << 20; +/// The HTML written for one copy never passes this many bytes. +pub const MAX_CLIP_HTML_BYTES: usize = 16 << 20; +/// A family name longer than this is not a family name. +const MAX_FAMILY_CHARS: usize = 64; + +/// How a span looks. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct ClipStyle { + /// The family a word processor knows ([`font_family`]); `None` keeps the target's own. + pub family: Option, + /// Size in points; zero or not finite leaves the target's size. + pub size: f64, + pub bold: bool, + pub italic: bool, + /// Fill colour as RGB in 0..=1. + pub color: [f64; 3], +} + +/// A run of text in one style. +#[derive(Clone, Debug, PartialEq)] +pub struct ClipSpan { + pub text: String, + pub style: ClipStyle, +} + +/// The copied text: paragraphs of spans, and whether a cap left some out. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct ClipText { + pub paragraphs: Vec>, + pub truncated: bool, +} + +/// Builds a [`ClipText`] piece by piece, merging pieces of equal style into one span and +/// stopping at the caps. +#[derive(Debug, Default)] +pub struct ClipBuilder { + out: ClipText, + /// The separator owed before the next piece in this paragraph (a space or a line break). + pending: Option, + new_paragraph: bool, + spans: usize, + bytes: usize, +} + +impl ClipBuilder { + pub fn new() -> Self { + Self { new_paragraph: true, ..Self::default() } + } + + /// A word space before the next piece (dropped at the start of a paragraph). + pub fn space(&mut self) { + self.pending.get_or_insert(' '); + } + + /// A line break inside the paragraph before the next piece. + pub fn line_break(&mut self) { + self.pending = Some('\n'); + } + + /// The next piece starts a new paragraph. + pub fn paragraph(&mut self) { + self.new_paragraph = true; + self.pending = None; + } + + /// Append `text` in `style`. Returns `false` once a cap is reached; nothing more is kept. + pub fn push(&mut self, text: &str, style: &ClipStyle) -> bool { + if self.out.truncated { + return false; + } + if text.is_empty() { + return true; + } + let sep = if self.new_paragraph { None } else { self.pending }; + let need = text.len().saturating_add(sep.map_or(0, char::len_utf8)); + let Some(bytes) = self.bytes.checked_add(need).filter(|b| *b <= MAX_CLIP_TEXT_BYTES) else { + self.out.truncated = true; + return false; + }; + if self.new_paragraph { + self.out.paragraphs.push(Vec::new()); + self.new_paragraph = false; + } + self.pending = None; + let Some(spans) = self.out.paragraphs.last_mut() else { return false }; + match spans.last_mut() { + Some(last) if last.style == *style => { + last.text.extend(sep); + last.text.push_str(text); + } + last => { + if self.spans >= MAX_CLIP_SPANS { + self.out.truncated = true; + return false; + } + // A separator belongs to the span before it, so a new style starts at its text. + if let (Some(last), Some(sep)) = (last, sep) { + last.text.push(sep); + } + spans.push(ClipSpan { text: text.to_owned(), style: style.clone() }); + self.spans += 1; + } + } + self.bytes = bytes; + true + } + + pub fn finish(self) -> ClipText { + self.out + } +} + +/// The family a word processor knows, from a font's `/BaseFont`: subset tag and style suffix +/// removed, `PSMT`/`MT`/`PS` dropped, camel case spaced (`ABCDEF+TimesNewRomanPSMT` → +/// `Times New Roman`). Only letters, digits, spaces, `-`, `_` and `.` are kept, so the name +/// cannot leave its CSS string. `None` when nothing is left. +pub fn font_family(base_font: &str) -> Option { + let name = match base_font.split_once('+') { + Some((tag, rest)) if tag.len() == 6 && tag.bytes().all(|b| b.is_ascii_uppercase()) => rest, + _ => base_font, + }; + let family = name.split_once([',', '-']).map_or(name, |(family, _)| family); + let family = family.strip_suffix("PSMT").or_else(|| family.strip_suffix("MT")).or_else(|| family.strip_suffix("PS")).unwrap_or(family); + let mut spaced = String::with_capacity(family.len().min(MAX_FAMILY_CHARS * 2)); + let mut prev_lower = false; + for c in family.chars().take(MAX_FAMILY_CHARS) { + if prev_lower && c.is_ascii_uppercase() { + spaced.push(' '); + } + prev_lower = c.is_ascii_lowercase(); + spaced.push(c); + } + safe_family(&spaced) +} + +/// `family` with only letters, digits, spaces, `-`, `_` and `.`, so it cannot leave a CSS +/// string; `None` when nothing is left. +fn safe_family(family: &str) -> Option { + let kept: String = family.chars().filter(|c| c.is_alphanumeric() || matches!(c, ' ' | '-' | '_' | '.')).take(MAX_FAMILY_CHARS * 2).collect(); + let kept = kept.trim(); + (!kept.is_empty()).then(|| kept.to_owned()) +} + +/// The copied text as an HTML fragment: one `

` per paragraph, a `` per run with its +/// family, size and colour inline, bold as `` and italic as `` (so a target that +/// drops style attributes still keeps them). Text is escaped; a line break is `
` and a run +/// of spaces keeps its width. Never longer than [`MAX_CLIP_HTML_BYTES`]: a span that would pass +/// it ends the fragment. +pub fn clip_html(text: &ClipText) -> String { + const OPEN: &str = "

"; + const CLOSE: &str = "

\n"; + let mut out = String::new(); + for spans in &text.paragraphs { + let mut p = String::from(OPEN); + for span in spans { + let s = span_html(span); + if out.len().saturating_add(p.len()).saturating_add(s.len()).saturating_add(CLOSE.len()) > MAX_CLIP_HTML_BYTES { + if p.len() > OPEN.len() { + out.push_str(&p); + out.push_str(CLOSE); + } + return out; + } + p.push_str(&s); + } + p.push_str(CLOSE); + out.push_str(&p); + } + out +} + +fn span_html(span: &ClipSpan) -> String { + let st = &span.style; + let mut css = Vec::new(); + if let Some(f) = st.family.as_deref().and_then(safe_family) { + css.push(format!("font-family:'{f}'")); + } + if st.size.is_finite() && st.size > 0.0 { + css.push(format!("font-size:{}pt", num(st.size))); + } + if let Some(h) = hex(st.color) { + css.push(format!("color:#{h}")); + } + let mut body = text_html(&span.text); + if st.italic { + body = format!("{body}"); + } + if st.bold { + body = format!("{body}"); + } + if css.is_empty() { body } else { format!("{body}", esc(&css.join(";"))) } +} + +/// Escape for HTML text (control characters dropped by [`esc`]); `\n` becomes `
` and a +/// run of spaces keeps its width with ` `. +fn text_html(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + let mut prev_space = false; + for (k, line) in text.split('\n').enumerate() { + if k > 0 { + out.push_str("
"); + prev_space = false; + } + for c in line.chars() { + if c == ' ' && prev_space { + out.push_str(" "); + } else { + out.push_str(&esc(c.encode_utf8(&mut [0; 4]))); + } + prev_space = c == ' '; + } + } + out +} + +/// A size with at most one decimal and no trailing `.0`. +fn num(v: f64) -> String { + let r = (v * 10.0).round() / 10.0; + if r.fract() == 0.0 { format!("{r:.0}") } else { format!("{r:.1}") } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn style(size: f64, bold: bool, italic: bool, color: [f64; 3]) -> ClipStyle { + ClipStyle { family: font_family("ABCDEF+Helvetica-Bold"), size, bold, italic, color } + } + + #[test] + fn heading_body_and_coloured_span_keep_their_formatting() { + let mut b = ClipBuilder::new(); + let heading = style(24.0, true, false, [0.0; 3]); + let body = style(11.5, false, false, [0.0; 3]); + let red = style(11.5, false, true, [1.0, 0.0, 0.0]); + b.push("Title", &heading); + b.paragraph(); + b.push("Body", &body); + b.space(); + b.push("text", &body); + b.space(); + b.push("red", &red); + let clip = b.finish(); + assert_eq!(clip.paragraphs.len(), 2); + assert_eq!(clip.paragraphs.get(1).map(Vec::len), Some(2), "{clip:?}"); + let html = clip_html(&clip); + assert_eq!( + html, + "

Title

\n\ +

Body text \ + red

\n" + ); + } + + #[test] + fn special_characters_are_escaped_and_cannot_leave_the_style() { + let st = ClipStyle { family: Some("Evil';}"#, - r#""#, + r#""#, ) .replace( r#"if (qty.rawValue === null) qty.rawValue = 2;"#, @@ -7337,9 +7338,11 @@ fn hostile_xfa_open_scripts_are_merged_capped_kept_from_printing_and_noted() { let mut s = Session::new().with_clock(|| 1_700_000_000); let started = std::time::Instant::now(); let id = s.open("hostile.pdf", None, bytes, None).expect("opens"); - assert!(started.elapsed() < std::time::Duration::from_secs(5), "{:?}", started.elapsed()); + // 30 s and not 5 s: the run is wall-clock timed and a loaded test machine (all suites in + // parallel) can take several seconds for the 20 000 iterations; a real hang still fails. + assert!(started.elapsed() < std::time::Duration::from_secs(30), "{:?}", started.elapsed()); let doc = s.get(id).unwrap(); - assert_eq!(doc.form.iter().find(|f| f.name == "total").unwrap().value, vec!["49999".to_string()]); + assert_eq!(doc.form.iter().find(|f| f.name == "total").unwrap().value, vec!["19999".to_string()]); // One datasets stream for the scripts' revision, not one per value set. assert!(datasets_streams(&doc.bytes) <= 3, "{} datasets streams", datasets_streams(&doc.bytes)); // What opening changed is noted with the form's other XFA notes. diff --git a/crates/engine/tests/it/ocr_edit.rs b/crates/engine/tests/it/ocr_edit.rs index 53ca309326..2d358e327e 100644 --- a/crates/engine/tests/it/ocr_edit.rs +++ b/crates/engine/tests/it/ocr_edit.rs @@ -12,7 +12,14 @@ const WORD: [u32; 4] = [40, 40, 100, 60]; /// A 200 × 100 pt "scan" on tinted paper, with OCR text over its word. fn scanned() -> (Session, pdfcraft_engine::DocId) { - scanned_with(vec![PlacedWord { text: "John".into(), origin: [40.0, 40.0], across: [60.0, 0.0], up: [0.0, 20.0] }]) + scanned_with(vec![PlacedWord { + text: "John".into(), + origin: [40.0, 40.0], + across: [60.0, 0.0], + up: [0.0, 20.0], + confidence: None, + source: "ocrs".into(), + }]) } fn scanned_with(words: Vec) -> (Session, pdfcraft_engine::DocId) { @@ -86,11 +93,18 @@ fn an_explicit_colour_is_kept_for_the_new_text() { /// descender line when it has descenders, up to the cap height. fn scanned_word(text: &str, x0: f64, x1: f64, baseline: f64, size: f64) -> PlacedWord { let low = if text.chars().any(|c| "gjpqy".contains(c)) { 0.21 * size } else { 0.0 }; - PlacedWord { text: text.into(), origin: [x0, baseline - low], across: [x1 - x0, 0.0], up: [0.0, 0.718 * size + low] } + PlacedWord { + text: text.into(), + origin: [x0, baseline - low], + across: [x1 - x0, 0.0], + up: [0.0, 0.718 * size + low], + confidence: None, + source: "ocrs".into(), + } } fn word(text: &str, x0: f64, x1: f64, baseline: f64) -> PlacedWord { - PlacedWord { text: text.into(), origin: [x0, baseline], across: [x1 - x0, 0.0], up: [0.0, 10.0] } + PlacedWord { text: text.into(), origin: [x0, baseline], across: [x1 - x0, 0.0], up: [0.0, 10.0], confidence: None, source: "ocrs".into() } } /// OCR words keep the scan's layout: a phrase is one paragraph, a value in the next table cell and diff --git a/crates/fonts/README.md b/crates/fonts/README.md index b20b930a91..096f1a263e 100644 --- a/crates/fonts/README.md +++ b/crates/fonts/README.md @@ -59,7 +59,12 @@ set). Without that variable `CRAFT_FONTS` is empty and everything below copes: and shaped Thai Add Text content. Harfrust positions stacked vowel/tone marks and decomposes Sara Am; cluster ToUnicode entries preserve the original text. The shared Type 3 writer embeds outlines, so reopening does not depend on installed fonts. +- `craft_glyph(face, ch)` and `face_has(bytes, ch)`: bounded outlines and glyph coverage for any bundled face. - `SHIPPORI_MINCHO`: Shippori Mincho's bytes, or `None`. +- `TextPlan::for_words`: how the OCR output writes text beyond WinAnsi — per word the first + face (document face first) whose glyphs cover it, planned once per page into Type 3 fonts + (codes, advances, `?` stand-ins) that `pdfcraft-edit` registers and the `pdfcraft-ocr` layers + draw with; characters no face has are counted (`missing`) so the output reports them. Generated Japanese Type 3 fonts record the fallback face's family and style in an indirect `FontDescriptor`. `pdf::Metrics::base_font` uses its `FontName` when the Type 3 dictionary has diff --git a/crates/fonts/src/lib.rs b/crates/fonts/src/lib.rs index 61061f5509..45cb843ab0 100644 --- a/crates/fonts/src/lib.rs +++ b/crates/fonts/src/lib.rs @@ -14,6 +14,7 @@ pub mod embedding; mod encodings; pub mod fallback; pub mod pdf; +pub mod plan; mod script; mod shaping; pub mod source_shaping; @@ -24,8 +25,10 @@ pub use craft::{ document_japanese_font_for_style, document_japanese_fonts_for_style, document_latin_fonts, document_thai_font, ui_arabic_fonts, ui_chinese_fonts, ui_cjk_fonts, ui_devanagari_fonts, ui_hebrew_fonts, ui_japanese_fonts, ui_telugu_fonts, ui_thai_fonts, }; +pub use plan::{MAX_PLAN_GLYPHS, MAX_TYPE3_GLYPHS, PlannedType3, TextPlan}; pub use script::{ - GlyphError, GlyphOutline, INTER_REGULAR, MAX_SIGNATURE_CHARS, ScriptOutline, japanese_glyph, japanese_glyph_from, script_missing, script_outline, + GlyphError, GlyphOutline, INTER_REGULAR, MAX_SIGNATURE_CHARS, ScriptOutline, craft_glyph, face_has, japanese_glyph, japanese_glyph_from, + script_missing, script_outline, }; pub use shaping::{ ShapedCluster, ShapedScript, arabic_glyph, arabic_has, hebrew_has, shape_arabic, shape_hebrew, shape_thai, shaped_glyph, thai_has, @@ -273,6 +276,16 @@ pub fn set_undrawn_mark(d: &mut pdfcraft_cos::Dict, c: Option) { } } +/// The byte used by Unicode text plans, from the shared WinAnsi table (tabs become spaces). +pub fn win_ansi_char(c: char) -> Option { + if c == '\t' { Some(b' ') } else { win_ansi_byte(c) } +} + +/// Whether every character can be represented without substitution in WinAnsi. +pub fn fits_win_ansi(s: &str) -> bool { + s.chars().all(win_ansi_encodable) +} + /// Bytes as a PDF literal string, `(` … `)`, with delimiters escaped. pub fn literal(bytes: &[u8]) -> Vec { let mut out = Vec::with_capacity(bytes.len() + 2); diff --git a/crates/fonts/src/plan.rs b/crates/fonts/src/plan.rs new file mode 100644 index 0000000000..9c22a38fd3 --- /dev/null +++ b/crates/fonts/src/plan.rs @@ -0,0 +1,338 @@ +//! Planning visible Unicode text output for OCR: which characters the standard WinAnsi +//! fonts show as is, which need a Type3 font drawn from the craft-fonts faces, and which have +//! no glyph anywhere (written as `?` — and counted, never silently dropped). +//! +//! The plan is a pure function of the words ([`TextPlan::for_words`]). The content generators +//! (`pdfcraft_ocr::text_layer` / `visible_text_layer`) read the per-character codes and +//! advances from it, and whoever registers the fonts (`pdfcraft_edit::add_type3_fonts`) fills +//! in each face's resource name and object. The build must work without craft-fonts too: there +//! [`CRAFT_FONTS`] is empty, nothing can be planned, and every non-WinAnsi character is +//! reported as a substitution instead. + +use std::collections::HashMap; + +use pdfcraft_cos::ObjRef; + +use crate::{CraftFont, GlyphOutline, craft_glyph, document_japanese_font, face_has, fits_win_ansi, win_ansi_char}; + +/// Unique characters per Type3 font: the same bound the text-editing fallback uses, so one +/// hostile word cannot balloon into thousands of glyph streams. A face with more characters +/// simply gets another font. +pub const MAX_TYPE3_GLYPHS: usize = 240; + +/// Planned glyphs per plan in total (10 full fonts). A page's recognition cannot realistically +/// exceed it; hostile input beyond it degrades to `?` substitutions, which are counted. +pub const MAX_PLAN_GLYPHS: usize = 2400; + +/// One planned Type3 font: the glyphs of one craft-fonts face, one character per code +/// (`code = position + 1`, as in the text-editing fallback). `name` and `object` are filled in +/// when the font is added to a page's resources; until then the content generators fall back to +/// the WinAnsi path for the words that reference it. +pub struct PlannedType3 { + /// The face family the glyphs are drawn from ("Shippori Mincho"). + pub family: &'static str, + /// (character, outline with its advance) per code, in code order. + pub glyphs: Vec<(char, GlyphOutline)>, + /// The resource name the content references (`PCOcr`, `PCOcr1`, …); empty until registered. + pub name: String, + /// The built font object, shared by every page that uses the font; set at registration. + pub object: Option, +} + +impl PlannedType3 { + fn new(family: &'static str) -> Self { + PlannedType3 { family, glyphs: Vec::new(), name: String::new(), object: None } + } + + fn push(&mut self, ch: char, glyph: GlyphOutline) -> u8 { + self.glyphs.push((ch, glyph)); + self.glyphs.len() as u8 + } +} + +/// How the text layers write a plan's characters: each planned character is a +/// `(font index, code)` pair in its font's encoding; a character no face had a glyph for is +/// either the font's `?` code or absent from the run (both counted in [`TextPlan::missing`]). +#[derive(Default)] +pub struct TextPlan { + /// One planned font per face chunk, in first-use order. + pub fonts: Vec, + /// Per input word: `(font, code)` per character; empty for a word the standard WinAnsi + /// fonts write as is (or for which nothing could be planned). + words: Vec>, + /// character → where its glyph lives, across all fonts (codes are font-local). + index: HashMap, + /// Characters with no code in their word's font: they appear as `?` (or, when not even a + /// `?` glyph exists, are left out) and are reported, never silently lost. + missing: usize, +} + +impl TextPlan { + /// Plan the output for `words`, in order: characters the WinAnsi fonts can't show get codes + /// in Type3 fonts built from the craft-fonts faces (the document face preferred, then the + /// rest of the build input in manifest order); a word whose face can't show a character + /// substitutes its `?`. Faces are chosen per word, so a word of one script stays one face; + /// characters already planned for an earlier word keep their code. Built without + /// craft-fonts, the plan reports every non-WinAnsi character as a substitution. + pub fn for_words<'a>(words: impl IntoIterator) -> TextPlan { + let faces = document_faces(); + let mut coverage = Coverage::default(); + let mut plan = TextPlan::default(); + for word in words { + if fits_win_ansi(word) { + // The WinAnsi path substitutes '?' behind the caller's back; count it. + plan.missing += word.chars().filter(|&c| win_ansi_char(c).is_none()).count(); + plan.words.push(Vec::new()); + continue; + } + let Some(face) = choose_face(&faces, word, &mut coverage) else { + // No face for this word's script: the WinAnsi path writes '?' for what it + // can't encode. + plan.missing += word.chars().filter(|&c| win_ansi_char(c).is_none()).count(); + plan.words.push(Vec::new()); + continue; + }; + let mut enc: Vec<(usize, u8)> = Vec::with_capacity(word.len()); + let mut run: Option = None; + for ch in word.chars() { + if let Some(&where_) = plan.index.get(&ch) { + enc.push(where_); + run = Some(where_.0); + continue; + } + if let Ok(glyph) = craft_glyph(face, ch) + && plan.planned_glyphs() < MAX_PLAN_GLYPHS + && let Some((fi, code)) = plan.add_glyph(face, ch, glyph) + { + plan.index.insert(ch, (fi, code)); + enc.push((fi, code)); + run = Some(fi); + continue; + } + // The face has no glyph for the character (or the plan is full): substitute + // the font's '?', in the font the surrounding run is set in. + if let Some((fi, code)) = plan.question_mark(face, run.filter(|&f| plan.fonts[f].family == face.family)) { + enc.push((fi, code)); + run = Some(fi); + } + plan.missing += 1; + } + plan.words.push(enc); + } + plan + } + + /// The encoding of word `i` (parallel to the slice [`Self::for_words`] planned): a list of + /// `(font, code)` per character, or an empty slice for a word the WinAnsi fonts write. + /// Out-of-range indexes degrade to the empty slice rather than panicking — callers pass the + /// same word list they planned with. + pub fn encoding(&self, word: usize) -> &[(usize, u8)] { + self.words.get(word).map(Vec::as_slice).unwrap_or_default() + } + + /// How many characters no font could show (they appear as `?`, or are left out when the + /// word's face has no `?` glyph either). + pub fn missing(&self) -> usize { + self.missing + } + + /// The advance of one planned code, in em units (the Type3 glyphs' own widths). `0.0` for + /// anything out of range. + pub fn advance(&self, font: usize, code: u8) -> f64 { + self.fonts.get(font).and_then(|f| f.glyphs.get(usize::from(code).checked_sub(1)?)).map_or(0.0, |(_, g)| g.width) + } + + /// The sum of the advances of a word's planned codes, in em units. + pub fn advance_of(&self, encoding: &[(usize, u8)]) -> f64 { + encoding.iter().map(|&(f, c)| self.advance(f, c)).sum() + } + + /// Whether every font the encoding references has been registered (has a resource name). + /// Unregistered plans fall back to the WinAnsi path, the same output as before the + /// Unicode mapping existed. + pub fn registered(&self, encoding: &[(usize, u8)]) -> bool { + encoding.iter().all(|&(f, _)| self.fonts.get(f).is_some_and(|font| !font.name.is_empty())) + } + + /// Characters with a planned glyph so far. + fn planned_glyphs(&self) -> usize { + self.fonts.iter().map(|f| f.glyphs.len()).sum() + } + + /// Add a glyph to the face's current font (a fresh one when that font is full). + fn add_glyph(&mut self, face: &CraftFont, ch: char, glyph: GlyphOutline) -> Option<(usize, u8)> { + let fi = self.font_of_face(face.family)?; + let code = self.fonts[fi].push(ch, glyph); + Some((fi, code)) + } + + /// The index of the face's open font, or a fresh one. `None` without craft-fonts (the + /// face itself comes from there, so this cannot happen for a planned face). + fn font_of_face(&mut self, family: &'static str) -> Option { + if let Some(fi) = self.fonts.iter().rposition(|f| f.family == family && f.glyphs.len() < MAX_TYPE3_GLYPHS) { + return Some(fi); + } + if self.fonts.len() >= MAX_PLAN_GLYPHS / MAX_TYPE3_GLYPHS { + return None; + } + self.fonts.push(PlannedType3::new(family)); + Some(self.fonts.len() - 1) + } + + /// The code of the font's `?` stand-in, adding the glyph when the font lacks it. + /// `prefer` is the run's current font; `None` without craft-fonts. + fn question_mark(&mut self, face: &CraftFont, prefer: Option) -> Option<(usize, u8)> { + if let Some(fi) = prefer + && let Some(code) = self.fonts[fi].glyphs.iter().position(|(c, _)| *c == '?') + { + return Some((fi, code as u8 + 1)); + } + let fi = self.font_of_face(face.family)?; + if let Some(code) = self.fonts[fi].glyphs.iter().position(|(c, _)| *c == '?') { + return Some((fi, code as u8 + 1)); + } + let glyph = craft_glyph(face, '?').ok()?; + let code = self.fonts[fi].push('?', glyph); + Some((fi, code)) + } +} + +/// The faces that can draw document text, in preference order: the Japanese document face +/// (serif document text) first, then the rest of the build input in manifest order. +fn document_faces() -> Vec<&'static CraftFont> { + let mut out = Vec::new(); + if let Some(d) = document_japanese_font() { + out.push(d); + } + for f in crate::CRAFT_FONTS { + if !out.iter().any(|d| std::ptr::eq(*d, f)) { + out.push(f); + } + } + out +} + +/// The face for a word: the first (in preference order) whose glyphs cover every character, +/// else the one covering the most characters (ties go to the earlier face). `None` when no +/// face covers any character of the word. +fn choose_face<'f>(faces: &[&'f CraftFont], word: &str, coverage: &mut Coverage) -> Option<&'f CraftFont> { + if let Some(f) = faces.iter().enumerate().find(|(i, f)| word.chars().all(|c| coverage.has(*i, f, c))).map(|(_, f)| *f) { + return Some(f); + } + faces + .iter() + .enumerate() + .map(|(i, f)| (word.chars().filter(|&c| coverage.has(i, f, c)).count(), std::cmp::Reverse(i), *f)) + .max_by(|a, b| (a.0, a.1).cmp(&(b.0, b.1))) + .filter(|(covered, _, _)| *covered > 0) + .map(|(_, _, f)| f) +} + +/// Which faces have a glyph for which characters, remembered for one plan: a page of words +/// asks the same (face, character) pairs over and over, and each fresh answer parses the font. +#[derive(Default)] +struct Coverage { + known: std::collections::HashMap<(usize, char), bool>, + lookups: usize, +} + +impl Coverage { + /// Whether face number `i` (`face`) has `ch`; asks the font once per pair. + fn has(&mut self, i: usize, face: &CraftFont, ch: char) -> bool { + *self.known.entry((i, ch)).or_insert_with(|| { + self.lookups += 1; + face_has(face.bytes, ch) + }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Face coverage is asked once per (face, character): repeating words cost no new font + /// lookups. + #[test] + fn face_coverage_is_remembered_per_face_and_character() { + let faces = document_faces(); + let Some(face) = faces.first() else { + eprintln!("skipping: built without craft-fonts (set CRAFT_FONTS_DIR to run it)"); + return; + }; + let mut coverage = Coverage::default(); + let first = (0..50).map(|_| choose_face(&faces, "日本語", &mut coverage)).last().flatten(); + assert!(first.is_some()); + assert!(coverage.lookups <= faces.len() * 3, "{} lookups for 3 characters", coverage.lookups); + assert_eq!(coverage.has(0, face, '日'), face_has(face.bytes, '日')); + } + + /// A page's vocabulary: German umlauts are WinAnsi, and a character no bundled face has + /// (here an emoji) is counted wherever it appears. + #[test] + fn win_ansi_words_get_no_encoding_and_count_only_real_losses() { + let plan = TextPlan::for_words(["Zürcher", "café", "size"]); + assert!(plan.words.iter().all(Vec::is_empty), "all WinAnsi: no Type3 fonts needed"); + assert!(plan.fonts.is_empty()); + assert_eq!(plan.missing(), 0); + let none = TextPlan::for_words(["plain text"]); + assert_eq!(none.missing(), 0); + // U+1F600 has no WinAnsi code and no glyph in any bundled face (with craft-fonts the + // JIS faces even cover the snowman): counted, shown as '?'. + let emoji = TextPlan::for_words(["\u{1f600}!"]); + assert_eq!(emoji.missing(), 1); + } + + #[test] + fn without_craft_fonts_every_non_win_ansi_character_is_reported() { + if !crate::CRAFT_FONTS.is_empty() { + eprintln!("built with craft-fonts: the planning tests with faces cover this instead"); + return; + } + let plan = TextPlan::for_words(["Привет", "日本", "abc"]); + assert!(plan.words.iter().all(Vec::is_empty)); + assert!(plan.fonts.is_empty()); + assert_eq!(plan.missing(), 8, "six Cyrillic + two CJK characters, each reported"); + } + + #[test] + fn encodings_are_aligned_and_total_the_advances() { + if crate::CRAFT_FONTS.is_empty() { + eprintln!("skipping: built without craft-fonts (set CRAFT_FONTS_DIR to run it)"); + assert!(TextPlan::for_words(["日本"]).fonts.is_empty()); + return; + } + let plan = TextPlan::for_words(["日本語", "語"]); + assert_eq!(plan.encoding(0).len(), 3); + assert_eq!(plan.encoding(1).len(), 1); + // 語 planned once keeps its code (and font) in both words. + let (f, c) = plan.encoding(1)[0]; + assert_eq!(plan.encoding(0)[2], (f, c), "the repeated character keeps its code"); + assert!(plan.advance(f, c) > 0.0); + assert_eq!(plan.advance(f, 0), 0.0, "codes start at 1"); + assert_eq!(plan.encoding(7), &[], "out of range degrades to the WinAnsi path"); + assert!(plan.advance_of(plan.encoding(0)) > plan.advance_of(plan.encoding(1))); + } + + #[test] + fn hostile_input_stays_bounded() { + // Thousands of DISTINCT characters: the covered ones fill fonts up to the caps, the + // rest are reported substitutions. + let word: String = (0..4000u32).filter_map(|i| char::from_u32(0x4e00 + i)).collect(); + let plan = TextPlan::for_words([word.as_str()]); + assert!(plan.planned_glyphs() <= MAX_PLAN_GLYPHS); + assert!(plan.fonts.iter().all(|f| f.glyphs.len() <= MAX_TYPE3_GLYPHS)); + assert!(plan.missing() > 0, "characters beyond the caps are reported, never silently dropped"); + } + + #[test] + fn substitution_and_unknown_scripts_come_out_counted() { + if crate::CRAFT_FONTS.is_empty() { + eprintln!("skipping: built without craft-fonts (set CRAFT_FONTS_DIR to run it)"); + return; + } + let plan = TextPlan::for_words(["日本\u{1f4a9}語"]); + // The emoji has no glyph in any bundled face; the kana/kanji do. + assert_eq!(plan.missing(), 1); + assert_eq!(plan.encoding(0).len(), 4, "the emoji still has an encoding: the font's ?"); + } +} diff --git a/crates/fonts/src/script.rs b/crates/fonts/src/script.rs index 70fc6ce092..94a8a93b8c 100644 --- a/crates/fonts/src/script.rs +++ b/crates/fonts/src/script.rs @@ -134,6 +134,17 @@ impl OutlinePen for Flatten { } } +/// Whether the face has a glyph for `ch` (a cheap character-map probe, no outline work). +pub fn face_has(bytes: &[u8], ch: char) -> bool { + FontRef::new(bytes).is_ok_and(|font| font.charmap().map(ch).is_some()) +} + +/// Return one glyph of `face` (a craft-fonts face), bounded so hostile replacement text cannot +/// allocate unbounded outline data. +pub fn craft_glyph(face: &crate::CraftFont, ch: char) -> Result { + glyph_in_face(face.bytes, ch) +} + /// Return one glyph of the Japanese document face ([`crate::document_japanese_font`], Shippori /// Mincho from craft-fonts), bounded so hostile replacement text cannot allocate unbounded /// outline data. [`GlyphError::NoFont`] when the build has no Japanese face. @@ -145,7 +156,11 @@ pub fn japanese_glyph(ch: char) -> Result { /// The bounded outline and advance from the explicitly selected craft-fonts face. /// Uses the same complexity and missing-glyph checks as [`japanese_glyph`]. pub fn japanese_glyph_from(face: &crate::CraftFont, ch: char) -> Result { - let Ok(font) = FontRef::new(face.bytes) else { return Err(GlyphError::Missing) }; + glyph_in_face(face.bytes, ch) +} + +fn glyph_in_face(bytes: &[u8], ch: char) -> Result { + let Ok(font) = FontRef::new(bytes) else { return Err(GlyphError::Missing) }; let loc = LocationRef::default(); let metrics = font.metrics(Size::unscaled(), loc); let scale = 1.0 / metrics.units_per_em.max(1) as f64; diff --git a/crates/ocr/Cargo.toml b/crates/ocr/Cargo.toml index 750e78b96d..1b04dbb522 100644 --- a/crates/ocr/Cargo.toml +++ b/crates/ocr/Cargo.toml @@ -16,5 +16,9 @@ ocrs = { version = "0.13.1", default-features = false, features = ["rten"] } rten = { version = "0.26.0", default-features = false, features = ["onnx_format", "rten_format"] } thiserror.workspace = true +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +# The page image is handed to the external tesseract process as a PNG. +png = "0.18" + [dev-dependencies] tempfile = "3.27" diff --git a/crates/ocr/README.md b/crates/ocr/README.md new file mode 100644 index 0000000000..cd33b186cf --- /dev/null +++ b/crates/ocr/README.md @@ -0,0 +1,104 @@ +# pdfcraft-ocr + +Layer L4 (may use `fonts` for the text layers' metrics): Scan & OCR — recognising the words in +a page raster and turning them back into page content (execution plan M10). + +```rust +// The caller renders a page to pixels and hands them over: +let image = OcrImage::new(w, h, rgba)?; // validated by construction +let lines = recognizer.recognize(&image, &options)?; // Vec: words in reading order +let placed: Vec = …; // PlacedWord::place maps boxes to user space +// The editable output draws real glyphs: plan the words, register the display faces into the +// plan (`pdfcraft_edit::add_type3_fonts`), then generate the layer. An unregistered plan falls +// back to WinAnsi, and characters no registered face has become '?'. +let plan = TextPlan::for_words(placed.iter().map(|w| w.text.as_str())); +add_type3_fonts(&mut doc, page, &mut plan)?; // names the plan's fonts on the page +let layer = visible_text_layer(&placed, &plan); // visible text: the editable output +// The searchable layer needs no display fonts: the engine's own layer (engine::ocr::layer) +// keeps every Unicode character with empty Type3 programs and a ToUnicode map beyond WinAnsi. +``` + +- **Engines behind one trait.** A [`Recognizer`] reports its `id`, whether it is `available`, + the languages it reads, and reads `Vec` from an image. Two exist: the pure-Rust + `OcrsRecognizer` (the `ocrs` crate, MIT/Apache-2.0, with its pre-trained models, + CC-BY-SA-4.0, fetched by `cargo xtask models`; found via `$PDFCRAFT_MODELS` or `models/` + beside the executable — see ATTRIBUTION.toml), and — off the web build — + `tesseract::TesseractCli`, which drives an installed `tesseract` binary as an external + process: never linked, bundled or downloaded. Implementations are shared across threads + (`Send + Sync`); nothing spawns on wasm32. +- **The tesseract process is fenced in.** The program comes from a user-configured location or + the PATH, never from document data (the engine validates a configured path as an absolute + path to an existing executable file before it is used). The language code is allow-listed + (`tesseract::language_code`, 12 codes) before anything reaches a process; the page image + goes to a private temp file (a fresh 0700 directory holding a 0600 file on Unix) that is + deleted afterwards; the run has a 120 s timeout, a 64 MiB output cap and a 50 000-word + reading cap (an incomplete reading is rejected rather than applied); every failure comes back through `OcrError` with its cause named. A configured + location that is invalid fails closed (it is never replaced by the PATH search). Availability + is probed with `--list-langs` (its list is read from stdout, or from stderr where older + versions print it); a failed probe is remembered for 30 s, so an install that appears later + is found without a restart. Known limit: at the timeout or the output cap only the direct + child is killed; a wrapper script's own children may outlive it. +- **Confidence drives review.** One classifier (`band_for`) puts a word in the High + (≥ 90), Medium (70–89) or Low (< 70) band — the review overlay's colours, the legend and the + counts; the batch flag rule (`low_confidence`, default threshold 60) is deliberately a + separate threshold. The `ocrs` models report no confidence (its words carry `None`); + a reading whose confidence is unknown is treated as unknown, never as good. +- **Two engines can be merged (ROVER).** Strategies: `PrimaryOnly` (the default), and two + ensembles — `ConfidenceWeighted` (the secondary engine joins when the primary's mean + confidence is under 70 or unknown; an empty primary keeps the fast path) and `RoverVote` + (both engines always run). The merge lets every primary word adopt the best-overlapping + unused secondary word at IoU ≥ 0.5; the higher confidence wins text, box and confidence (a + tie keeps the primary; against a primary word of unknown confidence — ocrs — the secondary + wins only at 70 or more, `UNKNOWN_CONFIDENCE_BAR`), the survivor is named `"ROVER"`, and + unclaimed secondary words are + appended under their engine's name. A merge is capped at 100 M word-pair comparisons (over + it, the primary reading stands alone), and a failing partner degrades to the primary with a + recorded note — a page is never skipped because a partner could not read it. +- **Preprocessing between the raster and recognition**, a pure function in fixed order: DPI + normalization to 300 (only when one of the steps below is on, since they are calibrated for + it; with all of them off the raster is read exactly as rendered, and a scan at or above 300 + is left alone), then the opt-in steps — + orientation (90°/180°/270° from projection-profile banding), deskew (± 5°, shear-scored, + below 0.1° is a no-op), denoise (3×3 median) and binarization (Sauvola, window 8, k 0.34). + Every geometric step composes one exact affine `InverseTransform`, and callers map every + word box back to the original raster before placing it. The heuristics are conservative and + panic-free: too small to judge, blank, or no clear winner is a no-op, never a guess. +- **Searchable output preserves Unicode without display fonts.** The engine uses empty + Type3 character programs and `/ToUnicode` maps for non-WinAnsi words, leaving the scan + unchanged. WinAnsi words use `/PCHelv`; the `/OCR` layer is fitted to each word's box. +- **Editable output uses display glyphs.** `visible_text_layer` writes visible text whose + size comes from the box height and whose horizontal scaling uses real font advances. + Beyond WinAnsi, attributed craft-fonts faces supply outline Type3 glyphs with `/ToUnicode`. + Uncovered characters become `?` and are reported in editable output notes; searchable + output retains them. The engine refuses nonfinite geometry or oversized input before + planning fonts: 4 MiB of text and 65,536 words, across all pages for editable export. + Empty geometry is skipped; rotated words carry `across`/`up` vectors and retain direction. +- **Hostile input is rejected, not panicked at.** `OcrImage` holds private, validated state: + non-zero sides, a 10 000-pixel side cap, a 64-megapixel area cap (checked arithmetic — the + counts come from documents, so they are untrusted), and an exact buffer length. + +## Where it is wired + +The engine's `ocr` module renders pages into `OcrImage`s, runs jobs (`OcrJob::run`) and applies +the words: searchable-image mode draws the invisible layer over the page's own image and leaves +that image untouched; editable mode writes a NEW `_ocr.pdf` of visible text (the scan +image is not carried over, the source is never overwritten). Skip probes keep pages — and +whole files — that already contain text out of the work unless Force OCR overrides them. A +searchable result applies as one undoable step; the Editable text output opens the OCR Verify +screen first, and Scan & OCR ▸ Correct recognized text reviews any recognition there before it +is applied (confidence-coloured boxes, corrections, a per-language user dictionary — a +plain-text file in the preferences folder, so words added from a document persist on disk +outside it until removed — Re-OCR of a region or a page, Accept/Reject), a +multiple-file run writes searchable copies as it goes, and the automation tools `ocr_recognize`, +`ocr_words`, `ocr_status` and `ocr_recognize_files` expose recognition headlessly. +`ocr_words` with `retain_review` returns a bounded session-local handle; `ocr_review_apply` +applies text corrections at the stored original boxes through its stale-document guard, +or discards the result. Empty corrections remove words. Searchable acceptance is one undo +step; editable acceptance writes a separate file. `ocr_dictionary` reads or atomically +updates the UI's per-language dictionary format through an explicitly root-confined path. + +Not yet: languages beyond the tested set (the `ocrs` models read only unaccented Latin; the +Tesseract side is bounded by its allow-list and whatever its installed language packs cover), +OCR in the browser (the tesseract module is compiled out for wasm32), the deep scan-cleanup +options (descreen, background removal, edge-shadow removal), and editable glyph coverage +beyond the available attributed faces. diff --git a/crates/ocr/src/confidence.rs b/crates/ocr/src/confidence.rs new file mode 100644 index 0000000000..1c0de605f5 --- /dev/null +++ b/crates/ocr/src/confidence.rs @@ -0,0 +1,106 @@ +//! Confidence: one classifier ([`band_for`]) drives the review overlay, the legend, the counts +//! and the uncertain-word navigation; the batch rule ([`low_confidence`]) is a separate +//! threshold with its own range check. The two rules are deliberately never merged. + +/// The three confidence bands (the review overlay's green / yellow / red). +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum ConfidenceBand { + Low, + Medium, + High, +} + +impl ConfidenceBand { + /// The legend line for this band, so the meaning never rests on colour alone. + pub fn legend(self) -> &'static str { + match self { + ConfidenceBand::High => "HIGH (>= 90%)", + ConfidenceBand::Medium => "MEDIUM (70-89%)", + ConfidenceBand::Low => "LOW (< 70%)", + } + } +} + +/// The one classifier: `>= 90` is High, `70..=89` Medium, `< 70` Low. Out-of-range values +/// clamp (negative → Low, above 100 → High); NaN is Low. +pub fn band_for(confidence: f32) -> ConfidenceBand { + if confidence >= 90.0 { + ConfidenceBand::High + } else if confidence >= 70.0 { + ConfidenceBand::Medium + } else { + ConfidenceBand::Low + } +} + +/// The batch rule, separate from [`band_for`]: a word is flagged when `0 <= conf <= threshold` +/// (default 60). Negative or above-threshold values are not flagged; NaN is not. +pub fn low_confidence(confidence: f32, threshold: f32) -> bool { + (0.0..=threshold).contains(&confidence) +} + +/// A word whose confidence lands in the Low band — a suspect for review. A word without a +/// reported confidence has no signal and is never a suspect. +pub fn is_suspect(confidence: Option) -> bool { + confidence.is_some_and(|c| band_for(c) == ConfidenceBand::Low) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The boundary matrix: 89 and 90, 69 and 70 straddle the bands. + #[test] + fn bands_straddle_the_boundaries() { + assert_eq!(band_for(89.0), ConfidenceBand::Medium); + assert_eq!(band_for(90.0), ConfidenceBand::High); + assert_eq!(band_for(69.0), ConfidenceBand::Low); + assert_eq!(band_for(70.0), ConfidenceBand::Medium); + assert_eq!(band_for(0.0), ConfidenceBand::Low); + assert_eq!(band_for(100.0), ConfidenceBand::High); + } + + /// Out-of-range values clamp, and NaN lands in the Low band instead of comparing equal to + /// nothing. + #[test] + fn out_of_range_confidences_clamp() { + assert_eq!(band_for(-0.5), ConfidenceBand::Low, "negative → Low"); + assert_eq!(band_for(1000.0), ConfidenceBand::High, "above 100 → High"); + assert_eq!(band_for(f32::NAN), ConfidenceBand::Low); + assert_eq!(band_for(f32::NEG_INFINITY), ConfidenceBand::Low); + assert_eq!(band_for(f32::INFINITY), ConfidenceBand::High); + } + + /// The batch threshold is its own rule: everything in `0..=threshold` flags, including + /// words band_for calls Medium (60 < 70), and negatives never do. + #[test] + fn batch_threshold_is_separate_from_the_bands() { + assert!(low_confidence(60.0, 60.0), "at the threshold it flags"); + assert!(low_confidence(0.0, 60.0)); + assert!(!low_confidence(60.5, 60.0)); + assert!(!low_confidence(-1.0, 60.0), "negative confidences are out of the rule's range"); + assert!(!low_confidence(f32::NAN, 60.0)); + // The rules disagree on real values: 65 is inside the Low band yet outside the batch + // rule with the default threshold (60), which is why the two must not be merged. + assert!(!low_confidence(65.0, 60.0)); + assert_eq!(band_for(65.0), ConfidenceBand::Low); + } + + /// Unconfident words are not suspects; Low-band words are. + #[test] + fn suspects_need_a_low_band_confidence() { + assert!(!is_suspect(None)); + assert!(!is_suspect(Some(95.0))); + assert!(!is_suspect(Some(70.0))); + assert!(is_suspect(Some(69.9))); + assert!(is_suspect(Some(0.0))); + } + + /// The legend names the thresholds, so the overlay never rests on colour alone. + #[test] + fn legend_text_matches_the_classifier() { + assert_eq!(ConfidenceBand::High.legend(), "HIGH (>= 90%)"); + assert_eq!(ConfidenceBand::Medium.legend(), "MEDIUM (70-89%)"); + assert_eq!(ConfidenceBand::Low.legend(), "LOW (< 70%)"); + } +} diff --git a/crates/ocr/src/lib.rs b/crates/ocr/src/lib.rs index 7f58270e1a..ff5b920dca 100644 --- a/crates/ocr/src/lib.rs +++ b/crates/ocr/src/lib.rs @@ -1,35 +1,50 @@ //! pdfcraft-ocr — Scan & OCR ▸ Recognize text (L4). //! -//! Recognition runs the ocrs engine (MIT/Apache-2.0) with its pre-trained models (CC-BY-SA-4.0, -//! fetched by `cargo xtask models`; see ATTRIBUTION.toml). The caller renders a page to pixels; -//! [`Ocr::recognize`] finds the words in them, and [`text_layer`] turns words placed in user space -//! into page content: invisible text (rendering mode 3) over each word, so the page becomes a +//! The caller renders a page to pixels ([`OcrImage`]) and hands it to a [`Recognizer`]. Two +//! engines exist: the pure-Rust `OcrsRecognizer` (ocrs, MIT/Apache-2.0, with its pre-trained +//! models, CC-BY-SA-4.0, fetched by `cargo xtask models`; see ATTRIBUTION.toml) and — off the +//! web — `tesseract::TesseractCli`, which drives an installed `tesseract` binary as an external +//! process. `merge` combines two engines' readings (ROVER), `preprocess` cleans the raster up +//! between rendering and recognition, and [`text_layer`] turns words placed in user space into +//! page content: invisible text (rendering mode 3) over each word, so the page becomes a //! searchable image (Acrobat's "Searchable Image (Exact)": the image is left untouched). //! -//! The models read the Latin alphabet (English and other languages written without accents). +//! Visible editable output uses [`pdfcraft_fonts::TextPlan`] and optional craft-fonts glyph +//! outlines; unsupported visible characters become `?` and are counted. The engine's invisible +//! searchable layer uses empty Type3 programs and ToUnicode maps, preserving Unicode without +//! display fonts. WinAnsi-only invisible text keeps the standard Helvetica layout here. +//! +//! The ocrs models read the Latin alphabet (English and other languages written without +//! accents); Tesseract (the `tesseract` module, off the web build) reads whatever its installed +//! language packs cover. #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] -use std::path::{Path, PathBuf}; +pub mod confidence; +pub mod merge; +mod ocrs; +pub mod preprocess; +#[cfg(not(target_arch = "wasm32"))] +pub mod tesseract; +pub use confidence::{ConfidenceBand, band_for, is_suspect, low_confidence}; +pub use merge::{MergeStrategy, StrategyOutcome, mean_confidence, recognize_with_strategy, rover_merge}; +pub use ocrs::{ALPHABET, DETECTION_MODEL, LANGUAGES, Models, Ocr, OcrsRecognizer, RECOGNITION_MODEL}; + +pub use pdfcraft_fonts::TextPlan; pub use pdfcraft_fonts::helvetica_width; -/// The model files, as named in ATTRIBUTION.toml. -pub const DETECTION_MODEL: &str = "text-detection.onnx"; -pub const RECOGNITION_MODEL: &str = "text-recognition.onnx"; +/// Hard cap on either side of an image any code in this crate will touch, in pixels; larger +/// images are rejected with [`OcrError::ImageTooLarge`] instead of being processed. +pub const MAX_SIDE: u32 = 10_000; -/// The recognition model's alphabet, as it was trained (ocrs-models' `DEFAULT_ALPHABET`): its -/// output classes follow this order. The ocrs crate's own copy has an `E` where the training had -/// `€`, which made a scanned "€250" read as "E250". -pub const ALPHABET: &str = " 0123456789!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~€ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; - -/// The languages the models read (ISO 639-1); all use the Latin alphabet without accents. -pub const LANGUAGES: &[(&str, &str)] = &[("en", "English")]; +/// Hard cap on an image's TOTAL area, in pixels (64 MP): sides under [`MAX_SIDE`] can still +/// multiply out to a raster no engine should have to read; larger images are rejected with +/// [`OcrError::ImageTooLarge`] instead of being processed. +pub const MAX_PIXELS: u32 = 64_000_000; #[derive(Debug, thiserror::Error)] pub enum OcrError { - /// Shown to end users (release packages ship the models; a source build fetches them with - /// `cargo xtask models`). The UI catalogs key this exact text. #[error("the text recognition models are not installed (reinstall PdfCraft, or set PDFCRAFT_MODELS to the folder that holds them)")] NoModels, #[error("loading {0}: {1}")] @@ -38,52 +53,93 @@ pub enum OcrError { Recognize(String), #[error("the image is empty")] EmptyImage, + #[error("the image is larger than the {MAX_SIDE}-pixel side or {MAX_PIXELS}-pixel limit")] + ImageTooLarge, + #[error("the image buffer holds {got} bytes, expected {want} for {width}x{height} RGBA")] + ImageSize { got: usize, want: usize, width: u32, height: u32 }, + #[error("the tesseract program is not available: {0}")] + NoTesseract(String), + #[error("tesseract did not finish within {0} seconds")] + Timeout(u64), + #[error("tesseract produced more than {0} bytes of output")] + OutputTooLarge(u64), + #[error("tesseract failed: {0}")] + Process(String), } -/// Where the two model files are. -#[derive(Clone, Debug, PartialEq)] -pub struct Models { - pub detection: PathBuf, - pub recognition: PathBuf, +/// An RGBA8 raster, row-major, top-left origin: what the engines read. The fields are private +/// so the invariants (non-zero, capped, exactly `width * height * 4` bytes) hold by +/// construction: an [`OcrImage`] in hand has always been validated, whatever produced it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct OcrImage { + width: u32, + height: u32, + rgba: Vec, } -impl Models { - /// A complete model pair in `dir`, preferring current ONNX files over legacy RTEN files. - /// An incomplete update never mixes formats or hides an intact legacy pair. - pub fn in_dir(dir: &Path) -> Option { - [(DETECTION_MODEL, RECOGNITION_MODEL), ("text-detection.rten", "text-recognition.rten")].into_iter().find_map(|(detection, recognition)| { - let m = Models { detection: dir.join(detection), recognition: dir.join(recognition) }; - (m.detection.is_file() && m.recognition.is_file()).then_some(m) - }) - } - - /// Look for the models: `$PDFCRAFT_MODELS`, then where the release packages install them - /// beside the executable ([`Models::dirs_beside_exe`]), then the source tree's `assets/models/`. - pub fn find() -> Option { - Self::search_dirs().iter().find_map(|d| Self::in_dir(d)) - } - - /// The directories [`Models::find`] looks in, in order. - pub fn search_dirs() -> Vec { - let mut dirs = Vec::new(); - if let Some(d) = std::env::var_os("PDFCRAFT_MODELS") { - dirs.push(PathBuf::from(d)); +impl OcrImage { + /// Validate the sides ([`MAX_SIDE`]) and total area ([`MAX_PIXELS`]) and that `rgba` holds + /// exactly `width * height * 4` bytes (checked arithmetic; the counts come from documents, + /// so they are untrusted). + pub fn new(width: u32, height: u32, rgba: Vec) -> Result { + if width == 0 || height == 0 { + return Err(OcrError::EmptyImage); } - if let Some(exe) = std::env::current_exe().ok().and_then(|e| e.parent().map(Path::to_path_buf)) { - dirs.extend(Self::dirs_beside_exe(&exe)); + if width > MAX_SIDE || height > MAX_SIDE { + return Err(OcrError::ImageTooLarge); + } + let pixels = (width as usize).checked_mul(height as usize).ok_or(OcrError::ImageTooLarge)?; + if pixels > MAX_PIXELS as usize { + return Err(OcrError::ImageTooLarge); + } + match pixels.checked_mul(4) { + Some(want) if rgba.len() == want => Ok(OcrImage { width, height, rgba }), + Some(want) => Err(OcrError::ImageSize { got: rgba.len(), want, width, height }), + None => Err(OcrError::ImageTooLarge), } - dirs.push(Path::new(env!("CARGO_MANIFEST_DIR")).join("../../assets/models")); - dirs } - /// Where the release packages put the models, relative to the directory holding the - /// executable (packaging/*): - /// - `models/` beside it: the Windows MSI and portable zip; - /// - `../Resources/models`: the macOS app bundle (`Contents/MacOS` → `Contents/Resources`); - /// - `../share/pdfcraft/models`: the FHS-style trees (`bin/` → `share/`) of the Linux - /// deb/rpm/tar.gz/AppImage/Flatpak and the FreeBSD tarball, wherever they are installed. - pub fn dirs_beside_exe(exe_dir: &Path) -> Vec { - vec![exe_dir.join("models"), exe_dir.join("../Resources/models"), exe_dir.join("../share/pdfcraft/models")] + /// The image's width, in pixels. + pub fn width(&self) -> u32 { + self.width + } + + /// The image's height, in pixels. + pub fn height(&self) -> u32 { + self.height + } + + /// The RGBA bytes, row-major, exactly `width * height * 4` of them. + pub fn rgba(&self) -> &[u8] { + &self.rgba + } + + /// The smallest side, in pixels. + pub fn min_side(&self) -> u32 { + self.width.min(self.height) + } + + /// One RGBA byte at integer pixel (`x`, `y`), edges clamped; 255 (white) outside. Only + /// whole-pixel positions and channel 0–3 make sense; anything else reads clamped. + pub fn byte(&self, x: f64, y: f64, ch: usize) -> u8 { + let stride = self.width as usize * 4; + if stride == 0 || self.height == 0 || ch > 3 { + return 255; + } + let xi = (x.round().max(0.0) as usize).min(self.width as usize - 1); + let yi = (y.round().max(0.0) as usize).min(self.height as usize - 1); + self.rgba.get(yi * stride + xi * 4 + ch).copied().unwrap_or(255) + } + + /// The grayscale value of the pixel at (`x`, `y`), 0 (ink) … 255 (paper), with the borders + /// clamped (edges replicate). `None` only when the image is empty. + pub fn gray(&self, x: i64, y: i64) -> Option { + let x = x.clamp(0, self.width as i64 - 1); + let y = y.clamp(0, self.height as i64 - 1); + let row = y as usize * self.width as usize * 4; + let o = row + x as usize * 4; + let px = self.rgba.get(o..o + 4)?; + Some(((299 * px[0] as u32 + 587 * px[1] as u32 + 114 * px[2] as u32) / 1000) as u8) } } @@ -92,6 +148,26 @@ impl Models { pub struct Word { pub text: String, pub rect: [f32; 4], + /// How sure the engine is, 0–100, when it reports a confidence at all (`None`: the ocrs + /// engine does not). + pub confidence: Option, + /// Which engine read this word: the recogniser's [`Recognizer::id`], or `"ROVER"` when two + /// engines were merged and the surviving text is the merged choice. + pub source: String, +} + +impl Word { + pub fn new(text: impl Into, rect: [f32; 4], source: &str) -> Word { + Word { text: text.into(), rect, confidence: None, source: source.into() } + } + + /// Set the confidence (builder style), clamped to the documented 0–100 scale (a negative + /// value is 0, anything over 100 is 100). NaN means no confidence was reported (`None`), + /// the same unknown the engines that report nothing carry. + pub fn with_confidence(mut self, confidence: f32) -> Word { + self.confidence = if confidence.is_nan() { None } else { Some(confidence.clamp(0.0, 100.0)) }; + self + } } /// A recognised line of words, in reading order. @@ -106,63 +182,34 @@ impl Line { } } -/// A loaded recogniser. Loading takes a moment; keep one and reuse it. -pub struct Ocr { - engine: ocrs::OcrEngine, +/// What a recogniser needs to know beyond the pixels. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct RecognizeOptions { + /// Language code from [`LANGUAGES`] (`"en"`, `"de"`, …). Engines map it to their own codes + /// (Tesseract: `tesseract::language_code`, allow-listed before anything reaches a process); + /// unknown codes fall back to the engine's default. + pub language: String, } -impl Ocr { - pub fn load(models: &Models) -> Result { - let load = |p: &Path| rten::Model::load_file(p).map_err(|e| OcrError::Load(p.display().to_string(), e.to_string())); - let params = ocrs::OcrEngineParams { - detection_model: Some(load(&models.detection)?), - recognition_model: Some(load(&models.recognition)?), - alphabet: Some(ALPHABET.into()), - ..Default::default() - }; - let engine = ocrs::OcrEngine::new(params).map_err(|e| OcrError::Load("ocr engine".into(), e.to_string()))?; - Ok(Ocr { engine }) +impl Default for RecognizeOptions { + fn default() -> Self { + RecognizeOptions { language: "en".into() } } +} - /// Load the models found by [`Models::find`]. - pub fn find() -> Result { - Self::load(&Models::find().ok_or(OcrError::NoModels)?) - } - - /// Recognise the text in an RGBA (or RGB, or grey) image, `width` × `height` pixels. - pub fn recognize(&self, pixels: &[u8], width: u32, height: u32) -> Result, OcrError> { - if width == 0 || height == 0 || pixels.is_empty() { - return Err(OcrError::EmptyImage); - } - // ocrs wants 1 or 3 channels. - let rgb: std::borrow::Cow<[u8]> = if pixels.len() == (width * height * 4) as usize { - pixels.as_chunks::<4>().0.iter().flat_map(|p| [p[0], p[1], p[2]]).collect::>().into() - } else { - pixels.into() - }; - let err = |e: &dyn std::fmt::Display| OcrError::Recognize(e.to_string()); - let source = ocrs::ImageSource::from_bytes(&rgb, (width, height)).map_err(|e| err(&e))?; - let input = self.engine.prepare_input(source).map_err(|e| err(&e))?; - let found = self.engine.detect_words(&input).map_err(|e| err(&e))?; - let lines = self.engine.find_text_lines(&input, &found); - let read = self.engine.recognize_text(&input, &lines).map_err(|e| err(&e))?; - use ocrs::TextItem; - Ok(read - .into_iter() - .flatten() - .map(|line| Line { - words: line - .words() - .map(|w| { - let r = w.bounding_rect(); - Word { text: w.to_string(), rect: [r.left() as f32, r.top() as f32, r.right() as f32, r.bottom() as f32] } - }) - .filter(|w| !w.text.trim().is_empty()) - .collect(), - }) - .filter(|l| !l.words.is_empty()) - .collect()) - } +/// One text-recognition engine. Implementations are shared across threads +/// (`Arc`); `available` and `recognize` may spawn processes only off the web +/// build (see the `tesseract` module). +pub trait Recognizer: Send + Sync { + /// The engine's short name (`"ocrs"`, `"tesseract"`): what [`Word::source`] carries and + /// what settings and tools name it by. + fn id(&self) -> &'static str; + /// Whether this engine can run right now (models installed, binary found). + fn available(&self) -> bool; + /// The languages it reads, as `(code, name)` pairs (`"en"`, `"English"`). + fn languages(&self) -> Vec<(String, String)>; + /// Read the words in `image`, in reading order. + fn recognize(&self, image: &OcrImage, options: &RecognizeOptions) -> Result, OcrError>; } /// A word placed on a page, in PDF user space: the bottom-left corner of its box, and the @@ -173,6 +220,10 @@ pub struct PlacedWord { pub origin: [f64; 2], pub across: [f64; 2], pub up: [f64; 2], + /// Confidence 0–100, when the engine reported one. + pub confidence: Option, + /// Which engine read this word (see [`Word::source`]). + pub source: String, } impl PlacedWord { @@ -183,7 +234,14 @@ impl PlacedWord { let o = to_user(l, b); let br = to_user(r, b); let tl = to_user(l, t); - PlacedWord { text: word.text.clone(), origin: o, across: [br[0] - o[0], br[1] - o[1]], up: [tl[0] - o[0], tl[1] - o[1]] } + PlacedWord { + text: word.text.clone(), + origin: o, + across: [br[0] - o[0], br[1] - o[1]], + up: [tl[0] - o[0], tl[1] - o[1]], + confidence: word.confidence, + source: word.source.clone(), + } } } @@ -214,32 +272,193 @@ fn num(v: f64) -> String { if s == "-0" { "0".into() } else { s.into() } } -/// Page content that writes `words` as invisible text in `/PCHelv` (standard Helvetica, -/// WinAnsiEncoding), each word stretched to its box so selection and search highlight the -/// right place. Marked content `/OCR` so it can be told apart from the page's own text. -pub fn text_layer(words: &[PlacedWord]) -> Vec { +/// One run of a word's planned characters in one Type3 font: the resource name, the codes to +/// show (one byte per character), and the run's start as an offset in em units from the word's +/// origin (the accumulated advances of the runs before it). +struct PlannedRun { + font: String, + codes: Vec, + start: f64, +} + +/// The planned runs of word `i` (the same index [`pdfcraft_fonts::TextPlan::for_words`] planned +/// the words with), or `None` when the word is drawn in Helvetica: every character has a +/// WinAnsi code, or nothing could be planned/registered for it (the pre-Unicode output). +fn planned_runs(plan: &pdfcraft_fonts::TextPlan, word: usize) -> Option<(Vec, f64)> { + let enc = plan.encoding(word); + if enc.is_empty() || !plan.registered(enc) { + return None; + } + let mut runs: Vec = Vec::new(); + let mut total = 0.0; + for &(f, code) in enc { + let font = plan.fonts[f].name.as_str(); + match runs.last_mut() { + Some(last) if last.font == font => last.codes.push(code), + _ => runs.push(PlannedRun { font: font.to_string(), codes: vec![code], start: total }), + } + total += plan.advance(f, code); + } + (total.is_finite() && total > 0.0).then_some((runs, total)) +} + +/// Page content that writes `words` as VISIBLE text at their original boxes: the +/// editable-text output's page content. WinAnsi words are set in `/PCHelv` (standard +/// Helvetica); words with characters beyond WinAnsi are set in the plan's Type3 fonts, one run +/// per font. Each word fills its box — the font size comes from the box height and the +/// horizontal scaling is computed from the real advances of the font that draws the word +/// (Helvetica's approximated, the Type3 faces' exact) and emitted as an explicit `Tz` +/// percentage. Words whose box is empty (or whose text has no measurable width) are skipped; a +/// page that ends up with nothing simply shows nothing. +pub fn visible_text_layer(words: &[PlacedWord], plan: &pdfcraft_fonts::TextPlan) -> Vec { + visible_text_layer_at(words, plan, 0) +} + +/// Visible text for words starting at `word_offset` in a shared document plan. +pub fn visible_text_layer_at(words: &[PlacedWord], plan: &pdfcraft_fonts::TextPlan, word_offset: usize) -> Vec { + // The rendering mode and the font are graphics state, which `ET` doesn't reset: without + // `q … Q` they would stay set for every later content stream on the page, so text added + // after OCR (a header or footer) drew invisibly (#3029). + let mut out = b"/OCR BMC\nq\nBT\n0 Tr\n/PCHelv 1 Tf\n".to_vec(); + // The font and size the text state is set in, so a Tf is written only when they change. + // (The header's size-1 Helvetica covers layers of WinAnsi words only.) + let mut current: Option<(String, f64)> = Some(("PCHelv".to_string(), 1.0)); + for (i, w) in words.iter().enumerate() { + let height = w.up[0].hypot(w.up[1]); + let across = w.across[0].hypot(w.across[1]); + let (ux, uy) = (w.across[0] / across, w.across[1] / across); + let (vx, vy) = (w.up[0] / height, w.up[1] / height); + if !across.is_finite() + || !height.is_finite() + || across <= 0.0 + || height <= 0.0 + || !ux.is_finite() + || !vx.is_finite() + || !w.origin.iter().all(|v| v.is_finite()) + { + continue; + } + let size = height / BOX_EM; + let runs = planned_runs(plan, word_offset.saturating_add(i)); + // The word's natural advance at this size: Helvetica's approximation, or the exact sum + // of the planned glyphs' widths. + let natural = runs.as_ref().map_or_else(|| helvetica_width(&w.text, size), |(_, total)| total * size); + if natural.is_nan() || natural <= 0.0 { + continue; + } + // Horizontal scaling as a percentage of the natural advance at this size, so the word + // lands exactly on its box (100 = no scaling; a long box on a short word stretches it). + let tz = (across / natural * 100.0).max(1.0); + // The baseline sits above the box's bottom edge by the descender depth. + let o = [w.origin[0] + vx * DESCENT * size, w.origin[1] + vy * DESCENT * size]; + out.extend_from_slice(format!("{} Tz ", num(tz)).as_bytes()); + match &runs { + None => { + if current.as_ref().is_some_and(|(f, s)| f != "PCHelv" || *s != size) { + out.extend_from_slice(format!("/PCHelv {} Tf ", num(size)).as_bytes()); + current = Some(("PCHelv".to_string(), size)); + } + out.extend_from_slice(format!("{} {} {} {} {} {} Tm ", num(ux), num(uy), num(vx), num(vy), num(o[0]), num(o[1])).as_bytes()); + out.extend(pdfcraft_fonts::literal(&pdfcraft_fonts::win_ansi(&w.text))); + out.extend_from_slice(b" Tj\n"); + } + Some((planned, total)) => { + for run in planned { + if current.as_ref().is_some_and(|(f, s)| *f != run.font || *s != size) { + out.extend_from_slice(format!("/{} {} Tf ", run.font, num(size)).as_bytes()); + current = Some((run.font.clone(), size)); + } + // The run's origin: the word's baseline plus the across vector scaled by + // the advances before this run (the stretch applies to runs alike). + let frac = run.start / total; + out.extend_from_slice( + format!( + "{} {} {} {} {} {} Tm ", + num(ux), + num(uy), + num(vx), + num(vy), + num(o[0] + w.across[0] * frac), + num(o[1] + w.across[1] * frac) + ) + .as_bytes(), + ); + out.extend(pdfcraft_fonts::literal(&run.codes)); + out.extend_from_slice(b" Tj\n"); + } + } + } + } + out.extend_from_slice(b"ET\nQ\nEMC\n"); + out +} + +/// Page content that writes `words` as invisible text (rendering mode 3), each word stretched +/// to its box so selection and search highlight the right place. WinAnsi words are set in +/// `/PCHelv` (standard Helvetica); words with characters beyond WinAnsi are set in the plan's +/// Type3 fonts, one run per font, with the same exact box fit. Marked content `/OCR` so it can +/// be told apart from the page's own text. +pub fn text_layer(words: &[PlacedWord], plan: &pdfcraft_fonts::TextPlan) -> Vec { // `3 Tr` and the font are graphics state, which `ET` doesn't reset: without `q … Q` they // would stay set for every later content stream on the page, so text added after OCR (a // header or footer) drew invisibly (#3029). let mut out = b"/OCR BMC\nq\nBT\n3 Tr\n/PCHelv 1 Tf\n".to_vec(); - for w in words { - let width = helvetica_width(&w.text, 1.0); + let mut current: Option<(String, f64)> = Some(("PCHelv".to_string(), 1.0)); + for (i, w) in words.iter().enumerate() { + let runs = planned_runs(plan, i); + let width = runs.as_ref().map_or_else(|| helvetica_width(&w.text, 1.0), |(_, total)| *total); let height = w.up[0].hypot(w.up[1]); - if width <= 0.0 || height <= 0.0 { + let across = w.across[0].hypot(w.across[1]); + // Only positive, finite geometry reaches the content stream: a zero-sized box cannot + // be drawn, and a NaN or infinite one would put non-finite numbers on the page. (NaN + // fails every comparison, so the positivity checks already refuse it; the finiteness + // checks keep that property explicit.) + if width <= 0.0 || !(height > 0.0 && height.is_finite()) || !(across > 0.0 && across.is_finite()) || !w.origin.iter().all(|v| v.is_finite()) { continue; } - // The em square: the word's own ink (cap height or x-height, descenders or not) spans the - // box, so the text is as tall as the scanned word; its width is stretched to fit the word. - let (top, bottom) = ink_extent(&w.text); + // The em square. Helvetica words: the word's own ink (cap height or x-height, + // descenders or not) spans the box, so the text is as tall as the scanned word. Planned + // (Type3) words: those Helvetica letter heights do not describe the face, so the box spans + // the ascender-descender height. Either way the width is stretched to fit the word. + let (top, bottom) = if runs.is_some() { (BOX_EM - DESCENT, -DESCENT) } else { ink_extent(&w.text) }; let em = [w.up[0] / (top - bottom), w.up[1] / (top - bottom)]; - let ax = [w.across[0] / width, w.across[1] / width]; let o = [w.origin[0] - em[0] * bottom, w.origin[1] - em[1] * bottom]; - let m = [ax[0], ax[1], em[0], em[1], o[0], o[1]]; - let nums: Vec = m.iter().map(|v| num(*v)).collect(); - out.extend_from_slice(nums.join(" ").as_bytes()); - out.extend_from_slice(b" Tm "); - out.extend(pdfcraft_fonts::literal(&pdfcraft_fonts::win_ansi(&w.text))); - out.extend_from_slice(b" Tj\n"); + match &runs { + None => { + let ax = [w.across[0] / width, w.across[1] / width]; + if current.as_ref().is_some_and(|(f, s)| f != "PCHelv" || *s != 1.0) { + out.extend_from_slice(b"/PCHelv 1 Tf "); + current = Some(("PCHelv".to_string(), 1.0)); + } + let m = [ax[0], ax[1], em[0], em[1], o[0], o[1]]; + let nums: Vec = m.iter().map(|v| num(*v)).collect(); + out.extend_from_slice(nums.join(" ").as_bytes()); + out.extend_from_slice(b" Tm "); + out.extend(pdfcraft_fonts::literal(&pdfcraft_fonts::win_ansi(&w.text))); + out.extend_from_slice(b" Tj\n"); + } + Some((planned, total)) => { + for run in planned { + if current.as_ref().is_some_and(|(f, s)| *f != run.font || *s != 1.0) { + out.extend_from_slice(format!("/{} 1 Tf ", run.font).as_bytes()); + current = Some((run.font.clone(), 1.0)); + } + let m = [ + w.across[0] / total, + w.across[1] / total, + em[0], + em[1], + o[0] + w.across[0] * run.start / total, + o[1] + w.across[1] * run.start / total, + ]; + let nums: Vec = m.iter().map(|v| num(*v)).collect(); + out.extend_from_slice(nums.join(" ").as_bytes()); + out.extend_from_slice(b" Tm "); + out.extend(pdfcraft_fonts::literal(&run.codes)); + out.extend_from_slice(b" Tj\n"); + } + } + } } out.extend_from_slice(b"ET\nQ\nEMC\n"); out @@ -253,10 +472,22 @@ mod tests { fn places_words_through_the_mapping() { // 2 pixels per point, page 612 × 792, y flipped. let to_user = |x: f32, y: f32| [x as f64 / 2.0, 792.0 - y as f64 / 2.0]; - let w = PlacedWord::place(&Word { text: "Hello".into(), rect: [100.0, 200.0, 300.0, 240.0] }, to_user); + let w = PlacedWord::place(&Word::new("Hello", [100.0, 200.0, 300.0, 240.0], "ocrs"), to_user); assert_eq!(w.origin, [50.0, 672.0]); assert_eq!(w.across, [100.0, 0.0]); assert_eq!(w.up, [0.0, 20.0]); + assert_eq!(w.source, "ocrs"); + assert_eq!(w.confidence, None); + } + + /// Confidence and source survive placement. + #[test] + fn placed_words_carry_confidence_and_source() { + let to_user = |x: f32, y: f32| [x as f64, y as f64]; + let w = Word::new("Hi", [1.0, 2.0, 3.0, 4.0], "tesseract").with_confidence(87.5); + let p = PlacedWord::place(&w, to_user); + assert_eq!(p.confidence, Some(87.5)); + assert_eq!(p.source, "tesseract"); } /// The text layer is Helvetica in WinAnsiEncoding: every character the model can recognise @@ -267,10 +498,27 @@ mod tests { assert_eq!(pdfcraft_fonts::first_undrawable(ALPHABET), None); } + /// `with_confidence` is clamped to the documented 0–100 scale: negatives (and −inf) are 0, + /// anything over 100 (and +inf) is 100, in-scale values pass through, and NaN means no + /// confidence was reported — the same `None` unknown the unreporting engines carry. + #[test] + fn with_confidence_clamps_and_treats_nan_as_unknown() { + let c = |v: f32| Word::new("w", [0.0; 4], "ocrs").with_confidence(v).confidence; + assert_eq!(c(0.0), Some(0.0)); + assert_eq!(c(50.0), Some(50.0)); + assert_eq!(c(100.0), Some(100.0)); + assert_eq!(c(-1.0), Some(0.0), "negatives clamp to 0"); + assert_eq!(c(f32::NEG_INFINITY), Some(0.0)); + assert_eq!(c(150.0), Some(100.0), "over 100 clamps to 100"); + assert_eq!(c(f32::INFINITY), Some(100.0)); + assert_eq!(c(f32::NAN), None, "NaN is unknown, not a confidence"); + assert_eq!(c(f32::EPSILON), Some(f32::EPSILON), "in-scale values are untouched"); + } + #[test] fn text_layer_is_invisible_text_fitted_to_each_box() { - let w = PlacedWord { text: "Hi".into(), origin: [10.0, 20.0], across: [30.0, 0.0], up: [0.0, 9.3] }; - let s = String::from_utf8(text_layer(&[w])).unwrap(); + let w = PlacedWord { text: "Hi".into(), origin: [10.0, 20.0], across: [30.0, 0.0], up: [0.0, 9.3], confidence: None, source: "ocrs".into() }; + let s = String::from_utf8(text_layer(&[w], &TextPlan::default())).unwrap(); assert!(s.contains("3 Tr") && s.contains("/PCHelv 1 Tf"), "{s}"); let width = helvetica_width("Hi", 1.0); // "Hi" has no descender: its box spans the cap height, from the baseline up. @@ -283,8 +531,8 @@ mod tests { /// after it on the page starts from the graphics state it had before. #[test] fn the_layer_restores_the_graphics_state_it_changes() { - let w = PlacedWord { text: "Hi".into(), origin: [10.0, 20.0], across: [30.0, 0.0], up: [0.0, 9.3] }; - let s = String::from_utf8(text_layer(&[w])).unwrap(); + let w = PlacedWord { text: "Hi".into(), origin: [10.0, 20.0], across: [30.0, 0.0], up: [0.0, 9.3], confidence: None, source: "ocrs".into() }; + let s = String::from_utf8(text_layer(&[w], &TextPlan::default())).unwrap(); let mut depth = 0_i32; for token in s.split_ascii_whitespace() { match token { @@ -309,78 +557,200 @@ mod tests { assert_eq!(ink_extent("Gallus.Ze@camtel.cm"), (CAP_HEIGHT, 0.0)); assert_eq!(ink_extent("Corporation"), (CAP_HEIGHT, -DESCENT)); assert_eq!(ink_extent("--"), (BOX_EM - DESCENT, -DESCENT)); - let w = PlacedWord { text: "gap".into(), origin: [0.0, 100.0], across: [20.0, 0.0], up: [0.0, 7.3] }; - let s = String::from_utf8(text_layer(&[w])).unwrap(); + let w = PlacedWord { text: "gap".into(), origin: [0.0, 100.0], across: [20.0, 0.0], up: [0.0, 7.3], confidence: None, source: "ocrs".into() }; + let s = String::from_utf8(text_layer(&[w], &TextPlan::default())).unwrap(); let em = 7.3 / (X_HEIGHT + DESCENT); assert!(s.contains(&format!(" 0 0 {} 0 {} Tm (gap) Tj", num(em), num(100.0 + em * DESCENT))), "{s}"); } + /// A word whose box is not positive, finite geometry never reaches the content stream: + /// NaN, infinite and zero-sized boxes are dropped, so the layer never carries non-finite + /// numbers. + #[test] + fn text_layer_refuses_non_finite_and_degenerate_boxes() { + let word = |across: [f64; 2], up: [f64; 2], origin: [f64; 2]| PlacedWord { + text: "Hi".into(), + origin, + across, + up, + confidence: Some(90.0), + source: "tesseract".into(), + }; + for (why, w) in [ + ("NaN across", word([f64::NAN, 0.0], [0.0, 9.3], [0.0; 2])), + ("NaN up", word([30.0, 0.0], [f64::NAN, 1.0], [0.0; 2])), + ("inf up", word([30.0, 0.0], [0.0, f64::INFINITY], [0.0; 2])), + ("inf origin", word([30.0, 0.0], [0.0, 9.3], [0.0, f64::INFINITY])), + ("zero across", word([0.0, 0.0], [0.0, 9.3], [0.0; 2])), + ("NaN origin", word([30.0, 0.0], [0.0, 9.3], [0.0, f64::NAN])), + ] { + let s = String::from_utf8(text_layer(&[w], &TextPlan::default())).unwrap(); + assert_eq!(s, "/OCR BMC\nq\nBT\n3 Tr\n/PCHelv 1 Tf\nET\nQ\nEMC\n", "{why}: {s}"); + assert!(!s.contains("NaN") && !s.contains("inf"), "{why}: {s}"); + } + } + + /// The editable-text layer draws visible text whose horizontal scaling is an explicit + /// percentage computed from real Helvetica metrics: `Tz` × natural advance = box width. + #[test] + fn visible_text_layer_scales_by_a_percentage_of_real_metrics() { + let w = PlacedWord { text: "Hi".into(), origin: [10.0, 20.0], across: [40.0, 0.0], up: [0.0, 9.3], confidence: None, source: "ocrs".into() }; + let s = String::from_utf8(visible_text_layer(&[w], &TextPlan::default())).unwrap(); + assert!(s.contains("0 Tr"), "visible, not invisible: {s}"); + let size = 9.3 / BOX_EM; + let natural = helvetica_width("Hi", size); + let tz = 40.0 / natural * 100.0; + assert!(s.contains(&format!("{} Tz", num(tz))), "{s}"); + assert!(s.contains("(Hi) Tj"), "{s}"); + assert!(s.starts_with("/OCR BMC") && s.ends_with("EMC\n")); + } + + /// Degenerate boxes and words with no measurable width are skipped, never divide by zero. + #[test] + fn visible_text_layer_skips_degenerate_words() { + let word = |across: [f64; 2], up: [f64; 2], text: &str| PlacedWord { + text: text.into(), + origin: [0.0; 2], + across, + up, + confidence: None, + source: "ocrs".into(), + }; + let words = vec![ + word([0.0, 0.0], [0.0, 9.0], "flat"), + word([10.0, 0.0], [0.0, 0.0], "short"), + word([f64::NAN, 0.0], [0.0, 9.0], "nan"), + word([10.0, 0.0], [0.0, 9.0], ""), + word([10.0, 0.0], [0.0, 9.0], "Hi"), + ]; + let s = String::from_utf8(visible_text_layer(&words, &TextPlan::default())).unwrap(); + assert_eq!(s.matches("Tj").count(), 1, "only the one drawable word is written: {s}"); + } + + /// A word whose ORIGIN is not finite never reaches the visible layer either: a NaN or + /// infinite origin would put "NaN"/"inf" into the content stream. + #[test] + fn visible_text_layer_refuses_non_finite_origins() { + let word = + |origin: [f64; 2]| PlacedWord { text: "Hi".into(), origin, across: [10.0, 0.0], up: [0.0, 9.0], confidence: None, source: "ocrs".into() }; + for (why, origin) in [("NaN x", [f64::NAN, 100.0]), ("inf y", [10.0, f64::INFINITY]), ("NaN both", [f64::NAN; 2])] { + let s = String::from_utf8(visible_text_layer(&[word(origin)], &TextPlan::default())).unwrap(); + assert!(!s.contains("NaN") && !s.contains("inf"), "{why}: {s}"); + assert_eq!(s.matches("Tj").count(), 0, "{why}: {s}"); + } + } + + /// After a Type3 word the text state goes back to Helvetica: the Latin words around a + /// planned word must not inherit its font. + #[test] + fn helvetica_is_restored_after_a_planned_word() { + if pdfcraft_fonts::CRAFT_FONTS.is_empty() { + eprintln!("skipping: built without craft-fonts (set CRAFT_FONTS_DIR to run it)"); + return; + } + let word = |text: &str| PlacedWord { + text: text.into(), + origin: [10.0, 20.0], + across: [60.0, 0.0], + up: [0.0, 9.3], + confidence: None, + source: "ocrs".into(), + }; + let words = [word("Hi"), word("日本"), word("Ho")]; + let mut plan = TextPlan::for_words(["Hi", "日本", "Ho"]); + for (i, font) in plan.fonts.iter_mut().enumerate() { + font.name = format!("PCOcr{i}"); + } + let s = String::from_utf8(text_layer(&words, &plan)).unwrap(); + assert!(s.contains("/PCOcr0 1 Tf"), "{s}"); + // The last Helvetica selection comes after the Type3 run, and the Latin word after it + // is drawn in Helvetica, not left in the planned font. + assert!(s.rfind("/PCOcr0 1 Tf").unwrap_or(0) < s.rfind("/PCHelv 1 Tf").unwrap_or(usize::MAX), "{s}"); + // The `Q` closes the `q … Q` scope that keeps `3 Tr` from leaking into later content. + assert!(s.ends_with("Tm (Ho) Tj\nET\nQ\nEMC\n"), "{s}"); + // The visible layer restores Helvetica with the word's size. + let v = String::from_utf8(visible_text_layer(&words, &plan)).unwrap(); + assert!(v.rfind("/PCOcr0").unwrap_or(0) < v.rfind("/PCHelv").unwrap_or(usize::MAX), "{v}"); + assert!(v.contains("(Ho) Tj"), "{v}"); + } + + /// Words with characters beyond WinAnsi draw in the plan's Type3 font, one run, with the + /// same exact box fit as the Helvetica words (the matrix x-axis spans across / advance). + #[test] + fn planned_words_draw_in_their_font_with_the_same_exact_fit() { + if pdfcraft_fonts::CRAFT_FONTS.is_empty() { + eprintln!("skipping: built without craft-fonts (set CRAFT_FONTS_DIR to run it)"); + return; + } + let w = + PlacedWord { text: "日本".into(), origin: [10.0, 20.0], across: [60.0, 0.0], up: [0.0, 9.3], confidence: None, source: "ocrs".into() }; + let mut plan = TextPlan::for_words(["日本"]); + // The names the registrar (pdfcraft_edit::add_type3_fonts) fills in — this crate + // cannot reach it, so the test sets them where it reads them. + for (i, font) in plan.fonts.iter_mut().enumerate() { + font.name = format!("PCOcr{i}"); + } + let codes: Vec = plan.encoding(0).iter().map(|&(_, c)| c).collect(); + assert_eq!(codes.len(), 2, "{codes:?}"); + let total = plan.advance_of(plan.encoding(0)); + let s = String::from_utf8(text_layer(std::slice::from_ref(&w), &plan)).unwrap(); + assert!(s.contains("/PCOcr0 1 Tf"), "{s}"); + assert!(s.contains(&format!("{} 0 0 10 10 22.1 Tm ", num(60.0 / total))), "{s}"); + assert!(s.contains(&format!("Tm {} Tj\n", String::from_utf8_lossy(&pdfcraft_fonts::literal(&codes)))), "{s}"); + // The visible layer: size from the box height, Tz from the Type3's own advance. + let v = String::from_utf8(visible_text_layer(std::slice::from_ref(&w), &plan)).unwrap(); + let size = 9.3 / BOX_EM; + assert!(v.contains(&format!("/PCOcr0 {} Tf ", num(size))), "{v}"); + assert!(v.contains(&format!("{} Tz", num(60.0 / (total * size) * 100.0))), "{v}"); + } + + /// Without planned characters (an empty plan: built without craft-fonts, or before the + /// fonts are registered) the WinAnsi path stands, '?' included. + #[test] + fn unplanned_words_fall_back_to_win_ansi_question_marks() { + let w = + PlacedWord { text: "日本".into(), origin: [10.0, 20.0], across: [60.0, 0.0], up: [0.0, 9.3], confidence: None, source: "ocrs".into() }; + let s = String::from_utf8(text_layer(&[w], &TextPlan::default())).unwrap(); + assert!(s.contains("(??) Tj"), "{s}"); + assert!(!s.contains("PCOcr"), "{s}"); + } + + /// A character no face has a glyph for becomes the font's '?', still in the run, counted + /// by the plan. + #[test] + fn substitutions_are_written_as_the_font_s_question_mark() { + if pdfcraft_fonts::CRAFT_FONTS.is_empty() { + eprintln!("skipping: built without craft-fonts (set CRAFT_FONTS_DIR to run it)"); + return; + } + let w = PlacedWord { + text: "日\u{1f4a9}語".into(), + origin: [0.0, 0.0], + across: [60.0, 0.0], + up: [0.0, 9.3], + confidence: None, + source: "ocrs".into(), + }; + let mut plan = TextPlan::for_words(["日\u{1f4a9}語"]); + for (i, font) in plan.fonts.iter_mut().enumerate() { + font.name = format!("PCOcr{i}"); + } + assert_eq!(plan.missing(), 1, "the emoji has no glyph anywhere"); + assert_eq!(plan.encoding(0).len(), 3, "it still gets an encoding: the font's ?"); + let codes: Vec = plan.encoding(0).iter().map(|&(_, c)| c).collect(); + let s = String::from_utf8(text_layer(&[w], &plan)).unwrap(); + assert!(s.contains(&format!("Tm {} Tj\n", String::from_utf8_lossy(&pdfcraft_fonts::literal(&codes)))), "{s}"); + } + #[test] fn rotated_pages_keep_the_reading_direction() { // A page turned 90°: image x runs up the page. let to_user = |x: f32, y: f32| [y as f64, x as f64]; - let w = PlacedWord::place(&Word { text: "Up".into(), rect: [0.0, 0.0, 50.0, 10.0] }, to_user); + let w = PlacedWord::place(&Word::new("Up", [0.0, 0.0, 50.0, 10.0], "ocrs"), to_user); assert_eq!(w.across, [0.0, 50.0]); assert_eq!(w.up, [-10.0, 0.0]); } - /// The current ONNX pair wins when an installation also retains both legacy models. - #[test] - fn model_discovery_prefers_the_complete_onnx_pair() { - let dir = tempfile::tempdir().unwrap(); - for file in ["text-detection.onnx", "text-recognition.onnx"] { - std::fs::write(dir.path().join(file), b"model").unwrap(); - } - let expected = Some(Models { detection: dir.path().join("text-detection.onnx"), recognition: dir.path().join("text-recognition.onnx") }); - assert_eq!(Models::in_dir(dir.path()), expected, "a complete ONNX pair is sufficient"); - for file in ["text-detection.rten", "text-recognition.rten"] { - std::fs::write(dir.path().join(file), b"model").unwrap(); - } - assert_eq!(Models::in_dir(dir.path()), expected, "the complete ONNX pair takes precedence over legacy models"); - } - - /// An interrupted ONNX update still permits an intact legacy pair in the same directory. - #[test] - fn model_discovery_falls_back_to_legacy_when_the_onnx_pair_is_incomplete() { - for current in ["text-detection.onnx", "text-recognition.onnx"] { - let dir = tempfile::tempdir().unwrap(); - for file in [current, "text-detection.rten", "text-recognition.rten"] { - std::fs::write(dir.path().join(file), b"model").unwrap(); - } - assert_eq!( - Models::in_dir(dir.path()), - Some(Models { detection: dir.path().join("text-detection.rten"), recognition: dir.path().join("text-recognition.rten") }), - "{current} alone must not hide a complete legacy pair" - ); - } - } - - /// Detection and recognition must come from one complete format pair, never a mixed update. - #[test] - fn model_discovery_refuses_mixed_format_pairs() { - for pair in [["text-detection.onnx", "text-recognition.rten"], ["text-detection.rten", "text-recognition.onnx"]] { - let dir = tempfile::tempdir().unwrap(); - for file in pair { - std::fs::write(dir.path().join(file), b"model").unwrap(); - } - assert_eq!(Models::in_dir(dir.path()), None, "{pair:?} is not a complete model pair"); - } - } - - /// Existing source builds and PDFCRAFT_MODELS overrides retain their complete RTEN pair. - #[test] - fn model_discovery_keeps_legacy_only_installations() { - let dir = tempfile::tempdir().unwrap(); - assert_eq!(Models::in_dir(dir.path()), None); - std::fs::write(dir.path().join("text-detection.rten"), b"model").unwrap(); - assert_eq!(Models::in_dir(dir.path()), None, "one legacy model is incomplete"); - std::fs::write(dir.path().join("text-recognition.rten"), b"model").unwrap(); - assert_eq!( - Models::in_dir(dir.path()), - Some(Models { detection: dir.path().join("text-detection.rten"), recognition: dir.path().join("text-recognition.rten") }) - ); - } - /// #103: every release package's layout puts the models where the installed app looks. /// Mirrors packaging/{windows/package.ps1,macos/package.sh,linux/package.sh,freebsd/package.sh}. #[test] @@ -443,6 +813,47 @@ mod tests { assert!(word.rect[0] >= 10.0 && word.rect[0] < 60.0, "{word:?}"); } + /// The recognised words name their engine. + #[test] + fn ocrs_recognizer_names_its_words() { + let Ok(rec) = OcrsRecognizer::find() else { + eprintln!("skipped: OCR models not installed"); + return; + }; + assert_eq!(rec.id(), "ocrs"); + assert!(rec.languages().iter().any(|(c, n)| c == "en" && n == "English"), "{:?}", rec.languages()); + let (w, h, px) = test_image::hello(); + let image = OcrImage::new(w, h, px).unwrap(); + let lines = rec.recognize(&image, &RecognizeOptions::default()).unwrap(); + let words: Vec<&Word> = lines.iter().flat_map(|l| &l.words).collect(); + assert!(!words.is_empty()); + assert!(words.iter().all(|w| w.source == "ocrs"), "{words:?}"); + } + + /// Hostile images are rejected with errors, never panics: zero sides, sides over the cap, + /// totals over the area cap (even with legal sides), buffers that do not fit, counts that + /// overflow. + #[test] + fn hostile_image_dimensions_are_rejected_not_panicked_at() { + assert!(matches!(OcrImage::new(0, 10, vec![]), Err(OcrError::EmptyImage))); + assert!(matches!(OcrImage::new(10, 0, vec![]), Err(OcrError::EmptyImage))); + assert!(matches!(OcrImage::new(u32::MAX, u32::MAX, vec![]), Err(OcrError::ImageTooLarge))); + assert!(matches!(OcrImage::new(MAX_SIDE + 1, 1, vec![0; 4]), Err(OcrError::ImageTooLarge))); + // 10 000 × 7 000 = 70 MP: legal on both sides, over the 64 MP area cap (no allocation + // is needed to reject it — the buffer is never inspected). + assert!(matches!(OcrImage::new(MAX_SIDE, 7000, vec![]), Err(OcrError::ImageTooLarge))); + // 8 000 × 8 000 = exactly the area cap: accepted shape (the short buffer proves the + // cap check passed), while one more row is refused. + assert!(matches!(OcrImage::new(8000, 8000, vec![0; 16]), Err(OcrError::ImageSize { .. }))); + assert!(matches!(OcrImage::new(8000, 8001, vec![0; 16]), Err(OcrError::ImageTooLarge))); + assert!(matches!(OcrImage::new(2, 2, vec![0; 15]), Err(OcrError::ImageSize { .. }))); + let ok = OcrImage::new(2, 2, vec![255; 16]).unwrap(); + assert_eq!((ok.width(), ok.height()), (2, 2)); + assert_eq!(ok.rgba().len(), 16); + assert_eq!(ok.gray(5, 5), Some(255), "clamped reads stay inside"); + assert_eq!(ok.gray(-1, -1), Some(255)); + } + mod test_image { /// "HELLO WORLD" drawn with blocky 5×7 letters, 4 pixels per dot, black on white. pub fn hello() -> (u32, u32, Vec) { diff --git a/crates/ocr/src/merge.rs b/crates/ocr/src/merge.rs new file mode 100644 index 0000000000..58fc17a4e5 --- /dev/null +++ b/crates/ocr/src/merge.rs @@ -0,0 +1,583 @@ +//! Combining engines. "ROVER" merges two engines' readings of one page: every primary word +//! adopts the best-overlapping unused secondary word when their boxes agree enough (IoU, below); +//! the higher confidence wins the text, box and confidence (a tie keeps the primary; against a +//! primary of unknown confidence the secondary must reach [`UNKNOWN_CONFIDENCE_BAR`]) and the +//! surviving word is named `"ROVER"`; secondary words nothing claimed are appended, keeping +//! their own engine's name. + +use crate::{Line, OcrError, OcrImage, RecognizeOptions, Recognizer, Word}; + +/// How the chosen engines combine. The default is [`MergeStrategy::PrimaryOnly`] with the +/// default engine (ocrs), which is today's behavior. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash)] +pub enum MergeStrategy { + /// Only the primary engine runs. + #[default] + PrimaryOnly, + /// The secondary engine joins only when the primary's reading looks weak: its mean word + /// confidence is under 70, or its confidence is unknown (words were read but none reports + /// a confidence — the ocrs engine). An unknown reading is never counted as 100; an empty + /// primary keeps the fast path (no words, nothing to double-check). + ConfidenceWeighted, + /// Both engines always run and their readings merge (with no secondary engine this + /// degrades to primary only). + RoverVote, +} + +/// The mean of the confidences the words report, 0–100; `None` when the reading's confidence +/// is UNKNOWN — words were read but none reports a confidence (the ocrs engine's words). +/// Unknown is never counted as 100: there is no low-confidence signal, but a reading whose +/// confidence nobody knows is not therefore confident, so a confidence-weighted strategy +/// answers it by asking the secondary engine. An empty reading is `Some(100.0)`: no words +/// were read, so there is nothing to double-check and the fast path stands. +pub fn mean_confidence(lines: &[Line]) -> Option { + let mut total = 0usize; + let mut reported = (0.0f64, 0usize); + for word in lines.iter().flat_map(|l| &l.words) { + total += 1; + if let Some(c) = word.confidence { + reported.0 += c as f64; + reported.1 += 1; + } + } + match (total, reported.1) { + (0, _) => Some(100.0), + (_, 0) => None, + (_, n) => Some((reported.0 / n as f64) as f32), + } +} + +/// Intersection over union of two `[left, top, right, bottom]` boxes. An empty intersection is +/// 0; degenerate or NaN boxes give 0 rather than panicking or poisoning the comparison. +pub fn iou(a: &[f32; 4], b: &[f32; 4]) -> f64 { + let l = a[0].max(b[0]); + let t = a[1].max(b[1]); + let r = a[2].min(b[2]); + let btm = a[3].min(b[3]); + if !(r > l && btm > t) { + return 0.0; // disjoint, touching, inverted, or NaN (all comparisons false) + } + let inter = (r - l) as f64 * (btm - t) as f64; + let area = |x: &[f32; 4]| ((x[2] - x[0]).max(0.0) as f64) * ((x[3] - x[1]).max(0.0) as f64); + let union = area(a) + area(b) - inter; + if union <= 0.0 { + return 0.0; + } + inter / union +} + +/// The most primary×secondary word comparisons one ROVER merge will attempt. Known limit: a +/// merge over this cap is skipped — the primary reading stands alone, and the skip is noted, +/// never silent — so two enormous readings cannot turn the merge into quadratic work without +/// end. A real page is a few hundred words on each side, orders of magnitude under the cap. +pub const ROVER_MAX_COMPARISONS: usize = 100_000_000; + +/// Whether the two readings are too large to merge: their word counts multiply past +/// [`ROVER_MAX_COMPARISONS`]. Shared by [`rover_merge`] (which then keeps the primary) and +/// [`recognize_with_strategy`] (which then notes the skip). +fn over_cap(primary: &[Line], secondary: &[Line]) -> bool { + let primary_words: usize = primary.iter().map(|l| l.words.len()).sum(); + let secondary_words: usize = secondary.iter().map(|l| l.words.len()).sum(); + primary_words.saturating_mul(secondary_words) > ROVER_MAX_COMPARISONS +} + +/// The confidence a secondary word needs to replace a primary word whose confidence is +/// unknown (the ocrs engine reports none). It is the bar under which a reading counts as weak +/// ([`MergeStrategy::ConfidenceWeighted`] asks the second engine below it): a word the second +/// engine itself calls weak does not displace a reading nobody scored. +pub const UNKNOWN_CONFIDENCE_BAR: f32 = 70.0; + +/// True when the secondary word's confidence `a` beats the primary's `b`. Both known: the +/// strictly higher one wins (a tie keeps the primary). A known confidence against an unknown +/// primary wins only at or above [`UNKNOWN_CONFIDENCE_BAR`]: an unknown confidence is neither +/// 0 nor 100, so a 5% reading must not replace it, while a confident one may. NaN never wins, +/// and with no confidence on the secondary side the primary stands. +fn higher(a: Option, b: Option) -> bool { + match (a, b) { + (Some(a), Some(b)) => a.partial_cmp(&b) == Some(std::cmp::Ordering::Greater), + (Some(a), None) => a >= UNKNOWN_CONFIDENCE_BAR, + _ => false, + } +} + +/// Merge the two engines' readings. Primary words keep their order and lines; a word is +/// replaced by its overlapping secondary (IoU >= 0.5) only when that secondary is more +/// confident, and either way the merged word's [`Word::source`] is `"ROVER"`. Secondary words +/// no primary claimed are appended, in their own reading order and lines, under their own +/// engine's name (`secondary_id`). Known limit: over [`ROVER_MAX_COMPARISONS`] comparisons +/// (word counts multiplied) the primary reading is kept as is, unmerged; the strategy runner +/// names that skip in a note. +pub fn rover_merge(primary: Vec, secondary: Vec, secondary_id: &str) -> Vec { + if over_cap(&primary, &secondary) { + return primary; // over the comparison cap: the primary stands alone + } + let mut primary = primary; + // The secondary's words, marked `None` once a primary word claimed them, grouped by their + // own lines so the unclaimed ones can be appended in order. + let mut unused: Vec>> = secondary.into_iter().map(|l| l.words.into_iter().map(Some).collect()).collect(); + for line in &mut primary { + for word in &mut line.words { + // The best-overlapping unused secondary word, if any beats IoU 0.5. + let mut best: Option<(f64, usize, usize)> = None; + for (li, words) in unused.iter().enumerate() { + for (wi, slot) in words.iter().enumerate() { + let Some(w) = slot else { continue }; + let score = iou(&word.rect, &w.rect); + if score >= 0.5 && best.is_none_or(|(b, _, _)| score > b) { + best = Some((score, li, wi)); + } + } + } + let Some((_, li, wi)) = best else { continue }; + let claimed = unused.get_mut(li).and_then(|l| l.get_mut(wi)).and_then(|slot| slot.take()); + let Some(claim) = claimed else { continue }; + if higher(claim.confidence, word.confidence) { + *word = Word { source: "ROVER".into(), ..claim }; + } else { + // Tie (or the primary wins): keep the primary's text, box and confidence, but + // record that this reading is the engines' agreed one. + word.source = "ROVER".into(); + } + } + } + // Append the secondary words nothing claimed, line by line, under their engine's name + // (`secondary_id` backfills words that name no source of their own). + let mut merged = primary; + for line in unused { + let words: Vec = line + .into_iter() + .flatten() + .map(|mut w| { + if w.source.is_empty() { + w.source = secondary_id.into(); + } + w + }) + .collect(); + if !words.is_empty() { + merged.push(Line { words }); + } + } + merged +} + +/// What a strategy run produced: the merged (or primary-only) reading, whether the secondary +/// engine's reading is part of it, and the notes recorded along the way. +#[derive(Clone, Debug, PartialEq, Default)] +pub struct StrategyOutcome { + pub lines: Vec, + /// Whether the secondary engine's reading is part of `lines`: true only when the merge + /// actually happened. A secondary that failed, or whose reading was discarded over the + /// comparison cap, is `false`, with a note in [`StrategyOutcome::notes`] saying why. + pub secondary_ran: bool, + /// Why the page's reading is partial, if it is. The secondary engine is an enhancement, + /// never a gate: when it is configured but fails, the primary reading is kept and the + /// failure is noted here — a page is never skipped because a partner could not read it. + /// Only a primary failure is an [`OcrError`]. + pub notes: Vec, +} + +/// The note recorded when an ensemble partner fails: the page keeps the primary reading and +/// the cause is named for the report. +fn degraded_note(id: &str, error: &OcrError) -> String { + format!("the {id} engine failed ({error}) — the page keeps the primary reading alone") +} + +/// The note recorded when the two readings are too large to merge: the page keeps the primary +/// reading alone and the discarded partner reading is named, never silent. +fn cap_note(id: &str, primary_words: usize, secondary_words: usize) -> String { + format!( + "the readings were too large to merge ({primary_words} × {secondary_words} comparisons, over the cap of {ROVER_MAX_COMPARISONS}) — the page keeps the primary reading alone and the {id} reading is unused" + ) +} + +/// Run the engines per `strategy` and merge. See [`StrategyOutcome`] for what comes back. +pub fn recognize_with_strategy( + primary: &dyn Recognizer, + secondary: Option<&dyn Recognizer>, + strategy: MergeStrategy, + image: &OcrImage, + options: &RecognizeOptions, +) -> Result { + // The shared shape of the ensemble strategies: the primary has read, the secondary joins + // or fails (a failure degrades to the primary, with a note). + let join = |p: Vec, sec: &dyn Recognizer| match sec.recognize(image, options) { + // Over the comparison cap the merge is skipped: the secondary's reading was read but + // is not part of the result, so the outcome says so — primary only, honestly not + // merged, with the skip named. + Ok(s) if over_cap(&p, &s) => { + let (primary_words, secondary_words) = (p.iter().map(|l| l.words.len()).sum::(), s.iter().map(|l| l.words.len()).sum::()); + StrategyOutcome { lines: p, secondary_ran: false, notes: vec![cap_note(sec.id(), primary_words, secondary_words)] } + } + Ok(s) => StrategyOutcome { lines: rover_merge(p, s, sec.id()), secondary_ran: true, notes: Vec::new() }, + Err(e) => StrategyOutcome { lines: p, secondary_ran: false, notes: vec![degraded_note(sec.id(), &e)] }, + }; + match strategy { + MergeStrategy::PrimaryOnly => Ok(StrategyOutcome { lines: primary.recognize(image, options)?, secondary_ran: false, notes: Vec::new() }), + MergeStrategy::RoverVote => match secondary { + None => Ok(StrategyOutcome { lines: primary.recognize(image, options)?, secondary_ran: false, notes: Vec::new() }), + Some(sec) => { + let p = primary.recognize(image, options)?; + Ok(join(p, sec)) + } + }, + MergeStrategy::ConfidenceWeighted => { + let p = primary.recognize(image, options)?; + match mean_confidence(&p) { + // Confident enough — or an empty reading: the primary stands alone. + Some(mean) if mean >= UNKNOWN_CONFIDENCE_BAR => Ok(StrategyOutcome { lines: p, secondary_ran: false, notes: Vec::new() }), + // A weak mean, or an unknown confidence (words, none reported): the secondary + // runs. + _ => match secondary { + None => Ok(StrategyOutcome { lines: p, secondary_ran: false, notes: Vec::new() }), + Some(sec) => Ok(join(p, sec)), + }, + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering}; + + /// A fake engine: returns canned lines, counts recognize calls, never available. + struct Fake { + id: &'static str, + lines: Vec, + calls: AtomicUsize, + fail: bool, + } + + impl Fake { + fn new(id: &'static str, words: &[(&str, [f32; 4], f32)]) -> Fake { + Fake { + id, + lines: vec![Line { words: words.iter().map(|(t, r, c)| Word::new(*t, *r, id).with_confidence(*c)).collect() }], + calls: AtomicUsize::new(0), + fail: false, + } + } + + fn empty(id: &'static str) -> Fake { + Fake { id, lines: Vec::new(), calls: AtomicUsize::new(0), fail: false } + } + + /// An engine like ocrs: words, but no confidence reported on any of them. + fn unknown(id: &'static str, n: usize) -> Fake { + Fake { + id, + lines: vec![Line { + words: (0..n).map(|i| Word::new(format!("w{i}"), [i as f32 * 10.0, 0.0, i as f32 * 10.0 + 8.0, 10.0], id)).collect(), + }], + calls: AtomicUsize::new(0), + fail: false, + } + } + } + + impl Recognizer for Fake { + fn id(&self) -> &'static str { + self.id + } + fn available(&self) -> bool { + !self.fail + } + fn languages(&self) -> Vec<(String, String)> { + vec![] + } + fn recognize(&self, _image: &OcrImage, _options: &RecognizeOptions) -> Result, OcrError> { + self.calls.fetch_add(1, Ordering::SeqCst); + if self.fail { + return Err(OcrError::Recognize("the fake engine failed".into())); + } + Ok(self.lines.clone()) + } + } + + fn image() -> OcrImage { + OcrImage::new(4, 4, vec![255; 64]).unwrap() + } + + fn words(lines: &[Line]) -> Vec<&Word> { + lines.iter().flat_map(|l| &l.words).collect() + } + + #[test] + fn iou_of_disjoint_touching_and_nan_boxes_is_zero() { + let a = [0.0, 0.0, 10.0, 10.0]; + assert_eq!(iou(&a, &[20.0, 0.0, 30.0, 10.0]), 0.0, "disjoint"); + assert_eq!(iou(&a, &[10.0, 0.0, 20.0, 10.0]), 0.0, "touching edges are an empty intersection"); + assert_eq!(iou(&a, &[f32::NAN, 0.0, 10.0, 10.0]), 0.0, "NaN boxes score 0, not a panic"); + assert_eq!(iou(&a, &a), 1.0, "identical boxes"); + assert!((iou(&a, &[5.0, 0.0, 15.0, 10.0]) - 1.0 / 3.0).abs() < 1e-9, "half overlap is IoU 1/3"); + } + + /// With nothing on the primary side, every secondary word is appended under the secondary + /// engine's own name. + #[test] + fn rover_appends_everything_when_the_primary_is_empty() { + let primary = Vec::new(); + let secondary = vec![Line { words: vec![Word::new("sole", [0.0, 0.0, 9.0, 9.0], "tesseract").with_confidence(80.0)] }]; + let merged = rover_merge(primary, secondary, "tesseract"); + assert_eq!(words(&merged).len(), 1); + assert_eq!(merged[0].words[0].text, "sole"); + assert_eq!(merged[0].words[0].source, "tesseract", "unclaimed words keep their own engine"); + } + + /// Non-overlapping secondary words never displace primary words; they are appended. + #[test] + fn rover_appends_non_overlapping_secondary_words() { + let primary = vec![Line { words: vec![Word::new("kept", [0.0, 0.0, 10.0, 10.0], "ocrs").with_confidence(50.0)] }]; + let secondary = vec![Line { words: vec![Word::new("extra", [100.0, 100.0, 120.0, 110.0], "tesseract").with_confidence(99.0)] }]; + let merged = rover_merge(primary, secondary, "tesseract"); + let w = words(&merged); + assert_eq!(w.len(), 2); + assert_eq!((w[0].text.as_str(), w[0].source.as_str()), ("kept", "ocrs"), "the more confident but disjoint word does not win"); + assert_eq!((w[1].text.as_str(), w[1].source.as_str()), ("extra", "tesseract")); + } + + /// The higher-confidence word of an overlapping pair wins text, box and confidence, and is + /// named ROVER. + #[test] + fn rover_higher_confidence_wins_and_is_named_rover() { + let primary = vec![Line { words: vec![Word::new("helo", [0.0, 0.0, 10.0, 10.0], "ocrs").with_confidence(60.0)] }]; + let secondary = vec![Line { words: vec![Word::new("hello", [1.0, 0.0, 11.0, 10.0], "tesseract").with_confidence(95.0)] }]; + let merged = rover_merge(primary, secondary, "tesseract"); + let w = words(&merged); + assert_eq!(w.len(), 1, "the secondary word was claimed, not appended"); + assert_eq!(w[0].text, "hello"); + assert_eq!(w[0].rect, [1.0, 0.0, 11.0, 10.0]); + assert_eq!(w[0].confidence, Some(95.0)); + assert_eq!(w[0].source, "ROVER"); + } + + /// A tie keeps the primary word (still named ROVER, as an agreed reading). + #[test] + fn rover_tie_keeps_the_primary() { + let primary = vec![Line { words: vec![Word::new("primary", [0.0, 0.0, 10.0, 10.0], "ocrs").with_confidence(80.0)] }]; + let secondary = vec![Line { words: vec![Word::new("secondary", [0.0, 0.0, 10.0, 10.0], "tesseract").with_confidence(80.0)] }]; + let merged = rover_merge(primary, secondary, "tesseract"); + let w = words(&merged); + assert_eq!(w.len(), 1); + assert_eq!(w[0].text, "primary"); + assert_eq!(w[0].source, "ROVER"); + } + + /// Boxes under the 0.5 IoU bar never merge, even when the secondary is more confident. + #[test] + fn rover_below_half_iou_does_not_merge() { + let primary = vec![Line { words: vec![Word::new("kept", [0.0, 0.0, 10.0, 10.0], "ocrs").with_confidence(50.0)] }]; + // IoU 10·10 / (100 + 169 - 25) = 100/244 ≈ 0.41 < 0.5. + let secondary = vec![Line { words: vec![Word::new("other", [5.0, 0.0, 23.0, 20.0], "tesseract").with_confidence(99.0)] }]; + let merged = rover_merge(primary, secondary, "tesseract"); + let w = words(&merged); + assert_eq!(w.len(), 2); + assert_eq!(w[0].text, "kept"); + assert_eq!(w[0].source, "ocrs"); + } + + /// Merged words keep their line; the merge never panics on NaN boxes. + #[test] + fn rover_survives_nan_boxes() { + let primary = vec![Line { words: vec![Word::new("nan", [f32::NAN, 0.0, 10.0, 10.0], "ocrs").with_confidence(50.0)] }]; + let secondary = vec![Line { words: vec![Word::new("real", [0.0, 0.0, 10.0, 10.0], "tesseract").with_confidence(90.0)] }]; + let merged = rover_merge(primary, secondary, "tesseract"); + let w = words(&merged); + assert_eq!(w.len(), 2, "a NaN primary box merges with nothing"); + assert_eq!(w[0].text, "nan"); + assert_eq!(w[1].text, "real"); + } + + /// Only reported confidences count. An empty reading is Some(100) (nothing to + /// double-check); words without a single confidence are UNKNOWN (`None`), never 100. + #[test] + fn mean_confidence_distinguishes_empty_from_unknown() { + assert_eq!(mean_confidence(&[]), Some(100.0)); + assert_eq!(mean_confidence(&[Line { words: vec![Word::new("a", [0.0; 4], "ocrs")] }]), None, "words, no confidences: unknown"); + let lines = vec![Line { + words: vec![ + Word::new("a", [0.0; 4], "e").with_confidence(40.0), + Word::new("b", [0.0; 4], "e").with_confidence(80.0), + Word::new("c", [0.0; 4], "e"), + ], + }]; + assert_eq!(mean_confidence(&lines), Some(60.0), "reported confidences are averaged"); + } + + /// ROVER's preference needs a meaningful comparison: a known confidence beats an unknown + /// one (the ocrs primary yields to a confident tesseract reading), and None vs None keeps + /// the primary. + #[test] + fn rover_known_confidence_beats_unknown_and_none_vs_none_keeps_the_primary() { + let primary = vec![Line { words: vec![Word::new("helo", [0.0, 0.0, 10.0, 10.0], "ocrs")] }]; + let secondary = vec![Line { words: vec![Word::new("hello", [0.0, 0.0, 10.0, 10.0], "tesseract").with_confidence(95.0)] }]; + let merged = rover_merge(primary, secondary, "tesseract"); + assert_eq!( + (merged[0].words[0].text.as_str(), merged[0].words[0].confidence, merged[0].words[0].source.as_str()), + ("hello", Some(95.0), "ROVER") + ); + let primary = vec![Line { words: vec![Word::new("primary", [0.0, 0.0, 10.0, 10.0], "ocrs")] }]; + let secondary = vec![Line { words: vec![Word::new("secondary", [0.0, 0.0, 10.0, 10.0], "tesseract")] }]; + let merged = rover_merge(primary, secondary, "tesseract"); + assert_eq!( + (merged[0].words[0].text.as_str(), merged[0].words[0].source.as_str()), + ("primary", "ROVER"), + "unknown vs unknown is not a comparison: the primary stands" + ); + } + + /// Over the comparison cap the merge is skipped and the primary reading stands alone (a + /// documented known limit, bounding the merge's quadratic core). + #[test] + fn rover_over_the_comparison_cap_keeps_the_primary() { + // 11 000 × 11 000 = 121 M comparisons > the 100 M cap. + let words = |id: &str| (0..11_000).map(|i| Word::new(format!("w{i}"), [i as f32, 0.0, i as f32 + 5.0, 10.0], id)).collect::>(); + let primary = vec![Line { words: words("ocrs") }]; + let secondary = vec![Line { words: words("tesseract") }]; + let merged = rover_merge(primary, secondary, "tesseract"); + assert_eq!(merged.len(), 1); + assert!(merged[0].words.iter().all(|w| w.source == "ocrs"), "every primary word stands, unmerged"); + assert_eq!(merged[0].words.len(), 11_000); + // Under the cap, merging still happens (a small overlap is replaced and named ROVER). + let small_primary = vec![Line { words: vec![Word::new("kept", [0.0, 0.0, 10.0, 10.0], "ocrs").with_confidence(50.0)] }]; + let small_secondary = vec![Line { words: vec![Word::new("wins", [0.0, 0.0, 10.0, 10.0], "tesseract").with_confidence(95.0)] }]; + let merged = rover_merge(small_primary, small_secondary, "tesseract"); + assert_eq!((merged[0].words[0].text.as_str(), merged[0].words[0].source.as_str()), ("wins", "ROVER")); + } + + /// Over the comparison cap the merge is skipped honestly at the strategy level: the page + /// keeps the primary reading alone, the skip is named in a note, and `secondary_ran` does + /// not claim the discarded reading is part of the result (though the secondary really did + /// run — its work was spent, its reading unused). + #[test] + fn rover_cap_hit_reports_primary_only_with_a_note_and_an_honest_flag() { + let mut primary = Fake::empty("ocrs"); + primary.lines = + vec![Line { words: (0..11_000).map(|i| Word::new(format!("w{i}"), [i as f32, 0.0, i as f32 + 5.0, 10.0], "ocrs")).collect() }]; + let mut secondary = Fake::empty("tesseract"); + secondary.lines = + vec![Line { words: (0..11_000).map(|i| Word::new(format!("w{i}"), [i as f32, 0.0, i as f32 + 5.0, 10.0], "tesseract")).collect() }]; + let outcome = recognize_with_strategy(&primary, Some(&secondary), MergeStrategy::RoverVote, &image(), &RecognizeOptions::default()).unwrap(); + assert_eq!(secondary.calls.load(Ordering::SeqCst), 1, "the secondary really ran"); + assert_eq!(outcome.lines.len(), 1); + assert_eq!(outcome.lines[0].words.len(), 11_000, "primary only"); + assert!(outcome.lines[0].words.iter().all(|w| w.source == "ocrs"), "every primary word stands, unmerged"); + assert!(!outcome.secondary_ran, "the secondary's reading is not part of the result"); + assert_eq!(outcome.notes.len(), 1, "{:?}", outcome.notes); + assert!(outcome.notes[0].contains("merge"), "{:?}", outcome.notes[0]); + assert!(outcome.notes[0].contains("tesseract"), "{:?}", outcome.notes[0]); + // Under the cap the same strategy merges and reports the secondary honestly. + let primary = Fake::new("ocrs", &[("kept", [0.0, 0.0, 10.0, 10.0], 50.0)]); + let secondary = Fake::new("tesseract", &[("wins", [0.0, 0.0, 10.0, 10.0], 95.0)]); + let outcome = recognize_with_strategy(&primary, Some(&secondary), MergeStrategy::RoverVote, &image(), &RecognizeOptions::default()).unwrap(); + assert!(outcome.secondary_ran && outcome.notes.is_empty()); + assert_eq!((outcome.lines[0].words[0].text.as_str(), outcome.lines[0].words[0].source.as_str()), ("wins", "ROVER")); + } + + #[test] + fn primary_only_never_runs_the_secondary() { + let primary = Fake::new("ocrs", &[("w", [0.0; 4], 90.0)]); + let secondary = Fake::empty("tesseract"); + let outcome = + recognize_with_strategy(&primary, Some(&secondary), MergeStrategy::PrimaryOnly, &image(), &RecognizeOptions::default()).unwrap(); + assert_eq!(primary.calls.load(Ordering::SeqCst), 1); + assert_eq!(secondary.calls.load(Ordering::SeqCst), 0); + assert!(!outcome.secondary_ran); + assert!(outcome.notes.is_empty()); + assert_eq!(words(&outcome.lines).len(), 1); + } + + #[test] + fn confidence_weighted_runs_the_secondary_only_on_a_weak_primary() { + let weak = Fake::new("ocrs", &[("w", [0.0; 4], 40.0)]); + let strong = Fake::new("ocrs", &[("w", [0.0; 4], 95.0)]); + let secondary = Fake::empty("tesseract"); + let outcome = + recognize_with_strategy(&weak, Some(&secondary), MergeStrategy::ConfidenceWeighted, &image(), &RecognizeOptions::default()).unwrap(); + assert!(outcome.secondary_ran, "mean 40 < 70 → the secondary joins"); + let outcome = + recognize_with_strategy(&strong, Some(&secondary), MergeStrategy::ConfidenceWeighted, &image(), &RecognizeOptions::default()).unwrap(); + assert!(!outcome.secondary_ran, "mean 95 >= 70 → primary alone"); + // An empty primary counts as 100: no signal, no secondary run. + let empty = Fake::empty("ocrs"); + let outcome = + recognize_with_strategy(&empty, Some(&secondary), MergeStrategy::ConfidenceWeighted, &image(), &RecognizeOptions::default()).unwrap(); + assert!(!outcome.secondary_ran); + assert_eq!(secondary.calls.load(Ordering::SeqCst), 1, "only the weak page ran the secondary"); + } + + /// The ocrs engine reports no confidences: unknown is not 100, so the secondary runs — + /// and its known reading wins the overlapping word (the ensemble's whole point). + #[test] + fn confidence_weighted_runs_the_secondary_on_an_unknown_primary() { + let ocrs_like = Fake::unknown("ocrs", 1); + let secondary = Fake::new("tesseract", &[("hello", [0.0, 0.0, 10.0, 10.0], 95.0)]); + let outcome = + recognize_with_strategy(&ocrs_like, Some(&secondary), MergeStrategy::ConfidenceWeighted, &image(), &RecognizeOptions::default()).unwrap(); + assert!(outcome.secondary_ran, "an unknown confidence asks the secondary"); + assert_eq!((outcome.lines[0].words[0].text.as_str(), outcome.lines[0].words[0].confidence), ("hello", Some(95.0))); + assert_eq!(outcome.lines[0].words[0].source, "ROVER"); + assert_eq!(secondary.calls.load(Ordering::SeqCst), 1); + } + + /// Against a primary word of unknown confidence, a secondary word wins only when it is + /// confident (at or above the weak-reading bar): a 5% reading does not replace a word nobody + /// scored, a 70% one does. The merged word is named ROVER either way. + #[test] + fn an_unknown_primary_word_yields_only_to_a_confident_secondary() { + let merge = |conf: f32| { + let secondary = vec![Line { words: vec![Word::new("other", [0.0, 0.0, 10.0, 10.0], "tesseract").with_confidence(conf)] }]; + let primary = Fake::unknown("ocrs", 1).lines; + rover_merge(primary, secondary, "tesseract").remove(0).words.remove(0) + }; + let weak = merge(5.0); + assert_eq!((weak.text.as_str(), weak.confidence, weak.source.as_str()), ("w0", None, "ROVER"), "a 5% word keeps the primary"); + let below = merge(UNKNOWN_CONFIDENCE_BAR - 0.5); + assert_eq!(below.text, "w0"); + let at_bar = merge(UNKNOWN_CONFIDENCE_BAR); + assert_eq!((at_bar.text.as_str(), at_bar.confidence), ("other", Some(UNKNOWN_CONFIDENCE_BAR)), "a confident word wins"); + let nan = merge(f32::NAN); + assert_eq!(nan.text, "w0", "NaN (unknown after clamping) never wins"); + } + + #[test] + fn rover_vote_always_runs_both_and_degrades_without_a_secondary() { + let primary = Fake::new("ocrs", &[("w", [0.0; 4], 95.0)]); + let secondary = Fake::empty("tesseract"); + let outcome = recognize_with_strategy(&primary, Some(&secondary), MergeStrategy::RoverVote, &image(), &RecognizeOptions::default()).unwrap(); + assert!(outcome.secondary_ran); + assert_eq!(secondary.calls.load(Ordering::SeqCst), 1); + // No secondary: primary only. + let outcome = recognize_with_strategy(&primary, None, MergeStrategy::RoverVote, &image(), &RecognizeOptions::default()).unwrap(); + assert!(!outcome.secondary_ran); + } + + /// A configured secondary that fails degrades to the primary reading, with the failure + /// recorded as a note — never a skipped page, never a silently partial reading without a + /// trace. Both ensemble strategies degrade; a primary failure is still an error. + #[test] + fn a_failing_secondary_degrades_to_the_primary_with_a_note() { + let mut secondary = Fake::empty("tesseract"); + secondary.fail = true; + let primary = Fake::new("ocrs", &[("w", [0.0; 4], 95.0)]); + let outcome = recognize_with_strategy(&primary, Some(&secondary), MergeStrategy::RoverVote, &image(), &RecognizeOptions::default()).unwrap(); + assert_eq!(words(&outcome.lines).len(), 1, "the primary reading stands"); + assert_eq!(outcome.lines[0].words[0].text, "w"); + assert!(!outcome.secondary_ran); + assert_eq!(outcome.notes.len(), 1); + assert!(outcome.notes[0].contains("tesseract"), "{:?}", outcome.notes[0]); + // The confidence-weighted strategy degrades the same way on its weak path. + let weak = Fake::new("ocrs", &[("w", [0.0; 4], 40.0)]); + let outcome = + recognize_with_strategy(&weak, Some(&secondary), MergeStrategy::ConfidenceWeighted, &image(), &RecognizeOptions::default()).unwrap(); + assert_eq!(words(&outcome.lines).len(), 1); + assert_eq!(outcome.notes.len(), 1, "{:?}", outcome.notes); + // A failing primary is still an error: there is nothing to degrade to. + let mut failing_primary = Fake::new("ocrs", &[("w", [0.0; 4], 95.0)]); + failing_primary.fail = true; + let r = recognize_with_strategy(&failing_primary, Some(&secondary), MergeStrategy::RoverVote, &image(), &RecognizeOptions::default()); + assert!(r.is_err()); + } +} diff --git a/crates/ocr/src/ocrs.rs b/crates/ocr/src/ocrs.rs new file mode 100644 index 0000000000..7f8feb47c4 --- /dev/null +++ b/crates/ocr/src/ocrs.rs @@ -0,0 +1,251 @@ +//! The pure-Rust engine: the ocrs crate with its pre-trained models, wrapped as a +//! [`Recognizer`] (`OcrsRecognizer`). Reading is offline and local; the models do not report a +//! confidence, so its words have `confidence: None`. + +use std::path::{Path, PathBuf}; + +use super::{Line, MAX_PIXELS, MAX_SIDE, OcrError, OcrImage, RecognizeOptions, Recognizer, Word}; + +/// The validated byte length of a `width` × `height` RGBA raster, with the same caps +/// [`OcrImage::new`] applies: zero sides are empty, sides over [`MAX_SIDE`] or totals over +/// [`MAX_PIXELS`] are too large. `Ocr::recognize` takes raw dimensions (they come from +/// callers, ultimately from documents, so they are untrusted), and the naive +/// `width * height * 4` overflows `u32` for absurd values — the caps are checked first, in +/// `usize`, so nothing overflows and nothing huge is ever touched. +fn checked_len(width: u32, height: u32) -> Result { + if width == 0 || height == 0 { + return Err(OcrError::EmptyImage); + } + if width > MAX_SIDE || height > MAX_SIDE { + return Err(OcrError::ImageTooLarge); + } + let pixels = (width as usize).checked_mul(height as usize).ok_or(OcrError::ImageTooLarge)?; + if pixels > MAX_PIXELS as usize { + return Err(OcrError::ImageTooLarge); + } + pixels.checked_mul(4).ok_or(OcrError::ImageTooLarge) +} + +/// The languages the models read (ISO 639-1); all use the Latin alphabet without accents. +pub const LANGUAGES: &[(&str, &str)] = &[("en", "English")]; + +/// The model files, as named in ATTRIBUTION.toml. +pub const DETECTION_MODEL: &str = "text-detection.onnx"; +pub const RECOGNITION_MODEL: &str = "text-recognition.onnx"; + +pub const ALPHABET: &str = " 0123456789!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~€ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; + +/// Where the two model files are. +#[derive(Clone, Debug, PartialEq)] +pub struct Models { + pub detection: PathBuf, + pub recognition: PathBuf, +} + +impl Models { + pub fn dirs_beside_exe(exe_dir: &Path) -> Vec { + vec![exe_dir.join("models"), exe_dir.join("../Resources/models"), exe_dir.join("../share/pdfcraft/models")] + } + + /// The models in `dir`, preferring current ONNX files over legacy RTEN files. An incomplete + /// update never mixes formats or hides an intact legacy pair. + pub fn in_dir(dir: &Path) -> Option { + [(DETECTION_MODEL, RECOGNITION_MODEL), ("text-detection.rten", "text-recognition.rten")].into_iter().find_map(|(detection, recognition)| { + let m = Models { detection: dir.join(detection), recognition: dir.join(recognition) }; + (m.detection.is_file() && m.recognition.is_file()).then_some(m) + }) + } + + /// Look for the models: `$PDFCRAFT_MODELS`, then `models/` beside the executable (and + /// `Resources/models` in a macOS bundle), then the source tree's `assets/models/`. + pub fn find() -> Option { + Self::search_dirs().iter().find_map(|d| Self::in_dir(d)) + } + + /// The directories [`Models::find`] looks in, in order. + pub fn search_dirs() -> Vec { + let mut dirs = Vec::new(); + if let Some(d) = std::env::var_os("PDFCRAFT_MODELS") { + dirs.push(PathBuf::from(d)); + } + if let Some(exe) = std::env::current_exe().ok().and_then(|e| e.parent().map(Path::to_path_buf)) { + dirs.extend(Self::dirs_beside_exe(&exe)); + } + dirs.push(Path::new(env!("CARGO_MANIFEST_DIR")).join("../../assets/models")); + dirs + } +} + +/// A loaded recogniser. Loading takes a moment; keep one and reuse it. +pub struct Ocr { + engine: ocrs::OcrEngine, +} + +impl Ocr { + pub fn load(models: &Models) -> Result { + let load = |p: &Path| rten::Model::load_file(p).map_err(|e| OcrError::Load(p.display().to_string(), e.to_string())); + let params = ocrs::OcrEngineParams { + detection_model: Some(load(&models.detection)?), + recognition_model: Some(load(&models.recognition)?), + alphabet: Some(ALPHABET.into()), + ..Default::default() + }; + let engine = ocrs::OcrEngine::new(params).map_err(|e| OcrError::Load("ocr engine".into(), e.to_string()))?; + Ok(Ocr { engine }) + } + + /// Load the models found by [`Models::find`]. + pub fn find() -> Result { + Self::load(&Models::find().ok_or(OcrError::NoModels)?) + } + + /// Recognise the text in an RGBA (or RGB, or grey) image, `width` × `height` pixels. + pub fn recognize(&self, pixels: &[u8], width: u32, height: u32) -> Result, OcrError> { + if pixels.is_empty() { + return Err(OcrError::EmptyImage); + } + let want = checked_len(width, height)?; + // ocrs wants 1 or 3 channels. + let rgb: std::borrow::Cow<[u8]> = if pixels.len() == want { + pixels.as_chunks::<4>().0.iter().flat_map(|p| [p[0], p[1], p[2]]).collect::>().into() + } else { + pixels.into() + }; + let err = |e: &dyn std::fmt::Display| OcrError::Recognize(e.to_string()); + let source = ocrs::ImageSource::from_bytes(&rgb, (width, height)).map_err(|e| err(&e))?; + let input = self.engine.prepare_input(source).map_err(|e| err(&e))?; + let found = self.engine.detect_words(&input).map_err(|e| err(&e))?; + let lines = self.engine.find_text_lines(&input, &found); + let read = self.engine.recognize_text(&input, &lines).map_err(|e| err(&e))?; + use ocrs::TextItem; + Ok(read + .into_iter() + .flatten() + .map(|line| Line { + words: line + .words() + .map(|w| { + let r = w.bounding_rect(); + Word::new(w.to_string(), [r.left() as f32, r.top() as f32, r.right() as f32, r.bottom() as f32], "ocrs") + }) + .filter(|w| !w.text.trim().is_empty()) + .collect(), + }) + .filter(|l| !l.words.is_empty()) + .collect()) + } +} + +/// The ocrs engine as a [`Recognizer`] (the default engine). +pub struct OcrsRecognizer { + engine: Ocr, +} + +impl OcrsRecognizer { + /// Load the models from `models`. + pub fn load(models: &Models) -> Result { + Ok(OcrsRecognizer { engine: Ocr::load(models)? }) + } + + /// Load the models found by [`Models::find`]. + pub fn find() -> Result { + Self::load(&Models::find().ok_or(OcrError::NoModels)?) + } +} + +impl Recognizer for OcrsRecognizer { + fn id(&self) -> &'static str { + "ocrs" + } + + fn available(&self) -> bool { + Models::find().is_some() + } + + fn languages(&self) -> Vec<(String, String)> { + LANGUAGES.iter().map(|(c, n)| ((*c).to_string(), (*n).to_string())).collect() + } + + fn recognize(&self, image: &OcrImage, _options: &RecognizeOptions) -> Result, OcrError> { + self.engine.recognize(image.rgba(), image.width(), image.height()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Raw dimensions are untrusted (they come from callers, ultimately from documents): zero + /// sides, sides or pixel totals over the caps, and products that would overflow the naive + /// `width * height * 4` are errors, never panics, before anything is touched. + #[test] + fn raw_dimensions_are_capped_and_checked_not_panicked_at() { + assert!(matches!(checked_len(0, 10), Err(OcrError::EmptyImage))); + assert!(matches!(checked_len(10, 0), Err(OcrError::EmptyImage))); + assert!(matches!(checked_len(MAX_SIDE + 1, 1), Err(OcrError::ImageTooLarge))); + // 65536² would wrap u32 in the old `width * height * 4`; the side cap refuses it first. + assert!(matches!(checked_len(u32::MAX, u32::MAX), Err(OcrError::ImageTooLarge))); + // Legal sides whose product is over the 64 MP area cap. + assert!(matches!(checked_len(MAX_SIDE, 7000), Err(OcrError::ImageTooLarge))); + assert_eq!(checked_len(2, 2).unwrap(), 16); + // Exactly the area cap fits: 8000 × 8000 = 64 MP. + assert_eq!(checked_len(8000, 8000).unwrap(), 8000usize * 8000 * 4); + } + + /// The current ONNX pair wins when an installation also retains both legacy models. + #[test] + fn model_discovery_prefers_the_complete_onnx_pair() { + let dir = tempfile::tempdir().unwrap(); + for file in ["text-detection.onnx", "text-recognition.onnx"] { + std::fs::write(dir.path().join(file), b"model").unwrap(); + } + let expected = Some(Models { detection: dir.path().join("text-detection.onnx"), recognition: dir.path().join("text-recognition.onnx") }); + assert_eq!(Models::in_dir(dir.path()), expected, "a complete ONNX pair is sufficient"); + for file in ["text-detection.rten", "text-recognition.rten"] { + std::fs::write(dir.path().join(file), b"model").unwrap(); + } + assert_eq!(Models::in_dir(dir.path()), expected, "the complete ONNX pair takes precedence over legacy models"); + } + + /// An interrupted ONNX update still permits an intact legacy pair in the same directory. + #[test] + fn model_discovery_falls_back_to_legacy_when_the_onnx_pair_is_incomplete() { + for current in ["text-detection.onnx", "text-recognition.onnx"] { + let dir = tempfile::tempdir().unwrap(); + for file in [current, "text-detection.rten", "text-recognition.rten"] { + std::fs::write(dir.path().join(file), b"model").unwrap(); + } + assert_eq!( + Models::in_dir(dir.path()), + Some(Models { detection: dir.path().join("text-detection.rten"), recognition: dir.path().join("text-recognition.rten") }), + "{current} alone must not hide a complete legacy pair" + ); + } + } + + /// Detection and recognition must come from one complete format pair, never a mixed update. + #[test] + fn model_discovery_refuses_mixed_format_pairs() { + for pair in [["text-detection.onnx", "text-recognition.rten"], ["text-detection.rten", "text-recognition.onnx"]] { + let dir = tempfile::tempdir().unwrap(); + for file in pair { + std::fs::write(dir.path().join(file), b"model").unwrap(); + } + assert_eq!(Models::in_dir(dir.path()), None, "{pair:?} is not a complete model pair"); + } + } + + /// Existing source builds and PDFCRAFT_MODELS overrides retain their complete RTEN pair. + #[test] + fn model_discovery_keeps_legacy_only_installations() { + let dir = tempfile::tempdir().unwrap(); + assert_eq!(Models::in_dir(dir.path()), None); + std::fs::write(dir.path().join("text-detection.rten"), b"model").unwrap(); + assert_eq!(Models::in_dir(dir.path()), None, "one legacy model is incomplete"); + std::fs::write(dir.path().join("text-recognition.rten"), b"model").unwrap(); + assert_eq!( + Models::in_dir(dir.path()), + Some(Models { detection: dir.path().join("text-detection.rten"), recognition: dir.path().join("text-recognition.rten") }) + ); + } +} diff --git a/crates/ocr/src/preprocess.rs b/crates/ocr/src/preprocess.rs new file mode 100644 index 0000000000..280ee73fec --- /dev/null +++ b/crates/ocr/src/preprocess.rs @@ -0,0 +1,973 @@ +//! Preprocessing: a pure function between the page raster and recognition, in fixed order — +//! (a) DPI normalization, then the four opt-in steps (b) orientation, (c) deskew, (d) denoise, +//! (e) binarization. Normalization exists for those steps (their windows and thresholds are +//! calibrated for 300 dpi), so it runs only when at least one of them is on: with every step +//! off the raster is handed to the engines exactly as it was rendered. (Enlarging a render +//! on its own changes what the models read: a 72 dpi scan of "due: €250" read as "ue: ?250" +//! after a 4× bilinear enlargement.) Everything is plain Rust on the RGBA raster: no +//! Leptonica, no process, no new dependencies. +//! +//! Every geometric step records its inverse; the result carries one exact +//! [`InverseTransform`] (an affine map, composed in f64), and callers map every word box +//! through it back to the *original* raster coordinates before placing the words. +//! +//! The heuristics are deliberately conservative and panic-free: an image too small to judge, +//! a blank page, or a score without a clear margin is a no-op, never a guess. + +use crate::{MAX_PIXELS, MAX_SIDE, OcrError, OcrImage}; + +/// The four opt-in steps. All off by default: a scan is passed through as rendered. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct PreprocessOptions { + /// Detect pages scanned at 90°/180°/270° and turn them upright. + pub auto_rotate: bool, + /// Straighten skewed text lines. + pub deskew: bool, + /// 3×3 median filter (salt-and-pepper noise). + pub denoise: bool, + /// Sauvola binarization (window 8, k 0.34). + pub binarize: bool, +} + +impl PreprocessOptions { + /// Whether any step is on (and so the raster is normalized to 300 dpi first). + pub fn any(&self) -> bool { + self.auto_rotate || self.deskew || self.denoise || self.binarize + } +} + +/// A preprocessed raster: the image to recognize, the exact way back to the input raster, and +/// the resolution the image now stands for. +#[derive(Debug)] +pub struct Preprocessed { + pub image: OcrImage, + /// Maps pixel coordinates in [`Preprocessed::image`] back to the input raster's. + pub inverse_transform: InverseTransform, + /// The image's resolution in dots per inch: 300 after normalization (only when a step is + /// on), or the source's own + /// when it already rendered at 300 or more. + pub dpi: f32, +} + +/// Preprocess `image` (rendered at `source_dpi`) per `options`, in the fixed order. The image +/// is consumed: the common path (every step off, or a no-op) hands the raster back without +/// copying it, and a step that rewrites pixels replaces it. +pub fn preprocess(image: OcrImage, source_dpi: f32, options: &PreprocessOptions) -> Result { + let mut current = image; + let mut inverse = InverseTransform::identity(); + // (a) DPI normalization: for the steps below, a scan under 300 dpi is enlarged by 300/src so + // they see letter-sized strokes. With every step off nothing is resampled. A broken dpi + // (zero, negative, NaN) is left alone. + let mut dpi = if source_dpi.is_finite() && source_dpi >= 1.0 { source_dpi } else { 300.0 }; + if options.any() && dpi < 300.0 { + let scale = f64::from(300.0 / dpi); + current = resize_bilinear(¤t, scale)?; + inverse = compose(&inverse, &InverseTransform::scale(1.0 / scale)); + dpi = 300.0; + } + // (b) Orientation: turn 90/180/270 scans upright (opt-in, skipped for small images). + if options.auto_rotate { + let (w, h) = (current.width, current.height); + let turns = detect_orientation(¤t); + if turns != 0 { + current = rotate_quarter_turns(¤t, turns); + inverse = compose(&inverse, &inverse_quarter_turns(turns, w, h)); + } + } + // (c) Deskew: straighten skewed lines (opt-in; no-op under 64 px, blank, or without a + // clear margin or below 0.1°). + if options.deskew + && let Some(angle) = estimate_skew(¤t) + { + let (straightened, back) = deskew_rotate(¤t, angle); + current = straightened; + inverse = compose(&inverse, &back); + } + // (d) Denoise: 3×3 median with edge clamping (opt-in; dimensions never change). + if options.denoise { + current = median3(¤t); + } + // (e) Binarize: Sauvola (window 8, k 0.34) via an integral image (opt-in); dark ink stays + // dark and paper stays light. + if options.binarize { + current = sauvola(¤t); + } + Ok(Preprocessed { image: current, inverse_transform: inverse, dpi }) +} + +/// An affine map `(x, y) → (a·x + c·y + e, b·x + d·y + f)`, composed in f64 so mapping a +/// word box back is exact up to float rounding. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct InverseTransform { + a: f64, + b: f64, + c: f64, + d: f64, + e: f64, + f: f64, +} + +impl InverseTransform { + pub fn identity() -> InverseTransform { + InverseTransform { a: 1.0, b: 0.0, c: 0.0, d: 1.0, e: 0.0, f: 0.0 } + } + + fn scale(s: f64) -> InverseTransform { + InverseTransform { a: s, b: 0.0, c: 0.0, d: s, e: 0.0, f: 0.0 } + } + + /// `compose(outer, inner)` maps `p` to `outer(inner(p))`: the accumulated inverse of the + /// earlier steps is the outer map (their spaces come later in the round trip), so each + /// step calls `compose(&inverse, &this_step_inverse)`. The scale step, for instance, + /// happens first, so its inverse is applied last: `scale⁻¹(rotate⁻¹(p))`. + fn compose_(outer: &InverseTransform, inner: &InverseTransform) -> InverseTransform { + InverseTransform { + a: outer.a * inner.a + outer.c * inner.b, + b: outer.b * inner.a + outer.d * inner.b, + c: outer.a * inner.c + outer.c * inner.d, + d: outer.b * inner.c + outer.d * inner.d, + e: outer.a * inner.e + outer.c * inner.f + outer.e, + f: outer.b * inner.e + outer.d * inner.f + outer.f, + } + } + + /// Map a point from the processed image back to the input raster. + pub fn apply(&self, x: f64, y: f64) -> (f64, f64) { + (self.a * x + self.c * y + self.e, self.b * x + self.d * y + self.f) + } + + /// Map a word box `[left, top, right, bottom]` back to the input raster: the four corners + /// go through [`Self::apply`] and their bounding box is rounded outward to whole pixels. + /// A box with no finite corners comes back unchanged (garbage in, garbage out — never a + /// panic). + pub fn map_rect(&self, rect: [f32; 4]) -> [f32; 4] { + let corners = [(rect[0], rect[1]), (rect[2], rect[1]), (rect[0], rect[3]), (rect[2], rect[3])]; + let mut lo = (f64::INFINITY, f64::INFINITY); + let mut hi = (f64::NEG_INFINITY, f64::NEG_INFINITY); + let mut finite = false; + for (x, y) in corners { + if !(x.is_finite() && y.is_finite()) { + continue; + } + let (mx, my) = self.apply(f64::from(x), f64::from(y)); + if mx.is_finite() && my.is_finite() { + finite = true; + lo = (lo.0.min(mx), lo.1.min(my)); + hi = (hi.0.max(mx), hi.1.max(my)); + } + } + if !finite { + return rect; + } + [lo.0.floor() as f32, lo.1.floor() as f32, hi.0.ceil() as f32, hi.1.ceil() as f32] + } +} + +/// The step inverses compose in pipeline order: each new step's inverse is applied *after* +/// the inverses accumulated so far. +fn compose(outer: &InverseTransform, inner: &InverseTransform) -> InverseTransform { + InverseTransform::compose_(outer, inner) +} + +/// The size `width` x `height` becomes under `scale`, refused (before any buffer exists) when +/// a side is under one pixel or over [`MAX_SIDE`], or the area is over [`MAX_PIXELS`]: two +/// sides under the side cap can still ask for hundreds of megabytes, which `OcrImage::new` +/// would only refuse after they were allocated. +fn scaled_size(width: u32, height: u32, scale: f64) -> Result<(u32, u32), OcrError> { + let new_w = (f64::from(width) * scale).ceil(); + let new_h = (f64::from(height) * scale).ceil(); + if !(1.0..=f64::from(MAX_SIDE)).contains(&new_w) || !(1.0..=f64::from(MAX_SIDE)).contains(&new_h) { + return Err(OcrError::ImageTooLarge); + } + let (nw, nh) = (new_w as u32, new_h as u32); + if (nw as usize).checked_mul(nh as usize).is_none_or(|pixels| pixels > MAX_PIXELS as usize) { + return Err(OcrError::ImageTooLarge); + } + Ok((nw, nh)) +} + +/// Bilinear resize by `scale`, clamped at the edges; the result stays inside [`scaled_size`]'s +/// limits (the side cap and the area cap). +fn resize_bilinear(image: &OcrImage, scale: f64) -> Result { + let (nw, nh) = scaled_size(image.width, image.height, scale)?; + let mut out = vec![255u8; (nw as usize).checked_mul(nh as usize).and_then(|p| p.checked_mul(4)).ok_or(OcrError::ImageTooLarge)?]; + for (y, row) in out.chunks_exact_mut(4 * nw as usize).enumerate() { + for (x, px) in row.as_chunks_mut::<4>().0.iter_mut().enumerate() { + // The source position of the destination pixel's centre, and the four + // neighbours bilinear interpolation blends (positions clamp at the edges). + let sx = (x as f64 + 0.5) / scale - 0.5; + let sy = (y as f64 + 0.5) / scale - 0.5; + let (x0, y0) = (sx.floor().max(0.0), sy.floor().max(0.0)); + let (fx, fy) = (sx - x0, sy - y0); + for (ch, o) in px.iter_mut().enumerate() { + let blend = |dx: f64, dy: f64| f64::from(image.byte(x0 + dx, y0 + dy, ch)); + let v = blend(0.0, 0.0) * (1.0 - fx) * (1.0 - fy) + + blend(1.0, 0.0) * fx * (1.0 - fy) + + blend(0.0, 1.0) * (1.0 - fx) * fy + + blend(1.0, 1.0) * fx * fy; + *o = v.round().clamp(0.0, 255.0) as u8; + } + } + } + OcrImage::new(nw, nh, out) +} + +/// Grayscale (Rec.601 luma) of every pixel. +fn grayscale(image: &OcrImage) -> Vec { + image.rgba.as_chunks::<4>().0.iter().map(|p| ((299 * p[0] as u32 + 587 * p[1] as u32 + 114 * p[2] as u32) / 1000).min(255) as u8).collect() +} + +/// How the page must be turned (quarter turns, clockwise) to read upright. +type QuarterTurns = u32; + +/// Detect the correction for pages scanned sideways or upside down. Method (documented, +/// panic-free, no OCR): ink is everything darker than the midpoint between the darkest and +/// lightest pixel; for each candidate correction (0/90/180/270 clockwise) the ink's row +/// profile is taken and scored by how sharply it bands (Σ Δ², the projection-profile score: +/// upright text lines band sharply, turned text does not). The best-banding axis wins when it +/// beats the other axis by a factor of 1.3; within the winning axis the line bands' +/// bottom-heaviness (real text carries more ink below each line's middle — baselines and +/// descenders) picks the quarter turn, with a 4%-of-ink margin. Anything less clear is a +/// no-op (`0`), and images under 32 px are never judged. +fn detect_orientation(image: &OcrImage) -> QuarterTurns { + if image.min_side() < 32 { + return 0; + } + let gray = grayscale(image); + let (Some(lo), Some(hi)) = (gray.iter().min(), gray.iter().max()) else { return 0 }; + let t = (*lo as u32 + *hi as u32) / 2; + let ink: Vec = gray.iter().map(|&g| (g as u32) < t).collect(); + let (w, h) = (image.width as usize, image.height as usize); + let total: u64 = ink.iter().filter(|i| **i).count() as u64; + if total == 0 { + return 0; + } + // The ink grid rotated by k quarter turns (clockwise), as row profiles. + let mut scores = [0.0f64; 4]; + let mut asyms = [0.0f64; 4]; + for k in 0u32..4 { + let k = k as usize; + let profile = rotated_row_profile(&ink, w, h, k as u32); + scores[k] = profile.windows(2).map(|p| (p[1] as f64 - p[0] as f64).powi(2)).sum(); + asyms[k] = band_bottom_heaviness(&profile); + } + // 0 and 180 share one profile shape (reversed), as do 90 and 270: pick the axis first. + let (vertical_pair, horizontal_pair) = ((scores[0], scores[2]), (scores[1], scores[3])); + let row_axis = vertical_pair.0.max(vertical_pair.1); + let column_axis = horizontal_pair.0.max(horizontal_pair.1); + if row_axis >= column_axis { + if row_axis < column_axis * 1.3 { + return 0; // the axes are too close to call + } + if asyms[0] > 0.15 * total as f64 { + 0 + } else if asyms[2] > 0.15 * total as f64 { + 2 + } else { + 0 // the bands are not clearly bottom-heavy either way: no-op + } + } else { + if column_axis < row_axis * 1.3 { + return 0; + } + if asyms[1] > 0.15 * total as f64 { + 1 + } else if asyms[3] > 0.15 * total as f64 { + 3 + } else { + 0 + } + } +} + +/// The row profile of the ink grid after `k` quarter turns clockwise. +fn rotated_row_profile(ink: &[bool], w: usize, h: usize, k: u32) -> Vec { + // The rotated image is `w` pixels tall for odd k and `h` for even k; its row index is + // always the new y of the source pixel. + let len = if k.is_multiple_of(2) { h } else { w }; + let mut profile = vec![0u64; len]; + for (y, row) in ink.chunks_exact(w).enumerate() { + for (x, on) in row.iter().enumerate() { + if !on { + continue; + } + // Where does (x, y) land after k quarter turns clockwise? + let ny = match k { + 1 => x, + 2 => h - 1 - y, + 3 => w - 1 - x, + _ => y, + }; + if let Some(p) = profile.get_mut(ny) { + *p += 1; + } + } + } + profile +} + +/// Sum over text-line bands of (ink in the band's lower half − ink in its upper half): +/// positive when the bands carry more ink near their bottoms, as real text does. +fn band_bottom_heaviness(profile: &[u64]) -> f64 { + let mut asym = 0.0; + let mut start: Option = None; + for (i, &p) in profile.iter().chain(std::iter::once(&0)).enumerate() { + match (start, p > 0) { + (None, true) => start = Some(i), + (Some(s), false) => { + let mid = (s + i) / 2; + let lower: u64 = profile.get(s..i).map_or(0, |b| b[mid.saturating_sub(s)..].iter().sum()); + let upper: u64 = profile.get(s..i).map_or(0, |b| b[..mid.saturating_sub(s)].iter().sum()); + asym += lower as f64 - upper as f64; // signed: a top-heavy band scores below zero + start = None; + } + _ => {} + } + } + asym +} + +/// Rotate the raster `k` quarter turns clockwise (an exact pixel permutation). +fn rotate_quarter_turns(image: &OcrImage, k: QuarterTurns) -> OcrImage { + let (w, h) = (image.width as usize, image.height as usize); + let (nw, nh) = if k.is_multiple_of(2) { (image.width, image.height) } else { (image.height, image.width) }; + let mut out = vec![255u8; nw as usize * nh as usize * 4]; + for (y, row) in image.rgba.chunks_exact(w * 4).enumerate() { + for (x, px) in row.as_chunks::<4>().0.iter().enumerate() { + let (nx, ny) = match k { + 1 => (h - 1 - y, x), + 2 => (w - 1 - x, h - 1 - y), + 3 => (y, w - 1 - x), + _ => (x, y), + }; + if let Some(dst) = out.get_mut((ny * nw as usize + nx) * 4..(ny * nw as usize + nx) * 4 + 4) { + dst.copy_from_slice(px); + } + } + } + OcrImage::new(nw, nh, out).unwrap_or_else(|_| image.clone()) +} + +/// The inverse of a `k`-quarter-turn correction: where each corrected pixel came from. +fn inverse_quarter_turns(k: QuarterTurns, old_w: u32, old_h: u32) -> InverseTransform { + let (w, h) = (f64::from(old_w), f64::from(old_h)); + match k { + // Forward (content moves): (x, y) ↦ (h−1−y, x) for one turn clockwise. + 1 => InverseTransform { a: 0.0, b: -1.0, c: 1.0, d: 0.0, e: 0.0, f: h - 1.0 }, + 2 => InverseTransform { a: -1.0, b: 0.0, c: 0.0, d: -1.0, e: w - 1.0, f: h - 1.0 }, + 3 => InverseTransform { a: 0.0, b: 1.0, c: -1.0, d: 0.0, e: w - 1.0, f: 0.0 }, + _ => InverseTransform::identity(), + } +} + +/// Estimate the skew of the text lines, in degrees (−5…5, positive: lines run downhill to the +/// right). Method (documented, panic-free): for each candidate angle the ink is sheared into a +/// row profile (`r = y − (x − cx)·tan φ`); the projection-profile score (Σ profile²) peaks +/// where the shear realigns the lines. `None` — leave the page alone — when the image is under +/// 64 px on a side, blank, the winner is under 0.1°, or the winner does not clearly beat +/// leaving the page as is (score < 1.02 × the 0° score). +fn estimate_skew(image: &OcrImage) -> Option { + if image.min_side() < 64 { + return None; + } + let gray = grayscale(image); + let (Some(lo), Some(hi)) = (gray.iter().min(), gray.iter().max()) else { return None }; + let t = (*lo as u32 + *hi as u32) / 2; + let ink: Vec = gray.iter().map(|&g| (g as u32) < t).collect(); + if !ink.iter().any(|i| *i) { + return None; // a blank page has no skew + } + let (w, h) = (image.width, image.height); + let cx = f64::from(w) / 2.0; + let max_shift = (f64::from(w) / 2.0 * 5f64.to_radians().tan()).ceil() as i64 + 2; + let score = |phi_deg: f64| -> f64 { + let slope = phi_deg.to_radians().tan(); + let mut profile = vec![0u64; (h as i64 + 2 * max_shift + 1) as usize]; + for (y, row) in ink.chunks_exact(w as usize).enumerate() { + for (x, on) in row.iter().enumerate() { + if !on { + continue; + } + let r = y as f64 - (x as f64 - cx) * slope; + let idx = r.round() as i64 + max_shift; + if let Some(p) = profile.get_mut(idx as usize) { + *p += 1; + } + } + } + profile.iter().map(|p| (*p as f64).powi(2)).sum() + }; + let mut best = (0.0f64, score(0.0)); + let mut phi = -5.0; + while phi <= 5.0 + 1e-9 { + let s = score(phi); + if s > best.1 { + best = (phi, s); + } + phi += 0.25; + } + let (angle, best_score) = best; + if angle.abs() < 0.1 || best_score < 1.02 * score(0.0) { + return None; // straight enough, or no clear winner: no-op + } + Some(angle) +} + +/// Rotate the content by `−angle` about the centre (the deskew correction), nearest-neighbour +/// with white fill; also returns the exact dest→src map (the correction's inverse). +fn deskew_rotate(image: &OcrImage, angle_deg: f64) -> (OcrImage, InverseTransform) { + let phi = angle_deg.to_radians(); + let (cos, sin) = (phi.cos(), phi.sin()); + let (w, h) = (image.width, image.height); + let (cx, cy) = (f64::from(w) / 2.0, f64::from(h) / 2.0); + let back = InverseTransform { a: cos, b: sin, c: -sin, d: cos, e: cx - cos * cx + sin * cy, f: cy - sin * cx - cos * cy }; + let mut out = vec![255u8; w as usize * h as usize * 4]; + for (y, row) in out.chunks_exact_mut(w as usize * 4).enumerate() { + for (x, px) in row.as_chunks_mut::<4>().0.iter_mut().enumerate() { + let (sx, sy) = back.apply(x as f64, y as f64); + let (xi, yi) = (sx.round() as i64, sy.round() as i64); + if xi < 0 || yi < 0 || xi >= w as i64 || yi >= h as i64 { + continue; // outside the scan: white fill + } + if let Some(src) = image.rgba.get((yi as usize * w as usize + xi as usize) * 4..(yi as usize * w as usize + xi as usize) * 4 + 4) { + px.copy_from_slice(src); + } + } + } + (OcrImage::new(w, h, out).unwrap_or_else(|_| image.clone()), back) +} + +/// 3×3 median filter on each colour channel, edges clamped (border pixels replicate); +/// dimensions and alpha are unchanged. +fn median3(image: &OcrImage) -> OcrImage { + let (w, h) = (image.width as usize, image.height as usize); + let mut out = image.rgba.clone(); + for (i, dst) in out.as_chunks_mut::<4>().0.iter_mut().enumerate() { + let (x, y) = (i % w, i / w); + let mut rgba = [255u8; 3]; + for (ch, v) in rgba.iter_mut().enumerate() { + let mut window = [0u8; 9]; + for (n, (dx, dy)) in [(-1i64, -1i64), (0, -1), (1, -1), (-1, 0), (0, 0), (1, 0), (-1, 1), (0, 1), (1, 1)].iter().enumerate() { + let nx = ((x as i64) + dx).clamp(0, w as i64 - 1) as usize; + let ny = ((y as i64) + dy).clamp(0, h as i64 - 1) as usize; + window[n] = image.byte(nx as f64, ny as f64, ch); + } + window.sort_unstable(); + *v = window[4]; + } + let alpha = dst[3]; + dst.copy_from_slice(&[rgba[0], rgba[1], rgba[2], alpha]); + } + OcrImage::new(image.width, image.height, out).unwrap_or_else(|_| image.clone()) +} + +/// The Sauvola ink mask: local thresholding with the documented window — 8 pixels wide, the +/// 8 pixels from −4 to +3 (the end is exclusive, so `x0..x1` is exactly `x−4..=x+3`), k 0.34 — +/// through an integral image. A pixel is ink when its grey is at or under the local +/// threshold. +fn sauvola_mask(image: &OcrImage) -> Vec { + let (w, h) = (image.width as usize, image.height as usize); + let gray = grayscale(image); + // Integral sums and square sums, (w+1)×(h+1), so any window sum is four lookups. + let stride = w + 1; + let mut sums = vec![0u64; stride * (h + 1)]; + let mut squares = vec![0u64; stride * (h + 1)]; + for (y, row) in gray.chunks_exact(w).enumerate() { + let (mut row_sum, mut row_sq) = (0u64, 0u64); + for (x, &g) in row.iter().enumerate() { + row_sum += u64::from(g); + row_sq += u64::from(g) * u64::from(g); + let above = sums.get(y * stride + x + 1).copied().unwrap_or(0); + let above_sq = squares.get(y * stride + x + 1).copied().unwrap_or(0); + if let (Some(s), Some(sq)) = (sums.get_mut((y + 1) * stride + x + 1), squares.get_mut((y + 1) * stride + x + 1)) { + *s = above + row_sum; + *sq = above_sq + row_sq; + } + } + } + let (k, r) = (0.34f64, 128.0f64); + let half = 4usize; + let mut ink = vec![false; w * h]; + for (y, row) in ink.chunks_exact_mut(w).enumerate() { + for (x, dst) in row.iter_mut().enumerate() { + let (x0, y0) = (x.saturating_sub(half), y.saturating_sub(half)); + // Exclusive end: the window covers −4..=+3, 8 pixels on a side (an end of + // `x + half + 1` would make it 9 and disagree with the documented window). + let (x1, y1) = ((x + half).min(w), (y + half).min(h)); + let n = ((x1 - x0) * (y1 - y0)) as f64; + let sum = rect_sum(&sums, stride, x0, y0, x1, y1); + let sq = rect_sum(&squares, stride, x0, y0, x1, y1); + let mean = sum / n; + let var = (sq / n - mean * mean).max(0.0); + let threshold = mean * (1.0 + k * (var.sqrt() / r - 1.0)); + *dst = matches!(gray.get(y * w + x), Some(&g) if f64::from(g) <= threshold); + } + } + ink +} + +/// Sauvola binarization through [`sauvola_mask`], followed by a one-pixel dilation of the ink: +/// ink goes to black, paper to white, alpha preserved. +fn sauvola(image: &OcrImage) -> OcrImage { + let (w, h) = (image.width as usize, image.height as usize); + // One dilation keeps anti-aliased strokes whole: a thresholded edge pixel that flaked off + // rejoins its stroke (slightly bold beats broken, for reading). + let ink = dilate1(&sauvola_mask(image), w, h); + let mut out = image.rgba.clone(); + for (i, px) in out.as_chunks_mut::<4>().0.iter_mut().enumerate() { + let v = if ink.get(i).copied().unwrap_or(false) { 0u8 } else { 255u8 }; + if let Some(dst) = px.get_mut(..3) { + dst.copy_from_slice(&[v, v, v]); + } + } + OcrImage::new(image.width, image.height, out).unwrap_or_else(|_| image.clone()) +} + +/// Grow the ink mask by one pixel (8-neighbourhood, edges clamped by the bounds checks). +fn dilate1(ink: &[bool], w: usize, h: usize) -> Vec { + let mut out = ink.to_vec(); + for y in 0..h { + for x in 0..w { + if !ink.get(y * w + x).copied().unwrap_or(false) { + continue; + } + for dy in [-1isize, 0, 1] { + for dx in [-1isize, 0, 1] { + let (nx, ny) = (x as isize + dx, y as isize + dy); + if nx < 0 || ny < 0 || nx >= w as isize || ny >= h as isize { + continue; + } + if let Some(o) = out.get_mut(ny as usize * w + nx as usize) { + *o = true; + } + } + } + } + } + out +} + +fn rect_sum(table: &[u64], stride: usize, x0: usize, y0: usize, x1: usize, y1: usize) -> f64 { + let a = table.get(y1 * stride + x1).copied().unwrap_or(0); + let b = table.get(y0 * stride + x1).copied().unwrap_or(0); + let c = table.get(y1 * stride + x0).copied().unwrap_or(0); + let d = table.get(y0 * stride + x0).copied().unwrap_or(0); + (a + d - b - c) as f64 +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Sides under MAX_SIDE whose area passes MAX_PIXELS (9500 × 9500, 90 MP) are refused + /// when the size is worked out, before the 360 MB buffer is allocated rather than after. + #[test] + fn an_upscale_over_the_area_cap_is_refused_before_allocating() { + assert!(matches!(scaled_size(1000, 1000, 9.5), Err(OcrError::ImageTooLarge))); + assert_eq!(scaled_size(1000, 1000, 2.0).unwrap(), (2000, 2000), "under the caps it still scales"); + assert_eq!(scaled_size(8000, 8000, 1.0).unwrap(), (8000, 8000), "exactly 64 MP is allowed"); + assert!(matches!(scaled_size(8001, 8000, 1.0), Err(OcrError::ImageTooLarge)), "one row over is not"); + assert!(matches!(scaled_size(1000, 1000, 10.5), Err(OcrError::ImageTooLarge)), "a side over MAX_SIDE"); + assert!(matches!(scaled_size(1000, 1000, 0.0), Err(OcrError::ImageTooLarge)), "a side under one pixel"); + assert!(matches!(scaled_size(1000, 1000, f64::NAN), Err(OcrError::ImageTooLarge))); + let small = OcrImage::new(100, 100, vec![255; 100 * 100 * 4]).unwrap(); + assert!(matches!(resize_bilinear(&small, 100.5), Err(OcrError::ImageTooLarge))); + assert_eq!(resize_bilinear(&small, 2.0).unwrap().width, 200); + } + + fn white(w: u32, h: u32) -> OcrImage { + OcrImage::new(w, h, vec![255; (w * h * 4) as usize]).unwrap() + } + + /// Ink bounding box of a raster (None when blank). + fn ink_bbox(image: &OcrImage) -> Option<[f32; 4]> { + let gray = grayscale(image); + let (Some(lo), Some(hi)) = (gray.iter().min(), gray.iter().max()) else { return None }; + let t = (*lo as u32 + *hi as u32) / 2; + let (mut l, mut tp, mut r, mut b) = (u32::MAX, u32::MAX, 0u32, 0u32); + let mut any = false; + for (i, &g) in gray.iter().enumerate() { + if (g as u32) >= t { + continue; + } + any = true; + let (x, y) = ((i as u32) % image.width, (i as u32) / image.width); + l = l.min(x); + r = r.max(x); + tp = tp.min(y); + b = b.max(y); + } + any.then_some([l as f32, tp as f32, r as f32, b as f32]) + } + + /// A horizontal line of three bottom-heavy block glyphs (L shapes) on white — the fixture + /// for orientation detection (240 × 64 at 300 dpi). + fn l_line() -> OcrImage { + let (w, h) = (240u32, 64u32); + let mut img = white(w, h); + for gx in [20u32, 100, 180] { + fill(&mut img, gx, 12, 12, 40); + fill(&mut img, gx, 40, 36, 12); + } + img + } + + fn fill(img: &mut OcrImage, x0: u32, y0: u32, w: u32, h: u32) { + for y in y0..y0 + h { + for x in x0..x0 + w { + let o = ((y * img.width + x) * 4) as usize; + if let Some(p) = img.rgba.get_mut(o..o + 3) { + p.copy_from_slice(&[0, 0, 0]); + } + } + } + } + + /// The same fixture pre-rotated by k quarter turns clockwise, built independently of the + /// crate's rotation (so the test cannot pass by a shared bug). + fn rotated(k: u32) -> OcrImage { + let src = l_line(); + let (w, h) = (src.width as usize, src.height as usize); + let (nw, nh) = if k.is_multiple_of(2) { (src.width, src.height) } else { (src.height, src.width) }; + let mut out = white(nw, nh); + for (y, row) in src.rgba.chunks_exact(w * 4).enumerate() { + for (x, px) in row.as_chunks::<4>().0.iter().enumerate() { + if px[0] != 0 { + continue; + } + let (nx, ny) = match k { + 1 => (h - 1 - y, x), + 2 => (w - 1 - x, h - 1 - y), + 3 => (y, w - 1 - x), + _ => (x, y), + }; + fill(&mut out, nx as u32, ny as u32, 1, 1); + } + } + out + } + + /// A ruled fixture sheared by `degrees` (lines running downhill to the right for positive + /// angles): four 8-px bars across a 300 × 120 page. + fn sheared(degrees: f64) -> OcrImage { + let (w, h) = (300u32, 120u32); + let mut img = white(w, h); + let slope = degrees.to_radians().tan(); + for (base, height) in [(20u32, 8u32), (45, 8), (70, 8), (95, 8)] { + for x in 20..280u32 { + let y = (base as f64 + (x as f64 - 150.0) * slope).round().clamp(0.0, f64::from(h - height - 1)) as u32; + fill(&mut img, x, y, 1, height); + } + } + img + } + + /// With the four opt-in flags off and a dpi at or above 300, the raster passes through + /// untouched and the inverse transform is exact. + #[test] + fn flags_off_and_adequate_dpi_is_an_identity_pass_through() { + let src = l_line(); + let p = preprocess(src.clone(), 300.0, &PreprocessOptions::default()).unwrap(); + assert_eq!(p.image.rgba, src.rgba); + assert_eq!(p.dpi, 300.0); + assert_eq!(p.inverse_transform, InverseTransform::identity()); + assert_eq!(p.inverse_transform.map_rect([3.0, 7.0, 41.0, 19.0]), [3.0, 7.0, 41.0, 19.0]); + } + + /// With a step on, a scan below 300 dpi is enlarged by 300/src first, and its boxes map + /// back to the original raster. + #[test] + fn low_dpi_scans_are_upscaled_for_the_steps_and_boxes_map_back() { + let src = white(100, 50); + let on = PreprocessOptions { denoise: true, ..Default::default() }; + let p = preprocess(src.clone(), 150.0, &on).unwrap(); + assert_eq!((p.image.width, p.image.height), (200, 100)); + assert_eq!(p.dpi, 300.0); + assert_eq!(p.inverse_transform.map_rect([0.0, 0.0, 200.0, 100.0]), [0.0, 0.0, 100.0, 50.0]); + assert_eq!(p.inverse_transform.apply(2.0, 4.0), (1.0, 2.0)); + // 300 and 600 dpi are not resampled; a broken dpi is left alone. + assert_eq!(preprocess(src.clone(), 600.0, &on).unwrap().image.width, 100); + assert_eq!(preprocess(src, f32::NAN, &on).unwrap().image.width, 100); + } + + /// With every step off a low-dpi render is read exactly as rendered: not resampled, with + /// the identity as its way back. (Reading a 72 dpi scan through a 4x enlargement changed + /// what the models read: "due: \€250" came back as "ue: ?250".) + #[test] + fn with_every_step_off_a_low_dpi_render_is_not_resampled() { + let src = l_line(); + let p = preprocess(src.clone(), 72.0, &PreprocessOptions::default()).unwrap(); + assert_eq!(p.image.rgba, src.rgba); + assert_eq!((p.image.width, p.image.height), (src.width, src.height)); + assert_eq!(p.dpi, 72.0, "the render's own resolution"); + assert_eq!(p.inverse_transform, InverseTransform::identity()); + } + + /// The upscale refuses to break the side cap. + #[test] + fn upscaling_respects_the_side_cap() { + let src = white(9999, 10); + let r = preprocess(src, 72.0, &PreprocessOptions { denoise: true, ..Default::default() }); + assert!(matches!(r, Err(OcrError::ImageTooLarge)), "upscale past the cap: {r:?}"); + } + + /// An upright page is left alone by the orientation detector. + #[test] + fn upright_pages_are_not_rotated() { + let p = preprocess(l_line(), 300.0, &PreprocessOptions { auto_rotate: true, ..Default::default() }).unwrap(); + assert_eq!(p.inverse_transform, InverseTransform::identity()); + } + + /// Pages scanned at 90/180/270 are detected, turned upright, and their word boxes map back + /// onto the scanned raster's ink. + #[test] + fn quarter_turns_are_detected_corrected_and_mapped_back() { + for k in 1..=3 { + let scanned = rotated(k); + let p = preprocess(scanned.clone(), 300.0, &PreprocessOptions { auto_rotate: true, ..Default::default() }).unwrap(); + let corrected = ink_bbox(&p.image).unwrap_or_else(|| panic!("k={k}: the corrected page still has ink")); + // The corrected reading is a horizontal line again: wider than tall. + assert!(corrected[2] - corrected[0] > corrected[3] - corrected[1], "k={k}: {corrected:?}"); + // And the box maps back onto the scanned raster's own ink box. + let back = p.inverse_transform.map_rect(corrected); + let original = ink_bbox(&scanned).unwrap(); + for i in 0..4 { + assert!((back[i] - original[i]).abs() <= 2.0, "k={k}: {back:?} vs {original:?}"); + } + } + } + + /// Images under 32 px on a side are never judged; the step is a no-op. + #[test] + fn tiny_images_skip_orientation() { + let src = white(31, 31); + let p = preprocess(src, 300.0, &PreprocessOptions { auto_rotate: true, ..Default::default() }).unwrap(); + assert_eq!(p.inverse_transform, InverseTransform::identity()); + } + + /// Blank pages are inconclusive: every step leaves them alone. + #[test] + fn blank_pages_are_a_noop() { + let src = white(200, 200); + let p = preprocess(src, 300.0, &PreprocessOptions { auto_rotate: true, deskew: true, denoise: true, binarize: true }).unwrap(); + assert_eq!(p.inverse_transform, InverseTransform::identity()); + } + + /// The skew estimator sees a 3° shear (either direction) and leaves a straight page alone. + #[test] + fn skew_is_estimated_and_straight_pages_are_left_alone() { + for degrees in [3.0, -3.0] { + let angle = estimate_skew(&sheared(degrees)).unwrap_or_else(|| panic!("{degrees}° should be found")); + assert!((angle - degrees).abs() <= 0.5, "{degrees}°: got {angle}"); + } + assert!(estimate_skew(&sheared(0.0)).is_none(), "a straight page is a no-op"); + assert!(estimate_skew(&l_line()).is_none(), "glyph lines without a shear are a no-op"); + } + + /// Deskew straightens ±3° lines (the vertical span tightens) and the centre maps back to + /// itself. + #[test] + fn deskew_corrects_plus_and_minus_three_degrees() { + for degrees in [3.0, -3.0] { + let src = sheared(degrees); + let before = ink_bbox(&src).unwrap(); + let p = preprocess(src.clone(), 300.0, &PreprocessOptions { deskew: true, ..Default::default() }).unwrap(); + let after = ink_bbox(&p.image).unwrap(); + assert!(after[3] - after[1] < before[3] - before[1], "{degrees}°: {before:?} → {after:?}"); + // The exact inverse maps the corrected centre back to the scanned centre. + let (cx, cy) = (f64::from(src.width) / 2.0, f64::from(src.height) / 2.0); + let (bx, by) = p.inverse_transform.apply(cx, cy); + assert!((bx - cx).abs() < 1.0 && (by - cy).abs() < 1.0, "{degrees}°: ({bx}, {by})"); + } + } + + /// Deskew no-ops on small images (under 64 px) — the detector is never consulted. + #[test] + fn small_images_skip_deskew() { + let src = white(63, 63); + let p = preprocess(src, 300.0, &PreprocessOptions { deskew: true, ..Default::default() }).unwrap(); + assert_eq!(p.inverse_transform, InverseTransform::identity()); + } + + /// The median filter removes isolated specks, keeps solid strokes, and never changes the + /// dimensions. + #[test] + fn denoise_removes_specks_and_keeps_dimensions() { + let mut src = white(32, 32); + fill(&mut src, 15, 15, 1, 1); // salt + fill(&mut src, 4, 4, 5, 5); // a solid stroke + let p = preprocess(src, 300.0, &PreprocessOptions { denoise: true, ..Default::default() }).unwrap(); + assert_eq!((p.image.width, p.image.height), (32, 32)); + assert_eq!(p.image.byte(15.0, 15.0, 0), 255, "an isolated speck is gone"); + assert_eq!(p.image.byte(6.0, 6.0, 0), 0, "the inside of a solid block stays"); + assert_eq!(p.inverse_transform, InverseTransform::identity()); + } + + /// Sauvola keeps the polarity (dark ink stays dark, paper stays light) and the metadata. + /// The fixture is a thin grey stroke: its windows mix ink and paper, so the local + /// threshold separates them (a perfectly flat mid-tone area has no contrast and whitens, + /// which is Sauvola working as designed). + #[test] + fn binarize_keeps_polarity_and_metadata() { + let (w, h) = (64u32, 64u32); + let mut src = white(w, h); + for y in 26..30 { + for x in 8..56 { + let o = ((y * w + x) * 4) as usize; + if let Some(p) = src.rgba.get_mut(o..o + 3) { + p.copy_from_slice(&[70, 70, 70]); // a grey scan's ink, far below paper + } + } + } + let p = preprocess(src, 300.0, &PreprocessOptions { binarize: true, ..Default::default() }).unwrap(); + assert_eq!((p.image.width, p.image.height), (w, h)); + assert_eq!(p.dpi, 300.0); + assert_eq!(p.image.byte(30.0, 28.0, 0), 0, "ink stays dark"); + assert_eq!(p.image.byte(2.0, 2.0, 0), 255, "paper stays light"); + for px in p.image.rgba.as_chunks::<4>().0.iter() { + assert!((px[0] == 0 || px[0] == 255) && px[0] == px[1] && px[1] == px[2], "{px:?}"); + } + } + + /// The inverse transform composes: an upscaled, turned, deskewed page maps corrected + /// points back inside the original raster. + #[test] + fn inverse_transform_maps_points_back_through_every_step() { + let scanned = rotated(1); // a page photographed on its side + let p = preprocess(scanned.clone(), 150.0, &PreprocessOptions { auto_rotate: true, deskew: true, ..Default::default() }).unwrap(); + let (w, h) = (f64::from(p.image.width), f64::from(p.image.height)); + for (x, y) in [(0.0, 0.0), (w - 1.0, 0.0), (0.0, h - 1.0), (w - 1.0, h - 1.0), (w / 2.0, h / 2.0)] { + let (bx, by) = p.inverse_transform.apply(x, y); + assert!((-2.0..=f64::from(scanned.width) + 2.0).contains(&bx), "x {x} → {bx}"); + assert!((-2.0..=f64::from(scanned.height) + 2.0).contains(&by), "y {y} → {by}"); + } + // The scale step, alone, is exact. + assert_eq!(InverseTransform::scale(0.5).apply(4.0, 10.0), (2.0, 5.0)); + } + + /// A realistic letter page (one line of type on white, at 300 dpi) reads as upright: the + /// row axis bands decisively sharper than the column axis, and the near-balanced + /// bottom-heaviness of real mixed-case text is below the 15% flip bar, so the page is left + /// alone rather than risk flipping a good scan upside down. + #[test] + fn realistic_letter_pages_are_left_upright() { + let (w, h) = (1200u32, 1650u32); + let mut img = white(w, h); + let mut x = 120u32; + let base = 300u32; + let mut seed = 7u64; + let mut rnd = || { + seed = seed.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + (seed >> 33) as u32 + }; + while x < 1000 { + let word = rnd() % 5 + 2; + for _ in 0..word { + let kind = rnd() % 4; + let wl = rnd() % 8 + 3; + match kind { + 0 => fill(&mut img, x, base - 20, wl, 20), // x-height + 1 => fill(&mut img, x, base - 34, wl, 34), // ascender + 2 => fill(&mut img, x, base - 20, wl, 28), // descender + _ => { + fill(&mut img, x, base - 20, 2, 20); + fill(&mut img, x + 3, base - 14, 2, 14); // stems + } + } + x += wl + 2; + } + x += 12; // space + } + assert_eq!(detect_orientation(&img), 0); + let p = preprocess(img, 300.0, &PreprocessOptions { auto_rotate: true, ..Default::default() }).unwrap(); + assert_eq!(p.inverse_transform, InverseTransform::identity()); + } + + /// Hostile inputs are errors or no-ops, never panics: 1×1 rasters, all-black pages, NaN + /// boxes through the inverse map. + #[test] + fn hostile_inputs_never_panic() { + let tiny = white(1, 1); + let all = PreprocessOptions { auto_rotate: true, deskew: true, denoise: true, binarize: true }; + let p = preprocess(tiny, 300.0, &all).unwrap(); + assert_eq!((p.image.width, p.image.height), (1, 1)); + // All-black: no paper, nothing to judge. + let mut black = white(80, 80); + for px in black.rgba.as_chunks_mut::<4>().0 { + px.copy_from_slice(&[0, 0, 0, 255]); + } + let p = preprocess(black, 300.0, &all).unwrap(); + assert_eq!(p.inverse_transform, InverseTransform::identity(), "all-black is inconclusive"); + // NaN boxes never poison downstream math: the finite corners bound the mapped box, + // and a box with no finite corners comes back unchanged. + let mapped = p.inverse_transform.map_rect([f32::NAN, 0.0, 5.0, 5.0]); + assert!(mapped.iter().all(|v| v.is_finite()), "{mapped:?}"); + let unchanged = InverseTransform::identity().map_rect([f32::NAN; 4]); + assert!(unchanged.iter().all(|v| v.is_nan()), "{unchanged:?}"); + } + + /// The one-pixel dilation is load-bearing, pinned with a fixture that fragments without + /// it: a thin anti-aliased-looking stroke on a grey scan (alternating dark and light + /// pixels) — the raw Sauvola mask drops the light runs (the local window sits on a grey + /// page, so the threshold falls below them), while the binarized image keeps the stroke + /// whole. + #[test] + fn binarize_without_the_dilation_the_stroke_fragments_and_with_it_stays_whole() { + let (w, h) = (64u32, 64u32); + let mut src = white(w, h); + let set = |img: &mut OcrImage, x: u32, y: u32, v: u8| { + let o = ((y * w + x) * 4) as usize; + if let Some(p) = img.rgba.get_mut(o..o + 3) { + p.copy_from_slice(&[v, v, v]); + } + }; + for px in src.rgba.as_chunks_mut::<4>().0.iter_mut() { + px[..3].copy_from_slice(&[210, 210, 210]); // a grey scan's paper + } + // The stroke: one pixel tall, alternating dark (reads as ink) and light (an + // anti-aliased shoulder the threshold alone drops) in runs of two. + for x in 8..56u32 { + let v = if (x / 2) % 2 == 0 { 110 } else { 170 }; + set(&mut src, x, 32, v); + } + let mask = sauvola_mask(&src); + let at = |x: u32, y: u32| mask[y as usize * w as usize + x as usize]; + let dropped: Vec = (12..52).filter(|&x| !at(x, 32)).collect(); + assert!(!dropped.is_empty(), "without the dilation the stroke fragments: these stroke pixels fall under the threshold at {dropped:?}"); + let p = preprocess(src, 300.0, &PreprocessOptions { binarize: true, ..Default::default() }).unwrap(); + // With the dilation the stroke is whole: every position on it has ink within its + // 8-neighbourhood. + for x in 12..=51u32 { + let ink_near = (-1i32..=1).any(|dy| { + (-1i32..=1).any(|dx| { + let (nx, ny) = (x as i32 + dx, 32 + dy); + p.image.byte(f64::from(nx), f64::from(ny), 0) == 0 + }) + }); + assert!(ink_near, "the stroke is broken at x={x}"); + } + } + + /// The deskew inverse, checked at OFF-CENTRE points (the centre maps to itself under any + /// rotation, so it cannot catch a wrong sign): a point on a straightened line maps back + /// onto the sheared source line, uphill and downhill. + #[test] + fn deskew_inverse_maps_off_centre_points_back_onto_the_source_line() { + for degrees in [3.0, -3.0] { + let src = sheared(degrees); + let p = preprocess(src, 300.0, &PreprocessOptions { deskew: true, ..Default::default() }).unwrap(); + let slope = degrees.to_radians().tan(); + // The corrected first line sits at y = 20 (the fixture's base); the same ink was + // at 20 + (x − 150)·slope in the source. Well off centre on purpose. The + // rotation moves x a little too (it mixes in y: at most sin(5°)·40 ≈ 3.5 px + // here), so the line is checked at the mapped x — a wrong sign still misses it + // by ±(x−150)·slope, 6+ px at these offsets. + for x in [30.0f64, 150.0, 270.0] { + let (sx, sy) = p.inverse_transform.apply(x, 20.0); + assert!((sx - x).abs() < 4.0, "{degrees}° x={x}: mapped to ({sx}, {sy})"); + let expected = 20.0 + (sx - 150.0) * slope; + assert!((sy - expected).abs() < 2.0, "{degrees}° x={x}: ({sx}, {sy}), the source line is at {expected}"); + } + } + } +} diff --git a/crates/ocr/src/tesseract.rs b/crates/ocr/src/tesseract.rs new file mode 100644 index 0000000000..9fbcc574d8 --- /dev/null +++ b/crates/ocr/src/tesseract.rs @@ -0,0 +1,1022 @@ +//! The Tesseract engine: an installed `tesseract` binary driven as an optional external +//! process (`tesseract stdout -l tsv`), never linked, never bundled, never +//! downloaded. The whole module is compiled out for wasm32, where nothing may spawn. +//! +//! Security posture (hard rules): the program path comes from the PATH or a user-configured +//! location, never from document data; the language code is allow-listed before it reaches a +//! process; the page image goes to a private temp file (a fresh 0700 directory holding a 0600 +//! file on Unix) that is deleted afterwards; the run has a timeout and an output size cap; and +//! every failure comes back through [`OcrError`] with its cause named. + +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; +use std::sync::atomic::{AtomicU8, Ordering}; +use std::time::{Duration, Instant}; + +use super::{Line, OcrError, OcrImage, RecognizeOptions, Recognizer, Word}; + +/// How long one recognition may run before the process is killed. +const TIMEOUT: Duration = Duration::from_secs(120); +/// How much TSV output is accepted before the run is abandoned. +const MAX_OUTPUT: u64 = 64 * 1024 * 1024; +/// How much of the program's stderr is read back (its end): enough for a `--list-langs` list +/// printed there, small enough that error text stays a message, not a memory DoS. +const MAX_STDERR: u64 = 256 * 1024; +/// The most words one TSV reading may hold. Known limit: a reading with more words than this +/// is truncated at the cap — corrupt or runaway output must not become unbounded memory in +/// the word list, the text layer and the review UI. +pub const MAX_WORDS: usize = 50_000; +/// Tesseract's code for each language PdfCraft offers (ISO 639-1 → tessdata name). Only these +/// codes ever reach the process, whatever the settings say. +pub const LANGUAGE_CODES: &[(&str, &str)] = &[ + ("en", "eng"), + ("de", "deu"), + ("fr", "fra"), + ("es", "spa"), + ("it", "ita"), + ("pt", "por"), + ("ru", "rus"), + ("zh", "chi_sim"), + ("ja", "jpn"), + ("ko", "kor"), + ("ar", "ara"), + ("nl", "nld"), +]; + +/// English names for [`LANGUAGE_CODES`] (dropdowns show code + name). +pub const LANGUAGE_NAMES: &[(&str, &str)] = &[ + ("en", "English"), + ("de", "German"), + ("fr", "French"), + ("es", "Spanish"), + ("it", "Italian"), + ("pt", "Portuguese"), + ("ru", "Russian"), + ("zh", "Chinese (Simplified)"), + ("ja", "Japanese"), + ("ko", "Korean"), + ("ar", "Arabic"), + ("nl", "Dutch"), +]; + +/// Map a settings language code to a Tesseract one: trimmed, case-insensitive, allow-listed; +/// anything unknown falls back to `"eng"`. +pub fn language_code(code: &str) -> &'static str { + let c = code.trim().to_ascii_lowercase(); + LANGUAGE_CODES.iter().find(|(k, _)| *k == c).map(|(_, v)| *v).unwrap_or("eng") +} + +/// The user-configured program location, when valid: `$PDFCRAFT_TESSERACT` (a user +/// preference, never taken from document data). An invalid value — relative, missing, a +/// directory, not executable — means unavailable: an explicit setting is never silently +/// replaced by a PATH search. +pub fn configured_path() -> Option { + let value = std::env::var_os("PDFCRAFT_TESSERACT")?; + let program = PathBuf::from(value); + program_is_executable(&program).then_some(program) +} + +/// The program [`TesseractCli::find`] would run: a set `configured` location (the value of +/// `PDFCRAFT_TESSERACT`) decides alone, so an invalid one yields `None` and `search` (the PATH) +/// is not even consulted; only when nothing is configured does `search` decide. +fn locate(configured: Option, search: impl FnOnce() -> Option) -> Option { + match configured { + Some(value) => Some(PathBuf::from(value)).filter(|program| program_is_executable(program)), + None => search(), + } +} + +/// The first `tesseract` on the PATH that is an existing executable file. Entries that are not +/// absolute are refused: a relative PATH entry resolves against the caller's working +/// directory, which is neither stable nor a safe place to find a program. Also named by +/// `ocr_status` (the engine reports where it would look, without probing). +pub fn path_search() -> Option { + path_search_in(std::env::split_paths(&std::env::var_os("PATH")?)) +} + +/// [`path_search`] over an explicit directory list (so tests can pass one). +fn path_search_in(dirs: impl Iterator) -> Option { + for dir in dirs.filter(|d| d.is_absolute()) { + for name in program_file_names() { + let candidate = dir.join(name); + if program_is_executable(&candidate) { + return Some(candidate); + } + } + } + None +} + +/// The file names an installed `tesseract` binary goes by. +fn program_file_names() -> &'static [&'static str] { + #[cfg(windows)] + { + &["tesseract.exe"] + } + #[cfg(not(windows))] + { + &["tesseract"] + } +} + +/// Whether `program` is a usable program location (see [`program_is_executable`]); callers +/// that run a validated program by path (the engine's `--version` query) check it first. +pub fn program_is_usable(program: &Path) -> bool { + program_is_executable(program) +} + +/// Whether `program` is a usable configured location: an absolute path to an existing, +/// executable FILE (on Unix the executable bits must be set; elsewhere being a file is what +/// the OS requires). A directory or a missing file is simply unavailable — never a panic. +fn program_is_executable(program: &Path) -> bool { + if !program.is_absolute() || !program.is_file() { + return false; + } + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::metadata(program).is_ok_and(|m| m.permissions().mode() & 0o111 != 0) + } + #[cfg(not(unix))] + { + true + } +} + +/// One row of Tesseract's TSV output. `level` 5 rows are the words; `conf` is −1 on the header +/// rows (page, block, paragraph, line), which are not words. +struct TsvRow<'a> { + level: &'a str, + block: u32, + par: u32, + line: u32, + left: f32, + top: f32, + width: f32, + height: f32, + conf: f32, + text: &'a str, +} + +/// Parse one TSV row; `None` for anything unreadable (short rows, non-numeric fields), which +/// is skipped rather than trusted. A row with non-finite numbers or a negative width or height +/// is not a word either: geometry like that is output corruption, and trusting it puts NaN +/// into word boxes and on to the page. +fn tsv_row(row: &str) -> Option> { + let c: Vec<&str> = row.split('\t').collect(); + if c.len() < 12 { + return None; + } + let (block, par, line) = (c[2].trim().parse().ok()?, c[3].trim().parse().ok()?, c[4].trim().parse().ok()?); + let (left, top): (f32, f32) = (c[6].trim().parse().ok()?, c[7].trim().parse().ok()?); + let (width, height, conf): (f32, f32, f32) = (c[8].trim().parse().ok()?, c[9].trim().parse().ok()?, c[10].trim().parse().ok()?); + if !left.is_finite() || !top.is_finite() || !width.is_finite() || !height.is_finite() || !conf.is_finite() { + return None; // NaN/inf anywhere in the numbers + } + if width < 0.0 || height < 0.0 { + return None; // a word box has a non-negative size + } + Some(TsvRow { level: c[0], block, par, line, left, top, width, height, conf, text: c[11] }) +} + +/// Parse `tesseract … tsv` output into lines of words. Rows with confidence −1 are the +/// structural header rows and never become words; consecutive word rows that share their +/// (block, paragraph, line) form one [`Line`], in output (reading) order. Confidence clamps +/// into 0–100. Known limit: past [`MAX_WORDS`] the rest of the reading is dropped (truncated, +/// not an error — the words before the cap are still placed). +pub fn parse_tsv(tsv: &str) -> Vec { + parse_tsv_bounded(tsv).0 +} + +/// The parser utility keeps its bounded-prefix API; recognition also checks completeness. +fn parse_tsv_bounded(tsv: &str) -> (Vec, bool) { + let mut lines: Vec = Vec::new(); + let mut key: Option<(u32, u32, u32)> = None; + let mut words = 0usize; + for row in tsv.lines() { + let Some(r) = tsv_row(row) else { continue }; + if r.level != "5" || r.conf < 0.0 || r.text.trim().is_empty() { + continue; + } + if words >= MAX_WORDS { + return (lines, true); + } + let this = (r.block, r.par, r.line); + if key != Some(this) { + lines.push(Line::default()); + key = Some(this); + } + let word = Word::new(r.text, [r.left, r.top, r.left + r.width, r.top + r.height], "tesseract").with_confidence(r.conf.clamp(0.0, 100.0)); + if let Some(last) = lines.last_mut() { + last.words.push(word); + words += 1; + } + } + (lines, false) +} + +/// Parse `tesseract --list-langs` output: the header line ("List of available languages…"), +/// then one code per line. Lines with spaces are not codes (a header, a warning) and are left +/// out. +pub fn parse_list_langs(out: &str) -> Vec { + out.lines() + .skip_while(|l| !l.trim().is_empty() && l.trim().starts_with("List of available")) + .map(|l| l.trim().to_string()) + .filter(|l| !l.is_empty() && !l.contains(char::is_whitespace)) + .collect() +} + +/// The languages a `--list-langs` run reported: its stdout, or, when that holds none, its +/// stderr (Tesseract 3.x prints the list there). +pub fn list_langs_from(stdout: &str, stderr: &str) -> Vec { + let codes = parse_list_langs(stdout); + if codes.is_empty() { parse_list_langs(stderr) } else { codes } +} + +/// The first non-empty line of what `tesseract --version` printed: on stdout, or on stderr +/// where older versions print it. +fn version_line(stdout: &str, stderr: &str) -> Option { + let first = |text: &str| text.lines().map(str::trim).find(|line| !line.is_empty()).map(str::to_owned); + first(stdout).or_else(|| first(stderr)) +} + +/// The Tesseract program. Build one (via [`TesseractCli::find`] for the PATH copy, or +/// [`TesseractCli::new`] for a user-configured path) and reuse it: `--list-langs` runs at most +/// once. +pub struct TesseractCli { + program: PathBuf, + /// The timeout and the output cap are fields so tests can shorten them. + timeout: Duration, + max_output: u64, + /// Whether `--list-langs` has succeeded yet: 0 untried, 1 yes, 2 no. + probed: AtomicU8, + langs: Mutex>, +} + +impl TesseractCli { + /// The program at `path` (a user preference; never taken from document data). + pub fn new(program: PathBuf) -> TesseractCli { + TesseractCli { program, timeout: TIMEOUT, max_output: MAX_OUTPUT, probed: AtomicU8::new(0), langs: Mutex::new(Vec::new()) } + } + + /// The program this engine runs (`ocr_status` names it). + pub fn program(&self) -> &std::path::Path { + &self.program + } + + /// The program at a user-configured `path`, if it is usable there: only an absolute path + /// to an existing, executable file is accepted (a directory, a missing file or a relative + /// path is unavailable, never a panic), and it must answer `--list-langs`. + pub fn at(program: PathBuf) -> Option { + if !program_is_executable(&program) { + return None; + } + let t = TesseractCli::new(program); + let langs = t.list_langs().ok()?; + *t.langs.lock().unwrap_or_else(|e| e.into_inner()) = langs; + t.probed.store(1, Ordering::Relaxed); + Some(t) + } + + /// The program this machine offers, if it answers `--list-langs`: the user-configured + /// location ([`configured_path`] / `$PDFCRAFT_TESSERACT`) when valid, else the first + /// absolute PATH entry that holds an executable `tesseract`. + /// + /// # The PATH-lookup risk + /// + /// Resolving through the PATH trusts every directory on it: a binary planted earlier on + /// the PATH runs as a subprocess with this application's privileges — it sees the page + /// image and anything the process can reach. The language code and the page image still + /// never come from document data, but a user who handles untrusted documents should pin + /// the exact program with `PDFCRAFT_TESSERACT`, which is validated (absolute, existing, + /// executable file) before any process starts. + /// + /// A `PDFCRAFT_TESSERACT` that is set but invalid fails closed: it is never replaced by + /// the PATH search. + pub fn find() -> Option { + TesseractCli::at(locate(std::env::var_os("PDFCRAFT_TESSERACT"), path_search)?) + } + + /// Run `tesseract --list-langs` and parse it; also decides [`TesseractCli::available`]. + fn list_langs(&self) -> Result, OcrError> { + let (out, err) = self.run_capture(&["--list-langs"], self.timeout, self.max_output)?; + let codes = list_langs_from(&out, &err); + if codes.is_empty() { + return Err(OcrError::NoTesseract("--list-langs reported no languages".into())); + } + Ok(codes + .into_iter() + .map(|c| { + let name = LANGUAGE_NAMES.iter().find(|(k, _)| *k == c.as_str()).map(|(_, n)| (*n).to_string()).unwrap_or_else(|| c.clone()); + (c, name) + }) + .collect()) + } + + /// The program's first `--version` line (named by `ocr_status`); `None` when the program + /// does not answer. Not cached: status asks rarely, and a re-install mid-session should + /// show up without a restart. + pub fn version(&self) -> Option { + let (out, err) = self.run_capture(&["--version"], self.timeout, self.max_output).ok()?; + version_line(&out, &err) + } + + /// Run the program with fixed argv, its stdout in a private temp file, under `timeout` + /// seconds and an output cap of `max_output` bytes. Returns the (capped) stdout. + fn run(&self, args: &[&str], timeout: Duration, max_output: u64) -> Result { + self.run_capture(args, timeout, max_output).map(|(out, _)| out) + } + + /// [`Self::run`], also returning what the program wrote to stderr (up to + /// [`MAX_STDERR`] bytes, its end kept): some Tesseract versions print the `--list-langs` + /// list there instead of on stdout. + fn run_capture(&self, args: &[&str], timeout: Duration, max_output: u64) -> Result<(String, String), OcrError> { + let dir = TempDir::create()?; + let out_path = dir.path.join("out.txt"); + let err_path = dir.path.join("err.txt"); + let out_file = std::fs::File::create(&out_path).map_err(|e| OcrError::Process(format!("creating the output file failed: {e}")))?; + // stderr goes to a file, not a pipe: an undrained pipe fills (around 64 KiB) and then + // blocks the child mid-write, which a watcher can only read as a timeout. A file never + // blocks; it is read back as a short tail after the run. + let err_file = std::fs::File::create(&err_path).map_err(|e| OcrError::Process(format!("creating the error file failed: {e}")))?; + // Known limit: only the direct child is killed at the timeout or on an early cap. A + // wrapper script's children (grandchildren of this process) may survive the kill; the + // cap and the timeout still bound what this run reads and how long it waits. + let mut child = + std::process::Command::new(&self.program) + .args(args) + .stdin(std::process::Stdio::null()) + .stdout(out_file) + .stderr(err_file) + .current_dir(&dir.path) + // A minimal environment: nothing from the document or the caller's shell beyond + // what an installed tesseract needs — its data prefix, the PATH (bare-name lookup + // and DLL loading), the Windows system root, and a temp dir it may write to. + .env_clear() + .envs(std::env::vars().filter(|(k, _)| { + k.eq_ignore_ascii_case("TESSDATA_PREFIX") || k.eq_ignore_ascii_case("PATH") || k.eq_ignore_ascii_case("SystemRoot") + })) + .env("TMP", &dir.path) + .env("TEMP", &dir.path) + .spawn() + .map_err(|e| OcrError::NoTesseract(format!("{} could not start: {e}", self.program.display())))?; + let started = Instant::now(); + let status = loop { + match child.try_wait() { + Ok(Some(status)) => break Ok(status), + Ok(None) => { + // The output caps are enforced while the child runs: a flood is killed as + // soon as a file passes its cap, not after it has finished writing. The + // read-back below caps what this run KEEPS of stderr; this cap bounds what + // the child can write there in the first place. + if std::fs::metadata(&out_path).map(|m| m.len()).unwrap_or(0) > max_output + || std::fs::metadata(&err_path).map(|m| m.len()).unwrap_or(0) > MAX_STDERR + { + let _ = child.kill(); + let _ = child.wait(); + break Err(OcrError::OutputTooLarge(max_output)); + } + if started.elapsed() >= timeout { + let _ = child.kill(); + let _ = child.wait(); + break Err(OcrError::Timeout(timeout.as_secs())); + } + std::thread::sleep(Duration::from_millis(20)); + } + Err(e) => { + let _ = child.kill(); + let _ = child.wait(); + break Err(OcrError::Process(format!("waiting for {} failed: {e}", self.program.display()))); + } + } + }; + let status = status?; + // Whatever the exit, the stderr tail names the cause (capped: error text stays a + // message, not a memory DoS). + let stderr = std::fs::File::open(&err_path) + .and_then(|mut f| { + use std::io::{Read, Seek, SeekFrom}; + let len = f.metadata()?.len().min(MAX_STDERR); + f.seek(SeekFrom::End(-(len as i64)))?; + let mut buf = Vec::with_capacity(len as usize); + f.take(len).read_to_end(&mut buf)?; + Ok::<_, std::io::Error>(buf) + }) + .unwrap_or_default(); + let stderr = String::from_utf8_lossy(&stderr).into_owned(); + // The message names only the end of it. + let tail = stderr.char_indices().rev().nth(2047).map_or(0, |(i, _)| i); + let note = stderr.get(tail..).unwrap_or_default().trim(); + if !status.success() { + return Err(OcrError::Process(format!( + "{} exited with {}: {}", + self.program.display(), + status.code().map(|c| c.to_string()).unwrap_or_else(|| "a signal".into()), + if note.is_empty() { "no error message" } else { note } + ))); + } + let len = std::fs::metadata(&out_path).map(|m| m.len()).unwrap_or(0); + if len > max_output { + return Err(OcrError::OutputTooLarge(max_output)); + } + let bytes = std::fs::read(&out_path).map_err(|e| OcrError::Process(format!("reading the output failed: {e}")))?; + Ok((String::from_utf8_lossy(&bytes).into_owned(), stderr)) + } + + /// Retry `--list-langs` once (an install may have appeared since the last try). + fn refresh(&self) -> Result<(), OcrError> { + let langs = self.list_langs()?; + *self.langs.lock().unwrap_or_else(|e| e.into_inner()) = langs; + self.probed.store(1, Ordering::Relaxed); + Ok(()) + } +} + +impl Recognizer for TesseractCli { + fn id(&self) -> &'static str { + "tesseract" + } + + /// Whether the program answers `--list-langs` (remembered; one process at most after the + /// first call). + fn available(&self) -> bool { + match self.probed.load(Ordering::Relaxed) { + 1 => true, + 2 => false, + _ => self.refresh().is_ok(), + } + } + + fn languages(&self) -> Vec<(String, String)> { + if self.probed.load(Ordering::Relaxed) == 2 { + return Vec::new(); + } + let langs = self.langs.lock().unwrap_or_else(|e| e.into_inner()); + if langs.is_empty() { + drop(langs); + return self.refresh().map_or(Vec::new(), |()| self.langs.lock().unwrap_or_else(|e| e.into_inner()).clone()); + } + langs.clone() + } + + /// Write the page as a PNG and read it back as TSV. + fn recognize(&self, image: &OcrImage, options: &RecognizeOptions) -> Result, OcrError> { + let lang = language_code(&options.language); + let png = encode_png(image)?; + let dir = TempDir::create()?; + let png_path = dir.path.join("page.png"); + { + let mut f = std::fs::File::create(&png_path).map_err(|e| OcrError::Process(format!("creating the image file failed: {e}")))?; + set_private(&mut f); + f.write_all(&png).map_err(|e| OcrError::Process(format!("writing the image failed: {e}")))?; + } + // Fixed argv, no shell: tesseract stdout -l tsv. + let tsv = self.run(&[&png_path.to_string_lossy(), "stdout", "-l", lang, "tsv"], self.timeout, self.max_output)?; + let (lines, truncated) = parse_tsv_bounded(&tsv); + if truncated { + return Err(OcrError::Recognize(format!("the {MAX_WORDS}-word limit was exceeded; the incomplete reading was rejected"))); + } + Ok(lines) + } +} + +/// Encode an RGBA raster as PNG (the interchange format Tesseract reads best). +fn encode_png(image: &OcrImage) -> Result, OcrError> { + let mut out = Vec::new(); + let mut enc = png::Encoder::new(&mut out, image.width, image.height); + enc.set_color(png::ColorType::Rgba); + enc.set_depth(png::BitDepth::Eight); + let mut writer = enc.write_header().map_err(|e| OcrError::Process(format!("encoding the page image failed: {e}")))?; + writer.write_image_data(&image.rgba).map_err(|e| OcrError::Process(format!("encoding the page image failed: {e}")))?; + writer.finish().map_err(|e| OcrError::Process(format!("encoding the page image failed: {e}")))?; + Ok(out) +} + +/// Make a just-created file owner-only, where the OS supports it (Unix). Windows relies on the +/// user's private %TEMP% ACLs; the fresh directory keeps other sessions' files out of sight. +fn set_private(f: &mut std::fs::File) { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let _ = f.set_permissions(std::fs::Permissions::from_mode(0o600)); + } + #[cfg(not(unix))] + let _ = f; +} + +/// A fresh directory under the system temp dir, deleted (with its contents) when dropped. +/// On Unix it is created 0700, so the page image inside is private. +struct TempDir { + path: PathBuf, +} + +impl TempDir { + fn create() -> Result { + // The recognizer is shared across threads (`Arc`), so two `recognize` calls can run at + // once, and on a coarse clock they can read the same nanosecond. One counter value per + // attempt keeps the names of one process apart without trusting the clock. + static SEQ: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let nanos = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_nanos()).unwrap_or(0); + // Created owner-only in one step on Unix (no window in which the directory exists with + // the default, wider mode); `create` refuses a path that already exists. + let mut builder = std::fs::DirBuilder::new(); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o700); + } + // Elsewhere the default mode stands (Windows relies on the user's private %TEMP% ACLs). + #[cfg(not(unix))] + let _ = &mut builder; + // A leftover directory of an earlier process with the same pid can still take a name; + // that one collision is retried with the next counter value, anything else is fatal. + for _ in 0..4 { + let seq = SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let path = std::env::temp_dir().join(format!("pdfcraft-ocr-{}-{nanos}-{seq}", std::process::id())); + match builder.create(&path) { + Ok(()) => return Ok(TempDir { path }), + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(e) => return Err(OcrError::Process(format!("creating the private temp directory failed: {e}"))), + } + } + Err(OcrError::Process("creating the private temp directory failed: every candidate name already exists".into())) + } +} + +impl Drop for TempDir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.path); + } +} + +#[cfg(test)] +mod tests { + use super::*; + // The Unix process tests below build fake scripts and name their paths. + #[cfg(unix)] + use std::path::Path; + + /// Several directories alive at once never share a name: the recognizer is shared across + /// threads (`Arc`), so two `recognize` calls can create their directories concurrently, + /// and the counter in the name — not the clock alone — tells them apart. + #[test] + fn temp_directories_alive_at_once_have_distinct_names() { + let dirs: Vec = (0..32).map(|_| TempDir::create().expect("create")).collect(); + let distinct: std::collections::BTreeSet = dirs.iter().map(|d| d.path.clone()).collect(); + assert_eq!(distinct.len(), dirs.len(), "{:?}", dirs.iter().map(|d| &d.path).collect::>()); + } + + /// A tiny image with one dark square (a synthetic "word"); the Unix process tests use it. + #[cfg(unix)] + fn blot(w: u32, h: u32, x0: u32, y0: u32, bw: u32, bh: u32) -> OcrImage { + let mut rgba = vec![255u8; (w * h * 4) as usize]; + for y in y0..(y0 + bh).min(h) { + for x in x0..(x0 + bw).min(w) { + let o = ((y * w + x) * 4) as usize; + rgba[o..o + 3].copy_from_slice(&[0, 0, 0]); + } + } + OcrImage::new(w, h, rgba).unwrap() + } + + /// The literal TSV fixture: two header rows (conf −1) and two lines of words. + const TSV: &str = "level\tpage\tblock\tpar\tline\tword\tleft\ttop\twidth\theight\tconf\ttext +1\t1\t0\t0\t0\t0\t0\t0\t100\t50\t-1\t +5\t1\t1\t1\t1\t1\t10\t10\t40\t12\t96.5\tHello +5\t1\t1\t1\t1\t2\t55\t10\t30\t12\t91.0\tworld +5\t1\t1\t2\t2\t1\t10\t30\t20\t12\t42.123\tsecond"; + + #[test] + fn tsv_parses_words_into_lines() { + let lines = parse_tsv(TSV); + assert_eq!(lines.len(), 2, "two (block, par, line) groups: {lines:?}"); + let first = &lines[0].words; + assert_eq!(first.len(), 2); + assert_eq!(first[0].text, "Hello"); + assert_eq!(first[0].rect, [10.0, 10.0, 50.0, 22.0], "left + width, top + height"); + assert_eq!(first[0].confidence, Some(96.5)); + assert_eq!(first[0].source, "tesseract"); + assert_eq!(first[1].text, "world"); + assert_eq!(lines[1].words.len(), 1, "a new par starts a new line"); + assert_eq!(lines[1].words[0].text, "second"); + assert_eq!(lines[1].words[0].confidence, Some(42.123)); + } + + /// Conf −1 rows and level ≠ 5 rows are structure, not words; blank text is dropped. + #[test] + fn tsv_treats_header_rows_and_blank_text_as_not_words() { + let tsv = "5\t1\t1\t1\t1\t1\t0\t0\t10\t10\t-1\tskipped\n\ + 4\t1\t1\t1\t1\t0\t0\t0\t10\t10\t95.0\tline row\n\ + 5\t1\t1\t1\t1\t1\t0\t0\t10\t10\t95.0\t \n\ + 5\t1\t1\t1\t1\t2\t0\t0\t10\t10\t88\tkept"; + let lines = parse_tsv(tsv); + assert_eq!(lines.len(), 1, "{lines:?}"); + assert_eq!(lines[0].words.len(), 1); + assert_eq!(lines[0].words[0].text, "kept"); + } + + /// Garbage rows (short, non-numeric) are skipped; CRLF endings survive; boxes saturate + /// instead of overflowing. + #[test] + fn tsv_skips_garbage_and_never_panics() { + let lines = parse_tsv("not a tsv row\n\n5\t1\t1\n5\t1\t1\t1\t1\t1\tx\ty\tw\th\tzzz\ttext\r\n"); + assert!(lines.is_empty(), "{lines:?}"); + let big = parse_tsv(&format!("5\t1\t1\t1\t1\t1\t{}\t0\t{}\t10\t50\tbig", u32::MAX, u32::MAX)); + let rect = big[0].words[0].rect; + assert!(rect[2].is_finite() && rect[3].is_finite(), "no overflow: {rect:?}"); + // A row whose text field is empty yields no words. + assert!(parse_tsv("5\t1\t1\t1\t1\t1\t1\t1\t2\t2\t50\t\t").is_empty()); + } + + /// Non-finite numbers and negative sizes are not words: a row with NaN/inf coordinates or + /// a negative width/height never reaches the output (and so never reaches the content + /// stream). `1e39` overflows f32 to infinity, so it is refused the same way. + #[test] + fn tsv_rejects_non_finite_and_negative_geometry() { + let row = |left: &str, top: &str, w: &str, h: &str, conf: &str| format!("5\t1\t1\t1\t1\t1\t{left}\t{top}\t{w}\t{h}\t{conf}\tword"); + for (why, r) in [ + ("NaN left", row("NaN", "0", "10", "10", "50")), + ("NaN width", row("0", "0", "NaN", "10", "50")), + ("NaN height", row("0", "0", "10", "NaN", "50")), + ("NaN confidence", row("0", "0", "10", "10", "NaN")), + ("inf left", row("inf", "0", "10", "10", "50")), + ("inf width", row("0", "0", "inf", "10", "50")), + ("1e39 width overflows to inf", row("0", "0", "1e39", "10", "50")), + ("negative width", row("0", "0", "-10", "10", "50")), + ("negative height", row("0", "0", "10", "-10", "50")), + ] { + let lines = parse_tsv(&r); + assert!(lines.is_empty(), "{why}: {lines:?}"); + } + // A sane row still parses. + assert_eq!(parse_tsv(&row("0", "0", "10", "10", "50"))[0].words[0].text, "word"); + } + + /// A reading longer than the word cap is truncated at [`MAX_WORDS`] (a known limit, not an + /// error): the words before the cap are kept, the rest is dropped. + #[test] + fn tsv_truncates_readings_past_the_word_cap() { + let mut tsv = String::new(); + for i in 0..MAX_WORDS + 5 { + tsv.push_str(&format!("5\t1\t1\t1\t1\t{}\t{}\t10\t40\t12\t96.5\tw{i}\n", i % 7 + 1, i % 500)); + } + let lines = parse_tsv(&tsv); + let count: usize = lines.iter().map(|l| l.words.len()).sum(); + assert_eq!(count, MAX_WORDS, "truncated at the cap"); + assert_eq!(lines.last().and_then(|l| l.words.last()).map(|w| w.text.as_str()), Some(&*format!("w{}", MAX_WORDS - 1))); + } + + #[test] + fn language_table_maps_and_falls_back() { + assert_eq!(language_code("en"), "eng"); + assert_eq!(language_code(" EN "), "eng", "trimmed, case-insensitive"); + assert_eq!(language_code("de"), "deu"); + assert_eq!(language_code("ZH"), "chi_sim"); + assert_eq!(language_code("ja"), "jpn"); + assert_eq!(language_code("ko"), "kor"); + assert_eq!(language_code("nl"), "nld"); + assert_eq!(language_code("klingon"), "eng", "unknown → eng"); + assert_eq!(language_code(""), "eng"); + // The security property: whatever the settings say, only allow-listed codes reach a + // process. + for code in ["en", "de", "fr", "es", "it", "pt", "ru", "zh", "ja", "ko", "ar", "nl", "klingon", "", " eng ", "EN/;rm"] { + let mapped = language_code(code); + assert!(LANGUAGE_CODES.iter().any(|(_, v)| *v == mapped), "{code:?} → {mapped:?} is not allow-listed"); + } + } + + #[test] + fn list_langs_come_from_stderr_when_stdout_has_none() { + let listing = "List of available languages in \"/usr/share/tessdata/\" (2):\neng\nosd\n"; + assert_eq!(list_langs_from(listing, ""), ["eng", "osd"]); + assert_eq!(list_langs_from("", listing), ["eng", "osd"], "3.x prints the list on stderr"); + assert_eq!(list_langs_from("", "Error opening data file\n"), Vec::::new(), "a warning is not a language"); + assert_eq!(list_langs_from("eng\n", "deu\n"), ["eng"], "stdout wins when it has the list"); + } + + /// The private temp directory is created owner-only (Unix) and removed on drop. + #[test] + fn temp_dir_is_private_and_removed() { + let dir = TempDir::create().unwrap(); + let path = dir.path.clone(); + assert!(path.is_dir()); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + assert_eq!(std::fs::metadata(&path).unwrap().permissions().mode() & 0o777, 0o700); + } + drop(dir); + assert!(!path.exists()); + } + + /// A set `PDFCRAFT_TESSERACT` decides alone: when it is invalid, [`TesseractCli::find`] does + /// not fall back to a PATH copy (which would run a program the user did not name); the PATH + /// search only runs when nothing is configured. + #[test] + fn an_invalid_configured_program_fails_closed() { + let Ok(real) = std::env::current_exe() else { return }; + let searched = std::cell::Cell::new(false); + let on_path = || { + searched.set(true); + Some(real.clone()) + }; + assert_eq!(locate(None, on_path), Some(real.clone()), "nothing configured: the PATH search decides"); + assert!(searched.get()); + searched.set(false); + assert_eq!(locate(Some(real.clone().into_os_string()), on_path), Some(real.clone()), "a valid location is used as is"); + assert!(!searched.get(), "and the PATH is not searched"); + let dir = real.parent().map(Path::to_path_buf).unwrap_or_default().into_os_string(); + for bad in [std::ffi::OsString::from("relative/tesseract"), std::ffi::OsString::from(""), dir, "/no/such/program-0123456789".into()] { + assert_eq!(locate(Some(bad.clone()), on_path), None, "{bad:?} is invalid, so there is no program"); + assert!(!searched.get(), "{bad:?}: an invalid location never falls back to the PATH"); + } + } + + #[test] + fn the_version_comes_from_stdout_or_stderr() { + assert_eq!(version_line("tesseract 5.4.0\n leptonica-1.84\n", ""), Some("tesseract 5.4.0".into())); + assert_eq!(version_line("", "\ntesseract 4.0.0\n"), Some("tesseract 4.0.0".into()), "older versions print it on stderr"); + assert_eq!(version_line(" \n", ""), None); + } + + #[test] + fn list_langs_output_parses() { + let out = "List of available languages in \"C:\\Program Files\\Tesseract-OCR\\tessdata/\" (2):\neng\nosd\n"; + assert_eq!(parse_list_langs(out), vec!["eng".to_string(), "osd".to_string()]); + assert_eq!(parse_list_langs("List of available languages (1):\neng\n"), vec!["eng".to_string()]); + assert!(parse_list_langs("").is_empty()); + } + + /// The real binary, when it is installed: a run end to end, with words that carry a + /// confidence and the engine's name (skipped where `tesseract` is missing). + #[test] + fn recognize_runs_the_installed_binary() { + let Some(t) = TesseractCli::find() else { + eprintln!("skipped: the tesseract program is not installed"); + return; + }; + assert_eq!(t.id(), "tesseract"); + assert!(t.available()); + // 300 dpi-ish block capitals, black on white. + let (w, h, rgba) = hello_world_image(); + let image = OcrImage::new(w, h, rgba).unwrap(); + let lines = t.recognize(&image, &RecognizeOptions::default()).unwrap(); + let text: Vec = lines.iter().map(Line::text).collect(); + assert!(text.iter().any(|t| t.to_uppercase().contains("HELL")), "{text:?}"); + let words: Vec<&Word> = lines.iter().flat_map(|l| &l.words).collect(); + assert!(words.iter().all(|w| w.source == "tesseract")); + assert!(words.iter().all(|w| matches!(w.confidence, Some(c) if (0.0..=100.0).contains(&c))), "{words:?}"); + } + + /// `available`/`languages` come from `--list-langs` (skipped without the binary). + #[test] + fn languages_come_from_list_langs() { + let Some(t) = TesseractCli::find() else { + eprintln!("skipped: the tesseract program is not installed"); + return; + }; + let langs = t.languages(); + assert!(!langs.is_empty()); + assert!(langs.iter().all(|(c, n)| !c.is_empty() && !n.is_empty()), "{langs:?}"); + } + + #[cfg(unix)] + #[test] + fn recognize_refuses_a_truncated_reading() { + let dir = TempDir::create().unwrap(); + let script = dir.path.join("synthetic-recognizer"); + let mut tsv = String::new(); + for i in 0..=MAX_WORDS { + use std::fmt::Write as _; + writeln!(tsv, "5\t1\t1\t1\t1\t{i}\t0\t0\t1\t1\t95\tword{i}").unwrap(); + } + let exact = tsv.lines().take(MAX_WORDS).collect::>().join("\n"); + std::fs::write(dir.path.join("reading.tsv"), tsv).unwrap(); + std::fs::write(&script, "#!/bin/sh\ncat \"$(dirname \"$0\")/reading.tsv\"\n").unwrap(); + make_executable(&script); + let recognizer = TesseractCli::new(script); + let result = recognizer.recognize(&blot(8, 8, 0, 0, 4, 4), &RecognizeOptions::default()); + let status = result.as_ref().map(|lines| lines.iter().map(|line| line.words.len()).sum::()).map_err(ToString::to_string); + assert!( + matches!(result, Err(OcrError::Recognize(ref reason)) if reason.contains("word limit")), + "a capped reading must be reported, never accepted as complete: {status:?}" + ); + std::fs::write(dir.path.join("reading.tsv"), exact).unwrap(); + let complete = recognizer.recognize(&blot(8, 8, 0, 0, 4, 4), &RecognizeOptions::default()).unwrap(); + assert_eq!(complete.iter().map(|line| line.words.len()).sum::(), MAX_WORDS); + } + + /// A hung binary is killed at the timeout and reported (Unix: a fake `tesseract` script; + /// this test needs a real process, so it is off the web and off Windows). + #[cfg(unix)] + #[test] + fn a_hung_program_is_killed_at_the_timeout() { + let dir = std::env::temp_dir().join(format!("pdfcraft-ocr-test-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let script = dir.join("tesseract-hang"); + std::fs::write(&script, "#!/bin/sh\nsleep 30\n").unwrap(); + make_executable(&script); + let mut t = TesseractCli::new(script); + t.timeout = Duration::from_millis(500); + let started = Instant::now(); + let r = t.recognize(&blot(8, 8, 0, 0, 4, 4), &RecognizeOptions::default()); + assert!(matches!(r, Err(OcrError::Timeout(_))), "{r:?}"); + assert!(started.elapsed() < Duration::from_secs(10), "killed, not waited out"); + let _ = std::fs::remove_dir_all(&dir); + } + + /// Output past the cap kills the run EARLY, while the child is still writing (Unix only, + /// same fake-script trick): the script floods far past the cap and would sleep for 30 s + /// after; a cap checked only after the exit would come back as a timeout instead. + #[cfg(unix)] + #[test] + fn flooding_output_is_capped_and_killed_early() { + let dir = std::env::temp_dir().join(format!("pdfcraft-ocr-test-cap-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let script = dir.join("tesseract-flood"); + std::fs::write(&script, "#!/bin/sh\nhead -c 100000000 /dev/zero | tr '\\0' 'x'\nsleep 30\n").unwrap(); + make_executable(&script); + let mut t = TesseractCli::new(script); + t.max_output = 1024; + t.timeout = Duration::from_secs(20); + let started = Instant::now(); + let r = t.recognize(&blot(8, 8, 0, 0, 4, 4), &RecognizeOptions::default()); + assert!(matches!(r, Err(OcrError::OutputTooLarge(_))), "{r:?}"); + assert!(started.elapsed() < Duration::from_secs(10), "killed early, at {:?}", started.elapsed()); + let _ = std::fs::remove_dir_all(&dir); + } + + /// An stderr flood does not block the child, so it does not cause a false timeout (Unix + /// only): the script writes megabytes of stderr — far more than a pipe would swallow — + /// then exits successfully with a small output, which must come back as a reading. + #[cfg(unix)] + #[test] + fn an_stderr_flood_does_not_cause_a_false_timeout() { + let dir = std::env::temp_dir().join(format!("pdfcraft-ocr-test-err-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let script = dir.join("tesseract-err-flood"); + // (below MAX_STDERR, past which the child is stopped: `an_stderr_flood_is_killed_while_the_child_runs`) + std::fs::write(&script, "#!/bin/sh\nhead -c 200000 /dev/zero | tr '\\0' 'e' >&2\necho flooded away\n").unwrap(); + make_executable(&script); + let mut t = TesseractCli::new(script); + t.timeout = Duration::from_secs(10); + let r = t.recognize(&blot(8, 8, 0, 0, 4, 4), &RecognizeOptions::default()); + assert!(r.is_ok(), "stderr flood must not fail the run: {r:?}"); + let _ = std::fs::remove_dir_all(&dir); + } + + /// The child runs with a minimal environment (its data prefix, the PATH, the Windows + /// system root, and a temp dir), never the caller's whole environ. This is checked with a + /// script that prints its own environ as the "TSV" (Unix only). + #[cfg(unix)] + #[test] + fn the_child_gets_a_minimal_environment() { + let dir = std::env::temp_dir().join(format!("pdfcraft-ocr-test-env-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let script = dir.join("tesseract-env"); + std::fs::write(&script, "#!/bin/sh\nenv | grep -v '^_='\n").unwrap(); + make_executable(&script); + let t = TesseractCli::new(script); + let png = encode_png(&blot(8, 8, 0, 0, 4, 4)).unwrap(); + std::fs::write(dir.join("page.png"), png).unwrap(); + let png_arg = dir.join("page.png").to_string_lossy().into_owned(); + let tsv = t.run(&[png_arg.as_str(), "stdout", "-l", "eng", "tsv"], t.timeout, t.max_output).unwrap(); + for (why, name) in [("the child's temp dir", "TMP="), ("temp", "TEMP="), ("lookup path", "PATH=")] { + assert!(tsv.contains(name), "{why} must be passed: {tsv}"); + } + // cargo always sets CARGO_* for the test process; env_clear must drop them. + assert!(!tsv.contains("CARGO_"), "the caller's environment must not leak: {tsv}"); + let _ = std::fs::remove_dir_all(&dir); + } + + #[cfg(unix)] + fn make_executable(p: &Path) { + use std::os::unix::fs::PermissionsExt; + let _ = std::fs::set_permissions(p, std::fs::Permissions::from_mode(0o755)); + } + + /// The temp directory and its contents are gone after a run (private files do not linger). + #[cfg(unix)] + #[test] + fn temp_files_are_deleted_and_private() { + let dir = TempDir::create().unwrap(); + assert!(dir.path.is_dir()); + std::fs::write(dir.path.join("x"), b"secret").unwrap(); + let p = dir.path.clone(); + drop(dir); + assert!(!p.exists(), "the temp directory is deleted after use"); + } + + /// A configured program location is only usable as an absolute path to an existing, + /// executable file: a relative path, a missing file and a directory are unavailable (and + /// [`TesseractCli::at`] refuses them without spawning anything). + #[test] + fn a_configured_program_must_be_an_absolute_executable_file() { + assert!(!program_is_executable(Path::new(".")), "relative"); + assert!(!program_is_executable(Path::new("definitely/missing/tesseract")), "missing"); + let temp = std::env::temp_dir(); + assert!(!program_is_executable(&temp), "a directory is not a program"); + let file = temp.join(format!("pdfcraft-ocr-not-a-program-{}", std::process::id())); + std::fs::write(&file, b"").unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let _ = std::fs::set_permissions(&file, std::fs::Permissions::from_mode(0o644)); + assert!(!program_is_executable(&file), "a file without executable bits is not a program"); + } + #[cfg(not(unix))] + assert!(program_is_executable(&file), "an existing file is usable on this platform"); + let _ = std::fs::remove_file(&file); + // `at` validates before any process starts, so these are None and not panics. + assert!(TesseractCli::at(PathBuf::from(".")).is_none()); + assert!(TesseractCli::at(temp).is_none()); + assert!(TesseractCli::at(PathBuf::from("definitely/missing/tesseract")).is_none()); + } + + /// The PATH search resolves through absolute entries only and skips relative ones: a + /// relative PATH entry would resolve against the caller's working directory. + #[test] + fn path_search_skips_relative_entries() { + let dir = std::env::temp_dir().join(format!("pdfcraft-ocr-path-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let program = dir.join(program_file_names()[0]); + std::fs::write(&program, b"").unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let _ = std::fs::set_permissions(&program, std::fs::Permissions::from_mode(0o755)); + } + let found = path_search_in([PathBuf::from("relative/dir"), PathBuf::from("definitely/missing"), dir.clone()].into_iter()); + assert_eq!(found.as_deref(), Some(program.as_path()), "the absolute entry with the program wins, relative entries are skipped"); + let none = path_search_in([PathBuf::from("relative/dir")].into_iter()); + assert_eq!(none, None, "only relative entries: nothing is found"); + let _ = std::fs::remove_dir_all(&dir); + } + + /// "HELLO" as blocky 5×7 glyphs, 8 pixels per dot with wide tracking, at roughly 300 dpi + /// scale (each dot ~10 px): big enough for Tesseract's word detector. + fn hello_world_image() -> (u32, u32, Vec) { + const GLYPHS: &[&str] = &[ + "X...X", "X...X", "X...X", "XXXXX", "X...X", "X...X", "X...X", // H + ]; + // Draw the five glyphs of HELLO with per-letter shapes for E, L, L, O. + const E: &[&str] = &["XXXXX", "X....", "X....", "XXXX.", "X....", "X....", "XXXXX"]; + const L: &[&str] = &["X....", "X....", "X....", "X....", "X....", "X....", "XXXXX"]; + const O: &[&str] = &[".XXX.", "X...X", "X...X", "X...X", "X...X", "X...X", ".XXX."]; + let letters: [&[&str]; 5] = [GLYPHS, E, L, L, O]; + let (dot, x0, y0, advance) = (8u32, 40u32, 40u32, 7u32 * 8); + let w = x0 * 2 + advance * 5; + let h = y0 * 2 + 7 * dot; + let mut rgba = vec![255u8; (w * h * 4) as usize]; + for (i, rows) in letters.iter().enumerate() { + for (ry, row) in rows.iter().enumerate() { + for (rx, b) in row.bytes().enumerate() { + if b != b'X' { + continue; + } + for dy in 0..dot { + for dx in 0..dot { + let x = x0 + i as u32 * advance + rx as u32 * dot + dx; + let y = y0 + ry as u32 * dot + dy; + let o = ((y * w + x) * 4) as usize; + rgba[o..o + 3].copy_from_slice(&[0, 0, 0]); + } + } + } + } + } + (w, h, rgba) + } +} + +#[cfg(all(test, unix))] +mod process_tests { + use super::*; + + /// A stderr flood is killed at the cap while the child still runs, not accepted until it + /// exits (an endless writer would otherwise run out the whole timeout and fill the disk). + /// The fixture needs a Unix shell, like the other process fixtures. + #[test] + fn a_stderr_flood_is_killed_while_the_child_runs() { + let dir = std::env::temp_dir().join(format!("pdfcraft-tesseract-flood-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let program = dir.join("flood-tesseract"); + std::fs::write(&program, "#!/bin/sh\nwhile true; do echo aaaaaaaaaaaaaaaaaaaaaaaa >&2; done\n").unwrap(); + use std::os::unix::fs::PermissionsExt; + let mut perms = std::fs::metadata(&program).unwrap().permissions(); + perms.set_mode(0o755); + std::fs::set_permissions(&program, perms).unwrap(); + let t = TesseractCli::new(program); + let started = std::time::Instant::now(); + let err = t.run_capture(&["--list-langs"], Duration::from_secs(60), 1024 * 1024).unwrap_err(); + assert!(matches!(err, OcrError::OutputTooLarge(_)), "{err}"); + assert!(started.elapsed() < Duration::from_secs(30), "killed at the cap, not at the timeout"); + let _ = std::fs::remove_dir_all(&dir); + } +} diff --git a/crates/redact/src/tests.rs b/crates/redact/src/tests.rs index d463303fcd..78cff889d3 100644 --- a/crates/redact/src/tests.rs +++ b/crates/redact/src/tests.rs @@ -3953,6 +3953,41 @@ fn a_reply_on_another_page_to_a_removed_popup_goes_too() { } } +/// An invisible OCR-style text layer drawn in a Type3 font (what the OCR output writes for +/// characters beyond WinAnsi) is redacted like any other text: its glyphs are hit-tested by +/// their real advances, removed, and the verifier passes. +#[test] +fn a_type3_text_layer_is_redacted_like_any_other() { + // One glyph (code 1): a filled box, 500 units wide, with a ToUnicode CMap that reads it + // back as U+3042. + let t3 = b"<< /Type /Font /Subtype /Type3 /Name /PCOcr /FontBBox [0 0 700 700] /FontMatrix [0.001 0 0 0.001 0 0] \ + /FirstChar 1 /LastChar 1 /Widths [500] /Encoding << /Type /Encoding /Differences [1 /g01] >> \ + /CharProcs << /g01 8 0 R >> /ToUnicode 9 0 R >>"; + let charproc = b"500 0 0 0 0 700 700 d1\n0 0 700 700 re f\n"; + let cmap = b"/CIDInit /ProcSet findresource begin\n12 dict begin\nbegincmap\n/CMapType 2 def\n1 begincodespacerange\n<01> <01>\nendcodespacerange\n1 beginbfchar\n<01> <3042>\nendbfchar\nendcmap\nCMapName currentdict /CMap defineresource pop\nend\nend\n"; + let layer = b"BT 3 Tr /T3 10 Tf 100 100 Td (\x01) Tj ET"; + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /Font << /T3 5 0 R >> >> >>".to_vec(), + stream("", layer), + t3.to_vec(), + FONT.replace("95 0 R", "6 0 R").into_bytes(), + widths(), + stream("", charproc), + stream("", cmap), + ]); + // The glyph box at size 10: 5 pt wide, from the font's descent to its ascent. + assert_eq!(under(&mut doc, 0, &[[98.0, 96.0, 112.0, 110.0]]), 1, "the Type3 glyph is found under the mark"); + assert_eq!(under(&mut doc, 0, &[[200.0, 200.0, 250.0, 250.0]]), 0, "elsewhere the page is clean"); + mark(&mut doc, 0, &[[98.0, 96.0, 112.0, 110.0]], ""); + let report = apply(&mut doc, None).unwrap(); + assert_eq!(report.glyphs, 1, "{report:?}"); + // The verified apply left no text-showing operator: the re-read page is empty of them. + assert!(!content(&doc, 0).contains("Tj"), "{}", content(&doc, 0)); + assert_eq!(under(&mut doc, 0, &[[98.0, 96.0, 112.0, 110.0]]), 0, "nothing is left under the mark"); +} + #[test] fn a_rewritten_form_xobject_drops_the_source_text_of_added_content() { // A form drawing invisible text whose dictionary repeats it in /PCAdded. diff --git a/crates/ui-egui/Cargo.toml b/crates/ui-egui/Cargo.toml index c859b93cf6..22e7323476 100644 --- a/crates/ui-egui/Cargo.toml +++ b/crates/ui-egui/Cargo.toml @@ -23,6 +23,7 @@ pdfcraft-fonts = { workspace = true } pdfcraft-engine = { workspace = true } pdfcraft-cos = { workspace = true } pdfcraft-content = { workspace = true } +pdfcraft-ocr = { workspace = true } pdfcraft-render = { workspace = true } pdfcraft-model = { workspace = true } pdfcraft-platform = { workspace = true } diff --git a/crates/ui-egui/examples/shot.rs b/crates/ui-egui/examples/shot.rs index 01e48e03d6..8058df40c8 100644 --- a/crates/ui-egui/examples/shot.rs +++ b/crates/ui-egui/examples/shot.rs @@ -6,18 +6,64 @@ //! `--width N` downscales the image to N pixels wide (README screenshots). Any other //! `--key value` pair is passed to `PdfCraftApp::set_option` //! (the same verbs as the desktop app's command-line flags and the future control channel). +//! `--ocr-verify on` ignores any file and instead builds a synthetic scanned page, recognizes +//! it with the real engines (models required: `cargo xtask models`) and opens the OCR Verify +//! screen on the result, as a reviewer sees it. use std::time::{Duration, Instant}; use egui_kittest::Harness; +use pdfcraft_engine::ocr::{MergeStrategy, OcrSettings}; +use pdfcraft_engine::{Session, export}; use pdfcraft_ui_egui::PdfCraftApp; +/// Build a one-page PDF that is only a picture of a sentence (as a scanner would produce), +/// recognize it with the engines named in `settings` and open the OCR Verify screen on the +/// result — the same path the review tests use, with a real recognition behind it. The toolbar +/// dropdowns are seeded to the ensemble the shot ran. +fn load_ocr_verify(app: &mut PdfCraftApp) -> Result<(), String> { + let mut s = Session::new(); + let text = s + .create_from_text( + "t", + "Scanned pages are pictures of words.\nThe Verify screen reads them back before anything is applied.\nReview the uncertain words, then Accept.", + ) + .map_err(|e| e.to_string())?; + let id = s.open("text.pdf", None, text, None).map_err(|e| e.to_string())?; + let png = export::Exporter::new(s.get(id).ok_or("the document is gone")?).png(0, 150.0).map_err(|e| e.to_string())?; + // Keep the text block of the rendered page, so the scan is a short note-sized picture: + // the whole page and the confidence legend fit in the shot's source-page pane. + let mut page = image::load_from_memory(&png).map_err(|e| e.to_string())?; + let mut scan_png = std::io::Cursor::new(Vec::new()); + page.crop(0, 0, page.width(), (page.height() as f32 * 0.34) as u32) + .write_to(&mut scan_png, image::ImageFormat::Png) + .map_err(|e| e.to_string())?; + let scan = s.create_from_images(&[("scan.png".into(), scan_png.into_inner())]).map_err(|e| e.to_string())?.to_vec(); + let settings = + OcrSettings { engine: pdfcraft_engine::ocr::EngineChoice::Auto, strategy: MergeStrategy::ConfidenceWeighted, ..OcrSettings::default() }; + app.ocr_draft.engine = settings.engine; + app.ocr_draft.strategy = settings.strategy; + app.open_bytes("scan.pdf", None, scan).map_err(|e| e.to_string())?; + app.run_inline = true; + if !app.execute("ocr.correct") { + return Err("ocr.correct did not run".into()); + } + let Some((_, id)) = app.active_ids() else { return Err("no open document".into()) }; + let job = app.session.ocr_job(id, &[], settings.clone()).ok_or("the OCR job could not be captured")?; + let recognizers = pdfcraft_engine::ocr::recognizers(&settings)?; + let guard = job.guard.clone(); + let pages = job.run(&recognizers, |_, _| true); + app.ocr_verify.load(id, guard, pages); + Ok(()) +} + fn main() -> Result<(), String> { let mut args = std::env::args().skip(1); let out = args.next().ok_or("usage: shot [file.pdf] [--option value …]")?; let (mut file, mut opts) = (None, Vec::new()); let (mut size, mut scale) = (egui::vec2(1440.0, 900.0), 2.0f32); let mut width: Option = None; + let mut ocr_verify = false; while let Some(a) = args.next() { match a.strip_prefix("--") { Some("size") => { @@ -27,13 +73,20 @@ fn main() -> Result<(), String> { } Some("scale") => scale = args.next().and_then(|v| v.parse().ok()).ok_or("bad --scale")?, Some("width") => width = Some(args.next().and_then(|v| v.parse().ok()).ok_or("bad --width")?), + Some("ocr-verify") => ocr_verify = args.next().is_some_and(|v| v == "on"), Some(k) => opts.push((k.to_string(), args.next().unwrap_or_default())), None => file = Some(a), } } let mut harness = Harness::builder().with_size(size).with_pixels_per_point(scale).build_eframe(move |_cc| { let mut app = PdfCraftApp::new(); - if let Some(f) = &file { + if ocr_verify { + if let Err(e) = load_ocr_verify(&mut app) { + eprintln!("shot: --ocr-verify: {e}"); + // an empty Verify screen must not pass for the documentation's screenshot + std::process::exit(1); + } + } else if let Some(f) = &file { let bytes = std::fs::read(f).unwrap_or_default(); let name = std::path::Path::new(f).file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(); if let Err(e) = app.open_bytes(&name, Some(f.clone()), bytes) { diff --git a/crates/ui-egui/src/app.rs b/crates/ui-egui/src/app.rs index 61b85cd7f1..3b3d5dd036 100644 --- a/crates/ui-egui/src/app.rs +++ b/crates/ui-egui/src/app.rs @@ -202,6 +202,7 @@ impl PdfCraftApp { ocr_draft: ocr_ui::OcrDraft::default(), ocr_run: None, ocr_batch: None, + ocr_verify: ocr_verify::OcrVerify::default(), run_inline: false, custom_actions: Vec::new(), wizard: Default::default(), @@ -624,6 +625,14 @@ impl eframe::App for PdfCraftApp { self.poll_optimize(); self.poll_scan_clean(); self.poll_action(); + // The OCR Verify screen's worker may have finished while no frame was drawn. + if self.ocr_verify.phase == workflows::ocr_verify::Phase::Running { + workflows::ocr_verify::with_state(self, workflows::ocr_verify::OcrVerify::poll); + } + // So may a page or region re-read's worker. + if self.ocr_verify.reread.is_some() { + workflows::ocr_verify::with_state(self, workflows::ocr_verify::OcrVerify::poll_reread); + } self.process_file_requests(); #[cfg(not(target_arch = "wasm32"))] self.process_picked(); @@ -680,6 +689,20 @@ impl eframe::App for PdfCraftApp { if self.draws_window_frame() { window_controls::resize_edges(&ctx); } + if self.ocr_verify.open { + // The OCR Verify screen owns the window (its four panes); dialogs and notices + // still show above it. + workflows::ocr_verify::show(self, ui); + dialogs::show(self, &ctx); + self.show_progress(&ctx); + widgets::toast(self, &ctx); + self.finish_render_frame(&ctx); + if let Some(mut control) = self.control.take() { + control.finish_frame(&ctx, self); + self.control = Some(control); + } + return; + } chrome::tab_strip(self, ui); chrome::mode_bar(self, ui); // Whichever way another tab came to the front, it shows instead of the expanded view. diff --git a/crates/ui-egui/src/app/prefs.rs b/crates/ui-egui/src/app/prefs.rs index 89789d8a4c..1a8c2d269e 100644 --- a/crates/ui-egui/src/app/prefs.rs +++ b/crates/ui-egui/src/app/prefs.rs @@ -144,6 +144,7 @@ impl PdfCraftApp { "combine_zoom": self.combine_zoom, "comments_panel_closed": self.comments_panel_closed, "right_panel_closed": self.right_panel_closed, + "ocr": self.ocr_draft.to_pref(), }); // Set apart: one more key inside the macro above passes serde_json's recursion limit. v["default_page_gaps"] = serde_json::Value::Bool(self.view_defaults.show_page_gaps); @@ -295,6 +296,9 @@ impl PdfCraftApp { if let Some(closed) = v["right_panel_closed"].as_bool() { self.right_panel_closed = closed; } + // Scan & OCR choices (language, engine, strategy, output mode, preprocessing and skip + // flags); unknown or malformed values keep the defaults. + self.ocr_draft.apply_pref(&v["ocr"]); if let Some(defaults) = v["default_zoom"].as_str().and_then(|zoom| self.view_defaults.with_zoom(zoom)) { self.view_defaults = defaults; } diff --git a/crates/ui-egui/src/commands.rs b/crates/ui-egui/src/commands.rs index 4601986e9a..7c0df87782 100644 --- a/crates/ui-egui/src/commands.rs +++ b/crates/ui-egui/src/commands.rs @@ -527,6 +527,7 @@ impl PdfCraftApp { "ocr.recognize" => self.dialog = Some(Dialog::RecognizeText), "ocr.enhance" => self.dialog = Some(Dialog::EnhanceScan), "ocr.recognize_batch" => self.ocr_files_dialog(), + "ocr.correct" => self.ocr_verify.open = true, "edit.edit_text" => { self.quick_tool = crate::QuickTool::EditText; self.left = crate::LeftPanel::Tool("edit"); diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 94e83f451a..2f393c61d1 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -69,8 +69,9 @@ use pdfcraft_engine::{DocId, Session}; pub use canvas::{DocView, RasterMemory}; pub use editing::{CloseRequest, SaveTarget}; pub use files::{ExtractDraft, FilePurpose, FileRequest, RotateDraft, SplitDraft, SplitMode, SplitPlan}; -pub use recovery::{AUTOSAVE_SECS, RecoveryMeta, RecoveryStore}; +pub use recovery::{AUTOSAVE_SECS, RecoveryMeta, RecoveryStore, default_dictionary_dir}; use theme::{ThemeKind, ThemePreference}; +pub use workflows::ocr_verify::{OcrVerify, Phase}; mod app; mod hooks; @@ -181,6 +182,7 @@ use workflows::export_ui; use workflows::files; use workflows::js_ui; use workflows::ocr_ui; +use workflows::ocr_verify; use workflows::optimize_ui; use workflows::print_ui; use workflows::scan_clean_ui; @@ -510,6 +512,8 @@ pub struct PdfCraftApp { pub ocr_draft: ocr_ui::OcrDraft, pub ocr_run: Option, pub ocr_batch: Option>>, + /// Scan & OCR ▸ Correct recognized text: the OCR Verify screen (its state, never persisted). + pub ocr_verify: ocr_verify::OcrVerify, /// Background jobs (OCR, actions, listing pinned folders) run inline instead (tests). pub run_inline: bool, /// Action Wizard: the user's actions, the dialog state, the running action and (tests) the diff --git a/crates/ui-egui/src/persist/recovery.rs b/crates/ui-egui/src/persist/recovery.rs index 71759c4599..f217781c07 100644 --- a/crates/ui-egui/src/persist/recovery.rs +++ b/crates/ui-egui/src/persist/recovery.rs @@ -184,7 +184,27 @@ impl RecoveryStore { pub fn dir(&self) -> &Path { &self.dir } +} +/// The OCR Verify screen's per-language user dictionary folder, next to the recovery store: +/// the portable data folder's `Dictionary` in portable mode ([`crate::portable`]), otherwise +/// `…/PdfCraft/Dictionary` in the platform's per-user data folder. `None` where files are not +/// kept (the web build). +pub fn default_dictionary_dir() -> Option { + if let Some(dir) = crate::portable::data_dir() { + return Some(dir.join("Dictionary")); + } + let env = |k: &str| std::env::var_os(k).filter(|v| !v.is_empty()).map(PathBuf::from); + if cfg!(target_os = "macos") { + env("HOME").map(|h| h.join("Library/Application Support/PdfCraft/Dictionary")) + } else if cfg!(windows) { + env("LOCALAPPDATA").map(|d| d.join("PdfCraft").join("Dictionary")) + } else { + env("XDG_DATA_HOME").or_else(|| env("HOME").map(|h| h.join(".local/share"))).map(|d| d.join("pdfcraft/dictionary")) + } +} + +impl RecoveryStore { /// Write (or replace) an entry. The PDF is written before its metadata, both atomically, so /// a listed entry always has complete bytes. pub fn write(&self, meta: &RecoveryMeta, bytes: &[u8]) -> std::io::Result<()> { diff --git a/crates/ui-egui/src/widgets.rs b/crates/ui-egui/src/widgets.rs index c4ca254948..f621c7832b 100644 --- a/crates/ui-egui/src/widgets.rs +++ b/crates/ui-egui/src/widgets.rs @@ -477,6 +477,9 @@ pub fn toast(app: &mut PdfCraftApp, ctx: &egui::Context) { // Above the progress notice while a job runs. let card = if app.progress_notice.is_some() { ctx.data(|d| d.get_temp::(egui::Id::unique(PROGRESS_HEIGHT))).unwrap_or(0.0) } else { 0.0 }; let lift = if card > 0.0 { 28.0 + card + 10.0 } else { 28.0 }; + // A running batch keeps its toast up (poll_ocr re-notifies), so it can carry the one + // control a long job needs: Cancel — the file in flight finishes, the rest are not run. + let batch_running = app.ocr_batch.is_some(); egui::Area::new(egui::Id::unique("toast")) .order(egui::Order::Tooltip) .pivot(Align2::CENTER_BOTTOM) @@ -494,7 +497,12 @@ pub fn toast(app: &mut PdfCraftApp, ctx: &egui::Context) { ui.set_max_width((screen.width() - 64.0).clamp(1.0, 560.0)); let ink = if t.dark() { Color32::from_rgb(0x22, 0x22, 0x26) } else { Color32::WHITE }; let text = |ui: &mut egui::Ui| { - ui.add(egui::Label::new(egui::RichText::new(msg).color(ink)).wrap()); + ui.horizontal_wrapped(|ui| { + ui.add(egui::Label::new(egui::RichText::new(msg).color(ink)).wrap()); + if batch_running && ui.small_button(crate::i18n::t("Cancel")).clicked() { + app.cancel_ocr_batch(); + } + }); }; match script { // A form script's message: its icon, then its title (when it gave one) diff --git a/crates/ui-egui/src/workflows.rs b/crates/ui-egui/src/workflows.rs index 9314914ec4..8701eac119 100644 --- a/crates/ui-egui/src/workflows.rs +++ b/crates/ui-egui/src/workflows.rs @@ -9,6 +9,7 @@ pub(super) mod export_ui; pub(super) mod files; pub(super) mod js_ui; pub(super) mod ocr_ui; +pub(super) mod ocr_verify; pub(super) mod optimize_ui; pub(super) mod print_ui; pub(super) mod scan_clean_ui; diff --git a/crates/ui-egui/src/workflows/ocr_ui.rs b/crates/ui-egui/src/workflows/ocr_ui.rs index 04e257d52d..7534915d10 100644 --- a/crates/ui-egui/src/workflows/ocr_ui.rs +++ b/crates/ui-egui/src/workflows/ocr_ui.rs @@ -6,7 +6,7 @@ use std::sync::{Arc, Mutex}; use egui::{Align, Layout}; use pdfcraft_engine::DocId; -use pdfcraft_engine::ocr::{LANGUAGES, OcrPage, OcrSettings}; +use pdfcraft_engine::ocr::{EngineChoice, LANGUAGES, MergeStrategy, OcrPage, OcrSettings, OutputMode}; use crate::theme::{self, Tokens}; use crate::{PdfCraftApp, widgets}; @@ -19,7 +19,10 @@ pub enum OcrPages { Range, } -/// The dialog's choices (kept between runs, as in Acrobat). +/// The dialog's choices (kept between runs, as in Acrobat). The Recognize Text dialog, the OCR +/// Verify screen and the batch options all read and write this one draft, and it is what the +/// preferences persist: language, output mode, the four preprocessing flags, engine, strategy +/// and the three skip flags — never review edits, verified flags or the selection. #[derive(Clone, Debug, PartialEq)] pub struct OcrDraft { pub pages: OcrPages, @@ -30,18 +33,135 @@ pub struct OcrDraft { pub dpi: u32, /// Also recognize pages that already have text (pictures on them; their text isn't added again). pub text_pages: bool, + /// Which engine reads the pages. + pub engine: EngineChoice, + /// How two engines' readings combine. + pub strategy: MergeStrategy, + /// Preprocessing flags, all off by default. + pub auto_rotate: bool, + pub deskew: bool, + pub denoise: bool, + pub binarize: bool, + /// What Accept produces. + pub output_mode: OutputMode, + /// Batch: leave files that already contain text alone. + pub skip_text_files: bool, + /// Batch: leave pages that already contain text alone. + pub skip_text_pages: bool, + /// Batch: read everything, overriding both skips. + pub force_ocr: bool, + /// Batch: where the searchable copies go (None: ask each time). Remembered from the last + /// batch's folder pick, so a repeated export into the same place does not re-ask. + pub output_folder: Option, } impl Default for OcrDraft { fn default() -> Self { - OcrDraft { pages: OcrPages::All, from: 1, to: 1, language: "en".into(), dpi: 300, text_pages: false } + OcrDraft { + pages: OcrPages::All, + from: 1, + to: 1, + language: "en".into(), + dpi: 300, + text_pages: false, + engine: EngineChoice::default(), + strategy: MergeStrategy::default(), + auto_rotate: false, + deskew: false, + denoise: false, + binarize: false, + output_mode: OutputMode::default(), + skip_text_files: false, + skip_text_pages: true, + force_ocr: false, + output_folder: None, + } } } impl OcrDraft { /// The recognition settings these choices make. pub fn settings(&self) -> OcrSettings { - OcrSettings { dpi: self.dpi as f32, language: self.language.clone(), skip_text_pages: !self.text_pages, ..Default::default() } + OcrSettings { + dpi: self.dpi as f32, + language: self.language.clone(), + engine: self.engine, + strategy: self.strategy, + auto_rotate: self.auto_rotate, + deskew: self.deskew, + denoise: self.denoise, + binarize: self.binarize, + output_mode: self.output_mode, + skip_text_files: self.skip_text_files, + skip_text_pages: self.skip_text_pages && !self.text_pages, + force_ocr: self.force_ocr, + ..Default::default() + } + } + + /// Restore from the preferences JSON (`ocr` object); unknown or malformed values keep the + /// defaults, like every other restored setting. + pub fn apply_pref(&mut self, v: &serde_json::Value) { + if let Some(l) = v["language"].as_str().filter(|l| LANGUAGES.iter().any(|x| x.0 == *l)) { + self.language = l.to_string(); + } + if let Some(e) = v["engine"].as_str() { + self.engine = EngineChoice::parse(e); + } + if let Some(s) = v["strategy"].as_str() { + self.strategy = match s.trim().to_ascii_lowercase().as_str() { + "confidence" | "confidence-weighted" | "confidence_weighted" => MergeStrategy::ConfidenceWeighted, + "rover" | "rover-vote" | "vote" => MergeStrategy::RoverVote, + "primary" | "primary-only" => MergeStrategy::PrimaryOnly, + _ => MergeStrategy::PrimaryOnly, + }; + } else if self.engine == EngineChoice::Auto { + // No strategy was saved with "Automatic": it reads the way Automatic reads by + // default, not as "Primary only" (which would never bring in the second engine). + self.strategy = self.engine.default_strategy(); + } + if let Some(m) = v["output_mode"].as_str() { + self.output_mode = OutputMode::parse(m); + } + for (key, flag) in + [("auto_rotate", &mut self.auto_rotate), ("deskew", &mut self.deskew), ("denoise", &mut self.denoise), ("binarize", &mut self.binarize)] + { + if let Some(on) = v[key].as_bool() { + *flag = on; + } + } + for (key, flag) in + [("skip_text_files", &mut self.skip_text_files), ("skip_text_pages", &mut self.skip_text_pages), ("force_ocr", &mut self.force_ocr)] + { + if let Some(on) = v[key].as_bool() { + *flag = on; + } + } + if let Some(f) = v["output_folder"].as_str().filter(|f| !f.is_empty()) { + self.output_folder = Some(f.to_string()); + } + } + + /// The preferences JSON for `persist`. + pub fn to_pref(&self) -> serde_json::Value { + serde_json::json!({ + "language": self.language, + "engine": self.engine.name(), + "strategy": match self.strategy { + MergeStrategy::PrimaryOnly => "primary", + MergeStrategy::ConfidenceWeighted => "confidence", + MergeStrategy::RoverVote => "rover", + }, + "output_mode": self.output_mode.name(), + "auto_rotate": self.auto_rotate, + "deskew": self.deskew, + "denoise": self.denoise, + "binarize": self.binarize, + "skip_text_files": self.skip_text_files, + "skip_text_pages": self.skip_text_pages, + "force_ocr": self.force_ocr, + "output_folder": self.output_folder, + }) } } @@ -54,12 +174,14 @@ pub struct OcrProgress { pub cancel: bool, } -/// Recognize text in multiple files: files done, total, and the summary once finished. +/// Recognize text in multiple files: files done, total, and the summary once finished. Cancel +/// is asked for between files: the file in flight finishes, the rest are counted as not run. #[derive(Default)] pub struct BatchProgress { pub done: usize, pub total: usize, pub message: Option, + pub cancel: bool, } /// A page Edit should read now (see [`PdfCraftApp::scan_to_read`]). @@ -74,10 +196,16 @@ struct ScanToRead { pub struct OcrRun { pub doc: DocId, pub progress: Arc>, + /// What the document looked like when the job started: a result is applied only while it + /// still does (the dialog closes on Go, so the user can edit, or redact, while this runs). + pub guard: pdfcraft_engine::ocr::OcrGuard, /// Set for a run Edit started on a scanned page, not asked for: it shows no progress, says nothing /// when it found no words (a page with text of its own is skipped, and Edit must not make noise), /// and its result waits for Read mode and dialogs and is dropped if the document changed. pub automatic: Option, + /// The dialog promised an editable document: the result goes to the OCR Verify screen + /// (whose Accept produces the editable document), not to an invisible layer. + pub editable: bool, } /// What an Edit-started run needs to know to refuse a result that no longer fits its document. @@ -90,9 +218,16 @@ pub struct AutoOcr { pub generation: u64, } +/// The Tesseract probe runs in the background (the interface never waits on a process): while +/// it runs, an engine choice reads as "still checking", so look again soon. +pub(crate) fn keep_asking_while_probing(ctx: &egui::Context) { + if pdfcraft_engine::ocr::engine_probe_running() { + ctx.request_repaint_after(std::time::Duration::from_millis(200)); + } +} + pub(crate) fn body(ui: &mut egui::Ui, app: &mut PdfCraftApp, t: &Tokens) -> (bool, bool) { let pages = app.active_ids().and_then(|(_, id)| app.session.get(id)).map_or(1, |d| d.info.pages.len().max(1)); - let available = pdfcraft_engine::ocr::available(); let d = &mut app.ocr_draft; d.to = d.to.clamp(1, pages); d.from = d.from.clamp(1, d.to); @@ -127,11 +262,55 @@ pub(crate) fn body(ui: &mut egui::Ui, app: &mut PdfCraftApp, t: &Tokens) -> (boo } }); ui.end_row(); + ui.label(tl!("Engine")); + let engine_name = match d.engine { + EngineChoice::Auto => tl!("Automatic"), + EngineChoice::Ocrs => tl!("Built-in"), + EngineChoice::Tesseract => tl!("Tesseract"), + }; + craft_ui::ComboBox::from_id_salt("ocr-engine").selected_text(engine_name).width(220.0).show_ui(ui, |ui| { + let choices: [(EngineChoice, &str); 3] = + [(EngineChoice::Auto, "Automatic"), (EngineChoice::Ocrs, "Built-in"), (EngineChoice::Tesseract, "Tesseract")]; + for (choice, name) in choices { + ui.selectable_value(&mut d.engine, choice, tl!(name)); + } + }); + ui.end_row(); + ui.label(tl!("Combine engines")); + let strategy_name = match d.strategy { + MergeStrategy::PrimaryOnly => tl!("Primary only"), + MergeStrategy::ConfidenceWeighted => tl!("Confidence weighted"), + MergeStrategy::RoverVote => tl!("ROVER vote"), + }; + craft_ui::ComboBox::from_id_salt("ocr-strategy").selected_text(strategy_name).width(220.0).show_ui(ui, |ui| { + let choices: [(MergeStrategy, &str); 3] = [ + (MergeStrategy::PrimaryOnly, "Primary only"), + (MergeStrategy::ConfidenceWeighted, "Confidence weighted"), + (MergeStrategy::RoverVote, "ROVER vote"), + ]; + for (choice, name) in choices { + ui.selectable_value(&mut d.strategy, choice, tl!(name)); + } + }); + ui.end_row(); ui.label(tl!("Output")); - craft_ui::ComboBox::from_id_salt("ocr-output").selected_text(tl!("Searchable Image (Exact)")).width(220.0).show_ui(ui, |ui| { - let _ = ui - .selectable_label(true, tl!("Searchable Image (Exact)")) + let mode_name = match d.output_mode { + OutputMode::Searchable => tl!("Searchable Image (Exact)"), + OutputMode::EditableText => tl!("Editable text"), + }; + craft_ui::ComboBox::from_id_salt("ocr-output").selected_text(mode_name).width(220.0).show_ui(ui, |ui| { + let s = ui + .selectable_label(d.output_mode == OutputMode::Searchable, tl!("Searchable Image (Exact)")) .on_hover_text(tl!("Adds invisible text over each word; the page image is not changed")); + if s.clicked() { + d.output_mode = OutputMode::Searchable; + } + let e = ui + .selectable_label(d.output_mode == OutputMode::EditableText, tl!("Editable text")) + .on_hover_text(tl!("Writes a new document of visible text; the scan is not changed")); + if e.clicked() { + d.output_mode = OutputMode::EditableText; + } }); ui.end_row(); ui.label(tl!("Downsample to")); @@ -145,20 +324,50 @@ pub(crate) fn body(ui: &mut egui::Ui, app: &mut PdfCraftApp, t: &Tokens) -> (boo ui.add_space(4.0); ui.add(crate::widgets::checkbox(&mut d.text_pages, tl!("Also recognize pages that already have text"))) .on_hover_text(tl!("Pictures on those pages are recognized too; words already in the page's text are not added again")); + ui.add(crate::widgets::checkbox(&mut d.auto_rotate, tl!("Straighten pages scanned sideways or upside down"))); + ui.add(crate::widgets::checkbox(&mut d.deskew, tl!("Straighten skewed text lines"))); + ui.add(crate::widgets::checkbox(&mut d.denoise, tl!("Remove scan speckle"))); + ui.add(crate::widgets::checkbox(&mut d.binarize, tl!("Black and white (binarize)"))); }); - if !available { - ui.label( - egui::RichText::new(tl!("Text recognition isn't installed: its model files are missing. Reinstall PdfCraft, or set PDFCRAFT_MODELS to the folder that holds them.")) - .small() - .color(t.text_muted), - ); + group(ui, "Skip", &mut |ui| { + ui.add(crate::widgets::checkbox(&mut d.skip_text_pages, tl!("Skip pages that already have text"))) + .on_hover_text(tl!("Pages with renderable text are left alone and reported as skipped")); + ui.add(crate::widgets::checkbox(&mut d.skip_text_files, tl!("Skip files that already have text"))) + .on_hover_text(tl!("In multiple files: a file with any text is its own skipped bucket, and nothing is written for it")); + ui.add(crate::widgets::checkbox(&mut d.force_ocr, tl!("Force OCR (override skip options)"))) + .on_hover_text(tl!("Reads pages and files even where the skip options would leave them alone")); + }); + group(ui, "Output folder", &mut |ui| { + ui.horizontal(|ui| match &d.output_folder { + Some(f) => { + ui.monospace(f).on_hover_text(tl!("The searchable copies are written here under the source names")); + if ui.small_button(tl!("Clear")).clicked() { + d.output_folder = None; + } + } + None => { + ui.label(tl!("Automatic")).on_hover_text(tl!("A folder is asked for when a multiple-file run starts")); + } + }); + }); + keep_asking_while_probing(ui.ctx()); + if let Some(why) = pdfcraft_engine::ocr::engine_unavailable_reason(d.engine) { + // Said in the interface's language: the built-in engine's missing models get the + // notice users know; Tesseract's reasons are catalog keys of their own. + let text = match d.engine { + pdfcraft_engine::ocr::EngineChoice::Tesseract => crate::i18n::t(&why).to_string(), + _ => tl!("Text recognition isn't installed: its model files are missing. Reinstall PdfCraft, or set PDFCRAFT_MODELS to the folder that holds them.") + .to_string(), + }; + ui.label(egui::RichText::new(text).small().color(t.text_muted)); ui.add_space(6.0); } ui.add_space(6.0); let (mut go, mut cancel) = (false, false); + let can_run = pdfcraft_engine::ocr::engine_unavailable_reason(d.engine).is_none(); ui.horizontal(|ui| { ui.with_layout(Layout::right_to_left(Align::Center), |ui| { - if ui.add_enabled_ui(available, |ui| widgets::pill_button(ui, tl!("Recognize text"), true)).inner.clicked() { + if ui.add_enabled_ui(can_run, |ui| widgets::pill_button(ui, tl!("Recognize text"), true)).inner.clicked() { go = true; } if widgets::pill_button(ui, tl!("Cancel"), false).clicked() { @@ -201,7 +410,9 @@ impl PdfCraftApp { /// is one undoable step, "Recognize text", and is not run again for that page after Undo. pub(crate) fn auto_ocr_scanned_page(&mut self) { let Some(scan) = self.scan_to_read() else { return }; - if !pdfcraft_engine::ocr::available() { + // the engine this run will use, not always the built-in one + let mut settings = self.ocr_draft.settings(); + if pdfcraft_engine::ocr::engine_unavailable_reason(settings.engine).is_some() { self.views[scan.view].auto_ocr_skipped.insert(scan.page, scan.generation); return; } @@ -209,7 +420,6 @@ impl PdfCraftApp { // Only a page with no text of its own is read. Comments and text boxes over the picture do // not count as its text (a scan with a few comments is still a scan, as in Acrobat), but // a page with real text and a background picture is left alone. - let mut settings = self.ocr_draft.settings(); settings.skip_text_pages = true; settings.native_text_only = true; self.start_ocr_pages(scan.doc, &[scan.page], settings, true); @@ -267,7 +477,9 @@ impl PdfCraftApp { } else { None }; - let Some(job) = self.session.ocr_job(id, pages, settings) else { return }; + let Some(job) = self.session.ocr_job(id, pages, settings.clone()) else { return }; + let guard = job.guard.clone(); + let editable = settings.output_mode == OutputMode::EditableText; let progress = Arc::new(Mutex::new(OcrProgress { total: job.pages.len(), ..Default::default() })); let p = progress.clone(); // A failure is written on the worker thread: in the UI's language. @@ -275,8 +487,8 @@ impl PdfCraftApp { let work = move || { crate::i18n::set_current(lang); let result = recognize_guarded(|| { - pdfcraft_engine::ocr::engine().map(|ocr| { - job.run(&ocr, |done, total| { + pdfcraft_engine::ocr::recognizers(&settings).map(|r| { + job.run(&r, |done, total| { let Ok(mut s) = p.lock() else { return false }; s.done = done; s.total = total; @@ -296,7 +508,7 @@ impl PdfCraftApp { } #[cfg(target_arch = "wasm32")] work(); - self.ocr_run = Some(OcrRun { doc: id, progress, automatic: auto }); + self.ocr_run = Some(OcrRun { doc: id, progress, guard, automatic: auto, editable }); self.poll_ocr(); } @@ -307,10 +519,11 @@ impl PdfCraftApp { self.notify_tr("Text recognition is already running"); return; } - // The settings showing now, not when the folder arrives. + // The settings showing now, not when the folder arrives. A remembered folder (the last + // batch's pick, persisted with the preferences) is reused; otherwise the user is asked. let settings = self.ocr_draft.settings(); #[cfg(not(target_arch = "wasm32"))] - match self.export_dir_override.clone() { + match self.export_dir_override.clone().or_else(|| self.ocr_draft.output_folder.clone()) { Some(d) => self.ocr_files_into(files, settings, d.into()), None => { let dialog = rfd::AsyncFileDialog::new().set_title(tl!("Choose a folder for the searchable files").to_string()); @@ -328,21 +541,66 @@ impl PdfCraftApp { self.notify_tr("Text recognition is already running"); return; } + // The folder is remembered (the next batch reuses it; the dialog's Clear un-remembers). + #[cfg(not(target_arch = "wasm32"))] + { + self.ocr_draft.output_folder = Some(dir.to_string_lossy().into_owned()); + } + // Two inputs of one file name (from different folders) would write the same output + // (or download under the same name): refused before anything is read. + let twice = pdfcraft_engine::ocr::duplicate_output_names(files.iter().map(|(name, _)| name.as_str())); + if !twice.is_empty() { + self.notify_error(crate::i18n::fmt( + tl!("Several files have the same name, so their results would overwrite each other; nothing was written: {list}"), + &[("list", &twice.join(", "))], + )); + return; + } + // Collision pre-flight: a searchable copy never silently overwrites. Every target is + // checked before the first page is read; a collision stops the batch here. + #[cfg(not(target_arch = "wasm32"))] + { + let hits: Vec = files + .iter() + .filter_map(|(name, _)| { + let target = dir.join(name); + target.exists().then(|| target.to_string_lossy().into_owned()) + }) + .take(5) + .collect(); + if !hits.is_empty() { + self.notify_error(crate::i18n::fmt( + tl!("The output folder already holds files of these names; nothing was written: {list}"), + &[("list", &hits.join(", "))], + )); + return; + } + } + // The engines are built (and named if they cannot be) before anything starts. + let recognizers = match pdfcraft_engine::ocr::recognizers(&settings) { + Ok(r) => r, + Err(why) => { + self.notify_error(why); + return; + } + }; let progress = Arc::new(Mutex::new(BatchProgress { total: files.len(), ..Default::default() })); let p = progress.clone(); // The summary is written on the worker thread: draw it in the UI's language. let lang = crate::i18n::current(); let work = move || { crate::i18n::set_current(lang); - // A panic in the recognizer still ends the batch with a summary (#3919). #[cfg(not(target_arch = "wasm32"))] - let summary = pdfcraft_engine::guard(|| recognize_files(files, &settings, &p, &dir)); + let write = move |name: &str, r: &pdfcraft_engine::ocr::FileResult| { + crate::editing::write_atomically(&dir.join(name).to_string_lossy(), &r.bytes).map_err(|e| e.to_string()) + }; #[cfg(target_arch = "wasm32")] - let summary = pdfcraft_engine::guard(|| recognize_files(files, &settings, &p)); - let summary = summary.unwrap_or_else(|panic| failed(&panic)); + let write = move |name: &str, r: &pdfcraft_engine::ocr::FileResult| crate::editing::download(name, &r.bytes); + // A panic in the recognizer still ends the batch with a summary (#3919). + let msg = pdfcraft_engine::guard(|| run_batch_files(files, settings, &recognizers, &p, write)).unwrap_or_else(|panic| failed(&panic)); if let Ok(mut s) = p.lock() { s.done = s.total; - s.message = Some(summary); + s.message = Some(msg); } }; #[cfg(not(target_arch = "wasm32"))] @@ -356,54 +614,17 @@ impl PdfCraftApp { self.ocr_batch = Some(progress); self.poll_ocr(); } -} -/// Recognize each file and write its searchable copy into `dir`; returns the summary to show. -fn recognize_files( - files: Vec<(String, Vec)>, - settings: &OcrSettings, - p: &Mutex, - #[cfg(not(target_arch = "wasm32"))] dir: &std::path::Path, -) -> String { - let (mut ok, mut words, mut failed) = (0, 0, Vec::new()); - match pdfcraft_engine::ocr::engine() { - Err(e) => failed.push(e), - Ok(ocr) => { - for (i, (name, bytes)) in files.into_iter().enumerate() { - if let Ok(mut s) = p.lock() { - s.done = i; - } - let r = pdfcraft_engine::ocr::recognize_file(&name, Arc::new(bytes), None, settings.clone(), &ocr, |_, _| true); - let saved = r.and_then(|r| { - #[cfg(not(target_arch = "wasm32"))] - crate::editing::write_atomically(&dir.join(&name).to_string_lossy(), &r.bytes).map_err(|e| e.to_string())?; - #[cfg(target_arch = "wasm32")] - crate::editing::download(&name, &r.bytes)?; - Ok(r.words()) - }); - match saved { - Ok(n) => { - ok += 1; - words += n; - } - Err(e) => failed.push(format!("{name}: {}", e)), - } - } + /// Ask the running batch to stop: the file in flight finishes, the rest are counted as not + /// run. + pub fn cancel_ocr_batch(&mut self) { + if let Some(b) = &self.ocr_batch + && let Ok(mut s) = b.lock() + { + s.cancel = true; } } - let done = if ok == 1 { - crate::i18n::fmt(tl!("Recognized {w} words in 1 file"), &[("w", &words.to_string())]) - } else { - crate::i18n::fmt(tl!("Recognized {w} words in {n} files"), &[("w", &words.to_string()), ("n", &ok.to_string())]) - }; - let mut msg = done; - if !failed.is_empty() { - msg.push_str(&crate::i18n::fmt(tl!("; failed: {list}"), &[("list", &failed.join("; "))])); - } - msg -} -impl PdfCraftApp { /// Stop a running recognition (the pages read so far are kept). pub fn cancel_ocr(&mut self) { if let Some(r) = &self.ocr_run @@ -442,7 +663,7 @@ impl PdfCraftApp { } } let Some(run) = self.ocr_run.as_ref() else { return }; - let (doc, progress, auto) = (run.doc, run.progress.clone(), run.automatic); + let (doc, progress, auto, guard, editable) = (run.doc, run.progress.clone(), run.automatic, run.guard.clone(), run.editable); let automatic = auto.is_some(); let mut s = match progress.lock() { Ok(s) => s, @@ -498,9 +719,20 @@ impl PdfCraftApp { return; } }; + // The dialog's "Editable text" choice is honored: the recognition is not written as an + // invisible layer here. It opens in the OCR Verify screen, whose Accept produces the + // editable document (a NEW file) the choice promises. + if editable && !automatic { + self.ocr_verify.load(doc, guard, found); + self.ocr_verify.open = true; + self.notify_tr("Review the recognition, then Accept to write the editable document"); + return; + } let read = found.iter().filter(|p| p.skipped.is_none()).count(); let skipped = found.len() - read; - match self.session.apply_ocr(doc, &found) { + // The guard refuses a result that no longer fits its document (the user edited, or + // redacted, while the worker was out); its reason says so. + match self.session.apply_ocr_checked(doc, &found, &guard) { Ok(words) => { if let Some(info) = self.session.get(doc).map(|d| d.info.clone()) && let Some(view) = self.views.iter_mut().find(|v| v.id == doc) @@ -525,6 +757,12 @@ impl PdfCraftApp { crate::i18n::fmt(tl!("; {n} pages already had text"), &[("n", &skipped.to_string())]) }); } + // The pages' own notes (the editable output's unrenderable-character warning, + // for one) are part of the result: said here, not dropped. + for note in found.iter().flat_map(|p| p.notes.iter()) { + msg.push_str("; "); + msg.push_str(note); + } self.notify(msg); } Err(e) => self.notify_error(e), @@ -532,6 +770,77 @@ impl PdfCraftApp { } } +/// The batch's worker: read each file, hand the searchable copy to `write` (the output folder, +/// or the web's download), count the buckets, and compose the summary. A file the skip-files +/// option passes over is its own bucket and `write` is never called for it. A cancel is asked +/// for between files: the file in flight finishes, the rest are counted as not run. +pub(crate) fn run_batch_files( + files: Vec<(String, Vec)>, + settings: OcrSettings, + recognizers: &pdfcraft_engine::ocr::Recognizers, + progress: &Mutex, + mut write: impl FnMut(&str, &pdfcraft_engine::ocr::FileResult) -> Result<(), String>, +) -> String { + let total = files.len(); + let (mut ok, mut skipped, mut words, mut failed, mut notes) = (0usize, 0usize, 0usize, Vec::new(), Vec::new()); + let mut not_run: Option = None; + for (i, (name, bytes)) in files.into_iter().enumerate() { + if progress.lock().map(|s| s.cancel).unwrap_or(false) { + not_run = Some(total - i); + break; + } + if let Ok(mut s) = progress.lock() { + s.done = i; + } + let r = pdfcraft_engine::ocr::recognize_file(&name, Arc::new(bytes), None, settings.clone(), recognizers, |_, _| true); + match r { + Ok(res) if res.skipped() => skipped += 1, + Ok(res) => match write(&name, &res) { + Ok(()) => { + ok += 1; + words += res.words(); + // The batch's low-confidence note: at or under the settings' threshold (60 by + // default), on 1-based sorted pages. + if let Some((count, pages)) = pdfcraft_engine::ocr::low_confidence_pages(&res.pages, settings.low_confidence_threshold) { + let list = pages.iter().map(|p| p.to_string()).collect::>().join(", "); + notes.push(crate::i18n::fmt( + tl!("{name}: {n} low-confidence word(s) on pages {pages}"), + &[("name", &name), ("n", &count.to_string()), ("pages", &list)], + )); + } + } + Err(e) => failed.push(format!("{name}: {e}")), + }, + Err(e) => failed.push(format!("{name}: {e}")), + } + } + batch_summary(ok, words, skipped, &failed, ¬es, not_run) +} + +/// The batch summary: what was read, the skipped files as their own bucket, the failures, the +/// low-confidence notes and (on a cancel) how many files were not run. +fn batch_summary(ok: usize, words: usize, skipped: usize, failed: &[String], notes: &[String], not_run: Option) -> String { + let mut msg = if ok == 1 { + crate::i18n::fmt(tl!("Recognized {w} words in 1 file"), &[("w", &words.to_string())]) + } else { + crate::i18n::fmt(tl!("Recognized {w} words in {n} files"), &[("w", &words.to_string()), ("n", &ok.to_string())]) + }; + if skipped > 0 { + msg.push_str(&crate::i18n::fmt(tl!("; {n} files already had text"), &[("n", &skipped.to_string())])); + } + if !failed.is_empty() { + msg.push_str(&crate::i18n::fmt(tl!("; failed: {list}"), &[("list", &failed.join("; "))])); + } + if let Some(rest) = not_run { + msg.push_str(&crate::i18n::fmt(tl!("; cancelled — {n} files were not run"), &[("n", &rest.to_string())])); + } + for note in notes { + msg.push_str("; "); + msg.push_str(note); + } + msg +} + /// "Recognize text failed: {e}", in the current language. fn failed(e: &str) -> String { crate::i18n::fmt(tl!("{label} failed: {e}"), &[("label", tl!("Recognize text")), ("e", e)]) @@ -547,6 +856,7 @@ fn recognize_guarded(recognize: impl FnOnce() -> Result) -> Result mod tests { use super::*; use crate::{Dialog, LeftPanel, Mode, QuickTool}; + use pdfcraft_engine::ocr::{Recognizer, Recognizers}; use pdfcraft_engine::{Edit, Session}; /// A one-page PDF that is one picture covering the page. @@ -629,12 +939,26 @@ mod tests { /// asked for is applied at once. The result is made up, so no OCR models are needed. #[test] fn the_result_of_an_automatic_run_waits_for_read_mode_and_dialogs() { - let word = || pdfcraft_engine::ocr::PlacedWord { text: "scanned".into(), origin: [72.0, 700.0], across: [60.0, 0.0], up: [0.0, 12.0] }; + let word = || pdfcraft_engine::ocr::PlacedWord { + text: "scanned".into(), + origin: [72.0, 700.0], + across: [60.0, 0.0], + up: [0.0, 12.0], + confidence: None, + source: "ocrs".into(), + }; let finished = |app: &PdfCraftApp, automatic: bool| { let generation = app.session.get(app.active_ids().unwrap().1).unwrap().edit_generation(); - let page = pdfcraft_engine::ocr::OcrPage { page: 0, words: vec![word()], already_on_page: 0, skipped: None }; + let page = pdfcraft_engine::ocr::OcrPage { page: 0, words: vec![word()], already_on_page: 0, skipped: None, notes: Vec::new() }; let progress = Arc::new(Mutex::new(OcrProgress { done: 1, total: 1, result: Some(Ok(vec![page])), cancel: false })); - OcrRun { doc: app.active_ids().unwrap().1, progress, automatic: automatic.then_some(AutoOcr { page: 0, generation }) } + let guard = app.session.ocr_guard(app.active_ids().unwrap().1).unwrap(); + OcrRun { + doc: app.active_ids().unwrap().1, + progress, + guard, + automatic: automatic.then_some(AutoOcr { page: 0, generation }), + editable: false, + } }; let label = |app: &PdfCraftApp| app.session.get(app.active_ids().unwrap().1).unwrap().can_undo().map(String::from); @@ -669,12 +993,21 @@ mod tests { let generation = app.session.get(id).unwrap().edit_generation(); let page = pdfcraft_engine::ocr::OcrPage { page: 0, - words: vec![pdfcraft_engine::ocr::PlacedWord { text: "scanned".into(), origin: [72.0, 700.0], across: [60.0, 0.0], up: [0.0, 12.0] }], + words: vec![pdfcraft_engine::ocr::PlacedWord { + text: "scanned".into(), + origin: [72.0, 700.0], + across: [60.0, 0.0], + up: [0.0, 12.0], + confidence: None, + source: "ocrs".into(), + }], already_on_page: 0, skipped: None, + notes: Vec::new(), }; let progress = Arc::new(Mutex::new(OcrProgress { done: 1, total: 1, result: Some(Ok(vec![page])), cancel: false })); - app.ocr_run = Some(OcrRun { doc: id, progress, automatic: Some(AutoOcr { page: 0, generation }) }); + let guard = app.session.ocr_guard(id).unwrap(); + app.ocr_run = Some(OcrRun { doc: id, progress, guard, automatic: Some(AutoOcr { page: 0, generation }), editable: false }); app.views[0].auto_ocr_tried.insert(0); // The document changes while the run is out. let mut png = std::io::Cursor::new(Vec::new()); @@ -712,9 +1045,13 @@ mod tests { fn a_run_whose_worker_died_ends_instead_of_waiting_forever() { let mut app = PdfCraftApp::new(); app.set_option("language", "en").unwrap(); + let pdf = pdfcraft_engine::Session::new().create_from_text("t", "guard fixture").unwrap(); + app.open_bytes("a.pdf", None, pdf.to_vec()).unwrap(); + let id = app.active_ids().unwrap().1; + let guard = app.session.ocr_guard(id).unwrap(); let progress = Arc::new(Mutex::new(OcrProgress { total: 3, ..Default::default() })); poison(&progress); - app.ocr_run = Some(OcrRun { doc: DocId(1), progress, automatic: None }); + app.ocr_run = Some(OcrRun { doc: id, progress, guard, automatic: None, editable: false }); app.poll_ocr(); assert!(app.ocr_run.is_none(), "the next run isn't refused as already running"); let toast = app.toast.take().map(|t| t.0).unwrap_or_default(); @@ -728,4 +1065,267 @@ mod tests { let toast = app.toast.take().map(|t| t.0).unwrap_or_default(); assert!(toast.starts_with("Recognize text failed"), "{toast}"); } + + /// A fake engine: one confident word per call, no filesystem, no models needed. + struct Fake { + confidence: f32, + } + + impl Recognizer for Fake { + fn id(&self) -> &'static str { + "fake" + } + fn available(&self) -> bool { + true + } + fn languages(&self) -> Vec<(String, String)> { + vec![("en".into(), "English".into())] + } + fn recognize( + &self, + _image: &pdfcraft_ocr::OcrImage, + _options: &pdfcraft_ocr::RecognizeOptions, + ) -> Result, pdfcraft_engine::ocr::OcrError> { + Ok(vec![pdfcraft_ocr::Line { + words: vec![pdfcraft_ocr::Word::new("word", [1.0, 1.0, 9.0, 9.0], "fake").with_confidence(self.confidence)], + }]) + } + } + + /// A batch over two text files (the engine never runs on them: they are skipped files): + /// the summary counts them in their own bucket, nothing is written, and a cancel before + /// the first file leaves the rest not run. + #[test] + fn batch_counts_skipped_files_and_honours_a_cancel() { + let s = pdfcraft_engine::Session::new(); + let pdf = s.create_from_text("t", "Already typed here").expect("fixture"); + let files = vec![("a.pdf".into(), pdf.to_vec()), ("b.pdf".into(), pdf.to_vec())]; + let progress = Mutex::new(BatchProgress { total: files.len(), cancel: true, ..Default::default() }); + let recognizers = Recognizers { primary: Arc::new(Fake { confidence: 95.0 }), secondary: None }; + let writes = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let w = writes.clone(); + let msg = run_batch_files(files, OcrSettings { skip_text_files: true, ..Default::default() }, &recognizers, &progress, move |_, _| { + w.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Ok(()) + }); + assert_eq!(writes.load(std::sync::atomic::Ordering::SeqCst), 0, "nothing is written for skipped files, and the cancel came first"); + assert!(msg.contains("cancelled — 2 files were not run"), "{msg}"); + // Without the cancel, the skipped bucket is named and still nothing is written. + let files = vec![("a.pdf".into(), pdf.to_vec()), ("b.pdf".into(), pdf.to_vec())]; + let progress = Mutex::new(BatchProgress { total: files.len(), ..Default::default() }); + let w = writes.clone(); + let msg = run_batch_files(files, OcrSettings { skip_text_files: true, ..Default::default() }, &recognizers, &progress, move |_, _| { + w.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Ok(()) + }); + assert_eq!(writes.load(std::sync::atomic::Ordering::SeqCst), 0); + assert!(msg.contains("; 2 files already had text"), "{msg}"); + assert!(msg.contains("Recognized 0 words in 0 files"), "{msg}"); + } + + /// A batch over scanned files: the words are counted, and a low-confidence reading lands + /// in the summary as a note with its 1-based sorted pages. + #[test] + fn batch_notes_low_confidence_pages() { + let s = pdfcraft_engine::Session::new(); + let pdf = s.create_from_text("t", "fake scan").expect("fixture"); + let files = vec![("scan.pdf".into(), pdf.to_vec()), ("clean.pdf".into(), pdf.to_vec())]; + let progress = Mutex::new(BatchProgress { total: files.len(), ..Default::default() }); + let recognizers = Recognizers { primary: Arc::new(Fake { confidence: 40.0 }), secondary: None }; + // The fixtures are text PDFs standing in for scans; the default skip-text-pages option + // would (correctly) leave them alone, so the batch here reads everything. + let settings = OcrSettings { skip_text_pages: false, ..Default::default() }; + let msg = run_batch_files(files, settings.clone(), &recognizers, &progress, |_, _| Ok(())); + assert!(msg.contains("Recognized 2 words in 2 files"), "{msg}"); + assert!(msg.contains("scan.pdf: 1 low-confidence word(s) on pages 1"), "{msg}"); + // The write failing is a per-file failure, named in the summary. + let files = vec![("scan.pdf".into(), pdf.to_vec())]; + let progress = Mutex::new(BatchProgress { total: files.len(), ..Default::default() }); + let msg = run_batch_files(files, settings, &recognizers, &progress, |_, _| Err("disk full".into())); + assert!(msg.contains("; failed: scan.pdf: disk full"), "{msg}"); + } + + /// The batch note follows the settings' low-confidence threshold (60 by default), not a + /// number of its own: a 40% word is flagged under 60 and under 40, and not under 30. + #[test] + fn the_batch_note_uses_the_settings_threshold() { + let s = pdfcraft_engine::Session::new(); + let pdf = s.create_from_text("t", "fake scan").expect("fixture"); + let recognizers = Recognizers { primary: Arc::new(Fake { confidence: 40.0 }), secondary: None }; + let run = |threshold: f32| { + let files = vec![("scan.pdf".into(), pdf.to_vec())]; + let progress = Mutex::new(BatchProgress { total: files.len(), ..Default::default() }); + let settings = OcrSettings { skip_text_pages: false, low_confidence_threshold: threshold, ..Default::default() }; + run_batch_files(files, settings, &recognizers, &progress, |_, _| Ok(())) + }; + assert!(run(OcrSettings::default().low_confidence_threshold).contains("1 low-confidence word(s)"), "the default flags a 40% word"); + assert!(run(40.0).contains("1 low-confidence word(s)"), "at the threshold counts"); + assert!(!run(30.0).contains("low-confidence"), "a lower threshold leaves the word alone: {}", run(30.0)); + } + + /// The remembered output folder survives the preferences round trip; an absent or empty + /// value keeps the default (ask each time). + #[test] + fn the_output_folder_is_persisted_with_the_draft() { + let mut d = OcrDraft::default(); + assert_eq!(d.output_folder, None, "by default the folder is asked for"); + d.output_folder = Some("out".into()); + let mut r = OcrDraft::default(); + r.apply_pref(&d.to_pref()); + assert_eq!(r.output_folder.as_deref(), Some("out")); + let mut e = OcrDraft::default(); + e.apply_pref(&serde_json::json!({})); + assert_eq!(e.output_folder, None, "an absent value keeps the default"); + e.output_folder = Some("kept".into()); + e.apply_pref(&serde_json::json!({ "output_folder": "" })); + assert_eq!(e.output_folder.as_deref(), Some("kept"), "an empty value is not a folder"); + } + + /// "Automatic" saved without a strategy reads the way Automatic reads by default (the + /// second engine joins when the first is unsure); an explicit choice always stands. + #[test] + fn an_automatic_engine_without_a_saved_strategy_is_confidence_weighted() { + let mut d = OcrDraft::default(); + d.apply_pref(&serde_json::json!({ "engine": "auto" })); + assert_eq!((d.engine, d.strategy), (EngineChoice::Auto, MergeStrategy::ConfidenceWeighted)); + let mut named = OcrDraft::default(); + named.apply_pref(&serde_json::json!({ "engine": "tesseract" })); + assert_eq!(named.strategy, MergeStrategy::PrimaryOnly, "a named engine reads alone"); + let mut explicit = OcrDraft::default(); + explicit.apply_pref(&serde_json::json!({ "engine": "auto", "strategy": "primary" })); + assert_eq!((explicit.engine, explicit.strategy), (EngineChoice::Auto, MergeStrategy::PrimaryOnly)); + } + + /// Two inputs of one file name (from different folders, or differing only in case) would + /// write the same output: the batch stops before anything is read or written. + #[test] + fn inputs_that_share_a_file_name_are_refused_before_anything_is_read() { + let mut app = PdfCraftApp::new(); + app.run_inline = true; + let dir = std::env::temp_dir().join(format!("pdfcraft-ocr-same-name-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("fixture"); + app.ocr_draft.output_folder = Some(dir.to_string_lossy().into_owned()); + let s = pdfcraft_engine::Session::new(); + let pdf = s.create_from_text("t", "scanned words here").expect("fixture"); + app.ocr_files(vec![("scan.pdf".into(), pdf.to_vec()), ("other.pdf".into(), pdf.to_vec()), ("Scan.PDF".into(), pdf.to_vec())]); + assert!(app.ocr_batch.is_none(), "the batch never starts"); + let toast = app.toast.clone().map(|t| t.0).unwrap_or_default(); + assert!(toast.contains("same name") && toast.contains("Scan.PDF"), "{toast}"); + assert_eq!(std::fs::read_dir(&dir).expect("fixture").count(), 0, "nothing was written"); + let _ = std::fs::remove_dir_all(&dir); + } + + /// A remembered output folder is used without asking, is remembered again after the run, + /// and its collision pre-flight still stops the batch before the first page is read. + #[test] + fn a_remembered_output_folder_is_used_and_still_preflights_collisions() { + let mut app = PdfCraftApp::new(); + app.run_inline = true; + let dir = std::env::temp_dir().join(format!("pdfcraft-ocr-folder-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("fixture"); + std::fs::write(dir.join("a.pdf"), b"would be overwritten").expect("fixture"); + app.ocr_draft.output_folder = Some(dir.to_string_lossy().into_owned()); + let s = pdfcraft_engine::Session::new(); + let pdf = s.create_from_text("t", "scanned words here").expect("fixture"); + app.ocr_files(vec![("a.pdf".into(), pdf.to_vec()), ("b.pdf".into(), pdf.to_vec())]); + assert!(app.ocr_batch.is_none(), "the collision stops the batch before anything is read"); + let toast = app.toast.clone().map(|t| t.0).unwrap_or_default(); + assert!(toast.contains("already holds files of these names"), "{toast}"); + assert!(toast.contains("a.pdf"), "the colliding names are in the error: {toast}"); + assert_eq!(app.ocr_draft.output_folder.as_deref(), Some(dir.to_string_lossy().as_ref()), "the folder stays remembered for the next batch"); + let _ = std::fs::remove_dir_all(&dir); + } + + /// A result that arrives after the document changed (here: a page turned while the worker was + /// reading; the same applies to a redaction applied meanwhile) is refused with the reason and + /// writes nothing, so words read before a redaction cannot come back as invisible text. + #[test] + fn a_result_that_arrives_after_the_document_changed_is_refused() { + let mut app = PdfCraftApp::new(); + let pdf = pdfcraft_engine::Session::new().create_from_text("t", "Typed text").expect("fixture"); + app.open_bytes("a.pdf", None, pdf.to_vec()).expect("opens"); + let id = app.active_ids().expect("a document").1; + let guard = app.session.ocr_guard(id).expect("a guard"); + app.session.apply(id, pdfcraft_engine::Edit::RotatePages { pages: vec![0], degrees: 90 }).expect("an edit while the worker runs"); + let after_edit = app.session.get(id).and_then(|d| d.can_undo().map(str::to_owned)); + let word = pdfcraft_engine::ocr::PlacedWord { + text: "123-45-6789".into(), + origin: [40.0, 40.0], + across: [50.0, 0.0], + up: [0.0, 12.0], + confidence: Some(95.0), + source: "fake".into(), + }; + let page = OcrPage { page: 0, words: vec![word], already_on_page: 0, skipped: None, notes: Vec::new() }; + let progress = Arc::new(Mutex::new(OcrProgress { total: 1, done: 1, result: Some(Ok(vec![page])), cancel: false })); + app.ocr_run = Some(OcrRun { doc: id, progress, guard, automatic: None, editable: false }); + app.poll_ocr(); + assert!(app.ocr_run.is_none(), "the run is over"); + let toast = app.toast.clone().map(|t| t.0).unwrap_or_default(); + assert!(toast.contains("changed since recognition started"), "{toast}"); + assert_eq!( + app.session.get(id).and_then(|d| d.can_undo().map(str::to_owned)), + after_edit, + "nothing was written: the last undoable step is still the edit" + ); + } + + /// A page's notes survive to the result the user sees: the editable output's + /// unrenderable-character warning ("N characters had no font … appear as ?") is said in + /// the run's summary, not dropped. The result is fabricated, so no models are needed. + #[test] + fn the_result_summary_names_the_substituted_characters() { + let mut app = app_with(scan()); + let id = app.active_ids().unwrap().1; + let word = pdfcraft_engine::ocr::PlacedWord { + text: "scanned".into(), + origin: [72.0, 700.0], + across: [60.0, 0.0], + up: [0.0, 12.0], + confidence: Some(95.0), + source: "ocrs".into(), + }; + let page = pdfcraft_engine::ocr::OcrPage { + page: 0, + words: vec![word], + already_on_page: 0, + skipped: None, + notes: vec!["3 characters had no font that could show them and appear as ? in the output".into()], + }; + let guard = app.session.ocr_guard(id).unwrap(); + let progress = Arc::new(Mutex::new(OcrProgress { total: 1, done: 1, result: Some(Ok(vec![page])), cancel: false })); + app.ocr_run = Some(OcrRun { doc: id, progress, guard, automatic: None, editable: false }); + app.poll_ocr(); + let toast = app.toast.take().map(|t| t.0).unwrap_or_default(); + assert!(toast.contains("3 characters had no font"), "{toast}"); + } + + /// The dialog's "Editable text" output is honored: a run that promised an editable document + /// does not quietly write the invisible searchable layer — its result opens in the OCR + /// Verify screen, whose Accept produces the editable document. The result is fabricated, + /// so no models are needed. + #[test] + fn an_editable_output_run_opens_the_review_instead_of_applying() { + let mut app = app_with(scan()); + app.run_inline = true; + let id = app.active_ids().unwrap().1; + let word = pdfcraft_engine::ocr::PlacedWord { + text: "scanned".into(), + origin: [72.0, 700.0], + across: [60.0, 0.0], + up: [0.0, 12.0], + confidence: Some(95.0), + source: "ocrs".into(), + }; + let page = OcrPage { page: 0, words: vec![word], already_on_page: 0, skipped: None, notes: Vec::new() }; + let guard = app.session.ocr_guard(id).unwrap(); + let progress = Arc::new(Mutex::new(OcrProgress { total: 1, done: 1, result: Some(Ok(vec![page])), cancel: false })); + app.ocr_draft.output_mode = pdfcraft_engine::ocr::OutputMode::EditableText; + app.ocr_run = Some(OcrRun { doc: id, progress, guard, automatic: None, editable: true }); + app.poll_ocr(); + assert!(app.ocr_run.is_none(), "the run is over"); + assert_eq!(app.session.get(id).unwrap().can_undo(), None, "no invisible layer was written"); + assert!(app.ocr_verify.open, "the review opens, to accept as an editable document"); + assert_eq!(app.ocr_verify.phase, crate::workflows::ocr_verify::Phase::ReviewReady); + } } diff --git a/crates/ui-egui/src/workflows/ocr_verify.rs b/crates/ui-egui/src/workflows/ocr_verify.rs new file mode 100644 index 0000000000..4054339de4 --- /dev/null +++ b/crates/ui-egui/src/workflows/ocr_verify.rs @@ -0,0 +1,2830 @@ +//! Scan & OCR ▸ Correct recognized text: the OCR Verify screen. A full-workspace surface with +//! four panes (page list, source page, recognized text, zoom + word inspector) where a +//! recognition is reviewed before it is applied: words are colored by confidence, corrected, +//! removed (and restored), sent to a per-language user dictionary, and navigated by their +//! uncertainty. Accept applies the review — as one undoable step on the open document +//! (searchable), or as a new `_ocr.pdf` of visible text (editable). Reject throws the +//! reading away. +//! +//! The state machine is [`Phase`]: recognition runs on a worker thread ([`Phase::Running`], a +//! real Cancel keeps the pages already read), results land in [`Phase::ReviewReady`], failures +//! land in [`Phase::RecoverableError`] with the cause named and Run enabled again, and +//! Accept/Reject/Re-OCR are ignored outside ReviewReady instead of acting on nothing. + +use std::collections::BTreeSet; +use std::path::PathBuf; +use std::sync::{Arc, Mutex}; + +use pdfcraft_engine::ocr::{ + ConfidenceBand, EngineChoice, MergeStrategy, OcrGuard, OcrPage, OcrSettings, OutputMode, PlacedWord, band_for, clamp_region, +}; +use pdfcraft_engine::{DocId, Session}; + +use super::ocr_ui; +use crate::theme::{self, Tokens}; +use crate::{PdfCraftApp, widgets}; + +/// Where the screen stands. Run is enabled in Idle, ReviewReady and RecoverableError; the +/// review tools (Accept, Reject, navigation, the inspector, re-OCR) only in ReviewReady. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Phase { + #[default] + Idle, + Running, + ReviewReady, + Saving, + RecoverableError, +} + +/// The four panes; Ctrl+Tab moves the focus ring. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Pane { + #[default] + Pages, + Source, + Text, + Inspector, +} + +impl Pane { + fn next(self) -> Pane { + match self { + Pane::Pages => Pane::Source, + Pane::Source => Pane::Text, + Pane::Text => Pane::Inspector, + Pane::Inspector => Pane::Pages, + } + } +} + +/// One review change, so it can be undone within the panel (review edits are not document +/// edits; they live until the review is applied or rejected). +#[derive(Clone, Debug)] +pub enum Op { + /// The correction text field committed (an empty commit removes instead). + Corrected { + key: (usize, usize), + before: String, + after: String, + }, + Removed { + key: (usize, usize), + }, + Restored { + key: (usize, usize), + }, + WordVerified { + key: (usize, usize), + on: bool, + }, + Dictionary { + word: String, + }, + /// A re-read (a region or the whole page) replaced words on the page. Review marks are + /// keyed by word position, so the page's marks as they were are kept for the undo. + Region { + page: usize, + before: Vec, + keys: PageKeys, + }, + PageVerified { + page: usize, + on: bool, + }, + /// "Reject entire page": the page's words were cleared (and its review marks with them). + PageCleared { + page: usize, + before: Vec, + keys: PageKeys, + }, +} + +/// One page's positional review marks (word indices), saved when its word list is replaced. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct PageKeys { + pub removed: Vec, + pub verified: Vec, +} + +/// The review state: which words are struck out, which are vouched for, which are in the +/// per-language dictionary, and the undo history. Never persisted. +#[derive(Clone, Debug, Default)] +pub struct Review { + pub removed: BTreeSet<(usize, usize)>, + pub verified: BTreeSet<(usize, usize)>, + pub verified_pages: BTreeSet, + pub dictionary: BTreeSet, + /// Pages whose words were already applied to the document ("Accept entire page"); a later + /// Accept of the whole review never writes them twice. + pub applied_pages: BTreeSet, + pub history: Vec, +} + +/// A word that asks for review: its confidence is in the Low band and the reviewer has not +/// dealt with it — it is not removed, not corrected into the dictionary, not marked verified, +/// and not on a page marked verified. A corrected word stays uncertain by its ORIGINAL +/// confidence: the reviewer changed the text, not the engine's confidence in it. +pub fn is_uncertain(page: &OcrPage, key: (usize, usize), review: &Review) -> bool { + let (_, wi) = key; + if review.removed.contains(&key) || review.verified.contains(&key) || review.verified_pages.contains(&key.0) { + return false; + } + let Some(w) = page.words.get(wi) else { return false }; + if w.text.trim().is_empty() || review.dictionary.contains(&w.text.to_lowercase()) { + return false; + } + pdfcraft_engine::ocr::is_suspect(w.confidence) +} + +/// Candidate corrections for `text`: dictionary words and the page's own words within edit +/// distance 2, nearest first, at most 5, never the word itself. +pub fn suggestions(text: &str, dictionary: &BTreeSet, page_words: &[PlacedWord]) -> Vec { + let lower = text.to_lowercase(); + let mut out: Vec<(usize, String)> = Vec::new(); + let consider = |candidate: &str, out: &mut Vec<(usize, String)>| { + let c = candidate.to_lowercase(); + if c.is_empty() || c == lower { + return; + } + if let Some(d) = edit_distance_within_2(&lower, &c, 2) { + out.push((d, candidate.to_owned())); + } + }; + for w in dictionary { + consider(w, &mut out); + } + for w in page_words { + consider(&w.text, &mut out); + } + // Nearest first; ties break alphabetically; at most five. + out.sort_by(|a, b| a.0.cmp(&b.0).then_with(|| a.1.cmp(&b.1))); + out.truncate(5); + out.into_iter().map(|(_, s)| s).collect() +} + +/// Levenshtein distance between `a` and `b`, abandoned as soon as it is sure to exceed `max` +/// (small bounded DP over char boundaries; `None` means "farther than `max`"). +pub fn edit_distance_within_2(a: &str, b: &str, max: usize) -> Option { + let a: Vec = a.chars().collect(); + let b: Vec = b.chars().collect(); + if a.len().abs_diff(b.len()) > max { + return None; + } + let mut prev: Vec = (0..=b.len()).collect(); + let mut cur = vec![0usize; b.len() + 1]; + for i in 1..=a.len() { + cur[0] = i; + let mut row_min = cur[0]; + for j in 1..=b.len() { + let cost = usize::from(a[i - 1] != b[j - 1]); + cur[j] = (prev[j] + 1).min(cur[j - 1] + 1).min(prev[j - 1] + cost); + row_min = row_min.min(cur[j]); + } + if row_min > max { + return None; + } + std::mem::swap(&mut prev, &mut cur); + } + (prev[b.len()] <= max).then_some(prev[b.len()]) +} + +/// The page preview render: cached until the document or page changes. +pub struct Preview { + pub doc: DocId, + pub page: usize, + /// Pixels per point the page was rendered at. + pub scale: f32, + pub width: u32, + pub height: u32, + pub(crate) texture: Option, +} + +/// A page or region re-read in flight ([`OcrVerify::start_reread`]): the page being read +/// again (and the drawn region, in view-space points, for a region re-read), with the +/// worker's progress. +pub struct Reread { + pub page: usize, + pub region: Option<[f32; 4]>, + pub(crate) progress: Arc>, +} + +/// A re-read's progress: the replacement words once the worker is done, and a cancel request. +/// A cancelled re-read is discarded when it lands: the page's words stay as they were. +#[derive(Default)] +pub struct RereadProgress { + pub cancel: bool, + pub(crate) result: Option, String>>, +} + +/// The OCR Verify screen. One instance per app; lives across frames. +pub struct OcrVerify { + pub open: bool, + pub phase: Phase, + /// The document being reviewed; `None` until a run starts. + pub doc: Option, + /// The recognition results, in page order, as read (review edits mutate them). + pub pages: Vec, + /// What applying the results still expects of the document. + pub guard: Option, + pub review: Review, + /// The selected word (page, index) and the page the source pane shows. + pub selected: Option<(usize, usize)>, + pub current: usize, + /// A region drawn on the source pane, in view-space points on the shown page, snapped + /// outward to whole pixels when it is used ([`Self::start_reread`]). + pub region: Option<[f32; 4]>, + /// Where a source-pane drag started (the live region rectangle is drawn from here). + pub(crate) drag: Option, + pub focus: Pane, + /// The status line: where the run stands, why something was refused. + pub message: String, + cancelled: bool, + pub progress: Option>>, + /// A page or region re-read running on a worker thread: the screen stays live (the review + /// edits and navigation work) and the reading is applied by [`Self::poll_reread`]. + pub reread: Option, + /// The correction text field's current contents. + pub correction: String, + /// Where the per-language dictionary files live (`None`: off, e.g. the web build). + pub dictionary_dir: Option, + /// Language of the captured recognition, independent of the interface locale. + pub dictionary_language: String, + pub(crate) preview: Option, +} + +fn dictionary_path(dir: &std::path::Path, language: &str) -> Result { + if language.is_empty() || language.len() > 32 || !language.bytes().all(|b| b.is_ascii_alphabetic() || b == b'-') { + return Err("invalid OCR dictionary language".into()); + } + Ok(dir.join(format!("ocr-dictionary-{language}.txt"))) +} + +fn read_dictionary(path: &std::path::Path) -> Result { + use std::io::Read as _; + let mut text = String::new(); + match std::fs::File::open(path) { + Ok(file) => { + file.take((pdfcraft_engine::ocr::dictionary::MAX_BYTES + 1) as u64).read_to_string(&mut text).map_err(|e| e.to_string())?; + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => return Err(e.to_string()), + } + Ok(text) +} + +impl Default for OcrVerify { + fn default() -> Self { + OcrVerify { + open: false, + phase: Phase::Idle, + doc: None, + pages: Vec::new(), + guard: None, + review: Review::default(), + selected: None, + current: 0, + region: None, + drag: None, + focus: Pane::Pages, + message: String::new(), + cancelled: false, + progress: None, + reread: None, + correction: String::new(), + dictionary_dir: None, + dictionary_language: "en".into(), + preview: None, + } + } +} + +impl OcrVerify { + /// Begin a review of `found` for `doc`: ReviewReady even when nothing was found (the + /// status then says so), with the guard remembered for Accept. + pub fn load(&mut self, doc: DocId, guard: OcrGuard, found: Vec) { + self.doc = Some(doc); + self.guard = Some(guard); + self.pages = found; + self.review = Review::default(); + self.selected = None; + self.region = None; + self.current = self.current.min(self.pages.len().saturating_sub(1)); + self.phase = Phase::ReviewReady; + self.load_dictionary_pref(); + } + + /// The count of live (not removed, not empty) words across the review. + pub fn word_count(&self) -> usize { + self.pages + .iter() + .enumerate() + .map(|(p, pg)| pg.words.iter().enumerate().filter(|(i, w)| !w.text.trim().is_empty() && !self.review.removed.contains(&(p, *i))).count()) + .sum() + } + + /// The words that ask for review, in reading order across pages. + pub fn uncertain(&self) -> Vec<(usize, usize)> { + self.pages + .iter() + .flat_map(|pg| pg.suspects().into_iter().map(move |i| (pg.page, i))) + .filter(|k| self.page(k.0).is_some_and(|p| is_uncertain(p, *k, &self.review))) + .collect() + } + + /// The count of Low-band live words (the info strip's LOW-CONFIDENCE WORDS). + pub fn suspect_count(&self) -> usize { + self.pages + .iter() + .enumerate() + .map(|(p, pg)| { + pg.words + .iter() + .enumerate() + .filter(|(i, w)| { + !self.review.removed.contains(&(p, *i)) + && !w.text.trim().is_empty() + && !self.review.dictionary.contains(&w.text.to_lowercase()) + && pdfcraft_engine::ocr::is_suspect(w.confidence) + }) + .count() + }) + .sum() + } + + /// The share of words the reviewer vouched for: marked verified, or on a page marked + /// verified (0.0 with no words). + pub fn verified_fraction(&self) -> f32 { + let (mut yes, mut all) = (0usize, 0usize); + for (p, pg) in self.pages.iter().enumerate() { + let page_done = self.review.verified_pages.contains(&pg.page); + for (i, w) in pg.words.iter().enumerate() { + if w.text.trim().is_empty() || self.review.removed.contains(&(p, i)) { + continue; + } + all += 1; + if page_done || self.review.verified.contains(&(p, i)) { + yes += 1; + } + } + } + if all == 0 { 0.0 } else { yes as f32 / all as f32 } + } + + pub fn page(&self, page: usize) -> Option<&OcrPage> { + self.pages.iter().find(|p| p.page == page) + } + + pub fn word(&self, key: (usize, usize)) -> Option<&PlacedWord> { + self.page(key.0).and_then(|p| p.words.get(key.1)) + } + + pub fn is_removed(&self, key: (usize, usize)) -> bool { + self.review.removed.contains(&key) + } + + pub fn is_verified(&self, key: (usize, usize)) -> bool { + self.review.verified.contains(&key) || self.review.verified_pages.contains(&key.0) + } + + /// The word's text with any correction applied. + pub fn text_of(&self, key: (usize, usize)) -> Option<&str> { + self.word(key).map(|w| w.text.as_str()) + } + + fn in_review(&self) -> bool { + self.phase == Phase::ReviewReady + } + + /// Commit the correction field for `key`. An empty (after trim) commit removes the word; + /// the box never moves. Only in ReviewReady. + pub fn correct(&mut self, key: (usize, usize), text: &str) -> bool { + if !self.in_review() || self.word(key).is_none() { + return false; + } + if text.trim().is_empty() { + return self.set_removed(key, true); + } + let Some(w) = self.page_mut(key.0).and_then(|p| p.words.get_mut(key.1)) else { return false }; + let before = w.text.clone(); + if before == text { + return false; + } + w.text = text.to_string(); + self.review.history.push(Op::Corrected { key, before, after: text.to_string() }); + true + } + + /// Strike a word out (or restore it). Removed words stay in place — reviewable, restorable, + /// and never written by Accept. + pub fn set_removed(&mut self, key: (usize, usize), removed: bool) -> bool { + if !self.in_review() || self.word(key).is_none() || self.is_removed(key) == removed { + return false; + } + if removed { + self.review.removed.insert(key); + self.review.history.push(Op::Removed { key }); + } else { + self.review.removed.remove(&key); + self.review.history.push(Op::Restored { key }); + } + true + } + + /// Vouch for one word. + pub fn set_word_verified(&mut self, key: (usize, usize), on: bool) -> bool { + if !self.in_review() || self.word(key).is_none() || self.review.verified.contains(&key) == on { + return false; + } + if on { + self.review.verified.insert(key); + } else { + self.review.verified.remove(&key); + } + self.review.history.push(Op::WordVerified { key, on }); + true + } + + /// Mark the page reviewed (the toolbar's Verified toggle, Ctrl+T). + pub fn set_page_verified(&mut self, page: usize, on: bool) -> bool { + if !self.in_review() || self.page(page).is_none() || self.review.verified_pages.contains(&page) == on { + return false; + } + if on { + self.review.verified_pages.insert(page); + } else { + self.review.verified_pages.remove(&page); + } + self.review.history.push(Op::PageVerified { page, on }); + true + } + + /// Add a word to the per-language dictionary (case-insensitive, duplicate-free) and + /// remember it in the dictionary file when one can be written. Privacy: that file + /// (`ocr-dictionary-.txt` in the preferences folder) is plain text outside any + /// document, so a word added from a document stays on disk after the document is closed, + /// until it is removed (the undo removes it again). + pub fn add_to_dictionary(&mut self, word: &str, language: &str) -> bool { + if !self.in_review() { + return false; + } + if language != self.dictionary_language { + self.message = "The dictionary language differs from the captured recognition".into(); + return false; + } + let lower = word.trim().to_lowercase(); + if self.review.dictionary.contains(&lower) { + return false; + } + match self.updated_dictionary(&[word], &[]) { + Ok((entries, changed)) => { + self.review.dictionary = entries; + if !changed { + return false; + } + self.review.history.push(Op::Dictionary { word: lower }); + true + } + Err(why) => { + self.message = format!("OCR dictionary: {why}"); + false + } + } + } + + /// Validate the complete file and persist atomically before changing review state. + fn updated_dictionary(&self, add: &[&str], remove: &[&str]) -> Result<(BTreeSet, bool), String> { + use pdfcraft_engine::ocr::dictionary; + let path = self.dictionary_dir.as_ref().map(|dir| dictionary_path(dir, &self.dictionary_language)).transpose()?; + let current = + if let Some(path) = &path { read_dictionary(path)? } else { self.review.dictionary.iter().cloned().collect::>().join("\n") }; + let before = dictionary::words(¤t)?; + let text = dictionary::update(¤t, add, remove)?; + let entries = dictionary::words(&text)?; + let changed = before != entries; + if let Some(path) = path.filter(|_| changed) { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).map_err(|e| e.to_string())?; + } + crate::editing::write_atomically(&path.to_string_lossy(), text.as_bytes()).map_err(|e| e.to_string())?; + } + Ok((entries, changed)) + } + + /// Read the captured recognition language's dictionary; errors preserve its live state. + pub fn load_dictionary_pref(&mut self) { + let Some(dir) = &self.dictionary_dir else { return }; + let path = match dictionary_path(dir, &self.dictionary_language) { + Ok(path) => path, + Err(why) => { + self.message = why; + return; + } + }; + match read_dictionary(&path).and_then(|text| pdfcraft_engine::ocr::dictionary::words(&text)) { + Ok(words) => self.review.dictionary = words, + Err(why) => self.message = format!("OCR dictionary: {why}"), + } + } + + /// Undo the last review change (Ctrl+Z / the inspector's Undo). + pub fn undo(&mut self) -> bool { + if !self.in_review() { + return false; + } + let Some(op) = self.review.history.pop() else { return false }; + match op { + Op::Corrected { key, before, .. } => { + if let Some(w) = self.page_mut(key.0).and_then(|p| p.words.get_mut(key.1)) { + w.text = before; + } + } + Op::Removed { key } => { + self.review.removed.remove(&key); + } + Op::Restored { key } => { + self.review.removed.insert(key); + } + Op::WordVerified { key, on } => { + if on { + self.review.verified.remove(&key); + } else { + self.review.verified.insert(key); + } + } + Op::Dictionary { word } => match self.updated_dictionary(&[], &[&word]) { + Ok((entries, _)) => self.review.dictionary = entries, + Err(why) => { + self.message = format!("OCR dictionary: {why}"); + self.review.history.push(Op::Dictionary { word }); + return false; + } + }, + Op::Region { page, before, keys } => { + if let Some(p) = self.page_mut(page) { + p.words = before; + } + self.restore_page_keys(page, keys); + } + Op::PageVerified { page, on } => { + if on { + self.review.verified_pages.remove(&page); + } else { + self.review.verified_pages.insert(page); + } + } + Op::PageCleared { page, before, keys } => { + if let Some(p) = self.page_mut(page) { + p.words = before; + } + self.restore_page_keys(page, keys); + } + } + true + } + + fn reread_settings(&self, draft: &super::ocr_ui::OcrDraft) -> OcrSettings { + let mut settings = draft.settings(); + settings.language = self.dictionary_language.clone(); + settings + } + + /// Whether this word's text was corrected in the review (the inspector says so, and the + /// source pane outlines it). + pub fn is_corrected(&self, key: (usize, usize)) -> bool { + // Newest first: a correction recorded before the page's words were replaced (a re-read, + // a cleared page) names a position in the OLD word list, not this word. + for op in self.review.history.iter().rev() { + match op { + Op::Corrected { key: k, .. } if *k == key => return true, + Op::Region { page, .. } | Op::PageCleared { page, .. } if *page == key.0 => return false, + _ => {} + } + } + false + } + + /// Take `page`'s positional review marks out of the review (its word list is about to be + /// replaced), returning them for the undo. + fn take_page_keys(&mut self, page: usize) -> PageKeys { + let take = |set: &mut BTreeSet<(usize, usize)>| { + let on_page: Vec = set.iter().filter(|k| k.0 == page).map(|k| k.1).collect(); + set.retain(|k| k.0 != page); + on_page + }; + PageKeys { removed: take(&mut self.review.removed), verified: take(&mut self.review.verified) } + } + + /// Put back `page`'s marks exactly as `keys` holds them (the undo of a replacement). + fn restore_page_keys(&mut self, page: usize, keys: PageKeys) { + self.review.removed.retain(|k| k.0 != page); + self.review.verified.retain(|k| k.0 != page); + self.review.removed.extend(keys.removed.into_iter().map(|i| (page, i))); + self.review.verified.extend(keys.verified.into_iter().map(|i| (page, i))); + } + + /// Accept ONE page ("Accept entire page"): the page's live words become one undoable step + /// on the open document, the page is marked applied, and a later whole-review Accept (or a + /// second click on the same page) never writes it again. Ignored outside ReviewReady; zero + /// live words on the page is a typed failure; a stale session is refused with its reason + /// (edits kept). + pub fn accept_page(&mut self, session: &mut Session, page: usize) -> Result { + if !self.in_review() { + return Ok(0); + } + let Some(guard) = self.guard.clone() else { return Ok(0) }; + let Some(doc) = self.doc else { return Ok(0) }; + let Some(pg) = self.live_pages().into_iter().find(|p| p.page == page) else { return Ok(0) }; + if pg.words.is_empty() { + return Err("no words are left to apply on this page".into()); + } + // The page's words are already in the document: never append the OCR layer twice. + // (A page accepted and then rejected still counts: its words were written, and the + // empty-words check above has already spoken for it.) + if self.review.applied_pages.contains(&page) { + return Ok(0); + } + match session.apply_ocr_checked(doc, &[pg], &guard) { + Ok(n) => { + self.review.applied_pages.insert(page); + // This apply is now the document's newest edit: the next page (or the final + // Accept) checks against the document as it stands after it, so only an edit + // made elsewhere in between makes it stale. + self.guard = session.ocr_guard(doc); + Ok(n) + } + Err(why) => { + self.phase = Phase::RecoverableError; + self.message = why.clone(); + Err(why) + } + } + } + + /// Reject ONE page ("Reject entire page"): clear its words (undoable within the panel), + /// drop the selection if it was there. + pub fn reject_page(&mut self, page: usize) -> bool { + if !self.in_review() { + return false; + } + let Some(pg) = self.page_mut(page) else { return false }; + if pg.words.is_empty() { + return false; + } + let before = std::mem::take(&mut pg.words); + let keys = self.take_page_keys(page); + self.review.history.push(Op::PageCleared { page, before, keys }); + if self.selected.is_some_and(|k| k.0 == page) { + self.selected = None; + } + true + } + + /// Replace a page's words wholesale ("Re-OCR entire page"), undoable within the panel. The + /// page's review marks belonged to the old words (they are kept by position), so they are + /// cleared with them and come back with the undo. + pub fn replace_page_words(&mut self, page: usize, words: Vec) -> bool { + if !self.in_review() || self.page(page).is_none() { + return false; + } + let Some(pg) = self.page_mut(page) else { return false }; + let before = std::mem::replace(&mut pg.words, words); + let keys = self.take_page_keys(page); + self.review.history.push(Op::Region { page, before, keys }); + if self.selected.is_some_and(|k| k.0 == page) { + self.selected = None; + } + true + } + + fn page_mut(&mut self, page: usize) -> Option<&mut OcrPage> { + self.pages.iter_mut().find(|p| p.page == page) + } + + /// Walk the uncertain words. Wraps around, skips removed words; with none to show it is a + /// no-op. A corrected word that is still Low-band stays in the walk. + pub fn goto_uncertain(&mut self, forward: bool) -> bool { + let list = self.uncertain(); + let Some(target) = (match self.selected { + None if forward => list.first().copied(), + None => list.last().copied(), + Some(sel) => { + let pos = list.iter().position(|k| *k == sel); + match pos { + Some(i) => { + let next = if forward { i + 1 } else { i + list.len() - 1 }; + list.get(next % list.len()).copied() + } + None if forward => list.first().copied(), + None => list.last().copied(), + } + } + }) else { + return false; + }; + self.select(target); + true + } + + /// Select a word and show its page. + pub fn select(&mut self, key: (usize, usize)) { + self.selected = Some(key); + self.current = key.0; + self.correction = self.text_of(key).unwrap_or_default().to_string(); + } + + /// The results Accept would write: the read pages with removed words stripped. + pub fn live_pages(&self) -> Vec { + self.pages + .iter() + .map(|p| OcrPage { + already_on_page: 0, + page: p.page, + skipped: p.skipped.clone(), + notes: p.notes.clone(), + words: p + .words + .iter() + .enumerate() + .filter(|(i, w)| !self.review.removed.contains(&(p.page, *i)) && !w.text.trim().is_empty()) + .map(|(_, w)| w.clone()) + .collect(), + }) + .collect() + } + + /// Accept in searchable mode: apply the reviewed words as ONE undoable step on the open + /// document, but only while the session is still the one that was read. Ignored outside + /// ReviewReady; zero words and a stale session are typed failures (edits kept). + /// Returns the number of words written. + pub fn accept(&mut self, session: &mut Session) -> Result { + if !self.in_review() { + return Ok(0); // ignored, not an error + } + let Some(guard) = self.guard.clone() else { return Ok(0) }; + let Some(doc) = self.doc else { return Ok(0) }; + // Pages accepted page-by-page are already in the document; never write them twice. + let applied = &self.review.applied_pages; + let pages: Vec = self.live_pages().into_iter().filter(|p| !applied.contains(&p.page)).collect(); + let words = pages.iter().map(|p| p.words.len()).sum::(); + if words == 0 { + if !applied.is_empty() { + self.finish(); + return Ok(0); + } + return Err("no words are left to apply: every word was removed or empty".into()); + } + match session.apply_ocr_checked(doc, &pages, &guard) { + Ok(n) => { + self.finish(); + Ok(n) + } + Err(why) => { + self.phase = Phase::RecoverableError; + self.message = why.clone(); + Err(why) + } + } + } + + /// Accept in editable-text mode: the NEW document's bytes and its default file name + /// (`_ocr.pdf`). The open document is not touched; refuses zero words. + pub fn editable_output(&self, session: &Session) -> Result<(Arc>, String), String> { + if !self.in_review() { + return Err("the review is not ready".into()); + } + let Some(doc) = self.doc else { return Err("the review is not ready".into()) }; + // The review can stay open while the document is edited (Exit keeps it): words read before a + // redaction must not be written out as visible text afterwards. + if let Some(why) = self.guard.as_ref().and_then(|g| session.stale_reason(g)) { + return Err(why); + } + let pages = self.live_pages(); + if pages.iter().map(|p| p.words.len()).sum::() == 0 { + return Err("no words are left to write: every word was removed or empty".into()); + } + let bytes = session.editable_ocr_bytes(doc, &pages)?; + let d = session.get(doc).ok_or("the document is no longer open")?; + let base = d.name.strip_suffix(".pdf").unwrap_or(&d.name).to_string(); + Ok((bytes, format!("{base}_ocr.pdf"))) + } + + /// Reject: clear the words, the session, the selection and the flags; back to Idle. + /// Ignored outside ReviewReady. + pub fn reject(&mut self) -> bool { + if !self.in_review() { + return false; + } + *self = OcrVerify { open: self.open, dictionary_dir: self.dictionary_dir.take(), ..Default::default() }; + true + } + + /// The state after a successful accept (or a saved editable copy): the review is done. + pub(crate) fn finish(&mut self) { + *self = OcrVerify { open: self.open, phase: Phase::Idle, doc: self.doc, dictionary_dir: self.dictionary_dir.take(), ..Default::default() }; + } + + /// Ask the running recognition to stop. The pages already read are kept; the rest of the + /// document is not read. + pub fn cancel(&mut self) { + if self.phase == Phase::Running { + self.cancelled = true; + if let Some(p) = &self.progress + && let Ok(mut s) = p.lock() + { + s.cancel = true; + } + } + } + + pub fn can_run(&self) -> bool { + matches!(self.phase, Phase::Idle | Phase::ReviewReady | Phase::RecoverableError) + } + + /// Start re-reading a whole page (`region` `None`) or a drawn region on a WORKER THREAD: + /// the UI thread only builds the job and refuses an empty or out-of-page region BEFORE + /// anything runs — the render, the recognition and the engine probe (a `--list-langs` + /// process for Tesseract) happen off it, with the status line and a Cancel while they run. + /// The reading is applied by [`Self::poll_reread`] when it lands. + pub fn start_reread(&mut self, app: &mut PdfCraftApp, page: usize, region: Option<[f32; 4]>) { + if self.phase != Phase::ReviewReady || self.reread.is_some() { + return; // one re-read at a time, and never under a run + } + let Some(doc) = self.doc else { return }; + let settings = self.reread_settings(&app.ocr_draft); + if let Some(region) = region { + let Some(info) = app.session.get(doc).and_then(|d| d.info.pages.get(page)) else { + app.notify_error("the page is gone"); + return; + }; + // Refuse before running: an empty, inverted or out-of-page rectangle never + // reaches the engine (the words are untouched). + let scale = app.session.ocr_job(doc, &[page], settings.clone()).map_or(1.0, |job| job.render_scale(page)); + let (w, h) = (info.width * scale, info.height * scale); + let px = [region[0] * scale, region[1] * scale, region[2] * scale, region[3] * scale]; + if clamp_region(px, w.max(1.0) as u32, h.max(1.0) as u32).is_none() { + app.notify_error("the selected region is empty or outside the page"); + return; + } + } + let Some(mut job) = app.session.ocr_job(doc, &[page], settings) else { + app.notify_error("the document is no longer open"); + return; + }; + if let Some(region) = region { + let scale = job.render_scale(page); + job.settings.region = Some([region[0] * scale, region[1] * scale, region[2] * scale, region[3] * scale]); + } + let is_region = region.is_some(); + let progress = Arc::new(Mutex::new(RereadProgress::default())); + self.message = if is_region { tl!("Re-reading the region…").to_string() } else { tl!("Re-reading the page…").to_string() }; + let p = progress.clone(); + let work = move || { + let result = pdfcraft_engine::ocr::recognizers(&job.settings).and_then(|r| { + let found = job.run(&r, |_, _| !p.lock().map(|s| s.cancel).unwrap_or(false)); + let words: Vec = found.into_iter().flat_map(|pg| pg.words).collect(); + if words.is_empty() { + Err(if is_region { + "the region re-read found no text; the existing words are unchanged".into() + } else { + "the re-read found no text; the existing words are unchanged".into() + }) + } else { + Ok(words) + } + }); + if let Ok(mut s) = p.lock() { + s.result = Some(result); + } + }; + #[cfg(not(target_arch = "wasm32"))] + if app.run_inline { + work(); + } else { + std::thread::Builder::new().name("pdfcraft-ocr-reread".into()).spawn(work).ok(); + } + #[cfg(target_arch = "wasm32")] + work(); + self.reread = Some(Reread { page, region, progress }); + } + + /// Ask a running re-read to stop and be discarded: when the worker lands, its reading is + /// thrown away and the page's words stay as they were. + pub fn cancel_reread(&mut self) { + if let Some(r) = &self.reread + && let Ok(mut s) = r.progress.lock() + { + s.cancel = true; + } + } + + /// Apply a finished re-read, once the worker is done: a page re-read replaces the page's + /// words, a region re-read the words inside the drawn rectangle. A cancel discards the + /// reading, and so does a refusal — a document changed since the review began no longer + /// matches the page the words were read from. Called every frame while a re-read is out. + pub(crate) fn poll_reread(&mut self, app: &mut PdfCraftApp) { + if self.phase != Phase::ReviewReady { + return; + } + let Some(reread) = &self.reread else { return }; + let (cancelled, result) = match reread.progress.lock() { + Ok(mut s) => (s.cancel, s.result.take()), + Err(_) => return, + }; + let Some(result) = result else { + // Still reading: keep the status line live and ask for another frame. + if let Some(ctx) = &app.ctx { + ctx.request_repaint_after(std::time::Duration::from_millis(200)); + } + return; + }; + let (page, region) = (reread.page, reread.region); + self.reread = None; + if cancelled { + self.message = tl!("The re-read was cancelled; the words are unchanged").to_string(); + return; + } + // The document changed since the review began: the reading may no longer fit the page + // it was read from, so it is refused with the reason and the words stay. + if let Some(why) = self.guard.as_ref().and_then(|g| app.session.stale_reason(g)) { + self.message = why.clone(); + app.notify_error(why); + return; + } + match result { + Err(why) => { + self.message = why.clone(); + app.notify_error(why); + } + Ok(words) => { + let applied = if let Some(region) = region { + self.replace_region_words(&app.session, page, region, words).map(|_| true) + } else if self.replace_page_words(page, words) { + Ok(false) + } else { + Err("the page is gone".to_string()) + }; + match applied { + Ok(is_region) => { + if is_region { + self.message = tl!("The region was read again; its words were replaced (undo in the panel)").to_string(); + app.notify_tr("The region was read again; its words were replaced (undo in the panel)"); + } else { + self.message = tl!("The page was read again; its words were replaced (undo in the panel)").to_string(); + app.notify_tr("The page was read again; its words were replaced (undo in the panel)"); + }; + } + Err(why) => app.notify_error(why), + } + } + } + } + + /// Put `words` (the worker's reading of the drawn rectangle) in place of the words inside + /// it: the rectangle's bbox in user space decides which old words go; the kept words' + /// review marks follow them to their new positions, the replacement is undoable within + /// the panel, and the first NEW word is selected. An empty reading was refused before the + /// run, so `words` is never empty here. + fn replace_region_words(&mut self, session: &Session, page: usize, region_view: [f32; 4], words: Vec) -> Result<(), String> { + let Some(doc) = self.doc else { return Ok(()) }; + let Some(info) = session.get(doc).and_then(|d| d.info.pages.get(page)) else { return Err("the page is gone".into()) }; + // The rectangle in user space (a bbox of its mapped corners) decides which old words + // the new reading replaces. + let (a, b, c, d) = ( + info.view_to_user(region_view[0], region_view[1]), + info.view_to_user(region_view[2], region_view[3]), + info.view_to_user(region_view[0], region_view[3]), + info.view_to_user(region_view[2], region_view[1]), + ); + let xs = [a[0] as f64, b[0] as f64, c[0] as f64, d[0] as f64]; + let ys = [a[1] as f64, b[1] as f64, c[1] as f64, d[1] as f64]; + let bbox = [ + xs.iter().cloned().fold(f64::INFINITY, f64::min), + ys.iter().cloned().fold(f64::INFINITY, f64::min), + xs.iter().cloned().fold(f64::NEG_INFINITY, f64::max), + ys.iter().cloned().fold(f64::NEG_INFINITY, f64::max), + ]; + let Some(pg) = self.page_mut(page) else { return Err("the page is gone".into()) }; + let before = pg.words.clone(); + let center = |w: &PlacedWord| [w.origin[0] + w.across[0] / 2.0 + w.up[0] / 2.0, w.origin[1] + w.across[1] / 2.0 + w.up[1] / 2.0]; + // The old words outside the rectangle stay, in order; `moved` maps each kept word's old + // position to its new one, so its review marks follow it. + let mut moved = std::collections::BTreeMap::new(); + let mut after: Vec = Vec::with_capacity(before.len().saturating_add(words.len())); + for (i, w) in before.iter().enumerate() { + let c = center(w); + if !(c[0] >= bbox[0] && c[0] <= bbox[2] && c[1] >= bbox[1] && c[1] <= bbox[3]) { + moved.insert(i, after.len()); + after.push(w.clone()); + } + } + // The new words follow the kept ones, so the first of them is at the kept words' count; + // selecting it shows the reviewer what the re-read produced. + let first_new = after.len(); + after.extend(words.iter().cloned()); + pg.words = after; + let keys = self.take_page_keys(page); + let remap = |old: &[usize]| old.iter().filter_map(|i| moved.get(i).map(|&j| (page, j))).collect::>(); + self.review.removed.extend(remap(&keys.removed)); + self.review.verified.extend(remap(&keys.verified)); + self.review.history.push(Op::Region { page, before, keys }); + self.select((page, first_new)); + Ok(()) + } + + /// Store a finished run's pages (the worker's result) and move to ReviewReady. A cancel is + /// honest: the pages already read are kept, and the status says the rest was not read. + pub fn finished(&mut self, found: Vec, cancelled: bool) { + let read = found.iter().filter(|p| p.skipped.is_none()).count(); + let found_words: usize = found.iter().map(|p| p.words.len()).sum(); + self.pages = found; + self.review = Review::default(); + self.selected = None; + self.region = None; + self.current = self.current.min(self.pages.len().saturating_sub(1)); + let total = self.pages.len(); + self.message = if cancelled { + crate::i18n::fmt( + tl!("Cancelled — {n} words were read on {m} of {t} pages; the rest were not read"), + &[("n", &found_words.to_string()), ("m", &read.to_string()), ("t", &total.to_string())], + ) + } else if found_words == 0 { + tl!("OCR complete — no text recognized on this page.").to_string() + } else { + crate::i18n::fmt( + tl!("Read {n} words on {m} of {t} pages"), + &[("n", &found_words.to_string()), ("m", &read.to_string()), ("t", &total.to_string())], + ) + }; + self.phase = Phase::ReviewReady; + self.load_dictionary_pref(); + } + + /// A run that failed: the cause is named and Run is enabled again. + pub fn failed(&mut self, why: &str) { + self.phase = Phase::RecoverableError; + self.message = why.to_string(); + self.progress = None; + } + + pub(crate) fn start_run(&mut self, app: &mut PdfCraftApp) { + let Some((_, id)) = app.active_ids() else { return }; + if !self.can_run() || self.phase == Phase::Running { + return; + } + if self.reread.is_some() { + return; // the page re-read in flight replaces these words; one worker at a time + } + if let Some(why) = app.session.get(id).and_then(|d| d.read_only_reason.clone()) { + self.message = why; + self.phase = Phase::RecoverableError; + return; + } + let d = &app.ocr_draft; + let pages: Vec = match d.pages { + super::ocr_ui::OcrPages::All => Vec::new(), + super::ocr_ui::OcrPages::Current => vec![app.active.map_or(0, |i| app.views.get(i).map_or(0, |v| v.current))], + super::ocr_ui::OcrPages::Range => (d.from.saturating_sub(1)..d.to).collect(), + }; + let settings = d.settings(); + let Some(job) = app.session.ocr_job(id, &pages, settings.clone()) else { return }; + self.dictionary_language = job.settings.language.clone(); + let progress = Arc::new(Mutex::new(ocr_ui::OcrProgress { total: job.pages.len(), ..Default::default() })); + self.doc = Some(id); + self.guard = Some(job.guard.clone()); + self.current = 0; + self.cancelled = false; + self.phase = Phase::Running; + self.message = String::new(); + let p = progress.clone(); + let work = move || { + let result = pdfcraft_engine::ocr::recognizers(&settings).map(|r| { + job.run(&r, |done, total| { + let Ok(mut s) = p.lock() else { return false }; + s.done = done; + s.total = total; + !s.cancel + }) + }); + if let Ok(mut s) = p.lock() { + s.result = Some(result); + } + }; + #[cfg(not(target_arch = "wasm32"))] + if app.run_inline { + work(); + } else { + std::thread::Builder::new().name("pdfcraft-ocr-verify".into()).spawn(work).ok(); + } + #[cfg(target_arch = "wasm32")] + work(); + self.progress = Some(progress); + self.poll(app); + } + + /// Take the worker's result, once it is done. + pub(crate) fn poll(&mut self, app: &mut PdfCraftApp) { + if self.phase != Phase::Running { + return; + } + let Some(progress) = self.progress.clone() else { return }; + let result = progress.lock().ok().and_then(|mut s| s.result.take()); + let Some(result) = result else { + // Still running: refresh the status line and ask for another frame. + if let Ok(s) = progress.lock() { + self.message = crate::i18n::fmt( + tl!("Recognizing text… page {d} of {t}"), + &[("d", &(s.done + 1).min(s.total.max(1)).to_string()), ("t", &s.total.max(1).to_string())], + ); + } + if let Some(ctx) = &app.ctx { + ctx.request_repaint_after(std::time::Duration::from_millis(200)); + } + return; + }; + self.progress = None; + match result { + Ok(found) => self.finished(found, self.cancelled), + Err(why) => self.failed(&why), + } + } +} + +/// What a frame's controls asked for, applied after the drawing (the panes read `app` while +/// the controls write it, so nothing mutates through an open borrow). +#[derive(Default)] +struct Intents { + run: bool, + cancel: bool, + cancel_reread: bool, + accept: bool, + reject: bool, + exit: bool, + undo: bool, + next_uncertain: bool, + prev_uncertain: bool, + toggle_page_verified: bool, + reocr_region: bool, + reocr_page: bool, + accept_page: bool, + reject_page: bool, + cycle_focus: bool, + select_page: Option, + select_word: Option<(usize, usize)>, + correct: Option, + set_removed: Option, + set_word_verified: Option, + dictionary_add: bool, +} + +/// Band colours: High green, Medium yellow, Low red (the fill at ~20% opacity, the outline +/// at ~60%; the legend carries the same thresholds as text). +fn band_color(band: ConfidenceBand) -> egui::Color32 { + match band { + ConfidenceBand::High => egui::Color32::from_rgb(34, 139, 74), + ConfidenceBand::Medium => egui::Color32::from_rgb(196, 148, 20), + ConfidenceBand::Low => egui::Color32::from_rgb(204, 61, 47), + } +} + +/// The whole-window OCR Verify screen (Scan & OCR ▸ Correct recognized text). +pub(crate) fn show(app: &mut PdfCraftApp, ui: &mut egui::Ui) { + use egui::{Key, Modifiers}; + let ctx = ui.ctx().clone(); + let t = Tokens::get(&ctx); + let wants_text = ctx.egui_wants_keyboard_input(); + let (next_u, prev_u, toggle_verified, cycle, undo_key) = ctx.input_mut(|i| { + ( + i.consume_key(Modifiers::NONE, Key::F8), + i.consume_key(Modifiers::SHIFT, Key::F8), + (!wants_text && i.consume_key(Modifiers::COMMAND, Key::T)), + (!wants_text && i.consume_key(Modifiers::COMMAND, Key::Tab)), + (!wants_text && i.consume_key(Modifiers::COMMAND, Key::Z)), + ) + }); + + let mut n = Intents { + next_uncertain: next_u, + prev_uncertain: prev_u, + toggle_page_verified: toggle_verified, + cycle_focus: cycle, + undo: undo_key, + ..Default::default() + }; + + egui::Panel::left("ocr-verify-pages").resizable(false).default_size(190.0).frame(egui::Frame::new().fill(t.panel).inner_margin(8.0)).show( + ui, + |ui| { + pane_title(ui, &t, tl!("Pages"), app.ocr_verify.focus == Pane::Pages); + pages_pane(app, ui, &mut n); + }, + ); + + egui::Panel::right("ocr-verify-right").resizable(false).default_size(340.0).frame(egui::Frame::new().fill(t.panel).inner_margin(8.0)).show( + ui, + |ui| { + pane_title(ui, &t, tl!("Recognized text"), app.ocr_verify.focus == Pane::Text); + text_pane(app, ui, &t, &mut n); + ui.add_space(10.0); + pane_title(ui, &t, tl!("Word"), app.ocr_verify.focus == Pane::Inspector); + inspector(app, ui, &t, &mut n); + }, + ); + + egui::CentralPanel::default().frame(egui::Frame::new().fill(t.pasteboard).inner_margin(egui::Margin::same(8))).show(ui, |ui| { + toolbar(app, ui, &mut n); + ui.add_space(4.0); + info_strip(app, ui, &t); + ui.add_space(6.0); + pane_title(ui, &t, tl!("Source page"), app.ocr_verify.focus == Pane::Source); + source_pane(app, ui, &t, &mut n); + ui.add_space(6.0); + ui.horizontal(|ui| { + ui.label(egui::RichText::new(&app.ocr_verify.message).small().color(if app.ocr_verify.phase == Phase::RecoverableError { + t.accent + } else { + t.text_muted + })); + }); + }); + + if n.cycle_focus { + app.ocr_verify.focus = app.ocr_verify.focus.next(); + } + apply(app, n); +} + +/// A pane's caption; the focused pane's name is underlined (Ctrl+Tab moves the focus). +fn pane_title(ui: &mut egui::Ui, t: &Tokens, name: &str, focused: bool) { + ui.add_space(2.0); + let font = theme::semibold(11.5); + let w = ui.fonts_mut(|f| f.layout_no_wrap(name.to_owned(), font.clone(), t.text).size().x); + let (rect, _) = ui.allocate_exact_size(egui::vec2(w, 14.0), egui::Sense::hover()); + ui.painter().text(rect.left_center(), egui::Align2::LEFT_CENTER, name.to_uppercase(), font, t.text_muted); + if focused { + let r = egui::Rect::from_min_max(rect.left_bottom() + egui::vec2(0.0, 1.0), rect.right_bottom() + egui::vec2(0.0, 2.0)); + ui.painter().rect_filled(r, egui::CornerRadius::same(1), t.accent); + } + ui.add_space(4.0); +} + +/// Toolbar: every control is live — the choices feed the next run (and are persisted), Run +/// starts a recognition, Accept/Reject/navigate/verify only act in ReviewReady. +fn toolbar(app: &mut PdfCraftApp, ui: &mut egui::Ui, n: &mut Intents) { + let v = &app.ocr_verify; + let d = &mut app.ocr_draft; + let review = v.phase == Phase::ReviewReady; + let run_enabled = v.can_run() && v.reread.is_none(); + let accept_enabled = review && v.word_count() + v.review.applied_pages.len() > 0; + + // Row 1: the settings (each written back into the persisted draft). + ui.horizontal_wrapped(|ui| { + craft_ui::ComboBox::from_id_salt("ocr-verify-language").width(150.0).selected_text(language_name(&d.language)).show_ui(ui, |ui| { + for (code, name) in pdfcraft_engine::ocr::LANGUAGES { + ui.selectable_value(&mut d.language, (*code).to_string(), *name); + } + }); + engine_combo(ui, &mut d.engine, &mut d.strategy); + strategy_combo(ui, &mut d.strategy); + output_combo(ui, &mut d.output_mode); + for (flag, label, why) in [ + (&mut d.auto_rotate, tl!("Auto-rotate"), tl!("Turns pages scanned sideways or upside down upright before reading")), + (&mut d.deskew, tl!("Deskew"), tl!("Straightens skewed text lines before reading")), + (&mut d.denoise, tl!("Denoise"), tl!("A 3×3 median filter against scan speckle")), + (&mut d.binarize, tl!("Binarize"), tl!("Sauvola binarization: ink to black, paper to white")), + ] { + ui.add(crate::widgets::checkbox(flag, label)).on_hover_text(format!("{label} \u{2014} {why}")); + } + }); + + // Row 2: scope and actions. + ui.add_space(4.0); + ui.horizontal_wrapped(|ui| { + crate::widgets::radio_value(ui, &mut d.pages, super::ocr_ui::OcrPages::All, tl!("All pages")); + crate::widgets::radio_value(ui, &mut d.pages, super::ocr_ui::OcrPages::Current, tl!("Current page")); + crate::widgets::radio_value(ui, &mut d.pages, super::ocr_ui::OcrPages::Range, tl!("From")); + let on = d.pages == super::ocr_ui::OcrPages::Range; + ui.add_enabled(on, egui::DragValue::new(&mut d.from).range(1..=9999)); + ui.label(tl!("to")); + ui.add_enabled(on, egui::DragValue::new(&mut d.to).range(1..=9999)); + + ui.separator(); + if ui.add_enabled_ui(run_enabled, |ui| widgets::pill_button(ui, tl!("Run OCR"), true)).inner.clicked() { + n.run = true; + } + if v.phase == Phase::Running && widgets::pill_button(ui, tl!("Cancel"), false).clicked() { + n.cancel = true; + } + if v.reread.is_some() && widgets::pill_button(ui, tl!("Cancel re-read"), false).clicked() { + n.cancel_reread = true; + } + if ui.add_enabled_ui(accept_enabled, |ui| widgets::pill_button(ui, tl!("Accept"), false)).inner.clicked() { + n.accept = true; + } + if ui.add_enabled_ui(review, |ui| widgets::pill_button(ui, tl!("Reject"), false)).inner.clicked() { + n.reject = true; + } + ui.separator(); + let page_done = review && v.review.verified_pages.contains(&v.current); + let mut verified = page_done; + if ui + .add_enabled(review, crate::widgets::checkbox(&mut verified, tl!("Page verified"))) + .on_hover_text(tl!("Mark this page as reviewed (Ctrl+T)")) + .changed() + { + n.toggle_page_verified = true; + } + if ui + .add_enabled_ui(review, |ui| widgets::ghost_button(ui, "chevron-right", tl!("Next uncertain"))) + .inner + .on_hover_text(tl!("F8: next word to review; Shift+F8 goes back")) + .clicked() + { + n.next_uncertain = true; + } + if ui + .add_enabled_ui(review, |ui| widgets::ghost_button(ui, "chevron-left", tl!("Previous uncertain"))) + .inner + .on_hover_text(tl!("Shift+F8: the previous word to review")) + .clicked() + { + n.prev_uncertain = true; + } + if widgets::ghost_button(ui, "x", tl!("Exit")).clicked() { + n.exit = true; + } + }); +} + +/// A language code's dropdown text: its name and its code ("English (en)"). +fn language_name(code: &str) -> String { + pdfcraft_engine::ocr::LANGUAGES.iter().find(|l| l.0 == code).map_or_else(|| code.to_owned(), |l| format!("{} ({})", l.1, l.0)) +} + +/// The Engine dropdown's text. +fn engine_label(engine: EngineChoice) -> &'static str { + match engine { + EngineChoice::Auto => tl!("Automatic"), + EngineChoice::Ocrs => "ocrs", + EngineChoice::Tesseract => "Tesseract", + } +} + +/// The Strategy dropdown's text. +fn strategy_label(strategy: MergeStrategy) -> &'static str { + match strategy { + MergeStrategy::PrimaryOnly => tl!("Primary only"), + MergeStrategy::ConfidenceWeighted => tl!("Confidence weighted"), + MergeStrategy::RoverVote => tl!("ROVER vote"), + } +} + +/// The Engine dropdown: Automatic, ocrs, Tesseract. An engine that cannot run is disabled and +/// says why (a tooltip names the missing models or program). The choice feeds the next run. +/// Choosing Automatic while the strategy is "Primary only" (which would never run the second +/// engine) moves the strategy to Automatic's own default, Confidence weighted. +fn engine_combo(ui: &mut egui::Ui, engine: &mut EngineChoice, strategy: &mut MergeStrategy) { + ocr_ui::keep_asking_while_probing(ui.ctx()); + craft_ui::ComboBox::from_id_salt("ocr-verify-engine").width(120.0).selected_text(engine_label(*engine)).show_ui(ui, |ui| { + for (choice, label) in [(EngineChoice::Auto, tl!("Automatic")), (EngineChoice::Ocrs, "ocrs"), (EngineChoice::Tesseract, "Tesseract")] { + let why = pdfcraft_engine::ocr::engine_unavailable_reason(choice); + let resp = ui.add_enabled(why.is_none(), egui::Button::selectable(*engine == choice, label)); + let clicked = resp.clicked(); + match why { + Some(reason) => { + resp.on_disabled_hover_text(crate::i18n::t(&reason)); + } + None => { + resp.on_hover_text(match choice { + EngineChoice::Auto => tl!("Both engines, combined as the Strategy says"), + EngineChoice::Ocrs => tl!("The built-in engine (pure Rust)"), + EngineChoice::Tesseract => tl!("The tesseract program, run as an external process"), + }); + } + } + if clicked { + *engine = choice; + if choice == EngineChoice::Auto && *strategy == MergeStrategy::PrimaryOnly { + *strategy = choice.default_strategy(); + } + } + } + }); +} + +/// The Strategy dropdown: how two engines' readings combine. With a single named engine the +/// ensemble strategies still run — they just degrade to the primary reading (the tooltip says +/// so), because only one engine was asked to read. +fn strategy_combo(ui: &mut egui::Ui, strategy: &mut MergeStrategy) { + craft_ui::ComboBox::from_id_salt("ocr-verify-strategy").width(150.0).selected_text(strategy_label(*strategy)).show_ui(ui, |ui| { + for (s, label, hint) in [ + (MergeStrategy::PrimaryOnly, tl!("Primary only"), tl!("Use the chosen engine's reading alone")), + (MergeStrategy::ConfidenceWeighted, tl!("Confidence weighted"), tl!("Add a second engine's reading when the first is unsure")), + (MergeStrategy::RoverVote, tl!("ROVER vote"), tl!("Always read with both engines and merge the words")), + ] { + let resp = ui.add(egui::Button::selectable(*strategy == s, label)).on_hover_text(hint); + if resp.clicked() { + *strategy = s; + } + } + }); +} + +/// The Output mode dropdown: what Accept produces, with tooltips that say what each mode +/// changes (searchable: the open page gains an invisible text layer; editable: a new file). +fn output_combo(ui: &mut egui::Ui, mode: &mut OutputMode) { + let name = match *mode { + OutputMode::Searchable => tl!("Searchable Image (Exact)"), + OutputMode::EditableText => tl!("Editable text"), + }; + craft_ui::ComboBox::from_id_salt("ocr-verify-output").width(180.0).selected_text(name).show_ui(ui, |ui| { + let s = ui + .selectable_label(*mode == OutputMode::Searchable, tl!("Searchable Image (Exact)")) + .on_hover_text(tl!("Adds invisible text over each word; the page image is not changed")); + if s.clicked() { + *mode = OutputMode::Searchable; + } + let e = ui + .selectable_label(*mode == OutputMode::EditableText, tl!("Editable text")) + .on_hover_text(tl!("Writes a new document of visible text; the scan is not changed")); + if e.clicked() { + *mode = OutputMode::EditableText; + } + }); +} + +/// The info strip: where the review stands, in facts — page, average confidence, how many +/// words ask for review, the share verified, and the engine · strategy that read it. +fn info_strip(app: &PdfCraftApp, ui: &mut egui::Ui, t: &Tokens) { + let v = &app.ocr_verify; + let read = v.pages.iter().filter(|p| p.skipped.is_none()).count(); + let avg = v.page(v.current).and_then(pdfcraft_engine::ocr::OcrPage::mean_confidence); + let cells = [ + (tl!("PAGE"), if v.pages.is_empty() { "—".into() } else { format!("{} / {}", v.current + 1, v.pages.len()) }), + (tl!("AVG CONFIDENCE"), avg.map_or_else(|| "—".into(), |c| format!("{c:.0}%"))), + (tl!("LOW-CONFIDENCE WORDS"), v.suspect_count().to_string()), + (tl!("VERIFIED %"), format!("{}%", (v.verified_fraction() * 100.0).round() as i64)), + ( + tl!("ENGINE · STRATEGY"), + if read == 0 { "—".into() } else { format!("{} · {}", engine_label(app.ocr_draft.engine), strategy_label(app.ocr_draft.strategy)) }, + ), + ]; + ui.horizontal(|ui| { + for (k, val) in cells { + ui.label(egui::RichText::new(k.to_string()).small().color(t.text_faint)); + ui.label(egui::RichText::new(val).small()); + ui.separator(); + } + }); +} + +/// The page list: one row per read page, with its live word count and a flag for the words +/// that still ask for review. Clicking shows the page. +fn pages_pane(app: &mut PdfCraftApp, ui: &mut egui::Ui, n: &mut Intents) { + let v = &app.ocr_verify; + egui::ScrollArea::vertical().show(ui, |ui| { + for pg in &v.pages { + let p = pg.page; + let (live, suspects) = ( + pg.words.iter().filter(|w| !w.text.trim().is_empty()).count(), + pg.words.iter().filter(|w| !w.text.trim().is_empty() && pdfcraft_engine::ocr::is_suspect(w.confidence)).count(), + ); + let marked = if pg.skipped.is_some() { + format!(" · {}", tl!("skipped")) + } else if v.review.verified_pages.contains(&p) { + format!(" · {}", tl!("verified")) + } else { + String::new() + }; + let text = format!("{} {}{} · {live}", tl!("Page"), p + 1, marked); + let label = ui.selectable_label(p == v.current, text); + let label = if suspects > 0 { + label.on_hover_text(crate::i18n::fmt(tl!("{n} word(s) to review"), &[("n", &suspects.to_string())])) + } else { + label + }; + if label.clicked() { + n.select_page = Some(p); + } + } + }); +} + +/// The SOURCE PAGE pane: the rendered page with one box per recognized word. A click selects +/// the word under the pointer; a drag draws the re-OCR region (in view-space points, clamped +/// to the page). Words are coloured by their confidence band; removed words are struck out and +/// dimmed, corrected words outlined. The context menu re-reads a region or the page, and +/// accepts or rejects the page. +fn source_pane(app: &mut PdfCraftApp, ui: &mut egui::Ui, t: &Tokens, n: &mut Intents) { + let page = app.ocr_verify.current; + let review = app.ocr_verify.phase == Phase::ReviewReady; + ensure_preview(app, page); + let Some(doc) = app.ocr_verify.doc else { return }; + let Some(info) = app.session.get(doc).and_then(|d| d.info.pages.get(page)).cloned() else { return }; + // Reserve the room the confidence legend (at the end of this pane) and the message row + // (after it) need, so the page cannot push them off the panel. + let size = (ui.available_size() - egui::vec2(0.0, 40.0)).max(egui::vec2(120.0, 160.0)); + let (rect, resp) = ui.allocate_exact_size(size, egui::Sense::click_and_drag()); + let painter = ui.painter_at(rect); + // The page fits the pane; the pasteboard shows around it. + let s = (rect.width() / info.width.max(1.0)).min(rect.height() / info.height.max(1.0)).max(0.01); + let size = egui::vec2(info.width * s, info.height * s); + let img_rect = egui::Rect::from_min_size(rect.min + (rect.size() - size) * 0.5, size); + painter.rect_filled(img_rect.expand(2.0), egui::CornerRadius::same(2), t.hover); + if let Some(preview) = &app.ocr_verify.preview + && let Some(tex) = &preview.texture + { + painter.image(tex.id(), img_rect, egui::Rect::from_min_max(egui::pos2(0.0, 0.0), egui::pos2(1.0, 1.0)), egui::Color32::WHITE); + } + // A word's box: its user-space corners, through the page's rotation, into pane pixels. + let to_pane = |p: [f64; 2]| { + let v = info.user_to_view(p[0] as f32, p[1] as f32); + egui::pos2(img_rect.left() + v[0] * s, img_rect.top() + v[1] * s) + }; + let corners = |w: &PlacedWord| { + [ + to_pane(w.origin), + to_pane([w.origin[0] + w.across[0], w.origin[1] + w.across[1]]), + to_pane([w.origin[0] + w.across[0] + w.up[0], w.origin[1] + w.across[1] + w.up[1]]), + to_pane([w.origin[0] + w.up[0], w.origin[1] + w.up[1]]), + ] + }; + if let Some(pg) = app.ocr_verify.page(page) { + for (wi, w) in pg.words.iter().enumerate() { + let key = (pg.page, wi); + let quad = corners(w); + if app.ocr_verify.is_removed(key) { + // Removed: dimmed, struck through — still there, still restorable. + painter.add(egui::Shape::convex_polygon(quad.to_vec(), egui::Color32::from_black_alpha(32), egui::Stroke::NONE)); + painter.line_segment([quad[0], quad[2]], egui::Stroke::new(1.0, t.text_faint)); + continue; + } + if let Some(band) = w.confidence.map(band_for) { + let (fill, _) = band_colors(band); + painter.add(egui::Shape::convex_polygon(quad.to_vec(), fill, egui::Stroke::NONE)); + } + let outline = if app.ocr_verify.selected == Some(key) { + egui::Stroke::new(2.0, egui::Color32::from_rgb(43, 122, 236)) + } else if app.ocr_verify.is_corrected(key) { + egui::Stroke::new(1.5, t.text) + } else { + // The band's outline; a word without a confidence reads as unmeasured. + w.confidence.map(|c| band_colors(band_for(c)).1).unwrap_or_else(|| egui::Stroke::new(0.5, t.text_faint)) + }; + painter.add(egui::Shape::convex_polygon(quad.to_vec(), egui::Color32::TRANSPARENT, outline)); + } + } + // The chosen region and the one being drawn right now. + let drag_now = resp.interact_pointer_pos().zip(app.ocr_verify.drag).map(|(p, start)| egui::Rect::from_two_pos(start, p).intersect(img_rect)); + let chosen = app.ocr_verify.region.and_then(|region| pane_region(region, &info, img_rect)); + for r in [chosen, drag_now].into_iter().flatten() { + painter.rect_stroke(r, 0.0, egui::Stroke::new(1.5, t.accent), egui::StrokeKind::Middle); + } + // A click selects the word under the pointer; dragging away from a click draws the region. + if resp.drag_started() { + app.ocr_verify.drag = resp.interact_pointer_pos(); + } + if resp.drag_stopped() { + let (start, end) = (app.ocr_verify.drag.take(), resp.interact_pointer_pos()); + if let (Some(a), Some(b)) = (start, end) + && egui::Rect::from_two_pos(a, b).intersect(img_rect).area() > 36.0 + { + // A region: normalize it into view-space points, clamped to the page. + let r = egui::Rect::from_two_pos(a, b).intersect(img_rect); + let region = [ + (r.left() - img_rect.left()) / s, + (r.top() - img_rect.top()) / s, + (r.right() - img_rect.left()) / s, + (r.bottom() - img_rect.top()) / s, + ]; + app.ocr_verify.region = (region[2] > region[0] && region[3] > region[1]).then_some(region); + } + } + if resp.clicked() + && let Some(at) = resp.interact_pointer_pos() + && let Some(pg) = app.ocr_verify.page(page) + { + n.select_word = pg + .words + .iter() + .enumerate() + .find(|(wi, w)| !app.ocr_verify.is_removed((pg.page, *wi)) && point_in_quad(at, &corners(w))) + .map(|(wi, _)| (pg.page, wi)); + } + resp.context_menu(|ui| { + let rereading = app.ocr_verify.reread.is_some(); + if ui.add_enabled(review && !rereading && app.ocr_verify.region.is_some(), egui::Button::new(tl!("Re-OCR this region"))).clicked() { + n.reocr_region = true; + ui.close(); + } + if ui.add_enabled(review && !rereading, egui::Button::new(tl!("Re-OCR entire page"))).clicked() { + n.reocr_page = true; + ui.close(); + } + ui.separator(); + if ui.add_enabled(review, egui::Button::new(tl!("Accept entire page"))).clicked() { + n.accept_page = true; + ui.close(); + } + if ui.add_enabled(review, egui::Button::new(tl!("Reject entire page"))).clicked() { + n.reject_page = true; + ui.close(); + } + ui.separator(); + ui.label(egui::RichText::new(tl!("Click a word to inspect it; drag a rectangle to re-read part of the page")).small().color(t.text_faint)); + }); + legend(ui, t); +} + +/// A region given in view-space points, back into pane pixels on `img_rect` (None when it +/// does not lie on the page). The region is already in view space (the drag stored it so), so +/// it is only scaled: mapping it through `user_to_view` again would mirror it vertically (and +/// misplace it on rotated or cropped pages). +fn pane_region(region: [f32; 4], info: &pdfcraft_render::PageInfo, img_rect: egui::Rect) -> Option { + let scale = img_rect.width() / info.width.max(1.0); + let to_pane = |v: [f32; 2]| egui::pos2(img_rect.left() + v[0] * scale, img_rect.top() + v[1] * scale); + let a = to_pane([region[0], region[1]]); + let b = to_pane([region[2], region[3]]); + let r = egui::Rect::from_two_pos(a, b).intersect(img_rect); + (r.area() > 1.0).then_some(r) +} + +/// Point in a convex quad (corners in order): the pointer is on the same side of all four +/// edges. +fn point_in_quad(p: egui::Pos2, quad: &[egui::Pos2; 4]) -> bool { + (0..4).all(|i| { + let (a, b) = (quad[i], quad[(i + 1) % 4]); + (b.x - a.x) * (p.y - a.y) - (b.y - a.y) * (p.x - a.x) <= 0.0 + }) +} + +/// Band colours: the fill at ~20% opacity, the outline at ~60% — the legend names the bands +/// as text, so meaning never rests on colour alone. +fn band_colors(band: ConfidenceBand) -> (egui::Color32, egui::Stroke) { + let c = band_color(band); + (c.gamma_multiply(0.2), egui::Stroke::new(1.0, c.gamma_multiply(0.6))) +} + +/// The overlay legend: the classifier's bands, with their thresholds spelled out. +fn legend(ui: &mut egui::Ui, t: &Tokens) { + ui.add_space(2.0); + ui.horizontal_wrapped(|ui| { + ui.label(egui::RichText::new(tl!("CONFIDENCE")).small().strong()); + for (band, text) in legend_rows() { + let (fill, stroke) = band_colors(band); + let (rect, _) = ui.allocate_exact_size(egui::vec2(12.0, 12.0), egui::Sense::hover()); + ui.painter().rect_filled(rect, 2.0, fill); + ui.painter().rect_stroke(rect, 2.0, stroke, egui::StrokeKind::Inside); + ui.label(egui::RichText::new(text).small().color(t.text_muted)); + } + }); +} + +/// The legend's rows: the same [`ConfidenceBand`] values the overlay, the counts and the +/// navigation use, with the classifier's thresholds as text (a test holds the drawing to +/// these numbers). +pub fn legend_rows() -> [(ConfidenceBand, String); 3] { + [ + (ConfidenceBand::High, tl!("HIGH (>= 90%)").to_string()), + (ConfidenceBand::Medium, tl!("MEDIUM (70-89%)").to_string()), + (ConfidenceBand::Low, tl!("LOW (< 70%)").to_string()), + ] +} + +/// Render (or re-render) the source pane's page preview when the shown page changed. One +/// synchronous render per page; the texture is cached until then. +fn ensure_preview(app: &mut PdfCraftApp, page: usize) { + let Some(doc) = app.ocr_verify.doc else { return }; + if app.ocr_verify.preview.as_ref().is_some_and(|p| p.doc == doc && p.page == page && p.texture.is_some()) { + return; + } + let Some(d) = app.session.get(doc) else { return }; + let Some(info) = d.info.pages.get(page) else { return }; + // Screen scale (the boxes are drawn on top of it); the recognition dpi is irrelevant here. + let scale = 2.0_f32.min((900.0 / info.width.max(1.0)).min(1200.0 / info.height.max(1.0))).max(0.2); + let config = pdfcraft_render::RenderConfig { password: d.password.as_deref().map(std::sync::Arc::from), ..Default::default() }; + let mut r = pdfcraft_render::PageRenderer::new(d.bytes.clone(), config); + let shot = r.render(pdfcraft_render::RenderRequest { page, kind: pdfcraft_render::RequestKind::Pixels, scale, ..Default::default() }); + if shot.error.is_some() || shot.width == 0 || shot.height == 0 { + return; // keep any earlier preview; a render failure names itself elsewhere + } + let scale = shot.width as f32 / info.width.max(1.0); + let texture = app.ctx.as_ref().map(|ctx| { + ctx.load_texture( + "ocr-verify-page", + egui::ColorImage::from_rgba_premultiplied([shot.width as usize, shot.height as usize], &shot.rgba), + egui::TextureOptions::LINEAR, + ) + }); + app.ocr_verify.preview = Some(Preview { doc, page, scale, width: shot.width, height: shot.height, texture }); +} + +/// The RECOGNIZED pane: the page's words as the engine read them — a preview, not +/// authoritative (the source pane's boxes and the inspector are where corrections happen). +/// Removed words show struck out; clicking a word selects it in the inspector. +fn text_pane(app: &mut PdfCraftApp, ui: &mut egui::Ui, t: &Tokens, n: &mut Intents) { + let v = &app.ocr_verify; + let Some(pg) = v.page(v.current) else { return }; + if let Some(why) = &pg.skipped { + ui.label(egui::RichText::new(why.as_str()).small().color(t.text_muted)); + return; + } + if pg.words.is_empty() { + ui.label(egui::RichText::new(tl!("OCR complete — no text recognized on this page.")).small().color(t.text_muted)); + return; + } + egui::ScrollArea::vertical().max_height(180.0).show(ui, |ui| { + ui.horizontal_wrapped(|ui| { + for (wi, w) in pg.words.iter().enumerate() { + let key = (pg.page, wi); + let mut label = egui::RichText::new(w.text.as_str()).small(); + if v.is_removed(key) { + label = label.strikethrough().color(t.text_faint); + } else if v.is_corrected(key) { + label = label.underline(); + } + if ui.selectable_label(v.selected == Some(key), label).clicked() { + n.select_word = Some(key); + } + } + }); + }); +} + +/// The ZOOM + word inspector: a magnified crop of the selected word, its facts (id, +/// confidence, engine, review state), the correction field (Enter commits; clearing removes +/// the word; the original box never moves), Remove/Restore, + Dictionary, the per-word +/// Verified toggle, and suggestions within edit distance 2. +fn inspector(app: &mut PdfCraftApp, ui: &mut egui::Ui, t: &Tokens, n: &mut Intents) { + let v = &app.ocr_verify; + let Some(key) = v.selected else { + ui.label(egui::RichText::new(tl!("Select a word on the page to inspect it")).small().color(t.text_muted)); + return; + }; + if v.word(key).is_none() { + ui.label(egui::RichText::new(tl!("The selected word is gone; choose another")).small().color(t.text_muted)); + return; + } + zoom_crop(app, ui, key, t); + ui.add_space(4.0); + let (id, conf, source) = { + let w = v.word(key); + ( + key.1 + 1, + w.and_then(|w| w.confidence).map_or_else(|| "—".to_owned(), |c| format!("{c:.0}%")), + w.map(|w| w.source.clone()).unwrap_or_default(), + ) + }; + let state = if v.is_removed(key) { + tl!("removed") + } else if v.is_corrected(key) { + tl!("corrected") + } else if v.is_verified(key) { + tl!("verified") + } else { + tl!("unreviewed") + }; + ui.label(egui::RichText::new(format!("{id} · {conf} · {source} · {state}")).small().color(t.text_muted)); + ui.add_space(4.0); + // The correction field: Enter commits, an empty commit removes, the box never moves. + let (mut correction, verified) = (app.ocr_verify.correction.clone(), app.ocr_verify.is_verified(key)); + let field = ui.add(egui::TextEdit::singleline(&mut correction).hint_text(tl!("Corrected text (Enter commits)"))); + let commit = field.lost_focus() && ui.input(|i| i.key_pressed(egui::Key::Enter)); + ui.horizontal(|ui| { + if ui.add(egui::Button::new(tl!("Remove")).small()).clicked() { + n.set_removed = Some(true); + } + if ui.add(egui::Button::new(tl!("Restore")).small()).clicked() { + n.set_removed = Some(false); + } + if ui.add(egui::Button::new(tl!("+ Dictionary")).small()).clicked() { + n.dictionary_add = true; + } + let mut on = verified; + if ui.add(crate::widgets::checkbox(&mut on, tl!("Verified"))).changed() { + n.set_word_verified = Some(on); + } + if ui.add(egui::Button::new(tl!("Undo")).small()).on_hover_text(tl!("Undo the last review change (Ctrl+Z)")).clicked() { + n.undo = true; + } + }); + app.ocr_verify.correction = correction; + if commit { + n.correct = Some(app.ocr_verify.correction.clone()); + } + // Suggestions: dictionary words and this page's own words within edit distance 2. + let (word_text, page_words) = { + let v = &app.ocr_verify; + (v.text_of(key).unwrap_or_default().to_owned(), v.page(key.0).map(|p| p.words.clone()).unwrap_or_default()) + }; + let sugg = suggestions(&word_text, &app.ocr_verify.review.dictionary, &page_words); + if !sugg.is_empty() { + ui.add_space(2.0); + ui.horizontal_wrapped(|ui| { + ui.label(egui::RichText::new(tl!("Suggestions")).small().color(t.text_muted)); + for s in sugg { + if ui.add(egui::Button::new(egui::RichText::new(s.clone()).small())).clicked() { + n.correct = Some(s); + } + } + }); + } +} + +/// The magnified crop of the selected word, drawn out of the page preview's texture. +fn zoom_crop(app: &PdfCraftApp, ui: &mut egui::Ui, key: (usize, usize), t: &Tokens) { + let Some(preview) = &app.ocr_verify.preview else { return }; + let Some(tex) = &preview.texture else { return }; + let Some(w) = app.ocr_verify.word(key) else { return }; + let Some(info) = app.ocr_verify.doc.and_then(|doc| app.session.get(doc)).and_then(|d| d.info.pages.get(preview.page)).cloned() else { + return; + }; + // The word's box in preview pixels (user → view → × preview scale), plus a margin. + let to_px = |p: [f64; 2]| { + let v = info.user_to_view(p[0] as f32, p[1] as f32); + [v[0] * preview.scale, v[1] * preview.scale] + }; + let a = to_px(w.origin); + let b = to_px([w.origin[0] + w.across[0] + w.up[0], w.origin[1] + w.across[1] + w.up[1]]); + let (x0, y0, x1, y1) = (a[0].min(b[0]), a[1].min(b[1]), a[0].max(b[0]), a[1].max(b[1])); + let (mw, mh) = ((x1 - x0).max(8.0) * 0.5, (y1 - y0).max(8.0) * 0.6); + let crop = egui::Rect::from_min_max( + egui::pos2((x0 - mw).max(0.0), (y0 - mh).max(0.0)), + egui::pos2((x1 + mw).min(preview.width as f32), (y1 + mh).min(preview.height as f32)), + ); + if crop.width() < 1.0 || crop.height() < 1.0 { + return; + } + // At most 300 × 90 points, at most 3× the preview's own resolution. + let k = (300.0 / crop.width()).min(90.0 / crop.height()).clamp(0.1, 3.0); + let (rect, _) = ui.allocate_exact_size(egui::vec2(crop.width() * k, crop.height() * k), egui::Sense::hover()); + let uv = egui::Rect::from_min_max( + egui::pos2(crop.left() / preview.width as f32, crop.top() / preview.height as f32), + egui::pos2(crop.right() / preview.width as f32, crop.bottom() / preview.height as f32), + ); + ui.painter().image(tex.id(), rect, uv, egui::Color32::WHITE); + ui.painter().rect_stroke(rect, 0.0, egui::Stroke::new(1.0, t.accent), egui::StrokeKind::Inside); +} + +/// Run `f` with the screen's state moved out of the app (a step that needs both the state and +/// the rest of the app would otherwise borrow it twice). The state is put back afterwards. +pub(crate) fn with_state(app: &mut PdfCraftApp, f: impl FnOnce(&mut OcrVerify, &mut PdfCraftApp)) { + let mut v = std::mem::take(&mut app.ocr_verify); + f(&mut v, app); + app.ocr_verify = v; +} + +/// Apply what the frame's controls asked for. The guards live in the [`OcrVerify`] methods: +/// outside ReviewReady, Accept/Reject/Re-OCR and edits are ignored, not acted on. +fn apply(app: &mut PdfCraftApp, n: Intents) { + with_state(app, |v, app| { + if n.exit { + v.open = false; + } + if let Some(p) = n.select_page { + v.current = p; + if v.selected.is_some_and(|k| k.0 != p) { + v.selected = None; + } + } + if let Some(k) = n.select_word { + v.select(k); + } + if n.toggle_page_verified { + let on = !v.review.verified_pages.contains(&v.current); + v.set_page_verified(v.current, on); + } + if n.next_uncertain { + v.goto_uncertain(true); + } + if n.prev_uncertain { + v.goto_uncertain(false); + } + if n.undo { + v.undo(); + } + if let Some(text) = n.correct + && let Some(k) = v.selected + { + v.correct(k, &text); + v.correction = v.text_of(k).unwrap_or_default().to_owned(); + } + if let Some(removed) = n.set_removed + && let Some(k) = v.selected + { + v.set_removed(k, removed); + } + if let Some(on) = n.set_word_verified + && let Some(k) = v.selected + { + v.set_word_verified(k, on); + } + if n.dictionary_add + && let Some(k) = v.selected + { + let word = v.text_of(k).unwrap_or_default().to_owned(); + let language = v.dictionary_language.clone(); + v.add_to_dictionary(&word, &language); + } + if n.reocr_page { + let page = v.current; + v.start_reread(app, page, None); + } + if n.reocr_region { + let (page, region) = (v.current, v.region); + if let Some(region) = region { + v.start_reread(app, page, Some(region)); + } + } + if n.cancel_reread { + v.cancel_reread(); + } + if n.accept_page { + let page = v.current; + match v.accept_page(&mut app.session, page) { + Ok(count) if count > 0 => { + if let Some(doc) = v.doc { + refresh_view(app, doc); + } + app.notify_fmt(tl!("The page's words were applied as one undoable step ({n} words)"), &[("n", &count.to_string())]); + } + Ok(_) => {} + Err(why) => app.notify_error(why), + } + } + if n.reject_page { + let page = v.current; + if v.reject_page(page) { + app.notify_tr("The page's words were cleared (undo in the panel)"); + } + } + if n.run { + v.start_run(app); + } + if n.cancel { + v.cancel(); + } + if n.accept { + do_accept(v, app); + } + if n.reject && v.reject() { + app.notify_tr("OCR results rejected."); + } + }); +} + +/// Accept: searchable mode applies the review to the open document as one undoable step; +/// editable mode asks where to write the new `_ocr.pdf`. Stale sessions and empty +/// reviews are typed failures the user sees (the edits are kept). +fn do_accept(v: &mut OcrVerify, app: &mut PdfCraftApp) { + let doc = v.doc; + match app.ocr_draft.output_mode { + OutputMode::Searchable => match v.accept(&mut app.session) { + Ok(count) => { + if let Some(doc) = doc { + refresh_view(app, doc); + } + if count > 0 { + app.notify_fmt(tl!("Recognized text applied ({n} words, one undoable step)"), &[("n", &count.to_string())]); + } + } + Err(why) => app.notify_error(why), + }, + OutputMode::EditableText => match v.editable_output(&app.session) { + Ok((bytes, name)) => save_editable(app, bytes, name, doc), + Err(why) => app.notify_error(why), + }, + } +} + +/// Write the editable-text output: into the tests' export folder when one is set, otherwise +/// ask where (the default name is `_ocr.pdf`). The source is never overwritten; a +/// cancelled dialog keeps the review. The write is the [`Phase::Saving`] state: it ends the +/// review when the file lands, and returns to ReviewReady when it does not, so a failed or +/// cancelled save can simply be tried again. +fn save_editable(app: &mut PdfCraftApp, bytes: Arc>, name: String, doc: Option) { + let source = doc.and_then(|id| app.session.get(id)).and_then(|d| d.path.clone()); + // The chosen file is the open source when both resolve to the same path (a new file does + // not exist yet, so fall back to its folder and name). + let is_source = |picked: &std::path::Path, source: &Option| { + let Some(src) = source else { return false }; + if let (Ok(a), Ok(b)) = (std::fs::canonicalize(picked), std::fs::canonicalize(src)) { + return a == b; + } + let same_name = picked.file_name().is_some_and(|n| n.to_string_lossy() == src.as_str()); + let same_folder = std::fs::canonicalize(src) + .ok() + .zip(picked.parent().map(std::path::Path::to_path_buf)) + .is_some_and(|(s, parent)| s.parent().is_some_and(|p| p == parent)); + same_name && same_folder + }; + #[cfg(not(target_arch = "wasm32"))] + if let Some(dir) = app.export_dir_override.clone() { + let target = std::path::Path::new(&dir).join(&name); + if is_source(&target, &source) { + app.notify_tr("That is the source file; choose another destination"); + return; + } + write_editable(app, &target, bytes); + return; + } + #[cfg(not(target_arch = "wasm32"))] + { + let dialog = rfd::AsyncFileDialog::new().add_filter("PDF", &["pdf"]).set_file_name(&name); + app.ask_one(crate::pickers::Ask::Save(dialog), None, move |app, picked: std::path::PathBuf| { + if is_source(&picked, &source) { + app.notify_tr("That is the source file; choose another destination"); + return; + } + write_editable(app, &picked, bytes); + }); + } + // The web build has no file system to write into: the new document downloads under its + // `_ocr.pdf` name, so it never lands on the source either. + #[cfg(target_arch = "wasm32")] + { + let _ = (&source, doc, is_source); + let saved = crate::editing::download(&name, &bytes); + with_state(app, |v, _| { + if saved.is_ok() { + v.finish(); + } else { + v.phase = Phase::ReviewReady; + } + }); + if let Err(e) = saved { + app.notify_error(e); + } + } +} + +/// The editable write itself: [`Phase::Saving`] while it runs, Idle once the file lands (the +/// review is done), back to ReviewReady when it fails — the reviewer sees the error and can +/// accept again without re-reading the document. Desktop only: the web build downloads instead. +#[cfg(not(target_arch = "wasm32"))] +fn write_editable(app: &mut PdfCraftApp, target: &std::path::Path, bytes: Arc>) { + with_state(app, |v, _| v.phase = Phase::Saving); + match crate::editing::write_atomically(&target.to_string_lossy(), &bytes) { + Ok(()) => { + with_state(app, |v, _| v.finish()); + app.notify(crate::i18n::fmt(tl!("Wrote {name}"), &[("name", &target.to_string_lossy())])); + } + Err(e) => { + with_state(app, |v, _| v.phase = Phase::ReviewReady); + app.notify_error(e); + } + } +} + +/// Pull the document's changed appearance into its view after an apply. +fn refresh_view(app: &mut PdfCraftApp, doc: DocId) { + if let Some(info) = app.session.get(doc).map(|d| d.info.clone()) + && let Some(view) = app.views.iter_mut().find(|view| view.id == doc) + { + view.document_changed(&info); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use pdfcraft_engine::Session; + + /// A word box that sits where a real placed word would (high on the page, reading left + /// to right). + fn word(text: &str, conf: Option) -> PlacedWord { + PlacedWord { text: text.into(), origin: [72.0, 700.0], across: [40.0, 0.0], up: [0.0, 12.0], confidence: conf, source: "ocrs".into() } + } + + /// One page of words with the given confidences, and a review loaded with it (the guard + /// comes from a real job, as it does on screen). + fn loaded(session: &mut Session, confs: &[Option]) -> (DocId, OcrVerify) { + let text = session.create_from_text("t", "Recognize these scanned words").expect("fixture"); + let id = session.open("t.pdf", None, text, None).expect("fixture"); + let job = session.ocr_job(id, &[0], OcrSettings::default()).expect("fixture"); + let page = OcrPage { already_on_page: 0, page: 0, words: confs.iter().map(|c| word("w", *c)).collect(), skipped: None, notes: Vec::new() }; + let mut v = OcrVerify::default(); + v.load(id, job.guard.clone(), vec![page]); + (id, v) + } + + /// The spec's status wording is fixed: a finished run that found nothing says so, and a + /// cancelled one never claims a complete run. + #[test] + fn finished_states_keep_the_spec_wording() { + let mut v = OcrVerify::default(); + v.finished(vec![OcrPage { already_on_page: 0, page: 0, words: vec![], skipped: None, notes: Vec::new() }], false); + assert_eq!(v.phase, Phase::ReviewReady); + assert_eq!(v.message, "OCR complete — no text recognized on this page."); + v.finished( + vec![OcrPage { + already_on_page: 0, + page: 0, + words: vec![word("Hi", Some(95.0)), word("there", Some(40.0))], + skipped: None, + notes: Vec::new(), + }], + true, + ); + assert!(v.message.starts_with("Cancelled — 2 words were read on 1 of 1 pages"), "{}", v.message); + assert_eq!(v.phase, Phase::ReviewReady, "a cancel keeps the pages already read, in review"); + assert_eq!(v.page(0).expect("the cancelled run's page is kept").words.len(), 2); + } + + /// The legend's text is the classifier's thresholds, spelled out — a drift between the + /// two would put the wrong words in front of the reviewer. + #[test] + fn legend_rows_carry_the_classifier_thresholds() { + let rows = legend_rows(); + assert_eq!(rows[0].0, ConfidenceBand::High); + assert_eq!(rows[1].0, ConfidenceBand::Medium); + assert_eq!(rows[2].0, ConfidenceBand::Low); + assert_eq!(rows[0].1, "HIGH (>= 90%)"); + assert_eq!(rows[1].1, "MEDIUM (70-89%)"); + assert_eq!(rows[2].1, "LOW (< 70%)"); + // The same classifier drives the overlay: 90 is High, 89 is not; 70 is Medium, 69 is + // not; the bands clamp out-of-range values. + assert_eq!(band_for(90.0), ConfidenceBand::High); + assert_eq!(band_for(89.0), ConfidenceBand::Medium); + assert_eq!(band_for(70.0), ConfidenceBand::Medium); + assert_eq!(band_for(69.0), ConfidenceBand::Low); + assert_eq!(band_for(-1.0), ConfidenceBand::Low); + assert_eq!(band_for(140.0), ConfidenceBand::High); + } + + /// F8/Shift+F8 walk the uncertain words: wrap around, skip removed words, and a page + /// marked verified drops out entirely. With nothing to show, the walk is a no-op. + #[test] + fn uncertain_walk_wraps_and_skips_removed_and_verified() { + let mut s = Session::new(); + // 95 is fine; 40 and 50 are Low; the third word is corrected into the dictionary. + let (_, mut v) = loaded(&mut s, &[Some(95.0), Some(40.0), Some(50.0), Some(95.0)]); + assert_eq!(v.uncertain(), vec![(0, 1), (0, 2)], "the Low-band words, in order"); + assert!(v.goto_uncertain(true)); + assert_eq!(v.selected, Some((0, 1))); + assert!(v.goto_uncertain(true)); + assert_eq!(v.selected, Some((0, 2))); + assert!(v.goto_uncertain(true), "wraps around"); + assert_eq!(v.selected, Some((0, 1))); + // A removed word leaves the walk; a corrected word stays by its original confidence. + v.set_removed((0, 1), true); + assert!(v.goto_uncertain(true)); + assert_eq!(v.selected, Some((0, 2))); + assert!(v.correct((0, 2), "fixed"), "the correction commits"); + assert!(v.uncertain().contains(&(0, 2)), "a corrected word stays uncertain by its original confidence"); + v.set_word_verified((0, 2), true); + assert!(v.uncertain().is_empty(), "a vouched-for word leaves the walk"); + assert!(!v.goto_uncertain(true), "nothing left: a no-op"); + // A page marked verified drops all its words from the walk. + v.set_word_verified((0, 2), false); + v.set_removed((0, 1), false); + v.set_page_verified(0, true); + assert!(v.uncertain().is_empty()); + } + + /// The review tools are guarded: outside ReviewReady they are ignored, not acted on. + #[test] + fn guards_ignore_review_actions_outside_reviewready() { + let mut v = OcrVerify::default(); + assert_eq!(v.phase, Phase::Idle); + assert!(v.can_run(), "Run is enabled in Idle"); + assert!(!v.reject(), "Reject is ignored in Idle"); + assert!(!v.set_removed((0, 0), true)); + assert!(!v.correct((0, 0), "x")); + assert!(!v.set_page_verified(0, true)); + assert!(!v.reject_page(0)); + let mut s = Session::new(); + assert_eq!(v.accept(&mut s).expect("ignored, not an error"), 0); + v.phase = Phase::Saving; + assert!(!v.can_run(), "Run is disabled while saving"); + v.phase = Phase::RecoverableError; + assert!(v.can_run(), "Run is enabled again after a recoverable error"); + } + + /// Accept: zero words left is a typed failure; a real review is applied as ONE undoable + /// step; a stale session is refused with its reason and the edits are kept. + #[test] + fn accept_zero_words_fails_and_a_live_review_applies_once() { + let mut s = Session::new(); + let (id, mut v) = loaded(&mut s, &[Some(95.0), Some(40.0)]); + v.set_removed((0, 0), true); + v.set_removed((0, 1), true); + assert!(v.accept(&mut s).is_err(), "zero words: a typed failure"); + assert_eq!(v.phase, Phase::ReviewReady, "the failure keeps the review"); + let live = v.live_pages(); + assert!(live[0].words.is_empty(), "removed words are not written"); + // Restore one word: the accept applies the whole review in one step. + v.set_removed((0, 0), false); + let n = v.accept(&mut s).expect("the accept applies"); + assert_eq!(n, 1); + assert_eq!(s.get(id).expect("open").can_undo(), Some("Recognize text"), "one undoable step"); + assert_eq!(v.phase, Phase::Idle, "a successful accept ends the review"); + // A second accept of the same (now spent) review has nothing to write. + let mut v2 = OcrVerify::default(); + let job = s.ocr_job(id, &[0], OcrSettings::default()).expect("fixture"); + v2.load( + id, + job.guard.clone(), + vec![OcrPage { already_on_page: 0, page: 0, words: vec![word("w", Some(95.0))], skipped: None, notes: Vec::new() }], + ); + v2.set_removed((0, 0), true); + assert!(v2.accept(&mut s).is_err()); + } + + /// Accept refuses a stale session: the document was edited since the job started, the + /// refusal names the reason, and the review edits survive. + #[test] + fn accept_refuses_a_stale_session_with_a_named_reason() { + let mut s = Session::new(); + let (id, mut v) = loaded(&mut s, &[Some(95.0)]); + assert!(v.correct((0, 0), "Corrected"), "the correction commits"); + // The document changed since the job started: the result no longer applies. + s.apply(id, pdfcraft_engine::Edit::RotatePages { pages: vec![0], degrees: 90 }).expect("fixture"); + let why = v.accept(&mut s).expect_err("the stale result is refused"); + assert!(why.contains("changed"), "{why}"); + assert_eq!(v.phase, Phase::RecoverableError, "Run is enabled again"); + assert_eq!(v.text_of((0, 0)), Some("Corrected"), "the edits are kept"); + } + + /// "Accept entire page" applies just that page and a later whole-review Accept never + /// writes it twice; "Reject entire page" clears the page (undoable in the panel). + #[test] + fn accept_page_then_accept_does_not_double_write() { + let mut s = Session::new(); + let (_, mut v) = loaded(&mut s, &[Some(95.0), Some(95.0)]); + v.pages = vec![OcrPage { + already_on_page: 0, + page: 0, + words: vec![word("one", Some(95.0)), word("two", Some(95.0))], + skipped: None, + notes: Vec::new(), + }]; + let n = v.accept_page(&mut s, 0).expect("the page applies"); + assert_eq!(n, 2); + assert!(v.review.applied_pages.contains(&0)); + // Reject entire page: the words clear (undoable), then come back. + assert!(v.reject_page(0)); + assert!(v.page(0).expect("page").words.is_empty()); + assert!(v.undo(), "the clear is undoable"); + assert_eq!(v.page(0).expect("page").words.len(), 2); + // A zero-word page cannot be accepted. + v.reject_page(0); + assert!(v.accept_page(&mut s, 0).is_err(), "zero words on the page: a typed failure"); + } + + /// The text the document's page `page` carries now, as the renderer extracts it. + fn page_text(s: &Session, id: DocId, page: usize) -> String { + let doc = s.get(id).expect("open"); + let mut r = pdfcraft_render::PageRenderer::new(doc.bytes.clone(), pdfcraft_render::RenderConfig::default()); + let request = pdfcraft_render::RenderRequest { page, kind: pdfcraft_render::RequestKind::Text, scale: 1.0, ..Default::default() }; + r.render(request).text.map(|t| t.plain_text()).unwrap_or_default() + } + + /// A blank document of `pages` pages, open, with a review of one word list per page loaded + /// against a real job's guard. + fn blank_review(s: &mut Session, words: Vec>) -> (DocId, OcrVerify) { + let blank = s.create_blank(612.0, 792.0, words.len()).expect("fixture"); + let id = s.open("blank.pdf", None, blank, None).expect("fixture"); + let job = s.ocr_job(id, &[], OcrSettings::default()).expect("fixture"); + let pages = words + .into_iter() + .enumerate() + .map(|(page, words)| OcrPage { already_on_page: 0, page, words, skipped: None, notes: Vec::new() }) + .collect(); + let mut v = OcrVerify::default(); + v.load(id, job.guard.clone(), pages); + (id, v) + } + + /// Pages accepted one at a time, then the whole review: each page-accept is the document's + /// newest edit, so the next one (and the final Accept) still applies, and every page is + /// written exactly once. Before the guard was refreshed, the second page-accept was refused + /// as "the document changed since recognition started" and the review was stuck. + #[test] + fn accepting_pages_one_by_one_then_the_review_applies_every_page_once() { + let mut s = Session::new(); + let (id, mut v) = blank_review(&mut s, vec![vec![word("one", Some(95.0))], vec![word("two", Some(95.0))], vec![word("three", Some(95.0))]]); + assert_eq!(v.accept_page(&mut s, 0), Ok(1)); + assert_eq!(v.accept_page(&mut s, 1), Ok(1), "the second page applies after the first"); + assert_eq!(v.phase, Phase::ReviewReady); + assert_eq!(v.accept(&mut s), Ok(1), "the final Accept writes only the page not yet applied"); + assert_eq!(v.phase, Phase::Idle, "the review ended"); + for (page, text) in ["one", "two", "three"].into_iter().enumerate() { + let on_page = page_text(&s, id, page); + assert_eq!(on_page.matches(text).count(), 1, "page {page} written exactly once: {on_page:?}"); + } + } + + /// A page already applied ("Accept entire page") is never applied again: a second + /// "Accept entire page" on it writes nothing (the OCR layer is not appended twice), and + /// corrections made after the page-accept are kept in the review. + #[test] + fn accepting_a_page_twice_writes_it_once_and_keeps_later_corrections() { + let mut s = Session::new(); + let (id, mut v) = blank_review(&mut s, vec![vec![word("one", Some(95.0))], vec![word("two", Some(95.0))]]); + assert_eq!(v.accept_page(&mut s, 0), Ok(1), "the first accept applies the page"); + // The reviewer corrects a word on the applied page afterwards. + assert!(v.correct((0, 0), "uno"), "the correction commits"); + // A second "Accept entire page" on the same page writes nothing. + assert_eq!(v.accept_page(&mut s, 0), Ok(0), "already applied: ignored, not applied twice"); + let text = page_text(&s, id, 0); + assert_eq!(text.matches("one").count(), 1, "the page was written exactly once: {text:?}"); + // The later correction is kept (not lost by the refusal), and the final Accept writes + // only the page not yet applied. + assert_eq!(v.text_of((0, 0)), Some("uno")); + assert_eq!(v.accept(&mut s), Ok(1), "the final Accept writes only page 2"); + let text = page_text(&s, id, 0); + assert_eq!(text.matches("one").count() + text.matches("uno").count(), 1, "still written once: {text:?}"); + assert_eq!(page_text(&s, id, 1).matches("two").count(), 1); + } + + /// The refreshed guard still catches an edit made elsewhere between two page-accepts: the + /// next page is refused with the reason, nothing is written, and the review is kept. + #[test] + fn an_outside_edit_between_page_accepts_is_still_refused() { + let mut s = Session::new(); + let (id, mut v) = blank_review(&mut s, vec![vec![word("one", Some(95.0))], vec![word("two", Some(95.0))]]); + assert_eq!(v.accept_page(&mut s, 0), Ok(1)); + s.apply(id, pdfcraft_engine::Edit::RotatePages { pages: vec![0], degrees: 90 }).expect("an unrelated edit"); + let why = v.accept_page(&mut s, 1).expect_err("stale after an outside edit"); + assert!(why.contains("changed since recognition started"), "{why}"); + assert_eq!(v.phase, Phase::RecoverableError); + assert_eq!(v.page(1).expect("page").words.len(), 1, "the review is kept"); + assert!(page_text(&s, id, 1).trim().is_empty(), "nothing was written to page 2"); + } + + /// A word at `x` on the first line of a US Letter page, with a Low confidence. + fn word_at(text: &str, x: f64) -> PlacedWord { + PlacedWord { text: text.into(), origin: [x, 700.0], across: [40.0, 0.0], up: [0.0, 12.0], confidence: Some(40.0), source: "ocrs".into() } + } + + /// Whole-page re-read: review marks are positional, so the page's marks go with the old + /// words (a struck-out slot must not strike the new word there), a correction of an old + /// word no longer shows on the new one, and the undo restores words and marks exactly. + #[test] + fn rereading_a_page_clears_its_marks_and_undo_restores_them() { + let mut s = Session::new(); + let (_, mut v) = blank_review(&mut s, vec![vec![word_at("keep", 72.0), word_at("gone", 300.0), word_at("struck", 400.0)]]); + assert!(v.set_removed((0, 2), true)); + assert!(v.set_word_verified((0, 0), true)); + assert!(v.correct((0, 1), "fixed")); + let marks = (v.review.removed.clone(), v.review.verified.clone()); + assert!(v.replace_page_words(0, vec![word_at("a", 72.0), word_at("b", 300.0), word_at("c", 400.0)])); + assert!(v.review.removed.is_empty() && v.review.verified.is_empty(), "the old marks are gone"); + assert!(!v.is_corrected((0, 1)), "the correction belonged to the old word list"); + let texts: Vec<_> = v.live_pages()[0].words.iter().map(|w| w.text.clone()).collect(); + assert_eq!(texts, ["a", "b", "c"], "no new word is struck out by an old mark"); + assert!(v.undo()); + assert_eq!((v.review.removed.clone(), v.review.verified.clone()), marks, "the undo restores the marks"); + assert!(v.is_corrected((0, 1))); + let texts: Vec<_> = v.live_pages()[0].words.iter().map(|w| w.text.clone()).collect(); + assert_eq!(texts, ["keep", "fixed"], "the struck word stays out"); + } + + /// A blank two-page-shaped review inside a real app (the document open, Run set to go + /// inline), one word list per page, against a real job's guard. + fn app_review(words: Vec>) -> (PdfCraftApp, DocId) { + let blank = Session::new().create_blank(612.0, 792.0, words.len()).expect("fixture").to_vec(); + let mut app = PdfCraftApp::new(); + app.run_inline = true; + app.open_bytes("blank.pdf", None, blank).expect("fixture"); + let id = app.active_ids().expect("open").1; + let job = app.session.ocr_job(id, &[], OcrSettings::default()).expect("fixture"); + let pages = words + .into_iter() + .enumerate() + .map(|(page, words)| OcrPage { already_on_page: 0, page, words, skipped: None, notes: Vec::new() }) + .collect(); + app.ocr_verify.load(id, job.guard.clone(), pages); + (app, id) + } + + /// The status the tests see, from the app's last toast. + fn toast(app: &mut PdfCraftApp) -> String { + app.toast.take().map(|t| t.0).unwrap_or_default() + } + + /// A region re-read whose worker has finished with `result`, already asked for. + fn finished_reread(region: Option<[f32; 4]>, result: Result, String>) -> Reread { + Reread { page: 0, region, progress: std::sync::Arc::new(Mutex::new(RereadProgress { cancel: false, result: Some(result) })) } + } + + /// A page re-read runs OFF the UI thread: start_reread returns with the reading in flight + /// (Run is refused while it is out), and the worker's result is applied by the next poll. + /// No OCR models are needed: the engine fails fast here, which still walks the whole + /// off-thread path — and a failed reading keeps the words. + #[test] + fn a_page_reread_runs_off_the_ui_thread_and_poll_applies_it() { + let (mut app, _) = app_review(vec![vec![word("one", Some(95.0))]]); + app.ocr_draft.engine = EngineChoice::Ocrs; // an ocrs-only job never probes tesseract + app.run_inline = false; // the point of the test: a real worker thread + with_state(&mut app, |v, app| v.start_reread(app, 0, None)); + assert!(app.ocr_verify.reread.is_some(), "the reading is in flight"); + assert!(app.ocr_verify.message.contains("Re-reading"), "{}", app.ocr_verify.message); + // A full run waits for the re-read. + with_state(&mut app, OcrVerify::start_run); + assert_eq!(app.ocr_verify.phase, Phase::ReviewReady, "Run is refused while a re-read is out"); + // Wait for the worker (it fails fast without models), then apply its result. + let progress = app.ocr_verify.reread.as_ref().expect("in flight").progress.clone(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + while progress.lock().expect("progress").result.is_none() && std::time::Instant::now() < deadline { + std::thread::sleep(std::time::Duration::from_millis(20)); + } + assert!(progress.lock().expect("progress").result.is_some(), "the worker finished"); + with_state(&mut app, OcrVerify::poll_reread); + assert!(app.ocr_verify.reread.is_none(), "the re-read is over"); + assert_eq!(app.ocr_verify.phase, Phase::ReviewReady, "the review survives a failed re-read"); + assert_eq!(app.ocr_verify.page(0).expect("page").words.len(), 1, "the words survive"); + assert!(!toast(&mut app).is_empty(), "the failure is said"); + } + + /// A re-read that finds nothing is a typed failure: the words stay, the review stays. + #[test] + fn a_reread_that_finds_nothing_keeps_the_words() { + let (mut app, _) = app_review(vec![vec![word("one", Some(95.0))]]); + app.ocr_verify.reread = Some(finished_reread(None, Err("the re-read found no text; the existing words are unchanged".into()))); + with_state(&mut app, OcrVerify::poll_reread); + assert!(app.ocr_verify.reread.is_none(), "the re-read is over"); + assert_eq!(app.ocr_verify.phase, Phase::ReviewReady); + assert_eq!(app.ocr_verify.page(0).expect("page").words.len(), 1, "the words survive the failed re-read"); + assert!(toast(&mut app).contains("found no text")); + } + + /// Cancel asks the worker to stop and discards the reading when it lands anyway: the + /// page's words stay as they were. + #[test] + fn a_cancelled_reread_is_discarded_and_keeps_the_words() { + let (mut app, _) = app_review(vec![vec![word("one", Some(95.0))]]); + app.ocr_verify.reread = Some(Reread { page: 0, region: None, progress: std::sync::Arc::new(Mutex::new(RereadProgress::default())) }); + with_state(&mut app, |v, _| v.cancel_reread()); + assert!(app.ocr_verify.reread.as_ref().expect("in flight").progress.lock().expect("progress").cancel, "the cancel reached the worker"); + // The worker finishes anyway; its (late) reading is discarded. + app.ocr_verify.reread.as_ref().expect("in flight").progress.lock().expect("progress").result = Some(Ok(vec![word("late", Some(95.0))])); + with_state(&mut app, OcrVerify::poll_reread); + assert!(app.ocr_verify.reread.is_none()); + let words: Vec<_> = app.ocr_verify.page(0).expect("page").words.iter().map(|w| w.text.clone()).collect(); + assert_eq!(words, ["one"], "the late reading is discarded"); + assert!(app.ocr_verify.message.contains("cancelled"), "{}", app.ocr_verify.message); + } + + /// A re-read of a document that changed since the review began is refused: the reading may + /// no longer fit the page it was read from, so the words stay and the reason is said. + #[test] + fn a_reread_of_a_changed_document_is_refused() { + let (mut app, id) = app_review(vec![vec![word("one", Some(95.0))], vec![word("two", Some(95.0))]]); + app.ocr_verify.reread = Some(finished_reread(None, Ok(vec![word("late", Some(95.0))]))); + app.ocr_verify.current = 1; + app.session.apply(id, pdfcraft_engine::Edit::RotatePages { pages: vec![1], degrees: 90 }).expect("an edit"); + with_state(&mut app, OcrVerify::poll_reread); + let words: Vec<_> = app.ocr_verify.page(1).expect("page").words.iter().map(|w| w.text.clone()).collect(); + assert_eq!(words, ["two"], "the late reading is not applied"); + assert!(toast(&mut app).contains("changed"), "the refusal names its reason"); + } + + /// Region re-read (the worker's reading applied by the poll): the words outside the + /// rectangle stay and their marks follow them to their new positions; the replaced word's + /// marks go; the first NEW word is the one selected (it follows the kept words); undo + /// restores everything. + #[test] + fn rereading_a_region_remaps_marks_and_selects_the_first_new_word() { + let (mut app, _) = app_review(vec![vec![word_at("keep", 72.0), word_at("gone", 300.0), word_at("struck", 400.0)]]); + // The drawn region and the worker's reading of it: one word, "new". + let region = [290.0, 70.0, 350.0, 100.0]; + let new_word = PlacedWord { + text: "new".into(), + origin: [300.0, 700.0], + across: [40.0, 0.0], + up: [0.0, 12.0], + confidence: Some(95.0), + source: "ocrs".into(), + }; + { + let v = &mut app.ocr_verify; + assert!(v.set_removed((0, 2), true)); + assert!(v.set_word_verified((0, 0), true)); + assert!(v.set_word_verified((0, 1), true)); + v.region = Some(region); + v.reread = Some(finished_reread(Some(region), Ok(vec![new_word]))); + } + let marks = { + let v = &app.ocr_verify; + (v.review.removed.clone(), v.review.verified.clone()) + }; + // "gone" spans x 300..340 and y 700..712 in user space: view y 80..92 on the 792 pt page. + with_state(&mut app, OcrVerify::poll_reread); + let v = &mut app.ocr_verify; + assert!(v.reread.is_none(), "the re-read is over"); + let texts: Vec<_> = v.page(0).expect("page").words.iter().map(|w| w.text.clone()).collect(); + assert_eq!(texts, ["keep", "struck", "new"]); + assert_eq!(v.selected, Some((0, 2)), "the first new word is selected"); + assert!(v.is_removed((0, 1)), "the struck word's mark moved with it"); + assert!(!v.is_removed((0, 2)), "the new word is not struck out"); + assert!(v.is_verified((0, 0)) && !v.is_verified((0, 1)) && !v.is_verified((0, 2)), "{:?}", v.review.verified); + let live: Vec<_> = v.live_pages()[0].words.iter().map(|w| w.text.clone()).collect(); + assert_eq!(live, ["keep", "new"], "the struck word is never written, the new one is"); + assert!(v.undo()); + assert_eq!((v.review.removed.clone(), v.review.verified.clone()), marks); + let texts: Vec<_> = v.page(0).expect("page").words.iter().map(|w| w.text.clone()).collect(); + assert_eq!(texts, ["keep", "gone", "struck"]); + } + + /// The editable output: refuses zero words, keeps the open document untouched, and names + /// the new file `_ocr.pdf`. + #[test] + fn editable_output_refuses_zero_words_and_never_touches_the_source() { + let mut s = Session::new(); + let (id, mut v) = loaded(&mut s, &[Some(95.0)]); + let before = s.get(id).expect("open").bytes.clone(); + let (bytes, name) = v.editable_output(&s).expect("the output builds"); + assert!(name.ends_with("_ocr.pdf"), "{name}"); + assert!(std::sync::Arc::ptr_eq(&s.get(id).expect("open").bytes, &before), "the source is untouched"); + assert!(!bytes.is_empty()); + v.set_removed((0, 0), true); + assert!(v.editable_output(&s).is_err(), "zero words: refused"); + assert_eq!(v.phase, Phase::ReviewReady); + } + + /// The editable output is a new file made from words read earlier: when the document changed + /// since (a redaction, say), it is refused with the reason, as Accept in searchable mode is. + #[test] + fn editable_output_refuses_a_document_that_changed_since_the_review_started() { + let mut s = Session::new(); + let (id, v) = loaded(&mut s, &[Some(95.0)]); + assert!(v.editable_output(&s).is_ok(), "an untouched document is fine"); + s.apply(id, pdfcraft_engine::Edit::RotatePages { pages: vec![0], degrees: 90 }).expect("an edit"); + let err = v.editable_output(&s).expect_err("a changed document is refused"); + assert!(err.contains("changed since recognition started"), "{err}"); + } + + /// The editable save is the Saving state: the file lands and the review ends (Idle); a + /// failed write returns to ReviewReady with the edits kept, and a cancelled or unanswered + /// save dialog never touches the review either — Accept can simply be tried again. + #[test] + fn editable_save_lands_and_a_failed_or_cancelled_one_keeps_the_review() { + let mut app = PdfCraftApp::new(); + app.run_inline = true; + let text = app.session.create_from_text("t", "Recognize these scanned words").expect("fixture"); + let id = app.session.open("t.pdf", None, text, None).expect("fixture"); + let job = app.session.ocr_job(id, &[0], OcrSettings::default()).expect("fixture"); + app.ocr_verify.load( + id, + job.guard.clone(), + vec![OcrPage { already_on_page: 0, page: 0, words: vec![word("w", Some(95.0))], skipped: None, notes: Vec::new() }], + ); + assert!(app.ocr_verify.correct((0, 0), "Edited"), "the correction commits"); + app.ocr_draft.output_mode = OutputMode::EditableText; + // A destination that cannot be written (the "folder" is a regular file): the write + // fails, the review stays in ReviewReady with the edit, Accept is enabled again. + let blocker = std::env::temp_dir().join(format!("pdfcraft-ocr-save-block-{}", std::process::id())); + std::fs::write(&blocker, b"not a folder").expect("fixture"); + app.export_dir_override = Some(blocker.to_string_lossy().into_owned()); + with_state(&mut app, do_accept); + assert_eq!(app.ocr_verify.phase, Phase::ReviewReady, "a failed write keeps the review"); + assert_eq!(app.ocr_verify.text_of((0, 0)), Some("Edited"), "the edits are kept"); + // A cancelled save dialog (the override answers with nothing): nothing is written, + // nothing changes. + app.export_dir_override = None; + app.pick_override = Some(Vec::new()); + with_state(&mut app, do_accept); + app.process_picked(); + assert_eq!(app.ocr_verify.phase, Phase::ReviewReady, "a cancelled save keeps the review"); + assert_eq!(app.ocr_verify.text_of((0, 0)), Some("Edited")); + assert_eq!(app.ocr_verify.doc, Some(id), "the review still belongs to its document"); + // A real destination: the file lands under `_ocr.pdf` and the review ends. + let dir = std::env::temp_dir().join(format!("pdfcraft-ocr-save-out-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("fixture"); + app.pick_override = Some(vec![dir.join("t_ocr.pdf").to_string_lossy().into_owned()]); + with_state(&mut app, do_accept); + app.process_picked(); + assert!(dir.join("t_ocr.pdf").exists(), "the editable copy is written"); + assert_eq!(app.ocr_verify.phase, Phase::Idle, "a successful save ends the review"); + let _ = std::fs::remove_file(&blocker); + let _ = std::fs::remove_dir_all(&dir); + } + + /// Region re-OCR refuses an empty, inverted or out-of-page rectangle BEFORE running: no + /// job goes out, no engine is probed, and the existing words are intact. + #[test] + fn reread_refuses_an_empty_region_before_running() { + let (mut app, id) = app_review(vec![vec![word("one", Some(95.0))]]); + app.ocr_draft.dpi = 72; // one pixel per point, so the arithmetic below is exact + // An empty rectangle never reaches the engine. + with_state(&mut app, |v, app| v.start_reread(app, 0, Some([10.0, 10.0, 10.0, 60.0]))); + assert!(app.ocr_verify.reread.is_none(), "nothing runs"); + assert_eq!(app.ocr_verify.page(0).expect("page").words.len(), 1, "the existing words are intact"); + assert!(toast(&mut app).contains("empty or outside")); + // Nor does one wholly outside the page. + with_state(&mut app, |v, app| v.start_reread(app, 0, Some([-50.0, -50.0, -1.0, -1.0]))); + assert!(app.ocr_verify.reread.is_none()); + assert_eq!(id, app.ocr_verify.doc.expect("the review keeps its document")); + } + + /// Review edits are undoable within the panel, in order. + #[test] + fn review_edits_undo_in_order() { + let mut s = Session::new(); + let (_, mut v) = loaded(&mut s, &[Some(40.0)]); + assert!(v.correct((0, 0), "Fixed"), "commits"); + assert_eq!(v.text_of((0, 0)), Some("Fixed")); + assert!(v.is_corrected((0, 0))); + assert!(v.add_to_dictionary("Fixed", "en")); + assert!(v.review.dictionary.contains("fixed")); + assert!(v.set_removed((0, 0), true)); + assert!(v.is_removed((0, 0))); + // Undo in order: the removal, then the dictionary add, then the correction. + assert!(v.undo()); + assert!(!v.is_removed((0, 0))); + assert!(v.undo()); + assert!(!v.review.dictionary.contains("fixed")); + assert!(v.undo()); + assert_eq!(v.text_of((0, 0)), Some("w")); + assert!(!v.undo(), "nothing left to undo"); + } + + /// Corrections commit on Enter; an empty (after trim) commit removes the word; the box + /// never moves. + #[test] + fn correction_commits_and_clearing_removes() { + let mut s = Session::new(); + let (_, mut v) = loaded(&mut s, &[Some(95.0)]); + let box_before = v.word((0, 0)).map(|w| (w.origin, w.across, w.up)); + assert!(v.correct((0, 0), " Fixed "), "commits"); + assert_eq!(v.text_of((0, 0)), Some(" Fixed "), "the reviewer's text is kept verbatim"); + assert_eq!(v.word((0, 0)).map(|w| (w.origin, w.across, w.up)), box_before, "the box never moves"); + assert!(v.correct((0, 0), " "), "an empty commit removes"); + assert!(v.is_removed((0, 0)), "clearing the text removes the word"); + } + + /// Suggestions: dictionary words and the page's own words within edit distance 2, nearest + /// first, at most 5, never the word itself. + #[test] + fn suggestions_are_near_dictionary_and_page_words() { + let mut dictionary = BTreeSet::new(); + dictionary.insert("recognize".into()); + dictionary.insert("these".into()); + let page_words = vec![word("recognise", None), word("scanned", None), word("words", None), word("completely", None)]; + let sugg = suggestions("recgnise", &dictionary, &page_words); + assert_eq!(sugg.first().map(String::as_str), Some("recognise"), "distance 1 beats distance 2"); + assert!(sugg.contains(&"recognize".to_string()), "distance 2 candidates are in"); + assert!(!sugg.contains(&"recgnise".to_string()), "never the word itself"); + assert!(sugg.len() <= 5); + assert!(suggestions("xyzzyx", &dictionary, &page_words).is_empty(), "nothing within distance 2"); + assert!(edit_distance_within_2("kitten", "sitting", 2).is_none(), "distance 3 is out"); + assert_eq!(edit_distance_within_2("kitten", "mitten", 2), Some(1)); + } + + /// The page list's data: a page marked verified, a skipped page, and the verified + /// fraction the info strip shows. + #[test] + fn verified_fraction_and_word_count_follow_the_review() { + let mut s = Session::new(); + let (_, mut v) = loaded(&mut s, &[Some(95.0), Some(95.0), Some(95.0)]); + assert!((v.verified_fraction() - 0.0).abs() < f32::EPSILON); + v.set_word_verified((0, 0), true); + assert!((v.verified_fraction() - 1.0 / 3.0).abs() < 0.01); + v.set_page_verified(0, true); + assert!((v.verified_fraction() - 1.0).abs() < f32::EPSILON, "a verified page vouches for all its words"); + assert_eq!(v.word_count(), 3); + v.set_removed((0, 1), true); + assert_eq!(v.word_count(), 2, "removed words are not counted"); + // Reject clears the words, the session, the selection and the flags; back to Idle. + assert!(v.reject()); + assert_eq!(v.phase, Phase::Idle); + assert!(v.pages.is_empty() && v.selected.is_none() && v.review.verified_pages.is_empty()); + assert!(!v.reject(), "Reject outside ReviewReady is ignored"); + } + + /// The suspects' count the info strip shows: live Low-band words that are not in the + /// dictionary. + #[test] + fn suspect_count_follows_bands_and_dictionary() { + let mut s = Session::new(); + let (_, mut v) = loaded(&mut s, &[Some(95.0), Some(40.0), Some(69.0)]); + assert_eq!(v.suspect_count(), 2, "69 is Low, 40 is Low, 95 is High"); + v.add_to_dictionary("w", "en"); + assert_eq!(v.suspect_count(), 0, "dictionary words do not ask for review"); + } + + #[test] + fn dictionary_uses_recognition_language_instead_of_interface_locale() { + let mut session = Session::new(); + let (_, mut v) = loaded(&mut session, &[Some(40.0)]); + let locale = crate::i18n::current().code(); + let language = if locale == "en" { "fr" } else { "en" }; + let dir = std::env::temp_dir().join(format!("pdfcraft-ocr-dictionary-language-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join(format!("ocr-dictionary-{locale}.txt")), "wrong\n").unwrap(); + let path = dir.join(format!("ocr-dictionary-{language}.txt")); + std::fs::write(&path, "recognized\n").unwrap(); + v.dictionary_dir = Some(dir.clone()); + v.dictionary_language = language.into(); + v.load_dictionary_pref(); + assert!(v.review.dictionary.contains("recognized")); + assert!(!v.review.dictionary.contains("wrong")); + assert!(v.add_to_dictionary("Added", language)); + assert!(std::fs::read_to_string(path).unwrap().contains("added\n")); + assert_eq!(std::fs::read_to_string(dir.join(format!("ocr-dictionary-{locale}.txt"))).unwrap(), "wrong\n"); + } + + #[test] + fn dictionary_limit_and_normalization_failures_preserve_file_and_review() { + let mut session = Session::new(); + let (_, mut v) = loaded(&mut session, &[Some(40.0)]); + let dir = std::env::temp_dir().join(format!("pdfcraft-ocr-dictionary-limits-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("ocr-dictionary-en.txt"); + let full: String = (0..65536).map(|i| format!("word{i}\n")).collect(); + std::fs::write(&path, &full).unwrap(); + v.dictionary_dir = Some(dir); + v.load_dictionary_pref(); + let before = v.review.dictionary.clone(); + assert!(!v.add_to_dictionary("extra", "en")); + assert!(v.message.contains("65536"), "{}", v.message); + assert_eq!(v.review.dictionary, before); + assert_eq!(std::fs::read_to_string(&path).unwrap(), full); + assert!(!v.add_to_dictionary(&"İ".repeat(512), "en")); + assert!(v.message.contains("normalized"), "{}", v.message); + assert_eq!(v.review.dictionary, before); + assert_eq!(std::fs::read_to_string(&path).unwrap(), full); + let full_bytes: String = (0..8192).map(|i| format!("{i:05}{}\n", "a".repeat(506))).collect(); + assert_eq!(full_bytes.len(), pdfcraft_engine::ocr::dictionary::MAX_BYTES); + std::fs::write(&path, &full_bytes).unwrap(); + v.load_dictionary_pref(); + let before = v.review.dictionary.clone(); + assert!(!v.add_to_dictionary("extra", "en")); + assert!(v.message.contains("4 MiB"), "{}", v.message); + assert_eq!(v.review.dictionary, before); + assert_eq!(std::fs::read_to_string(&path).unwrap(), full_bytes); + // A corrupt file is not silently treated as a fresh, empty dictionary. + std::fs::write(&path, [0xff]).unwrap(); + v.load_dictionary_pref(); + assert_eq!(v.review.dictionary, before); + assert!(v.message.contains("OCR dictionary")); + } + + #[test] + fn dictionary_undo_persists_and_failed_undo_keeps_history() { + let mut session = Session::new(); + let (_, mut v) = loaded(&mut session, &[Some(40.0)]); + let dir = std::env::temp_dir().join(format!("pdfcraft-ocr-dictionary-undo-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("ocr-dictionary-en.txt"); + std::fs::write(&path, "original\n").unwrap(); + v.dictionary_dir = Some(dir); + v.load_dictionary_pref(); + std::fs::write(&path, "original\nexternal\n").unwrap(); + assert!(!v.add_to_dictionary("external", "en"), "an externally added word creates no undo entry"); + assert!(v.review.dictionary.contains("external")); + assert!(v.add_to_dictionary("temporary", "en")); + assert!(v.undo()); + assert!(!std::fs::read_to_string(&path).unwrap().contains("temporary")); + assert!(v.add_to_dictionary("later", "en")); + assert!(!v.review.dictionary.contains("temporary")); + let before = v.review.dictionary.clone(); + std::fs::write(&path, [0xff]).unwrap(); + assert!(!v.undo()); + assert_eq!(v.review.dictionary, before); + assert_eq!(std::fs::read(&path).unwrap(), [0xff]); + std::fs::write(&path, "original\nexternal\nlater\n").unwrap(); + assert!(v.undo(), "failed undo keeps its operation for retry"); + assert_eq!(std::fs::read_to_string(&path).unwrap(), "external\noriginal\n"); + assert_eq!(v.review.dictionary, std::collections::BTreeSet::from(["external".into(), "original".into()])); + } + + #[test] + fn rereading_keeps_captured_language_when_the_draft_changes() { + let mut session = Session::new(); + let (_, mut v) = loaded(&mut session, &[Some(40.0)]); + v.dictionary_language = "ja".into(); + let draft = super::ocr_ui::OcrDraft { language: "fr".into(), ..Default::default() }; + let settings = v.reread_settings(&draft); + assert_eq!(settings.language, "ja"); + assert_eq!(settings.dpi, draft.settings().dpi); + assert!(dictionary_path(std::path::Path::new("prefs"), "../outside").is_err()); + } + + #[test] + fn finishing_or_rejecting_a_review_keeps_dictionary_configuration() { + let mut session = Session::new(); + let (_, mut v) = loaded(&mut session, &[Some(40.0)]); + let dir = PathBuf::from("configured-dictionaries"); + v.dictionary_dir = Some(dir.clone()); + assert!(v.reject()); + assert_eq!(v.dictionary_dir.as_ref(), Some(&dir)); + v.finish(); + assert_eq!(v.dictionary_dir.as_ref(), Some(&dir)); + } + + /// The drawn region is stored in view space, so its outline is that rectangle scaled onto + /// the pane: not mirrored on an upright page, and in place on a rotated, cropped one. + #[test] + fn the_region_outline_is_where_it_was_drawn() { + let img = egui::Rect::from_min_size(egui::pos2(10.0, 20.0), egui::vec2(306.0, 396.0)); + let upright = pdfcraft_render::PageInfo { width: 612.0, height: 792.0, label: "1".into(), crop: [0.0, 0.0, 612.0, 792.0], rotation: 0 }; + let r = pane_region([100.0, 50.0, 200.0, 150.0], &upright, img).expect("on the page"); + assert_eq!((r.left(), r.top(), r.right(), r.bottom()), (60.0, 45.0, 110.0, 95.0), "top region stays at the top"); + let turned = pdfcraft_render::PageInfo { width: 792.0, height: 612.0, label: "1".into(), crop: [36.0, 36.0, 648.0, 828.0], rotation: 90 }; + let img = egui::Rect::from_min_size(egui::pos2(0.0, 0.0), egui::vec2(396.0, 306.0)); + let r = pane_region([100.0, 50.0, 200.0, 150.0], &turned, img).expect("on the page"); + assert_eq!((r.left(), r.top(), r.right(), r.bottom()), (50.0, 25.0, 100.0, 75.0)); + } +} diff --git a/crates/ui-egui/tests/it/ocr.rs b/crates/ui-egui/tests/it/ocr.rs index d7a5501987..c1a987fbb3 100644 --- a/crates/ui-egui/tests/it/ocr.rs +++ b/crates/ui-egui/tests/it/ocr.rs @@ -3,6 +3,7 @@ use egui_kittest::Harness; use egui_kittest::kittest::Queryable; +use pdfcraft_engine::ocr::{OcrPage, OcrSettings, PlacedWord}; use pdfcraft_engine::{AddedText, Edit, Session, export}; use pdfcraft_ui_egui::PdfCraftApp; @@ -127,7 +128,14 @@ fn an_ocr_word_opens_for_editing_and_stays_open() { app.open_bytes("scan.pdf", None, scan()).unwrap(); let id = app.views[0].id; let ph = app.session.get(id).unwrap().info.pages[0].height as f64; - let words = vec![PlacedWord { text: "Recognize".into(), origin: [72.0, ph - 100.0], across: [80.0, 0.0], up: [0.0, 14.0] }]; + let words = vec![PlacedWord { + text: "Recognize".into(), + origin: [72.0, ph - 100.0], + across: [80.0, 0.0], + up: [0.0, 14.0], + confidence: None, + source: "ocrs".into(), + }]; app.session.apply(id, Edit::AddOcrText { page: 0, words }).unwrap(); app.set_option("zoom", "100").unwrap(); assert!(app.execute("edit.edit_text")); @@ -348,3 +356,112 @@ fn only_the_edit_tool_recognizes_a_scanned_page() { h.run_steps(6); assert_eq!(h.state().session.get(id).unwrap().can_undo(), None); } + +/// A placed word the review can hold (its box sits where a real one would). +fn placed(text: &str, confidence: f32) -> PlacedWord { + PlacedWord { text: text.into(), origin: [40.0, 40.0], across: [50.0, 0.0], up: [0.0, 12.0], confidence: Some(confidence), source: "ocrs".into() } +} + +/// The OCR Verify screen (Scan & OCR ▸ Correct recognized text) in the real shell: every +/// control is live, the keyboard walks the uncertain words, the inspector removes and restores +/// through clicks, and Accept applies the review as one undoable step. The recognition is +/// loaded directly, so this runs without the OCR models. +#[test] +fn ocr_verify_screen_controls_keyboard_and_accept() { + let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).build_eframe(|_cc| { + let mut app = PdfCraftApp::new(); + app.open_bytes("scan.pdf", None, scan()).unwrap(); + app.run_inline = true; + app + }); + h.run_steps(4); + assert!(h.state_mut().execute("ocr.correct")); + h.run_steps(3); + // The screen owns the window and its controls are all present (no dead controls). + h.get_by_label("Run OCR"); + h.get_by_label("Reject"); + h.get_by_label("Next uncertain"); + h.get_by_label("Previous uncertain"); + h.get_by_label("Page verified"); + h.get_by_label("Exit"); + use pdfcraft_ui_egui::Phase; + assert_eq!(h.state().ocr_verify.phase, Phase::Idle); + // The guards hold in the real UI: Reject outside ReviewReady is ignored, not acted on. + h.get_by_label("Reject").click(); + h.run_steps(2); + assert_eq!(h.state().ocr_verify.phase, Phase::Idle); + // A recognition result, as the worker would deliver it. + { + let app = h.state_mut(); + let id = app.active_ids().unwrap().1; + let job = app.session.ocr_job(id, &[0], OcrSettings::default()).unwrap(); + let page = + OcrPage { already_on_page: 0, page: 0, words: vec![placed("Helo", 40.0), placed("wrold", 95.0)], skipped: None, notes: Vec::new() }; + app.ocr_verify.load(id, job.guard.clone(), vec![page]); + } + h.run_steps(3); + assert_eq!(h.state().ocr_verify.phase, Phase::ReviewReady); + // F8 walks to the Low-band word; the inspector shows it. + h.key_press(egui::Key::F8); + h.run_steps(2); + assert_eq!(h.state().ocr_verify.selected, Some((0, 0))); + h.get_by_label("Remove").click(); + h.run_steps(2); + assert!(h.state().ocr_verify.is_removed((0, 0)), "Remove strikes the word out"); + h.get_by_label("Restore").click(); + h.run_steps(2); + assert!(!h.state().ocr_verify.is_removed((0, 0)), "Restore brings it back"); + // Ctrl+T marks the page reviewed (the checkbox flips). + h.key_press_modifiers(egui::Modifiers::COMMAND, egui::Key::T); + h.run_steps(2); + assert!(h.state().ocr_verify.review.verified_pages.contains(&0), "Ctrl+T marks the page verified"); + h.key_press_modifiers(egui::Modifiers::COMMAND, egui::Key::T); + h.run_steps(2); + assert!(!h.state().ocr_verify.review.verified_pages.contains(&0)); + // Accept applies the whole review as ONE undoable step on the open document. + h.get_by_label("Accept").click(); + h.run_steps(3); + assert_eq!(h.state().ocr_verify.phase, Phase::Idle, "a successful accept ends the review"); + let app = h.state(); + let id = app.active_ids().unwrap().1; + assert_eq!(app.session.get(id).unwrap().can_undo(), Some("Recognize text")); + let toast = app.toast.clone().map(|t| t.0).unwrap_or_default(); + assert!(toast.contains("one undoable step"), "{toast}"); +} + +/// Ctrl+Tab moves the pane focus ring, and F8 with nothing to review is a no-op: the walk +/// never invents a selection. The recognition is loaded directly (no models needed). +#[test] +fn ocr_verify_screen_focus_ring_and_empty_walk() { + let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).build_eframe(|_cc| { + let mut app = PdfCraftApp::new(); + app.open_bytes("scan.pdf", None, scan()).unwrap(); + app.run_inline = true; + app + }); + h.run_steps(4); + assert!(h.state_mut().execute("ocr.correct")); + h.run_steps(3); + { + let app = h.state_mut(); + let id = app.active_ids().unwrap().1; + let job = app.session.ocr_job(id, &[0], OcrSettings::default()).unwrap(); + let page = OcrPage { already_on_page: 0, page: 0, words: vec![placed("Fine", 95.0)], skipped: None, notes: Vec::new() }; + app.ocr_verify.load(id, job.guard.clone(), vec![page]); + } + h.run_steps(3); + // Nothing asks for review: F8 selects nothing instead of wrapping into an error. + h.key_press(egui::Key::F8); + h.run_steps(2); + assert_eq!(h.state().ocr_verify.selected, None); + // Ctrl+Tab cycles the four panes; four steps come back to the start. + let start = h.state().ocr_verify.focus; + let mut moved = false; + for _ in 0..4 { + h.key_press_modifiers(egui::Modifiers::COMMAND, egui::Key::Tab); + h.run_steps(2); + moved |= h.state().ocr_verify.focus != start; + } + assert!(moved, "Ctrl+Tab moves the focus"); + assert_eq!(h.state().ocr_verify.focus, start, "the ring returns after the four panes"); +} diff --git a/crates/ui-egui/tests/ocr.rs b/crates/ui-egui/tests/ocr.rs new file mode 100644 index 0000000000..a0af321a01 --- /dev/null +++ b/crates/ui-egui/tests/ocr.rs @@ -0,0 +1,281 @@ +//! Scan & OCR ▸ Recognize text in the real shell (egui_kittest): the dialog, the run and its +//! result (skipped without the OCR models: `cargo xtask models`). + +use egui_kittest::Harness; +use egui_kittest::kittest::Queryable; +use pdfcraft_engine::ocr::{OcrPage, OcrSettings, PlacedWord}; +use pdfcraft_engine::{AddedText, Edit, Session, export}; +use pdfcraft_ui_egui::PdfCraftApp; + +/// A one-page PDF that is only a picture of a sentence. +fn scan() -> Vec { + let mut s = Session::new(); + let text = s.create_from_text("t", "Recognize these scanned words").unwrap(); + let id = s.open("text.pdf", None, text, None).unwrap(); + let png = export::Exporter::new(s.get(id).unwrap()).png(0, 150.0).unwrap(); + s.create_from_images(&[("scan.png".into(), png)]).unwrap().to_vec() +} + +#[test] +fn recognize_text_dialog_adds_searchable_text() { + let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).build_eframe(|_cc| { + let mut app = PdfCraftApp::new(); + app.set_option("language", "en").unwrap(); + app.open_bytes("scan.pdf", None, scan()).unwrap(); + app.run_inline = true; + app + }); + h.run_steps(4); + assert!(h.state_mut().execute("ocr.recognize")); + h.run_steps(2); + h.get_by_label("Document language"); + h.get_by_label("Output"); + if !pdfcraft_engine::ocr::available() { + eprintln!("skipped: OCR models not installed"); + return; + } + h.get_by_label("Recognize text").click(); + h.run_steps(3); + assert!(h.state().ocr_run.is_none(), "finished"); + let toast = h.state().toast.clone().unwrap().0; + assert!(toast.starts_with("Recognized ") && toast.contains("on 1 page"), "{toast}"); + let app = h.state(); + let id = app.active_ids().unwrap().1; + assert_eq!(app.session.get(id).unwrap().can_undo(), Some("Recognize text")); +} + +#[test] +fn recognize_text_in_multiple_files_writes_searchable_copies() { + if !pdfcraft_engine::ocr::available() { + eprintln!("skipped: OCR models not installed"); + return; + } + let dir = std::env::temp_dir().join(format!("pdfcraft-ocr-ui-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let mut app = PdfCraftApp::new(); + app.set_option("language", "en").unwrap(); + app.run_inline = true; + app.export_dir_override = Some(dir.to_string_lossy().into_owned()); + app.use_files(pdfcraft_ui_egui::FilePurpose::Ocr, vec![("one.pdf".into(), scan()), ("two.pdf".into(), scan())]); + assert!(app.ocr_batch.is_none(), "finished"); + let toast = app.toast.clone().unwrap().0; + assert!(toast.starts_with("Recognized ") && toast.ends_with("in 2 files"), "{toast}"); + for name in ["one.pdf", "two.pdf"] { + let bytes = std::fs::read(dir.join(name)).unwrap(); + assert!(bytes.len() > scan().len(), "an incremental update was appended"); + } +} + +/// A placed word the review can hold (its box sits where a real one would). +fn placed(text: &str, confidence: f32) -> PlacedWord { + PlacedWord { text: text.into(), origin: [40.0, 40.0], across: [50.0, 0.0], up: [0.0, 12.0], confidence: Some(confidence), source: "ocrs".into() } +} + +/// The OCR Verify screen (Scan & OCR ▸ Correct recognized text) in the real shell: every +/// control is live, the keyboard walks the uncertain words, the inspector removes and restores +/// through clicks, and Accept applies the review as one undoable step. The recognition is +/// loaded directly, so this runs without the OCR models. +#[test] +fn ocr_verify_screen_controls_keyboard_and_accept() { + let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).build_eframe(|_cc| { + let mut app = PdfCraftApp::new(); + app.open_bytes("scan.pdf", None, scan()).unwrap(); + app.run_inline = true; + app + }); + h.run_steps(4); + assert!(h.state_mut().execute("ocr.correct")); + h.run_steps(3); + // The screen owns the window and its controls are all present (no dead controls). + h.get_by_label("Run OCR"); + h.get_by_label("Reject"); + h.get_by_label("Next uncertain"); + h.get_by_label("Previous uncertain"); + h.get_by_label("Page verified"); + h.get_by_label("Exit"); + use pdfcraft_ui_egui::Phase; + assert_eq!(h.state().ocr_verify.phase, Phase::Idle); + // The guards hold in the real UI: Reject outside ReviewReady is ignored, not acted on. + h.get_by_label("Reject").click(); + h.run_steps(2); + assert_eq!(h.state().ocr_verify.phase, Phase::Idle); + // A recognition result, as the worker would deliver it. + { + let app = h.state_mut(); + let id = app.active_ids().unwrap().1; + let job = app.session.ocr_job(id, &[0], OcrSettings::default()).unwrap(); + let page = + OcrPage { already_on_page: 0, page: 0, words: vec![placed("Helo", 40.0), placed("wrold", 95.0)], skipped: None, notes: Vec::new() }; + app.ocr_verify.load(id, job.guard.clone(), vec![page]); + } + h.run_steps(3); + assert_eq!(h.state().ocr_verify.phase, Phase::ReviewReady); + // F8 walks to the Low-band word; the inspector shows it. + h.key_press(egui::Key::F8); + h.run_steps(2); + assert_eq!(h.state().ocr_verify.selected, Some((0, 0))); + h.get_by_label("Remove").click(); + h.run_steps(2); + assert!(h.state().ocr_verify.is_removed((0, 0)), "Remove strikes the word out"); + h.get_by_label("Restore").click(); + h.run_steps(2); + assert!(!h.state().ocr_verify.is_removed((0, 0)), "Restore brings it back"); + // Ctrl+T marks the page reviewed (the checkbox flips). + h.key_press_modifiers(egui::Modifiers::COMMAND, egui::Key::T); + h.run_steps(2); + assert!(h.state().ocr_verify.review.verified_pages.contains(&0), "Ctrl+T marks the page verified"); + h.key_press_modifiers(egui::Modifiers::COMMAND, egui::Key::T); + h.run_steps(2); + assert!(!h.state().ocr_verify.review.verified_pages.contains(&0)); + // Accept applies the whole review as ONE undoable step on the open document. + h.get_by_label("Accept").click(); + h.run_steps(3); + assert_eq!(h.state().ocr_verify.phase, Phase::Idle, "a successful accept ends the review"); + let app = h.state(); + let id = app.active_ids().unwrap().1; + assert_eq!(app.session.get(id).unwrap().can_undo(), Some("Recognize text")); + let toast = app.toast.clone().map(|t| t.0).unwrap_or_default(); + assert!(toast.contains("one undoable step"), "{toast}"); +} + +/// Ctrl+Tab moves the pane focus ring, and F8 with nothing to review is a no-op: the walk +/// never invents a selection. The recognition is loaded directly (no models needed). +#[test] +fn ocr_verify_screen_focus_ring_and_empty_walk() { + let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).build_eframe(|_cc| { + let mut app = PdfCraftApp::new(); + app.open_bytes("scan.pdf", None, scan()).unwrap(); + app.run_inline = true; + app + }); + h.run_steps(4); + assert!(h.state_mut().execute("ocr.correct")); + h.run_steps(3); + { + let app = h.state_mut(); + let id = app.active_ids().unwrap().1; + let job = app.session.ocr_job(id, &[0], OcrSettings::default()).unwrap(); + let page = OcrPage { already_on_page: 0, page: 0, words: vec![placed("Fine", 95.0)], skipped: None, notes: Vec::new() }; + app.ocr_verify.load(id, job.guard.clone(), vec![page]); + } + h.run_steps(3); + // Nothing asks for review: F8 selects nothing instead of wrapping into an error. + h.key_press(egui::Key::F8); + h.run_steps(2); + assert_eq!(h.state().ocr_verify.selected, None); + // Ctrl+Tab cycles the four panes; four steps come back to the start. + let start = h.state().ocr_verify.focus; + let mut moved = false; + for _ in 0..4 { + h.key_press_modifiers(egui::Modifiers::COMMAND, egui::Key::Tab); + h.run_steps(2); + moved |= h.state().ocr_verify.focus != start; + } + assert!(moved, "Ctrl+Tab moves the focus"); + assert_eq!(h.state().ocr_verify.focus, start, "the ring returns after the four panes"); +} + +/// #1028: "Also recognize pages that already have text" reads a picture on a page that has its own +/// text, without adding that text a second time. +#[test] +fn recognize_text_can_read_pictures_on_pages_with_text() { + let hybrid = { + let mut s = Session::new(); + let id = s.open("scan.pdf", None, std::sync::Arc::new(scan()), None).unwrap(); + let (w, h) = { + let p = &s.get(id).unwrap().info.pages[0]; + (p.width as f64, p.height as f64) + }; + let note = + AddedText { rect: [w * 0.1, h * 0.75, w * 0.9, h * 0.85], text: "Invoice reference 4711".into(), size: 18.0, ..Default::default() }; + s.apply(id, Edit::AddText { page: 0, text: note }).unwrap(); + s.save_bytes(id).unwrap().to_vec() + }; + let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).build_eframe(|_cc| { + let mut app = PdfCraftApp::new(); + app.set_option("language", "en").unwrap(); + app.open_bytes("hybrid.pdf", None, hybrid).unwrap(); + app.run_inline = true; + app + }); + h.run_steps(4); + assert!(h.state_mut().execute("ocr.recognize")); + h.run_steps(2); + assert!(h.state().ocr_draft.settings().skip_text_pages, "pages with text are skipped by default"); + h.get_by_label("Also recognize pages that already have text").click(); + h.run_steps(2); + assert!(!h.state().ocr_draft.settings().skip_text_pages); + if !pdfcraft_engine::ocr::available() { + eprintln!("skipped: OCR models not installed"); + return; + } + h.get_by_label("Recognize text").click(); + h.run_steps(3); + assert!(h.state().ocr_run.is_none(), "finished"); + let toast = h.state().toast.clone().unwrap().0; + assert!(toast.starts_with("Recognized ") && toast.ends_with("on 1 page"), "{toast}"); + let app = h.state(); + let id = app.active_ids().unwrap().1; + let doc = app.session.get(id).unwrap(); + assert_eq!(doc.can_undo(), Some("Recognize text")); + let mut r = pdfcraft_render::PageRenderer::new(doc.bytes.clone(), Default::default()); + let text = r.render(pdfcraft_render::RenderRequest { page: 0, kind: pdfcraft_render::RequestKind::Text, scale: 1.0, ..Default::default() }); + let text = text.text.unwrap().plain_text().to_lowercase(); + assert_eq!((text.matches("invoice").count(), text.matches("scanned").count()), (1, 1), "{text}"); +} + +/// Edit text & images on a scanned page: an OCR word (invisible text over the scan) opens for +/// editing on a click, stays open when the pointer leaves it, and its editor hides the scanned +/// word instead of showing it through. +#[test] +fn an_ocr_word_opens_for_editing_and_stays_open() { + use pdfcraft_engine::ocr::PlacedWord; + let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).build_eframe(|_cc| { + let mut app = PdfCraftApp::new(); + app.set_option("language", "en").unwrap(); + app.open_bytes("scan.pdf", None, scan()).unwrap(); + let id = app.views[0].id; + let ph = app.session.get(id).unwrap().info.pages[0].height as f64; + let words = vec![PlacedWord { + text: "Recognize".into(), + origin: [72.0, ph - 100.0], + across: [80.0, 0.0], + up: [0.0, 14.0], + confidence: None, + source: "ocrs".into(), + }]; + app.session.apply(id, Edit::AddOcrText { page: 0, words }).unwrap(); + app.set_option("zoom", "100").unwrap(); + assert!(app.execute("edit.edit_text")); + app + }); + for _ in 0..60 { + h.run_steps(2); + if !h.state().render_pending() { + break; + } + std::thread::sleep(std::time::Duration::from_millis(5)); + } + let (r, k) = { + let app = h.state(); + let d = app.session.get(app.views[0].id).unwrap(); + assert!(d.text_blocks(0).first().is_some_and(|b| b.invisible)); + let r = app.views[0].page_screen_rect(0).unwrap(); + (r, r.width() / d.info.pages[0].width) + }; + // The word, 100 pt from the top of the page. + let p = r.min + egui::vec2(100.0, 93.0) * k; + h.hover_at(p); + h.run_steps(2); + for pressed in [true, false] { + h.event(egui::Event::PointerButton { pos: p, button: egui::PointerButton::Primary, pressed, modifiers: Default::default() }); + } + h.run_steps(4); + let opened = |h: &Harness<'_, PdfCraftApp>| h.state().views[0].line_editor.as_ref().map(|e| e.text.clone()); + assert_eq!(opened(&h).as_deref(), Some("Recognize")); + assert!(h.state().views[0].image_selection.is_none(), "the scan under the word isn't selected"); + h.hover_at(r.min + egui::vec2(300.0, 400.0) * k); + h.run_steps(4); + assert_eq!(opened(&h).as_deref(), Some("Recognize"), "moving the pointer away keeps the word open"); +} diff --git a/docs/images/pdfcraft-ocr-verify.png b/docs/images/pdfcraft-ocr-verify.png new file mode 100644 index 0000000000000000000000000000000000000000..14a5b3dd5c98ecbd1b8b95b4aaf943673630eee1 GIT binary patch literal 224336 zcmd44cUV>Dwl%&G1REk?M*#sL#DanprD-H0SV;6l5>yaThyo%K1IVV>C_y@kh=M0Z zJzft2A&4lwij*BiDFJDl5`)4n0((=xF_*c|y}##v_xt?;q@RRybEPY$+yK9w# zY2U%9=e76G>~eYJ+4}hJ|2uId^_p!)s=k57pybEgljrV*nYzEp_`}G4GS}31P~)(D zT=_TlhtIzGAAd|bX+(J6jO{Fsa5+T(Jh?>u$zT8R@X^!%?T@jd%bowPe}DSd^x_Nu zkKf0Irw`8h->**}Jmt0O{|`6uPoZ9Bolio!bc^g(C2pP3XhQjI>Nfcs;{AfOq>iZA;79Ba4+k%}8eCFRB308nr!{e($&Vgk4r?CA zS-D5eU9w(3joZ@R-hE!N~2G zv-ojtu2g1}*8jE}rUN{(p9MsmxRJT1>Ff&2pn3Q`FRwJ`rB9q(f6d9M=XXs;A zSpWFr;|Do2aF08?`TX$SGu4fa+9Ju#@fK|j=82zoQg~-`)=Dj@Oh<)7Pk)HuRmHp< zHD}USFVSeplVuk9gS!<9Lg}ta55#HHRX7o5>mVT5>yq~jvLU`ww z$5mRbWTp?`{;f{`&xnF3GtI$X-7t?MSQ5P2_w89@6&01WNxM!xeYeA+XM1~0idSKT z*F>ck&ue-_t+}N|w)4+F!*hejbNkZU|El{o{Q#$9baF%RL{;#POL)`y4Ws ze!^L~XeoA?X!M#~ue7ryrSP4K5P$E(zRc|ilIs}p?5Ok1eY-9FYRajn=SKrOMmw(^ zJk_T+E%XxO32hw*avU$lO#P`avlENP`Dh)iG&Ei~R9&Vv^_9gS&*Cz7dVDcn(7g?={^$i~N?f%BGcafyr zZsG@zf92@Ub61`G(^@b!H0^lPBJd}iux~Tu@XLkqI360<%ipf;d;ei2f0ddr=-&s{ z6kQo; zGvgn8`B$AJXYmql_y+aomKh;p&TMUNzCG}{QnPx=rq$Ng))NO?T#Zr`6|p-OM{a(M z7_qZk5;4oj*w|!?!_k!B2+y&?`vY6%D=5U37hUso%}#YNWxx0zd5m(~wCT>0lR4eT zoks5ZXN;b+KUFk2zi@D%@bb{qpjxMa1utP~Ra}_G#}dwK>%duM8)8pHuWXg-Y%Q7$ zkkG!e=+?9dKX@uHnT}2GuA+|*ld=1{*A%Yk7!9kZh*2F(;NRag983tnc(Ts7~bLb-j!mypQ%oT zsY^*g?~=3(BrR!^Upfxc2*--DCtgNh(vrUJ~q;Hw}rj8Gq z*6hkJkMu5+Z=(MR)^!MTR;*kpwLtOE@CBQ0EcTE|z{uM4)$!kZc8dkd`L#9qo06-s zf5r;?-`NFkp*Sdxw7jNc{?6uy!&P<09krVbwNkfRWgc2Dc~~^IMpbmZYC}q0X5HNQ zWDgGyKHlk)iMoyn`JrLQp=y)po!wtvW&a_dXms3+w~X8Qw*AB5@6r$K><*sWdQ4kA z)sh0||1Bv`FQ2WSkf39CTG%stZ>m|zu7cN*B0A~go}Qj(>I~idZkrC}M|Mn4TJ{vQ z54`nzV;{JhX`g)Cv1~z2Z#)uP#^fJf!XLdxujNk-6d5sIl51WsKIC?ka3|aJ1hf+; zutyb_tUuI~Z8^R_Z}6$Rl=y<4cx@2!!bR^DS#^DV+PQ6Klr6;=EsvqKEv+q_q%NGx7l&@tW)H;=c!Yl?hDz=x$MDy9TR_{jE-GuCv7h=BEY68 zeCoBNU`f(yWX)4gy3R4tYL=lg#vL5Zy9*h8WLMfhezzN!>+D_8kEm=et{q>FMiEGW7bveaq84*QJwLO4JpGD zWtL{qT=5ldmC?ZP4TbScg`N_H!pphm9vbu9g8G(-@xtahzOPU%qs$`UhgmM9T-eVw z%5J&BmE7d=BJ4OC3c=#CPP9^J?b z3;Hum@~mh)An33AL07nA|K|4n;hw=$DH04k`Qe0uiq@)##^t#9L9akr3cJtkrg~yT zzlT{Q4xf|XYQ>*O+$a7kV`@2~yIfY*#{TXK!}#!7p8(va4?^k#%t1uY==sKJ_!~d| zMYY;<1v`|c8(QuSY(Y>>1`NIA866#+S@)kob2vD`li9lSkSieJp<%TeBXzCy*-}@u zri!xVLy+_tY4wQdA^hiyZvFoPm%-*hdwINQQ+gn|)QO(^0-a!CfBx9RqR*L}1F8H= zI)dahiR8Cy^U8|v2mYaRTF|{dxq7?W*b(O5gRgD*gF{n$8M9tpJC~)hs)Fz^sSWd_ z13N^|tu1cSUIem>!PZZ4&nO6`gvcKao;C?}?sQfNn3c1b!4vo)6zwm>uD?nls>icq zWL;*R$RJwtaFYOEygiivq^6%!f8V+M8)QN} z9~F!&r?Z!y+g@2yG*NS==)?_V?mhA; zcH@hd?0yQB8uv~dwMY|CNX4QSbc}DuZ8#vFZVnt<`0UxUi4Pr9(n$Au=RLKgP7f8h zN!o`dS2YeS3J8$njQR!}Fvt`6rEBt6q#NQ=0MQ|oR{P&LO@6}$_o)f)bUgMCx2Mc< z;-cS&rXYF!J(r&RC5fEHv`24PR?&!^Fa zL-;oS2sIqYA%u&_ccd&SuqA1qdoM6`IDz@v9rtwa9!Z&%m6r#Ey;vlfXOp<^+x1Ci zU%CGJ?OMN5<uR-k#oR9 z)bY<8*{!;lr}uayBq#3s&uVPC`st&6?spRznd`d^qLmvq34V?*?sKBZG_~e9N~cJI z_EVb8+8WpE&e_iXg(=?auq)JA%OI|Nu0{mv1%tRtT2tm)&MDKU4mZ6#dv%u7jNR_l zoA&J9&$}E~E}hfJgwLAN%iN=TxiG_OA?hNXhSy80B5K*1X)q}nS7vW!Y`WGcutU(J zV|jH5*sHhkU|LB{;rm@81TR(Qn08h42mYK4zHoQLbCbOu9&4X(VvM-QRCg*d$+E>p z_+u;Tu~?a6U{XEN6}PtfhP0qHX&-IW$_$%scK!K%H8NsNKz@~}D5^v3Y4+FGG3)~M z;+1!`5rsJYzGKAHai`OHm*iz8>&^eWky#mc9zRxFXWnX$E7 z--e3PoCA)3-F3Wx!tLeD_<={y7Kb>j;hA{jIA?#yJ?#4yk?FLh4YPB`Rw@UJP!4W$ zS9Ytv*_U7`;r;paMScDJ$cmc*Lj^Gu*V0#~36`bB$lP5x^=HgfB5kPdv4ISp+EkZ! zajTGyzn3^PqtiO5M@N$5G`#=c96w*Wx&P)uUdmgTtBSB8w_Dd(vV$j7e$3DQ& z!-s1=%Kt8HxW(+l2|FIp?l7<(P=r8|HobXwL;0pXJNf1vq5@Q%H)F>B6>}`dK>&=n zeFBhW0g!8ciAugjnou`wx2t|ioft^>n;=P2aPr%?hb{QhmLa)81G%mRj#TH@x~>dH zsZpD3aNl2CrdD~*e1RgTzMr$cNMu@6!?EBU?ifDoU**e@13p1g<@w>K>XFjx)sV`C z|B*5FImNFThnp0oGWc`!xG3n-Ufzf+pT@gwZEfWYjRvVv%~&tsRb9UqF@AMlb89OP z$t>G1s>x6OS`C&r*s%4ciT9P=%^e-qO4BOMt`=zqDQ8iLoPJYOi#lH%b4;a`v64aI z2CBej!U=U{{%5HFrsv^X$Q;9#+@l-s4}8u|Ux+2%WC&#X*xy-8O8va2<3h!fqgh9C z9B^jlD3$3BW-NY=NR4(XY7MPYzmE*5F4BKjA;o)g%S<&eHCRFQa*W7Ip8i_vYxfGH zhE3j7xei;#E&TTZcSp%pr}GL^oufRUKNY{f_NPUAB9eC#2wm6S9p!)9J;Qu^`c>eG z2(z)5F}EA%6dQBOatR_NHU++II-2z@3U|JVqhOI!bc=^uHrecgGf-+kY6 zra<1RRrF$nK(ui2Ch%P3O*P;aK8uurL9c-EV3vV>%iD@MsEHfT7;V!avuD z2a2XlgG9!`VXa<831MbEB~0_7VZB&T3IA@=3=0cev0{b(JZVLZYyr~t8Y)?N$`)P9 z1bWJKIHfw?r%#7?S_m~QZ&00zmzeNqKn!rY_(xw~@Jyae>eR>kh^1y1_mw03s0l7R ziNDN8#kEpRpr}@fA=XMs)(zAVS&MJMmI&QW5K$R*t~@Ow{W%wP>P^JGtU+IG*UDn)tk0=V~Y6 zYKb^z`rDvBx1ei7lGLGIFjI{KarS^eWr22VGX!W2lt7CHa|DM zn0BR4nlf#!G@4%g*nb-?9=s0s68w29I5WUy+@gz0>x6ymCD1#TFDol=K6;qHA!o6X z(c*mdbVDErG4j9x#}98DFW$GDd{^6VpsyTK`%6ycBpvAZ=C^Nq>HasHnRTDnPdIVo z`*i1&C6z|$hV(_I{8m@}J-`8RCm5})+}tX9Yd)t>7f*ULI3O!JxFcWxUoQdj;pron ztm~RYha#Un8py1hba40Z@)F#!9COJ>z+LY(e88(LrATs6NF@*l31IbO;3E~L^|wYu z;@Td%p_2evc!yX?0m}c`Yz6_D_~|EPigBYvq2xKaN7eQO;YO3&ylyOi?Ef_|0l0j9rMdY)8kbF<6zNfTfk7@p@0vD`CTf7>n)Lo zOIh>;N6>=px{k!O@jRz)>)?+%{@u!H<@96x@uKN43JHcx z#?)UKrF)Fu7B2}F*bAxFA&hSBGfKV+F_4~g{NG+vA!Nxrh%Xbsi8`^$s zU1>J_z^>muc-UC-Q-*j8ks_f+ZLS(K zk>ZK-5_RiMed{P(?R0az0L;>2f-RL4`w4`+V2dci=`~G3AB<4}l(ZIpI7n2)0cM5i z(Wre-1c*8h%c*t z*P!En$M{^}6hc^dZBJgEtHD^IT<80iY=(~MVwCLLiBFRbbw1|no3w9^+K98-i<@<7EfbU?Kwkt`yQqzSk0n+3VdVX0A_5|1d^-Y-kr9eY?Sp#l`TBaIR}cYIr^Pya zO-tdOd=Tj7KMKLW|B*lbtY|J{$LUTWn%?E!1LqWEP=tWCJfA~8b%yiG!nLM!B4>fV z%zJk)MGTTczkQ&fJ;i&)yQk5+lGY1Q49p>1xEQD$yhG7GlvK6;!i2Qa`b9yVBvNI10--2lNv@eAq)=V!VM!cmwiqw*E648Dj#p@{d$D|O z{sXY$yvRBSZeYJ(#BWI7}#$N&fi z;Htn)A;WlhdV2rx!zIrtL#>VTq{}uS@l=akQkGoSnle46eRm;MH#`%K*)|jQ(1Ct; z;5lh(gwFzoli=+l>PebV;nIK8DvjQa1+m~}xz<5P0ZTtDS3 z01m1OJmt%?Geh2->_ov0X~?ImbS40fMH~)c+Hp6x$hk{?#PhgRJZs0g&cxm2Ds z%p|merIT=W=$1tO+-Te3`|l>{cqZ1$UD>zITaJpV+FgY`yH1mYfGW=*f8bf63Zi)j z7w7X^oX>a@=nYCqD_5+b?F^i~w5In)#ZlSfPakPTnQB{yJ#>=upy^Da&3aZTbWOGo z1(N)DL%UD3IkIA2i1T%b4Et<-(f$%v%Sv~+^E*WH#b1=LTRk+H)nXw zCR?^Z8^o$bMQvu1PoF-$m9)#5TQ7w#0JTHUW}pznzxZ8xBZ4SCS?**VaPEDUUAf4MuMHgdYV$uUm;DbNP^W)e)Z1F3i z+}zAZ_OZ=xeO90x53yMFd&H_g{;*OSP2u(6kl~nP!DNcDr8}EAMoAw zcTeX-Str=t4ahYR{uz`1;$glVGkDkjI zMV841GR^+hBeN*@xrdPpL~S~^sdp1g&*VXS(Ewx5bS~pQiMn%VyTugG@;sDD6Em}( zuVckoIoar|&q(i}6n)MC`f zGN$5aNp*sb1f-+c>uOFQq};ur(fwCq<^e1ER(1$Jw7e~_O26vU zU!V2u+UwkY=KY5c0Gf&=P7@VQXGY$CH&`&bHs=5kH&{m0iA0S0^lbvS?YtBWVp*By zqrAR(P}30w!*Dbj2Mke9>*1+>84AD`hnqI0ar5vks4zD-`^X16oml5zX>f{%vR=WD zXB@QsRH4L`-dCz|V1w~L2etspBYYyjQbr#6!#$bX4;fNRkXAw!(Qo(7qe!Es1hwXF zCJF~sylaGxL&K?H`^$r8w!U`wZhg`x9tWth;^;;7CNvuL&UHkhzcp>TFiDOfNQw{&wM)*EZ!;mKg$g`xc=}{)O{K=LQdQ;2^`bv5Se$>P=QPb z{uZLq0!89a>Ql|DoPgehf5Q+U@Y-^t_|NY0wA^JJCJkO{9Z*n?iT30^-`w&)!V`PiCqBl{h*C`h8+JM@w@WSPlj%gn} zc#!U^w#V`hocMv^t*&Q&_@NxE#C<`WnnV#gV37$DuaykgBE2~_6V`Y^-%J@|A1>Vi z?@sI5#dA?B1%){;75CUpX;Ekir$yHOf`mbmcFZQ<+4U2J$2`IIgW@3hj)b!?ud#bx zu6c)qKxDf4VEU9Z#{MJ2)>Q~yc^64^t4Kpzi5>Z*D|BG<76&Co6y|{)KK-|Rg=>n0 z4~gqHBPOUOrJyB+Mc@JL6|Id;dg8!}Zd zZterriA@8;%ZaP|;c%Fi<3Hlb;~-5&A*ggDfkkFJ#^g7;7i^*DD`ksecd(K9)hYQK z!GSd$_7#EDbisKSusE3jIo>#4tm^2KD#6?$ek25=g9UF63XkfI9Bux6)Lc<&Yx-3J zSG<=oLGvhu)$dOgZVG}UMjBW#R0gN9yG|u*3SO;2 zq!xaT&C7>GayB3DR7p>yuUB?5=L;q7It{?V;HXCr=aoH|wmAywu+~Mjj+TbKiTsL# z`MN_Qiy5Kkl^pZ$Sv zsH$g`DIzRJMvP(t+IZr3X;aG_+^Mdbtv7l`ujb`TaQDGr1*QybbrbhpzZY7bZMBfJ zEvj7V&gAu7b$qu-j1+hlDP!siQGLy_w}%1>aS6z~V4IL|H0auXD4wD4wup+XkX8J7D8Lah z{U+*h<2#KnoThr6gtW*=)$Af)uQCe-srjFf=KFXCO99`3osPzn4!OYJ$A3Mv9$Y-< zd04($!|T{5wV{t5g7>AIcd#AV5?#gZ`2 z^YNDs8f;335(N%Va%dGqC0nUtSXi8NXNFJ$>5=c?pT%vq4av86o@JR(h^YT;yr4zd zF$#qd-<-Vcsg+ewPyue#{`a#h8AVp!w&H)${M9q?#XGc*eWAS+iSe_X zSFf5&7bgSay&H?Z=6cd*Ikv`$>khA89s8om7X%BiD{^AR(Y+6@?!N92KRd8PSrM59 z-kQq^U`@t8o=_e8T~6>XYx4_L79$(Ug0vgr^o*};ON@RzS_WwbE&_+S&=OW56Zvg} zB@VmOKJZPF2nB5!Rk11^eEE*&&}r;ZWZ8g2LLPa(DW3FuR1gjK?RNtg+ z1BHX`Vrck^;Ghb_poyjthrTB73)rW_uWlaf%dy#^GM0`^{5m)lTpb_vg76-$4 z^8)!wj@Ee*QEG<7(v~ZyiM7ZPmYN&bn%fV<%3I>iaaz|Vtsy9R(KsMGw-ciC2Y)av z_JdCcPEle{w3Ga!(z|EBMQ;PW6mKHO!(rx9kR@%-Yy7cSKepl?5_{ZeFQ2Di0x2dT zdBTloZW0%O#2$486!r&5=)?y>`H5GX7NilAeqRDxnbZ_Y&!3t&bxpq{UhtfH2o{yt z>xv;1IHJ6?@y*0^tph3KRaq6-RwaTCrnG;Bu*Hg`- zTYwL5lftnSoE-dWs3#dh%5Z~BW*vlPRL1WyuDt&-w@ z{c{;U8^4624mn{}_IFieTtT78fXsxIKzx%cF~iY9!3?&sWAyJ|^nu|MwpaWj4o>`| zHNQ2YEMZ{N{m@+ziBtlm9GMyc3hx2<1fd$I378hbm+fN!$O#16wO*q?dU44aM82?V zoc8`e3-~7)@XdE}cHlU{273ae!=0Q^PvA98eBrD@eD)3yz^G1vrp54KafnJIMwe_~ z_{pSXVoc-kCls(A%6{xg(0WNMHgVYj0SPhVPh$Gzf;#2Ez2bVVujmt9g%htGB)1jD z76iSo3c3lhsM1@@pnA81`!VgubR(JuU|2p8vd$?iA40zspA0ziiKvAMI=jxJQ6P;P z9Qz->F4_7tOD@2~Gr}76gU#DIFZu}?D)U?7OAihTf$PEAW2YoMk`ZK`WDMJmbOJL(Y_WvZN zbBmA{#)n9-82oR+1k(MY?3SC4)=c04=~_Y#`Xc=*9KP!hb;8$^($diZOQDvcV)KTS zvn=_nHtMa$t%!U^+S?YAjJa=pJGM8S#rw(H-7;Vdoolw4g!tR8V@hz zVxFkpRDvY4hRD1yVxm!qNX1%R3ZI2Y0JGDi7%}N2q60zr#Uvs{LAE@vr7?q)Ve+h& zfLp$Q+RyJfG0p-MqK<^nVXF2IvYkDZV>>%F?ZgeRbjX_ZX+_WPs1DC^e0S9`4gj#@ zUSsJk-}Y+U^tga6=yJ||j zI_d3*E1V|2v{fV@Q#oFyn}{rfZ9(HJ>1?PGk6w-R@PYUM#ZnMefxCE1l|<@uCG z5;cgtj{rf z^rN&8WOg1kNGa55eYT(if3TN4F zLW_P)zJWSq*!u}`;{jV;D;XuOk|K=B@KO#u86ON_{CYIiN^+VUYlw^yz`6PzcB};= zU>b?m9Mp*FN>(zXkqg*D#tjZpkY>Y8PF%8%1}od3=&m*mW;vd?(*R>}i zkgCBo1K3+lavdKxl0zK^b;^x0krc%ivI7 zL90SZLGI3CSbkmd`~+Y=rYJ1~g3yB1vE>Drzqw}bofZ22{Jw%}I4beUHzum|n%F@s z50$(i7vCa3m{VOngPtW6vI5hcqzI1E1w>>O zAFuHMX@llCc)?ZB5$&4JVl90+RlWjn!BzRl0jh`Xb?-8n->!W^i2>F)B`6n4X4xVT0t1uA7(HeOM$1pZ;P>*>HFmmCLlt$=Ch zK*K_p8mkBqA!Q~mQDEWEF*K>PP7p;00z`r6MzB2`Z09BY^|$(g?=}CGL&dE0-&w+W z0@dM43iVZ>v@^z^X4FuLrXQ^d28{eHF!RcS3b~<8Hu^Xs_IlueKHL1qAARIhpo-!7 zZn*U&*cOHaRWINGtm6DG7pQI*@J9nJs0XqjZ47ec>$L-{U!oi-AaB+CLC0>|o_<6@ z-8)fAyc-%XvMCRL4k{o9!goSp8k|UgBCtK^SO{!5T}y&iDbAejk-Ay~%z8HrnA`nO zP_GGf-l{a+qMrboi2TT*bTqr}Sp0Rz58e&=Qw77zIDkJZ;>n9SZ~dX>;}&VqH~NF* z}Xm9TLzLY^3(%E=K8&!S9ffeK;Ba9Ps}d%Xhfm&q@$!nG2IYmP?V8c>s3NoTWVM{Dt-tpZ9|=% z;f>2tkfvYVl{6xO3}B9jwNPSOiPzy7Slpr#;Yjc_V}oED9es@uZCTlBp2@4xl>ou> z=SP1}o}lI|&%xY7kCuf>g;0-_=b-Noza71y4eG6uK&H?e=qSHuSD^{7nM7qbV(!Cm zjH=LJ)v#$*gxpi?bKSe@5put-`T|(m>NILl57E%-^IB8GpJX$zw*7ij-FlK*?!*uE zH#b+T$uF%D5hp-W{K6{1_I?j}X-4csB08UL^@MBTC2|2iDEA;Ox%ag8TUtdvaN_5KJQ+wrg}V; zO7G6vywabcsVQ06~E+vJ;*+4!l2|x{fV47)_$#Gjq6s0zUvh`>{fJ(@t0aneU>@lPG zS$@qkwoJ~iAIM+KGZC`PFkqLC5(k8l4oiolmbT7@h8AVqUY^UTG2U-0g{E0$#B-N2 zDl7QV(GD=2Mz_HAULSXOJx3Fiwf;P1@T^YcYrfA%e%;G(UM>mLW%-qd`7hS2Lgyd# z!|sOvLb?V0myi9`;j1NERGif%zuVy`JyGtEC-^a3$*4@xkID9p#-l6-GmduDt@PJP zG!nqCM2l95sWKt*O5DGj!H)hT72O6;hW7Eia}_>*(>#$Bf8=^?S;o+lcmJr6 z{o6b%q^&Ul+qnhLgaA}?nQA3`tj79#GyzbW&1%4|Lg{V$p<$R#zzVo$(yE9cfdj6( zOm$>~!!BxAf>DowNDMlQ@q6#0R)fu)E0p!(@7SrK!5(L9uYiDnW6Ni2krn{f!&rqn zZurPdlkhHZ0TE#boGXr1kZcG7@DDpd)+O@Bvpsji=g^thIB@>rMQJJkDfv^=33!d_ z_I7v?rnTdt7=fn};gzpv?FZBAAHN%a0a7D_Ivw{RVdnC0%e^gq*Kl6~9^JE@XWkw~ zeswM2V{?C{M!s`i-By51X8uS?5023k(B{Fem6^% z`wLBo^e=5l9r*!TaC@jEzkd(dMsoX`-_s=<q0BGsm{sksE*mFWL8vJ3ocf8KE8K@zq>rE$?= z(gU6Do4=6~$e{~VB~nk^alhx=RckNLeP$(tEiR?TpCXnUR!?HdhysI1TSvw%!Jdwf zb#3Pc(rGipG0(m>1d^2ezaAET4nc+5=jM?()uj+@LV_h~;AeiX@2FM)6IX58!`7ZL ze&w2e?~;OS$iEOqH_?af1+=e>ZLLo33VRP^K~#xG>I|&h>@D3TE4RbN#ZBS|vSW2E8`sn_Z!&64%tAxNPG44NELfPzF64bRs(xo=F zO?8SNCIqMcB@7`}e5*Q=8AgurnJD!>Q0!fcjSV!&(LvW<*{8>!Y$yt$ zUJ-VWP^X4Dv4XMWgP#f3nlyWRD3&+SnD|c)usj1UWC| z7ESf3b50tm%9rs#qTn zb=JUMphD2%Is81-4RQV!!!9HmqHRcbGa_F2xu%+ARv|*pC3Gq!kcq7qmh3lKb>zIJ zE{`6fL6BQ3$aG+dYOH!UiVTp=nnly&MO+X_YuNTKd=<=gBV^WLL)94v@r-@wMUufR zA1_b=l{2h6aI(PXL>V4!G5&c;0pC?-DWVjGV;Gp|%Z^bi8Q|h7%c+moyH}5cNJY4$S6vsz&-VhI_X>Kjn1d$`y1LsXM;z1a-NM5sA>tQHpVwDAI%1N zi0(h!d2%&0VE6$R5^rCJsLjSwp!i;p#}vjkgHN&KN<>Q}U{Q90owTz8Wg!8q(k22vsEa?7{G6{VK+Rb*cyN4P@+q|o8 z?Szdh@(K^kGpu$#fPF9mR%I6zRT0~RPKV~^<{s~ufE5H`e8I1u%t2!S!w1EAc@i!i zgdOO}DBx-Hz)1dpEj5XtEG1|ug#Z5r`GMBl-+Sw}A@ScZ-He`r8YhAyMYD|~ilUN< zT3DYwOY(!lp$0?a3Tx03>r+R5yDtFIXv=9y3fv55qzb*NPsjw3GR#a6Vb=rNDucDH zqJSWu*uFW_Z?rZ>YbMjQ1ccMz;;AVqY@#$Af>j9gWJK+yo<}8Az$n0X{feq)t*hC` ze<~0sO5q5`3|8s{$j&*?jwpWv9mO#&UPB@KK2JBKradaZ3IV2K&}W&;gGg#fl8M`+ zqA3T|`s+zLq1MieIKYzp+tF3&n|GqzI-Kb;8t3)z2o0Jzh zb{$`a=Bu7LqJ7{MP@FY=!(s`%^Q$LpRmYpFUX7ZQE8#iLSs71t)I`e}V-n1|6&3Ce z%%RHU?MZ2;(u}-5qm&5z?VC9 zL0+?SQ{3Xp36NA&-e}u{Lo5Gp>Z1HHr?nP1CJQI4m!GxnI+BBNWDm7FM>CJ z&b40tQ>T~LN^zT7!Sjgp+jk`y|M=pN$@iS{6jk0zC(p=D3%%dWU4=ea$w#~CO%!<@ zc4@u*qrAR;?2r(7Y%2eyN`WFvSh%CpgHKTb zYp)pMd+xq(4+(52I%6c)V}2eu2WMhFpvGESiaHYD_S^Dd3GPOE3Pr01A5du>rK%Gr zf4c_Ch-1;@7$9$Ly#~gS@FRJL1a#au&|?jPQF6S1G&sNyw>G-~{!OimDAVu-tO5_; z5ZxS(T{8T=SEwc?P4*HrY5QWM@$oR2VJ2kJ)ie|)^H|ccQz_jL9vyV;nO@xoJ*FFO ze+BSx&qzkI_wX4=&lSSp+zJ$*Z(zT4L$a*81_p%0C-2@L-|_?vtA02V-!bR`+3ejd zgT5I}s=55}&1GYSZUv6gkjAJ*F7J?j+UONt$gS7QaoAVV1Fo_Q9!rO;1=m3!L@V?f z?cOhqR`IOL1Nv_9$ur0-x6nKiAQe>D`JY=L-4&FHkx6L{hHu!ooQc z&SW$A5m4(meOGb%qMi*Eroi8wq&@?p1msfVw?AmMwx-ZRt`teGP!sDH7jpema=UM} zzdKCLs0Z0Tqwr43gAyoNyP1)Z^$_X=2^=_`)}bAicx4Xk5-X6}XfgrLkI9ekmx2r5 zFCnU$tQ3h!=N=@%-w9ip@zK0PGx#vEnA6#YshM|>VOg1W!OD(g(f~jW@Q`bXx}pO8 zI+^|Ysb0v#Lgbz9(jXajDiN;*Q$0qV&WU zxc%gFW2kI7%?c9x<^X7KjVGFlZ9IhEzntoOZ`_@S4>(F<1%>EAn)%x`n(7lg(`NU6 zp^~1Un+@V1N1ci>VdB~8hCRmQ|FON|;!@z4otw)gb{52H8M*9)Fre}#h}G)=x~wO% zK*XwY7+f)fKy(G+l zlwp0iFhj`cqjMga<#TK+nAU#T%>|s-aT9-%wqvld;qV7qH zJEJ8=P*Wl+*L6y;JDLwCnxbcx-Q56HH-0 z9PT<{7qF!&dL<8Coxd4L9Z8|4zoS{V$!*!A7~^MxjKM=?x{D?q7?Hpgd-39SDi>gS zR2%+Ujm!#`R9-w{^%~-HODJ>FhO+2$vA9MxPCv@Zm5`Lgf$f7I+Rqbc?+H9%mZ194k{hD zm;BVw&(uo~Az zRrEcSW8J&d@xg!$e(4~+RVi7!AyKLMUQVbCHC}XAIKJOVutkP^E|qN0FdJLJlv|LG z1FBO#AoxOZnWIS#{8uEXyinMV;7T>KOxC^-d=;H)8b{!!qplP>{Ts?Qyd1qsa!5B) z>p`4fv1_qe%5#qp@Zh1ykW^7`qSRCYP3@rojY>b!!gQluq(83%J-J`mcAeLEhU0|#np#meMH2#qqqjaQfpy>65rr-xN zr{8*fR`QMBh1#B8$HC+8o|Z}t9YOI{Kd z?6-m@-V_rn7e5>PtR}1lFi1PXEcg=%)F7BXeDG|C(zHMY{1lb|x{(kvj02=S{e;?{ zPnAxb^HT6?TDWK9w$m%=zH6u;CwlAL`w#0jBgc9PwY_AD(K!!jtVMHF{2ko?-Q>>D z3=ctUnA!D54<`WR)*?5!dX3xzj>PHnT8+*X-xsWwLWZ5e!tqmNl8SX$gKm6ufL()k zT@5n?ym}E`mKufpIX(wq-Gb{1pLS#Me4y!bR02p*=JW8OlA%-1a#m|dT=}A(b3(Nk zH2Z{5d3zLn?AW5xAdB~)aeRs)tZSXVXhS^QHE^aRa|1BKsSa%W4_fG3ab7Bmgf$f^ zXDKd$7F-v6Z{HkH9K-}EQMj!@Qs)vU{s5|jmY`!BNxn4ZOtd!Hrel=ajw*>J=EFc3 zy7TnNdw=SfrPpwYI=6b^L+xc~1W3H|Qn)+lW2ml_?S$0Q+=pVoN+uplQ#b^RwM2z| zg@0OG-tg&P51GM+V9@!1!Mr6QP-;(5Gh8PCttZAvB`T2>gjCdH+Ni5nVxJsE#fKgV z>RSVkN$p-G2+)y)W-eg{+N-lGN%-8^w1^1jXxc*<2NnP3WZluMPC-nt87w8^Ht1-i zWJ638E6>61@!(3d&iSD)GN-25$A}$_nfdP^`u> ztRoNd(*F9xogd#Nur^i9GBah$Ht>hY z12Bb?h@=S#yc8Y`+zjkVW|1h>d=p6KW~BejFGg!<3DH{u2!Pr<@oQ+yOe@n=6>K1B zEFLf1xJL&vB~2|_s;`oBAk?{=P#igPb~qxn&ZevL7wJLTK26FJre%F2^yIQ@3Mypuu*T()0xD&08RIKKJ-=Qs}lkwe*uoh3@Z0Z zVJ&5-O$5rWISm-FTz~iv9Sw0z>4p$Gd%9%Kq=!g@5vV15Q*Ox3x9dx^z+(b5Omg~@-tt6<@1n~xs7u@b_;44>FF zGhtlJF<9YJM za4fm{)4Cv=EuleD4yG8ygP9QkXAHTkcezKAINFWY4HE{Ik)lcvDDi&V2mqN*p`W`u z52o@Ibd>>2C=si>0G*-A67XY(7-FRr70DQ+!}zxQ8wZ2ytV}j=qx)F|wy?H}qopR= zs8)@TcTj|xD*LH~hbAM$HrY!r%#gZ}UeK`=L8@{#`*WMZ&xxggFaa}8qkTX`Nf^}Z z@1s{??6vPAkrG11Qc8O)5+}m-N_Z(*C1Bx<1vRk^Ucr5~{GU&z$DP0=v~=pt!h8^t zKH+A!C`560gMFn71;l$}03qSY5;WdQiE%poA(3bOGx37fpuPL?71pZ4SUETq*iHqi zAZF0gJS64-&MnM=pur$~Je?_>5PAZlDVGv|T1vIOkU%E2aij8%r`CR{P5?!X+_xk1 zfb!D~Q`K70irQFOw8I^0`e#2cK&O=~O^oY-DM9Xb;uluTSGMXiN7t;z_#8@lLYlKf zgX}aJ(%x#(0&pQ5SbfstGYHqSV2W6V88!LTGNC41r8cV==23-nzh7)_*<4A;6!og9 zG);lChy1xQrEY1c6pdK?8?8(#SohA1Ee_o@o&JTo_nV?p#7&antYjKCWfc|KVa_VX zQD|l5l2Ne}ZO4b| zuo4>rc?U-Ej(VfR_N-wI>lDU!RvRh7JVt21kgnIVYeo?hh++$|1UG5ENiWKs<q74c|FOiW~9T6GSZ)8hZQidiRR?-iV*JK`g z0J<&r>w;U|PQ=PB^fcw7@hy@i4RbK7q7%@#LQLc}4Ipz~VNIrF63zLD!={{aYmSNH z4xZMxB++7mkSx&*%}w%Ty1;-EDL3};q{XalfBevvV!ib416?n zj3bz$X>I=3*EEsTh#|y-ff?w;x(tgJ)K0A7qczQ$AaWszjH7LaA#{LN_I{eD#i$!R zF!V!-yP6=@jk-E{M?Bs&swT+rO%xG{M9zq2cr|RPPYpC6@8XbcsW(dAxDE0tMmB|_ z4kH~H^BbtbYRTMVwLozvJf;^gu?)$75%Fu+LE%x`Gy*l10%S&%y^b!x@CE6H9-<3Y zG8l?S17C>BAfYCZhzk;8{i*R4p&cbZ@6F z;#yiHLen>g&wbf63?NoO5*f8nDuI0uu94#uhf(t#DBvHBjC0ZL%_<1!DXQ4C8p9G2 z@EumNM=Jmtcj4VBgU?)^l~KCD})#`(1z7Tl-VZ)g6?*rh{^7- z5t@!R^7vMm!jJ*928N3v2OraxlN&#FK?@z9V3#nA8txx-nb1*tLF|+FVB2Oi@{xkf z8uXA!r?YC$nPV#4mxl8+;Zz`zs*~t(8as8?u|KDJcb{k(JojXwp_zc+5hB_`2&jzG zrhqI4VBljqL3?r7=D|-O~ak@J|o~x&5?3p@M6$K%Y`2@(yO1U2E~Er*R6xjcTHo z#49X!LN{3zP~iCKGI?V@%_l(50HQGxz`C8?L<>ra6($%OTXB_~e%)l(^#eiTj=j5= zq>EdW3;VKS0@t#gaCt8SQ-hP9)m?OIU9YIVk%%jP843FDM%lG(Gj6n?>_rt z2)tIK&m5*Fg?>Pj0)GrsGKP#{EdsZvY&Mb=(~IWvb!^OKy_)NXg!R~jLNY_s97`z# z{SXkbF_76p$^O$@1_HUx>-;BpX~on>&j$&1k*xoRo}4~^VtJel$xqm|3@PdJ zsO!9%&N9mJV3>)@N#u}53WJgVc4H}FeVxEpI>dEGV}>4CG6Y{beZ0(&&u^aE*gw_k z>9D--2)RI<}P5AD=1O2*&VqkkD7}BIx(Em*4AN|#p1~dmE@Y0 zD#km&R+XZ8qpSsGzCuiMhb7ZVj!3e$ttH+rwK1Md;F#Tqk3rtn^!`qU5F}z4L!&EG zDYW;vSoGZ?DsnlZ4{NhQBk40fZAp6V8lW44YeA2N1&ggE@UbnRHph$fq5m~0eJbpZ z@Qa;am47o`V}?{!`FVrhXwZkPvMSKweTWVQMjEaeqbc0aCgepQt`k5@c^T+5+bckR!8`H7)2$7|J7C`O&9 zfP}S2_v(q0%+8}rwP`;kAnY;U)qi2j0Zja9@^3yu7( zVXxG;>eZtY_9{E=|v?e|1WJvpzoHm5PxBjs5m62USgrSF5FhoDbd4&$reAx!|ZB}2oA1_cPJMk(F z49Wn4pr$whu``$q2adW2PO;C(idjQy5bP(qECB_tHiTU?*{Xf1gr{{##|-ib3}ZdE zbO*J7czamMP#h;};o+O*A#g9#9;l>!%I*QuTdC4mL#vFTyVO0?L;4^Zw8;MlQ)@4Z z?_f(PwQA|BNo1)f(;eF?i5A&-QULo`DfrJquzu~pw!x8t=nIE}tA!IoJz1n9(f=O^ zbeIMuqbGs7bv$VZ?OJ3ijOh4@_CAl6n`UrTQmH6{flht`^^MkJf|m2xLyl)9aEr~6 zBqF0Dm2ObcQTUo*8n;0a3_-={QIt?Gts_iMsmK@_7nU5$JCvQ5w^ix?(DvqGRhH}9 z{{oyf6&w>4C6vq*oKSPX9CFT`4W_9$g(;AtnV<+bL(VuNCVN+EZBsLIDg+c1Ns(Qa zV46$bTK5-Kbmv9G!vFh4r!i21o}?~8 zxJ4SZu5*v^@~2JV1Y2LS2)+8a>uZu5c8Z+Yx=Sa7Okt_>PvYX~7MO6OjkB{MUp3Uk zj3TdROpLQ(mujhx?y`1fYt|*E-4pUORttmqacx-p(isx=M5BYiIQ z2s)Rs+|=EC7>>af7#+!j(Vl6tAb?(CJzqv0kDwx7whgeKbiWH|WF2d7 zsvI11|G0Nul~PwNmy~{9rXdrfI$Q+W96lS*yWLJLkHUX6SyAuskBJhBAr_aj>m@#N zQ}L(_=J+-nIQl>Ef$lvTHJj+!sF{pFuc;B6om5cJQob}P!idAF2XZ{BS={`FINEm3 z&OYhs^!YsPR(i?#YO8jbh4!Ds9$W2C!8Ur`^bHG(rXT0m4tJ|QF`DK8 z`KYV)wB*Xh>ew4?CneYdWWLR)Nwv2?;Sh?|o3ohVVo7Go;9b;vE``8(i=I5ppx~mq zf+!*L-zdWlsWC}AsbCDa-|rgp2!V-E4`*^jojd1Xo(WM4XCB`JIGU{M3-_X=jbM=o zjkt4uzw;pACOwj9sp<#~<%eeqnf|^3OB;tHpNrUHU3lkFK-RAKBH@d2l=}j!guC={ z!_2IdY1`TA$M?~LF!ym?Wt&fFSd!Jrm3Gv0ET|S5S{dIol16E#IGfU&cGBWAoJ|%M zt@9v>upHEcuF)FJWBDu%f)VIrk_8$ygTBQJ;$VOUHh0kuKUb>S#FvLyD22D3J2!}S zFD8ib(uBbb>-QO~k#+vnjW>^FSfo#E1qnXvDwx21?XpqEd_nM@N|FwcJb5(>2{USd zbQwjH66>m!#`N3GOk3!#n3#GD^nV?UMl>&XjSW2Y>kZ460smW(^>7~=%?wM>CnWH} z+*kQc`z~MJus%&tW3w6`F1t(j|4fjwGW5weuJ^8*X`U#`<#E zkQks=FdXbUBOfL($g_O=a2>+Mc3UlYc7iOXc0ZfH>#a=uVvN9y9;Kfu?GNNzl27ry zDG2Y5B+X09g!8G-X{LvrHsE2V89;3oz&^9azjMqAJG&aqec$e<<6^U zkr?=0cKj|Af*WX4w$q|DdRSq`0{<6fOe;sKN(l}r+s$&aC9S$9gzTyu^d-0nd07np zyu17FdfZmk^v$nwQd^$wv~?9`(a#=Om=GZcTXAFznt;p42@6aat1n9l+*yLls10#0_S+soIK7%QmZwHz5lLnhR5E7G642JtipKc7n`WyQIev;!zVm77>d9pS*e zzqCAv(|v?+f);Y{qxg7RY&E&uda`}na_ha;PojO_9cf5#%;T-m-hJ)elO)t zc3s7(Ag#h~K_|H@N!DQU2+D|akLLUZ?1N=zvZ|{)UhfUUWZHV2ZkBa34V6p^8Ym8n z&eqD7Nbrl9q`#P|1Od>%2dD8U!(g2Wi8}TQsC1_Os`xzp@q6Y4u$LU`_U4frbN1f+ z+!%S|sW-E=^3V8X&?Lmo>)vEe5*IVCpA!`3t;v+OmBq-w2U~bK246tm;yW!eYBp$S zs<|$Y<1>Ge1CMJ-FqStCyjyNh)6s@yrbVxwk=;k1IfqGdnx%EhB|HHSH0M|lk}*%< zv|ac(xk6BrMYvrG%q?D+8Z!9f9oZFNBTS1Sff8*1>HX&*0IVkM-)Swga*GjT`r&{* zNDcsWn(PTD(AY@2$bu?wVlk{HolOmJCR7pA4n)6D)n-IJem(g?*o^@UJfGyb=-&73 zMMob}D?N`1FOR&K4w3qChN-hkbxqa zXLtwv0kO*fj8NYij%#9~={+bV)@e7$`Q?|bq9vM1VU2SeBPc4yiq0BZtWtR@*wZ{d z@BtV=(g*k$#v5O7s zw7~({@+{6y*<2$0)o4pNTlyN^v?<#51}8t${u*9JbzPaZugEH6#_|t#87_l^`7V`e zJkw>THMXrQ!w}?@E!m<=m=g>(%Tc=+WqkXqtyX;3s#t{*4?i;Z(HMYm2#8QWBMjsEBN&n2h689JP0}qZT+4Z~bubDk z5&A47q}7zSgp3u4b#JgPFo4#GbE`KhGGk+aE2B6VD%@$fDMqSiT-2qcx@4{s*%y~+ zX~idP!*xfIb~;y+YXv!sH;1w4rFn{N(`_`5l;Lj3~p^E!ahp)-98k9hj6W_MaH!8nr%qK(S3<8n+z5{NhMn;}aCY;t(6pM8nnYasYeF zM2(c6(R38#Aa;aPk|+HMLh4%OSH>Ou3~L#N?;4fjTLtvijnW=^J?UZ2QE+#vS5jONt3xM8=ciL`u~Iq~=YTRF-gnQhh7*nGu^ zHMjU*lo#j!$LoYgG<7_mFnrax#qWLf$&R62WY2hg%J=a0a)U?Z65GNEseJnyMst-T zYT;u{J_b}t`_UG_VjLZZIynky-p$Y$XUwhY~gjpHvh7;0)iDl3$;5u!v_Wof?%{xH_z`2X8fsF~w091*$D&nt%^QF?D$y5?B4^prBE{_A;CNPgbpM zLx)=h|FpdiL5gyd@%2A#k+Kp|cok!RORd)C-|$dRAcGC`Hu6{t!%!1tMV2amCFUuP zg#l=E9yiaLl?1_3fn0i%pc2q@gTW;6iLqOpJv7%%KVM!Bt>H4-tNk(h}9H*6LssI#j9j*N973mRx@VgqHW zD!{C%8U9H{R4Z931^Z$hOhO7(gHsJjm1@X~ntJp!pI^5p5TMg=6wLtzDmb1+L*DFa z{kc$dv>+(fEwJu!mmR{`Ds-{^D23;A24zUNOg`DSqNJPQclzWoJk+e46Y+dw<$G&Ssvm3F^8*f5%Re0=;1$0 z*o2q>jBB-3A$B)5w+EWznl_BRW{9u$7A(wOVN_&X?vZs6YiyaJEr&E-wh1~Gz8KSm zNR7_e1Chzy>I6Bfp%w<9=kJ5&Q zP%hIG82=#*ecLfz7{t6+KWpAY?GBi#QOqR%^%7)%P~jL)BU^w!paVC)vR%6)0FFBN z38nfMagT6wt^PP^{?H^Y!olSsPE>#I4G!IA{NhsivYV4 zWmY#lv3Dmd<|sJ8{gLY7Pknrk9YynSKHG;M23!hc&zcp3Rj&UHB9ONjXshrP6b8?6 zY|#k69rhqkuuwuX6U1iUjjyomqLlFR^4gR3f}GUZ(86p@M;m>Y=6Y8(KZvRiC9Oy) z8J8JL?bb+ZOc#<)0--z&>)A|+(*l`frz84ih(@5Bf#y%Vc{S+WBb|~wi8z4Pkm%)j z9xq4pAwR<3q;D%-4FX7(4Z<$oj_c8R)fJkg)N$NA26h%y$K_%!IU1;#;Dlo;h-9nz z?~rrbS;SS%obquC5lmps^6u`lE=)JbGTYfcmE%8@k36f zB_yufP`6;Ar`M!&^=du8EO6G?9oatb{oj{0%-A>{sOQYooUlYiUKgE$jKomEM;?G{ z)lM-5{8EZQVN}(9czkW6`Nw~t>BH52qh_^rcaxE0jk#IvtpR02GZ((^F?mzd+<%6x z?fplKqy1a13Hk>#mr)t7v;UYCVRPZJ#_G!iBiD+S$L$KA)HPrnB+S&r_5Jl3(P(bW zTT@HEl_tM6!9q~VfoD#%`A5C0uiM&;@Eo_WByrzQE&p6-4{yuJ_Hpd9#*L`>Un=1K zdllAv+Y^r$^Vhypf23bC`^7&d>{{9M`MSNlzH9NH3tOELQSU#0`s=H(F3))+_lw@u zp~Pc8p+=RO&4-<&U#0@YlFt9BTh9(3p0kqm7+$~j2iYTLCVB2X8u4G(>#r9wpTv8e z0_eTB1?Z~aw0V`W>gzLNV1VIg-r+fq;cXg4gxx%z*KS1<`$ml%+WzK_ElC)w(Al|N zOw7WdT8G~r=H$`f)~H#9K_j!yFAHD&WL>QeJ-5dn^ABjzd|?p93~Tp7y-^*~-mas_ zxPLPw|Ld!VMcY_vRO(>sqif!X9MVY^Z-)Pnt>>*#AMLOY$H~NnK=heMo6eZ-Il3yd zNp-&oVLxq8?d^E}nf4W}?-Y;EiZTz6ze644i!x)4=Oi}uQ+5u}8`55WdEYPA`w1cT z2~`cf)BNJEIg{LCTh!?OI=AVqdG;>%_AYDYo&IWURju|Mu=y<)$mkMlq%fwIVM@oYs&imBIMNnD{PdXE$~zrC_D2 zH`L?S6Rt!W-)^(3H@#ND+O8uAn-6W4P<|`nyyxd!}XC0p>MRa%Ij3ZRpmm3X|r>4?8wET zI?z_l7hG(*Az%9r^+)q1E~UCrzyhJCBAZ%F(I?ue9Vs#iAGmP~#YR1ag+3x0IH%Mi zYq%wZ{PGpAf^Lrf!uDg$-pQz4KVg1Z(^DvBS}5m?U;JRHvM!95pATWdG@2|bIAC+l z@a8~sjtTBCor_+bccapQ$U2Q)G7A!>5KvQ3X~9uOVCpH9D@voI88NmFwLI)gE{y(S z&w^r9XYnEVyF(lMJQ?qkbUL(XOUdB`Yp*KQ+?_Uz_~Q6?JsSD#Fa6|DVAhJc(H?}n zrOkQ;L)5}96PC4!mVM6Lq5)&ExUPs>nErEl@ehXvwyR%I_CYUh z&xFV0?%eHmXWiA|_K}4%+z}fRxiXnd?I*KCf8QO3bl|9qd|Sp^A1|o13w<&vbY+iKlp|F2ymwk!Fpn%|d*A%kc$ z#9jC4ThWnAMexNToIEk<>1b;>BHdiEbdlsm_?N5wEjg;*Xe0g&u+B5C7R^SJJe6aE z)O#t!WcqmlJ5sAbl!{W|i^*^Rb*-B8?KrXzApOfdRQ%hB4!vNjy&C(tx2!keD%1~G zFd$6`>h0Bb_rSFf2>~<~)^AL6QUpoc?*UTlL`)wP?uiD(%DkN8SeS5oZJC@T5=) zVaatxugyAAE?zRP2UT1$Zrp-iqxUyfDhz$DAo7&|iY)#A=2u%P*Ay!l_=`fErA|Nm zV*}0Y%*X`mSbELeN4JV3t{PrT<1o}i*D;o|xRJwhx)cVJ1G`T_!YV+yt9KbXJ!e?I zLw${bJH8C3OO!7am$_k3OLlATdOK|$qEoBybh_p-v;2Z%lhDh<&fY7iTe8D_(Ef~N zzwgd>538DF-Rofsxiph_(2@s5Pgcei)m_p0#N?dv(86--z6q86+>pL6>yN%7BWqzZ zhC#6@)uH4QRIRPLJ;C-Y_K)}?6)aA;>lgG(lFNnp{T&_q{bR?4Nnv?QHF5F`Egc!g z7VCXt@_{}`6W!c;G+y{*Q{R$&pYq@Loh^9bLI2pPVM*h0jX-yj^7Z*!aZkQkgva*H zbMJVaP9Em+XjEurMp*QZ&qdbTSD!rF|LxFP@?+RhDEx3$>>P@@0`oGv4I zDROvovG|~&47h`qZoBAz0vJ!oZ2)D90w7Av^j+kL^y9NrBxN^t|C)EmD|4S1pm`dA zqKgo_Ag*wEd|d&qpu4hfdrl^|SZiSra!hW)?v`R?;}@&ob#@m@uSZyk4cs|_D3|5< zA+ZXW&b0wW3WOdeue`2`r!(qf6mT#DMD5R^9dCw6HV=!DJY~e`^$dkuoWv3o%G!-> z_0{TM1+)M-3Sm6i-Iz5&z&un;!tq*QrBY=Cz^n~%QrU2jC(uf7-CpXj_16!jdp7bD zrEdR}hU!UE`eaN^Vtk-Zs++X4S44@1QVfcltCjuB*Nw&Q=b57oR~K)DA>ua^t+g5+ zUm4tu-!*&h!wlc&5?Z7!4#n7WR2T#Rt-6UMLJ^R!W>`dV5wrxCcF;_H71X{h*~T1I z8ur13iwAGfoH$1UV&fA7O-rnr0`cmMhJXPE47Ea-cN-$>3FKa{=9mG@AlVFVO2>&I zH(@BzXy!p7|0ETNYw2!LrS2MXANa)kOf{^lJgl?i(6~ts{YLn2S!Jtd4-yj)INvT~ z>gn^N}nE-=hc1s!0cXw?>)oa zUEV76x5Cg387o%z%_$En?EA~({YUJ*tCOv)e&v}=YRaG6cb@Zr?@iSp z6T|MjG2fFnD{Y;Fly?0kR&E3zBP*Rzx>;Ve*=&TOTUm&XI@$MYI$)=L%*?Bq{VlhK zHwp5_hc907X#ik_;#*sxw>>A{s8-=xb%oj4TVt(}F8Po$=2j^Dm8DU=*Pt_^jfojO z6X-ggKs;s1d&S zjWTpuc-$kE(qt85;{EsEmzzP*mhzy!YN>Y_LprfcOvq8~)Nkp7fui7itKo>GUWbifFR!0#q|Kia z213?AqrKLdo(=c>gx~>tg}sWGBtJhUVev6~LSPwLn+(nO4sZ5#eBlIJak+K#rt!GB zR2+z{tlz%k)0EOh$BMsg+wOed6&d9V=PkW>egES*5EnA{4Os3mVMS>(WL(!qQZRWo zF5X;Jx~nRAONjUVA3vxST9y&EEhRpAY+%WS;0JMeUhN;!3>1bwPC5Jg)29(r4BC^w z$QAM==B^J0?2qA8ywRrU;mZrI-H$&XQ{3ODs`hWwdUos>IrE*i?Kki1*Q0QYEqj6W z>toiTgKc9IHaBx`^;^;Sw(W=dE-HO3y>P|j!Y`MX*t`5O(^F0tn9c^I4S z6(ypQr&jt(BQmAUA7B-EQ(Wn-f@8JY*k?&Y7!$@Ehuk6j`gb=N?tCZv-T<+cq>IBCF0QdG{zSi_!SqyEtD3<+C8a+feMXAmQP zk#2pyPAWxqQJ$e)*sQd!yCUQo=SWdtoIpLWM(NCH04!X~48kxykY|T@lbm8mfmZ72 zwU>~}g_`8QrULwnF0)d}~IHp5okN&<;=(RrE7$pes z)?3mspPjmh!1+?Pw`hwopq2uwLMM-W)_QIycLzUL+xbtc>U#doFDG?E$UxD69X!fP zk6o=?@vo6RW0sEFUQvK8bZCKl*q#q(r}PgjZGPYl%ex7`zwz#wd%m7LnBOgNp1SnO zQ==AUW*29NdspUp-yOXrtnKyQ6aBF7`r%54+FqIm$$n~POhmQ{Z@5!F`{vau@sCb@ zbZsB2AZCj-acSYtS(OgK*AYy1|9vx+bwKXj&-NEp?J6or3A1zetFJFdLWC>IbaZUF3Dx_e`3zY`hkvuL!B+JCs5;0C=Thrdj ze=7tFV2cQ47nzcOQ%Y8nf>M<5>b;t34Jrpb5tqwS0~H$sEvhR==;9F_XkjuejAzc2 zTymDcq&Lj-KOF?%)>lnpNQv5aQG#shp;AqLz5qrcG*;CgcIYS6er#Fx!!wcfkj-N8 z$dAJ@Wgd%_%HJf1w1Dmg{cH8RIg1Au_F@E`D_cf%v}CZF3|?KNToD+9i9qYCXJ!u% zsVU5^hYTL+B$siqM=X+>@QX7KqP;DS4gWSQydHg1TuNnVQ^Ir^HJiR{atET zS1+(Jjkvae8n1dl@Q;(iI*nPGfxrQa@pr$jc+r&njzkNcpJlf+Yl(Z471n2Nc4rQk zdOzX2n6gNhvXxWcT-oe>kM=!$o^QR518KZd2R7!x)!V&S&RdoLT>NNu*n-rN zA&K9V{dCOMIi%`P=)jTd;i&KJ{PMovbZjh%f6z4{;#l>iqJp=5s=u51Qy<3`o_|7h zS$G~Nyw%44)7oAyywYHYCE2#@<#+1O?v+$;UOiU!2UlyMFwQF9F%kM)vhB4aouW4d z?%Q*){;lD)Y?FI7YkIQf*GE6z(cz;GCEi2*Mk{~QTRJGSMbBCN*N;avHhE0SNi zkq(Ie%?ogKs#>cCOBHRW6gK1@e4?PptS_eMI?_^oktHNR7Om5`x5iNW2!28 z#bG|*7@ykdaS_dp`m{bBK|&%jKaXM=sxVzBx@#P^idVY;aFxUc&__X_>1Q%@Fx)5> zGsDo_IiT3uEqCsHcxRk(Ju+^q|!@I#194(I>{k`uPOn*Nevn9a_44LFjMm!&WCe-r${c z+@pG8>!1y%L+i#rzT!4NYTW-eN2-X1h$Y zUh#Vy>px|YChR1i2-e7dKcw-@>ad_amVz15X;9;mR~7h}C(t$c^wQ<6EXK}UYfREX z>Z)_C6>?K65#Cmzlpsf{aD8x2UVa{hGNtWOJA;QMw5H-Z#PU zsx~vk``G2lv&s}%FYZ~Ls{n^17h^MHQPlTNP1lVFnR?cwwGCU@f#I(uy%VCA(aoy> zDHB^xMV%#^l4{VlWFCnvav5!7ay};3__R|NwuhP?mPlhewhWuOe>O_0muOlq)lrI* z=qDyx5TACE^vrWt+?ZPo6*y*xQwX4fyc6A-NZ?C-!3%<*)RVPUrlrvypmuagVcG&A zLVlBTX3w@mhCNroAq^<-L&FaZ%>T@g;F#fgG(pYcGZUalP=X8~#00A^zTZ_iQ&&oa zG$6q}o#LfaM|QTCB^M4vM{c?{+2k1Rrfj2ico~z&!M|T094+I&Ty;9;V~DhF5xrG{ z+4kXYy?Ztq{fkE)#&qk0Bv0jwx)Rp3t%*KUdHF?}dKraoXjo>828YA1q`cs2^mVCT z+e=z2&9a9@!I)yikr-~P&w$Fl?a{P~70q(j{`xgHB0m5jY-qzfwqbX6Z)o)r1H;UN zMGdpcr)6Dh5|Vxz-;86Ae?69zW&7Eu z2m$i`f-{lnl~L(=zch_6ACR!g1=m40SA}zY>|N3eQuOjZH}2Q(f*0nOXJ4;KEDvci zz4U<3<}KC9TME8^Flu;?QGmXgl3x9TNAYNvvKI$dIc~u%>Z#rDdw^P%RK&5MYZpB{ zT6D$5286J6{osA12Y=b=Ei^+{%Bxy4dLH*G{M_+F#`Zhj=~Sx=;X%{?&*gZ=Xe4># z)!$EN*+t;RV1)RIRZG82V+K9-C~^(_OFl5VempAs$bSi@hDIk-qJ3M}K?(O?*x{4( zPJfL_G1kKhHVt53*8RC_;RrDQ9fiO@vL1O$CU_R*tuQn7I8e?wcTQ*1x!(%5Cb?XC zBC=DVn4A?tcBQP?%7&I&MW6hF(!;KjNDyJH|7$`7Y%TbSu+iX9RZC66hpY#z1Wt{O zCxv!X<=Y%SPsn`AUdi`W^5)=RG8g;^8d~*XJ`ckvmUra8*X^s!g$DG?Jam5j`k;h+P}4+h>%{;YN@kI#cHKU{pL$YJoR>vNChpBQYr-hcV~ zGyJ|O{QiOCm5-l!dEdZ`A>B7`Zo?;z>X(^y=!b7-9Z&SKIgS2&QdjDdBjYn8n-ki5 zYrxgQhK(PUb2ry zlC&BD>0x4nCO?QI@*=WlxG1goTnCz?u;_`UG?Nuu6hl)Gm)e>bC#*x!&#En}XoQ2`S8G088}#cr(xHERCbV=y#?*nHWsM&{=h~r` zlkNW7L)%Qbu&np&)g!0-1Z8DRE`D&V$DyJbeTLMp9^Z32^k|v1DXrsG0}=vYPZSa9 zd6nsKgJ2ih<{+|P3COuYeA67s8~C%V?D9b?e~dn;8IU#O%co>gg9)K6Q86ijP_@J~ z+HaP))Rx$lOFwMN3_uSqu9Obc0Js%6!}T! z3-Awe63{>!-jIYk2LA0me#b;!gG9tp7uHQ_c&dU^@G1C6)gMf0C`$Ad>HHX$zC%8u zSP5b$gwqvSkL?iILe@C{lFfqQ1$Bm{S>2V}OfwNChHR3|C-`{e>)7AIv9dXh!8nJe z9nGIC2bx35wC_wpw>|U3JalzrHZ(H{p_w8=Qd7dJJ_$3UA9Ujb$!NzThAB>(wRaO+U ziObHhJ;mI_2kPeM?yl@V=n|I=D!!2E%N{w!!Qu?1Mvwh~5iI;v8@52vY0F$!)zzNo z+31<{EdeC*yxTqK-TMzOD>fM&!ADWaz45o2H8^%=Osm(&_B+)vv^?+`my&7AZ!h!e zf2!mBA#MCUZKvxE>F}cEuw&yl3eu*ZDm&1rb5EHvXD z={hX2(Cb1cSM45^OxmRI$r6+!+XYmu6VFW48RDm5U@|tJOI~4xn0_){+oFt;xdh&m zI%s_8TD^v`g$F!Dj|A{L%80(XthfIyrri2(P3G_|%)mKV z3=tOt%>Z4&(Q%)0YGQfDw7#5SEF%*3Ro!4vwTS(i7DO_EVsU!yW(XvK4@ zdH5xA@6SK+pB7Uh1k z(N2HuFWTayCmiH>JuUn(tl_FwA07LsmWR`5L{soh*emnChV%b$en{4qYB%qy>j{^d zR41nsNBgj4qhZ6GL_e2LJqOklIBQ}Xh`nvTEcs~b-;0tH)<1Q!B`eZG2;^`gA7vom zx(er)vf12QS$^vtHP*N#TEz&e7)<`!C`_ZL%m5oCYU(AG4lEra1=>rX4M8lN!q~+ zO!(x+g^O<*xh&K?QBs; z4|DN!4QTlUnrP0c0n6#zr5zN%N7}A$L0N4>H{WrA>o6^2n-c2o-0}7EYiPM;Av`lP zbBGgh4ew1G>9=W0SE_Qp@-6jA^*U8y`<8#ie%{I7Be`f$=-ut1+2P~%lzaB^v?Y8p zb4nfosK>IO$8W*qe+Z7ovw#r7a;nVfkFN3a>EAL&3p5YkGfWUljb|f1ZSzJFd^F6xLIhRgR0_>73%}q zKii8q3sXtUPDYx*VaEM6zKHY=Tqp1%&;)K3E9j{XJOL15frG2-gzMtSy~>4^#L_`4 z+?l*GL2T8C!`T#msRSm)4V!PNUa3hK_!y1lWq6vnIWK8~Z^l|*hH&&&+4NCv0rNE+O^`2!6C8wDl+~SFdf6D?-!kS(pML~k z--f7UVCM9Gw>W4OkGYh<90yrLTab&U?T=R-f{T~!7 z+pO2BYo~Z6dq)3QuVXdzMV$`S9`CG+zYsaTr0DfAe;ZR9wz^3ul#l%$O!dLjVCIWQcFt@~XZV$GDAJF$8Ay%`B*&93-HP(9A4i|0Nj&r@cE-cWv#X)6Q*IN)SyT&TQn|=!Y8AQdg1L+%Au;n|0kq{ z*3MP6X2$#U_QIrx0^|9_l@)R$mkn5=4k|O-Qr1Tw>HRp!p47Oh$bgvZJvbKDz6|S1 zWV<6*9f4r-u@b4L>hUq#3&wpH!UK6)8)PcB2t*XQ9Ons3JPEPf`ck>-v&gFuMpw$l z;w~(gu9DhL2>$j}D+phZ{C@luzlcaBD-mN!nFK1zMGveXyC9j#$rjwi{pfhl24$0F z^T)v#Fy7}4NSKXA5PjjIUI?}$vd`W;h-Ax}fdSV1lb|5J3VAJePFdHSp~vqbiFHFd zaPyB{R;X7Q#D)HUtVmHhIj*Vc#HZkgdq8Ps4UESYAe-i={bi9~N6ecfgd-J9yN8Qn z6azKyv&;fn&%h@O3WO}hG@*`8RnJB;a+4hrK~Vl>u@}8gzQLsT?9|e%@lGe-KpxuI zO}KqU?6XffpR+#7db8w*<<5srUHrk?YNsV|t)27tRfib;ygl!W;PxH2o_YQ$uk?wl z=k}N}Y;V|iKV(}ET2s9zmu)`QGw)>k>nBe4?i93gOnq7GMoBktSD^t0hZ%u+KMj7E z`z;s^K`)9-08I%I!%DeNb<*hP5}r9%aP-Wrh^&XI8&AbUK{2O;8_Gu8SPC}8=TM=q z^w{J*0^2OuDe5q%gg{A`5=WP-oT-KI9tTE_B;2dU#H5ke4^!@ajV-SIxPy3+&znCg zPC=0(rrNYKNK1p>TYlIb==75lb{UTNe*s$49%hAS`*b*e@5333J2(2I<5TlS4QuoL z)^)xAfBe$R|Br>f?h)Bp7rh*ZiqRSL?W_AQFbbQ*ri%*4tRb#k7t2J(p`!+F2mqi` zAR=pe|AoWnembmm|E=8j#FJ(VUEOMJ03**c_&|c+^%Jiw3A#cPXC?ZgP6*7%BkrDE|Da4#@w-U8RE2rPYN06-=Z`WP1MK_GB3^hU?H*YCtXR zIhS>?mgv`ucKm`#Xpsf`y~;7Zi-e;yYulyg5f3gCY$J!0e7O+RI7j0mn(&rSte1yQlH#(ai1E8~Wf$TJX)5X55RRCuq!QVygH0FWRBAG5ar|vAej_dp6`e%UTBi}6%7fZ)v=%@5ZL*Yp zn?&KtMg-SLrO*X43y=?vq5o-!zb9>3^$CXEN}ymq>3v0ui0Y>7^n26R2UPbm=|V!2 zvvmT}CDXT~gn|sdQ*4DCyYTj6t=6l(vaDUlu1j4L6wgF_)_F*lS&N`LLxHlMqAN$p!3g_6aaO7X zOx2yJ9kyd->a z)TN9GTjx^*G0{c=)zP1!B_5%iJ%CJ#ADSs*i5ZeaS+^izx$FHwK=sErJNK%HRc*83 zTQA#o`r?Z6FUR|qx}>S*xvak@d8Q(k&NDcaTTxRKIUs%cx1eCzw!Z~CuF3x4RObHV z`JS&X`my0F_5~mASyENk$>O#9=cQSbDI3YslKH}$#8^xQtj5gd)+md8qal>C<;F`b z!zZkr*6Gd6J!MSLO*Da%(PC~Giou=SHc5!Tql|L7yOuD2jyFBC@m9b(bL8Tj3nS!= zy@s;HU;GriViA{&AC}0!m#~Rdbz_8IslAa8D*36M`h|40rxf>$;|leq9RaNEuj#j! zz&Yuq(#ln?5PAvYklnUq$n_uIIf5@a)?OCn07`-KU@mA4$A*j*Hp}3#08!@;QvhP{ z1Wk)O?;K~xbug4*k!d)=;CG#CJK+6|+%cu~#^{&N*xOm|&wJQu>xAR{no%g%A|%>n z+i#*G)x^&oUiV!hFBBBr&r}%)##WlR0^^4vQt)wNitr*`lJT=el};Tocam9!F<)XJ z!-F^g9nc?X;yoxLQ^q%YC82G83RlGfad!H{^4E|G5gSo_ES&x8!Bp<5M*>>I{6IuQ zW5tHefkvl_?>#>ImZC&VEEHv>&t)1xq{`WH$l?}MOreYgeS3QP{gcx-eE4Cv=?`D` zSjIu^Pq@a;TcJ znF0=6@4YRW28e+bd4wgg9qCS6YMbt?L=e@V-}c&H(g-x__{=wQCRH!Xjvt%)S+^dY zJa(*fD&BnT)WwptzqbrLGp4%w)XIc!0%qKZbZYAKYA=h^j;62n;ve0{*Q?cgb=u;l zTi1PjGB|quA8~hAFJp@E_-=~{@yq8dcO?2aY|ivw&kyo=vUUH*=c^uG&lp(oAaaDq zse;^q>GNVnc+7X)-Pk$o;U6pAvj%5|>2W=CG~yf$RsNjDI1))H<8O}1 zxgjEW`mh*N#b7gLBRUxi@Z}Tx#uJRdhX$swg09U;^dJd8J1_z~Si{zwrnuMr696W2 z@XbF?HiKfp9hWNrQ;F!7QF40whh*>(kEL?)7@I)yPQT(zrr!(0VAWtFpY_?KPS zF%euOux<1~QjI1syw*Ob|9M^^WdHfq#IH$Dhp17!=J=tt#qDG+xSw-VdR0A9@q zXMAuGJkVU#Xhl(PO^RibfPsVSDyT)Cjr2TMcjCfe=)lDYe$kCkhA+|x2)XesOuUnT z=l=#$J(}+fTN^*r=i=+OhyKieEJl!T6f|;@VW;rLXkTA4(=bcUFH%Mk*x9ksmdm2{ z0;8n2X2hr$m-yh%>{tKl5l8wMm~hd1?+s6GnK50Rt#1QC|OZdtuGzG&VNaLFJcP_lheR+jo)bz>J~`K?GB zU%9q7SCdbSsItV6s(&|y1j86255;$CGO7p$n7Q~!!_at@OXD}u5-!B6{{CCNJ4HvP zz3qt44o6XAn6lgw<7;{~S@5d%EmuZkuB&G5Azvz-{sPht1NnMB~EP z1+y0rF3Cl(3N?sWwFY!KV3DLscJ3COO+Gz|)$3@t%v%FyoNF)Jo3SdBnWXaQYkSW1 ziHffBd%F!!Q)u)iPDMCT@oVYO_s`q|MBLzgCt^0cT4X9j!^X@C!7dNrUcYO+cxIOO zC(!Jq52uZfk3By980Rj|9385ADW}P#W+@ZEZ1;a|sDB;|r-fU(V&FzRdzzI5O@9J? zrtcO>rvOS_B~6m?x{w3ivnG7a_MLbOauhb6)@p+gq<+o2y`}e?PXhV6D^E zb!~Fr^1j>XO2^<|2Q?euoV%-6yVxg5t*i%q?l{co@0Q}Vy6NX57k@G4W&b)!b2i^d zEg1LPv5n!gonP!SqwM)Q-xnV!3XWdj<2!@P zr%fnzT<~?$&#jz0WhcK;e9wE~%44>RgW89TPwyL&J@3KIfPLqF=y$fozB^q)tfxQB zbGKQyrbUk`8@e-m?8t&9`#pvfg9z+zXbUVUnpM{1j$CHI;AV?hO|Bh5HNf#GPF|LQp<-iHbD^M zJ<*~xt2N++WP-b^7@xl}FT#zHWqgs^v70~toUeA@LPYE+WpT&}*62f2{(O1-_<1D| z_U5J^%b|A5{=r~Two;3xNAkWwdxV6l$cAKOPG zerPq^$s*q&{hwMYEM0?U>O1x$f*e0TbpJVz52kcok);?2=+f_(s^T)Xk}AOj2+9Yt zvDq$Fc@1;IH0HO~j#E#aUTO(vlbW4!%I?i0b>M-Xyo3eb-;0*F-0<{YUN3B8Eau~T zFyxB}@b49yz9)V8qeqP|YXtRaA83W%Z13#ZDZIkwcDL$e^R@AS}DZ z;El^p3-ui52LL1`wY(Z|@$0fya}#PA<7q{Cl0t+yZPPhJLy$sbm#lr&xyT&CnQ2_l z?UZnXu_pOYHUfph+C-q5jQbq@rp(aVo~odQeU57-zYG~^d~~wT5RRl9ZM?v4xH3u= zQ6}fJd>R}cbUFD^=^mgC?IcdRZ zws1g80}bj5HOnPk%~M%zpWH=FC*iSW>^q0;9yK}dx3GN0rReK9t?yQ zCBVw)F68Y6S`r~MCY%}=X{chk!L}*G5+01j$}i__S>8Ef$B=1nXTA{>gD}%i|LsCz zPJIqli|j@+v5xgc7oxg^h+V$vO-Kt|g8a2a)l#=LO z05L29BD)DE-HW16&%1f^e))s4NTtAM;(}o{es$<)Q;90ekM`l>Uy$;{p|70%HEbXj zbsF$#{fUjI;m$*&3^Fcaa1C9sVyK~#7${i2*-GYCw3-Qa3EYQzL+N5Q(IpI|hcWiG z=+pzxW0hh2KM+6E)pjOTLyTRH1vB8x}{MG;|7kgO<_mo z%~xh(+6&Xmm-wt#7nd&{a{m>N-SRSj`T7ES(7Fq+bn7cTX<~Tml$BE)!cWXikNQ)B zo_r!Qg1jMxXFT6>v6gM*cK0m~^p~}JI%7L;7w&enscbn!Myoe^;NK*wnI`{1a!IE* z2v?7hN5liY#VT3)6sFzzZM-bUU$Ynn5CU;VaU~?^UE1%g~%4Rzfaxao=XyO9c zOd47+7cWfrnZf)Ld@*P`>b8V8{5BPKa)ZZ@nW#}BfXm3;!gHLA9A_eFsr>zP*>lHA z$5`l>wkXHx_ut23uvdSyWt9E6;T8=GUpw{L$x=l&-e@D!kt$-+@lVrMAH>evHt8i@>u|qK7z^i6hZuIt^~`@Q%{_Hu zlY7?nU;gRov3Hkm|Ggenk<;RToxAkr&3c0${yw3&e^Ch2KWtG##^=A?{smZ`OJkFE z2LF~e=vaAF!}!9}kAkhAZJreT!@jJlZ#pfg>e+Bz{m@@0G(Pj*p7&nQ{F{CE_HHgu zTBcaT`)1}0OL*AU`(euZ9pMu~Z{>yDTfRT{v+{9Of5f&e4P3Bff_SG>sdQ5 z+eZmOkCW%GX_|hXA+N#tuikxS%QDpC*4<(-g#vbh#J^m|uxgee)Qi;`?j|xH7PqEF zR->c{9%a9b(9w&$kODKiNpxC)a=JP(C%#>Tp>$tWw(J&H$Lbb-Ymq@NTU%8F6UHj0 zKJ|`X|8k#RYkprZrZ2IB!eS7W6J*??2N1GS{#Y_niSj42QiHQQyw zu;uY!{HgqW!(&D`7XH_`Uexmv5$ma5{ffXdHqc|_c0{=^kQst{Ki|%=(kNVSC!EKPW z*!W>3$h%;znMBIq3;+yXL$g@xa5GQVu#Nh6wPcIOs&*<*x#HlUlcSgr6z zURT8C9^-e99{j+w2cVbjaHaN(sHZ~j?|pFgiU@#m`I29OqW-jD+lQl_FV?%5y7cFh z)XWSpS_1uBpyVw&JKPDhF9|2V70N}$J&Z$46|M@cEN&OJPZPP&y@hZe*zxl(i2J^yk%jZgwz_mjnZ>PrRxSa4-7MWaV=8(puxCo!Ki#6V!e-T{=`Mh@YvCnxpYp@YieM*Ds z@%W?kLdene2@KcDowYHZ!1yaU&y1mVT}zPgPOgPco&** zyMO(Z+;6iX)xasq$`{e|EISdkSZ19>##Ut#oDYw{b(6j`ewzvbJ4XE1iiO71(Dd2j z(~OGj1tR4vN-XI@=4{p#Xke{MArj05I>xYm)`~>s02`X zi9#DDy3@E3@KT@@9t?^ zXlh)O=@r|0-Kom&8?@fzn^w+ldz?V-Es>4t-tyLLd8NwXWvmF3g(Gvn^gG%=iSl6%u2Q+)W7b48wY34W`PY=v zAlc=p{;Js-yJo#^( zr5~xFZSxVEbf@N3Ihvy|fvlJ;m}0LWmIF7wl?F?LMGzC2?JS44@xFmzq=m!QXu;gJ zBXylbALA6y7P*L&qyaP)LETVBxulqG3``c#wgTsz#)c1MJaEL3BUJ^V+&br@g~FG3 z$8N5cvZ59pWxGQJP_m+Vit+n_+sQus%$rAc&1o|_?pe1l(tvfZlo=J0NT*2lib(Ty zJaMzpfaBxAMsywV#RzU-4n=m16ΉM*7B?sdqxmAe{c-j}dmbPkr_hVmg}AVp;D zF(De*JiN~R!bMUvtT&WCKfUPT^j4!y?ySyAt0F3mwnXu4aa`EiVa$~-3pF$HdbDgr zf2ijZ)NPdque{4OOfv!YP^>6rL?!xGDCok{yY=-ewjhnGeM>XOL7HbL3ymug1f>C( zyn)+|4k?%$>LsY|0zs!&m|2z=D@R2KfT^4{eILo3;plxa_kk5%&7r4TU#wSBu!!%1 zM+Z$)d2TwUkN!MUBo2s!eu2|-3w*HB+QwdD`SRtOT-SHgZ1X%4+P7vV86hn!p=a%5!(TtrREb#LmJ3qWvW+bYwJ>+%{IARaA>;E-FB zi6GSIN*2itP`u#GEDJC}tA=zcAZ2EZ7V_K6mv4~pSUyC4!8qK%E~gWj7As`OgaOWN zU1xr`uXd>cd{#-h!Kj{-Vd=jGL6b;@qxY?PbXf7W{p?-mb#@P?M#{c@woGDznn9w* ztBbKGhJk)bWQ-G2kIIzAUO~#M`LULK;Qfhu8o!)!M0LjNnw!`;yRK4l(wmw}`@#13 zS*at>2`5fx`Ixc`^?K_R$I-OvBt$6~5@bmd%ouq5=}Hkhsf;F-FTrFDUZvldv>1p4 z+bX$74zD$_7}m}QM%r-r$00{{-e?oONj$$95Vn-^$hb>}GBIQiS9N1EEgGA#1jpIM zIGE|lnfAXwIsaCC?5}(Nbz4_L5MMg@8MP3Ws`ZTo$sU8U>r?WG$Xqt4R0~Ma+>Sc4 zzQLLO`rhCTr-K?oa(((M-=pUqy};eIh%^TGx&-b&|-5m<}a$BD2A+gNX(2f z_Tl8%BoTv(;k@S%55)4g9#Ky|{F@C1r9TJ3p6t&g*_hgY{)S)A?X4~@I(w;LvPMS0 zJnHZZOg=VDg`TL}yKa{KZ>a?Ui5Ygq#j6?=wyQ&L`Yg`(g@HnpOfR!Qqs)yjyhXh9?f()k2OtJF==c! z)Uq(OQy#Rojzm50#RSp30#;IdaY`bnOd!!EM6oNk{$6bLJQH#%vdl#gl?=r+Rx6#W?~!O2 zH@YT(!v6k?xL^wtz}ET1Sbmhmj&`yw8QvLVzMk7zHcC;?z1Yz7?@6AFs=KR*(PpXC z^=TP1V{FeyS!1uRb6PlWUgI|o*4oo*liQ{G(;s#0b?4rXr-O^mMX=k79*m8wupR!gHrj`B6uWLr^vs0JG6&Fr;dQbUBEr0nZL;eMi^9NP` zk=oVE*Dmp}x9|A@aldwUDeatM-MF;hd6&U~X)AWOS<%7^GD5w6-|XJlY}AbXuMVHd zaJ3?2v)lCM7{16duSA`U`lx4gFevs!Y~0IFblR<9@8T{8XePN|pj9rTKg6}hY)vAt zJcM8F72wHli$7!h>3PgHs6ET&$vGtR!}jsyVeOM(NI#wP!)LF~*{j4eu_;ID<=ZhW zk*55bCMT*O+H^6-VrF1Ct{Kro+7yX7i!L~OvuVfomwFsOckUvbNN#cHYL!NLi1FPu z1A1Q-Vv}%uzS6UmI8|ZXw|MeaN{UVq=wZ+JCJ(T_PIx+9Nx}c}O+9_NFLd&f{LIKh~d?0|R0Afm> z0FmX(ox}C;%#8k*K9^GT^jgA0N64)D(j$!lP+C<06DDdw^T@|@04)l4Ybu5

_x(-SJAvdk!Uo|BVuP%E>{Y8-K*#qpRHJuCeOpmT6NLnAW$5RJc^4>)+e#l9eNq z#$X^@7p_9+206tQVolLaNoidr-Zd~KCXNqODz;RcoT7rM1mfx>nov&p_Kfzf6_(1$ z(m2s>S$;e3EOj%aK#JsDWFY|Ebyo&~W(>!&wV%sI@oxs^Koh151HL-44e-=gRI6J)*Qi@+v6WseT=ZW)Bcun(6*ZQZU8DnBPtFg3dztll3^4;txFt@GanJUI1!Y;0^{U_2==Y{g=(3g%x<#^~bZsVNn9t@UT04j*Utbp955?mc90j1x4ZNH#@nKdep(l9vfgWC-0KG zj9ndInsBl&I-Bfe;4W@VlM_?BE55At4T-DIiA*F7e!lA3GgT+s70>#?ig7&Dj87(YyiWV z2A2yQr%xd@H>pxjk{*1}Nx{$9Kd`p)DAX6v?hRdjh1mQ~|nd?{7 z!=Sk46^^IDKKq;M$o2eMT5XO9iX0S_)J)=l0D#7()Tu9O`VM$|#2WSyf_6>BbW@sc zXZ#Di9_JZdfXwXTte3m2Ep+cSx%(u=4Iw2J=b?RBYaK)zCpHirO)*xY)t6C34wDms z-0c{8H{_~f1pRzVLEMqTk8-2{BSnfldr3tMS~zMJH(jBKCJvZB8#eRO){$2rnyWgG zFS_~jxlvh@5Dw@>Y+VNwcZA<2TT!Dm9g-8q)VuL<@P?1 zg<(c*k8x}88SR6+x)4c;jdV*#f}W~$KC61g$!=xlZR&a$qjWR{FDYfCW4>{Ckm!yc@ap0&sVR-lqdq#UW_C|WY*)OS$UXnpcB<*m|iil68xUce&>-!kBq=gOZ5 z8c#J>Kc;>`pl)Dqj-0Y-x$UqO(=qnbg zghBEI&m%Wka9h7SoIRWW4`XK@S9O{G{ez-{O9-0#E`sKo3tHfonu=TLl!dwF7NVm_ zCL{{3K&B{Yim0@VR^#B35tckLE!O(WieIj}IHk%~ zC#r!1N;z>~o9|)-`GAHE`bV!6&hH}FD_OOeY?H8~7)q6T3m9oUqqSqFmaveZ+wSmY z6{Sp6dTSI2zC)e_ashbV`H1u$DiFiST<>@^j>^Bj% zoXQzrW~F}gMVCpL@*>o&`|r~bVzTp-K+e9DW#_MnaxSOb5P`ZNI7eg)(#=qdbdeS8 zdN%4lcACH#bfjKAF}h5FcA0`Ux37T}0{_`yO(zS&VzDtamCAc`|7Z>m`h6~_yn)8O z8}pJ^DHnIlu|kL1Uj_2uVkP-~zc;R~fe%|j#?UwGmLi-M?= zMVv59A~5~TS(WsSx9K@PN!R!i7>uSlMTblaB~ED3SE3zN5>AG@HslXI1tFH zVMCqTP_dkD(fB=}h~9@%z6)c$CbUY4W!Onw;iW_o-S z2cb+JVsv8XcSZCdtGvksQau9=>7$#p0p_%q<)rK4SR-2ysadgBnVHd528zVsBLUae zDMX*MWP?&v9>|!z#3Jaki9C!ixSzd|d4f8Ly`@0{pcMOzsP_E~Tn#<fhLWS8L@#j!C1B2!eZ;n5>E4Dgd`$bu+Sdj zn*)Pq9P>XuAOw5r1cclBHJ!|De>dkh|655Nu8V4gazvKyu#-nSu_;L#M1WK)j2r;KJ)mDEp>TF|J1G14(kG-T_Y zomKarE8F8$ft4T;HgLZ!it(kv!bO0eh2Bp&;z!<^<(t;Yj=|@OQYLBy$N6$Edrr*6 zuF9c8>;+z8;W|#eMQ+z88CTWRa>z_EREpiC^A<=NC^Jl>lvr`<<)@#+-``dPCoVTB zsoTuyU&c-KBy4How0W&ue&y1`agV)KaO%mW?TF$IzT#=4^v6(Ae&b%3Lljbc!kB`W zRv#YMsbu!28CS_@&>4-yFUoDio}fq%5+zM5QyvE7Ljy-_35AUAuGcedL0p1a6# zN!q1;BkfYp|KlKx-oB5*wgg^xcq{w%yUX$8*>k)#hvcc^}VwS)IfYlOr9oLc_R?zv;SeB5o{ z-4=1dY1Vgo6Mh(yzhcaxu%>-KRF+j0RHKGc6x9~sj`o6;*SQWW36sw)w;?CT#xnCJ zvt61ST-O!?Ign@y`H+p_JbX0GPz&PYli31lmE#Q0@qXhmRGJnFEi(5p(%KbXyWZpJ+4PWNKqS5Rjogqli zl9S0p52QlLCBo{bj<1%G<~1w8{Xc z?frtsTcvz!{j~vW4s<7=a6e~T6>lI%t6D$gEtmEw$Z^5m2-A;2I?G4bS~WdoB2d3q zi!Dvem{wzn(fALAwS3Ko&MR8Ab0O2sIA%-qIDkEfzM1%q4SIL8>WaKyZ~LjXE&Rk` zO?OMwp*YjTiOCByYE}dz5!HyHka=v-@wHZvi5_vJX2&*vnD|M);3pbbz&?8G(>rpk69X2ZyYnWkSgV^ArP&pI5UEt-mH>1 zjXi7lMdl>sG|KN!KHy<89lP*SoM(t|@w9{17!*6V6^Hx^h*WTwLP93U*bS>lL|${+ z^7@XsYmw+0<0~jK-vs4V#N z!GG2C-7$eFQy#3a{&F?sT*>o&I=GCf+5%CAXu;IKkF&N@`GTy`R9fd9f|t%}nkfqO zlx$&074!_x$p>W}Mf7ZeV7n#bq^@sg5w|u;0IFe*QfLY+iw>{Y6P~Ap7`FHO0t}_j zGj%P)Xdf%}{S1+u>70Ih821#>Rab8j_#U3kIm>0|uc;l&zjky^L|d&KZ7LQNG+ z#B8CG6eO$fog&&S#5GEU6QLc?k}u-kj;`q*g_ir~JH1ngga+k+5@KKU$6t#s2+}BJ z_z}VW|JCv53%QJN1(O@#pO&K6ENIUmXV0!#{G0#rmaCsdz5+mnggC8ab#kqEVO z8{AUtl$NWHx6Fe8k%!|;MPr?AJ2WQ;4ciVcAMbmvtc?-%sv$N?Bo+Df6Y%EVGB?`H zp+FRF2$B8EEa-d(3`s(<*ar~bot#bP_k)(NA~a<}BaySzluB{(h_Sv9lgNn2-Iswk zX3q9ncqV4o4vV2^SC5=9(XfAC6OO3;mWQg0nWHAFFYw&}O68T$Zg6&Z9*>5({O^0?(BNkPR8>uydr&5y`sU4DLUD}Ohe zF0@$3v+?`-JODa!Bg489wOi*YEd%WEEH z%4bX{O_XuHrd5qpPeR&6)z}K`vpp8a&fjZ?1InPTVRKu_P~+C|JX&?W-XtTUwWh^) zN>qq(DZk>%II!wjX{PnPU9LO+(>Qw$kb`lT2M!Xbhep?8a{X%7eUpJAa``<0k{Cd^g?@(F>zyP_gnSnv}>?9)`AXIo+EH^M`W#{ zP*FVZqjPzxGsFs=hj6O`B~u)3Bu{&E2L-BfgUuIId)JFo&-dSf) zO+yS2p0OAHAHq2Od@eh{eRcl>hB1TRDhmmzKIXgh{a|WCdNspJr2(0ptp2`)e)?yT z7dm5Y724_3Id%6idS24Cq#fE@d&-xmr(V5JSNvGxDJ$OcbQgiN8#?^nNwh?0W|-)T z+(b>Gdw5;FEr+rr#f(C5t51y2CE2zlXooX}rXXomdjRFW?ZWJ4HILvtKEv$4g}&+u zhkA_R{7|Ar_V5p!vCXTAp+c=+d%(}3CMk&-DR+c|)YjlD=k}{}&iQD@U-8#qha0Sc z@!(#&dy3S*Ptn0f#)ibR)yrVMAz?=c30iW(M95Igs;>Uo)`T5kI{<=7-=T zcJ@m`gPmTf)40<+_U>&H!>bc*T;tbISyS08@Jid?{`u_j_=$HPA1=!48}n_!qh`Be zsv}2DUfLk$;bhl5u<8|uVB2VG_Oh7c1(j=8$4m)0omX|S zWBcWmr^3!jIFI*f@wRgpa@sUa?;?h2{zuW!4U&gGuvbQR7f4W6ygCi_g`Hsf`v<4j znbXdd?uCbQWqD%9s*%HTnhpC%I7mJzZ))fFAJ4gUy#28JX~XP;*|Luof*XAn08Ghr zrYryjSd&=@ZPw9WiN^X76@Y{I^!Pc3t35cBabajrSr$s)=` zs?dGZPFW^zv!Kb{?1#qO=+teOF1TzfisWui`@0`a#P5_^SFUCi$2-n2nNztcC63RKgf7 z$oKATqrZCA68OW|HMR=Ky9+}y((d_~zVg{=XqdkTrZA$)5${q4K8}ndaAP?T6KcsO ze$h$&*5r+H!0mc#VXK#0Y`L&yajTbEB9*u4f+RGHIyB$Fcn6z9mpAY*Ll1_2$1(J3 zVyqeNKm35+tc3WQf&kVppEIL!zA28AqB*`b%sLUNk~w56i7Suh{M^q2Lr0Z|8k8P`AhAvT(D{cYcS+tpf8$6n?M>vW6( zdv~m-e6mLJZHKnkiE#LlPE`El7eD8iDAXMoZ(9-bZRD zwu$W)+wEGu6Nk{)(VaiF;DZAf^fQ0>Zx{7YP5l7cvxi)eQo44mu=5(gaADb1aeA%X z@(t3T2nY4Jv2&->4_-_{6j-2gTErLPBno0J<&0P5Mi4;p(-@x|xxKxRH$xZcs`oHp`s88m}iOuep;f62}{)T+*=tiX=#lBxcn|bf3Td*{rn! z$itGOPN$a;hx+rJjsb38yIrgXU1K}Go+;?LvKz?-ceE*wIM#LIsSl5^=-YMfTjJDr zo3m#RAVkEFnqzKY+M+qOq3Z*dr`OFE8yV2BX=l$bX_Xx7jlXl0FS>fbZBm97mNHg) z!J{H&N)Gbf!(yC?bUDRqf?AC&s>5^QX5AlTrnu$1cTNS5#J2RveQq9<1@Q!yL*>G9 zt>k|Kzjs~J5Z>*1QhmmaB!=C)ed*cJI4FgT61n-!O*aFVr1!1hMo>dDH^`^(u}~wp?6Ma-nwgd zV0m)FjKG|s9u-G+jB)LMDs^*!>$wLl`<6@#T5)C6(cj+j?hurG`;-ipQ^W9c+NJ*RtEI?xsQ!J)7|- zb+Jw^*?I8a%?wn}X9H67bL;yG;gz6%Z_^7^JvaESf7h}~C=_k987u#rs#_e%%aI3u zpHJl{!^BIVi74frEo%Mc1y_m?ISNWmhsXaKOW{0nAx0-zc$imcmtSVeAU<;7yC`z+5h`8yoBerYP}h|1xkQoZi;)$L0t7M5fj}$_4`@@eb;m0BK<{hw3|E)>H)CIr z4M$+5y4p7Q#H|fwo*5~F>_Rl*LkU@SCM8c8BP6P+VoSBxzLm~*4_TYb16 zX*-Q`pAmnE1#1#>SFdhhum@Ghi5hXB!SgW8k{n@S3RnOjsvcTvfxy)A0AtW)dH|;d zz$0u1!uhIa&TJn{z8^mxy14gSCztQQsp@7PfMHs>tTl0Cewj)@r?DZ=`XdRNC!+jr zk-v%24rnOfzjW?%xK)G+xeB&rSCDI^n=U(Dw;3J7U)PU_01>*qB?Q z4XXIPWo47aY?hLf#& zYaLS(MOjkSCw7tmeWw_dTnWO(LAa38*MXV|hOCeu74GRU5@}Onl1)EFU-X?2hepiV z<>!sP`k}DY%lHUE>>zL`sJae z1_yNqm6Z)fH@84MJeZ+$^)O%%G>xOx*cSahx#)_{s}d3Vm5kAjZuvZLvhk*&hcE#w z7%iEZZ#hkVFD)&Q(rI}}pcxT)+N-oZAWTK1#ov&hivu8wO-G9v2#8HpBK-%$o0V$$ z_${w!;(auX6!B5dP&<8}*JKR=c>ePRnY05KN)gIJ@RH5xP+_Ip*UxP=WXP=s(qGe~ zIBy3E-HsfSBOGE?f{jpZ; zdx7^XIy=kn{6zm>=XFdnmrcUpFrn;y%*!))Eu%(XT}MbS$PXhTQ39aOz?a6YfLvsdik(9 z>g5w;(tzOd{=)>sLH&kd`M6D*g*>(Yus+KJ0F9H2r79Ex>mu-#3TZb)h&tKm5g|Bf z7?c=zCwWb~Wu-35QrQ6|#|rt#{`=wTdOL*^S1-Zrd)2#Nf4UQ_ zXa*sAp0z&pSmO5?Up3>2ZT*o)N5AvxifszvYF}r-UE-D2Ev2cx|F8{9s%96tFHI2a zZXbOpWgq5~y0^`*iG7Q+JLDYV^5ZKP+4O zTfJ+EI5axT8c+;Wli#8q4$SsUC+QV`_%agp8UAn_B^r<@0U-=Yd2Mo8H?459_YSkMi$xq#?9807P+g~|2D2eJ>k z>Lm0>^U;kj+c6Oz6U80s3tk|UE%Nq1l1ah%Ys4}BoT2ziu^rP;7O^Y^Gc_A8YJ`C!$Wo2zmZM=Q>7HaMo{>GE!I!uUicH+rcj|Gxl^g8mw@&xZh%XyYF9 znh~NrERaMfXuOQ>5dt0ha_%VLwgWK_-^XH#=#=rdFcHwC0blIa!n6|~OMV&de=#ZG zbo^=`xUqLXq!InZSn3F!leK{`j`Cv;m2=d)Zo^-~F%Uw`vlyu?yP`xsggizQuhn`f z^X$)=9L&WHKg>fa9ah-F@l0CgnP>a{@pj+qkQC@#>L^*Q%^9XXy~R1cj-*eIdIb4< z{(cex_iH@gg{f*@Z!*k6<@77j>nVy@>JMb23OJQ7wkMqof2rKt|NEWo$U0^5rit&+ z+Pn##%~zoN+{fA@NfyFcJ-e_udr0V%D6ShFAvDXNSTFbM2hN5!82Yb~#-~usLWs`C znV*s zczrZcoJE7O&wH;y!3@1XL_*Gv07+@h>nwkYh&wF@O*dy z9PMXMRzJEgR)$8(oDChT=5TS7HE9yF>?7P>w#nITy8ou>*_vN*L~$7*6aAJSOIHyh4oSa%aNq~D zhsp%1TU`}kP?V6Nzs{A3RdvtNwwz32i>s=HGas!rrI;ofk5fhGD*f+u?U*QxmKYVR z5E%Jj0#kBj?63&&7UZM?_9X{0NpJw=6PO1cwn8%Aq(wlf#&DUQE_I=w_?%njC2yM> z%>#exEyGXx_1AWJPX*YJu@U(tid8Q~`RlqvRMDH|nIejvjPD9TirQb9Xsm8|4q6zQ zD@CbBHz4-IC=WWO!yn<%uxBrp`Ve`x2Cq~)9gGXEtXTwJo|5GTO<+CjKSI9u!a+E| zeC=q}!uVzxcZ?m4oy{6e>0guq;T0y}SVkDb3=FNr{L^D>&NMx$n&QI=5d!ucs@Z0- zgS>nMahudra<9!qLd)h~mD^MY)1JDS7f;^yo5?d2{6UMak__628Yj!a2{wusaW+Mj<=e%;bsl*9dvfaJGMYq>x{t z^nstk6iBSQE>ghN)=RH1d-xH^JB`Q;<5w*iA%&6rSkmCzO%NqcVcjb*ca@sR@CbA` z(_99>5$*4Bq$3y_B#jd<>B>3qAE|hlY{rh0 z$R=%@d8coea*Vn=AcHm{Z$+(GzOf$N z^R$}a7aRpdS}*(m$NeL{7mIG(NNJ+mFIq7TCPyhM1yKTS-T6NQH6mKck=9I>E2%nu z)!v_oivu|){CX|xeAafJ8)Nr%`LTS^%tt2#D)nzW>);n4+b>ygLPe_mc?;8IB_#2y z0wjmUH0unOgbt(k5?$fK9vet#%jP>RVeVX?7I82KZ8 zOT{EjE);yHZ`8hC`O%NWJGamr>|+ ze}fP+tW@)cbipJwI=Yde6xb;PZ{~E%o zWn|QUVl-okU|1=SF+GRbQK z7rf)*+ax44tTk<6lJkg^uDl_g6R`^{FZ*gM@m_UB3BXU2C4hGcJURY+y)qvt{OjhWzv_|tr*;9nCs{Wq|RlhB5mf7g} zLDgA@>-G6HI{f_`O@`fX@AkqQd+la*0%H|$FN=gO;!E#kE`5`21_gSJQZ|*R85En* z=3sLXP@&{3Duq((qDeE<3;2@wPhtimoJm;&H;WM$eWH%A`HdgGh%4@R$t?m0M`l(B zwRO>JF>w66gkq#rkWF3l5;ablv`p5D4}%(IdJIMieMKfq)KY`ROub4BaII#Wp_+7T zo9B*jntpb|FLF=O{* zHziD(zX1c3MPgfL&H-(P7wg+=b2RBsSlsa{<`e$M2m8T`H=pKdlRzdXj`O+0l`O{r z0k8&nL28c%5g@9ycZkdRGmEm*06_corkRZ24n0MK?B^M*lSDFwuEQ(~m=FwKq)nj# zP83RDBN7$QF74uC@_2G&2mraWd{P&|8ej%-^IZ%f*^qld1NIu00?DDUXE1Hjxu|fU z+;f!=WbHDlf(e?Jz(*Zl<%~w&k(7U(&>_AC%awtSETRZmRRq9m?j#S~C^4G^ZWaIy zMEi60O@Rx3H@xPrybGtEMG@kJcIY;|^5;j<(wD&`VX|7yyZP5Y-C9lHtK3l~(bjNH zxXeJ6A#!%*`=A&Krs;lCQw-iQ$8*T>)`5>Jm-;Pq9r8w$S9wru3q-wV@_VM1{}6vW z`<5$ui`sC(?H(j+w?uxG6r`4ST7)?74%lc;>8ge$@w7 z-eV>@pAgKxiSW{${G)Q0sL$s^KhE$RU0k(u>KAcNxPh`1`SCGML~b;7WDxOE?53#= z9TV%0;cqm@D2pT#fmYCt^MmQ+=#>{Wrd9eB7 z-nua6Qo0nBQT#+C3&E=qtC{0o*RJrsle4)7k9gd^UZ+6QKUftI4T9l4QyO8f=1Q^7 zF3wy>w&pA1rUgK?`~AprsgI@wqDSp623@lBZT* zTGf>JHJO9`U`Q^fTAu5NdvgbH9X^D$urKWN5u@is*DQfVcjQz-8@amhXasPzGN!0P zKv7Q<>xST_uxid)Mu(=FGC`;=V z&-tmNpA3U3CfSvEZb{qiQ(Hi!Oa$rYsCMmQIZ30(f~ZZTydHBNt1}x%8(it2U3&TK zb6?eAJhffTYt&1zgE-1hz3Uy$9QxPWC%_-Izw^V08n3N5<&5r-n@A6AqZ~F+3@kKg z3$I+-a#}~_uBQx+tX{TI4*u2lxvb)~nStf29{6P)oWHuW*ZQ=YwvNF$x$6RwcJC`W zSoBT35p7V=lt{oJB>GJh?~fp1c7FMjP363)-b<*ZGI3<-|)<;v0rWFh~d#G zI(Gs-YT&axANb9l>&u)rC`v&Lxt9i+JmG(HQULxIu*Vh4aOezD8YfY45;f~BhB&oa z4wZGLtp4dn1!&DUA!h4kYAE}&1!p^0to&M-8I>BzwuUNf5JZRtwv%U1oA;L*o#4;@}|JUQUY2CrQ&iatKFNvOsCVoh(SjUU_Y4E}7_?b&AKCBf>B92T{qs-oG|g4N1A`2YP^NA`We zjXW+5DqtAa5fd>C@ZX^;N4=O|?B%rrfKg-D@8VTN89ssyBbo)^5hXMi8$&lah^AQhqyP=&97w z%&ZuxH7lPk#a< zbRkHPI}ewW1?;>QV$CL2at;r$wQM%%9Gc&lLcXSTn# zj|!PRy*wv;*yXaCyB`!)|Mu*Z^G9E8^!sZ&gPoe@1`Vnk_0KjRdSAv6EPw2~aP<24 zf3|zy^KJtM+Lat_;RqN}jQ4Z|*^X1>tkklCy}^qp$d@HT+KZkv8Bfq8w8m)tVhDqk z*LClJpKkm$pCR_qiqq#;w}S()mtmqqo>;t$_)Pm4s-#wlJaY+y5lJW}A6@E}fbyh{ zS5@F@$npU_JJJz+7FM0mERMPqLl1ZHy3<65K5_Q!B?PpU!nqL@QOu)d_^*R72JRTf z7(T^*!^;|(#3k8yi7r=;w%|46@MT6Kt?COj=J~CxQ#`ro_ibo1YMP3HO|F(90vx%o z_Cok4jPsu@{!n!XsLzOG3EoA+v0Ek!;z@lLfJ2y%;O!0;_mt*zGJ|BEFg;Z7Q`cC~ zh~1>N6vfEraVO5G@0-07;zpx}NuP zbmCosOA!{c|MV-as{zjul95QNi7PM#_HeX!Rhwd2AEA@L+92JbOG~NZpk)7qn}D zv9DdF zxw6DX6z|A@=FTZUspKT2kpXGk>$86Qan>%)Fcz^*8Sj#U=w*c4O1D@d2URRRk|i6L zQtPU^u6B-_8a#Kfc=ne&Upi#-Mch=Me0gvND1;W~u&Rx=MsDbU^UorLA+lmHdt^Dn zllM4wV+Sd*5*Ym2;yAjj?F7O|z@N&|Xr$~KtVT)4SBRTC2u#h(V$;=r%QgKvzhmFh zA$4W)dWXQr^@^-7HhOU8pSOk%Ts`5$fz+CXb=(%c?bG?lZF~OHxZOWe4|bilzNV?7 zui%LJ;q$0X?+;F-+b0Uuu|dHg;LdhWP3w)dB6V6r(%qbSzC{k$&BTWzh)VKy!$ z%tHb`W_#Z}*S6CD^_m$GG}G|~GzT-@jJ^ATW}Xrw+|zkgeiRT)*VaZm=4I9UlDs#u zocUqGS!eC(~}mHNhDX6L$ZBn2m>}_2xlkt1(boeV3Oq ze1?3PG-3Q+Oz>TV1lnrxlO+Wbi9v%oOngK5v|sjQ1+!md-|di1;m+H}h>*?(m=cbhnX?jhp@B<*tc?lGnGL9O~4mgyBFby)#OwFp;9n8o#Ro6(h-oSWLrHv&m@>2AG{Xp978Q6EV5`zE=Fta(9fg`wuoqDyi&_nQw2P$v*FrgCRZ;DV*3^vmwdaGN^;q-#&D1+n$m#B8jWPxY2QCo5q$$KwIK{~l@e~Kp z8C0_cteU6cCFUXwKx~4WL~X4cV=Ep{Q-&+VL)MN;Tn3y{?zWlfR`(o#!A(gEd~j)V zBcntye=-UwKk5=jqMjgQUih9y!CpR~0%R#kJP_M$4iF`xOi;<8fo!7BQyB;Zaf_YblMoeu?b<`o-_e9Ei;UtUp!z((Q){=aYb^v`N~+dlpG|Me$p z;+y`*?)SgE(&Ma`gkdKw*gMgWf3_gejL#jkfv;}*5pfO=X!Z1r0D&5O}*5PCoGRYE2YAb`G03Y`31WL(R#I~7B;!ISZ<8klJ7s!A!|uzZ7bVY?+=Rf}-w z&_1FMV20i1-IpLeGn)^VE=<)^?&7 zHn-;D`OUQ-mr*XUr?)p=wDQ-MCDtlUzex*f@YR!RHMi_j{lDtZkE8KlUt^~`4dGli z0aRwpZ5DFf%OI=L1~uZwRgLAf{(ChBwR~>+?wVhO;2H7^If}^_S!eT^PG<3$3O1!o^z8BbnkkCSK%0Z# z7Rh-mobMS;ugS8JMt3d&zAP(QiGLA?N~l2336#=3he)gWG{cjCUJElW7)azr-q5LN@PRuK@*?*_w zWMq#`JyVBF=3BIokWT-5;`vEXk(Gu_%G8j_%u2sAbNJ0n2z}u5b_?z9fkZCMOz4P} zVED-m>K}RJz8St}v@sX{?#wX_2`9PngBdVtdhRoRBMbY>xH<%#z@TBOpL%bM$&=Ph zl!$M{HjFz>N>Z5J;#kc?s|T12_z`(15aC+5C>=(Tjta3na{tOi+UJe7Pu}lf9J!RX z7_zfJ>Z2|ikT--)VJqHHb>V|BFnYza0f%|#kv_06jQ zD-hc9Xr$F$K)!O~mcJLddrjyc`5$MY4#38@IS1n1S3T%5b2?hwu2M)>SKH{xmjl|7 z`w#`CzYZBky*Ym2iu3rOi|syS2(~|vG{3=objb;t0@sIw9 z;Vt}0oO4akGN0gYJJAJi@LoJxxnAQK4v~7;A*mm;L6mXF=eg5(!;2+j_3qtaaO{IG zy?vcr65P8oF#<8+OqL}8G0&{YI)M zs);w)df~z0St^W%7t=n@zy~n3Qy08Aai%%>?k`SMjLlrPfotgrn2#me95yzIxUk2B zs0VM1dXw?OOHOUuHiCeW2eFSQL#tX7UzGZJ$>Es95PA?9#h1x`7$GtoLs8#$kkFZl zM^MrVo;NwTTLVUC+_AphhOkCi4A=4-x?`3i4)5Gn zMpQ6|bZbxy1VYS_K{4jG;!NfV##NoMIaa=!PVrgPHp$S*;V+&!fYOV5WB6pUlNeRo z=;jTm*v*;}FP+xkpU8}f3lU6%5u=fzu{If02otzMlu7v@jh*=o#Y9*hor78Xp)5XxJ}-+6hppHZX)-OCwsmNuZIT^b9#_>36q9w6T5M{`N~BOoHuV|)~fk! zm{pBhc;djhBECIn+;4AKy0|xcN*zpZwS&tiH9@>%J3Lclrlxel8v|=y2U%*Uj)hRs zLEpY}XMVe}5XzY@^!pI5qRn!+34Fbj`B-|4w&j_yWP>KNlP3s?1k92}AKBkYZmd7r zZETu&D2dB8T@GLT+pawR^Y^yYHqRIS-A@-``E%%wSLP?r;T+Uyvk@ zvM0jFOhq8B1+XBLjKRdR8oXeyu>}y@9y2f<5n(NNEd>ZwV52f01eXZg1#J<1ID(@? z8FUOXqf+>;HCI3j9F%C}R8q7V>6BoFq<)x+f&8&A^CBohzuB>o(#&r-^;Vf6gjzx} zm*sa&oC>u~$F?>RB9`%u#z}UqXcI5Ev4QgFg?RPk(Dw3y5sEEf z`}VJT+SK-Bi}>7$&_p1C68pra{4(Fb$M>H*XzTW*zJEA80*1p&TBfzd{OsUT=qa~j z(q60i4fd*#ol^^3VCYscHn)-1`g(=6WqIcL#|?1ss@{yo8&d94uL+vP?U#jEp)gt6 zKP(_bsjfgQ{{jOn*&w>65K&>Ad_R;3oP7qrU~pMtK!{{=X%c*Whv@`Ooo1LfK*&=p!4du;IdW&D%^*k2v} zAKi#aBs-eogex`69Gf#=O|$w*Y*#@{*9;vAipy#07+j9Md^dq`TOo3I14~vtf6!Ku z0Z0|m@!iny(?fAw<9K2hGURVIJn83ES=SJ#yrR^+&C16$z#UvX2Xr@>&x|{V`-mdw zcd#Nv%Xpt3n4y93WlcnuD{Kt@$cONDs~INlm7#kfe#yer?%~zpD^cONT%gM>JR7-4 zcZPg(3ge{;Zs14*p|cHNj~Sk>$%Pe(`)>2`F>PBW5j(NO17s`FM?}#?4wryCc?S~6 zfv}aJtM;g%moh|j<@6ax2T=fIb=wuN`V8s0&QU4yp5PTqKQp;vhK{kJ!@G?%zR1Dp z!9f$}zZg1;gg}yXEP@y!H+wT^>Ff|-r`)@iFFW1%L7sv^KpiqAC?p>%Z?DghxuhNP z7(!mcce%O~qLQE_B!YQ*o@Ov$bwLHvM&spmShjl$MW&1dRbCq@R-;K>(Cq>8mKVlg ztQ5l+Fjk;kWH!DQ(*F*`CA0IDpD1<;zUX3QMo>1i%c?v({0i=CS;g|a)l&J0&#+Ks z0_m6H{iMJ()Ld$$Dvqw2b8Zpa1}QChrPfIvC1Fzvc+B%;t-_fSgIQk7LcqGy@~>c^ zy9pu6vU|SatLw;BL<}2b#QEO3OA_~h{*9N_hbxE z^w`37&G@AV1ze6E1MG7Ppo^rJcs&4K+;3yEZq(zRw=Bv4HUY!&f@Ci0di+*xT#x_tBZOA7hpbK4)gupzCN31$sqw0c$2J1g{Qsp+!H&;$J}`cYqfuw+;X;m{gN3klPhiSzGpBwi;YWssSx zl~XR8Ctaw-jdDQt7pzLE)p*%i@0XK~N9m66YzA?MPvvbpEj`Xc;^lf4Y*ZszSc}#>R$8MB3}w2{*Hn2P z9g4!4eA`ia8e0*0HC%yEbB$2?6gj6_A${x?6ZBcp*)Axl1kTuBaHQ#wbhhlS{IcSg zhXxh{QlBxV+%%w4;nUSpQWU39G#RPyDwp{2+)jyEqv z(LjeXx2{XhhkS4-KVJ2mOL*&!C<$2Zn5qrit$B(Ha?(-^2;1%7OErRmmuDLnYA7Y6 z)tJU^^4({EKbk|SNDDcetmw?k8=(yP&TdN2vt;2)+-s zouU=I8lhbYQyb5Ob_eEG#!G^E{iJ5!@Jxe5yd1krD9JVpug`NRoPy>bK9hB*VTaNj zCrs8zJrrnV9!z5qnewrRIw+y&#wJw4k*?vo#{=Z1e+F-c&GdT4x|{->fM2N~q#N?g zk$=LelFMz4R8PG`-%#5dKXmUF+Vp8jw+oi6eTywGm_Mkg_r+Te&%ZSK7%594B*;RE zXpjOJ3a*-Ycf)pQSg0iZ3*GK}_YNRC^twt978^9x7Z`H{B`^eCZ43s)DZ>P5@QF8; z6w4=PTrQi?Om!NdbDB%w^i6XyW7(SediJl&z$VTFSp_`l>yGbWX zPL4xR_t-(_C?iAV&XEm6beFu;2`F{fR(M@gDk(}xN0VO|#zW0ySLrAxuVg0dkQau^ zJhqo+W0#qZ_a%j(*k8P(ElRVOtAuu*xzyX5dt)kzFD;waVDS!mE&)1N`2EUV+}%}j zT?b0KF>r3%eiY_nL#UC&g{m7&)}UaV?`P+tfu8wBpXthzEZ|}`7YO>ALnCHUb+b**4Xk2WU6eB6_6nqc zs)tX_X8{~allZ3SO==o{bNo~)h%aCy+^(_y|J?A`9Hg(4c3i|6p>HYAYm#gwdzo1+ z3Kbb1M9@<~Ll1K=Iunfq|07D=_gGv+gE#}RcUL@-;g&PT6WLb(GB&N3)%~T*6I5!lrl}YZi}>ki{;UXPsEUX4jq? zmYi^#U~s~3Y~JY|fWhMorL#a*P>xb~scLg#vCa)o>L&s~>D-Qg*ucBnF*7xljR%C_rgV+bAoNvCLJ5K>ReHoTYaQdPBb#$3bz_ z&c^(3fX$VOe0t?Q%WTUel+uNik>CywMC=j9#l>%Vpvs+wsJ(VJ8IR>^ zIh1LzS0{~~FFX(u#&wl-uw9RDA{1{6j36d8hjLBP8CO@ zlcv-cj6bkkfDCJ}+pOG6ijfq|jgn`@ueaQ!ElOikO_unadlqsrs1*W>1Vi58nGC_Q zWae9|e;Y)8E^E7p{KEUE3ieJDb5S0V;-HApoe?bwc}TT({sTB}+#wZU{#N z3@8;!iN+I7yMUH2W=0N$&t;u_>vg*aAjdhj?I+i9Y^-vd-#3*C`Q3hpotxL zrmy#23_hj)a2GoCRi6<(7I~Y+4H(f{5z@BW8L!HL4|ko5qNo+?$u!HHae~(b4m6Y8 z$|_Cwr)7N>)yDBAtr+8Fld(1_;LF|b+k*<3xvv>h0Znl`wOBaqL>zPmGt{xGpSOu;rMJT&y1o~1}3nR&)QT9EP;dAyy z8K_(=$;+&w-G4^>1W6(~af#HU2gYHe46I@!8y8~`%p5M?c2YJAbI z)T{*#GuS^Wdq5UKdZSB}j<7Jf}a%_?J1 zqZa1a8@=SkMFf=!Gsft+VSgNVnL1dWkoMvXMbauAEu?H%;IAu_MVV-X5^;pV7(xbf z_5ZzE0d(Rj6oN&nYtEP8Pv^33(*C=@w3l1INF&Zl4eHkz*|WN_DBpcL(Y|`4*GX^z zz29|ADLP3Y2n$5{KcpDs!C%5bjhw4@a78ooD1B^p5)T|P76LdD1^|=Fz+&Q}>m_&= z(O8OsiN|AeIWkby$dU@{L~R&IuXb1q)2&M3I&#>UQDO{aLYl$pxzCFNFMqC3V2u)u z{sZHZ(50~ZX5O^vH)*TN0Y?mLkzQOo!6^{n&-JYQ&S2nB0grG;CKU-w#y+BU_ z0%TbEli630Ns}ZVQi_>$4oEPaSVJNB_lYN))C`M)D$TRDwa?TRlUX54YHT3Rzr68e zjPnwayxNKa-Xhpomtm0c5qtS|a^j#TDcdd$mR%WA_i;!V>)7+{It8~%mXLEB zj+Z;kv6MILLD%h;)Fl~K!3VSm!`MCS#$&PtgZkmgg1tU(9G#BuW}Sp38C~mW>g)y0UaC2`s#4A&LsaOYv8q}Y=6E4P*OZ_$F>?~DJGtKfm&1_xyN=Bbtz3} zW(lXG5x>qq+W|+XdQ2IHVwM<$tIZiYoM&(Ra4e03Y=%{aL{*$-y^TYS{vRCFjm_yW z#i@@l;07D*K~>(D1%Kfika2%~b7R-2y1W3oTN<`IZEq9&A;pW%luh_B60Xo8ec;yr z_w5XIF419aQ<`{5n%b%5Oc3zerJ+SP{{H(5Pjmp0wrgb%WeNn!yUlIlvY#m`{ziC1 zS2B(|YJ~39=){{2cJ55)adRo*lv^)0S0M+`XQ1DBVsOI?%wCMU%9|8~7a4XHp@qB5 ztwbCZYWL{eXZIUM4S*VGI|5FTSwAR5hFT<=(Q^OVjZ!Dx7xQim)@R)K& zjIDIEMjTisHpf(W_Lj<(GY@9Waf@aWJO_Yr6pq?W09st69Fdp7czTy58Ie)K>BAQ7 zAUcMJBet8y08QP#{z%?QXHqBpn|LmkquaPSt}3l1ni#aiO0|(8F~tjqf0Q@CV5T?B zBQCnFnOz})jbhG~G}ba{vlQBSdTNj5fV}T2r-lnO`O@1hbY?BD*{~x59r9 z)Wr18*cT)c6N?ua_XV&f{}#^{$ICJ+1j!bRe(Xv(4HuaMgNo%f$p7S=Mh4y+yVefL z#(-AI_C2=SwrZ-|eNAwnc{ngd64eKLtXT8v*yF!*d{1N)HUp4UCi!5SGmNuPZUSUi z-Hk!14xzcj7jg6VqAMNIGwWLrO-w8QBK%QPN+ibi zS<_{tyF<=Mu%9Q930l=_*~8bDC1Mr^Ow-f2+=-%?2rMrn1y>mWrwD85#@tj1_jX2Q zX}pT%JS8NO*_ncWLq!QSuF5himh6oFQABHP34`K)ey?rEGT)BLxOL62s;m!$MBRoW zD+dfo@o>)=Nut1$V`kf78d**l%4z1 zSweD6U~9G&1@*cytQPOoU5%-%Op~R0f7AGatEo?xHC~D0Jm$X9%u$5}eaihnzQQIP zOuZRF4Aw3zN22?RwR=qyRh}XBRqI8vzDLp2O!Y^(#4v0FjNxTkB@?e&UOx|e9c-f0 z_OH5o$=C&RSs7q2(|k~RmH-6r5CTvagV!Vz!=*gWU;z+1MBPmf#lomB&+ZRtoU@n- zg+N;euJ?478PG`mdRHnFh5b1nY!1WPOS8hwQqSF^zD~2z;OTWtsXQL|hUbZ`X9xl}bBQwJnsihQnC!eX`*My7U_k_n+JzPY}^@k4( z&(w@7Ce_VtbZCu@Af1i4{$lc_)LK6pOtT0P!9*Y@TfzLNgrnAlokpNAX}biUHSP+J zU|}l=+72o$+&Rnn1D65bu_ZdJnSjH|D#bEx#hE+T&JNUMr4Sn&5?7hP16tY5^^1bL zM18jI&v3T+acx3TuK@Dp$(;fPW@ASp@ElbE_;XTU%D9-6qV`GsY>v%xV1Qi0Wqw7A`SZBk z@{&BYI=K>)7WzLOX#!V3I$Cd87<1 zJ2BE^x8VXxS0wR0kf0gz*8V{SJlwmn^d5+-VAII0AU>^vi4YD3hXB%eg8 zqHWgXeVAb<1*P0)+}+{ zgi7BJ9rhAzNZR&e@ioEbrLTd$XfRGP%AuQvGo$SeQFQS&J}{4O-(`O+Ty{}YzmoFg z;Wpz9SVaqSFs!M30ZG_1bv*EUKd27|oX4|KB*4sIn*9Ncjoy?UpW?R|`5Z0=M2k|f zYzj2!!+lPq0kVYygkJ&#fRaBwkL>w_qRN1X+17!l@+m0|ywrD>hR?s!Y6B#S}KXWT;QTS z%ve-RM=)3qv}(y*b@!)Y*i!OKLumwxKFhd(&z+uUl{??qgf_+X3Ea;Cw|S%+=CqK6 zG-0x3goRcI+*S$cb1~4cO#n94sj42HNi!X@%+cSJ``=dW5K|knzhI5!T(Vw%8bF}m z?WL52E^x?OE*qKx+bV4)O}oR&S%q;rp_&@0CvypvnpP(apGM79fs-_Y^y?pq2KoFw;jO$1H&HdqF8^WSk6%oZvkc-1tz?0Z$iOeb6=qPwWpx`fWq-DuPpt7n+bHv}Muvx6MHhxO25C}%QDv{6*xB`HcuVRdX} z4%;!CujGC?_EfLR$a3$wqKcE@m2b%LV54s6J$5XsBs?Q1{j*@E`XjDrU{t^7iY)px zefp+Kda33~77kv@(EUd$pJLAb{}KG;Zo=wCVG-cgRr%rFO)P=7 zsTBnJIXbchl+t?c(o5!G=Nc`r&372jP%vYsZc4mG+sUc19_LV2?!G*?f|R zdk$Te6Df0z-6gt9fj@n|roIZzOWMo7^D3qM>8e9cV^Q^F4pQ7FD37lgCQF|xF10>0 zlbB>BaD9aXC=&@Zl-d`ZMRovP!e>k)#362`YB4t4AY(qkON$T3MnBVbuZmq3dRc>qCs~C>R3Pee5@8XBBb?S3}(vpx~)EW>2 z{}Y6vPRXaR_GQmDzAET@FQ?Azk`R`Saht3D29|GiG)e`U@gKaPvsB9IsVfF9DEsmQ z$aYw&nQsK#OLdZOM`NcDK#KQONnj(qQP19g?)2&1n!jHK-vIGzA)lQllKoB)O_ZH6 zYQX1e9Z?E-r;%;tv4v!%*1FY~coyNK^rLFBrxurY!hl-~v69yF!Ip1WMbz|9B-+j} zPrS_Gqe=`XprPv7@2OvjYoRdPF|oTyt`ZeMK5U{K(T^qLbvr%Ot0z9La-JI9DG^3l z-0NNwB;0}V+nZy1>MI{?Cm+yJxb$W1@Ky#XhanYWNQu{}rd?pQHwLbt@PKpc5i&6{ z%5rOAJ69kUY#X2@NS7O>@#Dp}pHFs+9mLUo=EoUlU*O?CpCCxXh`Y1OI4F@k-!g*e zuC=DKnv>vkdzE$ciVw999N93k_O_KXpdP4QX@iI+QqnacLD_#LOfC&}BzT`VZK{tY zmsWLb03~-bPB(T^!V6#2Fz0~e_KR9&sQ+xvgcz%Ybj9lG@(tnh`d_;5u;{r>nv zAk`j7qF(XDua?vTi-=YTBrw5hTj4O4tEyObYHynW~x1By0GZ z;n4DB5h$tyeqZ7y+DSVZp&Fh*jmAb79PblX=e&6F;my5Ya_SDf-BQ z!0%eFsVDOd?2+-Tj7pi;vGN_g0C6lAxf6&+F1XFXKIhh3F2Enc;F-Yhhd1P|X~lIu z@G`3HD{8cv`0GJzD$)3VedA1dNIBCNrT>EA=aEEnl(FP_u`s#z3k9d_?EANcE396g zF6Dc#ug;#Gq{jnEk%~`+Au?d8-X^=P9N83b11U6j=b*}tSKv+nV*;#q-;Ra}tQ%)F z$pHYXyv2w1*;83!kOO%?>TxYt#q70UygXZU$ma)~e*S#>Et zT#cwC?H0pl)Yn!I83pa%=DA&N+!85Ra}9=Pqcqx~#rZ&*bwi(16CrtlK!n^>`0BD@5h;K6gY61e^ z=yTKfw*v^YQ#6zY^UAk0pV$mjpFtKwJ#Dib6nBNFvpkPy-{}pgpbGHkpSQ@Yqa$&# zPrp#M=#5XMtp|25#j{BQYICrA3`PSO^qkS;rl-{p!!uFmz5ZBYDN=hCT8++oB=>$E z`n~QblxilUjPpCO)p^z)1hN+6ASuhfT60V0@_K7V3}-sIjjP%%<=@G?$*>R<`=j*i zwvNCl>zn^My@pYBK&~OK+C6L%AdU@J_wCC-1#(W=3U%T-Lw;X_Ui+`PVmTa3!+-y? z08C;Wsq!9X9wcI`6e}$b8NO3z$s(w858uxgwFfy(sR$5x z$*YF=z8!)sM7W`vwyJMLH6_f(B4i~ZQdPG->4lt|<3<-lcYA0O9*sAZ``H`rI`xWa zUvhC|h%cqWc#S>`*;=Tqu5bS%5AUzf8LhR0=~I*>`$4NQ>Q8b%GZaNT*;~_>SUdLa zf>dKZpNE=|9irk&2$ofa^ieT#I0krX1^yMQj8x&)bM{7v)y{zz?_2rXDp_%*UccT# zgLFE9WeSScHjbn!Pj^sV$@A#{>(kauIGKkS`(SeIq~?;jRX+(Au~+)*=gSKLJn6I>guoN(XA z)Nm<Os?2YKg!)_)pXAINV<$Hiq;0Yji-1D_JGMa zB>)J`GxL}E&-DllSdGZ6M1w*(3a&phWy4%C%s-H?mEeQFi{;d!i$DoYw>pZI!r)hk z4Fq=taK5G4(Wljmd(SqH@>}hT zOLS_RmSU+1p(9?ds-9Of2-lEf<7p1r=hcBM1p=e;NBnX%r;_;0l~8s5AtsiJO&FJG zmf=F7piw~1t`L)nnZ=8EB(h+XENtbtXfuL@i8 z{YQQr71eb?r_*U6s^7Sx#**|RRX1jUswzF1iK^%98nu_CUwUUHgfI=e)o18Jpv24x zjP$xfhA3S1-M4+WjPa`!hyWpEF@lJj;&BTQ`{rR|89QIiXLYL?tkx$>kd|B>c zlc63>2cX~-CLlSJ*%<(WEL{j4K0Ocb?llCxFBomY&R(i)A?rz|s8%Ju?T z&DtvX#m;(@Y=mb;zzePQaBnU2$AWZmK7Tw|63-VHWs9~~G3M3NtZ;2^-@vLJiFDCT z1pP$ye?@vyIde=VGPA%s(Z$o);Cz`6wvge>8fNbcC@cY3$!7e38-^i8{X{a(0-1Mg z(TN2;yb%=e(i+NONulr!fG~Yt^ zKADr^&j)O=bc((=i&#v#dedTf0BWYCMHvSj(eXp=c{H-F#?BQ#cCFy;EY6xLM5nYS zr}4mn1TMJigA+5|qM9*LxA^S?&sfc?%HP*<$Y1y-euZ^~97AbczrJUxZ{l3KBAP=u zAu0qqFjb$JYD`>+`}!8nE-xW-#Er|ms|iy+H!`sE&JeknN~XgQjoJ=%kxRrD^mR_NZ-a#4MU@gy&wm!#uaHkamUC(GWDA!i~tv? z>|RT@W-W%pvO)fdj08kv-8oh?;V4z(Y3gp{QYwKY*u`86C7Kuh z9c+Mjk6`y*vS*S@Fh)hipkf{0ihER3{v3$LB12D4k95S9JmTSc&}v$mkp-0|<_|(v zeLARKnk8UN#NupV2Dr;`-VajU5Mrx=L_fg06{)bOa!@BwBH7@=vA{iznit1|{xb?E zwlfAI(xlO09P}H|d>NpXdk!4}s%4J+mrpRtYUFiw2AY~cwy#KI#utclfcXZFbBZRm z>0(p-$0iRZpCB?!qIQfq3e$tnzgFJXli?Hf@-(KRc5t?p*TWDtLPOYBw0bRo>NKt^ zrBmFa_BU81D2;ao@-ceLmCd2;<-bjn!6Q+h%WTBXu+$iwsUS8KsUu zC=|pQg5S(+%zLP81~E-&eY>_4G7l)Z=K>wm{d{;$ z_G!Wt#x?%k5OJ;#ybhgCsMPV}OIS7K!31e@85bG#13qdssr%sl^b(aD8U(X;PM!`g zdtgl+;z}9J2l6qN=IjS6whPP`ikCwZw%W)p7>S8-T|wD|=>VXn1|NZA&}jh~x|MReY&4v)U*Py{ISf(BCAJN)mMvqsR?Nd+b<=kZhk8mwM1 z-Iz$4L5@&g4bK|{(_K%N+M+Iij4+_29Ct#PvLv}V{KLUCRP+lVyDGo+J+)TS?LE}- zr7?i0d&IhoJ9c=jt609BHq@yVn7>^3@`M}Id36NlgJ;BJ>rf;%u{?g7 z$Yv}>E=7%zcYN-ht(_3nshKzl;7yf^qeZClG_zqzbTjc!5AXP6H0a?BsnU=|OpYfC zCh~u40MRh3e53ssfI4SLBH(mNH34iZgL}0N35PfPPkY+s#;F1tI!0l9q1YK%1pC!eK#EfhB6>U({<)(fc$FgxTNjW$G)jN$|TE;&|h` zm>VB(XPdy^ucn{@Q&2GJ$zqySPm^VTo<}6Z=<$n?CJrG)xwN{RAsyox#AKVUhB{2e zuW4P(gb*Z3t+uLHq4f>)@H#hB;|AkllT1k165GqeIu(zQH#MQIp8n(T^ot^%!Mv=^ zgA6^8zo-AS(?xtQxgPhDJ>{=lY`~fQ?};|#sm!Sur%> zMv(sTfmQZuIcP^_!&~6cM4_Ca6&xEOaf6=y3)sA?m4Y%5H3s@J`WgZv4Cc%ISZSDB z2uHs@*OftD4gE$uXsxmx2VjyyUB((NeA(y6RT8Q65JV#QX`bYW@2LQs+MpHMMKUx4 z66yyuGxL}!orDzL)J#I%#zh{@S6uNi-9PJ136)w#p5uWE7qa>qdY=Q{n;d~TbmMYp z0B-wIfVE-%tu!w|bj`6GOFv5~oy<@vNo}qOU~U#Q5hM6;$g-#1m@b->DURi;qCzK& z?rn=1-p?;H^^V4qA7Wr&iQOq}bC;;Q^wUN@s_`nT%e0kpVBX+DVnNA1UH~^sC$wF6{s~n7X6*^Wa zm{NZ$TVeN`M)s>LI3pU8B#NRD3Hu-(I?AO?AzGS_T~LCZ;<5@53pfcCv^7IBMXT3w zA`5jXvXIJ^9||wwncCWZyshF0v&46lWMCFoF>s|k59dE8jgJHl{X|*WcwrhWo#4g( zhZY;FW570I#bkWi+GMMTA5K5BcB`UGYUBJLqxg<_L~-;B5&I(xKHfW;vb5F5F4QPa zeUv1?Vzt;KV%&qpEjhG`sP*8%pYZVk2cmzzzjH^o8e7I%7|`(~3QJl*tEW5y&BN?X zQmG0N%wt=swm1A5O&zkJP0THU?KnsZL|xArJUD@|u-FoB5Tl z(fz!9D~V>YVvZ+Q)i4^5TxF}^#0KYDa17TuE@9HBO| zl%1@^wA8l8zi6v`)b`5Y?kr6r0FJ~rIe7Wa)>-Z^q4u(ecaDa!4R1kIh$+dx4b0vZ zJ|#j_0oUV<$z?{jDZA;9E;GJvj&}y4e@aJ$mDGBoyeNd_C&M;Pwc74?D^Qz)(=hpN zM5)L!gfWa_HXV^&;5sRT8cO^loQj~CIDr1;0MnxKdaEYilvYx*KBBxB8Y$EY*&3-I z#cY*Jq%1FhcXEn`yDAYKoCzfX8(bBKa(RfXah9apRs4pBJzpbe3wfLz#%HF~ z8{^Rg7$ZraB8H-4;O&ai#a@If5C0n}s08azMTr-(W5)KRu1>;MZYRh#F}Tdya^ zxwdPuBiJ@x|8yxH*9E2qfFK2@2ApQy+zCB! zRKQC`mCEPd05Ar^0T+6t=fBGME6iiE0uL3g}j(LN``5tKUT)l_lM0Kp=foCe* zNHS=o&rr~T#q$jbT{<&mIqwRzuyU@R<24L4E_vH!7!VC+K6TP`x7!kmED^v&m5X1R z!A$_Q;UOl&dR9J-#@v~VRU{b$*P230n1!daaA2dtl454nTqDazdJeLygj+xdfswG` zv0)XA&V9s9%eWE{o~7!N@zYs4M}=-A!mp3-xNh?(=G725!RXnuMY;o?hH6;>YjR3z zFvOVPR0UKceu1p@Ga%DUMtG#c#DoM)ZUFgxgW3T$0k{z=nuHQvc1BwSOU+J^ore(YHeLj@;Hx5 z`+c5HISmvg+!vQ3{7^`y`kjY8zsY&W$fW<3yHdugNWCSWTYc>32Ad>V=mA@aIJSFb z_H^R+gr_ZU5>PEhE%8I((&R`iTdK`V3Im;_L< z)SmF_f4cnwC22cPbQFUBKuPq|q@IUMQa9Pnyu)M@+Q|rH4xGQ!I0*10nNt$Z#`CxD zLdm|&-C4rP$n{lUG<$c?m)ayKPc)u-Q+JZk8qpz{XR7J%VM)KQqLzr(HWeH@>@vRj zW+O}@b4vR`SL9&9#Bmy$SCq%&~H&kZ9yhR#7KoVY%;ujc2}#o32k zAjZTzaqxbCgJxDp9%u#MFIMeRX^$TJ+i65Y18%t!;(rZF+8(qS4BDPzig%u1yBlEbg?N?S1393 z_sWjLAJII=QIlQLK&iC9p6t3_H$Wp?k!E#mF7_C~7HM{CfKBZUSndL5EkxsmKVqA` z5fzWOznsz!o`R17?b*e&&e}3b=Y`*F@XP54wXMlfbfYY%m&;lnc%3Mx2}0FHoZL5V zH*8n7s}y!QEh69L-4*XFr{-Mv2sJdf5q?xz^v)F=92;=gXd`7!A84gM@z64k@TU^{ z)q(|UEZWOTJ4F@JhxQA80yE9V}-sF!$%B=2?aHl1%#9$tFgtZq6~!1ugl2 zUGmsBdluRi$?6is57JDeh_TlOE`w;&^W~8%Up$LBurX$nR!yV{;8wyrd>u4#EK%1C zO#ri1dbxkn`6kQHO`@^Bgg>9)W8x$yVv)Sft3}@1E_PDvIzXA^d5rN2#aPe~ zh22Q!D&&fhpj1LX!O!dBXz~$UMNx!I2(*5}$ml2$#Lf~J-UGeHRH=501;VPv-G0S) z{g_1Jxu3Ux5 z^k9Oc?}YJVL9~~^3E|SlNW?g4iz1oGBkBn;sTTsJ)1MGIRV2Qh;xB4*E9(2lCgIo# z-)v&Mx}rkJTU{{V;*Se;1ZmzSMZKetkz~09XiVdI) zv&5-9ivsa^Dw0z~Zf}*e|0DA%;{ND3h6Q#?SIoxDHq`zqxgGN+#a?--f$A>bl8a0% z!X*0dVd9*0MY2GQi)83D+mQOy0pF4^te9o!b%GOfJ-Y7hFE10T>81jfM8g!R((h!Q z9q~V@9SxnqfWI9c{Ha5No74-ZARly)#C){;?5oOd=%TJ$5zq1g$$lFE0qlvV!ZNl@ z+rmnGVf~$m(vcwTacl6(4i`BZ&HZeNj)%`(tB6nPBFxLg(j_sv%`K$#o1>LnT|qPjz$4;l*pwf0)=Hkp>mpG8%W=&MA+b1xYJi=ZeEWZ0Ak9mPNvNYc z8da5(u?nt5N5*Z6wD7d|CVZaL1=%h1WZWX=yojv=Hmn$NFoZB7?hs2o%Nk=XyRsyp zW|fZN{LD1B8T)!dYOt`H7@58p00=i+Sa38v9XHs?I`Kr_wv?P;EQ1#ush}}modMWW zrF~jjZ(i}h>x>}bM2l^QnIZ+7XM)o7mbxlxmt^CNLCw(RLYu*#O}>5$F)C}C(6t-Y zne2FeTvCfB8yrKP#%HG308e`m=D}5?6^uMr1YfUOQHd?5Dm|6_b^qgcnqe|}@y!EZ z_1}JWevTW|sw!zTUi4v+=N`?$lpNmk{15PoB0|yHhCCRPQamfBbgFFMpaSfi61$DN z0OUz`6_uiJ6dogjdoK^%g?vCNWG?;ASL1{CMgUOaiOe_T5>p{?y{^2&0fNu+m)>u@ zn8Sbp>W;in@$8}CdBhl*3rxq5G9%cAn)}nQs(ML#Ii=I!s^4P-DbHEFQ<15sx#eEJO>;rqhSxDuKFW?aV}flaM^SKaYa-9_?x-rbTYywicRjH*xXQem zhv`VWpT0c&H7A<}UUX#CDZ6h&z#G@hdp4COOp(feNBVxs?e^VZS651z0W)}t3OeAP zyAoWR9(L2M1>~bfoydz!oe9{{!Vzj@oh0o8xOX7Rt99A^V;XTm)4&=L*Ue&W zGHzX;K~IyULoB(6<{mZemZ%I`i?VFzng&imweD0}$I(9`A%Xm*H0Jes-0OY=uV;nLK;;@`sI{d#)l$n8<32 zxM*mjLncEQA?-W6N9U*%ZXUOR?*QOsOcZ|jVR+42TPcELw&;!}#^Uyskzwg8w3vSl zvX^qM4^c95Bh8NpTig};^S2MCdm7?o!AO!X*8OBzZ*WJT^i}*qt*!en%lcEV@7;wj z?O~52<<%U`7fJ-4ncNL&;rb{^P7qL!B@enHP+{5r>{FYsXy>B#pT(H!PdpD0v8`cc z{;8Ys>4j{S^)oNhpA6z383pP_s?n*XV@WJX(1fS}aJ`$X0()qtdJGV|gaI7GImO42QXbE;3<$jV_^=unWW#Bb;vAU0nX# z#hgJcWhbgQTNf`uq?-TfY;O}?5*d9F#q51Ewh@QSnTbtyab$E!&wfCYp+d)Qa;`;) zd)>(7Ul{uZ>dvFZR1BldsmXblmn$B-39uGwT8PYGWdJkZ%7Wb#4}I4-b#bf#R; z&*nQu=f{c)Uq^hRlX%#TECMA=sNiTq*h!XOO(CP%p*lfpn`%hf*&@O4JRwEY0A-@H z1zgMxcm_X+^z$mdxKULnRE>`5GTV*&(<=* zld6TpH>)Kf{kfyZ)BFo1DLe-0sFA6*41JRCuwCqdcDk1`y^B@>c1gFOm*!Nbf%s?V zH_j`#V=^{jt~Y@&npDEyR~3xB5g}@URGl)$>uBm7sXsMNTxSwMWwd^hLB^GNaz>Fv zM6wiH4Yv}7Z3)dQ0@rRPE;*z-+gEo#m^gr|fDN~;C3{m77XP-dZ-{GijGs9Q??;eD zE05uYYpZDgOxm=z{uO{nsHQUAz&&9TOFqBUL(LO=LMCpc3@*?XEu-_!&#!_9#-yNf z%M0v_!GgNXf@Okh-1!US9ew+X8cyVtZ`9n= zQi2G7BprPDi+~Fxpx1C_e#9m(QkhI$y|Or#gNbZDfamr@k17Hy4Q|j6eu~0MXi^B2 z>+dWNe?BiSi5=^d<2OFpY-OMx3CURMI{)prSI}L9$Ju!ebjDQ*ti2!E^|hP10$V|u zX{ruuPvs#<`Gwj&B4s+51AER!-}gc?j$E5zE}$){GHL@{v7aQZ1mHkkik{ve(8c)=CQbU zFO<9d_+w6}%RhVD4Am{E+^bI?+;u^3xN4-mES@(nm?1b}S%LbY=JHm4`Fs5Dp&Ls|RvvXr4 zQy)icekl-nRy;gv%Nfp&Q||bLedF5y><6d{EJjK#`HY-hFwdx1c;*q3pnFkt3glq& zB6z;=>?hi}CG8`KJYYBO{en%+bz@I5zn#Y0h5~U`{HPLMfOzFhO+(F9<&MH~T6TeG zAhh7@=bm= zEGeT-!n67Ff1akj(UhPX5*y*r3F^ocqc=MGL065)lj<`XHEwR$!!4E8B3J&D4o(Sb z^7kG+njG0>rY}N*mPUHQ1)PVGR^dcG%EgPFBvY&wjC$&RmW+QYBDHX(3YRIF+ADZ8 zcTSB!ftPGVb1++)QQ`SQxoamolxoA=AhPx;8w|45bv30qPEnwde#Gddv z2PmZ@DJ(hzzmk|gA1H)%?xu>RIu9k+Gy+H`)xjjrnz<&6u6&3H_h_PZmrp?F`#^*A zOc)`{fs7Ob738Fp5h8$zVsS0Fu5)3abLvQse4my?#A`7!!k^8cr74k78~lE!zirpt z_rs73ghvxzMS#$p-*tCN-~?g=(m#{n>PwN)&hAk7;Yg_VNvPK2{Q z$iDn_^Y8jB3)}9T=gzO=+rwXhdYj*MSEL*1PL2P}w%B|b7bGNOUQ*Fpc~?Ig4A*l z9y>GknCvlhAliJmdD(Wr-$#k_87_NljI=yLM!g_&YdIm1pCKkqI72|CAib!P7v4yy zu?A(wSnhjm>yl?Ru~$sa^IU5e%8+A<=oB$eJmmx}K)hhBy%A@o*zpY(*`oP*ekaE( z-S+JlKK;rW7M`RnS2xark^#C>yvNlsG+A~R8pxtKlrrN2PE=P)jeFbC`Nw4soBLW|ha7iOLj>;3hQtUS={WIhlkT zb`(5dAe$l${ZKzbJ-*L4r3pLLMEb2`1rB%*@myH)Qej6QlL1$x;lE!8mw=AZsZe-U z0&wtT8)qYUAz30-*ifm#7`yIsWE_G!i$UlfRn_uIKaG&B_x!lsw@cV;+i|U?9q^`{ zWMtNvXbfl#rfyCXHIy(CoaQwz6MHrjG0A$P?G@)C&%dl~%2DMr0y8GP`EOKRwJp=7mnP5H&YaorRE)|KE zYq<}mGYlt6N1DH=2uQo@9{H0mEt`lXgC%CKprEDB8hIs^6xH`lxe`zV%t=OqP6?kL zJKt8k!8c2(Eo_k1q(herhgQEh3t^t|h0{Di){cksPCPu@)l4XuaLY7ZW&!|HWR1D6 zb)~e2aw1E*Ecql6b~tF_k0op(1q{K$O9gT(k39w9eUkB?7eK27!%eefi@1>?KnaYR zj+!GOe0$4#wovG*IyI9;mXrNPxN07XtMWHgu$!)f52ozi#xm$!)}eSDb&dQPUlS4P z1ND=TyT;2Tbdw%qswt|vL81&+_A(eQg+Uzw4xMBg4>f5fh*g5|%!Ade_jC?6Jbk(B z6cgXrkMYkfyj{2BeAcDno6phM?9%;46Ysy1&MG}>WSv6n<(@#Hkqmlq+scvz+!aUii6gi z{|Q2SVkG0;=)WFtP_zCAzB55sG->5w3O4-Li~r)E%0_R&Elp+1n7;qJq8@*;@&4sQ z{l6`5(xf*|b#ZYGsY1M9q(jqzJlBzq{Q(LT$EQ^4b6^>_^W%PT#RDAe+X1{)_49S{VHmb;>x9-J7Aw-Xw3 zmNC$6=ZISiLR~FnA!motwlemC@kLDmYIA316m(`dS+l6mR0IMbr@VWoH6s^}R^W2C z+Y);CybsW^h6NNZ(t+X362ilD!;MI&8GjdO@92dT1o>E?^SY_n9EEboJ@Jv^CY6^r zP0RUf`J&v5TP1A2>J6)o@j%W{~TcGz_4Ob_~w;JoxLL zVPxoiym^_cNt2M@t)h^j`k&&MOzttl=pR&S9mE1@lO)*qdX7ObW9q_RUag=eKU=sw z265Ic@%Y@8_}ZvZ%=0a^i4xj`0se9MjJKm~D~|_%6WDw4+P;~LZT;Ht@lB=%0BL*R znZqzJN;sM7b&im1jH(~zM*4YvODdhjj@7^P7N7TZ+309Lu5fwhOl>oMjDenYtW%7}aW=I2l$i#U__J&eS7C;94j-c>%GGt{j=h^c*f}!Po{5B6Zw9GtB5`{GJOzI- zK`6NEll|c(4nTy^VN?M?Ez(V0Rxq}8V;2{8837u9xP#D=44lzd50vz_8y;b#6Dg!-5N-oN&% z^2PGpTc#;50Cb@QsK>r&d)#&p22SVonVKhPq%Uhru?MzZZivp_!6*MS9x3te#NpE zvgAL5mQZR1 zjrYsXqQ@^Om0T6*xJrike9|+k+y>Yytaa$+aFz#OsdoVrxL$1YBXiC6$H=IrY~JRT z7I)fFe7dy~S7K{OUn0(n6qz2l;EbXx8`abidP9;70iYDUEqbG1hjrpbRIn9D7Lh4Q z!MZf@^K35UEe{wN$!c3i$Bi^7E#u1;vwl zyI@(Y9m{I9Ai=tiQbrhTEf&+Mos0LhJ_Ke0dOSQnf2WajY9vAWzU~wTz#O z#3+)yA+qd_hDojtg(?!b$)X>wkpL=Jof~L%UtqYr^Xt1$5X)BS8!_E*^_ffz1q+O%y;g$Ja>uRDKvep{$pDg+3 zOeHkEWHOs^aCmr)_}YV{aRTKqI+4?tt(je;13o8XRo;e#Y`q5kOi45j!s{iaC0mzI zjq>)Y1M##IcY*OS!;-^5TeXW<9X_Ou={g(=dO$~=GWr+&Xo>p@DO)Lnl{Tg04AI$J zYSqcX^Dt895hs_l(G*3lJO2Ui(<`(PMU`e==yI z+kRLq+p0HXz{{Zrj}@o-B^VH@t$lo=Kl(^fpNu1+PfICvUIATNqy-R}s|LeObH9mV zW~ezvG=x}(Sx{1eYA;jEFd8&!+wnJUH6n4N0plc9NOEDS=CqR6RXW0rZ`Mx;Q_j;9 z%jxp&Y9~OzN$CvLJkC(+9T}f;wGr3OHq!}BEubU@)Pk$t*TNi@aVK4I^TX*Nq8Aq& z!6v5sEAC*4Xb@5Xg^Jk9id{&UOJEN%3(vu9(P&yC8)c}w+T;rml8lA=GLR0L*R({f zRpffmF2>`BQLq$W4SQrSS$f3e2F2O}KDOXiY2)^^ZQ5KfRMt8~AiF+}zJ^6_42;iT zG(Y{wXs`4vma>&sf-J`tRQj@Fwo{A{uNieKh$da*zx4MYes7%2`8rm}T8 z_I29vdQ@XTeCDQSukSvO!-*;nd)QTGr`5z=*pi|HTX#V>ZQLdDejMSGWWrKI*|69J zWRLt3EHXWB93lD>5g=sr+Jg)o{PIK!HEw05#%}=rja5cX0*}12uf{f#K!xW2Z?6x9 zN@M&>@I-f>q|i?hsO8mL$Wym!^0b*Gl8Tnf(>Rr?a%G4pZ#d6r(I{+>ZxswIPgh97rZ>m>)u&CJxeY*g)@c8fAi)B)2nV zL&dFD|0ge)Ts`n5toEvbKMoFBCmuPMP+oW&B=-YJt*nEz1e5bQXf6WRL*>1aElWys-L$elC?Ce-$=pxK z_(hB^Jd6pJJT#5xx!_r;*^*W87En#Dfu60oEWsQzi=f927thy0rMaTA4MP}0tux|OsqVWm{}L8ySir*9b06|E z28R@d{a^|9Cjn9%7f3QFE>Z_EzLP0#>W4=fWL}l-0`Xo*{T69_ot}gN%PxmA?MBqW zIu&Yd424BF5k1pMc3Sd40hw(SQah$4*nuAskUOh%I7F@obm%hy;L z_n9>4mc+6f;jx_WUvq6AflC_XZ4o5|n;V!Kz}W-k$@~vQy`75TY^|`v=_=z&6Y`RJ zj9`F*6QYzj9{+wx#`aq@$Q`bie9i^-=2Lpfpykw9vT9SAN3kJ?2PgSwgvoICrp6h# z(_hj^x4lWJmG4HMd$D1e*nsLmXvloYCso^A0T!$82+i%I5Xu#E`?GZ+J5J{d<1Uz< ztnqaqXU{s_&kQUFE)~OitE4SAjPg4#3zvi_v_FuXZWuFxxJx}Y2rtlFmbP;TwqdGN(3t1}&~#I02Ao|TP+Jbte5&Zdv)7wI zQ26>;Jju69KCktt$_ZZSeU@E^Oiq#_2!wUr?~Zm)kP}>iIVB*XTg<;#Ubrcch5HlC zK8!8+SQ}wG@E_%?kR_|#lKmqe4wDm5s8snGJp40Dz(;0iuDMjVJyX+=VtW7HJnwo8yfY#*kwmSb&cdhu*28s2?5?cf?g>SH;hBs zij2?`+076?7&}QaAdpuGON?P*h@NQD!Em5*4*`HLtFIuPhHL~^koe5m2WYq7iAo)8tMs%K2r91vL@a6^O8Mst z#Td-tL`B8#Pj?V1ogOZuO}fjK%5K$OBQ5{7jFe)&Aazj-bs;bSeo6n zWZ~8qOC}))rsd}KuqL~uFDXGXdbzP^oR^i)G4pcENhExb_(T3j4`tFH-qJKdYUJ?@ zef50F5W1;^<3>DAsk{{cL*&Q!((9>@`b+}&HOUI(m^HeQ`Geoh7ENsA%?n+huGChr zQ8Oh})MQE;m&7JPDT-F0+M6+i19h8_{goe{d5)HWL(o;Pl;kMbrXa3C!fWN9dW3|0 zxVb6pIJ-giJNb{{gm}C%M4YHVO+LkX0QCmX9<_`i0*H5;m7|u8k_i%o3(-fOaoX%G z8U@v?$G?8_Pn!p}8Ng3gbxL@0{d;KtWvdvtsNsoBmlL5#r8}A9pHYRe z9FI)W%4+oAMMUdO!V@5J7tw9?B(Ey$<~v{;22_QPTZzG?FQ%-6(3u4fgM=+_rvrgenZOpU4Ief=-Y>l)0k);xi+KUL@WxuKHHruL{TH_ zp&}t3PVczRbKE_7L*5T6uf^=30Kcv^(jXHPV`dW=u!a}AFUxo`qD3JYWD}JvETvGw zHpzN2CCfyp+p0NJ1Py|N%>a`w>KRG2jK?Z-d4GR@rYMnwXk7&c^uT$|hxT?kUKk{7H^*FuUIW(?4n!fTDe}3W-po!a~4zCQc^2yoo zGJ-mx;HgG`2@&D1lqF z=ve|G(Nn%E!B_`rwj=GnW#n&i_+(p5uy8&&W-qQo<=~W{US5`bluS~zSdnL-y0jKV z@ZO0$8^-NDgjU(8m8gD#lW2vUHl|}+N#S*(^h&3ux?XDy4@KewU*0kq^uLy*a+45* z+Q2f*Mt(^m<=|r+XDt-zRHt8Ngu)7`RW4Zn(x+^ustgrn3$~e+tDO7Ky(AqmQfi4H* zL0k4IV{7X%3!#H2FhYg-Z8gr@Bm7-?J*}TAY?ZH-zhG!JPd+tWLlSyXS^E+kZB!cPpgAaK$Tryc%sB?T0_Q>9W75lf zUedGJlt=tgc};1{iu$|hEgkzaeBhS{F>9I&tOljw$ix{_Tnwq8;x~^q<>v(vVpn;v zB*?nvHs;Zx-iz-vA3ug~^=EFncA+4 zruk(P|G41EP`oDS{XOzwbL!2?+dloOnh#TcsPTim~Lf<-i$$55a&H(3Ipa}c_7~TYC6fDT4~UH4DvXtYPrj`T_UOS za3{D)<)JB8wWmKUL;gv=OeXho7oW{BDv;s)?v3)dlpN(79#3J>}Sd zq_>TyabbrMUhP_1T4jXe0M)qCgOkUq+KVu}9+7y$i)GubJa2m}nec$+ox+8vi(R0_ z8`F*dKMh81{y?*W)1u2WD+hHg_j&pQ{YdHpaQ026tHR_5azSk;k^bk>GkMsH2yCHS zlsfDkUd~3mQUn74?mKa{GIow6yVzT3a*7X3eE1FtlTcE_fq(G#Zq9uFrx&7vo)lI7 zf)LQ7c`q8nFR8Gk^lk>js40?xA&v@`ka&PL?x#<}Tk%r*B)kBFek;1LO?09LceXh5 z{k6RQFL{c;oZ_0D#QCuTKL(0dhT3n$AoA7-phL1m0yrf_+Wb4$dx{_&scx(JKsocb zk!CDW(N6iY&!E*P9G?DTJClVcm0Joeh_ z3f?#^;}QrwwgAf&@K|v7=ZOdMG&u%T|n$r9a{)P^{6Q+)5(fa zs)XzZiYWvJC@JQBjiIu|#%8S5fX}u2f&2PjE6&rb3%Olnj`vLPL~>;^q_)SNT-lZ~ z*44cb2+NH0P$90UewGw{FIpxHLqChztFODfdN$y79(D_lB;8RPx%^2z!jM3gx+Nd< zUc7@i!@QPVMA)cfVQUJcga35{17mSTP7-uDn8Y&jmisM-v0=h!}+4M^iN|@J;}%+5{%(WCQsA)5-lD&ClM>v z>328Pl(?t;Ztv??Id~%9A(LV1ZNo_=No7Kq>%8XhJhysyoD-wYbj8u;oMqET6+0%?B!QU}{z0*e(5u~kStd16@*`S3GG+Rt# zKSvOHG4L2(Z9PxDc_2r@5=Z7`+8V*9vSBm_|BR#Ft-NLD8u7B`gj&PA$=*`iMT}SA zF}Upw^lcO68A~0hPgGAhzPwhDl^9$grW*rlHL=MU_fOT8oM)k&>Bs$!Zi5V7EG;aC z+Y97f#*n-IO!CC!abg#5P%i59Hpa|pYQLP1#1VMc(zE#9F z5XlT!OXR!_vr8c@#Guls3Dt~ORD%+uKT`~wqc+59@sy@Aabey)|AVt#$2^K2eCy}; z;6H9okd2oma&r!bN~K)9=M7g!rM0fFan!`uaS^Q%23BrS3t^4ZI|;5p@y`E47Z}op z#l^-gr{&5T@GCTEs7y560g;=W)qYqo(JSlkWH)$N8zYglqa#X7OY?-L(qls~(b5vbM(Z(vfw3^E99uCqxIQgVY|se97Ai!A5kSb;Vj zH`KvY>$(q3)bZm>Ivd{s{ON%A=teJT7zMc{TT^Vg z@aY%#MQUdacB-p4J1JqUf>j_6JrAK!!x8Cay)Btd*XyWW#z@p`4ri{q&3L@7m>*WW zBpOAU_s7J{mGNJ=j_?3bFxk%C)JVfZVR{ovRr!z#*P+gJ9HAsSZ#^KV@0;8wwThV< zOK~}-$}|bz0DFK;)bR1E?!H@NTlpOFZmfI~z(#8{Qd0o=ISILxQ}&uc!7Q`kLs(E8 zk3hRrEkK+6&-Og*WlT+SQ=@bBwCb*oK>#D`oimCQz)s|OnbgZMn7N}d*wwlm>Hp75 z#d1#+vQQcAh%fP2w#Kzp{O{@P(P6)`M}ylQyFB}ND|x}5necrqM4yGp?Yb8MX$|qT z&Cm|Ow8h_#rBT_qJqrJ>B&ImBw>$Tl!B#T%OEEXTo7+>$?|X0VIp1}-(RWMxcM{X` zG{WMQl4&Wb%tNd>Tbi5lNP1yo5F}q%d8M95O{+mB170HI0l+K;BR?0&JUW7C;Kex_ zm>I%L29cct4bc+eDE7zACxhJ5EYmJ4{|1kjp7>)#r~0z^#}n)^!{Dgm+ykCXRg>D* zy|9J*N#QBB`SATO9=mb-czXbzCZ~-Z4OAHim?f*!0p`HFur+!cjVYABZeg}YC`Z?c zKMtCBa`lp{gA2jvZx~{{rE@6N*!gI?H$mZ=OdbBNyw9{W=`pS!S^M_*LC!xl8~?=^ zz8e&ZmU!SYoB<1>p+^fi*N7XQcCd=7pZ=Y;F2q8i$yXGSlVRfG3>JTpNPY8FAh}~- zZiB8!X#&j4VXg^KXW$jkuW~%ZZ&zxyX{`{;eJa8V`GF4ooe$!`03g6N!~d#_+ck@g zV&T}B%j`3E?ssZZG~Z13PWAi#rQh$VObuj-*$y~!BJap|<&|D|@>dewb~U{xh<0_= zhbX>XC%Xj8o()YNV|p>mscxiZK|M7~!#o|bVwziQ_(%ic_UoV2cyr|g#h{(|m&;YD z@ogq8hYFa4pUT6=!4gjTg3p!?erB{B1 zsXmDJESnilPu5Yu}q2IdR#|!DVEAcRT%;9cUm4;kJk9J%bcz{E<%TOk>%mQ$ z!n`+2>a_89;~H-=6$pu7A2{C8$N3r;HzU(0_FM1&yW8;Qe|^Z6HaFZAkq*u>ws2Xr@o0QjIw( z1M48G(;%X!GCS1tDUd>wOX+ql^1(!Hf6jP+Ln8l!;@oPoQLaYRR5S~sT;Bk$f*sYT zG}s!PskOqHEmJ#oOXT`0KFbgMN`v6yWouAx8on03*Ryw2iNCaCQCp?}0O*jk9OD`E zziPOMax|pn{6XGv9Ebt!83jcrQ*Y-I_qw=%x#MtV4ZuoiyO8oKm3$$1f#^6)zcXHYe)oC_o$8?4iDJeCC z!@wK*$bhF~;Kly(>@rZw1D{ro!OF6NG^@4%E_6#N0Y8ufs-B!ux=G`d&oOzD1Qq#L zY}@64v6<}!Y|Sp%6Z(S+QG^4Gh>lmrh-m(E;6iu0Z?MDO7f!o*w(kDQvlYd&B?DdF z!K6Uw8O6~@e#esuyTbBrt28yAY-T+DiRc^BW%M-%xcc2w>WaxxDk(A(^1XcXf@a|u{vqinD zX=D-Ml*AEfbuDjM9>|mLc#dj{0=Wp&Kt>c}gLG#NNXw(~)t*5ms7t?3l_CirSUh2#u8;)ln-sgP=%gn` za8N}3zP_@g9WaA~oM2=KOp*>xcVqXzM~2cYW2-8SYToggl22zbjd>(nW!v~S4`Bbs z__0GOMzXS(rd7tt9M<#8`0<<_23RQjSy_7aW+&@VIoOE%%D?&_PFSkJF(j`h+_$MB zYASS@b{dp8Ki*2ip1zfq*OutdP&Fl>G!pqlH?crDk7jY5$k@RX2s`gSt3!07AGk05 zl~-%3!m&R_4IDUl`aC}fvZ!%U5DNY%deHx2FkBzv|NhxTc}M0>YfrkBZ@roWKw8$} z@0i)Tzh)r)p}*4b*P85tWfZ^X>?H%W@=M^W-ZKjOhZm>Qu$*p$32~8|UQ2xXsLq5MVb2)RfS^P8_T^sxWL1;iUuJ);lq%D1`fX~c-O49fn;FSz>SfX(kL?Z zY8)vk_&l0VZGOv@mnD{s$>YHitRH&P?1L^pL_kn#VG_mF)2rD`ipp(8Avw_5{JVaE zkD`CqyV*R`E{;*+^&LNc{K-5NR5qdjqy7IXsNt_v8qKtOhd?nNbE`|g>Tyb-`0Belw#=9*gv91Q> zc3seCS2=lI&r@zDhJRfSr{m2&9mdjwOFm>mUc|!PD@!p|#AY;%WcI!3G&PG+-!*Ws zlriw0p!{6vY_fY4_7{WWV%uXUlMAQ}7i&P`pkdgvt2TXFHu~4P7>X(Q?S1`grolc4 zmz^Q3Vk^l|IHD`kSLY0&e!D;%%b zK6iqg##qvX8!ak?(A!(U=?s$im^&;78X<1`c6ar5b<#x!&$9sp#wpIKZ&Mj|I5Lc_+q`;}kS!6JJR1^>v2r-3h zzgFho8HN1;x*JH&~?;@9xb^wZ3 zb8xq^6Ov82A;a<#0;T(5$m0%Qb9HM0zL!+Y-qNFvf)}287^xFQ?a~XkYc>VdR>|`P zkaq3_;Q8L0NnmWOzFZCzp>BTQn|78Y1L1l(yk)qMFt%-QJ1ImFv$_--{R;rJa(W_ph?C$pOWN+ewo)*W7&ACG4r=?l$@LW5dD+E`6(* zsUu4^r{=(=c9C0|2RclEne+;`xNI$yY-wqX zvV-RaGWa{4BQ#*hfD`l)JZspT(BZZph8R_hCa4J+5eaTq3=B_#K&dxJ=8o=nYpUOQ z5?B@3+4vg`lo-ZiDX1i1a&BwVCD+Hc&l1Ehe)wdig+?N{54(vr?tpTLoa79w9!Ct{ z>ac0)q@%O!`ehu~20o9$!eFGQBS}IXW|xm}?6wrb#9M|+oKO`SxQn`i1ZFyL{{f#5 zWdeoo!H_<$Bn#@)= zjH;>zf%6LxUpB1>e;kxeUe_Q6wH|#QW|yxXGJs6ihVq6*A=-~i%M^>34=ZHc_nrAp zqIaDk+XbGQ81dDOrcIAJ+2H-&S2L6a8M1kE2IY9qNw4N0mRbhZr!fZdP!{!hsfqJh z@3Gp?nv@qrtLzvBaTp?dj}gbxjvjAe`g>waklKVz<;;~^O72VjZaymcQ-=r7O}KMw z!uuBzvH4Nor2+2q=8{M^w}3f~jt3@tWlLd@jm+WJW`|VK7->6CD0RUbQ0s0)Xd@7h z^?*h#AP7H@75eX|!@#hiTsJ21!QeoY&-YZ|7$4Q%SDORX#39X8w4%!9Z!_}$~nIldvct^2L9jnv!(20p`Zk7*^RAn7I374YFs?Ra} zdhmFH*2KP^FUz+1ED?^ACTX~6{MCdkl?564Gr8Jx$ zCxKUn_8yJ0B;7q7nr#XBneOA$#HtNxbImZ+&hbd-V!AP|jS|wPoRf+~kgEh(@Io$3 z5)e}OJnVPAs&I4t)g#o9c!Vj(omeBJlCAm`i)o0e{vjK-o7qQrFMyD~3=I%pDKGnr0Gzwv@Z zI2Yh}PRu7i+7efAc86<5&C}nLJqcN0;Knml-4O)^JOdk*)412eZ|jNhdd1T&1vC4N074 zYQ0gEqv`T&waH+_x&N}#U#p3zYZzmZ%gZQ0yN@P`{@$P$C z(YLUUx(TA&O%}FL>alY4CKuGQ?5J?gbrpjihx%^DUT}e*RG_|ci($|w*}j-4>G?1- zPN~|q((qxVn&7Z#7KjdiH8PhxrA%(&IJmTVl%(G-=JSYCEhj>a%9rY=GL-~eLOIn* z=K8pJ?@3ncTf|g0Nm93GbR5f(*)dprBC+drkuurHZ*9zA;(SzN^w*?f+(M~+C3h)N z8gW4urqh%R3*xXfmcj1fWZjs24*yRz51)za!0y{5tx4im@ev^0H&juLor7ZJJ0!CA z1AW$)wA@8fg%y-uYo#EOmLJ=}fJV~G2w5Lfs7Z!OSh^vFl%9X{fI4QSA<;pk%s^%; zD$7zVxdFvKEqm#A|DCw^6jCf?XgW`Jg+A_7!lbvfWlqAeBU>X=S#q2(ULSh=%c1iq zz7)VGTet`z%4Ro#C)B-2u~2YjByu~=*xnaw*wNfatQ~U6@~Nv0SbKuXKCC0?i*yL7 zTUG3ia%zSx{F-nt+!qzvD<#G@MZ#qXm}GLx4uE~lUV114gkwM=;s-5(i7T@ts$01o zLfLlZvb>;tOW4p-F~_2riI)_BI15FlnGy>+*l8RCyP(&a9EHY8`kPZaJ@&nks*ohO z2QSS&SCHLXF?}f9l;q47POpxS1|0Q#z?DQL*xtMMOzROJ$SaDU`YE?2TctT?*LdLYNFf*luox?Yf^rJ4JGbKL=mOyAV!r^38V^%mo=#_DQN9^E-6->&5aAG z{Mh77<;P!~qm-IZnC&@R0e{l z0ni{RFnDkXLmh?W+2V*o6zY6X@Iu0~H!GlcSP=w?nko;)=g3rn4bj7vDOI~XbQ_*+ zh)U5dIHxJs@%uY&;RR^4TAeJ-i{3}@!S(QEXPc|ICA3tv-u=8Cwt|P0(iK8*WsUHV z+A251K|y)Z5!>V+Z;h9d`-h1(;-Jd?q@TW}P-XlCYB|1&0O*2G2 zA+^;{@U#2?g*)pHjE|qtA;TB|b&w?qzGa?SgXmlPyaM7Z$XaMug+1V7!oMJbl7l`E z-x7f+_@Uk)HC0ose1$nPBTOn}v0m}aYT;h^62A@TC~q-Em>5z#4(h1UFeMaCxj#6B za50?!9yetpZmY zE^{0F4*=EMtgL*VnJzdM?|tAh#^sZvhK(MAX&knKoO{?+$ww{}odHi4*lVO?2;S-< zaxe^*6Epbks38e%bb;oF$+ki6fd(&wghAI8E>1jEr zPdD2R0yCSl!}`l_|Bx8P`snf)Un;T-fbG9t9}z474*I^QusgiPbbJQ?{3a&EIWYq?pqx9%0@PO>Oikzsu#EN=?Jiy z1eQPZ^g6}N)XT>{ePU;*ie)~(eTWM7!EaaM@pSpOEAju{m5^Egq~&IcU-7t`)&KZ^ zg8o{??2>og?YDedHs<-3-#o0D_OxQM57PTN#{T`TyTy+u?Hk@^K>FgR{rIcdJ;G;I zn8ubJ{S{7~7)}2Vj7E)k?l&smW1G?6uEcLwf`$F~BK)>Ge!CL?{W$#gO#Jpt{Ps-z z=N9?fj`<&J;@_9!w|e67h57AD{B|Y&d)xf&nfUFQ`0bf^+EM!3>iGX~br{-Exm#4_ z&Z%B~f)9UpBywhiM4n+6;#MBHFs@l`Nl1R-M-o@%r{4>$%sk=K5L5?2BTL7IBtOv{ zDX6kR{mb8Yt4`1AlOkq)5!xm8$S`LHUQRa7WBNuhg1TgFKJn3ooN;%9qAu;f{O6th zL(20K-8$u*j$75~<5`hCYAoCRy4!>0cq%lINT&4ae8$MJo!{@Se!h!ThDAqw=Wo5+HGfL2 zK-VVsKg)Y5{k{m$LB46}`MHZr?UwwoH>LblTWH$;{#$=v%Z|8JhUnG2N@qVi`lG@P z9ST-G7wt23UF|0<;h^V#y2yUM!G{hWnU{R3I0EC|9Y5auWWl#zw7=AK@=NK9rHIQ| z-DkxZ-;%Mu#iyDiXr@$ERXHdneEL$X5TJjRSL@VI&RB+^?03 z<$8MG&XjkLOz{Z{!({^#m7TA;b{rFuJAHTU)EPUXY?|$eu$g0RteqO|*D5O6b#LTX zJK~NN6^#4+9h*t(YCG6hzjXH4y<`7qF=0!m);pqn-Rx`==VwWTeyHqjdeQ#Y?pwCI zj95_iWj9zrU7BW`SYFh%d*k`vP5#>_HfU}7MEQ=$sao(s^+|TY_QAdra%|@CR_83l z1POBweSI-v$J@Q{-&ojTPchzu!>_n+#$&N#aLPWyacfB>K{}WK# z@>oNSDxcY=>eiZJ#o5?h2bLTPT)X@B;adZ0Wx*Awbn494Yx|UVjm$d7dmVXXccb9P zZhdav^kO1d&_C-G&33W*oo%z8MhsEw?&-N*F3cHyzd-wF%SQcjJ zoGy7Y`aH_oUPWK#!56XJ(k9JmF|g^5lu0`-{p0Pk*`j(DxtES!vuyCOs~=sxeiic6 z!iZ&EqFVc&{!8WgU3XUrEuXcn&F^}Aws7mMueTn~JGXeR?_b7bCM3P}Vf5V<%Ytf5 zEz8-_`U!){N1qV3C7sf1Lrk1z{l(Xl|Mn?1l5ySS+I{NYJ-35z?0LPkWMXNhxCd?H zPEIVl7G01Pw&cZ6Ml{YXE~qmpqEAFh(Z<@r_HGV){ho20(>{WK4xYPrfNgaL*S$xE zks&^5PP_w*?~U;}ogE$fGX;n5Wi4E}!oF$yeVu}91Roi=v;XXkHPW`#A7MGTcAOOd zq2*dTB==M>KAf(u{RbYi9KHyC#d2=2(dyr)wVgwCdr#Yf(w37fyMZ>IA@xT%XGKt{M8GxZhC$8M(XD;Tk8_~1lf9BUy|OpG&8!aQxKwpk=yTH_e~yt`u8YC&@MzC zK8d2m^>n?1evNILS=A>Mm3rD*HV$@I`*fdbdAr5)%*-G3&;4_T#n=Q}r|Z33#j+o< zaGvkkm_vP+|2bjyi3hU_tL`cK>-B81wtX9~iXHy9#BR4|H61u^Mn?xb+h(;Vwcl5x z;)MeoSM(1)a=|V2rGeunM?HLY^pMGYYpm

D?)3H~n$Fue0-{-L)s}7(URltsd^# z`r9YO^E?dzBbJK9~g%sv~3$hoS zBkT*k{yD7`5e`yjUWaFh+(Ndpa3NFif5Q2}#sM2mEJEs|v8Y9w($yTeq~n4f-9;PN z4gQ?{{>ym4$Q>BY;zC?I6~+J!OH`di7oVGQ*2wdB?X-bF)5_?autJcjd2O$%CG=t)@ z?QR!q-LtEfK6O7U}+Dp z!Uf$=G}trm(%MaK&F06&t}ne_CvfY+)kpWw%HoTXfDqgGvWNZHsIhu2Sd>HBAR6<3 z=>NAdvth+^;6ln032Nkyk1sE)ajN594%?l_hfb1%E{7;~Jk`e?TC%2l(2uj5hvpuf zu%}1RojxNke)39gqlf+vA@6)`=|<^>;f!kQ>grVWGjF(FUt;g(I;Trb`D|uLS4`r~ zm)=m(t1Ddt)1($%WWRJMeUS@H+$Q(G_U}Nn9XiRw*pqEkKs(}V*Sell4or~5NAgu} z5!@R{sxpeTpy$C}^5=rx-V?Wjl(@)IRp-91DuSEa9fO)rE}!QKgI9eUHrF3<-Rm^> zeQai5)U(E4SI*$;jSD(eqt7>qQ6*a=Lna;ZUXXtLKA~0jxjjjoi29`DI<;GS#*r@Tp$8@%4KQa|*mym#i! za}iv}U2ckR<$?{q@{TflF{+N|IhH4~K2{`Tt|ap)a*YTd+2y8Bas2*`p)p&qE_rk> z4GyKaW@OY}x|+lKA_R3IxvrKn*;h^e@Vz0=*B031DUa z+#oBse}zW&ziP(vw_jBeS$?$o?`&PQe27KT1!wu-?BY6~{O0@@$Pd#xVjW^&u+d}o zdO6xl`wPe;arNkjBYp7DGW*+c%a>5w4odQO>@(UeazVZ%h!Td04oLYMNc+6$; zr?di(e8t8~QJ*988|+*N2RsY?_mpV1lBX~5_4S<5IVwfg;lA*1z2xNrQ9B$@nWJ|; z!=YwvT*kq)x3ZxfkE#k|uG-@Cck08&z4~%iNynylzIIRbDg8EY!IZwA+-ro>@ZC3u zCx3jq*NE-COhgX3xZM#ZTx)m~a{0}qzDry8EeK^_8uezgx*tcd0x|xYAQgg>R^x$paPznANMUDx~lQ%Tn+>N?gV{o|{*ywr2c&337f zTkYur7n+4LEnbIA^*Dt+zJn`N{P!sCEer@Uli%z(K*DNCl21TL{I*c2!mvn48P5Of zf-ht^RYlh=(QHuac+9|Tid7Xkzheaq{4MqkDL1$;ZQW~CGFG$hwV8neY2MHk?5JuB z2!Aow{P96q$P?j)3`hTSH!3Nx;F+RdVdpc#f|}c)R$}7?3<}I3@{r+n z?OcE)jQ}1CmMR5}Kd$ zhA1BdbL!fmHr4oQV--$QO$gzwVh~MmHpMwSKhoyQ59`ig>CM@ty&>2NAdh%8NZ!kn zUFWFst3bpoIs-L3&6WRZ;ceo@Qi_93AKCAf$M_G_VhCFXjfDpXOmxM z*#5y#c-*6TAxVq}1M6c>(oc8*6lndvF8y@DqE62#)|&Dp1GAHQ{WN5szu>ibaHH$v z{rkTd)kMpr8lD~7Cj0MA9b8cc#S7T$YoZ^+R5Fad*+Dd-jD`R81h5#p!Y-Qo(aCRS zK%rguI8ai9kcB-SSpAZ4=YkPqhUK|J}C25qLv}I-g5Y>nOiT@zy}_<7bfe%0=^_;4Ua^ z05wyoI6-Njk=XHoq>8mH?T7hm?crxg|7jE#5Zm;}3Yru;C%~qPZvUxVv9J%?#89XU ze3@H#-@qzF_?JT5{EWwr2mG(uTEAm=B52fH99qGpkFYymKO^i8Y{8fg;I2SD4vz!> zfUHm1YB^&Q*TW*TqT9P27ni{=W)ygOY%pUk)6UvlGI86Xm^ufI)VlDfh$MWkY zHyFJMaeEt;3%l*fc8L%NB`gd5+>rkg-fEdEs)i?AJ2Y9^UL3=na^=;p^v|OQ8e85 zDbWKA=uF6lts39rK*~P5L;fI#9F&?I)&Sco0>Y9b6aBnm%a7?L)deiEf9|>?cU5R( zi!2d&HZ4)B-pDz+e(upPu!(7(82b5ee96$^C4Xu*ZO`A<-2Fvr*oBU@p*5be4DPJT z8k|1ZZAp}+rfcgU-K3Jx<`rg5{_UYAmOM!pD|f@hQ8TptWlG)8uO^v&C|eieYSML> ze&`!4MW}z{Qc+*HS7T=ZTFpv% zzs=7Uhi2@*l`$MgG&{7}{Z~U#1q7v{EK1{ss~QLc4a7PvCIGC+3dSxdkpNg4a23Y_ z38M`qE?isUmB(F&rZ5cR!bb6hk6Nlqph5NZ@ClUm!WFYtW$cGG{u>6$h)IN^@Q_iE zPz=)zex&i@RRsw?d!n2+rHFC3iLjPjikMjg|3+U5D&FPCxbaDf{sU1C+Xdn)a8wL* zA@3Q$)cQ{>@n*xBoo&|jsARa1erMD1)aHEI(>`YJ0av*CdulM9F|PT0q}vf9$^;4j z8EQuq-KdtW#RtOm6l&{AkKLE{u_+p81Ft;jJXh1OW~|uV!+eB#Fog9E%1HVs{?098 zL+Shjtv3rh)=TAFQ~@2dD#-f7A!oC;!|rT)^FRfAC|By6`*Y<~ycaf<@cd*j{rv*$ zZuZAokpORK=y2ob&#A#8`}0&hOoh4?p2fAaSOi1AwJUc;Wa4vU?NVckc__EKLeCLQ zPh>8L+eH5d4X#*C!ag!nt{3I@fUu@LnY3oE!O?ywVmu|e;yzG;iq`-V(&n3PSr^;y_IGe?p)rl@S~-0B}Pe{ic7*8#`tu@j^>-Ex-AKr^3?CuD?SGqLuiGBFaTQ6eL>*`^%L9Mef%pDASI8Ow6EH~1M3Sf70#X{eRJP) znkHh?i>p5I&6V-R75nNd@}}ii{J3(wbc2h>GcD{eu3a79Fm3U~f~pGzp#Mc)h> zmhal~sTGz}BV{$=vN`Efm)LloFMEEfup_{{^u==pDnZg5pqaZB6_d&g5r);o+gh@n zOIoYC9d&)zlsYl(&z}qmoBv(!SMPCQvUDwK6q_@~xgl^YlJgvDbwp}aq*}>(chp=? ziFEs*u5v`jAjj$OIB5XXFPbgqQf?y*1pNU0-ah}r=D0$g#Mrmf;`N|HOhmnuzwgZ_ z$d5Xv!VW?9xGBFvMb=XtsD} z5K|0a0C$qP*dumquwgBn6`)*1oJpFS=yD7VZWTAqSR}{f(SNqQ_L<$ zCD&zNO-5dCyk6KittzHPlUfZ z*CRHgKrOO(^bxM7DT8Ld46h~KtKo>fj+S&cVTXnO9fdLXvcV5z3>8d?!{Frvt^0Vi z$EXJzyE33}!^45W$Ui~gQ@%prbXI$}37;ZpibrukXc86(PvoXuu_XsGm6K3BaSL(n zcC6qKF0B*|!_uDhDQY>IC!Z3M{U&p*DDDvRpCY!x`a^UnaCJT=ha^aG zKMV`-b@%Bf$L>=OdzY>GWkXzMVu^o&e<0su%13>1kKh7rlv1pqjU_JL`YX+?PR*Ll= zDAMhQy~(dX{)Wp0l)uw`mV?qSG<|DKsW>q219xvhE?khi)6v-x3qeOjQJwdXA-hFkuole^$Nflb`zS)@x?wDi?E}EP9l*sP>eS z=XjfsBW#XC;CtsxxGy$n@H0yX(jSX zp>5pl=mv}s=7hAoaps8?(FWo@?1#gGRd$3NWT3W*EnW0TpP)msqe)i_f{AM$?BM>a zlk3%)#L+RaLm1{94Yz9uc*F(}C4IirJc&Gg+nYbbh>2$!>xO;Mb8vz0LCCNIkvkWT zo}8w7kmZ4gK)*WnM%8r@6i#P z_XRM_Y_5ng^`r6`!4Nr~zL*dqa`TaRJWPD4`l{|-U_r(;4Vs&z+SC+S-DX099? zq1o;|ZyUieT#RUer&Y%WJgj~cEZW|WukO6G?z|X3Kkfz(LJ~x3;T9iky1~EE^ML-^ z0G$zN7+29g1K+8cjMdx3TM^!o9i$(Yh0!S*xY6Alp#8ulP$yS#_@r(8z9m%6U_9J5gG67QH|KtRsj^AU{xejp9%3 zjfA(0AW{kT2V`MA5F!2o4+Bq6&yQ7AwGcJ;fH}mE5dv$){laU_HPWIw*qI*j=tr2lA`l+sFCl!4r~$wdqmTqN zZCJENL%<}uuJzdv++%A59~A2nOqzjI5)sC++Y-SKDQBNWcP;cK@Zq6?9tYqy-UA#D z`cSXZY=};gv(8~gBMgXrz&rD|mH3*~{-U(GqbeSDm378`YJ=SkPrZT>%`iIALxSpu zT#oSp%RF>#p-F%fDd+gZ4k3~}N7NPr{yVhb4GBz8miFDz*AtzEQzmi7i*UIE)p zJ*717UGh)*%~P%VjxF769NDqA_WAFTU*TPi|re>0O@009gYS(CU7Evt1~Lb z#aW8afyp__Aly#9aTij`kKE4qEez7+nJ}b;(i7nYB^6#n4o0Gq0izP&1H2&1VvxqE zd$#!MmJPreup+?1!58ll9_qEI?Z}kSs$wl1AQBXoxTV1V=+=obiRjzl*>wTRS6CzA z`)7n1TdJYaqk9P^WCFiRnO@~e)BCZ+3W$+TTKDD8RiQvGs=9}2b?uIqd`io4&&J^R z7HVym0Jl&;s$%tvk56Yo*y><{Q#n=Qc*o`@))V6!bdjd6CZAPly|o+^!pa1cyTRAOJgDIUv26 zs^`Y&+nn&WMKjD-%&TD3&b9_UoTiNa|1wM+keW!dpfhQr~XJ=K(BN!D{DQ1{_5$v z<6fBc;Q4Hg09yw`B^nE?&1ItJFabd)g`hKE6>gt`WE$WQtR%83=z}HNJD}^|p!5d{ zrYT2aN1S&>7NZf6Na&r`-`_io(3LtvXbuY`tX9RE)UFGP4?y_L4W%4wapLsUMCsoP zxm=Oe9PjMr;0mkQU#%SwfVtn!RRhL@IEvU8hy%f;Ohs(;=&DOhiR=EN5%-aku`JY& zfB#DqEEOP0zngOlMw{jP=tYQ5lS=b$%(qme!;uEdPxaxVZ4As8#Z>{C(sT4mqA%7v zF$6(`7N}h|OsX{7UmHG^=T1czYF&bSP}q1U9xIgq?JkS{Zq4a?f*la1hGB;yuHf)jPeFYAW`5c~Wjh9);wfnUva`(Av z?)@}dx=;7uqpDo`E5hN~&^1`N=;~le%?MlMI5!u=;KP z=-2Yct@77+v*C8L<-B*#=11Yxo-7s6l|MPs2NONO2PiOU63J^=ua2fD&t7AmXj^ER z!Fn{o1ZZc+W4~V)6PMde*T^c445ALcs9l?~?xA>r`ykOzq9vRz)ir6r92DzK znc>0f69J$20_YRi>H)3**090Og?VW1FrSJMp(AunP0il5#?}iZL)e!zQtwCshvl1*Rf6bP<#ES@)ofq4^Rsqy)R4jQX-NDBBKG0;~`t zzX83_r>NXoEJdxw2z^38&QMgN`MWD4P#LgbAv(jtf<*;p>iiG_B9_%x$TD48Yj4)F z!H!!LD6!}w>3VR>P}V6{dtL;nhx<(4osnC55-UN;Du`UU4T1B)4ZHIrtsP>GPT@9l zjFMi8uvqj&I6n*+Lvv|o!gFC1dCre_XhLVt?TeD zd^g%(rt4a1k`}nSwsl9;?YKJ`7lPP$z``xW#w99na|$`zhQBHrhlB3LEg?(4q0hqeIs=rm+a17}9+Z z@qit{2nAol<}u<7S{YPdpao&UA6h4ZFn8hi!A1ZLLNpj70yZSD0TZ_F0^V`QG`JNt z*ElgP=9}oFW{_7?Ce9aP zZzHED95AwvWtZFz?L=5Aw zgJS_C><#BfqI?i7IpOlm&^RSLu_vdOMsI8H!jY5sD-I1&KM}&i$Wn%GYb}exS^;YaGz0zB$~iRc$q+lS(E7@q^H91wYshk*2{}3F(|d)Jx@2%AWK<7YluV$3)rr{pbO^uXv+u_s3dR> zizy=B#?CidRKD<*g#G=Du16l5twV@IsV!@9m!)x(&4CeKXe-2a@#yhOJw9L3%D}0* zmC=TcMF?CRk1=r(P{Rdh?T)kW^m&diAr9b?U!ut_=*obFqwnSrn?nGdMf`|S3fIhB zff9>QBv2JWk74teJrQIVE7f}*KMNDxQi374Dt--mLzMUU2`qiYkuTC}*8za=t!hQ) zg%XZU8exp~1E;>+zxc-?%_QDM*S4LxAjcs*rI${qTioU6UBeBmLP`Nu!xMU~XzbAw z0CHsa#g8Rsa-hecSwyXa?dpcTyQ%=kTa%W$%z&qd&+B9o`hl+$yp4d`IctDjxGU=~ zywdt;wo{2op-mx_#y8W{)D%M@)+3@pVxC~=K#=MJdu{t*)~`MgMb<<>@xbP@>8MT6Ym+Y4n^K24GnCTV(yv&OW4l(2_*7m!IFQZ z49h$7az&o#c+4(OVX<6bW%`c;9&vzaJ7@}M_0fZ{(6P{e6 z)gje)WnROnJizfGxRrpvn7=-Le8q;*&Z;ZVpomU(g8wq8}qp zwRYcVArB2-9qa>hl3$QT(HeCc~kL}P5_C%ajDhq3Dt6k z_;!c8i=7|HiA^`be$}BWwsMI$FXS@K>2-Yk$P4Or>3v4zy)Gz5RR!I|G0vZ1()WFc zsmDRTHC?kn%Rx3EqX4ahg)KBOU(=As#I`efcX z8~70XCXI6oA14Pn1oquDA0eu6H8%|`4l*ga*l7?B|6TJPi>B)locR=Afxll09iZ36azpDc>CI1 ztlZun6aDHt|+vgcUe}<-CHXLr)F~KnHt|m=P$F?^rwR^|6 zU621I3Le%foc2DijMq`}d_T`E^2R6?Gjo&EqiCcHBv}~A5*=FF8$gdJ=U$A7Qzg`k zP6>Dy{uk&wiOEc)AIK+KP%+LIR1D-*i6G>%nH?=6(9jgo;?@y3d$~WbLH$eUk>Nwzc%eI1R*>uLV>DlU)=@n+? zzu0;@1n2Ha47@m}h}@s1n6Ty{s5ybz*K9ZZayvG4SUkoBd4oX6dYw#E_dmub&(dEn;n3Me_laeX_-y) zCJfy#`uk-YERok5)@@&7>KQ8TUAU~(02sbv#3bB1ajtxG_w>_jKjC%zPGm;&k57l+ zFGxngA5&C5WSPT1^+8)YiyrrHbzEfKb2T+l@zef>wZ9lPfV|wU>9?;XSAA>QgkZ1G zX=#zeGQV@U()v{E&i|YVTxnDMAjRysLw%NUdxDFbvq>V7_bpBV6_-Zpq9#^yk(E4hyF2bip|GwR`!&iUZ!S#w0)L!K20et;B#HsSjz%mXtYEj*4g0L6j= z!FM9{lgS*v7qAZsuja~|AYn@tIYYWK0MlKzkgU1ktSWX_i@l+ZK5DUfdRmFoH~Aw- z?OS*)`ln07rrz+pFwb>;gYL3L7Zs0XSQeM|D!;NVdv--$c|&IVSXtEWsSk65R~OWb zh;B$azb@{R)w#FR5=|Q36u#bEamKmmwU;xh=TSk9FKO#KB{s4|Yx#Y*2cZ(5iWK$z z437B7M|-@j)kdFuOVlo3=X5ys7OKRXiJ6Fr3gwMxa)|UlQ$M1d9CziFBVYqz?6n`L zwt!Djd$K`;4g9_<14zur;(CYeXf8PP#l81-qBSobshBuoI;}uLu0Tb}{$ZJcz5(l6 zLqVk~iu9NEwMz2@u#0s{?!!YiP4CxNY3;U>WIN0YvB_o-WQ13KiBg$m;B{_IoS9-P3|t+QRLcM_Iz2we#N7SO$xz=?p-IpeG$1P;<5dm%DgGN zj>ionKX3 zd+xh_y>e^KSdWWUMIV>#3pAPZTJL_$mddAuI$oXnSEO;UYwgR?<3hHy{%+iUc=zVA zC(pZ;8}1z&o|CdFByYov;D40-&HUePrSg2o?1tgkF9NAp?vGwQj#p$IjBz|aa{2WU zSS1Mgmx;;byn@#oOv2Q@f1JEO8V=E`B35W-S!Nb?kC&L zp2wbUIOJ^`GWV*h{>AUMCTy{GIK9++sqzZ-fUy{fEgakP`R$WqM_*BaV&ioO2>*2e@P1-T+ERA%i&vDh$Ad zC}MGAAwSb&?=InRqzr*C#LVLq`azw(iH{vkIJP0Zu^7(Wlci9>cobB9D)2-2QBy%u z1ON>=nEmaQ-{9rG$c@1q5#tO4KtS7$Eu7^!gADHZ$>-hg=vVimu0K@9mjk}r7JoNp zbH_Jnt{JA5xXx`5zZ8B!@npO%J#xLqya~fpj#&48e>qBDcN7d>V}__7o#g4+vT-Ds zWf2*c3rDNpd7csD+%j=iz9E!tJ@z$(H{A&9(5dgJaxQ4UP;fxdJF7h%F)71X7z?k0 z(>IL|Tht;L6HM(qRuV*~^%2Hj4(TRxX6Hd;F=%!P(*<$~J^%_ufU5tW)kKtMM zzMX?ack_W2{}JJ9+~8B2F2S`f}u~Bco=i&N`l*gbsJ!Jg?JB(Tr`J zu})dXDDa9UCKTygQ!)&0TpBa{e{o~f*|WA8klC(moW4Q9OeAz82&5+h2J z*0JO?;MRrDK)X%eu1T0{Tff{pQE}b6yitFAwbFCRg&Fd!A-%$tjj|W-LE)G&u*WbEAVg2!bwvOL< z;{`FfZ<4V8=S{kDjRELN4_NfHbF`a%ja;Gi76*4+R36b=`$hM~rzIBNJGvCV>f0+B zcN!%i)Vo)sx_42wHOjLG^b(vQF_K)}lxSl3>|(I|o!aEuu(HMt!6nE3(HFG+Q0Jh( z`nCtajUdHygMB1&LpAMCb9qRTte-;hw+0rm%Booj`z^eO9$7Uuz4G|ClkJ{or7pG^ zplWNBwMg}RLZViyk!FW`$Kyg6{ckUNREK`0`y0a2BznS_|8;EZqIiwfb^R>N%)6u? zo4a4ZuKaDtqvhc2nH`gw)KwdV2E?TA$6(D|=Xs!ChI7L5_=I}LfQ+U=v-T?}bXPn# zMzKq}anP6~^wdKh@0a#3?p+s~bSDVH)>wSFeoTMPhLDFFZE(((4U$#2lwa? zo%*rGs#zK8!;)m$kf|3gF*B%p;#0BFW|Y`;{J#$$bTl1*mwO9hzbpP9nW5{EcP}k- zx4-qBcDLkA_?p-YRX-nDx^uxX z|6XAx&o^%L-x%yMxg;~LCEBCH+D3VS*}7u$(CEdn+TG`vhb5#Ad1L4?{eIxuebx2x z7ppxFNB2JdeqotUg>zZ*1qbO&eR&rj*afn7%5yC06x={U9IYMO$Yl_?p+z&@#CqL zFNfUMh}V=TcLelOs8O9f{U`T5e@?brsf0**CvT;)%9<}BHeb#L7Jt846Z`l6p`E|p zu4M85NpRTyJ1#t7u8MYhcJv5XX`krng;diTydN7bi7 z+s8HJ#JeOPJ8|J{gMX%M9`+`(o<}6|Fv}66Mnr!&qGo%%pK6|@{C_H}dj8WRUD2lX zRVdJlElJkj&cge;jk1pZ zxBv-0W7P44{f8T?H}&m~m6n~~y6Sks8XcV*&G(HUrianMJq0eDY2p_~W5eQn?+$6Q zt;K(hRytC?(rQj6#rs;c2CD8)I<`=?W0k(VwdVopfy0eA^^-a_-5amn?xoUG!B!G1 z^HJz^Si0}UT(-5l7WPVScLvJj#dKIVu~5 zC@7q2ES%Ld$?ZgvY^2VAIc;<(bz(08N4IQ@vmU=hZOI`EGnvVJ44k7XpDj*a|8u2h zGss4b{;Iu0SKHs#%;}p51>!5mxpQU~Hw{g}9=jv@7{Y?PatZPMmgbMI@>YKHYgFPn zM`#bho>o&qO8d^?TUEoK;6W1ahl^TVvfWpIYvF8fnUAz+Ru>Di;Ama1WYhlX_HO@v z+A*@y)mEeXgn)e}m#z<2?I?5n(FyIY^&%{1g5Vt061(@bFF!#>8gb^;Z{Tam?xuqo zj23d#2(U6JR(niV_r_@CG6&c#a2{eC>}~7dcFZ@U-&NO* z9q(9$BXwVks;$;Z+C5ZS<15M^#j{BMi{^NRgf81&X#t2tIsB8fGfN`dckxs%N zBQ@R1bz)y1vKcD27(}~O&)C;CTGmCqYwdhxW%UhEl_pIvaT68X- z-!F4?bXHZdwP+3R(`{bjsTDD0SEO&f;&fqq(-3@H9w%#ZgB#2Dmu%BzlY`Te2C8ak zXo4HKXe`resYlh38FdDVi(g5fr4IAB;Pt2rS5ma*6}J7$DsJ1fV^j6U zcR2MuwNlBUzM+2QxD_8Y-mJ&nbydfW)|k)G8X0*&zx{3Bo>yJ5>+0CzWp=2nF4#?* zU%aYsb?Z`>^v?VEpHaW#dS?i+n)*<7Aim>Re9^nnn_o?v`tuyy*@q?Tk7v*S!`coT z3rTMf;QZimnl(x0ZDVY((Cp;AM=vj-hgLMvW{^l9*u}B=wcDGjxA{@-T7g59Z7qy| z?1nuIPHC%Jx4q5BLHZ?kcR$UhpXYtI@Tt1%5vQv;z;Bv{?d!0pUxg<=)Rz7?*%s;m5hzpVzrZPF~U>YC`sQ`-ur;E z(#F2!h?!5+4|xni&Cng*AlMgc^KY4~dRgAbK_LaE$$yMgL=EG1MEd;Pn51Lr-Hj@$ zr$7zYuJyuwy5W}>h8a-?Sv4Bzchrq<8i(#^AJUo_XI1p4=I0mcdWRJh1_yeYz0Wt6 zHVxBCD@d3DLRC!6Obi&Rs^phtQe6@%JkX*{0XAwA;@i~4XYhrXS+PH+4=nZw8l(8j zyPMVfbZ>4eIp$i@UmZP|^0t&Cp3)B`p-CGC+kA<#xiDCkcj?zFH1$d7Ya1G15o_BW zwnAx&*k3G7b}}oGTfL*??w6Yl&1S9TUOcCLP42v|YrJDs@v-RJamL!Ai2+s8p|X~T zvgAcGK1aqjI+zJ`AZO@&Z&?tZtUSce?<~c%#Y+21vOKWrHAznk!>Qtof{-5TT5N$x z-BWmM7HnZ=SSP_)lD9Q$2{}UsoUR+AIwh)MXTW)KaR_k<3Z_H}JSfBT{^+)Sq30+}^ z4HJ}z_?D8CWedx8LKGzZIM#pt`H}F4QAho*Y}GO4(M8 zEn}NypC6v?Ddg;d(!VZUi`sBp()HYCHSfWhFkqtKYDGtl7Imuv#!bUq^(82HI`TJG zEp9p4&qSf}_y2s6b?>zGx@_G$H!S*S^!ngMX?tgZb7Wvv;>tfpzvbqlg=cfVY;{bFHp^9AEPN%unv@{sg_s_5mrPXiKO7G3=4 zP#?1%uB#wBNgw=8R<&D8T#q%o8||iiQ8mokGb6=B`-TxebMdFmF>P|Ik`Y?X>dvOS z`sF%Z;5z6NN!XQrNGg$Q9Ja6xUbMO*W>@8am?_&nG*q#CnoPVUdkL_j^iJywIF1Tqucg1+Zr(7Fj3kYilB&J0FPJsm)Xc)#-g$2uzth}PKG!*K%Iw^oNS~)93dtrW z@Yqt_FX_rVW{X$fLe)DBOVb@QO-vu<=?sX~Ui)_&Q(~gu}_@n+uBsvF@>}xa+G44N&$vc*EEcYuDIrIbYW^n><@U} z=+2dG9hGf|*=0I~6~B&HhLH@;L#|VpJyud5P*CUzV=4R&1x4J|geCS-o2T>(F@Igw zO>K2s_u$pJuJ$bh40n~UC+>l)$Us3nzUF|9NJl{ z)pDIfFP?sfPeq?Sylf1)lq7N{h6{K z3IJ05HQEfv1K^Fdk8)O`ZSXbA#U<3WoVGrZ zlsEFY$C56#CSiBLjY-sO+7{Y+-wNbaq>9=PHxDi(k+L}F4LOqXiTC`R>VC%ZG&=*o z@r(5u9A1$ee`@K$8@uo@7OyFXs@gR^UH`)8wONyL*8(!GsstEq>QO%N`ue9a$0|HN zWQQd?j<5JqSkq6MeP8MZSbb9eIm1SOe?qai_w1>u*CL1Jlpgii_hKZ-sq59jW29tN zP%Cv0R-BC(e_W4yaBozC%zAZWAS(hP3X0| z-TBhmr{`6Z=RdA|r|i2i-Dmu~36}TfI{5y1YvhGki!Ns#13)@kiA?$vE_S(Y7rRie zUp7r3$5H#N5NERe$#STaZ4SPLt{V;tX3GwyA z;xi+%q}^TGpc?kYE2^Ng&s+8}*1GUT8PQv5M0wTkV}8|GSFKECjekmF7p zD)UTSTCckFeT*T;7X4RqB}R91@a~;m;3K4n+M$6u#z9wmL2q&Hc2SqZ^MfKSf9g((DlITcqH?au2 z2dJO6=n%mD#^%C9STq>F18X)Z93uf3a4|`U{PW5ubnx$x^d6gDAIWO5>>5~K4+p!y zDtHZk5LoHq=56nxn;%ZKZaJj2fR4FCEsC0K(g==d8buz6*i>uIhxE?eK^<<{2kEiK?cLkg0>BiMf&cj@=Q+% zo<2^w@zykTU>9xR=2q2>)7oo>$Sh5`paE*z)VE1{im($g2u3M^A!5GwCP5lj4k3(#j z?iLwjZCCwb1l1mCrHzqxA2p^W|5iPI#l}V8$zcq2Yzo-Y+-(aP3FOsa_)*f@t-Ah& z%kMr<`zON<^3^G9N3OCd@U5S_Jn=jUd-(ZSbJ-)3{TgN>q!L1R0NR}!$*3^(0s9w( zVB}{w%F)mQ1T&Ii(z){aZ<{_yKe$WJG{dMb{|?fw7Y*E#IMNsdT%9>nS80wFG;cf| zFdhNa181IG^GC2&-!009-`P_UUzDJ*I+$8Fr^Cv#_AAfxnuq&!5{fDkZs`X1`n9gE z^k6k;WFS2U*z7?Psv&&FhH)fg`s?NSg9s@?SAS}XB5WA_>URs|#tS@)>f^zYMVGn`5dRj zP+X>B$vKDfW;<0!x-~R;u1x6-1LDwNeTa^noFk_cUOxSk@~-VWq|fNELDHj9UlN7+ z1A}kGtE9(|GN$iA{WNFD6sp$a4uh%&!ULrxkY?ZzBo#eZq2Q5HKV%@j9E?lZ9wGiykwI4>vfh=HFyPoEPghqnpn5aHV47^XB~D>FU7gJ17K^miF-7d&bKK#guv}oSE6G z5@39^`zR$5l??wM$w4G>gTLAX^vs=r5`B7-YE5z;_rqfHwLRc1;7Uw)pyt!9d zqMEhQ;^R#v=oQQ!wRoGiq?vDyTC>68@dmquMX+C}5dtgNUFPl22ncQ6*xnA!1E~ML z+%XNv7bm~F4X8W^dYFG)Pp)>Zf4nb)&Tq>d7@ENEEQ2P>BX7*xXYWfkGh;Pno3|N5 zIXVPy_pNJdQ$Rn~rX$075GPf3L?uEO!|=n|9h^FnBYw;6ih@P6^<59Dy_TBQp7g(* zV>Cd8lF8q#WhI5H3$-$zFqm=TkU>b z_`TApwo6?ZkOTo21P{gtX2USLERlyRM5)$ecqfHj(uAO;6}ScnYc;gdv$ zn74#p9I;Q*V$D)CUB_;^Q!-_UGDIPQ8vIeq{7iUZ^A3{rtzhj|%UK*J^hdUw7h*ZS z$WS$-_(%Oto1`(1ipRj8*F5A=wF3UD;N4?Wwdqk|D10Bt1^%E9N{?X^7 zmyEjuy$8tLa(YM#^ASYoIf^O?R72qM5-5D8nP>o}11V4+Kw4;+aUHfi+;S~g&=mZT zu8pWC_k`ID+Lg4p{4_ZkJs<#xGhyCQXI_CQpEFcN80YcE33DBmd#)Octu#8LV=u+t zWL*CmRlD_O{Z{7KymA`b!IQJG4+Zx*m6NUvU`>#yCEr%?*UrKd=}BfX>`it;wQkjT zJw@JVFe4O)NTI6sm!Owi$1es&wzd}6iDha$8I$Qs8KfXlIbDr|o^ZAz5e!4)#!TuS9B zaOgaNl!SwL1BE*8vI8FWou;1yjMM=2W_^OaL~Dji$7oqwgDkP&%5(`t#N=wnez^YV zUo<2mtCZ?&Qfd+EF6Cece6g_p1?!X>z009d`IF+fuXbRhb&7ft5rG;bUFt6wTRPn~ zp=q`rqom(!KP2aJkjg9PD2hW!Z<2+h`Hh0UE*SIRFww)b2$vI^pBASfxarW_-s<<5 zr9ffZ+d_FR5L}$q^}?nSSOp%h0#Ka)9Q$@;cM>C&aLJacS^n$@NwMeYQ1xJoY!qG&y@zOkm)`U_fSoA+Au_WIuM3VuMA&(-8 zItjXmyZc@Dp_JX9+@recWu_9{c?O*^;vdzm%ruw-IGQEBX4 z)@zDv zIBJ@Kdgk!jQgfNcvS8}KbDb}|JNn5u|JL4q)(Okni(Spi3^lEC-OXytd*D?@Ld>ss zx7sv*x#r%s*)jFW4shHc&(ff+$sE>_doNb%QHAC#;bTc@zi4`Zy{;YgLx4 zOrrD_gfR&sQ5|hkkv!-Wgokv%iM~$4&69aI4;AE%^)S6=;YCkr*d;(1du0Ir0rp}T zSV2VX<*MO90W^4)Yy_AayaMYze3}=UeIBn=(Uh74i!_blnn?MmxNvy4!-^U-Z3#Vc z>CH@fC59UGj$tK5iy3@;ibTM-^w(#1LqS3S&5_|ZO6|DrL16=n;Jz7kC`$$;GrTE0 zSdAQU4EmL$}@v-abAOlGZ`SRS<^G z}TSk9;7Yz+q2{44ZOTFA7Pes#rbp1Q3p5XWqpFlOG4b%^(vWNB2J&2VX z70M%ibSlnS9hO;;pvfi$Y<#3Wh8LP*==jj!)gECT&8tvD!TfC<$POXZ>EN{qb3s-(B-hhq0G?9lPFx$NySweN>rr?&A=7d-whSl)rEd2~ z3hT&FYiwKBHhfTUANCq7-nYk;3Pl*5`4G!_Ls6B{#z%+{8M^ohNq5K9fiOoMDBb_` zg%FTFJ7O>fax$wh4ALaH?<`D(nMxPe?dqE~J@j4;&9buJ4jEKcQ(E}83?E}uOK}=e z%iEi8cAV#vhuH~_KbjSWdlsPnDam>^Uq*QRb!_^DkqUCp+72N(8BK)l6@XI0yRQBN^L&w z7Rw`ap^C!95aMONgY-KViH@)qSlDwEspe*iq?!ah@tpsrka%=bNP3VH6oeLVme8Yk ze9GPDL|QnDby3ChObN9~Cwc{MHnRg1{SZ5(QXV3sFF)Y55oi0QRoejGDS4#dyx14V zBW~2!T|7rEe|a8L=!|)AUArv074UPIIbE{(OH+U!4+Qs?eW6VM}#?K zluc#Stqk_W!a9kvXjI7PeXa;2zrv10bMA?7Kv5hKVJUIc>!mCynOOTWeEOM|(B>UB zm0xUvC(G_kuAs{ivvkA0jt0mXHB*}2_W7tad_QzApZ#G6(&~oSx8Q?Q;dzP?5Sq|R zudAU~iy0(|rB6fU#j%ZxVig014IZ}^@WaibNj8;}kft;fuVsg03?R-zuB>HpOcm5h ziIext7G8$l??zkg?QPz0$2?_STgE!KPyKS`LyzTB5)t3l0^#$%%E>~9(nj~p_XF3d z=!Zh%Eq60-aYKOW7?@iq?RxW(bm!32=eLTVRDJtkRaRQulQhsM>t26!tga~yHwy}D zl!u`dZZ`BN-D#M%-9s_Oo){$^Tbfxwv*~F#Z-X&Rd zCU}8}l5f2W9BIDQwl#1)n>^3!lCGJ~O? z>cZ3bT?L;l3r|z3h$tnT#D!loKt{3i0MH36O52YjUdh?G2q#kH1yK=}V^HHhoi;tM zufP!|CQP#O8xv{GA+7YFMaK31{aNe?os*qrojO`hX!ywGZV*m$bhl+uEi5`~`oupL zHUNLkGe1x6Y_Kp_%B*hfFWev~mZE>!SM8~JgBu?^ip0ie7l;42+vR2*rCBT z_R|yA{#tF%wd7QokuhH$iSFDOcr73=ToL^vdL5A1|f~Ekdhlsa=ox_v!CN|@Zc&e@% zO}uneHZos6qKfhrdnYQzH6_3&@`(u)7a?)g-yqXXmL^Vq_k&H-58_HubLdNj`eA-8EXpx+5RyguHHl+MUvk2()9eYJTDxUOUylu^QELA+t;)?=R!i&Fu~ z&TvugPSG9+&vTkIp}-k^#WwiA5ax*jVg4cOsma=0IrlgS|0?c(f&2>g$^u8C^a?Km zEZpEKMa+o7QJ9X4MjbV|M0i5U@uH3kjekrp3eDEr{)!;Qw~LVlh8iLqb2P#ffQW{E zg9XZ!0YpPl>n)fAwkM|?!ka~Slszh4M4u2nAKcMedi1i^XWV2Z&0~D=^g(?v#)KCl zk(CLMv&2B}0Bzu_f&%CcP${A0WrMq3T}ls1x`gmtu`R-sL&E(%wCgpL=MAoSNe|X; zk(R+DuU!;vm4*B?6CSB3rB;<7^}}?PYGI3Do;KvFj%Lu zmUA7cu6uCUjraq?b_$&gSsCb@YZ)*xHW9EC-$&HM63s8q=E;u-?&;1c9-RmP4jX3R zb^MAIikKgXrvjKSuUth*rYQJbKn~C#go^zu%N)94)*^j*N%N^N5BW4HSJ#Ke!})u% zGxD2{Z>A(*YgCuV>N=Y9Rp(z0D1i|LV+8)*5=nq>;j+-$0R-#?5V#OQ4R@~Z0#W7D z*S*vDpVqZ{NiIPHY|v{_0og{iW#Co1D#~uSm2*mk9-Uq^tb~&&|Q;`9%Q5 z|6}Ksm4oln-bCOsMfxLr*=bG5=2YArLP(u1FLV!J_eoO=`Y|ht+sO~bOvG86l;*Fz ze^@JYPeFJ}3L8BZ`0&kTLC+CV>Epq!Pq-^m2sPCcy1 zh3pgqqdBTVsL0PF>eI^x(|v@80wiaFdtU_#2j5?m_KsqU3o#w<1Z!+op1hMF83Q&0 zQqpgMSiU4|njw9PTv@{hiRml{RBWW+w#4}mD^=!jj$FlB8o&-ygbh<|)u&(OT{eMS zb0faY9?8Y#2ra+prU(>1BX^Ib=+9^g!grPvHG(5-{s`>~r!E#Q66QOaS`Y&kP%mwV z2*Uy_2T*7|25lmBv_we&x4R*IiiI=4r!Rm~@SQ^$TN&=W`M=&V6xi4D^ugPe5?cmQ z|9zh?F-JvA5`Rn39x+59sdNrQRshyO?XUaWy_03=43k?wt^(pL(Uw?9<|qQ&S+O^2 zMOl=M{7>n#L{08Fx|M-%feATH0)RRTJ4GNn(2s91Cn}c$ptY85dtl+!n|m^oE2d_%Rp7y<8-p zfW?ix1n+mW3!nv8mh?mR$-^1(KRJP`hsM^ZuY?oUSe}cLt?wMGiY8)_XwJ^zhL}$! zUJLZJ%v>Yu*C}I`@De#87I+pg&54#9N8XzP#8R9?mcVF3I;1)is)+DPM4n26p z^IdS4X`TI!cN4?lN7WwOY|6O`9JKfO5es4a_$_2nCX}t1`jG_ij`$HeO0oTjkPQwj zEhT78DDY#Dxe$17Iz}s#IJt_yeP_z|ZPd2Fl}J2rd7yZMUPYQ>8SW19ai0 zg~I0PhMD%DK;YD3a01vKj5b^@x`eFR-Vh08Ug54RZM`XwAg@}YPlmg8sfsy(RW&@1Iai^K|Nkmf~mipK1hMetNhpc7c_>C9pZIzaJnsWuIavy zrUpwiSO)KhVHv3zn1_qSDS+2DIobuIEnHTG%I-O{G-v6=418SkG@&muh$=9Pp~sldm}4h}+|Jw_CO{I^E%?&BTwgwuL6aupnd#VtnBqHz34_dFYUd_kuoh5?!fC=EuQG^jkAB z)GK=wT@)BY&at4k9wS0IZ!}ICc^u3&Z1_0qU>lAbD?l?Q6Iy_An-Hge=sPgUz!VsG zA@3maqj2d&0Y=d=zZG3Lh)y^rsLD?UiBdMlrEL(KE5??u#@nmm`tuldPfG|YU0LnoRIrpcr0CEA@MH`GRa+-Q<9)Rw4Vc!y#G z{^%!@=u$^yJ7OUCa-6s@kqM*~Mu)(~M66;VFDPaeU>@OPlNkcl7TFH;L%iqH^Y8Hu zGD1e{lLh^i`YW6O309>yGM`fX`}OI!x5LIkK$m!gba@1m?j74y#4~8HPX@tNhO`D+ z#eptk0S|9FjUg)`mILWO6_k$#K}0k9Q3+%b1qe_nrep#Z+_AyGhhqgegG*1$$=xXi z>98PS0^+a&p8Ist#Y16}%xKv;&pqH2x;l3!b2-z}=ewJ1UN@0lZA&IFCEIY^CKPAROrjWCjrC+w~PPAN`&Y! zUU5|7`yol^0V90~pBztL?94<;!}Jc{$YLB|4i+bG;+r z*nsR9X*v4H(%x7Q9*nmK2S&mqED7?_C;EY|^y;)bJ_kTvemKOC9)Xh!w&^jEG3ili z!{!?!4u=vE!EtScE~hu{F1uXIdHxPo#J-5CLA80Yi*-3oq%^a zp8;J~OoKUd|LSDPc@svd0I?hHKnjVd+C^Cf-guh(kvcX+a?_MMk|+Q?Z}Ne>oT>(^ z0E}a4i#iNZj}ZYE3HTMqK$2I!578e$E^oUdY-4CmfV{`*0*Wf5#s7z{FAwB8-`D?I zMp}g$N(qUYrYMmnVj4}fXgjAGQz$J{2&q8|t!PovCZTCFa~s=8QL;p2B2$_WB}+^x zLiX+VJa5gp_ndowf6P7S&M9Bt&*#0oUeD`U3N60!TK8nrvjQ^9Mik2-1PvpZljN2W z(|{GY);=s-+!$QCm~X&!7?+W7+7-5sXmRw1;NS)lA$9|R12o@g*om*90sw6-4~GsK zL^?c+^eh5zNQA*SPgtdLXYlx;i)PJvgOsNZcEKI{gA#VWRV_H~EODxu0XtVzZdOpk z0L8smRwis&Piba9s^dM+YYhRu$ zUMW5w%^q(l4kf`5W410iQOK7DSZZ#8(w-K*_!vYmE>;46x3yJ#mA%?%KtTdLwOoug z$KslBylSlbnnzJD zv!e@=NC5_=Zp?h#F=OWQN8H42$7;bxKedTpY_W8~*Z6oG4xB}>&?9q(zL&Vd@LRHc z#5|XQKv;B=>F^LNhE)Pw`LUvLiw1y|A%O`ud?r}(>RAdeUntHW@&aiqjlN8_t+MECN!x&+AMN;R3-y8A($eu5{c&=|&hC zX&b=4yN2%f50M1Z?f4JeVeHy?vEx;M=<@Mu z)z6WC89@`BGgvF#nkgo17{oGVjY(fJR;oCMIE};313y7GO9XLnUN49O=+h94l_JH0ArFyZJ=!>xa7dsOd;T1>OaA+SS9=#sLbv@t z4rTmePNb8t_(5?LD32%V$+-+GXAFc&aZ2F`!O@7V5}>UueC<=Sj$Ao~tu&(22lYG{ z0362(b{W>7r4|lvlySrYIO_-dW+X7rv~*UPtVMYti31;J*qG~ z<6IrV5Tcr77$)v#G{Nw6^gYW0K-6bYwqzR7q8>j{M2_J4L|a@Va(U^!$9b86eqW(v zRi2&mwV&7CypfukSu!Qes9U}ON#Q7BSThNKPDaUL<7#7X!qW2{r zoJ-m4da-v4jCf&#Uu-$|-|aLy)BLGz+xh2UA++RNAD7GjpnIYPZ@bg*?z>h!+sf2+ zDjuefM>_5Iw8aEES6NpOIK?QIrHjpE*6yN*0%3$v2NJwrR?;*;!-P&irR;;8Wa!?8 zULM>PQo*3li_iBx!1jmTpnxEH8;>*T6XE56PkNpR3%IEf0~*)Jrt2Rib;;zt>!VzQ zC;<(sgPYjQm`nk+n|=JoOpr-L(k)9;ZukDKXrbe=JPRdq!buDillsuJUZV&wQ*2XO z*1-NAl`K0eE~FGXKQ`0jpJg%7Gdz*Rm8?kcjmQ)teNn)PXpeo^NdYS8Xb}yX-kd&6v)0y`ym(F@}fEuH~^Zy zJY2Chj)aO)xS`gi@9ZwWdjyN6ZYiK{L%i$k30IM>OcN27wQ zzDqdbdK0xzZdv(og7?{0cERzhB`9;)Wql$GkQBvyKe8UMuf*i|zhXQ83*P|^HcDUA zdYhi$+`$RQ=Hl;y2^B={A_CzY!E*G`%+auz7ZZFIq+5((W)S=&;R=Z%9Z+uc>eS2} zmziP@iC5QvE>OY&JP1!xs8dky6TV`*`-zB1E-C#^36Cu*lud_P1eBo8bthzxZ!g9`JDpe145zb6!z>55Qe-tr0 zXIEdqH{Yz9ElKxF_q(|G*qw&kN1*f`$Zvkjr}S3Er*h4b3LI=9)YLRo3dW5vJ%scp zkqbtYQc$t5?(pYQcsHYyLEHq6%}v3f#WiW?ol$(TyE(;lDvnr0dwUbXiPhkOEoz1Z zkdjpJa59DnPh&R2#)R+{$ihdOM}s4Sp=qzCZ&R1=1Leq%7V4dq$6;1Jt$Z~sJjz^1 zw8G_8b-TmrWqbb10|eOZ<5%PGX3dI26t&tRXArS5-RHz{d-9c2(r7B#WfIm9BN=Qs zG%grQpnYPMyjVGOU^?nzY*Ks_(Jb8I=p`WRhV?47ND!0JnRrm_he2X&$4pLcIEcQ8f`}Cm5Yy%AB4s=6|B$2kZMPUMO?l-P*J;T24WtbRzu5L^1B2{+Oy(7R$88h`UBYbAwdMQRE8AO08@ix(X@!^o^z$yuA%4VvSuEeNdRe8$>9e`7y zEf?B+gZ+bIpavqzGS9+e=wuOy5KfeeMAY!F4mt}?i=ZelZ3~F95D#U!haAwho!BUC zb+FV4>S@Cv9QnkyXXd61q6B}7%;Mz>?4$dydf#L!7lqR1gj>Is@X zQ87h>%_u!WDrn3?#ZcW>+9J}H?*sRPK%a2G13chpg>#HM z2TuL??KvFB+ZDhgf6n%vNnxafgU(-Vp~kwwO_E3Qdm4WdG~ zRmDpr-;{+hYK4AY=9|dgNze!}6APb%3OuzT5L%RxgeO?fHts8i+7Qfo#Q!W&lHBw{ zJl2CnxHH9eL^&psr|=>PwGq;S@pTR-&(p1ygY|0;PFq!6L)3^{v;!Zuh2%t}-V^#| zg-?3(VSBF|S463qw-LNkH;=RV44RDdZ=%09SLml%6aOwJNtS}(fwxda{w~j~MVp|I z6_g;(ewRBU(!fg|2bipe0ZDkG0_O0HPXj2jZvi(xpa2f@Iq3_{g+Uc!T~YJ)N0#B6 z5h%p3&M75}wok;c*xt9XO@AjTViW?hs}$NE-hJfy&^GKd3gfVR zMY6&pyQFku>;Et`+Pt(sYH+$vcyZv~8(T_#h(2jOJ>-U!npAPIPWY9O9fD6#7nF?vYV;Oer1K)taU2^u8F363bIHPFAWQ^S0al{R{t zQWo7_+fa&=`oy2Y4h)!NBej7cQPz)ljx05?+xL7YCIQY%d`Azc8{gX6mehN@4w_NS zoxHwyQ;qd6#ElN_`&GOVn)M9xB;i5X6x%%=y^6v0*TKS0J!p1=a9l20<2`gs;cu{W zwSJNfbf!J9%A_0AfST{O)im@~j0;dQsk*uO#FXTopi;1FI#vuD*6Q{2^0>t!E|vf7 zz8GA!^?hj5XgpfFEz_C|4bZuI{qEk~gQLPuY2FFzCsq%&!*MBLCttX+C@v$#TyDsf zkTZeKaQ$qC-~vz7gwr?3X)ASo-$r?&#@+%r9=(ySZx3Aj-=qH>I`gIZ`>(WE4^P4pkvE0UUH?gYsHsP}$*MdfE4w~B!m9{|G$kdppJ1**Jp`=;@{ zzznTJ%|<8thLgq*CD?V5B=%++BU;<-+LmfPpoO8=PO6RDF<$M+(23?vDY7|1<8Yuw z_|z44Ae?;9re-7x1RG5W*%560L~e-rS6fOREV9u-`9${IGz4wvk}ZD=uZz3{nVOrM z%d(j!kI7Ze!_=~eN)ygKw|JOs9CX7;)#013taPWStw5f`lfW;Z=onb(+8^*-=air> z7twx2x2Y6|F{@0Afg**BSC-ngl)X8r#SXLJz8LoX_nZibZyDQH1H=a&Zcpl`_8}OL zsG_zTPJ+zK-{4CAR=wlr`Hdu>0XFFyyKEnu7BY-ge0xb}d zJ{_HDyd55;&A-Ps{ciFeKr*A@i^CLBIva~*Wni(aG(`=i*Q+Ic?+R3-gdi9RA-uh| zyLvt#BZU^Hg)OEtU+x&M*RfnLmp`D1#6Gqk`H4jYQ3XJ7D#}EV)SmC0xbMudGgc3? zjNV*%`uj29LkmILfs0b-=I*z#J)_aVTP7v&=bs!%Fv!w!t(V z=c7`_ZcW;HeS`E01ZyM5w_}4U*WKm+8&r6^AR4 zlhDZClF02q(DfH9v*r>fqHz!w8{7XGiy`Gj`ZpV}9R$(y>s}*FB+fns3wA1yka5z<tKxv8Sc15%eK{oIN#cxWz6zc#uM2Me;v8o8jBfXZUb@;(D_G&MDc-uRg4H>hrqcce}t$@1@i~EE`D?7ierc62q6!-Zm(H~;(zouCN8dM?D`Da0)fiAj>6er8`$D%l!@N$)qf8@B zMhFf`mZoTfm?+A%!Wl%5I2=x?uS=auYNB`()Ei700Oqkz zcur5gnQL80#G$>JnKCzxZh5v8dH?febxU~wTUqo~$!9%BrEKd4!E*HRzEa;;qn3Dm zB|U7I0_;jE4a@&B^t@c;wI?RI)uukVYa8F*tu?;NQOCnK;Da`$Ni=}wDMuy0?hN}m z3}XhbuNO?hcVJw^Ok4M{-zVUg`pi0^M6cL0?}$NaZn`+FPU+4$*j1n6uauEdp_w?s zw#^XAfL;4%Rx2ua8FvjBkn++oby(g02_7czM|HowZ;HzSHg;7xL5<6g?{RxH#>{Hc z!8MC7%=t0+tat9&b$$0H2rhD3XJOi_fxd0V*M30-HS(|$U#ZvvhNViYgGR#dq2`X`gWPOIlX1Bp`!o?dm11>?sxEmiLd$Z@TR za5t%U>D_yF_s1jdFAzuA^TyHhkD)W&YKkY_tPOK&9h@$?Fw5usfVI8`Md7y(IjDWV z#=B`qEQIz8V|0Bm;|TY~^;k7(N%BZ16d59K#w)R?dZ=A(OLFUZ$K+a=$kb#Zfv{J{ zdRwG@Lb>GPyqw(Vshr+|GXTKRBvCakzTsJtT zlr2pCGPZ43?8C)Hwh9j0u|ps1torz9;LS~Hu+40quGhN!f_HdAcSb|6x=Q`*f~$8I z-f+9}acn_!!&mCN3-=qp8262mFGMN#^@EI&yb7)6YH0p*^9;qWzoPIX1z z<;`D>_9WEZ(3?#!&Wv5ERKwj@Z&bByXt9et>l1&LlLT+0ACnsMcK+IS+&FR1zGGiS zIR!+k&4l}sr*h54Y@SrO8XHT2bY;6`om=Jk%h@Uf@Er4u&q-0$xYlt#EGBtxc*p0u zw}KK3OYJY)*@u<<;^?`*XqV~pt(9OeE$P@_-eC&P&!}gPE9(qzmAM+YbY#0cdV7BE zl!LVzQzu__MLgXy^{&;utxI|j(U$6WD#rz8YEOx2upMuz81c@#Bm8|0&2k_gxVfG z0-H4aPzvS5xNc4)EpWDiv~ye=-+9q~1#}WCE zxho$w?JC|+9g<(aR?ls8$oXZDKkN|UYVCOtJ_-??D#vTt;(EgI(!zqKXauGi!Nn;c zC0Wp8dforM_TbPD+^glUc9i` z_K==%I|juiwU?YI;^gO84iLz)1u)&ktdq$kbhtc`#oMltDiD++2pl`FBfv;j6}A2J zZ$aT8xP9Z9m?@AmV6Wzfm&rIez)3JRz)T>EbxibOOGpkI%zdo{!? zt$AEh+ba`?F!NW7hHaCN*^@utwc=w|Y(-x3#Hs}Omx{feYj#^X|FM_d;)LClURl2H+|dI{^g zKmol2a1#V|wxv@>I%(TpypcR(T=DL}LX-CDg@k~1sZLb9Q1Y^dO<;t>LZ6N;z7Ufn z_}+HwaWp#*Sub(Qd9Izs(}Ez3Sn|}gthXAPOUFkMV~Tf`s3kSDAGX`|eB5$)TVqW1 zxN&4rO7{3aN|Z4+4vBW#?sf>%)hfqZ9s3h}=O-^cVexQ-ms4(HY=?a1yMe83P#Ljt-dw#xg@{+#sowL2u{Va8_mmGG8bu~cp31y4=K~8NsoEP@A zM0H%h9M!0BJPoaR^6KrK{~WjAjzC-y1q@bKYLlEeGo_lb9E|FUxzt zi8PtzTKP})JkJI>K&m4GL!y>&+`2XK!19heEe1)=zYuS~XPKGR6mGcl@uCP zFg334hS{aMFO1H^*w9B@hV;;UfJUB{sn@^8QZSL)}3y*gVl}yinS)tw%A7wnP{==QNZo}pm zx>}(28JpcnZ(Ox&c;T3Sak9Q$USQbGS?*O^I`1C^MWZP-;l`qvlGKP5MYfqG%8ouQ zqkQMm|F3V+_QJY3XGr(BW&8Ngpz!bzr!~G~!<16vGWCPHA0NzVtWHCS?3l9ENv($J zU(fOgBBA29%v{(?tnr?*+O?)?BIkqH0FG$6e!| zw+gkDaZB8lmU;Trl&W3uI_U4Zt4sS#2@>gA>L=WeseK-zdv2}!p|rIxey@t$t&7-@ zoV_IrZ8L`!g$}OO`^{#`FB^}91(}|6K=`OZ&7HEg8)n*_nC*i(h$AQ(Z`Ol1qM0kvwUHL9!Ld>i^hXyD~gW`pJx0Lp} zyXtx^4^%Mm_NDy_4q6h?fzJ3t*g_N(ifkNiJU1Po8e%tz#sYN$WF0`n`4DlmL|Hf>ITmkADF;pS2JY5-6Js3X;`ZrH_ zf|7|VNqoQ^Z>wNM?ir3{O4MF<=a*k##fm!#HGHw~uHp%fz;E24!J)YO{uR7LzCV>& z=vDad>XX}SeVSMKh98Tb(usTFi-IMA_uduDWja_-d!*7`6YiT^*?r;anG2p*2e@VC zpT4`$`)KszKb^Ogy7oN5oNIPWr@Cu|3PmFrO_0B~*3Micr7(pR6J~V}M z7ML@CGg7UI4)z<-3FH));F$RwVMd?Ei=`sfum*J7i)`FXzADZgl^hZ{J?neN4PwmmUm05xOQ^Om_VhE zJx2q~*Gx0Li5#u(rkuU;3}!xHFQR|DIV;(1h{P(z{vJ4C4A=&2eqqabdNgh_Kwr=P zpR7&}9*NZoXHGCoe?!NsBk)dN2o?sa{>eGr$r{jCx1Ow|`aoR$9Juah|W zFd#-K{ATo`XpZm)*O@N8^ShIguC_K(Ca`1?g?EK4rw*rV{UG}lj&KBkjV$&Myx|Is zm$Pz;D_*@6{a*A6iKVoLwnTkL_pX< zjBk<0W%gr!#beD({iT=iD+@z9U^|(+;9ZB6bJ$!7g(|*P=B~x3>>c7WcE?`?quwS= zb=)i*Bnc5VUs(FNo&WPlG@8RYk4NT;1qFBRc89sQE`|D*FFBlYE=XM@WV~Y;@jN2wiX$VQbMz|1*ctORA?UWXlsl3obd1< z-qj7{Lx#4Ky%>oyFb7Av1d6s9i@_v{h6bYUEA;Awr5Pk0(! zzH4s3FYI<%59y0kfD(R^j6M9h5njX}>h3f^le|q31ScmV6cDYDD_~zX|T^TgT+HjRmVq!43~T46!9vrW?;-N z+hEUlwQ(7JhH?iyFGl<9KbuJTf7+?Eo8c;yc7H;}*1x*Mm_4s@*j=S&#`n>rwa1>(G`@Ob=>mJ8)KdGq$ zZTa{Z%3q*O$?ASLWS(n(iqa=uL{H?Wc%)pT?Ct`sxs@FQ1HXHJt|(%XnqGisZ{Fag zgW8l=ovYQ3$T!iIs+l!U_TeP2-<2csg#Q5bPn@TI5!?Q|Nq2_A<^NiK=Uz(K<9V+; z6zz{X_ofZ{OmES1zmBxI@>V7@Wqxc*J3S(xwg4aHqIEz!BJze*bKk+?!Ja+`{#tPL z$e>w!lYjkr?q_)$<+LM0BrxF5w!D<=R}?x(kZ znnwR0M!174(mqB?A+tK8riC=2mz|JQIlTEG1%S!)4!}Igfk5?TY=a<+xJ(;)8{iglOud-#W zhMV{Y0LqC&i0t;}fcz441e@bNx)(%|i_a6q) z4I<5|CsKQ$X74TI(W|;&M*HTbJc`Tj`$zVHee1?}-m5S$PP@CQ`F_mn+kkz)pOurm zxqH@4slP65M>xi;1h0Ly^O1Pf(tS`ips!BWGNW%^@bJpM!zUzeBjI1!);kNCI;W2H z8EYlDZQB#LZj-fH^)43S?)jP9s+X((cFO}PQon8)rq}jW>>}S?A*C*N!hMr^axL{E z3%v3sr)ujpFW38~B=qs~!zwx>y5~EFb~nEF%`BW+_UN4QTobeGQxnT}?wI*v{E|ys zDsu}>k1px_1~GV^8V++a_H?zKyyX<<_I7l{llq@b-zfaB0#ph$d1K!jga4DDC{tpz zs^`{hk9w`Ot6K#cRjmfB@q6%0BYXtwYms4${|=<;qiW@`4T1!bxv8uK*4a@pJ?WDc4nx;_ z2Ux1<_uY}#M_h+?|Bv_Gt9^J--!EBx)|KB4GOw5w*k^QeT8-TjKd;9Dlk@sk{`BfU z5)<}|#Du;>$#c64l$A7k+Z6jhzo%9o|If67h23qz2Zr|<{qNYi9tCrrKs{-i<|biM zP|T<{`$l?I!(kh`Dr&91)<1$a&4eQMjBIsTK7wz_s*_<3hUCRHg7hW_Wd(!urBf^Cgjsy zX;1lOrS8oJ-^w@oo^5Y)gG0Cm>2!k{FCp_|d{*=L-m{hoHnEDB!hDgDb}$LiToGYX zZU;SW>t3^%-h3+NL;2S!u9aTN6RsmG;k$C>#1Fp5JTtOJB7W*We?`Pg$oGf9+!)t$ z#IAB&{qE`8bi&tYZk>W!17*P4_Y)JBFO-rx*Tl2zUp0%%97zv6RkBJ0=-k`QEm> z1xE$@W&WM~CXQ)=I-~VORnjSbm)!x4GJB6qJ<^O{rKY5lA3u8!%Au_PE2WWupN+< zfB+NUokdk0`!{zRuKkCTgSkX{GyTD`k#R4g(6prW`wU?Y{<332`MD6a99&XI71#Uj zd7#kqUe6$Lpp{?l4D?;t^HOnl6Z0_&U3HErW5C1AYX(W3u(qBW)c7Nekf3?^x8+Ik zaj!zZ1AqDH8oh`nkM3u&eL7tpwim-Zo<84uEP4I20-%y=UV#|aiUQSX^8l;v&HBUX zDPP$T>8VBsSm5P|OyV>^ImddbR(rwvi(eG4ecybl+2e%uMSD0r)Xz39*uJJYu5hNC zoA23}*1TE+C-+jg-Of+zo{hf!1Np3F-46yVTae+`eoL`Z!KKS_nE!YdK6{TUR<^$I zmNW(wDy%n*Ihob>nOycd`(ztUdq>AG`Nmkyz?>I7#=V^iy{E`~JLA%_(xko9KRA>! zyZn@}#-y%@-^ZSYE{VfwSKoz3*oBo`_j#+j!bG zaDH0#HyuU#*Nz0;X>F*w`QpyWESXW`vr00Lj5)T@_U-xk+t<0ARe}Yk1Eh6pqTAm` z_85{7EUa1lQ}0a%Y5@B`9*c&9(mI}+y>?x*J=g)BsTo^GC$&_Di?}$=6J)lzM0Af3 zPA!W-+-^Ca^2jmGD4}GCRks}m^k@cI)w24P+dAS$)F|mnuYrC_2YHj6j2f5sI7u6^Wf6;UjL9NJ3S|L*x2%yhvj&P!jWDwUZ5rIY zNcWug9?VR)E$tm5LbRWUj-P7lt9R$f5tGjI1DZdqxCLQOVIUwM!>AZ! zJ`A^8osi;wHafmmGslG#t30`YE~KjR3tCqWYVIDWZ@)-otglyXmFK#fo5M`HpH7j7 zMNFVd#Dj{lD!I**;o(){{l|BoU%$6M=I+Uuy0~Nb-7fUMhQ8@iHS@-&o9H2DC<4q4 zVL+E3*(*b;PJjDUov0!8$LZhq3{XjOXqfZwh!O9)K8^low)8`Ftq)mq z!IJ0pReoXWyA530bzJgr6zzOzP#v%xf|}4=<74&{6M1#F}9a=n5 z8o&O;K`W~cpE*{$!Q66vgZ=TYh>tsbnmv4uL_5I2;Njw>S+|ESvL2mVo&8t#j9*=K zmJQNeH>75km3`rFfLeZnFidOsAHOx;E8b)}bj;9&X9~|>dgUYMwEvhkF#$gmL`H^B zu3bMpQA)y~JXI+2oVQvG-1>`!Y_@&OaQSs%Tlh^bJsTD{Id9`lD>Z$^ufs-X??|lr z!7FQAA4Zp_qUIbt>s@lT>Z)z))k!7O5(iEi4&?iSj>yIyJYkTf{r*?ghEZNuSMAvd zT1oqxkYBGE-jo_H{q@Zaj8Hj+DdkVcQuISN84epWbn>%VO7apJ3b8a1wQ8?xWFfD4 zLJ~4%RMbGJ#FdvuNaB+1HtjQ4vM&5hGjRE!xt7v<3vW1CN@rR;*)Tfy#Q3@`=|=L? zL$+LauxP=gr!h*);*`;Y*&EzY3{zR8zer}uovxKWuZw(7uiA5Z)$23UwoHWU43v% z=&w;yewFtMUGAKYD(jq~-ul4by=&|(_kt2{uQKIv1;032CyWZau_#N%QvLyOA}!~w z2zMHOPc?5NxuDrwq>+4ibp!yFhuTtjZd;W#koF`$#9jv7kGX|x$>=KMeHN=A;W?V2 z5t#W}aY=H^7Q_w~dg1Hy7{;xAe#-E`BTW&@9bq;#OY5OF|jxIqWXQ)r{XX%R6WWbgye1yLGe zir>~uVI&R(3CU}~pv9+Cs{wpwYc#xZP#^t&9_d6FePMzG3xrRhE#SWj`6B3P}+n8 zbJA7UckRYgA}dYvZY5 z8oxFA*Hx4GZ=jgNlNRDtdOHabJ?5xTJ3)@683-s11SC8i!ufdH(DOs^3T+M40pQXD zQ*HW7ezUZOthwVcc4R2(AUVc*LHY@Xu+#$z?C$I+Qr)g4Zt!)99PIIKduwU}#Oe`8 zRHklk%CgcwAPuo6eS9)scwnpqe$*?zb|CC+rol<&{Z3N&RQ&7{SpaHCL%>}Fx$WYc zOfq_~2=xPSQqJdWqUo*+xx$56J4b8k>z&fJL}0xjyjP;ne~Y;^Zx4<~gh3!p%+L!e z0!3V0n1yf4cTY#D^T6LIG{AE~9}G_{2%WHeC}zP|JsFB(odhkQeh_8DBLF7CpB@21 z9!}7q1k7Syi3~Brzd|W=HSyc%(}2fWEn5II%g4+Lf=jTi5o2TQ8&L|=I2v$IEpy$A zugkwwTx_oRf0aZ2{pqjc^l!#vv_%>t+)Y@10LzDjmhYA%@TS?MpTav1ttp-nkw}QK zVt-#qqc?0Hda(F(=lEX03de-Y)}L&Ur#8%d5On9w(6uDPD@9lg;ib0&dD)}hbi!YZ z=V@=^N8X-UPoddsA+y79J~S6_q_DiohpCld)#IJgrKOn$_F{79@O|(eBkY2TTM12 z_78mmGz|jf(S3tm**RJ8zd3B`x6%L=XV)ZK`d?wJC+8;wV$39{It~hq@E_Zn2p(u6 zNR=d`Hqr^pUZy75YR!)omGgSv9q&7p_I>Z9*tTzD3rL(TVc<+Evjf1m(C3 zfT*t33|$bccf36=4vK%txYvd~MW&c zWk!!fl*xjuDS27F_~il4>x4frRY*c1APxyl97$f{sX?W@pMEqLdeUKtY+ojO2&+$u zC}>V7d^@?>L6W7z8uT9nO3QX2>Y&~+#i-f;g8j{DwaEh?KU4PZ>gw*+>o!?3yF(%6 zaoF8ytS50EQ!+@cAi1YC$6#WNGGo^S99=j|gr1q)NzQD=!odibIu&8<$j6JTOEH6s zj^CrzC<=0N$Bq>f zadL!6n2}-DQ8|P+>u_RG`pS3}iu$qsaE*Ia!)KdIWh zJdu(ND-;6ZFmbj$A~G2;F#z3_dNO(hHq*fq;F%0_+;|iw0I68z?G#a}c?h2YZ(B#+sZvzfwUokWYu|)`Y$ohAM36Yq-E&b-&r9)~4 zS<%S>SK24r9l@uiOox{Tra0my^W>r_a4D?M_!xxR93n7Y=Cqb1Vfw~1&%{;UhJcK4 z>#|1?ey{Wjhf;)ua^e5do~28xY@EWJ+jFZev=G8Zn8KlK=zxXoV&NEzQN0+9VF|M3 z7Fs*EF~1~obSC8r*Et+6Ln06e36(GvY>C;@er+fm3BDUN$|^CtaoSDeV*STEEYBtA-0Zt=ak3aXrK3 zO_+s2f(jccapeh9T(Z&_r%q1`)Oy148Wz7}{5Xw72?wl$w*iYzGE$B@|9uNjA0k7K zA^P~hLX{;H)0jj?#Qc<+g-w5{cn?8jj6`n1Ku~IV$Gx6gbsyIkrc@DNM$iF~FL*n= zZ<;95o^RIqn7;)Kjd7)VQ~XUT%k&TiM+knk@VTeN7qkuqVPL^!Ofi04YJFXk61t8J zIs)OsekAx!EgF3i=*)!!8T&~{`RERXP03Ry4xAn6Dfrw_%WOntV>21WV|Php&nO%c zzDO};K!XF$7VcP~DcGGC77#Ki+c1jb(=CiF!X z`Ui@Zl<9gvDqrp*G6tIw;Sna{8%P18KoYxx5(O&4sTju86qF)f{B$bZh~EU&+Xo8h zY=rz4HtoKdk7_=z$}~EeH5eWt!snh>0NxF+WlK6AipqxtB2AG|Yi2>yfhtqn)8b?T zXNkff>_h4v#ZVKSQI&BMWz-ZO(}n*PPdZ8x>En7bi-y2Z&GjCJrd%+La-gZ8Hy>6? z_aLDH#h;~zq9h44rgoMwC}+H`-`X{qEK~GFLTAhq@9(Sb^(vo@PR-Qb(2RH!?gE;s zAfYDuG^{KpUR~;V6^=5Je+?dX{yp&PiTw68fYMI0#hTMYXa-Z>-ctU`t?z@e_Q_C` zio7NTHwY&UKSaIr4}TggbGt(c=bJ~*Oj5Q4swphZP!z9XeYGw7J&NRPdI&VCE4tAlyea=XfNTY6+_H#bwVc z5m0#1#54Ec-L>L|rJp)EU~rupT#AzioCR?^v;5N*kuPGK!R5gB5#83}k^wj(n z`0ENAd@485MhWh(T3WAKcSmI3tQ@=E&$afcWF2}o`n=A&EF+kBIUd_`9)oMgV`#N` z2=kFQ3AZ`eUvE6GBUA^%bD!c%PoB-VaA;NuL74A-E-!K2QXf`J#WpndK`9JQ_+!Ky%jZ2QtNW?+0qgIN`!RyW0 zU0ZOnm#PyB!hVOz-L0s~z2YKZQ#|_fkCBj#qg)^$P?xEy* zQ9FwO=6$cX4@{q&a&Pj7pW!UYb!IbJ2+H_Ec@Z;}LzJ7anBrm}Hyh{#!?l=xDP-{Y z%~C2Sr)t3*-6a=|AP?rm7etCfz;uy&yKi`;my8>mO( zRUy3OsVbu~fW~JcvW0;=+<(sHtORneFQ%@Ymz=0dD6S^=6xWmZ{b(}+6{JFiND=^@Zm38y>5pH6t9vcEp16Q$w8r0@ zdiv{kpI>j}1qhZsi+*aC_KS7FKTJ6>j2G1Jv4BO+%uMt;DOgWUMqh<3H8(`u{vy%=&V<0ef=m!= z^><7a2t=rt1PlP~6Z{eNa)64&=K}-c`YQ0uGH6HS=ZbAfUWW=(G^7tC<)u8p$;U{B z0(!eq^@$Shs$J(*Q9!a-;yK%hQ>v&L$q~d4w+e^(!Y1IE!UO_6n_8#Zd+b z4#^*&$5w6l1|H`9LBVgX}q`OK06aqXHzYBH7pBgja9tZQ7e!XS>Iv*UK7sA`>VN*{JP-tPv*Q0Qm-grEv zC`J~AhLm7>v!10-k~m!X;o#Ylk4-mk@q%z#OKcIGM(u6$5i$;|Ir$|^|!`_PRdGA){*#q8(%}2S;On}Z;xm}2oIU*r7URDcq+<>VoPK? zDxh(u{m^&;YXT;*Y}css8zO_FALttG+qn-ZLuN70okFmer3uVHxTdjQGEfT*r2{mM zQ5G8f?o_&IX~m&pU}hrz3JNXyUhtd|WeUSk(M3?uh3*^1?dS{y7+3<=F*E_2%YIbb zY#mucH0{hj?T9dmm3%t(oMojC8;SmwM_JKA7M_@^zr;5I*r7k9-$f`F7af^=^x2d` z&Sy{%T`WqAR~Z}#ahAn zg9}c$cfziR(4C?b=MjJ|9AGvvAHWv&8=WBW8L%%wc<1dM0~r;JW0lP&c%F3XfMX%j z@H7JXYB|UW8c$C#RxrF2c&`3!8Z!&W83rM6LIu|j%v2!YxNM%<VPfpC;J+1a# zg%`#KiwMFpGC@lmXl#?X)#>NLxdR$g z7D@;gmrchp&ggG^W86fpz%E_!+CxHtZ4e&?mcO=0kqJZGci0#7d_wrCg{*|+S@P{! zr>!l*5ENHYm9Ut{w{0g_!poU#S8?2OW%Jo?o)ftOX7PQal`wCVz~~K*P`Lkj3UaoR z+CW=woWlSB82+j%a+|T9;PWHEG2((ChL_?VCS??lk;Nc7c5GFr1VHwbN$j2zC5#X8 zb0p}BXBij!hdX|4G5}7*FtOuV#X1Ey0097ct+P_c+lV$~heF_jU@`sjP${Aq1jNf8 zf84MCTfEC)KnZRus&J5qCtj5Z*o^VopDr3B;y^S_J$U~CdTt?Cw)07X)6TH!*#M>5D`Kx|6+xuEU9#?k8lJ$wE;I0*f9I4KX#5~`!7&9 zXkr(NHB5w!B|l9Ft4N$S3WF(HY4?#KazkJ_0n1k?5U_3`QUE6%^Z@$B(y0wLcc^0c zu0h+kfM)jDXr329b@)JVYGOLA1@hZmrNosd9-JAHDGy??e}#RY9~%@bU=9Za$j3K^ z_rxlmX9DLDG^YgGg?1I+R|u$rh2b?eDpm?0C>(&ONI*Krv8L^oCJcOeUhsNDbq#-L zqP8~C=7Ws+Kw&?^1 zrMDept5_&U8?sumaTIWpn2bQ5EgCvb<(=W{iz|o!su79e^g0aYh03!Avy$P8c$w@o zMK!|HfG0I8UYPw0MFEObOhE~zDrg5H1Ow|>V43`A>~f?D5e4Edgn=SpIDS7~KBD}l z+=&GX+bRqZm>DPjJ@ydbefkG*b}`@99309)ARzf9uH;gJt+)2F1)9QG|C3V*Xf`-M z0$-&%I4+o!qM@lc=(h=lPk)F8sm@5a*n-Dqg;}VD2E4jRb-XF{b0U0dVXU>?tQ7Z* zhzw+$2ry0p6)ergWIgTKXnZHmAdnG(_k{HsjC~Cv4Vx>09*fBymZ)phv6;*t=Hb-8 z18^i^qpE0Y@xv@-Y~+07ASgE`9b}mebAI~T(kfVzZUj3zOlz7&C=wJj!ctop-!adN zgBH|PQt51>T`@OUC)ulQJ3>0ESr>{N8b(v-fXBp}gzp`P8Z=ayMv`FM_8jYBzeo@c z@)K;6!nE~OWoka#5=vWC<~gy?U?>0{1s#MFS3+hpG@XzjYddOf@WkuBqRp7XQyY3S z!W0rj5FB0HxG=xQAr5i^tZMKGzukmRfy;^Cf>mJXKtG)F7P6m;6q+YVXm>WWloId4 z8L$Cn%vgpIY=c3eHlO={QEW%_H{?cc-w{~DC+CqdAhJd2tHUe|&Moc}Dr9m`zk7cb z1_7+%5K7m0&871iO9rIqp|B(-3>1!H#iE@O_6DNWrFV@0uW?oeISZ8h?%iX~>JJcC zgS#4eEg43FE~j?)u^Qkaa%s?6b89g#8VA8da5;*n@#bYc!{Zm~^ES39+p)}~(IXM# zV*5=~1S~wKTGd;2CJL1uN^_Oq!82XiO8Adqslr|e4G0=R;W|SHYBmeZy}_Bo1055y zhmFHnX65;L4qIFPxL{YSKWGN6;4YwzN7apI&D-2%LLe?vmOG!|EKCGJzr%Q0pzn$q zqK(4p#&@LQzb9J)JR2E5XSQSjno!4Jc`RK3hBADnf=^h`YzYvSfNg~#BY}ZGzrQG88IL7)uIBiY6N9HgX`ZgJ0tfuVt#V ziz;(!-d7BZR=jqs3{9ZrmAkV|@Nx`W23MUdq`2VrYn!4tK>oZTPJB{AQSa=4fVs^U;6T!;r$+=O8cs zqZH&ibM$GDIN>G7c7G9@Cx|PkrG*ZNF+oUlAsr4>HU>oSc)}^K4bC!hw<3No=*}zn zK+6r5ZaGH5+#sF+${-OHMu5J3a1%opa@^~kgK(Q~gI%W+tQk2(MJ-S04b==@O)wvd zKzMn52}3}h`q-arLGgD9sOVvz3eFg?RW#)+s^}e0Sd(<0o#-9#VK=>ehFDBu^TqxE z5co~6WjNW^mXn&rOXY6ci-!adM6?bA8qwQ#jP${b?F_1X|A`rcDcX(?AIqUo`5px{ z*ic;l4K;8?3y*$BUVkKI;M@}`b#&60E#T+{BuFR9Zbmr^e#oSQ?ALGt@R#L!!RtH8 zs1tT{W-Tks?h|cu@3&Nw1fR(2!CSPMoVXIG2FfnCF<9gH(fI#pc7lHZ`wuP!93c99 z);_keIAZc-&6V^5P-$mX-YlA0Gk=Fiym97kz4NwE|IIa=mj; zB$#!H>_q(`H=c2?iaHW+hcpV5@>e0SrNg(Cu$cucPQD7KS=1aHT8Cs3UR5HZm-xhD zWz=D)c!krPfmySGz`HZ`Q2-DBcj-QHHYh_*5OC?}Hg)aKQ2avW`m1%X4sf zp2!*ud=ODUSgw5ZxcW$~09oL%&3G*yIULY+(FG*6cCE+sAej;+iTmI4 zNDwpn#Zi5qOhA`#AogsuHG=pw7fdpF_me`ENF|_s9=HI5**nLFJHSeCh+vwPoZa)C zaNr-;6+f;@%x=-*@HiDMC1Gx!0`5fqXrRJMiyGVZ{bublu(4y^QDo~5q+uO<2NoIQ z4RGBE%$}_~*I38iaqUvP<9}P7ClbrZX*2kBz$XqWM0Z^NxKzZ9?NiS~j+`cuK=H!S zX~m>f$dQwosFBsYG2vW6( z-dzZaHlU|dR&R#X+Euf>>%Y9-a(D-vcH!x1{p8xl>4|~AEM5fbSf>!96-KazNIazM zG`zs>8tu4XvnNH1)kS@oe?qo)<%IEbxFWI703p{8gLn=ls&vVhTBi9o5$mNmQipC!8}WJm@#&! zfjkq6$Qo>CFi1P#r8|HGYo-%oX+m4r%}@x|GxsK^NaHL4$_kt389r63e2-n;R=G~E zP4$V@M12Icp0amz4|)H7is+#4uLmWI1^hD*&ew<4o(Toj8p$|RAc1J`JHZ&1Ux;kO zv)y ze&8IK!x2GH;Snc#!wGgx6U3M=viD776Aiah@X5xA1|bq8*aEjP@Z{d(hm*9$$(C>i zqv<(znh7T4g5(tv*}Z|1c7Ym`dG`dfQ5J)*xDIJ^&tZ&2U|z!gSR{Xt0RqfeGcd23 zbF^Fk$VH@$DYqo24kDuuty6dIYWHBRu_p_~?iaFrPh&qx7#Ia$zf>*EL=5p<_zt_2v_Z2M4KR zxGSeJy_g+%HzfPx&<2&Kw-O(8YNEzpTXO=dbUll|8JFEsw06lVawm@O$#W>a>v_`V z3uZ3Y{dqpX9+wg>Q``6dv;$wKTy(5lR)znts^koo&F87O%cNMFtQ6)6?>Z zY3qGrEcfl+-7kJwrj_5US$``rF?Xm!IarPF?k88Z&Q5AW7Wn!qKVS{`LS12FB!>8RcjFu&<} zY|HbqTM5+%0W0O0RGwU8wyn|Mw>77%^GcLk!`L=lKg$%njN$G1n_-OE&Y`8vQyz_( z4FjH!OJM;SBG9C< zAu`?h$Yx|37vzRliR6sf#<#H#;SLz#y*2TK;oz*P;ia!{V!(JO(oMHz zWEKq&-EMfN^Gs~2&M`yKM2i2J+!@DG+Q*N3_{JeB$+g!c;jhqNs`BDjm~@}E57FW{PLsDG5Q+yx)&#pEwb}Cw92z%)hn0YPM4(GdT%bh zq}Mgy(_U_P<br~0QgzAsvyE`P(pckWqMtmQr7<)Dew*L-+2 zyp&YVnvbKH$c$Mc0_U)T*O|IvO0TSV;VZAhHj_1_E?pnXfy7S=UK3S37X9*>ti8F$ zR5ky-94_Q|=3Q3DTwv{dPW(OL?!l{{^JoBnB6St4#Sdh-a?U|UCrN`=92S$aK`yU| z^uUbHA5l%B_}j16tr86Mn3LYvS~X(4_25a4CLX^kGwe643>I%!i(Gp$0chEb^?*P`~5xDJh zMZ0Ib###WktZ+u0SJfmotu;lF6^)tUCXH4e?rB>_C-3;m>yNrrSuEP9&N)*}dOd!K zjxT!e*8O_hCdczhuDh!}>bobO4>mV33r=$An^SYO>d-3R5wBCLel_y$-PJHDug1Mb z@5j;eD#oq6<({)*Dm=o_7fdgkUe%r(mVCGTb?vymJFl`1)i2Xc?r1;aR$kI&eCO7Z zF?tui>d;{IH#K}g_L&1&nB^bZQhl>=jPAId!H#vA$#swJBpDB(!}R<4Wq#^DM|*nq zM7-@vaxINaNGtRA-{xLD^Le=6s6UIneo;%VZx<}@gO^QjzYnUj zeW!TA=fVDD569ewvR;)d`Mnu)cC}A4Dsu1be6qsp_rm%K3p6we$JGBitE1^w-0jrg z&U#E0UZxX~ ztjDi#!ZR1OIninbAGGU0U&1G5T~_b;fyFzpY0NW#JEK|ZkG+P~xWJUU>b?Kk`nm#1 z&NE+lj#q*7N^uR2J(fH?S#do+35m>lf9u5~9GBeouW>gswIc&lE7zLlQPw;po7U^K zR+{uwhCh9-2|Izn&1-Il4Q^2i({mfE@D;r0BYyvGSVQ~5ri`#LX|0>KHRv^}-9jS8 z=aOu<%DHI|1J9-%Z22+AVK84I&WuiH$?o8AlOZUU$6F!~a&( zttCERA$_vUWOG$sVp)P$x{GO5!Ow}(`a{Oo8XYh8%ykTWzB*4uBO>SNX#UD0ZTpt} zJs(A47Vv!_0h3cJCllj_dBP?Gt$k##5DXQh?H@~>QB;ZpgohAEuk3OdJxcO{puiCA zCLj~=3MWV7E7I4or=v<4+q#aSGnMjSM?BP#h7X zRb$CWvNCE_5jE+GCiuf9m)1?3B91qpoNsLr9F9znjy}DxN-ER(YGgh_g8%L?|8;ov zjeKx3`EA;ZWOtP}w-199;M|$&5PuFmY2RqM2%q{NTUYrXfEX_TNpjApCB|f2W9F}= zGynvF$|I5%(Z3=E7%nr>6JT?xxMS^vORi>^9;3PufqsYw1Pd(c>8#Qcc)C(0F*_y(F0+n_0#Y| zV<`}NPu=T*Di&hitGe7+0IPUPYxsof3ExTWpcWX62Z*H1-*Lc9gZ_xc9fK+HrKpjHk zRPM23?UDm=$&1V%_Q#5-uqREi_N> zQ!@v@o7zS;4^v^IPh)JGb29H!{2s_m$RD+UX&-n595osT9jyv*rp+5syh(;7AblMhbv=rbw*m}7QN&Gh3x|GERxbAY_m@qHau0IDeXwnCJc$1xedmcO4C$>R;4{>OejjHO_@8KPK);Xz1~;MJ!9Oz|G$sN znLlV!=UnG=eLnB~dOexnl=zl^6wT91>UFVRW*8EArqQ)2<=xwgy~Q77Y3PIuAD#XG zQ;Zo+6F2PcuH4!4?>jDCYSr!uErFsclSpKM)AGl7&nJP3DhWHvTOMYoRq%Z#tWgM; zWG&OqxMQU(rINL@2s#WaB~P$CpB)IQE|RZEn?IIX4n3Wn$vc*=j$Eh&IgCb0)g^0Z<}z0Yvw>R#4j>zY=psQo?RV;;(ue=jZoiJS>4 z2_|BA)E>{-1o_ylHVl-=+o=M(z>*M$CPy9DP*Wx0f(Wz|3#Uosv4Ui{>fvoJ5tTY| zt`OB)c$}jLeaF?%!O`MchAjlYQS$Z%C-E9X4bKy+nzm1Jh~X$oXerBZOIv-+4ej-~ zmdXidMJCB|FNjopC3|5;o}skVB*di2T0!`6_m#?cuSv0z;p1g(>ik~YMiou#eVte! zU(1qQiGQAiKl^`P!nI(@a`Riat0ibEK!ZZe-0*aykDX`?%Ur`!B8qn?G_rDoqEEpZ zKo@YSlSL;5-B~TEMW6Jnh?a6;THIh<+uTAZci_S`AP%=Wf1<3o|Dof@Cx*#Yuv)dJ zB|KnBXxT_E2y6R_B__{OlVQHNxKLbVY91iIR})FB|MI@ukGOH?6J&#@;`1enR7?{Q zb^p@xT6EOPPZ;f0?`KfdR0?xVu@}r(VS$rHlSO={&2;^-QWE|F!$4~J9- z)-2R%Eg9W<1A$<%@vAd6*N@CjpDHuD7y_ZGI=t1NSe)g|;xLtoHkI&EkHS%-rc-P6 z+a@as7X0(aw~Tr)nkr-adLU4F--x#k*J8*_MmhYW>IOY35=uf(@4e`#n2I1b{-0Hl zTXws}|NNo<{`gm%29Un3*;YtWs;)8D# zu7q$UK8_NEdqTJ;gnL3*$9!)GgcaQnhe5a!!jyph%y5HJPR#M?aH@ms>AdxMkGx z8Q_EB|5RyZK5tKA5FRWXR@!Yf@8T0YKCC3>tPlqz^sSMJ&*m-o(MZ!OW7;wiL15fI zHG#<${GpnG%r1F2YcM+9g*JvvE1@MER8goWJ(Ue5 z&L!7RmmRe2IyW)1>Sw`_c~j^comq(eb+d@qPRWy#PEy55?eEPGdw_L`F{Dfdk_m5>kGs$WXF+Jo&Pi zvRw>x8Dcr#R){OBLPt;0Ts9V@J`E{Ser4fHz;z)zMk1@KQWl0^Fn{V>Jhq(y`V2m% znn=u^JY1D##6IA-@dMXI@E57ZVj)=W&TEddAWN|4QlH$jANYIj$<5%_i?#M|PCBGt zXVtA^7^y4>!2{*_U06jHUWOGHj+b;$sD*B7IizLUTohOqS@ZB{_4ru4-usW8kqTv^ zKR?+g9u-ANX>F)`{H>AND+b zKOS)O3Fr5&pKAxzxDD~B4jPJB;9Gc-4` zdBP5y2r?! zKu*7Pi`~mDuLh%D9(>@G3Z-|!(cIp_op*mJCu$r+bb%7u-Q-Sa+(%k>(2fGC z2Xccc5GYh5%Nenl1pp&Fk+ z@C|Yd451hp!b=t6&@pk#X?25wE#Q2CMQ+(o-JrzH&Iine(|I-#x`X%2#cqS~O~49b z!Ggw}(swE)?=ax2pdvuKWdX^@2JL&>3lXcCBnwVAefam%ckXAE`fBZ4GP3Xal$&Kw zkCPBwcHet*s9t9zHoHefsjH0t@Ziol;p2bakaQl*%J~G^c1vDfKui>R^RiTD1IRK? zCO)GJxKjm_tDWPm2*I-q>V&}jwENT)DBcH66pE}6(-yokGm#AnBfVkh$({vs))6x9 zmBUcTfa>8)NN`7!q%Wj4rm%;0)VP4JPW0AaZoDXigf!>zRYfc!HUpp%5DfL~_J0XE z!BpbIsu4A58N}vmP8)@K&Vn8XVSm1YPz?04*zBQ!lu>m8$Uq!&&bFkd=1o2|rn?UG z+WWJwnH0T?OgZLd8U`tr1v~xz6RlJ0BY8t{IH%?I_z93zhi<20F~pLfjB_mRIw7E} zp|b?h+Y6jYIr`lo6!SrX#`3~mz{W}O621$I8< z-W=!^xlp5o%&`t6)90VS+TO&E0%9K31#n9ybbdB~bgD}r7?vWq!fEO0mGqlMo~8hI zzk4^`1l-MXXoGqPrh62TaO0!HMn^WUf7$)VSyE{K?i|VCnJ%`gdkJE(8uWv9%S)&U zR%XOi`a*bkF%*Ay0E46hPOdb>M$|-zY7ze8g(dA!a8|#BIf94}6lG1txdrR|xz9XCSKO_tzAeyB3A(|S z;jVGw+XO~ED^RLT)%S7hbWl>|?9U+m$O?$sE5nLHOP=@>B$7;P`$KvMHp(ezn=gSB zCv?h}<1FEH6zM_4T5fP~JiR~5L4F-@wKs?x8Ue<9B)tOhw~K-3@wrm%vjOTXv4{}J ztO6_uF>!iJ)1=uMH`7T&x1(XaiDWG_pA8dmjv-5_Sn%+nUV9I5xFI9o3>%mA*fLxE z+SI*C;2doy6==}rBw?OG=E^N44k7f$mynhG*wO$o^LI-lnLJ>h5C73M&c5S>AK09R z4Db>Aq_+e3M-@(-K1U#ileUHNVg?w|l#$0y(_DHv_&6i&O;$jqO z_^PI7R^-Aw0IsCzZFh_YX=iFgQL6xW? zqU;n1kVWkyhY!YW{66>F`Fhglh>t|?5rm~^i(xsOHAyba}VXpKR7n0FsW z*}g*|S76`=oTO$F5vU>G7+6AHLeOP0YDuh{aWET1jqbvmHJb-Ry$I$6=gx*D5S4nZ zc5;nNt#r}8$QyH)FDl(*vC;CNZEO{rmG)-*qpfmh@-DV<{rJ5j%DnCkJPL7EQ{VuW zpGV8dfLCVQKX;*R9z`F#mB2WAtnZ!2-MtAP57beyw_kB@M}%wLm~<_zBcKW%&hip^ z`f_ABr^7`Bn)AGe3ay8h!k0w5rd~W72Rm#%(!%W^!)xnCODty6ImZNwMit*{^<>=9~ z5Qt7}(g_3B20sf`j$_eWStzmcr0D;>(}lAK&_7wopyTK zc5CpCpf;%rG!w4`?~6c$Fs(xk(j$_fGexRFvt2Hr@xr`oGVB@&WgQWygdztz0KOjZ zOJkh24||888H?lNC+6Uu#n5uLf_K30#~Z0XykfvbKu%|`A7G;Sq*9$oJXzcypu3>8 zZc=g0@~F8J^mq;US*MBV4N{k5{GW~G|F)Qayk)2MujBzGO`YXO+)pO2y0H0nYb=}n=fd0+T8oZZlKz68TN7jRQ;@gsb zTn-QoAqZeJHFS_jAgZXcWym@9f@$^;RU=?7dDYD!6)mJMXPMh=`3kv+U*32>Wr%s& z7r}wr;CD9QEcCtRj!aT~1)=eo^tSo5_ekP%K~~wVEZJ+~wY9a-&7)u|r(t4F47V@B zx{bi(tiFf*A;p|mr*ke=2fV3PR^{M~j5mJgY;tOF_Uxo>DdW`yb&xeH6JMi@uhcGH z1_c9RRgkbUg*w^h2~a3Jv}=5x(~=wQ3s^)%eBl1H%ObHmwZdg~=8Gd5_8sHw%bXRv zzQ$>^)zgFID{J1h-+E`q$14QEWx1p!khN+-I58FIyGsa`P6N;oO@4Uu`U8i4fkZ#R z(MU|g2^oZ-6H6a(o~URIKwWGB1@B%zBB~X3y@&1NaT`q>j1vV?i5+Kts-BZLc*nkk zu}L`E&kY6P)~$!_d()RKTh@SBac0W!^OT2(;G((${CY9Jt%@EcN+|F>A+mNpSb1`nEOD6~n z+v*3id67@8j+ZdzgoFZ{#EcuVZ#&-EDFBH337rO6evci0JPO!2 zPQk?lHtfiGX+SiUH3-f$2#qVii-gZI$tfV?-$_Q@59VtrL=!dH`GYz9qXJ#~uE~#W z`-gd>a}XPI;0Ud^-qK~~BbksyX~VPl3I%SCl(n^4hCE#;jat0Lx@Zy(xeN((iZ~rB$?~lZhapbsSpaALRbHawRJOC%}^wZeWa^Bk) z+rZp-;UXb&>B@xd6vmJ?KRgs<*L;b(kY<}iR(TK7tKQoB`lgPKFj7x{3E1v36kqUd zqSO$UYKbcH^B7n$sVw++5z_r82;9Gnp>G#?<;o%oD6Wv${@%+8+NkjZ4}OBfYB7?iNV zw+6-e`D`4iL@ri(umgb$p(<|DEthXz2E3)K4^oGRxM@2&L8a(ok-WO}@Nd!(Cg-S9 zD2?L@aYo>CkT8L8M=;H1{O#i5A#L%Co@cJ-tmT0thH{#Asyh9@Ne{~~fYlyfFkL{Z z+MZ&zKX-T|Vz>RaI><^vAXdpLFvv1W>{^~!JPphb_#U9<`xrDvJusOe6XlM8qperJFaHVE-SB&4D;n&&v1;tC4pEZ+6J?e}X&v{vB%q@;g; z?*3lgj>k8QBlVA_>93O=96X7m8tDf!U>xn&QZOS_H%S-2)Qn$3{-2r~h?BRH4Z3&} zXmTxdSM@Fo*KvncfxWk{q;pW!I~#)fK~QW2fau=63pFpY3-J|7EFf5dZs3NSyN`ts z2p^GXAYgu;CLw00Y{mI=NyPYb3YJ(LM5wxiuZ}bUwxKn${#8#u;+wg^{Nw{^dkd&8 zcK#R%5cDeAdSAMo+&A)7(>NFqEm`mnX_I|UE;v-YYAL=JR5Mwf>lC~(B1Ocr-{78? zlddG;RHW}5aJ-Pi_hF9XXrD%`)aIByEN#L&ZG<#yOP|0Q`BdZ(c{~M$i4PNV^FVEx zY6n?nAMxlqG z5S6+&`oW*}0_o}^*A+>1_2C+Z4+fOIVvwUeF*S)1U48=qrY`8vUWbaOn?=%jBC)WQ z1e?yG)>P8wU(W<7$F00fK%78nHZ&NhhUzVg?XpU+EW{$KBpc(!;p9Dis%H=Px`yWs z_Cx}ilP2I-iu-_cF{W(N(k)UF_hVdg?#POwR5nB41r+7<6cOCjA*2mcGwTN?y?eLV z6QG1npTyqfOijhuH;3QG%(4;idC}1iCIB*Pnyd{9n@b^#4-1WrBT`wRZ_k2kA z&$VlM0G1xy1NCK1r-A`fx>S}c%SN3RMs+#<6|$?yss$ijI1;&HMlMpoN-7_8 z?M-=CmW8Zu8RJJO{Y07Ww&NxB`nPZDNjz8uw+Cwtm>qs}0TX0Xl*R?62Qu|Im&Bda z!DJ-T8xZrH<<%8?elY96TQ$0a3x@Q&A8eO}#!YjKk5f+Y6?oAAdB;vFFgFE@R|+QA zST$sHK`hEMSvUr>C~T_4hCmSub~xI`|A|ctFgVpB?>&~_iT!XSn4P8_BmB|&bDLRV zl6x0ncRDR8)@ob5--*30Z4Qxne=YvI{rn2c ztisc9d4e6tk28PH%q%95bd_xYJ`~~FgiHHFf zQX>pwJO_F4zEH;&?9eu!CpP|b;Wc6$z!<}|VPQ7LQuYM&2`w19A5?z@G^49ib5whl zK}IFYO9oUxK_0LOg-Fd(3U<9lhSp*mFjCjPMnDm>sFHy|taO-3k7M*9$4zaUr ztzjnQp;21w14L*CF1~(Z5XztKdc_;vplONSLjdYeB2q1R1!yaVt%40|jM}T~*sfg;YS;{bqpv6ofiG0hkbKZ8^93>qTHP zQ7KX7t?L@X3xed@07S6oW#T>&vxPtibaNg8pJ!$Y^w$PA3m4Ij?TI}z{90WD^pv_+ zDU|@NTJ@M{Gl6|41I??af%p7v`w9Z&1rz<<0VHUWwf1{{!+!!Hh;~&2agN^0NZQ&k z_|o{CKNLEBQ+l7HaLjdD(uTaX9lFS91i>$$YLCx$7zqR67YLWU4~_?s!ek}@JQ~YI&C7BI7#47;^EH&DHQndf;s~}g#hfC(w zfLRDr4&{4f2zH<##mDczdj_djDHFND7&sS4;q4u_q5pkw_agN%yYH6;4Y~n(GQGU0Pc^kpS|5VD6)8eC(dR*g6M#U-T-Ei}~eg4DjyQ z#O)onDLwyTJ+`HzhBB0yPn!NwXS>04`(sG4XZTfn`zxU^tfODg5usO z&9NBZM=V6L>5@?GG#1@x zIa(#E6VR}VLb#<{eYq7C!h0N$Gb|8)!-b9i<8B$+X_>ya^xII55BoMf^wVA14?(vF zDs>OZ=XWCNyn;9Zy}c0Y{&UtZ4RX8gaj4EuX$93=5;iXYx;PZIL8&2R;GcW+f8&7; zstlV&$4Z^BwN#`X6E0c6lBt-)W z1{^QZlrr)kkNyzNzJpBm1~1~>C}z^(A1=xC11N&(LdG;_C{KE59&a$)q=cfOq-nL- zy!}!9K0Cy+FY4dS4x5K!aRL}5`QV%OkhHjN+VV~zGhUUz!kQwfFQ`x~+2$wS!R8ZM~iX~;x zeK&L_ z6=SNKWFGchVl4rZ;iXN$T7o48V|l0*cB#h~4Mx`~NprykosD$p`}FUh-bd$*DVCzx z*FI&FeCQc8jT*s#&-5G_^3l_4C5YQJ>TiHW00eh^J?`geRML@K!^AWF4L7&5SS^*B zR=}FDq~)3ZwcdgiDP6W%h<@17N9&rwv)4hzPW$WwCzKMsl-b06;h7ZZ|y|kbBysy z49>I_b^F?W+!_fceF9^bMZHf%1C%YYQicNNfDzl*=TkcK=4yr}UHa_OlPo<2i;-Qsj zIX3kkD5_@a%qUFsHlmF46`1yem|2J(Dv5TGSEmBO>FD8_=;nR(t#631ObrNy`u;1t za1`yTuj(SK1_iB#+{SU~z98uaY-uAORG;m&&stts45tF*!De8gk>fnF{k=CIJm37{ zfY-%eu5TnKi5^kY6ywHb8kC^@`0Y8g8e@8VAD9KSn0>MSC78k}Lf6)|%|WHQFWr9d zFhLY!{%9dVocoR5&Vd ztfg*mh-8Hq9{2gI$l-@geHYB((os9o+dY#&)xj*+yB z1%RiWqE&CAVE)z<+1eV1_@^_?%q;UT0fPXh755)3y*I?#n}xW#4OLbfGKVvOw6!BL zINDtB$wZ?gIVo$xqb_ zLExEb*Er2C3bh6pEoKuk>ERir{_oIkVPsoRV&?5X}1O^!pw$t9vX*S4- zW*`w+_3hc8p_hZQ!ddtl{_aca=u~dp|7)XfJ-Ql_L7ubzxXTVAoNpr<@#FjctvN6u z8DvQUFEjl<3N=B`oB>aaHDL@Lpi9;r=z8x(1^w_54qweb6 ziEd&T70rRzYBX$7kzOquA0O}cu+2{eNUGg|qd@@D_ByBAZeCAC&NU|8%~@z)D1t(W z+T&r$I>4gZLUa_MX=lSFB;=vxh8J>@UuQQ@Vf{D-_k?(zRjaQxSS4#|mNYi9c|E&$ zgeBaz_yXCvw#xd~k(`s>9EAX!>&sMqagmpxatD^|9_CUoHG(2WCWX#LMIjWpRkW;< zzI%eYgiP5x?mAn>M$&+y$A0QgATMVD(xUFs8A=6nRZwGcAJXaQR8i^_pYI9muejpq~}wTD}ckv^h0T4a&fd$GYTrkeaCo-V4w1O>wMZ?3}IGc0w0~i_1CF1 zS4xHDPR4dUzm=_hjcR(94C+cdYD(;^eDwtp0yNIeu#Cx=6Q|~s4>C}B`(le!GDckAj(bK2|Bmn_4Xwekx-i3h!u4`Su z;1x1qWb>$>fLj8@3Fc`~n+e8;MzFuu^f%k9Ps^w~OAPCKZ)b)&qQu#(*vo0{8i)5h z2JQ+o#KE{`4gcSZwI_+iauoG8$<=k)*jpXi;m+&6yUc8*q5k||`_3sjA9>w)*|&Qz zra*BXCx9%OC#dX1cgC2^x}*~n7!d>;XkiMRWAQarQ?Fp62mv?3h3_a^p$K+TL690H z%pkiRxV{JDiHIjbgU6_g*!A=lsvi)39FZ748b+C@f($k|F{~uhw=!?s*X{#-}%x^oFRSz6?}+aPNtaVJXrYk?(a3y5TwDTu&EA+a%G`c z``sw22T~h=DAX{j2i(P!!>O$ZhXe4f8Aj;xX+(!wIyAz8YOgp|XN-Uk*UaPvXi$zw zuM&CGl z3~g=6H#P;kwhM^W^Ki>5NZn{$3UIG^{yZ<9f3JX+a`S{}-^X*n@M3xzlFavj^zCnr zXVHaGjWpV);wm;1CK3zf{2yKDLg*e!j!yLCNi#&fAp7pTdni0iS@qXMHJbG}fF2>$ zk&z#*MrVgcTu=%mQX*((K-0r(Q*{_VjQlg$�qOp^-`;*kv@0NTsJD{EenRzX%mq z_Q#ZpQ9Y!!Qc~#m7cq|l1d8mg-Pyx?TX*O5pjiXlMM15l)y z0@xiP3xUIe*+;32nKMKv3g0$Xp+rf;tG2Fg_nn_p);l~+KF6TIhesfwPX0sX^B`v$AQviQKsUA6#A(-I4Wu@IeEpBi$C6f z@WhzcUu{xZKM=HdLF;lUbA_N@PN{ygqsD%hujj;K)COnpm#6qpfWW%w(9~dwyWSko znb<4~M7`^!F!z^jC-%kq@AyaI`+84H03%nKXyVfia{RY&#MB z@14m%q@(#CBQ9_8(^fZIH`t)>Sq_)fan%?Ebbsm$!^w;gez_hPHK3WKEKni8^<*ZN z)(gd!OX_!%=cwXyV6>N=kNZdHX*8$MRT~?oWK!ylNhhA{r5wDeKedw3Rtu7Jkxoc2 zP2`44;;j((i^#CM{V|3%HXn&wxcCB#Ox)$=UYB6Nw9mGcV-9NIrnOVAYv3f+n7r2I z=J%`2mmdk4!tYQlY@PmoDDuWR(I6a6M3Si+s<&X0QxC?`)(w}J_H2^DcJ!IqY}_+M zD%?dn&+6(LsPi#!oCVaN+X6jgH`*{)%xH#dho z*&C_mt`lE5Z!T)9sAAFpP#05^3LH+SKJJI-#^1O1M>5Y8F574h?DCSaN7N@O*S7vR zdZg{)uJ79y#P(g-t)6kl6ea+4!?A&F3^b!Dwa!$TuV#WJrZ49Gamf5?l@-^*|x!F z5-Z5lY>xb!C*SR#u&9|5Y2es z#y7!VyEgT1^3QhCHMlr#B{QatSyfB}V+PzxZ0s*8KGFy_D{VV>*~Z=Kn&%=15;DBQ z|9I^mM5;z%!R^UZ#>CKBbt6q6RdB{uVX;7--@g@@>lOrf~|- zP0O)}H_9+7KKci(Uf`L?e}DUG)!6C0U78^wD>G_CH)6Dk`BPP4{n}kG`=``!W?{mA zL?ac8GIEdp>Emg*f6Uzygn4w;YWcL8)mdMNS`1lFXAgNOhs(tsf6;NMaH6_QP+zW< z?C1{jrnTy;;t#}gzEg|oOT=9Mm*!1>9YOhX-v;Ue@zoP}Q7w5OalNwVlTpWyFXH_* z_3)05sV#i+WvJskIz8kcjpx5=EBrXR&U>xcIo$UPyv2xhs2EAf{- b5h&jCIDg7=Q?5G;|5>cvXqvp*A>{u7>CV+M literal 0 HcmV?d00001 diff --git a/docs/localization.md b/docs/localization.md index 2f9e98a676..96822877a3 100644 --- a/docs/localization.md +++ b/docs/localization.md @@ -20,7 +20,7 @@ Whole sentences that the layers below word — what the accessibility checker fo The Ukrainian catalog has 2,530 entries, written from the meaning of the English labels using Ukrainian computing vocabulary. Commands use infinitives; fixed-count messages use count-neutral wording. Plural-aware messages use the three [CLDR Ukrainian integer forms](https://www.unicode.org/cldr/charts/48/supplemental/language_plural_rules.html#uk): `1 сторінка`, `2 сторінки`, `5 сторінок`; 11–14 take the third form, while 21 and 22 return to the first and second. Calendar dates use genitive month names (`9 жовтня 2026`), while headings use standalone names (`Жовтень 2026`). Home recommendation cards grow to fit translated text; the toolbar search hint truncates before the shortcut, with the full label available on hover. -The Lithuanian catalog has 2,798 entries and three integer plural forms. Calendar headings use standalone month names; dates use genitive forms. Calendar, inspector and password-dialog UI regressions check dynamic labels and preserve document values. A native Lithuanian-speaking contributor has reviewed the translation. +The Lithuanian catalog has 2,798 entries and three integer plural forms. Calendar headings use standalone month names; dates use genitive forms. Calendar, inspector and password-dialog UI regressions check dynamic labels and preserve document values. A native Lithuanian-speaking contributor has reviewed the translation. The Scan & OCR review strings added afterwards (M10: the OCR Verify screen, the batch options and the engine choice) were written without native review. The Vietnamese catalog has 2,798 entries and one plural form (`plural_none`), covering the current catalog-key union. Preferences theme `Appearance` uses context `theme` → `Giao diện`; object/comment/signature appearance stays plain `Diện mạo`. Form fields use `Trường`; redaction apply wording states permanent removal. Calendar months are numeric (`Tháng 1`…`12` / `tháng 1`…`12`). A native Vietnamese-speaking contributor has reviewed the translation. Status stays `partial`: no translated help, and engine/OS details stay English inside a translated frame. diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index f71546b731..ec144d3a07 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -6683,7 +6683,7 @@ milestone = "M10" status = "shipped" commands = ["ocr.recognize"] tools = ["ocr_recognize", "ocr_status"] -evidence = ["crates/ocr/src/lib.rs::model_discovery_prefers_the_complete_onnx_pair", "crates/ocr/src/lib.rs::model_discovery_falls_back_to_legacy_when_the_onnx_pair_is_incomplete", "crates/ocr/src/lib.rs::model_discovery_refuses_mixed_format_pairs", "crates/ocr/src/lib.rs::model_discovery_keeps_legacy_only_installations", "crates/engine/src/tests.rs::recognize_text_makes_a_scanned_page_searchable", "crates/automation/tests/it/automation.rs::ocr_tools_make_a_scan_searchable", "crates/ui-egui/tests/it/ocr.rs::recognize_text_dialog_adds_searchable_text"] +evidence = ["crates/ocr/src/ocrs.rs::model_discovery_prefers_the_complete_onnx_pair", "crates/ocr/src/ocrs.rs::model_discovery_falls_back_to_legacy_when_the_onnx_pair_is_incomplete", "crates/ocr/src/ocrs.rs::model_discovery_refuses_mixed_format_pairs", "crates/ocr/src/ocrs.rs::model_discovery_keeps_legacy_only_installations", "crates/engine/src/tests.rs::recognize_text_makes_a_scanned_page_searchable", "crates/automation/tests/it/automation.rs::ocr_tools_make_a_scan_searchable", "crates/ui-egui/tests/it/ocr.rs::recognize_text_dialog_adds_searchable_text"] notes = "ocrs engine (MIT/Apache) with CC-BY-SA ONNX models fetched and SHA-256 checked by `cargo xtask models`; within each search directory, a complete ONNX pair takes precedence over a complete legacy RTEN pair, incomplete and mixed-format pairs are not combined, and directory precedence is unchanged; the desktop release packages ship them (#103; packaging/env.sh stage_models), the web build does not yet; runs on a worker thread with progress; one undoable step; pages with text are skipped unless asked (ocr.skip-renderable-text)." [[feature]] @@ -6708,7 +6708,7 @@ status = "shipped" commands = ["ocr.recognize_batch"] tools = ["ocr_recognize_files"] evidence = ["crates/automation/tests/it/automation.rs::ocr_recognize_files_writes_searchable_copies", "crates/ui-egui/tests/it/ocr.rs::recognize_text_in_multiple_files_writes_searchable_copies"] -notes = "Pick PDFs, then an output folder; copies are saved incrementally under the same names, in the background with progress." +notes = "Pick PDFs, then an output folder (the last folder is remembered and reused; Clear asks again); copies are saved incrementally under the same names, in the background with progress. Output names are pre-flighted (a collision aborts before the first page is read), files that already have text are skipped whole only when that option is on (skip_text_files, off by default; Force OCR overrides), and the summary counts recognized words, the skipped bucket, per-file failures, low-confidence notes and what a cancel left unrun." [[feature]] id = "ocr.ocr-languages" @@ -6719,8 +6719,8 @@ milestone = "M10" status = "partial" commands = ["ocr.recognize"] tools = ["ocr_recognize"] -evidence = ["crates/ui-egui/tests/it/ocr.rs::recognize_text_dialog_adds_searchable_text"] -notes = "Language choice exists, but the models read only the Latin alphabet without accents (English)." +evidence = ["crates/ui-egui/tests/it/ocr.rs::recognize_text_dialog_adds_searchable_text", "crates/ocr/src/tesseract.rs::language_table_maps_and_falls_back"] +notes = "The choice feeds both engines: ocrs reads only unaccented Latin (its model table holds English); tesseract maps 12 allow-listed codes (the language enum comes from the engine's own table) but only as far as the installed language packs cover. Recognition quality beyond the tested English set is unproven." [[feature]] id = "ocr.searchable-image" @@ -6731,8 +6731,8 @@ milestone = "M10" status = "partial" commands = ["ocr.recognize"] tools = ["ocr_recognize"] -evidence = ["crates/engine/src/tests.rs::recognize_text_makes_a_scanned_page_searchable"] -notes = "Invisible text (Tr 3) fitted to each word box; the image is not deskewed or downsampled (exact mode only)." +evidence = ["crates/engine/src/tests.rs::recognize_text_makes_a_scanned_page_searchable", "crates/engine/src/ocr.rs::searchable_layer_round_trips_non_win_ansi_words", "crates/engine/src/ocr.rs::preprocessing_flags_flow_through_the_job"] +notes = "Invisible text (Tr 3) fitted to each word box. The page's own image object is left untouched; the raster the engines read can be cleaned first (opt-in rotate/deskew/denoise/binarize), with word boxes mapped back to the original raster. Words beyond WinAnsi are written in Type3 fonts with empty character programs and a ToUnicode CMap, so extraction reads them back while nothing needs a display font (no craft-fonts input, and no character is ever substituted with '?'; those substitutions belong to the editable output). Missing: the downsampled Searchable Image variant (ocr.ocr-downsample)." [[feature]] id = "ocr.searchable-image-exact" @@ -6753,9 +6753,9 @@ area = "I" tier = "P1" milestone = "M10" status = "partial" -commands = ["edit.edit_text"] -tools = ["text_edit"] -evidence = ["crates/engine/tests/it/ocr_edit.rs::editing_an_ocr_word_covers_the_scan_and_shows_the_new_text", "crates/engine/tests/it/ocr_edit.rs::an_explicit_colour_is_kept_for_the_new_text", "crates/engine/tests/it/ocr_edit.rs::ocr_words_group_into_phrases_that_edit_in_place", "crates/engine/tests/it/ocr_edit.rs::a_longer_ocr_phrase_grows_rightwards", "crates/engine/tests/it/ocr_edit.rs::a_rewritten_ocr_phrase_keeps_its_word_widths", "crates/engine/tests/it/ocr_edit.rs::an_ocr_layer_under_the_scan_image_shows_its_edit", "crates/engine/tests/it/ocr_edit.rs::an_ocr_phrase_split_from_visible_text_under_the_scan_shows_its_edit", "crates/ui-egui/tests/it/ocr.rs::an_ocr_word_opens_for_editing_and_stays_open", "crates/ui-egui/tests/it/ocr.rs::editing_a_scanned_page_recognizes_its_text_first_and_only_once"] +commands = ["edit.edit_text", "ocr.correct"] +tools = ["text_edit", "ocr_recognize"] +evidence = ["crates/engine/tests/it/ocr_edit.rs::editing_an_ocr_word_covers_the_scan_and_shows_the_new_text", "crates/engine/tests/it/ocr_edit.rs::an_explicit_colour_is_kept_for_the_new_text", "crates/engine/tests/it/ocr_edit.rs::ocr_words_group_into_phrases_that_edit_in_place", "crates/engine/tests/it/ocr_edit.rs::a_longer_ocr_phrase_grows_rightwards", "crates/engine/tests/it/ocr_edit.rs::a_rewritten_ocr_phrase_keeps_its_word_widths", "crates/engine/tests/it/ocr_edit.rs::an_ocr_layer_under_the_scan_image_shows_its_edit", "crates/engine/tests/it/ocr_edit.rs::an_ocr_phrase_split_from_visible_text_under_the_scan_shows_its_edit", "crates/ui-egui/tests/it/ocr.rs::an_ocr_word_opens_for_editing_and_stays_open", "crates/ui-egui/tests/it/ocr.rs::editing_a_scanned_page_recognizes_its_text_first_and_only_once", "crates/engine/src/ocr.rs::editable_writer_refuses_empty_and_writes_visible_words", "crates/engine/src/ocr.rs::editable_output_round_trips_non_win_ansi_words", "crates/engine/src/ocr.rs::editable_writer_maps_rotated_pages", "crates/automation/tests/it/automation.rs::ocr_recognize_editable_output_writes_visible_text"] notes = "On demand, per edit: in Edit text & images, recognised text (the invisible OCR layer over a scan) is grouped into phrases (words across a word space; a wider gap such as a table cell, and each scanned line, stay separate) and opens for editing on a click. Applying draws the new text visibly in the scan's sampled ink colour (or the chosen one) and first covers the old words with the paper colour sampled around them, so the scanned word no longer shows; when an image or form is drawn after the OCR layer over where the edit lands (its old box, moved, rewrapped or grown to more lines), or an inline image or shading after it (a layer written under its scan, #3973), the cover and new text are drawn on top of the page instead, so they show; the phrase keeps each word's position and width and grows rightwards instead of wrapping. Turning Edit text & images on for a scanned page (an image covering at least half of a page that has no text of its own, once per page per window; comments and text boxes over it do not count as text, a page with real text and a background picture is left alone; one undoable \"Recognize text\" step) now recognizes its words first, as Acrobat's \"Converting scanned page to Editable Text and Images\" does (English only until more OCR languages exist, gap 7). Still missing: separating the photographs on the page from the lettering, matching the scanned typeface (the layer's Helvetica is used), covering on a textured or patterned background (one flat colour is used), a scan painted by a pattern fill or a filled rectangle over a layer written under it, and, for text drawn on top, the clip, optional content and marked content (tags) that enclosed the OCR layer." [[feature]] @@ -6779,8 +6779,8 @@ milestone = "M10" status = "shipped" commands = ["ocr.recognize"] tools = ["ocr_recognize"] -evidence = ["crates/engine/src/tests.rs::recognize_text_makes_a_scanned_page_searchable", "crates/automation/tests/it/automation.rs::ocr_tools_make_a_scan_searchable", "crates/engine/src/tests.rs::recognize_text_reads_a_picture_on_a_page_that_has_text", "crates/automation/tests/it/automation.rs::ocr_recognize_reads_pictures_on_pages_with_text", "crates/ui-egui/tests/it/ocr.rs::recognize_text_can_read_pictures_on_pages_with_text"] -notes = "Pages that already have text are skipped and reported by default. The dialog's \"Also recognize pages that already have text\" (skip_text_pages false, also for multiple files) reads pictures on such pages (#1028); a recognised word at least half covered by the page's own text (an earlier OCR layer included) is left out and counted (already_on_page). Missing: recognising only the image areas (the whole page is still read, so it takes as long as a scan), and a word in a picture that sits under real text is treated as that text." +evidence = ["crates/engine/src/tests.rs::recognize_text_makes_a_scanned_page_searchable", "crates/automation/tests/it/automation.rs::ocr_tools_make_a_scan_searchable", "crates/engine/src/tests.rs::recognize_text_reads_a_picture_on_a_page_that_has_text", "crates/automation/tests/it/automation.rs::ocr_recognize_reads_pictures_on_pages_with_text", "crates/ui-egui/tests/it/ocr.rs::recognize_text_can_read_pictures_on_pages_with_text", "crates/automation/tests/it/automation.rs::ocr_recognize_files_writes_searchable_copies"] +notes = "Text pages are skipped by default; skip_text_pages false or Force OCR reads pictures on them while omitting words overlapping existing text (already_on_page). Batch skip_text_files reports skipped files without writing them; Force OCR overrides both skips. Recognition still reads the whole page, and image text under real text can be omitted." [[feature]] id = "ocr.rotation-detection" @@ -6788,11 +6788,11 @@ title = "Detect and fix page rotation" area = "I" tier = "P1" milestone = "M10" -status = "planned" -commands = [] -tools = [] -evidence = [] -notes = "" +status = "partial" +commands = ["ocr.recognize"] +tools = ["ocr_recognize"] +evidence = ["crates/ocr/src/preprocess.rs::quarter_turns_are_detected_corrected_and_mapped_back", "crates/ocr/src/preprocess.rs::realistic_letter_pages_are_left_upright"] +notes = "Opt-in preprocessing: 0/90/180/270 detected from projection-profile banding (conservative: under 32 px, blank, or without a clear margin it is a no-op); word boxes map back to the scanned raster through the exact inverse. Correction is proven at the preprocessing layer and the flag flows through the job (needs the models), but correction of a genuinely rotated page through the workflow is not yet covered by a cited end-to-end test." [[feature]] id = "ocr.deskew" @@ -6800,11 +6800,11 @@ title = "Deskew" area = "I" tier = "P1" milestone = "M10" -status = "planned" -commands = [] -tools = [] -evidence = [] -notes = "" +status = "partial" +commands = ["ocr.recognize"] +tools = ["ocr_recognize"] +evidence = ["crates/ocr/src/preprocess.rs::skew_is_estimated_and_straight_pages_are_left_alone", "crates/ocr/src/preprocess.rs::deskew_inverse_maps_off_centre_points_back_onto_the_source_line"] +notes = "Opt-in: skew estimated over +/-5 degrees by a shear-scored projection profile; conservative no-ops (under 64 px, blank, below 0.1 degrees, no clear winner); boxes map back off-centre-exact through the composed inverse. Correction is proven at the preprocessing layer; a cited end-to-end test of a skewed page through the workflow is still owed." [[feature]] id = "ocr.despeckle" @@ -6813,9 +6813,9 @@ area = "I" tier = "P1" milestone = "M10" status = "partial" -commands = ["ocr.enhance_camera", "ocr.enhance"] -tools = ["page_enhance_camera", "page_enhance_scan"] -evidence = ["crates/engine/src/camera_clean.rs::a_shadow_and_a_yellow_cast_become_white_paper", "crates/engine/src/tests.rs::enhance_camera_page_at_a_level_whitens_a_shadowed_sheet", "crates/ui-egui/tests/it/camera.rs::the_clean_up_level_enhances_again_from_the_photo_and_keeps_the_turn", "crates/engine/src/scan_clean/tests.rs::despeckle_is_its_own_switch", "crates/automation/tests/it/automation.rs::enhance_scanned_file_cleans_the_scan_in_place_through_tools", "crates/ui-egui/tests/it/scan_clean.rs::enhance_scanned_file_cleans_the_chosen_pages_and_says_what_it_did"] +commands = ["ocr.enhance_camera", "ocr.enhance", "ocr.recognize"] +tools = ["page_enhance_camera", "page_enhance_scan", "ocr_recognize"] +evidence = ["crates/engine/src/camera_clean.rs::a_shadow_and_a_yellow_cast_become_white_paper", "crates/engine/src/tests.rs::enhance_camera_page_at_a_level_whitens_a_shadowed_sheet", "crates/ui-egui/tests/it/camera.rs::the_clean_up_level_enhances_again_from_the_photo_and_keeps_the_turn", "crates/engine/src/scan_clean/tests.rs::despeckle_is_its_own_switch", "crates/automation/tests/it/automation.rs::enhance_scanned_file_cleans_the_scan_in_place_through_tools", "crates/ui-egui/tests/it/scan_clean.rs::enhance_scanned_file_cleans_the_chosen_pages_and_says_what_it_did", "crates/ocr/src/preprocess.rs::denoise_removes_specks_and_keeps_dimensions"] notes = "Camera images, and scanned pages through Enhance scanned file (ocr.enhance, page_enhance_scan despeckle): isolated dark pixels (at most one dark neighbour) turn white, on grey, RGB and 1-bit scans, in place (the OCR text layer, comments and links stay). Missing: as an option of Recognize text; specks larger than two pixels; CMYK, indexed, JPEG 2000, JBIG2 and CCITT scans and scans inside form XObjects (reported, not changed)." [[feature]] @@ -6896,11 +6896,11 @@ title = "Correct recognized text (low-confidence suspects)" area = "I" tier = "P2" milestone = "M10" -status = "planned" -commands = [] -tools = [] -evidence = [] -notes = "" +status = "partial" +commands = ["ocr.correct"] +tools = ["ocr_words", "ocr_review_apply", "ocr_dictionary"] +evidence = ["crates/ui-egui/src/workflows/ocr_verify.rs::dictionary_uses_recognition_language_instead_of_interface_locale", "crates/ui-egui/src/workflows/ocr_verify.rs::dictionary_limit_and_normalization_failures_preserve_file_and_review", "crates/ui-egui/src/workflows/ocr_verify.rs::dictionary_undo_persists_and_failed_undo_keeps_history", "crates/ui-egui/src/workflows/ocr_verify.rs::rereading_keeps_captured_language_when_the_draft_changes", "crates/automation/tests/it/automation.rs::ocr_review_corrections_keep_boxes_are_guarded_and_apply_once", "crates/automation/tests/it/automation.rs::ocr_dictionary_updates_are_atomic_bounded_and_confined", "crates/automation/tests/it/automation.rs::ocr_retained_reviews_have_eight_document_cap_and_close_releases_slot", "crates/ui-egui/src/workflows/ocr_verify.rs::accept_zero_words_fails_and_a_live_review_applies_once", "crates/ui-egui/src/workflows/ocr_verify.rs::legend_rows_carry_the_classifier_thresholds", "crates/ui-egui/src/workflows/ocr_verify.rs::suggestions_are_near_dictionary_and_page_words", "crates/ui-egui/tests/it/ocr.rs::ocr_verify_screen_controls_keyboard_and_accept", "crates/ui-egui/src/workflows/ocr_verify.rs::accepting_pages_one_by_one_then_the_review_applies_every_page_once", "crates/ui-egui/src/workflows/ocr_verify.rs::rereading_a_region_remaps_marks_and_selects_the_first_new_word"] +notes = "The OCR Verify screen: page list, source page with confidence-coloured word boxes (click-select, drag a region), recognised-text preview, zoom + word inspector; corrections commit on Enter (an empty commit removes), Remove/Restore, +Dictionary (a per-language user file), per-word Verified, suggestions within edit distance 2; F8/Shift+F8 walk the low-confidence suspects, Ctrl+T marks the page reviewed, Ctrl+Tab moves the pane focus; Re-OCR of a region or a page; Accept/Reject entire page; a real Cancel keeps the pages already read; a stale session is refused with its reason; review edits are undoable in the panel; Accept (searchable) is one undoable step through the guard path, and pages accepted one by one each apply once (the guard follows each accepted page); a re-read keeps every review mark on the word it was made for. Headless: ocr_words (retain_review) returns the words and a review handle, ocr_review_apply applies corrections at the stored boxes (an empty one removes the word) through the same stale-document guard, or discards the review, and ocr_dictionary edits the dictionary file. Still missing: suggestions, per-word Verified marks and region re-reads are UI-only, and Acrobat's suspect walk across a whole saved document (outside one review) is not offered." [[feature]] id = "ocr.adaptive-compression" @@ -6932,11 +6932,11 @@ title = "Pluggable OCR engines (ocrs default; tesseract optional)" area = "I" tier = "P1" milestone = "M10" -status = "planned" -commands = [] -tools = [] -evidence = [] -notes = "" +status = "partial" +commands = ["ocr.recognize"] +tools = ["ocr_recognize"] +evidence = ["crates/engine/src/ocr.rs::selector_names_the_missing_engine", "crates/ocr/src/tesseract.rs::recognize_runs_the_installed_binary", "crates/ocr/src/merge.rs::rover_vote_always_runs_both_and_degrades_without_a_secondary", "crates/ocr/src/merge.rs::confidence_weighted_runs_the_secondary_on_an_unknown_primary", "crates/ocr/src/merge.rs::an_unknown_primary_word_yields_only_to_a_confident_secondary", "crates/ocr/src/tesseract.rs::an_invalid_configured_program_fails_closed", "crates/ocr/src/tesseract.rs::list_langs_come_from_stderr_when_stdout_has_none", "crates/engine/src/ocr.rs::automatic_defaults_to_the_confidence_weighted_strategy"] +notes = "ocrs (pure Rust, MIT/Apache, CC-BY-SA models fetched by `cargo xtask models`) is the default; an installed tesseract binary runs as an external process (fixed argv, allow-listed language codes, private temp dir, timeout and output cap, a user-validated PDFCRAFT_TESSERACT path or the PATH); Automatic is an ensemble (ROVER and confidence-weighted merge; a failing partner degrades to the primary with a note). Partial, because part of the evidence is conditional: `recognize_runs_the_installed_binary` runs only where tesseract is installed and skips elsewhere, so a build machine without it proves nothing about the process path, the absent-binary paths are exercised only on machines without tesseract, and the process-level tests (timeout, output cap, minimal environment) are Unix-only (fake scripts). What always runs: location validation and the fail-closed rule, the --list-langs parsing (stdout or stderr), the engine selector, the merge rules and the default strategy. Automatic defaults to the confidence-weighted strategy (Primary only would never run the second engine)." [[feature]] id = "ocr.ocr-web" diff --git a/xtask/src/license_check.rs b/xtask/src/license_check.rs index c845101db6..0231b4e853 100644 --- a/xtask/src/license_check.rs +++ b/xtask/src/license_check.rs @@ -54,7 +54,7 @@ fn validate(root: &Path) -> Result<()> { let docs: Documents = toml::from_str(&text).context("parsing LICENSES.toml")?; let attribution = assets::load(root)?; let mut problems = check(root, &docs, &attribution); - let ocr_source = std::fs::read_to_string(root.join("crates/ocr/src/lib.rs")).context("reading OCR model filename constants")?; + let ocr_source = std::fs::read_to_string(root.join("crates/ocr/src/ocrs.rs")).context("reading OCR model filename constants")?; problems.extend(check_required_ocr_metadata(&ocr_source, &attribution)?); if !problems.is_empty() { bail!("source licensing errors:\n{}", problems.join("\n")); @@ -72,7 +72,7 @@ fn check_required_ocr_metadata(source: &str, attribution: &assets::Manifest) -> let file = declarations.first().and_then(|value| value.strip_suffix("\";")); let Some(file) = file.filter(|file| declarations.len() == 1 && safe_path(file) && !file.contains('/')) else { bail!( - "OCR constant {name}: expected one literal filename declaration in crates/ocr/src/lib.rs; update the source validator if its syntax changed" + "OCR constant {name}: expected one literal filename declaration in crates/ocr/src/ocrs.rs; update the source validator if its syntax changed" ); }; let entries: Vec<_> = attribution.fetched.iter().filter(|f| f.file == file).collect(); @@ -407,7 +407,7 @@ mod tests { #[test] fn required_ocr_models_cannot_be_missing() { - let source = include_str!("../../crates/ocr/src/lib.rs"); + let source = include_str!("../../crates/ocr/src/ocrs.rs"); let original = assets::load(&assets::root()).unwrap(); assert_eq!(original.fetched.iter().filter(|f| f.kind == "model").count(), 2); assert!(check_required_ocr_metadata(source, &original).unwrap().is_empty()); @@ -420,7 +420,7 @@ mod tests { #[test] fn required_ocr_models_cannot_be_misclassified() { - let source = include_str!("../../crates/ocr/src/lib.rs"); + let source = include_str!("../../crates/ocr/src/ocrs.rs"); let original = assets::load(&assets::root()).unwrap(); assert_eq!(original.fetched.iter().filter(|f| f.kind == "model").count(), 2); for model in original.fetched.iter().filter(|f| f.kind == "model") { diff --git a/xtask/src/screenshots.rs b/xtask/src/screenshots.rs index 243d8b0bad..521d793f48 100644 --- a/xtask/src/screenshots.rs +++ b/xtask/src/screenshots.rs @@ -39,6 +39,7 @@ pub const SCENES: &[(&str, &str, &str, &[&str])] = &[ "README: layouts", &["--layout", "two-up", "--mode", "read", "--page", "4", "--theme", "dark", "--notice", "off"], ), + ("pdfcraft-ocr-verify", "OCR Verify screen", "README: OCR review", &["--size", "1680x1000", "--ocr-verify", "on"]), ]; const PDF: &str = "dist/demo/pdfcraft-showcase.pdf"; @@ -109,7 +110,10 @@ fn update_attribution(root: &Path) -> Result<()> { // The home screen shows the ArtCraft mark: say so (it is a Storyteller trademark, not open // source; AGENTS.md §1.2), so the screenshot's licence is not read as covering it. let shows_home = scene.is_some_and(|(.., args)| args.windows(2).any(|w| w[0] == "--home" && w[1] == "on")); - let contents = if shows_home { + let shows_ocr = scene.is_some_and(|(.., args)| args.windows(2).any(|w| w[0] == "--ocr-verify" && w[1] == "on")); + let contents = if shows_ocr { + "Shows only PdfCraft UI, its Lucide icons and a synthetic scanned page built from the embedded fonts, with a real recognition (the OCR models) loaded into the Verify screen" + } else if shows_home { "Shows PdfCraft UI, Lucide icons, the showcase PDF built from the [[fetched]] OFL fonts, and the ArtCraft mark (Storyteller trademark from docs/brand/, not covered by this licence)" } else { "Shows only PdfCraft UI, Lucide icons and the showcase PDF built from the [[fetched]] OFL fonts" From ce3857c2e5ec61dd6a5cf8565c7e699acc7618a7 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 5 Nov 2026 20:55:45 +0000 Subject: [PATCH 131/198] sign: classify changes by signed object role Judge each object changed after signing by the role it held in the signed revision, keyed by object number, and compare a slot's accepted generations across revisions. Adds `Document::accepted_generations` to cos. Co-Authored-By: Claude Opus 5.5 --- crates/cos/README.md | 4 +- crates/cos/src/document/objects.rs | 24 ++ crates/cos/tests/it/generations.rs | 35 +++ crates/sign/src/pdf.rs | 3 + crates/sign/src/pdf/changes.rs | 168 ++++++++------ .../src/pdf/revision_classification_tests.rs | 207 ++++++++++++++++++ crates/sign/tests/it/main.rs | 1 + .../sign/tests/it/revision_classification.rs | 158 +++++++++++++ 8 files changed, 536 insertions(+), 64 deletions(-) create mode 100644 crates/sign/src/pdf/revision_classification_tests.rs create mode 100644 crates/sign/tests/it/revision_classification.rs diff --git a/crates/cos/README.md b/crates/cos/README.md index 774f2c4367..9f5fb56725 100644 --- a/crates/cos/README.md +++ b/crates/cos/README.md @@ -62,7 +62,9 @@ another. Edits never touch the original bytes. not expand compact xref ranges; the winning xref entry remains authoritative for free slots. Compressed objects require generation zero. Explicit `set` overlays accept only the supplied generation; `set_existing` and `update_dict` retain both accepted generations of a - damaged live slot, including through snapshots, and require full rewrite. Headers beyond + damaged live slot, including through snapshots, and require full rewrite. + `accepted_generations(num)` lists the generations a slot accepts, so code comparing two + revisions can tell whether a reference still reaches the same slot. Headers beyond this bounded probe are not additionally recovered. Materialized primary and hybrid rows are diagnosed before folding, so a newer row does not hide a retained disagreement. - **Tolerant, never fatal.** The lexer works on a byte slice and a position; it allocates nothing diff --git a/crates/cos/src/document/objects.rs b/crates/cos/src/document/objects.rs index 8d9917d749..458840de3a 100644 --- a/crates/cos/src/document/objects.rs +++ b/crates/cos/src/document/objects.rs @@ -175,6 +175,30 @@ impl Document { } } + /// The generations a reference to `num` may carry and still resolve (see [`Self::get`]), + /// sorted: none for a free or unknown slot, else the slot's generation, and for a damaged + /// file slot (or a repair-aware edit of one) also the generation its object header carries. + /// Code comparing revisions (signature validation) uses it to tell whether the references + /// that reached an object in one revision still reach the same slot in another. + pub fn accepted_generations(&self, num: u32) -> Vec { + // Mirrors `reference_matches`, which stays allocation-free for every lookup. + let mut generations = match self.overlay.get(&num) { + Some(Slot::Set(generation, _)) => vec![*generation], + Some(Slot::RepairedSet(generation, alias, _)) => vec![*generation, *alias], + Some(Slot::Freed(_)) => Vec::new(), + None => match self.entries.lookup(num, &self.data) { + Some(XrefEntry::InFile { offset, generation }) => { + std::iter::once(generation).chain(self.indexed_header_reference(num, offset).map(|h| h.generation)).collect() + } + Some(XrefEntry::InStream { .. }) => vec![0], + None | Some(XrefEntry::Free { .. }) => Vec::new(), + }, + }; + generations.sort_unstable(); + generations.dedup(); + generations + } + pub(super) fn try_get_reference(&self, r: ObjRef) -> Result, CosError> { if !self.reference_matches(r) { return Ok(Arc::new(Object::Null)); diff --git a/crates/cos/tests/it/generations.rs b/crates/cos/tests/it/generations.rs index 3d56628c93..bbe2ff0b33 100644 --- a/crates/cos/tests/it/generations.rs +++ b/crates/cos/tests/it/generations.rs @@ -473,3 +473,38 @@ fn restoring_explicit_sets_removes_old_generation_aliases() { } } } + +/// `accepted_generations` lists exactly the generations `get` resolves, for every kind of slot: +/// a clean file slot, a damaged one (row and header disagree), a repair-aware edit, a plain +/// edit, a freed slot, a free row, an object stream member and an unknown number. +#[test] +fn accepted_generations_are_the_generations_get_resolves() { + let agrees = |doc: &Document, num: u32, expected: &[u16]| { + assert_eq!(doc.accepted_generations(num), expected, "object {num}"); + for generation in 0..8 { + let live = !matches!(*doc.get(ObjRef::new(num, generation)), Object::Null); + assert_eq!(live, expected.contains(&generation), "object {num} generation {generation}"); + } + }; + let doc = Document::open(contradictory_entry(false)).unwrap(); + agrees(&doc, 1, &[0]); + agrees(&doc, 3, &[0, 1]); + agrees(&doc, 9, &[]); + let mut repaired = doc.clone(); + repaired.set_existing(ObjRef::new(3, 0), Object::Int(77)).unwrap(); + agrees(&repaired, 3, &[0, 1]); + let mut narrowed = doc.clone(); + narrowed.set(ObjRef::new(3, 2), Object::Int(99)); + agrees(&narrowed, 3, &[2]); + let mut freed = doc.clone(); + freed.free(ObjRef::new(3, 0)); + agrees(&freed, 3, &[]); + agrees(&Document::open(contradictory_entry(true)).unwrap(), 3, &[]); + let mut doc = + Document::open(object_file(&[(1, 0, "<< /Type /Catalog /Pages 2 0 R >>"), (2, 0, "<< /Type /Pages /Kids [] /Count 0 >>")])).unwrap(); + let member = doc.add(Object::Dict(Default::default())); + doc.update_dict(ObjRef::new(1, 0), |c| c.set(b"Extra".to_vec(), Object::Ref(member))).unwrap(); + let packed = Document::open(Arc::new(write_full(&doc, &SaveOptions { object_streams: true, ..SaveOptions::default() }).unwrap())).unwrap(); + assert!(matches!(packed.xref_entry(member.num), Some(XrefEntry::InStream { .. })), "the member is in an object stream"); + agrees(&packed, member.num, &[0]); +} diff --git a/crates/sign/src/pdf.rs b/crates/sign/src/pdf.rs index 75b42f9905..8fafff7f01 100644 --- a/crates/sign/src/pdf.rs +++ b/crates/sign/src/pdf.rs @@ -45,6 +45,9 @@ use validate::{DssEvidence, MAX_DSS_BYTES, MAX_DSS_ENTRIES}; #[cfg(test)] #[path = "pdf_discovery_tests.rs"] mod discovery_tests; +#[cfg(test)] +#[path = "pdf/revision_classification_tests.rs"] +mod revision_classification_tests; /// One signature field. #[derive(Clone, Debug)] diff --git a/crates/sign/src/pdf/changes.rs b/crates/sign/src/pdf/changes.rs index 3a10a74e41..81467542e6 100644 --- a/crates/sign/src/pdf/changes.rs +++ b/crates/sign/src/pdf/changes.rs @@ -7,7 +7,11 @@ pub(super) fn classify_changes(doc: &Document, old: Option, p: Option< let Some(old) = old else { return Modification::Disallowed(vec![kind::UNREADABLE]); }; - let old_content: HashSet = pdfcraft_annot::page_refs(&old) + // Every role below is keyed by object number, not by the reference as written: a damaged + // file's slot accepts two generations (#1143), and a page signed as `3 5 R` and edited as + // `3 0 R` is still that page. The loop over changed objects refuses any slot whose accepted + // generations changed, so a number names the same slot in both revisions. + let old_content: HashSet = pdfcraft_annot::page_refs(&old) .unwrap_or_default() .into_iter() .flat_map(|pg| { @@ -21,24 +25,34 @@ pub(super) fn classify_changes(doc: &Document, old: Option, p: Option< _ => Vec::new(), } }) + .map(|r| r.num) .collect(); // The signed revision's document-level XMP stream. - let old_metadata = old.root().and_then(|root| old.get(root).as_dict().and_then(|c| c.get(b"Metadata").and_then(Object::as_ref))); + let old_metadata = old.root().and_then(|root| old.get(root).as_dict().and_then(|c| c.get(b"Metadata").and_then(Object::as_ref))).map(|r| r.num); // The signed revision's XFA data packets: filling the form rewrites them. let old_xfa_data = xfa_data_packets(&old); // What each object was in the signed revision: its pages, and its trailer /Info. A changed // object is judged by the role it held when signed, never by the labels or keys its new, // attacker-written version carries (relabelling a signed page /Type /Sig, or dropping its // /Type and adding /ModDate, must not reclassify it away from page content). - let signed_pages: HashSet = pdfcraft_annot::page_refs(&old).unwrap_or_default().into_iter().collect(); + let signed_page_refs = pdfcraft_annot::page_refs(&old).unwrap_or_default(); + let signed_pages: HashSet = signed_page_refs.iter().map(|r| r.num).collect(); // The signed pages' /Annots arrays that are objects of their own. Adding an annotation updates // such an array in place (the page dictionary does not change), so it is judged like a // page's inline /Annots: by the annotations it gained. - let signed_annots: HashSet = - signed_pages.iter().filter_map(|pg| old.get(*pg).as_dict().and_then(|d| d.get(b"Annots").and_then(Object::as_ref))).collect(); + let signed_annots: HashSet = signed_page_refs + .iter() + .filter_map(|pg| old.get(*pg).as_dict().and_then(|d| d.get(b"Annots").and_then(Object::as_ref))) + .map(|r| r.num) + .collect(); + // The annotations the signed pages listed. One of them listed again elsewhere was moved or + // copied there, not added. + let placed_annots: HashSet = + signed_page_refs.iter().flat_map(|pg| refs(&old, old.get(*pg).as_dict().and_then(|d| d.get(b"Annots")))).map(|r| r.num).collect(); // What each signed page draws from, by the role it held when signed (#1787). - let signed_resources = signed_page_resources(&old, &signed_pages); - let old_info = old.trailer().get(b"Info").and_then(Object::as_ref); + let signed_resources = signed_page_resources(&old, &signed_page_refs); + let old_info = old.trailer().get(b"Info").and_then(Object::as_ref).map(|r| r.num); + let signed_role = |num: u32| old_content.contains(&num) || signed_pages.contains(&num) || signed_resources.contains_key(&num); let (mut allowed, mut disallowed): (Vec<&'static str>, Vec<&'static str>) = (Vec::new(), Vec::new()); let dss = dss_parts(doc, &old); // Stored at the same place in both (and not edited since): the same bytes, unchanged. @@ -61,8 +75,7 @@ pub(super) fn classify_changes(doc: &Document, old: Option, p: Option< // while every comparison here finds the signed object untouched. No writer has a reason to // add such an entry after signing: it is never a permitted change. for (_, num) in doc.xref_oddities().iter().filter(|o| !old.xref_oddities().contains(o)) { - let r = ObjRef { num: *num, generation: doc.generation(*num) }; - let what = if old_content.contains(&r) || signed_pages.contains(&r) { kind::PAGE_CONTENT } else { kind::OTHER }; + let what = if old_content.contains(num) || signed_pages.contains(num) { kind::PAGE_CONTENT } else { kind::OTHER }; if !disallowed.contains(&what) { disallowed.push(what); } @@ -80,6 +93,7 @@ pub(super) fn classify_changes(doc: &Document, old: Option, p: Option< old_xfa_data: &old_xfa_data, signed_pages: &signed_pages, signed_annots: &signed_annots, + placed_annots: &placed_annots, signed_resources: &signed_resources, old_info, dss: &dss, @@ -88,6 +102,19 @@ pub(super) fn classify_changes(doc: &Document, old: Option, p: Option< if same_place(num) { continue; } + // The signed revision's references reached this slot by the generations it accepted. If + // it accepts others now, those references may reach nothing here while another reader + // follows them back through /Prev to an intermediate revision, or to another alias, and + // shows what that holds: the object compared below is not provably what is displayed. + // No writer changes a signed object's generation. + let signed_generations = old.accepted_generations(num); + if !signed_generations.is_empty() && signed_generations != doc.accepted_generations(num) { + let what = if signed_role(num) { kind::PAGE_CONTENT } else { kind::OTHER }; + if !disallowed.contains(&what) { + disallowed.push(what); + } + continue; + } let r = ObjRef { num, generation: doc.generation(num) }; let new = doc.get(r); let before = old.try_get(num).ok().filter(|o| !matches!(**o, Object::Null)); @@ -124,9 +151,8 @@ pub(super) fn classify_changes(doc: &Document, old: Option, p: Option< // rewritten by every save and is not a change. let current: HashSet = doc.object_numbers().into_iter().collect(); for num in old.object_numbers().into_iter().filter(|n| !current.contains(n)) { - let r = ObjRef { num, generation: old.generation(num) }; - let was = old.get(r); - let signed_role = old_content.contains(&r) || signed_pages.contains(&r) || signed_resources.contains_key(&r); + let was = old.get(ObjRef { num, generation: old.generation(num) }); + let signed_role = signed_role(num); // A signed page's content or resource is judged by that role, whatever /Type its stream carried. if !signed_role && matches!(&*was, Object::Stream(s) if matches!(s.dict.name(b"Type"), Some(b"XRef" | b"ObjStm"))) { continue; @@ -142,13 +168,14 @@ pub(super) fn classify_changes(doc: &Document, old: Option, p: Option< struct ChangedObjectClassifier<'a> { doc: &'a Document, old: &'a Document, - old_content: &'a HashSet, - old_metadata: Option, - old_xfa_data: &'a HashSet, - signed_pages: &'a HashSet, - signed_annots: &'a HashSet, - signed_resources: &'a HashMap, - old_info: Option, + old_content: &'a HashSet, + old_metadata: Option, + old_xfa_data: &'a HashSet, + signed_pages: &'a HashSet, + signed_annots: &'a HashSet, + placed_annots: &'a HashSet, + signed_resources: &'a HashMap, + old_info: Option, dss: &'a HashSet, } @@ -170,7 +197,7 @@ impl ChangedObjectClassifier<'_> { } fn resource_kind(&self, r: ObjRef, new: &Object, before: Option<&Object>) -> Option<&'static str> { - match (self.signed_resources.get(&r), new) { + match (self.signed_resources.get(&r.num), new) { (Some(false), _) => Some(kind::PAGE_CONTENT), (Some(true), Object::Dict(dict)) if self.keeps_previous_entries(dict, before) => None, (Some(true), _) => Some(kind::PAGE_CONTENT), @@ -183,32 +210,40 @@ impl ChangedObjectClassifier<'_> { } fn is_signed_annotation_array(&self, r: ObjRef, before: Option<&Object>) -> bool { - self.signed_annots.contains(&r) && before.is_some_and(|object| object.as_array().is_some()) + self.signed_annots.contains(&r.num) && before.is_some_and(|object| object.as_array().is_some()) } fn annotation_array_change(&self, now: &[Object], before: Option<&Object>) -> &'static str { - let was: HashSet = - before.and_then(Object::as_array).map(|items| items.iter().filter_map(Object::as_ref).collect()).unwrap_or_default(); - let kept: HashSet = now.iter().filter_map(Object::as_ref).collect(); - let removed: Vec = was.iter().filter(|reference| !kept.contains(reference)).copied().collect(); - if removed.is_empty() { - let added: Vec = now.iter().filter_map(Object::as_ref).filter(|reference| !was.contains(reference)).collect(); - added_annots(self.doc, &added) - } else if removed.iter().any(|reference| self.old.get(*reference).as_dict().is_some_and(|dict| dict.name(b"Subtype") == Some(b"Widget"))) { - // A form field or signature widget taken off the page: no permission covers it. + let was: Vec = before.and_then(Object::as_array).map(|items| items.iter().filter_map(Object::as_ref).collect()).unwrap_or_default(); + self.annots_change(&was, &now.iter().filter_map(Object::as_ref).collect::>()) + } + + /// A signed page's annotations going from `was` to `now`, compared by object number. + fn annots_change(&self, was: &[ObjRef], now: &[ObjRef]) -> &'static str { + let had: HashSet = was.iter().map(|r| r.num).collect(); + let kept: HashSet = now.iter().map(|r| r.num).collect(); + let removed: Vec = had.iter().filter(|num| !kept.contains(num)).copied().collect(); + let added: Vec = now.iter().filter(|r| !had.contains(&r.num)).copied().collect(); + let was_widget = |num: u32| self.old.try_get(num).ok().is_some_and(|o| o.as_dict().is_some_and(|d| d.name(b"Subtype") == Some(b"Widget"))); + // A signed annotation listed on another page was moved or copied there. + let moved: Vec = added.iter().map(|r| r.num).filter(|num| self.placed_annots.contains(num)).collect(); + if removed.iter().chain(&moved).any(|num| was_widget(*num)) { + // A form field or signature widget taken off its page, or onto another: no permission covers it. kind::OTHER - } else { - // Deleting comments is a comment change (permitted at P 3 only). + } else if !removed.is_empty() || !moved.is_empty() { + // Deleting or moving comments is a comment change (permitted at P 3 only). kind::COMMENTS + } else { + added_annots(self.doc, &added) } } fn dictionary_change(&self, r: ObjRef, dict: &Dict, before: Option<&Object>) -> &'static str { let previous = before.and_then(Object::as_dict); - if self.signed_pages.contains(&r) { + if self.signed_pages.contains(&r.num) { // A signed page is judged by what the page shows, whatever its new labels say. - page_change_kind(self.doc, self.old, dict, previous) - } else if previous.is_some() && Some(r) == self.old_info { + self.page_change_kind(dict, previous) + } else if previous.is_some() && Some(r.num) == self.old_info { // Only the signed revision's own /Info dictionary is a metadata change. kind::METADATA } else { @@ -216,17 +251,29 @@ impl ChangedObjectClassifier<'_> { } } + /// A changed object that was a page in the signed revision, judged by what the page shows + /// rather than by any `/Type` its new version carries. Only its annotations may change (see + /// [`Self::annots_change`]); any other difference (content, resources, a relabelled or dropped + /// `/Type`) is a page content change. + fn page_change_kind(&self, d: &Dict, before: Option<&Dict>) -> &'static str { + match before { + Some(b) if without(d, &[b"Annots"]) != without(b, &[b"Annots"]) => kind::PAGE_CONTENT, + Some(b) => self.annots_change(&refs(self.old, b.get(b"Annots")), &refs(self.doc, d.get(b"Annots"))), + None => kind::PAGE_CONTENT, + } + } + fn stream_change(&self, r: ObjRef, stream: &pdfcraft_cos::Stream, before: Option<&Object>) -> Option<&'static str> { // File plumbing is the stream's role when signed; a later /Type label cannot change it. let was_plumbing = matches!(before, Some(Object::Stream(previous)) if matches!(previous.dict.name(b"Type"), Some(b"XRef" | b"ObjStm"))); - if self.old_content.contains(&r) { + if self.old_content.contains(&r.num) { Some(kind::PAGE_CONTENT) } else if before.is_none() || was_plumbing { None - } else if stream.dict.name(b"Type") == Some(b"Metadata") && Some(r) == self.old_metadata { + } else if stream.dict.name(b"Type") == Some(b"Metadata") && Some(r.num) == self.old_metadata { // Only the signed catalog's XMP stream is metadata. Some(kind::METADATA) - } else if self.old_xfa_data.contains(&r) { + } else if self.old_xfa_data.contains(&r.num) { // Signed XFA data and state packets are changed by a form fill. Some(kind::FORM_FILL) } else { @@ -238,7 +285,7 @@ impl ChangedObjectClassifier<'_> { /// The streams of `doc`'s XFA form that hold its data and state: the `datasets` and `form` /// packets of the AcroForm's `/XFA` array (name, stream, name, stream, …). A form given as one /// XFA stream holds its template too, and is not listed. -fn xfa_data_packets(doc: &Document) -> HashSet { +fn xfa_data_packets(doc: &Document) -> HashSet { let Some(form) = acroform(doc) else { return HashSet::new() }; let Some(xfa) = form.get(b"XFA").map(|x| doc.resolve(x)) else { return HashSet::new() }; let Object::Array(xfa) = &*xfa else { return HashSet::new() }; @@ -247,6 +294,7 @@ fn xfa_data_packets(doc: &Document) -> HashSet { .iter() .filter(|[name, _]| doc.resolve(name).as_string().is_some_and(|s| matches!(s.to_text().as_str(), "datasets" | "form"))) .filter_map(|[_, packet]| packet.as_ref()) + .map(|r| r.num) .collect() } @@ -442,23 +490,24 @@ fn is_signature_dict(d: &Dict) -> bool { /// properties), where any change changes what the page shows. Resources are the page's own or /// inherited. Bounded by depth and by the number of objects visited; past the bound the rest is /// judged as before. -fn signed_page_resources(old: &Document, pages: &HashSet) -> HashMap { +fn signed_page_resources(old: &Document, pages: &[ObjRef]) -> HashMap { const CATEGORIES: [&[u8]; 7] = [b"Font", b"XObject", b"ExtGState", b"ColorSpace", b"Pattern", b"Shading", b"Properties"]; const MAX_VISITED: usize = 50_000; const MAX_DEPTH: u8 = 32; const MAX_INHERITANCE: usize = 64; /// Everything reachable from `o` is content the page draws (a leaf role, overriding a - /// container one reached by another path). A signed page or the catalog reached this way - /// (a `/P` or `/Parent` back-reference) keeps the role of its own: the walk stops there. - fn reach(old: &Document, o: &Object, depth: u8, stop: &HashSet, roles: &mut HashMap) { + /// container one reached by another path, under any generation alias). A signed page or the + /// catalog reached this way (a `/P` or `/Parent` back-reference) keeps the role of its own: + /// the walk stops there. + fn reach(old: &Document, o: &Object, depth: u8, stop: &HashSet, roles: &mut HashMap) { if roles.len() >= MAX_VISITED || depth > MAX_DEPTH { return; } let resolved; let o = match o { Object::Ref(r) => { - if stop.contains(r) || roles.insert(*r, false).is_some() { + if stop.contains(&r.num) || roles.insert(r.num, false).is_some() { return; } resolved = old.get(*r); @@ -490,14 +539,14 @@ fn signed_page_resources(old: &Document, pages: &HashSet) -> HashMap = pages.iter().map(|r| r.num).collect(); + stop.extend(old.root().map(|r| r.num)); for page in pages { let Some(res) = resources(old, *page) else { continue }; - let container = |o: &Object, roles: &mut HashMap| -> Arc { + let container = |o: &Object, roles: &mut HashMap| -> Arc { match o { Object::Ref(r) => { - roles.entry(*r).or_insert(true); + roles.entry(r.num).or_insert(true); old.get(*r) } other => Arc::new(other.clone()), @@ -516,21 +565,6 @@ fn signed_page_resources(old: &Document, pages: &HashSet) -> HashMap) -> &'static str { - match before { - Some(b) if without(d, &[b"Annots"]) != without(b, &[b"Annots"]) => kind::PAGE_CONTENT, - Some(b) => { - let was = refs(old, b.get(b"Annots")); - let added: Vec = refs(doc, d.get(b"Annots")).into_iter().filter(|r| !was.contains(r)).collect(); - added_annots(doc, &added) - } - None => kind::PAGE_CONTENT, - } -} - /// Whether the catalog `d` keeps every developer extension the signed catalog `b` declared, each /// with the same value: it may only declare more. fn extensions_kept(doc: &Document, old: &Document, d: &Dict, b: &Dict) -> bool { @@ -557,6 +591,14 @@ fn change_kind(doc: &Document, old: &Document, d: &Dict, before: Option<&Dict>) { return kind::PAGE_CONTENT; } + // A signed form field or widget keeps that role: its value, appearance or signature may + // change, but turning it into a comment, another kind of annotation or another type of + // field is no form fill. + if before.is_some_and(|b| { + (b.name(b"Subtype") == Some(b"Widget") || b.contains(b"FT")) && (d.name(b"Subtype") != b.name(b"Subtype") || d.name(b"FT") != b.name(b"FT")) + }) { + return kind::OTHER; + } match ty { // Only a new signature value or policy: rewriting a signed one is not a signature (#1788). Some(b"Sig" | b"DocTimeStamp" | b"SigRef" | b"TransformParams") if before.is_none() => return kind::SIGNATURE, diff --git a/crates/sign/src/pdf/revision_classification_tests.rs b/crates/sign/src/pdf/revision_classification_tests.rs new file mode 100644 index 0000000000..ed338c24de --- /dev/null +++ b/crates/sign/src/pdf/revision_classification_tests.rs @@ -0,0 +1,207 @@ +//! Changes made after signing are judged against what a reader resolves and by the role each +//! object held when signed: a newer generation, a repaired generation alias, or a widget +//! replaced, removed or moved never passes as a permitted change. + +use std::sync::Arc; + +use pdfcraft_cos::{Document, ObjRef, Object}; + +use super::Modification; +use super::changes::classify_changes; + +/// One cross-reference row: an object written with header generation `.0` under row generation `.1`. +enum Row<'a> { + Obj(u16, u16, &'a str), +} + +/// A classic-xref file built one revision at a time; each later revision points back with /Prev. +#[derive(Clone)] +struct File { + bytes: Vec, + size: u32, + xref: Option, +} + +impl File { + fn new() -> Self { + File { bytes: b"%PDF-1.7\n".to_vec(), size: 1, xref: None } + } + + fn revision(mut self, rows: &[(u32, Row<'_>)]) -> Self { + let mut table = Vec::new(); + for (num, row) in rows { + self.size = self.size.max(num + 1); + let Row::Obj(header, generation, body) = row; + table.push(format!("{num} 1\n{:010} {generation:05} n \n", self.bytes.len())); + self.bytes.extend_from_slice(format!("{num} {header} obj\n{body}\nendobj\n").as_bytes()); + } + let at = self.bytes.len(); + self.bytes.extend_from_slice(b"xref\n"); + if self.xref.is_none() { + self.bytes.extend_from_slice(b"0 1\n0000000000 65535 f \n"); + } + self.bytes.extend_from_slice(table.concat().as_bytes()); + let prev = self.xref.map(|p| format!(" /Prev {p}")).unwrap_or_default(); + self.bytes.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R{prev} >>\nstartxref\n{at}\n%%EOF\n", self.size).as_bytes()); + self.xref = Some(at); + self + } + + fn open(&self) -> Document { + Document::open(Arc::new(self.bytes.clone())).unwrap() + } +} + +const PAGES: &str = "<< /Type /Pages /Kids [3 0 R 4 0 R] /Count 2 >>"; +const TEXT_FIELD: &str = "<< /Type /Annot /Subtype /Widget /FT /Tx /T (Name) /V (Ann) /Rect [20 20 120 40] /P 3 0 R >>"; +const COMMENT: &str = "<< /Type /Annot /Subtype /Text /Rect [20 20 40 40] /Contents (note) >>"; + +fn page(extra: &str) -> String { + format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] {extra} >>") +} + +fn r(num: u32, generation: u16) -> ObjRef { + ObjRef { num, generation } +} + +fn assert_refused(doc: &Document, old: Document, p: u8) { + let verdict = classify_changes(doc, Some(old), Some(p)); + assert!(matches!(verdict, Modification::Disallowed(_)), "P={p}: main returns {verdict:?}"); +} + +/// Two pages and a text field on page 3, listed in /Fields and in the page's inline /Annots. +fn form(page3: &str) -> File { + File::new().revision(&[ + (1, Row::Obj(0, 0, "<< /Type /Catalog /Pages 2 0 R /AcroForm << /Fields [5 0 R] >> >>")), + (2, Row::Obj(0, 0, PAGES)), + (3, Row::Obj(0, 0, &page(page3))), + (4, Row::Obj(0, 0, &page("/Annots []"))), + (5, Row::Obj(0, 0, TEXT_FIELD)), + (6, Row::Obj(0, 0, "[5 0 R]")), + ]) +} + +#[test] +fn content_stream_intermediate_generation_cover() { + let signed = File::new().revision(&[ + (1, Row::Obj(0, 0, "<< /Type /Catalog /Pages 2 0 R >>")), + (2, Row::Obj(0, 0, PAGES)), + (3, Row::Obj(0, 0, &page(""))), + (4, Row::Obj(0, 0, &page("/Contents 5 0 R"))), + (5, Row::Obj(0, 0, "<< /Length 3 >>\nstream\nq Q\nendstream")), + ]); + let now = signed + .clone() + .revision(&[(5, Row::Obj(0, 0, "<< /Length 3 >>\nstream\nQ q\nendstream"))]) + .revision(&[(5, Row::Obj(1, 1, "<< /Length 3 >>\nstream\nq Q\nendstream"))]); + let (doc, old) = (now.open(), signed.open()); + assert_eq!(doc.revisions().len(), 3); + assert!(doc.xref_oddities().is_empty()); + assert_eq!(doc.get(r(5, 1)), old.get(r(5, 0)), "the newest generation holds the signed bytes"); + assert_refused(&doc, old, 2); +} + +#[test] +fn widget_removed_from_inline_annots() { + let signed = form("/Annots [5 0 R]"); + let mut doc = signed.open(); + doc.update_dict(r(3, 0), |p| p.set(b"Annots".to_vec(), Object::Array(Vec::new()))).unwrap(); + assert_refused(&doc, signed.open(), 3); +} + +#[test] +fn widget_moved_between_pages() { + let signed = form("/Annots [5 0 R]"); + let mut doc = signed.open(); + doc.update_dict(r(3, 0), |p| p.set(b"Annots".to_vec(), Object::Array(Vec::new()))).unwrap(); + doc.update_dict(r(4, 0), |p| p.set(b"Annots".to_vec(), Object::Array(vec![Object::Ref(r(5, 0))]))).unwrap(); + doc.update_dict(r(5, 0), |w| w.set(b"P".to_vec(), Object::Ref(r(4, 0)))).unwrap(); + assert_refused(&doc, signed.open(), 3); +} + +#[test] +fn annots_moved_to_new_object_dropping_widget() { + let signed = form("/Annots 6 0 R"); + let mut doc = signed.open(); + let empty = doc.add(Object::Array(Vec::new())); + doc.update_dict(r(3, 0), |p| p.set(b"Annots".to_vec(), Object::Ref(empty))).unwrap(); + assert_refused(&doc, signed.open(), 3); +} + +#[test] +fn repaired_page_relabelled_as_field() { + let signed = File::new().revision(&[ + (1, Row::Obj(0, 0, "<< /Type /Catalog /Pages 2 0 R >>")), + (2, Row::Obj(0, 0, "<< /Type /Pages /Kids [3 5 R 4 0 R] /Count 2 >>")), + (3, Row::Obj(5, 0, &page(""))), + (4, Row::Obj(0, 0, &page(""))), + ]); + let mut doc = signed.open(); + doc.update_dict(r(3, 0), |p| { + p.remove(b"Type"); + p.set(b"FT".to_vec(), Object::name("Tx")); + p.set(b"Rotate".to_vec(), Object::Int(180)); + }) + .unwrap(); + assert_refused(&doc, signed.open(), 2); +} + +#[test] +fn resource_container_and_drawn_alias() { + let signed = File::new().revision(&[ + (1, Row::Obj(0, 0, "<< /Type /Catalog /Pages 2 0 R >>")), + (2, Row::Obj(0, 0, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>")), + (3, Row::Obj(0, 0, &page("/Contents 6 0 R /Resources << /Font 4 0 R /ExtGState << /GS1 4 5 R >> >>"))), + (4, Row::Obj(5, 0, "<< /F1 5 0 R /CA 1 >>")), + (5, Row::Obj(0, 0, "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>")), + (6, Row::Obj(0, 0, "<< /Length 3 >>\nstream\nq Q\nendstream")), + ]); + let mut doc = signed.open(); + doc.update_dict(r(4, 0), |d| d.set(b"FT".to_vec(), Object::name("Tx"))).unwrap(); + assert_refused(&doc, signed.open(), 2); +} + +/// A field signed as `4 5 R` / `4 0 R` (header 5, row 0) that a later revision rewrites. +fn aliased_field(fields: &str, annots: &str) -> File { + File::new().revision(&[ + (1, Row::Obj(0, 0, &format!("<< /Type /Catalog /Pages 2 0 R /AcroForm << /Fields [{fields}] >> >>"))), + (2, Row::Obj(0, 0, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>")), + (3, Row::Obj(0, 0, &page(&format!("/Annots [{annots}]")))), + (4, Row::Obj(5, 0, TEXT_FIELD)), + ]) +} + +fn set_annots(doc: &mut Document, annot: ObjRef) { + doc.update_dict(r(3, 0), |p| p.set(b"Annots".to_vec(), Object::Array(vec![Object::Ref(annot)]))).unwrap(); +} + +#[test] +fn repaired_alias_set_changes_widget_to_comment() { + let signed = aliased_field("4 0 R", "4 0 R"); + let now = signed.clone().revision(&[(4, Row::Obj(7, 0, COMMENT))]); + let mut doc = now.open(); + set_annots(&mut doc, r(4, 7)); + assert_refused(&doc, signed.open(), 3); +} + +#[test] +fn repaired_slot_becomes_disjoint_generation() { + let signed = aliased_field("4 0 R", "4 0 R"); + let now = signed.clone().revision(&[(4, Row::Obj(7, 7, COMMENT))]); + let mut doc = now.open(); + set_annots(&mut doc, r(4, 7)); + assert_refused(&doc, signed.open(), 3); +} + +#[test] +fn hidden_shared_alias_form_fill() { + let signed = aliased_field("4 5 R", "4 0 R"); + let filled = TEXT_FIELD.replace("(Ann)", "(Bob)"); + let now = signed.clone().revision(&[(4, Row::Obj(5, 1, &filled))]); + let mut doc = now.open(); + set_annots(&mut doc, r(4, 1)); + // At P=3 a relisted annotation is permitted whatever it is: only the generations refuse it. + for p in [2, 3] { + assert_refused(&doc, signed.open(), p); + } +} diff --git a/crates/sign/tests/it/main.rs b/crates/sign/tests/it/main.rs index d526b01e7c..791b2b503b 100644 --- a/crates/sign/tests/it/main.rs +++ b/crates/sign/tests/it/main.rs @@ -7,4 +7,5 @@ mod cms; mod crypto; mod encrypted_signing; mod pdf; +mod revision_classification; mod store_keys; diff --git a/crates/sign/tests/it/revision_classification.rs b/crates/sign/tests/it/revision_classification.rs new file mode 100644 index 0000000000..9d36c3f57e --- /dev/null +++ b/crates/sign/tests/it/revision_classification.rs @@ -0,0 +1,158 @@ +//! A certification signature stays valid only while what a reader shows is what was signed: +//! revisions appended by hand that reach back to an older generation, or that replace a signed +//! form field in place, are refused. + +use std::sync::Arc; + +use pdfcraft_cos::{Document, Object}; +use pdfcraft_sign::{Modification, SignOptions, Status, TrustStore, pkcs12, signatures}; + +fn data(name: &str) -> Vec { + std::fs::read(format!("{}/tests/data/{name}", env!("CARGO_MANIFEST_DIR"))).unwrap() +} + +fn open(b: &[u8]) -> Document { + Document::open(Arc::new(b.to_vec())).unwrap() +} + +/// A classic-xref PDF of `objs`, numbered from 1, with object 1 as the catalog. +fn write_pdf(objs: &[&str]) -> Vec { + let mut out = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (i, o) in objs.iter().enumerate() { + offsets.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n{o}\nendobj\n", i + 1).as_bytes()); + } + let xref = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes()); + for o in offsets { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n", objs.len() + 1).as_bytes()); + out +} + +/// One row of a hand-written revision: `num header obj body` under row generation `row`, or a +/// free row with its next generation. +enum Row<'a> { + Obj { num: u32, header: u16, row: u16, body: &'a str }, + Free { num: u32, next: u16 }, +} + +/// `base` with one more revision of `rows`, written as a classic table pointing back with /Prev. +fn append(base: &[u8], rows: &[Row<'_>]) -> Vec { + let doc = open(base); + let size = doc.trailer().get(b"Size").and_then(Object::as_int).unwrap(); + let root = doc.root().unwrap(); + let tail = String::from_utf8_lossy(&base[base.len().saturating_sub(64)..]).into_owned(); + let prev: usize = tail.rsplit("startxref").next().unwrap().split_whitespace().next().unwrap().parse().unwrap(); + let mut out = base.to_vec(); + let mut table = String::new(); + for row in rows { + match row { + Row::Obj { num, header, row, body } => { + table.push_str(&format!("{num} 1\n{:010} {row:05} n \n", out.len())); + out.extend_from_slice(format!("{num} {header} obj\n{body}\nendobj\n").as_bytes()); + } + Row::Free { num, next } => table.push_str(&format!("{num} 1\n0000000000 {next:05} f \n")), + } + } + let xref = out.len(); + out.extend_from_slice( + format!("xref\n{table}trailer\n<< /Size {size} /Root {} {} R /Prev {prev} >>\nstartxref\n{xref}\n%%EOF\n", root.num, root.generation) + .as_bytes(), + ); + out +} + +fn opts(certify: u8) -> SignOptions { + SignOptions { + page: 0, + rect: Some([20.0, 100.0, 220.0, 150.0]), + date: "D:20261002120000+01'00'".into(), + certify: Some(certify), + ..SignOptions::default() + } +} + +/// The certification signature of `bytes`, which must be refused. +fn assert_refused(bytes: &[u8]) { + let id = pkcs12::open(&data("rsa-aes.p12"), "test").unwrap(); + let trust = TrustStore { certs: vec![id.certificate.clone()] }; + let s = signatures(&open(bytes), bytes, &trust).into_iter().find(|s| s.signed).unwrap(); + assert!(matches!(s.modification, Modification::Disallowed(_)) && s.status != Status::Valid, "main returns {:?}, {:?}", s.status, s.modification); +} + +fn certify(unsigned: &[u8], p: u8) -> Vec { + let id = pkcs12::open(&data("rsa-aes.p12"), "test").unwrap(); + pdfcraft_sign::sign(&open(unsigned), &id, &opts(p)).unwrap() +} + +const PAGE2: &str = "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 7 0 R /Resources << /Font << /F1 6 0 R >> >> >>"; + +/// Revision 2 rotates page 2 (`4 0 obj`); revision 3 writes the signed page back as `4 1 obj`. +/// The page tree still says `4 0 R`. +fn intermediate_generation_cover(p: u8) { + let unsigned = write_pdf(&[ + "<< /Type /Catalog /Pages 2 0 R >>", + "<< /Type /Pages /Kids [3 0 R 4 0 R] /Count 2 >>", + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 5 0 R /Resources << /Font << /F1 6 0 R >> >> >>", + PAGE2, + "<< /Length 41 >>\nstream\nBT /F1 14 Tf 20 250 Td (Page one) Tj ET\nendstream", + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + "<< /Length 41 >>\nstream\nBT /F1 14 Tf 20 250 Td (Page two) Tj ET\nendstream", + ]); + let signed = certify(&unsigned, p); + let rotated = PAGE2.replace(" >> >> >>", " >> >> /Rotate 180 >>"); + let two = append(&signed, &[Row::Obj { num: 4, header: 0, row: 0, body: &rotated }]); + let three = append(&two, &[Row::Obj { num: 4, header: 1, row: 1, body: PAGE2 }]); + let doc = open(&three); + assert!(doc.xref_oddities().is_empty()); + assert_eq!(doc.revisions().len(), open(&signed).revisions().len() + 2); + assert_eq!(doc.get(pdfcraft_cos::ObjRef { num: 4, generation: 1 }), open(&signed).get(pdfcraft_cos::ObjRef { num: 4, generation: 0 })); + assert_refused(&three); +} + +#[test] +fn intermediate_generation_cover_p1() { + intermediate_generation_cover(1); +} + +#[test] +fn intermediate_generation_cover_p2() { + intermediate_generation_cover(2); +} + +#[test] +fn intermediate_generation_cover_p3() { + intermediate_generation_cover(3); +} + +const COMMENT: &str = "<< /Type /Annot /Subtype /Text /Rect [20 200 40 220] /Contents (note) >>"; + +/// A page with text field `4 0 R`, listed in /Fields and in the page's inline /Annots, certified +/// allowing comments. +fn certified_form() -> Vec { + certify( + &write_pdf(&[ + "<< /Type /Catalog /Pages 2 0 R /AcroForm << /Fields [4 0 R] >> >>", + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Annots [4 0 R] >>", + "<< /Type /Annot /Subtype /Widget /FT /Tx /T (Name) /V (Ann) /Rect [20 200 120 220] /P 3 0 R >>", + ]), + 3, + ) +} + +#[test] +fn widget_replaced_in_place_by_comment() { + let signed = certified_form(); + assert_refused(&append(&signed, &[Row::Obj { num: 4, header: 0, row: 0, body: COMMENT }])); +} + +#[test] +fn widget_freed_then_reused_as_comment() { + let signed = certified_form(); + let freed = append(&signed, &[Row::Free { num: 4, next: 1 }]); + assert_refused(&append(&freed, &[Row::Obj { num: 4, header: 0, row: 0, body: COMMENT }])); +} From de344a4bb1dd02bad3a9cffd13cf69a5ac73292b Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 5 Nov 2026 22:44:26 +0000 Subject: [PATCH 132/198] sign: compare annotation lists by what resolves, within a bound An annotation list is compared by the entries that resolve in each revision, and an unchanged list is no change. Expanding annotation lists is bounded per classification. A damaged slot rewritten with fewer accepted generations is the same slot when nothing signed refers to a generation it gave up. Co-Authored-By: Claude Opus 5.5 --- crates/sign/src/pdf/changes.rs | 142 +++++++++++++++--- .../src/pdf/revision_classification_tests.rs | 77 ++++++++++ 2 files changed, 196 insertions(+), 23 deletions(-) diff --git a/crates/sign/src/pdf/changes.rs b/crates/sign/src/pdf/changes.rs index 81467542e6..ef4b8868c6 100644 --- a/crates/sign/src/pdf/changes.rs +++ b/crates/sign/src/pdf/changes.rs @@ -1,5 +1,11 @@ use super::types::{Modification, kind}; use super::{Arc, Dict, Document, HashMap, HashSet, ObjRef, Object}; +use std::cell::{Cell, OnceCell}; + +/// The most annotation-list entries one classification expands. Pages sharing one long /Annots +/// array, or many pages rewritten to list one, would otherwise cost pages × entries; past this +/// the lists are not compared, and the change is refused. +const MAX_ANNOT_ENTRIES: usize = 1 << 22; /// What later revisions changed, classified as Acrobat reports it, under DocMDP `p` (or none: /// an approval signature permits form fill, comments and further signatures). @@ -47,13 +53,31 @@ pub(super) fn classify_changes(doc: &Document, old: Option, p: Option< .collect(); // The annotations the signed pages listed. One of them listed again elsewhere was moved or // copied there, not added. - let placed_annots: HashSet = - signed_page_refs.iter().flat_map(|pg| refs(&old, old.get(*pg).as_dict().and_then(|d| d.get(b"Annots")))).map(|r| r.num).collect(); + let annot_budget = Cell::new(MAX_ANNOT_ENTRIES); + let (mut placed_annots, mut expanded, mut annots_unbounded) = (HashSet::new(), HashSet::new(), false); + for pg in &signed_page_refs { + let page = old.get(*pg); + let Some(list) = page.as_dict().and_then(|d| d.get(b"Annots")) else { continue }; + // Pages sharing one array object list the same annotations. + if list.as_ref().is_some_and(|r| !expanded.insert(r.num)) { + continue; + } + let Some(found) = annot_refs(&old, Some(list), &annot_budget) else { + annots_unbounded = true; + break; + }; + placed_annots.extend(found.into_iter().map(|r| r.num)); + } // What each signed page draws from, by the role it held when signed (#1787). let signed_resources = signed_page_resources(&old, &signed_page_refs); let old_info = old.trailer().get(b"Info").and_then(Object::as_ref).map(|r| r.num); let signed_role = |num: u32| old_content.contains(&num) || signed_pages.contains(&num) || signed_resources.contains_key(&num); let (mut allowed, mut disallowed): (Vec<&'static str>, Vec<&'static str>) = (Vec::new(), Vec::new()); + if annots_unbounded { + disallowed.push(kind::OTHER); + } + // Every reference the signed revision makes, read only when a slot's generations narrowed. + let signed_references = OnceCell::new(); let dss = dss_parts(doc, &old); // Stored at the same place in both (and not edited since): the same bytes, unchanged. let same_place = |num: u32| -> bool { @@ -94,6 +118,7 @@ pub(super) fn classify_changes(doc: &Document, old: Option, p: Option< signed_pages: &signed_pages, signed_annots: &signed_annots, placed_annots: &placed_annots, + annot_budget: &annot_budget, signed_resources: &signed_resources, old_info, dss: &dss, @@ -106,9 +131,19 @@ pub(super) fn classify_changes(doc: &Document, old: Option, p: Option< // it accepts others now, those references may reach nothing here while another reader // follows them back through /Prev to an intermediate revision, or to another alias, and // shows what that holds: the object compared below is not provably what is displayed. - // No writer changes a signed object's generation. + // No writer changes a signed object's generation. A damaged slot rewritten cleanly + // accepts fewer: that is the same slot when nothing signed refers to it by a generation + // it gave up. let signed_generations = old.accepted_generations(num); - if !signed_generations.is_empty() && signed_generations != doc.accepted_generations(num) { + let generations = doc.accepted_generations(num); + let narrowed_unreferenced = || { + !generations.is_empty() + && generations.iter().all(|g| signed_generations.contains(g)) + && signed_references.get_or_init(|| references(&old)).as_ref().is_some_and(|found: &HashSet| { + signed_generations.iter().filter(|g| !generations.contains(g)).all(|g| !found.contains(&ObjRef { num, generation: *g })) + }) + }; + if !signed_generations.is_empty() && signed_generations != generations && !narrowed_unreferenced() { let what = if signed_role(num) { kind::PAGE_CONTENT } else { kind::OTHER }; if !disallowed.contains(&what) { disallowed.push(what); @@ -174,6 +209,7 @@ struct ChangedObjectClassifier<'a> { signed_pages: &'a HashSet, signed_annots: &'a HashSet, placed_annots: &'a HashSet, + annot_budget: &'a Cell, signed_resources: &'a HashMap, old_info: Option, dss: &'a HashSet, @@ -188,8 +224,8 @@ impl ChangedObjectClassifier<'_> { return Some(kind::SECURITY_STORE); } match new { - Object::Array(now) if self.is_signed_annotation_array(r, before) => Some(self.annotation_array_change(now, before)), - Object::Dict(dict) => Some(self.dictionary_change(r, dict, before)), + Object::Array(_) if self.is_signed_annotation_array(r, before) => self.annotation_array_change(new, before), + Object::Dict(dict) => self.dictionary_change(r, dict, before), Object::Stream(stream) => self.stream_change(r, stream, before), _ if before.is_none() => None, _ => Some(kind::OTHER), @@ -213,13 +249,21 @@ impl ChangedObjectClassifier<'_> { self.signed_annots.contains(&r.num) && before.is_some_and(|object| object.as_array().is_some()) } - fn annotation_array_change(&self, now: &[Object], before: Option<&Object>) -> &'static str { - let was: Vec = before.and_then(Object::as_array).map(|items| items.iter().filter_map(Object::as_ref).collect()).unwrap_or_default(); - self.annots_change(&was, &now.iter().filter_map(Object::as_ref).collect::>()) + fn annotation_array_change(&self, now: &Object, before: Option<&Object>) -> Option<&'static str> { + match (annot_refs(self.old, before, self.annot_budget), annot_refs(self.doc, Some(now), self.annot_budget)) { + (Some(was), Some(now)) => self.annots_change(&was, &now), + _ => Some(kind::OTHER), + } } - /// A signed page's annotations going from `was` to `now`, compared by object number. - fn annots_change(&self, was: &[ObjRef], now: &[ObjRef]) -> &'static str { + /// A signed page's annotations going from `was` to `now`, compared by object number. Only an + /// entry that resolves is on the page: one rewritten to a generation its slot does not accept + /// draws nothing, whatever its number. `None` when the page shows the same annotations in + /// the same order. + fn annots_change(&self, was: &[ObjRef], now: &[ObjRef]) -> Option<&'static str> { + let live = + |doc: &Document, list: &[ObjRef]| -> Vec { list.iter().filter(|r| !matches!(*doc.get(**r), Object::Null)).copied().collect() }; + let (was, now) = (live(self.old, was), live(self.doc, now)); let had: HashSet = was.iter().map(|r| r.num).collect(); let kept: HashSet = now.iter().map(|r| r.num).collect(); let removed: Vec = had.iter().filter(|num| !kept.contains(num)).copied().collect(); @@ -227,27 +271,31 @@ impl ChangedObjectClassifier<'_> { let was_widget = |num: u32| self.old.try_get(num).ok().is_some_and(|o| o.as_dict().is_some_and(|d| d.name(b"Subtype") == Some(b"Widget"))); // A signed annotation listed on another page was moved or copied there. let moved: Vec = added.iter().map(|r| r.num).filter(|num| self.placed_annots.contains(num)).collect(); + // The annotations kept, in their new order: another order draws another one on top. + let reordered = !now.iter().map(|r| r.num).filter(|num| had.contains(num)).eq(was.iter().map(|r| r.num)); if removed.iter().chain(&moved).any(|num| was_widget(*num)) { // A form field or signature widget taken off its page, or onto another: no permission covers it. - kind::OTHER - } else if !removed.is_empty() || !moved.is_empty() { - // Deleting or moving comments is a comment change (permitted at P 3 only). - kind::COMMENTS + Some(kind::OTHER) + } else if !removed.is_empty() || !moved.is_empty() || reordered { + // Deleting, moving or restacking comments is a comment change (permitted at P 3 only). + Some(kind::COMMENTS) + } else if added.is_empty() { + None } else { - added_annots(self.doc, &added) + Some(added_annots(self.doc, &added)) } } - fn dictionary_change(&self, r: ObjRef, dict: &Dict, before: Option<&Object>) -> &'static str { + fn dictionary_change(&self, r: ObjRef, dict: &Dict, before: Option<&Object>) -> Option<&'static str> { let previous = before.and_then(Object::as_dict); if self.signed_pages.contains(&r.num) { // A signed page is judged by what the page shows, whatever its new labels say. self.page_change_kind(dict, previous) } else if previous.is_some() && Some(r.num) == self.old_info { // Only the signed revision's own /Info dictionary is a metadata change. - kind::METADATA + Some(kind::METADATA) } else { - change_kind(self.doc, self.old, dict, previous) + Some(change_kind(self.doc, self.old, dict, previous)) } } @@ -255,11 +303,14 @@ impl ChangedObjectClassifier<'_> { /// rather than by any `/Type` its new version carries. Only its annotations may change (see /// [`Self::annots_change`]); any other difference (content, resources, a relabelled or dropped /// `/Type`) is a page content change. - fn page_change_kind(&self, d: &Dict, before: Option<&Dict>) -> &'static str { + fn page_change_kind(&self, d: &Dict, before: Option<&Dict>) -> Option<&'static str> { match before { - Some(b) if without(d, &[b"Annots"]) != without(b, &[b"Annots"]) => kind::PAGE_CONTENT, - Some(b) => self.annots_change(&refs(self.old, b.get(b"Annots")), &refs(self.doc, d.get(b"Annots"))), - None => kind::PAGE_CONTENT, + Some(b) if without(d, &[b"Annots"]) != without(b, &[b"Annots"]) => Some(kind::PAGE_CONTENT), + Some(b) => match (annot_refs(self.old, b.get(b"Annots"), self.annot_budget), annot_refs(self.doc, d.get(b"Annots"), self.annot_budget)) { + (Some(was), Some(now)) => self.annots_change(&was, &now), + _ => Some(kind::OTHER), + }, + None => Some(kind::PAGE_CONTENT), } } @@ -439,6 +490,51 @@ fn added_annots(doc: &Document, added: &[ObjRef]) -> &'static str { } } +/// The references in the annotation list `o` (inline or indirect), its length spent from +/// `budget`. `None` past the budget. +fn annot_refs(doc: &Document, o: Option<&Object>, budget: &Cell) -> Option> { + let held; + let items = match o { + Some(Object::Ref(_)) => { + held = o.map(|o| doc.resolve(o)); + held.as_deref().and_then(Object::as_array) + } + other => other.and_then(Object::as_array), + }; + let Some(items) = items else { return Some(Vec::new()) }; + budget.set(budget.get().checked_sub(items.len())?); + Some(items.iter().filter_map(Object::as_ref).collect()) +} + +/// Every indirect reference in `doc`'s objects and trailer. `None` (nothing can be shown +/// unreferenced) when an object can't be read, or past a bound on references or nesting. +fn references(doc: &Document) -> Option> { + const MAX_REFERENCES: usize = 1 << 22; + const MAX_DEPTH: u8 = 64; + fn walk(o: &Object, depth: u8, found: &mut HashSet) -> bool { + if depth > MAX_DEPTH { + return false; + } + match o { + Object::Ref(r) => { + found.insert(*r); + found.len() <= MAX_REFERENCES + } + Object::Dict(d) => d.iter().all(|(_, v)| walk(v, depth + 1, found)), + Object::Stream(s) => s.dict.iter().all(|(_, v)| walk(v, depth + 1, found)), + Object::Array(a) => a.iter().all(|v| walk(v, depth + 1, found)), + _ => true, + } + } + let mut found = HashSet::new(); + for num in doc.object_numbers() { + if !walk(&*doc.try_get(num).ok()?, 0, &mut found) { + return None; + } + } + doc.trailer().iter().all(|(_, v)| walk(v, 0, &mut found)).then_some(found) +} + fn refs(doc: &Document, o: Option<&Object>) -> Vec { o.map(|o| doc.resolve(o)).and_then(|a| a.as_array().map(|a| a.iter().filter_map(Object::as_ref).collect())).unwrap_or_default() } diff --git a/crates/sign/src/pdf/revision_classification_tests.rs b/crates/sign/src/pdf/revision_classification_tests.rs index ed338c24de..b5e72fcae0 100644 --- a/crates/sign/src/pdf/revision_classification_tests.rs +++ b/crates/sign/src/pdf/revision_classification_tests.rs @@ -205,3 +205,80 @@ fn hidden_shared_alias_form_fill() { assert_refused(&doc, signed.open(), p); } } + +fn verdict(doc: &Document, old: Document, p: Option) -> Modification { + classify_changes(doc, Some(old), p) +} + +/// A widget whose /Annots entry is rewritten to a generation its slot never had no longer +/// resolves: it is taken off the page, inline or through an indirect array. +#[test] +fn widget_dropped_through_a_dead_generation() { + for (page3, num) in [("/Annots 6 0 R", 6), ("/Annots [5 0 R]", 3)] { + let signed = form(page3); + let body = if num == 6 { "[5 3 R]".to_owned() } else { page("/Annots [5 3 R]") }; + let now = signed.clone().revision(&[(num, Row::Obj(0, 0, &body))]); + for p in [Some(2), Some(3), None] { + let v = verdict(&now.open(), signed.open(), p); + assert!(matches!(v, Modification::Disallowed(_)), "{page3} P={p:?}: {v:?}"); + } + } +} + +/// The same annotations, listed in a new array object, change nothing on the page; listed in +/// another order they change which is drawn on top. +#[test] +fn an_unchanged_annotation_list_is_no_change_and_a_reordered_one_is_a_comment_change() { + let signed = File::new().revision(&[ + (1, Row::Obj(0, 0, "<< /Type /Catalog /Pages 2 0 R >>")), + (2, Row::Obj(0, 0, "<< /Type /Pages /Kids [3 0 R] /Count 1 >>")), + (3, Row::Obj(0, 0, &page("/Annots [4 0 R 5 0 R]"))), + (4, Row::Obj(0, 0, COMMENT)), + (5, Row::Obj(0, 0, COMMENT)), + ]); + let moved = signed.clone().revision(&[(3, Row::Obj(0, 0, &page("/Annots 6 0 R"))), (6, Row::Obj(0, 0, "[4 0 R 5 0 R]"))]); + assert_eq!(verdict(&moved.open(), signed.open(), Some(2)), Modification::Allowed(Vec::new())); + let reordered = signed.clone().revision(&[(3, Row::Obj(0, 0, &page("/Annots [5 0 R 4 0 R]")))]); + assert_eq!(verdict(&reordered.open(), signed.open(), Some(2)), Modification::Disallowed(vec![super::kind::COMMENTS])); +} + +/// Many signed pages sharing one long /Annots array, all rewritten to list it through a new +/// object, are judged in bounded time. +#[test] +fn shared_annotation_arrays_are_judged_in_bounded_time() { + const PAGES: u32 = 30_000; + let first_page = 10; + let kids: String = (0..PAGES).map(|i| format!("{} 0 R ", first_page + i)).collect(); + let annots: String = (0..PAGES).map(|i| format!("{} 0 R ", first_page + i)).collect(); + let mut rows = vec![ + (1, "<< /Type /Catalog /Pages 2 0 R >>".to_owned()), + (2, format!("<< /Type /Pages /Kids [{kids}] /Count {PAGES} >>")), + (3, format!("[{annots}]")), + ]; + rows.extend((0..PAGES).map(|i| (first_page + i, page("/Annots 3 0 R")))); + let rows: Vec<(u32, Row<'_>)> = rows.iter().map(|(n, b)| (*n, Row::Obj(0, 0, b))).collect(); + let signed = File::new().revision(&rows); + let repointed: Vec = (0..PAGES).map(|_| page("/Annots 4 0 R")).collect(); + let mut later: Vec<(u32, Row<'_>)> = repointed.iter().enumerate().map(|(i, b)| (first_page + i as u32, Row::Obj(0, 0, b))).collect(); + let copy = format!("[{annots}]"); + later.push((4, Row::Obj(0, 0, ©))); + let now = signed.clone().revision(&later); + let (doc, old) = (now.open(), signed.open()); + let started = std::time::Instant::now(); + let _ = verdict(&doc, old, Some(3)); + assert!(started.elapsed() < std::time::Duration::from_secs(20), "took {:?}", started.elapsed()); +} + +/// A damaged signed slot (header 5, row 0) rewritten cleanly as `4 0 obj` keeps the generation +/// every signed reference uses: a form fill. Had anything signed referred to it as `4 5 R`, that +/// reference would no longer resolve, and the change is refused. +#[test] +fn a_repaired_slot_rewritten_cleanly_is_judged_by_its_references() { + let filled = TEXT_FIELD.replace("(Ann)", "(Bob)"); + let signed = aliased_field("4 0 R", "4 0 R"); + let now = signed.clone().revision(&[(4, Row::Obj(0, 0, &filled))]); + assert_eq!(verdict(&now.open(), signed.open(), Some(2)), Modification::Allowed(vec![super::kind::FORM_FILL])); + let signed = aliased_field("4 5 R", "4 0 R"); + let now = signed.clone().revision(&[(4, Row::Obj(0, 0, &filled))]); + assert!(matches!(verdict(&now.open(), signed.open(), Some(2)), Modification::Disallowed(_))); +} From e1a3c033aaf749a758010bd1c9342c426eee2b39 Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Fri, 6 Nov 2026 00:19:58 +0000 Subject: [PATCH 133/198] sign: under no-changes certification, only invisible document timestamps may follow DocMDP P=1 allowed any change classified as a signature. A signature added after signing now breaks the certification unless it is a document timestamp drawing nothing (no positive-area /Rect, or Hidden/NoView). The added objects are judged directly, so a widget with no /T or one only in a page's /Annots is covered. The document security store and a genuine timestamp still pass. Co-Authored-By: Claude Opus 4.8 --- crates/sign/src/pdf/changes.rs | 51 ++++++++++++++++++++++++ crates/sign/tests/it/pdf.rs | 71 ++++++++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+) diff --git a/crates/sign/src/pdf/changes.rs b/crates/sign/src/pdf/changes.rs index 3a10a74e41..3f77fd0554 100644 --- a/crates/sign/src/pdf/changes.rs +++ b/crates/sign/src/pdf/changes.rs @@ -136,9 +136,60 @@ pub(super) fn classify_changes(doc: &Document, old: Option, p: Option< disallowed.push(what); } } + // DocMDP P=1 ("no changes allowed", ISO 32000-2 §12.8.2.2) permits only validation data to + // follow: the document security store, and document timestamps added for PAdES long-term + // archiving (§12.8.5). Those timestamps draw nothing. Any other signature added since — an + // approval signature, an unsigned signature widget, or a widget that paints over the signed + // page — is a change the certification forbids, however it is labelled. + if p == Some(1) && allowed.contains(&kind::SIGNATURE) && !only_invisible_timestamps_added(doc, &old) { + allowed.retain(|k| *k != kind::SIGNATURE); + if !disallowed.contains(&kind::SIGNATURE) { + disallowed.push(kind::SIGNATURE); + } + } if disallowed.is_empty() { Modification::Allowed(allowed) } else { Modification::Disallowed(disallowed) } } +/// Whether every signature object `doc` gained or changed since the signed revision `old` is an +/// invisible document timestamp. A signature annotation (a widget, or a dictionary carrying a +/// signature value or role) that the signed revision did not hold byte for byte must be a +/// `/SubFilter /ETSI.RFC3161` value in a widget that shows nothing — no positive-area `/Rect`, +/// or the Hidden or NoView flag set. An approval signature, an empty signature widget, or a +/// visible one fails, whatever `/Type` or `/SubFilter` it claims. Judged on the objects +/// themselves, not on the `/AcroForm /Fields` listing, so a widget with no `/T` or one reached +/// only through a page's `/Annots` is covered too. +fn only_invisible_timestamps_added(doc: &Document, old: &Document) -> bool { + /// A widget shows nothing: no `/Rect` enclosing a positive area, or Hidden (2) / NoView (32). + fn invisible(doc: &Document, w: &Dict) -> bool { + let flags = w.get(b"F").and_then(|o| doc.resolve(o).as_int()).unwrap_or(0); + let area = + super::listing::nums(doc, w, b"Rect").filter(|r| r.len() == 4).is_some_and(|r| (r[0] - r[2]).abs() > 0.5 && (r[1] - r[3]).abs() > 0.5); + !area || flags & (2 | 32) != 0 + } + for num in doc.object_numbers() { + let new = doc.get(ObjRef { num, generation: doc.generation(num) }); + let Some(d) = new.as_dict() else { continue }; + // A signature object: a signature value, or a field or widget that carries one. + let is_value = matches!(d.name(b"Type"), Some(b"Sig" | b"DocTimeStamp")); + let is_field = d.name(b"FT") == Some(b"Sig") || d.name(b"Subtype") == Some(b"Widget"); + if !is_value && !is_field { + continue; + } + // Unchanged since signing: the certification already covered it. + if old.try_get(num).ok().filter(|o| !matches!(**o, Object::Null)).as_deref() == Some(&*new) { + continue; + } + let value = if is_value { Some(d.clone()) } else { d.get(b"V").map(|v| doc.resolve(v)).and_then(|v| v.as_dict().cloned()) }; + // A new or changed signature object passes only as an invisible document timestamp. A + // widget with no value (an empty signature field placed on the page) never does. + let timestamp = value.as_ref().is_some_and(super::validate::is_doc_timestamp); + if !timestamp || (is_field && !invisible(doc, d)) { + return false; + } + } + true +} + struct ChangedObjectClassifier<'a> { doc: &'a Document, old: &'a Document, diff --git a/crates/sign/tests/it/pdf.rs b/crates/sign/tests/it/pdf.rs index 869a4bebd5..57efadbaf3 100644 --- a/crates/sign/tests/it/pdf.rs +++ b/crates/sign/tests/it/pdf.rs @@ -351,6 +351,77 @@ fn later_changes_are_classified_under_the_signature_permissions() { assert_eq!((s.certify, s.status), (Some(1), Status::Invalid), "{:?}", s.modification); } +/// Certify `fixture` with "no changes allowed" (DocMDP P=1). +fn certified_p1() -> (DigitalId, TrustStore, Vec) { + let id = pkcs12::open(&data("rsa-aes.p12"), "test").unwrap(); + let trust = TrustStore { certs: vec![id.certificate.clone()] }; + let signed = pdfcraft_sign::sign(&open(&fixture()), &id, &SignOptions { certify: Some(1), ..opts() }).unwrap(); + (id, trust, signed) +} + +/// Add a signature widget to page 3's /Annots (not to /Fields), with value `v` and rectangle +/// `rect`. +fn add_sig_widget(doc: &mut Document, v: Option<&[(&[u8], Object)]>, rect: [i64; 4]) { + let mut w = Dict::new(); + w.set(b"Type".to_vec(), Object::name("Annot")); + w.set(b"Subtype".to_vec(), Object::name("Widget")); + w.set(b"FT".to_vec(), Object::name("Sig")); + w.set(b"Rect".to_vec(), Object::Array(rect.map(Object::Int).to_vec())); + w.set(b"P".to_vec(), Object::Ref(ObjRef { num: 3, generation: 0 })); + if let Some(entries) = v { + let mut value = Dict::new(); + for (k, o) in entries { + value.set(k.to_vec(), o.clone()); + } + let vr = doc.add(Object::Dict(value)); + w.set(b"V".to_vec(), Object::Ref(vr)); + } + let wr = doc.add(Object::Dict(w)); + doc.update_dict(ObjRef { num: 3, generation: 0 }, |pg| { + let mut a = pg.get(b"Annots").and_then(Object::as_array).cloned().unwrap_or_default(); + a.push(Object::Ref(wr)); + pg.set(b"Annots".to_vec(), Object::Array(a)); + }) + .unwrap(); +} + +/// Under P=1, a signature added after signing breaks the certification unless it is an invisible +/// document timestamp. A visible overlay, an unsigned widget, an approval signature and a visible +/// widget wearing a timestamp /SubFilter each break it; a genuine document timestamp does not. +#[test] +fn only_an_invisible_document_timestamp_survives_a_no_changes_certification() { + let (_, trust, signed) = certified_p1(); + let check = |bytes: &[u8]| signatures(&open(bytes), bytes, &trust).into_iter().find(|s| s.signed && s.certify == Some(1)).unwrap(); + assert_eq!(check(&signed).status, Status::Valid, "{:?}", check(&signed).details); + let approval: &[(&[u8], Object)] = &[(b"Type", Object::name("Sig")), (b"SubFilter", Object::name("adbe.pkcs7.detached"))]; + let timestamp: &[(&[u8], Object)] = &[(b"Type", Object::name("DocTimeStamp")), (b"SubFilter", Object::name("ETSI.RFC3161"))]; + // Each of these breaks the certification: it is a change P=1 does not allow. + for (what, v, rect) in [ + ("visible overlay widget", None, [0, 0, 300, 300]), + ("invisible unsigned widget", None, [0, 0, 0, 0]), + ("visible approval signature", Some(approval), [0, 0, 300, 300]), + ("invisible approval signature", Some(approval), [0, 0, 0, 0]), + ("visible widget wearing a timestamp subfilter", Some(timestamp), [0, 0, 300, 300]), + ] { + let edited = edit_after(&signed, |doc| add_sig_widget(doc, v, rect)); + let s = check(&edited); + assert!(matches!(&s.modification, Modification::Disallowed(k) if k.contains(&kind::SIGNATURE)), "{what}: {:?}", s.modification); + assert_ne!(s.status, Status::Valid, "{what}"); + } +} + +/// A genuine document timestamp, which draws nothing, is validation data P=1 permits: the +/// certification stays valid. +#[test] +fn a_document_timestamp_keeps_a_no_changes_certification_valid() { + let (_, trust, signed) = certified_p1(); + let tsa = TestTsa { id: tsa_id(), time: Time { year: 2026, month: 10, day: 6, hour: 12, minute: 0, second: 0 } }; + let stamped = pdfcraft_sign::timestamp_document(&open(&signed), &tsa, "D:20261006120000Z").unwrap(); + let s = signatures(&open(&stamped), &stamped, &trust).into_iter().find(|s| s.certify == Some(1)).unwrap(); + assert_eq!(s.status, Status::Valid, "{:?}", s.details); + assert!(matches!(&s.modification, Modification::Allowed(k) if k.contains(&kind::SIGNATURE)), "{:?}", s.modification); +} + #[test] fn invisible_signatures_and_refusals() { let id = pkcs12::open(&data("ec-p256.p12"), "test").unwrap(); From ddeca878dff1ea6cc2fa55adb2db84513eecda1f Mon Sep 17 00:00:00 2001 From: Martin Ball Date: Thu, 5 Nov 2026 21:40:30 -0500 Subject: [PATCH 134/198] sign: judge the signed page tree, catalog and form by their signed roles Rebased onto 102cae3b/9afa1266 for landing (#6358). The signed page tree's nodes, a /Kids array of its own, the catalog by object number and an indirect AcroForm keep the roles they had when signed; a field dropped from the form is no permitted change at any level; a signed widget's /Rect or /F is page content. Co-Authored-By: Martin Ball --- crates/sign/src/pdf/changes.rs | 117 ++++++++++--- crates/sign/tests/it/main.rs | 1 + crates/sign/tests/it/signed_structure.rs | 199 +++++++++++++++++++++++ 3 files changed, 295 insertions(+), 22 deletions(-) create mode 100644 crates/sign/tests/it/signed_structure.rs diff --git a/crates/sign/src/pdf/changes.rs b/crates/sign/src/pdf/changes.rs index 03253b79fb..a44c08bb7c 100644 --- a/crates/sign/src/pdf/changes.rs +++ b/crates/sign/src/pdf/changes.rs @@ -68,14 +68,26 @@ pub(super) fn classify_changes(doc: &Document, old: Option, p: Option< }; placed_annots.extend(found.into_iter().map(|r| r.num)); } + // The signed page tree's nodes: a change to one changes the pages, whatever it is labelled + // now. Every node, and every /Kids array object of its own, is read once; past the bound the + // tree is not known in full, and no change is judged against it. + let page_tree = page_tree_nodes(&old); + let page_tree_unbounded = page_tree.is_none(); + let (page_nodes, kids_arrays) = page_tree.unwrap_or_default(); // What each signed page draws from, by the role it held when signed (#1787). let signed_resources = signed_page_resources(&old, &signed_page_refs); let old_info = old.trailer().get(b"Info").and_then(Object::as_ref).map(|r| r.num); + // The signed catalog's /AcroForm, when it is an object of its own. + let old_form = old.root().and_then(|c| old.get(c).as_dict().and_then(|c| c.get(b"AcroForm").and_then(Object::as_ref))).map(|r| r.num); let signed_role = |num: u32| old_content.contains(&num) || signed_pages.contains(&num) || signed_resources.contains_key(&num); let (mut allowed, mut disallowed): (Vec<&'static str>, Vec<&'static str>) = (Vec::new(), Vec::new()); if annots_unbounded { disallowed.push(kind::OTHER); } + if page_tree_unbounded { + // Past the bound the tree is not known in full: no change is judged against it. + disallowed.push(kind::PAGES); + } // Every reference the signed revision makes, read only when a slot's generations narrowed. let signed_references = OnceCell::new(); let dss = dss_parts(doc, &old); @@ -117,6 +129,9 @@ pub(super) fn classify_changes(doc: &Document, old: Option, p: Option< old_xfa_data: &old_xfa_data, signed_pages: &signed_pages, signed_annots: &signed_annots, + page_nodes: &page_nodes, + kids_arrays: &kids_arrays, + old_form, placed_annots: &placed_annots, annot_budget: &annot_budget, signed_resources: &signed_resources, @@ -259,10 +274,13 @@ struct ChangedObjectClassifier<'a> { old_xfa_data: &'a HashSet, signed_pages: &'a HashSet, signed_annots: &'a HashSet, + page_nodes: &'a HashSet, + kids_arrays: &'a HashSet, placed_annots: &'a HashSet, annot_budget: &'a Cell, signed_resources: &'a HashMap, old_info: Option, + old_form: Option, dss: &'a HashSet, } @@ -275,6 +293,8 @@ impl ChangedObjectClassifier<'_> { return Some(kind::SECURITY_STORE); } match new { + // A signed page-tree node's /Kids array of its own lists the pages. + Object::Array(_) if before.is_some() && self.kids_arrays.contains(&r.num) => Some(kind::PAGES), Object::Array(_) if self.is_signed_annotation_array(r, before) => self.annotation_array_change(new, before), Object::Dict(dict) => self.dictionary_change(r, dict, before), Object::Stream(stream) => self.stream_change(r, stream, before), @@ -342,6 +362,15 @@ impl ChangedObjectClassifier<'_> { if self.signed_pages.contains(&r.num) { // A signed page is judged by what the page shows, whatever its new labels say. self.page_change_kind(dict, previous) + } else if previous.is_some() && self.page_nodes.contains(&r.num) { + // A signed page-tree node is a page-tree change, whatever its new version is labelled. + Some(kind::PAGES) + } else if let Some(previous) = previous.filter(|_| self.old.root().is_some_and(|root| root.num == r.num)) { + // The signed catalog, whatever /Type its new version carries. + Some(catalog_kind(self.doc, self.old, dict, previous)) + } else if let Some(previous) = previous.filter(|_| self.old_form == Some(r.num)) { + // The signed form dictionary, held in an object of its own, whatever keys it carries now. + Some(acroform_kind(self.doc, self.old, dict, Some(previous))) } else if previous.is_some() && Some(r.num) == self.old_info { // Only the signed revision's own /Info dictionary is a metadata change. Some(kind::METADATA) @@ -601,6 +630,10 @@ fn acroform(doc: &Document) -> Option { fn acroform_kind(doc: &Document, old: &Document, now: &Dict, before: Option<&Dict>) -> &'static str { let fallback = acroform(old).unwrap_or_default(); let before = before.unwrap_or(&fallback); + // A field taken out of the form: no DocMDP permission covers dropping a signed field. + if lost_entries(doc, old, before.get(b"Fields"), now.get(b"Fields")) { + return kind::OTHER; + } let strip = |d: &Dict| -> Vec<(Vec, Object)> { let mut v: Vec<(Vec, Object)> = d.iter().filter(|(k, _)| !matches!(k.as_slice(), b"Fields" | b"SigFlags")).map(|(k, o)| (k.to_vec(), o.clone())).collect(); @@ -724,6 +757,62 @@ fn extensions_kept(doc: &Document, old: &Document, d: &Dict, b: &Dict) -> bool { } } +/// A change to the signed catalog `b`, now `d`. +fn catalog_kind(doc: &Document, old: &Document, d: &Dict, b: &Dict) -> &'static str { + let keys: &[&[u8]] = &[b"AcroForm", b"DSS", b"Perms", b"Metadata", b"NeedsRendering", b"Extensions"]; + if without(d, keys) != without(b, keys) { + kind::STRUCTURE + } else if !extensions_kept(doc, old, d, b) { + // Developer extensions (ISO 32000-2 §7.12) only declare what the file uses. A writer + // adding a /DSS also declares the ESIC extension it belongs to, so a declaration may be + // added, but none that was signed may change or go. + kind::STRUCTURE + } else if d.get(b"AcroForm") != b.get(b"AcroForm") { + // Compare the forms themselves, inline or not. + let now = acroform(doc).unwrap_or_default(); + acroform_kind(doc, old, &now, None) + } else if b.get(b"Perms") != d.get(b"Perms") { + kind::SIGNATURE + } else { + kind::METADATA + } +} + +/// The nodes of `doc`'s page tree and the /Kids arrays that are objects of their own, by number: +/// every dictionary with /Kids from the catalog's /Pages down. Each node, and each /Kids array +/// object, is read once. `None` past a million nodes. +fn page_tree_nodes(doc: &Document) -> Option<(HashSet, HashSet)> { + const MAX_NODES: usize = 1_000_000; + let (mut nodes, mut arrays) = (HashSet::new(), HashSet::new()); + let top = doc.root().and_then(|r| doc.get(r).as_dict().and_then(|c| c.get(b"Pages").and_then(Object::as_ref))); + let mut stack: Vec = top.into_iter().collect(); + while let Some(r) = stack.pop() { + if nodes.len() >= MAX_NODES { + return None; + } + let node = doc.get(r); + // A leaf (a page) has no /Kids. + let Some(kids) = node.as_dict().and_then(|d| d.get(b"Kids")) else { continue }; + if !nodes.insert(r.num) || kids.as_ref().is_some_and(|a| !arrays.insert(a.num)) { + continue; + } + if let Some(kids) = doc.resolve(kids).as_array() { + stack.extend(kids.iter().filter_map(Object::as_ref)); + } + } + Some((nodes, arrays)) +} + +/// Whether a list of references (`/Fields`) no longer holds an entry it held when signed. An +/// entry is still held only by a reference that resolves now and that the signed revision +/// resolved too, so to the same slot (a repaired alias counts): one rewritten to a generation +/// the slot does not accept names nothing, and one to a newer generation names another object. +fn lost_entries(doc: &Document, old: &Document, was: Option<&Object>, now: Option<&Object>) -> bool { + let live = |d: &Document, r: &ObjRef| !matches!(*d.get(*r), Object::Null); + let now: HashSet = refs(doc, now).into_iter().filter(|r| live(doc, r) && live(old, r)).map(|r| r.num).collect(); + refs(old, was).iter().filter(|r| live(old, r)).any(|r| !now.contains(&r.num)) +} + fn change_kind(doc: &Document, old: &Document, d: &Dict, before: Option<&Dict>) -> &'static str { let ty = d.name(b"Type"); // A later revision cannot make an existing object a signature by relabelling it: a signature @@ -746,31 +835,15 @@ fn change_kind(doc: &Document, old: &Document, d: &Dict, before: Option<&Dict>) }) { return kind::OTHER; } + // Where a signed widget sits and whether it shows are what the page shows: filling the form + // changes its value and appearance, never its rectangle or its flags. + if before.is_some_and(|b| b.name(b"Subtype") == Some(b"Widget") && (d.get(b"Rect") != b.get(b"Rect") || d.get(b"F") != b.get(b"F"))) { + return kind::PAGE_CONTENT; + } match ty { // Only a new signature value or policy: rewriting a signed one is not a signature (#1788). Some(b"Sig" | b"DocTimeStamp" | b"SigRef" | b"TransformParams") if before.is_none() => return kind::SIGNATURE, - Some(b"Catalog") => { - let keys: &[&[u8]] = &[b"AcroForm", b"DSS", b"Perms", b"Metadata", b"NeedsRendering", b"Extensions"]; - return match before { - Some(b) if without(d, keys) != without(b, keys) => kind::STRUCTURE, - // Developer extensions (ISO 32000-2 §7.12) only declare what the file uses. A - // writer adding a /DSS also declares the ESIC extension it belongs to, so a - // declaration may be added, but none that was signed may change or go. - Some(b) if !extensions_kept(doc, old, d, b) => kind::STRUCTURE, - Some(b) => { - if d.get(b"AcroForm") != b.get(b"AcroForm") { - // Compare the forms themselves, inline or not. - let now = acroform(doc).unwrap_or_default(); - acroform_kind(doc, old, &now, None) - } else if b.get(b"Perms") != d.get(b"Perms") { - kind::SIGNATURE - } else { - kind::METADATA - } - } - None => kind::FORM_FILL, - }; - } + Some(b"Catalog") => return before.map_or(kind::FORM_FILL, |b| catalog_kind(doc, old, d, b)), Some(b"Page") => { return match before { Some(b) if without(d, &[b"Annots"]) != without(b, &[b"Annots"]) => kind::PAGE_CONTENT, diff --git a/crates/sign/tests/it/main.rs b/crates/sign/tests/it/main.rs index 791b2b503b..bcd6334719 100644 --- a/crates/sign/tests/it/main.rs +++ b/crates/sign/tests/it/main.rs @@ -8,4 +8,5 @@ mod crypto; mod encrypted_signing; mod pdf; mod revision_classification; +mod signed_structure; mod store_keys; diff --git a/crates/sign/tests/it/signed_structure.rs b/crates/sign/tests/it/signed_structure.rs new file mode 100644 index 0000000000..471a43c8fe --- /dev/null +++ b/crates/sign/tests/it/signed_structure.rs @@ -0,0 +1,199 @@ +//! A certified document's page tree, catalog and form keep the roles they had when signed: a +//! later revision that relabels them, drops a field from the form, or hides or moves a signed +//! field's widget is no form fill. + +use std::sync::Arc; + +use pdfcraft_cos::{Document, ObjRef, Object, PdfString, SaveOptions, write_incremental}; +use pdfcraft_sign::{Modification, SignOptions, Status, TrustStore, kind, pkcs12, signatures}; + +fn data(name: &str) -> Vec { + std::fs::read(format!("{}/tests/data/{name}", env!("CARGO_MANIFEST_DIR"))).unwrap() +} + +fn open(b: &[u8]) -> Document { + Document::open(Arc::new(b.to_vec())).unwrap() +} + +/// A classic-xref PDF of `objs`, numbered from 1, with object 1 as the catalog. +fn write_pdf(objs: &[&str]) -> Vec { + let mut out = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (i, o) in objs.iter().enumerate() { + offsets.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n{o}\nendobj\n", i + 1).as_bytes()); + } + let xref = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes()); + for o in offsets { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n", objs.len() + 1).as_bytes()); + out +} + +fn r(num: u32) -> ObjRef { + ObjRef { num, generation: 0 } +} + +/// Two pages, listed by the /Kids array object 6; page 3 holds text field 5, listed in /Fields. +fn form() -> Vec { + write_pdf(&[ + "<< /Type /Catalog /Pages 2 0 R /AcroForm << /Fields [5 0 R] >> >>", + "<< /Type /Pages /Kids 6 0 R /Count 2 >>", + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Annots [5 0 R] >>", + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] >>", + "<< /Type /Annot /Subtype /Widget /FT /Tx /T (Name) /V (Ann) /Rect [20 200 120 220] /P 3 0 R >>", + "[3 0 R 4 0 R]", + ]) +} + +fn certified(p: u8) -> Vec { + let id = pkcs12::open(&data("rsa-aes.p12"), "test").unwrap(); + let opts = SignOptions { + page: 1, + rect: Some([20.0, 100.0, 220.0, 150.0]), + date: "D:20261002120000+01'00'".into(), + certify: Some(p), + ..SignOptions::default() + }; + pdfcraft_sign::sign(&open(&form()), &id, &opts).unwrap() +} + +fn edit_after(signed: &[u8], f: impl FnOnce(&mut Document)) -> Vec { + let mut doc = open(signed); + f(&mut doc); + write_incremental(&doc, &SaveOptions { object_streams: false, ..SaveOptions::default() }).unwrap() +} + +/// The certification signature of `bytes`: its status and modification verdict. +fn verdict(bytes: &[u8]) -> (Status, Modification) { + let id = pkcs12::open(&data("rsa-aes.p12"), "test").unwrap(); + let trust = TrustStore { certs: vec![id.certificate.clone()] }; + let s = signatures(&open(bytes), bytes, &trust).into_iter().find(|s| s.signed && s.certify.is_some()).unwrap(); + (s.status, s.modification) +} + +/// `bytes` is refused, for (among others) `expected`. +fn assert_refused(what: &str, bytes: &[u8], expected: &str) { + let (status, modification) = verdict(bytes); + assert!( + matches!(&modification, Modification::Disallowed(kinds) if kinds.contains(&expected)) && status != Status::Valid, + "{what}: {status:?} {modification:?}, expected {expected}" + ); +} + +#[test] +fn a_signed_page_tree_node_is_judged_as_the_page_tree() { + let signed = certified(2); + // Relabelled as a field and dropping a page. + assert_refused( + "node relabelled", + &edit_after(&signed, |doc| { + doc.update_dict(r(2), |n| { + n.remove(b"Type"); + n.set(b"T".to_vec(), PdfString::text("x")); + n.set(b"Kids".to_vec(), Object::Array(vec![Object::Ref(r(4))])); + n.set(b"Count".to_vec(), Object::Int(1)); + }) + .unwrap(); + }), + kind::PAGES, + ); + // Turning every page through an inherited attribute. + assert_refused( + "inherited rotation", + &edit_after(&signed, |doc| { + doc.update_dict(r(2), |n| { + n.set(b"DR".to_vec(), Object::Dict(Default::default())); + n.set(b"Rotate".to_vec(), Object::Int(180)); + }) + .unwrap(); + }), + kind::PAGES, + ); + // Reordering the pages through the node's /Kids array object. + assert_refused( + "pages reordered", + &edit_after(&signed, |doc| doc.set(r(6), Object::Array(vec![Object::Ref(r(4)), Object::Ref(r(3))]))), + kind::PAGES, + ); +} + +#[test] +fn a_signed_catalog_is_judged_as_the_catalog_whatever_its_type() { + let signed = certified(2); + assert_refused( + "catalog relabelled", + &edit_after(&signed, |doc| { + let content = doc.add(Object::Stream(pdfcraft_cos::Stream::from_raw(Default::default(), b"0 0 m 300 300 l S".to_vec()))); + let pages = doc.add(Object::Dict(Default::default())); + let mut page = pdfcraft_cos::Dict::new(); + page.set(b"Parent".to_vec(), Object::Ref(pages)); + page.set(b"MediaBox".to_vec(), Object::Array([0, 0, 300, 300].map(Object::Int).to_vec())); + page.set(b"Contents".to_vec(), Object::Ref(content)); + let page = doc.add(Object::Dict(page)); + doc.update_dict(pages, |n| { + n.set(b"Kids".to_vec(), Object::Array(vec![Object::Ref(page)])); + n.set(b"Count".to_vec(), Object::Int(1)); + }) + .unwrap(); + let root = doc.root().unwrap(); + doc.update_dict(root, |c| { + c.remove(b"Type"); + c.set(b"FT".to_vec(), Object::name("Tx")); + c.set(b"Pages".to_vec(), Object::Ref(pages)); + }) + .unwrap(); + }), + kind::STRUCTURE, + ); +} + +#[test] +fn a_field_dropped_from_the_form_is_no_change_any_level_permits() { + for p in [1, 2] { + let signed = certified(p); + let set_fields = |fields: Vec| { + edit_after(&signed, move |doc| { + let root = doc.root().unwrap(); + let mut form = + doc.get(root).as_dict().and_then(|c| c.get(b"AcroForm")).map(|f| doc.resolve(f)).and_then(|f| f.as_dict().cloned()).unwrap(); + let kept: Vec = fields + .into_iter() + .chain( + form.get(b"Fields") + .and_then(Object::as_array) + .into_iter() + .flatten() + .filter(|f| f.as_ref().is_some_and(|f| f.num != 5)) + .cloned(), + ) + .collect(); + form.set(b"Fields".to_vec(), Object::Array(kept)); + doc.update_dict(root, |c| c.set(b"AcroForm".to_vec(), Object::Dict(form))).unwrap(); + }) + }; + assert_refused(&format!("P={p} field dropped"), &set_fields(Vec::new()), kind::OTHER); + let dead = ObjRef { num: 5, generation: 3 }; + assert_refused(&format!("P={p} field listed by a dead generation"), &set_fields(vec![Object::Ref(dead)]), kind::OTHER); + // The field's slot taken by a newer generation: another object, listed in its place. + let newer = ObjRef { num: 5, generation: 1 }; + let replaced = edit_after(&set_fields(vec![Object::Ref(newer)]), |doc| { + let mut field = doc.get(r(5)).as_dict().cloned().unwrap(); + field.set(b"V".to_vec(), PdfString::text("Eve")); + doc.set(newer, Object::Dict(field)); + }); + assert_refused(&format!("P={p} field replaced by a newer generation"), &replaced, kind::OTHER); + } +} + +#[test] +fn a_signed_fields_widget_may_be_filled_but_not_hidden_or_moved() { + let signed = certified(2); + let edit = |f: &dyn Fn(&mut pdfcraft_cos::Dict)| edit_after(&signed, |doc| doc.update_dict(r(5), |w| f(w)).unwrap()); + let filled = edit(&|w| w.set(b"V".to_vec(), PdfString::text("Bob"))); + assert_eq!(verdict(&filled), (Status::Valid, Modification::Allowed(vec![kind::FORM_FILL]))); + assert_refused("hidden", &edit(&|w| w.set(b"F".to_vec(), Object::Int(2))), kind::PAGE_CONTENT); + assert_refused("moved", &edit(&|w| w.set(b"Rect".to_vec(), Object::Array(vec![Object::Int(0); 4]))), kind::PAGE_CONTENT); +} From 560c68db899ae332b9fda18677583ae9a45e3043 Mon Sep 17 00:00:00 2001 From: LetoCoder26 Date: Thu, 5 Nov 2026 21:41:29 -0500 Subject: [PATCH 135/198] fix(sign): an existing object is judged by the role it had when signed Rebased #5187 onto 9afa1266. Landing the signed-role rule only: an existing object that gains form or annotation keys is judged by what it was when signed, and a signed annotation turned into another kind is no comment. The two P1 regressions the original PR carried are not landed: the signature decision still reads the new /FT (a signed widget relabelled as a text field is not signing), and the dropped-/Fields check runs before the form-key comparison (already carried by #6358's lost_entries). Co-Authored-By: LetoCoder26 --- crates/sign/src/pdf/changes.rs | 23 ++++-- crates/sign/tests/it/pdf.rs | 139 +++++++++++++++++++++++++++++++++ 2 files changed, 157 insertions(+), 5 deletions(-) diff --git a/crates/sign/src/pdf/changes.rs b/crates/sign/src/pdf/changes.rs index a44c08bb7c..8cfd823fb0 100644 --- a/crates/sign/src/pdf/changes.rs +++ b/crates/sign/src/pdf/changes.rs @@ -858,19 +858,32 @@ fn change_kind(doc: &Document, old: &Document, d: &Dict, before: Option<&Dict>) Some(b"Pages") => return if before.is_some() { kind::PAGES } else { kind::STRUCTURE }, _ => {} } - match d.name(b"Subtype") { + // The role an existing object plays (annotation, form field, the form) is the one it had when + // signed: keys its new version carries are written by whoever changed it. An indirect + // /OCProperties rewritten with an added /FT still decides which layers show (#1787 judges a + // signed page's resources this way; this applies it to every other object). + let role = before.unwrap_or(d); + // Nor does an existing annotation or widget turned into another kind of annotation edit the + // one it was: an empty signature widget rewritten as a /Square that covers signed text is no + // signature change, and no comment either. + if before.is_some_and(|b| b.name(b"Subtype") != d.name(b"Subtype")) { + return kind::OTHER; + } + // The new version's /FT still decides whether this is a signature: a signed widget relabelled + // as a text field is not a signature change, whatever it was when signed. + match role.name(b"Subtype") { Some(b"Widget") => return if d.name(b"FT") == Some(b"Sig") && signature_ok { kind::SIGNATURE } else { kind::FORM_FILL }, Some(b"Link") => return kind::LINKS, - Some(_) if ty == Some(b"Annot") || d.contains(b"Rect") => return kind::COMMENTS, + Some(_) if role.name(b"Type") == Some(b"Annot") || role.contains(b"Rect") => return kind::COMMENTS, _ => {} } - if d.name(b"FT") == Some(b"Sig") && signature_ok { + if role.name(b"FT") == Some(b"Sig") && signature_ok { return kind::SIGNATURE; } - if d.contains(b"Fields") { + if role.contains(b"Fields") { return acroform_kind(doc, old, d, before); } - if d.contains(b"FT") || (d.contains(b"T") && d.contains(b"Kids")) || d.contains(b"DR") { + if role.contains(b"FT") || (role.contains(b"T") && role.contains(b"Kids")) || role.contains(b"DR") { return kind::FORM_FILL; } if before.is_none() { diff --git a/crates/sign/tests/it/pdf.rs b/crates/sign/tests/it/pdf.rs index 57efadbaf3..8f327115d2 100644 --- a/crates/sign/tests/it/pdf.rs +++ b/crates/sign/tests/it/pdf.rs @@ -1620,6 +1620,145 @@ fn a_dictionary_labelled_metadata_is_not_a_metadata_change() { }); } +/// A one-page document whose text sits in an optional-content group (object 8) that the +/// catalog's indirect `/OCProperties` (object 7) lists as shown. +fn fixture_with_layer() -> Vec { + let content = b"/OC /oc1 BDC BT /F1 14 Tf 20 250 Td (Clause 4: the buyer pays) Tj ET EMC"; + let objs: Vec> = vec![ + b"<< /Type /Catalog /Pages 2 0 R /AcroForm << /Fields [6 0 R] >> /OCProperties 7 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> /Properties << /oc1 8 0 R >> >> /Annots [6 0 R] >>".to_vec(), + [format!("<< /Length {} >>\nstream\n", content.len()).as_bytes(), content, b"\nendstream"].concat(), + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>".to_vec(), + b"<< /Type /Annot /Subtype /Widget /FT /Sig /T (Approval) /Rect [150 20 280 70] /P 3 0 R /F 4 >>".to_vec(), + b"<< /OCGs [8 0 R] /D << /ON [8 0 R] >> >>".to_vec(), + b"<< /Type /OCG /Name (Clause 4) >>".to_vec(), + ]; + let mut out = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (i, o) in objs.iter().enumerate() { + offsets.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n", i + 1).as_bytes()); + out.extend_from_slice(o); + out.extend_from_slice(b"\nendobj\n"); + } + let xref = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes()); + for o in offsets { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n", objs.len() + 1).as_bytes()); + out +} + +/// Hide the clause: rewrite `/OCProperties` (object 7) so its group starts off, adding `extra`. +fn hide_layer(doc: &mut Document, extra: &[(&str, Object)]) { + let mut d = dict_of(vec![("OCGs", Object::Array(vec![oref(8)])), ("D", Object::Dict(dict_of(vec![("OFF", Object::Array(vec![oref(8)]))])))]); + for (k, v) in extra { + d.set(k.as_bytes().to_vec(), v.clone()); + } + doc.set(pdfcraft_cos::ObjRef { num: 7, generation: 0 }, Object::Dict(d)); +} + +#[test] +fn an_existing_object_given_form_keys_after_signing_is_still_judged() { + // An existing object is judged by the role it had when signed, not only when a signed page + // draws it (#1787): the catalog's indirect /OCProperties decides which layers show, and a + // rewrite that hides the clause stays a change whatever form-field keys it adds. + assert_refused_at_every_level(&fixture_with_layer(), "other changes", |doc| hide_layer(doc, &[])); + let zeros = Object::Array(vec![Object::Int(0); 4]); + for extra in [ + vec![("FT", Object::name("Tx"))], + vec![("DR", Object::Dict(pdfcraft_cos::Dict::new()))], + vec![("T", Object::String(PdfString::text("x"))), ("Kids", Object::Array(Vec::new()))], + vec![("Subtype", Object::name("Widget")), ("Rect", zeros.clone())], + vec![("Subtype", Object::name("Square")), ("Rect", zeros)], + ] { + assert_refused_at_every_level(&fixture_with_layer(), "other changes", |doc| hide_layer(doc, &extra)); + } +} + +#[test] +fn an_unsigned_signature_widget_turned_into_a_comment_is_no_edit_of_it() { + // The empty signature field "Approval" (object 6) rewritten as a white /Square annotation that + // covers the signed text. Judged by what it was, it is a signature widget; but an annotation + // of another kind is no edit that widget permits, at any level. + assert_refused_at_every_level(&fixture(), "other changes", |doc| { + let white = Object::Array(vec![Object::Int(1); 3]); + doc.set( + pdfcraft_cos::ObjRef { num: 6, generation: 0 }, + Object::Dict(dict_of(vec![ + ("Type", Object::name("Annot")), + ("Subtype", Object::name("Square")), + ("Rect", Object::Array(vec![Object::Int(0), Object::Int(0), Object::Int(300), Object::Int(300)])), + ("C", white.clone()), + ("IC", white), + ("P", oref(3)), + ])), + ); + }); +} + +/// `fixture()` with the form an indirect dictionary (object 7) listing the unsigned field. +fn fixture_with_indirect_acroform() -> Vec { + let objs: Vec<&[u8]> = vec![ + b"<< /Type /Catalog /Pages 2 0 R /AcroForm 7 0 R >>", + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> /Annots [6 0 R] >>", + b"<< /Length 44 >>\nstream\nBT /F1 14 Tf 20 250 Td (Contract text) Tj ET\nendstream", + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + b"<< /Type /Annot /Subtype /Widget /FT /Sig /T (Approval) /Rect [150 20 280 70] /P 3 0 R /F 4 >>", + b"<< /Fields [6 0 R] >>", + ]; + let mut out = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (i, o) in objs.iter().enumerate() { + offsets.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n", i + 1).as_bytes()); + out.extend_from_slice(o); + out.extend_from_slice(b"\nendobj\n"); + } + let xref = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes()); + for o in offsets { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n", objs.len() + 1).as_bytes()); + out +} + +#[test] +fn the_forms_signed_fields_are_not_dropped_after_signing() { + // Signed, then timestamped. A later revision drops the form's /Fields, or empties it: the + // fields the timestamp covered, the signature among them, are gone from the form. That is no + // signature change, so the timestamp reports it as not permitted. + let id = pkcs12::open(&data("rsa-aes.p12"), "test").unwrap(); + let tsa = TestTsa { + id: pkcs12::open(&data("rsa-aes.p12"), "test").unwrap(), + time: Time { year: 2026, month: 10, day: 6, hour: 12, minute: 0, second: 0 }, + }; + let trust = TrustStore { certs: vec![id.certificate.clone()] }; + let signed = pdfcraft_sign::sign(&open(&fixture_with_indirect_acroform()), &id, &opts()).unwrap(); + let stamped = pdfcraft_sign::timestamp_document(&open(&signed), &tsa, "D:20261006120000Z").unwrap(); + let mut failures = Vec::new(); + for (how, fields) in [("dropped", None), ("emptied", Some(Object::Array(Vec::new())))] { + let edited = edit_after(&stamped, |doc| { + doc.update_dict(pdfcraft_cos::ObjRef { num: 7, generation: 0 }, |af| match &fields { + Some(f) => af.set(b"Fields".to_vec(), f.clone()), + None => { + af.remove(b"Fields"); + } + }) + .unwrap(); + }); + let stamp = signatures(&open(&edited), &edited, &trust).into_iter().find(|s| s.doc_timestamp).expect("the document timestamp"); + if !matches!(stamp.modification, Modification::Disallowed(_)) { + failures.push(format!("/Fields {how}: {:?}, {:?}", stamp.status, stamp.modification)); + } + } + assert!(failures.is_empty(), "{}", failures.join("\n")); +} + /// A one-page document like `fixture()` (objects 1–6) plus a stream, object 7, with the given /// dictionary entries and body. `catalog_extra` goes into the catalog; the page draws object 7 /// as a Form XObject when `drawn`. From fbe079c953d3f2644ac59d40894a07fb061f21b0 Mon Sep 17 00:00:00 2001 From: LetoCoder26 Date: Thu, 5 Nov 2026 21:43:14 -0500 Subject: [PATCH 136/198] fix(sign): compare the catalog's /Perms by content under P=1 Rebased #5173 onto 9afa1266. Its later-signature predicate is superseded by #6364's object-based check, so only the /Perms rule lands: under DocMDP P=1 a /Perms that differs from the signed catalog's, by content, is a structure change however it was rewritten. Co-Authored-By: LetoCoder26 --- crates/sign/src/pdf/changes.rs | 73 +++++++++++++++++++++++++++++++++- crates/sign/tests/it/pdf.rs | 29 ++++++++++++++ parity/acrobat-features.toml | 4 +- 3 files changed, 102 insertions(+), 4 deletions(-) diff --git a/crates/sign/src/pdf/changes.rs b/crates/sign/src/pdf/changes.rs index 8cfd823fb0..712c9fcfff 100644 --- a/crates/sign/src/pdf/changes.rs +++ b/crates/sign/src/pdf/changes.rs @@ -217,13 +217,30 @@ pub(super) fn classify_changes(doc: &Document, old: Option, p: Option< // archiving (§12.8.5). Those timestamps draw nothing. Any other signature added since — an // approval signature, an unsigned signature widget, or a widget that paints over the signed // page — is a change the certification forbids, however it is labelled. - if p == Some(1) && allowed.contains(&kind::SIGNATURE) && !only_invisible_timestamps_added(doc, &old) { + if p == Some(1) { + only_validation_data(&mut allowed, &mut disallowed, doc, &old); + } + if disallowed.is_empty() { Modification::Allowed(allowed) } else { Modification::Disallowed(disallowed) } +} + +/// DocMDP P=1 ("no changes allowed") permits only validation data to follow the signed revision: +/// the security store, and document timestamps that draw nothing. A later approval signature, a +/// visible widget or a changed catalog `/Perms` breaks the certification. +fn only_validation_data(allowed: &mut Vec<&'static str>, disallowed: &mut Vec<&'static str>, doc: &Document, old: &Document) { + if allowed.contains(&kind::SIGNATURE) && !only_invisible_timestamps_added(doc, old) { allowed.retain(|k| *k != kind::SIGNATURE); if !disallowed.contains(&kind::SIGNATURE) { disallowed.push(kind::SIGNATURE); } } - if disallowed.is_empty() { Modification::Allowed(allowed) } else { Modification::Disallowed(disallowed) } + // The catalog's /Perms names the certification signature itself. Elsewhere a change to it is + // judged as signing; under P=1 dropping or redirecting it is a change like any other. + // Compared by content: an indirect /Perms rewritten in place keeps its reference (and, shared + // with the trailer's /Info, would otherwise pass as a metadata change). + let perms = |d: &Document| d.root().and_then(|r| d.get(r).as_dict().and_then(|c| c.get(b"Perms").map(|p| (*d.resolve(p)).clone()))); + if perms(doc) != perms(old) && !disallowed.contains(&kind::STRUCTURE) { + disallowed.push(kind::STRUCTURE); + } } /// Whether every signature object `doc` gained or changed since the signed revision `old` is an @@ -892,3 +909,55 @@ fn change_kind(doc: &Document, old: &Document, d: &Dict, before: Option<&Dict>) } kind::OTHER } + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Arc; + + /// A PDF of `objs` (numbered from 1) whose trailer also holds `trailer_extra`. + fn pdf(objs: &[&str], trailer_extra: &str) -> Vec { + let mut out = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (i, o) in objs.iter().enumerate() { + offsets.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n{o}\nendobj\n", i + 1).as_bytes()); + } + let xref = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes()); + for o in offsets { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R {trailer_extra} >>\nstartxref\n{xref}\n%%EOF\n", objs.len() + 1).as_bytes()); + out + } + + /// A P=1 signed revision whose catalog `/Perms` and trailer `/Info` are one indirect + /// dictionary (object 3). A later revision points its `/DocMDP` at another signature + /// dictionary already in the file, in place: the reference stays, no object is added, and + /// as the signed `/Info` the object reads as metadata, but the certification's own record + /// has changed. + #[test] + fn a_perms_dictionary_shared_with_info_rewritten_in_place_is_a_structure_change() { + let signed = pdf( + &[ + "<< /Type /Catalog /Pages 2 0 R /Perms 3 0 R >>", + "<< /Type /Pages /Kids [] /Count 0 >>", + "<< /DocMDP 4 0 R >>", + "<< /Type /Sig >>", + "<< /Type /Sig >>", + ], + "/Info 3 0 R", + ); + let old = Document::open(Arc::new(signed.clone())).unwrap(); + let mut doc = Document::open(Arc::new(signed)).unwrap(); + let other = ObjRef { num: 5, generation: 0 }; + doc.update_dict(ObjRef { num: 3, generation: 0 }, |p| p.set(b"DocMDP".to_vec(), Object::Ref(other))).unwrap(); + let edited = pdfcraft_cos::write_incremental(&doc, &pdfcraft_cos::SaveOptions { object_streams: false, ..Default::default() }).unwrap(); + let doc = Document::open(Arc::new(edited)).unwrap(); + match classify_changes(&doc, Some(old), Some(1)) { + Modification::Disallowed(kinds) => assert!(kinds.contains(&kind::STRUCTURE), "{kinds:?}"), + other => panic!("the certification's /Perms changed in place under P=1: {other:?}"), + } + } +} diff --git a/crates/sign/tests/it/pdf.rs b/crates/sign/tests/it/pdf.rs index 8f327115d2..8c9a827e6d 100644 --- a/crates/sign/tests/it/pdf.rs +++ b/crates/sign/tests/it/pdf.rs @@ -3164,3 +3164,32 @@ fn signing_a_field_with_a_null_value_after_another_signature_keeps_the_first_val all.iter().map(|s| (&s.status, &s.modification)).collect::>() ); } + +#[test] +fn the_catalogs_perms_do_not_change_under_no_changes() { + // /Perms names the certification signature (DocMDP). Under P=1 nothing may change, so a later + // revision that drops it, or points it elsewhere, breaks the certification. + let id = pkcs12::open(&data("rsa-aes.p12"), "test").unwrap(); + let trust = TrustStore { certs: vec![id.certificate.clone()] }; + let certified = pdfcraft_sign::sign(&open(&fixture()), &id, &SignOptions { certify: Some(1), ..opts() }).unwrap(); + let elsewhere = dict_of(vec![("DocMDP", oref(5))]); + for (how, perms) in [("dropped", None), ("pointed elsewhere", Some(Object::Dict(elsewhere)))] { + let edited = edit_after(&certified, |doc| { + let root = doc.root().unwrap(); + doc.update_dict(root, |c| match &perms { + Some(p) => c.set(b"Perms".to_vec(), p.clone()), + None => { + c.remove(b"Perms"); + } + }) + .unwrap(); + }); + let s = signatures(&open(&edited), &edited, &trust).into_iter().find(|s| s.field == "Signature1").unwrap(); + assert!( + s.status == Status::Invalid && matches!(&s.modification, Modification::Disallowed(k) if k.contains(&kind::STRUCTURE)), + "/Perms {how} under P=1: {:?}, {:?}", + s.status, + s.modification + ); + } +} diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index ec144d3a07..10b040e09f 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -6535,8 +6535,8 @@ milestone = "M9" status = "shipped" commands = ["sign.panel"] tools = ["sign_list"] -evidence = ["crates/sign/tests/it/pdf.rs::later_changes_are_classified_under_the_signature_permissions", "crates/sign/tests/it/pdf.rs::signing_an_existing_field_and_counter_signing", "crates/engine/src/tests.rs::signing_saving_trusting_and_commenting_afterwards", "crates/sign/tests/it/pdf.rs::validation_data_and_xmp_added_after_signing_are_not_tampering", "crates/sign/tests/it/pdf.rs::a_dss_entry_does_not_make_the_object_it_names_part_of_the_store", "crates/sign/tests/it/pdf.rs::an_existing_dictionary_is_not_a_store_part_because_the_store_names_it", "crates/sign/tests/it/pdf.rs::an_existing_array_stays_what_it_is_when_the_store_points_at_it", "crates/sign/tests/it/pdf.rs::a_signed_store_may_grow_but_not_lose_or_swap_entries", "crates/sign/tests/it/pdf.rs::a_dictionary_labelled_metadata_is_not_a_metadata_change", "crates/sign/tests/it/pdf.rs::a_stream_already_labelled_metadata_when_signed_is_not_metadata_unless_the_catalog_names_it", "crates/sign/tests/it/pdf.rs::the_catalogs_own_xmp_stream_may_still_be_rewritten_after_signing", "crates/sign/tests/it/pdf.rs::rewritten_page_resources_are_page_content_whatever_keys_they_carry", "crates/sign/tests/it/pdf.rs::rewriting_a_certified_forms_xfa_data_is_a_form_fill"] -notes = "revision diff classified as signing, form fill, comments, metadata, page content or structure. Later revisions that add DSS data (new /Certs, /CRLs, /OCSPs, /VRI objects, or ones of the signed store that only grew) or rewrite the catalog's XMP are permitted changes; a /DSS entry that names other objects, or a /Type /DSS or /Type /Metadata label on an existing dictionary or stream, does not excuse changing them (negative tests at every DocMDP level). A signed page's resources (fonts, XObjects, graphics states and what they reach) are page content by the role they held when signed, whatever keys a later revision writes into them (#1787). Rewriting the XFA `datasets` or `form` packet the signed revision named is form fill, as a Reader Extensions (UR3) save of a certified form does (#2759); the packets' roles come from the signed revision" +evidence = ["crates/sign/tests/it/pdf.rs::later_changes_are_classified_under_the_signature_permissions", "crates/sign/tests/it/pdf.rs::signing_an_existing_field_and_counter_signing", "crates/engine/src/tests.rs::signing_saving_trusting_and_commenting_afterwards", "crates/sign/tests/it/pdf.rs::validation_data_and_xmp_added_after_signing_are_not_tampering", "crates/sign/tests/it/pdf.rs::a_dss_entry_does_not_make_the_object_it_names_part_of_the_store", "crates/sign/tests/it/pdf.rs::an_existing_dictionary_is_not_a_store_part_because_the_store_names_it", "crates/sign/tests/it/pdf.rs::an_existing_array_stays_what_it_is_when_the_store_points_at_it", "crates/sign/tests/it/pdf.rs::a_signed_store_may_grow_but_not_lose_or_swap_entries", "crates/sign/tests/it/pdf.rs::a_dictionary_labelled_metadata_is_not_a_metadata_change", "crates/sign/tests/it/pdf.rs::a_stream_already_labelled_metadata_when_signed_is_not_metadata_unless_the_catalog_names_it", "crates/sign/tests/it/pdf.rs::the_catalogs_own_xmp_stream_may_still_be_rewritten_after_signing", "crates/sign/tests/it/pdf.rs::rewritten_page_resources_are_page_content_whatever_keys_they_carry", "crates/sign/tests/it/pdf.rs::rewriting_a_certified_forms_xfa_data_is_a_form_fill", "crates/sign/tests/it/pdf.rs::the_catalogs_perms_do_not_change_under_no_changes"] +notes = "revision diff classified as signing, form fill, comments, metadata, page content or structure. Later revisions that add DSS data (new /Certs, /CRLs, /OCSPs, /VRI objects, or ones of the signed store that only grew) or rewrite the catalog's XMP are permitted changes; a /DSS entry that names other objects, or a /Type /DSS or /Type /Metadata label on an existing dictionary or stream, does not excuse changing them (negative tests at every DocMDP level). A signed page's resources (fonts, XObjects, graphics states and what they reach) are page content by the role they held when signed, whatever keys a later revision writes into them (#1787). Rewriting the XFA `datasets` or `form` packet the signed revision named is form fill, as a Reader Extensions (UR3) save of a certified form does (#2759); the packets' roles come from the signed revision. Under P=1 a change to the catalog's /Perms breaks the certification." [[feature]] id = "sign.view-signed-version" From d7ce875dd692109789bcdad1b683ffc5a2d07dc7 Mon Sep 17 00:00:00 2001 From: eliets2 Date: Fri, 6 Nov 2026 10:15:07 +0200 Subject: [PATCH 137/198] redact: independent redaction proof machinery (snapshot, sweep, budget) (#5557) * redact: add the independent redaction proof machinery * redact: harden the proof's paint check and count removed codes The manifest counts character codes, not raw bytes: a composite font's codes are several bytes, so the byte length overstated what was cut. Annotation appearances are checked whatever their storage (a direct dictionary has no reference of its own to dedupe on), and a stream shared by several annotations is checked at each annotation's own placement. The graphics-state stack and the clip holes of one walk are bounded; deeper or more is unverifiable, never unbounded memory. Even-odd clips cut holes only where the rule can be told cheaply - all subpaths axis-aligned, disjoint, few - and anything else keeps its paint reported, failing closed. The /W widths of a descendant font cost bounded work, like a CMap. * redact: settle the survivors a page sweep counted before its work ran out A page that the search budget stops on left the counts made on earlier pages unsettled: the survivors were seen but never reported, and the regions that owned them were never marked. The sweep now stops and still settles, so a located survivor is reported where it sits (the page reads unswept either way, and the proof still fails). * redact: prove the redacted output of every supported encryption The authenticated-reopen regression now covers all four crypt algorithms the writer supports, not only AES-128: the caller's credentials prove the saved output, and empty or wrong credentials still fail closed. * redact: answer the review round on the proof A glyph lying under several regions is now accounted for by every one of them, so overlapping marks each carry their own share in a shared proof instead of the first one absorbing the rest and the others reading as if no text was there. Each finding is recorded once per region as it is found, so a page that paints over a region a thousand times neither grows the report nor the memory with repeats. The byte search walks the haystack once (the pattern's prefix function is built once), so a needle of one letter over a haystack full of it can no longer spend comparisons the search budget never charged for. A numeric XML character reference may carry any number of leading zeros; the padded form now decodes like the short one, so a survivor spelled that way in the XMP is still found. The /W widths of a descendant font are paid for from the shared work budget: a form invoked again rebuilds its fonts, and a per-font counter alone let the same array be paid for many times over. * redact: sweep the proof's survivors out of every place a reader could look Plant the removed text where the review listed it and expect the sweep to report it: a compressed object stream, a page thumbnail, a form field value, an XFA packet, and an object nothing refers to. A file whose page content is bigger than the proof may decode for one stream is now shown to come out unverified on its surfaces and its region, never clean. * redact: spare the names a redacted word legitimately sits in, and charge failed decodes A redacted word that is also a structural name (a /Subtype, a /Filter, a /BaseFont) is structure, not a leftover copy: the name surface is now swept against the matches the input itself had, and only a surplus of matches is a survivor. The input's names are counted when the snapshot is taken, for the region's needles and for strings added later alike. A decode that fails after doing part of its work paid nothing: the proof's cumulative decode allowance now charges every attempt, failures included, so the same bomb cannot be decoded (and fail) again and again for free. * redact: count the input's names once each, and say so when the count runs out The name allowance counted one entry per occurrence, so a document whose pages repeat the same structural name paid for the same scan again and again; names are now kept once, with their count. A name search that runs out of work read as no matches, which would have swept names with none allowed: that is reported on the text surface now, and the proof fails rather than pass on an uncounted allowance. * Bound redaction proof searches across many needles Assisted-by: OpenAI Codex * Redaction proof: drop find_all, which nothing calls Co-Authored-By: Claude Opus 5.5 * Cargo.lock: put aho-corasick in its sorted place Co-Authored-By: Claude Opus 5.5 * Redaction proof: review notes A word split across a TJ array is looked for whole. Overlapping marks count a removed copy once. A spent decode budget is a problem, not a raw-only stream. Short needles stay out of the attachment text search, and an indirect /Decode counts. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: eliets2 <287273517+eliets2@users.noreply.github.com> Co-authored-by: FredApps Co-authored-by: FredApps <153173854+FredApps@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 --- Cargo.lock | 11 + crates/redact/Cargo.toml | 2 + crates/redact/src/image.rs | 2 +- crates/redact/src/interp.rs | 100 +- crates/redact/src/lib.rs | 11 +- crates/redact/src/tests_verify.rs | 1380 ++++++++++++++++++++++ crates/redact/src/verify/extract.rs | 711 +++++++++++ crates/redact/src/verify/geometry.rs | 500 ++++++++ crates/redact/src/verify/limits.rs | 42 + crates/redact/src/verify/mod.rs | 389 ++++++ crates/redact/src/verify/search_tests.rs | 34 + crates/redact/src/verify/snapshot.rs | 532 +++++++++ crates/redact/src/verify/sweep.rs | 759 ++++++++++++ crates/redact/src/verify/util.rs | 301 +++++ 14 files changed, 4759 insertions(+), 15 deletions(-) create mode 100644 crates/redact/src/tests_verify.rs create mode 100644 crates/redact/src/verify/extract.rs create mode 100644 crates/redact/src/verify/geometry.rs create mode 100644 crates/redact/src/verify/limits.rs create mode 100644 crates/redact/src/verify/mod.rs create mode 100644 crates/redact/src/verify/search_tests.rs create mode 100644 crates/redact/src/verify/snapshot.rs create mode 100644 crates/redact/src/verify/sweep.rs create mode 100644 crates/redact/src/verify/util.rs diff --git a/Cargo.lock b/Cargo.lock index 4816a44ec0..e1f271ed12 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -143,6 +143,15 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + [[package]] name = "aligned-vec" version = "0.6.4" @@ -4610,6 +4619,7 @@ dependencies = [ name = "pdfcraft-redact" version = "0.18.1" dependencies = [ + "aho-corasick", "pdfcraft-annot", "pdfcraft-content", "pdfcraft-cos", @@ -4620,6 +4630,7 @@ dependencies = [ "pdfcraft-model", "pdfcraft-testkit", "pdfcraft-xfa", + "sha2 0.11.1", "thiserror 2.0.21", ] diff --git a/crates/redact/Cargo.toml b/crates/redact/Cargo.toml index fe21d13b32..c02335e3c4 100644 --- a/crates/redact/Cargo.toml +++ b/crates/redact/Cargo.toml @@ -9,6 +9,7 @@ license.workspace = true workspace = true [dependencies] +aho-corasick = "1" pdfcraft-cos = { workspace = true } pdfcraft-content = { workspace = true } pdfcraft-fonts = { workspace = true } @@ -20,6 +21,7 @@ pdfcraft-xfa = { workspace = true } # CCITT Group 4 and JBIG2 encoding for cleared one-bit images. pdfcraft-filters = { workspace = true } thiserror = { workspace = true } +sha2 = "0.11" [dev-dependencies] pdfcraft-testkit = { workspace = true } diff --git a/crates/redact/src/image.rs b/crates/redact/src/image.rs index 4b3c44b0fd..a99e179895 100644 --- a/crates/redact/src/image.rs +++ b/crates/redact/src/image.rs @@ -12,7 +12,7 @@ use pdfcraft_cos::{Dict, Document, ImageBudget, Object, Stream}; /// Most pixels an image may have to be cleared in place (larger ones are removed), or to be stored /// again without shared `/JBIG2Globals` (see [`without_globals`]). -const MAX_PIXELS: u64 = pdfcraft_cos::MAX_REDACTED_IMAGE_PIXELS; +pub(crate) const MAX_PIXELS: u64 = pdfcraft_cos::MAX_REDACTED_IMAGE_PIXELS; fn components(doc: &Document, s: &Stream) -> Option { let cs = doc.resolve(s.dict.get(b"ColorSpace")?); diff --git a/crates/redact/src/interp.rs b/crates/redact/src/interp.rs index 460085a320..7a2cd26eca 100644 --- a/crates/redact/src/interp.rs +++ b/crates/redact/src/interp.rs @@ -53,6 +53,23 @@ pub(crate) enum Mode { Verify, } +/// A run of glyphs cut out under a region, kept for the redaction proof +/// ([`Snapshot::capture`](crate::verify::Snapshot::capture)). +pub(crate) struct Removed { + /// Index of the region (in `Scope::rects`) the run lay under. + pub(crate) rect: usize, + /// The character codes as they stood in the string. + pub(crate) raw: Vec, + /// What the font says they showed (codes without a known meaning are left out). + pub(crate) text: String, + /// How many character codes the run holds (a composite font's codes are several bytes). + pub(crate) codes: usize, + /// Which text operator showed it, counted from the start of the page. + pub(crate) op: usize, + /// Where in that operator it starts: the operand's place in a `TJ` array, and the byte in it. + pub(crate) at: (usize, usize), +} + #[derive(Clone)] struct Gs { ctm: Matrix, @@ -124,6 +141,12 @@ pub(crate) struct Scope<'a> { xobject_oc_cache: HashMap, /// Content removed from hidden layers (blocks and XObjects). pub layer_blocks: usize, + /// Collects the glyph runs removed under a region in apply mode (`None` = not collected). + /// The redaction proof's snapshot is what sets it: it must know what applying is about to + /// remove, by the interpreter's own judgement, without trusting anything else to it. + pub removed: Option>, + /// How many text operators have been shown, for [`Removed::op`]. + shown: usize, fonts: HashMap, Rc>, used_names: Vec>, depth: usize, @@ -149,6 +172,8 @@ impl<'a> Scope<'a> { ocmd_cache: HashMap::new(), xobject_oc_cache: HashMap::new(), layer_blocks: 0, + removed: None, + shown: 0, fonts: HashMap::new(), used_names: Vec::new(), depth: 0, @@ -169,21 +194,33 @@ impl<'a> Scope<'a> { self.rects.iter().any(|r| contains(*r, b, 0.01)) } - /// Does a glyph box count as under a region? Any real overlap does (more than a fifth of + /// Which regions a glyph box counts as under? Any real overlap does (more than a fifth of /// the glyph or a point, whichever is less, in both directions), so neighbours that only /// touch an edge survive while partly covered glyphs go. Degenerate boxes (zero-size text) - /// count when their origin is inside. - fn glyph_hit(&self, b: [f64; 4], origin: (f64, f64)) -> bool { + /// count when their origin is inside. Empty = under no region (a glyph kept). A glyph may + /// lie under several regions at once (overlapping marks); each is reported. + fn glyph_rects(&self, b: [f64; 4], origin: (f64, f64)) -> Vec { let (gw, gh) = (b[2] - b[0], b[3] - b[1]); if gw < 0.01 || gh < 0.01 { - return self.rects.iter().any(|r| origin.0 >= r[0] && origin.0 <= r[2] && origin.1 >= r[1] && origin.1 <= r[3]); + return self + .rects + .iter() + .enumerate() + .filter(|(_, r)| origin.0 >= r[0] && origin.0 <= r[2] && origin.1 >= r[1] && origin.1 <= r[3]) + .map(|(i, _)| i) + .collect(); } let (mx, my) = ((gw * 0.2).min(1.0), (gh * 0.2).min(1.0)); - self.rects.iter().any(|r| { - let ox = b[2].min(r[2]) - b[0].max(r[0]); - let oy = b[3].min(r[3]) - b[1].max(r[1]); - ox > mx && oy > my - }) + self.rects + .iter() + .enumerate() + .filter(|(_, r)| { + let ox = b[2].min(r[2]) - b[0].max(r[0]); + let oy = b[3].min(r[3]) - b[1].max(r[1]); + ox > mx && oy > my + }) + .map(|(i, _)| i) + .collect() } } @@ -223,6 +260,8 @@ struct Glyph { /// Advance in unscaled text space (before `Th`). advance: f64, hit: bool, + /// The regions it lies under (empty for hidden text). + rects: Vec, } /// Process a scope's content streams (state flows from one stream to the next). @@ -644,11 +683,14 @@ fn clean_replacements(doc: &Document, resources: &Dict, original: &[Op], process /// Lay out a shown string (or `TJ` array) glyph by glyph, advancing `tm`. Returns the `TJ` /// items with the glyphs under a region replaced by displacements, and how many were removed. -fn show(scope: &Scope<'_>, gs: &Gs, tm: &mut Matrix, items: &[Object]) -> (Vec, usize) { +/// In apply mode the runs removed under each region are also handed to `scope.removed`. +fn show(scope: &mut Scope<'_>, gs: &Gs, tm: &mut Matrix, items: &[Object]) -> (Vec, usize) { let f = &gs.font; let size = gs.size; let mut out: Vec = Vec::new(); let mut removed = 0; + scope.shown += 1; + let op = scope.shown; let push_num = |out: &mut Vec, v: f64| { if v == 0.0 { return; @@ -661,7 +703,7 @@ fn show(scope: &Scope<'_>, gs: &Gs, tm: &mut Matrix, items: &[Object]) -> (Vec { let bytes = &s.bytes; @@ -674,18 +716,50 @@ fn show(scope: &Scope<'_>, gs: &Gs, tm: &mut Matrix, items: &[Object]) -> (Vec { let degenerate = b[2] - b[0] < 0.01 && b[3] - b[1] < 0.01; matches!(gs.render_mode, 3 | 7) || !(overlaps(page, b, 0.0) || degenerate) } - None => scope.glyph_hit(b, origin), + None => { + rects = scope.glyph_rects(b, origin); + !rects.is_empty() + } }; - glyphs.push(Glyph { bytes: pos..pos + len, advance, hit }); + glyphs.push(Glyph { bytes: pos..pos + len, advance, hit, rects }); *tm = Matrix::translate(advance * gs.scale, 0.0).then(tm); pos += len; } + if scope.mode == Mode::Apply + && let Some(sink) = scope.removed.as_mut() + { + // Consecutive removed glyphs under one region are one run, recorded for + // every region the glyphs lie under: overlapping marks each account for + // their own share of what was cut. + let mut under: Vec = glyphs.iter().flat_map(|g| g.rects.iter().copied()).collect(); + under.sort_unstable(); + under.dedup(); + for r in under { + let mut k = 0; + while let Some(g) = glyphs.get(k) { + if !g.rects.contains(&r) { + k += 1; + continue; + } + let mut end = k; + while glyphs.get(end + 1).is_some_and(|n| n.rects.contains(&r)) { + end += 1; + } + let span = g.bytes.start..glyphs.get(end).map_or(g.bytes.end, |l| l.bytes.end); + let at = (place, span.start); + let raw = bytes.get(span).unwrap_or_default().to_vec(); + sink.push(Removed { rect: r, text: f.decode(&raw), raw, codes: end - k + 1, op, at }); + k = end + 1; + } + } + } let mut run: Vec = Vec::new(); for g in &glyphs { if g.hit { diff --git a/crates/redact/src/lib.rs b/crates/redact/src/lib.rs index b6a8885c03..6bb8e5cc06 100644 --- a/crates/redact/src/lib.rs +++ b/crates/redact/src/lib.rs @@ -27,10 +27,14 @@ pub mod sanitize; mod tags; #[cfg(test)] mod tests; +#[cfg(test)] +mod tests_verify; pub(crate) mod type3; pub(crate) mod type3_tables; +pub mod verify; use interp::{Mode, Scope, process}; +pub use verify::{Proof, ProofOptions, Snapshot}; pub type Rgb = [f64; 3]; @@ -59,6 +63,11 @@ pub enum RedactError { "redaction appearance has invalid geometry or exceeds the 8 MiB budget; reduce the marked area, turn off repeated text or apply fewer pages at once (nothing was redacted)" )] OverlayLimit, + /// A stream the redaction created to draw the marks' boxes and labels can't be read back, so + /// the proof can't tell it apart from content that was there before and nothing can be + /// exempted from the sweep. + #[error("a stream the redaction created can't be read back, so the result can't be proven")] + OverlayUnreadable, /// The structure tree couldn't be cleaned completely (#3302): an element written in place that /// would need changing, an element whose inherited page can't be found, or a tree too large to /// walk. Tags that kept the removed text or annotation would undo the redaction. @@ -164,7 +173,7 @@ pub struct Report { pub overlays: Vec, } -fn rect_of(doc: &Document, o: Option<&Object>) -> Option<[f64; 4]> { +pub(crate) fn rect_of(doc: &Document, o: Option<&Object>) -> Option<[f64; 4]> { let v: Vec = doc.resolve(o?).as_array()?.iter().filter_map(|x| doc.resolve(x).as_f64()).collect(); (v.len() == 4 && v.iter().all(|x| x.is_finite())).then(|| [v[0].min(v[2]), v[1].min(v[3]), v[0].max(v[2]), v[1].max(v[3])]) } diff --git a/crates/redact/src/tests_verify.rs b/crates/redact/src/tests_verify.rs new file mode 100644 index 0000000000..ee443f3209 --- /dev/null +++ b/crates/redact/src/tests_verify.rs @@ -0,0 +1,1380 @@ +//! The independent redaction proof: what it catches, what it refuses to call clean. + +use std::sync::Arc; + +use pdfcraft_annot::{Meta, NewAnnotation, Shape, Style, add_annotation, rect_quad}; +use pdfcraft_cos::{Algorithm, Dict, Document, NewEncryption, Object, SaveOptions, Stream, write_full, write_incremental}; +use sha2::Digest; + +use super::*; +use crate::verify::tests_support::{self, count_text_operators, survivor_encodings}; +use crate::verify::{Status, Surface, SurfaceReport, Verdict}; + +const SECRET: &str = "SECRETVALUE"; + +pub(super) fn stream(dict: &str, data: &[u8]) -> Vec { + let mut v = format!("<< {dict} /Length {} >>\nstream\n", data.len()).into_bytes(); + v.extend_from_slice(data); + v.extend_from_slice(b"\nendstream"); + v +} + +pub(super) fn pdf_bytes(objs: Vec>, trailer: &str) -> Vec { + let mut out = b"%PDF-1.7\n".to_vec(); + let mut offs = Vec::new(); + for (i, o) in objs.iter().enumerate() { + offs.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n", i + 1).as_bytes()); + out.extend_from_slice(o); + out.extend_from_slice(b"\nendobj\n"); + } + let x = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes()); + for o in offs { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R {trailer} >>\nstartxref\n{x}\n%%EOF\n", objs.len() + 1).as_bytes()); + out +} + +pub(super) fn open(bytes: Vec) -> Document { + Document::open(Arc::new(bytes)).unwrap() +} + +/// Every glyph is 500 units wide (5 pt at 10 pt). +pub(super) const FONT: &str = + "<< /Type /Font /Subtype /TrueType /BaseFont /Arial /FirstChar 32 /LastChar 126 /Widths 6 0 R /FontDescriptor << /Ascent 800 /Descent -200 >> >>"; + +/// A composite font without a /ToUnicode map: its text can't be decoded. +const OPAQUE_FONT: &str = "<< /Type /Font /Subtype /Type0 /BaseFont /X /Encoding /Identity-H /DescendantFonts [<< /Type /Font /Subtype /CIDFontType2 /BaseFont /X /CIDSystemInfo << /Registry (Adobe) /Ordering (Identity) /Supplement 0 >> /DW 1000 >>] >>"; + +/// Options of a one-page file: catalog, page and trailer entries, page resources, and more +/// objects (numbered from 7). +#[derive(Default)] +struct Spec<'a> { + catalog: &'a str, + page: &'a str, + trailer: &'a str, + resources: &'a str, + extra: Vec>, +} + +/// "Public" at the top, the secret lower down at x 10..65, y 100..110. +const CONTENT: &[u8] = b"BT /F1 10 Tf 10 200 Td (Public) Tj ET BT /F1 10 Tf 10 100 Td (SECRETVALUE) Tj ET"; + +pub(super) const REGION: [f64; 4] = [8.0, 95.0, 70.0, 112.0]; + +fn build_with(content: &[u8], spec: Spec<'_>) -> Vec { + let mut objs: Vec> = vec![ + format!("<< /Type /Catalog /Pages 2 0 R {} >>", spec.catalog).into_bytes(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + format!( + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R {} /Resources << /Font << /F1 5 0 R >> {} >> >>", + spec.page, spec.resources + ) + .into_bytes(), + stream("", content), + FONT.as_bytes().to_vec(), + format!("[{}]", vec!["500"; 95].join(" ")).into_bytes(), + ]; + objs.extend(spec.extra); + pdf_bytes(objs, spec.trailer) +} + +pub(super) fn mark(doc: &mut Document, rects: &[[f64; 4]]) { + let shape = Shape::Redact { quads: rects.iter().map(|r| rect_quad(*r)).collect(), overlay: String::new(), look: Default::default() }; + let style = Style::default_for(&shape); + add_annotation(doc, &NewAnnotation { page: 0, shape, style, contents: String::new(), author: "Tester".into() }, &Meta::default()).unwrap(); +} + +/// The standard page with `spec`, the secret's region marked. +fn marked(spec: Spec<'_>) -> Document { + marked_content(CONTENT, spec) +} + +fn marked_content(content: &[u8], spec: Spec<'_>) -> Document { + let mut doc = open(build_with(content, spec)); + mark(&mut doc, &[REGION]); + doc +} + +/// Apply, and tell the snapshot which streams the apply created (its overlay boxes and labels). +fn applied(doc: &mut Document, snap: &mut Snapshot) -> Result { + let result = apply(doc, None); + if let Ok(report) = &result { + snap.set_overlay_streams(doc, &report.overlays)?; + } + result +} + +/// Capture, apply, prove. +pub(super) fn run(doc: &mut Document) -> (Result, Proof) { + let mut snap = Snapshot::capture(doc, None).unwrap(); + let result = applied(doc, &mut snap); + let proof = snap.prove(doc, None, &ProofOptions::default()); + (result, proof) +} + +/// Prove a document the redaction was never applied to (as a full save would write it). +pub(super) fn prove_unapplied(doc: &mut Document) -> Proof { + let snap = Snapshot::capture(doc, None).unwrap(); + doc.require_full_save(); + snap.prove(doc, None, &ProofOptions::default()) +} + +pub(super) fn surface(p: &Proof, s: Surface) -> &SurfaceReport { + p.surfaces.iter().find(|r| r.surface == s).unwrap() +} + +fn residue(r: &Result) -> bool { + matches!(r, Err(RedactError::Residue(n)) if *n > 0) +} + +#[test] +fn the_proof_reads_a_pages_streams_as_the_one_stream_they_are() { + // The TJ array of the secret ends the first piece and its operator starts the second + // (#153): the proof reads the pieces as the one stream they are, so the removed text is + // neither reported as a survivor nor does the page count as unreadable. + let objs: Vec> = vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents [4 0 R 7 0 R] /Resources << /Font << /F1 5 0 R >> >> >>".to_vec(), + stream("", b"BT /F1 10 Tf 10 100 Td [(SECRETVALUE)]"), + FONT.as_bytes().to_vec(), + format!("[{}]", vec!["500"; 95].join(" ")).into_bytes(), + stream("", b"TJ ET BT /F1 10 Tf 10 200 Td (Public) Tj ET"), + ]; + let mut doc = open(pdf_bytes(objs, "/Root 1 0 R")); + mark(&mut doc, &[REGION]); + let (r, proof) = run(&mut doc); + let r = r.unwrap(); + assert_eq!(r.glyphs, 11, "every glyph of the split string is removed: {r:?}"); + assert!(proof.passed(), "{}", proof.to_text()); + assert_eq!(proof.survivors(), 0); + let saved = write_incremental(&doc, &SaveOptions::default()).unwrap(); + assert!(!saved.windows(SECRET.len()).any(|w| w == SECRET.as_bytes()), "the secret is gone from the file"); +} + +#[test] +fn a_clean_redaction_passes_and_the_neighbour_is_untouched() { + let mut doc = marked(Spec::default()); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + assert_eq!(proof.entries.len(), 1); + let e = &proof.entries[0]; + assert_eq!((e.status, e.page, e.region), (Status::Verified, 0, REGION)); + assert_eq!((e.runs_removed, e.codes_removed), (1, SECRET.len())); + let content = page_content(&doc); + assert!(content.contains("(Public) Tj") && !content.contains(SECRET), "{content}"); + for s in &proof.surfaces { + assert!(matches!(s.verdict, Verdict::Clean | Verdict::Absent), "{s:?}"); + } + assert_eq!(surface(&proof, Surface::Revisions).verdict, Verdict::Clean); +} + +#[test] +fn the_manifest_has_digests_and_counts_but_no_plaintext() { + let mut doc = marked(Spec::default()); + // The digest of the original content: a shared proof must not carry it. + let original: String = sha2::Sha256::digest(page_content(&doc).as_bytes()).iter().map(|b| format!("{b:02x}")).collect(); + let (r, proof) = run(&mut doc); + r.unwrap(); + let e = &proof.entries[0]; + let after = e.page_sha256_after.as_deref().unwrap(); + assert!(after.len() == 64 && after.bytes().all(|c| c.is_ascii_hexdigit())); + assert_ne!(after, original); + for shown in [proof.to_text(), format!("{proof:?}")] { + assert!(!shown.contains(&original), "{shown}"); + } + // Two glyph-carrying operators before; the removed one left a number-only TJ, which carries + // no glyphs and is not counted. + assert_eq!((e.text_ops_before, e.text_ops_after), (Some(2), Some(1))); + assert!(e.removed_text.is_none()); + for shown in [proof.to_text(), format!("{proof:?}")] { + assert!(!shown.contains(SECRET), "{shown}"); + } + + // Plaintext only on request. + let mut doc = marked(Spec::default()); + let mut snap = Snapshot::capture(&doc, None).unwrap(); + assert!(!format!("{snap:?}").contains(SECRET)); + applied(&mut doc, &mut snap).unwrap(); + let with = snap.prove(&doc, None, &ProofOptions { include_plaintext: true }); + assert_eq!(with.entries[0].removed_text.as_deref(), Some(&[SECRET.to_string()][..])); +} + +#[test] +fn survivors_do_not_echo_the_text_either() { + let mut doc = marked(Spec { trailer: "/Info 7 0 R", extra: vec![b"<< /Title (SECRETVALUE) >>".to_vec()], ..Spec::default() }); + let (_, proof) = run(&mut doc); + assert!(!proof.passed()); + for shown in [proof.to_text(), format!("{proof:?}")] { + assert!(!shown.contains(SECRET), "{shown}"); + } +} + +#[test] +fn numeric_only_tj_is_not_glyph_carrying() { + assert_eq!(count_text_operators(b"BT (Tj) Tj [ -250 ] TJ [(x) -3 (y)] TJ () Tj [] TJ ET"), 2); + assert_eq!(count_text_operators(b"BT (a) ' 1 2 (b) \" [ 10 -10 ] TJ ET"), 2); +} + +#[test] +fn a_survivor_in_the_info_dictionary() { + let mut doc = + marked(Spec { trailer: "/Info 7 0 R", extra: vec![b"<< /Title (Report SECRETVALUE) /Author (Someone) >>".to_vec()], ..Spec::default() }); + let (_, proof) = run(&mut doc); + assert_eq!(surface(&proof, Surface::Info).verdict, Verdict::Survivor); + assert_eq!(proof.entries[0].status, Status::Failed); + assert!(!proof.passed()); +} + +#[test] +fn a_survivor_in_xmp() { + let xmp = stream("/Type /Metadata /Subtype /XML", b"Quarterly & SECRETVALUE"); + let mut doc = marked(Spec { catalog: "/Metadata 7 0 R", extra: vec![xmp], ..Spec::default() }); + let (_, proof) = run(&mut doc); + assert_eq!(surface(&proof, Surface::Xmp).verdict, Verdict::Survivor, "{}", proof.to_text()); + assert!(!proof.passed()); +} + +#[test] +fn a_long_numeric_reference_in_xmp_is_a_survivor() { + // A character reference may carry any number of leading zeros: the padded form reads the + // same, and is swept the same. + let xmp = stream("/Type /Metadata /Subtype /XML", b"SECRETVALUE"); + let mut doc = marked(Spec { catalog: "/Metadata 7 0 R", extra: vec![xmp], ..Spec::default() }); + let (_, proof) = run(&mut doc); + assert_eq!(surface(&proof, Surface::Xmp).verdict, Verdict::Survivor, "{}", proof.to_text()); + assert!(!proof.passed()); +} + +#[test] +fn a_survivor_in_an_annotation_string() { + let annot = b"<< /Type /Annot /Subtype /Text /Rect [200 200 220 220] /Contents (note: SECRETVALUE) >>".to_vec(); + let mut doc = marked(Spec { page: "/Annots [7 0 R]", extra: vec![annot], ..Spec::default() }); + let (_, proof) = run(&mut doc); + assert_eq!(surface(&proof, Surface::ObjectStrings).verdict, Verdict::Survivor); + assert_eq!(proof.entries[0].status, Status::Failed); + assert!(!proof.passed()); +} + +// ── Every place a reader could recover the removed text from ──────────────────────────────── +// Each test plants the removed text in one location and expects the sweep to report it there. + +#[test] +fn a_survivor_in_a_compressed_object_stream_is_found() { + // A full save packs non-stream objects into compressed object streams: a copy of the + // removed text left in one (an annotation away from the region) is neither in the stored + // bytes nor on any page, only inside the /ObjStm. The sweep reads the saved file like any + // reader, objects in object streams included, and reports the copy. + let annot = b"<< /Type /Annot /Subtype /Text /Rect [200 200 220 220] /Contents (note: SECRETVALUE) >>".to_vec(); + let mut doc = marked(Spec { page: "/Annots [7 0 R]", extra: vec![annot], ..Spec::default() }); + let mut snap = Snapshot::capture(&doc, None).unwrap(); + applied(&mut doc, &mut snap).unwrap(); + let saved = write_full(&doc, &SaveOptions::default()).unwrap(); + assert!(saved.windows(b"/ObjStm".len()).any(|w| w == b"/ObjStm"), "the output packs objects into object streams"); + assert!(!saved.windows(SECRET.len()).any(|w| w == SECRET.as_bytes()), "the plaintext is compressed away"); + let proof = snap.prove_saved(&saved, None, &ProofOptions::default()); + let s = surface(&proof, Surface::ObjectStrings); + assert_eq!(s.verdict, Verdict::Survivor, "{}", proof.to_text()); + assert_eq!(surface(&proof, Surface::RawBytes).verdict, Verdict::Clean, "the copy sits in the object stream, not the stored bytes"); + assert!(!proof.passed(), "{}", proof.to_text()); +} + +#[test] +fn a_survivor_in_a_page_thumbnail_is_found() { + // The page's /Thumb holds a picture of the page as it was, secret included (uncompressed + // pixels, so the stored bytes are the pixels). A thumbnail is not page content, so only + // the object sweep can find it. + let thumb = stream("/Type /XObject /Subtype /Image /Width 7 /Height 3 /ColorSpace /DeviceGray /BitsPerComponent 8", b"junk SECRETVALUE junk"); + let mut doc = marked(Spec { page: "/Thumb 7 0 R", extra: vec![thumb], ..Spec::default() }); + let (_, proof) = run(&mut doc); + let s = surface(&proof, Surface::DecodedStreams); + assert_eq!(s.verdict, Verdict::Survivor, "{}", proof.to_text()); + assert!(s.locations.iter().any(|l| l.contains("(raw)")), "{:?}", s.locations); + assert!(!proof.passed(), "{}", proof.to_text()); +} + +#[test] +fn a_survivor_in_a_form_field_value_is_found() { + // The redacted text also sits in the form: a text field whose /V holds it. The field + // dictionary is an object like any other, and its strings are swept wherever the field + // hangs (AcroForm and the page's annotations both point at it). + let field = b"<< /Type /Annot /Subtype /Widget /Rect [200 20 280 40] /FT /Tx /T (form) /V (note: SECRETVALUE) >>".to_vec(); + let spec = Spec { catalog: "/AcroForm << /Fields [7 0 R] >>", page: "/Annots [7 0 R]", extra: vec![field], ..Spec::default() }; + let mut doc = marked(spec); + let (_, proof) = run(&mut doc); + let s = surface(&proof, Surface::ObjectStrings); + assert_eq!(s.verdict, Verdict::Survivor, "{}", proof.to_text()); + // The field's copy is the only place left (the save renumbers objects, so the place is + // named, not numbered from the input file; both views of the string count as survivors). + assert!(s.survivors >= 1, "{}", proof.to_text()); + assert_eq!(s.locations.len(), 1, "{:?}", s.locations); + assert!(!proof.passed(), "{}", proof.to_text()); +} + +#[test] +fn a_survivor_in_an_xfa_packet_is_found() { + // An XFA form mirrors its field values in the /XFA packets, XML of their own. Redaction + // strips an XFA form outright (crates/xfa does that, not this machinery); the proof must + // not take the stripper on trust: whatever it left (a shape it refused, a leftover + // packet) is swept like any stream, and what the packet holds is reported. + let xfa = stream("", b"
SECRETVALUE
"); + let spec = Spec { catalog: "/AcroForm << /XFA 7 0 R /Fields [] >>", extra: vec![xfa], ..Spec::default() }; + let doc = marked(spec); + let snap = Snapshot::capture(&doc, None).unwrap(); + let proof = snap.prove_saved(doc.bytes(), None, &ProofOptions::default()); + let s = surface(&proof, Surface::DecodedStreams); + assert_eq!(s.verdict, Verdict::Survivor, "{}", proof.to_text()); + assert!(s.locations.iter().any(|l| l == "object 7 (decoded)"), "{:?}", s.locations); + assert!(!proof.passed(), "{}", proof.to_text()); +} + +#[test] +fn a_survivor_in_an_unused_object_is_found() { + // Object 7 is reachable from nothing: no catalog entry, no page resource. A reader that + // only walks the document tree misses it; the sweep walks every object the + // cross-reference table knows, orphans included, and names the object it sits in. + let orphan = format!("<< /Type /Annot /Subtype /Text /Contents (note: {SECRET}) >>").into_bytes(); + let doc = marked(Spec { extra: vec![orphan], ..Spec::default() }); + let snap = Snapshot::capture(&doc, None).unwrap(); + let proof = snap.prove_saved(doc.bytes(), None, &ProofOptions::default()); + let s = surface(&proof, Surface::ObjectStrings); + assert_eq!(s.verdict, Verdict::Survivor, "{}", proof.to_text()); + assert!(s.locations.iter().any(|l| l == "object 7"), "{:?}", s.locations); + assert!(!proof.passed(), "{}", proof.to_text()); +} + +/// The page draws "(Image)" under the marked region, and keeps an image XObject whose +/// `/Subtype /Image` is the same word. +fn image_word_doc() -> Document { + let image = stream("/Type /XObject /Subtype /Image /Width 20 /Height 20 /ColorSpace /DeviceGray /BitsPerComponent 8", &[0x80u8; 400]); + let content = b"BT /F1 10 Tf 10 100 Td (Image) Tj ET"; + let spec = Spec { resources: "/XObject << /Im1 7 0 R >>", extra: vec![image], ..Spec::default() }; + let mut doc = open(build_with(content, spec)); + mark(&mut doc, &[[0.0, 90.0, 40.0, 115.0]]); + doc +} + +#[test] +fn a_word_that_remains_in_a_structural_name_does_not_fail_the_proof() { + // "Image" is redacted from the page, and is also the /Subtype of the image the page keeps. + // The name is structure redaction never edits, not a leftover copy: the proof must pass, + // and the name surface must read clean, not unswept. + let mut doc = image_word_doc(); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + assert_eq!(surface(&proof, Surface::ObjectStrings).verdict, Verdict::Clean); + + // But a name the input did not have is a copy like any other: a surplus of name matches + // is a survivor, wherever it sits. + let snap = Snapshot::capture(&image_word_doc(), None).unwrap(); + let image = stream("/Type /XObject /Subtype /Image /Width 20 /Height 20 /ColorSpace /DeviceGray /BitsPerComponent 8", &[0x80u8; 400]); + let spec = Spec { resources: "/XObject << /Im1 7 0 R >>", catalog: "/Extra << /Stolen /Image >>", extra: vec![image], ..Spec::default() }; + let mut shown = open(build_with(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", spec)); + mark(&mut shown, &[[0.0, 90.0, 40.0, 115.0]]); + shown.require_full_save(); + let proof = snap.prove(&shown, None, &ProofOptions::default()); + assert_eq!(surface(&proof, Surface::ObjectStrings).verdict, Verdict::Survivor, "{}", proof.to_text()); + assert!(!proof.passed(), "{}", proof.to_text()); +} + +fn attachment_spec(payload: &[u8]) -> Spec<'static> { + Spec { + catalog: "/Names << /EmbeddedFiles << /Names [(a.bin) 7 0 R] >> >>", + extra: vec![b"<< /Type /Filespec /F (a.bin) /EF << /F 8 0 R >> >>".to_vec(), stream("/Type /EmbeddedFile", payload)], + ..Spec::default() + } +} + +#[test] +fn a_survivor_in_an_attachment() { + let mut doc = marked(attachment_spec(b"the SECRETVALUE is attached")); + let (_, proof) = run(&mut doc); + assert_eq!(surface(&proof, Surface::EmbeddedFiles).verdict, Verdict::Survivor); + assert!(!proof.passed()); + + // UTF-16LE with a byte order mark. + let mut le = vec![0xFF, 0xFE]; + le.extend(SECRET.encode_utf16().flat_map(u16::to_le_bytes)); + let mut doc = marked(attachment_spec(&le)); + let (_, proof) = run(&mut doc); + assert_eq!(surface(&proof, Surface::EmbeddedFiles).verdict, Verdict::Survivor); + assert!(!proof.passed()); +} + +#[test] +fn a_survivor_in_a_nested_attachment() { + let inner = build_with(b"BT /F1 10 Tf 10 100 Td (SECRETVALUE) Tj ET", Spec::default()); + let middle = build_with(b"", attachment_spec(&inner)); + let mut doc = marked(attachment_spec(&middle)); + let (_, proof) = run(&mut doc); + let s = surface(&proof, Surface::EmbeddedFiles); + assert_eq!(s.verdict, Verdict::Survivor); + assert!(s.locations.iter().any(|l| l.matches(" > ").count() >= 1), "{:?}", s.locations); + assert!(!proof.passed(), "{}", proof.to_text()); +} + +#[test] +fn a_clean_nested_attachment_is_swept_and_clean() { + let inner = build_with(b"BT /F1 10 Tf 10 100 Td (harmless) Tj ET", Spec::default()); + let mut doc = marked(attachment_spec(&inner)); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + assert_eq!(surface(&proof, Surface::EmbeddedFiles).verdict, Verdict::Clean); +} + +#[test] +fn unsweepable_attachments_fail_closed() { + for payload in [&b"PK\x03\x04 zipped"[..], b"%PDF-1.7 not really a pdf", b"\x1f\x8b gzip"] { + let mut doc = marked(attachment_spec(payload)); + let (r, proof) = run(&mut doc); + assert!(r.is_ok() || residue(&r), "only a residue refusal may stop the apply itself: {r:?}"); + assert_eq!(surface(&proof, Surface::EmbeddedFiles).verdict, Verdict::Unswept, "{}", String::from_utf8_lossy(payload)); + assert!(!proof.passed(), "a proof that could not sweep must not pass"); + } +} + +#[test] +fn an_undecodable_stream_fails_closed() { + let junk = stream("/Filter /NoSuchDecode", b"\x00\x01\x02 junk"); + let mut doc = marked(Spec { catalog: "/Extra 7 0 R", extra: vec![junk], ..Spec::default() }); + let (_, proof) = run(&mut doc); + assert_eq!(surface(&proof, Surface::DecodedStreams).verdict, Verdict::Unswept, "{}", proof.to_text()); + assert!(!proof.passed()); +} + +#[test] +fn an_incremental_remnant_fails_the_proof() { + let mut doc = marked(Spec::default()); + let mut snap = Snapshot::capture(&doc, None).unwrap(); + // A "redaction" saved as an incremental update: the page shows something else now, but the + // first revision still holds the original content. + let mut other = open(write_incremental(&doc, &SaveOptions::default()).unwrap()); + let new = other.add(Object::Stream(Stream::flate(Dict::new(), b"BT /F1 10 Tf 10 200 Td (Public) Tj ET"))); + let page = pdfcraft_model::pages(&other).swap_remove(0); + other.update_dict(page.obj, |d| d.set(b"Contents".to_vec(), Object::Ref(new))).unwrap(); + let saved = write_incremental(&other, &SaveOptions::default()).unwrap(); + let proof = snap.prove_saved(&saved, None, &ProofOptions::default()); + assert!(!proof.passed()); + assert_eq!(surface(&proof, Surface::Revisions).verdict, Verdict::Survivor); + assert_eq!(surface(&proof, Surface::RawBytes).verdict, Verdict::Survivor); + // The page text itself is clean: only the file structure gives it away. + assert_eq!(surface(&proof, Surface::ExtractedText).verdict, Verdict::Clean); + + // The real redaction, saved as apply() requires, is a single clean revision. + applied(&mut doc, &mut snap).unwrap(); + let full = write_incremental(&doc, &SaveOptions::default()).unwrap(); + let proof = snap.prove_saved(&full, None, &ProofOptions::default()); + assert!(proof.passed(), "{}", proof.to_text()); +} + +#[test] +fn the_secret_in_each_encoding_is_found() { + // Spelled as PDF syntax in a catalog entry. + let utf16: String = SECRET.encode_utf16().flat_map(u16::to_be_bytes).map(|b| format!("{b:02X}")).collect(); + let utf16le: String = SECRET.encode_utf16().flat_map(u16::to_le_bytes).map(|b| format!("{b:02X}")).collect(); + let hex8: String = SECRET.bytes().map(|b| format!("{b:02x}")).collect(); + let forms = [ + format!("({SECRET})"), + format!(""), + format!(""), + format!("", hex8.to_uppercase()), + format!("<{hex8}>"), + format!("<{utf16}>"), + "(\\123ECRETVALUE)".to_string(), + ]; + for f in forms { + let mut doc = marked(Spec { catalog: &format!("/Extra {f}"), ..Spec::default() }); + let (_, proof) = run(&mut doc); + assert!(!proof.passed(), "{f}: {}", proof.to_text()); + assert_eq!(surface(&proof, Surface::ObjectStrings).verdict, Verdict::Survivor, "{f}: {}", proof.to_text()); + } + + // Spelled as bytes in the file, including the backslash-escaped literal form. + let paren = "SECR(ET)X\\Y"; + let enc = survivor_encodings(paren); + assert!(enc.len() >= 10); + for spelling in enc { + let mut hay = b"junk ".to_vec(); + hay.extend(&spelling); + hay.extend(b" junk"); + assert!(tests_support::found(paren, &hay), "{}", String::from_utf8_lossy(&spelling)); + } + assert!(!tests_support::found(paren, b"nothing to see")); +} + +#[test] +fn a_rotated_page_is_proven_too() { + let mut doc = marked(Spec { page: "/Rotate 90", ..Spec::default() }); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + assert_eq!(proof.entries[0].status, Status::Verified); +} + +#[test] +fn a_region_without_text_is_recorded_as_such() { + let mut doc = open(build_with(CONTENT, Spec::default())); + mark(&mut doc, &[[150.0, 20.0, 200.0, 40.0]]); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert_eq!(proof.entries[0].status, Status::VerifiedNoTextInRegion); + assert!(proof.passed()); +} + +fn opaque_spec() -> Spec<'static> { + Spec { resources: "/Font << /F1 5 0 R /F2 7 0 R >>", extra: vec![OPAQUE_FONT.as_bytes().to_vec()], ..Spec::default() } +} + +#[test] +fn text_that_cannot_be_decoded_makes_a_region_unverifiable() { + let mut doc = open(build_with(b"BT /F2 10 Tf 10 200 Td <00410042> Tj ET", opaque_spec())); + mark(&mut doc, &[[150.0, 20.0, 200.0, 40.0]]); + let (r, proof) = run(&mut doc); + assert!(r.is_ok() || residue(&r), "only a residue refusal may stop the apply itself: {r:?}"); + assert_eq!(proof.entries[0].status, Status::Unverifiable); + assert!(!proof.passed(), "no PASS while a region is unverifiable"); +} + +#[test] +fn removed_text_of_an_undecodable_font_is_swept_by_its_codes() { + let content = b"BT /F2 10 Tf 10 100 Td <0041004200430044> Tj ET"; + let mut doc = open(build_with(content, opaque_spec())); + mark(&mut doc, &[[0.0, 90.0, 100.0, 120.0]]); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert_eq!(proof.entries[0].runs_removed, 1); + assert_eq!(proof.entries[0].status, Status::Verified, "{}", proof.to_text()); + + // Left in place, the codes are still on the page and the proof says so. + let mut doc = open(build_with(content, opaque_spec())); + mark(&mut doc, &[[0.0, 90.0, 100.0, 120.0]]); + let proof = prove_unapplied(&mut doc); + assert!(!proof.passed()); + assert_eq!(surface(&proof, Surface::ExtractedText).verdict, Verdict::Survivor); +} + +#[test] +fn codes_removed_counts_codes_not_bytes() { + // A composite font's codes are two bytes each: the manifest counts the codes, not the bytes. + let content = b"BT /F2 10 Tf 10 100 Td <0041004200430044> Tj ET"; + let mut doc = open(build_with(content, opaque_spec())); + mark(&mut doc, &[[0.0, 90.0, 100.0, 120.0]]); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert_eq!(proof.entries[0].runs_removed, 1); + assert_eq!(proof.entries[0].codes_removed, 4, "{}", proof.to_text()); + + // A simple font's codes are one byte each: the two counts agree. + let mut doc = marked(Spec::default()); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert_eq!((proof.entries[0].runs_removed, proof.entries[0].codes_removed), (1, SECRET.len())); +} + +#[test] +fn overlapping_marks_each_account_for_their_own_share() { + // Two regions over the same text: each gets the runs, codes and status it is owed — the + // second one is not waved through as "no text in the region". + let mut doc = open(build_with(CONTENT, Spec::default())); + mark(&mut doc, &[REGION, [7.0, 94.0, 71.0, 113.0]]); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert_eq!(proof.entries.len(), 2); + for e in &proof.entries { + assert_eq!((e.runs_removed, e.codes_removed), (1, SECRET.len()), "{}", proof.to_text()); + assert_eq!(e.status, Status::Verified, "{}", proof.to_text()); + } + + // Left in place, both regions name the survivor as theirs. + let mut doc = open(build_with(CONTENT, Spec::default())); + mark(&mut doc, &[REGION, [7.0, 94.0, 71.0, 113.0]]); + let proof = prove_unapplied(&mut doc); + assert!(!proof.passed()); + assert!(proof.entries.iter().all(|e| matches!(e.status, Status::Failed | Status::Unverifiable)), "{}", proof.to_text()); +} + +#[test] +fn nothing_applied_means_survivors_where_the_text_was() { + let mut doc = marked(Spec::default()); + let proof = prove_unapplied(&mut doc); + assert!(!proof.passed()); + assert_eq!(surface(&proof, Surface::ExtractedText).verdict, Verdict::Survivor); + assert_eq!(surface(&proof, Surface::DecodedStreams).verdict, Verdict::Survivor); + assert_eq!(proof.entries[0].status, Status::Failed); +} + +#[test] +fn a_document_that_would_save_incrementally_is_not_proven() { + let mut doc = marked(Spec::default()); + let mut snap = Snapshot::capture(&doc, None).unwrap(); + applied(&mut doc, &mut snap).unwrap(); + // A reopened copy no longer carries the full-save requirement. + let reopened = open(write_incremental(&doc, &SaveOptions::default()).unwrap()); + let proof = snap.prove(&reopened, None, &ProofOptions::default()); + assert_eq!(surface(&proof, Surface::Revisions).verdict, Verdict::Survivor); +} + +fn image_page() -> Document { + let image = stream("/Type /XObject /Subtype /Image /Width 20 /Height 20 /ColorSpace /DeviceGray /BitsPerComponent 8", &[0xFFu8; 400]); + let spec = Spec { resources: "/XObject << /Im1 7 0 R >>", extra: vec![image], ..Spec::default() }; + marked_region(b"q 100 0 0 100 100 100 cm /Im1 Do Q 0 0 1 rg 120 120 60 60 re f", spec) +} + +fn marked_region(content: &[u8], spec: Spec<'_>) -> Document { + let mut doc = open(build_with(content, spec)); + mark(&mut doc, &[[100.0, 100.0, 150.0, 150.0]]); + doc +} + +#[test] +fn images_and_paths_under_a_region_are_checked_independently() { + let mut doc = image_page(); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + assert_eq!(proof.entries[0].status, Status::VerifiedNoTextInRegion); + + // The same page unredacted: the image and the path are both still there. + let proof = prove_unapplied(&mut image_page()); + let e = &proof.entries[0]; + assert_eq!(e.status, Status::Failed); + assert!(e.detail.contains("image") && e.detail.contains("path"), "{}", e.detail); +} + +#[test] +fn a_shading_under_a_region_must_be_clipped() { + let sh = + b"<< /ShadingType 2 /ColorSpace /DeviceRGB /Coords [0 0 300 0] /Function << /FunctionType 2 /Domain [0 1] /C0 [1 0 0] /C1 [0 0 1] /N 1 >> >>" + .to_vec(); + let spec = || Spec { resources: "/Shading << /Sh1 7 0 R >>", extra: vec![sh.clone()], ..Spec::default() }; + let mut doc = marked_region(b"/Sh1 sh", spec()); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + + let proof = prove_unapplied(&mut marked_region(b"/Sh1 sh", spec())); + assert_eq!(proof.entries[0].status, Status::Failed); +} + +#[test] +fn an_even_odd_clip_around_the_region_hides_what_it_cuts_out() { + // How kept content is told to hide: an even-odd clip of the page and the region. A path + // painted inside the region under that clip is clipped away; the same path under a clip + // the rule cannot read (a triangle, overlapping inners), or with no clip, is reported. + let blank = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()); + let snap = Snapshot::capture(&blank, None).unwrap(); + let prove = |tail: &str| -> Proof { + let content = format!("BT /F1 10 Tf 10 200 Td (Public) Tj ET {tail}"); + let mut shown = marked_content(content.as_bytes(), Spec::default()); + shown.require_full_save(); + snap.prove(&shown, None, &ProofOptions::default()) + }; + let proof = prove("q 0 0 300 300 re 8 95 62 17 re W* n 0 0 0 rg 10 100 20 10 re f Q"); + assert!(proof.passed(), "{}", proof.to_text()); + + // A triangle is not a rectangle: no hole is cut, and the paint is reported. + let proof = prove("q 0 0 300 300 re 8 95 m 70 95 l 8 112 l h W* n 0 0 0 rg 10 100 20 10 re f Q"); + assert_eq!(proof.entries[0].status, Status::Failed, "{}", proof.to_text()); + assert!(proof.entries[0].detail.contains("path"), "{}", proof.to_text()); + + // Overlapping inners change the parity: no hole is cut either. + let proof = prove("q 0 0 300 300 re 8 95 62 17 re 5 90 70 25 re W* n 0 0 0 rg 10 100 20 10 re f Q"); + assert_eq!(proof.entries[0].status, Status::Failed, "{}", proof.to_text()); + + // Unclipped, the same path is a plain finding. + let proof = prove("0 0 0 rg 10 100 20 10 re f"); + assert_eq!(proof.entries[0].status, Status::Failed, "{}", proof.to_text()); +} + +#[test] +fn a_graphics_state_stack_too_deep_to_follow_is_unverifiable() { + // Saved states carry their clip holes with them: a `q` stack deep enough to grow without + // bound is refused, not followed. + let blank = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()); + let snap = Snapshot::capture(&blank, None).unwrap(); + let mut deep = Vec::new(); + for _ in 0..crate::verify::limits::MAX_GS_STACK + 8 { + deep.extend_from_slice(b"q "); + } + deep.extend_from_slice(b"0 0 0 rg 10 100 20 10 re f"); + let mut shown = marked_content(&deep, Spec::default()); + shown.require_full_save(); + let proof = snap.prove(&shown, None, &ProofOptions::default()); + assert_eq!(proof.entries[0].status, Status::Unverifiable, "{}", proof.to_text()); + assert!(proof.entries[0].detail.contains("too deeply"), "{}", proof.to_text()); + assert!(!proof.passed()); +} + +#[test] +fn a_page_full_of_paint_reports_its_finding_once() { + // Fifty paths over a region are one finding: the report names the problem once, not once + // per operator, and the report does not grow with the operator count. + let blank = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()); + let snap = Snapshot::capture(&blank, None).unwrap(); + let mut content = b"BT /F1 10 Tf 10 200 Td (Public) Tj ET ".to_vec(); + for k in 0..50 { + content.extend_from_slice(&format!("0 0 1 rg {} 100 4 4 re f ", 10 + k).into_bytes()); + } + let mut shown = marked_content(&content, Spec::default()); + shown.require_full_save(); + let proof = snap.prove(&shown, None, &ProofOptions::default()); + assert_eq!(proof.entries[0].status, Status::Failed, "{}", proof.to_text()); + assert_eq!(proof.entries[0].detail.matches("a path still paints").count(), 1, "{}", proof.to_text()); +} + +#[test] +fn text_kept_elsewhere_does_not_fail_the_proof() { + // The same word is redacted in one place and legitimately remains in another. + let content = b"BT /F1 10 Tf 10 200 Td (SECRETVALUE) Tj ET BT /F1 10 Tf 10 100 Td (SECRETVALUE) Tj ET"; + let mut doc = marked_content(content, Spec::default()); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + assert!(proof.entries[0].detail.contains("remain in retained content")); + + // A copy that should have gone is still caught, by count. + let proof = prove_unapplied(&mut marked_content(content, Spec::default())); + assert_eq!(surface(&proof, Surface::ExtractedText).verdict, Verdict::Survivor); +} + +#[test] +fn the_proof_text_keeps_its_honesty_wording() { + // The closing caveat of a proof is a promise to whoever reads a shared proof: pin it so the + // wording cannot silently drop out of the text for either verdict. + let honesty = "Evidence, not a guarantee"; + let passed = Proof { entries: vec![], surfaces: vec![], failures: vec![] }; + assert!(passed.passed()); + assert!(passed.to_text().contains(honesty), "{}", passed.to_text()); + + let failed = Proof { entries: vec![], surfaces: vec![], failures: vec!["the marked text is still readable".into()] }; + assert!(!failed.passed()); + assert!(failed.to_text().contains(honesty), "{}", failed.to_text()); +} + +// ── Independence, scope and bounds of the sweep ───────────────────────────────────────────── + +/// Hang `obj` off the catalog (as `/Extra`) so the saved file keeps it. +pub(super) fn attach(doc: &mut Document, obj: Object) { + let r = doc.add(obj); + let root = doc.root().unwrap(); + doc.update_dict(root, |d| d.set(b"Extra".to_vec(), Object::Ref(r))).unwrap(); +} + +/// Add a content stream (with `dict`) after the page's own. +fn append_content(doc: &mut Document, dict: Dict, data: &[u8]) { + let new = doc.add(Object::Stream(Stream::from_raw(dict, data.to_vec()))); + let page = pdfcraft_model::pages(doc).swap_remove(0); + let first = page.dict.get(b"Contents").cloned().unwrap(); + doc.update_dict(page.obj, |d| d.set(b"Contents".to_vec(), Object::Array(vec![first, Object::Ref(new)]))).unwrap(); +} + +#[test] +fn a_glyph_the_interpreter_kept_is_found_by_placement_alone() { + // The snapshot is taken where the region holds no text, so it has nothing to search for: + // only the independent placement of glyphs can notice the text in the output. + let blank = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()); + let snap = Snapshot::capture(&blank, None).unwrap(); + let mut shown = marked(Spec::default()); + shown.require_full_save(); + let proof = snap.prove(&shown, None, &ProofOptions::default()); + assert!(!proof.passed()); + let e = &proof.entries[0]; + assert_eq!(e.status, Status::Failed, "{}", proof.to_text()); + assert!(e.detail.contains("glyph(s) still show"), "{}", e.detail); + assert_eq!(proof.survivors(), 0, "no needle was involved"); + // The neighbour at y 200 is outside the region and is not reported. + assert!(!e.detail.contains("12 glyph"), "{}", e.detail); +} + +#[test] +fn glyphs_are_placed_with_the_text_matrix_and_the_form_matrix() { + // The secret drawn through a rotated/translated text matrix inside a scaled form: the + // placement has to follow `Tm` and the form's `/Matrix`, not the raw operands. + let form = stream( + "/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Matrix [1 0 0 1 0 0] /Resources << /Font << /F1 5 0 R >> >>", + b"BT /F1 10 Tf 1 0 0 1 10 100 Tm (SECRETVALUE) Tj ET", + ); + let blank = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()); + let snap = Snapshot::capture(&blank, None).unwrap(); + let mut doc = marked_content(b"/Fm1 Do", Spec { resources: "/XObject << /Fm1 7 0 R >>", extra: vec![form], ..Spec::default() }); + doc.require_full_save(); + let proof = snap.prove(&doc, None, &ProofOptions::default()); + assert_eq!(proof.entries[0].status, Status::Failed, "{}", proof.to_text()); + assert!(proof.entries[0].detail.contains("glyph(s) still show")); +} + +#[test] +fn an_ambiguous_needle_is_still_swept_outside_page_content() { + // The word also stays on the page legitimately (so page text, the raw file and decoded + // streams can only count it), but a copy in the Info dictionary is not page content. + let content = b"BT /F1 10 Tf 10 200 Td (SECRETVALUE) Tj ET BT /F1 10 Tf 10 100 Td (SECRETVALUE) Tj ET"; + let spec = Spec { trailer: "/Info 7 0 R", extra: vec![b"<< /Title (SECRETVALUE) >>".to_vec()], ..Spec::default() }; + let mut doc = marked_content(content, spec); + let (_, proof) = run(&mut doc); + assert_eq!(surface(&proof, Surface::Info).verdict, Verdict::Survivor, "{}", proof.to_text()); + assert!(!proof.passed()); + + // Likewise an XMP packet and an annotation string. + let xmp = stream("/Type /Metadata /Subtype /XML", b"SECRETVALUE"); + let mut doc = marked_content(content, Spec { catalog: "/Metadata 7 0 R", extra: vec![xmp], ..Spec::default() }); + assert_eq!(surface(&run(&mut doc).1, Surface::Xmp).verdict, Verdict::Survivor); + let annot = b"<< /Type /Annot /Subtype /Text /Rect [200 200 220 220] /Contents (SECRETVALUE) >>".to_vec(); + let mut doc = marked_content(content, Spec { page: "/Annots [7 0 R]", extra: vec![annot], ..Spec::default() }); + assert_eq!(surface(&run(&mut doc).1, Surface::ObjectStrings).verdict, Verdict::Survivor); +} + +#[test] +fn a_pcmark_key_in_the_input_does_not_exempt_a_stream() { + // A content stream that claims to be a redaction box paints inside the region, and was not + // created by this run: it counts as paint like any other. + let blank = b"BT /F1 10 Tf 10 200 Td (Public) Tj ET"; + let mut doc = marked_content(blank, Spec::default()); + let mut d = Dict::new(); + d.set(b"PCMark".to_vec(), Object::Name(b"Redaction".to_vec())); + append_content(&mut doc, d, b"0 0 0 rg 8 95 62 17 re f"); + let proof = prove_unapplied(&mut doc); + let e = &proof.entries[0]; + assert_eq!(e.status, Status::Failed, "{}", proof.to_text()); + assert!(e.detail.contains("path"), "{}", e.detail); +} + +#[test] +fn only_the_streams_this_run_created_are_exempt() { + // The same stream, declared as created by this run, is the box that was drawn on purpose. + let blank = b"BT /F1 10 Tf 10 200 Td (Public) Tj ET"; + let mut doc = marked_content(blank, Spec::default()); + let mut snap = Snapshot::capture(&doc, None).unwrap(); + let new = doc.add(Object::Stream(Stream::from_raw(Dict::new(), b"0 0 0 rg 8 95 62 17 re f".to_vec()))); + let page = pdfcraft_model::pages(&doc).swap_remove(0); + let first = page.dict.get(b"Contents").cloned().unwrap(); + doc.update_dict(page.obj, |d| d.set(b"Contents".to_vec(), Object::Array(vec![first, Object::Ref(new)]))).unwrap(); + doc.require_full_save(); + snap.set_overlay_streams(&doc, &[new]).unwrap(); + let proof = snap.prove(&doc, None, &ProofOptions::default()); + assert!(proof.passed(), "{}", proof.to_text()); +} + +#[test] +fn overlay_text_does_not_collide_with_what_was_removed() { + // "Account" is redacted and the label says "Account [redacted]": the label is not a survivor. + let content = b"BT /F1 10 Tf 10 100 Td (Account 1234) Tj ET"; + let mut doc = open(build_with(content, Spec::default())); + let shape = Shape::Redact { quads: vec![rect_quad([8.0, 95.0, 100.0, 112.0])], overlay: "Account [redacted]".into(), look: Default::default() }; + let style = Style::default_for(&shape); + add_annotation(&mut doc, &NewAnnotation { page: 0, shape, style, contents: String::new(), author: "T".into() }, &Meta::default()).unwrap(); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + assert_eq!(surface(&proof, Surface::ExtractedText).verdict, Verdict::Clean); +} + +#[test] +fn removed_annotation_and_field_text_is_searched_for_too() { + let spec = || Spec { trailer: "/Info 7 0 R", extra: vec![b"<< /Subject (call back ANNOTWORDS) >>".to_vec()], ..Spec::default() }; + // Not asked for: the proof does not know about it. + let mut doc = marked(spec()); + let (_, proof) = run(&mut doc); + assert!(proof.passed(), "{}", proof.to_text()); + // Asked for: it is found where it was left. + let mut doc = marked(spec()); + let mut snap = Snapshot::capture(&doc, None).unwrap(); + snap.add_removed_strings(&[" call back ANNOTWORDS ".to_string(), "x".to_string(), String::new()]); + let result = applied(&mut doc, &mut snap); + assert!(result.is_ok() || residue(&result)); + let proof = snap.prove(&doc, None, &ProofOptions::default()); + assert!(!proof.passed()); + assert_eq!(surface(&proof, Surface::Info).verdict, Verdict::Survivor, "{}", proof.to_text()); + // And an absent string is clean. + let mut doc = marked(Spec::default()); + let mut snap = Snapshot::capture(&doc, None).unwrap(); + snap.add_removed_strings(&["never present anywhere".to_string()]); + applied(&mut doc, &mut snap).unwrap(); + assert!(snap.prove(&doc, None, &ProofOptions::default()).passed()); +} + +#[test] +fn a_damaged_stream_tail_is_unswept_not_truncated() { + let text: Vec = (0..4000u32).map(|i| b'a' + (i * 7 % 23) as u8).collect(); + let mut s = Stream::flate(Dict::new(), &text); + let cut = s.raw.len() - 8; + s.raw = Arc::new(s.raw[..cut].to_vec()).into(); + // Nothing was removed (the snapshot's region is blank), so the only issue is the stream. + let snap = Snapshot::capture(&marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()), None).unwrap(); + let mut doc = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()); + attach(&mut doc, Object::Stream(s)); + doc.require_full_save(); + let proof = snap.prove(&doc, None, &ProofOptions::default()); + assert_eq!(surface(&proof, Surface::DecodedStreams).verdict, Verdict::Unswept, "{}", proof.to_text()); + assert!(!proof.passed()); +} + +#[test] +fn a_damaged_page_content_tail_makes_the_page_unreadable() { + let text: Vec = b"BT /F1 10 Tf 10 100 Td (filler) Tj ET\n".repeat(200); + let mut s = Stream::flate(Dict::new(), &text); + let cut = s.raw.len() - 8; + s.raw = Arc::new(s.raw[..cut].to_vec()).into(); + let mut doc = marked(Spec::default()); + let page = pdfcraft_model::pages(&doc).swap_remove(0); + let new = doc.add(Object::Stream(s)); + doc.update_dict(page.obj, |d| d.set(b"Contents".to_vec(), Object::Ref(new))).unwrap(); + doc.require_full_save(); + let snap = Snapshot::capture(&marked(Spec::default()), None).unwrap(); + let proof = snap.prove(&doc, None, &ProofOptions::default()); + assert!(!proof.passed()); + assert_eq!(surface(&proof, Surface::ExtractedText).verdict, Verdict::Unswept, "{}", proof.to_text()); +} + +#[test] +fn a_malformed_form_matrix_or_bbox_is_unswept() { + for dict in ["/Matrix [1 0 0]", "/Matrix [1 0 0 1 0 (x)]", "/BBox [0 0 300]"] { + let form = stream(&format!("/Type /XObject /Subtype /Form /BBox [0 0 300 300] {dict}"), b"BT /F1 10 Tf 10 100 Td (x) Tj ET"); + let spec = Spec { resources: "/XObject << /Fm1 7 0 R >>", extra: vec![form], ..Spec::default() }; + let mut doc = marked_content(b"/Fm1 Do", spec); + let proof = prove_unapplied(&mut doc); + assert_eq!(surface(&proof, Surface::ExtractedText).verdict, Verdict::Unswept, "{dict}: {}", proof.to_text()); + assert!(!proof.passed()); + assert!(proof.entries[0].detail.contains("malformed"), "{dict}: {}", proof.to_text()); + } +} + +#[test] +fn images_hidden_in_patterns_soft_masks_and_type3_glyphs_are_not_skipped() { + let image = "BI /W 1 /H 1 /CS /G /BPC 8 ID \u{ff} EI"; + let tile = + |data: &str| stream("/PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 10 10] /XStep 10 /YStep 10 /Resources << >>", data.as_bytes()); + let group = stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Group << /S /Transparency >> /Resources << >>", image.as_bytes()); + let proc = stream("", format!("1000 0 d0 {image}").as_bytes()); + let cases: Vec<(&str, Spec)> = vec![ + ("pattern", Spec { resources: "/Pattern << /P1 7 0 R >>", extra: vec![tile(image)], ..Spec::default() }), + ( + "soft mask", + Spec { resources: "/ExtGState << /G1 << /SMask << /Type /Mask /S /Luminosity /G 7 0 R >> >> >>", extra: vec![group], ..Spec::default() }, + ), + ( + "type3", + Spec { + resources: "/Font << /F3 << /Type /Font /Subtype /Type3 /FontBBox [0 0 1000 1000] /FontMatrix [0.001 0 0 0.001 0 0] /CharProcs << /a 7 0 R >> /Encoding << /Type /Encoding /Differences [97 /a] >> /FirstChar 97 /LastChar 97 /Widths [1000] >> >>", + extra: vec![proc], + ..Spec::default() + }, + ), + ]; + for (what, spec) in cases { + // The region holds no text and no visible image: only the hidden image is in question. + let mut doc = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", spec); + let proof = prove_unapplied(&mut doc); + let e = &proof.entries[0]; + assert_eq!(e.status, Status::Unverifiable, "{what}: {}", proof.to_text()); + assert!(e.detail.contains("pattern, soft mask or Type3"), "{what}: {}", e.detail); + assert!(!proof.passed()); + } + // The same resources without an image are fine. + let spec = Spec { resources: "/Pattern << /P1 7 0 R >>", extra: vec![tile("0 0 5 5 re f")], ..Spec::default() }; + let mut doc = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", spec); + let proof = prove_unapplied(&mut doc); + assert_eq!(proof.entries[0].status, Status::VerifiedNoTextInRegion, "{}", proof.to_text()); +} + +#[test] +fn the_sweep_is_bounded() { + // A CMap whose ranges rewrite the same 64Ki codes again and again costs work, whatever the map size. + let mut cmap = b"1 begincodespacerange <0000> endcodespacerange\n".to_vec(); + for _ in 0..200 { + cmap.extend_from_slice(b"100 beginbfrange\n"); + for _ in 0..100 { + cmap.extend_from_slice(b"<0000> <0041>\n"); + } + cmap.extend_from_slice(b"endbfrange\n"); + } + let start = std::time::Instant::now(); + assert!(!tests_support::cmap_is_complete(&cmap)); + assert!(start.elapsed().as_secs() < 20, "bounded work"); + // A small CMap is read in full. + assert!(tests_support::cmap_is_complete(b"1 beginbfchar <01> <0041> endbfchar")); + // Nested objects, cumulative decoding and searching all stop and say so. + assert!(tests_support::walks_in_full(60)); + assert!(!tests_support::walks_in_full(200)); + assert_eq!(tests_support::decode_budget_runs_out(), (true, true)); + assert_eq!(tests_support::search_budget_runs_out(), (true, true)); + // A decode that fails past the one-stream cap pays for the attempt: the same bomb cannot + // be decoded (and fail) again and again for free. + assert_eq!(tests_support::failed_decode_charges_the_budget(), (true, true, true)); +} + +#[test] +fn the_search_finds_what_is_there_whatever_the_prefix() { + assert!(tests_support::found("aaaab", b"xxaaaaaaab")); + assert!(tests_support::found("needle", b"needle")); + assert!(!tests_support::found("needle", b"needl")); + assert!(!tests_support::found("aaab", b"aaaaaaaaaa")); + assert!(tests_support::found("abc", b"zabc")); + // The worst shape for a skip-based search (a needle of one letter and a haystack full of + // it) is walked once per byte, and still finds what is there. + let needle = format!("{}b", "a".repeat(64)); + let mut hay = vec![b'a'; 20_000]; + assert!(!tests_support::found(&needle, &hay)); + hay.extend_from_slice(needle.as_bytes()); + assert!(tests_support::found(&needle, &hay)); +} + +#[test] +fn the_sweep_charge_stays_fair_for_a_bulk_of_needles() { + // 80 removed strings swept over one 5 MiB haystack (a stream decoded in full): the search + // really is one pass per needle, but the budget must not charge the haystack once per + // needle — that charge (haystack × needle × spelling: 80 × 20 × 5 MiB, about twice the 4 GiB + // work budget) would refuse an ordinary bulk redaction out of hand. The haystack is charged + // once per scan and a needle's own charge is capped, so the sweep stays under the budget. + let hay = vec![b'x'; 5 << 20]; + let word = "SECRETVALUE-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + assert_eq!(tests_support::bulk_counts(&hay, word, 80, 0), Some(0)); + // Genuinely unbounded work still fails closed: the region scanned is charged in full, so a + // haystack that no longer fits in the remaining budget is an unswept surface, never a pass. + assert_eq!(tests_support::bulk_counts(&hay, word, 80, crate::verify::tests_support::work_budget() - (1 << 20)), None); +} + +#[test] +fn survivors_seen_before_the_work_limit_are_still_reported() { + // The page sweep stops when the work budget runs out, but the survivors it has already + // counted are settled: a survivor on an earlier page is reported where it sits, not hidden + // behind the limit (the page still reads unswept, so the proof still fails). + let (reported, unswept) = tests_support::page_sweep_survivors_survive_the_limit(); + assert!(reported, "a survivor counted before the limit is reported"); + assert!(unswept, "the page the limit stopped on is unswept"); +} + +#[test] +fn a_file_past_the_proof_budget_is_unverified_never_clean() { + // A page whose content is bigger than the proof may decode for one stream: the budget + // stops the sweep, and the result must say so. The surfaces read unswept (a page content + // stream is never excused as raw-only), the region reads unverifiable, the proof fails. + // Work that stopped is "not verified", never a pass. + let big = vec![0u8; crate::verify::limits::MAX_STREAM + (6 << 20)]; + let content = Stream::flate(Dict::new(), &big); + drop(big); + let mut doc = open(build_with(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default())); + let page = pdfcraft_model::pages(&doc).swap_remove(0); + let new = doc.add(Object::Stream(content)); + doc.update_dict(page.obj, |d| d.set(b"Contents".to_vec(), Object::Ref(new))).unwrap(); + doc.require_full_save(); + let snap = Snapshot::capture(&marked(Spec::default()), None).unwrap(); + let proof = snap.prove(&doc, None, &ProofOptions::default()); + assert!(!proof.passed(), "{}", proof.to_text()); + assert!(!proof.failures.is_empty(), "{}", proof.to_text()); + for s in [Surface::ExtractedText, Surface::DecodedStreams] { + let r = surface(&proof, s); + assert_eq!(r.verdict, Verdict::Unswept, "{s:?}: {}", proof.to_text()); + assert_eq!((r.survivors, r.raw_only), (0, 0), "{s:?}: {}", proof.to_text()); + } + assert!(matches!(proof.entries[0].status, Status::Unverifiable | Status::Failed), "the region is not verified: {}", proof.to_text()); +} + +#[test] +fn font_widths_parsing_shares_the_work_budget() { + // A form invoked again rebuilds its fonts, so the `/W` array is paid for from the shared + // work budget, not from a per-font counter that every rebuild resets. + let (full, refused) = tests_support::font_widths_charge_shares_the_budget(); + assert!(full, "a small /W array is read in full"); + assert!(refused, "a spent work budget refuses the /W parsing"); +} + +#[test] +fn an_unreadable_overlay_stream_is_its_own_error() { + let mut doc = marked(Spec::default()); + let mut snap = Snapshot::capture(&doc, None).unwrap(); + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::Name(b"FlateDecode".to_vec())); + let broken = doc.add(Object::Stream(Stream::from_raw(dict, b"not deflate data".to_vec()))); + let err = snap.set_overlay_streams(&doc, &[broken]).unwrap_err(); + assert_eq!(err, RedactError::OverlayUnreadable); + assert!(!err.to_string().contains("page 0"), "{err}"); +} + +// ── The caller's credentials ───────────────────────────────────────────────────────────────── + +#[test] +fn a_nonempty_user_password_document_is_proven_with_those_credentials() { + // The redacted document is protected with a user password and saved: the proof's subject is + // now an encrypted file, and only the caller's own credentials can read it back. Every + // supported encryption algorithm. + for alg in [Algorithm::Rc4_40, Algorithm::Rc4_128, Algorithm::Aes128, Algorithm::Aes256] { + let mut doc = marked(Spec::default()); + let mut snap = Snapshot::capture(&doc, None).unwrap(); + applied(&mut doc, &mut snap).unwrap(); + doc.set_encryption(&NewEncryption { + algorithm: alg, + user_password: "pw-secret", + owner_password: "owner-secret", + permissions: -1, + encrypt_metadata: true, + seed: [7; 32], + }) + .unwrap(); + + // The credentials the caller holds prove the saved output. + let proof = snap.prove(&doc, Some("pw-secret"), &ProofOptions::default()); + assert!(proof.passed(), "{alg:?}: {}", proof.to_text()); + + // An already saved file too. + let saved = write_full(&doc, &SaveOptions::default()).unwrap(); + let proof = snap.prove_saved(&saved, Some("pw-secret"), &ProofOptions::default()); + assert!(proof.passed(), "{alg:?}: {}", proof.to_text()); + + // Empty and wrong credentials fail closed: the output can't be read back, which is an + // unswept proof, never a pass. + for pw in [None, Some("wrong")] { + let proof = snap.prove_saved(&saved, pw, &ProofOptions::default()); + assert!(!proof.passed(), "{alg:?} {pw:?}: {}", proof.to_text()); + for s in &proof.surfaces { + assert_eq!(s.verdict, Verdict::Unswept, "{alg:?} {pw:?}: {}", proof.to_text()); + } + } + } +} + +// ── Compressed copies inside binary streams ────────────────────────────────────────────────── + +/// A stream object whose stored bytes are the Flate encoding of `data` (as `Stream::flate` +/// writes them), with the extra dictionary entries `dict`. +fn flate_object(dict: &str, data: &[u8]) -> Vec { + stream(&format!("{dict} /Filter /FlateDecode"), &Stream::flate(Dict::new(), data).raw) +} + +/// A one-page file whose resources point at a fake font: /F2 is a /TrueType font whose +/// /FontDescriptor carries `font_file` as its /FontFile2 (objects 7, 8 and 9). The font carries +/// `/Widths` (object 6) so the interpreter resolves it; nothing on the page draws with it. +fn font_file_spec(font_file: Vec) -> Spec<'static> { + Spec { + resources: "/Font << /F1 5 0 R /F2 7 0 R >>", + extra: vec![ + b"<< /Type /Font /Subtype /TrueType /BaseFont /X /FirstChar 32 /LastChar 126 /Widths 6 0 R /FontDescriptor 8 0 R >>".to_vec(), + b"<< /Type /FontDescriptor /FontName /X /Flags 4 /FontFile2 9 0 R >>".to_vec(), + font_file, + ], + ..Spec::default() + } +} + +#[test] +fn a_compressed_copy_in_a_fake_font_file_is_found() { + // The visible copy is redacted normally, but a Flate-compressed copy of the secret hides in + // a fake /FontFile2. /Length1 makes the sweep class the stream as a binary font program, so + // its stored bytes hold no plaintext: only the decoded data gives the copy away. + let spec = font_file_spec(flate_object("/Length1 900", SECRET.as_bytes())); + let mut doc = marked(spec); + let (_, proof) = run(&mut doc); + assert_eq!(surface(&proof, Surface::DecodedStreams).verdict, Verdict::Survivor, "{}", proof.to_text()); + let s = surface(&proof, Surface::DecodedStreams); + assert!(s.locations.iter().any(|l| l.contains("(decoded)")), "{:?}", s.locations); + assert!(!proof.passed()); +} + +#[test] +fn a_legitimate_font_file_is_swept_and_clean() { + // A font program that holds none of what was removed (as real ones do not) is searched and + // lets the redaction pass: no false survivor. + let spec = font_file_spec(flate_object("/Length1 900", b"\x00\x01\x00\x00\x00\x10glyph tables")); + let mut doc = marked(spec); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + assert_eq!(surface(&proof, Surface::DecodedStreams).verdict, Verdict::Clean, "{}", proof.to_text()); +} + +#[test] +fn a_font_program_that_will_not_decode_is_searched_as_stored_bytes() { + // An unrelated, damaged font program the page reaches is not a reason to refuse: nothing + // redacted touched it, so its stored bytes are searched and it is counted as raw only. + let raw = Stream::flate(Dict::new(), b"a font program without the secret").raw.as_ref().to_vec(); + let cut = raw.len() - 8; + let spec = font_file_spec(stream("/Length1 900 /Filter /FlateDecode", &raw[..cut])); + let mut doc = marked(spec); + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + let s = surface(&proof, Surface::DecodedStreams); + assert_eq!((s.verdict, s.raw_only), (Verdict::Clean, 1), "{}", proof.to_text()); +} + +/// The standard page plus an image (object 7) drawn away from the secret. +fn image_doc(image: Vec) -> Document { + let mut content = CONTENT.to_vec(); + content.extend_from_slice(b" q 20 0 0 20 200 200 cm /Im1 Do Q"); + marked_content(&content, Spec { resources: "/XObject << /Im1 7 0 R >>", extra: vec![image], ..Spec::default() }) +} + +const IMG: &str = "/Type /XObject /Subtype /Image /Width 20 /Height 20 /ColorSpace /DeviceGray /BitsPerComponent 8 /Filter /FlateDecode"; + +fn assert_redacted_cleanly(mut doc: Document) -> Proof { + let (r, proof) = run(&mut doc); + r.unwrap(); + assert!(proof.passed(), "{}", proof.to_text()); + let saved = write_incremental(&doc, &SaveOptions::default()).unwrap(); + assert!(!saved.windows(SECRET.len()).any(|w| w == SECRET.as_bytes()), "the secret is gone from the file"); + proof +} + +#[test] +fn an_unrelated_truncated_flate_image_does_not_fail_the_proof() { + let raw = Stream::flate(Dict::new(), &[0x80u8; 400]).raw.as_ref().to_vec(); + let proof = assert_redacted_cleanly(image_doc(stream(IMG, &raw[..raw.len() - 8]))); + assert_eq!(surface(&proof, Surface::DecodedStreams).raw_only, 1, "{}", proof.to_text()); +} + +#[test] +fn an_unrelated_image_over_the_stream_cap_does_not_fail_the_proof() { + // Highly compressible: stored in a few dozen KiB, decoded past MAX_STREAM. + let big = vec![0u8; crate::verify::limits::MAX_STREAM + (6 << 20)]; + let raw = Stream::flate(Dict::new(), &big).raw.as_ref().to_vec(); + drop(big); + let proof = assert_redacted_cleanly(image_doc(stream(IMG, &raw))); + assert_eq!(surface(&proof, Surface::DecodedStreams).raw_only, 1, "{}", proof.to_text()); +} + +#[test] +fn an_unrelated_truncated_font_program_does_not_fail_the_redaction() { + let raw = Stream::flate(Dict::new(), b"glyph tables").raw.as_ref().to_vec(); + let doc = marked(font_file_spec(stream("/Length1 900 /Filter /FlateDecode", &raw[..raw.len() - 6]))); + assert_redacted_cleanly(doc); +} + +#[test] +fn a_secret_in_the_stored_bytes_of_a_font_program_is_still_found() { + // The font program cannot be decoded (it claims Flate but holds plain bytes), so only its + // stored bytes are searched: the copy of the secret in them is a survivor. + let spec = font_file_spec(stream("/Length1 900 /Filter /FlateDecode", format!("not deflate {SECRET} not deflate").as_bytes())); + let mut doc = marked(spec); + let (_, proof) = run(&mut doc); + let s = surface(&proof, Surface::DecodedStreams); + assert_eq!((s.verdict, s.raw_only), (Verdict::Survivor, 1), "{}", proof.to_text()); + assert!(s.locations.iter().any(|l| l.contains("(raw)")), "{:?}", s.locations); + assert!(!proof.to_text().contains(SECRET)); + assert!(!proof.passed()); +} + +/// The standard page plus an /Indexed colour space whose lookup table is object 7. +fn indexed_doc(lookup: Vec) -> Document { + marked_content(CONTENT, Spec { resources: "/ColorSpace << /CS1 [/Indexed /DeviceRGB 255 7 0 R] >>", extra: vec![lookup], ..Spec::default() }) +} + +#[test] +fn an_indexed_lookup_stream_is_searched_as_stored_bytes_only() { + // Unrelated and damaged: nothing redacted touched it, so it must not fail the proof. + let raw = Stream::flate(Dict::new(), &[0x40u8; 300]).raw.as_ref().to_vec(); + let proof = assert_redacted_cleanly(indexed_doc(stream("/Filter /FlateDecode", &raw[..raw.len() - 8]))); + assert_eq!(surface(&proof, Surface::DecodedStreams).raw_only, 1, "{}", proof.to_text()); + // A secret stored raw inside one that will not decode is still found. + let mut doc = indexed_doc(stream("/Filter /FlateDecode", format!("not deflate {SECRET} not deflate").as_bytes())); + let (_, proof) = run(&mut doc); + let s = surface(&proof, Surface::DecodedStreams); + assert_eq!((s.verdict, s.raw_only), (Verdict::Survivor, 1), "{}", proof.to_text()); + assert!(!proof.passed()); +} + +/// Capture a sound document, then damage the stream `num` (a Flate stream cut short, with the +/// extra dictionary entries `dict`) and prove it. +fn prove_damaged(mut doc: Document, num: u32, dict: &str, data: &[u8]) -> Proof { + let snap = Snapshot::capture(&doc, None).unwrap(); + let raw = Stream::flate(Dict::new(), data).raw.as_ref().to_vec(); + let cut = raw[..raw.len() - 8].to_vec(); + let broken = open(pdf_bytes(vec![b"<< /Type /Catalog >>".to_vec(), stream(&format!("{dict} /Filter /FlateDecode"), &cut)], "")); + let obj = broken.get(ObjRef::new(2, 0)).as_ref().clone(); + doc.set(ObjRef::new(num, 0), obj); + doc.require_full_save(); + snap.prove(&doc, None, &ProofOptions::default()) +} + +#[test] +fn damaged_content_appearance_and_metadata_still_fail_closed() { + // A truncated second content stream. + let objs: Vec> = vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents [4 0 R 7 0 R] /Resources << /Font << /F1 5 0 R >> >> >>".to_vec(), + stream("", CONTENT), + FONT.as_bytes().to_vec(), + format!("[{}]", vec!["500"; 95].join(" ")).into_bytes(), + stream("", b"BT /F1 10 Tf 10 200 Td (more text here) Tj ET"), + ]; + let mut doc = open(pdf_bytes(objs, "/Root 1 0 R")); + mark(&mut doc, &[REGION]); + let proof = prove_damaged(doc, 7, "", b"BT /F1 10 Tf 10 200 Td (more text here) Tj ET"); + assert!(!proof.passed()); + assert!(!surface(&proof, Surface::ExtractedText).problems.is_empty(), "{}", proof.to_text()); + assert!(!surface(&proof, Surface::DecodedStreams).problems.is_empty(), "{}", proof.to_text()); + + // A truncated appearance stream. + let annot = b"<< /Type /Annot /Subtype /Square /Rect [200 200 250 250] /AP << /N 8 0 R >> >>".to_vec(); + let ap = stream("/Type /XObject /Subtype /Form /BBox [0 0 50 50]", b"0 0 50 50 re f 0 0 1 rg 1 0 0 RG 2 w"); + let mut doc = marked(Spec { page: "/Annots [7 0 R]", extra: vec![annot, ap], ..Spec::default() }); + doc.require_full_save(); + let proof = prove_damaged(doc, 8, "/Type /XObject /Subtype /Form /BBox [0 0 50 50]", b"0 0 50 50 re f 0 0 1 rg 1 0 0 RG 2 w"); + assert!(!proof.passed()); + assert!(!surface(&proof, Surface::ExtractedText).problems.is_empty(), "{}", proof.to_text()); + + // A truncated XMP packet. + let xmp = stream("/Type /Metadata /Subtype /XML", b"some metadata of the file"); + let doc = marked(Spec { catalog: "/Metadata 7 0 R", extra: vec![xmp], ..Spec::default() }); + let proof = prove_damaged(doc, 7, "/Type /Metadata /Subtype /XML", b"some metadata of the file"); + assert!(!proof.passed()); + assert!(!surface(&proof, Surface::Xmp).problems.is_empty(), "{}", proof.to_text()); +} + +#[test] +fn an_annotation_appearance_is_checked_however_it_is_stored() { + // The region holds no text, so only the paint check can speak. An annotation written in + // place (a direct dictionary, no reference of its own) is checked like an indirect one, and + // a stream shared by two annotations is checked at each annotation's own placement, so the + // copy that lands over the region is seen even though the first placement does not reach it. + let blank = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()); + let snap = Snapshot::capture(&blank, None).unwrap(); + let prove = |spec: Spec<'_>| -> Proof { + let mut shown = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", spec); + shown.require_full_save(); + snap.prove(&shown, None, &ProofOptions::default()) + }; + let ap = stream("/Type /XObject /Subtype /Form /BBox [0 0 62 17] /Resources << >>", b"0 0 0 rg 10 5 20 10 re f"); + + let proof = prove(Spec { + page: "/Annots [<< /Type /Annot /Subtype /Square /Rect [8 95 70 112] /AP << /N 7 0 R >> >>]", + extra: vec![ap], + ..Spec::default() + }); + assert_eq!(proof.entries[0].status, Status::Failed, "{}", proof.to_text()); + assert!(proof.entries[0].detail.contains("path"), "{}", proof.to_text()); + + let ap = stream("/Type /XObject /Subtype /Form /BBox [0 0 62 17] /Resources << >>", b"0 0 0 rg 10 5 20 10 re f"); + let far = b"<< /Type /Annot /Subtype /Square /Rect [200 200 250 250] /AP << /N 7 0 R >> >>".to_vec(); + let near = b"<< /Type /Annot /Subtype /Square /Rect [8 95 70 112] /AP << /N 7 0 R >> >>".to_vec(); + let proof = prove(Spec { page: "/Annots [8 0 R 9 0 R]", extra: vec![ap, far, near], ..Spec::default() }); + assert_eq!(proof.entries[0].status, Status::Failed, "{}", proof.to_text()); + assert!(proof.entries[0].detail.contains("path"), "{}", proof.to_text()); +} + +#[test] +fn a_null_entry_in_the_contents_array_is_an_empty_piece() { + let objs: Vec> = vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents [4 0 R null] /Resources << /Font << /F1 5 0 R >> >> >>".to_vec(), + stream("", CONTENT), + FONT.as_bytes().to_vec(), + format!("[{}]", vec!["500"; 95].join(" ")).into_bytes(), + ]; + let mut doc = open(pdf_bytes(objs, "/Root 1 0 R")); + mark(&mut doc, &[REGION]); + assert_redacted_cleanly(doc); +} + +/// The page's decoded content streams, joined (what the redaction left on the page). +fn page_content(doc: &Document) -> String { + let p = &pdfcraft_model::pages(doc)[0]; + let (_, data) = page_streams(doc, &p.dict, 0).unwrap(); + data.iter().map(|d| String::from_utf8_lossy(d).into_owned()).collect::>().join("\n") +} diff --git a/crates/redact/src/verify/extract.rs b/crates/redact/src/verify/extract.rs new file mode 100644 index 0000000000..f6db96229b --- /dev/null +++ b/crates/redact/src/verify/extract.rs @@ -0,0 +1,711 @@ +//! A minimal text extractor, independent of the interpreter's font metrics and hit test. + +use std::collections::HashMap; + +use pdfcraft_content::{Matrix, parse}; +use pdfcraft_cos::{Dict, Document, ObjRef, Object, Stream}; + +use super::util::{Ctx, latin1, resolve_strict, squash}; +use crate::annots_of; +use crate::verify::limits::{MAX_CMAP, MAX_CMAP_ENTRIES, MAX_CMAP_WORK, MAX_DEPTH}; + +/// Is this one of the glyph-carrying text-showing operators (`Tj`, `'`, `"`, and `TJ` with at +/// least one non-empty string)? A `TJ` of numbers only (the gap redaction leaves for a removed +/// string) moves the cursor but shows nothing, so it does not count. +pub(super) fn carries_glyphs(op: &pdfcraft_content::Op) -> bool { + let nonempty = |o: &Object| o.as_string().is_some_and(|s| !s.bytes.is_empty()); + match op.op.as_slice() { + b"Tj" | b"'" | b"\"" => op.operands.last().is_some_and(nonempty), + b"TJ" => op.operands.first().and_then(Object::as_array).is_some_and(|a| a.iter().any(nonempty)), + _ => false, + } +} + +/// A page's text as two whitespace-free views: decoded text, and the character codes as Latin-1 +/// (what fonts without a usable mapping show). +#[derive(Default)] +pub(super) struct Extracted { + pub(super) text: String, + pub(super) raw: String, + pub(super) glyph_ops: usize, + /// Glyph-carrying operators whose text could not be decoded. + pub(super) undecodable_ops: usize, + /// Per region asked about: glyphs whose centre this extractor places inside it. + pub(super) shown_in: Vec, + /// Glyph-carrying operators drawn where this extractor can't place them (tiling patterns, + /// or an annotation appearance without a usable `/Rect`). + pub(super) unplaced: usize, +} + +/// How one font turns codes into text. +#[derive(Clone, Default)] +pub(super) struct Decoder { + pub(super) map: HashMap, + /// Bytes per code. + pub(super) width: usize, + /// No way to read these codes as text (composite font without a mapping). + pub(super) opaque: bool, +} + +impl Decoder { + fn latin1() -> Self { + Decoder { map: HashMap::new(), width: 1, opaque: false } + } + + /// `Err` when the font's own data can't be read in full (a damaged or oversized CMap). + fn new(doc: &Document, font: &Dict, ctx: &Ctx) -> Result { + let composite = font.name(b"Subtype") == Some(b"Type0"); + let mut d = Decoder { map: HashMap::new(), width: if composite { 2 } else { 1 }, opaque: false }; + let tu = font.get(b"ToUnicode").map(|t| doc.resolve(t)); + if let Some(Object::Stream(s)) = tu.as_deref() { + let data = ctx.decode(s)?; + if data.len() > MAX_CMAP { + return Err(()); + } + let (map, width) = parse_cmap(&data).ok_or(())?; + if !map.is_empty() { + d.map = map; + if let Some(w) = width { + d.width = w; + } + return Ok(d); + } + } + if composite { + d.opaque = true; + return Ok(d); + } + // Simple font: Latin-1, with the /Differences of the encoding. + if let Some(enc) = font.get(b"Encoding").map(|e| doc.resolve(e)) + && let Some(diff) = enc.as_dict().and_then(|e| e.get(b"Differences")).map(|x| doc.resolve(x)) + && let Some(items) = diff.as_array() + { + let mut code = 0u32; + for it in items { + match &*doc.resolve(it) { + Object::Int(i) => code = u32::try_from(*i).unwrap_or(0), + Object::Name(n) => { + if let Some(c) = std::str::from_utf8(n).ok().and_then(pdfcraft_fonts::pdf::glyph_unicode) { + d.map.insert(code, c.to_string()); + } + code = code.saturating_add(1); + } + _ => {} + } + } + } + Ok(d) + } + + /// Each code of a shown string, with the text it gives (`None`: no way to read it as text). + fn split(&self, s: &[u8]) -> Vec<(u32, Option)> { + let w = self.width.max(1); + s.chunks(w) + .map(|c| { + let code = code_of(c); + let text = if self.opaque { + None + } else { + match self.map.get(&code) { + Some(t) => Some(t.clone()), + None if self.map.is_empty() || w == 1 && code < 256 => Some(char::from(u8::try_from(code).unwrap_or(b'?')).to_string()), + None => None, + } + }; + (code, text) + }) + .collect() + } + + /// The text of a shown string. + fn decode(&self, s: &[u8]) -> String { + self.split(s).into_iter().filter_map(|(_, t)| t).collect() + } +} + +/// A font as the geometric check sees it: its decoder and its own simple width model. +#[derive(Clone)] +pub(super) struct Font { + pub(super) dec: Decoder, + /// Advance per code, in text space units per 1 em of font size (0.5 = half an em). + pub(super) widths: HashMap, + /// Advance of a code `widths` doesn't list. + pub(super) default_width: f64, + pub(super) vertical: bool, +} + +/// The width a font without usable `/Widths` is assumed to have (half an em). +pub(super) const FLAT_WIDTH: f64 = 0.5; + +impl Font { + fn flat() -> Self { + Font { dec: Decoder::latin1(), widths: HashMap::new(), default_width: FLAT_WIDTH, vertical: false } + } + + fn new(doc: &Document, font: &Dict, ctx: &Ctx) -> Result { + let dec = Decoder::new(doc, font, ctx)?; + let mut f = Font { dec, widths: HashMap::new(), default_width: FLAT_WIDTH, vertical: false }; + let num = |o: &Object| doc.resolve(o).as_f64(); + if font.name(b"Subtype") == Some(b"Type0") { + f.vertical = font.get(b"Encoding").map(|e| doc.resolve(e)).and_then(|e| e.as_name().map(|n| n.ends_with(b"-V"))).unwrap_or(false); + let desc = font.get(b"DescendantFonts").map(|d| doc.resolve(d)).and_then(|d| d.as_array().and_then(|a| a.first().cloned())); + let desc = desc.and_then(|d| doc.resolve(&d).as_dict().cloned()); + if let Some(d) = desc { + f.default_width = d.get(b"DW").and_then(num).map_or(1.0, |w| w / 1000.0); + // The array is charged to the shared work budget as it is taken (a form + // invoked again rebuilds its fonts, so a per-font counter alone would let the + // same work be paid for many times), and every entry is paid for there too, + // whether or not it grows the map (a range may rewrite the same codes again + // and again, and the array is bounded only by the file). + let items: Vec = match d.get(b"W").map(|w| doc.resolve(w)).and_then(|w| w.as_array().cloned()) { + Some(a) => { + if !ctx.charge(a.len()) { + return Err(()); + } + a + } + None => Vec::new(), + }; + let mut work = 0usize; + let mut step = |n: usize| -> Result<(), ()> { + work = work.saturating_add(n); + if work <= MAX_CMAP_WORK && ctx.charge(n) { Ok(()) } else { Err(()) } + }; + let mut i = 0; + while let Some(first) = items.get(i).and_then(num) { + let first = first.max(0.0) as u32; + match items.get(i + 1).map(|o| doc.resolve(o)) { + Some(a) if a.as_array().is_some() => { + step(a.as_array().map_or(0, Vec::len))?; + for (k, w) in a.as_array().into_iter().flatten().enumerate() { + if f.widths.len() >= MAX_CMAP_ENTRIES { + return Err(()); + } + if let (Some(w), Ok(k)) = (num(w), u32::try_from(k)) { + f.widths.insert(first.saturating_add(k), w / 1000.0); + } + } + i += 2; + } + Some(last) => { + let (Some(last), Some(w)) = (last.as_f64(), items.get(i + 2).and_then(num)) else { break }; + let last = (last.max(0.0) as u32).min(first.saturating_add(0xFFFF)); + // Each code the range covers is paid for, grown or not (a span is at + // most 0x10000 codes, so the count always fits). + step(last.saturating_sub(first) as usize + 1)?; + for c in first..=last { + if f.widths.len() >= MAX_CMAP_ENTRIES { + return Err(()); + } + f.widths.insert(c, w / 1000.0); + } + i += 3; + } + None => break, + } + } + } + } else { + // Glyph space units per em: 1000, or what a Type3 font's matrix says. + let scale = if font.name(b"Subtype") == Some(b"Type3") { + font.get(b"FontMatrix").map(|m| doc.resolve(m)).and_then(|m| m.as_array().and_then(|a| a.first().and_then(num))).unwrap_or(0.001) + } else { + 0.001 + }; + let first = font.get(b"FirstChar").and_then(num).map_or(0, |v| v.max(0.0) as u32); + let missing = font + .get(b"FontDescriptor") + .map(|d| doc.resolve(d)) + .and_then(|d| d.as_dict().and_then(|d| d.get(b"MissingWidth").and_then(num))) + .map(|w| w * scale); + if let Some(ws) = font.get(b"Widths").map(|w| doc.resolve(w)).and_then(|w| w.as_array().cloned()) { + if ws.len() > MAX_CMAP_ENTRIES { + return Err(()); + } + for (k, w) in ws.iter().enumerate() { + if let (Some(w), Ok(k)) = (num(w), u32::try_from(k)) { + f.widths.insert(first.saturating_add(k), w * scale); + } + } + f.default_width = missing.unwrap_or(0.0); + } else if let Some(std) = pdfcraft_fonts::Std14::from_base_font(&String::from_utf8_lossy(font.name(b"BaseFont").unwrap_or(b""))) { + // No /Widths: a standard font has exact published widths, by WinAnsi code (Symbol, + // ZapfDingbats and other encodings are refused before the proof is reached). + for code in 0..=255u8 { + f.widths.insert(u32::from(code), std.width(code) / 1000.0); + } + f.default_width = if std.is_fixed_pitch() { 0.6 } else { 0.0 }; + } + } + Ok(f) + } + + fn width(&self, code: u32) -> f64 { + self.widths.get(&code).copied().unwrap_or(self.default_width) + } +} + +pub(super) fn hex_token(tok: &[u8]) -> Option> { + let digits: Vec = tok.iter().filter_map(|c| (*c as char).to_digit(16).map(|d| d as u8)).collect(); + if digits.is_empty() || !digits.len().is_multiple_of(2) { + return None; + } + Some(digits.chunks(2).map(|p| p.iter().fold(0u8, |a, d| (a << 4) | d)).collect()) +} + +pub(super) fn utf16_text(b: &[u8]) -> String { + if let [c] = b { + return char::from(*c).to_string(); + } + let units: Vec = b.as_chunks::<2>().0.iter().map(|c| u16::from_be_bytes(*c)).collect(); + String::from_utf16_lossy(&units) +} + +pub(super) fn code_of(b: &[u8]) -> u32 { + b.iter().fold(0u32, |a, x| a.wrapping_shl(8) | u32::from(*x)) +} + +pub(super) enum Tok { + Hex(Vec), + Word(String), + Open, + Close, +} + +pub(super) fn cmap_tokens(data: &[u8]) -> Vec { + let mut out = Vec::new(); + let mut i = 0; + while let Some(&c) = data.get(i) { + match c { + b'<' if data.get(i + 1) != Some(&b'<') => { + let end = data.get(i..).and_then(|d| d.iter().position(|x| *x == b'>')).map_or(data.len(), |e| i + e); + if let Some(h) = data.get(i + 1..end).and_then(hex_token) { + out.push(Tok::Hex(h)); + } + i = end + 1; + } + b'[' => { + out.push(Tok::Open); + i += 1; + } + b']' => { + out.push(Tok::Close); + i += 1; + } + b'%' => { + while data.get(i).is_some_and(|x| !matches!(x, b'\n' | b'\r')) { + i += 1; + } + } + c if c.is_ascii_alphabetic() => { + let start = i; + while data.get(i).is_some_and(|x| x.is_ascii_alphanumeric()) { + i += 1; + } + out.push(Tok::Word(String::from_utf8_lossy(data.get(start..i).unwrap_or_default()).into_owned())); + } + _ => i += 1, + } + } + out +} + +/// `bfchar` / `bfrange` entries of a ToUnicode CMap, and the code length of its codespace. +/// `None` when the CMap is bigger than [`MAX_CMAP_ENTRIES`] entries or costs more than +/// [`MAX_CMAP_WORK`] steps: a map that is cut short would read as "no such code". +pub(super) fn parse_cmap(data: &[u8]) -> Option<(HashMap, Option)> { + let toks = cmap_tokens(data); + let mut map: HashMap = HashMap::new(); + let mut width = None; + // Every step is paid for, whether or not it grows the map (a range may rewrite the same + // codes again and again). + let mut work = 0usize; + let mut step = |n: usize| -> Option<()> { + work = work.saturating_add(n); + (work <= MAX_CMAP_WORK).then_some(()) + }; + let mut i = 0; + while let Some(t) = toks.get(i) { + i += 1; + let Tok::Word(w) = t else { continue }; + match w.as_str() { + "begincodespacerange" => { + if let Some(Tok::Hex(h)) = toks.get(i) { + width = Some(h.len()); + } + } + "beginbfchar" => { + while let (Some(Tok::Hex(src)), Some(Tok::Hex(dst))) = (toks.get(i), toks.get(i + 1)) { + step(1)?; + map.insert(code_of(src), utf16_text(dst)); + if map.len() > MAX_CMAP_ENTRIES { + return None; + } + i += 2; + } + } + "beginbfrange" => { + while let (Some(Tok::Hex(lo)), Some(Tok::Hex(hi))) = (toks.get(i), toks.get(i + 1)) { + let (lo, hi) = (code_of(lo), code_of(hi)); + let span = hi.saturating_sub(lo).min(0xFFFF); + match toks.get(i + 2) { + Some(Tok::Hex(dst)) => { + let base = utf16_text(dst); + step(usize::try_from(span).ok()?.saturating_add(1))?; + for k in 0..=span { + let mut chars: Vec = base.chars().collect(); + if let Some(last) = chars.last_mut() { + *last = char::from_u32(u32::from(*last).saturating_add(k)).unwrap_or('\u{fffd}'); + } + map.insert(lo.saturating_add(k), chars.into_iter().collect()); + } + if map.len() > MAX_CMAP_ENTRIES { + return None; + } + i += 3; + } + Some(Tok::Open) => { + i += 3; + let mut k = 0u32; + while let Some(Tok::Hex(d)) = toks.get(i) { + step(1)?; + map.insert(lo.saturating_add(k), utf16_text(d)); + if map.len() > MAX_CMAP_ENTRIES { + return None; + } + k = k.saturating_add(1); + i += 1; + } + i += 1; // the closing bracket + } + _ => break, + } + } + } + _ => {} + } + } + Some((map, width)) +} + +pub(super) fn res_dict(doc: &Document, res: &Dict, key: &[u8]) -> Dict { + res.get(key).map(|o| doc.resolve(o)).and_then(|o| o.as_dict().cloned()).unwrap_or_default() +} + +pub(super) fn push_squashed(into: &mut String, s: &str) { + into.extend(s.chars().filter(|c| !c.is_whitespace() && *c != '\u{feff}')); +} + +/// A form's or pattern's `/Matrix`: absent is the identity, present but malformed is an error +/// (guessing the identity would put its content in the wrong place). +pub(super) fn matrix_of(doc: &Document, dict: &Dict) -> Result { + match dict.get(b"Matrix") { + None => Ok(Matrix::IDENTITY), + Some(m) => { + let m = doc.resolve(m); + let items: Vec = m.as_array().ok_or(())?.iter().map(|x| (*doc.resolve(x)).clone()).collect(); + Matrix::from_operands(&items).filter(|m| m.0.iter().all(|v| v.is_finite())).ok_or(()) + } + } +} + +/// A form's `/BBox` as `[x0 y0 x1 y1]`: absent is `None`, present but malformed is an error. +pub(super) fn bbox_of(doc: &Document, dict: &Dict) -> Result, ()> { + let Some(b) = dict.get(b"BBox") else { return Ok(None) }; + let b = doc.resolve(b); + let v: Vec = + b.as_array().ok_or(())?.iter().map(|x| doc.resolve(x).as_f64().filter(|v| v.is_finite()).ok_or(())).collect::>()?; + match v[..] { + [a, b, c, d] => Ok(Some([a.min(c), b.min(d), a.max(c), b.max(d)])), + _ => Err(()), + } +} + +/// The content streams of a page as `(raw object, decoded)`, a missing or unreadable entry an +/// error. `skip_overlay` leaves out the streams this run drew the boxes and labels with. +pub(super) fn page_content(doc: &Document, page: &Dict, ctx: &Ctx, skip_overlay: bool) -> Result, ()> { + let list: Vec = match page.get(b"Contents") { + None => Vec::new(), + Some(c) => match &*resolve_strict(doc, c)? { + Object::Array(a) => a.clone(), + _ => vec![c.clone()], + }, + }; + let mut data: Vec = Vec::new(); + for o in &list { + match &*resolve_strict(doc, o)? { + Object::Stream(s) => { + let d = ctx.decode(s)?; + if skip_overlay && ctx.is_overlay(&d) { + continue; + } + data.extend(d); + data.push(b'\n'); + } + // A null entry is an empty piece, as for the redaction itself (ISO 32000-2 7.3.9). + Object::Null => {} + _ => return Err(()), + } + } + Ok(data) +} + +/// The text of a page: its content, the forms and patterns it reaches (visible or not), and the +/// appearance streams of its annotations; and which of `regions` still hold glyphs. +pub(super) fn extract_page(doc: &Document, page: &Dict, ctx: &Ctx, regions: &[[f64; 4]]) -> Result { + let data = page_content(doc, page, ctx, true)?; + let res = page.get(b"Resources").and_then(|r| doc.resolve(r).as_dict().cloned()).unwrap_or_default(); + let mut ex = Extractor { doc, ctx, regions, out: Extracted { shown_in: vec![0; regions.len()], ..Extracted::default() }, stack: Vec::new() }; + ex.scope(&data, &res, Matrix::IDENTITY, 0, true)?; + for a in annots_of(doc, page) { + let Some(d) = doc.resolve(&a).as_dict().cloned() else { continue }; + let Some(ap) = d.get(b"AP").and_then(|x| doc.resolve(x).as_dict().cloned()) else { continue }; + let rect = + d.get(b"Rect").and_then(|r| doc.resolve(r).as_array().map(|a| a.iter().filter_map(|x| doc.resolve(x).as_f64()).collect::>())); + for (_, state) in ap.iter() { + let s = doc.resolve(state); + let streams: Vec<(Option, Object)> = match &*s { + Object::Stream(_) => vec![(state.as_ref(), (*s).clone())], + Object::Dict(sub) => sub.iter().map(|(_, v)| (v.as_ref(), (*doc.resolve(v)).clone())).collect(), + _ => Vec::new(), + }; + for (r, o) in streams { + if let Object::Stream(st) = o { + // Where the appearance lands on the page (ISO 32000-2 §12.5.5), if it can be told. + let ctm = rect.as_deref().and_then(|r| appearance_matrix(doc, &st, r)); + // A font of the appearance is looked up in the appearance's own resources + // first (what the redaction itself does), then in the page's. + let own = st.dict.get(b"Resources").and_then(|x| doc.resolve(x).as_dict().cloned()); + let parent = own.as_ref().unwrap_or(&res); + ex.stream(&st, r, parent, ctm.unwrap_or_default(), 1, ctm.is_some())?; + } + } + } + } + Ok(ex.out) +} + +/// The matrix that maps an appearance stream's form space onto an annotation's `/Rect` +/// (ISO 32000-2 §12.5.5): the appearance's `/Matrix`, then the scale that fits the mapped +/// `/BBox` into the rectangle. `None` when that can't be told (no usable `/Rect`, `/BBox` or +/// `/Matrix`, or a mapped box that degenerates). +pub(crate) fn appearance_matrix(doc: &Document, ap: &Stream, rect: &[f64]) -> Option { + let [rx0, ry0, rx1, ry1] = rect[..] else { return None }; + let bbox = bbox_of(doc, &ap.dict).ok()??; + let m = matrix_of(doc, &ap.dict).ok()?; + let t = m.bbox(bbox); + let (tw, th) = (t[2] - t[0], t[3] - t[1]); + if tw <= 0.0 || th <= 0.0 { + return None; + } + let (rx0, rx1, ry0, ry1) = (rx0.min(rx1), rx0.max(rx1), ry0.min(ry1), ry0.max(ry1)); + let (sx, sy) = ((rx1 - rx0) / tw, (ry1 - ry0) / th); + let a = Matrix([sx, 0.0, 0.0, sy, rx0 - t[0] * sx, ry0 - t[1] * sy]); + Some(m.then(&a)) +} + +/// Text-state parameters that `q`/`Q` save and restore. +#[derive(Clone)] +pub(super) struct TextGs { + pub(super) ctm: Matrix, + pub(super) tc: f64, + pub(super) tw: f64, + pub(super) th: f64, + pub(super) tl: f64, + pub(super) rise: f64, + pub(super) size: f64, + pub(super) font: Vec, +} + +pub(super) struct Extractor<'a> { + pub(super) doc: &'a Document, + pub(super) ctx: &'a Ctx, + pub(super) regions: &'a [[f64; 4]], + pub(super) out: Extracted, + pub(super) stack: Vec, +} + +impl Extractor<'_> { + /// A form or pattern stream. `ctm` maps its space onto the page's; `placed` is false where + /// that mapping is not known. + fn stream(&mut self, s: &Stream, id: Option, parent: &Dict, ctm: Matrix, depth: usize, placed: bool) -> Result<(), ()> { + if let Some(r) = id { + if self.stack.contains(&r) { + return Ok(()); + } + self.stack.push(r); + } + let result = (|| { + let data = self.ctx.decode(s)?; + let own = s.dict.get(b"Resources").and_then(|r| self.doc.resolve(r).as_dict().cloned()); + self.scope(&data, own.as_ref().unwrap_or(parent), ctm, depth, placed) + })(); + if id.is_some() { + self.stack.pop(); + } + result + } + + /// Show the strings of one text-showing operator: extract their text, and place each glyph + /// to count the ones inside a region. `tm` moves along with the pen. + fn show_text(&mut self, font: &Font, op: &pdfcraft_content::Op, gs: &TextGs, tm: &mut Matrix, placed: bool) { + let o = op.op.as_slice(); + // Strings and TJ adjustments, in order. + let items: Vec<&Object> = match o { + b"TJ" => op.operands.first().and_then(Object::as_array).map(|a| a.iter().collect()).unwrap_or_default(), + _ => op.operands.last().into_iter().collect(), + }; + let mut any_text = false; + for it in items { + if let Some(n) = it.as_f64() { + let d = -n / 1000.0 * gs.size; + let m = if font.vertical { Matrix::translate(0.0, d) } else { Matrix::translate(d * gs.th, 0.0) }; + *tm = m.then(tm); + continue; + } + let Some(s) = it.as_string() else { continue }; + let t = font.dec.decode(&s.bytes); + any_text |= !squash(&t).is_empty(); + push_squashed(&mut self.out.text, &t); + push_squashed(&mut self.out.raw, &latin1(&s.bytes)); + for (code, text) in font.dec.split(&s.bytes) { + let w = font.width(code); + let space = text.as_deref().is_some_and(|t| t == " "); + // The glyph's centre, in text space (the box runs from 0.2 em below the + // baseline to 0.8 em above), and where the pen moves to. + let (centre, advance) = if font.vertical { + let adv = -gs.size + gs.tc + if space { gs.tw } else { 0.0 }; + ((0.0, gs.rise - 0.5 * gs.size), Matrix::translate(0.0, adv)) + } else { + let adv = (w * gs.size + gs.tc + if space { gs.tw } else { 0.0 }) * gs.th; + ((0.5 * w * gs.size * gs.th, gs.rise + 0.3 * gs.size), Matrix::translate(adv, 0.0)) + }; + let visible = text.as_deref().is_none_or(|t| !squash(t).is_empty()); + if placed && visible && !self.regions.is_empty() { + let (x, y) = tm.then(&gs.ctm).apply(centre.0, centre.1); + for (r, n) in self.regions.iter().zip(self.out.shown_in.iter_mut()) { + if x >= r[0] && x <= r[2] && y >= r[1] && y <= r[3] { + *n += 1; + } + } + } + *tm = advance.then(tm); + } + } + if carries_glyphs(op) { + self.out.glyph_ops += 1; + if !any_text { + self.out.undecodable_ops += 1; + } + if !placed { + self.out.unplaced += 1; + } + } + } + + fn scope(&mut self, data: &[u8], res: &Dict, ctm: Matrix, depth: usize, placed: bool) -> Result<(), ()> { + if depth > MAX_DEPTH { + return Err(()); + } + let doc = self.doc; + let fonts = res_dict(doc, res, b"Font"); + let xobjects = res_dict(doc, res, b"XObject"); + let mut cache: HashMap, Font> = HashMap::new(); + let mut gs = TextGs { ctm, tc: 0.0, tw: 0.0, th: 1.0, tl: 0.0, rise: 0.0, size: 0.0, font: Vec::new() }; + let mut saved: Vec = Vec::new(); + let (mut tm, mut tlm) = (Matrix::IDENTITY, Matrix::IDENTITY); + for op in parse(data).ops { + let o = op.op.as_slice(); + match o { + b"q" => { + if saved.len() >= 4096 { + return Err(()); + } + saved.push(gs.clone()); + } + b"Q" => { + if let Some(g) = saved.pop() { + gs = g; + } + } + b"cm" => { + if let Some(m) = op.nums::<6>() { + gs.ctm = Matrix(m).then(&gs.ctm); + } + } + b"BT" => { + tm = Matrix::IDENTITY; + tlm = Matrix::IDENTITY; + } + b"Tc" => gs.tc = op.num(0).unwrap_or(0.0), + b"Tw" => gs.tw = op.num(0).unwrap_or(0.0), + b"Tz" => gs.th = op.num(0).unwrap_or(100.0) / 100.0, + b"TL" => gs.tl = op.num(0).unwrap_or(0.0), + b"Ts" => gs.rise = op.num(0).unwrap_or(0.0), + b"Tf" => { + gs.font = op.name(0).map(<[u8]>::to_vec).unwrap_or_default(); + gs.size = op.num(1).unwrap_or(0.0); + } + b"Td" | b"TD" => { + if let Some([x, y]) = op.nums::<2>() { + if o == b"TD" { + gs.tl = -y; + } + tlm = Matrix::translate(x, y).then(&tlm); + tm = tlm; + } + } + b"Tm" => { + if let Some(m) = op.nums::<6>() { + tm = Matrix(m); + tlm = tm; + } + } + b"T*" => { + tlm = Matrix::translate(0.0, -gs.tl).then(&tlm); + tm = tlm; + } + b"Tj" | b"TJ" | b"'" | b"\"" => { + if matches!(o, b"'" | b"\"") { + tlm = Matrix::translate(0.0, -gs.tl).then(&tlm); + tm = tlm; + } + if o == b"\"" + && let (Some(aw), Some(ac)) = (op.num(0), op.num(1)) + { + (gs.tw, gs.tc) = (aw, ac); + } + if !cache.contains_key(&gs.font) { + let f = match fonts.get(&gs.font).map(|f| doc.resolve(f)) { + Some(f) => f.as_dict().map_or_else(|| Ok(Font::flat()), |d| Font::new(doc, d, self.ctx))?, + None => Font::flat(), + }; + cache.insert(gs.font.clone(), f); + } + let Some(font) = cache.get(&gs.font) else { continue }; + self.show_text(font, &op, &gs, &mut tm, placed); + } + b"Do" => { + let Some(name) = op.name(0) else { continue }; + let Some(entry) = xobjects.get(name) else { continue }; + if let Object::Stream(s) = &*doc.resolve(entry) + && s.dict.name(b"Subtype") == Some(b"Form") + { + let m = matrix_of(doc, &s.dict)?; + bbox_of(doc, &s.dict)?; + self.stream(s, entry.as_ref(), res, m.then(&gs.ctm), depth + 1, placed)?; + } + } + _ => {} + } + } + // Tiling patterns draw content of their own, repeated where the pattern is used: its + // glyphs can't be placed. + for (_, p) in res_dict(doc, res, b"Pattern").iter() { + if let Object::Stream(s) = &*doc.resolve(p) { + self.stream(s, p.as_ref(), res, Matrix::IDENTITY, depth + 1, false)?; + } + } + Ok(()) + } +} diff --git a/crates/redact/src/verify/geometry.rs b/crates/redact/src/verify/geometry.rs new file mode 100644 index 0000000000..fb69f91351 --- /dev/null +++ b/crates/redact/src/verify/geometry.rs @@ -0,0 +1,500 @@ +//! The non-text content under the regions: images, paths and shadings, and the placement of the +//! glyphs still shown, checked independently of the interpreter. + +use std::collections::HashSet; + +use pdfcraft_content::{Matrix, contains, overlaps, parse}; +use pdfcraft_cos::{Dict, Document, ObjRef, Object, Stream}; + +use super::extract::{appearance_matrix, bbox_of, matrix_of, page_content, res_dict}; +use super::util::{Ctx, image_codec}; +use crate::verify::limits::{MAX_CLIP_SUBPATHS, MAX_DEPTH, MAX_GS_STACK, MAX_HOLES, MAX_PIXELS}; + +#[derive(Clone, Debug, PartialEq, Eq, Hash)] +pub(super) enum Finding { + /// This many glyphs are placed inside the region. + Text(usize), + Image, + Path, + Shading, + Missing, + Unverifiable(&'static str), +} + +#[derive(Clone)] +pub(super) struct Gs { + pub(super) ctm: Matrix, + /// Device-space boxes cut out of the clip (even-odd clips with a hole). + pub(super) holes: Vec<[f64; 4]>, +} + +pub(super) struct Geo<'a> { + pub(super) doc: &'a Document, + pub(super) ctx: &'a Ctx, + pub(super) regions: &'a [[f64; 4]], + pub(super) found: Vec<(usize, Finding)>, + /// What `found` already holds: each finding is recorded once per region, so a page that + /// paints thousands of times cannot grow the report (or the memory) with repeats. + reported: HashSet<(usize, Finding)>, + /// Pattern, soft-mask and Type3 resources already searched for images. + pub(super) searched: HashSet, +} + +/// What still paints inside `regions` on a page of the output (the overlay boxes excepted): +/// images with pixels there, paths and shadings not clipped away from there. +pub(super) fn check_geometry(doc: &Document, page: &Dict, regions: &[[f64; 4]], ctx: &Ctx) -> Vec<(usize, Finding)> { + let unverifiable = |why: &'static str| (0..regions.len()).map(|i| (i, Finding::Unverifiable(why))).collect(); + // The redaction boxes are painted on purpose. + let Ok(data) = page_content(doc, page, ctx, true) else { return unverifiable("the page content can't be read in full") }; + let mut geo = Geo { doc, ctx, regions, found: Vec::new(), reported: HashSet::new(), searched: HashSet::new() }; + let res = page.get(b"Resources").and_then(|r| doc.resolve(r).as_dict().cloned()).unwrap_or_default(); + geo.run(&data, &res, Matrix::IDENTITY, Vec::new(), 0); + geo.appearances(page, &res); + geo.found +} + +impl Geo<'_> { + /// Record a finding for region `i`, once: a page that paints over a region a thousand + /// times is one finding, not a thousand. + fn report(&mut self, i: usize, f: Finding) { + if self.reported.insert((i, f.clone())) { + self.found.push((i, f)); + } + } + + /// The appearances of the annotations still on the page, checked like the page itself: their + /// content is placed with the appearance matrix (ISO 32000-2 §12.5.5) and swept for images, + /// paths and shadings under the regions. An appearance that has content but can't be placed + /// makes every region unverifiable: the redaction refuses such documents, so one in the + /// output means what it paints can't be vouched for. + fn appearances(&mut self, page: &Dict, res: &Dict) { + let doc = self.doc; + for a in crate::annots_of(doc, page) { + // A direct dictionary has no reference of its own; it is checked like an indirect + // one (what its appearance paints does not depend on how it is stored). + let Some(d) = doc.resolve(&a).as_dict().cloned() else { continue }; + // A redaction mark's appearance outlines the marked area on purpose (and applying + // removes the mark whole): it is exempt like the boxes drawn over the regions. + if d.name(b"Subtype") == Some(b"Redact") { + continue; + } + let Some(ap) = d.get(b"AP").and_then(|x| doc.resolve(x).as_dict().cloned()) else { continue }; + let rect = crate::rect_of(doc, d.get(b"Rect")); + for (_, state) in ap.iter() { + let s = doc.resolve(state); + let streams: Vec = match &*s { + Object::Stream(st) => vec![st.clone()], + Object::Dict(sub) => sub + .iter() + .filter_map(|(_, v)| match &*doc.resolve(v) { + Object::Stream(st) => Some(st.clone()), + _ => None, + }) + .collect(), + _ => Vec::new(), + }; + for st in streams { + // A stream shared by several annotations is checked at each annotation's + // own placement: where it lands comes from each `/Rect`, so the first + // placement tells nothing about the others. Repeated decodes spend the + // same bounded budget as everything else. + let Ok(data) = self.ctx.decode(&st) else { + self.unverifiable("an annotation appearance can't be read in full"); + continue; + }; + if data.is_empty() { + continue; + } + let Some(ctm) = rect.as_ref().and_then(|r| appearance_matrix(doc, &st, r)) else { + self.unverifiable("an annotation appearance can't be placed, so what it paints can't be checked"); + continue; + }; + let own = st.dict.get(b"Resources").and_then(|x| doc.resolve(x).as_dict().cloned()); + self.run(&data, own.as_ref().unwrap_or(res), ctm, Vec::new(), 1); + } + } + } + } + + fn touching(&self, b: [f64; 4]) -> Vec { + (0..self.regions.len()).filter(|&i| self.regions.get(i).is_some_and(|r| overlaps(*r, b, 0.0))).collect() + } + + /// Does something painting over `within` reach region `i` through the clip? Only the part of + /// the region the painting could touch has to be cut out. + fn unclipped(&self, i: usize, within: [f64; 4], gs: &Gs) -> bool { + let Some(r) = self.regions.get(i) else { return false }; + let reach = [r[0].max(within[0]), r[1].max(within[1]), r[2].min(within[2]), r[3].min(within[3])]; + !gs.holes.iter().any(|h| contains(*h, reach, 0.01)) + } + + fn run(&mut self, data: &[u8], res: &Dict, ctm: Matrix, holes: Vec<[f64; 4]>, depth: usize) { + if depth > MAX_DEPTH { + for i in 0..self.regions.len() { + self.report(i, Finding::Unverifiable("forms are nested too deep to check")); + } + return; + } + let xobjects = res_dict(self.doc, res, b"XObject"); + self.hidden_images(res, depth); + let mut gs = Gs { ctm, holes }; + let mut stack: Vec = Vec::new(); + // The current path in device space: its subpaths, and the clip operator it carries. + let mut sub: Vec> = Vec::new(); + let mut clip: Option = None; + for op in parse(data).ops { + let o = op.op.as_slice(); + if matches!(o, b"m" | b"l" | b"c" | b"v" | b"y" | b"h" | b"re" | b"W" | b"W*") { + extend_path(&op, &gs.ctm, &mut sub, &mut clip); + continue; + } + if matches!(o, b"S" | b"s" | b"f" | b"F" | b"f*" | b"B" | b"B*" | b"b" | b"b*" | b"n") { + self.paint_path(o, &mut gs, &sub, clip); + sub.clear(); + clip = None; + continue; + } + sub.clear(); + clip = None; + match o { + b"q" => { + // The saved state carries the holes with it, so an unbounded stack of them + // is unbounded memory: past the cap the walk says so instead of growing. + if stack.len() >= MAX_GS_STACK { + return self.unverifiable("the content stacks graphics states too deeply to check"); + } + stack.push(gs.clone()); + } + b"Q" => { + if let Some(g) = stack.pop() { + gs = g; + } + } + b"cm" => { + if let Some(m) = op.nums::<6>() { + gs.ctm = Matrix(m).then(&gs.ctm); + } + } + b"sh" => { + for i in 0..self.regions.len() { + if self.unclipped(i, [f64::MIN, f64::MIN, f64::MAX, f64::MAX], &gs) { + self.report(i, Finding::Shading); + } + } + } + b"BI" => { + for i in self.touching(gs.ctm.bbox([0.0, 0.0, 1.0, 1.0])) { + self.report(i, Finding::Image); + } + } + b"Do" => { + let Some(entry) = op.name(0).and_then(|n| xobjects.get(n)) else { continue }; + let obj = self.doc.resolve(entry); + let Object::Stream(s) = &*obj else { continue }; + match s.dict.name(b"Subtype") { + Some(b"Image") => { + for i in self.touching(gs.ctm.bbox([0.0, 0.0, 1.0, 1.0])) { + let Some(region) = self.regions.get(i) else { continue }; + match image_cleared(self.doc, self.ctx, s, &gs.ctm, *region) { + Ok(true) => {} + Ok(false) => self.report(i, Finding::Image), + Err(()) => self.report(i, Finding::Unverifiable("an image under the region can't be inspected")), + } + } + } + Some(b"Form") => self.form(s, &gs, res, depth), + _ => {} + } + } + _ => {} + } + } + } + + /// A path-painting operator `o` ends the current path: report what it paints over a region, + /// and let an even-odd clip add its holes to `gs`. + fn paint_path(&mut self, o: &[u8], gs: &mut Gs, sub: &[Vec<(f64, f64)>], clip: Option) { + let pts: Vec<(f64, f64)> = sub.iter().flatten().copied().collect(); + let bbox = pts.iter().fold(None, |b: Option<[f64; 4]>, &(u, v)| { + Some(match b { + None => [u, v, u, v], + Some(b) => [b[0].min(u), b[1].min(v), b[2].max(u), b[3].max(v)], + }) + }); + if o != b"n" + && let Some(b) = bbox + { + for i in self.touching(b) { + if self.unclipped(i, b, gs) { + self.report(i, Finding::Path); + } + } + } + // An even-odd clip whose subpaths are all axis-aligned rectangles lets through what an + // odd number of the inner rectangles covers. Where a region lies under the outer box + // and an odd number of inner ones, it is cut out of the clip; under an even number it + // is painted and must be reported. Anything that is not a stock of disjoint rectangles + // (a triangle, a rotated box, overlapping inners, too many of them) cannot be told + // cheaply, so no hole is cut: what the clip lets through is then reported as paint + // (fail closed). This is how redaction itself clips regions out of what it leaves. + if clip != Some(true) { + return; + } + let Some(rects) = sub.iter().map(|s| axis_aligned_rect(s)).collect::>>() else { return }; + if rects.len() < 2 || rects.len() > MAX_CLIP_SUBPATHS { + return; + } + let (outer, inners) = (rects[0], &rects[1..]); + // Inner rectangles must be disjoint, or their overlaps would change the parity the + // reader draws with. + let disjoint = inners.iter().enumerate().all(|(x, a)| inners.iter().skip(x + 1).all(|b| !overlaps(*a, *b, 0.0))); + let regions: Vec<[f64; 4]> = self.regions.to_vec(); + let mut cut = Vec::new(); + for r in regions { + if contains(outer, r, 0.0) && disjoint && inners.iter().filter(|b| contains(**b, r, 0.0)).count() % 2 == 1 { + cut.push(r); + } + } + if gs.holes.len() + cut.len() > MAX_HOLES { + return self.unverifiable("the content cuts clips with too many pieces to check"); + } + gs.holes.extend(cut); + } + + fn unverifiable(&mut self, why: &'static str) { + for i in 0..self.regions.len() { + self.report(i, Finding::Unverifiable(why)); + } + } + + fn form(&mut self, s: &Stream, gs: &Gs, parent: &Dict, depth: usize) { + let (Ok(m), Ok(bbox)) = (matrix_of(self.doc, &s.dict), bbox_of(self.doc, &s.dict)) else { + self.unverifiable("a form under the region has a malformed /Matrix or /BBox"); + return; + }; + let fctm = m.then(&gs.ctm); + if let Some(bb) = bbox + && self.touching(fctm.bbox(bb)).is_empty() + { + return; + } + let Ok(data) = self.ctx.decode(s) else { + self.unverifiable("a form under the region can't be decoded in full"); + return; + }; + let own = s.dict.get(b"Resources").and_then(|r| self.doc.resolve(r).as_dict().cloned()); + self.run(&data, own.as_ref().unwrap_or(parent), fctm, gs.holes.clone(), depth + 1); + } + + /// Images drawn from tiling patterns, soft-mask groups and Type3 glyph procedures can't be + /// placed or cleared by this check: if `res` reaches any, every region is unverifiable. + fn hidden_images(&mut self, res: &Dict, depth: usize) { + let doc = self.doc; + let mut streams: Vec<(Option, Stream)> = Vec::new(); + let take = |o: &Object, streams: &mut Vec<(Option, Stream)>| { + if let Object::Stream(s) = &*doc.resolve(o) { + streams.push((o.as_ref(), s.clone())); + } + }; + for (_, p) in res_dict(doc, res, b"Pattern").iter() { + take(p, &mut streams); + } + for (_, g) in res_dict(doc, res, b"ExtGState").iter() { + if let Some(Object::Dict(sm)) = doc.resolve(g).as_dict().and_then(|g| g.get(b"SMask")).map(|m| (*doc.resolve(m)).clone()).as_ref() + && let Some(group) = sm.get(b"G") + { + take(group, &mut streams); + } + } + for (_, f) in res_dict(doc, res, b"Font").iter() { + let Some(font) = doc.resolve(f).as_dict().cloned() else { continue }; + if font.name(b"Subtype") != Some(b"Type3") { + continue; + } + if let Some(procs) = font.get(b"CharProcs").and_then(|c| doc.resolve(c).as_dict().cloned()) { + for (_, p) in procs.iter() { + take(p, &mut streams); + } + } + } + for (id, s) in streams { + if let Some(r) = id + && !self.searched.insert(r) + { + continue; + } + match self.has_image(&s, res, depth + 1) { + Ok(false) => {} + Ok(true) => self.unverifiable("an image inside a pattern, soft mask or Type3 glyph can't be placed or cleared"), + Err(()) => self.unverifiable("a pattern, soft mask or Type3 glyph can't be read in full"), + } + } + } + + /// Does the content of `s` (forms it draws included) draw an image? + fn has_image(&mut self, s: &Stream, parent: &Dict, depth: usize) -> Result { + if depth > MAX_DEPTH { + return Err(()); + } + let data = self.ctx.decode(s)?; + let own = s.dict.get(b"Resources").and_then(|r| self.doc.resolve(r).as_dict().cloned()); + let res = own.as_ref().unwrap_or(parent); + let xobjects = res_dict(self.doc, res, b"XObject"); + for op in parse(&data).ops { + match op.op.as_slice() { + b"BI" => return Ok(true), + b"Do" => { + let Some(entry) = op.name(0).and_then(|n| xobjects.get(n)) else { continue }; + let Object::Stream(x) = &*self.doc.resolve(entry) else { continue }; + match x.dict.name(b"Subtype") { + Some(b"Image") => return Ok(true), + Some(b"Form") => { + if let Some(r) = entry.as_ref() + && !self.searched.insert(r) + { + continue; + } + if self.has_image(x, res, depth + 1)? { + return Ok(true); + } + } + _ => {} + } + } + _ => {} + } + } + Ok(false) + } +} + +/// Extend the current path (`sub`: its subpaths in device space; `clip`: the clip operator it +/// carries) by one construction operator. +fn extend_path(op: &pdfcraft_content::Op, ctm: &Matrix, sub: &mut Vec>, clip: &mut Option) { + let o = op.op.as_slice(); + let nums: Vec = op.operands.iter().filter_map(Object::as_f64).collect(); + let at = |x: f64, y: f64| ctm.apply(x, y); + match o { + b"m" => { + if let [x, y] = nums[..] { + sub.push(vec![at(x, y)]); + } + } + b"l" | b"c" | b"v" | b"y" => { + if let Some(cur) = sub.last_mut() { + cur.extend(nums.as_chunks::<2>().0.iter().map(|p| at(p[0], p[1]))); + } + } + b"re" => { + if let [x, y, w, h] = nums[..] { + sub.push(vec![at(x, y), at(x + w, y), at(x + w, y + h), at(x, y + h)]); + } + } + b"W" => *clip = Some(false), + b"W*" => *clip = Some(true), + _ => {} + } +} + +/// The axis-aligned rectangle a four-corner subpath draws, in device space: `None` for anything +/// else (open paths, more or fewer corners, skewed or rotated boxes, non-finite coordinates). +fn axis_aligned_rect(pts: &[(f64, f64)]) -> Option<[f64; 4]> { + // `re` leaves its rectangle open: the four corners are the whole subpath. A rectangle drawn + // by hand closes by repeating the first point at the end. Both spell the same four corners. + let pts = match pts { + [a, _, _, _, e] if a == e => &pts[..4], + _ => pts, + }; + let [p0, p1, p2, p3] = pts else { return None }; + let finite = |p: &(f64, f64)| p.0.is_finite() && p.1.is_finite(); + if !finite(p0) || !finite(p1) || !finite(p2) || !finite(p3) { + return None; + } + // The edges alternate between horizontal and vertical, whichever corner they start from and + // whichever way the rectangle is wound. + let flat = |a: &(f64, f64), b: &(f64, f64)| a.1 == b.1 && a.0 != b.0; + let steep = |a: &(f64, f64), b: &(f64, f64)| a.0 == b.0 && a.1 != b.1; + let even = flat(p0, p1) && steep(p1, p2) && flat(p2, p3) && steep(p3, p0); + let odd = steep(p0, p1) && flat(p1, p2) && steep(p2, p3) && flat(p3, p0); + if !even && !odd { + return None; + } + let (x0, x1) = (p0.0.min(p2.0), p0.0.max(p2.0)); + let (y0, y1) = (p0.1.min(p2.1), p0.1.max(p2.1)); + (x0 < x1 && y0 < y1).then_some([x0, y0, x1, y1]) +} + +/// Are the samples of image `s` whose centres fall in `region` all clear (zero; for image masks, +/// the sample that does not paint)? `Err` when the image can't be inspected. +#[allow(clippy::result_unit_err)] +pub(super) fn image_cleared(doc: &Document, ctx: &Ctx, s: &Stream, ctm: &Matrix, region: [f64; 4]) -> Result { + if image_codec(&s.dict) { + // Still encoded pixels: nothing was cleared. + return Ok(false); + } + let (w, h) = (s.dict.int(b"Width").ok_or(())?, s.dict.int(b"Height").ok_or(())?); + if w <= 0 || h <= 0 || (w as u64).saturating_mul(h as u64) > MAX_PIXELS { + return Err(()); + } + let (w, h) = (usize::try_from(w).map_err(|_| ())?, usize::try_from(h).map_err(|_| ())?); + let mask = matches!(s.dict.get(b"ImageMask"), Some(Object::Bool(true))); + let (ncomp, bpc) = if mask { + (1, 1) + } else { + let cs = doc.resolve(s.dict.get(b"ColorSpace").ok_or(())?); + let name = match &*cs { + Object::Name(n) => n.clone(), + Object::Array(a) => a.first().and_then(Object::as_name).ok_or(())?.to_vec(), + _ => return Err(()), + }; + let n = match name.as_slice() { + b"DeviceGray" | b"G" | b"CalGray" | b"Indexed" | b"I" | b"Separation" => 1, + b"DeviceRGB" | b"RGB" | b"CalRGB" | b"Lab" => 3, + b"DeviceCMYK" | b"CMYK" => 4, + // One component per colorant name. + b"DeviceN" => doc.resolve(cs.as_array().and_then(|a| a.get(1)).ok_or(())?).as_array().ok_or(())?.len(), + b"ICCBased" => match &*doc.resolve(cs.as_array().and_then(|a| a.get(1)).ok_or(())?) { + Object::Stream(icc) => usize::try_from(icc.dict.int(b"N").ok_or(())?).map_err(|_| ())?, + _ => return Err(()), + }, + _ => return Err(()), + }; + (n, usize::try_from(s.dict.int(b"BitsPerComponent").ok_or(())?).map_err(|_| ())?) + }; + if !matches!(bpc, 1 | 2 | 4 | 8 | 16) || ncomp == 0 || ncomp > 32 { + return Err(()); + } + let data = ctx.decode(s)?; + let bits = ncomp.checked_mul(bpc).ok_or(())?; + let row = w.checked_mul(bits).ok_or(())?.div_ceil(8); + if data.len() < row.checked_mul(h).ok_or(())? { + return Err(()); + } + let inverted = s.dict.get(b"Decode").is_some_and(|d| { + let d = doc.resolve(d); + d.as_array().and_then(|a| a.first()).is_some_and(|first| doc.resolve(first).as_f64() == Some(1.0)) + }); + let want = u8::from(mask && !inverted); + for j in 0..h { + let v = 1.0 - (j as f64 + 0.5) / h as f64; + // Rows that don't reach the region hold nothing to check. + let ((ax, ay), (bx, by)) = (ctm.apply(0.0, v), ctm.apply(1.0, v)); + if !overlaps(region, [ax.min(bx), ay.min(by) - 0.01, ax.max(bx), ay.max(by) + 0.01], 0.0) { + continue; + } + for i in 0..w { + let (x, y) = ctm.apply((i as f64 + 0.5) / w as f64, v); + if x < region[0] || x > region[2] || y < region[1] || y > region[3] { + continue; + } + let start = j.checked_mul(row).and_then(|v| v.checked_mul(8)).and_then(|v| v.checked_add(i.checked_mul(bits)?)).ok_or(())?; + for bit in start..start.checked_add(bits).ok_or(())? { + let byte = data.get(bit / 8).ok_or(())?; + if (byte >> (7 - bit % 8)) & 1 != want { + return Ok(false); + } + } + } + } + Ok(true) +} diff --git a/crates/redact/src/verify/limits.rs b/crates/redact/src/verify/limits.rs new file mode 100644 index 0000000000..e2972bb5f6 --- /dev/null +++ b/crates/redact/src/verify/limits.rs @@ -0,0 +1,42 @@ +//! Bounds of the redaction proof (`verify`). Hitting one is a failure the proof reports, never +//! a quiet skip: a proof that cannot sweep everything it must is not a passing proof. + +/// Most decoded bytes of one stream the proof reads. +pub(crate) const MAX_STREAM: usize = 64 << 20; +/// Most decoded bytes the whole proof reads, over every stream and every pass. +pub(crate) const MAX_TOTAL_DECODED: u64 = 2 << 30; +/// Most units of linear search work (input, patterns and reported matches) in one proof. +pub(crate) const MAX_WORK: u64 = 4 << 30; +/// Nested attachment levels that are swept (deeper ones are reported as unswept). +pub(crate) const MAX_NESTING: usize = 4; +/// Nesting bound of object walks (deeper is reported, never skipped silently). +pub(crate) const MAX_WALK: usize = 64; +/// Deepest graphics-state stack the paint check follows (saved states carry their clip holes +/// with them, so an unbounded stack is unbounded memory). Deeper is unverifiable. +pub(crate) const MAX_GS_STACK: usize = 4096; +/// Most clip holes one walk may cut (each is kept until the state that made it is restored). +/// More is unverifiable. +pub(crate) const MAX_HOLES: usize = 4096; +/// Most subpaths of one even-odd clip the hole rule examines (it compares them pairwise). +/// More, and no hole is cut: what the clip lets through is reported as paint. +pub(crate) const MAX_CLIP_SUBPATHS: usize = 64; +/// Most decoded bytes of an embedded CMap. +pub(crate) const MAX_CMAP: usize = 4 << 20; +/// Most CMap range steps one font may cost (a range that never grows the map still costs). +pub(crate) const MAX_CMAP_WORK: usize = 4 << 20; +/// Most entries one ToUnicode CMap may define. +pub(crate) const MAX_CMAP_ENTRIES: usize = 1 << 20; +/// Needles shorter than this (in characters) are too common to look for in raw bytes. +pub(crate) const MIN_NEEDLE: usize = 3; +/// Smallest needle looked for in the raw file and in decoded streams. Those hold object syntax, +/// numbers and binary data, where three characters match by chance (`/Length 612`, a +/// `/MediaBox`, pixels); shorter needles are still swept on page text, strings, names, Info, XMP +/// and attachment text. +pub(crate) const MIN_STRUCTURED_NEEDLE: usize = 6; +/// Deepest nesting of forms, patterns and appearance streams the proof's independent walks +/// follow: the bound the interpreter itself walks forms at, so the proof reaches everything a +/// redaction could have removed and the two cannot drift apart. +pub(crate) const MAX_DEPTH: usize = crate::interp::MAX_DEPTH; +/// Most pixels an image may have for the proof to inspect it (larger ones are removed by +/// [`apply`](crate::apply), and the proof's pixel look has the same ceiling: `crate::image`). +pub(crate) const MAX_PIXELS: u64 = crate::image::MAX_PIXELS; diff --git a/crates/redact/src/verify/mod.rs b/crates/redact/src/verify/mod.rs new file mode 100644 index 0000000000..b374f93fdc --- /dev/null +++ b/crates/redact/src/verify/mod.rs @@ -0,0 +1,389 @@ +//! An independent proof that redaction removed what it claimed to remove. +//! +//! [`apply`](crate::apply) cuts glyphs out with its own interpreter, so asking that interpreter +//! whether anything is left would only repeat its own mistakes. The proof therefore works from +//! the other end, and does not trust the interpreter's font metrics or hit test: +//! +//! 1. **Before** applying, [`Snapshot::capture`] records, per redaction region, the glyph runs the +//! interpreter is about to remove (their character codes and the text the font gives them), a +//! digest of each page's content, and the text of the whole document as a separate extractor +//! sees it. The strings the sweep looks for therefore *start* from what the interpreter +//! removed; a glyph it wrongly kept is caught by the geometric check below, not by a needle. +//! 2. **After** applying, [`Snapshot::prove`] serializes the document and sweeps the *output* +//! for those strings on every surface a reader could recover them from: the raw file, every +//! string object, every decoded stream (orphans included), the text of every page (read by a +//! minimal extractor of its own, not the interpreter's font metrics or hit test), the Info +//! dictionary, the XMP packet, embedded files (PDF attachments are swept recursively) and the +//! revision structure. Each string is looked for in every encoding a PDF can hold it in. +//! 3. It places every glyph still shown on the redacted pages with its own text-state tracking +//! and width model (the font's `/Widths`, else a flat guess) and fails a region when the +//! centre of a surviving glyph lies inside it, whatever the interpreter decided. +//! 4. It also re-checks the non-text content under each region — on the page and in the +//! appearance streams of the annotations left on it: no image still carries pixels there, and +//! no path or shading paints there unclipped. +//! +//! The proof fails closed: a survivor, but also anything it could not sweep (page content, a form +//! or an appearance stream, XMP, or an attachment that won't decode in full or can't be opened; a +//! nesting or work limit hit; an appearance with content that can't be placed on its page), makes +//! it fail. The one exception is binary data that redaction +//! never edits: a font program, colour profile, `/Indexed` lookup table, function sample table or +//! Flate/LZW image that is damaged or too long to decode is searched as stored bytes and reported +//! as `raw_only`, not as unswept. A proof that cannot fail is worthless. +//! +//! Work is bounded: every stream is decoded strictly under `limits::MAX_STREAM`, all decodes together +//! under `limits::MAX_TOTAL_DECODED`, and all searching under `limits::MAX_WORK`. A document that needs more +//! is reported as unswept, not skipped (binary data as above excepted). +//! +//! The result is a [`Proof`]: a per-region manifest (page, region, glyph counts, SHA-256 of +//! the page content after) and a verdict per surface. It holds counts and digests, +//! never the redacted text, unless [`ProofOptions::include_plaintext`] asks for it. +//! +//! Limits, stated plainly: text that exists only as pixels of an image is not text; a string +//! that also appears in content that was *not* redacted is counted, over the whole document, on +//! the page text, the raw file and the decoded streams (those hold the kept copy too) and fails +//! on a surplus; names are swept against the matches the input itself had, since a redacted word +//! that is also a `/Subtype` or a `/BaseFont` is structure, not a leftover copy, and a surplus of +//! matches still fails; every other surface is swept with no allowance. A word of six characters or more +//! that also spells a structural name (`/Filter`, `/MediaBox`) can still be reported by the raw-file and +//! decoded-stream surfaces, which search stored bytes and cannot tell a name from a copy: that is +//! over-reporting, never a pass. The raw file and the decoded +//! streams are searched for strings of at least six characters only (shorter ones match object +//! syntax and numbers by chance), skipping the trailer's file identifier and the +//! pixels of an image codec (DCT, JPX, JBIG2, CCITT), which are searched as stored bytes only; shorter strings are still found on the page text, +//! in strings, names, Info, XMP and attachment text. A region without extractable text is reported as such, and as +//! unverifiable when the page draws text this extractor cannot decode. The geometric check +//! places glyphs with the font's `/Widths` only: for a font without them the widths are a guess, +//! so a neighbouring glyph may be reported when the guess puts its centre inside a region. +//! `runs_removed` and `codes_removed` are exact counts of what was cut: a proof that is shared +//! discloses how much text sat under each region (never what it said). + +use std::fmt::Write as _; + +use pdfcraft_cos::ObjRef; + +mod extract; +mod geometry; +pub(crate) mod limits; +#[cfg(test)] +mod search_tests; +mod snapshot; +mod sweep; +mod util; + +pub use snapshot::Snapshot; +/// What the proof established for one region. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Status { + /// What was removed was swept for on every surface and is gone. + Verified, + /// The region held no extractable text: nothing was removed, and that is recorded as such. + VerifiedNoTextInRegion, + /// Neither "text was removed" nor "there was none" can be checked (the page draws text this + /// extractor can't decode, or an image under the region can't be inspected). The proof can't + /// pass while any region is unverifiable. + Unverifiable, + /// A survivor was found, or the mechanical evidence is missing. + Failed, +} + +/// The places a redacted string is looked for. +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum Surface { + RawBytes, + ObjectStrings, + DecodedStreams, + ExtractedText, + Info, + Xmp, + EmbeddedFiles, + Revisions, +} + +const SURFACES: [Surface; 8] = [ + Surface::RawBytes, + Surface::ObjectStrings, + Surface::DecodedStreams, + Surface::ExtractedText, + Surface::Info, + Surface::Xmp, + Surface::EmbeddedFiles, + Surface::Revisions, +]; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Verdict { + /// Swept, nothing found. + Clean, + /// Swept, a redacted string (or an earlier revision) is still there. + Survivor, + /// Could not be swept; the proof fails. + Unswept, + /// There is nothing of this kind in the document. + Absent, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct SurfaceReport { + pub surface: Surface, + pub verdict: Verdict, + /// Streams, strings, pages or files looked at. + pub scanned: usize, + /// Streams scanned as stored bytes only: image-codec pixels (not text), and font, colour or + /// image streams that could not be decoded in full. + pub raw_only: usize, + pub survivors: usize, + /// Where each survivor sits ("object 12", "page 1", "attachment #1 > object 4"); never the text. + pub locations: Vec, + /// Why the surface could not be swept. + pub problems: Vec, +} + +/// The manifest entry of one redaction region. +#[derive(Clone, Debug, PartialEq)] +pub struct Excision { + /// 0-based page. + pub page: usize, + /// The region in user space. + pub region: [f64; 4], + /// The Redact annotation it came from. + pub mark: ObjRef, + pub status: Status, + pub detail: String, + /// Glyph runs (consecutive glyphs of one string) removed under the region, and their + /// character codes. + pub runs_removed: usize, + pub codes_removed: usize, + /// SHA-256 of the page's decoded content streams, joined, after redaction. There is no digest + /// of the content before: a proof is meant to be shared, and a hash of the original page + /// would let a reader confirm a guess at what it said. + pub page_sha256_after: Option, + /// Glyph-carrying text operators on the page (forms included) before and after. A `TJ` of + /// numbers only (the gap left for a removed string) carries no glyphs and does not count. + pub text_ops_before: Option, + pub text_ops_after: Option, + /// The removed text, only when [`ProofOptions::include_plaintext`] is set. + pub removed_text: Option>, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Proof { + pub entries: Vec, + pub surfaces: Vec, + /// Located, plain reasons for every failure (never the redacted text). + pub failures: Vec, +} + +impl Proof { + /// Survivors found across all surfaces. + pub fn survivors(&self) -> usize { + self.surfaces.iter().map(|s| s.survivors).sum() + } + + /// Surfaces that could not be swept. + pub fn unswept(&self) -> usize { + self.surfaces.iter().filter(|s| s.verdict == Verdict::Unswept || !s.problems.is_empty()).count() + } + + pub fn unverifiable(&self) -> usize { + self.entries.iter().filter(|e| e.status == Status::Unverifiable).count() + } + + /// Pass only when nothing failed, nothing is unverifiable and no surface went unswept. + pub fn passed(&self) -> bool { + self.failures.is_empty() + && self.survivors() == 0 + && self.unswept() == 0 + && self.entries.iter().all(|e| matches!(e.status, Status::Verified | Status::VerifiedNoTextInRegion)) + } + + /// A human-readable report: the verdict, the manifest and the surfaces. It names places and + /// counts, and the removed text only if it was asked for. + pub fn to_text(&self) -> String { + let mut s = String::new(); + let _ = writeln!(s, "Redaction proof: {}", if self.passed() { "PASS" } else { "FAIL" }); + for f in &self.failures { + let _ = writeln!(s, " failure: {f}"); + } + for (i, e) in self.entries.iter().enumerate() { + let _ = writeln!( + s, + " region {} page {} {:?} {:?}: {} run(s), {} code(s) removed; text ops {:?} -> {:?}; page sha256 after {}{}", + i + 1, + e.page + 1, + e.region, + e.status, + e.runs_removed, + e.codes_removed, + e.text_ops_before, + e.text_ops_after, + e.page_sha256_after.as_deref().unwrap_or("-"), + if e.detail.is_empty() { String::new() } else { format!(" ({})", e.detail) }, + ); + } + for r in &self.surfaces { + let _ = writeln!( + s, + " {:?}: {:?} ({} scanned, {} raw only, {} survivor(s)){}", + r.surface, + r.verdict, + r.scanned, + r.raw_only, + r.survivors, + if r.locations.is_empty() { String::new() } else { format!(" at {}", r.locations.join(", ")) }, + ); + for p in &r.problems { + let _ = writeln!(s, " unswept: {p}"); + } + } + let _ = writeln!(s, " Evidence, not a guarantee: text that exists only as pixels of an image is not text, and is not searched."); + s + } +} + +/// What a proof may contain. +#[derive(Clone, Debug, Default)] +pub struct ProofOptions { + /// Include the removed text in the manifest. Off by default: a proof is meant to be shared, + /// and must not carry what it proves was removed. + pub include_plaintext: bool, +} + +/// Handles for the tests to reach the byte sweep without a whole document. +#[cfg(test)] +pub(crate) mod tests_support { + use pdfcraft_content::parse; + use pdfcraft_cos::{Dict, Object, Stream}; + + use super::extract::{carries_glyphs, parse_cmap}; + use super::limits::{MAX_STREAM, MAX_TOTAL_DECODED, MAX_WORK, MIN_STRUCTURED_NEEDLE}; + use super::sweep::{Needle, add_needle, bytes_counts, bytes_hits, walk_strings}; + use super::util::Ctx; + pub(crate) use super::util::survivor_encodings; + + /// The proof's whole search-work budget (for tests that pre-spend part of it). + pub(crate) fn work_budget() -> u64 { + MAX_WORK + } + + /// The byte sweep of `hay` for a bulk of `count` distinct needles, on a context already + /// `spent` units into its work budget: `Some` (with the needles that occur) while the sweep + /// stays under the budget, `None` once it ran out. + pub(crate) fn bulk_counts(hay: &[u8], text: &str, count: usize, spent: u64) -> Option { + let mut needles: Vec = Vec::new(); + for i in 0..count { + add_needle(&mut needles, false, format!("{text}{i:04}"), Some(i)); + } + let ctx = Ctx::new(Default::default()); + ctx.work.set(spent); + bytes_counts(&needles, hay, MIN_STRUCTURED_NEEDLE, &ctx).map(|hits| hits.len()) + } + + /// The glyph-carrying text-showing operators of a decoded content stream. + pub fn count_text_operators(decoded: &[u8]) -> usize { + parse(decoded).ops.iter().filter(|op| carries_glyphs(op)).count() + } + + /// Does the byte sweep for `text` find it in `hay`? + pub fn found(text: &str, hay: &[u8]) -> bool { + let mut needles: Vec = Vec::new(); + add_needle(&mut needles, false, text.to_string(), Some(0)); + bytes_hits(&needles, hay, 3, &Ctx::new(Default::default())).is_some_and(|h| !h.is_empty()) + } + + /// Does this CMap parse completely (not cut short by a bound)? + pub fn cmap_is_complete(data: &[u8]) -> bool { + parse_cmap(data).is_some() + } + + /// Is an object nested `depth` levels deep walked in full? + pub fn walks_in_full(depth: usize) -> bool { + let mut o = Object::Null; + for _ in 0..depth { + o = Object::Array(vec![o]); + } + walk_strings(&o, 0, &mut |_| {}) + } + + /// Does decoding fail once the cumulative decode budget is used up (and not before)? + pub fn decode_budget_runs_out() -> (bool, bool) { + let ctx = Ctx::new(Default::default()); + let s = Stream::from_raw(Dict::new(), vec![0u8; 100]); + let before = ctx.decode(&s).is_ok(); + let spent = Ctx::with_decode_left(50); + (before, spent.decode(&s).is_err()) + } + + /// Does a decode that fails past the one-stream cap pay for the attempt, so the same bomb + /// cannot be decoded (and fail) again and again for free? + /// `(refused, charged, then refused for want of allowance)`. + pub fn failed_decode_charges_the_budget() -> (bool, bool, bool) { + let big = vec![0u8; MAX_STREAM + (1 << 20)]; + let bomb = Stream::flate(Dict::new(), &big); + drop(big); + let small = Stream::from_raw(Dict::new(), vec![0u8; 100]); + let ctx = Ctx::with_decode_left(u64::try_from(MAX_STREAM + 50).unwrap_or(u64::MAX)); + let refused = ctx.decode(&bomb).is_err(); + let charged = ctx.decoded.get() == MAX_TOTAL_DECODED - 50; + let gone = ctx.decode(&small).is_err(); + (refused, charged, gone) + } + + /// Does the byte search report `None` once the work budget is used up (and not before)? + pub fn search_budget_runs_out() -> (bool, bool) { + let mut needles: Vec = Vec::new(); + add_needle(&mut needles, false, "secret".to_string(), Some(0)); + let ctx = Ctx::new(Default::default()); + let before = bytes_hits(&needles, b"some bytes", 3, &ctx).is_some(); + ctx.work.set(MAX_WORK); + (before, bytes_hits(&needles, b"some bytes", 3, &ctx).is_none()) + } + + /// Is a survivor counted on an earlier page still reported when the work budget runs out on + /// a later one? `(survivor reported, page left unswept)`. + pub fn page_sweep_survivors_survive_the_limit() -> (bool, bool) { + use super::Surface; + use super::sweep::{Acc, sweep_pages}; + let objs = vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R 4 0 R] /Count 2 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 5 0 R /Resources << /Font << /F1 6 0 R >> >> >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 7 0 R /Resources << /Font << /F1 6 0 R >> >> >>".to_vec(), + crate::tests_verify::stream("", b"BT /F1 10 Tf 10 100 Td (SECRETVALUE) Tj ET"), + b"<< /Type /Font /Subtype /TrueType /BaseFont /Arial >>".to_vec(), + crate::tests_verify::stream("", &format!("BT /F1 10 Tf 10 100 Td ({}) Tj ET", "x".repeat(700_000)).into_bytes()), + ]; + let doc = pdfcraft_cos::Document::open(std::sync::Arc::new(crate::tests_verify::pdf_bytes(objs, ""))).unwrap(); + let mut needles = Vec::new(); + add_needle(&mut needles, false, "SECRETVALUE".to_string(), Some(0)); + let mut acc = Acc::new(Ctx::new(Default::default())); + // Page 1 (the survivor's) fits in what is left; page 2 does not. + acc.ctx.work.set(MAX_WORK - 100_000); + sweep_pages(&needles, &doc, 0, "", &mut acc); + let text = acc.reps.get(&Surface::ExtractedText).unwrap(); + (text.survivors > 0, !text.problems.is_empty()) + } + + /// Does a font's `/W` parsing answer to the shared work budget, and not only to its own + /// counter (a form invoked again rebuilds its fonts, and would pay for the same array + /// again)? `(read in full, refused on a spent budget)`. + pub fn font_widths_charge_shares_the_budget() -> (bool, bool) { + use super::extract::extract_page; + let font = b"<< /Type /Font /Subtype /Type0 /BaseFont /X /Encoding /Identity-H /DescendantFonts [<< /Type /Font /Subtype /CIDFontType2 /BaseFont /X /DW 1000 /W [1 65535 250] >>] >>".to_vec(); + let objs = vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /Font << /F2 5 0 R >> >> >>".to_vec(), + crate::tests_verify::stream("", b"BT /F2 10 Tf 10 100 Td <00410042> Tj ET"), + font, + ]; + let bytes = std::sync::Arc::new(crate::tests_verify::pdf_bytes(objs, "")); + let probe = |spent: u64| -> bool { + let doc = pdfcraft_cos::Document::open(bytes.clone()).unwrap(); + let ctx = Ctx::new(Default::default()); + ctx.work.set(spent); + let page = &pdfcraft_model::pages(&doc)[0]; + extract_page(&doc, &page.dict, &ctx, &[]).is_ok() + }; + (probe(0), !probe(MAX_WORK - 10_000)) + } +} diff --git a/crates/redact/src/verify/search_tests.rs b/crates/redact/src/verify/search_tests.rs new file mode 100644 index 0000000000..4b8d84c243 --- /dev/null +++ b/crates/redact/src/verify/search_tests.rs @@ -0,0 +1,34 @@ +use super::{ + sweep::{add_needle, bytes_counts, bytes_hits}, + util::Ctx, +}; +use std::collections::HashSet; + +#[test] +fn a_thousand_needles_share_one_multimegabyte_scan() { + let mut needles = Vec::new(); + for i in 0..1000 { + add_needle(&mut needles, false, format!("SECRET_{i:04}"), None); + } + let hay = vec![b'z'; 8 << 20]; + assert_eq!(bytes_hits(&needles, &hay, 6, &Ctx::new(HashSet::new())), Some(Vec::new())); + assert_eq!(bytes_counts(&needles, &hay, 6, &Ctx::new(HashSet::new())), Some(Vec::new())); +} + +#[test] +fn overlapping_encodings_count_one_occurrence() { + let mut needles = Vec::new(); + add_needle(&mut needles, false, "SECRET42".into(), None); + let text: Vec = "SECRET42".encode_utf16().flat_map(u16::to_be_bytes).collect(); + let mut hay = vec![0xfe, 0xff]; + hay.extend(text); + assert_eq!(bytes_counts(&needles, &hay, 6, &Ctx::new(HashSet::new())), Some(vec![(0, 1)])); +} + +#[test] +fn match_count_exhaustion_is_incomplete_not_a_truncated_pass() { + let mut needles = Vec::new(); + add_needle(&mut needles, true, "SECRET42".into(), None); + let hay = b"SECRET42 ".repeat((1 << 16) + 1); + assert!(bytes_counts(&needles, &hay, 6, &Ctx::new(HashSet::new())).is_none()); +} diff --git a/crates/redact/src/verify/snapshot.rs b/crates/redact/src/verify/snapshot.rs new file mode 100644 index 0000000000..fe74571bb4 --- /dev/null +++ b/crates/redact/src/verify/snapshot.rs @@ -0,0 +1,532 @@ +//! The snapshot taken before applying (what is about to go) and the proof run on the output. + +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::sync::Arc; + +use pdfcraft_content::Matrix; +use pdfcraft_cos::{Dict, Document, ObjRef, Object, SaveOptions, write_full}; + +use super::extract::extract_page; +use super::geometry::{Finding, check_geometry}; +use super::sweep::{Acc, Needle, add_needle, bytes_hits, sweep_doc, walk_strings}; +use super::util::{Ctx, count, latin1, sha256, squash, words}; +use super::{Excision, Proof, ProofOptions, SURFACES, Status, Surface}; +use crate::interp::{Mode, Removed, Scope, process}; +use crate::verify::limits::MIN_NEEDLE; +use crate::{Mark, RedactError, Report, page_streams}; + +pub(super) struct RegionSnap { + pub(super) page: usize, + pub(super) region: [f64; 4], + pub(super) mark: ObjRef, + pub(super) runs: Vec<(Vec, String)>, + /// The runs of one text operator joined, where it split them: `[(sec) (ret)] TJ` reads as + /// `secret`, which a copy elsewhere in the file holds whole. + pub(super) joined: Vec<(Vec, String)>, + /// The character codes the runs hold (a composite font's codes are several bytes). + pub(super) codes: usize, +} + +pub(super) struct PageSnap { + pub(super) sha: Option, + pub(super) text_ops: Option, + pub(super) undecodable_ops: usize, +} + +/// What redaction is about to remove, recorded before it does. +pub struct Snapshot { + regions: Vec, + pages: BTreeMap, + needles: Vec, + /// Things that made the "before" side unreadable (reported on the text surface). + problems: Vec, + /// The document's page text before redaction (decoded view, code view), for the allowance + /// of needles added later. + src: (String, String), + /// The names of the document (structure redaction never edits), for the name allowance of + /// needles added later. + names: BTreeMap, usize>, + /// Digests of the decoded content of the streams this run created to draw the redaction + /// boxes and labels (see [`Snapshot::set_overlay_streams`]). + overlay: HashSet, +} + +impl std::fmt::Debug for Snapshot { + // The snapshot holds what is about to be redacted: never print it. + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Snapshot").field("regions", &self.regions.len()).field("needles", &self.needles.len()).finish() + } +} + +impl Snapshot { + /// Record what redacting the marks on `pages` (0-based; `None` = all) will remove. Call it + /// before [`apply`](crate::apply); it changes nothing. + pub fn capture(doc: &Document, pages: Option<&[usize]>) -> Result { + let all = crate::marks(doc); + let chosen: Vec<&Mark> = all.iter().filter(|m| pages.is_none_or(|p| p.contains(&m.page))).collect(); + if chosen.is_empty() { + return Err(RedactError::NothingToApply); + } + Snapshot::capture_marks(doc, &chosen) + } + + pub(crate) fn capture_marks(doc: &Document, chosen: &[&Mark]) -> Result { + let mut by_page: Vec = chosen.iter().map(|m| m.page).collect(); + by_page.sort_unstable(); + by_page.dedup(); + let page_list = pdfcraft_model::pages(doc); + let ctx = Ctx::new(HashSet::new()); + let mut regions: Vec = Vec::new(); + let mut pages = BTreeMap::new(); + let mut images = pdfcraft_cos::ImageBudget::new(doc); + // What was cut, each copy once, however many overlapping marks it lay under: the allowance + // for what stays is what the source holds less this. + let (mut cut_text, mut cut_raw) = (String::new(), String::new()); + for &pi in &by_page { + let page = page_list.get(pi).ok_or(RedactError::Unreadable(pi + 1))?; + let page_marks: Vec<&&Mark> = chosen.iter().filter(|m| m.page == pi).collect(); + let rects: Vec<[f64; 4]> = page_marks.iter().flat_map(|m| m.rects.iter().copied()).collect(); + let (_, data) = page_streams(doc, &page.dict, pi)?; + let resources = page.dict.get(b"Resources").and_then(|r| doc.resolve(r).as_dict().cloned()).unwrap_or_default(); + // The interpreter's own removal path, on a scratch copy: what it takes out is what + // has to be gone. + let mut scratch = doc.clone(); + let mut sink = Report::default(); + let removed: Vec = { + let mut scope = Scope::new(&rects, Mode::Apply, &mut sink, pdfcraft_cos::DecodeBudget::page(), &mut images); + scope.removed = Some(Vec::new()); + process(&mut scratch, &mut scope, &data, &resources, Matrix::IDENTITY); + scope.removed.take().unwrap_or_default() + }; + let lines = joined(&removed); + let mut seen = HashSet::new(); + for line in lines.iter().filter(|l| seen.insert((l.op, l.at, l.raw.clone()))) { + cut_text.push('\u{1}'); + cut_text.push_str(&squash(&line.text)); + cut_raw.push('\u{1}'); + cut_raw.push_str(&squash(&latin1(&line.raw))); + } + let before = extract_page(doc, &page.dict, &ctx, &[]); + pages.insert( + pi, + PageSnap { + sha: Some(sha256(&data)), + text_ops: before.as_ref().ok().map(|e| e.glyph_ops), + undecodable_ops: before.as_ref().map_or(0, |e| e.undecodable_ops), + }, + ); + let mut k = 0; + for m in &page_marks { + for r in &m.rects { + let hits: Vec<&Removed> = removed.iter().filter(|x| x.rect == k).collect(); + let runs = hits.iter().map(|x| (x.raw.clone(), x.text.clone())).collect(); + let codes = hits.iter().map(|x| x.codes).sum(); + let joined = lines.iter().filter(|l| l.rect == k && l.pieces > 1).map(|l| (l.raw.clone(), l.text.clone())).collect(); + regions.push(RegionSnap { page: pi, region: *r, mark: m.obj, runs, joined, codes }); + k += 1; + } + } + } + // The document's text as the independent extractor reads it, for the allowance of + // strings that legitimately remain elsewhere. + let (mut src_text, mut src_raw) = (String::new(), String::new()); + let mut problems = Vec::new(); + for (i, p) in page_list.iter().enumerate() { + match extract_page(doc, &p.dict, &ctx, &[]) { + Ok(e) => { + src_text.push_str(&e.text); + src_raw.push_str(&e.raw); + } + Err(()) => { + problems.push(format!("page {} of the source can't be read, so strings left on it can't be told from removed ones", i + 1)) + } + } + } + let mut needles: Vec = Vec::new(); + for (ri, r) in regions.iter().enumerate() { + for (raw, text) in r.runs.iter().chain(&r.joined) { + let raw_s = latin1(raw); + for (raw_view, whole) in [(false, text.trim().to_string()), (true, raw_s)] { + let mut shown: Vec = Vec::new(); + if !squash(&whole).is_empty() { + shown.push(whole.clone()); + shown.extend(words(&whole).into_iter().filter(|w| *w != whole).map(str::to_string)); + } + for s in shown { + add_needle(&mut needles, raw_view, s, Some(ri)); + } + } + } + } + for (src, cut, raw) in [(&src_text, &cut_text, false), (&src_raw, &cut_raw, true)] { + match (super::sweep::key_counts(&needles, src, raw, &ctx), super::sweep::key_counts(&needles, cut, raw, &ctx)) { + (Some(source), Some(removed)) => { + let removed: BTreeMap = removed.into_iter().collect(); + for (i, total) in source { + if let Some(n) = needles.get_mut(i) { + n.allowance = total.saturating_sub(removed.get(&i).copied().unwrap_or(0)); + } + } + } + _ => problems.push("the source text allowances could not be counted in full".into()), + } + } + // The names the removed text legitimately sits in (a redacted word that is also a + // `/Subtype`, a `/Filter`, a `/BaseFont`): names are structure redaction never edits, + // so the output may hold the input's own matches, and only a surplus is a survivor. + let names = all_names(doc); + let name_allow: Vec> = needles.iter().map(|n| count_name_matches(&names, n, &ctx)).collect(); + if name_allow.iter().any(Option::is_none) { + problems.push("the name allowances could not be counted in full, so names are swept with none of them allowed".into()); + } + for (n, allow) in needles.iter_mut().zip(name_allow) { + n.name_allowance = allow.unwrap_or(0); + } + Ok(Snapshot { regions, pages, needles, problems, src: (src_text, src_raw), names, overlay: HashSet::new() }) + } + + /// Tell the proof which streams this run created to draw the redaction boxes and labels + /// (`created`: references into `doc`, the document after `apply`). Call it before + /// [`prove`](Snapshot::prove) / [`prove_saved`](Snapshot::prove_saved). + /// + /// Those streams paint over the regions on purpose and may show label text, so the proof + /// leaves them out of the text sweep and the paint check. They are recognised by the digest + /// of their decoded content (a save renumbers objects, so references would not survive it), + /// and *only* those: a stream that merely carries a `/PCMark /Redaction` key in the input is + /// not exempt. Streams that can't be read here are an error; nothing is then exempted. + pub fn set_overlay_streams(&mut self, doc: &Document, created: &[ObjRef]) -> Result<(), RedactError> { + let ctx = Ctx::new(HashSet::new()); + let mut digests = HashSet::new(); + for r in created { + let obj = doc.try_get(r.num).map_err(|_| RedactError::OverlayUnreadable)?; + let Object::Stream(s) = &*obj else { continue }; + let data = ctx.decode(s).map_err(|_| RedactError::OverlayUnreadable)?; + digests.insert(sha256(&[data])); + } + self.overlay = digests; + Ok(()) + } + + /// Also look for these strings in the output: text that left the document by another route + /// than a page's glyphs (a removed annotation's `/Contents` or `/RC`, a form field's value). + /// Each is searched whole, on every surface. A string that also legitimately remains as page + /// text is counted on the page-content surfaces only (as for the removed glyph runs). + /// Strings that are blank or shorter than the minimum needle are ignored. + pub fn add_removed_strings(&mut self, strings: &[String]) { + for s in strings { + let text = s.trim(); + let key = squash(text); + if key.chars().count() < MIN_NEEDLE { + continue; + } + let known = self.needles.iter().any(|n| !n.raw_view && n.key == key); + // No owner: a survivor fails the proof through its surface; no region carries it. + add_needle(&mut self.needles, false, text.to_string(), None); + if !known { + let ctx = Ctx::new(HashSet::new()); + let matches = self.needles.last().map(|n| count_name_matches(&self.names, n, &ctx)); + if let Some(n) = self.needles.last_mut() { + n.allowance = count(&self.src.0, &n.key); + n.name_allowance = matches.flatten().unwrap_or(0); + } + if matches == Some(None) { + self.problems.push("the name allowances could not be counted in full, so names are swept with none of them allowed".into()); + } + } + } + } + + /// Serialize `doc` the way a save would and sweep the result. The output is re-opened with + /// `password`, the credentials the caller opened the document with (`None` tries the empty + /// password): a save of an encrypted document is itself encrypted, and a proof that could + /// not read its own subject would prove nothing. + pub fn prove(&self, doc: &Document, password: Option<&str>, opts: &ProofOptions) -> Proof { + match write_full(doc, &SaveOptions::default()) { + Ok(bytes) => self.prove_inner(&bytes, Some(doc), password, opts), + Err(e) => { + let mut p = self.prove_inner(&[], None, password, opts); + p.failures.push(format!("the document could not be serialized for the sweep: {e}")); + p + } + } + } + + /// Sweep an already saved file (for example an incrementally saved one, whose earlier + /// revision still holds the original content). The file is opened with `password`, the + /// credentials it is read with. + pub fn prove_saved(&self, bytes: &[u8], password: Option<&str>, opts: &ProofOptions) -> Proof { + self.prove_inner(bytes, None, password, opts) + } + + fn prove_inner(&self, bytes: &[u8], live: Option<&Document>, password: Option<&str>, opts: &ProofOptions) -> Proof { + let mut acc = Acc::new(Ctx::new(self.overlay.clone())); + let mut failures: Vec = Vec::new(); + for p in &self.problems { + acc.problem(Surface::ExtractedText, p.clone()); + } + // Always the caller's credentials, never unconditionally the empty password: a + // document whose user password is not empty must be provable with that password, and + // must fail closed (an unswept output, never a false pass) without it. + let parsed = if bytes.is_empty() { None } else { Document::open_with_password(Arc::new(bytes.to_vec()), password).ok() }; + match &parsed { + Some(out) => sweep_doc(&self.needles, out, bytes, 0, "", &mut acc), + None => { + let why = "the output could not be read back".to_string(); + for s in SURFACES { + acc.problem(s, why.clone()); + } + failures.push(why); + } + } + if live.is_some_and(|d| !d.full_save_required()) { + acc.survivor(Surface::Revisions, "the document would be saved as an incremental update, keeping the original revision".into(), None); + } + + // The non-text content under each region, on the output. + let (findings, after) = match &parsed { + Some(out) => self.check_output(out, &acc.ctx), + None => Default::default(), + }; + let mut entries = Vec::with_capacity(self.regions.len()); + for (i, r) in self.regions.iter().enumerate() { + let (entry, failure) = self.entry(i, r, &findings, &after, &acc.failed_needles, opts); + failures.extend(failure); + entries.push(entry); + } + let surfaces = acc.finish(); + for s in &surfaces { + if s.survivors > 0 { + failures.push(format!("{:?}: {} survivor(s) at {}", s.surface, s.survivors, s.locations.join(", "))); + } + for p in &s.problems { + failures.push(format!("{:?} could not be swept: {p}", s.surface)); + } + } + Proof { entries, surfaces, failures } + } + + /// The non-text content under each region of the output (what still paints or shows there), + /// and per page the digest and glyph-operator count of what `apply` left. + fn check_output(&self, out: &Document, ctx: &Ctx) -> (HashMap>, PagesAfter) { + let mut findings: HashMap> = HashMap::new(); + let mut after: PagesAfter = BTreeMap::new(); + let out_pages = pdfcraft_model::pages(out); + for &pi in self.pages.keys() { + let idx: Vec = (0..self.regions.len()).filter(|&i| self.regions.get(i).is_some_and(|r| r.page == pi)).collect(); + let Some(page) = out_pages.get(pi) else { + for &i in &idx { + findings.entry(i).or_default().push(Finding::Missing); + } + continue; + }; + let rects: Vec<[f64; 4]> = idx.iter().filter_map(|&i| self.regions.get(i).map(|r| r.region)).collect(); + let (sha, ops, shown) = page_after(out, &page.dict, ctx, &rects); + after.insert(pi, (sha, ops)); + match shown { + Some(shown) => { + for (k, n) in shown.into_iter().enumerate() { + if let (Some(&i), true) = (idx.get(k), n > 0) { + findings.entry(i).or_default().push(Finding::Text(n)); + } + } + } + None => { + for &i in &idx { + findings + .entry(i) + .or_default() + .push(Finding::Unverifiable("the page text can't be placed, so what still shows under the region can't be checked")); + } + } + } + for (k, f) in check_geometry(out, &page.dict, &rects, ctx) { + if let Some(&i) = idx.get(k) { + findings.entry(i).or_default().push(f); + } + } + } + (findings, after) + } + + /// The manifest entry of region `i`, and why it failed when it did. + fn entry( + &self, + i: usize, + r: &RegionSnap, + findings: &HashMap>, + after: &PagesAfter, + failed_needles: &HashSet, + opts: &ProofOptions, + ) -> (Excision, Option) { + let mut failure = None; + let snap = self.pages.get(&r.page); + let (sha_after, ops_after) = after.get(&r.page).cloned().unwrap_or((None, None)); + let sha_before = snap.and_then(|s| s.sha.clone()); + let owned: Vec<&Needle> = self.needles.iter().filter(|n| n.owners.contains(&i)).collect(); + let survived = self.needles.iter().enumerate().filter(|(ni, n)| n.owners.contains(&i) && failed_needles.contains(ni)).count(); + let mut detail: Vec = Vec::new(); + let mut status = if r.runs.is_empty() { Status::VerifiedNoTextInRegion } else { Status::Verified }; + let fs = findings.get(&i).map(Vec::as_slice).unwrap_or_default(); + if survived > 0 { + status = Status::Failed; + detail.push(format!("{survived} removed string(s) still found")); + } + for f in fs { + match f { + Finding::Text(n) => detail.push(format!("{n} glyph(s) still show inside the region (placed independently of the redaction)")), + Finding::Image => detail.push("an image still carries pixels under the region".into()), + Finding::Path => detail.push("a path still paints inside the region".into()), + Finding::Shading => detail.push("a shading still paints inside the region".into()), + Finding::Missing => detail.push("the page is missing from the output".into()), + Finding::Unverifiable(why) => detail.push((*why).to_string()), + } + if !matches!(f, Finding::Unverifiable(_)) { + status = Status::Failed; + } else if status != Status::Failed { + status = Status::Unverifiable; + } + } + if sha_before.is_none() || sha_after.is_none() { + status = Status::Failed; + detail.push("the page content can't be read on one side, so it can't be compared".into()); + } else if !r.runs.is_empty() && sha_before == sha_after { + status = Status::Failed; + detail.push("glyphs were removed but the page content is unchanged".into()); + } + if r.runs.is_empty() && snap.is_some_and(|s| s.undecodable_ops > 0) && status != Status::Failed { + status = Status::Unverifiable; + detail.push("the page draws text that can't be decoded, so \"no text in the region\" can't be checked".into()); + } + let ambiguous = owned.iter().filter(|n| n.allowance > 0).count(); + if ambiguous > 0 { + detail.push(format!("{ambiguous} string(s) also remain in retained content; only a surplus over those copies counts")); + } + if matches!(status, Status::Failed | Status::Unverifiable) { + failure = Some(format!("page {} region {:?}: {}", r.page + 1, r.region, detail.join("; "))); + } + let entry = Excision { + page: r.page, + region: r.region, + mark: r.mark, + status, + detail: detail.join("; "), + runs_removed: r.runs.len(), + codes_removed: r.codes, + page_sha256_after: sha_after, + text_ops_before: snap.and_then(|s| s.text_ops), + text_ops_after: ops_after, + removed_text: opts.include_plaintext.then(|| r.runs.iter().map(|(_, t)| t.clone()).collect()), + }; + (entry, failure) + } +} + +/// Per page: the digest of its content in the output and its glyph-operator count. +type PagesAfter = BTreeMap, Option)>; + +/// Every name of the document (the values, wherever an object holds them), with how often +/// each occurs. +fn all_names(doc: &Document) -> BTreeMap, usize> { + let mut names: BTreeMap, usize> = BTreeMap::new(); + for num in doc.object_numbers() { + if let Ok(obj) = doc.try_get(num) { + walk_strings(&obj, 0, &mut |o| { + if let Object::Name(n) = o { + let seen = names.entry(n.clone()).or_default(); + *seen = seen.saturating_add(1); + } + }); + } + } + names +} + +/// How many of `names` carry `needle`, by the same search the sweep runs on a name. `None` +/// when the search work ran out: an unknown allowance must not read as "none allowed", so the +/// caller reports that (and sweeps names with no allowance). +fn count_name_matches(names: &BTreeMap, usize>, needle: &Needle, ctx: &Ctx) -> Option { + if needle.short() { + return Some(0); + } + let mut total = 0usize; + for (name, seen) in names { + if !bytes_hits(std::slice::from_ref(needle), name, MIN_NEEDLE, ctx)?.is_empty() { + total = total.saturating_add(*seen); + } + } + Some(total) +} + +/// The page's content digest and glyph-operator count in the output, and per region the glyphs +/// still placed inside it (`None` when the page's text can't be placed). The digest covers the +/// page's content as `apply` left it, the overlay streams included. +pub(super) fn page_after(doc: &Document, page: &Dict, ctx: &Ctx, regions: &[[f64; 4]]) -> (Option, Option, Option>) { + let sha = page_streams(doc, page, 0).ok().map(|(_, d)| sha256(&d)); + let e = extract_page(doc, page, ctx, regions).ok(); + (sha, e.as_ref().map(|e| e.glyph_ops), e.map(|e| e.shown_in)) +} + +/// One text operator's removed runs under one region, as a reader takes them: joined. +struct Line { + rect: usize, + op: usize, + at: (usize, usize), + raw: Vec, + text: String, + pieces: usize, +} + +/// The runs of `removed` joined by text operator and region, in the order they were shown. +fn joined(removed: &[Removed]) -> Vec { + let mut lines: Vec = Vec::new(); + let mut found: HashMap<(usize, usize), usize> = HashMap::new(); + for r in removed { + let line = found.get(&(r.op, r.rect)).copied().and_then(|i| lines.get_mut(i)); + match line { + Some(l) => { + l.raw.extend_from_slice(&r.raw); + l.text.push_str(&r.text); + l.pieces += 1; + } + None => { + found.insert((r.op, r.rect), lines.len()); + lines.push(Line { rect: r.rect, op: r.op, at: r.at, raw: r.raw.clone(), text: r.text.clone(), pieces: 1 }); + } + } + } + lines +} + +#[cfg(test)] +mod tests { + use super::*; + + fn run(rect: usize, op: usize, place: usize, text: &str) -> Removed { + Removed { rect, raw: text.as_bytes().to_vec(), text: text.to_string(), codes: text.len(), op, at: (place, 0) } + } + + #[test] + fn a_word_split_across_a_tj_array_is_one_line_for_each_region() { + let lines = joined(&[run(0, 1, 0, "sec"), run(1, 1, 0, "sec"), run(0, 1, 1, "ret"), run(1, 1, 1, "ret"), run(0, 2, 0, "other")]); + let shown: Vec<(usize, usize, &str, usize)> = lines.iter().map(|l| (l.rect, l.op, l.text.as_str(), l.pieces)).collect(); + assert_eq!(shown, [(0, 1, "secret", 2), (1, 1, "secret", 2), (0, 2, "other", 1)]); + } + + #[test] + fn overlapping_marks_are_one_copy_and_two_copies_stay_two() { + // Under two marks at once: the same operator, place and bytes, so one copy was cut. + let both = joined(&[run(0, 1, 0, "word"), run(1, 1, 0, "word")]); + let mut seen = HashSet::new(); + assert_eq!(both.iter().filter(|l| seen.insert((l.op, l.at, l.raw.clone()))).count(), 1); + // The same word twice in one array, one under each mark: two copies were cut. + let apart = joined(&[run(0, 1, 0, "word"), run(1, 1, 2, "word")]); + let mut seen = HashSet::new(); + assert_eq!(apart.iter().filter(|l| seen.insert((l.op, l.at, l.raw.clone()))).count(), 2); + } + + #[test] + fn a_spent_decode_budget_is_told_apart() { + assert!(!Ctx::new(HashSet::new()).decode_spent()); + assert!(Ctx::with_decode_left(0).decode_spent()); + } +} diff --git a/crates/redact/src/verify/sweep.rs b/crates/redact/src/verify/sweep.rs new file mode 100644 index 0000000000..73b44c005d --- /dev/null +++ b/crates/redact/src/verify/sweep.rs @@ -0,0 +1,759 @@ +//! Sweeping the serialized output for the removed strings, on every surface a reader could +//! recover them from. + +use std::collections::{BTreeMap, BTreeSet, HashSet}; +use std::sync::Arc; + +use pdfcraft_cos::{Dict, Document, ObjRef, Object, PdfString}; + +use super::extract::extract_page; +use super::util::{Ctx, decode_text, find, image_codec, spellings, squash, survivor_encodings, xml_unescape}; +use super::{SURFACES, Surface, SurfaceReport, Verdict}; +use crate::verify::limits::{MAX_NESTING, MAX_WALK, MIN_NEEDLE, MIN_STRUCTURED_NEEDLE}; + +/// A string the sweep looks for. +pub(super) struct Needle { + /// The code bytes as Latin-1 text (the "raw view" of a page) rather than decoded text. + pub(super) raw_view: bool, + /// As removed, whitespace included (for the byte encodings). + pub(super) shown: String, + /// `shown` without whitespace (for page text, which is compared whitespace-blind). + pub(super) key: String, + /// What the raw file may spell it as. + pub(super) bytes: Vec>, + pub(super) owners: BTreeSet, + /// Occurrences that legitimately remain in content that was not redacted. The surfaces that + /// hold page content (the page text, the raw file, decoded streams) count occurrences over + /// the whole document and fail on a surplus; every other surface (strings, Info, XMP, + /// attachments) is swept with no allowance. + pub(super) allowance: usize, + /// Names of the input that carry this needle (a redacted word that is also a `/Subtype`, a + /// `/Filter`, a `/BaseFont`). Names are structure redaction never edits, so the name surface + /// is swept against the input's own matches: the output may hold as many as the input did, + /// and only a surplus is a survivor. + pub(super) name_allowance: usize, +} + +impl Needle { + pub(super) fn short(&self) -> bool { + self.key.chars().count() < MIN_NEEDLE + } + + /// Long enough to be looked for in bytes (`min`: the shortest the haystack allows). + pub(super) fn at_least(&self, min: usize) -> bool { + self.key.chars().count() >= min + } +} + +pub(super) fn add_needle(set: &mut Vec, raw_view: bool, shown: String, owner: Option) { + let key = squash(&shown); + if key.is_empty() { + return; + } + if let Some(n) = set.iter_mut().find(|n| n.raw_view == raw_view && n.key == key) { + n.owners.extend(owner); + return; + } + let bytes = if raw_view { + // The code bytes themselves (Latin-1 round-trips them). + let raw: Vec = shown.chars().filter_map(|c| u8::try_from(u32::from(c)).ok()).collect(); + spellings([raw]) + } else { + survivor_encodings(&shown) + }; + set.push(Needle { raw_view, shown, key, bytes, owners: owner.into_iter().collect(), allowance: 0, name_allowance: 0 }); +} + +pub(super) struct Acc { + pub(super) reps: BTreeMap, + pub(super) failed_needles: HashSet, + pub(super) ctx: Ctx, +} + +impl Acc { + pub(super) fn new(ctx: Ctx) -> Self { + let mut a = Acc { reps: BTreeMap::new(), failed_needles: HashSet::new(), ctx }; + for s in SURFACES { + a.rep(s); + } + a + } + + pub(super) fn rep(&mut self, surface: Surface) -> &mut SurfaceReport { + self.reps.entry(surface).or_insert_with(|| SurfaceReport { + surface, + verdict: Verdict::Absent, + scanned: 0, + raw_only: 0, + survivors: 0, + locations: Vec::new(), + problems: Vec::new(), + }) + } + + pub(super) fn scanned(&mut self, s: Surface) { + self.rep(s).scanned += 1; + } + + pub(super) fn raw_only(&mut self, s: Surface) { + self.rep(s).raw_only += 1; + } + + pub(super) fn survivor(&mut self, s: Surface, loc: String, needle: Option) { + self.survivors(s, loc, needle, 1); + } + + /// `count` survivors at one place. + pub(super) fn survivors(&mut self, s: Surface, loc: String, needle: Option, count: usize) { + if let Some(n) = needle { + self.failed_needles.insert(n); + } + let r = self.rep(s); + r.survivors = r.survivors.saturating_add(count); + if !r.locations.contains(&loc) && r.locations.len() < 64 { + r.locations.push(loc); + } + } + + pub(super) fn problem(&mut self, s: Surface, msg: String) { + let r = self.rep(s); + if r.problems.len() < 64 { + r.problems.push(msg); + } + } + + /// The needles (at least `min` long) found in `hay`; running out of search work is a + /// problem of `surface`. + pub(super) fn bytes(&mut self, surface: Surface, needles: &[Needle], hay: &[u8], min: usize) -> Vec { + bytes_hits(needles, hay, min, &self.ctx).unwrap_or_else(|| { + self.problem(surface, "the search work limit was reached".into()); + Vec::new() + }) + } + + /// How often each needle (at least `min` long) occurs in `hay`, as `(needle, count)`. + pub(super) fn counts(&mut self, surface: Surface, needles: &[Needle], hay: &[u8], min: usize) -> Vec<(usize, usize)> { + bytes_counts(needles, hay, min, &self.ctx).unwrap_or_else(|| { + self.problem(surface, "the search work limit was reached".into()); + Vec::new() + }) + } + + /// Survivors and problems so far, over all surfaces. + pub(super) fn totals(&self) -> (usize, usize) { + (self.reps.values().map(|r| r.survivors).sum(), self.reps.values().map(|r| r.problems.len()).sum()) + } + + pub(super) fn finish(mut self) -> Vec { + for r in self.reps.values_mut() { + r.verdict = if r.survivors > 0 { + Verdict::Survivor + } else if !r.problems.is_empty() { + Verdict::Unswept + } else if r.scanned + r.raw_only == 0 { + Verdict::Absent + } else { + Verdict::Clean + }; + } + self.reps.into_values().collect() + } +} + +/// Occurrences of needles on one surface, per place, so that what legitimately remains +/// (a needle's allowance) is taken off the document-wide total once, not once per place. +#[derive(Default)] +pub(super) struct Tally(BTreeMap>); + +impl Tally { + pub(super) fn add(&mut self, needle: usize, loc: &str, count: usize) { + if count > 0 { + self.0.entry(needle).or_default().push((loc.to_string(), count)); + } + } + + /// Report what is left of each needle's occurrences after its allowance (none inside an + /// attachment, where nothing legitimately remains). + pub(super) fn settle(self, surface: Surface, needles: &[Needle], depth: usize, acc: &mut Acc) { + self.settle_with(surface, needles, depth, acc, |n| n.allowance) + } + + /// The name surface: the input's own name matches are the allowance (redaction never edits + /// names), so what is left is a surplus of matches over the input. + pub(super) fn settle_names(self, surface: Surface, needles: &[Needle], depth: usize, acc: &mut Acc) { + self.settle_with(surface, needles, depth, acc, |n| n.name_allowance) + } + + fn settle_with(self, surface: Surface, needles: &[Needle], depth: usize, acc: &mut Acc, allow_of: impl Fn(&Needle) -> usize) { + for (ni, places) in self.0 { + let mut allow = if depth == 0 { needles.get(ni).map_or(0, &allow_of) } else { 0 }; + for (loc, count) in places { + let used = count.min(allow); + allow -= used; + if count > used { + acc.survivors(surface, loc, Some(ni), count - used); + } + } + } + } +} + +/// Search all encodings in one bounded pass, including overlapping matches. +fn pattern_matches<'a>( + patterns: impl IntoIterator, + hay: &[u8], + ctx: &Ctx, + mut found: impl FnMut(usize, usize, usize) -> bool, +) -> Option<()> { + const MAX_PATTERNS: usize = 100_000; + const MAX_PATTERN_BYTES: usize = 16 << 20; + const MAX_MATCHES: usize = 1 << 20; + if !ctx.charge(hay.len()) { + return None; + } + let mut map: BTreeMap, Vec> = BTreeMap::new(); + let (mut bytes, mut links) = (0usize, 0usize); + for (owner, pattern) in patterns { + if pattern.is_empty() { + continue; + } + if !ctx.charge(pattern.len().saturating_add(1)) { + return None; + } + if !map.contains_key(pattern) { + bytes = bytes.checked_add(pattern.len())?; + if bytes > MAX_PATTERN_BYTES || map.len() >= MAX_PATTERNS { + return None; + } + } + let owners = map.entry(pattern.to_vec()).or_default(); + if owners.last() != Some(&owner) { + links = links.checked_add(1)?; + if links > MAX_PATTERNS { + return None; + } + owners.push(owner); + } + } + if map.is_empty() { + return Some(()); + } + let matcher = aho_corasick::AhoCorasick::new(map.keys()).ok()?; + let owners: Vec> = map.into_values().collect(); + let mut matches = 0usize; + for hit in matcher.find_overlapping_iter(hay) { + for &owner in owners.get(hit.pattern().as_usize())? { + matches = matches.checked_add(1)?; + if matches > MAX_MATCHES || !ctx.charge(1) || !found(owner, hit.start(), hit.end()) { + return None; + } + } + } + Some(()) +} + +/// Count normalized text keys in one pass, with the same non-overlap rule as `str::matches`. +pub(super) fn key_counts(needles: &[Needle], hay: &str, raw: bool, ctx: &Ctx) -> Option> { + key_counts_from(needles, hay, raw, ctx, 0) +} + +/// [`key_counts`] of the needles at least `min` long, so a one-letter needle can't use up the match +/// limits of a search it takes no part in. +fn key_counts_from(needles: &[Needle], hay: &str, raw: bool, ctx: &Ctx, min: usize) -> Option> { + let patterns = needles.iter().enumerate().filter(|(_, n)| n.raw_view == raw && n.at_least(min)).map(|(i, n)| (i, n.key.as_bytes())); + let mut spans: BTreeMap> = BTreeMap::new(); + pattern_matches(patterns, hay.as_bytes(), ctx, |i, a, b| { + let hits = spans.entry(i).or_default(); + if hits.len() >= MAX_COUNTED { + return false; + } + hits.push((a, b)); + true + })?; + Some( + spans + .into_iter() + .map(|(i, mut hits)| { + hits.sort_unstable(); + let (mut count, mut end) = (0, 0); + for (a, b) in hits { + if a >= end { + count += 1; + end = b; + } + } + (i, count) + }) + .collect(), + ) +} + +pub(super) fn bytes_hits(needles: &[Needle], hay: &[u8], min: usize, ctx: &Ctx) -> Option> { + let patterns = needles.iter().enumerate().filter(|(_, n)| n.at_least(min)).flat_map(|(i, n)| n.bytes.iter().map(move |b| (i, b.as_slice()))); + let mut found = BTreeSet::new(); + pattern_matches(patterns, hay, ctx, |i, _, _| { + found.insert(i); + true + })?; + Some(found.into_iter().collect()) +} + +/// A larger count is incomplete evidence, never a silently truncated count. +pub(super) const MAX_COUNTED: usize = 1 << 16; + +pub(super) fn bytes_counts(needles: &[Needle], hay: &[u8], min: usize, ctx: &Ctx) -> Option> { + let patterns = needles.iter().enumerate().filter(|(_, n)| n.at_least(min)).flat_map(|(i, n)| n.bytes.iter().map(move |b| (i, b.as_slice()))); + let mut spans: BTreeMap> = BTreeMap::new(); + pattern_matches(patterns, hay, ctx, |i, a, b| { + let hits = spans.entry(i).or_default(); + if hits.len() >= MAX_COUNTED { + return false; + } + hits.push((a, b)); + true + })?; + Some( + spans + .into_iter() + .map(|(i, mut spans)| { + spans.sort_unstable(); + spans.dedup(); + let (mut count, mut end) = (0usize, 0usize); + for (a, b) in spans { + if count == 0 || a >= end { + count += 1; + } + end = end.max(b); + } + (i, count) + }) + .collect(), + ) +} + +pub(super) fn string_hits(needles: &[Needle], value: &PdfString, ctx: &Ctx) -> Option> { + let text = value.to_text(); + let key = squash(&text); + if !ctx.charge(value.bytes.len().saturating_add(key.len())) { + return None; + } + let mut found: BTreeSet = bytes_hits(needles, &value.bytes, MIN_NEEDLE, ctx)?.into_iter().collect(); + let patterns = needles.iter().enumerate().filter(|(_, n)| !n.raw_view && !n.short()).map(|(i, n)| (i, n.key.as_bytes())); + pattern_matches(patterns, key.as_bytes(), ctx, |i, _, _| { + found.insert(i); + true + })?; + let tokens: BTreeSet<&str> = text.split(|c: char| !c.is_alphanumeric()).filter(|s| !s.is_empty()).collect(); + for (i, needle) in needles.iter().enumerate().filter(|(_, n)| n.short() && n.allowance == 0 && !n.raw_view) { + if !ctx.charge(needle.shown.len().saturating_add(1)) { + return None; + } + if tokens.contains(needle.shown.as_str()) { + found.insert(i); + } + } + Some(found.into_iter().collect()) +} + +/// Streams whose data is binary rather than text: images, embedded font programs, colour +/// profiles, cross-reference and object streams (the objects in the latter are swept as +/// objects). Their decoded data is swept like any other stream's; the pixels of an image codec +/// stay undecoded (text that exists only as pixels is not text), and one that won't decode (or is +/// past the cap) is searched as stored bytes only. A short needle "found" in stored bytes is +/// chance. +pub(super) fn binary_stream(d: &Dict) -> bool { + let subtype = d.name(b"Subtype"); + subtype == Some(b"Image".as_slice()) + || matches!(d.name(b"Type"), Some(b"XRef" | b"ObjStm")) + || matches!(subtype, Some(b"Type1C" | b"CIDFontType0C" | b"OpenType")) + || [b"Length1".as_slice(), b"Length2", b"Length3"].iter().any(|k| d.contains(k)) + || (d.contains(b"N") && !d.contains(b"Type") && subtype.is_none()) +} + +/// What a file identifier (`/ID` of the trailer) is spelled as in the file. +pub(super) fn id_spellings(doc: &Document) -> Vec> { + let Some(Object::Array(ids)) = doc.trailer().get(b"ID").map(|o| doc.resolve(o)).as_deref().cloned() else { return Vec::new() }; + ids.iter().filter_map(Object::as_string).flat_map(|s| spellings([s.bytes.to_vec()])).collect() +} + +/// Visit `o` and everything nested in it. False when something lay deeper than [`MAX_WALK`] +/// and was not visited: the caller reports that as unswept. +pub(super) fn walk_strings(o: &Object, depth: usize, f: &mut impl FnMut(&Object)) -> bool { + if depth > MAX_WALK { + return false; + } + f(o); + // Every child is visited even after one is too deep: nothing is skipped silently. + let mut ok = true; + match o { + Object::Array(a) => a.iter().for_each(|x| ok &= walk_strings(x, depth + 1, f)), + Object::Dict(d) => d.iter().for_each(|(_, v)| ok &= walk_strings(v, depth + 1, f)), + Object::Stream(s) => s.dict.iter().for_each(|(_, v)| ok &= walk_strings(v, depth + 1, f)), + _ => {} + } + ok +} + +/// Sweep a document (the output, or an attachment) on every surface. +pub(super) fn sweep_doc(needles: &[Needle], doc: &Document, bytes: &[u8], depth: usize, tag: &str, acc: &mut Acc) { + let (embedded, overlay_raw) = sweep_objects(needles, doc, depth, tag, acc); + sweep_file(needles, doc, bytes, overlay_raw, depth, tag, acc); + sweep_pages(needles, doc, depth, tag, acc); + sweep_info(needles, doc, tag, acc); + sweep_xmp(needles, doc, tag, acc); + + // Revision structure: one revision, no earlier content kept. + acc.scanned(Surface::Revisions); + let revs = doc.revisions().len(); + if revs > 1 || doc.trailer().contains(b"Prev") { + acc.survivor(Surface::Revisions, format!("{tag}{revs} revisions: earlier ones keep the original content"), None); + } + + sweep_attachments(needles, doc, &embedded, depth, tag, acc); +} + +/// Every object: strings, names and stream data (orphans included). Returns the embedded file +/// streams found, and the raw bytes of the overlay streams (to leave out of the raw sweep). +fn sweep_objects(needles: &[Needle], doc: &Document, depth: usize, tag: &str, acc: &mut Acc) -> (BTreeSet, Vec>) { + let mut embedded: BTreeSet = BTreeSet::new(); + let mut overlay_raw: Vec> = Vec::new(); + let mut in_streams = Tally::default(); + let mut in_names = Tally::default(); + let lookup = lookup_streams(doc); + for num in doc.object_numbers() { + let loc = format!("{tag}object {num}"); + let obj = match doc.try_get(num) { + Ok(o) => o, + Err(_) => { + acc.problem(Surface::ObjectStrings, format!("{loc} can't be read")); + continue; + } + }; + let mut hits: BTreeSet = BTreeSet::new(); + let mut spent = false; + let ctx = &acc.ctx; + // The file identifier is random bytes, never text: a short needle must not "read" it. + let without_id = match &*obj { + Object::Stream(s) if s.dict.name(b"Type") == Some(b"XRef") => { + let mut d = s.dict.clone(); + d.remove(b"ID"); + Some(Object::Dict(d)) + } + _ => None, + }; + let complete = walk_strings(without_id.as_ref().unwrap_or(&obj), 0, &mut |o| match o { + Object::String(s) => match string_hits(needles, s, ctx) { + Some(h) => hits.extend(h), + None => spent = true, + }, + Object::Name(n) => match bytes_hits(needles, n, MIN_NEEDLE, ctx) { + // A name the input had its own matches in is structure, not a leftover copy: + // the matches are tallied and settled against the input's count. + Some(h) => h.into_iter().for_each(|ni| in_names.add(ni, &loc, 1)), + None => spent = true, + }, + _ => {} + }); + if !complete { + acc.problem(Surface::ObjectStrings, format!("{loc} is nested too deeply to sweep in full")); + } + if spent { + acc.problem(Surface::ObjectStrings, "the search work limit was reached".into()); + } + acc.scanned(Surface::ObjectStrings); + for n in hits { + acc.survivor(Surface::ObjectStrings, loc.clone(), Some(n)); + } + // File specifications, wherever they sit (a name tree keeps them as separate objects, + // annotations and collections may inline them). + walk_strings(&obj, 0, &mut |o| { + if let Object::Dict(d) = o + && let Some(Object::Dict(ef)) = d.get(b"EF") + { + embedded.extend(ef.iter().filter_map(|(_, v)| v.as_ref()).map(|r| r.num)); + } + }); + if let Object::Stream(s) = &*obj { + if s.dict.name(b"Type") == Some(b"EmbeddedFile") { + embedded.insert(num); + } + acc.scanned(Surface::DecodedStreams); + if binary_stream(&s.dict) && image_codec(&s.dict) { + // The pixels of an image codec are still encoded and are not text: their stored + // bytes are searched for whole strings only. + acc.raw_only(Surface::DecodedStreams); + for n in acc.bytes(Surface::DecodedStreams, needles, &s.raw, MIN_STRUCTURED_NEEDLE) { + acc.survivor(Surface::DecodedStreams, format!("{loc} (raw)"), Some(n)); + } + } else if binary_stream(&s.dict) { + // A compressed copy of the removed text can hide in a font program or colour + // profile (with /Length1 in its dict, its stored bytes hold no plaintext): the + // stored bytes and the decoded data are both searched. Such a stream that won't + // decode (damaged, or past the cap) is searched as stored bytes only and counted + // as raw only: it holds no page content, and redaction never touches it. A + // cross-reference or object stream that won't decode is a problem of the proof. + for n in acc.bytes(Surface::DecodedStreams, needles, &s.raw, MIN_STRUCTURED_NEEDLE) { + acc.survivor(Surface::DecodedStreams, format!("{loc} (raw)"), Some(n)); + } + match acc.ctx.decode(s) { + Ok(d) => { + let loc = format!("{loc} (decoded)"); + for (n, c) in acc.counts(Surface::DecodedStreams, needles, &d, MIN_STRUCTURED_NEEDLE) { + in_streams.add(n, &loc, c); + } + } + Err(()) if acc.ctx.decode_spent() || matches!(s.dict.name(b"Type"), Some(b"XRef" | b"ObjStm")) => { + acc.problem(Surface::DecodedStreams, format!("{loc} can't be decoded in full")); + } + Err(()) => acc.raw_only(Surface::DecodedStreams), + } + } else { + match acc.ctx.decode(s) { + // The boxes and labels this run drew are not text to look for. + Ok(d) if acc.ctx.is_overlay(&d) => overlay_raw.push(s.raw.to_vec()), + Ok(d) => { + let loc = format!("{loc} (decoded)"); + for (n, c) in acc.counts(Surface::DecodedStreams, needles, &d, MIN_STRUCTURED_NEEDLE) { + in_streams.add(n, &loc, c); + } + } + // Colour-space lookup data and function samples are data tables, not content. + Err(()) if !acc.ctx.decode_spent() && (lookup.contains(&num) || s.dict.contains(b"FunctionType")) => { + acc.raw_only(Surface::DecodedStreams); + for n in acc.bytes(Surface::DecodedStreams, needles, &s.raw, MIN_STRUCTURED_NEEDLE) { + acc.survivor(Surface::DecodedStreams, format!("{loc} (raw)"), Some(n)); + } + } + Err(()) => acc.problem(Surface::DecodedStreams, format!("{loc} can't be decoded in full")), + } + } + } + } + in_names.settle_names(Surface::ObjectStrings, needles, depth, acc); + in_streams.settle(Surface::DecodedStreams, needles, depth, acc); + (embedded, overlay_raw) +} + +/// The streams that hold the lookup table of an `/Indexed` colour space (object numbers). +fn lookup_streams(doc: &Document) -> BTreeSet { + let mut out = BTreeSet::new(); + for num in doc.object_numbers() { + let Ok(obj) = doc.try_get(num) else { continue }; + walk_strings(&obj, 0, &mut |o| { + if let Object::Array(a) = o + && matches!(a.first(), Some(Object::Name(n)) if n == b"Indexed" || n == b"I") + && let Some(Object::Ref(r)) = a.get(3) + { + out.insert(r.num); + } + }); + } + out +} + +/// The raw bytes of the whole file, the overlay streams (stored as they are, or not at all) and +/// the file identifier (random bytes, regenerated on every write) aside. +fn sweep_file(needles: &[Needle], doc: &Document, bytes: &[u8], overlay_raw: Vec>, depth: usize, tag: &str, acc: &mut Acc) { + acc.scanned(Surface::RawBytes); + let mut mask = overlay_raw; + mask.extend(id_spellings(doc)); + let masked = mask_all(bytes, &mask, &acc.ctx); + let mut in_file = Tally::default(); + for (n, c) in acc.counts(Surface::RawBytes, needles, masked.as_deref().unwrap_or(bytes), MIN_STRUCTURED_NEEDLE) { + in_file.add(n, &format!("{tag}file"), c); + } + in_file.settle(Surface::RawBytes, needles, depth, acc); + if masked.is_none() && !mask.is_empty() { + acc.problem(Surface::RawBytes, "the search work limit was reached".into()); + } +} + +/// Embedded files, PDF attachments swept in full. +fn sweep_attachments(needles: &[Needle], doc: &Document, embedded: &BTreeSet, depth: usize, tag: &str, acc: &mut Acc) { + for &num in embedded { + let loc = format!("{tag}attachment object {num}"); + acc.scanned(Surface::EmbeddedFiles); + let obj = doc.get(ObjRef::new(num, doc.generation(num))); + let Object::Stream(s) = &*obj else { + acc.problem(Surface::EmbeddedFiles, format!("{loc} is not a stream")); + continue; + }; + let payload = match acc.ctx.decode(s) { + Ok(p) => p, + Err(()) => { + acc.problem(Surface::EmbeddedFiles, format!("{loc} can't be decoded in full")); + continue; + } + }; + sweep_payload(needles, &payload, depth, &loc, acc); + } +} + +/// `bytes` with every occurrence of `pats` zeroed (`None` when the work limit ran out, or +/// there is nothing to mask). +pub(super) fn mask_all(bytes: &[u8], pats: &[Vec], ctx: &Ctx) -> Option> { + if pats.iter().all(Vec::is_empty) { + return None; + } + let mut out = bytes.to_vec(); + for p in pats.iter().filter(|p| !p.is_empty()) { + if !ctx.charge(bytes.len()) { + return None; + } + let mut at = 0; + while let Some(w) = bytes.get(at..at + p.len()) { + if w == p.as_slice() { + if let Some(z) = out.get_mut(at..at + p.len()) { + z.fill(0); + } + at += p.len(); + } else { + at += 1; + } + } + } + Some(out) +} + +/// One attachment: a PDF is swept as a document, text is searched, containers are reported as +/// unswept (their content can't be seen), images are searched as raw bytes only. +pub(super) fn sweep_payload(needles: &[Needle], payload: &[u8], depth: usize, loc: &str, acc: &mut Acc) { + let head = payload.get(..1024).unwrap_or(payload); + let tag = format!("{loc} > "); + if find(head, b"%PDF-") { + if depth >= MAX_NESTING { + acc.problem(Surface::EmbeddedFiles, format!("{loc} is nested too deep to sweep")); + return; + } + match Document::open(Arc::new(payload.to_vec())) { + Ok(inner) => { + let before = acc.totals(); + sweep_doc(needles, &inner, payload, depth + 1, &tag, acc); + // What the attachment's own sweep found counts against the attachment too. + let after = acc.totals(); + if after.0 > before.0 { + acc.survivor(Surface::EmbeddedFiles, loc.to_string(), None); + } + if after.1 > before.1 { + acc.problem(Surface::EmbeddedFiles, format!("{loc} could not be swept in full")); + } + } + Err(_) => acc.problem(Surface::EmbeddedFiles, format!("{loc} is a PDF that can't be opened")), + } + return; + } + let starts = |sig: &[u8]| payload.starts_with(sig); + let container = starts(b"PK\x03\x04") + || starts(b"PK\x05\x06") + || starts(&[0x1F, 0x8B]) + || starts(b"BZh") + || starts(&[0xFD, b'7', b'z', b'X', b'Z', 0]) + || starts(&[b'7', b'z', 0xBC, 0xAF, 0x27, 0x1C]) + || starts(b"Rar!") + || starts(&[0xD0, 0xCF, 0x11, 0xE0]) + || payload.get(257..262) == Some(b"ustar"); + if container { + acc.problem(Surface::EmbeddedFiles, format!("{loc} is an archive or container that can't be opened")); + return; + } + let image = starts(&[0x89, b'P', b'N', b'G']) || starts(&[0xFF, 0xD8, 0xFF]) || starts(b"GIF8"); + if image { + acc.raw_only(Surface::EmbeddedFiles); + } + let min = if image { MIN_STRUCTURED_NEEDLE } else { MIN_NEEDLE }; + for n in acc.bytes(Surface::EmbeddedFiles, needles, payload, min) { + acc.survivor(Surface::EmbeddedFiles, loc.to_string(), Some(n)); + } + if !image { + let key = squash(&decode_text(payload)); + match key_counts_from(needles, &key, false, &acc.ctx, MIN_NEEDLE) { + Some(counts) => { + for (i, count) in counts { + if count > 0 { + acc.survivor(Surface::EmbeddedFiles, loc.to_string(), Some(i)); + } + } + } + None => acc.problem(Surface::EmbeddedFiles, format!("{loc} could not be searched in full")), + } + } +} + +pub(super) fn sweep_pages(needles: &[Needle], doc: &Document, depth: usize, tag: &str, acc: &mut Acc) { + let mut shown = Tally::default(); + 'pages: for (i, p) in pdfcraft_model::pages(doc).iter().enumerate() { + acc.scanned(Surface::ExtractedText); + let loc = format!("{tag}page {}", i + 1); + let Ok(e) = extract_page(doc, &p.dict, &acc.ctx, &[]) else { + acc.problem(Surface::ExtractedText, format!("{loc} can't be read")); + continue; + }; + for (hay, raw) in [(&e.text, false), (&e.raw, true)] { + let Some(counts) = key_counts(needles, hay, raw, &acc.ctx) else { + acc.problem(Surface::ExtractedText, "the search work limit was reached".into()); + break 'pages; + }; + for (ni, count) in counts { + shown.add(ni, &loc, count); + } + } + } + // A surplus over what legitimately remains anywhere in the document is a survivor. The + // counts made before the work ran out are settled too: a survivor already seen is + // reported where it sits, not hidden behind the limit. + shown.settle(Surface::ExtractedText, needles, depth, acc); +} + +pub(super) fn sweep_info(needles: &[Needle], doc: &Document, tag: &str, acc: &mut Acc) { + let Some(info) = doc.trailer().get(b"Info").map(|i| doc.resolve(i)) else { return }; + let Some(d) = info.as_dict() else { + acc.problem(Surface::Info, format!("{tag}Info is not a dictionary")); + return; + }; + for (k, v) in d.iter() { + acc.scanned(Surface::Info); + let v = doc.resolve(v); + let mut hits: BTreeSet = BTreeSet::new(); + let mut spent = false; + let ctx = &acc.ctx; + let complete = walk_strings(&v, 0, &mut |o| { + if let Object::String(s) = o { + match string_hits(needles, s, ctx) { + Some(h) => hits.extend(h), + None => spent = true, + } + } + }); + if !complete || spent { + acc.problem(Surface::Info, format!("{tag}Info /{} can't be swept in full", String::from_utf8_lossy(k))); + } + for n in hits { + acc.survivor(Surface::Info, format!("{tag}Info /{}", String::from_utf8_lossy(k)), Some(n)); + } + } +} + +pub(super) fn sweep_xmp(needles: &[Needle], doc: &Document, tag: &str, acc: &mut Acc) { + let Some(root) = doc.root() else { return }; + let Some(meta) = doc.get(root).as_dict().and_then(|c| c.get(b"Metadata")).map(|m| doc.resolve(m)) else { return }; + acc.scanned(Surface::Xmp); + let loc = format!("{tag}XMP metadata"); + let Object::Stream(s) = &*meta else { + acc.problem(Surface::Xmp, format!("{loc} is not a stream")); + return; + }; + let Ok(data) = acc.ctx.decode(s) else { + acc.problem(Surface::Xmp, format!("{loc} can't be decoded in full")); + return; + }; + let mut hits: BTreeSet = acc.bytes(Surface::Xmp, needles, &data, MIN_NEEDLE).into_iter().collect(); + let key = squash(&xml_unescape(&decode_text(&data))); + for (i, n) in needles.iter().enumerate() { + if n.at_least(MIN_NEEDLE) && !n.raw_view && key.contains(&n.key) { + hits.insert(i); + } + } + for n in hits { + acc.survivor(Surface::Xmp, loc.clone(), Some(n)); + } +} diff --git a/crates/redact/src/verify/util.rs b/crates/redact/src/verify/util.rs new file mode 100644 index 0000000000..77c4309cdd --- /dev/null +++ b/crates/redact/src/verify/util.rs @@ -0,0 +1,301 @@ +//! Shared helpers of the proof: what one proof may spend, text and byte search, and the +//! spellings a string can have in a PDF. + +use std::cell::Cell; +use std::collections::HashSet; +use std::fmt::Write as _; +use std::sync::Arc; + +use pdfcraft_cos::{DecodeBudget, Dict, Document, Object, Stream}; +use sha2::{Digest, Sha256}; + +use crate::verify::limits::{MAX_STREAM, MAX_TOTAL_DECODED, MAX_WORK, MIN_NEEDLE}; + +/// What one proof may spend, and what it knows about the document it checks. +pub(super) struct Ctx { + /// Digests of the decoded content of the overlay streams (their boxes and labels are drawn + /// on purpose and are neither text to look for nor paint to flag). + pub(super) overlay: HashSet, + /// Decoded bytes of the proof so far: one allowance for the whole proof, failures charged + /// too (see [`Ctx::decode`]), so a stream referenced many times, or one that fails after + /// decoding part of itself, cannot decode again for free. + pub(super) decoded: Cell, + pub(super) work: Cell, +} + +impl Ctx { + pub(super) fn new(overlay: HashSet) -> Self { + Ctx { overlay, decoded: Cell::new(0), work: Cell::new(0) } + } + + /// Decode `s` strictly: a stream that is damaged, longer than [`MAX_STREAM`], or one more + /// than [`MAX_TOTAL_DECODED`] allows is an error (never silently truncated data). Every + /// attempt is charged to the cumulative allowance, failures included: a decode that fails + /// past the one-stream cap pays the cap, and one that fails mid-way pays what it decoded + /// before the error — the work of finding out is not work a repeated reference gets for free. + pub(super) fn decode(&self, s: &Stream) -> Result, ()> { + let left = MAX_TOTAL_DECODED.saturating_sub(self.decoded.get()); + let cap = usize::try_from(left).unwrap_or(usize::MAX).min(MAX_STREAM); + let mut budget = DecodeBudget::new(cap, usize::MAX); + let result = budget.decode_strict(s); + // The attempt's own accounting covers a success (what it decoded), a stream cut short + // by corruption (what it decoded first) and one refused past the cap (the cap): charge + // it through, whichever way the attempt went. + let spent = cap.saturating_sub(budget.remaining()); + self.decoded.set(self.decoded.get().saturating_add(spent as u64)); + result.map_err(|_| ()) + } + + /// Is the cumulative decode allowance used up? A decode refused then says nothing about the + /// stream, so it must not pass for a stream that can't be decoded. + pub(super) fn decode_spent(&self) -> bool { + self.decoded.get() >= MAX_TOTAL_DECODED + } + + /// Spend `n` units of search work; false once [`MAX_WORK`] is used up. + pub(super) fn charge(&self, n: usize) -> bool { + let used = self.work.get().saturating_add(n as u64); + self.work.set(used); + used <= MAX_WORK + } + + /// Is this decoded content one of the overlay streams this run created? + pub(super) fn is_overlay(&self, data: &[u8]) -> bool { + !self.overlay.is_empty() && self.overlay.contains(&sha256(&[data])) + } + + /// (tests) A context whose cumulative decode allowance has `left` bytes left. + #[cfg(test)] + pub(super) fn with_decode_left(left: u64) -> Self { + Ctx { overlay: HashSet::new(), decoded: Cell::new(MAX_TOTAL_DECODED.saturating_sub(left)), work: Cell::new(0) } + } +} + +/// Follow `o` to its object; a reference that can't be loaded is an error (a missing object is +/// `Null` for [`Document::resolve`], which would read as "empty"). +pub(super) fn resolve_strict(doc: &Document, o: &Object) -> Result, ()> { + match o { + Object::Ref(r) => doc.try_get(r.num).map_err(|_| ()), + other => Ok(Arc::new(other.clone())), + } +} + +pub(super) fn hex(bytes: &[u8]) -> String { + bytes.iter().fold(String::with_capacity(bytes.len() * 2), |mut s, b| { + let _ = write!(s, "{b:02x}"); + s + }) +} + +pub(super) fn sha256>(parts: &[T]) -> String { + let mut h = Sha256::new(); + for p in parts { + let p = p.as_ref(); + h.update((p.len() as u64).to_be_bytes()); + h.update(p); + } + hex(h.finalize().as_slice()) +} + +/// The text of `s` without whitespace (page text is compared whitespace-blind: a gap in a `TJ` +/// is a space in one extraction and nothing in another). +pub(super) fn squash(s: &str) -> String { + s.chars().filter(|c| !c.is_whitespace() && *c != '\u{feff}').collect() +} + +pub(super) fn latin1(b: &[u8]) -> String { + b.iter().map(|&c| char::from(c)).collect() +} + +/// The words of a run worth looking for on their own (a run that survives in pieces). +pub(super) fn words(s: &str) -> Vec<&str> { + s.split_whitespace().filter(|w| w.chars().count() >= MIN_NEEDLE).collect() +} + +pub(super) fn count(hay: &str, needle: &str) -> usize { + if needle.is_empty() { 0 } else { hay.matches(needle).count() } +} + +/// Is `pat` in `hay`? +pub(super) fn find(hay: &[u8], pat: &[u8]) -> bool { + find_from(hay, pat, 0).is_some() +} + +/// Where `pat` first occurs in `hay` at or after `from`. The pattern's prefix function is +/// built once and the haystack is walked once: the search is worst-case linear, so +/// adversarial input (a needle of one letter and a haystack full of it) can neither make it +/// quadratic nor spend more comparisons than the caller charges for. +pub(super) fn find_from(hay: &[u8], pat: &[u8], from: usize) -> Option { + let m = pat.len(); + if m == 0 { + return None; + } + // fail[i]: the longest proper prefix of `pat` that ends at `i` and is also a suffix there. + let mut fail = vec![0usize; m]; + let mut k = 0; + for (i, &c) in pat.iter().enumerate().skip(1) { + while k > 0 && pat[k] != c { + k = fail[k - 1]; + } + if pat[k] == c { + k += 1; + } + fail[i] = k; + } + let mut k = 0; + for (at, &c) in hay.iter().enumerate().skip(from) { + while k > 0 && pat[k] != c { + k = fail[k - 1]; + } + if pat[k] == c { + k += 1; + if k == m { + return Some(at + 1 - m); + } + } + } + None +} + +pub(super) fn escape_literal(b: &[u8], octal: bool) -> Vec { + let mut out = Vec::with_capacity(b.len() * 2); + for &c in b { + if octal { + out.extend(format!("\\{c:03o}").bytes()); + continue; + } + match c { + b'(' | b')' | b'\\' => out.extend([b'\\', c]), + b'\n' => out.extend(*b"\\n"), + b'\r' => out.extend(*b"\\r"), + b'\t' => out.extend(*b"\\t"), + _ => out.push(c), + } + } + out +} + +/// Every way `text` can be spelled in a PDF: UTF-8, UTF-16BE (with and without a byte order +/// mark) and UTF-16LE, each as raw bytes, as a hex string (either case) and as a backslash- +/// escaped literal string (minimal escapes, and octal for every byte). +pub(crate) fn survivor_encodings(text: &str) -> Vec> { + let utf16 = |be: bool, bom: bool| -> Vec { + let mut v = Vec::new(); + if bom { + v.extend(if be { [0xFE, 0xFF] } else { [0xFF, 0xFE] }); + } + for u in text.encode_utf16() { + v.extend(if be { u.to_be_bytes() } else { u.to_le_bytes() }); + } + v + }; + let forms = [text.as_bytes().to_vec(), utf16(true, true), utf16(true, false), utf16(false, true), utf16(false, false)]; + spellings(forms) +} + +/// The spellings of a byte string (see [`survivor_encodings`]). +pub(super) fn spellings(forms: impl IntoIterator>) -> Vec> { + let mut out: Vec> = Vec::new(); + for f in forms { + if f.is_empty() { + continue; + } + let h = hex(&f); + out.push(h.to_uppercase().into_bytes()); + out.push(h.into_bytes()); + out.push(escape_literal(&f, false)); + out.push(escape_literal(&f, true)); + out.push(f); + } + let mut seen = HashSet::new(); + out.retain(|b| seen.insert(b.clone())); + out +} + +/// Text of a payload: UTF-16 when it has a byte order mark, else UTF-8 (lossy). +pub(super) fn decode_text(b: &[u8]) -> String { + let units = |be: bool, rest: &[u8]| -> String { + let u: Vec = rest.as_chunks::<2>().0.iter().map(|c| if be { u16::from_be_bytes(*c) } else { u16::from_le_bytes(*c) }).collect(); + String::from_utf16_lossy(&u) + }; + match b { + [0xFE, 0xFF, rest @ ..] => units(true, rest), + [0xFF, 0xFE, rest @ ..] => units(false, rest), + _ => String::from_utf8_lossy(b).into_owned(), + } +} + +pub(super) fn xml_unescape(s: &str) -> String { + let mut out = String::with_capacity(s.len()); + let mut rest = s; + while let Some(at) = rest.find('&') { + out.push_str(rest.get(..at).unwrap_or_default()); + rest = rest.get(at..).unwrap_or_default(); + match xml_entity(rest) { + Some((c, len)) => { + out.push(c); + rest = rest.get(len..).unwrap_or_default(); + } + None => { + out.push('&'); + rest = rest.get(1..).unwrap_or_default(); + } + } + } + out.push_str(rest); + out +} + +/// The entity starting at `&`: the character it reads as, and how far it runs (through the +/// closing `;`). A numeric reference may carry any number of leading zeros, so its digits +/// are padded without a bound; more significant digits than a character value can hold, or +/// a missing `;`, is not an entity and the `&` stays as it is. Walking only the digits the +/// reference itself holds keeps the decode linear in the input. +fn xml_entity(s: &str) -> Option<(char, usize)> { + for (name, c) in [("&", '&'), ("<", '<'), (">", '>'), (""", '"'), ("'", '\'')] { + if s.starts_with(name) { + return Some((c, name.len())); + } + } + let body = s.strip_prefix("&#")?; + let (radix, digits) = match body.as_bytes().first() { + Some(b'x' | b'X') => (16u32, &body[1..]), + _ => (10, body), + }; + let mut value: u32 = 0; + let mut digits_held = false; + let mut end = None; + for (i, c) in digits.char_indices() { + match c.to_digit(radix) { + Some(d) => { + digits_held = true; + // Leading zeros only pad; past a character's worth of significant digits the + // multiplication overflows and this was never an entity. + value = value.checked_mul(radix)?.checked_add(d)?; + } + None => { + if c == ';' && digits_held { + end = Some(i); + } + break; + } + } + } + let end = end?; + let ch = char::from_u32(value)?; + Some((ch, 2 + usize::from(radix == 16) + end + 1)) +} + +/// Names of the filters of a stream dictionary. +pub(super) fn filter_names(d: &Dict) -> Vec> { + match d.get(b"Filter") { + Some(Object::Name(n)) => vec![n.clone()], + Some(Object::Array(a)) => a.iter().filter_map(|o| o.as_name().map(<[u8]>::to_vec)).collect(), + _ => Vec::new(), + } +} + +/// Image codecs: their data is pixels, still encoded. +pub(super) fn image_codec(d: &Dict) -> bool { + filter_names(d).iter().any(|n| matches!(n.as_slice(), b"DCTDecode" | b"DCT" | b"JPXDecode" | b"CCITTFaxDecode" | b"CCF" | b"JBIG2Decode")) +} From 97cef90e016fc81d6683996e88c6eed13c1b8be7 Mon Sep 17 00:00:00 2001 From: Hamza Naveed <45464935+Hamza0101@users.noreply.github.com> Date: Fri, 6 Nov 2026 14:01:42 +0500 Subject: [PATCH 138/198] Redaction: a pattern nothing paints with any more leaves the page (#3680) * Redaction: a pattern nothing paints with any more leaves the page A tiling pattern's cell is a content stream of its own and can draw anything, text included. Redacting a shape filled with such a pattern removed the fill from the page's content -- the page looked right, a black box -- but left the pattern in /Resources /Pattern, and its cell, with the text that was under the mark, was written into the saved file unchanged. The pass does not enter pattern cells, so it cannot redact inside one. What it can do is notice that nothing paints with the pattern any more. The graphics state now carries the fill and stroke pattern a `scn` or `SCN` selected (and `cs`, `g`, `rg`, `k` and their stroke forms clear it), every painting operator that survives the rewrite records the pattern it drew with, and a pattern the page selected but no surviving paint used leaves the page's resources. Measured on the report's file: before applied: 1, (SECRET) in the saved bytes after applied: 1, no /Pattern on the page, (SECRET) gone A pattern some surviving paint still uses is kept, and so is its cell: with two boxes filled from one pattern and only the first marked, patterns_removed is 0 and the resources keep the entry -- the second box still shows the word, so it is not a leak. Report gains patterns_removed. Fixes #3647. * Keep a pattern every operator that can paint with it still uses Two defects CodeRabbit found in the first version of this change, both of which could remove a pattern the page still draws with. Pattern use was recorded for path painting only. Text shows with the fill pattern too -- and with the stroke pattern in the stroking render modes -- as do image masks and anything a form draws, and a form's content is interpreted with a graphics state of its own, so a pattern it inherits is never seen inside it. Content like /Pattern cs /P0 scn 20 140 100 30 re f BT (KEEP) Tj ET with the box redacted therefore lost the fill of text still on the page. Every operator that can paint with the current pattern now records it: text (except mode 3, which paints nothing, and mode 7, which only clips), inline images, and image or form XObjects. Only path painting knows exactly what it drew and whether the paint survived, so the others are recorded as they are met. That can keep a pattern whose paint was in fact all redacted, which costs an unused resource entry, while dropping one something still draws with takes paint off the page. /Resources /Pattern may be an indirect object, like any value. Reading the reference itself as a dictionary found none and pruned nothing, so on such a file the cell -- with the text that was under the mark -- stayed in the saved bytes, which is the leak this change is for. It is resolved now, and what is written back is a dictionary of this page's own, so pruning never reaches through a /Pattern dictionary another page shares. The controls in the test: without the first fix the surviving text's pattern is removed (left: 1, right: 0), and without the second the indirect file prunes nothing (left: 0, right: 1). * M8: refuse retained unsafe tiling patterns * M8: guard shared appearances and narrow pattern scans * M8: update TLS dependencies for new advisories * Redaction: the comment on prune_patterns said more than the code does `prune_patterns` drops the `/Pattern` resource entry; the `scn` that selected it stays in the content, so the page names a pattern its resources no longer hold. The comment claimed the selector went too (noted in review). ISO 32000-1 8.6.6.2 makes that operand a key of the `/Pattern` subdictionary, so the open question is whether a reader stops reading the stream at the stale name. `a_page_whose_pruned_pattern_is_still_selected_renders` answers it with the renderer rather than with prose: the redacted page is saved, reopened and rasterized, and * it renders with no error and the same 300x300 page box, * the text written *after* the stale selector is still painted -- which is exactly what a reader would drop if it gave up at the bad operand, * the redaction overlay is black over the box the pattern filled. Control: with the drawing after the selector removed, the pixel assertion fails ("the text after the stale selector is still painted"), so it is reading that text and not a stray pixel. The selector is left in place rather than cut out: removing operators is a rewrite of the content this pass does not otherwise do, and nothing surviving paints with that colour. `pdfcraft-render` is a dev dependency of this crate only -- layer 4 on layer 3, which the layering allows. * WIP: pattern guard depth bound sized from the interpreter's * Pattern tests: keep the guard's cases reaching the guard now a hit path is removed whole * Pattern automation tests: keep the pattern in use where they test the guard, since a hit path is now removed whole * Redaction: a form without resources keeps the page pattern it still paints A form with no /Resources of its own paints with its caller's patterns, but its surviving pattern paint was recorded only in the form's own output and never reached the page. With the page's own fill under the mark removed whole, prune_patterns then dropped /P0 from the page while the form outside the mark still painted with it, so that fill lost its paint in the saved file. Pass the painted names up from a form that inherits its caller's resources. Only the painted side goes up, so this can keep a pattern, never prune one. * Redaction: a pruned pattern does not stay named by the page tree prune_patterns writes only the page's own /Resources, and the guard walked only pages' effective resources. A /Pages node that still named the pruned pattern kept its cell reachable from Root, so a full save wrote the cell, text included, while apply returned Ok. Three shapes did it: a page with no /Parent inheriting from the root, a /Parent that disagrees with the /Kids tree, and a root /Resources shadowed by a nearer holder that was retired. Patterns pruned from a page are now unlinked from page-tree holders no page inherits from, as unlink_doomed does for XObjects, when every /Parent agrees with the tree. Whenever a pattern was pruned (or a selected page had any), the guard also inspects the /Resources of every /Pages node Root reaches down /Kids and up each page's /Parent chain, so whatever stays reachable is proven safe or the redaction is refused, leaving the document unchanged. * Redaction: a Type 3 glyph without resources keeps the page pattern it paints A Type 3 font with no /Resources looks the names in its glyph descriptions up in the resources of the page or form showing it, so a glyph outside the mark can paint with the page's pattern. Text recorded only the fill and stroke pattern of the graphics state, so with the page's own fill under the mark removed whole, prune_patterns dropped the pattern and the glyph lost its paint. Showing such a font now keeps every pattern of the scope, as a form without /Resources keeps what it paints. * Redaction: say what text-recorded pattern use keeps Text, inline images and XObjects record their pattern before the hit test, so a pattern painted only by redacted text keeps its cell. The guard refuses a cell with text or images; a vector-only cell stays in the file. The comment said this cost only an unused resource entry. Pin the behaviour in a test. * Redaction: the page tree is followed up every /Parent link A full save writes every node a /Parent link reaches, but the guard followed /Parent only from pages and stopped at the first node it had already seen, and unlinking never followed it. A /Pages node reached only through the /Parent of a /Kids node -- a root /Pages with a /Parent, or an intermediate node whose /Parent leads out of the tree -- kept naming the pruned pattern, and its cell, text included, stayed in the saved file while apply returned Ok. The guard now follows /Parent from every node it visits, page or /Pages, with the same seen-set against cycles. Unlinking treats a root with a /Parent as an inconsistent tree, like a disagreeing link, and leaves the decision to the guard, which inspects every node those links reach and refuses an unsafe cell. * Redaction: a long page tree does not exhaust the pattern guard's budget The page-tree walk counted every node, pages included, against the guard's 16,384-object budget, so redacting a pattern page of a 17,000-page document was refused with a limit error where it had applied before. Only the holder nodes the guard inspects count against that budget now; the walk itself is bounded at a million nodes, as unlink_doomed bounds the same tree. * Redaction: a Type 3 font set by an ExtGState or shown in a form keeps the page patterns The resourceless Type 3 check covered only a font set by Tf in the scope that paints. A Type 3 font without /Resources set through an ExtGState /Font entry was not flagged, and a glyph shown inside a form with /Resources of its own recorded its patterns in the form's scope, which the page never saw. In both cases the page's own fill under the mark was removed whole, prune_patterns dropped the pattern, and the glyph outside the mark lost its paint. An ExtGState font is now checked the same way, and a glyph shown in any nested form marks the page as keeping every pattern it selected. This can only keep patterns; whatever is kept is checked by the guard. * Redaction: a pruned pattern a full save would still write is proven safe or refused The walks that check retained pattern cells follow the references they know -- resource categories, the page tree, appearances -- but a full save writes whatever the trailer reaches by any reference. A direct /Parent dictionary, a /Kids array on a page, or a stray resource entry (/Shading naming the pattern stream) kept a pruned cell, text included, in the saved file while apply returned Ok. pdfcraft_cos gains reachable_objects, which runs the same breadth-first walk write_full uses (factored out as full_save_order, so the two cannot drift). Before apply succeeds, every pruned pattern that walk still reaches is inspected like a retained cell: vector-only passes, anything else refuses with the existing pattern error. It runs only when a pattern was pruned and costs one walk of the reachable objects. * Redaction: a direct /Parent dictionary is part of the page tree too A /Parent given as a dictionary rather than a reference is written inline by a full save, but the page-tree walk followed /Parent only as a reference, and unlinking took such a tree for consistent. A root /Pages whose direct /Parent held /Resources naming the pruned pattern kept its cell in the saved file. The guard now walks a direct /Parent like any node (its /Resources, /Parent and /Kids), and unlinking counts any /Parent that is not the reference /Kids came from as an inconsistent tree, leaving the decision to the guard. * Redaction: page-tree walks queue each node once and borrow /Kids On a hostile tree whose /Pages nodes share one long /Kids array, the guard's page-tree walk and pattern unlinking pushed every /Kids entry of every node before checking whether it had been seen, and copied the array on each visit. Both now queue only nodes not seen yet and read /Kids in place; unlinking also stops at the first node reached twice, since its /Parent can agree with at most one of the nodes above it, and leaves that tree to the guard. * Redaction: a Type 3 font with /Resources null is resourceless too /Resources null is the same as no /Resources, but the check counted any entry, so such a font's glyph outside the mark lost the page pattern it painted with. A Type 3 glyph shown by a kept annotation appearance is a known limit, pinned as an ignored test: appearances are not interpreted for pattern use, and under ISO 32000 the glyph resolves in the appearance's own resources. * Redaction: a pruned pattern is recorded along its whole reference chain A pattern name can lead to a reference-valued object whose value is the cell (/P0 7 0 R, object 7 being 6 0 R). Pruning recorded only object 7, so the reachability backstop checked an object nothing reached while a stray reference kept the cell, object 6, in the saved file. Every object along the chain is recorded now, bounded like the other walks. * Pattern tests: pin the file identifier where two saves are compared main now gives a redacted rewrite a new identifier from fresh entropy, so an incremental and a full save after Apply differ in /ID alone. The test that checks they are the same save passes fixed entropy to both. * Redaction: replies and pop-ups removed on other pages feed the shared-appearance check Apply checks every page for retained unsafe pattern cells when a removed annotation's appearance is shared with one that stays on an unselected page. Only annotations removed under a mark fed that check. Replies and pop-ups that go with them in step 2b (removed_everywhere), often listed on another page, did not, so a text-bearing cell in an appearance a kept annotation shares with them escaped the all-pages check and Apply returned Ok. Their appearance references now go into removed_appearances too, with the same limit. Raised by FredApps' review of the rebase. --------- Co-authored-by: skylar taylor-barrick --- Cargo.lock | 1 + ROADMAP.md | 2 + crates/automation/tests/pattern_redaction.rs | 146 ++++ crates/cos/src/lib.rs | 2 +- crates/cos/src/writer.rs | 40 +- crates/redact/Cargo.toml | 3 + crates/redact/README.md | 9 + crates/redact/src/interp.rs | 127 ++- crates/redact/src/interp/prune.rs | 136 ++++ crates/redact/src/lib.rs | 137 +++- crates/redact/src/pattern_guard.rs | 496 ++++++++++++ crates/redact/src/tests.rs | 194 +++++ crates/redact/tests/pattern_page_tree.rs | 221 +++++ .../redact/tests/pattern_page_tree_shapes.rs | 275 +++++++ crates/redact/tests/pattern_reachability.rs | 304 +++++++ crates/redact/tests/pattern_redaction.rs | 758 ++++++++++++++++++ docs/roadmap.md | 2 +- 17 files changed, 2837 insertions(+), 16 deletions(-) create mode 100644 crates/automation/tests/pattern_redaction.rs create mode 100644 crates/redact/src/pattern_guard.rs create mode 100644 crates/redact/tests/pattern_page_tree.rs create mode 100644 crates/redact/tests/pattern_page_tree_shapes.rs create mode 100644 crates/redact/tests/pattern_reachability.rs create mode 100644 crates/redact/tests/pattern_redaction.rs diff --git a/Cargo.lock b/Cargo.lock index e1f271ed12..8617c45522 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4628,6 +4628,7 @@ dependencies = [ "pdfcraft-fonts", "pdfcraft-forms", "pdfcraft-model", + "pdfcraft-render", "pdfcraft-testkit", "pdfcraft-xfa", "sha2 0.11.1", diff --git a/ROADMAP.md b/ROADMAP.md index 4fcdadaee9..820ceb0179 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -357,6 +357,8 @@ Ranked; detail and the full milestone table in [docs/roadmap.md](docs/roadmap.md - **2026-10-20 (M5, arrow keys move comments):** With a comment selected, the arrow keys move it a screen pixel and ⌘/Ctrl or Shift and an arrow ten, instead of scrolling the view or turning the page; up is up on screen at any zoom and rotation, and each press is one undo step. Text markup and locked comments stay put. Estimates unchanged. +- **2026-10-20 (M8, #3647):** Apply prunes fully removed tiling patterns, then checks retained pattern resources for text, images and unreadable content before reporting success. Shared patterns, Form-owned and inherited patterns, and cells kept by another page cause an atomic refusal when safe removal cannot be shown. Synthetic saved-file and automation regressions cover refusal, preserved marks and undo, and vector-only controls. Estimates unchanged. + - **2026-10-20 (M1, PR #1143 review):** Repair-aware dictionary/preflight edits and snapshot restoration preserve both accepted generations of a damaged live slot. Such edits use the existing full-save fallback; classic and packed rewrites coalesce aliases so page-tree parent links retain one object identity. Explicit generation replacement remains single-generation. Materialized xref diagnostics inspect retained primary and hybrid rows before folding, retaining the 256-object cap plus summary and discarding abandoned-index notes. Synthetic red/green controls cover preflight, undo restoration, equal values, explicit narrowing, encryption packing and four save variants. Estimates unchanged. - **2026-10-19 (M14, #3373):** The main menu and its long submenus scroll in short windows, keeping their commands reachable. Estimates unchanged. diff --git a/crates/automation/tests/pattern_redaction.rs b/crates/automation/tests/pattern_redaction.rs new file mode 100644 index 0000000000..667779cef6 --- /dev/null +++ b/crates/automation/tests/pattern_redaction.rs @@ -0,0 +1,146 @@ +//! Rectangle redaction must not report a pattern-cell secret safely removed (#3647). + +use std::sync::Arc; + +use pdfcraft_automation::{Automation, Content}; +use pdfcraft_cos::{Dict, Document, Object, Stream}; +use pdfcraft_engine::DocId; +use serde_json::{Value, json}; + +fn tool(automation: &mut Automation, name: &str, args: Value) -> Value { + match automation.call(name, &args).unwrap().remove(0) { + Content::Json(value) => value, + other => panic!("expected JSON: {other:?}"), + } +} + +/// `retained`: paint with the pattern once more, away from the mark (x 290..340 lies right of it). +/// A path the mark hits is removed whole (F-008) and a pattern nothing surviving paints with is +/// pruned, so only a pattern that some surviving paint still uses reaches the guard. +fn fixture(cell: &[u8], partial: bool, compressed: bool, retained: bool) -> Vec { + let mut pattern = Dict::from_iter([ + (b"Type".to_vec(), Object::name("Pattern")), + (b"PatternType".to_vec(), Object::Int(1)), + (b"PaintType".to_vec(), Object::Int(1)), + (b"TilingType".to_vec(), Object::Int(1)), + (b"BBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 500.into(), 300.into()])), + (b"XStep".to_vec(), Object::Int(500)), + (b"YStep".to_vec(), Object::Int(300)), + ]); + pattern.set( + b"Resources".to_vec(), + Object::Dict(Dict::from_iter([( + b"Font".to_vec(), + Object::Dict(Dict::from_iter([(b"F1".to_vec(), Object::Ref(pdfcraft_cos::ObjRef::new(4, 0)))])), + )])), + ); + let width = if partial { 400 } else { 170 }; + let survivor = if retained { " 290 115 50 30 re f" } else { "" }; + let content = format!("BT /F1 20 Tf 40 250 Td (PUBLIC) Tj ET /Pattern cs /P0 scn 35 110 {width} 35 re f{survivor}"); + let mut page_stream = Vec::new(); + pdfcraft_cos::serialize(&Object::Stream(Stream::from_raw(Dict::new(), content.into_bytes())), &mut page_stream); + let mut pattern_stream = Vec::new(); + let stream = if compressed { Stream::flate(pattern, cell) } else { Stream::from_raw(pattern, cell.to_vec()) }; + pdfcraft_cos::serialize(&Object::Stream(stream), &mut pattern_stream); + let objects = [ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources << /Font << /F1 4 0 R >> /Pattern << /P0 6 0 R >> >> >>" + .to_vec(), + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>".to_vec(), + page_stream, + pattern_stream, + ]; + let mut bytes = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (index, object) in objects.iter().enumerate() { + offsets.push(bytes.len()); + bytes.extend(format!("{} 0 obj\n", index + 1).as_bytes()); + bytes.extend(object); + bytes.extend(b"\nendobj\n"); + } + let xref = bytes.len(); + bytes.extend(format!("xref\n0 {}\n0000000000 65535 f \n", objects.len() + 1).as_bytes()); + for offset in offsets { + bytes.extend(format!("{offset:010} 00000 n \n").as_bytes()); + } + bytes.extend(format!("trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n", objects.len() + 1).as_bytes()); + bytes +} + +fn saved_has_secret(bytes: Vec) -> bool { + let document = Document::open(Arc::new(bytes)).unwrap(); + document.object_numbers().into_iter().any(|num| { + let object = document.try_get(num).unwrap(); + matches!(&*object, Object::Stream(stream) if stream.decoded().unwrap().windows(10).any(|part| part == b"CELLSECRET")) + }) +} + +#[test] +fn pattern_redaction_tools_remove_or_refuse_without_leaking_or_changing_undo() { + let dir = tempfile::tempdir().unwrap(); + for (partial, retained) in [(false, false), (true, false), (true, true)] { + for compressed in [false, true] { + let cell = b"BT /F1 20 Tf 40 120 Td (CELLSECRET) Tj ET BT /F1 20 Tf 300 120 Td (SAFE) Tj ET"; + let bytes = fixture(cell, partial, compressed, retained); + assert!(saved_has_secret(bytes.clone())); + std::fs::write(dir.path().join("input.pdf"), bytes).unwrap(); + let mut automation = Automation::new().with_root(dir.path()).unwrap().with_clock(|| 1_700_000_000); + let id = tool(&mut automation, "doc_open", json!({"path": "input.pdf"}))["doc"].as_u64().unwrap(); + tool(&mut automation, "redact_mark", json!({"doc": id, "page": 1, "rect": [30, 150, 215, 195]})); + let before = automation.session().get(DocId(id)).unwrap().bytes.clone(); + let pixels = automation.render_pixels(id, 1, 1.0).unwrap(); + match automation.call("redact_apply", &json!({"doc": id})) { + Ok(_) => { + assert!(!retained, "a text-bearing cell still in use must not be reported safe (partial: {partial})"); + tool(&mut automation, "doc_save", json!({"doc": id, "path": "redacted.pdf"})); + assert!( + !saved_has_secret(std::fs::read(dir.path().join("redacted.pdf")).unwrap()), + "successful apply/save left the pattern-cell text recoverable" + ); + } + Err(error) => { + assert!(retained, "a pattern nothing surviving paints with should apply (partial: {partial}): {error}"); + assert!(error.to_string().contains("tiling pattern"), "{error}"); + assert!(!error.to_string().contains("CELLSECRET")); + let document = automation.session().get(DocId(id)).unwrap(); + assert_eq!(document.bytes, before); + assert_eq!(document.redaction_marks(), 1); + assert_eq!(document.can_undo(), Some("Add redaction mark")); + assert_eq!(automation.render_pixels(id, 1, 1.0).unwrap(), pixels); + tool(&mut automation, "edit_undo", json!({"doc": id})); + assert_eq!(automation.session().get(DocId(id)).unwrap().redaction_marks(), 0); + } + } + } + } +} + +#[test] +fn pattern_redaction_vector_control_saves_reopens_and_preserves_outside_pixels() { + let dir = tempfile::tempdir().unwrap(); + // The guard's control: the cell is plain vectors and a paint outside the mark still uses it, + // so it is inspected and kept, not refused, and what that paint draws is unchanged. + let bytes = fixture(b"0.2 0.4 0.6 rg 40 120 15 15 re f 300 120 30 20 re f", true, true, true); + std::fs::write(dir.path().join("vectors.pdf"), bytes).unwrap(); + let mut automation = Automation::new().with_root(dir.path()).unwrap().with_clock(|| 1_700_000_000); + let id = tool(&mut automation, "doc_open", json!({"path": "vectors.pdf"}))["doc"].as_u64().unwrap(); + let (width, height, original) = automation.render_pixels(id, 1, 1.0).unwrap(); + tool(&mut automation, "redact_mark", json!({"doc": id, "page": 1, "rect": [30, 150, 215, 195]})); + let applied = tool(&mut automation, "redact_apply", json!({"doc": id})); + assert_eq!(applied["applied"], 1); + let save = tool(&mut automation, "doc_save", json!({"doc": id, "path": "vectors-redacted.pdf"})); + assert_eq!(save["incremental"], false); + let reopened = tool(&mut automation, "doc_open", json!({"path": "vectors-redacted.pdf"}))["doc"].as_u64().unwrap(); + let (new_width, new_height, actual) = automation.render_pixels(reopened, 1, 1.0).unwrap(); + assert_eq!((new_width, new_height), (width, height)); + for (index, (before, after)) in original.as_chunks::<4>().0.iter().zip(actual.as_chunks::<4>().0).enumerate() { + let (x, y) = (index % width as usize, index / width as usize); + if x >= 280 { + assert_eq!(before, after, "unmarked pattern pixels changed at {x},{y}"); + } + if (35..205).contains(&x) && (155..190).contains(&y) { + assert_eq!(after, &[0, 0, 0, 255], "marked interior must be black"); + } + } +} diff --git a/crates/cos/src/lib.rs b/crates/cos/src/lib.rs index 8e051e4578..32a6f6d8b3 100644 --- a/crates/cos/src/lib.rs +++ b/crates/cos/src/lib.rs @@ -30,7 +30,7 @@ pub use parser::{Lexer, is_whitespace, parse_indirect}; pub use pdfcraft_crypt::{ Algorithm, Auth, Method as CryptMethod, NewEncryption, Permissions, SecurityHandler, effective_password, pdfdoc_byte, pdfdoc_char, }; -pub use writer::{SaveOptions, pdf_date, serialize, write_full, write_incremental}; +pub use writer::{SaveOptions, pdf_date, reachable_objects, serialize, write_full, write_incremental}; #[derive(Debug, thiserror::Error, Clone, PartialEq)] pub enum CosError { diff --git a/crates/cos/src/writer.rs b/crates/cos/src/writer.rs index e0762612dc..0af74f7bab 100644 --- a/crates/cos/src/writer.rs +++ b/crates/cos/src/writer.rs @@ -273,16 +273,12 @@ pub fn write_incremental(doc: &Document, opts: &SaveOptions) -> Result, Ok(file) } -/// Write only reachable objects, renumbered from 1, with a table or cross-reference stream. -pub fn write_full(doc: &Document, opts: &SaveOptions) -> Result, CosError> { - if doc.stream_limited() { - return Err(CosError::ReadOnlyLimit); - } - // A full traversal must not fill the editor's shared caches (which undo snapshots also - // own). Parsed objects and decoded object streams are temporary working data here. - let mut reader = doc.object_reader(); - stamp_mod_date(&mut reader.document, opts); - let trailer_in = reader.document.trailer().clone(); +/// The objects [`write_full`] writes: everything reachable from the trailer's /Root, /Info and +/// /Encrypt and from its unknown entries, breadth-first, with the new number each gets +/// (aliases of one slot included) and, in visit order, whether it is a stream. +type SaveOrder = (HashMap, Vec<(ObjRef, bool)>); + +fn full_save_order(reader: &mut ObjectReader, trailer_in: &Dict) -> Result { // Breadth-first walk from the trailer, assigning new numbers in visit order. let mut map: HashMap = HashMap::new(); // Record the type while traversing so classifying packed objects needs no second parse. @@ -326,6 +322,30 @@ pub fn write_full(doc: &Document, opts: &SaveOptions) -> Result, CosErro order.push((canonical, matches!(*o, Object::Stream(_)))); visit_refs(&o, &mut queue); } + Ok((map, order)) +} + +/// The object numbers a full save of `doc` would write: exactly the objects [`write_full`] +/// reaches from the trailer, object streams included. For a check that something removed from a +/// document cannot reach the saved file through a reference nobody looked for. +pub fn reachable_objects(doc: &Document) -> Result, CosError> { + let mut reader = doc.object_reader(); + let trailer_in = reader.document.trailer().clone(); + let (_, order) = full_save_order(&mut reader, &trailer_in)?; + Ok(order.into_iter().map(|(r, _)| r.num).collect()) +} + +/// Write only reachable objects, renumbered from 1, with a table or cross-reference stream. +pub fn write_full(doc: &Document, opts: &SaveOptions) -> Result, CosError> { + if doc.stream_limited() { + return Err(CosError::ReadOnlyLimit); + } + // A full traversal must not fill the editor's shared caches (which undo snapshots also + // own). Parsed objects and decoded object streams are temporary working data here. + let mut reader = doc.object_reader(); + stamp_mod_date(&mut reader.document, opts); + let trailer_in = reader.document.trailer().clone(); + let (map, order) = full_save_order(&mut reader, &trailer_in)?; let version = reader.document.version().chars().take(3).collect::(); let version = if opts.object_streams && version.as_str() < "1.5" { "1.5".to_string() } else { version }; let mut out = Vec::new(); diff --git a/crates/redact/Cargo.toml b/crates/redact/Cargo.toml index c02335e3c4..149d8d8295 100644 --- a/crates/redact/Cargo.toml +++ b/crates/redact/Cargo.toml @@ -25,3 +25,6 @@ sha2 = "0.11" [dev-dependencies] pdfcraft-testkit = { workspace = true } +# The redacted page is rendered in `tests::a_page_whose_pruned_pattern_is_still_selected_renders`: +# layer 3 from layer 4, a dev dependency only. +pdfcraft-render = { workspace = true } diff --git a/crates/redact/README.md b/crates/redact/README.md index db9f46b8c9..f78f819c03 100644 --- a/crates/redact/README.md +++ b/crates/redact/README.md @@ -60,6 +60,15 @@ resume after it would be a guess. A Redact annotation with neither a usable `/Qu `/Rect` makes it fail with `RedactError::UnreadableMark` before anything changes, instead of skipping that mark. The count covers the marks on the pages being applied. +Tiling-pattern cells are separate streams. Applying a mark removes a pattern resource when no +surviving paint uses it. If a cell remains reachable after rewriting the selected pages and may +contain text, images, replacement descriptions, or unreadable content, Apply refuses with a +`RedactError::Pattern` and restores the document and pending marks. This includes patterns owned +by Forms and patterns inherited by Forms. Vector-only retained cells remain supported. The +bounded check is conservative: an unrelated retained pattern on a selected page can also cause +refusal when its content cannot be proven safe. If a selected page had a pattern resource, the +check covers all pages after rewriting so another page cannot keep the same cell in the saved PDF. + Before changing any page, `apply` prepares the redaction appearances with a shared 8 MiB budget for their uncompressed PDF content across the selected pages. It bounds repeated-text counts and working line buffers before allocating them, and measures and encodes each repeated diff --git a/crates/redact/src/interp.rs b/crates/redact/src/interp.rs index 7a2cd26eca..b0a7257cb3 100644 --- a/crates/redact/src/interp.rs +++ b/crates/redact/src/interp.rs @@ -31,7 +31,7 @@ mod prune; use layers::strip_hidden_layers; use prune::{clip_out, extgstate_font, prune_form_resources}; -pub(crate) use prune::{prune, prune_properties, resource_ops, unlink_doomed}; +pub(crate) use prune::{prune, prune_patterns, prune_properties, resource_ops, unlink_doomed, unlink_pruned_patterns}; /// How deep this pass follows form XObjects. A form past it is removed rather than walked, which /// is the fail-safe choice for a redaction -- marked content inside a form we cannot read would @@ -85,6 +85,17 @@ struct Gs { scale: f64, leading: f64, rise: f64, + /// The tiling or shading pattern selected as the fill colour (`/Name scn`), and as the stroke + /// colour (`/Name SCN`). Part of the graphics state, so `q`/`Q` save and restore them with + /// everything else. A pattern's cell is a content stream of its own and this pass does not + /// enter it, so a pattern whose only painting was removed has to leave the page's resources + /// or its drawing stays in the file (#3647). + fill_pattern: Option>, + stroke_pattern: Option>, + /// The current font is a Type 3 font without `/Resources`: its glyph descriptions look their + /// names up in this scope's resources (ISO 32000-2 §9.6.4), so a shown glyph may paint with + /// any of this scope's patterns. + type3_inherits: bool, } /// What processing a scope produced. @@ -102,6 +113,16 @@ pub(crate) struct Output { /// dictionaries may name them too (an inherited or shared /Resources, or the caller's when a /// form has none of its own): [`unlink_doomed`] removes those entries. pub doomed: Vec, + /// Pattern resource names a painting operator that survived the rewrite still uses. A name + /// this scope saw selected but never painted with afterwards is pruned from the page's + /// `/Resources /Pattern`, so a cell whose only drawing was redacted does not stay in the + /// saved file (#3647). + pub patterns_seen: Vec>, + pub patterns_painted: Vec>, + /// A Type 3 glyph without resources of its own was shown in this scope or a nested form, + /// through a font set by `Tf` or an ExtGState: which of the page's patterns it paints with is + /// not tracked, so the page keeps them all. + pub keeps_all_patterns: bool, /// Original property values retired by this scope or a nested form. Other resource holders /// may still name them; unlink only entries no surviving content uses. pub retired_properties: Vec, @@ -264,6 +285,22 @@ struct Glyph { rects: Vec, } +/// Record the patterns an operator paints with, so [`prune_patterns`] keeps them. Only path +/// painting knows exactly what it draws and whether the paint survived; text, images and forms +/// are recorded as they are met, which can keep a pattern whose paint was in fact all redacted, +/// and with it the pattern's cell. A cell that carries text or images is then refused by +/// `pattern_guard`; a vector-only cell stays in the file though nothing shows it +/// (`a_pattern_painted_only_by_redacted_text_keeps_its_vector_cell`). Dropping a pattern something +/// still draws with would lose paint from the page instead. +fn uses_patterns(out: &mut Output, gs: &Gs, fills: bool, strokes: bool) { + if fills && let Some(n) = &gs.fill_pattern { + out.patterns_painted.push(n.clone()); + } + if strokes && let Some(n) = &gs.stroke_pattern { + out.patterns_painted.push(n.clone()); + } +} + /// Process a scope's content streams (state flows from one stream to the next). pub(crate) fn process(doc: &mut Document, scope: &mut Scope<'_>, streams: &[Vec], resources: &Dict, ctm: Matrix) -> Output { let gs = Gs { @@ -277,6 +314,9 @@ pub(crate) fn process(doc: &mut Document, scope: &mut Scope<'_>, streams: &[Vec< scale: 1.0, leading: 0.0, rise: 0.0, + fill_pattern: None, + stroke_pattern: None, + type3_inherits: false, }; process_from(doc, scope, streams, resources, gs) } @@ -377,6 +417,8 @@ struct Walk { fonts_res: Dict, ext_gstates: Dict, xobjects: Dict, + /// This scope's pattern names, which a Type 3 glyph without resources may paint with. + patterns: Vec>, fallback: Rc, /// The current path: its operators, its bounding box in user space, and whether it clips. path: Vec, @@ -395,6 +437,7 @@ impl Walk { fonts_res: res_dict(doc, resources, b"Font"), ext_gstates: res_dict(doc, resources, b"ExtGState"), xobjects: res_dict(doc, resources, b"XObject"), + patterns: res_dict(doc, resources, b"Pattern").iter().map(|(n, _)| n.to_vec()).collect(), fallback: Rc::new(Metrics::fallback()), path: Vec::new(), path_box: None, @@ -433,6 +476,39 @@ impl Walk { b"BI" => Self::inline_image(&self.gs, scope, &mut self.out, op, piece), b"sh" if scope.mode == Mode::Apply => Self::shading(&self.gs, scope, op, piece), b"Do" => self.draw_xobject(doc, scope, resources, op, piece), + // `/Name scn` selects a pattern as the fill colour; numbers select a plain colour + // and clear it. `cs` with a space other than /Pattern clears it too. + b"scn" | b"SCN" => { + let fill = o == b"scn"; + let picked = op.operands.last().and_then(Object::as_name).map(<[u8]>::to_vec); + if let Some(name) = &picked { + self.out.patterns_seen.push(name.clone()); + } + if fill { + self.gs.fill_pattern = picked; + } else { + self.gs.stroke_pattern = picked; + } + Some(op) + } + b"cs" | b"CS" => { + if op.name(0) != Some(b"Pattern") { + if o == b"cs" { + self.gs.fill_pattern = None; + } else { + self.gs.stroke_pattern = None; + } + } + Some(op) + } + b"g" | b"rg" | b"k" => { + self.gs.fill_pattern = None; + Some(op) + } + b"G" | b"RG" | b"K" => { + self.gs.stroke_pattern = None; + Some(op) + } _ => Some(op), }; piece.ops.extend(kept); @@ -464,6 +540,12 @@ impl Walk { let clips = std::mem::replace(&mut self.clip, false); let hit = scope.mode == Mode::Apply && op.op.as_slice() != b"n" && bbox.is_some_and(|b| scope.hits(b)); if !hit { + // The paint survives, so the pattern it draws with is still in use. `n` paints + // nothing, so it uses none. A hit path is removed whole (F-008), pattern and all. + let o = op.op.as_slice(); + let strokes = matches!(o, b"S" | b"s" | b"B" | b"B*" | b"b" | b"b*"); + let fills = matches!(o, b"f" | b"F" | b"f*" | b"B" | b"B*" | b"b" | b"b*"); + uses_patterns(&mut self.out, &self.gs, fills, strokes); piece.ops.extend(body); piece.ops.push(op); return; @@ -497,6 +579,8 @@ impl Walk { _ => { if let Some((font, vertical, size)) = op.name(0).and_then(|n| extgstate_font(doc, scope, &self.ext_gstates, n, &self.fallback)) { (self.gs.font, self.gs.vertical, self.gs.size) = (font, vertical, size); + // As for `Tf`: a Type 3 font without /Resources paints through this scope's. + self.gs.type3_inherits = op.name(0).is_some_and(|n| extgstate_type3_inherits(doc, &self.ext_gstates, n)); } } } @@ -507,6 +591,7 @@ impl Walk { let Some(name) = op.name(0) else { return }; let font_dict = self.fonts_res.get(name).and_then(|f| doc.resolve(f).as_dict().cloned()); self.gs.vertical = font_dict.as_ref().is_some_and(|d| is_vertical(doc, d)); + self.gs.type3_inherits = font_dict.as_ref().is_some_and(|d| type3_without_resources(doc, d)); self.gs.font = match scope.fonts.get(name) { Some(f) => f.clone(), None => { @@ -555,6 +640,16 @@ impl Walk { /// `Tj`, `TJ`, `'` and `"`: glyphs under a region are cut out (or counted, when verifying). fn show_text(&mut self, scope: &mut Scope<'_>, op: Op, piece: &mut Piece) -> Option { + // Text paints with the fill pattern, and with the stroke pattern in the stroking render + // modes. Mode 3 paints nothing and mode 7 only clips. + let mode = self.gs.render_mode; + uses_patterns(&mut self.out, &self.gs, matches!(mode, 0 | 2 | 4 | 6), matches!(mode, 1 | 2 | 5 | 6)); + // A Type 3 glyph without resources of its own paints through this scope's: keep every + // pattern it could select, as with a form that has no /Resources. + if self.gs.type3_inherits { + self.out.patterns_painted.extend(self.patterns.iter().cloned()); + self.out.keeps_all_patterns = true; + } let o = op.op.as_slice(); if o == b"\"" { self.gs.word_spacing = op.num(0).unwrap_or(self.gs.word_spacing); @@ -608,6 +703,8 @@ impl Walk { /// An inline image under a region is removed (or counted, when verifying). fn inline_image(gs: &Gs, scope: &mut Scope<'_>, out: &mut Output, op: Op, piece: &mut Piece) -> Option { + // An image mask paints with the fill colour, pattern included. + uses_patterns(out, gs, true, false); if !scope.hits(gs.ctm.bbox([0.0, 0.0, 1.0, 1.0])) { return Some(op); } @@ -632,6 +729,9 @@ impl Walk { } fn draw_xobject(&mut self, doc: &mut Document, scope: &mut Scope<'_>, resources: &Dict, op: Op, piece: &mut Piece) -> Option { + // An image mask paints with the fill colour, and a form's content is interpreted with a + // graphics state of its own, so a pattern it inherits from here is never recorded inside it. + uses_patterns(&mut self.out, &self.gs, true, true); let Some(new) = xobject(doc, scope, &op, &self.gs, resources, &mut self.xobjects, &mut self.out) else { return Some(op) }; piece.changed = true; piece.ops.extend(new); @@ -924,8 +1024,18 @@ fn xobject( out.ambiguous |= inner.ambiguous; out.doomed.extend_from_slice(&inner.doomed); out.retired_properties.extend(inner.retired_properties); + // A Type 3 glyph without resources shown inside the form may paint with the page's + // patterns (the renderer resolves its names there), whether or not the form has + // /Resources of its own. + out.keeps_all_patterns |= inner.keeps_all_patterns; out.tag_changes.extend(inner.tag_changes); out.error = inner.error.or(out.error.take()); + // A form without /Resources of its own paints with its caller's patterns, so what it + // still paints keeps the caller's entry. Only the painted side is passed up: a name + // the form selected but no longer paints with is left for the caller to keep. + if own.is_none() { + out.patterns_painted.extend(inner.patterns_painted); + } if scope.mode == Mode::Verify { return None; } @@ -992,3 +1102,18 @@ fn xobject( _ => None, } } + +/// Whether the ExtGState `name` sets a Type 3 font that has no `/Resources` of its own. +fn extgstate_type3_inherits(doc: &Document, ext_gstates: &Dict, name: &[u8]) -> bool { + let Some(state) = ext_gstates.get(name).map(|s| doc.resolve(s)) else { return false }; + let Some(font) = state.as_dict().and_then(|d| d.get(b"Font")).map(|f| doc.resolve(f)) else { return false }; + let Some(reference) = font.as_array().and_then(|a| a.first()) else { return false }; + let dict = doc.resolve(reference); + dict.as_dict().is_some_and(|d| type3_without_resources(doc, d)) +} + +/// A Type 3 font whose glyph descriptions use the resources of whatever shows them: no +/// `/Resources`, or `/Resources null`, which is the same thing. +fn type3_without_resources(doc: &Document, font: &Dict) -> bool { + font.name(b"Subtype") == Some(b"Type3") && font.get(b"Resources").is_none_or(|r| matches!(&*doc.resolve(r), Object::Null)) +} diff --git a/crates/redact/src/interp/prune.rs b/crates/redact/src/interp/prune.rs index 6ca5ea2803..188af03abe 100644 --- a/crates/redact/src/interp/prune.rs +++ b/crates/redact/src/interp/prune.rs @@ -355,3 +355,139 @@ pub(super) fn extgstate_font( }; Some((metrics, vertical, size)) } + +/// Drop every `/Pattern` entry this scope saw selected but no surviving painting operator drew +/// with. The `scn` that selected it stays in the content, so the page names a pattern its +/// resources no longer hold; nothing surviving paints with that colour, and +/// `a_page_whose_pruned_pattern_is_still_selected_renders` pins that the page still renders and +/// that operators after the stale selector still draw. A pattern's cell is a content stream of its own which +/// this pass does not enter, so a pattern whose only painting was redacted would otherwise keep +/// its drawing -- text included -- in the saved file (#3647). A pattern some surviving paint +/// still uses, or one the scope never saw, is left alone. +/// `/Pattern` may be an indirect object like any other value, so it is resolved: reading the +/// reference itself as a dictionary found none and pruned nothing, leaving the cell in the file. +/// What is written back is a dictionary of this page's own, so pruning never reaches through a +/// `/Pattern` dictionary another page shares. The pattern objects dropped are added to `pruned`, +/// for [`unlink_pruned_patterns`]. +pub(crate) fn prune_patterns(doc: &Document, res: &mut Dict, seen: &[Vec], painted: &[Vec], pruned: &mut Vec) -> usize { + let Some(mut pat) = res.get(b"Pattern").map(|p| doc.resolve(p)).and_then(|p| p.as_dict().cloned()) else { return 0 }; + let mut dropped = 0; + for n in seen { + if !painted.contains(n) + && let Some(value) = pat.get(n) + { + // Every object along the reference chain: a name can lead to a reference-valued + // object whose value is the cell, and the cell is what must not stay reachable. + let mut next = value.as_ref(); + for _ in 0..MAX_DEPTH { + let Some(reference) = next else { break }; + pruned.push(reference); + next = match &*doc.get(reference) { + Object::Ref(r) => Some(*r), + _ => None, + }; + } + pat.remove(n); + dropped += 1; + } + } + if dropped > 0 { + if pat.is_empty() { + res.remove(b"Pattern"); + } else { + res.set(b"Pattern".to_vec(), Object::Dict(pat)); + } + } + dropped +} + +/// Unlink the patterns a page's rewrite pruned (`pruned`) from page-tree nodes whose +/// `/Resources` no page inherits any more: a full save keeps whatever Root still reaches, so a +/// `/Pages` node above the page that still names the cell would keep it in the file. That happens +/// when the page's nearest holder was retired but a holder further up, which it shadowed, still +/// names the pattern. A holder some page still inherits from is left alone; whatever stays +/// reachable is checked by `pattern_guard`, which refuses a retained cell it cannot prove safe. +/// When any node's `/Parent` disagrees with where the `/Kids` tree puts it (or the root has +/// one), which holder a page inherits from is ambiguous and a node may be reached only through +/// `/Parent`, so nothing is unlinked and the guard, which follows `/Parent` too, decides. +pub(crate) fn unlink_pruned_patterns(doc: &mut Document, pruned: &[ObjRef]) -> Result<(), RedactError> { + if pruned.is_empty() { + return Ok(()); + } + let pruned: HashSet = pruned.iter().copied().collect(); + let Some(top) = doc.root().and_then(|root| doc.get(root).as_dict().and_then(|d| d.reference(b"Pages"))) else { return Ok(()) }; + // Every node with a /Resources of its own, and whether any page inherits from it (it is the + // nearest holder above a page, or the page itself). Cycles are cut as in `unlink_doomed`. + let mut holders: Vec<(ObjRef, bool)> = Vec::new(); + let mut seen = HashSet::new(); + let mut consistent = true; + let mut stack: Vec<(ObjRef, Option, Option)> = vec![(top, None, None)]; + seen.insert(top); + while let Some((node, parent, mut holder)) = stack.pop() { + let obj = doc.get(node); + let Some(d) = obj.as_dict() else { continue }; + // Every node's /Parent must be the node /Kids reached it from, and the root has none. + // A node reached only through a /Parent link is then impossible here; when the links + // disagree, the guard inspects every node they reach instead. A /Parent that is not a + // reference (a direct dictionary, written inline by a full save) never agrees. + let agrees = match d.get(b"Parent") { + None => parent.is_none(), + Some(Object::Ref(r)) => Some(*r) == parent, + Some(_) => false, + }; + if !agrees { + consistent = false; + break; + } + if d.contains(b"Resources") { + holders.push((node, false)); + holder = Some(holders.len() - 1); + } + let is_pages = d.name(b"Type") == Some(b"Pages") || (d.contains(b"Kids") && d.name(b"Type") != Some(b"Page")); + if is_pages { + // Borrowed, not copied: a /Kids array can be shared by many hostile nodes. + let kids_object = d.get(b"Kids").map(|k| doc.resolve(k)); + for kid in kids_object.as_deref().and_then(Object::as_array).into_iter().flatten().filter_map(Object::as_ref) { + // A node reached twice (a shared /Kids entry or a cycle) has a /Parent that agrees + // with at most one of the nodes above it. + if !seen.insert(kid) || seen.len() > 1_000_000 { + consistent = false; + break; + } + stack.push((kid, Some(node), holder)); + } + if !consistent { + break; + } + } else if let Some(h) = holder.and_then(|i| holders.get_mut(i)) { + h.1 = true; + } + } + if !consistent { + return Ok(()); + } + for (node, inherited) in holders { + if inherited { + continue; + } + let Some(mut res) = doc.get(node).as_dict().and_then(|d| d.get(b"Resources")).and_then(|r| doc.resolve(r).as_dict().cloned()) else { + continue; + }; + let mut pat = res_dict(doc, &res, b"Pattern"); + let names: Vec> = pat.iter().filter(|(_, v)| v.as_ref().is_some_and(|r| pruned.contains(&r))).map(|(n, _)| n.to_vec()).collect(); + if names.is_empty() { + continue; + } + for n in &names { + pat.remove(n); + } + if pat.is_empty() { + res.remove(b"Pattern"); + } else { + res.set(b"Pattern".to_vec(), Object::Dict(pat)); + } + // A private copy on this node: the dictionary object may be shared with other nodes. + doc.update_dict(node, |d| d.set(b"Resources".to_vec(), Object::Dict(res)))?; + } + Ok(()) +} diff --git a/crates/redact/src/lib.rs b/crates/redact/src/lib.rs index 6bb8e5cc06..6d323004ad 100644 --- a/crates/redact/src/lib.rs +++ b/crates/redact/src/lib.rs @@ -22,6 +22,7 @@ pub mod codes; mod image; mod interp; mod overlay; +mod pattern_guard; pub mod patterns; pub mod sanitize; mod tags; @@ -34,6 +35,7 @@ pub(crate) mod type3_tables; pub mod verify; use interp::{Mode, Scope, process}; +pub use pattern_guard::PatternProblem; pub use verify::{Proof, ProofOptions, Snapshot}; pub type Rgb = [f64; 3]; @@ -53,6 +55,8 @@ pub enum RedactError { #[error("redaction could not be verified: {0} item(s) were still found under the marks, so nothing was changed")] Residue(usize), /// A mark's overlay text has a character Helvetica can't draw; nothing was redacted. + #[error("page {page} cannot be redacted safely: {reason}; nothing was changed")] + Pattern { page: usize, reason: PatternProblem }, #[error("{0}")] Undrawable(String), #[error( @@ -146,6 +150,8 @@ fn overlay_look(doc: &Document, d: &Dict) -> pdfcraft_annot::OverlayLook { /// What [`apply`] removed. #[derive(Clone, Debug, Default, PartialEq)] pub struct Report { + /// Patterns dropped from a page's resources because nothing painted with them any more. + pub patterns_removed: usize, pub marks: usize, pub pages: usize, pub glyphs: usize, @@ -397,6 +403,38 @@ pub fn apply(doc: &mut Document, pages: Option<&[usize]>) -> Result Option> { + const MAX_REFERENCES: usize = 16_384; + let mut references = HashSet::new(); + let Some(ap) = annotation.get(b"AP") else { return Some(references) }; + if let Some(reference) = ap.as_ref() { + references.insert(reference); + } + let resolved_ap = doc.resolve(ap); + let Some(appearances) = resolved_ap.as_dict() else { return Some(references) }; + for key in [b"N".as_slice(), b"R", b"D"] { + let Some(value) = appearances.get(key) else { continue }; + if let Some(reference) = value.as_ref() { + references.insert(reference); + } + let resolved = doc.resolve(value); + if let Object::Dict(states) = &*resolved { + for (_, state) in states.iter() { + if let Some(reference) = state.as_ref() { + references.insert(reference); + if references.len() > MAX_REFERENCES { + return None; + } + } + } + } + } + Some(references) +} + /// [`apply`] without the copy: an error can leave `doc` part redacted. fn apply_to(doc: &mut Document, pages: Option<&[usize]>) -> Result { let images = pdfcraft_cos::ImageBudget::new(doc); @@ -429,6 +467,16 @@ fn apply_within(doc: &mut Document, pages: Option<&[usize]>, images: pdfcraft_co // content rewriting and verification instead of scanning all marks for every page. let groups: Vec<_> = chosen.chunk_by(|a, b| a.page == b.page).filter_map(|group| group.first().map(|mark| (mark.page, group))).collect(); let by_page: Vec<_> = groups.iter().map(|(page, _)| *page).collect(); + // A pattern pruned from a selected page can remain reachable through another page's + // resources. When a selected page had patterns, inspect every page after the rewrite. + let source_pages = pdfcraft_model::pages(doc); + let shared_pattern_possible = by_page.iter().any(|&pi| { + source_pages + .get(pi) + .and_then(|page| page.dict.get(b"Resources")) + .map(|resources| doc.resolve(resources)) + .is_some_and(|resources| resources.as_dict().is_some_and(|dict| dict.get(b"Pattern").is_some())) + }); // Budget failures must leave even a direct library caller's document untouched. let mut remaining = overlay::MAX_BYTES; let overlays = groups @@ -448,6 +496,9 @@ fn apply_within(doc: &mut Document, pages: Option<&[usize]>, images: pdfcraft_co widget_owner, removed_everywhere: HashSet::new(), images, + removed_appearances: HashSet::new(), + appearance_scan_limit: false, + pruned_patterns: Vec::new(), }; for &(pi, page_marks) in &groups { let page = pdfcraft_model::pages(doc).swap_remove(pi); @@ -458,7 +509,18 @@ fn apply_within(doc: &mut Document, pages: Option<&[usize]>, images: pdfcraft_co if applying.images.refused() > 0 { return Err(RedactError::ImagesPastAllowance); } - let Applying { mut report, doomed_fields, doomed_xobjects, retired_properties, mut removed_everywhere, mut images, .. } = applying; + let Applying { + mut report, + doomed_fields, + doomed_xobjects, + retired_properties, + mut removed_everywhere, + mut images, + mut removed_appearances, + mut appearance_scan_limit, + pruned_patterns, + .. + } = applying; // 2b. Replies and pop-ups of a removed annotation can be listed on another page than their // parent: their `/IRT` or `/Parent` would keep the removed comment reachable. @@ -481,6 +543,19 @@ fn apply_within(doc: &mut Document, pages: Option<&[usize]>, images: pdfcraft_co d.get(b"IRT").and_then(Object::as_ref).is_some_and(|x| removed_everywhere.contains(&x)) && d.name(b"Subtype") != Some(b"Widget"); let is_popup = d.get(b"Parent").and_then(Object::as_ref).is_some_and(|x| removed_everywhere.contains(&x)); if removed_everywhere.contains(&r) || is_reply || is_popup { + // A reply or pop-up removed here can share its appearance with an annotation + // that stays on an unselected page, like one removed under a mark: the + // all-pages pattern check must see it. + if !appearance_scan_limit { + match appearance_refs(doc, d) { + Some(references) => removed_appearances.extend(references), + None => appearance_scan_limit = true, + } + if removed_appearances.len() > 16_384 { + appearance_scan_limit = true; + removed_appearances.clear(); + } + } if removed_everywhere.insert(r) { grew = true; if is_reply { @@ -541,6 +616,8 @@ fn apply_within(doc: &mut Document, pages: Option<&[usize]>, images: pdfcraft_co // A removed image or form must not stay reachable through another resource dictionary (an // inherited or shared /Resources): a full save keeps whatever is still referenced. interp::unlink_doomed(doc, &doomed_xobjects, &retired_properties)?; + // Likewise a pruned pattern: a /Pages node above the page can still name its cell. + interp::unlink_pruned_patterns(doc, &pruned_patterns)?; report.images_restored = restore_shared_jbig2(doc, &doomed_xobjects, &mut images)?; // 5. Verify: nothing readable may remain under a region. @@ -563,6 +640,29 @@ fn apply_within(doc: &mut Document, pages: Option<&[usize]>, images: pdfcraft_co if residue > 0 { return Err(RedactError::Residue(residue)); } + // The interpreter can remove a pattern's paint while another use keeps the cell reachable. + // Inspect the finished selected pages, including rewritten Forms, and all pages if a removed + // pattern or appearance may still be shared. Retained unsafe cells refuse the redaction; + // `apply` runs on a copy, so the refusal leaves the caller's document as it was. + let shared_appearance_possible = appearance_scan_limit + || !removed_appearances.is_empty() + && pdfcraft_model::pages(doc).iter().enumerate().any(|(index, page)| { + !by_page.contains(&index) + && annots_of(doc, &page.dict).iter().any(|annotation| { + doc.resolve(annotation).as_dict().is_some_and(|dict| { + appearance_refs(doc, dict).is_none_or(|refs| refs.iter().any(|reference| removed_appearances.contains(reference))) + }) + }) + }); + let first_page = by_page.first().map_or(1, |p| p + 1); + let check_pages = + if shared_pattern_possible || shared_appearance_possible { (0..pdfcraft_model::pages(doc).len()).collect::>() } else { by_page }; + // A pruned pattern may also stay named by a /Pages node no page inherits from, or by one a + // page with a broken /Parent link still reaches: then every page-tree node is inspected too. + pattern_guard::check(doc, &check_pages, shared_pattern_possible || !pruned_patterns.is_empty())?; + // Whatever reference keeps a pruned pattern in the file -- one no walk above follows -- the + // full save would write it: prove it safe or refuse. + pattern_guard::check_pruned_unreachable(doc, &pruned_patterns, first_page)?; // The previous revision still holds the removed content: the next save must rewrite, and the // rewritten file is a different file from its source, so it gets a new identifier. doc.require_full_save_with_new_id(); @@ -650,6 +750,13 @@ struct Applying { removed_everywhere: HashSet, /// What clearing images may spend, across the run's pages. images: pdfcraft_cos::ImageBudget, + /// Appearance streams of the annotations removed, on any page: one shared with a kept + /// annotation keeps a pruned pattern's cell reachable (see `pattern_guard`). + removed_appearances: HashSet, + /// Too many appearance references to track: the pattern guard inspects every page. + appearance_scan_limit: bool, + /// Pattern objects pruned from a page's resources, which page-tree nodes may still name. + pruned_patterns: Vec, } /// Step 1 for one page: rewrites its content so nothing readable stays under a mark, and cleans the tags it no longer shows. @@ -660,7 +767,7 @@ fn redact_page_content( page: &pdfcraft_model::Page, rects: &[[f64; 4]], ) -> Result<(), RedactError> { - let Applying { report, doomed_xobjects, retired_properties, images, .. } = applying; + let Applying { report, doomed_xobjects, retired_properties, images, pruned_patterns, .. } = applying; // 1. Content. let mut budget = pdfcraft_cos::DecodeBudget::page(); let (list, data) = page_streams_within(doc, &page.dict, pi, &mut budget)?; @@ -733,6 +840,11 @@ fn redact_page_content( }, pi, )?; + // A pattern this page selected but no surviving paint draws leaves its resources, so + // its cell -- a content stream this pass does not enter -- cannot stay in the file. + // A Type 3 glyph without resources paints with patterns nobody tracked: keep them all. + let painted = if out.keeps_all_patterns { &out.patterns_seen } else { &out.patterns_painted }; + report.patterns_removed += interp::prune_patterns(doc, &mut res, &out.patterns_seen, painted, pruned_patterns); retired_properties.extend(interp::prune_properties(doc, &mut res, &cleaned_data)?); type3::prune_type3_from_resources(doc, &mut res, &cleaned_data)?; doc.update_dict(page.obj, |d| { @@ -761,7 +873,7 @@ fn remove_page_annotations( rects: &[[f64; 4]], page_marks: &[&Mark], ) -> Result<(), RedactError> { - let Applying { report, doomed_fields, widget_owner, removed_everywhere, .. } = applying; + let Applying { report, doomed_fields, widget_owner, removed_everywhere, removed_appearances, appearance_scan_limit, .. } = applying; // 2. Annotations: the marks, and whatever lies under them (with their pop-ups). let annots = annots_of(doc, &doc.get(page.obj).as_dict().cloned().unwrap_or_default()); let mut removed: Vec = Vec::new(); @@ -819,6 +931,25 @@ fn remove_page_annotations( } } removed_everywhere.extend(removed.iter().copied()); + // A removed annotation's appearance streams can be shared with a kept one on another page, + // which would keep a pruned pattern's cell reachable: remember them for the pattern guard. + for reference in &removed { + if *appearance_scan_limit { + break; + } + if let Some(annotation) = doc.get(*reference).as_dict() { + match appearance_refs(doc, annotation) { + Some(references) => { + removed_appearances.extend(references); + if removed_appearances.len() > 16_384 { + *appearance_scan_limit = true; + removed_appearances.clear(); + } + } + None => *appearance_scan_limit = true, + } + } + } // Pop-ups dropped with their parent below count too: a reply on another page may answer one. for a in &annots { let Some(r) = a.as_ref() else { continue }; diff --git a/crates/redact/src/pattern_guard.rs b/crates/redact/src/pattern_guard.rs new file mode 100644 index 0000000000..96b865e3da --- /dev/null +++ b/crates/redact/src/pattern_guard.rs @@ -0,0 +1,496 @@ +//! Refuse pattern content that remains reachable after redaction (#3647). +//! +//! Pattern cells paint through a path, not `Do`, so clipping/removing the path does not remove +//! their bytes. Inspect the rewritten pages' resource graphs before committing the +//! working copy. This is deliberately conservative: an unused text/image-bearing pattern is +//! refused too. A plain vector pattern remains supported. The bounded read-only pass follows +//! the fail-closed approach proposed in PR #162. + +use std::collections::HashSet; + +use pdfcraft_cos::{Dict, Document, ObjRef, Object, Stream}; + +use crate::RedactError; + +/// How deep the walk may recurse. Every step down (a reference, an array, a dictionary, a stream, +/// a resources map) is one frame, and a nested form costs about four of them: the 50-deep chain +/// in `a_deeply_nested_form_keeps_the_text_nobody_marked` peaks at 200. The interpreter follows +/// forms as deep as the renderer and Edit text read (`interp::MAX_DEPTH`, #3645), so this must +/// reach at least that far, or the guard would refuse a document the redaction itself handles. +/// Total work is bounded separately by `MAX_OBJECTS`, and 512 frames stay far inside a thread's +/// stack. +const MAX_DEPTH: usize = 512; +const _: () = assert!(MAX_DEPTH >= 8 * crate::interp::MAX_DEPTH, "the guard must walk past the deepest form the interpreter follows"); +const MAX_OBJECTS: usize = 16_384; +const MAX_STREAM_BYTES: usize = 1 << 20; +const MAX_TOTAL_BYTES: usize = 8 << 20; +const MAX_FILTERS: usize = 16; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +pub enum PatternProblem { + #[error("a tiling pattern carries text or images that cannot be removed safely")] + Content, + #[error("pattern resources or content cannot be read completely")] + Unreadable, + #[error("pattern inspection exceeded its resource, depth or decoded-content budget")] + Limit, +} + +struct Guard { + left: usize, + bytes_left: usize, + seen: HashSet<(ObjRef, bool)>, + active: HashSet<(ObjRef, bool)>, +} + +impl Guard { + fn new() -> Self { + Self { left: MAX_OBJECTS, bytes_left: MAX_TOTAL_BYTES, seen: HashSet::new(), active: HashSet::new() } + } + + fn charge(&mut self, depth: usize) -> Result<(), PatternProblem> { + self.left = self.left.checked_sub(1).ok_or(PatternProblem::Limit)?; + if depth > MAX_DEPTH { + return Err(PatternProblem::Limit); + } + Ok(()) + } + + fn walk(&mut self, doc: &Document, object: &Object, inside: bool, depth: usize) -> Result<(), PatternProblem> { + self.charge(depth)?; + match object { + Object::Ref(reference) => { + let key = (*reference, inside); + if inside && self.active.contains(&key) { + return Err(PatternProblem::Unreadable); + } + if !self.seen.insert(key) { + return Ok(()); + } + let value = doc.try_get(reference.num).map_err(|_| PatternProblem::Unreadable)?; + self.active.insert(key); + let result = self.walk(doc, &value, inside, depth + 1); + self.active.remove(&key); + result + } + Object::Array(values) => { + for value in values { + self.walk(doc, value, inside, depth + 1)?; + } + Ok(()) + } + Object::Dict(dict) => self.dictionary(doc, dict, inside, false, depth + 1), + Object::Stream(stream) => { + if inside { + let subtype = resolved(doc, stream.dict.get(b"Subtype"))?; + let pattern_type = resolved(doc, stream.dict.get(b"PatternType"))?; + if subtype.as_name() == Some(b"Image") { + return Err(PatternProblem::Content); + } + if subtype.as_name() == Some(b"Form") { + if resolved(doc, stream.dict.get(b"Subtype2"))? != Object::Null || resolved(doc, stream.dict.get(b"Ref"))? != Object::Null { + return Err(PatternProblem::Unreadable); + } + self.content(doc, stream)?; + } else if pattern_type.as_int() == Some(1) { + self.content(doc, stream)?; + } + } + self.dictionary(doc, &stream.dict, inside, true, depth + 1) + } + _ => Ok(()), + } + } + + fn dictionary(&mut self, doc: &Document, dict: &Dict, inside: bool, stream: bool, depth: usize) -> Result<(), PatternProblem> { + // Stream dictionaries include appearance and soft-mask streams whose consumers + // may infer Form semantics without /Subtype. Resource-map names, in contrast, + // are arbitrary and must not be mistaken for dictionary field names. + let subtype = resolved(doc, dict.get(b"Subtype"))?; + let pattern_type = resolved(doc, dict.get(b"PatternType"))?; + let owns_resources = stream || matches!(subtype.as_name(), Some(b"Form" | b"Type3")) || pattern_type.as_int() == Some(1); + for (key, value) in dict.iter() { + if owns_resources && key.as_slice() == b"Resources" { + self.resources(doc, value, inside, depth + 1)?; + } else if inside || !owns_resources { + self.walk(doc, value, inside, depth + 1)?; + } + } + Ok(()) + } + + fn resources(&mut self, doc: &Document, object: &Object, inside: bool, depth: usize) -> Result<(), PatternProblem> { + self.charge(depth)?; + let object = resolved(doc, Some(object))?; + // A Form's null Resources is equivalent to an absent entry: it uses the already + // inspected caller resources. Tiling-pattern resources are required separately. + if object == Object::Null { + return Ok(()); + } + let Some(resources) = object.as_dict() else { return Err(PatternProblem::Unreadable) }; + for (category, value) in resources.iter() { + // Resource dictionaries may include large embedded fonts and colour tables. Only + // structures that can own another pattern cell need recursive inspection. + if !matches!(category.as_slice(), b"Pattern" | b"XObject" | b"Font" | b"Properties" | b"ExtGState") { + continue; + } + if category.as_slice() == b"Font" { + let fonts = resolved(doc, Some(value))?; + let Some(fonts) = fonts.as_dict() else { return Err(PatternProblem::Unreadable) }; + for (_, font) in fonts.iter() { + self.charge(depth)?; + let font = resolved(doc, Some(font))?; + let Some(dict) = font.as_dict() else { return Err(PatternProblem::Unreadable) }; + if resolved(doc, dict.get(b"Subtype"))?.as_name() == Some(b"Type3") + && let Some(resources) = dict.get(b"Resources") + { + self.resources(doc, resources, inside, depth + 1)?; + } + } + continue; + } + if matches!(category.as_slice(), b"Pattern" | b"XObject" | b"Properties" | b"ExtGState") && resolved(doc, Some(value))? == Object::Null { + continue; + } + if category.as_slice() == b"Pattern" { + let object = resolved(doc, Some(value))?; + let Some(patterns) = object.as_dict() else { return Err(PatternProblem::Unreadable) }; + for (_, pattern) in patterns.iter() { + self.charge(depth)?; + let object = resolved(doc, Some(pattern))?; + if object == Object::Null { + continue; + } + let Some(dict) = object.as_dict() else { return Err(PatternProblem::Unreadable) }; + let kind = resolved(doc, dict.get(b"PatternType"))?; + match (kind.as_int(), &object) { + (Some(1), Object::Stream(_)) => { + // Do not guess a fallback for a malformed cell, which different + // readers may resolve differently. Pattern resources are mandatory. + if resolved(doc, dict.get(b"Resources"))?.as_dict().is_none() { + return Err(PatternProblem::Unreadable); + } + } + (Some(2), Object::Dict(_)) => {} + _ => return Err(PatternProblem::Unreadable), + } + self.walk(doc, pattern, true, depth + 1)?; + } + } else { + if (inside && matches!(category.as_slice(), b"XObject" | b"Properties")) || category.as_slice() == b"ExtGState" { + let object = resolved(doc, Some(value))?; + let Some(entries) = object.as_dict() else { return Err(PatternProblem::Unreadable) }; + for (_, entry) in entries.iter() { + self.charge(depth)?; + let object = resolved(doc, Some(entry))?; + if object == Object::Null { + continue; + } + if category.as_slice() == b"Properties" { + let Some(properties) = object.as_dict() else { return Err(PatternProblem::Unreadable) }; + replacement_text(doc, properties)?; + } else if category.as_slice() == b"ExtGState" { + let Some(state) = object.as_dict() else { return Err(PatternProblem::Unreadable) }; + match resolved(doc, state.get(b"SMask"))? { + Object::Null => {} + Object::Name(name) if name == b"None" => {} + Object::Dict(mask) => { + let group = resolved(doc, mask.get(b"G"))?; + let Object::Stream(group) = group else { return Err(PatternProblem::Unreadable) }; + readable_form(doc, &group)?; + if let Some(value) = mask.get(b"G") { + self.walk(doc, value, inside, depth + 1)?; + } + } + _ => return Err(PatternProblem::Unreadable), + } + } else { + let Object::Stream(stream) = object else { return Err(PatternProblem::Unreadable) }; + let subtype = resolved(doc, stream.dict.get(b"Subtype"))?; + if subtype.as_name() == Some(b"Image") { + return Err(PatternProblem::Content); + } + readable_form(doc, &stream)?; + } + } + } + if category.as_slice() == b"XObject" { + let entries = resolved(doc, Some(value))?; + let Some(entries) = entries.as_dict() else { return Err(PatternProblem::Unreadable) }; + for (_, entry) in entries.iter() { + self.charge(depth)?; + let object = resolved(doc, Some(entry))?; + if let Object::Stream(stream) = object + && resolved(doc, stream.dict.get(b"Subtype"))?.as_name() == Some(b"Form") + { + self.walk(doc, entry, inside, depth + 1)?; + } + } + } + } + } + Ok(()) + } + + fn content(&mut self, doc: &Document, stream: &Stream) -> Result<(), PatternProblem> { + // Strict decoding with a small bound, unlike the tolerant general reader. Normalize + // indirect filter declarations before using Stream::filters, which accepts direct data. + let normalized = normalized_stream(doc, stream)?; + let chain = normalized.filters(); + let max = MAX_STREAM_BYTES.min(self.bytes_left); + if chain.is_empty() && normalized.raw.len() > max { + return Err(PatternProblem::Limit); + } + let data = pdfcraft_filters::decode(&chain, &normalized.raw, max).map_err(|error| match error { + pdfcraft_filters::FilterError::LimitExceeded(_) => PatternProblem::Limit, + _ => PatternProblem::Unreadable, + })?; + self.bytes_left = self.bytes_left.checked_sub(data.len()).ok_or(PatternProblem::Limit)?; + let parsed = pdfcraft_content::parse(&data); + if parsed.skipped != 0 { + return Err(PatternProblem::Unreadable); + } + for op in parsed.ops { + self.charge(0)?; + if matches!(op.op.as_slice(), b"BDC" | b"DP") { + for operand in &op.operands { + if let Some(properties) = resolved(doc, Some(operand))?.as_dict() { + replacement_text(doc, properties)?; + } + } + } + match op.op.as_slice() { + b"Tj" | b"TJ" | b"'" | b"\"" | b"BI" => return Err(PatternProblem::Content), + // Unknown operators cannot establish that no hidden content was skipped. + b"q" | b"Q" | b"cm" | b"w" | b"J" | b"j" | b"M" | b"d" | b"ri" | b"i" | b"gs" | b"m" | b"l" | b"c" | b"v" | b"y" | b"h" | b"re" + | b"S" | b"s" | b"f" | b"F" | b"f*" | b"B" | b"B*" | b"b" | b"b*" | b"n" | b"W" | b"W*" | b"BT" | b"ET" | b"Tc" | b"Tw" | b"Tz" + | b"TL" | b"Tf" | b"Tr" | b"Ts" | b"Td" | b"TD" | b"Tm" | b"T*" | b"d0" | b"d1" | b"CS" | b"cs" | b"SC" | b"SCN" | b"sc" | b"scn" + | b"G" | b"g" | b"RG" | b"rg" | b"K" | b"k" | b"sh" | b"Do" | b"MP" | b"DP" | b"BMC" | b"BDC" | b"EMC" | b"BX" | b"EX" => {} + _ => return Err(PatternProblem::Unreadable), + } + } + Ok(()) + } +} + +fn readable_form(doc: &Document, stream: &Stream) -> Result<(), PatternProblem> { + // PS, Form/Subtype2 PS, reference XObjects and unknown/missing subtypes cannot be + // proven free of recoverable pattern content by the PDF content parser. + if resolved(doc, stream.dict.get(b"Subtype"))?.as_name() != Some(b"Form") + || resolved(doc, stream.dict.get(b"Subtype2"))? != Object::Null + || resolved(doc, stream.dict.get(b"Ref"))? != Object::Null + { + return Err(PatternProblem::Unreadable); + } + Ok(()) +} + +fn replacement_text(doc: &Document, properties: &Dict) -> Result<(), PatternProblem> { + for key in [b"ActualText".as_slice(), b"Alt", b"E"] { + if resolved(doc, properties.get(key))? != Object::Null { + return Err(PatternProblem::Content); + } + } + Ok(()) +} + +fn resolved(doc: &Document, value: Option<&Object>) -> Result { + let mut value = value.cloned().unwrap_or(Object::Null); + for _ in 0..MAX_DEPTH { + let Object::Ref(reference) = value else { return Ok(value) }; + value = (*doc.try_get(reference.num).map_err(|_| PatternProblem::Unreadable)?).clone(); + } + Err(PatternProblem::Limit) +} + +fn normalized_stream(doc: &Document, stream: &Stream) -> Result { + if stream.dict.contains(b"F") || stream.dict.contains(b"FFilter") || stream.dict.contains(b"FDecodeParms") || stream.dict.contains(b"DP") { + return Err(PatternProblem::Unreadable); + } + let mut normalized = stream.clone(); + let filter = resolved(doc, stream.dict.get(b"Filter"))?; + let names = match &filter { + Object::Null => Vec::new(), + Object::Name(_) => vec![filter], + Object::Array(values) if values.len() <= MAX_FILTERS => values.iter().map(|v| resolved(doc, Some(v))).collect::, _>>()?, + _ => return Err(PatternProblem::Unreadable), + }; + if names.iter().any(|name| name.as_name().is_none()) { + return Err(PatternProblem::Unreadable); + } + let parms = resolved(doc, stream.dict.get(b"DecodeParms"))?; + let parms = match parms { + Object::Null => vec![Object::Null; names.len()], + Object::Dict(_) if names.len() == 1 => vec![parms], + Object::Array(values) if values.len() == names.len() => values, + _ => return Err(PatternProblem::Unreadable), + }; + let parms = parms + .iter() + .map(|value| { + let value = resolved(doc, Some(value))?; + let Object::Dict(mut dict) = value else { + return if value == Object::Null { Ok(value) } else { Err(PatternProblem::Unreadable) }; + }; + for key in [b"Predictor".as_slice(), b"Colors", b"BitsPerComponent", b"Columns", b"EarlyChange"] { + if let Some(value) = dict.get(key) { + let value = resolved(doc, Some(value))?; + if value.as_int().is_none() { + return Err(PatternProblem::Unreadable); + } + dict.set(key.to_vec(), value); + } + } + if !matches!(dict.int(b"Predictor").unwrap_or(1), 1 | 2 | 10..=15) + || dict.int(b"Colors").unwrap_or(1) <= 0 + || dict.int(b"Columns").unwrap_or(1) <= 0 + || !matches!(dict.int(b"BitsPerComponent").unwrap_or(8), 1 | 2 | 4 | 8 | 16) + || !matches!(dict.int(b"EarlyChange").unwrap_or(1), 0 | 1) + { + return Err(PatternProblem::Unreadable); + } + Ok(Object::Dict(dict)) + }) + .collect::, _>>()?; + normalized.dict.set(b"Filter".to_vec(), Object::Array(names)); + normalized.dict.set(b"DecodeParms".to_vec(), Object::Array(parms)); + Ok(normalized) +} + +/// Page-tree nodes a walk of the tree may visit, pages included, before it refuses: the page tree +/// is walked whole elsewhere too (`pdfcraft_model::pages`, `unlink_doomed`), so a long document +/// must not refuse here. +const MAX_TREE_NODES: usize = 1_000_000; + +/// Every node a full save reaches through the page tree -- down `/Kids` and up the `/Parent` of +/// every node visited, page or `/Pages`, so a node reached only through a `/Parent` link counts +/// too -- that has `/Resources` of its own and is not one of the document's pages: those are +/// walked with their effective resources. Cycles are cut by the seen-set. Holders are bounded by +/// `MAX_OBJECTS`, pages are not. +fn tree_holders(doc: &Document) -> Result, PatternProblem> { + let Some(top) = doc.root().and_then(|root| doc.get(root).as_dict().and_then(|d| d.reference(b"Pages"))) else { return Ok(Vec::new()) }; + let pages: HashSet = pdfcraft_model::pages(doc).iter().map(|p| p.obj).collect(); + // The `/Resources` values to inspect. A direct `/Parent` dictionary is written inline by a + // full save, so it is a node like any other: its resources, `/Parent` and `/Kids` count. + let mut holders = Vec::new(); + let mut seen = HashSet::new(); + seen.insert(top); + let mut stack = vec![top]; + let mut inline: Vec = Vec::new(); + // One node: queue what it links to that was not seen yet (so a /Kids array shared by many + // nodes costs its length once per node, never a copy), and keep its resources. + let mut visit = + |dict: &Dict, is_page: bool, stack: &mut Vec, inline: &mut Vec, holders: &mut Vec| -> Result<(), PatternProblem> { + match dict.get(b"Parent") { + Some(Object::Ref(parent)) => { + if seen.insert(*parent) { + stack.push(*parent); + } + } + Some(Object::Dict(parent)) => { + if inline.len() >= MAX_DEPTH { + return Err(PatternProblem::Limit); + } + inline.push(parent.clone()); + } + _ => {} + } + if let Some(resources) = dict.get(b"Resources") + && !is_page + { + holders.push(resources.clone()); + if holders.len() > MAX_OBJECTS { + return Err(PatternProblem::Limit); + } + } + let is_pages = dict.name(b"Type") == Some(b"Pages") || (dict.contains(b"Kids") && dict.name(b"Type") != Some(b"Page")); + if is_pages { + let kids_object = match dict.get(b"Kids") { + Some(Object::Ref(reference)) => Some(doc.try_get(reference.num).map_err(|_| PatternProblem::Unreadable)?), + _ => None, + }; + let kids = match &kids_object { + Some(object) => object.as_array(), + None => dict.get(b"Kids").and_then(Object::as_array), + }; + for kid in kids.into_iter().flatten().filter_map(Object::as_ref) { + if seen.insert(kid) { + stack.push(kid); + } + } + } + if seen.len() > MAX_TREE_NODES { + return Err(PatternProblem::Limit); + } + Ok(()) + }; + loop { + if let Some(node) = stack.pop() { + let object = doc.try_get(node.num).map_err(|_| PatternProblem::Unreadable)?; + if let Some(dict) = object.as_dict() { + visit(dict, pages.contains(&node), &mut stack, &mut inline, &mut holders)?; + } + } else if let Some(dict) = inline.pop() { + visit(&dict, false, &mut stack, &mut inline, &mut holders)?; + } else { + break; + } + } + Ok(holders) +} + +/// Validate the requested pages before `apply` commits its working copy. With `tree`, the +/// `/Resources` of every page-tree node above the pages is inspected as well: a node no page +/// inherits from, or one a page with a broken `/Parent` link still reaches, can keep a pruned +/// cell in the saved file. +pub(crate) fn check(doc: &Document, selected: &[usize], tree: bool) -> Result<(), RedactError> { + if tree && let Some(&first) = selected.first() { + let page = first.saturating_add(1); + let holders = tree_holders(doc).map_err(|reason| RedactError::Pattern { page, reason })?; + for resources in holders { + let mut guard = Guard::new(); + guard.resources(doc, &resources, false, 0).map_err(|reason| RedactError::Pattern { page, reason })?; + } + } + let pages = pdfcraft_model::pages(doc); + for &index in selected { + let Some(page) = pages.get(index) else { return Err(RedactError::Unreadable(index.saturating_add(1))) }; + let mut guard = Guard::new(); + let checked = (|| { + if let Some(resources) = page.dict.get(b"Resources") { + guard.resources(doc, resources, false, 0)?; + } + // Kept appearance streams may own their own pattern resources. + for annotation in crate::annots_of(doc, &page.dict) { + let object = resolved(doc, Some(&annotation))?; + if let Some(appearance) = object.as_dict().and_then(|dict| dict.get(b"AP")) { + guard.walk(doc, appearance, false, 0)?; + } + } + Ok(()) + })(); + checked.map_err(|reason| RedactError::Pattern { page: index.saturating_add(1), reason })?; + } + Ok(()) +} + +/// The backstop after every other check: a full save writes whatever the trailer reaches, by any +/// reference at all -- a stray resource category, a key on a page, a direct /Parent, `/AcroForm +/// /DR`, `/PieceInfo` -- and the walks above follow only the references they know. A pattern +/// pruned from a page that a full save would still write must therefore be proven safe like any +/// retained cell (vector-only), or the redaction is refused. Costs one walk of the reachable +/// objects, and only when a pattern was pruned. +pub(crate) fn check_pruned_unreachable(doc: &Document, pruned: &[ObjRef], page: usize) -> Result<(), RedactError> { + if pruned.is_empty() { + return Ok(()); + } + let reachable = pdfcraft_cos::reachable_objects(doc)?; + let mut guard = Guard::new(); + let mut checked = HashSet::new(); + for &pattern in pruned { + if reachable.contains(&pattern.num) && checked.insert(pattern.num) { + guard.walk(doc, &Object::Ref(pattern), true, 0).map_err(|reason| RedactError::Pattern { page, reason })?; + } + } + Ok(()) +} diff --git a/crates/redact/src/tests.rs b/crates/redact/src/tests.rs index 78cff889d3..8978e0e7d6 100644 --- a/crates/redact/src/tests.rs +++ b/crates/redact/src/tests.rs @@ -290,6 +290,200 @@ fn vectors_are_removed_or_clipped_and_inline_images_go() { assert!(c.contains("150 150 m"), "the line elsewhere stays"); } +/// A tiling pattern's cell is a content stream of its own and can draw text. Redacting a shape +/// filled with such a pattern removed the fill from the page but left the pattern in the page's +/// resources, so its cell -- with the text that was under the mark -- stayed in the saved file +/// (#3647). A pattern nothing paints with any more now leaves the resources; one some surviving +/// paint still uses stays. +#[test] +fn a_pattern_whose_only_painting_was_redacted_leaves_the_page() { + fn page_with(fills: &str) -> Document { + let cell = stream( + "/Type /Pattern /PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 300 300] /XStep 300 /YStep 300 /Resources << /Font << /F1 5 0 R >> >>", + b"BT /F1 10 Tf 10 150 Td (SECRET) Tj ET", + ); + let content = format!("BT /F1 10 Tf 10 280 Td (PUBLIC) Tj ET {fills}"); + one_page(content.as_bytes(), "/Pattern << /P0 7 0 R >>", vec![cell]) + } + let has_pattern = |doc: &Document| { + pdfcraft_model::pages(doc)[0] + .dict + .get(b"Resources") + .and_then(|r| doc.resolve(r).as_dict().cloned()) + .is_some_and(|res| res.get(b"Pattern").is_some()) + }; + + // One box, marked: the pattern is no longer painted, so it goes with the fill. + let mut doc = page_with("/Pattern cs /P0 scn 20 140 100 30 re f"); + mark(&mut doc, 0, &[[15.0, 135.0, 130.0, 175.0]], ""); + let report = apply(&mut doc, None).unwrap(); + assert_eq!(report.patterns_removed, 1, "the pattern left the page"); + assert!(!has_pattern(&doc), "the page has no /Pattern left"); + let saved = write_incremental(&doc, &SaveOptions::default()).unwrap(); + assert!(!saved.windows(8).any(|w| w == b"(SECRET)"), "the cell's text is gone from the saved file"); + + // Two boxes share a text-bearing cell. The second still needs its pattern, but it is not + // safe to infer that it visibly displays every byte of the cell. + let mut doc = page_with("/Pattern cs /P0 scn 20 140 100 30 re f /Pattern cs /P0 scn 20 40 100 30 re f"); + mark(&mut doc, 0, &[[15.0, 135.0, 130.0, 175.0]], ""); + let before = write_full(&doc, &SaveOptions::default()).unwrap(); + assert!(matches!(apply(&mut doc, None), Err(RedactError::Pattern { reason: PatternProblem::Content, .. }))); + assert_eq!(write_full(&doc, &SaveOptions::default()).unwrap(), before, "refusal keeps the pending mark and original bytes"); + + // Paint is not only path painting: text shows with the fill pattern too, and the text here + // survives the mark. Dropping the pattern would take the fill away from text still on the + // page, so a pattern is kept for every operator that can paint with it. + let mut doc = page_with("/Pattern cs /P0 scn 20 140 100 30 re f BT /F1 10 Tf 10 240 Td (KEEP) Tj ET"); + mark(&mut doc, 0, &[[15.0, 135.0, 130.0, 175.0]], ""); + assert!(matches!(apply(&mut doc, None), Err(RedactError::Pattern { reason: PatternProblem::Content, .. }))); + assert!(has_pattern(&doc), "the refused operation keeps the original pattern"); + assert!(content(&doc, 0).contains("(KEEP)"), "the text is still there: {}", content(&doc, 0)); + + // Text in render mode 3 paints nothing, so it keeps no pattern: the unpainted pattern goes. + let mut doc = page_with("/Pattern cs /P0 scn 20 140 100 30 re f BT 3 Tr /F1 10 Tf 10 240 Td (KEEP) Tj ET"); + mark(&mut doc, 0, &[[15.0, 135.0, 130.0, 175.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().patterns_removed, 1, "invisible text paints no pattern"); + + // `/Pattern` may be an indirect object, like any value. Reading the reference itself as a + // dictionary found none, so nothing was pruned and the cell stayed in the saved file. + let cell = stream( + "/Type /Pattern /PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 300 300] /XStep 300 /YStep 300 /Resources << /Font << /F1 5 0 R >> >>", + b"BT /F1 10 Tf 10 150 Td (SECRET) Tj ET", + ); + let mut doc = one_page(b"/Pattern cs /P0 scn 20 140 100 30 re f", "/Pattern 8 0 R", vec![cell, b"<< /P0 7 0 R >>".to_vec()]); + mark(&mut doc, 0, &[[15.0, 135.0, 130.0, 175.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().patterns_removed, 1, "an indirect /Pattern is resolved before it is pruned"); + assert!(!has_pattern(&doc), "the page has no /Pattern left"); + let saved = write_incremental(&doc, &SaveOptions::default()).unwrap(); + assert!(!saved.windows(8).any(|w| w == b"(SECRET)"), "the cell's text is gone from the saved file"); +} + +/// Pruning takes the pattern out of `/Resources` but leaves the `/P0 scn` that selected it, so +/// the saved page names a pattern that is no longer there. ISO 32000-1 (8.6.6.2) says that +/// operand is a key of the `/Pattern` subdictionary, so this is what a reader is handed: the +/// question is whether it stops reading the stream there. Asked in the review of #3647. +#[test] +fn a_page_whose_pruned_pattern_is_still_selected_renders() { + let cell = stream( + "/Type /Pattern /PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 300 300] /XStep 300 /YStep 300 /Resources << /Font << /F1 5 0 R >> >>", + b"BT /F1 10 Tf 10 150 Td (SECRET) Tj ET", + ); + // `0 g` after the fill puts the text back on a plain colour: with the pattern still selected + // the text would paint with it, and a pattern something still paints with is never pruned. + // The text is *after* the selector on purpose -- that is what a reader would lose if it gave + // up on the stale `/P0`. + let content_stream = b"/Pattern cs /P0 scn 20 140 100 30 re f 0 g BT /F1 24 Tf 10 40 Td (PUBLIC) Tj ET"; + let mut doc = one_page(content_stream, "/Pattern << /P0 7 0 R >>", vec![cell]); + mark(&mut doc, 0, &[[15.0, 135.0, 130.0, 175.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().patterns_removed, 1, "the pattern is pruned"); + + // The state this test is about: the selector outlived the resource it names. + let after = content(&doc, 0); + assert!(after.contains("/P0 scn"), "the selector is still in the content: {after}"); + assert!(!after.contains("20 140 100 30 re"), "the fill that used it is gone: {after}"); + + let bytes = write_full(&doc, &SaveOptions::default()).unwrap(); + assert!(!bytes.windows(8).any(|w| w == b"(SECRET)"), "the cell's text is gone from the saved file"); + let mut r = pdfcraft_render::PageRenderer::new(std::sync::Arc::new(bytes), pdfcraft_render::RenderConfig::default()); + let out = r.render(pdfcraft_render::RenderRequest { page: 0, scale: 1.0, ..Default::default() }); + assert!(out.error.is_none(), "the page still renders: {:?}, warnings {:?}", out.error, out.warnings); + assert_eq!((out.width, out.height), (300, 300), "the page box is unchanged"); + + // The text after the stale selector is still drawn: 24 pt at y 40, so the top-left origin + // puts it in the rows around 240..265. Without it the reader gave up at the bad operand. + let painted = |rows: std::ops::Range, cols: std::ops::Range| { + rows.flat_map(|y| cols.clone().map(move |x| (y, x))).any(|(y, x)| { + let i = ((y * out.width + x) * 4) as usize; + out.rgba[i..i + 3] != [255, 255, 255] + }) + }; + assert!(painted(236..268, 8..120), "the text after the stale selector is still painted"); + // The mark itself is drawn over the area the pattern filled -- `apply` writes a black + // overlay there (`q 0 0 0 rg 15 135 115 40 re f Q`), so the cell cannot show through it + // even if a reader kept the stale selector. + let black = |y: u32, x: u32| { + let i = ((y * out.width + x) * 4) as usize; + out.rgba[i..i + 3] == [0, 0, 0] + }; + assert!(black(145, 70), "the redaction overlay is painted over the pattern's box"); +} + +#[test] +fn redaction_cannot_leave_hidden_cell_text_when_pattern_is_used_elsewhere() { + let cell = stream( + "/Type /Pattern /PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 300 300] /XStep 300 /YStep 300 /Resources << /Font << /F1 5 0 R >> >>", + b"0.9 g 0 0 300 300 re f 0 g BT /F1 10 Tf 30 150 Td (SECRET) Tj ET", + ); + let content = b"/Pattern cs /P0 scn 20 140 100 30 re f 20 40 100 30 re f"; + let mut doc = one_page(content, "/Pattern << /P0 7 0 R >>", vec![cell]); + mark(&mut doc, 0, &[[15.0, 135.0, 130.0, 175.0]], ""); + let before = write_full(&doc, &SaveOptions::default()).unwrap(); + assert!(matches!(apply(&mut doc, None), Err(RedactError::Pattern { reason: PatternProblem::Content, .. }))); + assert_eq!(write_full(&doc, &SaveOptions::default()).unwrap(), before, "refusal must be atomic"); +} + +#[test] +fn redaction_cannot_leave_hidden_cell_text_on_an_unselected_page() { + let cell = stream( + "/Type /Pattern /PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 300 300] /XStep 300 /YStep 300 /Resources << /Font << /F1 5 0 R >> >>", + b"BT /F1 10 Tf 30 150 Td (SECRET) Tj ET", + ); + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R 8 0 R] /Count 2 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /Pattern << /P0 7 0 R >> >> >>".to_vec(), + stream("", b"/Pattern cs /P0 scn 20 140 100 30 re f"), + FONT.replace("95 0 R", "6 0 R").into_bytes(), + widths(), + cell, + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 9 0 R /Resources << /Pattern << /P0 7 0 R >> >> >>".to_vec(), + stream("", b"/Pattern cs /P0 scn 20 40 100 30 re f"), + ]); + mark(&mut doc, 0, &[[15.0, 135.0, 130.0, 175.0]], ""); + let before = write_full(&doc, &SaveOptions::default()).unwrap(); + match apply(&mut doc, Some(&[0])) { + Ok(_) => { + let saved = write_full(&doc, &SaveOptions::default()).unwrap(); + assert!(!saved.windows(8).any(|w| w == b"(SECRET)"), "the unselected page kept the hidden cell text"); + } + Err(RedactError::Pattern { reason: PatternProblem::Content, .. }) => { + assert_eq!(write_full(&doc, &SaveOptions::default()).unwrap(), before, "refusal must keep both pages and the mark"); + } + Err(error) => panic!("unexpected refusal: {error}"), + } +} + +#[test] +fn redacted_form_must_not_retain_its_own_pattern_cell_text() { + let cell = stream( + "/Type /Pattern /PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 300 300] /XStep 300 /YStep 300 /Resources << /Font << /F1 5 0 R >> >>", + b"BT /F1 10 Tf 30 150 Td (SECRET) Tj ET", + ); + let form = stream( + "/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Resources << /Pattern << /P0 7 0 R >> >>", + b"/Pattern cs /P0 scn 20 140 100 30 re f", + ); + let mut doc = one_page(b"/Fm Do", "/XObject << /Fm 8 0 R >>", vec![cell, form]); + mark(&mut doc, 0, &[[15.0, 135.0, 130.0, 175.0]], ""); + let before = write_full(&doc, &SaveOptions::default()).unwrap(); + assert!(matches!(apply(&mut doc, None), Err(RedactError::Pattern { reason: PatternProblem::Content, .. }))); + assert_eq!(write_full(&doc, &SaveOptions::default()).unwrap(), before, "form-owned pattern refusal must be atomic"); +} + +#[test] +fn redacted_form_must_not_retain_inherited_pattern_cell_text() { + let cell = stream( + "/Type /Pattern /PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 300 300] /XStep 300 /YStep 300 /Resources << /Font << /F1 5 0 R >> >>", + b"BT /F1 10 Tf 30 150 Td (SECRET) Tj ET", + ); + let form = stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300]", b"20 140 100 30 re f"); + let mut doc = one_page(b"/Pattern cs /P0 scn /Fm Do", "/Pattern << /P0 7 0 R >> /XObject << /Fm 8 0 R >>", vec![cell, form]); + mark(&mut doc, 0, &[[15.0, 135.0, 130.0, 175.0]], ""); + let before = write_full(&doc, &SaveOptions::default()).unwrap(); + assert!(matches!(apply(&mut doc, None), Err(RedactError::Pattern { reason: PatternProblem::Content, .. }))); + assert_eq!(write_full(&doc, &SaveOptions::default()).unwrap(), before, "inherited pattern refusal must be atomic"); +} + #[test] fn partially_covered_vector_paths_do_not_leak_drawing_commands() { // A vector path with detailed drawing commands (e.g. outlined text or confidential geometry) diff --git a/crates/redact/tests/pattern_page_tree.rs b/crates/redact/tests/pattern_page_tree.rs new file mode 100644 index 0000000000..c2b68a4485 --- /dev/null +++ b/crates/redact/tests/pattern_page_tree.rs @@ -0,0 +1,221 @@ +//! Pattern redaction through the page tree and through glyphs: synthetic files whose pattern +//! cells could stay reachable, or whose surviving paint could lose its pattern, after Apply. +//! Each test reads the saved bytes back (full and incremental, with and without object +//! streams) and searches every decoded stream. + +use std::sync::Arc; + +use pdfcraft_annot::{Meta, NewAnnotation, Shape, Style, add_annotation, rect_quad}; +use pdfcraft_cos::{Document, Object, SaveOptions, write_full, write_incremental}; +use pdfcraft_redact::{PatternProblem, RedactError, apply, marks}; +use pdfcraft_testkit::{Pdf, stream_obj}; + +const SECRET: &[u8] = b"ADV2CELLSECRET"; +const TEXT_CELL: &[u8] = b"BT /F1 20 Tf 40 120 Td (ADV2CELLSECRET) Tj ET\n"; +const PATTERN: &str = + "/Type /Pattern /PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 500 300] /XStep 500 /YStep 300 /Resources << /Font << /F1 4 0 R >> >>"; +const FONT: &[u8] = b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>"; +const RES: &str = "<< /Font << /F1 4 0 R >> /Pattern << /P0 6 0 R >> >>"; +const AREA: [f64; 4] = [30.0, 105.0, 215.0, 150.0]; +/// Public text outside the mark, and a pattern fill wholly under it. +const CONTENT: &[u8] = b"BT /F1 20 Tf 40 250 Td (PUBLIC) Tj ET\n/Pattern cs /P0 scn 35 110 170 35 re f\n"; + +fn open(objs: Vec>) -> Document { + Document::open(Arc::new(Pdf::new().objs(objs).build())).unwrap() +} + +fn mark(doc: &mut Document, page: usize) { + let shape = Shape::Redact { quads: vec![rect_quad(AREA)], overlay: String::new(), look: Default::default() }; + let style = Style::default_for(&shape); + add_annotation(doc, &NewAnnotation { page, shape, style, contents: String::new(), author: "Synthetic test".into() }, &Meta::default()).unwrap(); +} + +fn decoded_streams_contain(bytes: Vec, needle: &[u8]) -> bool { + let doc = Document::open(Arc::new(bytes)).unwrap(); + doc.object_numbers().into_iter().any(|num| { + let object = doc.try_get(num).unwrap(); + matches!(&*object, Object::Stream(s) if s.decoded().unwrap_or_default().windows(needle.len()).any(|w| w == needle)) + }) +} + +/// The needle in any decoded stream, with and without object streams, and in an incremental save. +fn saved_contains(doc: &Document, needle: &[u8]) -> bool { + [true, false].into_iter().any(|object_streams| { + let opts = SaveOptions { object_streams, ..SaveOptions::default() }; + decoded_streams_contain(write_full(doc, &opts).unwrap(), needle) || decoded_streams_contain(write_incremental(doc, &opts).unwrap(), needle) + }) +} + +/// A successful apply must leave no trace of the cell; a refusal must leave the document as it was. +fn assert_confidential_or_refused(mut doc: Document) { + mark(&mut doc, 0); + assert!(saved_contains(&doc, SECRET), "the fixture must carry the secret"); + let before = write_full(&doc, &SaveOptions::default()).unwrap(); + match apply(&mut doc, None) { + Ok(report) => { + assert_eq!(report.paths_removed, 1, "{report:?}"); + assert!(!saved_contains(&doc, SECRET), "Apply succeeded but the saved file still holds the pattern cell's text ({report:?})"); + } + Err(error) => { + assert!(matches!(error, RedactError::Pattern { .. }), "{error}"); + assert_eq!(write_full(&doc, &SaveOptions::default()).unwrap(), before); + assert_eq!(marks(&doc).len(), 1); + } + } +} + +fn catalog() -> Vec { + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec() +} + +/// The page inherits its resources, but its `/Parent` does not lead back to the node it inherits +/// from (missing here). Readers resolve inheritance down the `/Kids` tree, so the page draws with +/// the root's `/Pattern`; that root keeps its `/Resources` after the page's copy is pruned. +#[test] +fn a_page_whose_parent_link_is_missing_does_not_keep_the_pruned_cell_in_the_page_tree() { + let doc = open(vec![ + catalog(), + format!("<< /Type /Pages /Kids [3 0 R] /Count 1 /Resources {RES} >>").into_bytes(), + b"<< /Type /Page /MediaBox [0 0 500 300] /Contents 5 0 R >>".to_vec(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + ]); + assert_confidential_or_refused(doc); +} + +/// As above with a `/Parent` that names a different, resource-less `/Pages` node. +#[test] +fn a_page_whose_parent_link_disagrees_with_the_tree_does_not_keep_the_pruned_cell() { + let doc = open(vec![ + catalog(), + format!("<< /Type /Pages /Kids [3 0 R] /Count 1 /Resources {RES} >>").into_bytes(), + b"<< /Type /Page /Parent 7 0 R /MediaBox [0 0 500 300] /Contents 5 0 R >>".to_vec(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + b"<< /Type /Pages /Kids [] /Count 0 >>".to_vec(), + ]); + assert_confidential_or_refused(doc); +} + +/// Two ancestors both hold `/Resources` naming the pattern; the page inherits from the nearer one, +/// which is retired once the page gets its own pruned copy. The root's dictionary, which no page +/// inherits, still names the cell. +#[test] +fn a_shadowed_ancestor_resource_dictionary_does_not_keep_the_pruned_cell() { + let doc = open(vec![ + catalog(), + format!("<< /Type /Pages /Kids [7 0 R] /Count 1 /Resources {RES} >>").into_bytes(), + b"<< /Type /Page /Parent 7 0 R /MediaBox [0 0 500 300] /Contents 5 0 R >>".to_vec(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + format!("<< /Type /Pages /Parent 2 0 R /Kids [3 0 R] /Count 1 /Resources {RES} >>").into_bytes(), + ]); + // The tree is consistent and no page inherits the root's dictionary any more, so the pruned + // pattern is unlinked from it and the redaction applies rather than refusing. + let mut doc = doc; + mark(&mut doc, 0); + let report = apply(&mut doc, None).unwrap(); + assert_eq!(report.patterns_removed, 1, "{report:?}"); + assert!(!saved_contains(&doc, SECRET), "the root's shadowed /Resources still names the cell ({report:?})"); +} + +/// Control: the same shape with only the nearer ancestor holding resources applies cleanly. +#[test] +fn control_a_single_inheriting_ancestor_is_retired_with_the_cell() { + let doc = open(vec![ + catalog(), + b"<< /Type /Pages /Kids [7 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 7 0 R /MediaBox [0 0 500 300] /Contents 5 0 R >>".to_vec(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + format!("<< /Type /Pages /Parent 2 0 R /Kids [3 0 R] /Count 1 /Resources {RES} >>").into_bytes(), + ]); + assert_confidential_or_refused(doc); +} + +/// A hostile chain of tiling patterns, each cell selecting the next, read from a file: the guard +/// must stop with an error (or apply), never overflow the stack. +#[test] +fn a_hostile_chain_of_nested_patterns_is_bounded() { + const N: usize = 3000; + let mut objs = vec![ + catalog(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources {RES} >>").into_bytes(), + FONT.to_vec(), + // The fill outside the mark keeps /P0 in use, so the guard must inspect the chain. + stream_obj("", b"/Pattern cs /P0 scn 35 110 170 35 re f /P0 scn 300 200 100 40 re f\n"), + ]; + for i in 0..N { + let num = 6 + i; + let (res, body) = if i + 1 < N { + (format!("/Resources << /Pattern << /Q {} 0 R >> >>", num + 1), b"/Pattern cs /Q scn 0 0 10 10 re f\n".to_vec()) + } else { + ("/Resources << /Font << /F1 4 0 R >> >>".to_string(), TEXT_CELL.to_vec()) + }; + objs.push(stream_obj(&format!("/Type /Pattern /PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 10 10] /XStep 10 /YStep 10 {res}"), body)); + } + let mut doc = open(objs); + mark(&mut doc, 0); + let before = write_full(&doc, &SaveOptions::default()).unwrap(); + let error = apply(&mut doc, None).unwrap_err(); + assert!(matches!(error, RedactError::Pattern { reason: PatternProblem::Limit | PatternProblem::Content, .. }), "{error}"); + assert_eq!(write_full(&doc, &SaveOptions::default()).unwrap(), before); +} +/// A Type 3 font without `/Resources` draws its glyph with the page's `/P0` (ISO 32000-1 9.6.5: +/// names are then looked up in the page's resources). The glyph is outside the mark; the page's +/// own fill with `/P0` is under it. Pruning `/P0` loses the glyph's paint. +#[test] +fn a_type3_glyph_without_resources_keeps_the_page_pattern_it_paints_with() { + let doc = open(vec![ + catalog(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources << /Font << /F1 4 0 R /T3 7 0 R >> /Pattern << /P0 6 0 R >> >> >>".to_vec(), + FONT.to_vec(), + stream_obj("", b"BT /T3 1 Tf 300 230 Td (a) Tj ET\n/Pattern cs /P0 scn 35 110 170 35 re f\n"), + stream_obj(PATTERN, b"0 0 1 rg 0 0 500 300 re f\n"), + b"<< /Type /Font /Subtype /Type3 /FontBBox [0 0 40 40] /FontMatrix [1 0 0 1 0 0] /CharProcs << /a 8 0 R >> /Encoding << /Differences [97 /a] >> /FirstChar 97 /LastChar 97 /Widths [40] >>".to_vec(), + stream_obj("", b"40 0 d0 /Pattern cs /P0 scn 0 0 40 40 re f\n"), + ]); + let mut doc = doc; + let blue_glyph = |doc: &Document| { + let bytes = write_full(doc, &SaveOptions::default()).unwrap(); + let mut r = pdfcraft_render::PageRenderer::new(Arc::new(bytes), pdfcraft_render::RenderConfig::default()); + let out = r.render(pdfcraft_render::RenderRequest { page: 0, scale: 1.0, ..Default::default() }); + // The glyph covers x 300..340, y 230..270 (rows 30..70 from the top). + let i = ((50 * out.width + 320) * 4) as usize; + out.rgba[i..i + 3].to_vec() + }; + let before = blue_glyph(&doc); + assert_eq!(before, vec![0, 0, 255], "the glyph paints with the page pattern before the redaction"); + mark(&mut doc, 0); + let report = apply(&mut doc, None).unwrap(); + assert_eq!(blue_glyph(&doc), before, "the unmarked glyph lost its pattern paint ({report:?})"); +} + +/// Text, inline images and XObjects record their fill pattern as they are met, before the hit +/// test, so a pattern whose only paint was text wholly under the mark is kept, and so is its cell. +/// That is the documented direction of error (keeping paint over losing it): a cell that carries +/// text or images is then refused by the guard, so what can stay is vector drawing nothing shows. +#[test] +fn a_pattern_painted_only_by_redacted_text_keeps_its_vector_cell() { + const DRAWING: &[u8] = b"0.1 0.2 0.3 rg 111.11 22.22 m 133.33 44.44 l 155.55 22.22 l f\n"; + let mut doc = open(vec![ + catalog(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources {RES} >>").into_bytes(), + FONT.to_vec(), + stream_obj("", b"BT /F1 20 Tf 40 250 Td (PUBLIC) Tj ET\nBT /F1 20 Tf /Pattern cs /P0 scn 40 120 Td (HIDDEN) Tj ET\n"), + stream_obj(PATTERN, DRAWING), + ]); + mark(&mut doc, 0); + let report = apply(&mut doc, None).unwrap(); + assert!(report.glyphs >= 6, "{report:?}"); + assert_eq!(report.patterns_removed, 0, "text paint keeps its pattern"); + assert!(!saved_contains(&doc, b"(HIDDEN)"), "the glyphs are gone"); + assert!(saved_contains(&doc, b"133.33 44.44 l"), "the vector cell is kept, as documented"); +} diff --git a/crates/redact/tests/pattern_page_tree_shapes.rs b/crates/redact/tests/pattern_page_tree_shapes.rs new file mode 100644 index 0000000000..dafedd56b4 --- /dev/null +++ b/crates/redact/tests/pattern_page_tree_shapes.rs @@ -0,0 +1,275 @@ +//! Pattern redaction over unusual page trees -- cyclic, deep, wide, inconsistent, packed in +//! object streams, or reached only through `/Parent` -- and over Type 3 fonts reached other than +//! by `Tf` in the scope that paints. Each test reads the saved bytes back. + +use std::sync::Arc; + +use pdfcraft_annot::{Meta, NewAnnotation, Shape, Style, add_annotation, rect_quad}; +use pdfcraft_cos::{Document, Object, SaveOptions, write_full, write_incremental}; +use pdfcraft_redact::{RedactError, apply, marks}; +use pdfcraft_testkit::{Pdf, stream_obj}; + +const SECRET: &[u8] = b"ADV2CELLSECRET"; +const TEXT_CELL: &[u8] = b"BT /F1 20 Tf 40 120 Td (ADV2CELLSECRET) Tj ET\n"; +const PATTERN: &str = + "/Type /Pattern /PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 500 300] /XStep 500 /YStep 300 /Resources << /Font << /F1 4 0 R >> >>"; +const FONT: &[u8] = b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>"; +const RES: &str = "<< /Font << /F1 4 0 R >> /Pattern << /P0 6 0 R >> >>"; +const AREA: [f64; 4] = [30.0, 105.0, 215.0, 150.0]; +const CONTENT: &[u8] = b"BT /F1 20 Tf 40 250 Td (PUBLIC) Tj ET\n/Pattern cs /P0 scn 35 110 170 35 re f\n"; + +fn open(objs: Vec>) -> Document { + Document::open(Arc::new(Pdf::new().objs(objs).build())).unwrap() +} + +fn mark(doc: &mut Document, page: usize) { + let shape = Shape::Redact { quads: vec![rect_quad(AREA)], overlay: String::new(), look: Default::default() }; + let style = Style::default_for(&shape); + add_annotation(doc, &NewAnnotation { page, shape, style, contents: String::new(), author: "Synthetic test".into() }, &Meta::default()).unwrap(); +} + +fn decoded_streams_contain(bytes: Vec, needle: &[u8]) -> bool { + let doc = Document::open(Arc::new(bytes)).unwrap(); + doc.object_numbers().into_iter().any(|num| { + let object = doc.try_get(num).unwrap(); + matches!(&*object, Object::Stream(s) if s.decoded().unwrap_or_default().windows(needle.len()).any(|w| w == needle)) + }) +} + +fn saved_contains(doc: &Document, needle: &[u8]) -> bool { + [true, false].into_iter().any(|object_streams| { + let opts = SaveOptions { object_streams, ..SaveOptions::default() }; + decoded_streams_contain(write_full(doc, &opts).unwrap(), needle) || decoded_streams_contain(write_incremental(doc, &opts).unwrap(), needle) + }) +} + +/// Round-trip through a full save with object streams, so every dictionary lives in one. +fn packed(doc: &Document) -> Document { + Document::open(Arc::new(write_full(doc, &SaveOptions { object_streams: true, ..SaveOptions::default() }).unwrap())).unwrap() +} + +/// Ok means confidential; Err must be a pattern refusal that leaves the document unchanged. +/// Returns whether it applied. +fn confidential_or_refused(mut doc: Document, page: usize) -> bool { + mark(&mut doc, page); + assert!(saved_contains(&doc, SECRET), "the fixture must carry the secret"); + let before = write_full(&doc, &SaveOptions::default()).unwrap(); + match apply(&mut doc, None) { + Ok(report) => { + assert_eq!(report.paths_removed, 1, "{report:?}"); + assert!(!saved_contains(&doc, SECRET), "Apply succeeded but the saved file still holds the pattern cell's text ({report:?})"); + true + } + Err(error) => { + assert!(matches!(error, RedactError::Pattern { .. }), "{error}"); + assert_eq!(write_full(&doc, &SaveOptions::default()).unwrap(), before); + assert_eq!(marks(&doc).len(), 1); + false + } + } +} + +fn catalog() -> Vec { + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec() +} + +/// The root `/Pages` node carries a `/Parent` to a node outside the `/Kids` tree that names the +/// pattern. A full save follows that `/Parent`, so the node and the cell are written. +#[test] +fn a_node_reached_only_through_the_roots_parent_does_not_keep_the_pruned_cell() { + let doc = open(vec![ + catalog(), + format!("<< /Type /Pages /Kids [3 0 R] /Count 1 /Parent 7 0 R /Resources {RES} >>").into_bytes(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R >>".to_vec(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + format!("<< /Type /Pages /Kids [] /Count 0 /Resources {RES} >>").into_bytes(), + ]); + confidential_or_refused(doc, 0); +} +/// As above, one level down: an intermediate node in `/Kids` whose `/Parent` leads elsewhere. +#[test] +fn a_node_reached_only_through_an_intermediate_parent_does_not_keep_the_pruned_cell() { + let doc = open(vec![ + catalog(), + b"<< /Type /Pages /Kids [8 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 8 0 R /MediaBox [0 0 500 300] /Contents 5 0 R >>".to_vec(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + format!("<< /Type /Pages /Kids [] /Count 0 /Resources {RES} >>").into_bytes(), + format!("<< /Type /Pages /Parent 7 0 R /Kids [3 0 R] /Count 1 /Resources {RES} >>").into_bytes(), + ]); + confidential_or_refused(doc, 0); +} +/// `/Parent` cycles: the root's parent is itself, and the page's chain loops through it. +#[test] +fn a_parent_cycle_is_confidential_or_refused() { + let doc = open(vec![ + catalog(), + format!("<< /Type /Pages /Kids [3 0 R] /Count 1 /Parent 2 0 R /Resources {RES} >>").into_bytes(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R >>".to_vec(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + ]); + confidential_or_refused(doc, 0); +} +/// A page 5,000 levels below the root that holds the pattern (past `resource_source`'s 4,096). +#[test] +fn a_very_deep_tree_is_confidential_or_refused() { + const DEPTH: usize = 5000; + let mut objs = vec![ + catalog(), + format!("<< /Type /Pages /Kids [7 0 R] /Count 1 /Resources {RES} >>").into_bytes(), + format!("<< /Type /Page /Parent {} 0 R /MediaBox [0 0 500 300] /Contents 5 0 R >>", 7 + DEPTH - 1).into_bytes(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + ]; + for i in 0..DEPTH { + let num = 7 + i; + let parent = if i == 0 { 2 } else { num - 1 }; + let kid = if i + 1 == DEPTH { 3 } else { num + 1 }; + objs.push(format!("<< /Type /Pages /Parent {parent} 0 R /Kids [{kid} 0 R] /Count 1 >>").into_bytes()); + } + confidential_or_refused(doc_or(objs), 0); +} +fn doc_or(objs: Vec>) -> Document { + open(objs) +} +/// The shadowed-root and missing-parent shapes again, with every dictionary in object streams. +#[test] +fn page_tree_shapes_in_object_streams_are_confidential_or_refused() { + let shadowed = open(vec![ + catalog(), + format!("<< /Type /Pages /Kids [7 0 R] /Count 1 /Resources {RES} >>").into_bytes(), + b"<< /Type /Page /Parent 7 0 R /MediaBox [0 0 500 300] /Contents 5 0 R >>".to_vec(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + format!("<< /Type /Pages /Parent 2 0 R /Kids [3 0 R] /Count 1 /Resources {RES} >>").into_bytes(), + ]); + assert!(confidential_or_refused(packed(&shadowed), 0), "the shadowed root applies"); + let missing = open(vec![ + catalog(), + format!("<< /Type /Pages /Kids [3 0 R] /Count 1 /Resources {RES} >>").into_bytes(), + b"<< /Type /Page /MediaBox [0 0 500 300] /Contents 5 0 R >>".to_vec(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + ]); + confidential_or_refused(packed(&missing), 0); +} +/// The shadowed root's `/Resources` object is also a form's resources, drawn outside the mark on +/// the redacted page: unlinking from the root must not reach through to the form. +#[test] +fn a_root_resource_dictionary_shared_with_a_form_is_confidential_or_refused() { + let doc = open(vec![ + catalog(), + b"<< /Type /Pages /Kids [7 0 R] /Count 1 /Resources 9 0 R >>".to_vec(), + b"<< /Type /Page /Parent 7 0 R /MediaBox [0 0 500 300] /Contents 5 0 R >>".to_vec(), + FONT.to_vec(), + stream_obj("", b"BT /F1 20 Tf 40 250 Td (PUBLIC) Tj ET\n/Pattern cs /P0 scn 35 110 170 35 re f\nq 1 0 0 1 300 200 cm /Fm Do Q\n"), + stream_obj(PATTERN, TEXT_CELL), + b"<< /Type /Pages /Parent 2 0 R /Kids [3 0 R] /Count 1 /Resources << /Font << /F1 4 0 R >> /Pattern << /P0 6 0 R >> /XObject << /Fm 8 0 R >> >> >>".to_vec(), + stream_obj("/Type /XObject /Subtype /Form /BBox [0 0 100 50] /Resources 9 0 R", b"0 0 1 rg 0 0 50 20 re f\n"), + RES.as_bytes().to_vec(), + ]); + confidential_or_refused(doc, 0); +} +/// A document of 17,000 pages whose first page names a pattern: only that page has patterns. +/// Redacting it should apply as it does in a short document. +#[test] +fn a_long_document_with_a_pattern_on_one_page_still_applies() { + const PAGES: usize = 17_000; + let kids: String = (0..PAGES).map(|i| format!("{} 0 R ", if i == 0 { 3 } else { 7 + i - 1 })).collect(); + let mut objs = vec![ + catalog(), + format!("<< /Type /Pages /Kids [{kids}] /Count {PAGES} >>").into_bytes(), + format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources {RES} >>").into_bytes(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + ]; + for _ in 1..PAGES { + objs.push(b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] >>".to_vec()); + } + let mut doc = open(objs); + mark(&mut doc, 0); + let result = apply(&mut doc, None); + assert!(result.is_ok(), "a long document refuses: {result:?}"); + assert!(!saved_contains(&doc, SECRET)); +} +/// Control for the above: the same with 1,000 pages. +#[test] +fn control_a_shorter_document_with_a_pattern_on_one_page_applies() { + const PAGES: usize = 1000; + let kids: String = (0..PAGES).map(|i| format!("{} 0 R ", if i == 0 { 3 } else { 7 + i - 1 })).collect(); + let mut objs = vec![ + catalog(), + format!("<< /Type /Pages /Kids [{kids}] /Count {PAGES} >>").into_bytes(), + format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources {RES} >>").into_bytes(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + ]; + for _ in 1..PAGES { + objs.push(b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] >>".to_vec()); + } + let mut doc = open(objs); + mark(&mut doc, 0); + assert!(apply(&mut doc, None).is_ok()); + assert!(!saved_contains(&doc, SECRET)); +} +fn glyph_pixel(doc: &Document, x: u32, row: u32) -> Vec { + let bytes = write_full(doc, &SaveOptions::default()).unwrap(); + let mut r = pdfcraft_render::PageRenderer::new(Arc::new(bytes), pdfcraft_render::RenderConfig::default()); + let out = r.render(pdfcraft_render::RenderRequest { page: 0, scale: 1.0, ..Default::default() }); + let i = ((row * out.width + x) * 4) as usize; + out.rgba[i..i + 3].to_vec() +} +const T3: &[u8] = b"<< /Type /Font /Subtype /Type3 /FontBBox [0 0 40 40] /FontMatrix [1 0 0 1 0 0] /CharProcs << /a 8 0 R >> /Encoding << /Differences [97 /a] >> /FirstChar 97 /LastChar 97 /Widths [40] >>"; +const BLUE_CELL: &[u8] = b"0 0 1 rg 0 0 500 300 re f\n"; + +/// The Type 3 font without `/Resources` is selected through an ExtGState `/Font` entry, not `Tf`. +#[test] +fn a_type3_font_set_by_extgstate_keeps_the_page_pattern_it_paints_with() { + let mut doc = open(vec![ + catalog(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources << /Font << /F1 4 0 R >> /ExtGState << /G 9 0 R >> /Pattern << /P0 6 0 R >> >> >>".to_vec(), + FONT.to_vec(), + stream_obj("", b"/G gs BT 300 230 Td (a) Tj ET\n/Pattern cs /P0 scn 35 110 170 35 re f\n"), + stream_obj(PATTERN, BLUE_CELL), + T3.to_vec(), + stream_obj("", b"40 0 d0 /Pattern cs /P0 scn 0 0 40 40 re f\n"), + b"<< /Type /ExtGState /Font [7 0 R 1] >>".to_vec(), + ]); + let before = glyph_pixel(&doc, 320, 50); + assert_eq!(before, vec![0, 0, 255], "the glyph paints with the page pattern before the redaction"); + mark(&mut doc, 0); + let report = apply(&mut doc, None).unwrap(); + assert_eq!(glyph_pixel(&doc, 320, 50), before, "the unmarked glyph lost its pattern paint ({report:?})"); +} +/// The Type 3 font without `/Resources` is shown inside a form that has its own `/Resources` +/// (with no `/Pattern`); the page paints `/P0` under the mark. +#[test] +fn a_type3_glyph_in_a_form_with_resources_renders_the_same_after_the_redaction() { + let mut doc = open(vec![ + catalog(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources << /Font << /F1 4 0 R >> /XObject << /Fm 9 0 R >> /Pattern << /P0 6 0 R >> >> >>".to_vec(), + FONT.to_vec(), + stream_obj("", b"/Fm Do\n/Pattern cs /P0 scn 35 110 170 35 re f\n"), + stream_obj(PATTERN, BLUE_CELL), + T3.to_vec(), + stream_obj("", b"40 0 d0 /Pattern cs /P0 scn 0 0 40 40 re f\n"), + stream_obj("/Type /XObject /Subtype /Form /BBox [0 0 500 300] /Resources << /Font << /T3 7 0 R >> >>", b"BT /T3 1 Tf 300 230 Td (a) Tj ET\n"), + ]); + let before = glyph_pixel(&doc, 320, 50); + mark(&mut doc, 0); + let report = apply(&mut doc, None).unwrap(); + assert_eq!(glyph_pixel(&doc, 320, 50), before, "the unmarked glyph renders differently ({report:?}, before {before:?})"); +} diff --git a/crates/redact/tests/pattern_reachability.rs b/crates/redact/tests/pattern_reachability.rs new file mode 100644 index 0000000000..9ca4be0c27 --- /dev/null +++ b/crates/redact/tests/pattern_reachability.rs @@ -0,0 +1,304 @@ +//! Pattern redaction against references no walk follows -- a direct `/Parent`, `/Kids` on a +//! page, a stray resource entry -- Type 3 fonts reached in other ways, and hostile wide trees. +//! Each test reads the saved bytes back. + +use std::sync::Arc; + +use pdfcraft_annot::{Meta, NewAnnotation, Shape, Style, add_annotation, rect_quad}; +use pdfcraft_cos::{Document, Object, SaveOptions, write_full, write_incremental}; +use pdfcraft_redact::{RedactError, apply, marks}; +use pdfcraft_testkit::{Pdf, stream_obj}; + +const SECRET: &[u8] = b"ADV2CELLSECRET"; +const TEXT_CELL: &[u8] = b"BT /F1 20 Tf 40 120 Td (ADV2CELLSECRET) Tj ET\n"; +const PATTERN: &str = + "/Type /Pattern /PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 500 300] /XStep 500 /YStep 300 /Resources << /Font << /F1 4 0 R >> >>"; +const FONT: &[u8] = b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>"; +const RES: &str = "<< /Font << /F1 4 0 R >> /Pattern << /P0 6 0 R >> >>"; +const AREA: [f64; 4] = [30.0, 105.0, 215.0, 150.0]; +const CONTENT: &[u8] = b"BT /F1 20 Tf 40 250 Td (PUBLIC) Tj ET\n/Pattern cs /P0 scn 35 110 170 35 re f\n"; + +fn open(objs: Vec>) -> Document { + Document::open(Arc::new(Pdf::new().objs(objs).build())).unwrap() +} + +fn mark(doc: &mut Document, page: usize) { + let shape = Shape::Redact { quads: vec![rect_quad(AREA)], overlay: String::new(), look: Default::default() }; + let style = Style::default_for(&shape); + add_annotation(doc, &NewAnnotation { page, shape, style, contents: String::new(), author: "Synthetic test".into() }, &Meta::default()).unwrap(); +} + +fn decoded_streams_contain(bytes: Vec, needle: &[u8]) -> bool { + let doc = Document::open(Arc::new(bytes)).unwrap(); + doc.object_numbers().into_iter().any(|num| { + let object = doc.try_get(num).unwrap(); + matches!(&*object, Object::Stream(s) if s.decoded().unwrap_or_default().windows(needle.len()).any(|w| w == needle)) + }) +} + +fn saved_contains(doc: &Document, needle: &[u8]) -> bool { + [true, false].into_iter().any(|object_streams| { + let opts = SaveOptions { object_streams, ..SaveOptions::default() }; + decoded_streams_contain(write_full(doc, &opts).unwrap(), needle) || decoded_streams_contain(write_incremental(doc, &opts).unwrap(), needle) + }) +} + +/// Ok means confidential; Err must be a pattern refusal that leaves the document unchanged. +/// Returns whether it applied. +fn confidential_or_refused(mut doc: Document, page: usize) -> bool { + mark(&mut doc, page); + assert!(saved_contains(&doc, SECRET), "the fixture must carry the secret"); + let before = write_full(&doc, &SaveOptions::default()).unwrap(); + match apply(&mut doc, None) { + Ok(report) => { + assert_eq!(report.paths_removed, 1, "{report:?}"); + assert!(!saved_contains(&doc, SECRET), "Apply succeeded but the saved file still holds the pattern cell's text ({report:?})"); + true + } + Err(error) => { + assert!(matches!(error, RedactError::Pattern { .. }), "{error}"); + assert_eq!(write_full(&doc, &SaveOptions::default()).unwrap(), before); + assert_eq!(marks(&doc).len(), 1); + false + } + } +} + +fn catalog() -> Vec { + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec() +} + +fn glyph_pixel(doc: &Document, x: u32, row: u32) -> Vec { + let bytes = write_full(doc, &SaveOptions::default()).unwrap(); + let mut r = pdfcraft_render::PageRenderer::new(Arc::new(bytes), pdfcraft_render::RenderConfig::default()); + let out = r.render(pdfcraft_render::RenderRequest { page: 0, scale: 1.0, ..Default::default() }); + let i = ((row * out.width + x) * 4) as usize; + out.rgba[i..i + 3].to_vec() +} + +const T3: &[u8] = b"<< /Type /Font /Subtype /Type3 /FontBBox [0 0 40 40] /FontMatrix [1 0 0 1 0 0] /CharProcs << /a 8 0 R >> /Encoding << /Differences [97 /a] >> /FirstChar 97 /LastChar 97 /Widths [40] >>"; +const BLUE_CELL: &[u8] = b"0 0 1 rg 0 0 500 300 re f\n"; + +/// The root's `/Parent` is a direct dictionary, not a reference: a full save writes it inline +/// with the root, and the pattern it names with it. +#[test] +fn a_direct_parent_dictionary_on_the_root_does_not_keep_the_pruned_cell() { + let doc = open(vec![ + catalog(), + format!("<< /Type /Pages /Kids [3 0 R] /Count 1 /Parent << /Type /Pages /Kids [] /Count 0 /Resources {RES} >> /Resources {RES} >>") + .into_bytes(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R >>".to_vec(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + ]); + confidential_or_refused(doc, 0); +} +/// A leaf (`/Type /Page`) that also carries `/Kids`: readers stop at the leaf, a full save does +/// not, and the hidden node names the pattern. +#[test] +fn a_node_hidden_under_a_leaf_s_kids_does_not_keep_the_pruned_cell() { + let doc = open(vec![ + catalog(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources {RES} /Kids [7 0 R] >>").into_bytes(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + format!("<< /Type /Pages /Parent 3 0 R /Kids [] /Count 0 /Resources {RES} >>").into_bytes(), + ]); + confidential_or_refused(doc, 0); +} +/// A stray reference to the cell from a resource category the guard does not inspect +/// (`/Shading`): the class #3999 tracks, shown here for patterns. +#[test] +fn a_stray_resource_reference_to_the_cell_does_not_keep_it() { + let doc = open(vec![ + catalog(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources << /Font << /F1 4 0 R >> /Pattern << /P0 6 0 R >> /Shading << /Sh 6 0 R >> >> >>".to_vec(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + ]); + confidential_or_refused(doc, 0); +} +fn type3_page(page_res_extra: &str, content: &[u8], extra: Vec>) -> Document { + let mut objs = vec![ + catalog(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources << /Font << /F1 4 0 R /T3 7 0 R >> /Pattern << /P0 6 0 R >> {page_res_extra} >> >>").into_bytes(), + FONT.to_vec(), + stream_obj("", content), + stream_obj(PATTERN, BLUE_CELL), + T3.to_vec(), + stream_obj("", b"40 0 d0 /Pattern cs /P0 scn 0 0 40 40 re f\n"), + ]; + objs.extend(extra); + open(objs) +} +fn assert_glyph_kept(mut doc: Document) { + let before = glyph_pixel(&doc, 320, 50); + assert_eq!(before, vec![0, 0, 255], "the glyph paints with the page pattern before the redaction"); + mark(&mut doc, 0); + let report = apply(&mut doc, None).unwrap(); + assert_eq!(glyph_pixel(&doc, 320, 50), before, "the unmarked glyph lost its pattern paint ({report:?})"); +} +/// The Type 3 font is shown in a form without `/Resources`, which draws through the page's. +#[test] +fn a_type3_glyph_in_a_resourceless_form_keeps_the_page_pattern() { + assert_glyph_kept(type3_page( + "/XObject << /Fm 9 0 R >>", + b"/Fm Do\n/Pattern cs /P0 scn 35 110 170 35 re f\n", + vec![stream_obj("/Type /XObject /Subtype /Form /BBox [0 0 500 300]", b"BT /T3 1 Tf 300 230 Td (a) Tj ET\n")], + )); +} +/// The Type 3 font is shown inside a pattern cell the page paints outside the mark: the cell +/// carries text, so the guard refuses rather than dropping anything. +#[test] +fn a_type3_glyph_inside_a_pattern_cell_is_refused() { + let mut doc = open(vec![ + catalog(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources << /Font << /F1 4 0 R >> /Pattern << /P0 6 0 R /P1 9 0 R >> >> >>".to_vec(), + FONT.to_vec(), + stream_obj("", b"/Pattern cs /P1 scn 300 200 100 40 re f\n/Pattern cs /P0 scn 35 110 170 35 re f\n"), + stream_obj(PATTERN, BLUE_CELL), + T3.to_vec(), + stream_obj("", b"40 0 d0 /Pattern cs /P0 scn 0 0 40 40 re f\n"), + stream_obj("/Type /Pattern /PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 500 300] /XStep 500 /YStep 300 /Resources << /Font << /T3 7 0 R >> >>", b"BT /T3 1 Tf 300 230 Td (a) Tj ET\n"), + ]); + mark(&mut doc, 0); + let before = write_full(&doc, &SaveOptions::default()).unwrap(); + let error = apply(&mut doc, None).unwrap_err(); + assert!(matches!(error, RedactError::Pattern { .. }), "{error}"); + assert_eq!(write_full(&doc, &SaveOptions::default()).unwrap(), before); +} +/// A hostile wide tree: `K` `/Pages` nodes that all share one `/Kids` array of `M` references +/// back to themselves, beside the real page. Time and memory are measured from outside +/// (`PDF_WIDE_K`, `PDF_WIDE_M`). Ignored by default. +#[test] +#[ignore] +fn hostile_wide_tree_probe() { + let k: usize = std::env::var("PDF_WIDE_K").ok().and_then(|v| v.parse().ok()).unwrap_or(200); + let m: usize = std::env::var("PDF_WIDE_M").ok().and_then(|v| v.parse().ok()).unwrap_or(500_000); + let mut array = String::with_capacity(m * 9); + for i in 0..m { + array.push_str(&format!("{} 0 R ", 10 + i % k)); + } + let mut objs = vec![ + catalog(), + b"<< /Type /Pages /Kids [3 0 R 10 0 R] /Count 1 >>".to_vec(), + format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources {RES} >>").into_bytes(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + format!("[{array}]").into_bytes(), + b"null".to_vec(), + b"null".to_vec(), + ]; + for _ in 0..k { + objs.push(b"<< /Type /Pages /Kids 7 0 R /Count 0 >>".to_vec()); + } + let mut doc = open(objs); + mark(&mut doc, 0); + let start = std::time::Instant::now(); + let result = apply(&mut doc, None); + eprintln!("WIDE k={k} m={m} apply={:?} in {:?}", result.as_ref().map(|r| r.patterns_removed), start.elapsed()); +} +/// The Type 3 font has `/Resources null`, which readers treat as absent. +#[test] +fn a_type3_font_with_null_resources_keeps_the_page_pattern() { + let mut doc = type3_page("", b"BT /T3 1 Tf 300 230 Td (a) Tj ET\n/Pattern cs /P0 scn 35 110 170 35 re f\n", vec![]); + doc.update_dict(pdfcraft_cos::ObjRef::new(7, 0), |d| d.set(b"Resources".to_vec(), Object::Null)).unwrap(); + assert_glyph_kept(doc); +} +/// Known limit (fidelity only, never a leak: a missed use can only prune more). Kept annotation +/// appearances are not interpreted for pattern use, so a Type 3 font without resources shown in +/// one does not keep the page's patterns. ISO 32000 resolves the glyph's names in the +/// appearance's own resources, where `/P0` does not exist, so only a lenient renderer draws it. +/// The Type 3 font is shown by a kept annotation's appearance stream (outside the mark). +#[test] +#[ignore = "known limit: kept annotation appearances are not interpreted, so a resourceless Type 3 glyph in one does not keep the page pattern; ISO 32000 resolves it in the appearance's own resources, which pdfcraft's renderer does not"] +fn a_type3_glyph_in_an_annotation_appearance_renders_the_same() { + let mut doc = type3_page( + "", + b"/Pattern cs /P0 scn 35 110 170 35 re f\n", + vec![ + stream_obj("/Type /XObject /Subtype /Form /BBox [0 0 40 40] /Resources << /Font << /T3 7 0 R >> >>", b"BT /T3 1 Tf 0 0 Td (a) Tj ET\n"), + b"<< /Type /Annot /Subtype /Stamp /Rect [300 230 340 270] /F 4 /AP << /N 9 0 R >> >>".to_vec(), + ], + ); + doc.update_dict(pdfcraft_cos::ObjRef::new(3, 0), |d| { + d.set(b"Annots".to_vec(), Object::Array(vec![Object::Ref(pdfcraft_cos::ObjRef::new(10, 0))])) + }) + .unwrap(); + let before = glyph_pixel(&doc, 320, 50); + mark(&mut doc, 0); + let report = apply(&mut doc, None).unwrap(); + assert_eq!(glyph_pixel(&doc, 320, 50), before, "the annotation renders differently ({report:?}, before {before:?})"); +} +/// A form wholly under the mark is removed; its own `/Pattern` cell carries text. Nothing prunes +/// that pattern, so the backstop never looks, and a stray reference (`/Shading`) keeps the cell. +#[test] +fn a_removed_form_s_pattern_does_not_survive_through_a_stray_reference() { + let doc = open(vec![ + catalog(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources << /Font << /F1 4 0 R >> /XObject << /Fm 7 0 R >> /Shading << /Sh 6 0 R >> >> >>".to_vec(), + FONT.to_vec(), + stream_obj("", b"BT /F1 20 Tf 40 250 Td (PUBLIC) Tj ET\nq 1 0 0 1 35 110 cm /Fm Do Q\n"), + stream_obj(PATTERN, TEXT_CELL), + stream_obj("/Type /XObject /Subtype /Form /BBox [0 0 170 35] /Resources << /Pattern << /P0 6 0 R >> >>", b"/Pattern cs /P0 scn 0 0 170 35 re f\n"), + ]); + let mut doc = doc; + mark(&mut doc, 0); + let before = write_full(&doc, &SaveOptions::default()).unwrap(); + match apply(&mut doc, None) { + Ok(report) => assert!(!saved_contains(&doc, SECRET), "Apply succeeded but the removed form's cell is in the saved file ({report:?})"), + Err(error) => { + assert!(matches!(error, RedactError::Pattern { .. }), "{error}"); + assert_eq!(write_full(&doc, &SaveOptions::default()).unwrap(), before); + } + } +} +/// `/P0` names an indirect object whose value is a reference to the cell; the cell is also named +/// by a stray reference. The backstop checks the number it pruned, not what it points at. +#[test] +fn a_pruned_pattern_named_through_a_reference_object_does_not_dodge_the_backstop() { + let doc = open(vec![ + catalog(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources << /Font << /F1 4 0 R >> /Pattern << /P0 7 0 R >> /Shading << /Sh 6 0 R >> >> >>".to_vec(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + b"6 0 R".to_vec(), + ]); + confidential_or_refused(doc, 0); +} +/// The backstop's reachability is a full save's: after Apply an incremental save must be that +/// full save, byte for byte, never an appended revision over the old bytes. +#[test] +fn an_incremental_save_after_apply_is_the_full_save() { + let doc = open(vec![ + catalog(), + format!("<< /Type /Pages /Kids [3 0 R] /Count 1 /Resources {RES} >>").into_bytes(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R >>".to_vec(), + FONT.to_vec(), + stream_obj("", CONTENT), + stream_obj(PATTERN, TEXT_CELL), + ]); + let mut doc = doc; + mark(&mut doc, 0); + apply(&mut doc, None).unwrap(); + for object_streams in [true, false] { + // A redacted rewrite gets a new identifier from fresh entropy; pin it so the two saves + // can be compared byte for byte. + let opts = SaveOptions { object_streams, id_entropy: Some([7; 16]), ..SaveOptions::default() }; + assert_eq!(write_incremental(&doc, &opts).unwrap(), write_full(&doc, &opts).unwrap()); + } + // A later edit after a save-and-reopen starts from the clean file. + let reopened = Document::open(Arc::new(write_incremental(&doc, &SaveOptions::default()).unwrap())).unwrap(); + assert!(!reopened.bytes().windows(SECRET.len()).any(|w| w == SECRET)); +} diff --git a/crates/redact/tests/pattern_redaction.rs b/crates/redact/tests/pattern_redaction.rs new file mode 100644 index 0000000000..4fd898c9ff --- /dev/null +++ b/crates/redact/tests/pattern_redaction.rs @@ -0,0 +1,758 @@ +//! Saved-output confidentiality regressions for #3647, using only synthetic PDFs. + +use std::sync::Arc; + +use pdfcraft_annot::{Meta, NewAnnotation, Shape, Style, add_annotation, rect_quad}; +use pdfcraft_cos::{Dict, Document, ObjRef, Object, SaveOptions, Stream, write_full}; +use pdfcraft_redact::{PatternProblem, RedactError, apply, marks}; +use pdfcraft_testkit::{Pdf, stream_obj}; + +const SECRET: &[u8] = b"CELLSECRET"; +const CELL: &[u8] = b"BT /F1 20 Tf 40 120 Td (CELLSECRET) Tj ET\n"; +const PATTERN: &str = + "/Type /Pattern /PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 500 300] /XStep 500 /YStep 300 /Resources << /Font << /F1 4 0 R >> >>"; +const AREA: [f64; 4] = [30.0, 105.0, 215.0, 150.0]; + +fn fixture(cell: &[u8], compressed: bool, partial: bool, inherited: bool) -> Document { + let resources = "/Resources << /Font << /F1 4 0 R >> /Pattern << /P0 6 0 R >> >>"; + let parent_res = if inherited { resources } else { "" }; + let own_res = if inherited { "" } else { resources }; + let width = if partial { 400 } else { 170 }; + let content = format!("BT /F1 20 Tf 40 250 Td (PUBLIC) Tj ET\n/Pattern cs /P0 scn 35 110 {width} 35 re f\n"); + let bytes = Pdf::new() + .objs(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + format!("<< /Type /Pages /Kids [3 0 R] /Count 1 {parent_res} >>").into_bytes(), + format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R {own_res} >>").into_bytes(), + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>".to_vec(), + stream_obj("", content.as_bytes()), + stream_obj(PATTERN, cell), + ]) + .build(); + let mut doc = Document::open(Arc::new(bytes)).unwrap(); + if compressed { + let pattern = doc.get(ObjRef::new(6, 0)); + let Object::Stream(pattern) = &*pattern else { panic!("fixture pattern must be a stream") }; + doc.set(ObjRef::new(6, 0), Object::Stream(Stream::flate(pattern.dict.clone(), cell))); + } + doc +} + +fn mark(doc: &mut Document, page: usize) { + let shape = Shape::Redact { quads: vec![rect_quad(AREA)], overlay: String::new(), look: Default::default() }; + let style = Style::default_for(&shape); + add_annotation(doc, &NewAnnotation { page, shape, style, contents: String::new(), author: "Synthetic test".into() }, &Meta::default()).unwrap(); +} + +fn saved(doc: &Document) -> Vec { + write_full(doc, &SaveOptions::default()).unwrap() +} + +fn has_decoded_secret(bytes: Vec) -> bool { + let doc = Document::open(Arc::new(bytes)).unwrap(); + doc.object_numbers().into_iter().any(|num| { + let object = doc.try_get(num).unwrap(); + matches!(&*object, Object::Stream(s) if s.decoded().unwrap().windows(SECRET.len()).any(|w| w == SECRET)) + }) +} + +/// Success must remove the payload from the saved object graph. Refusal must be atomic even +/// for the direct library caller, not just the engine's working clone. +fn assert_confidential_or_refused(mut doc: Document, expect_refusal: bool) { + mark(&mut doc, 0); + let before = saved(&doc); + assert!(has_decoded_secret(before.clone()), "the fixture must carry the secret"); + match apply(&mut doc, None) { + Ok(report) => { + assert!(!expect_refusal, "a retained text-bearing cell must refuse"); + assert_eq!(report.patterns_removed, 1); + assert!(!has_decoded_secret(saved(&doc)), "successful redaction retained the pattern cell's secret in the saved PDF"); + } + Err(error) => { + assert!(expect_refusal, "a fully removed pattern should apply: {error}"); + assert!(matches!(error, RedactError::Pattern { reason: PatternProblem::Content, .. }), "{error}"); + assert_eq!(saved(&doc), before, "a refused pattern must leave the entire document unchanged"); + assert_eq!(marks(&doc).len(), 1, "the pending mark must survive refusal"); + } + } +} + +#[test] +fn pattern_whole_fill_saved_output_is_confidential_or_refused() { + for compressed in [false, true] { + assert_confidential_or_refused(fixture(CELL, compressed, false, false), false); + } +} + +#[test] +fn pattern_partial_fill_saved_output_is_confidential_or_refused() { + // The mark hits part of the patterned path, which is removed whole (F-008), so nothing paints + // with the pattern any more: it is pruned with its cell, and the redaction is confidential + // rather than refused. Once a hit path was clipped instead, its paint kept the cell in use. + let cell = [CELL, b"BT /F1 20 Tf 300 120 Td (SAFE) Tj ET\n"].concat(); + for compressed in [false, true] { + assert_confidential_or_refused(fixture(&cell, compressed, true, false), false); + } +} + +#[test] +fn pattern_inherited_resources_saved_output_is_confidential_or_refused() { + assert_confidential_or_refused(fixture(CELL, true, false, true), false); +} + +#[test] +fn pattern_vector_only_control_still_applies_and_preserves_the_cell() { + let cell = b"0.2 0.4 0.6 rg 40 120 15 15 re f\n"; + let mut doc = fixture(cell, true, true, false); + // The guard's control: a plain vector cell some surviving paint still uses is inspected and + // kept, not refused. The hit path is removed whole (F-008), so keep another paint in use. + keep_pattern_in_use(&mut doc, 0); + mark(&mut doc, 0); + let report = apply(&mut doc, None).unwrap(); + assert_eq!(report.marks, 1); + assert_eq!((report.paths_removed, report.paths_clipped, report.patterns_removed), (1, 0, 0)); + assert!(marks(&doc).is_empty()); + let reopened = Document::open(Arc::new(saved(&doc))).unwrap(); + let page = &pdfcraft_model::pages(&reopened)[0]; + let res = reopened.dict(page.dict.get(b"Resources").unwrap()).unwrap(); + let patterns = reopened.dict(res.get(b"Pattern").unwrap()).unwrap(); + let pattern = reopened.resolve(patterns.get(b"P0").unwrap()); + let Object::Stream(pattern) = &*pattern else { panic!("pattern must be preserved") }; + assert_eq!(pattern.decoded().unwrap(), cell); +} + +#[test] +fn unrelated_font_widths_do_not_exhaust_pattern_budget() { + let mut doc = fixture(b"40 120 15 15 re f\n", false, true, false); + doc.update_dict(ObjRef::new(4, 0), |font| { + font.set(b"W".to_vec(), Object::Array(vec![Object::Int(500); 17_000])); + }) + .unwrap(); + mark(&mut doc, 0); + assert_eq!(apply(&mut doc, None).unwrap().marks, 1); +} + +#[test] +fn shared_annotation_appearance_on_unselected_page_is_refused() { + let mut doc = two_pages(); + let pages = pdfcraft_model::pages(&doc); + for page in &pages { + let mut resources = doc.dict(page.dict.get(b"Resources").unwrap()).unwrap(); + resources.remove(b"Pattern"); + doc.update_dict(page.obj, |dict| dict.set(b"Resources".to_vec(), Object::Dict(resources.clone()))).unwrap(); + } + let ap = doc.add(Object::Stream(Stream::from_raw( + Dict::from_iter([ + (b"Subtype".to_vec(), Object::name("Form")), + (b"BBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 500.into(), 300.into()])), + ( + b"Resources".to_vec(), + Object::Dict(Dict::from_iter([( + b"Pattern".to_vec(), + Object::Dict(Dict::from_iter([(b"P0".to_vec(), Object::Ref(ObjRef::new(6, 0)))])), + )])), + ), + ]), + b"/Pattern cs /P0 scn 35 110 170 35 re f".to_vec(), + ))); + for page in pages { + let annotation = doc.add(Object::Dict(Dict::from_iter([ + (b"Type".to_vec(), Object::name("Annot")), + (b"Subtype".to_vec(), Object::name("Stamp")), + (b"Rect".to_vec(), Object::Array(vec![30.into(), 105.into(), 215.into(), 150.into()])), + (b"AP".to_vec(), Object::Dict(Dict::from_iter([(b"N".to_vec(), Object::Ref(ap))]))), + ]))); + doc.update_dict(page.obj, |dict| dict.set(b"Annots".to_vec(), Object::Array(vec![Object::Ref(annotation)]))).unwrap(); + } + mark(&mut doc, 0); + let before = saved(&doc); + let error = apply(&mut doc, Some(&[0])).unwrap_err(); + assert_eq!(error, RedactError::Pattern { page: 2, reason: PatternProblem::Content }); + assert_eq!(saved(&doc), before); + assert_eq!(marks(&doc).len(), 1); +} + +#[test] +fn pattern_refusal_on_second_page_does_not_change_the_first_page() { + let mut doc = two_pages(); + // The second page's pattern must stay in use for it to refuse: its hit path alone would be + // removed whole (F-008) and the pattern pruned. + keep_pattern_in_use(&mut doc, 1); + mark(&mut doc, 0); + mark(&mut doc, 1); + let before = saved(&doc); + let result = apply(&mut doc, None); + assert!(result.is_err(), "text-carrying pattern must refuse the whole apply"); + assert_eq!(saved(&doc), before); + assert_eq!(marks(&doc).len(), 2); +} + +fn two_pages() -> Document { + let mut doc = fixture(CELL, false, true, false); + let source_page = pdfcraft_model::pages(&doc)[0].dict.clone(); + let mut clean = source_page.clone(); + let mut clean_resources = doc.dict(clean.get(b"Resources").unwrap()).unwrap(); + clean_resources.remove(b"Pattern"); + clean.set(b"Resources".to_vec(), Object::Dict(clean_resources)); + let content = doc.add(Object::Stream(Stream::from_raw(Default::default(), b"BT /F1 20 Tf 40 120 Td (DIRECTSECRET) Tj ET".to_vec()))); + clean.set(b"Contents".to_vec(), Object::Ref(content)); + let first = doc.add(Object::Dict(clean)); + doc.update_dict(ObjRef::new(2, 0), |pages| { + pages.set(b"Kids".to_vec(), Object::Array(vec![Object::Ref(first), Object::Ref(ObjRef::new(3, 0))])); + pages.set(b"Count".to_vec(), Object::Int(2)); + }) + .unwrap(); + doc +} + +fn pattern_stream(doc: &Document) -> Stream { + let object = doc.get(ObjRef::new(6, 0)); + let Object::Stream(stream) = &*object else { panic!("fixture pattern must be a stream") }; + stream.clone() +} + +/// Paint with the pattern once more, well away from the mark. A path the mark hits is removed +/// whole (F-008), and a pattern nothing surviving paints with is pruned along with its cell, so +/// the guard only ever inspects a pattern that some surviving paint still uses. This keeps one, +/// so the guard has to inspect this cell rather than the pattern simply leaving the file. +fn keep_pattern_in_use(doc: &mut Document, page: usize) { + let page = pdfcraft_model::pages(doc).swap_remove(page); + let extra = doc.add(Object::Stream(Stream::from_raw(Dict::new(), b"/Pattern cs /P0 scn 300 200 100 40 re f".to_vec()))); + let mut list = match page.dict.get(b"Contents") { + Some(Object::Array(items)) => items.clone(), + Some(other) => vec![other.clone()], + None => Vec::new(), + }; + list.push(Object::Ref(extra)); + doc.update_dict(page.obj, |d| d.set(b"Contents".to_vec(), Object::Array(list))).unwrap(); +} + +fn assert_refused(mut doc: Document, reason: PatternProblem) { + keep_pattern_in_use(&mut doc, 0); + mark(&mut doc, 0); + let before = saved(&doc); + let error = apply(&mut doc, None).unwrap_err(); + assert_eq!(error, RedactError::Pattern { page: 1, reason }); + assert!(!error.to_string().contains("CELLSECRET"), "errors must not disclose page text"); + assert_eq!(saved(&doc), before, "even direct callers must be unchanged"); + assert_eq!(marks(&doc).len(), 1); +} + +#[test] +fn pattern_refusal_does_not_block_redacting_other_selected_pages() { + let mut doc = two_pages(); + mark(&mut doc, 0); + mark(&mut doc, 1); + let report = apply(&mut doc, Some(&[0])).unwrap(); + assert_eq!(report.marks, 1); + assert_eq!(marks(&doc).len(), 1); + assert_eq!(marks(&doc)[0].page, 1); + assert!(has_decoded_secret(saved(&doc)), "the unselected page must remain intact"); +} + +#[test] +fn pattern_nested_form_and_nested_pattern_text_are_refused() { + for nested_pattern in [false, true] { + let mut doc = fixture(CELL, false, true, false); + let mut inner = pattern_stream(&doc); + if !nested_pattern { + inner.dict.remove(b"PatternType"); + inner.dict.set(b"Type".to_vec(), Object::name("XObject")); + inner.dict.set(b"Subtype".to_vec(), Object::name("Form")); + } + let reference = doc.add(Object::Stream(inner)); + let resources = Dict::from_iter([( + if nested_pattern { b"Pattern".to_vec() } else { b"XObject".to_vec() }, + Object::Dict(Dict::from_iter([(b"Inner".to_vec(), Object::Ref(reference))])), + )]); + let mut outer = pattern_stream(&doc); + outer.dict.set(b"Resources".to_vec(), Object::Dict(resources)); + let content = if nested_pattern { b"/Pattern cs /Inner scn 35 110 400 35 re f".as_slice() } else { b"/Inner Do" }; + doc.set(ObjRef::new(6, 0), Object::Stream(Stream::flate(outer.dict, content))); + assert_refused(doc, PatternProblem::Content); + } +} + +#[test] +fn pattern_inside_form_resources_is_refused() { + let mut doc = fixture(CELL, false, false, false); + let page = pdfcraft_model::pages(&doc)[0].clone(); + let resources = doc.dict(page.dict.get(b"Resources").unwrap()).unwrap(); + let form = doc.add(Object::Stream(Stream::from_raw( + Dict::from_iter([ + (b"Type".to_vec(), Object::name("XObject")), + (b"Subtype".to_vec(), Object::name("Form")), + (b"BBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 500.into(), 300.into()])), + (b"Resources".to_vec(), Object::Dict(resources)), + ]), + b"/Pattern cs /P0 scn 35 110 170 35 re f".to_vec(), + ))); + let resources = Dict::from_iter([(b"XObject".to_vec(), Object::Dict(Dict::from_iter([(b"Fm".to_vec(), Object::Ref(form))])))]); + doc.update_dict(page.obj, |dict| dict.set(b"Resources".to_vec(), Object::Dict(resources))).unwrap(); + doc.set(ObjRef::new(5, 0), Object::Stream(Stream::from_raw(Dict::new(), b"/Fm Do".to_vec()))); + assert_refused(doc, PatternProblem::Content); +} + +#[test] +fn pattern_images_and_uncolored_text_are_refused() { + let inline = b"q 10 0 0 10 40 120 cm BI /W 1 /H 1 /CS /G /BPC 8 ID \x00 EI Q"; + assert_refused(fixture(inline, true, true, false), PatternProblem::Content); + let mut uncolored = fixture(CELL, false, true, false); + uncolored.update_dict(ObjRef::new(6, 0), |dict| dict.set(b"PaintType".to_vec(), Object::Int(2))).unwrap(); + assert_refused(uncolored, PatternProblem::Content); +} + +#[test] +fn pattern_indirect_filter_and_parameters_are_decoded_not_skipped() { + let mut doc = fixture(CELL, true, true, false); + let filter = doc.add(Object::name("FlateDecode")); + let predictor = doc.add(Object::Int(1)); + let parms = doc.add(Object::Dict(Dict::from_iter([(b"Predictor".to_vec(), Object::Ref(predictor))]))); + doc.update_dict(ObjRef::new(6, 0), |dict| { + dict.set(b"Filter".to_vec(), Object::Ref(filter)); + dict.set(b"DecodeParms".to_vec(), Object::Ref(parms)); + }) + .unwrap(); + assert_refused(doc, PatternProblem::Content); +} + +#[test] +fn pattern_damaged_filters_and_unparsed_content_fail_closed() { + for bad in [b"BT (unterminated".as_slice(), b"] BT /F1 20 Tf (CELLSECRET) Tj ET", b"BX UnknownPaint EX"] { + assert_refused(fixture(bad, false, true, false), PatternProblem::Unreadable); + } + let mut damaged = fixture(CELL, true, true, false); + let stream = pattern_stream(&damaged); + let cut = stream.raw.get(..stream.raw.len() / 2).unwrap().to_vec(); + damaged.set(ObjRef::new(6, 0), Object::Stream(Stream::from_raw(stream.dict, cut))); + assert_refused(damaged, PatternProblem::Unreadable); + let mut parameters = fixture(CELL, true, true, false); + parameters + .update_dict(ObjRef::new(6, 0), |dict| { + dict.set(b"DecodeParms".to_vec(), Object::Dict(Dict::from_iter([(b"Predictor".to_vec(), Object::Int(99))]))); + }) + .unwrap(); + assert_refused(parameters, PatternProblem::Unreadable); +} + +#[test] +fn pattern_oversized_decoding_and_resource_cycles_fail_closed() { + let large = vec![b' '; (1 << 20) + 1]; + assert_refused(fixture(&large, true, true, false), PatternProblem::Limit); + let mut cycle = fixture(b"0 0 20 20 re f", false, true, false); + cycle + .update_dict(ObjRef::new(6, 0), |dict| { + dict.set( + b"Resources".to_vec(), + Object::Dict(Dict::from_iter([( + b"Pattern".to_vec(), + Object::Dict(Dict::from_iter([(b"Self".to_vec(), Object::Ref(ObjRef::new(6, 0)))])), + )])), + ); + }) + .unwrap(); + assert_refused(cycle, PatternProblem::Unreadable); +} + +#[test] +fn pattern_plain_nested_form_control_still_applies() { + let mut doc = fixture(b"/Plain Do", false, true, false); + let form = doc.add(Object::Stream(Stream::from_raw( + Dict::from_iter([ + (b"Type".to_vec(), Object::name("XObject")), + (b"Subtype".to_vec(), Object::name("Form")), + (b"BBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 500.into(), 300.into()])), + ]), + b"40 120 15 15 re f".to_vec(), + ))); + doc.update_dict(ObjRef::new(6, 0), |dict| { + dict.set( + b"Resources".to_vec(), + Object::Dict(Dict::from_iter([(b"XObject".to_vec(), Object::Dict(Dict::from_iter([(b"Plain".to_vec(), Object::Ref(form))])))])), + ); + }) + .unwrap(); + mark(&mut doc, 0); + assert_eq!(apply(&mut doc, None).unwrap().marks, 1); + assert!(marks(&doc).is_empty()); +} + +#[test] +fn pattern_unused_and_annotation_appearance_resources_are_checked() { + let mut unused = fixture(CELL, false, false, false); + unused.set(ObjRef::new(5, 0), Object::Stream(Stream::from_raw(Dict::new(), b"0 0 20 20 re f".to_vec()))); + assert_refused(unused, PatternProblem::Content); + + let mut appearance = fixture(CELL, false, false, false); + let page = pdfcraft_model::pages(&appearance)[0].clone(); + let mut resources = appearance.dict(page.dict.get(b"Resources").unwrap()).unwrap(); + let patterns = resources.remove(b"Pattern").unwrap(); + appearance.update_dict(page.obj, |dict| dict.set(b"Resources".to_vec(), Object::Dict(resources))).unwrap(); + appearance.set(ObjRef::new(5, 0), Object::Stream(Stream::from_raw(Dict::new(), b"0 0 20 20 re f".to_vec()))); + let ap = appearance.add(Object::Stream(Stream::from_raw( + Dict::from_iter([ + (b"Subtype".to_vec(), Object::name("Form")), + (b"BBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 500.into(), 300.into()])), + (b"Resources".to_vec(), Object::Dict(Dict::from_iter([(b"Pattern".to_vec(), patterns)]))), + ]), + b"/Pattern cs /P0 scn 35 110 170 35 re f".to_vec(), + ))); + let annotation = appearance.add(Object::Dict(Dict::from_iter([ + (b"Type".to_vec(), Object::name("Annot")), + (b"Subtype".to_vec(), Object::name("Stamp")), + (b"Rect".to_vec(), Object::Array(vec![300.into(), 200.into(), 400.into(), 250.into()])), + (b"AP".to_vec(), Object::Dict(Dict::from_iter([(b"N".to_vec(), Object::Ref(ap))]))), + ]))); + appearance.update_dict(page.obj, |dict| dict.set(b"Annots".to_vec(), Object::Array(vec![Object::Ref(annotation)]))).unwrap(); + let mut implicit = appearance.clone(); + implicit + .update_dict(ap, |dict| { + dict.remove(b"Subtype"); + }) + .unwrap(); + assert_refused(implicit, PatternProblem::Content); + assert_refused(appearance, PatternProblem::Content); +} + +#[test] +fn pattern_image_xobjects_and_missing_resources_are_refused() { + let mut doc = fixture(b"/Picture Do", false, true, false); + let image = doc.add(Object::Stream(Stream::from_raw( + Dict::from_iter([ + (b"Type".to_vec(), Object::name("XObject")), + (b"Subtype".to_vec(), Object::name("Image")), + (b"Width".to_vec(), Object::Int(1)), + (b"Height".to_vec(), Object::Int(1)), + (b"ColorSpace".to_vec(), Object::name("DeviceGray")), + (b"BitsPerComponent".to_vec(), Object::Int(8)), + ]), + vec![0], + ))); + doc.update_dict(ObjRef::new(6, 0), |dict| { + dict.set( + b"Resources".to_vec(), + Object::Dict(Dict::from_iter([(b"XObject".to_vec(), Object::Dict(Dict::from_iter([(b"Picture".to_vec(), Object::Ref(image))])))])), + ); + }) + .unwrap(); + assert_refused(doc, PatternProblem::Content); + let mut missing = fixture(b"0 0 20 20 re f", false, true, false); + missing + .update_dict(ObjRef::new(6, 0), |dict| { + dict.remove(b"Resources"); + }) + .unwrap(); + assert_refused(missing, PatternProblem::Unreadable); +} + +#[test] +fn pattern_indirect_vector_filter_control_still_applies() { + let mut doc = fixture(b"40 120 15 15 re f", true, true, false); + let filter = doc.add(Object::name("FlateDecode")); + let parms = doc.add(Object::Dict(Dict::from_iter([(b"Predictor".to_vec(), Object::Int(1))]))); + doc.update_dict(ObjRef::new(6, 0), |dict| { + dict.set(b"Filter".to_vec(), Object::Array(vec![Object::Ref(filter)])); + dict.set(b"DecodeParms".to_vec(), Object::Array(vec![Object::Ref(parms)])); + }) + .unwrap(); + mark(&mut doc, 0); + assert_eq!(apply(&mut doc, None).unwrap().marks, 1); +} + +#[test] +fn pattern_depth_and_total_decoded_budgets_fail_closed() { + let mut deep = fixture(b"0 0 20 20 re f", false, true, false); + // Two frames a level (the array, then the reference): 300 levels run past the 512-frame + // depth budget long before the object budget, so it is the depth budget that refuses, and it + // does so without overflowing the stack. + let mut nested = Object::Int(1); + for _ in 0..300 { + nested = Object::Array(vec![Object::Ref(deep.add(nested))]); + } + deep.update_dict(ObjRef::new(6, 0), |dict| dict.set(b"Extra".to_vec(), nested)).unwrap(); + assert_refused(deep, PatternProblem::Limit); + + let mut total = fixture(b"0 0 20 20 re f", false, true, false); + let blank = vec![b' '; 1 << 20]; + let template = pattern_stream(&total).dict; + let mut patterns = Dict::new(); + for index in 0..9 { + let cell = total.add(Object::Stream(Stream::flate(template.clone(), &blank))); + patterns.set(format!("P{index}").into_bytes(), Object::Ref(cell)); + } + total + .update_dict(ObjRef::new(3, 0), |dict| { + dict.set(b"Resources".to_vec(), Object::Dict(Dict::from_iter([(b"Pattern".to_vec(), Object::Dict(patterns))]))); + }) + .unwrap(); + assert_refused(total, PatternProblem::Limit); +} + +#[test] +fn pattern_replacement_text_without_text_showing_is_refused() { + assert_refused(fixture(b"/Span << /ActualText (CELLSECRET) >> BDC 40 120 20 20 re f EMC", false, true, false), PatternProblem::Content); + let mut named = fixture(b"/Span /Description BDC 40 120 20 20 re f EMC", true, true, false); + named + .update_dict(ObjRef::new(6, 0), |dict| { + dict.set( + b"Resources".to_vec(), + Object::Dict(Dict::from_iter([( + b"Properties".to_vec(), + Object::Dict(Dict::from_iter([( + b"Description".to_vec(), + Object::Dict(Dict::from_iter([(b"Alt".to_vec(), Object::String(pdfcraft_cos::PdfString::literal(b"CELLSECRET".to_vec())))])), + )])), + )])), + ); + }) + .unwrap(); + assert_refused(named, PatternProblem::Content); +} + +#[test] +fn pattern_resource_names_are_not_dictionary_categories_or_descriptions() { + for name in ["Pattern", "Alt", "E", "Resources"] { + for has_pattern in [false, true] { + let mut doc = fixture(b"40 120 15 15 re f", false, true, false); + let font = Object::Dict(Dict::from_iter([(name.as_bytes().to_vec(), Object::Ref(ObjRef::new(4, 0)))])); + if has_pattern { + doc.update_dict(ObjRef::new(6, 0), |dict| { + dict.set(b"Resources".to_vec(), Object::Dict(Dict::from_iter([(b"Font".to_vec(), font)]))); + }) + .unwrap(); + } else { + doc.update_dict(ObjRef::new(3, 0), |dict| { + dict.set(b"Resources".to_vec(), Object::Dict(Dict::from_iter([(b"Font".to_vec(), font)]))); + }) + .unwrap(); + doc.set(ObjRef::new(5, 0), Object::Stream(Stream::from_raw(Dict::new(), b"35 110 400 35 re f".to_vec()))); + } + mark(&mut doc, 0); + assert_eq!(apply(&mut doc, None).unwrap().marks, 1, "resource named {name}, pattern: {has_pattern}"); + } + } +} + +#[test] +fn pattern_opaque_and_external_xobjects_are_refused() { + for kind in ["PS", "Bogus", "missing", "Form-PS", "Form-Ref"] { + let mut doc = fixture(b"/X Do", false, true, false); + let mut dict = Dict::from_iter([(b"Type".to_vec(), Object::name("XObject"))]); + match kind { + "missing" => {} + "Form-PS" => { + dict.set(b"Subtype".to_vec(), Object::name("Form")); + dict.set(b"Subtype2".to_vec(), Object::name("PS")); + } + "Form-Ref" => { + dict.set(b"Subtype".to_vec(), Object::name("Form")); + dict.set( + b"Ref".to_vec(), + Object::Dict(Dict::from_iter([(b"F".to_vec(), Object::String(pdfcraft_cos::PdfString::literal(b"external.pdf".to_vec())))])), + ); + } + subtype => { + dict.set(b"Subtype".to_vec(), Object::name(subtype)); + } + } + // Reference XObjects can have an empty fallback: that cannot establish safety of + // the external representation. No external file is read by this test or the guard. + let data = if kind == "Form-Ref" { b"".as_slice() } else { b"/Helvetica findfont 20 scalefont setfont (CELLSECRET) show" }; + let object = doc.add(Object::Stream(Stream::from_raw(dict, data.to_vec()))); + doc.update_dict(ObjRef::new(6, 0), |dict| { + dict.set( + b"Resources".to_vec(), + Object::Dict(Dict::from_iter([(b"XObject".to_vec(), Object::Dict(Dict::from_iter([(b"X".to_vec(), Object::Ref(object))])))])), + ); + }) + .unwrap(); + assert_refused(doc, PatternProblem::Unreadable); + } +} + +#[test] +fn pattern_named_form_resources_still_apply() { + for name in ["Pattern", "Alt", "E", "Resources"] { + for inside in [false, true] { + let content = format!("/{name} Do"); + let mut doc = fixture(content.as_bytes(), false, true, false); + let form = doc.add(Object::Stream(Stream::from_raw( + Dict::from_iter([ + (b"Type".to_vec(), Object::name("XObject")), + (b"Subtype".to_vec(), Object::name("Form")), + (b"BBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 500.into(), 300.into()])), + ]), + b"35 110 400 35 re f".to_vec(), + ))); + let resources = Object::Dict(Dict::from_iter([( + b"XObject".to_vec(), + Object::Dict(Dict::from_iter([(name.as_bytes().to_vec(), Object::Ref(form))])), + )])); + doc.update_dict(ObjRef::new(if inside { 6 } else { 3 }, 0), |dict| dict.set(b"Resources".to_vec(), resources)).unwrap(); + if !inside { + doc.set(ObjRef::new(5, 0), Object::Stream(Stream::from_raw(Dict::new(), content.into_bytes()))); + } + mark(&mut doc, 0); + assert_eq!(apply(&mut doc, None).unwrap().marks, 1, "form named {name}, inside: {inside}"); + } + } +} + +#[test] +fn pattern_soft_mask_groups_are_checked_by_resource_context() { + for subtype in [None, Some("Bogus"), Some("Form")] { + for text in [false, true] { + let mut doc = fixture(b"q /GS gs 40 120 20 20 re f Q", false, true, false); + let mut dict = Dict::from_iter([ + (b"BBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 500.into(), 300.into()])), + (b"Group".to_vec(), Object::Dict(Dict::from_iter([(b"S".to_vec(), Object::name("Transparency"))]))), + (b"Resources".to_vec(), Object::Dict(Dict::new())), + ]); + if let Some(subtype) = subtype { + dict.set(b"Subtype".to_vec(), Object::name(subtype)); + } + let group = doc.add(Object::Stream(Stream::from_raw(dict, if text { CELL.to_vec() } else { b"40 120 20 20 re f".to_vec() }))); + let state = Object::Dict(Dict::from_iter([( + b"SMask".to_vec(), + Object::Dict(Dict::from_iter([(b"S".to_vec(), Object::name("Luminosity")), (b"G".to_vec(), Object::Ref(group))])), + )])); + doc.update_dict(ObjRef::new(6, 0), |dict| { + dict.set( + b"Resources".to_vec(), + Object::Dict(Dict::from_iter([(b"ExtGState".to_vec(), Object::Dict(Dict::from_iter([(b"GS".to_vec(), state)])))])), + ); + }) + .unwrap(); + match (subtype, text) { + (Some("Form"), false) => { + mark(&mut doc, 0); + assert_eq!(apply(&mut doc, None).unwrap().marks, 1); + } + (Some("Form"), true) => assert_refused(doc, PatternProblem::Content), + _ => assert_refused(doc, PatternProblem::Unreadable), + } + } + } +} + +#[test] +fn pattern_null_resource_categories_and_entries_are_absent() { + for null_entries in [false, true] { + let mut doc = fixture(b"40 120 15 15 re f", false, true, false); + let mut resources = Dict::new(); + for category in ["Pattern", "XObject", "Properties", "ExtGState"] { + let value = if null_entries { Object::Dict(Dict::from_iter([(b"Unused".to_vec(), Object::Null)])) } else { Object::Null }; + resources.set(category.as_bytes().to_vec(), value); + } + doc.update_dict(ObjRef::new(6, 0), |dict| dict.set(b"Resources".to_vec(), Object::Dict(resources))).unwrap(); + mark(&mut doc, 0); + assert_eq!(apply(&mut doc, None).unwrap().marks, 1); + } + let mut plain = fixture(b"", false, false, false); + plain + .update_dict(ObjRef::new(3, 0), |dict| { + dict.set(b"Resources".to_vec(), Object::Dict(Dict::from_iter([(b"Pattern".to_vec(), Object::Null)]))); + }) + .unwrap(); + plain.set(ObjRef::new(5, 0), Object::Stream(Stream::from_raw(Dict::new(), b"40 120 20 20 re f".to_vec()))); + mark(&mut plain, 0); + assert_eq!(apply(&mut plain, None).unwrap().marks, 1); +} + +#[test] +fn a_pattern_whose_only_paint_was_redacted_leaves_the_file_with_its_cell() { + // The mark hits the patterned path, which is removed whole (F-008). Nothing paints with the + // pattern any more, so it leaves the page's resources and its cell -- text included -- leaves + // the saved file: the redaction applies rather than refusing. + for compressed in [false, true] { + let mut doc = fixture(CELL, compressed, true, false); + mark(&mut doc, 0); + let report = apply(&mut doc, None).unwrap(); + assert_eq!(report.patterns_removed, 1, "compressed: {compressed}"); + assert!(!has_decoded_secret(saved(&doc)), "the cell's text is not in the saved file (compressed: {compressed})"); + } +} + +#[test] +fn a_form_without_resources_keeps_the_page_pattern_it_still_paints() { + // The page fills a box under the mark with /P0, then draws a form that has no /Resources of + // its own, so it paints with the page's: its /P0 fill lies outside the mark and survives. + // The page's own fill is removed whole, but the pattern must stay for the form's. + let cell = b"0.2 0.4 0.6 rg 40 120 15 15 re f\n"; + let bytes = Pdf::new() + .objs(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 500 300] /Contents 5 0 R /Resources << /Font << /F1 4 0 R >> /Pattern << /P0 6 0 R >> /XObject << /Fm0 7 0 R >> >> >>".to_vec(), + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>".to_vec(), + stream_obj("", b"/Pattern cs /P0 scn 35 110 170 35 re f\n0 g\n/Fm0 Do\n"), + stream_obj(PATTERN, cell), + stream_obj("/Type /XObject /Subtype /Form /BBox [0 0 500 300]", b"/Pattern cs /P0 scn 300 200 100 40 re f\n"), + ]) + .build(); + let mut doc = Document::open(Arc::new(bytes)).unwrap(); + mark(&mut doc, 0); + let report = apply(&mut doc, None).unwrap(); + assert_eq!((report.paths_removed, report.patterns_removed), (1, 0)); + let reopened = Document::open(Arc::new(saved(&doc))).unwrap(); + let page = &pdfcraft_model::pages(&reopened)[0]; + let res = reopened.dict(page.dict.get(b"Resources").unwrap()).unwrap(); + let patterns = reopened.dict(res.get(b"Pattern").unwrap()).unwrap(); + assert!(patterns.get(b"P0").is_some(), "the form still paints with /P0 outside the mark"); +} + +#[test] +fn a_shared_appearance_of_a_reply_or_pop_up_removed_on_another_page_is_refused() { + // The marked comment on page 1 has no appearance; its reply (or pop-up) is listed on page 2, + // goes with it, and shares its appearance with a stamp that stays on page 2. That stamp keeps + // the text-bearing cell reachable, so the all-pages check must run and refuse. + for link in ["IRT", "Parent"] { + let mut doc = two_pages(); + let pages = pdfcraft_model::pages(&doc); + for page in &pages { + let mut resources = doc.dict(page.dict.get(b"Resources").unwrap()).unwrap(); + resources.remove(b"Pattern"); + doc.update_dict(page.obj, |dict| dict.set(b"Resources".to_vec(), Object::Dict(resources.clone()))).unwrap(); + } + let ap = doc.add(Object::Stream(Stream::from_raw( + Dict::from_iter([ + (b"Subtype".to_vec(), Object::name("Form")), + (b"BBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 500.into(), 300.into()])), + ( + b"Resources".to_vec(), + Object::Dict(Dict::from_iter([( + b"Pattern".to_vec(), + Object::Dict(Dict::from_iter([(b"P0".to_vec(), Object::Ref(ObjRef::new(6, 0)))])), + )])), + ), + ]), + b"/Pattern cs /P0 scn 35 110 170 35 re f".to_vec(), + ))); + let comment = doc.add(Object::Dict(Dict::from_iter([ + (b"Type".to_vec(), Object::name("Annot")), + (b"Subtype".to_vec(), Object::name("Text")), + (b"Rect".to_vec(), Object::Array(vec![40.into(), 110.into(), 60.into(), 130.into()])), + ]))); + let follower = doc.add(Object::Dict(Dict::from_iter([ + (b"Type".to_vec(), Object::name("Annot")), + (b"Subtype".to_vec(), Object::name(if link == "IRT" { "Text" } else { "Popup" })), + (b"Rect".to_vec(), Object::Array(vec![300.into(), 200.into(), 400.into(), 260.into()])), + (link.as_bytes().to_vec(), Object::Ref(comment)), + (b"AP".to_vec(), Object::Dict(Dict::from_iter([(b"N".to_vec(), Object::Ref(ap))]))), + ]))); + let stamp = doc.add(Object::Dict(Dict::from_iter([ + (b"Type".to_vec(), Object::name("Annot")), + (b"Subtype".to_vec(), Object::name("Stamp")), + (b"Rect".to_vec(), Object::Array(vec![30.into(), 105.into(), 215.into(), 150.into()])), + (b"AP".to_vec(), Object::Dict(Dict::from_iter([(b"N".to_vec(), Object::Ref(ap))]))), + ]))); + doc.update_dict(pages[0].obj, |dict| dict.set(b"Annots".to_vec(), Object::Array(vec![Object::Ref(comment)]))).unwrap(); + doc.update_dict(pages[1].obj, |dict| dict.set(b"Annots".to_vec(), Object::Array(vec![Object::Ref(follower), Object::Ref(stamp)]))).unwrap(); + mark(&mut doc, 0); + let before = saved(&doc); + let error = apply(&mut doc, Some(&[0])).unwrap_err(); + assert_eq!(error, RedactError::Pattern { page: 2, reason: PatternProblem::Content }, "/{link}"); + assert_eq!(saved(&doc), before, "/{link}"); + assert_eq!(marks(&doc).len(), 1, "/{link}"); + } +} diff --git a/docs/roadmap.md b/docs/roadmap.md index 023688ddc4..736d53bb7a 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -245,7 +245,7 @@ total matches the parity estimate. | M6 | Forms + JavaScript | 160–320 | 50% | 40–75 | Done: filling all field types with regenerated appearances; Prepare a form (every field type, multi-select/shared properties, move/resize/delete, Field Properties General/Appearance/Position/Options/Format/Validate/Calculate); Acrobat's AF format/keystroke/validate/calculate functions and simplified field notation run natively in Acrobat's event order; tab order; push-button actions; flatten; agent tools. Form data exchange (FDF, XFDF, XML, CSV, text) done; XFDF values retain text across XML comments and processing instructions.
Done since: JavaScript engine (boa, sandboxed) running custom keystroke/validate/calculate/format and button scripts, document JavaScripts, console. Missing: Actions tab, wider object model (annotations, layers, dialogs), XFA
Done since: Existing simple field-font encodings and Differences are respected when generating appearances; an unencodable value uses a separate Helvetica appearance font without changing the source font. Synthetic reopen regressions cover punctuation, subset fallback and multiline text. Exact font metrics/layout and broader Unicode fallback remain incomplete. | | M6 | Forms + JavaScript | 160–320 | 50% | 40–75 | Done: filling all field types with regenerated appearances; Prepare a form (every field type, multi-select/shared properties, move/resize/delete, Field Properties General/Appearance/Position/Options/Format/Validate/Calculate); Acrobat's AF format/keystroke/validate/calculate functions and simplified field notation run natively in Acrobat's event order; tab order; push-button actions; flatten; agent tools. Form data exchange (FDF, XFDF, XML, CSV, text) done; XFDF values retain text across XML comments and processing instructions.
Done since: JavaScript engine (boa, sandboxed) running custom keystroke/validate/calculate/format and button scripts, document JavaScripts, console. Missing: Actions tab, wider object model (annotations, layers, dialogs), XFA
Done since: Tab and Shift-Tab wrapping to the same field or another widget of that field refresh the newly opened editor from the accepted value before Save. Synthetic UI regressions cover both directions, same-frame Save, validation refusal and undo/redo. Renaming a field renames it in the calculations that read it (field lists, notation, literal `getField` names), in one undo step, and says how many changed. Prepare a form ▸ Calculation order, a page of its own: each calculated field's formula and value, what is wrong (fields that calculate too early, loops, fields missing from `/CO`) and Fix order, Up/Down, Alt+↑/↓ and drag, and on the page each field's step and arrows to and from the field in focus (`form_calculation_order` headlessly). | | M7 | Content editing (text, images, header/footer, watermark) | 250–500 | 20% | 100–180 | Done: header & footer, watermarks, backgrounds, Bates; added text and images that stay editable (move, resize, format, rotate, flip, crop, replace); links (Link tool, Link Properties, create from URLs, remove all). Missing: editing existing text and images in place (the longest pole), image/PDF watermarks | -| M8 | Security + redaction | 100–180 | 68% | 15–30 | Done: opening protected documents, permissions, Protect Using Password, Remove security; redaction (mark text/areas/pages, Search & Redact with patterns, apply removing glyphs, image pixels, vectors, XObject content, annotations and fields, verification, full rewrite on save); Remove hidden information and Sanitize.
Done since: redaction codes (U.S. FOIA and Privacy Act sets as overlay text, in Redaction properties and `redact_mark`). Redaction cleans indirect and form structure references, preserving surviving form tags. Redaction retires inherited resource dictionaries, so a parent `/Pages` can't keep a removed XObject reachable; unredacted pages are left untouched. Image redaction removes alternate images and unsupported masks only on affected images. Changed marked-content spans lose replacement descriptions; untouched spans keep theirs. Hidden-text cleanup retires unused named descriptions across inherited/shared page and form resources (64 focused tests and scoped Rust 1.96 Clippy/wasm checks pass; publication CI pending). The redaction audit counts a filled cover that hides half of any one word of a long line (#2662), applied or not. Phone-pattern redaction also matches labelled Italian geographic numbers with `+39` and a slash after the area code (#3205 follow-up); unlabelled international slash numbers remain unsupported. Missing: certificate security, custom redaction code sets and pattern locales, DCT re-encoding | +| M8 | Security + redaction | 100–180 | 68% | 15–30 | Done: opening protected documents, permissions, Protect Using Password, Remove security; redaction (mark text/areas/pages, Search & Redact with patterns, apply removing glyphs, image pixels, vectors, XObject content, annotations and fields, verification, full rewrite on save); Remove hidden information and Sanitize.
Done since: redaction codes (U.S. FOIA and Privacy Act sets as overlay text, in Redaction properties and `redact_mark`). Redaction cleans indirect and form structure references, preserving surviving form tags. Redaction retires inherited resource dictionaries, so a parent `/Pages` can't keep a removed XObject reachable; unredacted pages are left untouched. Image redaction removes alternate images and unsupported masks only on affected images. Changed marked-content spans lose replacement descriptions; untouched spans keep theirs. Hidden-text cleanup retires unused named descriptions across inherited/shared page and form resources (64 focused tests and scoped Rust 1.96 Clippy/wasm checks pass; publication CI pending). Retained text/image-bearing or unreadable tiling-pattern cells now make Apply refuse atomically after a bounded check of rewritten resources; fully removed cells still apply, and cross-page retention is checked. The redaction audit counts a filled cover that hides half of any one word of a long line (#2662), applied or not. Phone-pattern redaction also matches labelled Italian geographic numbers with `+39` and a slash after the area code (#3205 follow-up); unlabelled international slash numbers remain unsupported. Missing: certificate security, custom redaction code sets and pattern locales, DCT re-encoding | | M9 | Signatures (PAdES, validation) | 160–280 | 60% | 40–70 | Done: new `sign` crate (DER, X.509, CMS, PKCS #12 on RustCrypto; aws-lc-rs for RSA private keys); PAdES B-B signing (visible, invisible, existing fields, certification with DocMDP); validation with trust store and changes-after-signing classification; self-signed digital IDs; Signatures panel, message bar, sign dialogs; agent tools. Checked with pdfsig and OpenSSL.
Done since: macOS Keychain and Windows Current User Personal store signing (software-backed CNG keys), certificate viewer, signed documents protected from rewrites, page validation overlays separate from original signature artwork (baked Not Verified captions remain; #79 open), snapshot tabs that reuse instead of multiplying. Missing: timestamps (B-T), LTV (DSS, OCSP, CRL), FieldMDP, smart cards, PKCS #11 | | M10 | OCR, create, export, print | 200–350 | 33% | 120–215 | Done: create from blank/text/PNG/JPEG/TIFF (multi-page)/GIF/BMP; export PNG/JPEG/TIFF and text; Print (Acrobat's sizing, n-up, booklet, poster, comments & forms, preview, the CUPS and Windows spoolers, print-ready PDF).
Done since: embedded/72/custom DPI choices for image imports; export all images; OCR (searchable image for pages, ranges and multiple files); single-sided cut-and-stack imposition with cut marks, through Print and doc_print; architectural/ANSI/large ISO sheet presets, Match page at 100% and PDF-scale preview (#599 follow-up); Create PDF from multiple files (PDFs, images and text, edited as pages in the grid); OCR can also read pictures on pages that have text, without adding that text twice (#1028). Model staging fetches SHA-256-verified ONNX detection/recognition files after the upstream RTEN URL brownout; runtime discovery prefers complete ONNX pairs per directory, then complete legacy RTEN pairs, without mixing formats or changing directory precedence. Done since: Word invoice export uses borderless inferred tables, floating positioned text/artwork only where text overlaps an image or is rotated, and per-page sizes, covered by synthetic invoice automation regressions (#773); printing on the web through the browser's print dialog, sheets drawn by PdfCraft (gap 4). Lossless image export applies per-component Decode, including Indexed mapping (#1201 partial). Existing 16-bit precision and color-management limits remain. #3532 partial: vector printing retains existing appearances with absent Subtype through a print-owned Form copy; source artwork stays unchanged. #3120: Office export reads form XObjects 50 levels deep, as the renderer does, and up to 50 000 a page; pages past that are listed in `truncated_pages`. Word table cells use the grid columns they span, including cells after a merged cell. Done since: text in a table slot covered by another cell's span remains as paragraph text in Word, HTML, RTF and PowerPoint. Missing: OCR languages beyond Latin, editable-text OCR output, full Office layout fidelity/import. Done since: dotted-leader contents entries stay in separate complete paragraphs (#1757) #2156 candidate: last selected print paper persists across sessions; other print preferences unchanged. Image export applies soft-mask Decode opacity intervals before resizing, including indirect entries and inverted ranges. Vector printing gives annotation appearances unoccupied XObject names so existing page artwork survives. Done since: Enhance scanned file cleans scanned pages in place (background removal, edge shadow removal, despeckle; the OCR text layer and comments stay; #4119 partial). | | M11 | Optimize, preflight, PDF/A/X/UA, print production | 200–350 | 18% | 80–140 | Done: new `optimize` crate: Reduce File Size and the PDF Optimizer (images measured where drawn, bicubic downsampling, JPEG/ZIP recompression only when smaller, discard objects and user data, Flate clean-up, resource merging, object streams). Target-version compatibility is partial: newer content is not detected or converted. Content stream boundaries preserve comments, split operands and graphics state when image fragments merge. Image downsampling includes page-local `/UserUnit` scaling when measuring physical resolution. Done since: reject invalid image and soft-mask dimensions with checked sizing (#2178); black-and-white images stored losslessly as CCITT Group 4 when smaller, averaged down to 300 ppi above 450 (halftones aside), Group 3 stored again as Group 4, and black-and-white RGB and indexed images that use only palette entries 0 and 1 packed to one bit. Missing: fonts and transparency panels, space audit, preflight, PDF/A/X/UA | From af76b5ca89ed0902c30f4da97677cea38b58709d Mon Sep 17 00:00:00 2001 From: eliets2 Date: Fri, 6 Nov 2026 11:10:16 +0200 Subject: [PATCH 139/198] Delete key removes selected content object in edit mode (#6384) * Delete key removes selected content object in edit mode The reporter says Delete does nothing for a selected textbox or image in edit mode; the only Delete/Backspace handlers in the UI are for organize pages and comments, so a selected content object has no delete path. Add Delete/Backspace handling to the object selection input in object_ui.rs that deletes the currently selected object(s), and cover it with a unit test that selects an object and presses Delete. Test: delete_key_removes_selected_object failed before the change and passes after. * Tests: the Delete key test takes PageToolContext and imports PagePointer Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: eliets2 <287273517+eliets2@users.noreply.github.com> Co-authored-by: FredApps <153173854+FredApps@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 --- crates/ui-egui/src/tools/object_ui.rs | 72 +++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/crates/ui-egui/src/tools/object_ui.rs b/crates/ui-egui/src/tools/object_ui.rs index 67a077bc7b..a35bf795e1 100644 --- a/crates/ui-egui/src/tools/object_ui.rs +++ b/crates/ui-egui/src/tools/object_ui.rs @@ -218,6 +218,22 @@ pub(crate) fn page_input(cx: PageToolContext<'_>, view: &mut DocView) -> bool { } return false; } + // Delete or Backspace removes the selected objects on this page as one edit, + // like Edit::DeletePages removes pages. The key is consumed even when the pointer + // is off the page, because canvas keys belong to the selection while it exists. + if view.objects.page == Some(page) + && !view.objects.selected.is_empty() + && view.objects.gesture.is_none() + && view.pending_edit.is_none() + && !ui.ctx().egui_wants_keyboard_input() + { + let deleted = + ui.input_mut(|i| i.consume_key(egui::Modifiers::NONE, egui::Key::Delete) || i.consume_key(egui::Modifiers::NONE, egui::Key::Backspace)); + if deleted { + view.pending_edit = delete_edit(page, view.objects.selected.iter().map(|o| o.target).collect()); + return true; + } + } let mut selection = std::mem::take(&mut view.objects); let result = input(&mut selection, cx, view); view.objects = selection; @@ -527,4 +543,60 @@ mod tests { assert!(matches!(delete_edit(2, vec![t(ObjectKind::Image, 4)]), Some(Edit::EditPageImage { page: 2, index: 4, .. }))); assert!(delete_edit(0, Vec::new()).is_none()); } + + /// Selecting content objects on a page and pressing Delete must queue one edit that removes + /// them: `page_input` selects and moves content objects but never consumes Delete or + /// Backspace, so pressing Delete currently does nothing (issue #3780). + #[test] + fn delete_key_removes_selected_object() { + use crate::canvas::page_pointer::PagePointer; + use crate::{PdfCraftApp, canvas::ViewDefaults}; + use pdfcraft_engine::{DocId, ImageEdit}; + use pdfcraft_render::PageInfo; + + let mut app = PdfCraftApp::new(); + let bytes = app.session.create_blank(300.0, 400.0, 1).unwrap(); + app.open_bytes("blank.pdf", None, bytes.as_ref().clone()).unwrap(); + let doc = app.session.get(app.views[0].id).unwrap(); + + let info = pdfcraft_render::DocInfo { + pages: vec![PageInfo { width: 300.0, height: 400.0, label: String::new(), crop: [0.0, 0.0, 300.0, 400.0], rotation: 0 }], + ..Default::default() + }; + let mut view = DocView::new(DocId(1), &info, ViewDefaults::default()); + view.objects.page = Some(0); + view.objects.selected.push(EditableObject { + target: ObjectTarget { kind: ObjectKind::Image, index: 0 }, + rect: [50.0, 60.0, 100.0, 110.0], + content_type: "image", + text: None, + }); + + let ctx = egui::Context::default(); + let xf = PageXform { rect: Rect::from_min_size(Pos2::ZERO, egui::vec2(800.0, 600.0)), rot: 0, pw: 800.0, ph: 600.0 }; + let ptr = PagePointer { pos: None, ..Default::default() }; + let mut output = ctx.run_ui( + egui::RawInput { + screen_rect: Some(Rect::from_min_size(Pos2::ZERO, egui::vec2(800.0, 600.0))), + events: vec![egui::Event::Key { + key: egui::Key::Delete, + physical_key: None, + pressed: true, + repeat: false, + modifiers: egui::Modifiers::NONE, + }], + ..Default::default() + }, + |ui| { + let resp = ui.interact(ui.available_rect_before_wrap(), egui::Id::unique("page"), egui::Sense::click()); + page_input(PageToolContext { ui, resp: &resp, xf: &xf, ptr: &ptr, page: 0, doc, info: &info }, &mut view); + }, + ); + output.textures_delta.clear(); + + assert!( + matches!(&view.pending_edit, Some(Edit::EditPageImage { page: 0, index: 0, change: ImageEdit::Delete })), + "pressing Delete with one object selected must queue that object's deletion as one edit" + ); + } } From a0811650dec177caf06a1b143a6cc7590c85dff6 Mon Sep 17 00:00:00 2001 From: eliets2 Date: Fri, 6 Nov 2026 11:29:00 +0200 Subject: [PATCH 140/198] Add Pages range field to Compare Files dialog (#6336) * Add Pages range field to Compare Files dialog Users cannot limit a comparison to selected pages, so the dialog needs a validated Pages field that blocks Compare on invalid ranges. Add a Pages text field under New file, keep its text in pages_text, parse comma-separated numbers and ranges to 0-based pages, show an error and disable Compare for zero, reversed, or out-of-range pages, and leave the comparison itself unchanged. Test: compare_dialog_pages_field_rejects_out_of_range failed before the change and passes after. * Compare Files: the Pages field limits the comparison, and says what is wrong with a bad range Co-Authored-By: Claude Opus 5.5 * Compare: a change across a page break counts on both pages; a bad range is refused, not read as all pages Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: eliets2 <287273517+eliets2@users.noreply.github.com> Co-authored-by: FredApps <153173854+FredApps@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 --- crates/compare/src/lib.rs | 14 ++++++++ crates/engine/src/compare.rs | 23 ++++++++++++-- crates/engine/src/tests.rs | 4 +++ crates/ui-egui/src/app.rs | 1 + crates/ui-egui/src/lib.rs | 2 ++ crates/ui-egui/src/workflows/compare_ui.rs | 35 ++++++++++++++++---- crates/ui-egui/tests/it/compare.rs | 37 ++++++++++++++++++++++ 7 files changed, 107 insertions(+), 9 deletions(-) diff --git a/crates/compare/src/lib.rs b/crates/compare/src/lib.rs index 91245cbc21..26a50eaf71 100644 --- a/crates/compare/src/lib.rs +++ b/crates/compare/src/lib.rs @@ -41,6 +41,8 @@ pub struct Side { /// The page the change is on (for a pure insertion or deletion, where the other side's /// words would have been). pub page: usize, + /// The page its last word is on: words changed across a page break are one change. + pub last_page: usize, pub rects: Vec<[f64; 4]>, /// For an empty side (the other side's words were inserted or deleted): the box of the /// word next to where they would be. @@ -245,6 +247,7 @@ fn side(words: &[&Word], neighbour: Option<&Word>) -> Side { Side { text: words.iter().map(|w| w.text.as_str()).collect::>().join(" "), page: words.first().or(neighbour.as_ref()).map_or(0, |w| w.page), + last_page: words.last().or(neighbour.as_ref()).map_or(0, |w| w.page), rects: rects(words), near: if words.is_empty() { neighbour.map(|w| w.rect) } else { None }, } @@ -349,6 +352,17 @@ mod tests { assert_eq!(c.count(Kind::Inserted), 2); } + #[test] + fn a_change_across_a_page_break_knows_both_pages() { + let mut old = words("alpha beta", 0); + old.extend(words("gamma delta", 1)); + let mut new = words("alpha one", 0); + new.extend(words("two delta", 1)); + let c = compare(&old, &new); + assert_eq!(c.changes.len(), 1); + assert_eq!((c.changes[0].new.page, c.changes[0].new.last_page), (0, 1)); + } + #[test] fn deletions_across_pages_keep_their_pages() { let mut old = words("alpha beta gamma", 0); diff --git a/crates/engine/src/compare.rs b/crates/engine/src/compare.rs index 9be1954b09..ec5cf3faf3 100644 --- a/crates/engine/src/compare.rs +++ b/crates/engine/src/compare.rs @@ -63,6 +63,15 @@ impl Session { Ok(pdfcraft_compare::compare(&a.words(), &b.words())) } + /// [`Session::compare`], keeping the changes on `pages` of `new` (0-based; empty: every page). + pub fn compare_pages(&self, old: DocId, new: DocId, pages: &[usize]) -> Result { + let mut c = self.compare(old, new)?; + if !pages.is_empty() { + c.changes.retain(|ch| pages.iter().any(|page| (ch.new.page..=ch.new.last_page).contains(page))); + } + Ok(c) + } + /// Visual compare: regions where page n of `new` looks different from page n of `old` /// (rendered at `dpi`), as (page, user-space box in `new`). pub fn compare_visual(&self, old: DocId, new: DocId, dpi: f32) -> Result, EditError> { @@ -95,7 +104,12 @@ impl Session { /// The compare report as a new PDF (not opened). pub fn compare_report(&self, old: DocId, new: DocId) -> Result>, EditError> { - let c = self.compare(old, new)?; + self.compare_report_of(old, new, &[]) + } + + /// [`Session::compare_report`] of the changes on `pages` ([`Session::compare_pages`]). + pub fn compare_report_of(&self, old: DocId, new: DocId, pages: &[usize]) -> Result>, EditError> { + let c = self.compare_pages(old, new, pages)?; let name = |id| self.get(id).map(|d| d.name.clone()).unwrap_or_default(); self.create_report("Compare Report", &pdfcraft_compare::report(&c, &name(old), &name(new))) } @@ -104,7 +118,12 @@ impl Session { /// (blue, green) and a note where text was deleted (red), authored "Compare". One undoable /// step; returns how many comments were added. pub fn mark_differences(&mut self, old: DocId, new: DocId) -> Result { - let c = self.compare(old, new)?; + self.mark_differences_of(old, new, &[]) + } + + /// [`Session::mark_differences`] of the changes on `pages` ([`Session::compare_pages`]). + pub fn mark_differences_of(&mut self, old: DocId, new: DocId, pages: &[usize]) -> Result { + let c = self.compare_pages(old, new, pages)?; let mut edits = Vec::new(); for ch in &c.changes { let color = colour(ch.kind); diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 688dcec049..5b6fb01b9e 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -5841,6 +5841,10 @@ fn comparing_two_versions_of_a_document() { let got: Vec<(compare::Kind, &str, &str)> = c.changes.iter().map(|x| (x.kind, x.old.text.as_str(), x.new.text.as_str())).collect(); assert_eq!(got, [(compare::Kind::Replaced, "Monday.", "Tuesday."), (compare::Kind::Replaced, "days.", "days of invoice.")], "{got:?}"); assert!(s.compare(old, old).unwrap().identical()); + // limited to pages of the new file: both changes are on the first + assert_eq!(s.compare_pages(old, new, &[]).unwrap().changes.len(), 2); + assert_eq!(s.compare_pages(old, new, &[0]).unwrap().changes.len(), 2); + assert!(s.compare_pages(old, new, &[1]).unwrap().identical()); let report = s.compare_report(old, new).unwrap(); let rid = s.open("report.pdf", None, report, None).unwrap(); diff --git a/crates/ui-egui/src/app.rs b/crates/ui-egui/src/app.rs index 3b3d5dd036..04dbd19447 100644 --- a/crates/ui-egui/src/app.rs +++ b/crates/ui-egui/src/app.rs @@ -210,6 +210,7 @@ impl PdfCraftApp { action_files_override: None, pdfa: Default::default(), compare_old: None, + compare_pages: String::new(), compare: None, js_console: Default::default(), doc_js: Default::default(), diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 2f393c61d1..380f84a644 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -526,6 +526,8 @@ pub struct PdfCraftApp { pub pdfa: standards_ui::PdfaState, /// Compare files: the chosen older document and the last result. pub compare_old: Option, + /// The Compare Files dialog's Pages field: the new file's pages to compare, empty for all. + pub compare_pages: String, pub compare: Option, /// The JavaScript console and the Document JavaScripts draft. pub js_console: js_ui::JsConsole, diff --git a/crates/ui-egui/src/workflows/compare_ui.rs b/crates/ui-egui/src/workflows/compare_ui.rs index 939d5fed3e..7bf4c9f24b 100644 --- a/crates/ui-egui/src/workflows/compare_ui.rs +++ b/crates/ui-egui/src/workflows/compare_ui.rs @@ -14,6 +14,8 @@ pub struct CompareState { pub old: DocId, pub new: DocId, pub result: Comparison, + /// The pages of the new file this comparison covers (0-based; empty: every page). + pub pages: Vec, /// Regions that look different (page, user-space box), page n against page n. pub visual: Vec<(usize, [f64; 4])>, pub selected: Option, @@ -52,13 +54,14 @@ pub fn colour(kind: Kind) -> Color32 { pub(crate) fn body(ui: &mut egui::Ui, app: &mut PdfCraftApp, t: &Tokens) -> (bool, bool) { ui.label(egui::RichText::new(tl!("Compare Files")).font(theme::semibold(18.0))); ui.add_space(8.0); - let Some((_, new)) = app.active_ids() else { return (false, true) }; + let Some((active_i, new)) = app.active_ids() else { return (false, true) }; let others: Vec<(DocId, String)> = app.views.iter().filter(|v| v.id != new).filter_map(|v| app.session.get(v.id).map(|d| (d.id, d.name.clone()))).collect(); if app.compare_old.is_none_or(|o| !others.iter().any(|(id, _)| *id == o)) { app.compare_old = others.first().map(|(id, _)| *id); } let new_name = app.session.get(new).map(|d| d.name.clone()).unwrap_or_default(); + let page_count = app.views.get(active_i).map_or(0, |v| v.page_count()); egui::Grid::new("compare-files").num_columns(2).spacing([12.0, 10.0]).show(ui, |ui| { ui.label(tl!("Old file:")); let shown = @@ -72,14 +75,23 @@ pub(crate) fn body(ui: &mut egui::Ui, app: &mut PdfCraftApp, t: &Tokens) -> (boo ui.label(tl!("New file:")); ui.label(egui::RichText::new(new_name).strong()); ui.end_row(); + ui.label(tl!("Pages:")); + let field = ui.add(egui::TextEdit::singleline(&mut app.compare_pages).desired_width(320.0).hint_text(format!("1-{page_count}"))); + field.widget_info(|| egui::WidgetInfo::labeled(egui::Role::TextInput, ui.is_enabled(), tl!("Pages"))); + ui.end_row(); }); + let pages_error = crate::page_selection_ui::parse(&app.compare_pages, page_count).err(); + if let Some(e) = &pages_error { + ui.label(e.said(&|s| crate::i18n::t(s).to_string())); + } ui.add_space(4.0); ui.label(egui::RichText::new(tl!("Text is compared word by word; the differences are shown on the new file.")).small().color(t.text_muted)); ui.add_space(10.0); let (mut go, mut cancel) = (false, false); ui.horizontal(|ui| { ui.with_layout(Layout::right_to_left(Align::Center), |ui| { - if ui.add_enabled_ui(app.compare_old.is_some(), |ui| widgets::pill_button(ui, tl!("Compare"), true)).inner.clicked() { + let ready = app.compare_old.is_some() && pages_error.is_none(); + if ui.add_enabled_ui(ready, |ui| widgets::pill_button(ui, tl!("Compare"), true)).inner.clicked() { go = true; } if widgets::pill_button(ui, tl!("Cancel"), false).clicked() { @@ -191,9 +203,17 @@ impl PdfCraftApp { pub fn run_compare(&mut self) { let Some((i, new)) = self.active_ids() else { return }; let Some(old) = self.compare_old else { return }; - match self.session.compare(old, new) { + let count = self.views.get(i).map_or(0, |v| v.page_count()); + let pages = match crate::page_selection_ui::parse(&self.compare_pages, count) { + Ok(pages) => pages, + Err(e) => return self.notify_error(e.said(&|s| crate::i18n::t(s).to_string())), + }; + match self.session.compare_pages(old, new, &pages) { Ok(result) => { - let visual = self.session.compare_visual(old, new, 72.0).unwrap_or_default(); + let mut visual = self.session.compare_visual(old, new, 72.0).unwrap_or_default(); + if !pages.is_empty() { + visual.retain(|(page, _)| pages.contains(page)); + } let n = result.changes.len(); // Side by side when the old version is open in a tab and the window has the room. let side_by_side = self.views.iter().any(|v| v.id == old) && self.ctx.as_ref().is_none_or(|c| c.content_rect().width() >= 1000.0); @@ -201,6 +221,7 @@ impl PdfCraftApp { old, new, result, + pages, visual, selected: None, side_by_side, @@ -225,7 +246,7 @@ impl PdfCraftApp { pub(crate) fn compare_action(&mut self, index: usize, a: PanelAction) { let Some(s) = self.compare.as_mut() else { return }; - let (old, new) = (s.old, s.new); + let (old, new, pages) = (s.old, s.new, s.pages.clone()); match a { PanelAction::Select(i) => { s.selected = Some(i); @@ -252,7 +273,7 @@ impl PdfCraftApp { s.show_visual = visual; self.shade_compare(index); } - PanelAction::Mark => match self.session.mark_differences(old, new) { + PanelAction::Mark => match self.session.mark_differences_of(old, new, &pages) { Ok(n) => { if let Some(info) = self.session.get(new).map(|d| d.info.clone()) { self.views[index].document_changed(&info); @@ -265,7 +286,7 @@ impl PdfCraftApp { } Err(e) => self.notify_error(e), }, - PanelAction::Report => match self.session.compare_report(old, new) { + PanelAction::Report => match self.session.compare_report_of(old, new, &pages) { Ok(bytes) => { if let Err(e) = self.open_bytes("Compare Report.pdf", None, bytes.to_vec()) { self.notify_error(e); diff --git a/crates/ui-egui/tests/it/compare.rs b/crates/ui-egui/tests/it/compare.rs index 52acae545f..25a4616727 100644 --- a/crates/ui-egui/tests/it/compare.rs +++ b/crates/ui-egui/tests/it/compare.rs @@ -160,6 +160,43 @@ fn compare_filters_hide_text_or_visual_changes() { assert_eq!(kinds(&h, old_i), (text_old, 0)); } +/// The Compare Files dialog's Pages field rejects an out-of-range page: the +/// comparison does not run and the dialog stays open with the typed range kept. +#[test] +fn compare_dialog_pages_field_rejects_out_of_range() { + let s = Session::new(); + let v1 = s.create_from_text("t", "Delivery within five days. Returns accepted.").unwrap().to_vec(); + let v2 = s.create_from_text("t", "Delivery within three days. Returns accepted for a week.").unwrap().to_vec(); + let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).build_eframe(move |_cc| { + let mut app = PdfCraftApp::new(); + app.set_option("language", "en").unwrap(); + app.open_bytes("v1.pdf", None, v1.clone()).unwrap(); + app.open_bytes("v2.pdf", None, v2.clone()).unwrap(); + app + }); + h.run_steps(4); + assert!(h.state_mut().execute("doc.compare")); + h.run_steps(2); + h.get_by_label("Compare Files"); + let pages = h.query_by_role_and_label(egui::accesskit::Role::TextInput, "Pages"); + assert!(pages.is_some(), "the dialog has a Pages text input"); + assert!(h.query_by_label("Pages:").is_some(), "the dialog has a Pages field"); + let Some(pages) = pages else { return }; + pages.focus(); + pages.type_text("99"); + h.run_steps(2); + assert_eq!( + h.query_by_role_and_label(egui::accesskit::Role::TextInput, "Pages").and_then(|n| n.value()), + Some("99".to_string()), + "the typed range is kept" + ); + h.get_by_label("Compare").click(); + h.run_steps(3); + assert!(h.query_by_label("Compare Files").is_some(), "the dialog stays open for an out-of-range page"); + assert!(h.state().compare.is_none(), "the comparison did not run"); + assert_ne!(h.state().right, Some(RightPanel::Compare)); +} + /// Writes a screenshot of the Compare panel when PDFCRAFT_SHOT is set (for review). #[test] fn compare_panel_screenshot() { From 81b0833b245fee444e4703b4298aea949995a9aa Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Fri, 6 Nov 2026 11:50:02 +0100 Subject: [PATCH 141/198] Copy the shared house rules into AGENTS.md. The blocks come from craft-contrib at dde09ea31e2e457f5b4653a6e76f7cdf55944bdf. CLAUDE.md imports AGENTS.md. The house-rules workflow checks the copy. --- .github/workflows/house-rules.yml | 28 ++++++++++++ AGENTS.md | 71 ++++++++++++++++++++++++++++--- CLAUDE.md | 66 +--------------------------- 3 files changed, 95 insertions(+), 70 deletions(-) create mode 100644 .github/workflows/house-rules.yml diff --git a/.github/workflows/house-rules.yml b/.github/workflows/house-rules.yml new file mode 100644 index 0000000000..de64ad3ee4 --- /dev/null +++ b/.github/workflows/house-rules.yml @@ -0,0 +1,28 @@ +name: House rules + +on: + push: + branches: [main] + paths: + - AGENTS.md + - CLAUDE.md + - .github/workflows/house-rules.yml + pull_request: + paths: + - AGENTS.md + - CLAUDE.md + - .github/workflows/house-rules.yml + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: house-rules-${{ github.ref }} + cancel-in-progress: true + +jobs: + house-rules: + uses: storytold/craft-common-actions/.github/workflows/house-rules.yml@main + with: + rust: true diff --git a/AGENTS.md b/AGENTS.md index 3ab900aa95..f55ab534a8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,3 +1,56 @@ + +## House rules + +These apply in every Craft repository. A later rule may be stricter, not weaker. The source is `standards/house-rules-core.md` in `storytold/craft-contrib`. Propose changes there, not in a copy. + +1. **A person is accountable.** Whoever runs an agent answers for what it opens and for its review comments. +2. **Check, then claim.** Before writing code, read the issue comments, search pull requests for `#N` and the keywords, and check recent commits on `main`. Comment on the issue first. Release a claim you will not finish. +3. **Verify, don't guess.** Open a definition before using it. Confirm the cause in the code, even when someone names a location. +4. **Keep fixes narrow and tested.** Fix what the issue describes, where the bug is. Add a test that fails without the fix. +5. **Review the diff before opening a pull request.** Identifiers exist as used, the test fails without the fix, nothing unrelated changed, and the description is true. +6. **Look for a better pull request.** If another fixes the same thing, close yours when it is better or equal, and say why. +7. **Report only what ran.** Say what you tested and what you could not run. Without a local build or test, say so and rely on CI before marking ready. +8. **`main` changes through a pull request.** Do not push to it. Branch protection enforces this. There is no exception. +9. **Stop and ask only about** licensing outside the allowlist, publishing, releases, a new remote, secrets, spending money, destructive git, and anything else hard to undo. Otherwise keep working. `plan/` is a gitignored notebook, not rules. Without it, do not ask for it and do not stop. +10. **Keep secrets and personal details out** of the repository, logs and requests. +11. **CI stays shared.** Workflows call `storytold/craft-common-actions`. Do not change CI in a feature or fix pull request. If CI cannot cover a change, open an issue there. +12. **Guards say why.** A check that looks removable says what it prevents. Read that before changing it. +13. **Instructions must be readable by everyone.** No notes about one machine, and no links to files a reader cannot open. `craftrules` is retired. Do not link to it. +14. **Keep pull requests moving.** Concurrent pull requests are fine; unanswered actionable feedback is not. Respond in 7 days, a reminder at 14, closure eligible at 30, from when an actionable request waits on the author. Suggestions and unconfirmed bot findings do not start the clock. A fix, explanation or agreed plan resets it; unrelated activity does not. Waiting on a reviewer or maintainer pauses it and does not count against the author. A maintainer decides closure; it is never automatic. Reuse search results, do not poll, and stop at a rate limit. +15. **No agent attribution.** A pull request does not say an agent wrote it. A repository rule may not require that. + + +## Rust Craft repositories + +These apply to native Craft apps and to shared Rust libraries. Select them by repository type. A repository that merely contains some Rust does not take this block. Rules later in a repository's file may be stricter, but not weaker. The source is `standards/house-rules-rust.md` in [storytold/craft-contrib](https://github.com/storytold/craft-contrib). + +1. **Never crash.** Non-test code does not panic and has no `unsafe`, except where the repository file names an exception. Return errors, treat input as hostile, and bound recursion and allocations. New production crate roots copy the `deny` attribute from an existing crate: `clippy::unwrap_used`, `clippy::expect_used`, `clippy::panic`, `clippy::unimplemented`, `clippy::todo`, `clippy::unreachable`. Every crash fix comes with a test that panicked before the fix. +2. **Clean room.** Learn from observed behaviour and public specifications. Do not copy another product's code, assets, strings or artwork. Check a dependency's licence before adding it. +3. **Assets and fonts.** Every asset is original or suitably licensed, and is recorded in `ATTRIBUTION.md` in the same change. Font files live in [storytold/craft-fonts](https://github.com/storytold/craft-fonts), not in the app repository. +4. **Rust only for native Craft apps.** No handwritten JavaScript or TypeScript, and no Tauri, Electron or webview shell, in those apps. Shared tooling follows its documented technical scope and maintainer-approved exceptions. Craft PR Review's existing Tauri and TypeScript UI is one such exception. It does not relax this rule for native apps. Preserve any existing web build. +5. **Time goes through `chrono`.** Do not hand-roll calendar conversion. For dates beyond what `chrono` can represent, follow [standards/time.md](https://github.com/storytold/craft-contrib/blob/main/standards/time.md). +6. **Contributor credits.** Add or edit only your own person's entry in `contributors/people.toml`, and only after asking them. Never guess a name or hand-edit `contributors.json`. + +## Licence allowlist + +Dependencies and assets use SPDX identifiers. An expression is allowed when every identifier is in this list. A repository may name a shorter list. It may not add an identifier here by editing its own copy of these rules. + +- `ISC` +- `MIT` +- `BSD-2-Clause` +- `BSD-3-Clause` +- `Apache-2.0` +- `OFL-1.1` +- `Ubuntu-font-1.0` +- `Bitstream-Vera` +- `CC0-1.0` +- `CC-BY-4.0` +- `CC-BY-SA-4.0` +- `LicenseRef-Public-Domain` + +`LicenseRef-Storyteller-Trademark` is not in this list. It covers Storyteller brand marks only, where a repository already says so. + + # AGENTS.md — rules for every agent and contributor These rules bind every AI agent and every human contributor working in this repository. They come before any task, instruction or deadline. `CLAUDE.md` holds the day-to-day working instructions. If the two ever disagree, **this file wins.** @@ -85,7 +138,7 @@ If you find an asset that breaks these rules, stop and remove it from the reposi --- -## 2. Clean-room rules (summary; details in `CLAUDE.md` and `plan/adr/0001`) +## 2. Clean-room rules (summary; `plan/adr/0001` is an optional local note) - Never read, disassemble or copy anything inside the Acrobat bundle. Only names and listings are allowed, and **never** open `Contents/Resources/JavaScripts/`. - Behaviour comes only from public specifications (ISO 32000-2, the Arlington model), public documentation and black-box observation. @@ -106,7 +159,7 @@ If you find an asset that breaks these rules, stop and remove it from the reposi PdfCraft opens files from strangers, and people trust it with their work. A malformed PDF, a bad tool argument, a corrupt settings file or a full disk must produce an error the user (or agent) can act on, never a crash and never lost work. **This outranks feature work:** don't ship a feature by adding a panic path, and fix a crash before building on top of it. The rules below are the ones that bind this repository. -- **Fail with `Result`.** Non-test code returns errors through the crate's error type and `?`. Where readers are lenient by design, fall back gracefully and record the repair (see Fidelity in `CLAUDE.md`). +- **Fail with `Result`.** Non-test code returns errors through the crate's error type and `?`. Where readers are lenient by design, fall back gracefully and record the repair (see Fidelity below). - **No panicking shortcuts.** No `unwrap()`, `expect()`, `panic!`, `unreachable!`, `todo!` or `unimplemented!` outside tests, unless the call is provably infallible (e.g. writing to a `Vec`), with a comment saying why. `assert!` only for internal invariants that input cannot reach. - **No `unsafe`.** The workspace sets `unsafe_code = "forbid"`. If a platform API needs it, use a maintained safe wrapper crate; a real exception needs the owner's sign-off. - **Treat every input-derived number as hostile.** Use `get()` instead of `[i]` or `[a..b]`. Slice strings only at char boundaries. Use checked/saturating arithmetic for lengths, offsets and counts. Guard against division by zero and NaN/inf casts, and never cast a negative value to `usize`. Cap allocations sized by the file. @@ -115,10 +168,18 @@ PdfCraft opens files from strangers, and people trust it with their work. A malf - **Prove it.** Every crash fix comes with a small synthetic regression test that panicked before the fix. Turn fuzzer findings (`cargo xtask fuzz`) into such tests before fixing them. - **Enforced by clippy.** Root `clippy.toml` lets tests use `unwrap`/`expect`/`panic!`. A crate that is clean carries `#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]`; new crates start with it. -## 5. Everything else +## 5. Page interaction, fidelity and gates - -Follow `CLAUDE.md`. It covers quality gates, layering, commit rules and how to run and look at the app. +- **Layering.** Nothing below L7 depends on egui/winit/eframe/rfd. `cos`, `filters`, `crypt` and `arlington` stay standalone. +- **Page input.** Page modules read `PagePointer` (`crates/ui-egui/src/page_pointer.rs`), never egui's raw pointer state. `page_pointer::tests::page_modules_read_the_pointer_only_through_page_pointer` enforces it. +- **Page cursor.** Page modules claim the cursor for what is under the pointer (`view.cursor.claim`, `crates/ui-egui/src/canvas/page_cursor.rs`) and never set it themselves. `page_pointer::tests::page_modules_claim_the_cursor_and_never_set_it` enforces it. +- **Page presses.** A page module decides what is under a point in one hit test (`Hit` in `crates/ui-egui/src/canvas/page_router.rs`) and acts on a press only through that router. `page_pointer::tests::page_modules_act_on_a_press_only_through_the_router` enforces it. +- **Fidelity.** The PDF object graph is the model. Preserve unknown data. Saves are incremental unless a full rewrite is required. Never silently drop or repair data without recording it. +- **Fonts** are the optional build input `CRAFT_FONTS_DIR`. Code using `pdfcraft_fonts::CRAFT_FONTS` must work when it is empty. +- **Quality gates** before every commit: `cargo fmt --check`, clippy with `-D warnings`, the tests, `cargo xtask layers`, `cargo xtask assets`, and the wasm check. New workspace crates include `[lints] workspace = true`. Scope `cargo clippy -p ` and `cargo test -p ` to the crates you touched; the workspace sweep is CI's job. +- **Review at four zooms** as you go: the clause, the function, the module, and the project (layering, wasm, undo, save fidelity, a matching automation tool, an honest `parity/` entry). Prove a test fails before the fix by reverting the fix once. +- `plan/` is an optional gitignored notebook. Do not ask for it and do not stop when it is absent. Public status is [`ROADMAP.md`](ROADMAP.md), [`docs/gaps.md`](docs/gaps.md) and [`docs/roadmap.md`](docs/roadmap.md). "Shipped" in `parity/` means exists and tested, not parity with Acrobat. +- `cargo run -p pdfcraft -- ` opens the app. `pdfcraft-cli run` drives the engine headlessly. `cargo xtask fuzz` findings stay in `fuzz-out/` and become synthetic tests. For UI work, look at a screenshot from `--control` or the `shot` example. ## Contributor credits (About window) diff --git a/CLAUDE.md b/CLAUDE.md index 5c7034b7a4..43c994c2d3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,65 +1 @@ -# PdfCraft — instructions for agents - -PdfCraft is a clean-room, open-source, Rust-native PDF application targeting Adobe Acrobat Pro parity. It runs natively on macOS, Windows, Linux and FreeBSD, and on the web via WASM. It is the sibling of `../photocraft` (a Photoshop-class editor) and follows the same conventions. - -## Start every session here -1. Read `plan/STATUS.md`: the current milestone, the next unchecked task and any blockers. `ROADMAP.md` is the one-page summary (stage, numbers, progress log) and `docs/roadmap.md` the milestone table; at the end of every session update the milestone row and add a line to the ROADMAP.md progress log. - - Read `docs/gaps.md` (the ranked gaps: Acrobat interop, the open-issue backlog, the renderer and a fidelity harness, editing existing content, Pro workflows, 1.0 polish) and `docs/target-app-parity.md` (the numbers and how they are measured) before choosing work. Prefer the gaps over new P2/P3 features, and keep both documents true when things change (`docs/` progress docs carry a status line and a revision history). - - "Shipped" in `parity/` means "exists and tested", not "as good as Acrobat". Don't mark a feature shipped on a generic test, and say in its notes what is still missing. -2. Read that task in `plan/execution-plan.md` §3, the relevant section of `plan/architecture.md`, and the README of the crate you're touching. -3. Follow the **autonomous operation protocol** in `plan/execution-plan.md` §7 (orient → plan → implement + test → verify → record → commit). Don't stop to ask unless §7 lists the decision as the user's. - -`plan/` is gitignored (local-only, like PhotoCraft). The machine-readable parity checklist lives in `parity/` (committed). - -## Non-negotiables -- **Assets: read `AGENTS.md` §1 before adding or showing any icon, image, font or document.** No assets from Adobe products, ever. Only openly licensed or contributor-original assets are allowed, each with an entry in `ATTRIBUTION.toml`. `cargo xtask assets` enforces this. `AGENTS.md` overrides this file. -- **Fonts live in [storytold/craft-fonts](https://github.com/storytold/craft-fonts).** Never commit font files here (`AGENTS.md` §1.4). It is the optional build input `CRAFT_FONTS_DIR`: `git clone https://github.com/storytold/craft-fonts ../craft-fonts && CRAFT_FONTS_DIR=../craft-fonts cargo test --workspace` embeds the Japanese fonts (UI fallback, Japanese text in edited PDFs) and runs their tests, which otherwise skip. Code using `pdfcraft_fonts::CRAFT_FONTS` must work when it is empty. -- **Clean-room.** - - Never read, disassemble or copy anything inside the Acrobat bundle (names and listings only). **Never open `Contents/Resources/JavaScripts/`.** - - Behaviour comes from public docs, specs (ISO 32000-2, the Arlington model) and black-box observation (`plan/acrobat/`). - - Never copy GPL/AGPL code. MuPDF, Ghostscript, Poppler, veraPDF and DSS run only as external oracle processes. - - See `plan/README.md` §Clean-room and `plan/adr/0001`. -- **Privacy.** When observing Acrobat, use synthetic fixtures only. Never capture the Home view, recent files or account info. Capture by window id (`plan/acrobat/tools/`). Never commit Acrobat outputs, corpus files or personal data. -- **Layering.** Nothing below L7 depends on egui/winit/eframe/rfd (`plan/architecture.md` §3). `cos`/`filters`/`crypt`/`arlington` stay standalone. -- **Page input.** Page modules read `PagePointer` (`crates/ui-egui/src/page_pointer.rs`), never egui's raw pointer state (`hover_pos`, `interact_pos`, `latest_pos`, `press_origin`, button reads); `page_pointer::tests::page_modules_read_the_pointer_only_through_page_pointer` enforces it. -- **Page cursor.** Page modules claim the cursor for what is under the pointer (`view.cursor.claim(Layer::…, icon)`, `crates/ui-egui/src/canvas/page_cursor.rs`), never set it (`set_cursor_icon`, `cursor_icon =`, `on_hover_cursor`); `document_area` sets it once, from the claim on the highest layer the active tool reaches. `page_pointer::tests::page_modules_claim_the_cursor_and_never_set_it` enforces it. -- **Page presses.** A page module decides what of its own is under a point in one hit test, which returns a `Hit` (owner, layer, cursor, tip; `crates/ui-egui/src/canvas/page_router.rs`) and gives its cursor, its tip and its press. It acts on a press only when the press is its own or, for the active tool, nobody's: the page view tells it which (`PagePress`: its own, nobody's, or another page's), and it never tests the press point again or asks another module whether it wants it. Escape cancels any press the pages hold (`cancel_on_escape`), and that press is ignored until the button is up. `page_pointer::tests::page_modules_act_on_a_press_only_through_the_router` enforces it. -- **Fidelity.** The PDF object graph is the model. Preserve unknown data. Saves are incremental unless a full rewrite is required. Never silently drop or repair data without recording it. -- **Never panic.** Every PDF, script, MCP call, settings file and keystroke is untrusted input, and none of it may crash the app or lose the user's work. This outranks feature work. Return `Result` (the crate's error enum) and propagate with `?`, or fall back leniently and record it. In non-test code: - - No `unwrap`/`expect`/`panic!`/`unreachable!`/`todo!` unless it is provably infallible, with a comment saying why. - - No indexing or slicing with input-derived positions (use `get`, and slice strings only at char boundaries). - - Use checked or saturating arithmetic on input-derived numbers, and guard against division by zero and NaN/inf casts. - - Cap allocations sized by input, and bound recursion with depth limits or seen-sets. - - Handle lock poisoning. - - No `unsafe` (`unsafe_code = "forbid"`). - - Every crash fix gets a synthetic regression test. See `AGENTS.md` §4. -- **Rust only** in the product and build (`xtask`). No handwritten JS/TS. -- **Time goes through `chrono`** (workspace dependency). Never hand-roll UTC/calendar conversion (days-to-date algorithms, leap-year math); a crate owns only its output format string. The one exception is range extension: past the years chrono can represent, shift the date by whole 400-year Gregorian cycles (146,097 days, after which the calendar repeats), let chrono compute the month and day, and add the cycles back to the year (splitting seconds into hours, minutes and seconds is plain arithmetic). -- **Review every update as you make it, at four zooms.** Not an end-of-task pass — a per-update habit, so a defect never gets built on. **Clause:** checked arithmetic, `get()` not `[i]`, char-boundary slicing, the right error through `?`, `<` against `<=`, the 1-based/0-based seam, capped allocations. **Function:** is it total — name an input that panics; walk empty, zero, max, malformed, cyclic; is the return type honest. **File/module:** two sources of truth, dead or duplicated code, is the public surface still smallest and the doc comment still true. **Project:** layering, wasm, undo/redo, save fidelity, encryption and permissions, a matching `automation` tool or registry entry, and whether a `parity/` status or note just became untrue. Scale the zoom to the change, and fix what a level turns up before widening. - - Tests mirror the zooms: property → unit → crate integration → end-to-end. - - **Prove a test fails before the fix by actually reverting the fix.** A test nobody has seen fail is not evidence. This has caught two worthless tests in this repo: one confinement test passed with path resolution switched off entirely, because the tools failed on a nonexistent document id before reaching the path. - - **Rebasing is a review step, not a mechanical one.** Read every hunk you resolve, and the auto-merged hunks in the files you touched too — `main` moves fast enough that a file can change by hundreds of lines under a branch. -- **External PRs are held to safety, security and completeness before style.** Provenance first (no Adobe-derived asset, no committed font, no copied GPL/AGPL code, behaviour from the spec and not from reading another implementation), then no new panic path or `unsafe`, then `--root` confinement on every path including ones reached by nested dispatch, opt-in-only MCP and control channel, no new outbound network, then a headless `automation` tool, tests at the right zoom and an honest `parity/` entry. Provenance is the part a contributor cannot be expected to know, and the only part that cannot be undone after merge. -- **Quality gates** before every commit: `cargo fmt --check`, clippy with `-D warnings`, the tests, `cargo xtask layers`, `cargo xtask assets`, and the wasm check once `xtask ci` exists (M0). - - New workspace crates must include `[lints] workspace = true` in `Cargo.toml` to inherit `[workspace.lints]`. Production crate roots must declare the required crate-level lint attributes before modules or other crate items; crate-level `//!` documentation may precede them. - - **Scope builds and tests to the crates you touched** — `cargo clippy -p -j 2 --all-targets` and `cargo test -p -j 2`, under a `timeout`. A full `--workspace` build has locked up a development machine's CPU; it is CI's job and CI runs it on every PR. Run the workspace sweep locally only when you have a reason to and the machine is free. -- **Check for existing work before starting a fix or feature.** Search the issues and PRs, open and closed, with `gh issue list -R storytold/pdfcraft --state all --search ""` and the same with `gh pr list` (try a few phrasings), and check whether a merged fix is already on `main` (`git merge-base --is-ancestor origin/main`). If one is, the report is likely from an older build: say so rather than fixing it again. If an open PR covers it, review or build on that PR instead of opening a competing one. If an issue exists, reference it in the PR (`Fixes #123`). -- **Commits and PRs:** `main` is the only integration branch and is **never pushed to directly**. Branch from an up-to-date `main` as `/` (`feat/`, `fix/`, `docs/`, `chore/`, `refactor/`), one task id per commit (e.g. `M1.4: xref stream reader`), commit only green states, and open a PR into `main` using `.github/pull_request_template.md`. Merge only when CI is green. - - **`main` moves fast enough that PRs rot.** It took 396 commits in the three days to 2026-10-12, and most open PRs are conflicting at any given time. Rebase your own branch the same day `main` moves under it, and ask the author to rebase theirs — never rewrite a contributor's branch. Conflict risk tracks how *contested* the files are, not how large the change is: tooling, tests and docs are nearly free, while `crates/fonts`, `ui-egui` and the i18n catalogues are where several people collide at once. - -## Running and looking at the app -- `cargo run -p pdfcraft -- ` opens the desktop app. -- `cargo run -p pdfcraft-cli -- run --script steps.json --root DIR` drives the engine headlessly through the automation tools (`pdfcraft-cli tools` lists them). Use it, together with `page_render`, to check engine changes. `pdfcraft-cli mcp` is the opt-in MCP server (AGENTS.md §3). -- `cargo xtask fuzz --time 300` mutation-fuzzes open/render/edit/save in child processes. Findings land in `fuzz-out/findings/` (git-ignored; never commit corpus-derived files). Turn every real finding into a small synthetic regression test before fixing it. -- `cargo xtask parity [--partial]` reports Acrobat-parity progress from `parity/acrobat-features.toml`. Update the entry when a feature ships. -- `cargo xtask demo-pdf` builds `dist/demo/pdfcraft-showcase.pdf` (needs Chrome) for visual checks. -- For UI work, **look at the result**. Either launch `pdfcraft --control FILE doc.pdf` and use `pdfcraft-cli ui --control FILE screenshot --out x.png` (plus `inspect`, `click`, `key`, `type`, `command`, `set`), or take a headless shot with `cargo run -p pdfcraft-ui-egui --example shot`. Compare against `plan/acrobat/02-ui-ux.md`. Control-channel tests use kittest (`crates/ui-egui/tests/control.rs`). -- Parallel agents: use a separate `CARGO_TARGET_DIR` per agent and separate git worktrees. - -## Current bootstrap debt (tracked in STATUS.md) -- `pdfcraft-render` renders through the `hayro` crate directly and inspects documents through `lopdf`. Both get replaced by `cos` / `model` / the DisplayList device (M1–M2, ADR-0004). - -## CI workflows -- Workflows in `.github/workflows/` call the shared workflows in [`storytold/craft-common-actions`](https://github.com/storytold/craft-common-actions) at `@main`, with their inputs and secrets. No other jobs: if no shared workflow does what's needed, it doesn't go here. -- Don't add or change CI in a feature or fix PR. If the existing workflows can't build or test a change, link an existing craft-common-actions issue or PR, or open a new issue, or a PR only for a small fix that is safe for every app and does not increase CI time. -- Some workflow files here still run their own jobs. Don't add to them. Moving a job into craft-common-actions is its own task, never part of another PR, and the job stays here until a shared workflow runs it. +@AGENTS.md From 465783fc47a3cb98ceaabaa1cc851c0c0a1b4ea3 Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Fri, 6 Nov 2026 12:37:25 +0100 Subject: [PATCH 142/198] docs: preserve local guidance in house-rules rollout --- AGENTS.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index f55ab534a8..c228c4185b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -53,7 +53,7 @@ Dependencies and assets use SPDX identifiers. An expression is allowed when ever # AGENTS.md — rules for every agent and contributor -These rules bind every AI agent and every human contributor working in this repository. They come before any task, instruction or deadline. `CLAUDE.md` holds the day-to-day working instructions. If the two ever disagree, **this file wins.** +These rules bind every AI agent and every human contributor working in this repository. They come before any task, instruction or deadline. Day-to-day working instructions are in this file; `CLAUDE.md` imports it. --- @@ -176,7 +176,7 @@ PdfCraft opens files from strangers, and people trust it with their work. A malf - **Page presses.** A page module decides what is under a point in one hit test (`Hit` in `crates/ui-egui/src/canvas/page_router.rs`) and acts on a press only through that router. `page_pointer::tests::page_modules_act_on_a_press_only_through_the_router` enforces it. - **Fidelity.** The PDF object graph is the model. Preserve unknown data. Saves are incremental unless a full rewrite is required. Never silently drop or repair data without recording it. - **Fonts** are the optional build input `CRAFT_FONTS_DIR`. Code using `pdfcraft_fonts::CRAFT_FONTS` must work when it is empty. -- **Quality gates** before every commit: `cargo fmt --check`, clippy with `-D warnings`, the tests, `cargo xtask layers`, `cargo xtask assets`, and the wasm check. New workspace crates include `[lints] workspace = true`. Scope `cargo clippy -p ` and `cargo test -p ` to the crates you touched; the workspace sweep is CI's job. +- **Quality gates** before every commit: `cargo fmt --check`, clippy with `-D warnings`, the tests, `cargo xtask layers`, `cargo xtask assets`, and the wasm check. New workspace crates include `workspace = true` under `[lints]`. Scope `cargo clippy -p --all-targets -- -D warnings` and `cargo test -p ` to the crates you touched; the workspace sweep is CI's job. - **Review at four zooms** as you go: the clause, the function, the module, and the project (layering, wasm, undo, save fidelity, a matching automation tool, an honest `parity/` entry). Prove a test fails before the fix by reverting the fix once. - `plan/` is an optional gitignored notebook. Do not ask for it and do not stop when it is absent. Public status is [`ROADMAP.md`](ROADMAP.md), [`docs/gaps.md`](docs/gaps.md) and [`docs/roadmap.md`](docs/roadmap.md). "Shipped" in `parity/` means exists and tested, not parity with Acrobat. - `cargo run -p pdfcraft -- ` opens the app. `pdfcraft-cli run` drives the engine headlessly. `cargo xtask fuzz` findings stay in `fuzz-out/` and become synthetic tests. For UI work, look at a screenshot from `--control` or the `shot` example. From 1d71d6039f43a645ec9a1b711d6f080f3e5b6cca Mon Sep 17 00:00:00 2001 From: eliets2 Date: Fri, 6 Nov 2026 14:16:36 +0200 Subject: [PATCH 143/198] Add PCX image support to Create PDF conversion (#6396) * Add PCX image support to Create PDF conversion PCX is listed as an accepted Create source elsewhere but PdfCraft rejects it, so users cannot convert PCX images to PDF. Detect PCX by its header in is_image and decode it in embed, adding pcx to CONVERTIBLE so existing from_images and source_kind paths handle it. Test: pcx_images_become_pages failed before the change and passes after. * Create: PCX scanlines hold every plane's row, 8-bit indexed images, older versions, standard test files Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: eliets2 <287273517+eliets2@users.noreply.github.com> Co-authored-by: FredApps <153173854+FredApps@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 --- crates/create/src/lib.rs | 134 ++++++++++++++++++++++++++++++++++++- crates/create/src/tests.rs | 76 +++++++++++++++++++++ 2 files changed, 209 insertions(+), 1 deletion(-) diff --git a/crates/create/src/lib.rs b/crates/create/src/lib.rs index 53a587728f..149bd99d97 100644 --- a/crates/create/src/lib.rs +++ b/crates/create/src/lib.rs @@ -516,6 +516,129 @@ fn decoded(name: &str, bytes: &[u8], format: image::ImageFormat) -> Result Option { + b.get(at..at + 2).map(|x| u16::from_le_bytes([x[0], x[1]])) +} + +/// How a PCX image holds its pixels: bits per plane and planes. +#[derive(Clone, Copy, PartialEq)] +enum PcxLayout { + /// One byte per pixel per colour: each scanline has a red, a green and a blue row. + Rgb, + /// One byte per pixel, an index into the palette after the image (grey without one). + Indexed, + /// One bit per pixel in each of 1 to 4 planes, an index into the header's palette. + Bits(usize), +} + +impl PcxLayout { + fn of(bits: u8, planes: usize) -> Option { + match (bits, planes) { + (8, 3) => Some(PcxLayout::Rgb), + (8, 1) => Some(PcxLayout::Indexed), + (1, 1..=4) => Some(PcxLayout::Bits(planes)), + _ => None, + } + } +} + +/// The palette of a PCX image, and where its run-length coded scanlines end. An 8-bit image ends +/// with a 0x0C byte and 256 colours; a one-plane bit image is black and white whatever its header +/// holds; other bit images use the 16 colours at byte 16. +fn pcx_palette(bytes: &[u8], layout: PcxLayout) -> (Vec, usize) { + match layout { + PcxLayout::Rgb => (Vec::new(), bytes.len()), + PcxLayout::Indexed => match bytes.len().checked_sub(769).filter(|at| *at >= 128 && bytes.get(*at) == Some(&0x0C)) { + Some(at) => (bytes.get(at + 1..).unwrap_or_default().to_vec(), at), + None => ((0..=255u8).flat_map(|v| [v, v, v]).collect(), bytes.len()), + }, + PcxLayout::Bits(1) => (vec![0, 0, 0, 255, 255, 255], bytes.len()), + PcxLayout::Bits(_) => (bytes.get(16..64).unwrap_or_default().to_vec(), bytes.len()), + } +} + +/// The `rows` scanlines of `line` bytes each, decoded. A run may cross from one plane's row into the +/// next, but not out of its scanline. +fn pcx_scanlines(mut body: &[u8], rows: usize, line: usize) -> Option> { + let mut out = Vec::new(); + out.try_reserve_exact(rows.checked_mul(line)?).ok()?; + for row in 1..=rows { + let end = row * line; + while out.len() < end { + let (&c, rest) = body.split_first()?; + body = rest; + if c < 0xC0 { + out.push(c); + continue; + } + let (&value, rest) = body.split_first()?; + body = rest; + let count = usize::from(c & 0x3F); + if count > end - out.len() { + return None; + } + out.resize(out.len() + count, value); + } + } + Some(out) +} + +/// The RGB samples of the `w` pixels of scanline `row` (planes `stride` bytes apart). +fn pcx_row(out: &mut Vec, row: &[u8], w: usize, stride: usize, layout: PcxLayout, palette: &[u8]) { + let at = |i: usize| row.get(i).copied().unwrap_or(0); + for x in 0..w { + let index = match layout { + PcxLayout::Rgb => { + out.extend([at(x), at(stride + x), at(2 * stride + x)]); + continue; + } + PcxLayout::Indexed => usize::from(at(x)), + PcxLayout::Bits(planes) => (0..planes).fold(0, |index, k| index | (usize::from(at(k * stride + x / 8) >> (7 - x % 8) & 1) << k)), + }; + out.extend((0..3).map(|c| palette.get(index * 3 + c).copied().unwrap_or(0))); + } +} + +/// A PCX image: a 128-byte header (size at 4-11, resolution at 12 and 14, plane count at 65, bytes +/// per plane row at 66), then run-length coded scanlines, each holding one row of every plane. +fn pcx(name: &str, bytes: &[u8]) -> Result { + let bad = |m: &str| CreateError::Image(name.into(), m.into()); + let header = bytes.get(..128).ok_or_else(|| bad("a damaged PCX header"))?; + let word = |at: usize| le16(header, at).unwrap_or(0); + let byte = |at: usize| header.get(at).copied().unwrap_or(0); + let (bits, planes, stride) = (byte(3), usize::from(byte(65)), usize::from(word(66))); + let (Some(w), Some(h)) = (word(8).checked_sub(word(4)), word(10).checked_sub(word(6))) else { + return Err(bad("a damaged PCX header")); + }; + let (w, h) = (usize::from(w) + 1, usize::from(h) + 1); + if planes == 0 || stride == 0 { + return Err(bad("a damaged PCX header")); + } + let layout = PcxLayout::of(bits, planes).ok_or_else(|| bad("this PCX plane layout is not supported"))?; + if stride < (w * usize::from(bits)).div_ceil(8) { + return Err(bad("a damaged PCX header")); + } + let line = planes * stride; + if line.saturating_mul(h) > MAX_DECODED_BYTES || w.saturating_mul(h).saturating_mul(3) > MAX_DECODED_BYTES { + return Err(bad("unusual PCX image size")); + } + let (palette, end) = pcx_palette(bytes, layout); + let body = bytes.get(128..end).unwrap_or_default(); + let scanlines = pcx_scanlines(body, h, line).ok_or_else(|| bad("a damaged PCX body"))?; + let mut data = Vec::new(); + data.try_reserve_exact(w * h * 3).map_err(|_| bad("not enough memory for this image"))?; + for row in scanlines.chunks_exact(line) { + pcx_row(&mut data, row, w, stride, layout, &palette); + } + let dpi = (f64::from(word(12)), f64::from(word(14))); + let dpi = if dpi.0 > 0.0 && dpi.1 > 0.0 { dpi } else { (72.0, 72.0) }; + let mut d = Dict::new(); + d.set(b"ColorSpace".to_vec(), Object::name("DeviceRGB")); + d.set(b"BitsPerComponent".to_vec(), Object::Int(8)); + let px = (u32::try_from(w).unwrap_or(0), u32::try_from(h).unwrap_or(0)); + Ok(Embedded { dict: d, data, filtered: false, smask: None, icc: None, px, dpi }) +} + /// Largest GIF logical screen, in pixels (4096×4096). const MAX_GIF_PIXELS: u64 = 1 << 24; @@ -623,7 +746,13 @@ pub enum SourceKind { } /// File extensions Create converts to PDF (images and plain text). -pub const CONVERTIBLE: [&str; 12] = ["png", "jpg", "jpeg", "tif", "tiff", "gif", "bmp", "jp2", "j2k", "jpx", "txt", "text"]; +pub const CONVERTIBLE: [&str; 13] = ["png", "jpg", "jpeg", "tif", "tiff", "gif", "bmp", "jp2", "j2k", "jpx", "pcx", "txt", "text"]; + +/// A PCX file: its manufacturer byte, a version (1 was never used), run-length coding and 1 or 8 +/// bits per plane. `pcx` says when the rest of the header isn't one it can read. +fn is_pcx(bytes: &[u8]) -> bool { + bytes.first() == Some(&0x0A) && matches!(bytes.get(1), Some(0 | 2..=5)) && bytes.get(2) == Some(&1) && matches!(bytes.get(3), Some(1 | 8)) +} /// A classic TIFF or a BigTIFF (version 43), in either byte order. fn is_tiff(bytes: &[u8]) -> bool { @@ -640,6 +769,7 @@ fn is_image(bytes: &[u8]) -> bool { || bytes.starts_with(b"\x89PNG\r\n\x1a\n") || is_tiff(bytes) || is_gif(bytes) + || is_pcx(bytes) // JPEG 2000: a JP2 file or a raw codestream. || bytes.starts_with(JP2_SIGNATURE) || bytes.starts_with(&[0xFF, 0x4F, 0xFF, 0x51]) @@ -675,6 +805,8 @@ fn embed(name: &str, bytes: &[u8], wanted: usize) -> Result, Creat Ok(vec![decoded(name, &pdfcraft_xfa::image::bmp_with_pixel_offset(bytes), image::ImageFormat::Bmp)?]) } else if is_gif(bytes) { Ok(vec![decoded(name, bytes, image::ImageFormat::Gif)?]) + } else if is_pcx(bytes) { + Ok(vec![pcx(name, bytes)?]) } else if bytes.starts_with(JP2_SIGNATURE) || bytes.starts_with(&[0xFF, 0x4F, 0xFF, 0x51]) { Ok(vec![jpx(name, bytes)?]) } else { diff --git a/crates/create/src/tests.rs b/crates/create/src/tests.rs index 9e620c19c3..bea96c887d 100644 --- a/crates/create/src/tests.rs +++ b/crates/create/src/tests.rs @@ -485,6 +485,82 @@ fn bigtiff_images_become_pages() { assert_eq!(media(&pages(&doc)[0]), [0.0, 0.0, 2.0, 1.0], "4×2 px at 144 dpi"); } +/// A PCX header for a `w` by `h` image at 72 dpi. +fn pcx_header(version: u8, bits: u8, planes: u8, stride: u16, w: u16, h: u16) -> Vec { + let mut pcx = vec![0u8; 128]; + pcx[..4].copy_from_slice(&[0x0A, version, 1, bits]); + pcx[8..10].copy_from_slice(&(w - 1).to_le_bytes()); + pcx[10..12].copy_from_slice(&(h - 1).to_le_bytes()); + pcx[12..14].copy_from_slice(&72u16.to_le_bytes()); + pcx[14..16].copy_from_slice(&72u16.to_le_bytes()); + pcx[65] = planes; + pcx[66..68].copy_from_slice(&stride.to_le_bytes()); + pcx +} + +/// A 24-bit PCX: each scanline holds a red, a green and a blue row. +#[test] +fn pcx_images_become_pages() { + let mut pcx = pcx_header(5, 8, 3, 2, 2, 2); + pcx.extend_from_slice(&[10, 11, 20, 21, 30, 31]); + pcx.extend_from_slice(&[40, 41, 50, 51, 60, 61]); + let embedded = super::pcx("a.pcx", &pcx).unwrap(); + assert_eq!(embedded.data, [10, 20, 30, 11, 21, 31, 40, 50, 60, 41, 51, 61]); + let doc = reopen(&from_images(&[("a.pcx".into(), pcx)]).unwrap()); + assert_eq!(pages(&doc).len(), 1); + assert_eq!(media(&pages(&doc)[0]), [0.0, 0.0, 2.0, 2.0]); + assert_eq!(image_of(&doc, 0).name(b"ColorSpace"), Some(&b"DeviceRGB"[..])); +} + +#[test] +fn a_pcx_run_may_cross_planes_but_not_scanlines() { + // 0xC4 repeats 7 four times: the red row and half of the green one + let mut pcx = pcx_header(5, 8, 3, 2, 2, 1); + pcx.extend_from_slice(&[0xC4, 7, 8, 9]); + assert_eq!(super::pcx("a.pcx", &pcx).unwrap().data[..], [7, 7, 8, 7, 7, 9]); + let mut pcx = pcx_header(5, 8, 3, 2, 2, 2); + pcx.extend_from_slice(&[0xC8, 7, 1, 2, 3, 4]); + assert!(super::pcx("a.pcx", &pcx).is_err(), "eight bytes in a six-byte scanline"); +} + +#[test] +fn an_8_bit_pcx_uses_its_palette_and_is_grey_without_one() { + let mut pcx = pcx_header(5, 8, 1, 2, 2, 1); + pcx.extend_from_slice(&[1, 2]); + assert_eq!(super::pcx("a.pcx", &pcx).unwrap().data[..], [1, 1, 1, 2, 2, 2], "no palette: grey"); + pcx.push(0x0C); + let mut palette = vec![0u8; 768]; + palette[3..9].copy_from_slice(&[9, 8, 7, 6, 5, 4]); + pcx.extend_from_slice(&palette); + assert_eq!(super::pcx("a.pcx", &pcx).unwrap().data[..], [9, 8, 7, 6, 5, 4]); +} + +#[test] +fn bit_plane_pcx_images_read_their_palette() { + // version 2, one plane: black and white + let mut pcx = pcx_header(2, 1, 1, 2, 2, 1); + pcx.extend_from_slice(&[0b1000_0000, 0]); + assert!(from_images(&[("a.pcx".into(), pcx.clone())]).is_ok(), "an old version is still a PCX"); + assert_eq!(super::pcx("a.pcx", &pcx).unwrap().data[..], [255, 255, 255, 0, 0, 0]); + // two planes: colour 3 is at byte 16 + 9 + let mut pcx = pcx_header(5, 1, 2, 1, 1, 1); + pcx[25..28].copy_from_slice(&[1, 2, 3]); + pcx.extend_from_slice(&[0x80, 0x80]); + assert_eq!(super::pcx("a.pcx", &pcx).unwrap().data[..], [1, 2, 3]); +} + +#[test] +fn damaged_pcx_headers_and_bodies_are_refused() { + let no_planes = pcx_header(5, 8, 0, 2, 2, 1); + let no_stride = pcx_header(5, 8, 3, 0, 2, 1); + let short_stride = pcx_header(5, 8, 3, 1, 2, 1); + let short_body = pcx_header(5, 8, 3, 2, 2, 1); + for bytes in [no_planes, no_stride, short_stride, short_body] { + assert!(super::pcx("a.pcx", &bytes).is_err()); + } + assert!(super::pcx("a.pcx", &[0x0A, 5, 1, 8]).is_err(), "no header"); +} + /// A synthetic ICC profile: a 128-byte header (size, device class, colour space, `acsp`) and /// some payload. Only the header is read. fn icc_profile(space: &[u8; 4]) -> Vec { From dc5cb122db4e79e795e92073340262f5c40d1344 Mon Sep 17 00:00:00 2001 From: eliets2 Date: Fri, 6 Nov 2026 14:23:18 +0200 Subject: [PATCH 144/198] trust certificate key type when CNG algorithm group is unreadable (#6391) * trust certificate key type when CNG algorithm group is unreadable When the algorithm group can't be read from the key store due to a provider bug, the certificate should not be rejected: its SubjectPublicKeyInfo already states the key type, and a real mismatch will surface at signing time instead of at enumeration. Change matches_key to fall back to Ok(()) when key.algorithm_group() returns an error, trusting the certificate's key type; extract the group comparison into key_group_matches(group: Option<&AlgorithmGroup>, want: StoreKey) so the fallback and mismatch cases can be unit tested without a live CNG key. Test: key_group_matches_trusts_when_group_unreadable failed before the change and passes after. * Sign: back to LF line endings, shorter test comment Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: eliets2 <287273517+eliets2@users.noreply.github.com> Co-authored-by: FredApps <153173854+FredApps@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 --- crates/sign/src/windows.rs | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/crates/sign/src/windows.rs b/crates/sign/src/windows.rs index 120fea3dab..61620ad384 100644 --- a/crates/sign/src/windows.rs +++ b/crates/sign/src/windows.rs @@ -178,7 +178,27 @@ fn entries() -> Result { /// Whether the store's key is what the certificate needs (RSA; ECDSA on its curve). fn matches_key(key: &NCryptKey, want: StoreKey) -> Result<(), String> { - let group = key.algorithm_group().map_err(|e| format!("its private key's algorithm couldn't be read: {}", describe(&e)))?; + // Some KSPs (Certisign's desktopID, for one) report NCRYPT_ALGORITHM_GROUP_PROPERTY + // as a character count instead of a byte count, so the group can't be read even though + // the key works. The certificate already names the key type, so an unreadable group is + // trusted: a working key is accepted instead of wrongly rejected. The key size, when it + // can be read, still has to agree with the certificate. + matches_key_from_group(key.algorithm_group().ok(), key, want) +} + +/// Core check, decoupled from the NCryptKey handle so the algorithm group can be injected +/// (e.g. forced to `None` to simulate a KSP that reports the property as a character count +/// instead of a byte count, as Certisign's desktopID KSP does). +fn matches_key_from_group(group: Option, key: &NCryptKey, want: StoreKey) -> Result<(), String> { + let Some(group) = group else { + return match want { + StoreKey::Rsa => Ok(()), + StoreKey::Ecdsa(bits) => match key.bits() { + Ok(actual) if actual != bits => Err(format!("its certificate holds a P-{bits} key but the store's private key has {actual} bits")), + _ => Ok(()), + }, + }; + }; let name = |g: &AlgorithmGroup| match g { AlgorithmGroup::Rsa => "an RSA".to_string(), AlgorithmGroup::Ecdsa => "an ECDSA".to_string(), @@ -272,4 +292,13 @@ mod tests { assert_eq!(describe(&CngError::WindowsError(0x57)), "Windows error 0x00000057"); assert_eq!(describe(&CngError::InvalidHashLength), "Invalid hash length"); } + + #[test] + fn key_group_matches_trusts_when_group_unreadable() { + // Certisign's desktopID KSP reports the algorithm-group property as a character + // count (3 for "RSA") instead of a byte count (8), so `algorithm_group` fails. An + // unreadable group must not make a usable key look unusable. + let key = NCryptKey::new_owned(0); + assert_eq!(matches_key(&key, StoreKey::Rsa), Ok(())); + } } From 7e8551e4bcacef6ddb8cbf76d97d0e5afee7635a Mon Sep 17 00:00:00 2001 From: Zinadin Zidan Date: Sat, 7 Nov 2026 01:01:33 +0600 Subject: [PATCH 145/198] Add text: gray or CMYK colour, painted with g or k (#6440) page_add_text and content_update took only RGB, so text added to a CMYK print file painted 0 0 0 rg, rich black on press. They now also take gray (0 to 1) or cmyk ([c, m, y, k]), painted with g or k and kept in /PCAdded as /Gray or /CMYK beside /Color, which holds the matching look. An ink applies only while the colour still shows its look: picking another colour, also in the Format panel, drops it for good. RGB text is written byte for byte as before, and a stored ink that doesn't match its colour is ignored. --- crates/automation/README.md | 5 + crates/automation/src/content.rs | 41 +++++++- crates/automation/src/tools.rs | 10 +- crates/automation/tests/it/automation.rs | 53 ++++++++++ crates/edit/src/added.rs | 2 +- crates/edit/src/added/draw.rs | 14 ++- crates/edit/src/added/store.rs | 24 ++++- crates/edit/src/added/types.rs | 89 ++++++++++++++++- crates/edit/src/lib.rs | 4 +- crates/edit/src/tests.rs | 101 ++++++++++++++++++++ crates/engine/src/lib.rs | 3 +- crates/engine/src/tests.rs | 3 + crates/ui-egui/src/editing/tests.rs | 63 ++++++++++++ crates/ui-egui/src/panels/edit.rs | 6 +- crates/ui-egui/src/panels/format_text_ui.rs | 3 +- parity/acrobat-features.toml | 4 +- 16 files changed, 400 insertions(+), 25 deletions(-) diff --git a/crates/automation/README.md b/crates/automation/README.md index db5b988394..2fbbff8ccb 100644 --- a/crates/automation/README.md +++ b/crates/automation/README.md @@ -26,6 +26,11 @@ its undo/redo history. `page_add_text`, `content_update` and `text_edit` take per-line `aligns`. An `"inherit"` entry follows the box-level `align`; legacy `null` entries remain accepted with the same meaning. +`page_add_text` and `content_update` take one of `color` (`#RRGGBB` or a name, painted in +DeviceRGB), `gray` (0 black to 1 white, DeviceGray `g`) and `cmyk` (`[c, m, y, k]`, each 0 to 1, +DeviceCMYK `k`), so text added to a print file can be 100 K black. `content_list` reports `gray` or +`cmyk` for such text. + `Content` is `Json(Value)` or `Png { data, width, height }`. Errors are `UnknownTool`, `InvalidArgs` (the call didn't match the schema) or `Failed` (a readable message for the agent). ## Conventions diff --git a/crates/automation/src/content.rs b/crates/automation/src/content.rs index 6b25c31f61..1f26dcc26a 100644 --- a/crates/automation/src/content.rs +++ b/crates/automation/src/content.rs @@ -2,7 +2,7 @@ //! move/resize/retype/restyle it, delete it. Geometry is in points from the top-left of the //! displayed page (y down), like every other tool. -use pdfcraft_engine::{ADDED_CHAR_SPACING_LIMIT, AddedContent, AddedText, Edit, FontFamily, TextAlign}; +use pdfcraft_engine::{ADDED_CHAR_SPACING_LIMIT, AddedContent, AddedText, Edit, FontFamily, TextAlign, TextInk}; use serde_json::{Value, json}; use crate::comments::parse_color; @@ -30,6 +30,27 @@ fn rect_arg(a: &Args, key: &str) -> Result> { .transpose() } +const INK_ARGS: &str = "give one of color (#RRGGBB or a name), gray (0 black to 1 white) and cmyk ([c, m, y, k], each 0 to 1)"; + +/// `gray` or `cmyk`: the text painted in DeviceGray or DeviceCMYK rather than RGB. At most one of +/// them and `color`. +fn ink_arg(a: &Args) -> Result> { + if ["color", "gray", "cmyk"].iter().filter(|k| a.get(k).is_some()).count() > 1 { + return Err(bad(INK_ARGS)); + } + let unit = |v: Option, key: &str| v.filter(|v| (0.0..=1.0).contains(v)).ok_or_else(|| bad(format!("{key} takes numbers from 0 to 1"))); + if let Some(g) = a.get("gray") { + return Ok(Some(TextInk::Gray(unit(g.as_f64(), "gray")?))); + } + let Some(c) = a.get("cmyk") else { return Ok(None) }; + let parts = c.as_array().filter(|p| p.len() == 4).ok_or_else(|| bad("cmyk must be [c, m, y, k]"))?; + let mut cmyk = [0.0; 4]; + for (out, v) in cmyk.iter_mut().zip(parts) { + *out = unit(v.as_f64(), "cmyk")?; + } + Ok(Some(TextInk::Cmyk(cmyk))) +} + /// Apply the style arguments to `t`. fn style(a: &Args, t: &mut AddedText) -> Result<()> { if let Some(s) = a.opt_num("size")? { @@ -49,8 +70,12 @@ fn style(a: &Args, t: &mut AddedText) -> Result<()> { if let Some(i) = a.opt_bool("italic")? { t.italic = i; } - if let Some(c) = a.opt_str("color")? { - t.color = parse_color(c)?; + if let Some(ink) = ink_arg(a)? { + t.set_ink(Some(ink)); + } else if let Some(c) = a.opt_str("color")? { + // An RGB colour paints in RGB, even one that is the ink's look. + t.set_ink(None); + t.set_color(parse_color(c)?); } if let Some(al) = a.opt_str("align")? { t.align = parse_align(al)?; @@ -124,6 +149,12 @@ impl Automation { v["bold"] = json!(t.bold); v["italic"] = json!(t.italic); v["char_spacing"] = json!(t.char_spacing); + // Text painted in a print colour space says so; RGB text has neither. + match t.painted_ink() { + Some(TextInk::Gray(g)) => v["gray"] = json!(g), + Some(TextInk::Cmyk(c)) => v["cmyk"] = json!(c), + None => {} + } } AddedContent::Image(_) => v["type"] = json!("image"), } @@ -205,8 +236,8 @@ impl Automation { if a.get("text").is_some() { return Err(bad("images have no text")); } - if ["font", "size", "bold", "italic", "color", "align", "aligns", "char_spacing"].iter().any(|k| a.get(k).is_some()) { - return Err(bad("font, size, bold, italic, color, align, aligns and char_spacing apply to text")); + if ["font", "size", "bold", "italic", "color", "gray", "cmyk", "align", "aligns", "char_spacing"].iter().any(|k| a.get(k).is_some()) { + return Err(bad("font, size, bold, italic, color, gray, cmyk, align, aligns and char_spacing apply to text")); } if let Some(deg) = a.opt_int("rotate")? { if deg.rem_euclid(90) != 0 { diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index af5146377a..b0954ede5e 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -718,13 +718,13 @@ pub fn tools() -> Vec { }), &["doc"], )), - t("content_list", "List added content", "Text and images added with page_add_text/page_add_image (or Edit a PDF ▸ Add content), per page with a 1-based index, rect (points from the top-left of the page), and text style.") + t("content_list", "List added content", "Text and images added with page_add_text/page_add_image (or Edit a PDF ▸ Add content), per page with a 1-based index, rect (points from the top-left of the page), and text style; text painted in gray or CMYK also has gray (its level) or cmyk ([c, m, y, k]).") .ro() .with(schema(json!({ "doc": doc() }), &["doc"])), t( "page_add_text", "Add text to a page", - "Add text as page content (not a comment). Place it with at [x, y] (top-left, points from the top-left of the displayed page) and width (wrap width, default 200), or rect. Newlines start new lines; long lines wrap. Style: font helvetica/times/courier, size, bold, italic, color (#RRGGBB or a name), align left/center/right, char_spacing (points after each character). Text must be Western European (WinAnsi) characters, or Arabic, Hebrew or Thai when the build has their craft-fonts faces (drawn right to left where the text is); others are refused. It stays editable with content_update. Undoable.", + "Add text as page content (not a comment). Place it with at [x, y] (top-left, points from the top-left of the displayed page) and width (wrap width, default 200), or rect. Newlines start new lines; long lines wrap. Style: font helvetica/times/courier, size, bold, italic, color (#RRGGBB or a name, painted in RGB) or, for print, gray (0 black to 1 white, painted in DeviceGray) or cmyk ([c, m, y, k], each 0 to 1, painted in DeviceCMYK; [0, 0, 0, 1] is 100 K black), one of the three, align left/center/right, char_spacing (points after each character). Text must be Western European (WinAnsi) characters, or Arabic, Hebrew or Thai when the build has their craft-fonts faces (drawn right to left where the text is); others are refused. It stays editable with content_update. Undoable.", ) .cmd("edit.text") .with(schema( @@ -735,6 +735,8 @@ pub fn tools() -> Vec { "rect": { "type": "array", "items": { "type": "number" }, "minItems": 4, "maxItems": 4 }, "font": { "type": "string", "enum": ["helvetica", "times", "courier"] }, "size": { "type": "number", "minimum": 1, "maximum": 500 }, "bold": { "type": "boolean" }, "italic": { "type": "boolean" }, "color": { "type": "string" }, + "gray": { "type": "number", "minimum": 0, "maximum": 1, "description": "DeviceGray level, 0 black to 1 white (instead of color)." }, + "cmyk": { "type": "array", "items": { "type": "number", "minimum": 0, "maximum": 1 }, "minItems": 4, "maxItems": 4, "description": "DeviceCMYK [c, m, y, k], each 0 to 1 (instead of color); [0, 0, 0, 1] is 100 K black." }, "align": { "type": "string", "enum": ["left", "center", "right", "justify"] }, "aligns": { "type": "array", "maxItems": 4096, "items": { "type": "string", "enum": ["left", "center", "right", "justify", "inherit"] }, "description": "Each paragraph's (line's) own alignment, by index; inherit (or legacy null) follows align." }, "char_spacing": { "type": "number", "minimum": -500, "maximum": 500, "description": "Points after each character." }, @@ -751,13 +753,15 @@ pub fn tools() -> Vec { json!({ "doc": doc(), "page": { "type": "integer", "minimum": 1 }, "path": { "type": "string" }, "rect": { "type": "array", "items": { "type": "number" }, "minItems": 4, "maxItems": 4 } }), &["doc", "page", "path"], )), - t("content_update", "Edit added content", "Move/resize (rect), retype (text) or restyle (font, size, bold, italic, color, align, char_spacing in points) an added item (page, index from content_list). Images: rotate (degrees, multiple of 90, counter-clockwise), flip_h / flip_v (toggle), crop [left, bottom, right, top] as fractions trimmed, image (a file that replaces the picture). Undoable.") + t("content_update", "Edit added content", "Move/resize (rect), retype (text) or restyle (font, size, bold, italic, color or gray or cmyk as page_add_text takes them, align, char_spacing in points) an added item (page, index from content_list). Images: rotate (degrees, multiple of 90, counter-clockwise), flip_h / flip_v (toggle), crop [left, bottom, right, top] as fractions trimmed, image (a file that replaces the picture). Undoable.") .with(schema( json!({ "doc": doc(), "page": { "type": "integer", "minimum": 1 }, "index": { "type": "integer", "minimum": 1 }, "rect": { "type": "array", "items": { "type": "number" }, "minItems": 4, "maxItems": 4 }, "text": { "type": "string" }, "font": { "type": "string", "enum": ["helvetica", "times", "courier"] }, "size": { "type": "number", "minimum": 1, "maximum": 500 }, "bold": { "type": "boolean" }, "italic": { "type": "boolean" }, "color": { "type": "string" }, + "gray": { "type": "number", "minimum": 0, "maximum": 1, "description": "DeviceGray level, 0 black to 1 white (instead of color)." }, + "cmyk": { "type": "array", "items": { "type": "number", "minimum": 0, "maximum": 1 }, "minItems": 4, "maxItems": 4, "description": "DeviceCMYK [c, m, y, k], each 0 to 1 (instead of color); [0, 0, 0, 1] is 100 K black." }, "align": { "type": "string", "enum": ["left", "center", "right", "justify"] }, "aligns": { "type": "array", "maxItems": 4096, "items": { "type": "string", "enum": ["left", "center", "right", "justify", "inherit"] }, "description": "Each paragraph's (line's) own alignment, by index; inherit (or legacy null) follows align." }, "char_spacing": { "type": "number", "minimum": -500, "maximum": 500, "description": "Points after each character." }, diff --git a/crates/automation/tests/it/automation.rs b/crates/automation/tests/it/automation.rs index 96649c8e80..b2bf8ba25d 100644 --- a/crates/automation/tests/it/automation.rs +++ b/crates/automation/tests/it/automation.rs @@ -5214,6 +5214,59 @@ fn adding_content_through_tools() { assert!(matches!(a.call("content_delete", &json!({ "doc": doc, "page": 1, "index": 5 })), Err(ToolError::Failed(_)))); } +/// #6440: `page_add_text` paints text in CMYK or gray for a print file, so its black is 100 K +/// (`0 0 0 1 k`) rather than DeviceRGB; `content_update` restyles it either way. +#[test] +fn added_text_takes_a_cmyk_or_gray_ink_through_tools() { + let dir = workdir("content-ink"); + let mut a = auto(&dir); + let doc = ok(&mut a, "doc_open", json!({ "path": "a.pdf" }))["doc"].as_u64().unwrap(); + ok(&mut a, "page_add_text", json!({ "doc": doc, "page": 1, "text": "PROOF", "at": [20, 20], "cmyk": [0, 0, 0, 1] })); + ok(&mut a, "page_add_text", json!({ "doc": doc, "page": 1, "text": "Note", "at": [20, 80], "gray": 0.5 })); + let items = ok(&mut a, "content_list", json!({ "doc": doc }))["items"].clone(); + assert_eq!((&items[0]["cmyk"], &items[1]["gray"]), (&json!([0.0, 0.0, 0.0, 1.0]), &json!(0.5))); + // The item streams of the saved file. + let saved_items = |a: &mut Automation, name: &str| -> Vec { + ok(a, "doc_save", json!({ "doc": doc, "path": name })); + let saved = pdfcraft_cos::Document::open(std::sync::Arc::new(std::fs::read(dir.join(name)).unwrap())).unwrap(); + let page = &pdfcraft_model::pages(&saved)[0]; + let list = saved.resolve(page.dict.get(b"Contents").unwrap()).as_array().cloned().unwrap(); + list.iter() + .filter_map(|c| match &*saved.resolve(c) { + pdfcraft_cos::Object::Stream(s) if s.dict.name(b"PCMark") == Some(&b"Added"[..]) => { + Some(String::from_utf8_lossy(&s.decoded().unwrap()).into_owned()) + } + _ => None, + }) + .collect() + }; + let streams = saved_items(&mut a, "ink.pdf"); + assert!(streams[0].contains("0 0 0 1 k") && !streams[0].contains("rg"), "{}", streams[0]); + assert!(streams[1].contains("0.5 g") && !streams[1].contains("rg"), "{}", streams[1]); + // Restyled: to RGB with color, and back to CMYK. + ok(&mut a, "content_update", json!({ "doc": doc, "page": 1, "index": 2, "color": "#000000" })); + ok(&mut a, "content_update", json!({ "doc": doc, "page": 1, "index": 1, "cmyk": [1, 0, 0, 0] })); + let items = ok(&mut a, "content_list", json!({ "doc": doc }))["items"].clone(); + assert_eq!((&items[0]["cmyk"], items[1].get("gray")), (&json!([1.0, 0.0, 0.0, 0.0]), None)); + let streams = saved_items(&mut a, "restyled.pdf"); + assert!(streams[0].contains("1 0 0 0 k") && streams[1].contains("0 0 0 rg"), "{streams:?}"); + // One colour at a time, four numbers from 0 to 1. + for bad in [ + json!({ "color": "red", "cmyk": [0, 0, 0, 1] }), + json!({ "gray": 0.5, "cmyk": [0, 0, 0, 1] }), + json!({ "gray": 1.5 }), + json!({ "cmyk": [0, 0, 1] }), + json!({ "cmyk": [0, 0, 0, 1, 0] }), + json!({ "cmyk": [0, "x", 0, 1] }), + json!({ "cmyk": [0, 0, 0, -0.1] }), + ] { + let mut args = json!({ "doc": doc, "page": 1, "text": "x", "at": [20, 200] }); + args.as_object_mut().unwrap().extend(bad.as_object().unwrap().clone()); + assert!(matches!(a.call("page_add_text", &args), Err(ToolError::InvalidArgs(_))), "{bad}"); + } + assert_eq!(ok(&mut a, "content_list", json!({ "doc": doc }))["count"], 2, "nothing refused was added"); +} + /// Edit ▸ Copy and Paste headlessly (#2636): an added text box or image, as it is, onto its own /// page beside it, onto a point, and into another document. #[test] diff --git a/crates/edit/src/added.rs b/crates/edit/src/added.rs index a603a37060..0fc2aa7359 100644 --- a/crates/edit/src/added.rs +++ b/crates/edit/src/added.rs @@ -31,7 +31,7 @@ use layout::Piece; pub(crate) use layout::WRAP_CALLS; pub use layout::{PASTE_STEP, first_undrawable, lines, paste_place, paste_rect, text_rect, text_width}; pub use store::{add_content, delete_content, isolate_content, list_added, scan_markers, update_content}; -pub use types::{Added, AddedImage, AddedText, Align, CHAR_SPACING_LIMIT, Family, MarkerScan}; +pub use types::{Added, AddedImage, AddedText, Align, CHAR_SPACING_LIMIT, Family, Ink, MarkerScan}; #[derive(Clone, Debug, PartialEq)] pub enum Content { diff --git a/crates/edit/src/added/draw.rs b/crates/edit/src/added/draw.rs index 9404f939b1..f3f9b55859 100644 --- a/crates/edit/src/added/draw.rs +++ b/crates/edit/src/added/draw.rs @@ -1,7 +1,7 @@ use super::layout::{ Piece, arabic_error, hebrew_error, is_arabic, is_hebrew, is_thai, latin_width, norm, shaped_layout, text_rect, thai_error, wrapped, }; -use super::types::{AddedText, Align, Family}; +use super::types::{AddedText, Align, Family, Ink}; use super::{ Content, Dict, Document, EditError, GlyphError, GlyphOutline, HashMap, HashSet, ObjRef, Object, PdfString, ShapedCluster, ShapedScript, Stream, align_code, literal, matrix_number, n, shaped_glyph, win_ansi, @@ -105,8 +105,7 @@ pub(super) fn draw_laid_shaped( fonts.set(name.clone().into_bytes(), Object::Ref(font)); names.push(name); } - let [cr, cg, cb] = t.color.map(|v| v.clamp(0.0, 1.0)); - out.extend(format!("BT {} {} {} rg\n", n(cr), n(cg), n(cb)).bytes()); + out.extend(format!("BT {}\n", t.fill_op()).bytes()); if t.char_spacing != 0.0 { out.extend(format!("{} Tc\n", n(t.char_spacing)).bytes()); } @@ -293,8 +292,7 @@ pub(super) fn draw(doc: &mut Document, c: &Content, view: [f64; 6], taken: &Dict } res.set(b"Font".to_vec(), Object::Dict(fonts)); let r = text_rect(t); - let [cr, cg, cb] = t.color.map(|v| v.clamp(0.0, 1.0)); - out.extend(format!("BT /{name} {} Tf {} {} {} rg\n", n(t.size), n(cr), n(cg), n(cb)).bytes()); + out.extend(format!("BT /{name} {} Tf {}\n", n(t.size), t.fill_op()).bytes()); if t.char_spacing != 0.0 { out.extend(format!("{} Tc\n", n(t.char_spacing)).bytes()); } @@ -355,6 +353,12 @@ pub(super) fn params(c: &Content) -> Dict { d.set(b"Font".to_vec(), Object::name(t.family.base_font(t.bold, t.italic))); d.set(b"Size".to_vec(), Object::Real(t.size)); d.set(b"Color".to_vec(), arr(&t.color)); + // The ink beside its RGB look, which a reader that knows no ink keeps showing. + match t.painted_ink() { + Some(Ink::Gray(g)) => d.set(b"Gray".to_vec(), Object::Real(g)), + Some(Ink::Cmyk(c)) => d.set(b"CMYK".to_vec(), arr(&c)), + None => {} + } d.set(b"Align".to_vec(), Object::Int(align_code(t.align))); // Each paragraph's own alignment, or null where it follows /Align. let aligns = crate::paragraph_align::tidy(t.aligns.clone(), t.align); diff --git a/crates/edit/src/added/store.rs b/crates/edit/src/added/store.rs index 7dae55596f..186935b9f2 100644 --- a/crates/edit/src/added/store.rs +++ b/crates/edit/src/added/store.rs @@ -1,8 +1,19 @@ use super::draw::{draw, params}; use super::layout::{first_undrawable, is_thai, nums, shaped_arabic, thai_error}; -use super::types::{Added, AddedImage, AddedText, CHAR_SPACING_LIMIT, Family, MarkerScan, TAG}; +use super::types::{Added, AddedImage, AddedText, CHAR_SPACING_LIMIT, Family, Ink, MarkerScan, TAG}; use super::{Content, Dict, Document, EditError, GlyphError, MarkKind, ObjRef, Object, Stream, align_from, check, contents, page_list, place_tagged}; +/// A text item's gray or CMYK ink. Items written before inks, and an ink out of range, have none: +/// the text paints in its RGB `/Color`. +fn ink(doc: &Document, d: &Dict) -> Option { + let ink = match (d.get(b"Gray").map(|g| doc.resolve(g)).and_then(|g| g.as_f64()), d.get(b"CMYK")) { + (Some(g), _) => Ink::Gray(g), + (None, Some(c)) => Ink::Cmyk(nums(doc, Some(c)).try_into().ok()?), + (None, None) => return None, + }; + Some(ink).filter(|i| i.valid()) +} + fn parse(doc: &Document, d: &Dict) -> Option { let r = nums(doc, d.get(b"Rect")); let rect: [f64; 4] = r.try_into().ok()?; @@ -21,7 +32,7 @@ fn parse(doc: &Document, d: &Dict) -> Option { .map(|a| a.iter().take(crate::paragraph_align::count(&text)).map(|v| v.as_int().map(|c| align_from(Some(c)))).collect()) }) .unwrap_or_default(); - Some(Content::Text(AddedText { + let mut t = AddedText { rect, text, family, @@ -29,11 +40,15 @@ fn parse(doc: &Document, d: &Dict) -> Option { italic, size: d.get(b"Size").and_then(Object::as_f64).unwrap_or(12.0), color: if c.len() == 3 { [c[0], c[1], c[2]] } else { [0.0; 3] }, + ink: ink(doc, d), align, aligns: crate::paragraph_align::tidy(aligns, align), // Items written before character spacing have none. char_spacing: d.get(b"CharSpacing").and_then(Object::as_f64).filter(|v| v.is_finite()).unwrap_or(0.0), - })) + }; + // An ink whose look isn't the stored colour (a file edited by hand) is none. + t.drop_stale_ink(); + Some(Content::Text(t)) } b"Image" => { let crop = nums(doc, d.get(b"Crop")); @@ -138,6 +153,9 @@ fn validate(c: &Content) -> Result<(), EditError> { "the character spacing must be between -{CHAR_SPACING_LIMIT} and {CHAR_SPACING_LIMIT} points" ))); } + if t.ink.is_some_and(|i| !i.valid()) { + return Err(EditError::Invalid("a gray level and each CMYK value must be from 0 to 1".into())); + } if (r[2] - r[0]).abs() < 1.0 { return Err(EditError::Invalid("the text box is too narrow".into())); } diff --git a/crates/edit/src/added/types.rs b/crates/edit/src/added/types.rs index 0caaaa863b..43480774a3 100644 --- a/crates/edit/src/added/types.rs +++ b/crates/edit/src/added/types.rs @@ -1,5 +1,5 @@ use super::layout::norm; -use super::{Content, MarkKind, ObjRef, Std14}; +use super::{Content, MarkKind, ObjRef, Std14, n}; pub(super) const TAG: &str = "Added"; @@ -90,6 +90,50 @@ pub enum Align { Justify, } +/// A device colour added text is painted in instead of RGB, for print files: a text's black in +/// CMYK stays 100 K rather than a DeviceRGB black the printer separates into rich black. +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum Ink { + /// A DeviceGray level, from 0 (black) to 1 (white): painted with `g`. + Gray(f64), + /// DeviceCMYK `[c, m, y, k]`, each from 0 to 1 (`[0, 0, 0, 1]` is 100 K): painted with `k`. + Cmyk([f64; 4]), +} + +impl Ink { + /// Whether every component is a number from 0 to 1. + pub fn valid(self) -> bool { + self.components().iter().all(|v| v.is_finite() && (0.0..=1.0).contains(v)) + } + + fn components(&self) -> &[f64] { + match self { + Ink::Gray(g) => std::slice::from_ref(g), + Ink::Cmyk(c) => c, + } + } + + /// How the ink looks on screen: gray is the same level in each of red, green and blue + /// (ISO 32000-2 §10.4.2.2), CMYK takes each colour plus black from 1 (§10.4.2.5). + pub fn rgb(self) -> [f64; 3] { + match self { + Ink::Gray(g) => [g.clamp(0.0, 1.0); 3], + Ink::Cmyk([c, m, y, k]) => [c, m, y].map(|v| 1.0 - (v + k).clamp(0.0, 1.0)), + } + } + + /// The operator that makes this ink the fill colour, without a line end. + pub(super) fn fill_op(self) -> String { + match self { + Ink::Gray(g) => format!("{} g", n(g.clamp(0.0, 1.0))), + Ink::Cmyk(c) => { + let [c, m, y, k] = c.map(|v| v.clamp(0.0, 1.0)); + format!("{} {} {} {} k", n(c), n(m), n(y), n(k)) + } + } + } +} + #[derive(Clone, Debug, PartialEq)] pub struct AddedText { /// The text box in display space; its width wraps the text and its top is the first line's @@ -100,7 +144,14 @@ pub struct AddedText { pub bold: bool, pub italic: bool, pub size: f64, + /// The RGB colour: what the text is painted in, unless it has an [`AddedText::ink`] this is + /// the look of. pub color: [f64; 3], + /// A gray or CMYK ink the text is painted in, with `color` its look ([`Ink::rgb`]). Set it + /// with [`AddedText::set_ink`] and the colour with [`AddedText::set_color`], which drops an + /// ink the colour no longer shows; one left behind by setting `color` directly paints nothing + /// ([`AddedText::painted_ink`]) until [`AddedText::drop_stale_ink`] forgets it. + pub ink: Option, pub align: Align, /// Extra space after every character, in points: the PDF's character spacing (`Tc`). pub char_spacing: f64, @@ -114,6 +165,41 @@ impl AddedText { pub fn paragraph_align(&self, i: usize) -> Align { crate::paragraph_align::align_of(&self.aligns, i, self.align) } + + /// Paint the text in `ink` (its look becomes [`AddedText::color`]), or in RGB with `None`. + pub fn set_ink(&mut self, ink: Option) { + if let Some(i) = ink { + self.color = i.rgb(); + } + self.ink = ink; + } + + /// Paint the text in RGB `rgb`. Its ink stays only when `rgb` is the ink's own look; once + /// gone it stays gone, so picking that look again later paints in RGB. + pub fn set_color(&mut self, rgb: [f64; 3]) { + self.color = rgb; + self.drop_stale_ink(); + } + + /// Forget an ink `color` no longer shows, so two styles that paint the same compare equal. + pub fn drop_stale_ink(&mut self) { + self.ink = self.painted_ink(); + } + + /// The ink the text is painted in, or `None` for RGB: the one it has while `color` is still + /// its look. + pub fn painted_ink(&self) -> Option { + // Within rounding: a look read back from a file, or from a colour picker, is the same. + self.ink.filter(|i| i.rgb().iter().zip(self.color).all(|(a, b)| (a - b).abs() < 1e-6)) + } + + /// The operator that sets the text's fill colour, without a line end. + pub(super) fn fill_op(&self) -> String { + match self.painted_ink() { + Some(ink) => ink.fill_op(), + None => crate::marks::rgb(self.color), + } + } } impl Default for AddedText { @@ -126,6 +212,7 @@ impl Default for AddedText { italic: false, size: 12.0, color: [0.0; 3], + ink: None, align: Align::Left, aligns: Vec::new(), char_spacing: 0.0, diff --git a/crates/edit/src/lib.rs b/crates/edit/src/lib.rs index 2a0add8d12..0cf1886afd 100644 --- a/crates/edit/src/lib.rs +++ b/crates/edit/src/lib.rs @@ -61,8 +61,8 @@ pub use text::{ }; pub mod added; pub use added::{ - Added, AddedImage, AddedText, Align, Content, Family, MarkerScan, add_content, delete_content, first_undrawable, isolate_content, list_added, - scan_markers, update_content, + Added, AddedImage, AddedText, Align, Content, Family, Ink, MarkerScan, add_content, delete_content, first_undrawable, isolate_content, + list_added, scan_markers, update_content, }; mod apply; diff --git a/crates/edit/src/tests.rs b/crates/edit/src/tests.rs index 22fbd6e8ae..8aadcef420 100644 --- a/crates/edit/src/tests.rs +++ b/crates/edit/src/tests.rs @@ -1048,6 +1048,107 @@ fn added_text_and_images_are_page_content_that_stays_editable() { assert!(marks_present(&doc).is_empty()); } +/// #6440: added text in a print file can be gray or CMYK, painted with `g` / `k`, so its black +/// stays 100 K. The ink survives save and reopen; another colour drops it for good. +#[test] +fn gray_and_cmyk_added_text_writes_g_and_k_until_it_is_recoloured() { + use crate::added::Ink; + let item_stream = |doc: &Document, i: usize| { + let obj = list_added(doc)[i].obj; + let Object::Stream(s) = &*doc.get(obj) else { panic!("an item is a stream") }; + String::from_utf8_lossy(&s.decoded().unwrap()).into_owned() + }; + let mut doc = fixture(); + let base = AddedText { rect: [72.0, 600.0, 300.0, 700.0], text: "Proof".into(), size: 14.0, ..AddedText::default() }; + let mut black = base.clone(); + black.set_ink(Some(Ink::Cmyk([0.0, 0.0, 0.0, 1.0]))); + assert_eq!(black.color, [0.0; 3], "100 K looks black on screen"); + let mut gray = AddedText { rect: [72.0, 400.0, 300.0, 500.0], ..base.clone() }; + gray.set_ink(Some(Ink::Gray(0.25))); + add_content(&mut doc, 0, &Content::Text(black)).unwrap(); + add_content(&mut doc, 0, &Content::Text(gray)).unwrap(); + add_content(&mut doc, 0, &Content::Text(AddedText { rect: [72.0, 200.0, 300.0, 300.0], ..base.clone() })).unwrap(); + + let doc = reopen(&doc); + let (k, g, rgb) = (item_stream(&doc, 0), item_stream(&doc, 1), item_stream(&doc, 2)); + assert!(k.contains("/PCFHelvetica 14 Tf 0 0 0 1 k\n") && !k.contains("rg"), "{k}"); + assert!(g.contains("/PCFHelvetica 14 Tf 0.25 g\n") && !g.contains("rg"), "{g}"); + assert!(rgb.contains("/PCFHelvetica 14 Tf 0 0 0 rg\n"), "RGB text paints as before: {rgb}"); + let inks: Vec> = list_added(&doc).iter().map(|a| if let Content::Text(t) = &a.content { t.painted_ink() } else { None }).collect(); + assert_eq!(inks, [Some(Ink::Cmyk([0.0, 0.0, 0.0, 1.0])), Some(Ink::Gray(0.25)), None]); + + // Another colour drops the ink, and picking the ink's look again later doesn't bring it back; + // picking that look while the ink is there keeps it. + let Content::Text(cmyk) = list_added(&doc)[0].content.clone() else { panic!() }; + let mut same = cmyk.clone(); + same.set_color([0.0; 3]); + assert_eq!(same, cmyk, "black on 100 K stays 100 K"); + let mut recoloured = cmyk.clone(); + recoloured.set_color([1.0, 0.0, 0.0]); + assert_eq!(recoloured.ink, None); + recoloured.set_color([0.0; 3]); + assert_eq!((recoloured.ink, recoloured.painted_ink()), (None, None), "the ink stays gone"); + + // Set directly, a colour paints in RGB too, and the item reads back without the ink: what a + // draft with its stale ink dropped compares equal to. + let mut doc = doc; + let red = AddedText { color: [1.0, 0.0, 0.0], ..cmyk }; + update_content(&mut doc, 0, 0, &Content::Text(red.clone())).unwrap(); + let stream = item_stream(&doc, 0); + assert!(stream.contains("1 0 0 rg") && !stream.contains(" k\n"), "{stream}"); + let mut settled = red; + settled.drop_stale_ink(); + for doc in [&doc, &reopen(&doc)] { + let Content::Text(back) = list_added(doc)[0].content.clone() else { panic!() }; + // The stored box is the text's own height. + assert_eq!(back, AddedText { rect: back.rect, ..settled.clone() }); + } + + // CMYK looks as ISO 32000-2 §10.4.2.5 maps it to RGB; an ink out of range is refused. + assert_eq!(Ink::Cmyk([0.5, 0.0, 1.0, 0.25]).rgb(), [0.25, 0.75, 0.0]); + let mut wrong = base; + wrong.ink = Some(Ink::Gray(1.5)); + assert!(add_content(&mut doc, 0, &Content::Text(wrong)).is_err()); +} + +/// #6440: a file edited by hand paints its added text in RGB unless the stored gray or CMYK is +/// in range, the right length and still the look of `/Color`; nothing in it panics. +#[test] +fn a_hand_edited_ink_reads_back_only_when_it_is_the_stored_colours_look() { + use crate::added::Ink; + let mut doc = fixture(); + add_content(&mut doc, 0, &Content::Text(AddedText { rect: [72.0, 600.0, 300.0, 700.0], text: "Proof".into(), ..AddedText::default() })).unwrap(); + let obj = list_added(&doc)[0].obj; + let real = |v: &[f64]| Object::Array(v.iter().map(|x| Object::Real(*x)).collect()); + let read = |entries: &[(&[u8], Object)]| { + let mut edited = doc.clone(); + let Object::Stream(mut s) = (*edited.get(obj)).clone() else { panic!("an item is a stream") }; + let mut params = s.dict.get(b"PCAdded").and_then(|p| p.as_dict().cloned()).unwrap(); + for (key, value) in entries { + params.set(key.to_vec(), value.clone()); + } + s.dict.set(b"PCAdded".to_vec(), Object::Dict(params)); + edited.set(obj, Object::Stream(s)); + let Content::Text(t) = list_added(&edited)[0].content.clone() else { panic!() }; + t.ink + }; + // As PDFCraft writes it: the ink stands. + assert_eq!(read(&[(b"Color", real(&[0.5; 3])), (b"Gray", Object::Real(0.5))]), Some(Ink::Gray(0.5))); + assert_eq!(read(&[(b"Color", real(&[0.0; 3])), (b"CMYK", real(&[0.0, 0.0, 0.0, 1.0]))]), Some(Ink::Cmyk([0.0, 0.0, 0.0, 1.0]))); + for (what, entries) in [ + ("a colour the ink isn't the look of", vec![(&b"Color"[..], real(&[1.0, 0.0, 0.0])), (b"CMYK", real(&[0.0, 0.0, 0.0, 1.0]))]), + ("three CMYK numbers", vec![(b"Color", real(&[0.0; 3])), (b"CMYK", real(&[0.0, 0.0, 1.0]))]), + ("five CMYK numbers", vec![(b"Color", real(&[0.0; 3])), (b"CMYK", real(&[0.0, 0.0, 0.0, 1.0, 0.0]))]), + ("CMYK not an array", vec![(b"Color", real(&[0.0; 3])), (b"CMYK", Object::Real(1.0))]), + ("gray out of range", vec![(b"Color", real(&[1.0; 3])), (b"Gray", Object::Real(2.0))]), + ("gray not a number", vec![(b"Color", real(&[0.0; 3])), (b"Gray", Object::name("Black"))]), + ("gray NaN", vec![(b"Color", real(&[0.0; 3])), (b"Gray", Object::Real(f64::NAN))]), + ("CMYK infinite", vec![(b"Color", real(&[0.0; 3])), (b"CMYK", real(&[0.0, 0.0, 0.0, f64::INFINITY]))]), + ] { + assert_eq!(read(&entries), None, "{what}"); + } +} + /// #4464: a page left with one added item draws that item and nothing else. #[test] fn an_isolated_item_is_all_that_is_left_on_its_page() { diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 1c113d0d82..f99d8ff816 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -91,7 +91,8 @@ pub use pdfcraft_edit::added::{ }; pub use pdfcraft_edit::paragraph_align; pub use pdfcraft_edit::{ - Added, AddedImage, AddedText, Align as TextAlign, Background, Content as AddedContent, Family as FontFamily, HeaderFooter, MarkKind, Watermark, + Added, AddedImage, AddedText, Align as TextAlign, Background, Content as AddedContent, Family as FontFamily, HeaderFooter, Ink as TextInk, + MarkKind, Watermark, }; pub use pdfcraft_forms::{ BorderStyle, CalculatedField, CalculationIssue, CalculationOrder, CheckStyle, Field as FormField, FieldAction, FieldChange, FieldFont, diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 5b6fb01b9e..c109a0bc80 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -4182,6 +4182,7 @@ fn pasted_added_content_keeps_its_style_and_picture() { italic: true, size: 12.0, color: [0.8, 0.1, 0.2], + ink: None, align: TextAlign::Center, aligns: Vec::new(), char_spacing: 0.0, @@ -8112,6 +8113,7 @@ fn content_edit_keeps_pages_annotations_fields_and_links() { italic: false, size: 12.0, color: [0.0, 0.0, 0.0], + ink: None, align: TextAlign::Left, aligns: Vec::new(), char_spacing: 0.0, @@ -8153,6 +8155,7 @@ fn content_refresh_lists_match_the_storages_an_edit_touches() { italic: false, size: 12.0, color: [0.0, 0.0, 0.0], + ink: None, align: TextAlign::Left, aligns: Vec::new(), char_spacing: 0.0, diff --git a/crates/ui-egui/src/editing/tests.rs b/crates/ui-egui/src/editing/tests.rs index 76886ff54f..c0e35329c0 100644 --- a/crates/ui-egui/src/editing/tests.rs +++ b/crates/ui-egui/src/editing/tests.rs @@ -391,3 +391,66 @@ fn save_asks_where_to_keep_an_email_attachment_and_leaves_the_temporary_copy_alo assert!(app.save_active(SaveTarget::InPlace)); let _ = std::fs::remove_dir_all(&dir); } + +/// #6440: recolouring gray or CMYK text in its open box drops the ink at once. Finishing the box +/// then adds no empty Undo step (the draft equals the item the page reads back), picking the +/// ink's own look again in the same box paints RGB rather than bringing the ink back, and new +/// boxes don't take the ink. +#[test] +fn recolouring_print_ink_text_in_its_box_drops_the_ink_without_an_empty_step() { + // 100 K text, open in its box. + let open = || { + let mut app = crate::PdfCraftApp::new(); + app.set_option("language", "en").unwrap(); + app.open_bytes("a.pdf", None, one_page_pdf()).unwrap(); + let mut text = pdfcraft_engine::AddedText { rect: [20.0, 200.0, 180.0, 220.0], text: "Proof".into(), ..Default::default() }; + text.set_ink(Some(pdfcraft_engine::TextInk::Cmyk([0.0, 0.0, 0.0, 1.0]))); + assert!(app.apply_edit(pdfcraft_engine::Edit::AddText { page: 0, text })); + let stored = item(&app); + assert!(stored.painted_ink().is_some()); + app.views[0].content.draft = Some(crate::content_ui::TextDraft { + page: 0, + index: Some(0), + rect: stored.rect, + text: stored.text.clone(), + style: stored, + focus: true, + selection: None, + typing_from: None, + }); + app + }; + fn item(app: &crate::PdfCraftApp) -> pdfcraft_engine::AddedText { + match &app.session.get(app.views[0].id).unwrap().added()[0].content { + pdfcraft_engine::AddedContent::Text(t) => t.clone(), + other => panic!("{other:?}"), + } + } + // The box's style with another colour, set as the Format panel hands it over. + let recolour = |app: &mut crate::PdfCraftApp, color: [f64; 3]| { + let style = app.views[0].content.draft.as_ref().map(|d| d.style.clone()).unwrap(); + app.restyle_draft(0, pdfcraft_engine::AddedText { color, ..style }); + }; + let finish = |app: &mut crate::PdfCraftApp| { + let added = app.session.get(app.views[0].id).unwrap().added().to_vec(); + app.views[0].content.finish_draft(&added) + }; + + // Red, then Done. + let mut app = open(); + recolour(&mut app, [1.0, 0.0, 0.0]); + assert_eq!(item(&app).ink, None, "the page paints red in RGB"); + let finished = finish(&mut app); + assert!(finished.is_none(), "an empty Undo step: {finished:?}"); + assert_eq!(app.text_style.ink, None, "new boxes don't take the ink"); + + // Red, then black in the same box: RGB black, not the 100 K back. + let mut app = open(); + recolour(&mut app, [1.0, 0.0, 0.0]); + recolour(&mut app, [0.0; 3]); + let now = item(&app); + assert_eq!((now.ink, now.color), (None, [0.0; 3])); + assert_eq!(app.text_style.ink, None); + let finished = finish(&mut app); + assert!(finished.is_none(), "an empty Undo step: {finished:?}"); +} diff --git a/crates/ui-egui/src/panels/edit.rs b/crates/ui-egui/src/panels/edit.rs index 2c2fb7a09a..883009df8e 100644 --- a/crates/ui-egui/src/panels/edit.rs +++ b/crates/ui-egui/src/panels/edit.rs @@ -406,7 +406,11 @@ impl PdfCraftApp { /// style at once and stays open. Text already on the page is updated with what has been typed, /// as one undoable change; new text waits until it is finished. Text typed after this styling /// starts from it too. - pub(crate) fn restyle_draft(&mut self, index: usize, style: pdfcraft_engine::AddedText) { + pub(crate) fn restyle_draft(&mut self, index: usize, mut style: pdfcraft_engine::AddedText) { + // A gray or CMYK ink the colour no longer shows goes now (#6440): kept, the draft would + // differ from the item the page reads back without it, and a later pick of the ink's + // look would bring it back. + style.drop_stale_ink(); let Some(view) = self.views.get_mut(index) else { return }; // Text the standard fonts can't draw stays in the box, unwritten, until it is corrected (#125). let blocked = view.content.blocked().is_some(); diff --git a/crates/ui-egui/src/panels/format_text_ui.rs b/crates/ui-egui/src/panels/format_text_ui.rs index 99ffc054e1..412563b026 100644 --- a/crates/ui-egui/src/panels/format_text_ui.rs +++ b/crates/ui-egui/src/panels/format_text_ui.rs @@ -266,7 +266,8 @@ pub(crate) fn panel( let swatch = colour_button(ui, t, look.color); egui::Popup::menu(&swatch).close_behavior(egui::PopupCloseBehavior::CloseOnClickOutside).show(|ui| { if let Some(picked) = crate::comments::swatch_grid(ui, Some(look.color)) { - look.color = picked; + // Drops a gray or CMYK ink the new colour isn't the look of (#6440). + look.set_color(picked); ui.close(); } }); diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 10b040e09f..bd7545175d 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -2047,8 +2047,8 @@ milestone = "M7" status = "shipped" commands = ["edit.text"] tools = ["page_add_text", "content_update"] -evidence = ["crates/ui-egui/tests/it/content.rs::typing_moving_styling_and_deleting_added_content", "crates/ui-egui/tests/it/content.rs::clicking_elsewhere_keeps_the_text_and_starts_a_new_box", "crates/automation/tests/it/automation.rs::adding_content_through_tools", "crates/edit/src/tests.rs::added_text_and_images_are_page_content_that_stays_editable", "crates/engine/src/tests.rs::added_text_and_images_render_and_stay_editable", "crates/edit/src/tests.rs::added_arabic_text_is_shaped_in_display_order_and_stays_searchable", "crates/automation/tests/it/automation.rs::thai_text_through_tools_saves_reopens_extracts_and_renders", "crates/ui-egui/tests/it/content.rs::typing_thai_text_keeps_marks_and_commits_to_page", "crates/automation/tests/it/automation.rs::added_arabic_text_keeps_copy_search_and_editor_order_after_retyping_and_reopen", "crates/edit/src/tests.rs::added_hebrew_text_is_drawn_right_to_left_and_stays_searchable", "crates/automation/tests/it/automation.rs::hebrew_text_through_tools_saves_reopens_extracts_and_renders"] -notes = "page content, editable later (retype, move, restyle); standard fonts, Windows-1252 text. Arabic text is shaped with the craft-fonts Arab face (Type 3 outlines with ToUnicode) and laid out in bidirectional display order; per-line ActualText keeps copy, search and editor text in typing order through retyping and save/reopen; it is drawn regular whatever the bold/italic setting, and typing in the box is still left to right (misc.rtl-ui). Hebrew (#128) takes the same path with the craft-fonts Hebr face (Noto Sans Hebrew): right-to-left display order, points and cantillation kept on their letter as one glyph, ToUnicode and per-line ActualText; regular style only, and right-to-left scripts other than Arabic and Hebrew are not covered. Thai Add Text uses the craft-fonts Thai face, with shaped clusters and ToUnicode; Thai is regular style, wrapping preserves clusters but is not dictionary-based, and forms/Fill and Sign are not covered by this change. Each click on the page keeps the box being typed and starts a new one; Done and Discard beside the box" +evidence = ["crates/ui-egui/tests/it/content.rs::typing_moving_styling_and_deleting_added_content", "crates/ui-egui/tests/it/content.rs::clicking_elsewhere_keeps_the_text_and_starts_a_new_box", "crates/automation/tests/it/automation.rs::adding_content_through_tools", "crates/edit/src/tests.rs::added_text_and_images_are_page_content_that_stays_editable", "crates/engine/src/tests.rs::added_text_and_images_render_and_stay_editable", "crates/edit/src/tests.rs::added_arabic_text_is_shaped_in_display_order_and_stays_searchable", "crates/automation/tests/it/automation.rs::thai_text_through_tools_saves_reopens_extracts_and_renders", "crates/ui-egui/tests/it/content.rs::typing_thai_text_keeps_marks_and_commits_to_page", "crates/automation/tests/it/automation.rs::added_arabic_text_keeps_copy_search_and_editor_order_after_retyping_and_reopen", "crates/edit/src/tests.rs::added_hebrew_text_is_drawn_right_to_left_and_stays_searchable", "crates/automation/tests/it/automation.rs::hebrew_text_through_tools_saves_reopens_extracts_and_renders", "crates/edit/src/tests.rs::gray_and_cmyk_added_text_writes_g_and_k_until_it_is_recoloured", "crates/automation/tests/it/automation.rs::added_text_takes_a_cmyk_or_gray_ink_through_tools", "crates/ui-egui/src/editing/tests.rs::recolouring_print_ink_text_in_its_box_drops_the_ink_without_an_empty_step"] +notes = "page content, editable later (retype, move, restyle); standard fonts, Windows-1252 text. The colour is RGB, or for print files gray or CMYK (#6440: page_add_text / content_update gray or cmyk, painted with g / k so black can be 100 K, kept through save and reopen; choosing another colour later, also from the Format panel, drops the ink for good and paints in RGB, while choosing the colour the ink already shows (black on 100 K) keeps it; content_update color always paints in RGB). Arabic text is shaped with the craft-fonts Arab face (Type 3 outlines with ToUnicode) and laid out in bidirectional display order; per-line ActualText keeps copy, search and editor text in typing order through retyping and save/reopen; it is drawn regular whatever the bold/italic setting, and typing in the box is still left to right (misc.rtl-ui). Hebrew (#128) takes the same path with the craft-fonts Hebr face (Noto Sans Hebrew): right-to-left display order, points and cantillation kept on their letter as one glyph, ToUnicode and per-line ActualText; regular style only, and right-to-left scripts other than Arabic and Hebrew are not covered. Thai Add Text uses the craft-fonts Thai face, with shaped clusters and ToUnicode; Thai is regular style, wrapping preserves clusters but is not dictionary-based, and forms/Fill and Sign are not covered by this change. Each click on the page keeps the box being typed and starts a new one; Done and Discard beside the box" [[feature]] id = "edit.find-replace" From c44896df952116be25208d0be3d98ea22889fbcd Mon Sep 17 00:00:00 2001 From: Zinadin Zidan Date: Sat, 7 Nov 2026 02:08:46 +0600 Subject: [PATCH 146/198] Add text: an ink is read only from numbers and kept as painted (review) A stored /CMYK with a non-numeric entry was read from its numbers alone, so [0 /Bad 0 0 1] became 100 K; a /Gray that isn't a number with a /CMYK beside it fell through to the CMYK. Both now give no ink, and the text paints in its /Color. g and k are written to the thousandth, as rg is, so an ink set or read with finer digits painted other than what the item, its stored /Gray or /CMYK and content_list held. The ink is now kept as painted, with its look, when set and when read back. RGB text is unchanged. --- crates/edit/src/added/store.rs | 19 ++++++++---- crates/edit/src/added/types.rs | 15 +++++++++- crates/edit/src/tests.rs | 53 ++++++++++++++++++++++++++++++++-- 3 files changed, 78 insertions(+), 9 deletions(-) diff --git a/crates/edit/src/added/store.rs b/crates/edit/src/added/store.rs index 186935b9f2..df0b697ff9 100644 --- a/crates/edit/src/added/store.rs +++ b/crates/edit/src/added/store.rs @@ -3,13 +3,16 @@ use super::layout::{first_undrawable, is_thai, nums, shaped_arabic, thai_error}; use super::types::{Added, AddedImage, AddedText, CHAR_SPACING_LIMIT, Family, Ink, MarkerScan, TAG}; use super::{Content, Dict, Document, EditError, GlyphError, MarkKind, ObjRef, Object, Stream, align_from, check, contents, page_list, place_tagged}; -/// A text item's gray or CMYK ink. Items written before inks, and an ink out of range, have none: -/// the text paints in its RGB `/Color`. +/// A text item's gray or CMYK ink. Items written before inks, and an ink out of range or not +/// made only of numbers (a `/Gray` name, a `/CMYK` entry that isn't a number), have none: the +/// text paints in its RGB `/Color`. fn ink(doc: &Document, d: &Dict) -> Option { - let ink = match (d.get(b"Gray").map(|g| doc.resolve(g)).and_then(|g| g.as_f64()), d.get(b"CMYK")) { - (Some(g), _) => Ink::Gray(g), - (None, Some(c)) => Ink::Cmyk(nums(doc, Some(c)).try_into().ok()?), - (None, None) => return None, + let ink = if let Some(g) = d.get(b"Gray") { + Ink::Gray(doc.resolve(g).as_f64()?) + } else { + let cmyk = doc.resolve(d.get(b"CMYK")?); + let parts: Vec = cmyk.as_array()?.iter().map(Object::as_f64).collect::>()?; + Ink::Cmyk(parts.try_into().ok()?) }; Some(ink).filter(|i| i.valid()) } @@ -48,6 +51,10 @@ fn parse(doc: &Document, d: &Dict) -> Option { }; // An ink whose look isn't the stored colour (a file edited by hand) is none. t.drop_stale_ink(); + // One that stands is the ink as painted, finer digits dropped, and its look. + if let Some(i) = t.ink { + t.set_ink(Some(i)); + } Some(Content::Text(t)) } b"Image" => { diff --git a/crates/edit/src/added/types.rs b/crates/edit/src/added/types.rs index 43480774a3..24101b2eea 100644 --- a/crates/edit/src/added/types.rs +++ b/crates/edit/src/added/types.rs @@ -106,6 +106,16 @@ impl Ink { self.components().iter().all(|v| v.is_finite() && (0.0..=1.0).contains(v)) } + /// The ink as its operator paints it: each component to the thousandth that `g` and `k` are + /// written with (as RGB's `rg`), so the item, what it reads back as and the page agree. + pub fn painted(self) -> Ink { + let round = |v: f64| (v * 1000.0).round() / 1000.0; + match self { + Ink::Gray(g) => Ink::Gray(round(g)), + Ink::Cmyk(c) => Ink::Cmyk(c.map(round)), + } + } + fn components(&self) -> &[f64] { match self { Ink::Gray(g) => std::slice::from_ref(g), @@ -166,8 +176,11 @@ impl AddedText { crate::paragraph_align::align_of(&self.aligns, i, self.align) } - /// Paint the text in `ink` (its look becomes [`AddedText::color`]), or in RGB with `None`. + /// Paint the text in `ink` (its look becomes [`AddedText::color`]), or in RGB with `None`. An + /// ink in range is kept as painted ([`Ink::painted`]); one out of range stays as given, for + /// saving to refuse. pub fn set_ink(&mut self, ink: Option) { + let ink = ink.map(|i| if i.valid() { i.painted() } else { i }); if let Some(i) = ink { self.color = i.rgb(); } diff --git a/crates/edit/src/tests.rs b/crates/edit/src/tests.rs index 8aadcef420..1edcb65c25 100644 --- a/crates/edit/src/tests.rs +++ b/crates/edit/src/tests.rs @@ -1120,7 +1120,7 @@ fn a_hand_edited_ink_reads_back_only_when_it_is_the_stored_colours_look() { add_content(&mut doc, 0, &Content::Text(AddedText { rect: [72.0, 600.0, 300.0, 700.0], text: "Proof".into(), ..AddedText::default() })).unwrap(); let obj = list_added(&doc)[0].obj; let real = |v: &[f64]| Object::Array(v.iter().map(|x| Object::Real(*x)).collect()); - let read = |entries: &[(&[u8], Object)]| { + let read_text = |entries: &[(&[u8], Object)]| { let mut edited = doc.clone(); let Object::Stream(mut s) = (*edited.get(obj)).clone() else { panic!("an item is a stream") }; let mut params = s.dict.get(b"PCAdded").and_then(|p| p.as_dict().cloned()).unwrap(); @@ -1130,11 +1130,15 @@ fn a_hand_edited_ink_reads_back_only_when_it_is_the_stored_colours_look() { s.dict.set(b"PCAdded".to_vec(), Object::Dict(params)); edited.set(obj, Object::Stream(s)); let Content::Text(t) = list_added(&edited)[0].content.clone() else { panic!() }; - t.ink + t }; + let read = |entries: &[(&[u8], Object)]| read_text(entries).ink; // As PDFCraft writes it: the ink stands. assert_eq!(read(&[(b"Color", real(&[0.5; 3])), (b"Gray", Object::Real(0.5))]), Some(Ink::Gray(0.5))); assert_eq!(read(&[(b"Color", real(&[0.0; 3])), (b"CMYK", real(&[0.0, 0.0, 0.0, 1.0]))]), Some(Ink::Cmyk([0.0, 0.0, 0.0, 1.0]))); + // Finer than it paints, it reads back as painted (`0.5 g`), its look with it. + let fine = read_text(&[(b"Color", real(&[0.50049; 3])), (b"Gray", Object::Real(0.50049))]); + assert_eq!((fine.ink, fine.color), (Some(Ink::Gray(0.5)), [0.5; 3])); for (what, entries) in [ ("a colour the ink isn't the look of", vec![(&b"Color"[..], real(&[1.0, 0.0, 0.0])), (b"CMYK", real(&[0.0, 0.0, 0.0, 1.0]))]), ("three CMYK numbers", vec![(b"Color", real(&[0.0; 3])), (b"CMYK", real(&[0.0, 0.0, 1.0]))]), @@ -1142,6 +1146,17 @@ fn a_hand_edited_ink_reads_back_only_when_it_is_the_stored_colours_look() { ("CMYK not an array", vec![(b"Color", real(&[0.0; 3])), (b"CMYK", Object::Real(1.0))]), ("gray out of range", vec![(b"Color", real(&[1.0; 3])), (b"Gray", Object::Real(2.0))]), ("gray not a number", vec![(b"Color", real(&[0.0; 3])), (b"Gray", Object::name("Black"))]), + ( + "a CMYK entry not a number", + vec![ + (b"Color", real(&[0.0; 3])), + (b"CMYK", Object::Array(vec![Object::Real(0.0), Object::name("Bad"), Object::Real(0.0), Object::Real(0.0), Object::Real(1.0)])), + ], + ), + ( + "gray not a number beside a CMYK", + vec![(b"Color", real(&[0.0; 3])), (b"Gray", Object::name("Black")), (b"CMYK", real(&[0.0, 0.0, 0.0, 1.0]))], + ), ("gray NaN", vec![(b"Color", real(&[0.0; 3])), (b"Gray", Object::Real(f64::NAN))]), ("CMYK infinite", vec![(b"Color", real(&[0.0; 3])), (b"CMYK", real(&[0.0, 0.0, 0.0, f64::INFINITY]))]), ] { @@ -1149,6 +1164,40 @@ fn a_hand_edited_ink_reads_back_only_when_it_is_the_stored_colours_look() { } } +/// #6440: an ink finer than the thousandth `g` and `k` are written to is kept as painted, so the +/// item, its stored `/CMYK` and what it reads back as are what the page shows. +#[test] +fn an_ink_is_kept_as_it_is_painted() { + use crate::added::Ink; + let mut doc = fixture(); + let mut t = AddedText { rect: [72.0, 600.0, 300.0, 700.0], text: "Proof".into(), size: 14.0, ..AddedText::default() }; + t.set_ink(Some(Ink::Cmyk([0.12345, 0.00049, 0.0, 0.99951]))); + let painted = Ink::Cmyk([0.123, 0.0, 0.0, 1.0]); + assert_eq!((t.ink, t.color), (Some(painted), painted.rgb())); + add_content(&mut doc, 0, &Content::Text(t)).unwrap(); + let doc = reopen(&doc); + let item = &list_added(&doc)[0]; + let Object::Stream(s) = &*doc.get(item.obj) else { panic!("an item is a stream") }; + let stream = String::from_utf8_lossy(&s.decoded().unwrap()).into_owned(); + assert!( + stream.contains( + "/PCFHelvetica 14 Tf 0.123 0 0 1 k +" + ), + "{stream}" + ); + let stored = s + .dict + .get(b"PCAdded") + .and_then(|p| p.as_dict()) + .and_then(|p| p.get(b"CMYK")) + .and_then(Object::as_array) + .map(|a| a.iter().filter_map(Object::as_f64).collect::>()); + assert_eq!(stored, Some(vec![0.123, 0.0, 0.0, 1.0])); + let Content::Text(back) = &item.content else { panic!() }; + assert_eq!(back.painted_ink(), Some(painted)); +} + /// #4464: a page left with one added item draws that item and nothing else. #[test] fn an_isolated_item_is_all_that_is_left_on_its_page() { From 2153da064438b461ded6e5d605a65752d301a4de Mon Sep 17 00:00:00 2001 From: Zinadin Zidan Date: Sat, 7 Nov 2026 03:06:01 +0600 Subject: [PATCH 147/198] Add text: a stored ink is read without copying a long array (review) Reading an item's /CMYK resolved the value, which copied a direct array, then collected every entry before checking there were four. A file with a huge numeric /CMYK paid for that on each content listing and item lookup. Now a direct value is read where it is, a reference is followed without a copy, and the array's length is checked before any entry is read. /Gray is read the same way. A long /CMYK, direct or by reference, and a /Gray that is a long array still read as no ink. --- crates/edit/src/added/store.rs | 19 +++++++++++++++---- crates/edit/src/tests.rs | 8 ++++++++ 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/crates/edit/src/added/store.rs b/crates/edit/src/added/store.rs index df0b697ff9..3555910f04 100644 --- a/crates/edit/src/added/store.rs +++ b/crates/edit/src/added/store.rs @@ -8,15 +8,26 @@ use super::{Content, Dict, Document, EditError, GlyphError, MarkKind, ObjRef, Ob /// text paints in its RGB `/Color`. fn ink(doc: &Document, d: &Dict) -> Option { let ink = if let Some(g) = d.get(b"Gray") { - Ink::Gray(doc.resolve(g).as_f64()?) + Ink::Gray(read_value(doc, g, Object::as_f64)?) } else { - let cmyk = doc.resolve(d.get(b"CMYK")?); - let parts: Vec = cmyk.as_array()?.iter().map(Object::as_f64).collect::>()?; - Ink::Cmyk(parts.try_into().ok()?) + // Four entries or none: a longer array is never copied or read through. + read_value(doc, d.get(b"CMYK")?, |cmyk| { + let [c, m, y, k] = cmyk.as_array()?.as_slice() else { return None }; + Some(Ink::Cmyk([c.as_f64()?, m.as_f64()?, y.as_f64()?, k.as_f64()?])) + })? }; Some(ink).filter(|i| i.valid()) } +/// `f` of `o`, a reference followed. A direct value is read where it is, never copied, so a +/// large one a file puts there costs nothing to look at. +fn read_value(doc: &Document, o: &Object, f: impl FnOnce(&Object) -> T) -> T { + match o { + Object::Ref(_) => f(&doc.resolve(o)), + direct => f(direct), + } +} + fn parse(doc: &Document, d: &Dict) -> Option { let r = nums(doc, d.get(b"Rect")); let rect: [f64; 4] = r.try_into().ok()?; diff --git a/crates/edit/src/tests.rs b/crates/edit/src/tests.rs index 1edcb65c25..8d9f5c58d2 100644 --- a/crates/edit/src/tests.rs +++ b/crates/edit/src/tests.rs @@ -1120,6 +1120,10 @@ fn a_hand_edited_ink_reads_back_only_when_it_is_the_stored_colours_look() { add_content(&mut doc, 0, &Content::Text(AddedText { rect: [72.0, 600.0, 300.0, 700.0], text: "Proof".into(), ..AddedText::default() })).unwrap(); let obj = list_added(&doc)[0].obj; let real = |v: &[f64]| Object::Array(v.iter().map(|x| Object::Real(*x)).collect()); + // A `/CMYK` far longer than four numbers, as a hostile file might store it: an ink first. + let long = real(&[0.0, 0.0, 0.0, 1.0].repeat(50_000)); + let long_ref = Object::Ref(doc.add(long.clone())); + let ink_ref = Object::Ref(doc.add(real(&[0.0, 0.0, 0.0, 1.0]))); let read_text = |entries: &[(&[u8], Object)]| { let mut edited = doc.clone(); let Object::Stream(mut s) = (*edited.get(obj)).clone() else { panic!("an item is a stream") }; @@ -1136,6 +1140,7 @@ fn a_hand_edited_ink_reads_back_only_when_it_is_the_stored_colours_look() { // As PDFCraft writes it: the ink stands. assert_eq!(read(&[(b"Color", real(&[0.5; 3])), (b"Gray", Object::Real(0.5))]), Some(Ink::Gray(0.5))); assert_eq!(read(&[(b"Color", real(&[0.0; 3])), (b"CMYK", real(&[0.0, 0.0, 0.0, 1.0]))]), Some(Ink::Cmyk([0.0, 0.0, 0.0, 1.0]))); + assert_eq!(read(&[(b"Color", real(&[0.0; 3])), (b"CMYK", ink_ref)]), Some(Ink::Cmyk([0.0, 0.0, 0.0, 1.0])), "by reference"); // Finer than it paints, it reads back as painted (`0.5 g`), its look with it. let fine = read_text(&[(b"Color", real(&[0.50049; 3])), (b"Gray", Object::Real(0.50049))]); assert_eq!((fine.ink, fine.color), (Some(Ink::Gray(0.5)), [0.5; 3])); @@ -1143,6 +1148,9 @@ fn a_hand_edited_ink_reads_back_only_when_it_is_the_stored_colours_look() { ("a colour the ink isn't the look of", vec![(&b"Color"[..], real(&[1.0, 0.0, 0.0])), (b"CMYK", real(&[0.0, 0.0, 0.0, 1.0]))]), ("three CMYK numbers", vec![(b"Color", real(&[0.0; 3])), (b"CMYK", real(&[0.0, 0.0, 1.0]))]), ("five CMYK numbers", vec![(b"Color", real(&[0.0; 3])), (b"CMYK", real(&[0.0, 0.0, 0.0, 1.0, 0.0]))]), + ("a long CMYK", vec![(b"Color", real(&[0.0; 3])), (b"CMYK", long.clone())]), + ("a long CMYK by reference", vec![(b"Color", real(&[0.0; 3])), (b"CMYK", long_ref.clone())]), + ("gray a long array", vec![(b"Color", real(&[0.0; 3])), (b"Gray", long.clone())]), ("CMYK not an array", vec![(b"Color", real(&[0.0; 3])), (b"CMYK", Object::Real(1.0))]), ("gray out of range", vec![(b"Color", real(&[1.0; 3])), (b"Gray", Object::Real(2.0))]), ("gray not a number", vec![(b"Color", real(&[0.0; 3])), (b"Gray", Object::name("Black"))]), From a9e98e0db6fc428773711dd52c15e513fa592b9a Mon Sep 17 00:00:00 2001 From: Helen Kwok <107752876+helenkwok@users.noreply.github.com> Date: Fri, 6 Nov 2026 09:24:29 +0800 Subject: [PATCH 148/198] pdfcraft-web: window.pdfcraft, the page command API from craft-web-api, opt-in with ?control ?control attaches the UI control channel (the one --control serves) and installs window.pdfcraft: request, execute (ui.command), commands (ui.commands), docInspect (ui.state), uiInspect (ui.inspect) and screenshot. The channel keeps the accessibility tree of every frame, so it stays off unless the page asks. Opening, saving, the file name and the unsaved flag stay with the host messages. --- Cargo.lock | 15 +++- apps/pdfcraft-web/Cargo.toml | 5 +- apps/pdfcraft-web/src/api.rs | 127 ++++++++++++++++++++++++++++++++++ apps/pdfcraft-web/src/main.rs | 8 +++ packaging/web/README.md | 12 ++++ 5 files changed, 165 insertions(+), 2 deletions(-) create mode 100644 apps/pdfcraft-web/src/api.rs diff --git a/Cargo.lock b/Cargo.lock index 8617c45522..0d315535a4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1312,10 +1312,21 @@ dependencies = [ "zip", ] +[[package]] +name = "craft-web-api" +version = "0.1.0" +source = "git+https://github.com/storytold/craft-libs?rev=fd5e2feff6ea25bed371a99cc732b3968d621505#fd5e2feff6ea25bed371a99cc732b3968d621505" +dependencies = [ + "js-sys", + "serde_json", + "wasm-bindgen", + "wasm-bindgen-futures", +] + [[package]] name = "craft-web-host" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=2e2312ddaf21bc3d85cd3395d0955c7d239306f6#2e2312ddaf21bc3d85cd3395d0955c7d239306f6" +source = "git+https://github.com/storytold/craft-libs?rev=fd5e2feff6ea25bed371a99cc732b3968d621505#fd5e2feff6ea25bed371a99cc732b3968d621505" dependencies = [ "wasm-bindgen", "web-sys", @@ -4754,11 +4765,13 @@ dependencies = [ name = "pdfcraft-web" version = "0.18.1" dependencies = [ + "craft-web-api", "craft-web-host", "eframe", "egui", "js-sys", "pdfcraft-ui-egui", + "serde_json", "wasm-bindgen-futures", "web-sys", ] diff --git a/apps/pdfcraft-web/Cargo.toml b/apps/pdfcraft-web/Cargo.toml index 56e10c0c29..ad5b62229c 100644 --- a/apps/pdfcraft-web/Cargo.toml +++ b/apps/pdfcraft-web/Cargo.toml @@ -12,9 +12,12 @@ workspace = true [dependencies] pdfcraft-ui-egui = { workspace = true } # The host messages (storytold/craft-libs): one contract and one bridge for the web builds. -craft-web-host = { git = "https://github.com/storytold/craft-libs", rev = "2e2312ddaf21bc3d85cd3395d0955c7d239306f6", version = "0.1.0" } +# The page command API (storytold/craft-libs#413): window.pdfcraft over the UI control channel. +craft-web-api = { git = "https://github.com/storytold/craft-libs", rev = "fd5e2feff6ea25bed371a99cc732b3968d621505", version = "0.1.0" } +craft-web-host = { git = "https://github.com/storytold/craft-libs", rev = "fd5e2feff6ea25bed371a99cc732b3968d621505", version = "0.1.0" } eframe = { workspace = true, features = ["default_fonts", "glow", "persistence", "accesskit"] } egui = { workspace = true, features = ["default_fonts", "monochrome_emoji_fonts"] } +serde_json = { workspace = true } [target.'cfg(target_arch = "wasm32")'.dependencies] wasm-bindgen-futures = "0.4" diff --git a/apps/pdfcraft-web/src/api.rs b/apps/pdfcraft-web/src/api.rs new file mode 100644 index 0000000000..379d092f72 --- /dev/null +++ b/apps/pdfcraft-web/src/api.rs @@ -0,0 +1,127 @@ +//! `window.pdfcraft`: the page command API (`craft-web-api`, storytold/craft-libs#413) over the app's +//! UI control channel (`pdfcraft_ui_egui::control`), opt-in with `?control`. +//! +//! The crate owns the JavaScript side (`request`, `execute`, `commands`, `docInspect`, `uiInspect`, +//! `screenshot`, as promises with a timeout). This module is PdfCraft's part: a [`Request`] that +//! sends each call through the app's [`ControlClient`] and waits for the reply, and the table that +//! names PdfCraft's methods: its control channel drives the UI, so `commands` is `ui.commands`, +//! `execute` runs a registry command with `ui.command`, `docInspect` is `ui.state` and `uiInspect` +//! is `ui.inspect`. +//! +//! The channel keeps the accessibility tree of every frame, so it is off unless the page asks for it +//! (`?control`), as FigCraft's is. PdfCraft's control requests cannot be withdrawn, so a call the page +//! gave up on still runs when the app reaches it, as it does for a desktop client that timed out. + +use serde_json::{Value, json}; + +/// The reply of a control request as the protocol's `{ok, result}` / `{ok: false, error}` object. +fn reply_object(reply: Result) -> Value { + match reply { + Ok(result) => json!({"ok": true, "result": result}), + Err(error) => json!({"ok": false, "error": error}), + } +} + +/// The params of `ui.command`. `execute(command, params)` sends `{command, params}`; `ui.command` +/// takes `{id}` (and `params` for `ui.panel`). A request that already names an `id` is the page's own +/// `request("ui.command", {id})` and goes through unchanged. +fn command_params(params: Value) -> Value { + match params.get("command").and_then(Value::as_str) { + Some(command) if params.get("id").is_none() => json!({"id": command, "params": params.get("params").cloned().unwrap_or(Value::Null)}), + _ => params, + } +} + +#[cfg(target_arch = "wasm32")] +pub use web::install; + +#[cfg(target_arch = "wasm32")] +mod web { + use std::rc::Rc; + use std::sync::mpsc::{Receiver, TryRecvError}; + + use craft_web_api::methods::Methods; + use craft_web_api::page::{PageApi, Request}; + use pdfcraft_ui_egui::control::{ControlClient, Reply}; + use serde_json::{Value, json}; + use wasm_bindgen_futures::JsFuture; + + use super::{command_params, reply_object}; + + /// The app id in `window.`. + const APP: &str = "pdfcraft"; + + /// How often a waiting call looks for the app's reply, in milliseconds (about a frame). + const POLL_MS: i32 = 16; + + /// PdfCraft's control methods behind each function. + fn methods() -> Methods { + Methods { + commands: Some("ui.commands"), + execute: Some("ui.command"), + doc_inspect: Some("ui.state"), + ui_inspect: Some("ui.inspect"), + screenshot: Some("ui.screenshot"), + } + } + + /// Put `window.pdfcraft` on the page, sending its calls through `client`. An existing + /// `window.pdfcraft` keeps the functions it already has. + pub fn install(client: ControlClient) { + let request: Request = Rc::new(move |method, params| { + let params = if method == "ui.command" { command_params(params) } else { params }; + Box::pin(wait(client.send(&method, params))) + }); + if let Err(e) = Rc::new(PageApi::new(APP, methods(), request)).install() { + eframe::web_sys::console::warn_1(&format!("PdfCraft: couldn't install window.{APP}: {e:?}").into()); + } + } + + /// The app's reply, polled about once a frame. + async fn wait(reply: Receiver) -> Value { + loop { + match reply.try_recv() { + Ok(reply) => return reply_object(reply), + Err(TryRecvError::Empty) => sleep(POLL_MS).await, + Err(TryRecvError::Disconnected) => return json!({"ok": false, "error": "the app dropped the request"}), + } + } + } + + async fn sleep(ms: i32) { + let promise = js_sys::Promise::new(&mut |resolve, _| { + if let Some(window) = web_sys::window() { + let _ = window.set_timeout_with_callback_and_timeout_and_arguments_0(&resolve, ms); + } + }); + let _ = JsFuture::from(promise).await; + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn execute_becomes_a_ui_command_by_id() { + assert_eq!(command_params(json!({"command": "file.word_count", "params": {"a": 1}})), json!({"id": "file.word_count", "params": {"a": 1}})); + assert_eq!(command_params(json!({"command": "file.save", "params": {}})), json!({"id": "file.save", "params": {}})); + } + + #[test] + fn a_request_that_names_an_id_goes_through_unchanged() { + let own = json!({"id": "file.save"}); + assert_eq!(command_params(own.clone()), own); + let both = json!({"id": "ui.panel", "command": "x", "params": {"reset": true}}); + assert_eq!(command_params(both.clone()), both, "an id wins over a command"); + assert_eq!(command_params(json!({})), json!({})); + } + + #[test] + fn replies_become_the_protocols_objects() { + assert_eq!(reply_object(Ok(json!({"n": 1}))), json!({"ok": true, "result": {"n": 1}})); + assert_eq!(reply_object(Err("boom".to_string())), json!({"ok": false, "error": "boom"})); + } +} diff --git a/apps/pdfcraft-web/src/main.rs b/apps/pdfcraft-web/src/main.rs index ffa5a06068..d23dd15a57 100644 --- a/apps/pdfcraft-web/src/main.rs +++ b/apps/pdfcraft-web/src/main.rs @@ -3,6 +3,9 @@ #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +#[cfg(any(target_arch = "wasm32", test))] +mod api; + #[cfg(target_arch = "wasm32")] fn main() { use pdfcraft_ui_egui::PdfCraftApp; @@ -68,6 +71,11 @@ fn main() { &"PdfCraft: ?host=parent needs a framing page on the same origin; saves download instead".into(), ); } + // `?control`: `window.pdfcraft`, the page's command API (`api`). Opt-in: the control + // channel keeps the accessibility tree of every frame. + if query_param("control").is_some() { + api::install(app.attach_control(&cc.egui_ctx)); + } Ok(Box::new(app)) }), ) diff --git a/packaging/web/README.md b/packaging/web/README.md index 3149dc75d5..a1fe64c952 100644 --- a/packaging/web/README.md +++ b/packaging/web/README.md @@ -136,6 +136,18 @@ The messages and both transports are the shared contract of [`craft-web-host`](h - **Files from the host are untrusted input.** An opened file goes through the same reading as one picked with File ▸ Open or dropped on the window, with the same format limits; before it is copied into the app's memory, its size is checked against the host open limit (512 MiB) and a larger file is refused, and a name is cut to 255 characters. - **The event and message names are the interface.** `pdfcraft:ready`, `pdfcraft:open`, `pdfcraft:save`, `pdfcraft:dirty` and `pdfcraft:failed`, and the fields they carry (the same as the `?host=parent` messages), stay stable; changing one is a breaking change. +### Driving it from the page (`window.pdfcraft`) + +`?control` on the page's URL puts `window.pdfcraft` on the page, from [`craft-web-api`](https://github.com/storytold/craft-libs/tree/main/crates/web-api) (storytold/craft-libs#413), the same shape in every Craft app (`index.html?control`, which can be combined with other flags). It is opt-in because the control channel keeps the accessibility tree of every frame. The browser already exposes PdfCraft's canvas (`id="pdfcraft"`) as `window.pdfcraft`, so the functions are added to that element. `pdfcraft.request(method, params)` calls any method of the UI control channel the desktop app serves with `--control` (`ui.state`, `ui.inspect`, `ui.click`, `ui.key`, `ui.set`, `ui.open`…), and `execute(command, params)` (`ui.command`: a registry command id), `commands()` (`ui.commands`), `docInspect(params)` (`ui.state`), `uiInspect(params)` (`ui.inspect`) and `screenshot(params)` are shortcuts for the common ones. Each returns a Promise that resolves with the method's result and rejects with an `Error`; a call the app does not answer within 30 seconds rejects, and the app still runs it when it reaches it, as it does for a desktop client that timed out. + +```js +const state = await pdfcraft.docInspect(); // open documents, page, zoom, panels, dialog +await pdfcraft.request("ui.set", { key: "show_page_edge_shadows", value: "false" }); +const { data } = await pdfcraft.screenshot(); // the window as PNG (base64) +``` + +Opening a PDF, handing a saved or exported file to the page, the file name and the unsaved flag stay with the host messages above. + ## Renderer selection and fallback flags PdfCraft renders with wgpu. It uses **WebGPU** when the browser has it and falls back to From 9db177c1637d79348a6ca3b51ff207f0ebdf6772 Mon Sep 17 00:00:00 2001 From: Zinadin Zidan Date: Sat, 7 Nov 2026 08:05:36 +0600 Subject: [PATCH 149/198] Add text: the ink tests reach the non-number and content_update paths (review) The hand-edited ink test's "CMYK entry not a number" case had five entries, so the length check refused it before any entry was read; it now has four. content_update gets its own refusals: two inks at once, an ink out of range beside another style change, and gray or cmyk on an image, with the item list unchanged after each. --- crates/automation/tests/it/automation.rs | 24 ++++++++++++++++++++++++ crates/edit/src/tests.rs | 2 +- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/crates/automation/tests/it/automation.rs b/crates/automation/tests/it/automation.rs index b2bf8ba25d..db47eeb37b 100644 --- a/crates/automation/tests/it/automation.rs +++ b/crates/automation/tests/it/automation.rs @@ -5265,6 +5265,30 @@ fn added_text_takes_a_cmyk_or_gray_ink_through_tools() { assert!(matches!(a.call("page_add_text", &args), Err(ToolError::InvalidArgs(_))), "{bad}"); } assert_eq!(ok(&mut a, "content_list", json!({ "doc": doc }))["count"], 2, "nothing refused was added"); + // content_update refuses them too, an image takes no ink, and a refused update changes nothing. + let mut png = Vec::new(); + { + let mut enc = png::Encoder::new(&mut png, 40, 20); + enc.set_color(png::ColorType::Rgb); + let mut w = enc.write_header().unwrap(); + w.write_image_data(&[200u8; 2400]).unwrap(); + } + std::fs::write(dir.join("logo.png"), png).unwrap(); + ok(&mut a, "page_add_image", json!({ "doc": doc, "page": 1, "path": "logo.png", "rect": [100, 200, 180, 240] })); + let before = ok(&mut a, "content_list", json!({ "doc": doc }))["items"].clone(); + for (index, bad) in [ + (1, json!({ "gray": 0.5, "cmyk": [0, 0, 0, 1] })), + (2, json!({ "color": "red", "gray": 0.5 })), + (1, json!({ "size": 30, "cmyk": [0, 0, 0, 2] })), + (2, json!({ "bold": true, "gray": -1 })), + (3, json!({ "gray": 0.5 })), + (3, json!({ "cmyk": [0, 0, 0, 1] })), + ] { + let mut args = json!({ "doc": doc, "page": 1, "index": index }); + args.as_object_mut().unwrap().extend(bad.as_object().unwrap().clone()); + assert!(matches!(a.call("content_update", &args), Err(ToolError::InvalidArgs(_))), "{bad}"); + } + assert_eq!(ok(&mut a, "content_list", json!({ "doc": doc }))["items"], before, "a refused update changed nothing"); } /// Edit ▸ Copy and Paste headlessly (#2636): an added text box or image, as it is, onto its own diff --git a/crates/edit/src/tests.rs b/crates/edit/src/tests.rs index 8d9f5c58d2..49ab8abbc7 100644 --- a/crates/edit/src/tests.rs +++ b/crates/edit/src/tests.rs @@ -1158,7 +1158,7 @@ fn a_hand_edited_ink_reads_back_only_when_it_is_the_stored_colours_look() { "a CMYK entry not a number", vec![ (b"Color", real(&[0.0; 3])), - (b"CMYK", Object::Array(vec![Object::Real(0.0), Object::name("Bad"), Object::Real(0.0), Object::Real(0.0), Object::Real(1.0)])), + (b"CMYK", Object::Array(vec![Object::Real(0.0), Object::name("Bad"), Object::Real(0.0), Object::Real(1.0)])), ], ), ( From 6ec23789c1a6ab072195cc6a69774d26f09667af Mon Sep 17 00:00:00 2001 From: Helen Kwok <107752876+helenkwok@users.noreply.github.com> Date: Sat, 7 Nov 2026 10:28:05 +0800 Subject: [PATCH 150/198] pdfcraft-web: say a timed-out call is not guaranteed to have been skipped --- apps/pdfcraft-web/src/api.rs | 2 +- packaging/web/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/pdfcraft-web/src/api.rs b/apps/pdfcraft-web/src/api.rs index 379d092f72..3f1d265377 100644 --- a/apps/pdfcraft-web/src/api.rs +++ b/apps/pdfcraft-web/src/api.rs @@ -10,7 +10,7 @@ //! //! The channel keeps the accessibility tree of every frame, so it is off unless the page asks for it //! (`?control`), as FigCraft's is. PdfCraft's control requests cannot be withdrawn, so a call the page -//! gave up on still runs when the app reaches it, as it does for a desktop client that timed out. +//! gave up on may still run (or be rejected) when the app reaches it, as it does for a desktop client that timed out. use serde_json::{Value, json}; diff --git a/packaging/web/README.md b/packaging/web/README.md index a1fe64c952..375f1f4ba0 100644 --- a/packaging/web/README.md +++ b/packaging/web/README.md @@ -138,7 +138,7 @@ The messages and both transports are the shared contract of [`craft-web-host`](h ### Driving it from the page (`window.pdfcraft`) -`?control` on the page's URL puts `window.pdfcraft` on the page, from [`craft-web-api`](https://github.com/storytold/craft-libs/tree/main/crates/web-api) (storytold/craft-libs#413), the same shape in every Craft app (`index.html?control`, which can be combined with other flags). It is opt-in because the control channel keeps the accessibility tree of every frame. The browser already exposes PdfCraft's canvas (`id="pdfcraft"`) as `window.pdfcraft`, so the functions are added to that element. `pdfcraft.request(method, params)` calls any method of the UI control channel the desktop app serves with `--control` (`ui.state`, `ui.inspect`, `ui.click`, `ui.key`, `ui.set`, `ui.open`…), and `execute(command, params)` (`ui.command`: a registry command id), `commands()` (`ui.commands`), `docInspect(params)` (`ui.state`), `uiInspect(params)` (`ui.inspect`) and `screenshot(params)` are shortcuts for the common ones. Each returns a Promise that resolves with the method's result and rejects with an `Error`; a call the app does not answer within 30 seconds rejects, and the app still runs it when it reaches it, as it does for a desktop client that timed out. +`?control` on the page's URL puts `window.pdfcraft` on the page, from [`craft-web-api`](https://github.com/storytold/craft-libs/tree/main/crates/web-api) (storytold/craft-libs#413), the same shape in every Craft app (`index.html?control`, which can be combined with other flags). It is opt-in because the control channel keeps the accessibility tree of every frame. The browser already exposes PdfCraft's canvas (`id="pdfcraft"`) as `window.pdfcraft`, so the functions are added to that element. `pdfcraft.request(method, params)` calls any method of the UI control channel the desktop app serves with `--control` (`ui.state`, `ui.inspect`, `ui.click`, `ui.key`, `ui.set`, `ui.open`…), and `execute(command, params)` (`ui.command`: a registry command id), `commands()` (`ui.commands`), `docInspect(params)` (`ui.state`), `uiInspect(params)` (`ui.inspect`) and `screenshot(params)` are shortcuts for the common ones. Each returns a Promise that resolves with the method's result and rejects with an `Error`; a call the app does not answer within 30 seconds rejects, and the app may still reach it later and run it (or reject it), as it does for a desktop client that timed out, so a timed-out call is not guaranteed to have been skipped. ```js const state = await pdfcraft.docInspect(); // open documents, page, zoom, panels, dialog From a9ccca5b71c70ffb20d1ea71e33e9bdea618ad49 Mon Sep 17 00:00:00 2001 From: Helen Kwok <107752876+helenkwok@users.noreply.github.com> Date: Sat, 7 Nov 2026 15:30:00 +0800 Subject: [PATCH 151/198] pdfcraft-web: say ui.open answers with an error on the web --- packaging/web/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packaging/web/README.md b/packaging/web/README.md index 375f1f4ba0..7d305b801d 100644 --- a/packaging/web/README.md +++ b/packaging/web/README.md @@ -138,7 +138,7 @@ The messages and both transports are the shared contract of [`craft-web-host`](h ### Driving it from the page (`window.pdfcraft`) -`?control` on the page's URL puts `window.pdfcraft` on the page, from [`craft-web-api`](https://github.com/storytold/craft-libs/tree/main/crates/web-api) (storytold/craft-libs#413), the same shape in every Craft app (`index.html?control`, which can be combined with other flags). It is opt-in because the control channel keeps the accessibility tree of every frame. The browser already exposes PdfCraft's canvas (`id="pdfcraft"`) as `window.pdfcraft`, so the functions are added to that element. `pdfcraft.request(method, params)` calls any method of the UI control channel the desktop app serves with `--control` (`ui.state`, `ui.inspect`, `ui.click`, `ui.key`, `ui.set`, `ui.open`…), and `execute(command, params)` (`ui.command`: a registry command id), `commands()` (`ui.commands`), `docInspect(params)` (`ui.state`), `uiInspect(params)` (`ui.inspect`) and `screenshot(params)` are shortcuts for the common ones. Each returns a Promise that resolves with the method's result and rejects with an `Error`; a call the app does not answer within 30 seconds rejects, and the app may still reach it later and run it (or reject it), as it does for a desktop client that timed out, so a timed-out call is not guaranteed to have been skipped. +`?control` on the page's URL puts `window.pdfcraft` on the page, from [`craft-web-api`](https://github.com/storytold/craft-libs/tree/main/crates/web-api) (storytold/craft-libs#413), the same shape in every Craft app (`index.html?control`, which can be combined with other flags). It is opt-in because the control channel keeps the accessibility tree of every frame. The browser already exposes PdfCraft's canvas (`id="pdfcraft"`) as `window.pdfcraft`, so the functions are added to that element. `pdfcraft.request(method, params)` calls any method of the UI control channel the desktop app serves with `--control` (`ui.state`, `ui.inspect`, `ui.click`, `ui.key`, `ui.set`…; `ui.open` needs a file system, so on the web it answers with an error and a file is dropped on the page instead), and `execute(command, params)` (`ui.command`: a registry command id), `commands()` (`ui.commands`), `docInspect(params)` (`ui.state`), `uiInspect(params)` (`ui.inspect`) and `screenshot(params)` are shortcuts for the common ones. Each returns a Promise that resolves with the method's result and rejects with an `Error`; a call the app does not answer within 30 seconds rejects, and the app may still reach it later and run it (or reject it), as it does for a desktop client that timed out, so a timed-out call is not guaranteed to have been skipped. ```js const state = await pdfcraft.docInspect(); // open documents, page, zoom, panels, dialog From 712345d7a1ed2fb794d9dab5b6fa59d8bc632d30 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Thu, 5 Nov 2026 07:47:29 +0100 Subject: [PATCH 152/198] M2.6: an edit never rewrites a content stream from the part before its damage A Flate stream damaged partway decodes to what came before the damage, and `DecodeBudget::decode` gave that back as the stream's data: the decoder's partial flag was dropped in `Stream::decoded_within_counted`. Editing a text line, a block, an image or an added item in such a stream wrote it again from that part, so everything after the damage was lost on save, though other readers may still read it. - `DecodeBudget::decode_with_damage` decodes as `decode_within` does and says whether the stream is damaged partway. A missing checksum or end-of-data marker is no damage, as before. - Edit reads the page's streams with it (`ContentStream`), and `rewritten_contents` refuses, before adding anything, a change to a damaged stream. Changes in the page's other streams go on, and the damaged one stays as stored. - Selecting objects (`editable_objects`, moving added items) refuses a page with a damaged stream, as it refuses one that doesn't decode: what follows the damage could change every later item's graphics state. - Marking a tag as an artifact or a figure as decorative refuses when the tag's content is in a damaged stream (`ArtifactError::Damaged`, `AltError::Damaged`), and changes nothing, the structure tree included. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/a11y/src/alt.rs | 65 ++++++++++++++++++------- crates/a11y/src/tests.rs | 42 ++++++++++++++++ crates/cos/src/budget.rs | 63 +++++++++++++++++++----- crates/cos/src/object.rs | 15 ++++-- crates/edit/src/background.rs | 4 +- crates/edit/src/images.rs | 45 +++++++---------- crates/edit/src/objects.rs | 23 +++++---- crates/edit/src/tests.rs | 60 +++++++++++++++++++++++ crates/edit/src/text.rs | 4 +- crates/edit/src/text/blocks.rs | 8 +-- crates/edit/src/text/lines.rs | 67 +++++++++++++++++--------- crates/edit/src/text/rewrite.rs | 20 ++++---- crates/edit/src/text_regions/styled.rs | 4 +- 13 files changed, 308 insertions(+), 112 deletions(-) diff --git a/crates/a11y/src/alt.rs b/crates/a11y/src/alt.rs index fde1048e47..973dc29e33 100644 --- a/crates/a11y/src/alt.rs +++ b/crates/a11y/src/alt.rs @@ -25,6 +25,8 @@ pub enum AltError { NotAFigure(u32), #[error("the page has more content than can be edited")] TooLarge, + #[error("the figure's content is in a content stream damaged partway, which can't be rewritten without losing what follows the damage")] + Damaged, #[error("{0}")] Cos(#[from] pdfcraft_cos::CosError), } @@ -52,6 +54,8 @@ pub enum ArtifactError { ParentTree(String), #[error("the page has more content than can be edited")] TooLarge, + #[error("the tag's marked content is in a content stream damaged partway, which can't be rewritten without losing what follows the damage")] + Damaged, #[error(transparent)] Cos(#[from] pdfcraft_cos::CosError), } @@ -372,7 +376,9 @@ fn content_entries(doc: &Document, page: &Dict) -> Vec { } } -type DecodedStream = (Object, Option<(Stream, Vec)>); +/// A `/Contents` entry and, for a stream, the stream, its decoded data and whether it is damaged +/// partway (the data is then what decoded before the damage). +type DecodedStream = (Object, Option<(Stream, Vec, bool)>); /// Decode the page's content streams within the page decode budget. `None` when a stream is /// rejected because the budget is exhausted. @@ -383,14 +389,14 @@ fn decode_page_streams(doc: &Document, page: &Dict) -> Option for entry in entries { let decoded = match &*doc.resolve(&entry) { Object::Stream(s) => { - let data = match budget.decode(s) { - Ok(data) => data, + let (data, damaged) = match budget.decode_with_damage(s, usize::MAX) { + Ok(read) => read, // Content past the budget may hold more of the tag: editing the rest // would leave that part marked as a tag the tree no longer has. Err(_) if budget.exhausted() => return None, - Err(_) => Vec::new(), + Err(_) => (Vec::new(), false), }; - Some((s.clone(), data)) + Some((s.clone(), data, damaged)) } _ => None, }; @@ -403,6 +409,18 @@ fn marked_id(doc: &Document, props: &Dict, op: &pdfcraft_content::Op, ids: &Hash (op.op == b"BDC").then(|| bdc_mcid(doc, props, op)).flatten().filter(|n| ids.contains(n)) } +/// What [`artifact_marked_content`] did. +#[derive(Debug, PartialEq)] +enum Marked { + /// The streams holding the content were rewritten. + Done, + /// None of the content is on the page, or with `require_all` not all of it: nothing changed. + NotFound, + /// Some of it is in a stream damaged partway, which can't be rewritten without losing what + /// follows the damage: nothing changed. + Damaged, +} + /// Turn the tag's marked-content blocks into artifacts, rewriting only the streams that /// contain them. Other content streams, and unknown entries on a rewritten stream, stay. /// With `require_all`, every id in `ids` must appear on the page before any stream is rewritten. @@ -412,9 +430,9 @@ fn artifact_marked_content( streams: Vec, ids: &HashSet, require_all: bool, -) -> Result { +) -> Result { let props = page_properties(doc, &page.dict); - let decoded: Vec<&[u8]> = streams.iter().filter_map(|(_, s)| s.as_ref().map(|(_, bytes)| bytes.as_slice())).collect(); + let decoded: Vec<&[u8]> = streams.iter().filter_map(|(_, s)| s.as_ref().map(|(_, bytes, _)| bytes.as_slice())).collect(); let pieces = pdfcraft_content::Pieces::join(&decoded); let mut found = HashSet::new(); let mut edits = Vec::new(); @@ -425,12 +443,17 @@ fn artifact_marked_content( } } if if require_all { found != *ids } else { found.is_empty() } { - return Ok(false); + return Ok(Marked::NotFound); } - let mut spliced = pieces.splice(edits).into_iter(); + let spliced = pieces.splice(edits); + // The pieces are the streams that decoded, in order. + if streams.iter().filter_map(|(_, s)| s.as_ref()).zip(&spliced).any(|((_, old, damaged), new)| *damaged && old != new) { + return Ok(Marked::Damaged); + } + let mut spliced = spliced.into_iter(); let mut next = Vec::with_capacity(streams.len()); for (entry, stream) in streams { - let Some((stream, old)) = stream else { + let Some((stream, old, _)) = stream else { next.push(entry); continue; }; @@ -456,7 +479,7 @@ fn artifact_marked_content( _ => Object::Array(next), }; doc.update_dict(page.obj, |p| p.set(b"Contents".to_vec(), contents))?; - Ok(true) + Ok(Marked::Done) } /// Decorative figure: its marked content on the page becomes an artifact, and the element @@ -468,11 +491,15 @@ pub fn mark_decorative(doc: &mut Document, obj: ObjRef) -> Result<(), AltError> let pg = d.get(b"Pg").and_then(Object::as_ref).or_else(|| elements(doc).into_iter().find(|e| e.object == obj).and_then(|e| e.page)); if let Some(page) = pg.and_then(|p| pages.iter().find(|x| x.obj == p)).filter(|_| !ids.is_empty()) { let streams = decode_page_streams(doc, &page.dict).ok_or(AltError::TooLarge)?; - // The parent tree no longer points those ids at the figure. - if artifact_marked_content(doc, page, streams, &ids, false)? - && let Some(key) = page.dict.get(b"StructParents").and_then(Object::as_int) - { - clear_parent_tree(doc, key, obj)?; + match artifact_marked_content(doc, page, streams, &ids, false)? { + // The parent tree no longer points those ids at the figure. + Marked::Done => { + if let Some(key) = page.dict.get(b"StructParents").and_then(Object::as_int) { + clear_parent_tree(doc, key, obj)?; + } + } + Marked::NotFound => {} + Marked::Damaged => return Err(AltError::Damaged), } } // Out of its parent's kids. @@ -513,8 +540,10 @@ pub fn mark_tag_artifact(doc: &mut Document, obj: ObjRef) -> Result<(), Artifact let parent = element.dict.get(b"P").and_then(Object::as_ref).ok_or(ArtifactError::NoParent)?; let replacement = replacement_kids(doc, parent, obj)?; let streams = decode_page_streams(doc, &page.dict).ok_or(ArtifactError::TooLarge)?; - if !artifact_marked_content(doc, page, streams, &ids, true)? { - return Err(ArtifactError::ContentNotFound); + match artifact_marked_content(doc, page, streams, &ids, true)? { + Marked::Done => {} + Marked::NotFound => return Err(ArtifactError::ContentNotFound), + Marked::Damaged => return Err(ArtifactError::Damaged), } if let Some(key) = page.dict.get(b"StructParents").and_then(Object::as_int) { clear_parent_tree(doc, key, obj).map_err(|e| ArtifactError::ParentTree(e.to_string()))?; diff --git a/crates/a11y/src/tests.rs b/crates/a11y/src/tests.rs index 3024319ec3..b551c2cb42 100644 --- a/crates/a11y/src/tests.rs +++ b/crates/a11y/src/tests.rs @@ -1087,6 +1087,48 @@ fn mark_tag_artifact_validates_parent_and_all_mcids_before_mutating_document() { assert_eq!(*absent.get(ObjRef::new(9, 0)), before_pt); } +/// zlib data that reads as `readable` and is then damaged, with more of the stream after the +/// damage: a stored block, then a block of the reserved type (RFC 1951 §3.2.3), then `rest`. +fn damaged_flate(readable: &[u8], rest: &[u8]) -> pdfcraft_cos::Stream { + let len = u16::try_from(readable.len()).unwrap(); + let mut raw = vec![0x78, 0x01, 0x00]; + raw.extend_from_slice(&len.to_le_bytes()); + raw.extend_from_slice(&(!len).to_le_bytes()); + raw.extend_from_slice(readable); + raw.push(0x07); + raw.extend_from_slice(rest); + let mut dict = pdfcraft_cos::Dict::new(); + dict.set(b"Filter".to_vec(), pdfcraft_cos::Object::name("FlateDecode")); + pdfcraft_cos::Stream::from_raw(dict, raw) +} + +/// A tag's content in a stream damaged partway stays tagged: rewriting the stream from what +/// decoded would lose what follows the damage. Nothing changes, the structure tree included. +/// Content in a whole stream beside a damaged one is marked, and the damaged one kept as stored. +#[test] +fn a_tags_content_in_a_stream_damaged_partway_is_not_rewritten() { + use pdfcraft_cos::{ObjRef, Object}; + let content = b"/Figure << /MCID 0 >> BDC q 40 0 0 20 10 150 cm /Im Do Q EMC /Photo << /MCID 1 >> BDC 100 100 50 30 re f EMC\n"; + let damaged = damaged_flate(content, b"0 0 1 rg 0 0 10 10 re f"); + let mut doc = figures_doc(); + doc.set(ObjRef::new(4, 0), damaged.clone()); + let contents = |doc: &Document| doc.get(ObjRef::new(3, 0)).as_dict().unwrap().get(b"Contents").cloned(); + let (before_contents, before_pt, before_kids) = (contents(&doc), doc.get(ObjRef::new(9, 0)), doc.get(ObjRef::new(6, 0))); + assert_eq!(figures(&doc).len(), 2, "read up to the damage"); + assert_eq!(mark_tag_artifact(&mut doc, ObjRef::new(7, 0)), Err(ArtifactError::Damaged)); + assert_eq!(mark_decorative(&mut doc, ObjRef::new(7, 0)), Err(AltError::Damaged)); + assert_eq!((contents(&doc), doc.get(ObjRef::new(9, 0)), doc.get(ObjRef::new(6, 0))), (before_contents, before_pt, before_kids)); + + let mut doc = figures_doc(); + let other = doc.add(damaged_flate(b"0 g\n", b"0 0 1 rg 0 0 10 10 re f")); + doc.update_dict(ObjRef::new(3, 0), |p| p.set(b"Contents".to_vec(), Object::Array(vec![Object::Ref(other), Object::Ref(ObjRef::new(4, 0))]))) + .unwrap(); + mark_tag_artifact(&mut doc, ObjRef::new(7, 0)).unwrap(); + let Some(Object::Array(list)) = contents(&doc) else { panic!("contents {:?}", contents(&doc)) }; + assert_eq!(list.first(), Some(&Object::Ref(other)), "kept as stored"); + assert_ne!(list.get(1), Some(&Object::Ref(ObjRef::new(4, 0))), "the tag's stream is rewritten"); +} + fn ref_(n: u32) -> Option { Some(pdfcraft_cos::ObjRef::new(n, 0)) } diff --git a/crates/cos/src/budget.rs b/crates/cos/src/budget.rs index 65b750e7b3..3211efb273 100644 --- a/crates/cos/src/budget.rs +++ b/crates/cos/src/budget.rs @@ -38,9 +38,9 @@ const JPEG_WORK: u64 = 2; /// and images can share one, so reading it again for each has to count. const JPEG_READ_WORK: u64 = 4; -/// A stream decode within a byte bound, with the bytes its filter stages decoded: the data or -/// the error, each with that count. -type Decoder = fn(&Stream, usize) -> Result<(Vec, usize), (FilterError, usize)>; +/// A stream decode within a byte bound, with the bytes its filter stages decoded: the data and +/// whether it stopped at damage, or the error, each with that count. +type Decoder = fn(&Stream, usize) -> Result<(Vec, bool, usize), (FilterError, usize)>; /// What one operation over a document's images may still spend decoding them: exporting them, /// optimizing them, and storing LZW streams with Flate as it does, clearing them for redaction. @@ -263,45 +263,56 @@ impl DecodeBudget { } /// `stream` decoded like [`Stream::decoded`], charged to the allowance. Fails, without - /// decoding, once the allowance is spent. + /// decoding, once the allowance is spent. A stream damaged partway gives what decoded before + /// the damage, so code that writes the stream back reads it with + /// [`DecodeBudget::decode_with_damage`] instead. pub fn decode(&mut self, stream: &Stream) -> Result, CosError> { - self.charge(stream, usize::MAX, Stream::decoded_within_counted) + self.charge(stream, usize::MAX, Stream::decoded_within_counted).map(|(data, _)| data) } /// Like [`DecodeBudget::decode`], for a stream with a tighter bound of its own: decoding /// past `max` bytes is an error that charges `max`, the work it took to find out. pub fn decode_within(&mut self, stream: &Stream, max: usize) -> Result, CosError> { + self.charge(stream, max, Stream::decoded_within_counted).map(|(data, _)| data) + } + + /// [`DecodeBudget::decode_within`] (`usize::MAX` for no bound of its own), also saying + /// whether the stream is damaged: `true` when a filter stopped at corrupt data, so the bytes + /// are only what decoded before it. A stream written back from them loses everything after + /// the damage, which other readers may still read. + pub fn decode_with_damage(&mut self, stream: &Stream, max: usize) -> Result<(Vec, bool), CosError> { self.charge(stream, max, Stream::decoded_within_counted) } /// `stream` decoded like [`Stream::decoded_strict`] (any corruption is an error), charged to /// the allowance. pub fn decode_strict(&mut self, stream: &Stream) -> Result, CosError> { - self.charge(stream, usize::MAX, Stream::decoded_strict_within_counted) + let strict: Decoder = |s, max| s.decoded_strict_within_counted(max).map(|(data, spent)| (data, false, spent)); + self.charge(stream, usize::MAX, strict).map(|(data, _)| data) } /// Decodes `stream` within `max` and the allowance, charging what every filter stage /// decoded: a stage's output counts even when a later stage shrinks it, and a corrupt stream /// pays for the bytes decoded before the error, so referencing a stream again and again /// spends the allowance by the work each decode does. - fn charge(&mut self, stream: &Stream, max: usize, decode: Decoder) -> Result, CosError> { + fn charge(&mut self, stream: &Stream, max: usize, decode: Decoder) -> Result<(Vec, bool), CosError> { if self.bytes == 0 && !stream.raw.is_empty() { return Err(CosError::Filter(FilterError::LimitExceeded(self.limit).to_string())); } if max < self.bytes { let result = decode(stream, max); let spent = match &result { - Ok((_, spent)) => *spent, + Ok((_, _, spent)) => *spent, Err((FilterError::LimitExceeded(_), _)) => max, Err((_, spent)) => *spent, }; self.bytes = self.bytes.saturating_sub(spent); - return result.map(|(data, _)| data).map_err(|(e, _)| CosError::Filter(e.to_string())); + return result.map(|(data, damaged, _)| (data, damaged)).map_err(|(e, _)| CosError::Filter(e.to_string())); } match decode(stream, self.bytes) { - Ok((data, spent)) => { + Ok((data, damaged, spent)) => { self.bytes = self.bytes.saturating_sub(spent); - Ok(data) + Ok((data, damaged)) } Err((FilterError::LimitExceeded(_), _)) => { self.bytes = 0; @@ -801,6 +812,36 @@ mod tests { assert_eq!(budget.remaining(), (10 << 20) - partial); } + /// A stream damaged partway decodes to what came before the damage, and says so, so that code + /// writing it back can leave it as it is. What every reader takes is no damage: a missing + /// checksum, no filter at all. + #[test] + fn a_stream_damaged_partway_says_so() { + let read = |budget: &mut DecodeBudget, s: &Stream, max| budget.decode_with_damage(s, max).map(|(d, damaged)| (d.len(), damaged)).ok(); + let whole = Stream::flate(Dict::new(), &vec![b' '; 1 << 20]); + let cut = Stream::from_raw(whole.dict.clone(), whole.raw.get(..whole.raw.len() / 2).unwrap_or_default().to_vec()); + let partial = cut.decoded().map(|d| d.len()).unwrap_or(0); + assert!(partial > 0, "a cut flate stream decodes in part"); + let mut budget = DecodeBudget::new(10 << 20, 1); + assert_eq!(read(&mut budget, &cut, usize::MAX), Some((partial, true))); + assert_eq!(budget.remaining(), (10 << 20) - partial); + // Within a bound of its own, under what is left. + assert_eq!(read(&mut budget, &cut, 4 << 20), Some((partial, true))); + assert_eq!(budget.decode(&cut).map(|d| d.len()).ok(), Some(partial)); + + let mut budget = DecodeBudget::new(10 << 20, 1); + assert_eq!(read(&mut budget, &whole, usize::MAX), Some((1 << 20, false))); + assert_eq!(read(&mut budget, &whole, 4 << 20), Some((1 << 20, false))); + let unchecked = Stream::from_raw(whole.dict.clone(), whole.raw.get(..whole.raw.len() - 4).unwrap_or_default().to_vec()); + assert_eq!(read(&mut budget, &unchecked, usize::MAX), Some((1 << 20, false))); + assert_eq!(read(&mut budget, &Stream::from_raw(Dict::new(), b"0 0 m".to_vec()), usize::MAX), Some((5, false))); + // A later stage's damage is the stream's. + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::Array(vec![Object::name("FlateDecode"), Object::name("ASCIIHexDecode")])); + let hex = Stream::from_raw(dict, Stream::flate(Dict::new(), b"4142zz4344>").raw.to_vec()); + assert_eq!(budget.decode_with_damage(&hex, usize::MAX).ok(), Some((b"AB".to_vec(), true))); + } + /// A stream that fails its reading as zlib, and then as raw deflate after decoding all of it, /// gives nothing back, and each reference still pays for what it decoded. #[test] diff --git a/crates/cos/src/object.rs b/crates/cos/src/object.rs index 5d437faa82..bcb188afb1 100644 --- a/crates/cos/src/object.rs +++ b/crates/cos/src/object.rs @@ -297,13 +297,18 @@ impl Stream { /// [`Stream::decoded_within`] for a caller charging an allowance: the filter error is kept /// typed, so a caller can tell a spent limit from corrupt data, `max` bounds all filter - /// stages together, and the result gives the bytes they decoded (the work done, on success - /// and on failure). - pub(crate) fn decoded_within_counted(&self, max: usize) -> Result<(Vec, usize), (pdfcraft_filters::FilterError, usize)> { + /// stages together, and the result gives whether the data stopped at damage (it is then what + /// decoded before it) and the bytes the stages decoded (the work done, on success and on + /// failure). + pub(crate) fn decoded_within_counted(&self, max: usize) -> Result<(Vec, bool, usize), (pdfcraft_filters::FilterError, usize)> { + let mut damaged = false; let decode = |chain: &[(Filter, Params)], data: &[u8], max| { - pdfcraft_filters::decode_tolerant_counted(chain, data, max).map(|(v, _, spent)| (v, spent)) + pdfcraft_filters::decode_tolerant_counted(chain, data, max).map(|(v, partial, spent)| { + damaged = partial; + (v, spent) + }) }; - self.decode_chain(max, decode) + self.decode_chain(max, decode).map(|(v, spent)| (v, damaged, spent)) } /// [`Stream::decoded_within`] spending from `work`, what an operation may still do across a diff --git a/crates/edit/src/background.rs b/crates/edit/src/background.rs index 376cb5a165..27073ddc09 100644 --- a/crates/edit/src/background.rs +++ b/crates/edit/src/background.rs @@ -1,7 +1,7 @@ //! A page's background colour: the colour the page is painted with before anything else, as //! Edit a PDF ▸ Background and many producers do it, for exports that keep a page colour (#2992). -use pdfcraft_content::{Matrix, Op, Pieces}; +use pdfcraft_content::{Matrix, Op}; use pdfcraft_cos::{Dict, Document}; use crate::{EditError, Rgb}; @@ -51,7 +51,7 @@ pub fn page_background(doc: &Document, page: usize) -> Result, EditE let resources = p.dict.get(b"Resources").map(|r| doc.resolve(r)).and_then(|r| r.as_dict().cloned()).unwrap_or_default(); let gstates = resources.get(b"ExtGState").map(|g| doc.resolve(g)).and_then(|g| g.as_dict().cloned()).unwrap_or_default(); let streams = crate::text::content_streams(doc, &p.dict)?; - let pieces = Pieces::join(&streams.iter().map(|(_, d)| d.as_slice()).collect::>()); + let pieces = crate::text::joined(&streams); let ops = pieces.parse().ops; let mut st = State { ctm: Matrix::IDENTITY, space: Space::Gray, fill: Some([0.0; 3]), alpha: 1.0, plain: true, clip: Some(Vec::new()) }; diff --git a/crates/edit/src/images.rs b/crates/edit/src/images.rs index 72656fa1bb..201612e5b2 100644 --- a/crates/edit/src/images.rs +++ b/crates/edit/src/images.rs @@ -4,12 +4,12 @@ //! or delete. Only the stream that draws it is rewritten, as a new object. use kurbo::{BezPath, Point, Shape}; -use pdfcraft_content::{Matrix, Op, Pieces, parse, serialize_ops}; +use pdfcraft_content::{Matrix, Op, parse, serialize_ops}; use pdfcraft_cos::{Dict, Document, ObjRef, Object}; use std::sync::Arc; use crate::EditError; -use crate::text::content_streams; +use crate::text::{ContentStream, content_streams, joined}; /// The identity of a soft mask (`/SMask`) in the PDF graphics state. #[derive(Clone, Debug)] @@ -47,7 +47,7 @@ pub struct PageImage { pub width: u32, pub height: u32, /// The content stream its `Do` (its first token) is in, and the `Do`'s index among the - /// operators of the page's streams joined ([`Pieces`]). + /// operators of the page's streams joined ([`pdfcraft_content::Pieces`]). pub stream: usize, op: usize, /// CTM before `Do`, excluding a Form's own Matrix. @@ -103,11 +103,6 @@ fn too_many_images() -> EditError { EditError::Invalid(format!("the page draws more than {MAX_PAGE_IMAGES} images")) } -/// The page's streams joined into the one stream they are: an operator may span two of them. -fn joined(streams: &[(Object, Vec)]) -> Pieces { - Pieces::join(&streams.iter().map(|(_, data)| data.as_slice()).collect::>()) -} - fn page_of(doc: &Document, page: usize) -> Result { pdfcraft_model::pages(doc).into_iter().nth(page).ok_or(EditError::NoSuchPage(page)) } @@ -1191,13 +1186,7 @@ fn build_relocated_unit_ops( } /// Change the stacking order of `img`, drawn on page `p` by its content streams `all`. -fn arrange_image( - doc: &mut Document, - p: &pdfcraft_model::Page, - all: &[(Object, Vec)], - img: &PageImage, - order: ArrangeOrder, -) -> Result<(), EditError> { +fn arrange_image(doc: &mut Document, p: &pdfcraft_model::Page, all: &[ContentStream], img: &PageImage, order: ArrangeOrder) -> Result<(), EditError> { let pieces = joined(all); let ops = pieces.parse().ops; let target = img.op; @@ -1264,7 +1253,7 @@ fn arrange_image( ops.get(dest_unit.start).map(|op| pieces.piece_of(op.span.start)) }; let owned_destination = destination_piece - .filter(|piece| all.get(*piece).is_some_and(|(obj, _)| doc.resolve(obj).as_dict().is_some_and(|d| d.get(b"PCAdded").is_some()))); + .filter(|piece| all.get(*piece).is_some_and(|s| doc.resolve(&s.entry).as_dict().is_some_and(|d| d.get(b"PCAdded").is_some()))); if let Some(piece) = owned_destination { let whole_unit = ops.iter().enumerate().all(|(index, op)| { let (first, last) = pieces.pieces_of(op); @@ -1440,7 +1429,7 @@ fn arrange_image( if let Some(piece) = owned_destination { let changed = pieces.splice([(target_span, removal_replacement)]); - let contents = crate::text::rewritten_contents(doc, all, changed); + let contents = crate::text::rewritten_contents(doc, all, changed)?; let mut list = match contents { Object::Array(list) => list, one => vec![one], @@ -1467,7 +1456,7 @@ fn arrange_image( } let changed = pieces.splice(edits); - let contents = crate::text::rewritten_contents(doc, all, changed); + let contents = crate::text::rewritten_contents(doc, all, changed)?; doc.update_dict(p.obj, |d| d.set(b"Contents".to_vec(), contents))?; Ok(()) } @@ -1606,7 +1595,7 @@ fn app_crop_ranges(ops: &[Op], start: usize, target: usize) -> Vec)], img: &PageImage) -> Result<(), EditError> { +fn remove_image_crop(doc: &mut Document, p: &pdfcraft_model::Page, all: &[ContentStream], img: &PageImage) -> Result<(), EditError> { let pieces = joined(all); let ops = pieces.parse().ops; let target = img.op; @@ -1619,7 +1608,7 @@ fn remove_image_crop(doc: &mut Document, p: &pdfcraft_model::Page, all: &[(Objec return Err(EditError::Invalid("the image has no PdfCraft crop to remove".into())); } let changed = crate::text::splice(&pieces, &ops, |index| (Vec::new(), !crops.iter().any(|r| r.contains(&index)))); - let contents = crate::text::rewritten_contents(doc, all, changed); + let contents = crate::text::rewritten_contents(doc, all, changed)?; doc.update_dict(p.obj, |d| d.set(b"Contents".to_vec(), contents))?; Ok(()) } @@ -1666,7 +1655,7 @@ pub fn change_image(doc: &mut Document, page: usize, index: usize, change: &Imag } }; let changed = crate::text::splice(&pieces, &ops, |index| if index == img.op { (replacement.clone(), false) } else { (Vec::new(), true) }); - let contents = crate::text::rewritten_contents(doc, &all, changed); + let contents = crate::text::rewritten_contents(doc, &all, changed)?; doc.update_dict(p.obj, |d| { d.set(b"Contents".to_vec(), contents); if let Some(res) = resources { @@ -1687,7 +1676,7 @@ pub(crate) fn selectable_images( Ok(page_images(doc, page)? .into_iter() .enumerate() - .filter(|(_, image)| !all.get(image.stream).and_then(|(o, _)| o.as_ref()).is_some_and(|r| excluded.contains(&r))) + .filter(|(_, image)| !all.get(image.stream).and_then(|s| s.entry.as_ref()).is_some_and(|r| excluded.contains(&r))) .collect()) } @@ -1700,7 +1689,7 @@ pub(crate) fn painted_over(doc: &Document, page: usize, rect: [f64; 4], from: us fn transform_image( doc: &mut Document, page: &pdfcraft_model::Page, - streams: &[(Object, Vec)], + streams: &[ContentStream], image: &PageImage, transform: [f64; 6], ) -> Result<(), EditError> { @@ -1733,7 +1722,7 @@ fn transform_image( let changed = crate::text::splice(&pieces, &ops, |index| { if index == at { (vec![Op::new("cm", folded.0.map(Object::Real).to_vec())], false) } else { (Vec::new(), true) } }); - let contents = crate::text::rewritten_contents(doc, streams, changed); + let contents = crate::text::rewritten_contents(doc, streams, changed)?; doc.update_dict(page.obj, |d| d.set(b"Contents".to_vec(), contents))?; return Ok(()); } @@ -1754,7 +1743,7 @@ fn transform_image( (Vec::new(), true) } }); - let contents = crate::text::rewritten_contents(doc, streams, changed); + let contents = crate::text::rewritten_contents(doc, streams, changed)?; doc.update_dict(page.obj, |d| d.set(b"Contents".to_vec(), contents))?; Ok(()) } @@ -1766,7 +1755,7 @@ fn transform_image( fn crop_image_impl( doc: &mut Document, page: &pdfcraft_model::Page, - streams: &[(Object, Vec)], + streams: &[ContentStream], image: &PageImage, keep: [f64; 4], ) -> Result<(), EditError> { @@ -1844,7 +1833,7 @@ fn crop_image_impl( (Vec::new(), true) } }); - let contents = crate::text::rewritten_contents(doc, streams, changed); + let contents = crate::text::rewritten_contents(doc, streams, changed)?; doc.update_dict(page.obj, |d| d.set(b"Contents".to_vec(), contents))?; Ok(()) } @@ -1977,7 +1966,7 @@ pub(crate) fn translate_images(doc: &mut Document, page: usize, indexes: &[usize out.push(Op::new("Q", vec![])); (out, false) }); - let contents = crate::text::rewritten_contents(doc, &all, data); + let contents = crate::text::rewritten_contents(doc, &all, data)?; doc.update_dict(p.obj, |d| d.set(b"Contents".to_vec(), contents))?; Ok(()) } diff --git a/crates/edit/src/objects.rs b/crates/edit/src/objects.rs index cdd289457d..dbdbf495ae 100644 --- a/crates/edit/src/objects.rs +++ b/crates/edit/src/objects.rs @@ -4,8 +4,9 @@ use std::collections::{BTreeMap, BTreeSet}; use pdfcraft_content::{Matrix, Op, Pieces}; -use pdfcraft_cos::{Dict, Document, Object}; +use pdfcraft_cos::{DecodeBudget, Dict, Document, Object}; +use crate::text::ContentStream; use crate::{Content, EditError}; pub const MAX_MOVE_OBJECTS: usize = 1_000; @@ -152,7 +153,7 @@ fn added_geometry(doc: &Document, page: usize) -> Result<(Pieces, Vec>()); + let pieces = crate::text::joined(&streams); let ops = pieces.parse().ops; let owned: BTreeSet<_> = added.iter().map(|a| a.obj).collect(); let mut geometry = BTreeMap::new(); @@ -161,7 +162,7 @@ fn added_geometry(doc: &Document, page: usize) -> Result<(Pieces, Vec, EditError>>()?; let data = pieces.splice(edits); - let contents = crate::text::rewritten_contents(doc, &streams, data); + let contents = crate::text::rewritten_contents(doc, &streams, data)?; let refs = match &contents { Object::Array(a) => a.clone(), one => vec![one.clone()], @@ -288,9 +289,9 @@ fn translate_added(doc: &mut Document, page: usize, indexes: &[usize], offset: [ Ok(()) } -/// A missing or undecodable preceding stream can hide graphics state that affects every +/// A missing, undecodable or damaged preceding stream can hide graphics state that affects every /// later item. Refuse group editing rather than inventing an inventory from partial data. -fn checked_streams(doc: &Document, page: usize) -> Result)>, EditError> { +fn checked_streams(doc: &Document, page: usize) -> Result, EditError> { let p = pdfcraft_model::pages(doc).into_iter().nth(page).ok_or(EditError::NoSuchPage(page))?; let contents = match p.dict.get(b"Contents") { None => Vec::new(), @@ -305,6 +306,7 @@ fn checked_streams(doc: &Document, page: usize) -> Result)> }, }; let mut remaining = MAX_CONTENT_BYTES; + let mut decode = DecodeBudget::new(usize::MAX, 0); let mut depth = 0usize; let mut operations = 0usize; let mut streams = Vec::with_capacity(contents.len()); @@ -313,9 +315,12 @@ fn checked_streams(doc: &Document, page: usize) -> Result)> let Object::Stream(stream) = &*resolved else { return Err(EditError::Invalid("a page content stream is missing".into())); }; - let bytes = stream - .decoded_within(remaining.min(64 * 1024 * 1024)) + let (bytes, damaged) = decode + .decode_with_damage(stream, remaining.min(64 * 1024 * 1024)) .map_err(|_| EditError::Invalid("the page content cannot be decoded within its size limit".into()))?; + if damaged { + return Err(EditError::Invalid("a page content stream is damaged partway".into())); + } remaining = remaining.checked_sub(bytes.len()).ok_or_else(|| EditError::Invalid("the page content is too large for group selection".into()))?; let parsed = pdfcraft_content::parse(&bytes); @@ -333,7 +338,7 @@ fn checked_streams(doc: &Document, page: usize) -> Result)> return Err(EditError::Invalid("the page graphics state is nested too deeply".into())); } } - streams.push((object, bytes)); + streams.push(ContentStream { entry: object, data: bytes, damaged: false }); } Ok(streams) } diff --git a/crates/edit/src/tests.rs b/crates/edit/src/tests.rs index 22fbd6e8ae..2f7a310648 100644 --- a/crates/edit/src/tests.rs +++ b/crates/edit/src/tests.rs @@ -5148,6 +5148,66 @@ fn the_graphics_state_carries_over_between_content_streams() { assert!(close(l.rect[0], 100.0) && close(l.size, 10.0), "{l:?}"); } +/// zlib data that reads as `readable` and is then damaged, with more of the stream after the +/// damage: a stored block, then a block of the reserved type (RFC 1951 §3.2.3), then `rest`. +fn damaged_flate(readable: &[u8], rest: &[u8]) -> pdfcraft_cos::Stream { + let len = u16::try_from(readable.len()).unwrap(); + let mut raw = vec![0x78, 0x01, 0x00]; + raw.extend_from_slice(&len.to_le_bytes()); + raw.extend_from_slice(&(!len).to_le_bytes()); + raw.extend_from_slice(readable); + raw.push(0x07); + raw.extend_from_slice(rest); + let mut dict = pdfcraft_cos::Dict::new(); + dict.set(b"Filter".to_vec(), Object::name("FlateDecode")); + pdfcraft_cos::Stream::from_raw(dict, raw) +} + +/// A page whose first content stream is damaged after its text, and whose second is whole. +fn damaged_page() -> (Document, pdfcraft_cos::Stream) { + let mut doc = split_page("", "BT /F1 12 Tf 72 600 Td (Whole) Tj ET"); + let damaged = damaged_flate(b"BT /F1 12 Tf 72 700 Td (Damaged) Tj ET\n", b"BT /F1 12 Tf 72 500 Td (After the damage) Tj ET"); + doc.set(pdfcraft_cos::ObjRef::new(4, 0), damaged.clone()); + (doc, damaged) +} + +/// An edit that would rewrite a content stream damaged partway is refused: written from what +/// decoded, it would lose what follows the damage, which other readers may still read. An edit +/// in the page's other stream goes on, and leaves the damaged one as it is stored. +#[test] +fn an_edit_never_rewrites_a_content_stream_damaged_partway() { + let (mut doc, damaged) = damaged_page(); + let texts = |doc: &Document| text::text_lines(doc, 0).unwrap().into_iter().map(|l| l.text).collect::>(); + assert_eq!(texts(&doc), ["Damaged", "Whole"], "read up to the damage"); + let contents = |doc: &Document| pdfcraft_model::pages(doc)[0].dict.get(b"Contents").cloned(); + let listed = contents(&doc); + + let refused = text::replace_line(&mut doc, 0, 0, "Edited").unwrap_err(); + assert!(refused.to_string().contains("damaged partway"), "{refused}"); + assert_eq!(contents(&doc), listed); + let refused = text::rewrite_block(&mut doc, 0, 0, Some("Edited"), &text::BlockStyle::default()).unwrap_err(); + assert!(refused.to_string().contains("damaged partway"), "{refused}"); + assert_eq!(contents(&doc), listed); + + text::replace_line(&mut doc, 0, 1, "Edited").unwrap(); + let doc = reopen(&doc); + assert_eq!(texts(&doc), ["Damaged", "Edited"]); + let Some(Object::Array(list)) = contents(&doc) else { panic!("contents {:?}", contents(&doc)) }; + assert_eq!(list.first(), Some(&Object::Ref(pdfcraft_cos::ObjRef::new(4, 0)))); + let Object::Stream(kept) = &*doc.get(pdfcraft_cos::ObjRef::new(4, 0)) else { panic!("not a stream") }; + assert_eq!(kept.raw, damaged.raw, "stored as it was"); +} + +/// Objects are selected only from content read whole: what a damaged stream hides past the +/// damage could change the graphics state of every later item. +#[test] +fn objects_are_not_listed_from_a_content_stream_damaged_partway() { + let (doc, _) = damaged_page(); + let refused = editable_objects(&doc, 0).unwrap_err(); + assert!(refused.to_string().contains("damaged partway"), "{refused}"); + assert!(editable_objects(&split_page("", "BT /F1 12 Tf 72 600 Td (Whole) Tj ET"), 0).is_ok()); +} + fn two_stream_image_page(first: &str, second: &str) -> Document { let objs: Vec = vec![ "<< /Type /Catalog /Pages 2 0 R >>".into(), diff --git a/crates/edit/src/text.rs b/crates/edit/src/text.rs index 3d1626a576..01c0acdb70 100644 --- a/crates/edit/src/text.rs +++ b/crates/edit/src/text.rs @@ -55,7 +55,9 @@ use fallback::{add_type3_font, is_type3_fallback}; pub(crate) use fallback::{glyph_path, type3_font_dict, unique_font_name}; use interp::{Origin, Paint, union}; use lines::page_dict; -pub(crate) use lines::{FormBudget, content_streams, content_streams_within, dict_in_place, form_call, resource_dict, rewritten_contents, splice}; +pub(crate) use lines::{ + ContentStream, FormBudget, content_streams, content_streams_within, dict_in_place, form_call, joined, resource_dict, rewritten_contents, splice, +}; pub use lines::{MAX_FORM_DEPTH, MAX_FORM_VISITS, glyph_text_lines, text_lines}; pub use rewrite::{LineEdit, replace_line, rewrite_block}; use rewrite::{coincident_ops, rewrite_block_from}; diff --git a/crates/edit/src/text/blocks.rs b/crates/edit/src/text/blocks.rs index 48fe956f4f..fd2ca0ed18 100644 --- a/crates/edit/src/text/blocks.rs +++ b/crates/edit/src/text/blocks.rs @@ -1,7 +1,7 @@ use super::interp::{Carry, interpret}; -use super::lines::{content_streams, page_dict, reading_lines, rewritten_contents, splice, text_lines}; +use super::lines::{content_streams, joined, page_dict, reading_lines, rewritten_contents, splice, text_lines}; use super::rewrite::{LineEdit, coincident_ops, rewrite_block}; -use super::{Document, EditError, HashMap, Object, Op, Pieces, TextLine, group_blocks, regions}; +use super::{Document, EditError, HashMap, Object, Op, TextLine, group_blocks, regions}; /// An editable text region: adjacent runs on visual rows, followed by aligned rows. /// A region may contain several source fonts, sizes, colours and weights. @@ -270,7 +270,7 @@ pub fn clear_block(doc: &mut Document, page: usize, block: usize) -> Result<(), let p = page_dict(doc, page)?; let res = p.dict.get(b"Resources").and_then(|r| doc.resolve(r).as_dict().cloned()).unwrap_or_default(); let streams = content_streams(doc, &p.dict)?; - let pieces = Pieces::join(&streams.iter().map(|(_, d)| d.as_slice()).collect::>()); + let pieces = joined(&streams); let ops = pieces.parse().ops; let shown = interpret(doc, &ops, &res, &mut HashMap::new(), &mut Carry::new(), None, None)?; let mut patches = HashMap::new(); @@ -297,7 +297,7 @@ pub fn clear_block(doc: &mut Document, page: usize, block: usize) -> Result<(), patches.insert(s.op, replacement); } let data = splice(&pieces, &ops, |i| patches.remove(&i).map_or((Vec::new(), true), |v| (v, false))); - let contents = rewritten_contents(doc, &streams, data); + let contents = rewritten_contents(doc, &streams, data)?; doc.update_dict(p.obj, |d| d.set(b"Contents".to_vec(), contents))?; Ok(()) } diff --git a/crates/edit/src/text/lines.rs b/crates/edit/src/text/lines.rs index 85209a5472..a28806078b 100644 --- a/crates/edit/src/text/lines.rs +++ b/crates/edit/src/text/lines.rs @@ -61,16 +61,27 @@ impl TextLine { } } -/// The page's content streams as listed, each with its decoded data (empty when it doesn't -/// decode), within one page's [`DecodeBudget`]. An error when the content runs past it: reading -/// or rewriting a page from part of its content would lose the rest. -pub(crate) fn content_streams(doc: &Document, page: &Dict) -> Result)>, EditError> { +/// One entry of a page's `/Contents`, and what it decoded to. +pub(crate) struct ContentStream { + /// The entry as listed. + pub(crate) entry: Object, + /// Its decoded data, empty when it doesn't decode. + pub(crate) data: Vec, + /// The stream is damaged partway: `data` is what decoded before the damage, so the stream + /// can't be written again from it without losing the rest ([`rewritten_contents`]). + pub(crate) damaged: bool, +} + +/// The page's content streams as listed, each with its decoded data, within one page's +/// [`DecodeBudget`]. An error when the content runs past it: reading or rewriting a page from +/// part of its content would lose the rest. +pub(crate) fn content_streams(doc: &Document, page: &Dict) -> Result, EditError> { content_streams_within(doc, page, &mut DecodeBudget::page()) } /// [`content_streams`], decoded against `budget`, which a walk of the page's forms then goes on /// spending. -pub(crate) fn content_streams_within(doc: &Document, page: &Dict, budget: &mut DecodeBudget) -> Result)>, EditError> { +pub(crate) fn content_streams_within(doc: &Document, page: &Dict, budget: &mut DecodeBudget) -> Result, EditError> { let list: Vec = match page.get(b"Contents") { None => Vec::new(), Some(c) => match &*doc.resolve(c) { @@ -79,20 +90,25 @@ pub(crate) fn content_streams_within(doc: &Document, page: &Dict, budget: &mut D }, }; let mut out = Vec::with_capacity(list.len()); - for o in list { - let data = match &*doc.resolve(&o) { - Object::Stream(s) => match budget.decode(s) { - Ok(data) => data, + for entry in list { + let (data, damaged) = match &*doc.resolve(&entry) { + Object::Stream(s) => match budget.decode_with_damage(s, usize::MAX) { + Ok(read) => read, Err(e) if budget.remaining() == 0 => return Err(EditError::Invalid(format!("the page content is too large to read: {e}"))), - Err(_) => Vec::new(), + Err(_) => (Vec::new(), false), }, - _ => Vec::new(), + _ => (Vec::new(), false), }; - out.push((o, data)); + out.push(ContentStream { entry, data, damaged }); } Ok(out) } +/// The page's streams joined into the one stream they are: an operator may span two of them. +pub(crate) fn joined(streams: &[ContentStream]) -> Pieces { + Pieces::join(&streams.iter().map(|s| s.data.as_slice()).collect::>()) +} + /// Rewrite a page's content (its streams joined, see [`Pieces`]): `edit(i)` gives the operators /// to insert before operator `i` of `ops` (parsed from the joined pieces) and whether to keep it. /// Everything else is copied byte for byte. A page's streams are one stream in pieces, split @@ -121,25 +137,32 @@ pub(crate) fn splice(pieces: &Pieces, ops: &[Op], mut edit: impl FnMut(usize) -> } /// The page's `/Contents` after a [`splice`]: pieces whose bytes changed become new streams -/// (keeping their dictionaries), the others stay as they were. -pub(crate) fn rewritten_contents(doc: &mut Document, streams: &[(Object, Vec)], new: Vec>) -> Object { +/// (keeping their dictionaries), the others stay as they were. An error, before anything is +/// added, when a piece that changed is a damaged stream: written from what decoded, it would +/// lose everything after the damage. +pub(crate) fn rewritten_contents(doc: &mut Document, streams: &[ContentStream], new: Vec>) -> Result { + if streams.iter().zip(&new).any(|(s, data)| s.damaged && s.data != *data) { + return Err(EditError::Invalid( + "this change is in a content stream that is damaged partway, and rewriting it would lose what follows the damage".into(), + )); + } let mut contents: Vec = Vec::with_capacity(streams.len()); - for ((obj, old), data) in streams.iter().zip(new) { - if *old == data { - contents.push(obj.clone()); + for (s, data) in streams.iter().zip(new) { + if s.data == data { + contents.push(s.entry.clone()); continue; } - let mut dict = match &*doc.resolve(obj) { + let mut dict = match &*doc.resolve(&s.entry) { Object::Stream(s) => s.dict.clone(), _ => Dict::new(), }; dict.remove(b"Length"); contents.push(Object::Ref(doc.add(Object::Stream(Stream::flate(dict, &data))))); } - match <[Object; 1]>::try_from(contents) { + Ok(match <[Object; 1]>::try_from(contents) { Ok([one]) => one, Err(many) => Object::Array(many), - } + }) } pub(crate) fn page_dict(doc: &Document, page: usize) -> Result { @@ -201,7 +224,7 @@ fn text_lines_in(doc: &Document, p: &pdfcraft_model::Page, actual_text: bool) -> let mut carry = Carry::new(); // The streams are one stream in pieces (§7.8.2), split between any two tokens: parse them // joined, so an operator whose operands end one piece still has them. - let pieces = Pieces::join(&content_streams(doc, &p.dict)?.into_iter().map(|(_, d)| d).collect::>()); + let pieces = joined(&content_streams(doc, &p.dict)?); let ops = pieces.parse().ops; let properties = res.get(b"Properties").and_then(|o| doc.dict(o)).unwrap_or_default(); let shown = interpret(doc, &ops, &res, &mut cache, &mut carry, None, actual_text.then_some(&properties))?; @@ -608,7 +631,7 @@ pub(super) fn reading_lines(doc: &Document, page: usize) -> Result<(Vec>()); + let pieces = joined(&streams); let ops = pieces.parse().ops; let stream_of = |i: usize| ops.get(i).map_or(0, |op| pieces.pieces_of(op).0); walk(&mut w, &ops, &res, &mut carry, &stream_of)?; diff --git a/crates/edit/src/text/rewrite.rs b/crates/edit/src/text/rewrite.rs index d27d23e0b5..e7fde8545a 100644 --- a/crates/edit/src/text/rewrite.rs +++ b/crates/edit/src/text/rewrite.rs @@ -4,10 +4,10 @@ use super::fallback::{ source_program_completes, source_show, source_space, source_width, type3_encode, type3_font, }; use super::interp::{Carry, Origin, TextState, interpret}; -use super::lines::{content_streams, page_dict, rewritten_contents, splice, text_lines}; +use super::lines::{content_streams, joined, page_dict, rewritten_contents, splice, text_lines}; use super::{ - Dict, Document, EditError, HashMap, Matrix, Metrics, Object, Op, PdfString, Pieces, TextLine, TextSession, alignment_frame, group_blocks, - is_centred, parse, rewrite_styled_block, + Dict, Document, EditError, HashMap, Matrix, Metrics, Object, Op, PdfString, TextLine, TextSession, alignment_frame, group_blocks, is_centred, + parse, rewrite_styled_block, }; /// What replacing a line did. @@ -56,7 +56,7 @@ pub(crate) fn replace_line_with_limit(doc: &mut Document, page: usize, line: usi let mut res = p.dict.get(b"Resources").map(|r| doc.resolve(r)).and_then(|r| r.as_dict().cloned()).unwrap_or_default(); let mut fonts_res = res.get(b"Font").map(|f| doc.resolve(f)).and_then(|f| f.as_dict().cloned()).unwrap_or_default(); let streams = content_streams(doc, &p.dict)?; - let pieces = Pieces::join(&streams.iter().map(|(_, d)| d.as_slice()).collect::>()); + let pieces = joined(&streams); let ops = pieces.parse().ops; let first = target.ops.first().copied().filter(|i| *i < ops.len()).ok_or_else(|| EditError::Invalid("the page's content changed".into()))?; // The line's own font, when it can show every character. @@ -127,7 +127,7 @@ pub(crate) fn replace_line_with_limit(doc: &mut Document, page: usize, line: usi } // Rebuild: the line's first operator becomes the replacement, its others go. let new_data = splice(&pieces, &ops, |i| if i == first { (std::mem::take(&mut replacement), false) } else { (Vec::new(), !drops.contains(&i)) }); - let contents = rewritten_contents(doc, &streams, new_data); + let contents = rewritten_contents(doc, &streams, new_data)?; let page_ref = p.obj; if substituted.is_some() { res.set(b"Font".to_vec(), Object::Dict(fonts_res)); @@ -834,7 +834,7 @@ pub(crate) fn rewrite_block_from( } let upright = dispatch.upright; let streams = content_streams(doc, &p.dict)?; - let pieces = Pieces::join(&streams.iter().map(|(_, d)| d.as_slice()).collect::>()); + let pieces = joined(&streams); let ops = pieces.parse().ops; let text = paragraph_text(text.unwrap_or(&b.text)); // The characters to show, for choosing fonts: the breaks are moves, not glyphs. @@ -970,7 +970,7 @@ pub(crate) fn rewrite_block_from( let insert = if i == at { std::mem::take(&mut block_ops) } else { Vec::new() }; (insert, !dropped.contains(&i)) }); - let contents = rewritten_contents(doc, &streams, new_data); + let contents = rewritten_contents(doc, &streams, new_data)?; let new_font = !reuse; if new_font { res.set(b"Font".to_vec(), Object::Dict(fonts_res)); @@ -1040,7 +1040,7 @@ pub(crate) fn selectable_blocks( !b.lines .iter() .filter_map(|i| lines.get(*i)) - .any(|l| streams.get(l.stream).and_then(|(o, _)| o.as_ref()).is_some_and(|r| excluded.contains(&r))) + .any(|l| streams.get(l.stream).and_then(|s| s.entry.as_ref()).is_some_and(|r| excluded.contains(&r))) }) .collect()) } @@ -1134,7 +1134,7 @@ pub(crate) fn translate_blocks(doc: &mut Document, page: usize, indexes: &[usize } let res = p.dict.get(b"Resources").map(|r| doc.resolve(r)).and_then(|r| r.as_dict().cloned()).unwrap_or_default(); let fonts = res.get(b"Font").map(|r| doc.resolve(r)).and_then(|r| r.as_dict().cloned()).unwrap_or_default(); - let pieces = Pieces::join(&streams.iter().map(|(_, d)| d.as_slice()).collect::>()); + let pieces = joined(&streams); let ops = pieces.parse().ops; let shown = interpret(doc, &ops, &res, &mut HashMap::new(), &mut Carry::new(), None, None)?; let mut patches = HashMap::new(); @@ -1205,7 +1205,7 @@ pub(crate) fn translate_blocks(doc: &mut Document, page: usize, indexes: &[usize return Err(EditError::Invalid("a selected text operation cannot be moved".into())); } let data = splice(&pieces, &ops, |i| patches.remove(&i).map_or((Vec::new(), true), |v| (v, false))); - let contents = rewritten_contents(doc, &streams, data); + let contents = rewritten_contents(doc, &streams, data)?; doc.update_dict(p.obj, |d| d.set(b"Contents".to_vec(), contents))?; Ok(()) } diff --git a/crates/edit/src/text_regions/styled.rs b/crates/edit/src/text_regions/styled.rs index 7a9204a8c0..442913b61b 100644 --- a/crates/edit/src/text_regions/styled.rs +++ b/crates/edit/src/text_regions/styled.rs @@ -962,7 +962,7 @@ pub(crate) fn rewrite_styled_block( }; let (rows, row_ranges, row_lines, row_soft, row_gap) = wrap_styled_rows(&chars, width, measure)?; let streams = content_streams(doc, &p.dict)?; - let pieces = Pieces::join(&streams.iter().map(|(_, d)| d.as_slice()).collect::>()); + let pieces = crate::text::joined(&streams); let ops = pieces.parse().ops; // A row the edit doesn't reach keeps its source operators, so its kerning, TJ word gaps and // glyph widths stay exactly as they were instead of being rebuilt from the text state. A row @@ -1206,7 +1206,7 @@ pub(crate) fn rewrite_styled_block( // when the rewrite splits one, and a kept row's operators come later in the stream. let marker = if marked { None } else { paragraph_marker(block, lines, &ops) }; let data = splice_with_marker(&pieces, &ops, marker, at, block_ops, &drop); - let contents = rewritten_contents(doc, &streams, data); + let contents = rewritten_contents(doc, &streams, data)?; res.set(b"Font".to_vec(), Object::Dict(fonts)); doc.update_dict(p.obj, |d| { d.set(b"Contents".to_vec(), contents); From 480fdb114abcb039aea93828085574b27045e0a5 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Thu, 5 Nov 2026 07:55:04 +0100 Subject: [PATCH 153/198] M2.6: Optimize leaves a page damaged partway as it is stored Optimize read a page's content up to any damage in its streams as the whole of it. Merging a page's image strips then wrote the page's content again as one stream, without what followed the damage, and the image pass measured each image by its uses before the damage only, so one drawn larger past it could be resampled below that use. - `DecodeBudget::decode_contents_with_damage` reads a page's `/Contents` as `decode_contents` does, each stream also saying whether it is damaged partway; `decode_contents` is it without the flag. - `content_bytes` gives `None` for a page with a damaged stream, as for one that doesn't decode: the content is not all known. Its strips aren't merged (`merge_page` already leaves such a page alone), and no image is resampled (`effective_resolutions` gives every image `INFINITY`). - A form damaged partway leaves the measuring incomplete in the same way. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/cos/src/budget.rs | 28 +++++++++++++- crates/optimize/README.md | 5 ++- crates/optimize/src/images.rs | 13 ++++--- crates/optimize/tests/optimize.rs | 63 +++++++++++++++++++++++++++++++ 4 files changed, 100 insertions(+), 9 deletions(-) diff --git a/crates/cos/src/budget.rs b/crates/cos/src/budget.rs index 3211efb273..41dff04ed4 100644 --- a/crates/cos/src/budget.rs +++ b/crates/cos/src/budget.rs @@ -348,12 +348,18 @@ impl DecodeBudget { /// A stream that fails to decode, or arrives after the allowance is spent, is an `Err` in its /// place, so the caller chooses between skipping it and giving up. pub fn decode_contents(&mut self, doc: &Document, contents: &Object) -> Vec, CosError>> { + self.decode_contents_with_damage(doc, contents).into_iter().map(|read| read.map(|(data, _)| data)).collect() + } + + /// [`DecodeBudget::decode_contents`], each stream also saying whether it is damaged partway + /// ([`DecodeBudget::decode_with_damage`]). + pub fn decode_contents_with_damage(&mut self, doc: &Document, contents: &Object) -> Vec, bool), CosError>> { match &*doc.resolve(contents) { - Object::Stream(s) => vec![self.decode(s)], + Object::Stream(s) => vec![self.decode_with_damage(s, usize::MAX)], Object::Array(list) => list .iter() .filter_map(|entry| match &*doc.resolve(entry) { - Object::Stream(s) => Some(self.decode(s)), + Object::Stream(s) => Some(self.decode_with_damage(s, usize::MAX)), _ => None, }) .collect(), @@ -842,6 +848,24 @@ mod tests { assert_eq!(budget.decode_with_damage(&hex, usize::MAX).ok(), Some((b"AB".to_vec(), true))); } + /// A page's content streams each say whether they are damaged, in their places. + #[test] + fn a_pages_streams_each_say_whether_they_are_damaged() { + let mut doc = Document::new_empty(); + let whole = doc.add(Object::Stream(Stream::flate(Dict::new(), b"0 0 m 1 1 l S"))); + let Object::Stream(s) = &*doc.get(whole) else { panic!("not a stream") }; + let cut = Stream::from_raw(s.dict.clone(), s.raw.get(..s.raw.len() - 6).unwrap_or_default().to_vec()); + let cut = doc.add(Object::Stream(cut)); + let contents = Object::Array(vec![Object::Ref(whole), Object::Null, Object::Ref(cut)]); + let mut budget = DecodeBudget::page(); + let read: Vec<_> = budget.decode_contents_with_damage(&doc, &contents).into_iter().map(|r| r.map(|(_, damaged)| damaged).ok()).collect(); + assert_eq!(read, [Some(false), Some(true)]); + let one: Vec<_> = + budget.decode_contents_with_damage(&doc, &Object::Ref(cut)).into_iter().map(|r| r.map(|(_, damaged)| damaged).ok()).collect(); + assert_eq!(one, [Some(true)]); + assert_eq!(budget.decode_contents(&doc, &contents).iter().filter(|r| r.is_ok()).count(), 2, "read up to the damage"); + } + /// A stream that fails its reading as zlib, and then as raw deflate after decoding all of it, /// gives nothing back, and each reference still pays for what it decoded. #[test] diff --git a/crates/optimize/README.md b/crates/optimize/README.md index 9acb076908..fd7c60ae85 100644 --- a/crates/optimize/README.md +++ b/crates/optimize/README.md @@ -22,11 +22,14 @@ let report = optimize_with_progress(&mut doc, &settings, &mut |stage| keep_going printed this way went from 4.3 MB to 231 KB (Standard) and 161 KB (Mobile). Not merged yet: side-by-side tiles, rotated strips, masks. Content arrays use the shared space boundary policy, so trailing comments, split operands and graphics state keep their meaning after merging. + A page whose content doesn't decode, or is damaged partway, is left as it is: written again from + what decoded, it would lose the rest. - **Images:** the effective resolution of each image is measured where pages draw it (the CTM through form XObjects, up to 12 deep; the smallest over all uses). Page-local `/UserUnit` scaling is included, so large-format pages are measured in physical inches. If a use can't be - measured (forms nested deeper, or drawn too many times), no image is resampled. Colour and grayscale images above a + measured (forms nested deeper or drawn too many times, or content that doesn't decode or is + damaged partway, which may draw an image larger past the damage), no image is resampled. Colour and grayscale images above a threshold are resampled with a bicubic (Catmull-Rom) filter to a target resolution and stored as JPEG (Acrobat's quality steps) or ZIP; soft masks are resampled with their image. A stencil `/Mask` has its own resolution, so the picture changes under it and it stays as it is. diff --git a/crates/optimize/src/images.rs b/crates/optimize/src/images.rs index 159ba04e8c..21f6953a8d 100644 --- a/crates/optimize/src/images.rs +++ b/crates/optimize/src/images.rs @@ -32,9 +32,10 @@ fn held_page_resources(doc: &Document, page: ObjRef) -> Option<(Dict, ObjRef)> { } /// The page content `contents` names, decoded against `budget` and joined. `None` when a stream -/// doesn't decode or the budget runs out: the content is not all known. +/// doesn't decode, is damaged partway or the budget runs out: the content is not all known. pub(crate) fn content_bytes(doc: &Document, contents: &Object, budget: &mut DecodeBudget) -> Option> { - let pieces = budget.decode_contents(doc, contents).into_iter().collect::, _>>().ok()?; + let reads = budget.decode_contents_with_damage(doc, contents); + let pieces = reads.into_iter().map(|read| read.ok().filter(|(_, damaged)| !damaged).map(|(data, _)| data)).collect::>>()?; Some(Pieces::join(&pieces).data().to_vec()) } @@ -214,7 +215,7 @@ impl Walk<'_> { self.complete = false; return; } - let Ok(data) = self.budget.decode(s) else { + let Ok((data, false)) = self.budget.decode_with_damage(s, usize::MAX) else { self.complete = false; return; }; @@ -245,9 +246,9 @@ fn remaps_device_spaces(doc: &Document, resources: &Dict) -> bool { /// The effective resolution (ppi) of every image drawn by `pages`: the smallest over its uses. /// -/// If forms nest too deeply, are drawn too many times, or a page's content doesn't decode within -/// its budget, every image is given `INFINITY` (never resampled), since its largest use may be -/// among those missed. +/// If forms nest too deeply, are drawn too many times, or a page's or form's content doesn't +/// decode within its budget or is damaged partway, every image is given `INFINITY` (never +/// resampled), since its largest use may be among those missed. pub fn effective_resolutions(doc: &Document, pages: &[ObjRef]) -> HashMap { walk_pages(doc, pages).0 } diff --git a/crates/optimize/tests/optimize.rs b/crates/optimize/tests/optimize.rs index 1387cae528..7250e9a0a0 100644 --- a/crates/optimize/tests/optimize.rs +++ b/crates/optimize/tests/optimize.rs @@ -3157,6 +3157,69 @@ fn images_that_do_not_tile_a_picture_are_left_alone() { unmerged(&mut off, &Settings { merge_fragments: false, ..Settings::default() }); } +/// zlib data that reads as `readable` and is then damaged, with more of the stream after the +/// damage: a stored block, then a block of the reserved type (RFC 1951 §3.2.3), then `rest`. +fn damaged_flate(readable: &[u8], rest: &[u8]) -> Stream { + let len = u16::try_from(readable.len()).unwrap(); + let mut raw = vec![0x78, 0x01, 0x00]; + raw.extend_from_slice(&len.to_le_bytes()); + raw.extend_from_slice(&(!len).to_le_bytes()); + raw.extend_from_slice(readable); + raw.push(0x07); + raw.extend_from_slice(rest); + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::name("FlateDecode")); + Stream::from_raw(dict, raw) +} + +/// A page whose content is damaged partway isn't all known: its strips aren't merged, which would +/// write its content again without what follows the damage. +#[test] +fn strips_on_a_page_damaged_partway_are_left_as_they_are() { + let (mut doc, _) = printed_page(40, |_| false, |_, s| s); + let p = pdfcraft_annot::page_refs(&doc).unwrap()[0]; + let bad = damaged_flate(b"0 g\n", b"q 1000 0 0 1 0 0 cm /R0 Do Q"); + let damaged = doc.add(Object::Stream(bad.clone())); + let first = doc.get(p).as_dict().unwrap().get(b"Contents").cloned().unwrap(); + let contents = Object::Array(vec![first, Object::Ref(damaged)]); + doc.update_dict(p, |d| d.set(b"Contents".to_vec(), contents.clone())).unwrap(); + let r = optimize(&mut doc, &Settings::default()).unwrap(); + assert_eq!((r.merged_images, r.fragments_merged), (0, 0), "{r:?}"); + assert_eq!(doc.get(p).as_dict().unwrap().get(b"Contents"), Some(&contents)); + assert_eq!(stream(&doc, damaged), bad); +} + +/// An image may be drawn larger past the damage in a page's or a form's content, so no image is +/// resampled for the uses before it. +#[test] +fn content_damaged_partway_leaves_every_image_at_its_resolution() { + for in_form in [false, true] { + let mut doc = Document::new_empty(); + let img = image(&mut doc, 720, 720, 1, false, None); + let p = page(&mut doc, &[("Im", img)], "q 72 0 0 72 0 0 cm /Im Do Q"); + assert_eq!(effective_resolutions(&doc, &[p])[&img], 720.0); + let mut bad = damaged_flate(b"q 72 0 0 72 0 0 cm /Im Do Q\n", b"q 720 0 0 720 0 0 cm /Im Do Q"); + let added = if in_form { + bad.dict.set(b"Subtype".to_vec(), Object::name("Form")); + bad.dict.set(b"BBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 1.into(), 1.into()])); + bad.dict.set(b"Resources".to_vec(), Object::Dict(resources(xobjects(&[("Im", img)])))); + let f = doc.add(Object::Stream(bad)); + let res = Object::Dict(resources(xobjects(&[("Im", img), ("F", f)]))); + doc.update_dict(p, |d| d.set(b"Resources".to_vec(), res)).unwrap(); + content(&mut doc, "/F Do") + } else { + doc.add(Object::Stream(bad)) + }; + let first = doc.get(p).as_dict().unwrap().get(b"Contents").cloned().unwrap(); + doc.update_dict(p, |d| d.set(b"Contents".to_vec(), Object::Array(vec![first, Object::Ref(added)]))).unwrap(); + // Not measured, or measured as never resampled: the image pass reads only what was measured. + let ppi = effective_resolutions(&doc, &[p]).get(&img).copied().unwrap_or(f64::INFINITY); + assert_eq!(ppi, f64::INFINITY, "in a form: {in_form}"); + let r = optimize(&mut doc, &Settings::default()).unwrap(); + assert_eq!(r.images_resampled, 0, "{r:?}"); + } +} + /// `levels` nested forms with their own resources, the innermost drawing `img` over the unit square. fn nested_forms(doc: &mut Document, img: ObjRef, levels: usize) -> ObjRef { let mut inner = form(doc, Some(Object::Dict(resources(xobjects(&[("Im", img)])))), "/Im Do"); From 50d0fcc403ff57bba98bf0a3285de692a72bf356 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Thu, 5 Nov 2026 07:55:04 +0100 Subject: [PATCH 154/198] M2.6: pruning keeps what a page or form damaged partway may draw Extracting, splitting and combining pages drops the `/XObject` entries no page draws. A page or form damaged partway was read up to the damage as the whole of it, so an image named only after the damage was dropped, though the stream still names it and other readers may still read that far. - `content_bytes` says whether a page's stream is damaged, and pruning keeps everything such a page could reach, as it does for content that doesn't decode. A form is read the same way. - A page made a form XObject (stamps, watermarks, backgrounds from a PDF) still draws what decoded before the damage. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/organize/src/import.rs | 5 +++-- crates/organize/src/prune.rs | 29 +++++++++++++++---------- crates/organize/src/tests.rs | 41 +++++++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 14 deletions(-) diff --git a/crates/organize/src/import.rs b/crates/organize/src/import.rs index 9f86d6455a..b52c9175b3 100644 --- a/crates/organize/src/import.rs +++ b/crates/organize/src/import.rs @@ -499,9 +499,10 @@ pub fn page_as_form(dst: &mut Document, src: &Document, page: usize) -> Result<( let bbox = rect(b"CropBox").or_else(|| rect(b"MediaBox")).unwrap_or([0.0, 0.0, 612.0, 792.0]); let (w, h) = (bbox[2] - bbox[0], bbox[3] - bbox[1]); let rotate = d.get(b"Rotate").and_then(|r| src.resolve(r).as_int()).unwrap_or(0).rem_euclid(360); - // Content: the page's streams, decoded and joined. + // Content: the page's streams, decoded and joined. Of a stream damaged partway, the copy draws + // what decoded before the damage. let content = match own.get(b"Contents") { - Some(c) => crate::prune::content_bytes(src, c).map_err(|e| OrganizeError::Invalid(format!("page {} content: {e}", page + 1)))?, + Some(c) => crate::prune::content_bytes(src, c).map_err(|e| OrganizeError::Invalid(format!("page {} content: {e}", page + 1)))?.0, None => Vec::new(), }; let mut copier = Copier::new(src, HashMap::new(), HashMap::new()); diff --git a/crates/organize/src/prune.rs b/crates/organize/src/prune.rs index 5a2a319fb4..8683b4d810 100644 --- a/crates/organize/src/prune.rs +++ b/crates/organize/src/prune.rs @@ -219,10 +219,10 @@ impl Pruner<'_> { } #[cfg(test)] crate::work(1); - match s.decoded() { - Ok(data) => self.content(&data, &inner, inner_owner, depth + 1), - // What the form draws is unknown: keep everything it could reach. - Err(_) => { + match DecodeBudget::new(usize::MAX, 0).decode_with_damage(s, usize::MAX) { + Ok((data, false)) => self.content(&data, &inner, inner_owner, depth + 1), + // What the form draws is unknown, past damage too: keep everything it could reach. + Ok((_, true)) | Err(_) => { self.protect_type3_resources(&inner, inner_owner); self.distrust(&inner, inner_owner); } @@ -232,16 +232,20 @@ impl Pruner<'_> { } /// The page's content, concatenated (`/Contents` may be one stream or an array) with a space -/// between pieces as the renderer joins them, decoded against one page's [`DecodeBudget`]. An +/// between pieces as the renderer joins them, decoded against one page's [`DecodeBudget`], and +/// whether a stream is damaged partway (its part is then what decoded before the damage). An /// error when a stream can't be decoded or the budget runs out: the page may draw names that /// can't be seen. -pub(crate) fn content_bytes(doc: &Document, contents: &Object) -> Result, CosError> { +pub(crate) fn content_bytes(doc: &Document, contents: &Object) -> Result<(Vec, bool), CosError> { let mut out = Vec::new(); - for data in DecodeBudget::page().decode_contents(doc, contents) { - out.extend(data?); + let mut damaged = false; + for read in DecodeBudget::page().decode_contents_with_damage(doc, contents) { + let (data, partial) = read?; + damaged |= partial; + out.extend(data); out.push(b' '); } - Ok(out) + Ok((out, damaged)) } /// The resources of a page (as `walk` reports them) with their owner. @@ -324,9 +328,10 @@ pub(crate) fn prune_unused_xobjects(doc: &mut Document) -> Result {} - Ok(data) => pruner.content(&data, &res_dict, owner, 0), - Err(_) => { + Ok((data, false)) if data.is_empty() => {} + Ok((data, false)) => pruner.content(&data, &res_dict, owner, 0), + // What the page draws past damage is unknown too. + Ok((_, true)) | Err(_) => { pruner.protect_type3_resources(&res_dict, owner); pruner.distrust(&res_dict, owner); } diff --git a/crates/organize/src/tests.rs b/crates/organize/src/tests.rs index d5ff2116ff..40bb4210d4 100644 --- a/crates/organize/src/tests.rs +++ b/crates/organize/src/tests.rs @@ -2628,6 +2628,47 @@ fn extract_keeps_images_an_undecodable_page_might_draw() { assert!(names.iter().any(|n| n.as_slice() == b"ImB"), "an image the unreadable page may draw must stay"); } +/// zlib data that reads as `readable` and is then damaged, with more of the stream after the +/// damage: a stored block, then a block of the reserved type (RFC 1951 §3.2.3), then `rest`. +fn damaged_flate(readable: &[u8], rest: &[u8]) -> pdfcraft_cos::Stream { + let len = u16::try_from(readable.len()).unwrap(); + let mut raw = vec![0x78, 0x01, 0x00]; + raw.extend_from_slice(&len.to_le_bytes()); + raw.extend_from_slice(&(!len).to_le_bytes()); + raw.extend_from_slice(readable); + raw.push(0x07); + raw.extend_from_slice(rest); + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::name("FlateDecode")); + pdfcraft_cos::Stream::from_raw(dict, raw) +} + +/// A page or form damaged partway may draw images past the damage, which other readers may still +/// read: pruning keeps them, as it does for content that doesn't decode. +#[test] +fn extract_keeps_images_content_damaged_partway_might_draw() { + use pdfcraft_cos::ObjRef; + let mut page = doc_images_with(&raw("/ImA Do"), ""); + page.set(ObjRef::new(6, 0), damaged_flate(b"/ImA Do\n", b"/ImB Do")); + + let mut form = doc_images_with(&raw("/Fm Do"), ""); + let mut fm = damaged_flate(b"/ImA Do\n", b"/ImB Do"); + fm.dict.set(b"Subtype".to_vec(), Object::name("Form")); + fm.dict.set(b"BBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 1.into(), 1.into()])); + let fm = form.add(fm); + let xobjects: Dict = + [(b"ImA".to_vec(), Object::Ref(ObjRef::new(8, 0))), (b"ImB".to_vec(), Object::Ref(ObjRef::new(9, 0))), (b"Fm".to_vec(), Object::Ref(fm))] + .into_iter() + .collect(); + form.update_dict(ObjRef::new(7, 0), |d| d.set(b"XObject".to_vec(), Object::Dict(xobjects))).unwrap(); + + for (src, what) in [(page, "page"), (form, "form")] { + let part = full_roundtrip(&extract_pages(&src, &[0, 1]).unwrap()); + let names = xobject_names(&part, 0); + assert!(names.iter().any(|n| n.as_slice() == b"ImB"), "{what}: an image drawn past the damage must stay: {names:?}"); + } +} + #[test] fn extract_keeps_images_type3_glyphs_may_draw() { // A Type 3 font without its own /Resources draws through the page's: its glyphs could `Do`. From 3fec35e1ea1f910219b9c2f429643d400c73a3e4 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Thu, 5 Nov 2026 08:14:47 +0100 Subject: [PATCH 155/198] M2.6: tests pin that content which doesn't decode is kept and edited around A stream that doesn't decode at all is not damage to write over: edit and accessibility keep it as stored and go on with the page's other streams, and only running past the page's decode allowance refuses. Nothing pinned either, nor that selecting objects reads a stream of a few MB. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/a11y/src/tests.rs | 17 +++++++++++++++++ crates/edit/src/tests.rs | 20 ++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/crates/a11y/src/tests.rs b/crates/a11y/src/tests.rs index b551c2cb42..a129f207cd 100644 --- a/crates/a11y/src/tests.rs +++ b/crates/a11y/src/tests.rs @@ -1129,6 +1129,23 @@ fn a_tags_content_in_a_stream_damaged_partway_is_not_rewritten() { assert_ne!(list.get(1), Some(&Object::Ref(ObjRef::new(4, 0))), "the tag's stream is rewritten"); } +/// A content stream that doesn't decode at all is kept as stored, and a tag in the page's other +/// streams is still marked: only running past the decode budget refuses. +#[test] +fn a_tag_beside_a_stream_that_does_not_decode_is_marked() { + use pdfcraft_cos::{Dict, ObjRef, Object, Stream}; + let mut doc = figures_doc(); + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::name("NoSuchFilter")); + let other = doc.add(Object::Stream(Stream::from_raw(dict, b"0 g".to_vec()))); + doc.update_dict(ObjRef::new(3, 0), |p| p.set(b"Contents".to_vec(), Object::Array(vec![Object::Ref(other), Object::Ref(ObjRef::new(4, 0))]))) + .unwrap(); + mark_tag_artifact(&mut doc, ObjRef::new(7, 0)).unwrap(); + let contents = doc.get(ObjRef::new(3, 0)).as_dict().unwrap().get(b"Contents").cloned(); + let Some(Object::Array(list)) = contents else { panic!("contents {contents:?}") }; + assert_eq!(list.first(), Some(&Object::Ref(other)), "kept as stored"); +} + fn ref_(n: u32) -> Option { Some(pdfcraft_cos::ObjRef::new(n, 0)) } diff --git a/crates/edit/src/tests.rs b/crates/edit/src/tests.rs index 2f7a310648..dc80cb6a29 100644 --- a/crates/edit/src/tests.rs +++ b/crates/edit/src/tests.rs @@ -5198,6 +5198,26 @@ fn an_edit_never_rewrites_a_content_stream_damaged_partway() { assert_eq!(kept.raw, damaged.raw, "stored as it was"); } +/// A content stream that doesn't decode at all is no damage to write over: it is kept as stored, +/// and the page's other streams are read and edited around it. A stream of a few MB is read for +/// selecting objects. +#[test] +fn a_content_stream_that_does_not_decode_is_kept_and_edited_around() { + let mut doc = split_page("", "BT /F1 12 Tf 72 600 Td (Whole) Tj ET"); + let mut dict = pdfcraft_cos::Dict::new(); + dict.set(b"Filter".to_vec(), Object::name("NoSuchFilter")); + let unknown = pdfcraft_cos::Stream::from_raw(dict, b"BT /F1 12 Tf 72 700 Td (Unread) Tj ET".to_vec()); + doc.set(pdfcraft_cos::ObjRef::new(4, 0), unknown.clone()); + text::replace_line(&mut doc, 0, 0, "Edited").unwrap(); + let doc = reopen(&doc); + assert_eq!(text::text_lines(&doc, 0).unwrap().into_iter().map(|l| l.text).collect::>(), ["Edited"]); + let Object::Stream(kept) = &*doc.get(pdfcraft_cos::ObjRef::new(4, 0)) else { panic!("not a stream") }; + assert_eq!(kept.raw, unknown.raw, "stored as it was"); + + let large = format!("{}0 0 m 10 10 l S", " ".repeat(2 << 20)); + assert!(editable_objects(&split_page(&large, "BT /F1 12 Tf 72 600 Td (Whole) Tj ET"), 0).is_ok()); +} + /// Objects are selected only from content read whole: what a damaged stream hides past the /// damage could change the graphics state of every later item. #[test] From 69e26167e4035d385e905f2563d6b6156266fbf8 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Thu, 5 Nov 2026 08:44:05 +0100 Subject: [PATCH 156/198] M2.6: a page's content is read with its /DecodeParms given by reference `decode_with_damage` and `decode_contents_with_damage` read a stream's `/DecodeParms` only when direct, as `Stream::decoded` does, so content behind a predictor whose parameters, or a number in them, are an indirect object decoded with the predictor's tag bytes left in, and was taken as whole. Edit then listed and wrote back the wrong bytes, and pruning dropped an image such a form draws. ISO 32000-2 lets those values be indirect. - `decode_with_damage` takes the document and reads the parameters through it, and so does `decode_contents_with_damage`, and `decode_contents` with it: printing, accessibility's reading of a page and a page copied as a form read them too. Code that calls `decode` on a stream itself doesn't yet. - It reads through `Document::decoding_view`: `with_direct_parms`, but a copy of the stream holds only its `/Filter` and the parameters, its data shared, so a stream a page's `/Contents` lists again and again isn't copied whole for each. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/a11y/src/alt.rs | 2 +- crates/cos/src/budget.rs | 68 +++++++++++++++++++++++------- crates/cos/src/document/objects.rs | 33 +++++++++++---- crates/edit/src/objects.rs | 2 +- crates/edit/src/tests.rs | 32 ++++++++++++++ crates/edit/src/text/lines.rs | 2 +- crates/optimize/src/images.rs | 2 +- crates/organize/src/prune.rs | 2 +- crates/organize/src/tests.rs | 31 ++++++++++++++ 9 files changed, 146 insertions(+), 28 deletions(-) diff --git a/crates/a11y/src/alt.rs b/crates/a11y/src/alt.rs index 973dc29e33..dd4b43a641 100644 --- a/crates/a11y/src/alt.rs +++ b/crates/a11y/src/alt.rs @@ -389,7 +389,7 @@ fn decode_page_streams(doc: &Document, page: &Dict) -> Option for entry in entries { let decoded = match &*doc.resolve(&entry) { Object::Stream(s) => { - let (data, damaged) = match budget.decode_with_damage(s, usize::MAX) { + let (data, damaged) = match budget.decode_with_damage(doc, s, usize::MAX) { Ok(read) => read, // Content past the budget may hold more of the tag: editing the rest // would leave that part marked as a tag the tree no longer has. diff --git a/crates/cos/src/budget.rs b/crates/cos/src/budget.rs index 41dff04ed4..d5d67503ee 100644 --- a/crates/cos/src/budget.rs +++ b/crates/cos/src/budget.rs @@ -276,12 +276,13 @@ impl DecodeBudget { self.charge(stream, max, Stream::decoded_within_counted).map(|(data, _)| data) } - /// [`DecodeBudget::decode_within`] (`usize::MAX` for no bound of its own), also saying - /// whether the stream is damaged: `true` when a filter stopped at corrupt data, so the bytes - /// are only what decoded before it. A stream written back from them loses everything after - /// the damage, which other readers may still read. - pub fn decode_with_damage(&mut self, stream: &Stream, max: usize) -> Result<(Vec, bool), CosError> { - self.charge(stream, max, Stream::decoded_within_counted) + /// [`DecodeBudget::decode_within`] (`usize::MAX` for no bound of its own), with the stream's + /// `/DecodeParms` given by reference read in `doc`, also saying whether the stream is damaged: + /// `true` when a filter stopped at corrupt data, so the bytes are only what decoded before it. + /// A stream written back from them loses everything after the damage, which other readers may + /// still read. + pub fn decode_with_damage(&mut self, doc: &Document, stream: &Stream, max: usize) -> Result<(Vec, bool), CosError> { + self.charge(&doc.decoding_view(stream), max, Stream::decoded_within_counted) } /// `stream` decoded like [`Stream::decoded_strict`] (any corruption is an error), charged to @@ -343,23 +344,24 @@ impl DecodeBudget { self.seen.insert(r) } - /// Each stream `contents` names, decoded against the allowance, in order. `contents` is a - /// page's `/Contents`: one stream, or an array whose non-stream entries are left out. - /// A stream that fails to decode, or arrives after the allowance is spent, is an `Err` in its - /// place, so the caller chooses between skipping it and giving up. + /// Each stream `contents` names, decoded against the allowance, in order, its `/DecodeParms` + /// given by reference read in `doc`. `contents` is a page's `/Contents`: one stream, or an + /// array whose non-stream entries are left out. A stream that fails to decode, or arrives + /// after the allowance is spent, is an `Err` in its place, so the caller chooses between + /// skipping it and giving up. pub fn decode_contents(&mut self, doc: &Document, contents: &Object) -> Vec, CosError>> { self.decode_contents_with_damage(doc, contents).into_iter().map(|read| read.map(|(data, _)| data)).collect() } /// [`DecodeBudget::decode_contents`], each stream also saying whether it is damaged partway - /// ([`DecodeBudget::decode_with_damage`]). + /// ([`DecodeBudget::decode_with_damage`], which reads `/DecodeParms` given by reference). pub fn decode_contents_with_damage(&mut self, doc: &Document, contents: &Object) -> Vec, bool), CosError>> { match &*doc.resolve(contents) { - Object::Stream(s) => vec![self.decode_with_damage(s, usize::MAX)], + Object::Stream(s) => vec![self.decode_with_damage(doc, s, usize::MAX)], Object::Array(list) => list .iter() .filter_map(|entry| match &*doc.resolve(entry) { - Object::Stream(s) => Some(self.decode_with_damage(s, usize::MAX)), + Object::Stream(s) => Some(self.decode_with_damage(doc, s, usize::MAX)), _ => None, }) .collect(), @@ -823,7 +825,8 @@ mod tests { /// checksum, no filter at all. #[test] fn a_stream_damaged_partway_says_so() { - let read = |budget: &mut DecodeBudget, s: &Stream, max| budget.decode_with_damage(s, max).map(|(d, damaged)| (d.len(), damaged)).ok(); + let doc = Document::new_empty(); + let read = |budget: &mut DecodeBudget, s: &Stream, max| budget.decode_with_damage(&doc, s, max).map(|(d, damaged)| (d.len(), damaged)).ok(); let whole = Stream::flate(Dict::new(), &vec![b' '; 1 << 20]); let cut = Stream::from_raw(whole.dict.clone(), whole.raw.get(..whole.raw.len() / 2).unwrap_or_default().to_vec()); let partial = cut.decoded().map(|d| d.len()).unwrap_or(0); @@ -845,7 +848,42 @@ mod tests { let mut dict = Dict::new(); dict.set(b"Filter".to_vec(), Object::Array(vec![Object::name("FlateDecode"), Object::name("ASCIIHexDecode")])); let hex = Stream::from_raw(dict, Stream::flate(Dict::new(), b"4142zz4344>").raw.to_vec()); - assert_eq!(budget.decode_with_damage(&hex, usize::MAX).ok(), Some((b"AB".to_vec(), true))); + assert_eq!(budget.decode_with_damage(&doc, &hex, usize::MAX).ok(), Some((b"AB".to_vec(), true))); + } + + /// `/DecodeParms` given by reference, or a parameter given by reference in them, is read: a + /// predicted stream decodes as with them direct. Of the stream, decoding copies no more than + /// its filters and their parameters. + #[test] + fn parameters_given_by_reference_are_read() { + let content = b"q 10 0 0 10 0 0 cm /Im Do Q"; + // PNG Up, one column: each byte a row of its own, tagged 2. + let mut rows = Vec::new(); + let mut above = 0u8; + for &b in content { + rows.extend([2, b.wrapping_sub(above)]); + above = b; + } + let raw = Stream::flate(Dict::new(), &rows).raw; + let mut doc = Document::new_empty(); + let direct: Dict = [(b"Predictor".to_vec(), Object::Int(12))].into_iter().collect(); + let by_reference = Object::Ref(doc.add(Object::Dict(direct.clone()))); + let predictor = doc.add(Object::Int(12)); + let inner = Object::Dict([(b"Predictor".to_vec(), Object::Ref(predictor))].into_iter().collect()); + for parms in [Object::Dict(direct), by_reference, inner] { + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::name("FlateDecode")); + dict.set(b"DecodeParms".to_vec(), parms.clone()); + dict.set(b"Other".to_vec(), Object::Array(vec![Object::Int(0); 1000])); + let s = Stream::from_raw(dict, raw.to_vec()); + assert_eq!(DecodeBudget::page().decode_with_damage(&doc, &s, usize::MAX).ok(), Some((content.to_vec(), false)), "{parms:?}"); + let r = doc.add(Object::Stream(s.clone())); + let read = DecodeBudget::page().decode_contents_with_damage(&doc, &Object::Array(vec![Object::Ref(r)])); + assert_eq!(read, [Ok((content.to_vec(), false))], "{parms:?}"); + assert_eq!(DecodeBudget::page().decode_contents(&doc, &Object::Ref(r)), [Ok(content.to_vec())], "{parms:?}"); + let view = doc.decoding_view(&s); + assert!(matches!(view, std::borrow::Cow::Borrowed(_)) || view.dict.len() == 2, "{:?}", view.dict); + } } /// A page's content streams each say whether they are damaged, in their places. diff --git a/crates/cos/src/document/objects.rs b/crates/cos/src/document/objects.rs index 458840de3a..7ce9474018 100644 --- a/crates/cos/src/document/objects.rs +++ b/crates/cos/src/document/objects.rs @@ -288,7 +288,29 @@ impl Document { /// ([`pdfcraft_filters::MAX_CHAIN`]). Only small values are copied, so a file can't have a /// large object copied once for each key and filter. pub fn with_direct_parms<'s>(&self, s: &'s Stream) -> Cow<'s, Stream> { - let Some(p) = s.dict.get(b"DecodeParms") else { return Cow::Borrowed(s) }; + let Some(made) = self.direct_parms(s) else { return Cow::Borrowed(s) }; + let mut s = s.clone(); + s.dict.set(b"DecodeParms".to_vec(), made); + Cow::Owned(s) + } + + /// [`Document::with_direct_parms`] for decoding alone: where that copies `s`, this makes a + /// stream of `s`'s data, shared, and a dictionary of its `/Filter` and the parameters made + /// direct, all its filters read. A stream named again and again, as a page's `/Contents` may + /// list one, isn't copied whole each time. + pub(crate) fn decoding_view<'s>(&self, s: &'s Stream) -> Cow<'s, Stream> { + let Some(made) = self.direct_parms(s) else { return Cow::Borrowed(s) }; + let mut dict = Dict::new(); + if let Some(f) = s.dict.get(b"Filter") { + dict.set(b"Filter".to_vec(), f.clone()); + } + dict.set(b"DecodeParms".to_vec(), made); + Cow::Owned(Stream { dict, raw: s.raw.clone() }) + } + + /// The `/DecodeParms` [`Document::with_direct_parms`] gives `s`, or `None` when it keeps `s`. + fn direct_parms(&self, s: &Stream) -> Option { + let p = s.dict.get(b"DecodeParms")?; let crypt = |f: &Object| matches!(f.as_name(), Some(b"Crypt")); let filters: Vec = match s.dict.get(b"Filter") { Some(Object::Array(a)) => a.iter().map(crypt).collect(), @@ -296,7 +318,7 @@ impl Document { None => Vec::new(), }; if filters.iter().filter(|c| !**c).count() > pdfcraft_filters::MAX_CHAIN { - return Cow::Borrowed(s); + return None; } let mut changed = matches!(p, Object::Ref(_)); // Its numbers and flags kept, resolved; anything else is no filter's parameters (null). @@ -323,12 +345,7 @@ impl Document { } _ => one(p, &mut changed), }; - if !changed { - return Cow::Borrowed(s); - } - let mut s = s.clone(); - s.dict.set(b"DecodeParms".to_vec(), made); - Cow::Owned(s) + changed.then_some(made) } /// `o`, or the object it refers to, without copying either. diff --git a/crates/edit/src/objects.rs b/crates/edit/src/objects.rs index dbdbf495ae..c0a57c358a 100644 --- a/crates/edit/src/objects.rs +++ b/crates/edit/src/objects.rs @@ -316,7 +316,7 @@ fn checked_streams(doc: &Document, page: usize) -> Result, Ed return Err(EditError::Invalid("a page content stream is missing".into())); }; let (bytes, damaged) = decode - .decode_with_damage(stream, remaining.min(64 * 1024 * 1024)) + .decode_with_damage(doc, stream, remaining.min(64 * 1024 * 1024)) .map_err(|_| EditError::Invalid("the page content cannot be decoded within its size limit".into()))?; if damaged { return Err(EditError::Invalid("a page content stream is damaged partway".into())); diff --git a/crates/edit/src/tests.rs b/crates/edit/src/tests.rs index dc80cb6a29..df9bce657d 100644 --- a/crates/edit/src/tests.rs +++ b/crates/edit/src/tests.rs @@ -5218,6 +5218,38 @@ fn a_content_stream_that_does_not_decode_is_kept_and_edited_around() { assert!(editable_objects(&split_page(&large, "BT /F1 12 Tf 72 600 Td (Whole) Tj ET"), 0).is_ok()); } +/// A content stream whose `/DecodeParms`, or a parameter in them, is given by reference is read +/// through the reference: its text is listed and edited, not read with the predictor's tag bytes +/// left in it. +#[test] +fn content_with_parameters_given_by_reference_is_read_and_edited() { + use pdfcraft_cos::{Dict, ObjRef, Stream}; + let content = b"BT /F1 12 Tf 72 700 Td (Predicted) Tj ET"; + // PNG Up, one column: each byte a row of its own, tagged 2. + let mut rows = Vec::new(); + let mut above = 0u8; + for &b in content { + rows.extend([2, b.wrapping_sub(above)]); + above = b; + } + let texts = |doc: &Document| text::text_lines(doc, 0).unwrap().into_iter().map(|l| l.text).collect::>(); + for given in ["parameters", "predictor"] { + let mut doc = split_page("", "BT /F1 12 Tf 72 600 Td (Whole) Tj ET"); + let predictor = Object::Int(12); + let parms = if given == "parameters" { + Object::Ref(doc.add(Object::Dict([(b"Predictor".to_vec(), predictor)].into_iter().collect::()))) + } else { + Object::Dict([(b"Predictor".to_vec(), Object::Ref(doc.add(predictor)))].into_iter().collect()) + }; + let mut s = Stream::flate(Dict::new(), &rows); + s.dict.set(b"DecodeParms".to_vec(), parms); + doc.set(ObjRef::new(4, 0), s); + assert_eq!(texts(&doc), ["Predicted", "Whole"], "{given} by reference"); + text::replace_line(&mut doc, 0, 0, "Edited").unwrap(); + assert_eq!(texts(&reopen(&doc)), ["Edited", "Whole"], "{given} by reference"); + } +} + /// Objects are selected only from content read whole: what a damaged stream hides past the /// damage could change the graphics state of every later item. #[test] diff --git a/crates/edit/src/text/lines.rs b/crates/edit/src/text/lines.rs index a28806078b..bf66cc0f8b 100644 --- a/crates/edit/src/text/lines.rs +++ b/crates/edit/src/text/lines.rs @@ -92,7 +92,7 @@ pub(crate) fn content_streams_within(doc: &Document, page: &Dict, budget: &mut D let mut out = Vec::with_capacity(list.len()); for entry in list { let (data, damaged) = match &*doc.resolve(&entry) { - Object::Stream(s) => match budget.decode_with_damage(s, usize::MAX) { + Object::Stream(s) => match budget.decode_with_damage(doc, s, usize::MAX) { Ok(read) => read, Err(e) if budget.remaining() == 0 => return Err(EditError::Invalid(format!("the page content is too large to read: {e}"))), Err(_) => (Vec::new(), false), diff --git a/crates/optimize/src/images.rs b/crates/optimize/src/images.rs index 21f6953a8d..6ad6413696 100644 --- a/crates/optimize/src/images.rs +++ b/crates/optimize/src/images.rs @@ -215,7 +215,7 @@ impl Walk<'_> { self.complete = false; return; } - let Ok((data, false)) = self.budget.decode_with_damage(s, usize::MAX) else { + let Ok((data, false)) = self.budget.decode_with_damage(doc, s, usize::MAX) else { self.complete = false; return; }; diff --git a/crates/organize/src/prune.rs b/crates/organize/src/prune.rs index 8683b4d810..591e804b1e 100644 --- a/crates/organize/src/prune.rs +++ b/crates/organize/src/prune.rs @@ -219,7 +219,7 @@ impl Pruner<'_> { } #[cfg(test)] crate::work(1); - match DecodeBudget::new(usize::MAX, 0).decode_with_damage(s, usize::MAX) { + match DecodeBudget::new(usize::MAX, 0).decode_with_damage(self.doc, s, usize::MAX) { Ok((data, false)) => self.content(&data, &inner, inner_owner, depth + 1), // What the form draws is unknown, past damage too: keep everything it could reach. Ok((_, true)) | Err(_) => { diff --git a/crates/organize/src/tests.rs b/crates/organize/src/tests.rs index 40bb4210d4..8d41f474cc 100644 --- a/crates/organize/src/tests.rs +++ b/crates/organize/src/tests.rs @@ -2669,6 +2669,37 @@ fn extract_keeps_images_content_damaged_partway_might_draw() { } } +/// A form whose `/DecodeParms` is given by reference is read through it: the image it draws is +/// kept, not lost to content read with the predictor's tag bytes left in it. +#[test] +fn extract_keeps_images_a_form_with_parameters_by_reference_draws() { + use pdfcraft_cos::ObjRef; + let content = b"q 10 0 0 10 0 0 cm /ImB Do Q"; + // PNG Up, one column: each byte a row of its own, tagged 2. + let mut rows = Vec::new(); + let mut above = 0u8; + for &b in content { + rows.extend([2, b.wrapping_sub(above)]); + above = b; + } + let mut src = doc_images_with(&raw("/Fm Do"), ""); + let parms = src.add(Object::Dict([(b"Predictor".to_vec(), Object::Int(12))].into_iter().collect::())); + let mut fm = pdfcraft_cos::Stream::flate(Dict::new(), &rows); + fm.dict.set(b"DecodeParms".to_vec(), Object::Ref(parms)); + fm.dict.set(b"Subtype".to_vec(), Object::name("Form")); + fm.dict.set(b"BBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 1.into(), 1.into()])); + let fm = src.add(fm); + let xobjects: Dict = + [(b"ImA".to_vec(), Object::Ref(ObjRef::new(8, 0))), (b"ImB".to_vec(), Object::Ref(ObjRef::new(9, 0))), (b"Fm".to_vec(), Object::Ref(fm))] + .into_iter() + .collect(); + src.update_dict(ObjRef::new(7, 0), |d| d.set(b"XObject".to_vec(), Object::Dict(xobjects))).unwrap(); + let part = full_roundtrip(&extract_pages(&src, &[1]).unwrap()); + let names = xobject_names(&part, 0); + assert!(names.iter().any(|n| n.as_slice() == b"ImB"), "{names:?}"); + assert!(!names.iter().any(|n| n.as_slice() == b"ImA"), "what the part doesn't draw still goes: {names:?}"); +} + #[test] fn extract_keeps_images_type3_glyphs_may_draw() { // A Type 3 font without its own /Resources draws through the page's: its glyphs could `Do`. From 3bf6c6f027f5bad86a4a16fccf955b87d449266e Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Thu, 5 Nov 2026 08:44:13 +0100 Subject: [PATCH 157/198] M2.6: marking content as an artifact refuses what may be past the damage A tag's content not found before the damage in a damaged stream may be past it. Marking a figure decorative took it for absent: it succeeded, took the figure out of the structure tree and left its content tagged. Marking a tag as an artifact said its content wasn't on the page. Both now refuse with `Damaged`, as when the content is before the damage, and change nothing. Clearing the parent tree read the page's `/StructParents` only when direct, so with it given by reference (ISO 32000-2 lets it be) the tree went on pointing the content at the retired tag. It is read through the reference. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/a11y/src/alt.rs | 11 ++++++++--- crates/a11y/src/tests.rs | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/crates/a11y/src/alt.rs b/crates/a11y/src/alt.rs index dd4b43a641..bdc72d2e33 100644 --- a/crates/a11y/src/alt.rs +++ b/crates/a11y/src/alt.rs @@ -417,7 +417,7 @@ enum Marked { /// None of the content is on the page, or with `require_all` not all of it: nothing changed. NotFound, /// Some of it is in a stream damaged partway, which can't be rewritten without losing what - /// follows the damage: nothing changed. + /// follows the damage, or may be past the damage: nothing changed. Damaged, } @@ -442,6 +442,11 @@ fn artifact_marked_content( edits.push((op.span, b"/Artifact BMC".to_vec())); } } + // Content not found may be past the damage in a damaged stream: whether it is on the page + // isn't known. + if found != *ids && streams.iter().any(|(_, s)| s.as_ref().is_some_and(|(_, _, damaged)| *damaged)) { + return Ok(Marked::Damaged); + } if if require_all { found != *ids } else { found.is_empty() } { return Ok(Marked::NotFound); } @@ -494,7 +499,7 @@ pub fn mark_decorative(doc: &mut Document, obj: ObjRef) -> Result<(), AltError> match artifact_marked_content(doc, page, streams, &ids, false)? { // The parent tree no longer points those ids at the figure. Marked::Done => { - if let Some(key) = page.dict.get(b"StructParents").and_then(Object::as_int) { + if let Some(key) = page.dict.get(b"StructParents").and_then(|o| doc.resolve(o).as_int()) { clear_parent_tree(doc, key, obj)?; } } @@ -545,7 +550,7 @@ pub fn mark_tag_artifact(doc: &mut Document, obj: ObjRef) -> Result<(), Artifact Marked::NotFound => return Err(ArtifactError::ContentNotFound), Marked::Damaged => return Err(ArtifactError::Damaged), } - if let Some(key) = page.dict.get(b"StructParents").and_then(Object::as_int) { + if let Some(key) = page.dict.get(b"StructParents").and_then(|o| doc.resolve(o).as_int()) { clear_parent_tree(doc, key, obj).map_err(|e| ArtifactError::ParentTree(e.to_string()))?; } doc.update_dict(parent, |p| p.set(b"K".to_vec(), replacement))?; diff --git a/crates/a11y/src/tests.rs b/crates/a11y/src/tests.rs index a129f207cd..78a6d96589 100644 --- a/crates/a11y/src/tests.rs +++ b/crates/a11y/src/tests.rs @@ -1129,6 +1129,42 @@ fn a_tags_content_in_a_stream_damaged_partway_is_not_rewritten() { assert_ne!(list.get(1), Some(&Object::Ref(ObjRef::new(4, 0))), "the tag's stream is rewritten"); } +/// A tag's content not found before the damage in a damaged stream may be past it: it isn't known +/// to be off the page, so neither marking changes anything. +#[test] +fn a_tags_content_past_the_damage_is_not_taken_for_absent() { + use pdfcraft_cos::ObjRef; + let damaged = + damaged_flate(b"/Photo << /MCID 1 >> BDC 100 100 50 30 re f EMC\n", b"/Figure << /MCID 0 >> BDC q 40 0 0 20 10 150 cm /Im Do Q EMC"); + let mut doc = figures_doc(); + doc.set(ObjRef::new(4, 0), damaged); + let contents = |doc: &Document| doc.get(ObjRef::new(3, 0)).as_dict().unwrap().get(b"Contents").cloned(); + let before = (contents(&doc), doc.get(ObjRef::new(9, 0)), doc.get(ObjRef::new(6, 0))); + assert_eq!(mark_decorative(&mut doc, ObjRef::new(7, 0)), Err(AltError::Damaged)); + assert_eq!(mark_tag_artifact(&mut doc, ObjRef::new(7, 0)), Err(ArtifactError::Damaged)); + assert_eq!((contents(&doc), doc.get(ObjRef::new(9, 0)), doc.get(ObjRef::new(6, 0))), before); +} + +/// A page's `/StructParents` given by reference is read through it: the parent tree no longer +/// points the marked content at the tag. +#[test] +fn struct_parents_given_by_reference_are_read() { + use pdfcraft_cos::{ObjRef, Object}; + let by_reference = || { + let mut doc = figures_doc(); + let key = doc.add(Object::Int(0)); + doc.update_dict(ObjRef::new(3, 0), |p| p.set(b"StructParents".to_vec(), Object::Ref(key))).unwrap(); + doc + }; + let nums = |doc: &Document| doc.get(ObjRef::new(9, 0)).as_dict().and_then(|d| d.get(b"Nums").cloned()); + let mut doc = by_reference(); + mark_decorative(&mut doc, ObjRef::new(7, 0)).unwrap(); + assert_eq!(nums(&doc), Some(Object::Array(vec![Object::Int(0), Object::Array(vec![Object::Null, Object::Ref(ObjRef::new(8, 0))])]))); + let mut doc = by_reference(); + mark_tag_artifact(&mut doc, ObjRef::new(8, 0)).unwrap(); + assert_eq!(nums(&doc), Some(Object::Array(vec![Object::Int(0), Object::Array(vec![Object::Ref(ObjRef::new(7, 0)), Object::Null])]))); +} + /// A content stream that doesn't decode at all is kept as stored, and a tag in the page's other /// streams is still marked: only running past the decode budget refuses. #[test] From 91968610c35dc20189f1ebb0c27d23eec8ba50bb Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Thu, 5 Nov 2026 12:19:30 +0100 Subject: [PATCH 158/198] M2.6: redaction reads content with its /DecodeParms given by reference `DecodeBudget::decode_strict` read a stream's `/DecodeParms` only when direct, so content behind a predictor whose parameters are an indirect object decoded with the predictor's tag bytes left in. Redaction then found no glyph under a mark, left the text in the file and reported the mark applied. ISO 32000-2 lets those parameters be indirect. `decode_strict` takes the document and reads through `decoding_view`, as `decode_with_damage` does. Its other readers read the parameters too: Remove Hidden Information's hidden text and layers, the audit's covered-text check, and edit's backdrop scan. The accessibility_mark_tag_artifact description says that content which is, or may be, in a stream damaged partway is refused. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/audit/src/cover.rs | 4 +-- crates/automation/src/tools.rs | 2 +- crates/cos/src/budget.rs | 14 +++++----- crates/edit/src/backdrop.rs | 8 +++--- crates/redact/src/interp.rs | 2 +- crates/redact/src/lib.rs | 2 +- crates/redact/src/tests.rs | 48 ++++++++++++++++++++++++++++++++++ 7 files changed, 65 insertions(+), 15 deletions(-) diff --git a/crates/audit/src/cover.rs b/crates/audit/src/cover.rs index bb5adf7ff8..9f771f7040 100644 --- a/crates/audit/src/cover.rs +++ b/crates/audit/src/cover.rs @@ -283,7 +283,7 @@ impl<'a> Walker<'a> { return Err(AuditError::TooManyForms); } // A form stream that cannot be decoded fails the audit: it might hide a cover. - let data = self.budget.decode_strict(stream)?; + let data = self.budget.decode_strict(self.doc, stream)?; // The XObject matrix maps form space into the user space active at `Do`. let mut sub_gs = gs.clone(); sub_gs.ctm = matrix.then(&gs.ctm); @@ -326,7 +326,7 @@ pub fn filled_rects(doc: &Document, page: &Dict) -> Result, Audi let mut streams = Vec::with_capacity(contents.len()); for c in &contents { if let Object::Stream(s) = &*doc.resolve(c) { - streams.push(budget.decode_strict(s)?); + streams.push(budget.decode_strict(doc, s)?); } } let pieces = Pieces::join(&streams); diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index af5146377a..e69a57dc33 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -1285,7 +1285,7 @@ pub fn tools() -> Vec { "property": { "type": "string", "enum": ["type", "title", "actual_text", "alternate_text", "language", "summary", "scope"] }, "value": { "type": "string" }, }), &["doc", "object", "generation", "property"])), - t("accessibility_mark_tag_artifact", "Mark tag content as artifact", "Mark a reachable leaf structure tag's direct marked content as an artifact and remove the tag from the reading structure. Use object and generation from accessibility_tags. Tags with child structure elements, no direct marked content, or content not found on its page are rejected. Undoable.") + t("accessibility_mark_tag_artifact", "Mark tag content as artifact", "Mark a reachable leaf structure tag's direct marked content as an artifact and remove the tag from the reading structure. Use object and generation from accessibility_tags. Tags with child structure elements, no direct marked content, content not found on its page, or content that is, or may be, in a content stream damaged partway are rejected. Undoable.") .with(schema(json!({ "doc": doc(), "object": { "type": "integer", "minimum": 1 }, "generation": { "type": "integer", "minimum": 0, "maximum": 65535 }, diff --git a/crates/cos/src/budget.rs b/crates/cos/src/budget.rs index d5d67503ee..51b05e034d 100644 --- a/crates/cos/src/budget.rs +++ b/crates/cos/src/budget.rs @@ -286,10 +286,10 @@ impl DecodeBudget { } /// `stream` decoded like [`Stream::decoded_strict`] (any corruption is an error), charged to - /// the allowance. - pub fn decode_strict(&mut self, stream: &Stream) -> Result, CosError> { + /// the allowance, with its `/DecodeParms` given by reference read in `doc`. + pub fn decode_strict(&mut self, doc: &Document, stream: &Stream) -> Result, CosError> { let strict: Decoder = |s, max| s.decoded_strict_within_counted(max).map(|(data, spent)| (data, false, spent)); - self.charge(stream, usize::MAX, strict).map(|(data, _)| data) + self.charge(&doc.decoding_view(stream), usize::MAX, strict).map(|(data, _)| data) } /// Decodes `stream` within `max` and the allowance, charging what every filter stage @@ -788,7 +788,7 @@ mod tests { assert_eq!(budget.remaining(), 0); let mut budget = DecodeBudget::new((1 << 20) - 1, 1); let Object::Stream(stream) = &*doc.get(bomb) else { panic!("not a stream") }; - assert!(budget.decode_strict(stream).is_err()); + assert!(budget.decode_strict(&doc, stream).is_err()); assert_eq!(budget.remaining(), 0); let mut budget = DecodeBudget::new(10 << 20, 1); @@ -812,8 +812,9 @@ mod tests { let corrupt = Stream::from_raw(whole.dict.clone(), whole.raw.get(..cut).unwrap_or_default().to_vec()); let partial = corrupt.decoded().map(|d| d.len()).unwrap_or(0); assert!(partial > 0, "a cut flate stream decodes in part"); + let doc = Document::new_empty(); let mut budget = DecodeBudget::new(10 << 20, 1); - assert!(budget.decode_strict(&corrupt).is_err()); + assert!(budget.decode_strict(&doc, &corrupt).is_err()); assert_eq!(budget.remaining(), (10 << 20) - partial); let mut budget = DecodeBudget::new(10 << 20, 1); assert!(budget.decode_within(&corrupt, 1 << 30).is_ok()); @@ -877,6 +878,7 @@ mod tests { dict.set(b"Other".to_vec(), Object::Array(vec![Object::Int(0); 1000])); let s = Stream::from_raw(dict, raw.to_vec()); assert_eq!(DecodeBudget::page().decode_with_damage(&doc, &s, usize::MAX).ok(), Some((content.to_vec(), false)), "{parms:?}"); + assert_eq!(DecodeBudget::page().decode_strict(&doc, &s).ok(), Some(content.to_vec()), "{parms:?}"); let r = doc.add(Object::Stream(s.clone())); let read = DecodeBudget::page().decode_contents_with_damage(&doc, &Object::Array(vec![Object::Ref(r)])); assert_eq!(read, [Ok((content.to_vec(), false))], "{parms:?}"); @@ -935,7 +937,7 @@ mod tests { assert_eq!(budget.remaining(), (10 << 20) - charged); assert!(budget.decode_within(&chained, 2 << 20).is_ok()); assert_eq!(budget.remaining(), (10 << 20) - 2 * charged); - assert!(budget.decode_strict(&chained).is_ok()); + assert!(budget.decode_strict(&Document::new_empty(), &chained).is_ok()); assert_eq!(budget.remaining(), (10 << 20) - 3 * charged); } } diff --git a/crates/edit/src/backdrop.rs b/crates/edit/src/backdrop.rs index 43e16b40ec..f6788bb51c 100644 --- a/crates/edit/src/backdrop.rs +++ b/crates/edit/src/backdrop.rs @@ -208,7 +208,7 @@ pub(crate) fn restore(doc: &mut Document, page: &pdfcraft_model::Page) -> Result let same = |w: &ObjRef| match &*doc.get(*w) { Object::Stream(s) => { [TAG, &b"PCMark"[..], b"Subtype"].iter().all(|k| s.dict.get(k).is_none()) - && DecodeBudget::new(SCAN_BYTES, 0).decode_strict(s).ok().as_deref() == Some(bytes.as_slice()) + && DecodeBudget::new(SCAN_BYTES, 0).decode_strict(doc, s).ok().as_deref() == Some(bytes.as_slice()) } _ => false, }; @@ -248,7 +248,7 @@ fn restored_copy(doc: &Document, page: &pdfcraft_model::Page, copy: ObjRef) -> O return None; } let hash = note.get(b"Before")?.as_string()?; - let bytes = DecodeBudget::new(SCAN_BYTES, 0).decode_strict(s).ok()?; + let bytes = DecodeBudget::new(SCAN_BYTES, 0).decode_strict(doc, s).ok()?; let parsed = pdfcraft_content::parse(head(&bytes)); // The `n` standing in for the fill, after exactly the operators that came before it. let fill = parsed.ops.get(index)?; @@ -384,11 +384,11 @@ fn scan(doc: &Document, p: &pdfcraft_model::Page, swap: Option<(ObjRef, &[u8])>) if b.len() > SCAN_BYTES.saturating_sub(total) { return None; } - budget.decode_strict(&Stream::from_raw(Dict::new(), b.to_vec())).ok()? + budget.decode_strict(doc, &Stream::from_raw(Dict::new(), b.to_vec())).ok()? } // Tolerant decoding can return an empty or partial stream after corruption. // That cannot prove which paint preceded the candidate backdrop. - _ => budget.decode_strict(s).ok()?, + _ => budget.decode_strict(doc, s).ok()?, }; if bytes.len() > SCAN_BYTES.saturating_sub(total).saturating_sub(1) { return None; diff --git a/crates/redact/src/interp.rs b/crates/redact/src/interp.rs index b0a7257cb3..4cb95c81d8 100644 --- a/crates/redact/src/interp.rs +++ b/crates/redact/src/interp.rs @@ -979,7 +979,7 @@ fn xobject( } // Strict, like page content: a Form read only in part counts as undecodable. let data = if scope.budget.enter_form() { - scope.budget.decode_strict(s) + scope.budget.decode_strict(doc, s) } else { Err(CosError::Filter(format!("the page draws more than {MAX_PAGE_FORMS} form XObjects"))) }; diff --git a/crates/redact/src/lib.rs b/crates/redact/src/lib.rs index 6d323004ad..8785cb357a 100644 --- a/crates/redact/src/lib.rs +++ b/crates/redact/src/lib.rs @@ -290,7 +290,7 @@ pub(crate) fn page_streams_within( match &*doc.resolve(o) { // Strict: a tolerant decode keeps what precedes corruption and drops the rest, which // still draws in viewers that read further; anything not read can't be cleaned. - Object::Stream(s) => data.push(budget.decode_strict(s).map_err(|_| RedactError::Unreadable(index + 1))?), + Object::Stream(s) => data.push(budget.decode_strict(doc, s).map_err(|_| RedactError::Unreadable(index + 1))?), Object::Null => data.push(Vec::new()), _ => return Err(RedactError::Unreadable(index + 1)), } diff --git a/crates/redact/src/tests.rs b/crates/redact/src/tests.rs index 8978e0e7d6..c98a21c14f 100644 --- a/crates/redact/src/tests.rs +++ b/crates/redact/src/tests.rs @@ -81,6 +81,54 @@ fn glyphs_under_a_mark_go_and_the_rest_stays_put() { assert!(!content(&doc, 0).contains("1234")); } +/// Content whose `/DecodeParms` is given by reference is read through the reference, on the page +/// and in a form: the text under a mark goes. Read with the predictor's tag bytes left in, no glyph +/// was found under the mark, and the redaction left the text in the file and said it was done. +#[test] +fn content_with_parameters_given_by_reference_is_redacted() { + use pdfcraft_cos::{Dict, ObjRef, Object, Stream}; + // PNG Up, one column: each byte a row of its own, tagged 2. + let predicted = |doc: &mut Document, mut dict: Dict| { + let mut rows = Vec::new(); + let mut above = 0u8; + for &b in b"BT /F1 10 Tf 10 100 Td (SECRET) Tj ET" { + rows.extend([2, b.wrapping_sub(above)]); + above = b; + } + let parms = doc.add(Object::Dict([(b"Predictor".to_vec(), Object::Int(12))].into_iter().collect::())); + dict.set(b"DecodeParms".to_vec(), Object::Ref(parms)); + let mut s = Stream::flate(Dict::new(), &rows); + for (k, v) in dict.iter() { + s.dict.set(k.to_vec(), v.clone()); + } + s + }; + let marked = [[5.0, 90.0, 120.0, 115.0]]; + let mut page = one_page(b"", "", vec![]); + let s = predicted(&mut page, Dict::new()); + page.set(ObjRef::new(4, 0), s); + // The parameters take object 7 and the form 8. + let mut form = one_page(b"/Fm Do", "/XObject << /Fm 8 0 R >>", vec![]); + let mut dict = Dict::new(); + dict.set(b"Subtype".to_vec(), Object::name("Form")); + dict.set(b"BBox".to_vec(), Object::Array([0, 0, 300, 300].into_iter().map(Object::Int).collect())); + dict.set( + b"Resources".to_vec(), + Object::Dict( + [(b"Font".to_vec(), Object::Dict([(b"F1".to_vec(), Object::Ref(ObjRef::new(5, 0)))].into_iter().collect()))].into_iter().collect(), + ), + ); + let s = predicted(&mut form, dict); + form.set(ObjRef::new(8, 0), s); + for (mut doc, what) in [(page, "page"), (form, "form")] { + assert_eq!(under(&mut doc, 0, &marked), 6, "{what}"); + mark(&mut doc, 0, &marked, ""); + let r = apply(&mut doc, None).unwrap(); + assert_eq!((r.marks, r.glyphs), (1, 6), "{what}"); + assert_eq!(under(&mut reopen(&doc), 0, &marked), 0, "{what}"); + } +} + /// #2130: a font with no /Widths is drawn with its embedded program's advances. Measured with a /// guess from its name instead (Courier's 600 units for "Mono"), the end of the line seemed to /// lie past a mark that covers all of it on screen, and was kept. From 2d672d951754dc2b69fa5819a55274ce091bcc13 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Thu, 5 Nov 2026 12:45:11 +0100 Subject: [PATCH 159/198] M2.6: redaction keeps an image the page still draws behind parameters by reference Once a redaction removes an image or form under a mark, it takes the name out of the page's resources unless the page's content still draws it elsewhere. It read the page's streams again for that with `decoded`, which reads `/DecodeParms` only when direct, so a stream behind a predictor whose parameters are an indirect object seemed not to draw it, and the page lost the image there. It now looks in what it read for the redaction, with the parameters by reference, and decodes again only the streams it has just written, which have none. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/redact/src/lib.rs | 7 +++- crates/redact/src/tests.rs | 68 +++++++++++++++++++++++++++----------- 2 files changed, 55 insertions(+), 20 deletions(-) diff --git a/crates/redact/src/lib.rs b/crates/redact/src/lib.rs index 8785cb357a..db729715f1 100644 --- a/crates/redact/src/lib.rs +++ b/crates/redact/src/lib.rs @@ -820,7 +820,12 @@ fn redact_page_content( // longer draws so the removed image or form bytes cannot survive in the saved file. let page_content: Vec> = new_list .iter() - .map(|o| match &*doc.resolve(o) { + .zip(&list) + .zip(&data) + .map(|((now, was), read)| match &*doc.resolve(now) { + // Left as it was: what was read above, with its parameters by reference. + _ if now == was => read.clone(), + // Written above by `Stream::flate`, which leaves no `/DecodeParms`. Object::Stream(s) => s.decoded().unwrap_or_default(), _ => Vec::new(), }) diff --git a/crates/redact/src/tests.rs b/crates/redact/src/tests.rs index c98a21c14f..2de12ba374 100644 --- a/crates/redact/src/tests.rs +++ b/crates/redact/src/tests.rs @@ -81,31 +81,36 @@ fn glyphs_under_a_mark_go_and_the_rest_stays_put() { assert!(!content(&doc, 0).contains("1234")); } +/// `content` behind a PNG Up predictor whose `/DecodeParms` is an indirect object added to `doc`, +/// in a stream with `dict`'s entries too. +fn predicted(doc: &mut Document, dict: pdfcraft_cos::Dict, content: &[u8]) -> pdfcraft_cos::Stream { + use pdfcraft_cos::{Dict, Object, Stream}; + // One column: each byte a row of its own, tagged 2. + let mut rows = Vec::new(); + let mut above = 0u8; + for &b in content { + rows.extend([2, b.wrapping_sub(above)]); + above = b; + } + let parms = doc.add(Object::Dict([(b"Predictor".to_vec(), Object::Int(12))].into_iter().collect::())); + let mut s = Stream::flate(Dict::new(), &rows); + for (k, v) in dict.iter() { + s.dict.set(k.to_vec(), v.clone()); + } + s.dict.set(b"DecodeParms".to_vec(), Object::Ref(parms)); + s +} + /// Content whose `/DecodeParms` is given by reference is read through the reference, on the page /// and in a form: the text under a mark goes. Read with the predictor's tag bytes left in, no glyph /// was found under the mark, and the redaction left the text in the file and said it was done. #[test] fn content_with_parameters_given_by_reference_is_redacted() { - use pdfcraft_cos::{Dict, ObjRef, Object, Stream}; - // PNG Up, one column: each byte a row of its own, tagged 2. - let predicted = |doc: &mut Document, mut dict: Dict| { - let mut rows = Vec::new(); - let mut above = 0u8; - for &b in b"BT /F1 10 Tf 10 100 Td (SECRET) Tj ET" { - rows.extend([2, b.wrapping_sub(above)]); - above = b; - } - let parms = doc.add(Object::Dict([(b"Predictor".to_vec(), Object::Int(12))].into_iter().collect::())); - dict.set(b"DecodeParms".to_vec(), Object::Ref(parms)); - let mut s = Stream::flate(Dict::new(), &rows); - for (k, v) in dict.iter() { - s.dict.set(k.to_vec(), v.clone()); - } - s - }; + use pdfcraft_cos::{Dict, ObjRef, Object}; + let secret = b"BT /F1 10 Tf 10 100 Td (SECRET) Tj ET"; let marked = [[5.0, 90.0, 120.0, 115.0]]; let mut page = one_page(b"", "", vec![]); - let s = predicted(&mut page, Dict::new()); + let s = predicted(&mut page, Dict::new(), secret); page.set(ObjRef::new(4, 0), s); // The parameters take object 7 and the form 8. let mut form = one_page(b"/Fm Do", "/XObject << /Fm 8 0 R >>", vec![]); @@ -118,7 +123,7 @@ fn content_with_parameters_given_by_reference_is_redacted() { [(b"Font".to_vec(), Object::Dict([(b"F1".to_vec(), Object::Ref(ObjRef::new(5, 0)))].into_iter().collect()))].into_iter().collect(), ), ); - let s = predicted(&mut form, dict); + let s = predicted(&mut form, dict, secret); form.set(ObjRef::new(8, 0), s); for (mut doc, what) in [(page, "page"), (form, "form")] { assert_eq!(under(&mut doc, 0, &marked), 6, "{what}"); @@ -129,6 +134,31 @@ fn content_with_parameters_given_by_reference_is_redacted() { } } +/// An image removed under a mark stays in the page's resources while another of its content +/// streams, with `/DecodeParms` given by reference, still draws it. Read without the parameters, +/// that stream seemed not to draw it, and the page lost the image there. +#[test] +fn an_image_drawn_again_in_content_with_parameters_by_reference_stays() { + use pdfcraft_cos::{Dict, ObjRef, Object}; + let img = stream("/Type /XObject /Subtype /Image /Width 4 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8", [255, 255, 255, 255]); + let mut doc = one_page(b"q 10 0 0 10 10 10 cm /Im1 Do Q", "/XObject << /Im1 7 0 R >>", vec![img]); + let again = predicted(&mut doc, Dict::new(), b"q 10 0 0 10 200 200 cm /Im1 Do Q"); + let again = doc.add(Object::Stream(again)); + doc.update_dict(ObjRef::new(3, 0), |d| d.set(b"Contents".to_vec(), Object::Array(vec![Object::Ref(ObjRef::new(4, 0)), Object::Ref(again)]))) + .unwrap(); + mark(&mut doc, 0, &[[5.0, 5.0, 25.0, 25.0]], ""); + let r = apply(&mut doc, None).unwrap(); + assert_eq!(r.images_removed, 1, "{r:?}"); + let doc = reopen(&doc); + assert!(content(&doc, 0).contains("200 200 cm /Im1 Do"), "{}", content(&doc, 0)); + let p = pdfcraft_model::pages(&doc).swap_remove(0); + let res = doc.resolve(p.dict.get(b"Resources").unwrap()).as_dict().cloned().unwrap(); + let xo = doc.resolve(res.get(b"XObject").unwrap()).as_dict().cloned().unwrap(); + // The save numbers objects afresh: the image is told by what it is. + let Some(Object::Stream(image)) = xo.reference(b"Im1").map(|r| (*doc.get(r)).clone()) else { panic!("no /Im1: {xo:?}") }; + assert_eq!((image.dict.name(b"Subtype"), image.decoded().ok()), (Some(&b"Image"[..]), Some(vec![255; 4]))); +} + /// #2130: a font with no /Widths is drawn with its embedded program's advances. Measured with a /// guess from its name instead (Courier's 600 units for "Mono"), the end of the line seemed to /// lie past a mark that covers all of it on screen, and was kept. From ebf9c7a3ce19eb71d58dc723181914f843dfb56d Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Thu, 5 Nov 2026 13:07:06 +0100 Subject: [PATCH 160/198] M2.6: a damaged-stream refusal says the content may be past the damage Since marking a figure decorative or a tag as an artifact refuses content not found beside a damaged stream, the refusal also covers content that only may be past the damage. Its message said the content is in a damaged stream; it now says it is, or may be. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/a11y/src/alt.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/crates/a11y/src/alt.rs b/crates/a11y/src/alt.rs index bdc72d2e33..607a1ec5c4 100644 --- a/crates/a11y/src/alt.rs +++ b/crates/a11y/src/alt.rs @@ -25,7 +25,9 @@ pub enum AltError { NotAFigure(u32), #[error("the page has more content than can be edited")] TooLarge, - #[error("the figure's content is in a content stream damaged partway, which can't be rewritten without losing what follows the damage")] + #[error( + "the figure's content is, or may be, in a content stream damaged partway, which can't be rewritten without losing what follows the damage" + )] Damaged, #[error("{0}")] Cos(#[from] pdfcraft_cos::CosError), @@ -54,7 +56,9 @@ pub enum ArtifactError { ParentTree(String), #[error("the page has more content than can be edited")] TooLarge, - #[error("the tag's marked content is in a content stream damaged partway, which can't be rewritten without losing what follows the damage")] + #[error( + "the tag's marked content is, or may be, in a content stream damaged partway, which can't be rewritten without losing what follows the damage" + )] Damaged, #[error(transparent)] Cos(#[from] pdfcraft_cos::CosError), From 0a350f6809ded06bdc519f5fdbea86d1e78e5482 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Fri, 6 Nov 2026 01:00:43 +0100 Subject: [PATCH 161/198] M2.6: nothing after a damaged content stream, or in one that doesn't decode, is rewritten A page's content streams are read as one stream, so what a stream damaged partway hides past the damage is unknown to the streams after it too. A string or comment the damage leaves open runs on into the next stream, and operators the damage hides can change the graphics state the later streams draw in. Marking a tag's content as an artifact took what read as a BDC in a later stream for the tag's own, and could rewrite bytes inside a string. Moving an image after the damage worked the move out without a cm the damage may hide. Now an edit refuses a change to any stream after one damaged partway (rewritten_contents), and an image isn't arranged into a new stream placed after one. Marking an artifact or a decorative figure refuses content that reads as the tag's in a stream after the damage. Edits and marking before the damage go on as before. A content stream that doesn't decode at all was read as empty and whole. A styled edit whose paragraph marker closes past the page's last operator appended the close to it, so the edit replaced that stream with one holding only the close. Edits now refuse a change to such a stream, and still go on around it. Marking an artifact or a decorative figure whose content isn't found takes it as possibly in such a stream, as it does for content past the damage, and refuses rather than take it for absent. Tests: an_edit_never_rewrites_a_content_stream_damaged_partway (an edit after the damage refused, one before it made), text_a_damaged_stream_runs_on_into_is_not_edited, images_after_a_content_stream_damaged_partway_are_not_moved, a_content_stream_that_does_not_decode_is_never_written_over, a_tags_content_after_a_damaged_stream_is_not_rewritten, a_tags_content_not_found_beside_a_stream_that_does_not_decode_is_not_taken_for_absent, and a_tags_content_in_a_stream_damaged_partway_is_not_rewritten now checks the damaged stream itself and marks content before the damage. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/a11y/src/alt.rs | 49 ++++++++++----- crates/a11y/src/tests.rs | 64 +++++++++++++++++--- crates/automation/src/tools.rs | 2 +- crates/edit/src/images.rs | 3 +- crates/edit/src/objects.rs | 2 +- crates/edit/src/tests.rs | 107 ++++++++++++++++++++++++++++++--- crates/edit/src/text.rs | 3 +- crates/edit/src/text/lines.rs | 74 ++++++++++++++++++----- 8 files changed, 255 insertions(+), 49 deletions(-) diff --git a/crates/a11y/src/alt.rs b/crates/a11y/src/alt.rs index 607a1ec5c4..dfe083bec9 100644 --- a/crates/a11y/src/alt.rs +++ b/crates/a11y/src/alt.rs @@ -26,7 +26,7 @@ pub enum AltError { #[error("the page has more content than can be edited")] TooLarge, #[error( - "the figure's content is, or may be, in a content stream damaged partway, which can't be rewritten without losing what follows the damage" + "the figure's content is, or may be, in a content stream that is damaged partway or doesn't decode, or after the damage, where it can't be rewritten without losing or misreading what wasn't read" )] Damaged, #[error("{0}")] @@ -57,7 +57,7 @@ pub enum ArtifactError { #[error("the page has more content than can be edited")] TooLarge, #[error( - "the tag's marked content is, or may be, in a content stream damaged partway, which can't be rewritten without losing what follows the damage" + "the tag's marked content is, or may be, in a content stream that is damaged partway or doesn't decode, or after the damage, where it can't be rewritten without losing or misreading what wasn't read" )] Damaged, #[error(transparent)] @@ -380,9 +380,19 @@ fn content_entries(doc: &Document, page: &Dict) -> Vec { } } -/// A `/Contents` entry and, for a stream, the stream, its decoded data and whether it is damaged -/// partway (the data is then what decoded before the damage). -type DecodedStream = (Object, Option<(Stream, Vec, bool)>); +/// How much of a content stream decoded. +#[derive(Clone, Copy, Debug, PartialEq)] +enum Read { + Whole, + /// Damaged partway: the data is what decoded before the damage. What follows it is unknown, + /// the syntax the later streams start in too. + Damaged, + /// Not at all (a filter this reader lacks, say): the data is empty. + Unread, +} + +/// A `/Contents` entry and, for a stream, the stream, its decoded data and how much of it decoded. +type DecodedStream = (Object, Option<(Stream, Vec, Read)>); /// Decode the page's content streams within the page decode budget. `None` when a stream is /// rejected because the budget is exhausted. @@ -393,14 +403,14 @@ fn decode_page_streams(doc: &Document, page: &Dict) -> Option for entry in entries { let decoded = match &*doc.resolve(&entry) { Object::Stream(s) => { - let (data, damaged) = match budget.decode_with_damage(doc, s, usize::MAX) { - Ok(read) => read, + let (data, read) = match budget.decode_with_damage(doc, s, usize::MAX) { + Ok((data, damaged)) => (data, if damaged { Read::Damaged } else { Read::Whole }), // Content past the budget may hold more of the tag: editing the rest // would leave that part marked as a tag the tree no longer has. Err(_) if budget.exhausted() => return None, - Err(_) => (Vec::new(), false), + Err(_) => (Vec::new(), Read::Unread), }; - Some((s.clone(), data, damaged)) + Some((s.clone(), data, read)) } _ => None, }; @@ -420,8 +430,8 @@ enum Marked { Done, /// None of the content is on the page, or with `require_all` not all of it: nothing changed. NotFound, - /// Some of it is in a stream damaged partway, which can't be rewritten without losing what - /// follows the damage, or may be past the damage: nothing changed. + /// Some of it is, or may be, in a stream damaged partway or one that doesn't decode, neither of + /// which can be rewritten without losing what wasn't read, or after the damage: nothing changed. Damaged, } @@ -436,27 +446,34 @@ fn artifact_marked_content( require_all: bool, ) -> Result { let props = page_properties(doc, &page.dict); + // The pieces are the streams, in order. let decoded: Vec<&[u8]> = streams.iter().filter_map(|(_, s)| s.as_ref().map(|(_, bytes, _)| bytes.as_slice())).collect(); + let read: Vec = streams.iter().filter_map(|(_, s)| s.as_ref().map(|(_, _, read)| *read)).collect(); + let damage = read.iter().position(|r| *r == Read::Damaged); let pieces = pdfcraft_content::Pieces::join(&decoded); let mut found = HashSet::new(); let mut edits = Vec::new(); for op in pieces.parse().ops { if let Some(id) = marked_id(doc, &props, &op, ids) { + // What follows the damage is unknown: a string or comment the damage left open runs + // on into the next stream, so what reads as an operator after it may not be one. + if damage.is_some_and(|d| pieces.piece_of(op.span.start) > d) { + return Ok(Marked::Damaged); + } found.insert(id); edits.push((op.span, b"/Artifact BMC".to_vec())); } } - // Content not found may be past the damage in a damaged stream: whether it is on the page - // isn't known. - if found != *ids && streams.iter().any(|(_, s)| s.as_ref().is_some_and(|(_, _, damaged)| *damaged)) { + // Content not found may be past the damage, or in a stream that doesn't decode: whether it + // is on the page isn't known. + if found != *ids && read.iter().any(|r| *r != Read::Whole) { return Ok(Marked::Damaged); } if if require_all { found != *ids } else { found.is_empty() } { return Ok(Marked::NotFound); } let spliced = pieces.splice(edits); - // The pieces are the streams that decoded, in order. - if streams.iter().filter_map(|(_, s)| s.as_ref()).zip(&spliced).any(|((_, old, damaged), new)| *damaged && old != new) { + if read.iter().zip(&decoded).zip(&spliced).any(|((read, old), new)| *read != Read::Whole && *old != new.as_slice()) { return Ok(Marked::Damaged); } let mut spliced = spliced.into_iter(); diff --git a/crates/a11y/src/tests.rs b/crates/a11y/src/tests.rs index 78a6d96589..7e719acabb 100644 --- a/crates/a11y/src/tests.rs +++ b/crates/a11y/src/tests.rs @@ -1103,8 +1103,9 @@ fn damaged_flate(readable: &[u8], rest: &[u8]) -> pdfcraft_cos::Stream { } /// A tag's content in a stream damaged partway stays tagged: rewriting the stream from what -/// decoded would lose what follows the damage. Nothing changes, the structure tree included. -/// Content in a whole stream beside a damaged one is marked, and the damaged one kept as stored. +/// decoded would lose what follows the damage. Nothing changes, the structure tree and the stream +/// included. Content in a whole stream before a damaged one is marked, and the damaged one kept +/// as stored. #[test] fn a_tags_content_in_a_stream_damaged_partway_is_not_rewritten() { use pdfcraft_cos::{ObjRef, Object}; @@ -1113,20 +1114,69 @@ fn a_tags_content_in_a_stream_damaged_partway_is_not_rewritten() { let mut doc = figures_doc(); doc.set(ObjRef::new(4, 0), damaged.clone()); let contents = |doc: &Document| doc.get(ObjRef::new(3, 0)).as_dict().unwrap().get(b"Contents").cloned(); - let (before_contents, before_pt, before_kids) = (contents(&doc), doc.get(ObjRef::new(9, 0)), doc.get(ObjRef::new(6, 0))); + let state = |doc: &Document| (contents(doc), doc.get(ObjRef::new(9, 0)), doc.get(ObjRef::new(6, 0)), doc.get(ObjRef::new(4, 0))); + let before = state(&doc); assert_eq!(figures(&doc).len(), 2, "read up to the damage"); assert_eq!(mark_tag_artifact(&mut doc, ObjRef::new(7, 0)), Err(ArtifactError::Damaged)); assert_eq!(mark_decorative(&mut doc, ObjRef::new(7, 0)), Err(AltError::Damaged)); - assert_eq!((contents(&doc), doc.get(ObjRef::new(9, 0)), doc.get(ObjRef::new(6, 0))), (before_contents, before_pt, before_kids)); + assert_eq!(state(&doc), before); let mut doc = figures_doc(); let other = doc.add(damaged_flate(b"0 g\n", b"0 0 1 rg 0 0 10 10 re f")); - doc.update_dict(ObjRef::new(3, 0), |p| p.set(b"Contents".to_vec(), Object::Array(vec![Object::Ref(other), Object::Ref(ObjRef::new(4, 0))]))) + doc.update_dict(ObjRef::new(3, 0), |p| p.set(b"Contents".to_vec(), Object::Array(vec![Object::Ref(ObjRef::new(4, 0)), Object::Ref(other)]))) .unwrap(); mark_tag_artifact(&mut doc, ObjRef::new(7, 0)).unwrap(); let Some(Object::Array(list)) = contents(&doc) else { panic!("contents {:?}", contents(&doc)) }; - assert_eq!(list.first(), Some(&Object::Ref(other)), "kept as stored"); - assert_ne!(list.get(1), Some(&Object::Ref(ObjRef::new(4, 0))), "the tag's stream is rewritten"); + assert_ne!(list.first(), Some(&Object::Ref(ObjRef::new(4, 0))), "the tag's stream is rewritten"); + assert_eq!(list.get(1), Some(&Object::Ref(other)), "kept as stored"); +} + +/// What follows the damage is unknown, the syntax the next stream starts in too, so a tag's +/// content in a stream after a damaged one stays tagged. Here a comment the damage leaves open +/// runs on into the next stream, up to its first end of line, and a string starting there reads +/// as operators, among them what looks like the figure's content: rewritten, it would change the +/// string. Nothing changes, the structure tree included. +#[test] +fn a_tags_content_after_a_damaged_stream_is_not_rewritten() { + use pdfcraft_cos::{Dict, ObjRef, Object, Stream}; + let in_a_string = b"(a note\n/Figure << /MCID 0 >> BDC 0 g EMC\n) pop /Photo << /MCID 1 >> BDC 100 100 50 30 re f EMC"; + for (damaged, after) in [(b"0 g\n".as_slice(), None), (b"0 g % a comment".as_slice(), Some(in_a_string.as_slice()))] { + let mut doc = figures_doc(); + if let Some(after) = after { + doc.set(ObjRef::new(4, 0), Stream::from_raw(Dict::new(), after.to_vec())); + } + let first = doc.add(damaged_flate(damaged, b"\n0 0 1 rg 0 0 10 10 re f")); + doc.update_dict(ObjRef::new(3, 0), |p| p.set(b"Contents".to_vec(), Object::Array(vec![Object::Ref(first), Object::Ref(ObjRef::new(4, 0))]))) + .unwrap(); + let state = |doc: &Document| { + (doc.get(ObjRef::new(3, 0)).as_dict().unwrap().get(b"Contents").cloned(), doc.get(ObjRef::new(9, 0)), doc.get(ObjRef::new(6, 0))) + }; + let before = state(&doc); + assert_eq!(mark_tag_artifact(&mut doc, ObjRef::new(7, 0)), Err(ArtifactError::Damaged), "{damaged:?}"); + assert_eq!(mark_decorative(&mut doc, ObjRef::new(7, 0)), Err(AltError::Damaged), "{damaged:?}"); + assert_eq!(state(&doc), before, "{damaged:?}"); + } +} + +/// A tag's content not found on the page may be in a stream that doesn't decode: it isn't known +/// to be off the page, so neither marking changes anything, the structure tree included. +#[test] +fn a_tags_content_not_found_beside_a_stream_that_does_not_decode_is_not_taken_for_absent() { + use pdfcraft_cos::{Dict, ObjRef, Object, Stream}; + let mut doc = figures_doc(); + doc.update_dict(ObjRef::new(7, 0), |d| d.set(b"K".to_vec(), Object::Int(2))).unwrap(); + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::name("NoSuchFilter")); + let other = doc.add(Object::Stream(Stream::from_raw(dict, b"/Figure << /MCID 2 >> BDC 0 g EMC".to_vec()))); + doc.update_dict(ObjRef::new(3, 0), |p| p.set(b"Contents".to_vec(), Object::Array(vec![Object::Ref(other), Object::Ref(ObjRef::new(4, 0))]))) + .unwrap(); + let state = |doc: &Document| { + (doc.get(ObjRef::new(3, 0)).as_dict().unwrap().get(b"Contents").cloned(), doc.get(ObjRef::new(9, 0)), doc.get(ObjRef::new(6, 0))) + }; + let before = state(&doc); + assert_eq!(mark_decorative(&mut doc, ObjRef::new(7, 0)), Err(AltError::Damaged)); + assert_eq!(mark_tag_artifact(&mut doc, ObjRef::new(7, 0)), Err(ArtifactError::Damaged)); + assert_eq!(state(&doc), before); } /// A tag's content not found before the damage in a damaged stream may be past it: it isn't known diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index e69a57dc33..4f3b0c374e 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -1285,7 +1285,7 @@ pub fn tools() -> Vec { "property": { "type": "string", "enum": ["type", "title", "actual_text", "alternate_text", "language", "summary", "scope"] }, "value": { "type": "string" }, }), &["doc", "object", "generation", "property"])), - t("accessibility_mark_tag_artifact", "Mark tag content as artifact", "Mark a reachable leaf structure tag's direct marked content as an artifact and remove the tag from the reading structure. Use object and generation from accessibility_tags. Tags with child structure elements, no direct marked content, content not found on its page, or content that is, or may be, in a content stream damaged partway are rejected. Undoable.") + t("accessibility_mark_tag_artifact", "Mark tag content as artifact", "Mark a reachable leaf structure tag's direct marked content as an artifact and remove the tag from the reading structure. Use object and generation from accessibility_tags. Tags with child structure elements, no direct marked content, content not found on its page, or content that is, or may be, in a content stream damaged partway or that doesn't decode, or after the damage, are rejected. Undoable.") .with(schema(json!({ "doc": doc(), "object": { "type": "integer", "minimum": 1 }, "generation": { "type": "integer", "minimum": 0, "maximum": 65535 }, diff --git a/crates/edit/src/images.rs b/crates/edit/src/images.rs index 201612e5b2..5e5c44fab7 100644 --- a/crates/edit/src/images.rs +++ b/crates/edit/src/images.rs @@ -1428,13 +1428,14 @@ fn arrange_image(doc: &mut Document, p: &pdfcraft_model::Page, all: &[ContentStr }; if let Some(piece) = owned_destination { + let insertion = piece.saturating_add(usize::from(insert_after)); + crate::text::drawn_before_damage(all, insertion)?; let changed = pieces.splice([(target_span, removal_replacement)]); let contents = crate::text::rewritten_contents(doc, all, changed)?; let mut list = match contents { Object::Array(list) => list, one => vec![one], }; - let insertion = piece.saturating_add(usize::from(insert_after)); if insertion > list.len() { return Err(EditError::Invalid("the destination content stream is missing".into())); } diff --git a/crates/edit/src/objects.rs b/crates/edit/src/objects.rs index c0a57c358a..6a64e3cfce 100644 --- a/crates/edit/src/objects.rs +++ b/crates/edit/src/objects.rs @@ -338,7 +338,7 @@ fn checked_streams(doc: &Document, page: usize) -> Result, Ed return Err(EditError::Invalid("the page graphics state is nested too deeply".into())); } } - streams.push(ContentStream { entry: object, data: bytes, damaged: false }); + streams.push(ContentStream { entry: object, data: bytes, read: crate::text::Read::Whole }); } Ok(streams) } diff --git a/crates/edit/src/tests.rs b/crates/edit/src/tests.rs index df9bce657d..f3b4e3cc2f 100644 --- a/crates/edit/src/tests.rs +++ b/crates/edit/src/tests.rs @@ -5172,8 +5172,10 @@ fn damaged_page() -> (Document, pdfcraft_cos::Stream) { } /// An edit that would rewrite a content stream damaged partway is refused: written from what -/// decoded, it would lose what follows the damage, which other readers may still read. An edit -/// in the page's other stream goes on, and leaves the damaged one as it is stored. +/// decoded, it would lose what follows the damage, which other readers may still read. So is an +/// edit in a later stream: what the damage hides can change how that stream reads, a string or +/// comment it leaves open, or the graphics state. An edit in a stream before the damage goes on, +/// and leaves the damaged one as it is stored. #[test] fn an_edit_never_rewrites_a_content_stream_damaged_partway() { let (mut doc, damaged) = damaged_page(); @@ -5183,21 +5185,64 @@ fn an_edit_never_rewrites_a_content_stream_damaged_partway() { let listed = contents(&doc); let refused = text::replace_line(&mut doc, 0, 0, "Edited").unwrap_err(); - assert!(refused.to_string().contains("damaged partway"), "{refused}"); + assert!(refused.to_string().contains("is damaged partway"), "{refused}"); assert_eq!(contents(&doc), listed); let refused = text::rewrite_block(&mut doc, 0, 0, Some("Edited"), &text::BlockStyle::default()).unwrap_err(); - assert!(refused.to_string().contains("damaged partway"), "{refused}"); + assert!(refused.to_string().contains("is damaged partway"), "{refused}"); + assert_eq!(contents(&doc), listed); + let refused = text::replace_line(&mut doc, 0, 1, "Edited").unwrap_err(); + assert!(refused.to_string().contains("comes after a content stream that is damaged partway"), "{refused}"); assert_eq!(contents(&doc), listed); - text::replace_line(&mut doc, 0, 1, "Edited").unwrap(); + let mut doc = split_page("BT /F1 12 Tf 72 600 Td (Whole) Tj ET", ""); + doc.set(pdfcraft_cos::ObjRef::new(5, 0), damaged.clone()); + text::replace_line(&mut doc, 0, 0, "Edited").unwrap(); let doc = reopen(&doc); - assert_eq!(texts(&doc), ["Damaged", "Edited"]); + assert_eq!(texts(&doc), ["Edited", "Damaged"]); let Some(Object::Array(list)) = contents(&doc) else { panic!("contents {:?}", contents(&doc)) }; - assert_eq!(list.first(), Some(&Object::Ref(pdfcraft_cos::ObjRef::new(4, 0)))); - let Object::Stream(kept) = &*doc.get(pdfcraft_cos::ObjRef::new(4, 0)) else { panic!("not a stream") }; + assert_eq!(list.get(1), Some(&Object::Ref(pdfcraft_cos::ObjRef::new(5, 0)))); + let Object::Stream(kept) = &*doc.get(pdfcraft_cos::ObjRef::new(5, 0)) else { panic!("not a stream") }; assert_eq!(kept.raw, damaged.raw, "stored as it was"); } +/// A comment the damage leaves open runs on into the next stream, up to its first end of line, +/// so a string starting there reads as operators: a line of text inside it. That line is in a +/// stream after the damage, and isn't edited. +#[test] +fn text_a_damaged_stream_runs_on_into_is_not_edited() { + let mut doc = split_page("", "(note\nBT /F1 12 Tf 72 600 Td (Inside a string) Tj ET\n) pop"); + doc.set(pdfcraft_cos::ObjRef::new(4, 0), damaged_flate(b"0 g % a comment", b"\n")); + let texts = |doc: &Document| text::text_lines(doc, 0).unwrap().into_iter().map(|l| l.text).collect::>(); + assert_eq!(texts(&doc), ["Inside a string"], "the comment runs on into the next stream"); + let listed = pdfcraft_model::pages(&doc)[0].dict.get(b"Contents").cloned(); + let refused = text::replace_line(&mut doc, 0, 0, "Edited").unwrap_err(); + assert!(refused.to_string().contains("comes after"), "{refused}"); + assert_eq!(pdfcraft_model::pages(&doc)[0].dict.get(b"Contents").cloned(), listed); +} + +/// A content stream that doesn't decode is never written over. A partial rewrite of the page's +/// last paragraph closes its marker past the last operator read, in the page's last stream, here +/// one that doesn't decode: the rewrite is refused, not that stream replaced by the close. An +/// empty stream there takes the close. +#[test] +fn a_content_stream_that_does_not_decode_is_never_written_over() { + let src = "BT /F1 12 Tf 14.4 TL 1 0 0 1 72 700 Tm (Alpha first row.) Tj T* (Alpha next row.) Tj ET"; + let mut doc = split_page(src, ""); + let mut dict = pdfcraft_cos::Dict::new(); + dict.set(b"Filter".to_vec(), Object::name("NoSuchFilter")); + doc.set(pdfcraft_cos::ObjRef::new(5, 0), pdfcraft_cos::Stream::from_raw(dict, b"0 0 1 rg 0 0 10 10 re f".to_vec())); + let contents = |doc: &Document| pdfcraft_model::pages(doc)[0].dict.get(b"Contents").cloned(); + let listed = contents(&doc); + let refused = text::replace_block(&mut doc, 0, 0, "One.\nAlpha next row.").unwrap_err(); + assert!(refused.to_string().contains("can't be decoded"), "{refused}"); + assert_eq!(contents(&doc), listed); + + let mut doc = split_page(src, ""); + text::replace_block(&mut doc, 0, 0, "One.\nAlpha next row.").unwrap(); + let texts: Vec<_> = text::text_blocks(&reopen(&doc), 0).unwrap().into_iter().map(|b| b.text).collect(); + assert_eq!(texts, ["One. Alpha next row."]); +} + /// A content stream that doesn't decode at all is no damage to write over: it is kept as stored, /// and the page's other streams are read and edited around it. A stream of a few MB is read for /// selecting objects. @@ -5310,6 +5355,52 @@ fn arrange_and_crop_image_in_second_content_stream_succeeds() { assert_eq!(after_crop[1].visible_rect, Some(after_crop[1].rect)); } +/// What follows the damage in a content stream is unknown, the graphics state the later streams +/// draw in too: here it hides a `cm` that doubles them. An image after the damage isn't moved, as +/// the move is worked out without that `cm`, and one before it isn't arranged to after it, beside +/// an added item there. An image before the damage is moved. +#[test] +fn images_after_a_content_stream_damaged_partway_are_not_moved() { + use pdfcraft_cos::{Dict, ObjRef, Stream}; + let image = "q 100 0 0 100 200 200 cm /Im1 Do Q"; + let hidden_scale = || damaged_flate(b"0 g\n", b"2 0 0 2 0 0 cm"); + let shift = images::ImageChange::Transform([1.0, 0.0, 0.0, 1.0, 10.0, 0.0]); + let contents = |doc: &Document| pdfcraft_model::pages(doc)[0].dict.get(b"Contents").cloned(); + + let mut doc = two_stream_image_page("", image); + doc.set(ObjRef::new(4, 0), hidden_scale()); + let listed = contents(&doc); + let refused = images::change_image(&mut doc, 0, 0, &shift).unwrap_err(); + assert!(refused.to_string().contains("comes after"), "{refused}"); + assert_eq!(contents(&doc), listed); + + let mut doc = two_stream_image_page(image, ""); + let damaged = doc.add(hidden_scale()); + let mut added = Dict::new(); + added.set(b"PCAdded".to_vec(), Object::Dict(Dict::new())); + let added = doc.add(Object::Stream(Stream::from_raw(added, b"q 0 0 1 rg 0 0 10 10 re f Q".to_vec()))); + doc.update_dict(ObjRef::new(3, 0), |p| { + p.set(b"Contents".to_vec(), Object::Array(vec![Object::Ref(ObjRef::new(4, 0)), Object::Ref(damaged), Object::Ref(added)])) + }) + .unwrap(); + let listed = contents(&doc); + let refused = images::change_image(&mut doc, 0, 0, &images::ImageChange::Arrange(images::ArrangeOrder::BringToFront)).unwrap_err(); + assert!(refused.to_string().contains("comes after"), "{refused}"); + assert_eq!(contents(&doc), listed); + // Arranged within the content before the damage, it goes on. + doc.update_dict(ObjRef::new(3, 0), |p| { + p.set(b"Contents".to_vec(), Object::Array(vec![Object::Ref(ObjRef::new(4, 0)), Object::Ref(added), Object::Ref(damaged)])) + }) + .unwrap(); + images::change_image(&mut doc, 0, 0, &images::ImageChange::Arrange(images::ArrangeOrder::BringToFront)).unwrap(); + + let mut doc = two_stream_image_page(image, ""); + doc.set(ObjRef::new(5, 0), hidden_scale()); + images::change_image(&mut doc, 0, 0, &shift).unwrap(); + let moved = images::page_images(&reopen(&doc), 0).unwrap(); + assert!(close(moved[0].rect, [210.0, 200.0, 310.0, 300.0]), "{:?}", moved[0].rect); +} + /// One page whose font draws a few Hebrew letters (ToUnicode): A ב, B ד, C י, D ו, E ק, F ש, /// G ל, H ם. Content streams draw Hebrew in visual order, so "בדיוק" is `(EDCBA)`. fn hebrew_page(content: &str) -> Document { diff --git a/crates/edit/src/text.rs b/crates/edit/src/text.rs index 01c0acdb70..40675b4892 100644 --- a/crates/edit/src/text.rs +++ b/crates/edit/src/text.rs @@ -56,7 +56,8 @@ pub(crate) use fallback::{glyph_path, type3_font_dict, unique_font_name}; use interp::{Origin, Paint, union}; use lines::page_dict; pub(crate) use lines::{ - ContentStream, FormBudget, content_streams, content_streams_within, dict_in_place, form_call, joined, resource_dict, rewritten_contents, splice, + ContentStream, FormBudget, Read, content_streams, content_streams_within, dict_in_place, drawn_before_damage, form_call, joined, resource_dict, + rewritten_contents, splice, }; pub use lines::{MAX_FORM_DEPTH, MAX_FORM_VISITS, glyph_text_lines, text_lines}; pub use rewrite::{LineEdit, replace_line, rewrite_block}; diff --git a/crates/edit/src/text/lines.rs b/crates/edit/src/text/lines.rs index bf66cc0f8b..58b4ac6289 100644 --- a/crates/edit/src/text/lines.rs +++ b/crates/edit/src/text/lines.rs @@ -67,9 +67,19 @@ pub(crate) struct ContentStream { pub(crate) entry: Object, /// Its decoded data, empty when it doesn't decode. pub(crate) data: Vec, - /// The stream is damaged partway: `data` is what decoded before the damage, so the stream - /// can't be written again from it without losing the rest ([`rewritten_contents`]). - pub(crate) damaged: bool, + /// How much of it decoded. + pub(crate) read: Read, +} + +/// How much of a content stream decoded ([`rewritten_contents`] says what each allows). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Read { + Whole, + /// Damaged partway: the data is what decoded before the damage. What follows it is unknown, + /// the graphics state and syntax the later streams start in too. + Damaged, + /// Not at all (a filter this reader lacks, say): the data is empty. + Unread, } /// The page's content streams as listed, each with its decoded data, within one page's @@ -91,15 +101,16 @@ pub(crate) fn content_streams_within(doc: &Document, page: &Dict, budget: &mut D }; let mut out = Vec::with_capacity(list.len()); for entry in list { - let (data, damaged) = match &*doc.resolve(&entry) { + let (data, read) = match &*doc.resolve(&entry) { Object::Stream(s) => match budget.decode_with_damage(doc, s, usize::MAX) { - Ok(read) => read, + Ok((data, damaged)) => (data, if damaged { Read::Damaged } else { Read::Whole }), Err(e) if budget.remaining() == 0 => return Err(EditError::Invalid(format!("the page content is too large to read: {e}"))), - Err(_) => (Vec::new(), false), + Err(_) => (Vec::new(), Read::Unread), }, - _ => (Vec::new(), false), + // Nothing to lose: a null or missing entry draws nothing. + _ => (Vec::new(), Read::Whole), }; - out.push(ContentStream { entry, data, damaged }); + out.push(ContentStream { entry, data, read }); } Ok(out) } @@ -136,15 +147,50 @@ pub(crate) fn splice(pieces: &Pieces, ops: &[Op], mut edit: impl FnMut(usize) -> pieces.splice(edits) } +/// An error when content at `/Contents` index `at` (a stream there, or one inserted there) is drawn +/// after a stream damaged partway: what the damage hides can change how it reads. +pub(crate) fn drawn_before_damage(streams: &[ContentStream], at: usize) -> Result<(), EditError> { + if at > first_damaged(streams) { + return Err(EditError::Invalid( + "this change comes after a content stream that is damaged partway, and what the damage hides can change how the rest of the page reads" + .into(), + )); + } + Ok(()) +} + +/// The index of the first stream damaged partway, or the number of streams. +fn first_damaged(streams: &[ContentStream]) -> usize { + streams.iter().position(|s| s.read == Read::Damaged).unwrap_or(streams.len()) +} + /// The page's `/Contents` after a [`splice`]: pieces whose bytes changed become new streams /// (keeping their dictionaries), the others stay as they were. An error, before anything is -/// added, when a piece that changed is a damaged stream: written from what decoded, it would -/// lose everything after the damage. +/// added, when a piece that changed is a stream that didn't decode whole, or comes after one +/// damaged partway. Written from what decoded, a damaged stream would lose everything after the +/// damage, and one that doesn't decode all of it. What follows the damage is unknown, so a later +/// stream may not read as it seems: a string or comment the damage left open runs on into it, and +/// operators the damage hides can change the graphics state it draws in. pub(crate) fn rewritten_contents(doc: &mut Document, streams: &[ContentStream], new: Vec>) -> Result { - if streams.iter().zip(&new).any(|(s, data)| s.damaged && s.data != *data) { - return Err(EditError::Invalid( - "this change is in a content stream that is damaged partway, and rewriting it would lose what follows the damage".into(), - )); + // The last stream that changed: when it is drawn before the damage, so are the others. + if let Some(i) = streams.iter().zip(&new).rposition(|(s, data)| s.data != *data) { + drawn_before_damage(streams, i)?; + } + for (s, data) in streams.iter().zip(&new) { + if s.data == *data { + continue; + } + match s.read { + Read::Whole => {} + Read::Damaged => { + return Err(EditError::Invalid( + "this change is in a content stream that is damaged partway, and rewriting it would lose what follows the damage".into(), + )); + } + Read::Unread => { + return Err(EditError::Invalid("this change is in a content stream that can't be decoded, and rewriting it would lose it".into())); + } + } } let mut contents: Vec = Vec::with_capacity(streams.len()); for (s, data) in streams.iter().zip(new) { From 8bf741222016dc9fd997f96100897edc95c4049e Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Fri, 6 Nov 2026 01:01:24 +0100 Subject: [PATCH 162/198] M2.6: pruning reads every form that damaged content may draw Earlier in this change, a page or form damaged partway, like one that doesn't decode, kept its own /XObject dictionary whole and was read no further. So a form it draws was never read: when the form's resources are shared with another page, the names the form draws there went, and extracted or split pages lost those images. On main the part before the damage was read, so forms drawn there kept theirs, and forms drawn past it didn't. Now every form the resources of such content name is read as drawn, as any of them may be drawn before the damage or past it. Each set of resources has its forms read once. Test: extract_keeps_images_forms_drawn_around_damage_draw_through_shared_resources, with a page and a form each damaged before and past the form they draw, and each not decoding. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/organize/src/prune.rs | 92 +++++++++++++++++++++--------------- crates/organize/src/tests.rs | 63 ++++++++++++++++++++++++ 2 files changed, 118 insertions(+), 37 deletions(-) diff --git a/crates/organize/src/prune.rs b/crates/organize/src/prune.rs index 591e804b1e..bac8ad5f90 100644 --- a/crates/organize/src/prune.rs +++ b/crates/organize/src/prune.rs @@ -53,6 +53,9 @@ struct Pruner<'a> { /// Forms already analysed, with the owner of the resources they drew against: drawing one /// again records nothing new, and re-reading it on every `Do` is exponential in the nesting. forms_seen: HashSet<(ObjRef, Owner)>, + /// Owners of resources that content not read whole drew against, whose every form was read + /// as drawn ([`Pruner::unread`]): once is enough. + partly_read: HashSet, } impl Pruner<'_> { @@ -194,39 +197,57 @@ impl Pruner<'_> { } let Some(name) = op.name(0) else { continue }; self.used.entry(key).or_default().insert(name.to_vec()); - let Some(r) = xobjects.get(name).and_then(Object::as_ref) else { continue }; - let Object::Stream(s) = &*self.doc.get(r) else { continue }; - if s.dict.name(b"Subtype") != Some(b"Form") { - continue; - } - let (inner, inner_owner) = match s.dict.get(b"Resources") { - Some(res) => match res.as_ref() { - Some(rr) => match self.doc.get(rr).as_dict().cloned() { - Some(d) => (d, Owner::Resources(rr)), - None => continue, - }, - None => match self.doc.resolve(res).as_dict().cloned() { - // The form carries its resources inline: filter them on the form itself. - Some(d) => (d, Owner::Resources(r)), - None => continue, - }, + self.form(&xobjects, name, resources, owner, depth); + } + } + + /// For content that can't be read whole (damaged partway, or not decoding at all): what it + /// draws is unknown, so its `/XObject` dictionary is kept whole, and every form there is read + /// as drawn, which also reads every form the part before any damage draws. A form whose + /// resources are shared keeps what it draws from them too. + fn unread(&mut self, resources: &Dict, owner: Owner, depth: u8) { + self.protect_type3_resources(resources, owner); + let Some((xobjects, key)) = self.xobjects(resources, owner) else { return }; + self.unsure.insert(key); + if depth > MAX_FORM_DEPTH || !self.partly_read.insert(owner) { + return; + } + for (name, _) in xobjects.iter() { + self.form(&xobjects, name, resources, owner, depth); + } + } + + /// Record what the XObject `name` of `xobjects` draws when it is a form, drawn from content + /// with `resources`. + fn form(&mut self, xobjects: &Dict, name: &[u8], resources: &Dict, owner: Owner, depth: u8) { + let Some(r) = xobjects.get(name).and_then(Object::as_ref) else { return }; + let Object::Stream(s) = &*self.doc.get(r) else { return }; + if s.dict.name(b"Subtype") != Some(b"Form") { + return; + } + let (inner, inner_owner) = match s.dict.get(b"Resources") { + Some(res) => match res.as_ref() { + Some(rr) => match self.doc.get(rr).as_dict().cloned() { + Some(d) => (d, Owner::Resources(rr)), + None => return, }, - // A form without its own resources draws against the ones it was invoked with. - None => (resources.clone(), owner), - }; - if !self.forms_seen.insert((r, inner_owner)) { - continue; - } - #[cfg(test)] - crate::work(1); - match DecodeBudget::new(usize::MAX, 0).decode_with_damage(self.doc, s, usize::MAX) { - Ok((data, false)) => self.content(&data, &inner, inner_owner, depth + 1), - // What the form draws is unknown, past damage too: keep everything it could reach. - Ok((_, true)) | Err(_) => { - self.protect_type3_resources(&inner, inner_owner); - self.distrust(&inner, inner_owner); - } - } + None => match self.doc.resolve(res).as_dict().cloned() { + // The form carries its resources inline: filter them on the form itself. + Some(d) => (d, Owner::Resources(r)), + None => return, + }, + }, + // A form without its own resources draws against the ones it was invoked with. + None => (resources.clone(), owner), + }; + if !self.forms_seen.insert((r, inner_owner)) { + return; + } + #[cfg(test)] + crate::work(1); + match DecodeBudget::new(usize::MAX, 0).decode_with_damage(self.doc, s, usize::MAX) { + Ok((data, false)) => self.content(&data, &inner, inner_owner, depth + 1), + Ok((_, true)) | Err(_) => self.unread(&inner, inner_owner, depth + 1), } } } @@ -321,6 +342,7 @@ pub(crate) fn prune_unused_xobjects(doc: &mut Document) -> Result Result {} Ok((data, false)) => pruner.content(&data, &res_dict, owner, 0), - // What the page draws past damage is unknown too. - Ok((_, true)) | Err(_) => { - pruner.protect_type3_resources(&res_dict, owner); - pruner.distrust(&res_dict, owner); - } + Ok((_, true)) | Err(_) => pruner.unread(&res_dict, owner, 0), } } let Pruner { used, unsure, glyph_budget_exhausted, .. } = pruner; diff --git a/crates/organize/src/tests.rs b/crates/organize/src/tests.rs index 8d41f474cc..d6273cc65b 100644 --- a/crates/organize/src/tests.rs +++ b/crates/organize/src/tests.rs @@ -2669,6 +2669,69 @@ fn extract_keeps_images_content_damaged_partway_might_draw() { } } +/// Two pages: the first draws `/Fm` through resources of its own, `/Fm` draws `/Nested` through +/// resources of its own too, and `/Nested` draws `/ImA` through resources it shares with the second +/// page, which draws `/ImB` through them. The three streams are the first page's content, `/Fm` +/// and `/Nested`. +fn doc_shared_form_resources(page: pdfcraft_cos::Stream, fm: pdfcraft_cos::Stream, nested: pdfcraft_cos::Stream) -> Document { + use pdfcraft_cos::ObjRef; + let b: Vec = vec![ + "<< /Type /Catalog /Pages 2 0 R >>".into(), + "<< /Type /Pages /Kids [3 0 R 4 0 R] /Count 2 /MediaBox [0 0 200 200] >>".into(), + "<< /Type /Page /Parent 2 0 R /Contents 5 0 R /Resources << /XObject << /Fm 7 0 R >> >> >>".into(), + "<< /Type /Page /Parent 2 0 R /Contents 6 0 R /Resources 8 0 R >>".into(), + raw(""), + raw("/ImB Do"), + raw(""), + "<< /XObject << /ImA 9 0 R /ImB 10 0 R >> >>".into(), + image(1, 1, 1), + image(1, 1, 2000), + raw(""), + ]; + let mut doc = open(build(&b, "/Root 1 0 R")); + doc.set(ObjRef::new(5, 0), page); + let nested_res: Dict = + [(b"XObject".to_vec(), Object::Dict([(b"Nested".to_vec(), Object::Ref(ObjRef::new(11, 0)))].into_iter().collect()))].into_iter().collect(); + for (num, mut form, resources) in [(7, fm, Object::Dict(nested_res)), (11, nested, Object::Ref(ObjRef::new(8, 0)))] { + form.dict.set(b"Subtype".to_vec(), Object::name("Form")); + form.dict.set(b"BBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 1.into(), 1.into()])); + form.dict.set(b"Resources".to_vec(), resources); + doc.set(ObjRef::new(num, 0), form); + } + doc +} + +/// Every form the resources of a page or form damaged partway name may be drawn, before the +/// damage or past it, so each is read: one whose resources another page shares keeps what it draws +/// from them. So is each one content that doesn't decode may draw. +#[test] +fn extract_keeps_images_forms_drawn_around_damage_draw_through_shared_resources() { + let whole = |content: &[u8]| pdfcraft_cos::Stream::from_raw(Dict::new(), content.to_vec()); + let undecodable = |content: &[u8]| { + let mut s = whole(content); + s.dict.set(b"Filter".to_vec(), Object::name("NoSuchFilter")); + s + }; + let cases = [ + ("page, before the damage", damaged_flate(b"/Fm Do\n", b"0 g"), whole(b"/Nested Do")), + ("page, past the damage", damaged_flate(b"0 g\n", b"/Fm Do"), whole(b"/Nested Do")), + ("page that doesn't decode", undecodable(b"/Fm Do"), whole(b"/Nested Do")), + ("form, before the damage", whole(b"/Fm Do"), damaged_flate(b"/Nested Do\n", b"0 g")), + ("form, past the damage", whole(b"/Fm Do"), damaged_flate(b"0 g\n", b"/Nested Do")), + ("form that doesn't decode", whole(b"/Fm Do"), undecodable(b"/Nested Do")), + ]; + for (what, page, fm) in cases { + let part = full_roundtrip(&extract_pages(&doc_shared_form_resources(page, fm, whole(b"/ImA Do")), &[0, 1]).unwrap()); + let names = xobject_names(&part, 1); + assert!(names.iter().any(|n| n.as_slice() == b"ImA"), "{what}: {names:?}"); + assert!(names.iter().any(|n| n.as_slice() == b"ImB"), "{what}: {names:?}"); + } + // What nothing draws still goes. + let src = doc_shared_form_resources(whole(b"0 g"), whole(b"/Nested Do"), whole(b"/ImA Do")); + let part = full_roundtrip(&extract_pages(&src, &[0, 1]).unwrap()); + assert_eq!(xobject_names(&part, 1), [b"ImB".to_vec()]); +} + /// A form whose `/DecodeParms` is given by reference is read through it: the image it draws is /// kept, not lost to content read with the predictor's tag bytes left in it. #[test] From 749a3233422614f3a9c97cf7d71a40f5696935b0 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Fri, 6 Nov 2026 01:01:24 +0100 Subject: [PATCH 163/198] M2.6: tests pin damage in an earlier filter stage, on another page, and in the saved file - cos: a filter chain whose first stage stops at damage and whose second reads all it was given is damaged. - optimize: a page damaged partway stops resampling for the whole document, so another page drawing the same image whole doesn't get it resampled. - redact: after redacting content with /DecodeParms by reference, no stream in the saved file holds the text, read with its parameters. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/cos/src/budget.rs | 11 +++++++++++ crates/optimize/tests/optimize.rs | 18 ++++++++++++++++++ crates/redact/src/tests.rs | 9 ++++++++- 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/crates/cos/src/budget.rs b/crates/cos/src/budget.rs index 51b05e034d..23bcaf1af7 100644 --- a/crates/cos/src/budget.rs +++ b/crates/cos/src/budget.rs @@ -850,6 +850,17 @@ mod tests { dict.set(b"Filter".to_vec(), Object::Array(vec![Object::name("FlateDecode"), Object::name("ASCIIHexDecode")])); let hex = Stream::from_raw(dict, Stream::flate(Dict::new(), b"4142zz4344>").raw.to_vec()); assert_eq!(budget.decode_with_damage(&doc, &hex, usize::MAX).ok(), Some((b"AB".to_vec(), true))); + // So is an earlier stage's, though the stage after it reads all it was given: a stored + // block of whole hex, then a block of the reserved type (RFC 1951 §3.2.3). + let readable = b"4142>"; + let len = u16::try_from(readable.len()).unwrap_or_default(); + let mut raw = vec![0x78, 0x01, 0x00]; + raw.extend_from_slice(&len.to_le_bytes()); + raw.extend_from_slice(&(!len).to_le_bytes()); + raw.extend_from_slice(readable); + raw.push(0x07); + let flate_damaged = Stream::from_raw(hex.dict.clone(), raw); + assert_eq!(budget.decode_with_damage(&doc, &flate_damaged, usize::MAX).ok(), Some((b"AB".to_vec(), true))); } /// `/DecodeParms` given by reference, or a parameter given by reference in them, is read: a diff --git a/crates/optimize/tests/optimize.rs b/crates/optimize/tests/optimize.rs index 7250e9a0a0..2785ddac6c 100644 --- a/crates/optimize/tests/optimize.rs +++ b/crates/optimize/tests/optimize.rs @@ -3220,6 +3220,24 @@ fn content_damaged_partway_leaves_every_image_at_its_resolution() { } } +/// One page damaged partway stops resampling for the whole document: another page drawing the same +/// image at its full resolution, whole, doesn't get it resampled either. +#[test] +fn content_damaged_partway_on_one_page_leaves_the_image_another_page_draws() { + let mut doc = Document::new_empty(); + let img = image(&mut doc, 720, 720, 1, false, None); + let damaged = page(&mut doc, &[("Im", img)], ""); + let bad = doc.add(Object::Stream(damaged_flate(b"0 g\n", b"q 720 0 0 720 0 0 cm /Im Do Q"))); + doc.update_dict(damaged, |d| d.set(b"Contents".to_vec(), Object::Ref(bad))).unwrap(); + let whole = page(&mut doc, &[("Im", img)], "q 72 0 0 72 0 0 cm /Im Do Q"); + assert_eq!(effective_resolutions(&doc, &[whole])[&img], 720.0); + let ppi = effective_resolutions(&doc, &[damaged, whole]).get(&img).copied().unwrap_or(f64::INFINITY); + assert_eq!(ppi, f64::INFINITY); + let r = optimize(&mut doc, &Settings::default()).unwrap(); + assert_eq!(r.images_resampled, 0, "{r:?}"); + assert_eq!(doc.get(img).as_dict().and_then(|d| d.int(b"Width")), Some(720)); +} + /// `levels` nested forms with their own resources, the innermost drawing `img` over the unit square. fn nested_forms(doc: &mut Document, img: ObjRef, levels: usize) -> ObjRef { let mut inner = form(doc, Some(Object::Dict(resources(xobjects(&[("Im", img)])))), "/Im Do"); diff --git a/crates/redact/src/tests.rs b/crates/redact/src/tests.rs index 2de12ba374..f080df0e46 100644 --- a/crates/redact/src/tests.rs +++ b/crates/redact/src/tests.rs @@ -130,7 +130,14 @@ fn content_with_parameters_given_by_reference_is_redacted() { mark(&mut doc, 0, &marked, ""); let r = apply(&mut doc, None).unwrap(); assert_eq!((r.marks, r.glyphs), (1, 6), "{what}"); - assert_eq!(under(&mut reopen(&doc), 0, &marked), 0, "{what}"); + let mut saved = reopen(&doc); + assert_eq!(under(&mut saved, 0, &marked), 0, "{what}"); + // Nor is the text left in a stream the page no longer draws, read with its parameters. + for (r, o) in saved.scan_objects() { + let Object::Stream(s) = &*o else { continue }; + let data = pdfcraft_cos::DecodeBudget::new(usize::MAX, 0).decode_with_damage(&saved, s, usize::MAX).map(|(d, _)| d).unwrap_or_default(); + assert!(!data.windows(6).any(|w| w == b"SECRET"), "{what}: {r:?}"); + } } } From 75bf5ed264ba3995247e779308d1d67a689d1fc6 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Fri, 6 Nov 2026 01:35:15 +0100 Subject: [PATCH 164/198] M2.6: a test pins that pruning reads the forms of damaged content at the depth limit Forms are read 8 deep. A damaged form that deep has every form it names read as drawn, so one drawing through resources another page shares keeps what it draws: a_damaged_form_at_the_depth_limit_has_its_forms_read. Mutation testing found nothing pinned that boundary. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/organize/src/tests.rs | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/crates/organize/src/tests.rs b/crates/organize/src/tests.rs index d6273cc65b..3d956727e3 100644 --- a/crates/organize/src/tests.rs +++ b/crates/organize/src/tests.rs @@ -2732,6 +2732,34 @@ fn extract_keeps_images_forms_drawn_around_damage_draw_through_shared_resources( assert_eq!(xobject_names(&part, 1), [b"ImB".to_vec()]); } +/// Forms are read 8 deep (`prune.rs`'s `MAX_FORM_DEPTH`). A damaged form that deep still has +/// every form it names read as drawn, and one drawing through resources another page shares keeps +/// what it draws from them. +#[test] +fn a_damaged_form_at_the_depth_limit_has_its_forms_read() { + use pdfcraft_cos::ObjRef; + let whole = |content: &[u8]| pdfcraft_cos::Stream::from_raw(Dict::new(), content.to_vec()); + let mut src = doc_shared_form_resources(whole(b"/C Do"), damaged_flate(b"0 g\n", b"/Nested Do"), whole(b"/ImA Do")); + // Seven forms between the page and `/Fm`, each drawing the next as `/C`: `/Fm` is 8 deep. + let mut next = ObjRef::new(7, 0); + for _ in 1..8 { + let mut form = whole(b"/C Do"); + form.dict.set(b"Subtype".to_vec(), Object::name("Form")); + form.dict.set(b"BBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 1.into(), 1.into()])); + let names: Dict = [(b"C".to_vec(), Object::Ref(next))].into_iter().collect(); + form.dict.set(b"Resources".to_vec(), Object::Dict([(b"XObject".to_vec(), Object::Dict(names))].into_iter().collect())); + next = src.add(Object::Stream(form)); + } + let names: Dict = [(b"C".to_vec(), Object::Ref(next))].into_iter().collect(); + src.update_dict(ObjRef::new(3, 0), |p| { + p.set(b"Resources".to_vec(), Object::Dict([(b"XObject".to_vec(), Object::Dict(names))].into_iter().collect())) + }) + .unwrap(); + let part = full_roundtrip(&extract_pages(&src, &[0, 1]).unwrap()); + let names = xobject_names(&part, 1); + assert!(names.iter().any(|n| n.as_slice() == b"ImA"), "{names:?}"); +} + /// A form whose `/DecodeParms` is given by reference is read through it: the image it draws is /// kept, not lost to content read with the predictor's tag bytes left in it. #[test] From 084fa0927edbb387bd7cee5365224e22142096b9 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Fri, 6 Nov 2026 01:37:51 +0100 Subject: [PATCH 165/198] M2.6: say why pruning reads a form's /Subtype only when direct The sweep flags the read in Pruner::form, moved there by this change. A /Subtype given by reference reads as no form wherever the workspace tells forms from images; resolving it is a change across all of them, and the comment says so. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/organize/src/prune.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/organize/src/prune.rs b/crates/organize/src/prune.rs index bac8ad5f90..85ddb5f743 100644 --- a/crates/organize/src/prune.rs +++ b/crates/organize/src/prune.rs @@ -222,6 +222,9 @@ impl Pruner<'_> { fn form(&mut self, xobjects: &Dict, name: &[u8], resources: &Dict, owner: Owner, depth: u8) { let Some(r) = xobjects.get(name).and_then(Object::as_ref) else { return }; let Object::Stream(s) = &*self.doc.get(r) else { return }; + // A `/Subtype` given by reference reads as no form here, as wherever the workspace tells + // forms from images (redaction, edit and Optimize among them): resolving it is a change + // across all of them. if s.dict.name(b"Subtype") != Some(b"Form") { return; } From d289038ca01432a97e164e29af181c55500bbb76 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Fri, 6 Nov 2026 01:52:43 +0100 Subject: [PATCH 166/198] M2.6: autotagging reads page content with its /DecodeParms given by reference Autotagging a text-only page rewrites its content, marked, from what it read. It read each stream through its filters with /DecodeParms only when direct, so content behind a predictor whose parameters are given by reference was read with the predictor's tag bytes in it, and written back so: with eight-byte rows, a \0 every eight bytes split `24 Tf` into `2 4 Tf` and the text's string, and the tool returned ok. Found by the consumers review of the CodeRabbit round; inherited from main. It now reads through DecodeBudget::decode_strict, which reads the parameters in the document and is strict on damage as before, within the same 16 MiB. pdfcraft-a11y no longer uses pdfcraft-filters. Test: autotag_text_reads_content_with_parameters_given_by_reference. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- Cargo.lock | 1 - crates/a11y/Cargo.toml | 1 - crates/a11y/src/autotag.rs | 19 +++++++++---------- crates/a11y/src/tests.rs | 33 +++++++++++++++++++++++++++++++++ 4 files changed, 42 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8617c45522..520413f003 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4254,7 +4254,6 @@ version = "0.18.1" dependencies = [ "pdfcraft-content", "pdfcraft-cos", - "pdfcraft-filters", "pdfcraft-fonts", "pdfcraft-model", "thiserror 2.0.21", diff --git a/crates/a11y/Cargo.toml b/crates/a11y/Cargo.toml index 42a86bbad2..adef6053a1 100644 --- a/crates/a11y/Cargo.toml +++ b/crates/a11y/Cargo.toml @@ -14,6 +14,5 @@ workspace = true pdfcraft-cos = { workspace = true } pdfcraft-model = { workspace = true } pdfcraft-content = { workspace = true } -pdfcraft-filters = { workspace = true } pdfcraft-fonts = { workspace = true } thiserror.workspace = true diff --git a/crates/a11y/src/autotag.rs b/crates/a11y/src/autotag.rs index 9b65f6972b..5f3a75ceb0 100644 --- a/crates/a11y/src/autotag.rs +++ b/crates/a11y/src/autotag.rs @@ -4,7 +4,7 @@ //! which can then be retagged with the existing tag-property tool after human review. use pdfcraft_content::{Op, parse, serialize_ops}; -use pdfcraft_cos::{Dict, Document, ObjRef, Object, Stream}; +use pdfcraft_cos::{DecodeBudget, Dict, Document, ObjRef, Object, Stream}; const MAX_PAGES: usize = 1_000; const MAX_PAGE_BYTES: usize = 16 * 1024 * 1024; @@ -59,16 +59,15 @@ fn content_bytes(doc: &Document, page: &pdfcraft_model::Page, index: usize) -> R if remaining == 0 { return Err(unsupported(index, "decoded content exceeds 16 MiB")); } - let chain = stream.filters(); - let decoded = if chain.is_empty() { - if stream.raw.len() > remaining { - return Err(unsupported(index, "decoded content exceeds 16 MiB")); + // Strictly, and with `/DecodeParms` given by reference: the content is written back from it. + let mut budget = DecodeBudget::new(remaining, 0); + let decoded = budget.decode_strict(doc, stream).map_err(|e| { + if budget.remaining() == 0 { + unsupported(index, "decoded content exceeds 16 MiB") + } else { + unsupported(index, format!("content cannot be decoded strictly: {e}")) } - stream.raw.to_vec() - } else { - pdfcraft_filters::decode(&chain, &stream.raw, remaining) - .map_err(|e| unsupported(index, format!("content cannot be decoded strictly: {e}")))? - }; + })?; all.extend_from_slice(&decoded); all.push(b' '); } diff --git a/crates/a11y/src/tests.rs b/crates/a11y/src/tests.rs index 7e719acabb..304e4ad8c4 100644 --- a/crates/a11y/src/tests.rs +++ b/crates/a11y/src/tests.rs @@ -79,6 +79,39 @@ fn autotag_text_writes_marked_content_and_parent_tree_across_pages() { assert_eq!(status(&check(&reopened, &Options::default()), Rule::TaggedContent), Status::Passed); } +/// Page content behind a predictor whose `/DecodeParms` are given by reference is read through +/// them: autotagging writes the text back as it was, marked, not with the predictor's tag bytes in +/// it (a `\0` every eight bytes, which split `24` into `2 4`). +#[test] +fn autotag_text_reads_content_with_parameters_given_by_reference() { + use pdfcraft_cos::{Dict, ObjRef}; + let mut content = b"BT /F1 24 Tf 72 700 Td (Hello World) Tj ET".to_vec(); + content.resize(content.len().div_ceil(8) * 8, b' '); + // PNG rows of eight bytes, each tagged 0 (no prediction). + let rows: Vec = content.chunks(8).flat_map(|row| std::iter::once(0).chain(row.iter().copied())).collect(); + let mut doc = pdf( + &[ + "<< /Type /Catalog /Pages 2 0 R >>".into(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>".into(), + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 600 800] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >>".into(), + stream(""), + FONT.into(), + ], + "", + ); + let parms: Dict = [(b"Predictor".to_vec(), Object::Int(12)), (b"Columns".to_vec(), Object::Int(8))].into_iter().collect(); + let parms = doc.add(Object::Dict(parms)); + let mut predicted = pdfcraft_cos::Stream::flate(Dict::new(), &rows); + predicted.dict.set(b"DecodeParms".to_vec(), Object::Ref(parms)); + doc.set(ObjRef::new(4, 0), predicted); + assert_eq!(autotag_text(&mut doc).unwrap(), AutoTagResult { pages: 1, paragraphs: 1 }); + let page = &pdfcraft_model::pages(&doc)[0]; + let written = doc.resolve(page.dict.get(b"Contents").unwrap()); + let Object::Stream(written) = &*written else { panic!("page content") }; + let text = String::from_utf8_lossy(&written.decoded_strict().unwrap()).into_owned(); + assert!(text.contains("/F1 24 Tf") && text.contains("(Hello World) Tj"), "{text}"); +} + #[test] fn autotag_text_refuses_paint_and_does_not_modify_the_document() { let mut doc = pdf( From 477f0abd49ea318bede4314e67d4842fd41fa9b1 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Fri, 6 Nov 2026 01:57:10 +0100 Subject: [PATCH 167/198] M2.6: a test pins why autotagging refuses a page it cannot read Content past 16 MiB says so, and content that does not decode strictly says that: autotag_text_says_why_it_cannot_read_a_page. Mutation testing found nothing pinned which message each gets. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/a11y/src/tests.rs | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/crates/a11y/src/tests.rs b/crates/a11y/src/tests.rs index 304e4ad8c4..6275c50437 100644 --- a/crates/a11y/src/tests.rs +++ b/crates/a11y/src/tests.rs @@ -112,6 +112,35 @@ fn autotag_text_reads_content_with_parameters_given_by_reference() { assert!(text.contains("/F1 24 Tf") && text.contains("(Hello World) Tj"), "{text}"); } +/// Autotagging refuses a page whose content it can't read whole, and says why: content past +/// 16 MiB, or content that doesn't decode strictly. +#[test] +fn autotag_text_says_why_it_cannot_read_a_page() { + use pdfcraft_cos::{Dict, ObjRef, Stream}; + let reason = |content: Stream| { + let mut doc = pdf( + &[ + "<< /Type /Catalog /Pages 2 0 R >>".into(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>".into(), + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 600 800] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >>".into(), + stream(""), + FONT.into(), + ], + "", + ); + doc.set(ObjRef::new(4, 0), content); + match autotag_text(&mut doc) { + Err(AutoTagError::Unsupported { reason, .. }) => reason, + other => panic!("{other:?}"), + } + }; + assert_eq!(reason(Stream::from_raw(Dict::new(), vec![b' '; 17 << 20])), "decoded content exceeds 16 MiB"); + let mut corrupt = Stream::from_raw(Dict::new(), b"not zlib data".to_vec()); + corrupt.dict.set(b"Filter".to_vec(), Object::name("FlateDecode")); + let why = reason(corrupt); + assert!(why.starts_with("content cannot be decoded strictly"), "{why}"); +} + #[test] fn autotag_text_refuses_paint_and_does_not_modify_the_document() { let mut doc = pdf( From 79ea9154cfd63fff70391f63eeaf87b480bf4148 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Fri, 6 Nov 2026 02:23:10 +0100 Subject: [PATCH 168/198] M2.6: autotagging says decoding takes more than 16 MiB, not that the content does Read through DecodeBudget::decode_strict since cbc3fc221, a stream counts what each of its filter stages decodes against the 16 MiB, as a page's decode allowance does, where main bounded each stage alone. A page of 12 MiB behind ASCIIHex and Flate, with 9.1 MiB between them, was tagged on main and is refused now; the refusal said the decoded content exceeds 16 MiB, which it does not. It now says decoding the content takes more than that. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/a11y/src/autotag.rs | 5 +++-- crates/a11y/src/tests.rs | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/a11y/src/autotag.rs b/crates/a11y/src/autotag.rs index 5f3a75ceb0..b9ffa46669 100644 --- a/crates/a11y/src/autotag.rs +++ b/crates/a11y/src/autotag.rs @@ -57,13 +57,14 @@ fn content_bytes(doc: &Document, page: &pdfcraft_model::Page, index: usize) -> R let Object::Stream(stream) = &*value else { return Err(unsupported(index, "contents include a non-stream object")) }; let remaining = MAX_PAGE_BYTES.saturating_sub(all.len().saturating_add(1)); if remaining == 0 { - return Err(unsupported(index, "decoded content exceeds 16 MiB")); + return Err(unsupported(index, "decoding the content takes more than 16 MiB")); } // Strictly, and with `/DecodeParms` given by reference: the content is written back from it. let mut budget = DecodeBudget::new(remaining, 0); let decoded = budget.decode_strict(doc, stream).map_err(|e| { + // The allowance counts what every filter stage decodes, as a page's does. if budget.remaining() == 0 { - unsupported(index, "decoded content exceeds 16 MiB") + unsupported(index, "decoding the content takes more than 16 MiB") } else { unsupported(index, format!("content cannot be decoded strictly: {e}")) } diff --git a/crates/a11y/src/tests.rs b/crates/a11y/src/tests.rs index 6275c50437..0ad6ecd3e5 100644 --- a/crates/a11y/src/tests.rs +++ b/crates/a11y/src/tests.rs @@ -134,7 +134,7 @@ fn autotag_text_says_why_it_cannot_read_a_page() { other => panic!("{other:?}"), } }; - assert_eq!(reason(Stream::from_raw(Dict::new(), vec![b' '; 17 << 20])), "decoded content exceeds 16 MiB"); + assert_eq!(reason(Stream::from_raw(Dict::new(), vec![b' '; 17 << 20])), "decoding the content takes more than 16 MiB"); let mut corrupt = Stream::from_raw(Dict::new(), b"not zlib data".to_vec()); corrupt.dict.set(b"Filter".to_vec(), Object::name("FlateDecode")); let why = reason(corrupt); From c3ca0633737be8a96a3e5d83b5e6751ae0c64d70 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Fri, 6 Nov 2026 02:35:33 +0100 Subject: [PATCH 169/198] M2.6: a test pins that pruning reads a long chain of forms only to the depth limit A chain of 100,000 forms, whole or each damaged partway, is read only down to the depth limit, so the walk's recursion stays bounded: a_long_chain_of_forms_is_read_only_to_the_depth_limit. Nothing pinned it: with the depth step dropped in either of the two calls that go one form deeper (reading a form whole, and reading one that is damaged), or the depth check in the walk of damaged content made to apply only with the read-once guard, the walk follows the chain to its end and the stack overflows. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/organize/src/tests.rs | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/crates/organize/src/tests.rs b/crates/organize/src/tests.rs index 3d956727e3..dfd9eecf35 100644 --- a/crates/organize/src/tests.rs +++ b/crates/organize/src/tests.rs @@ -2760,6 +2760,29 @@ fn a_damaged_form_at_the_depth_limit_has_its_forms_read() { assert!(names.iter().any(|n| n.as_slice() == b"ImA"), "{names:?}"); } +/// A chain of forms nested far past the depth limit, whole or each damaged partway, is read only +/// down to it: the walk's depth stays bounded however long the chain is, and pruning goes on. +#[test] +fn a_long_chain_of_forms_is_read_only_to_the_depth_limit() { + use pdfcraft_cos::ObjRef; + let whole = |content: &[u8]| pdfcraft_cos::Stream::from_raw(Dict::new(), content.to_vec()); + let xobjects = |names: Dict| Object::Dict([(b"XObject".to_vec(), Object::Dict(names))].into_iter().collect()); + for damaged in [false, true] { + let mut doc = doc_shared_form_resources(whole(b"/C Do"), whole(b""), whole(b"")); + let mut names = Dict::new(); + for _ in 0..100_000 { + let mut form = if damaged { damaged_flate(b"", b"/C Do") } else { whole(b"/C Do") }; + form.dict.set(b"Subtype".to_vec(), Object::name("Form")); + form.dict.set(b"BBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 1.into(), 1.into()])); + form.dict.set(b"Resources".to_vec(), xobjects(names)); + let next = doc.add(Object::Stream(form)); + names = [(b"C".to_vec(), Object::Ref(next))].into_iter().collect(); + } + doc.update_dict(ObjRef::new(3, 0), |p| p.set(b"Resources".to_vec(), xobjects(names))).unwrap(); + assert!(crate::prune::prune_unused_xobjects(&mut doc).is_ok(), "damaged: {damaged}"); + } +} + /// A form whose `/DecodeParms` is given by reference is read through it: the image it draws is /// kept, not lost to content read with the predictor's tag bytes left in it. #[test] From 104e22904ae34c7fbd421b8dcc830ea89c5c5eca Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Sat, 7 Nov 2026 03:23:18 +0100 Subject: [PATCH 170/198] M2.6: shorter wording for content in or after damage Marking a figure decorative or a tag's content as an artifact refuses content that may be in a damaged or unreadable content stream, or after the damage. The two errors now say so in one clause, and that nothing was changed, which holds: both return before touching the document. The mark-as-artifact tool's description names the case in a few words. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/a11y/src/alt.rs | 8 ++------ crates/automation/src/tools.rs | 2 +- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/crates/a11y/src/alt.rs b/crates/a11y/src/alt.rs index dfe083bec9..77dda106ed 100644 --- a/crates/a11y/src/alt.rs +++ b/crates/a11y/src/alt.rs @@ -25,9 +25,7 @@ pub enum AltError { NotAFigure(u32), #[error("the page has more content than can be edited")] TooLarge, - #[error( - "the figure's content is, or may be, in a content stream that is damaged partway or doesn't decode, or after the damage, where it can't be rewritten without losing or misreading what wasn't read" - )] + #[error("the figure's content may be in a damaged or unreadable content stream, or after the damage, so nothing was changed")] Damaged, #[error("{0}")] Cos(#[from] pdfcraft_cos::CosError), @@ -56,9 +54,7 @@ pub enum ArtifactError { ParentTree(String), #[error("the page has more content than can be edited")] TooLarge, - #[error( - "the tag's marked content is, or may be, in a content stream that is damaged partway or doesn't decode, or after the damage, where it can't be rewritten without losing or misreading what wasn't read" - )] + #[error("the tag's marked content may be in a damaged or unreadable content stream, or after the damage, so nothing was changed")] Damaged, #[error(transparent)] Cos(#[from] pdfcraft_cos::CosError), diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index 4f3b0c374e..e206f042bc 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -1285,7 +1285,7 @@ pub fn tools() -> Vec { "property": { "type": "string", "enum": ["type", "title", "actual_text", "alternate_text", "language", "summary", "scope"] }, "value": { "type": "string" }, }), &["doc", "object", "generation", "property"])), - t("accessibility_mark_tag_artifact", "Mark tag content as artifact", "Mark a reachable leaf structure tag's direct marked content as an artifact and remove the tag from the reading structure. Use object and generation from accessibility_tags. Tags with child structure elements, no direct marked content, content not found on its page, or content that is, or may be, in a content stream damaged partway or that doesn't decode, or after the damage, are rejected. Undoable.") + t("accessibility_mark_tag_artifact", "Mark tag content as artifact", "Mark a reachable leaf structure tag's direct marked content as an artifact and remove the tag from the reading structure. Use object and generation from accessibility_tags. Tags with child structure elements, no direct marked content, content not found on its page, or content that may be in or after a damaged content stream are rejected. Undoable.") .with(schema(json!({ "doc": doc(), "object": { "type": "integer", "minimum": 1 }, "generation": { "type": "integer", "minimum": 0, "maximum": 65535 }, From d1de0a4badd11203ef19ebf7e881c2c9a5c74484 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Sat, 7 Nov 2026 03:52:04 +0100 Subject: [PATCH 171/198] M2.6: autotagging's reason for content it can't decode reads as on main MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since autotagging reads page content through DecodeBudget::decode_strict, the reason it gave for content that doesn't decode carried that error's own text, so it said twice that the data couldn't be decoded: "content cannot be decoded strictly: stream data could not be decoded: FlateDecode: …". It now gives the decoder's reason right after, as main does: "content cannot be decoded strictly: FlateDecode: …". Test: autotag_text_says_why_it_cannot_read_a_page now checks the decoder's reason follows the prefix. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/a11y/src/autotag.rs | 8 +++++++- crates/a11y/src/tests.rs | 4 ++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/crates/a11y/src/autotag.rs b/crates/a11y/src/autotag.rs index b9ffa46669..5c0dc2bd19 100644 --- a/crates/a11y/src/autotag.rs +++ b/crates/a11y/src/autotag.rs @@ -66,7 +66,13 @@ fn content_bytes(doc: &Document, page: &pdfcraft_model::Page, index: usize) -> R if budget.remaining() == 0 { unsupported(index, "decoding the content takes more than 16 MiB") } else { - unsupported(index, format!("content cannot be decoded strictly: {e}")) + // The decoder's own reason: `CosError::Filter`'s text would say again that the data + // couldn't be decoded. + let why = match e { + pdfcraft_cos::CosError::Filter(why) => why, + other => other.to_string(), + }; + unsupported(index, format!("content cannot be decoded strictly: {why}")) } })?; all.extend_from_slice(&decoded); diff --git a/crates/a11y/src/tests.rs b/crates/a11y/src/tests.rs index 0ad6ecd3e5..960eb39d3d 100644 --- a/crates/a11y/src/tests.rs +++ b/crates/a11y/src/tests.rs @@ -113,7 +113,7 @@ fn autotag_text_reads_content_with_parameters_given_by_reference() { } /// Autotagging refuses a page whose content it can't read whole, and says why: content past -/// 16 MiB, or content that doesn't decode strictly. +/// 16 MiB, or content that doesn't decode strictly, with the decoder's reason right after. #[test] fn autotag_text_says_why_it_cannot_read_a_page() { use pdfcraft_cos::{Dict, ObjRef, Stream}; @@ -138,7 +138,7 @@ fn autotag_text_says_why_it_cannot_read_a_page() { let mut corrupt = Stream::from_raw(Dict::new(), b"not zlib data".to_vec()); corrupt.dict.set(b"Filter".to_vec(), Object::name("FlateDecode")); let why = reason(corrupt); - assert!(why.starts_with("content cannot be decoded strictly"), "{why}"); + assert!(why.starts_with("content cannot be decoded strictly: FlateDecode: "), "{why}"); } #[test] From 1d0c3034662aad0b697094afda59753237dba3af Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Sat, 7 Nov 2026 13:15:45 +0100 Subject: [PATCH 172/198] M2.6: the redaction proof reads /DecodeParms given by reference The proof that came with #5557 decodes every stream it sweeps through `Ctx::decode`, which called `decode_strict` with the stream alone. This PR's `decode_strict` also takes the document, to read `/DecodeParms` given by reference, so the branch merged with main no longer built. `Ctx::decode` now takes the document, and each of its callers passes the one it reads, so the proof reads such a stream as redaction reads page content and forms. Before, it decoded the stream with the parameters' defaults: a PNG predictor's rows kept their tag bytes, and a copy of the removed text split by them went unreported (`a_survivor_behind_parameters_given_by_reference_is_found`). The test helpers' streams are direct, so they pass an empty document. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/redact/src/tests_verify.rs | 18 ++++++++++++++++++ crates/redact/src/verify/extract.rs | 6 +++--- crates/redact/src/verify/geometry.rs | 8 ++++---- crates/redact/src/verify/mod.rs | 10 +++++----- crates/redact/src/verify/snapshot.rs | 2 +- crates/redact/src/verify/sweep.rs | 8 ++++---- crates/redact/src/verify/util.rs | 5 +++-- 7 files changed, 38 insertions(+), 19 deletions(-) diff --git a/crates/redact/src/tests_verify.rs b/crates/redact/src/tests_verify.rs index ee443f3209..dc3c3c89d9 100644 --- a/crates/redact/src/tests_verify.rs +++ b/crates/redact/src/tests_verify.rs @@ -332,6 +332,24 @@ fn a_survivor_in_an_xfa_packet_is_found() { assert!(!proof.passed(), "{}", proof.to_text()); } +#[test] +fn a_survivor_behind_parameters_given_by_reference_is_found() { + // Object 7 holds the text behind Flate and a PNG predictor whose `/DecodeParms` is object 8, + // a reference. Read without them, its rows keep their tag bytes, which split the text; the + // sweep reads them as redaction does, and reports the copy. + let text = format!("note: {SECRET} xx"); + let rows: Vec = text.as_bytes().chunks(4).flat_map(|row| std::iter::once(0).chain(row.iter().copied())).collect(); + let packed = stream("/Filter /FlateDecode /DecodeParms 8 0 R", Stream::flate(Dict::new(), &rows).raw.as_ref()); + let parms = b"<< /Predictor 12 /Columns 4 >>".to_vec(); + let doc = marked(Spec { extra: vec![packed, parms], ..Spec::default() }); + let snap = Snapshot::capture(&doc, None).unwrap(); + let proof = snap.prove_saved(doc.bytes(), None, &ProofOptions::default()); + let s = surface(&proof, Surface::DecodedStreams); + assert_eq!(s.verdict, Verdict::Survivor, "{}", proof.to_text()); + assert!(s.locations.iter().any(|l| l == "object 7 (decoded)"), "{:?}", s.locations); + assert!(!proof.passed(), "{}", proof.to_text()); +} + #[test] fn a_survivor_in_an_unused_object_is_found() { // Object 7 is reachable from nothing: no catalog entry, no page resource. A reader that diff --git a/crates/redact/src/verify/extract.rs b/crates/redact/src/verify/extract.rs index f6db96229b..fe92e4e78d 100644 --- a/crates/redact/src/verify/extract.rs +++ b/crates/redact/src/verify/extract.rs @@ -58,7 +58,7 @@ impl Decoder { let mut d = Decoder { map: HashMap::new(), width: if composite { 2 } else { 1 }, opaque: false }; let tu = font.get(b"ToUnicode").map(|t| doc.resolve(t)); if let Some(Object::Stream(s)) = tu.as_deref() { - let data = ctx.decode(s)?; + let data = ctx.decode(doc, s)?; if data.len() > MAX_CMAP { return Err(()); } @@ -435,7 +435,7 @@ pub(super) fn page_content(doc: &Document, page: &Dict, ctx: &Ctx, skip_overlay: for o in &list { match &*resolve_strict(doc, o)? { Object::Stream(s) => { - let d = ctx.decode(s)?; + let d = ctx.decode(doc, s)?; if skip_overlay && ctx.is_overlay(&d) { continue; } @@ -536,7 +536,7 @@ impl Extractor<'_> { self.stack.push(r); } let result = (|| { - let data = self.ctx.decode(s)?; + let data = self.ctx.decode(self.doc, s)?; let own = s.dict.get(b"Resources").and_then(|r| self.doc.resolve(r).as_dict().cloned()); self.scope(&data, own.as_ref().unwrap_or(parent), ctm, depth, placed) })(); diff --git a/crates/redact/src/verify/geometry.rs b/crates/redact/src/verify/geometry.rs index fb69f91351..7ea0de5699 100644 --- a/crates/redact/src/verify/geometry.rs +++ b/crates/redact/src/verify/geometry.rs @@ -98,7 +98,7 @@ impl Geo<'_> { // own placement: where it lands comes from each `/Rect`, so the first // placement tells nothing about the others. Repeated decodes spend the // same bounded budget as everything else. - let Ok(data) = self.ctx.decode(&st) else { + let Ok(data) = self.ctx.decode(self.doc, &st) else { self.unverifiable("an annotation appearance can't be read in full"); continue; }; @@ -278,7 +278,7 @@ impl Geo<'_> { { return; } - let Ok(data) = self.ctx.decode(s) else { + let Ok(data) = self.ctx.decode(self.doc, s) else { self.unverifiable("a form under the region can't be decoded in full"); return; }; @@ -336,7 +336,7 @@ impl Geo<'_> { if depth > MAX_DEPTH { return Err(()); } - let data = self.ctx.decode(s)?; + let data = self.ctx.decode(self.doc, s)?; let own = s.dict.get(b"Resources").and_then(|r| self.doc.resolve(r).as_dict().cloned()); let res = own.as_ref().unwrap_or(parent); let xobjects = res_dict(self.doc, res, b"XObject"); @@ -464,7 +464,7 @@ pub(super) fn image_cleared(doc: &Document, ctx: &Ctx, s: &Stream, ctm: &Matrix, if !matches!(bpc, 1 | 2 | 4 | 8 | 16) || ncomp == 0 || ncomp > 32 { return Err(()); } - let data = ctx.decode(s)?; + let data = ctx.decode(doc, s)?; let bits = ncomp.checked_mul(bpc).ok_or(())?; let row = w.checked_mul(bits).ok_or(())?.div_ceil(8); if data.len() < row.checked_mul(h).ok_or(())? { diff --git a/crates/redact/src/verify/mod.rs b/crates/redact/src/verify/mod.rs index b374f93fdc..3a36c9e08e 100644 --- a/crates/redact/src/verify/mod.rs +++ b/crates/redact/src/verify/mod.rs @@ -252,7 +252,7 @@ pub struct ProofOptions { #[cfg(test)] pub(crate) mod tests_support { use pdfcraft_content::parse; - use pdfcraft_cos::{Dict, Object, Stream}; + use pdfcraft_cos::{Dict, Document, Object, Stream}; use super::extract::{carries_glyphs, parse_cmap}; use super::limits::{MAX_STREAM, MAX_TOTAL_DECODED, MAX_WORK, MIN_STRUCTURED_NEEDLE}; @@ -308,9 +308,9 @@ pub(crate) mod tests_support { pub fn decode_budget_runs_out() -> (bool, bool) { let ctx = Ctx::new(Default::default()); let s = Stream::from_raw(Dict::new(), vec![0u8; 100]); - let before = ctx.decode(&s).is_ok(); + let before = ctx.decode(&Document::new_empty(), &s).is_ok(); let spent = Ctx::with_decode_left(50); - (before, spent.decode(&s).is_err()) + (before, spent.decode(&Document::new_empty(), &s).is_err()) } /// Does a decode that fails past the one-stream cap pay for the attempt, so the same bomb @@ -322,9 +322,9 @@ pub(crate) mod tests_support { drop(big); let small = Stream::from_raw(Dict::new(), vec![0u8; 100]); let ctx = Ctx::with_decode_left(u64::try_from(MAX_STREAM + 50).unwrap_or(u64::MAX)); - let refused = ctx.decode(&bomb).is_err(); + let refused = ctx.decode(&Document::new_empty(), &bomb).is_err(); let charged = ctx.decoded.get() == MAX_TOTAL_DECODED - 50; - let gone = ctx.decode(&small).is_err(); + let gone = ctx.decode(&Document::new_empty(), &small).is_err(); (refused, charged, gone) } diff --git a/crates/redact/src/verify/snapshot.rs b/crates/redact/src/verify/snapshot.rs index fe74571bb4..8421fc1136 100644 --- a/crates/redact/src/verify/snapshot.rs +++ b/crates/redact/src/verify/snapshot.rs @@ -200,7 +200,7 @@ impl Snapshot { for r in created { let obj = doc.try_get(r.num).map_err(|_| RedactError::OverlayUnreadable)?; let Object::Stream(s) = &*obj else { continue }; - let data = ctx.decode(s).map_err(|_| RedactError::OverlayUnreadable)?; + let data = ctx.decode(doc, s).map_err(|_| RedactError::OverlayUnreadable)?; digests.insert(sha256(&[data])); } self.overlay = digests; diff --git a/crates/redact/src/verify/sweep.rs b/crates/redact/src/verify/sweep.rs index 73b44c005d..3c607ed8e7 100644 --- a/crates/redact/src/verify/sweep.rs +++ b/crates/redact/src/verify/sweep.rs @@ -495,7 +495,7 @@ fn sweep_objects(needles: &[Needle], doc: &Document, depth: usize, tag: &str, ac for n in acc.bytes(Surface::DecodedStreams, needles, &s.raw, MIN_STRUCTURED_NEEDLE) { acc.survivor(Surface::DecodedStreams, format!("{loc} (raw)"), Some(n)); } - match acc.ctx.decode(s) { + match acc.ctx.decode(doc, s) { Ok(d) => { let loc = format!("{loc} (decoded)"); for (n, c) in acc.counts(Surface::DecodedStreams, needles, &d, MIN_STRUCTURED_NEEDLE) { @@ -508,7 +508,7 @@ fn sweep_objects(needles: &[Needle], doc: &Document, depth: usize, tag: &str, ac Err(()) => acc.raw_only(Surface::DecodedStreams), } } else { - match acc.ctx.decode(s) { + match acc.ctx.decode(doc, s) { // The boxes and labels this run drew are not text to look for. Ok(d) if acc.ctx.is_overlay(&d) => overlay_raw.push(s.raw.to_vec()), Ok(d) => { @@ -578,7 +578,7 @@ fn sweep_attachments(needles: &[Needle], doc: &Document, embedded: &BTreeSet p, Err(()) => { acc.problem(Surface::EmbeddedFiles, format!("{loc} can't be decoded in full")); @@ -742,7 +742,7 @@ pub(super) fn sweep_xmp(needles: &[Needle], doc: &Document, tag: &str, acc: &mut acc.problem(Surface::Xmp, format!("{loc} is not a stream")); return; }; - let Ok(data) = acc.ctx.decode(s) else { + let Ok(data) = acc.ctx.decode(doc, s) else { acc.problem(Surface::Xmp, format!("{loc} can't be decoded in full")); return; }; diff --git a/crates/redact/src/verify/util.rs b/crates/redact/src/verify/util.rs index 77c4309cdd..2051ae15a5 100644 --- a/crates/redact/src/verify/util.rs +++ b/crates/redact/src/verify/util.rs @@ -33,11 +33,12 @@ impl Ctx { /// attempt is charged to the cumulative allowance, failures included: a decode that fails /// past the one-stream cap pays the cap, and one that fails mid-way pays what it decoded /// before the error — the work of finding out is not work a repeated reference gets for free. - pub(super) fn decode(&self, s: &Stream) -> Result, ()> { + /// `/DecodeParms` given by reference are read in `doc`, as redaction itself reads them. + pub(super) fn decode(&self, doc: &Document, s: &Stream) -> Result, ()> { let left = MAX_TOTAL_DECODED.saturating_sub(self.decoded.get()); let cap = usize::try_from(left).unwrap_or(usize::MAX).min(MAX_STREAM); let mut budget = DecodeBudget::new(cap, usize::MAX); - let result = budget.decode_strict(s); + let result = budget.decode_strict(doc, s); // The attempt's own accounting covers a success (what it decoded), a stream cut short // by corruption (what it decoded first) and one refused past the cap (the cap): charge // it through, whichever way the attempt went. From 0916bec6effba931a260da5215906699d6946233 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Sat, 7 Nov 2026 13:15:45 +0100 Subject: [PATCH 173/198] M2.6: a font's streams read /DecodeParms given by reference The redaction proof now reads a font's /ToUnicode with its /DecodeParms given by reference, and `Metrics::from_dict`, which redaction takes the removed text from, read it only with direct ones. Behind a predictor so described, the two got different text for the same codes, so a copy the redaction rightly left counted as a survivor and the proof failed a sound redaction (`a_unicode_map_behind_parameters_given_by_reference_is_read_alike`). Every stream `from_dict` reads now goes through `Document::with_direct_parms`: the /ToUnicode; a composite font's /Encoding, which sets how its codes split (`cmap_streams_read_their_parameters_given_by_reference`); a subset's program and /CIDToGIDMap, which say which of its CIDs have a glyph (`a_subsets_program_and_cid_map_read_their_parameters_given_by_reference`); and the program a simple font without /Widths is measured with (`a_font_without_widths_measures_with_its_embedded_program`). Read without them, as on `main`, such a font was measured with widths guessed from its name, so redaction removed glyphs the mark doesn't cover and left some it does, and the proof, which also guesses from the name, passed. Redaction now measures such a font with its program. The proof still guesses, so it fails that redaction, as it already does on `main` when the program's parameters are direct. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/fonts/src/pdf.rs | 13 +-- crates/fonts/src/pdf/tests.rs | 133 ++++++++++++++++++++++++++---- crates/redact/src/tests_verify.rs | 21 +++++ 3 files changed, 144 insertions(+), 23 deletions(-) diff --git a/crates/fonts/src/pdf.rs b/crates/fonts/src/pdf.rs index 240b1c4d55..e9a909ea5c 100644 --- a/crates/fonts/src/pdf.rs +++ b/crates/fonts/src/pdf.rs @@ -156,13 +156,13 @@ fn cids_with_outlines(doc: &Document, font: &Dict) -> Vec { let Some(program) = dict(doc, descendant.get(b"FontDescriptor")).and_then(|d| d.get(b"FontFile2").cloned()) else { return Vec::new() }; let program = doc.resolve(&program); let Object::Stream(stream) = program.as_ref() else { return Vec::new() }; - let Ok(bytes) = stream.decoded_within(16 << 20) else { return Vec::new() }; + let Ok(bytes) = doc.with_direct_parms(stream).decoded_within(16 << 20) else { return Vec::new() }; let Some(loca) = skrifa::FontRef::from_index(&bytes, 0).ok().and_then(|face| face.loca(None).ok()) else { return Vec::new() }; let outlined = |gid: u32| loca.get_raw(gid as usize).zip(loca.get_raw(gid as usize + 1)).is_some_and(|(start, end)| end > start); match descendant.get(b"CIDToGIDMap").map(|o| doc.resolve(o)).as_deref() { None | Some(Object::Name(_)) => bit_set_below_64k((0..loca.len().min(0x1_0000) as u32).filter(|&gid| outlined(gid))), Some(Object::Stream(map)) => { - let Ok(map) = map.decoded_within(1 << 17) else { return Vec::new() }; + let Ok(map) = doc.with_direct_parms(map).decoded_within(1 << 17) else { return Vec::new() }; bit_set_below_64k( map.as_chunks::<2>() .0 @@ -189,7 +189,7 @@ fn embedded_widths(doc: &Document, font: &Dict, m: &mut Metrics) { let Some(object) = descriptor.get(key) else { continue }; let object = doc.resolve(object); let Object::Stream(stream) = object.as_ref() else { continue }; - let Ok(bytes) = stream.decoded_within(16 << 20) else { continue }; + let Ok(bytes) = doc.with_direct_parms(stream).decoded_within(16 << 20) else { continue }; let Ok(face) = skrifa::FontRef::from_index(&bytes, 0) else { continue }; let upem = f64::from(face.metrics(Size::unscaled(), LocationRef::default()).units_per_em); if upem == 0.0 { @@ -691,7 +691,7 @@ impl Metrics { m.codes = Codes::Two; if let Some(e) = font.get(b"Encoding").map(|e| doc.resolve(e)) && let Object::Stream(s) = &*e - && let Ok(data) = s.decoded_within(MAX_CMAP_BYTES) + && let Ok(data) = doc.with_direct_parms(s).decoded_within(MAX_CMAP_BYTES) { let (spaces, cids) = parse_cmap(&data); if !spaces.is_empty() { @@ -1293,7 +1293,10 @@ fn unicode_map(doc: &Document, font: &Dict, composite: bool) -> Unicode { } match font.get(b"ToUnicode").map(|t| doc.resolve(t)).as_deref() { Some(Object::Stream(s)) => { - if let Ok(data) = s.decoded_within(MAX_CMAP_BYTES) { + // With `/DecodeParms` given by reference, as every stream of the font is read here, + // and as the redaction proof reads the map: the two must get the same text, or a copy + // redaction rightly left counts as a survivor. + if let Ok(data) = doc.with_direct_parms(s).decoded_within(MAX_CMAP_BYTES) { parse_to_unicode(&data, &mut out); } } diff --git a/crates/fonts/src/pdf/tests.rs b/crates/fonts/src/pdf/tests.rs index c2acef191c..69c309c0e9 100644 --- a/crates/fonts/src/pdf/tests.rs +++ b/crates/fonts/src/pdf/tests.rs @@ -104,6 +104,18 @@ fn font(doc: &mut Document, entries: Vec<(&str, Object)>) -> Dict { d } +/// `data` behind Flate and a PNG predictor whose `/DecodeParms` is a reference: read without +/// it, every row of four bytes keeps its tag byte. +fn behind_parms_by_reference(doc: &mut Document, data: &[u8]) -> Object { + let parms = doc.add(Object::Dict(Dict::from_iter([(b"Predictor".to_vec(), Object::Int(12)), (b"Columns".to_vec(), Object::Int(4))]))); + let mut data = data.to_vec(); + data.resize(data.len().next_multiple_of(4), 0); + let rows: Vec = data.chunks(4).flat_map(|row| std::iter::once(0).chain(row.iter().copied())).collect(); + let mut stream = Stream::flate(Dict::new(), &rows); + stream.dict.set(b"DecodeParms".to_vec(), Object::Ref(parms)); + Object::Ref(doc.add(Object::Stream(stream))) +} + #[test] fn embedded_editor_fonts_are_validated_without_installing_or_exporting() { use pdfcraft_cos::Stream; @@ -124,34 +136,46 @@ fn embedded_editor_fonts_are_validated_without_installing_or_exporting() { } /// #2130: with no `/Widths`, an embedded program's advances win over a guess from the name -/// ("Mono" would mean Courier, 600 units for every glyph). +/// ("Mono" would mean Courier, 600 units for every glyph), the program's `/DecodeParms` given +/// directly or by reference. #[test] fn a_font_without_widths_measures_with_its_embedded_program() { use pdfcraft_cos::Stream; use skrifa::MetadataProvider as _; - let mut doc = Document::new_empty(); let bytes = include_bytes!("../../../../assets/fonts/Inter-Regular.ttf"); let face = skrifa::FontRef::new(bytes).unwrap(); let (size, location) = (skrifa::instance::Size::unscaled(), skrifa::instance::LocationRef::default()); let upem = f64::from(face.metrics(size, location).units_per_em); let advance = |c: char| f64::from(face.glyph_metrics(size, location).advance_width(face.charmap().map(c).unwrap()).unwrap()) / upem; - let program = doc.add(Object::Stream(Stream::flate(Dict::new(), bytes))); - let mut descriptor = Dict::new(); - descriptor.set(b"FontFile2".to_vec(), Object::Ref(program)); - let f = font( - &mut doc, - vec![ - ("Subtype", Object::name("TrueType")), - ("BaseFont", Object::name("HouseMono")), - ("Encoding", Object::name("WinAnsiEncoding")), - ("FontDescriptor", Object::Dict(descriptor)), - ], - ); - let m = Metrics::from_dict(&doc, &f); - for c in ['i', 'l', 'W', ' '] { - assert!((m.width(u32::from(c)) - advance(c)).abs() < 1e-9, "{c}: {} vs {}", m.width(u32::from(c)), advance(c)); + for by_reference in [false, true] { + let mut doc = Document::new_empty(); + let program = if by_reference { + behind_parms_by_reference(&mut doc, bytes) + } else { + Object::Ref(doc.add(Object::Stream(Stream::flate(Dict::new(), bytes)))) + }; + let mut descriptor = Dict::new(); + descriptor.set(b"FontFile2".to_vec(), program); + let f = font( + &mut doc, + vec![ + ("Subtype", Object::name("TrueType")), + ("BaseFont", Object::name("HouseMono")), + ("Encoding", Object::name("WinAnsiEncoding")), + ("FontDescriptor", Object::Dict(descriptor)), + ], + ); + let m = Metrics::from_dict(&doc, &f); + for c in ['i', 'l', 'W', ' '] { + assert!( + (m.width(u32::from(c)) - advance(c)).abs() < 1e-9, + "{c}, by reference {by_reference}: {} vs {}", + m.width(u32::from(c)), + advance(c) + ); + } + assert!(m.width(u32::from('i')) < 0.4, "not Courier's 0.6, by reference {by_reference}"); } - assert!(m.width(u32::from('i')) < 0.4, "not Courier's 0.6"); } #[test] @@ -342,6 +366,79 @@ fn cmap_streams_decode_within_a_size_cap() { } } +#[test] +fn cmap_streams_read_their_parameters_given_by_reference() { + // Both maps behind a predictor whose `/DecodeParms` is a reference: read without it, every + // row keeps its tag byte inside the map's tokens. + let to_unicode = b"1 begincodespacerange <0000> endcodespacerange 1 beginbfrange <0024> <0026> <0041> endbfrange"; + let encoding = b"1 begincodespacerange <0000> endcodespacerange 1 begincidrange <0000> 0 endcidrange"; + let mut doc = Document::new_empty(); + let (tu, enc) = (behind_parms_by_reference(&mut doc, to_unicode), behind_parms_by_reference(&mut doc, encoding)); + let mut desc = Dict::new(); + desc.set(b"Subtype".to_vec(), Object::name("CIDFontType2")); + let f = font( + &mut doc, + vec![("Subtype", Object::name("Type0")), ("Encoding", enc), ("DescendantFonts", Object::Array(vec![Object::Dict(desc)])), ("ToUnicode", tu)], + ); + let m = Metrics::from_dict(&doc, &f); + assert_eq!(m.decode(&[0, 0x24]), "A", "ToUnicode"); + assert!(matches!(m.codes, Codes::Ranges(_)), "Encoding"); +} + +/// A Type 0 subset over Inter with `/DW` alone, whose CIDs 1 to 7 draw "A B C D" (ToUnicode +/// gives CIDs 1 and 2), its program and its `/CIDToGIDMap` each behind a predictor whose +/// `/DecodeParms` is a reference, or behind Flate alone. +fn inter_subset(program_by_reference: bool, map_by_reference: bool) -> Metrics { + use skrifa::MetadataProvider as _; + let bytes = include_bytes!("../../../../assets/fonts/Inter-Regular.ttf"); + let face = skrifa::FontRef::new(bytes).unwrap(); + let gid = |c: char| u16::try_from(face.charmap().map(c).unwrap().to_u32()).unwrap(); + let gids: Vec = std::iter::once(0).chain("A B C D".chars().map(gid)).flat_map(u16::to_be_bytes).collect(); + let stream = |doc: &mut Document, data: &[u8], by_reference: bool| { + if by_reference { behind_parms_by_reference(doc, data) } else { Object::Ref(doc.add(Object::Stream(Stream::flate(Dict::new(), data)))) } + }; + let mut doc = Document::new_empty(); + let program = stream(&mut doc, bytes, program_by_reference); + let map = stream(&mut doc, &gids, map_by_reference); + let to_unicode = b"1 begincodespacerange <0000> endcodespacerange 2 beginbfchar <0001> <0041> <0002> <0020> endbfchar"; + let to_unicode = stream(&mut doc, to_unicode, false); + let mut descriptor = Dict::new(); + descriptor.set(b"FontFile2".to_vec(), program); + let mut desc = Dict::new(); + desc.set(b"Subtype".to_vec(), Object::name("CIDFontType2")); + desc.set(b"DW".to_vec(), Object::Int(600)); + desc.set(b"FontDescriptor".to_vec(), Object::Dict(descriptor)); + desc.set(b"CIDToGIDMap".to_vec(), map); + let f = font( + &mut doc, + vec![ + ("Subtype", Object::name("Type0")), + ("BaseFont", Object::name("ABCDEF+Inter")), + ("Encoding", Object::name("Identity-H")), + ("DescendantFonts", Object::Array(vec![Object::Dict(desc)])), + ("ToUnicode", to_unicode), + ], + ); + Metrics::from_dict(&doc, &f) +} + +/// A subset with `/DW` alone has a glyph for a CID when the program's glyph it maps to has an +/// outline, so the space, Inter's empty glyph, is refused. Its program, or its `/CIDToGIDMap`, +/// behind parameters given by reference says the same: read without them, no program says +/// which glyphs are empty, or the map points the CIDs at other glyphs. +#[test] +fn a_subsets_program_and_cid_map_read_their_parameters_given_by_reference() { + let direct = inter_subset(false, false); + assert!(!direct.outlined_cids.is_empty()); + assert_eq!(direct.encode("A").as_deref(), Some(&[0, 1][..])); + assert_eq!(direct.encode(" "), None, "the space's glyph is empty"); + for (program, map) in [(true, false), (false, true)] { + let m = inter_subset(program, map); + assert_eq!(m.outlined_cids, direct.outlined_cids, "program by reference: {program}, map: {map}"); + assert_eq!(m.encode(" "), None, "program by reference: {program}, map: {map}"); + } +} + #[test] fn composite_fonts_use_to_unicode() { let mut doc = Document::new_empty(); diff --git a/crates/redact/src/tests_verify.rs b/crates/redact/src/tests_verify.rs index dc3c3c89d9..28853cc4a4 100644 --- a/crates/redact/src/tests_verify.rs +++ b/crates/redact/src/tests_verify.rs @@ -350,6 +350,27 @@ fn a_survivor_behind_parameters_given_by_reference_is_found() { assert!(!proof.passed(), "{}", proof.to_text()); } +#[test] +fn a_unicode_map_behind_parameters_given_by_reference_is_read_alike() { + // The font's /ToUnicode (object 7) reads codes A..H as "SECRETVA", behind Flate and a PNG + // predictor whose `/DecodeParms` is object 8, a reference. The page draws the codes twice, + // once under the mark. Redaction takes the removed text from that map, and the proof the page + // text it allows for: read by one and not the other, the copy that stays counted as a + // survivor of a sound redaction. + let cmap = b"/CIDInit /ProcSet findresource begin\n12 dict begin\nbegincmap\n/CMapName /Test def\n1 begincodespacerange\n<00> \nendcodespacerange\n8 beginbfchar\n<41> <0053>\n<42> <0045>\n<43> <0043>\n<44> <0052>\n<45> <0045>\n<46> <0054>\n<47> <0056>\n<48> <0041>\nendbfchar\nendcmap\nCMapName currentdict /CMap defineresource pop\nend\nend\n"; + let mut data = cmap.to_vec(); + data.resize(data.len().next_multiple_of(4), b' '); + let rows: Vec = data.chunks(4).flat_map(|row| std::iter::once(0).chain(row.iter().copied())).collect(); + let to_unicode = stream("/Filter /FlateDecode /DecodeParms 8 0 R", Stream::flate(Dict::new(), &rows).raw.as_ref()); + let parms = b"<< /Predictor 12 /Columns 4 >>".to_vec(); + let content = b"BT /F1 10 Tf 10 200 Td (ABCDEFGH) Tj ET BT /F1 10 Tf 10 100 Td (ABCDEFGH) Tj ET"; + let mut doc = marked_content(content, Spec { extra: vec![to_unicode, parms], ..Spec::default() }); + doc.update_dict(ObjRef::new(5, 0), |font| font.set(b"ToUnicode".to_vec(), Object::Ref(ObjRef::new(7, 0)))).unwrap(); + let (result, proof) = run(&mut doc); + assert!(result.is_ok(), "{result:?}"); + assert!(proof.passed(), "{}", proof.to_text()); +} + #[test] fn a_survivor_in_an_unused_object_is_found() { // Object 7 is reachable from nothing: no catalog entry, no page resource. A reader that From 8ac51f833b891e29a40e3a8563e53a08c9320fb5 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sat, 7 Nov 2026 16:21:28 +0100 Subject: [PATCH 174/198] cos: charge the object-stream pre-scan to one budget per rebuild Overlapping unterminated dictionaries could each be read up to 1 MiB. A shared budget (the file length) is now taken off on every path; a spent budget keeps the object a candidate, as before. Co-Authored-By: Claude Sonnet 5.5 --- crates/cos/src/document/reader.rs | 163 +++++++++++++++++------------- crates/cos/src/document/tests.rs | 27 ++++- 2 files changed, 112 insertions(+), 78 deletions(-) diff --git a/crates/cos/src/document/reader.rs b/crates/cos/src/document/reader.rs index 7e8a4e3b49..6a325ec933 100644 --- a/crates/cos/src/document/reader.rs +++ b/crates/cos/src/document/reader.rs @@ -623,6 +623,9 @@ impl Document { // `/ObjStm` type; the others are not parsed for the object-stream check below. let mut candidates: HashSet = HashSet::new(); let headers = object_headers(&data); + // What the dictionary pre-check may read in all, so overlapping unterminated dictionaries + // cost no more than one more pass over the file. + let mut scan_budget = data.len(); // Where `/XRef` occurs in the file, found in one pass. let mut xref_marks = Vec::new(); let mut mark_from = 0; @@ -633,7 +636,7 @@ impl Document { for &(num, offset, generation, body) in &headers { entries.insert(num, XrefEntry::InFile { offset, generation }); objstms.push(num); - if may_be_object_stream(&data, body) { + if may_be_object_stream(&data, body, &mut scan_budget) { candidates.insert(num); } else { candidates.remove(&num); @@ -758,83 +761,86 @@ const DICTIONARY_SCAN_LIMIT: usize = 1 << 20; /// parser does: strings, hex strings, comments, names and nested dictionaries. `Some(None)`: the /// object is not a dictionary, so it is never a stream. `None`: the end is not certain (no end /// within the limit, a hex string with a bad digit, a `>>` inside an array); the caller must -/// then treat the object as a possible object stream. -fn dictionary_end(data: &[u8], from: usize) -> Option> { - let limit = data.len().min(from.saturating_add(DICTIONARY_SCAN_LIMIT)); +/// then treat the object as a possible object stream. `limit` ends the read; the second value is +/// how far the read got, for the caller's budget. +fn dictionary_end(data: &[u8], from: usize, limit: usize) -> (Option>, usize) { let mut i = from; - while i < limit && (is_whitespace(data[i]) || data[i] == b'%') { - if data[i] == b'%' { - while i < limit && !matches!(data[i], b'\n' | b'\r') { - i += 1; - } - } else { - i += 1; - } - } - if data.get(i..i + 2) != Some(b"<<") { - return if i < limit { Some(None) } else { None }; - } - i += 2; - let (mut dicts, mut arrays) = (1usize, 0usize); - while i < limit { - match data[i] { - b'(' => { - let mut depth = 1usize; - i += 1; - while i < limit && depth > 0 { - match data[i] { - b'\\' => i += 1, - b'(' => depth += 1, - b')' => depth -= 1, - _ => {} - } - i += 1; - } - if depth > 0 { - return None; - } - } - b'<' if data.get(i + 1) == Some(&b'<') => { - dicts += 1; - i += 2; - } - b'<' => { - i += 1; - while i < limit && data[i] != b'>' { - if !data[i].is_ascii_hexdigit() && !is_whitespace(data[i]) { - return None; - } - i += 1; - } - i += 1; - } - b'>' if data.get(i + 1) == Some(&b'>') => { - if arrays > 0 { - return None; - } - dicts -= 1; - i += 2; - if dicts == 0 { - return Some(Some(i)); - } - } - b'%' => { + let result = 'scan: { + while i < limit && (is_whitespace(data[i]) || data[i] == b'%') { + if data[i] == b'%' { while i < limit && !matches!(data[i], b'\n' | b'\r') { i += 1; } - } - b'[' => { - arrays += 1; + } else { i += 1; } - b']' => { - arrays = arrays.saturating_sub(1); - i += 1; - } - _ => i += 1, } - } - None + if data.get(i..i + 2) != Some(b"<<") { + break 'scan if i < limit { Some(None) } else { None }; + } + i += 2; + let (mut dicts, mut arrays) = (1usize, 0usize); + while i < limit { + match data[i] { + b'(' => { + let mut depth = 1usize; + i += 1; + while i < limit && depth > 0 { + match data[i] { + b'\\' => i += 1, + b'(' => depth += 1, + b')' => depth -= 1, + _ => {} + } + i += 1; + } + if depth > 0 { + break 'scan None; + } + } + b'<' if data.get(i + 1) == Some(&b'<') => { + dicts += 1; + i += 2; + } + b'<' => { + i += 1; + while i < limit && data[i] != b'>' { + if !data[i].is_ascii_hexdigit() && !is_whitespace(data[i]) { + break 'scan None; + } + i += 1; + } + i += 1; + } + b'>' if data.get(i + 1) == Some(&b'>') => { + if arrays > 0 { + break 'scan None; + } + dicts -= 1; + i += 2; + if dicts == 0 { + break 'scan Some(Some(i)); + } + } + b'%' => { + while i < limit && !matches!(data[i], b'\n' | b'\r') { + i += 1; + } + } + b'[' => { + arrays += 1; + i += 1; + } + b']' => { + arrays = arrays.saturating_sub(1); + i += 1; + } + _ => i += 1, + } + } + None + }; + (result, i.min(limit)) } /// Whether the object whose body starts at `data[from..]` could be an object stream: its @@ -843,8 +849,19 @@ fn dictionary_end(data: &[u8], from: usize) -> Option> { /// dictionary ends is read by [`dictionary_end`], so text inside a string (an object header, a /// `stream` keyword) is not taken for syntax; when that is not certain the object counts as a /// candidate, so a doubtful object is parsed as it always was. -pub(super) fn may_be_object_stream(data: &[u8], from: usize) -> bool { - match dictionary_end(data, from) { +/// +/// `budget` is the bytes all the calls of one rebuild may still read. A call reads at most +/// [`DICTIONARY_SCAN_LIMIT`] bytes or what is left, takes what it read off the budget on every +/// path, and says "maybe" once the budget cannot cover it. Dictionaries that do not overlap +/// add up to no more than the file, so only unterminated ones can use the budget up. +pub(super) fn may_be_object_stream(data: &[u8], from: usize, budget: &mut usize) -> bool { + let limit = data.len().min(from.saturating_add(DICTIONARY_SCAN_LIMIT.min(*budget))); + if *budget == 0 { + return true; + } + let (end, read) = dictionary_end(data, from, limit); + *budget = budget.saturating_sub(read.saturating_sub(from).max(1)); + match end { Some(None) => false, None => true, Some(Some(end)) => { diff --git a/crates/cos/src/document/tests.rs b/crates/cos/src/document/tests.rs index d2234fca41..ddd7965eda 100644 --- a/crates/cos/src/document/tests.rs +++ b/crates/cos/src/document/tests.rs @@ -1193,6 +1193,11 @@ fn parses_as_object_stream(file: &[u8]) -> bool { matches!(parse_indirect(file, 0, &|_| None), Ok((_, Object::Stream(s))) if s.dict.name(b"Type") == Some(b"ObjStm")) } +/// A scan budget that never runs out. +fn unlimited() -> usize { + usize::MAX +} + #[test] fn the_object_stream_pre_check_never_rejects_what_parses_as_one() { // Two inputs from review: header-shaped text, and `>>` + `stream` + newline, inside a @@ -1205,14 +1210,26 @@ fn the_object_stream_pre_check_never_rejects_what_parses_as_one() { ] { let file = format!("1 0 obj\n{dict}\nstream\nxx\nendstream\nendobj\n").into_bytes(); assert!(parses_as_object_stream(&file), "test input must be an object stream: {dict}"); - assert!(may_be_object_stream(&file, 7), "{dict}"); + assert!(may_be_object_stream(&file, 7, &mut unlimited()), "{dict}"); } // Ordinary streams stay out; their data is never read. let plain = b"1 0 obj\n<< /Length 9 >>\nstream\nObjStm # \nendstream\nendobj\n"; - assert!(!may_be_object_stream(plain, 7)); - assert!(!may_be_object_stream(b"1 0 obj\n[ /ObjStm ]\nendobj\n", 7)); + assert!(!may_be_object_stream(plain, 7, &mut unlimited())); + assert!(!may_be_object_stream(b"1 0 obj\n[ /ObjStm ]\nendobj\n", 7, &mut unlimited())); // Unterminated: not certain, so a candidate. - assert!(may_be_object_stream(b"1 0 obj\n<< /Note (open ", 7)); + assert!(may_be_object_stream(b"1 0 obj\n<< /Note (open ", 7, &mut unlimited())); + // The scan is charged to the shared budget on every path, and a spent budget says "maybe" + // without reading. + let open = b"1 0 obj\n<< /Note (open "; + let mut budget = 1000; + assert!(may_be_object_stream(open, 7, &mut budget)); + assert_eq!(budget, 1000 - (open.len() - 7)); + let mut budget = 3; + assert!(may_be_object_stream(plain, 7, &mut budget), "a budget too small for the dictionary is a candidate"); + assert_eq!(budget, 0); + let mut budget = 0; + assert!(may_be_object_stream(plain, 7, &mut budget)); + assert_eq!(budget, 0); // Random dictionaries built from tricky pieces: whatever parses as an object stream is // a candidate. let pieces: [&str; 22] = [ @@ -1257,7 +1274,7 @@ fn the_object_stream_pre_check_never_rejects_what_parses_as_one() { let file = format!("1 0 obj\n{dict}\nstream\nxx\nendstream\nendobj\n").into_bytes(); if parses_as_object_stream(&file) { hits += 1; - assert!(may_be_object_stream(&file, 7), "rejected an object stream: {dict:?}"); + assert!(may_be_object_stream(&file, 7, &mut unlimited()), "rejected an object stream: {dict:?}"); } } assert!(hits > 1000, "the generator must produce object streams ({hits})"); From a77cd9cb784fda453650f812ccdeb3787d5d7338 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sat, 7 Nov 2026 16:23:12 +0100 Subject: [PATCH 175/198] cos: reachable_objects keeps no parsed objects full_save_order retained up to 64 MiB of parsed objects for the write pass, which reachable_objects then dropped unused. It now asks for no retention. Co-Authored-By: Claude Sonnet 5.5 --- crates/cos/src/writer.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/crates/cos/src/writer.rs b/crates/cos/src/writer.rs index 6f8fbc92c1..a90ecea525 100644 --- a/crates/cos/src/writer.rs +++ b/crates/cos/src/writer.rs @@ -279,7 +279,10 @@ pub fn write_incremental(doc: &Document, opts: &SaveOptions) -> Result, /// object where the budget allowed keeping it. type SaveOrder = (HashMap, Vec<(ObjRef, bool)>, Vec>>); -fn full_save_order(reader: &mut ObjectReader, trailer_in: &Dict) -> Result { +/// Number the objects reachable from the trailer. `retain` keeps parsed objects for the write +/// pass, up to the budget. A caller that needs only the numbers passes `false`, so the traversal +/// holds no parsed objects beyond the current one. +fn full_save_order(reader: &mut ObjectReader, trailer_in: &Dict, retain: bool) -> Result { // Breadth-first walk from the trailer, assigning new numbers in visit order. let mut map: HashMap = HashMap::new(); // Record the type while traversing so classifying packed objects needs no second parse. @@ -326,7 +329,7 @@ fn full_save_order(reader: &mut ObjectReader, trailer_in: &Dict) -> Result Result Result, CosError> { let mut reader = doc.object_reader(); let trailer_in = reader.document.trailer().clone(); - let (_, order, _) = full_save_order(&mut reader, &trailer_in)?; + let (_, order, _) = full_save_order(&mut reader, &trailer_in, false)?; Ok(order.into_iter().map(|(r, _)| r.num).collect()) } @@ -351,7 +354,7 @@ pub fn write_full(doc: &Document, opts: &SaveOptions) -> Result, CosErro let mut reader = doc.object_reader(); stamp_mod_date(&mut reader.document, opts); let trailer_in = reader.document.trailer().clone(); - let (map, order, mut kept) = full_save_order(&mut reader, &trailer_in)?; + let (map, order, mut kept) = full_save_order(&mut reader, &trailer_in, true)?; let version = reader.document.version().chars().take(3).collect::(); let version = if opts.object_streams && version.as_str() < "1.5" { "1.5".to_string() } else { version }; let mut out = Vec::new(); From 82c5fe2e42c1c393f9479f04fe2e11b9581c0c00 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sat, 7 Nov 2026 16:24:20 +0100 Subject: [PATCH 176/198] cos tests: keep the whole-real property inside the integer range i32::MIN is outside Annex C's integer range and stays a real token (-2147483648.). The generator now stops at -i32::MAX and a separate test pins the i32::MIN output. Co-Authored-By: Claude Sonnet 5.5 --- crates/cos/tests/it/serialize_format.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/cos/tests/it/serialize_format.rs b/crates/cos/tests/it/serialize_format.rs index 19781a802f..8cfba2baf1 100644 --- a/crates/cos/tests/it/serialize_format.rs +++ b/crates/cos/tests/it/serialize_format.rs @@ -51,6 +51,12 @@ fn every_byte_value_in_hex_strings_octal_escapes_and_names() { assert_eq!(name, want); } +#[test] +fn whole_real_at_i32_min_stays_a_real_token() { + // -2^31 is outside the Annex C integer range (±2^31-1), so it keeps its trailing point. + assert_eq!(bytes_of(&Object::Real(f64::from(i32::MIN))), b"-2147483648."); +} + proptest! { #[test] fn integers_match_the_formatter(i in any::()) { @@ -58,7 +64,7 @@ proptest! { } #[test] - fn whole_reals_match_the_formatter(i in any::()) { + fn whole_reals_match_the_formatter(i in -i32::MAX..=i32::MAX) { prop_assert_eq!(bytes_of(&Object::Real(f64::from(i))), format!("{i}").into_bytes()); } From 442a6b56e3343a6a9d78a9a77a32ef8c5734c3b0 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sat, 7 Nov 2026 16:38:17 +0100 Subject: [PATCH 177/198] automation: skip optimize-to-target rungs that repeat earlier settings A rung equal to settings already run is not optimized again, and attempts counts only runs. The target test uses a TempDir so it cleans up even when it fails. Co-Authored-By: Claude Sonnet 5.5 --- crates/automation/src/documents.rs | 6 ++++++ crates/automation/tests/it/automation.rs | 8 +++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/crates/automation/src/documents.rs b/crates/automation/src/documents.rs index a3a517d298..f6cb33d4c4 100644 --- a/crates/automation/src/documents.rs +++ b/crates/automation/src/documents.rs @@ -391,6 +391,8 @@ impl Automation { let (mut bytes, mut r) = self.session.optimized_bytes(id, &settings, &discard).map_err(failed)?; let mut attempts = 1; let mut step_used = 0; + // Settings already optimized with; a rung that changes nothing is not run again. + let mut tried = vec![settings.clone()]; if let Some(t) = target { // Stronger image settings, one rung per attempt, until the copy fits. Every attempt // starts from the open document, so a failed one changes nothing; the number of @@ -409,7 +411,11 @@ impl Automation { img.compression = Compression::Jpeg(q.min(quality)); } } + if tried.contains(&s) { + continue; + } let (b2, r2) = self.session.optimized_bytes(id, &s, &discard).map_err(failed)?; + tried.push(s); attempts += 1; if b2.len() < bytes.len() { (bytes, r, step_used) = (b2, r2, k + 1); diff --git a/crates/automation/tests/it/automation.rs b/crates/automation/tests/it/automation.rs index a54791f5f0..e8042fe9ce 100644 --- a/crates/automation/tests/it/automation.rs +++ b/crates/automation/tests/it/automation.rs @@ -7196,7 +7196,9 @@ fn a_page_printed_in_strips_gets_small_through_tools() { #[test] fn optimizing_to_a_target_size() { - let dir = workdir("optimize-target"); + // Dropped at the end of the test, also when an assertion fails. + let tmp = tempfile::TempDir::new().unwrap(); + let dir = tmp.path().to_path_buf(); let (w, h) = (2400u32, 1600u32); let px: Vec = (0..w * h) .flat_map(|i| { @@ -7241,6 +7243,10 @@ fn optimizing_to_a_target_size() { // A JPEG caller gets the quality step. let r = ok(&mut a, "doc_optimize", json!({ "doc": doc, "path": "jpg.pdf", "target_bytes": 1 })); assert!(r["target_settings"]["max_jpeg_quality"].is_number(), "{r}"); + // Settings already at the ladder's floor leave every rung equal to the first run: no retries. + let floor = json!({ "ppi": 9, "above_ppi": 9, "compression": "jpeg", "quality": 1 }); + let r = ok(&mut a, "doc_optimize", json!({ "doc": doc, "path": "floor.pdf", "color": floor, "gray": floor, "target_bytes": 1 })); + assert_eq!((r["target_reached"].as_bool(), r["attempts"].as_u64()), (Some(false), Some(1)), "{r}"); // Confinement still applies. assert!(a.call("doc_optimize", &json!({ "doc": doc, "path": "../out.pdf", "target_bytes": 1000 })).is_err()); } From cc4bd4914caa11f6db73d029edd7ff7dcafcf57d Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sat, 7 Nov 2026 16:40:37 +0100 Subject: [PATCH 178/198] regex: enable perf-dfa so the dfa_size_limit on user patterns applies Without the feature the lazy-DFA cache does not exist and the limit set in UserRegex::new does nothing. Matching stays linear-time. Co-Authored-By: Claude Sonnet 5.5 --- Cargo.toml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 29d28051d8..a684ca9857 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -80,8 +80,9 @@ hayro-jbig2 = "0.3" # foundation thiserror = "2" -# Linear-time matching (no backtracking) for caller-supplied redaction patterns. -regex = { version = "1", default-features = false, features = ["std", "unicode"] } +# Linear-time matching (no backtracking) for caller-supplied redaction patterns. `perf-dfa` gives +# the lazy-DFA cache that `RegexBuilder::dfa_size_limit` bounds (the limit is inert without it). +regex = { version = "1", default-features = false, features = ["std", "unicode", "perf-dfa"] } serde = { version = "1", features = ["derive", "rc"] } serde_json = "1" log = "0.4" From 95f8eaa63f3a2a9661f2cd168ff704367338b3a8 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sat, 7 Nov 2026 16:42:02 +0100 Subject: [PATCH 179/198] ui: count the third item gap in the mode bar's right-hand width The print, save and info buttons each have 4 pt after them (three gaps), so the reserved width is 116 pt, not 112; the search field no longer overlaps the tabs. Co-Authored-By: Claude Sonnet 5.5 --- crates/ui-egui/src/chrome.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/ui-egui/src/chrome.rs b/crates/ui-egui/src/chrome.rs index 5e1fef8860..02b78d31fb 100644 --- a/crates/ui-egui/src/chrome.rs +++ b/crates/ui-egui/src/chrome.rs @@ -470,8 +470,9 @@ fn tab( theme::hand(resp).on_hover_text(if dirty { crate::i18n::fmt(tl!("{name} — unsaved changes"), &[("name", &shown)]) } else { shown }) } -/// Width the print, save and info buttons and their gaps take in the mode bar. -const MODE_BAR_RIGHT: f32 = 3.0 * 32.0 + 2.0 * 4.0 + 8.0; +/// Width the print, save and info buttons and their gaps take in the mode bar: three buttons, the +/// 4 pt `item_spacing` after each of them (three gaps, not two), and the 8 pt spacer. +const MODE_BAR_RIGHT: f32 = 3.0 * 32.0 + 3.0 * 4.0 + 8.0; /// Narrowest search field the mode bar shows; below it the field becomes an icon button. const MODE_BAR_SEARCH_MIN: f32 = 140.0; From 7b6baec065f29f4fe15158542575fbd94fd9f4a3 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sat, 7 Nov 2026 16:54:58 +0100 Subject: [PATCH 180/198] sign: an empty /Contents is not an earlier signature Only a non-empty signature value counts when deciding whether the first real signature may drop /NeedAppearances. Co-Authored-By: Claude Sonnet 5.5 --- crates/sign/src/pdf/signing.rs | 3 ++- crates/sign/tests/it/pdf.rs | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/crates/sign/src/pdf/signing.rs b/crates/sign/src/pdf/signing.rs index b41d7fe91a..fb3569602b 100644 --- a/crates/sign/src/pdf/signing.rs +++ b/crates/sign/src/pdf/signing.rs @@ -153,7 +153,8 @@ fn sign_inner( } check_signing_permissions(doc, opts)?; let mut doc = doc.clone(); - let earlier_signatures = !super::validate::signature_contents(&doc).is_empty(); + // A signature field whose /V holds an empty /Contents is not a signature yet. + let earlier_signatures = super::validate::signature_contents(&doc).iter().any(|der| !der.is_empty()); let root = doc.root().ok_or_else(|| SignError::Pdf("the document has no catalog".into()))?; let alg = id.key.preferred_digest(); let reserve = diff --git a/crates/sign/tests/it/pdf.rs b/crates/sign/tests/it/pdf.rs index 137960e83b..bcfe41631e 100644 --- a/crates/sign/tests/it/pdf.rs +++ b/crates/sign/tests/it/pdf.rs @@ -150,6 +150,22 @@ fn signing_drops_need_appearances() { } } +/// A field whose /V has an empty /Contents is no earlier signature: the first real one still +/// drops `/NeedAppearances`. +#[test] +fn an_empty_signature_value_does_not_count_as_an_earlier_signature() { + let id = pkcs12::open(&data("ec-p256.p12"), "test").unwrap(); + let bytes = fixture_with( + b"<< /Type /Catalog /Pages 2 0 R /AcroForm << /Fields [6 0 R 7 0 R] /NeedAppearances true >> >>", + &[ + b"<< /Type /Annot /Subtype /Widget /FT /Sig /T (Empty) /Rect [10 20 100 70] /P 3 0 R /F 4 /V 8 0 R >>", + b"<< /Type /Sig /Filter /Adobe.PPKLite /SubFilter /ETSI.CAdES.detached /Contents <> >>", + ], + ); + let signed = pdfcraft_sign::sign(&open(&bytes), &id, &SignOptions { field: Some("Approval".into()), ..opts() }).unwrap(); + assert!(form_of(&signed).get(b"NeedAppearances").is_none()); +} + /// A second signer leaves the form's flags alone: changing them would count as a change to /// what the first signature covered. #[test] From 761fd4ff5112c454eaefa8fcc9b10452b599b1d6 Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sat, 7 Nov 2026 18:35:28 +0100 Subject: [PATCH 181/198] Notice bar: stack the buttons only where the message still has room Below about 100 pt of content width the stacked message wrapped to a column thousands of points tall and pushed the hotbar off screen, which broke the narrow-viewport hotbar picker test merged from main. That width is under the smallest supported window, so the single row stays there. Co-Authored-By: Claude Sonnet 5.5 --- crates/ui-egui/src/canvas/bars.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/ui-egui/src/canvas/bars.rs b/crates/ui-egui/src/canvas/bars.rs index e588013a74..3e12bd6ce3 100644 --- a/crates/ui-egui/src/canvas/bars.rs +++ b/crates/ui-egui/src/canvas/bars.rs @@ -253,9 +253,13 @@ fn notice_bar(ui: &mut egui::Ui, t: &Tokens, icon: &str, icon_color: Color32, te const ICON: f32 = 16.0; /// Content width below which the buttons stack under the message. const STACK_BELOW: f32 = 200.0; + /// Content width under which stacking is not tried: the message would wrap to a column of a + /// few characters, thousands of points tall, and push the hotbar off the screen. Such a width + /// is below the smallest supported window, so the single row stays as it was. + const STACK_ABOVE: f32 = 100.0; egui::Frame::NONE.fill(t.accent_soft).inner_margin(egui::Margin::symmetric(14, 7)).show(ui, |ui| { let width = ui.available_width(); - if width < STACK_BELOW { + if (STACK_ABOVE..STACK_BELOW).contains(&width) { // A narrow document area (a half-width window): the message takes the full width and // the buttons sit on a row of their own below it, rather than squeezing it to nothing. ui.with_layout(egui::Layout::left_to_right(egui::Align::Min), |ui| { From 86cb375869428d0a69c0638321bfde366fb117c1 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Sat, 7 Nov 2026 18:41:02 +0100 Subject: [PATCH 182/198] M2.6: edits stop at a content stream that doesn't decode, as at damage An edit was refused after a content stream damaged partway, since what the damage hides can change how the later streams read, the graphics state they draw in among it. A stream that doesn't decode at all hides all of its operators, and an edit after one went ahead. It now stops there too: text lines and blocks, and images (arrange, crop, transform, move), as selecting objects on `main` already does for a page with such a stream. An edit in a stream before it goes on, and the stream is kept as stored. A file may give a stream's /Filter by reference, which isn't read here, so the case needs no damage: `content_after_a_stream_that_does_not_decode_is_not_edited` has such a stream move the text and image after it with a `cm`. `a_content_stream_that_does_not_decode_is_kept_and_edited_around` becomes `..._kept_and_edited_before`, its edit now in the stream before. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/edit/src/tests.rs | 43 ++++++++++++++++++++++++++++++----- crates/edit/src/text/lines.rs | 20 ++++++++-------- 2 files changed, 48 insertions(+), 15 deletions(-) diff --git a/crates/edit/src/tests.rs b/crates/edit/src/tests.rs index f3b4e3cc2f..5f9787a8e6 100644 --- a/crates/edit/src/tests.rs +++ b/crates/edit/src/tests.rs @@ -5191,7 +5191,7 @@ fn an_edit_never_rewrites_a_content_stream_damaged_partway() { assert!(refused.to_string().contains("is damaged partway"), "{refused}"); assert_eq!(contents(&doc), listed); let refused = text::replace_line(&mut doc, 0, 1, "Edited").unwrap_err(); - assert!(refused.to_string().contains("comes after a content stream that is damaged partway"), "{refused}"); + assert!(refused.to_string().contains("comes after a content stream that is damaged or can't be decoded"), "{refused}"); assert_eq!(contents(&doc), listed); let mut doc = split_page("BT /F1 12 Tf 72 600 Td (Whole) Tj ET", ""); @@ -5244,25 +5244,56 @@ fn a_content_stream_that_does_not_decode_is_never_written_over() { } /// A content stream that doesn't decode at all is no damage to write over: it is kept as stored, -/// and the page's other streams are read and edited around it. A stream of a few MB is read for +/// and the page's streams before it are read and edited. A stream of a few MB is read for /// selecting objects. #[test] -fn a_content_stream_that_does_not_decode_is_kept_and_edited_around() { - let mut doc = split_page("", "BT /F1 12 Tf 72 600 Td (Whole) Tj ET"); +fn a_content_stream_that_does_not_decode_is_kept_and_edited_before() { + let mut doc = split_page("BT /F1 12 Tf 72 600 Td (Whole) Tj ET", ""); let mut dict = pdfcraft_cos::Dict::new(); dict.set(b"Filter".to_vec(), Object::name("NoSuchFilter")); let unknown = pdfcraft_cos::Stream::from_raw(dict, b"BT /F1 12 Tf 72 700 Td (Unread) Tj ET".to_vec()); - doc.set(pdfcraft_cos::ObjRef::new(4, 0), unknown.clone()); + doc.set(pdfcraft_cos::ObjRef::new(5, 0), unknown.clone()); text::replace_line(&mut doc, 0, 0, "Edited").unwrap(); let doc = reopen(&doc); assert_eq!(text::text_lines(&doc, 0).unwrap().into_iter().map(|l| l.text).collect::>(), ["Edited"]); - let Object::Stream(kept) = &*doc.get(pdfcraft_cos::ObjRef::new(4, 0)) else { panic!("not a stream") }; + let Object::Stream(kept) = &*doc.get(pdfcraft_cos::ObjRef::new(5, 0)) else { panic!("not a stream") }; assert_eq!(kept.raw, unknown.raw, "stored as it was"); let large = format!("{}0 0 m 10 10 l S", " ".repeat(2 << 20)); assert!(editable_objects(&split_page(&large, "BT /F1 12 Tf 72 600 Td (Whole) Tj ET"), 0).is_ok()); } +/// A content stream that doesn't decode hides its operators as damage does. Here its `/Filter` is +/// given by reference, which a file may do and which isn't read here, and it holds a `cm` that +/// moves what the page draws after it. Text and images there aren't edited, as the edit would be +/// worked out without the `cm`. +#[test] +fn content_after_a_stream_that_does_not_decode_is_not_edited() { + use pdfcraft_cos::{Dict, ObjRef, Stream}; + let hidden_move = |doc: &mut Document| { + let filter = doc.add(Object::name("FlateDecode")); + let mut s = Stream::flate(Dict::new(), b"1 0 0 1 0 -300 cm"); + s.dict.set(b"Filter".to_vec(), Object::Ref(filter)); + doc.set(ObjRef::new(4, 0), s); + }; + let contents = |doc: &Document| pdfcraft_model::pages(doc)[0].dict.get(b"Contents").cloned(); + + let mut doc = split_page("", "BT /F1 12 Tf 72 600 Td (After) Tj ET"); + hidden_move(&mut doc); + let listed = contents(&doc); + let refused = text::replace_line(&mut doc, 0, 0, "Edited").unwrap_err(); + assert!(refused.to_string().contains("comes after a content stream that is damaged or can't be decoded"), "{refused}"); + assert_eq!(contents(&doc), listed); + + let mut doc = two_stream_image_page("", "q 100 0 0 100 200 200 cm /Im1 Do Q"); + hidden_move(&mut doc); + let listed = contents(&doc); + let shift = images::ImageChange::Transform([1.0, 0.0, 0.0, 1.0, 10.0, 0.0]); + let refused = images::change_image(&mut doc, 0, 0, &shift).unwrap_err(); + assert!(refused.to_string().contains("comes after"), "{refused}"); + assert_eq!(contents(&doc), listed); +} + /// A content stream whose `/DecodeParms`, or a parameter in them, is given by reference is read /// through the reference: its text is listed and edited, not read with the predictor's tag bytes /// left in it. diff --git a/crates/edit/src/text/lines.rs b/crates/edit/src/text/lines.rs index 58b4ac6289..bc0812e00d 100644 --- a/crates/edit/src/text/lines.rs +++ b/crates/edit/src/text/lines.rs @@ -148,29 +148,31 @@ pub(crate) fn splice(pieces: &Pieces, ops: &[Op], mut edit: impl FnMut(usize) -> } /// An error when content at `/Contents` index `at` (a stream there, or one inserted there) is drawn -/// after a stream damaged partway: what the damage hides can change how it reads. +/// after a stream not read whole, damaged partway or not decoding at all: what wasn't read can +/// change how it reads, the graphics state it draws in among it. pub(crate) fn drawn_before_damage(streams: &[ContentStream], at: usize) -> Result<(), EditError> { if at > first_damaged(streams) { return Err(EditError::Invalid( - "this change comes after a content stream that is damaged partway, and what the damage hides can change how the rest of the page reads" + "this change comes after a content stream that is damaged or can't be decoded, and what wasn't read can change how the rest of the page reads" .into(), )); } Ok(()) } -/// The index of the first stream damaged partway, or the number of streams. +/// The index of the first stream not read whole, or the number of streams. fn first_damaged(streams: &[ContentStream]) -> usize { - streams.iter().position(|s| s.read == Read::Damaged).unwrap_or(streams.len()) + streams.iter().position(|s| s.read != Read::Whole).unwrap_or(streams.len()) } /// The page's `/Contents` after a [`splice`]: pieces whose bytes changed become new streams /// (keeping their dictionaries), the others stay as they were. An error, before anything is -/// added, when a piece that changed is a stream that didn't decode whole, or comes after one -/// damaged partway. Written from what decoded, a damaged stream would lose everything after the -/// damage, and one that doesn't decode all of it. What follows the damage is unknown, so a later -/// stream may not read as it seems: a string or comment the damage left open runs on into it, and -/// operators the damage hides can change the graphics state it draws in. +/// added, when a piece that changed is a stream that didn't decode whole, or comes after one. +/// Written from what decoded, a damaged stream would lose everything after the damage, and one +/// that doesn't decode all of it. What follows the damage is unknown, so a later stream may not +/// read as it seems: a string or comment the damage left open runs on into it, and operators the +/// damage hides can change the graphics state it draws in. A stream that doesn't decode hides its +/// operators all the same. pub(crate) fn rewritten_contents(doc: &mut Document, streams: &[ContentStream], new: Vec>) -> Result { // The last stream that changed: when it is drawn before the damage, so are the others. if let Some(i) = streams.iter().zip(&new).rposition(|(s, data)| s.data != *data) { From ae677644e0b54860a16b2053a0044e19366df18d Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Sat, 7 Nov 2026 18:41:21 +0100 Subject: [PATCH 183/198] M2.6: Extract and Split prune nothing past the form depth limit Pruning reads forms 8 deep. Past that a form's /XObject dictionary was kept whole, but the forms it draws weren't read, nor what they draw through resources of their own. When those resources are shared with content that was read, they were cut to the names that content draws, and the images only the deep forms draw were dropped from the part. This was so on `main` for a form read whole, and in this change's reading of a damaged form too. A walk the depth limit stops now leaves the file unpruned, as an overlarge Type 3 resource graph already does: `nothing_is_pruned_past_the_depth_limit` (a form 9 deep, whole or damaged partway, drawing a form that draws through resources the second page shares). That also settles a form first reached past the limit and drawn again higher up, whose second reading `forms_seen` skips. Down to the limit pruning goes on, under a page whole or damaged partway and with the form 8 deep whole or damaged: `pruning_goes_on_down_to_the_depth_limit`. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/organize/src/prune.rs | 22 ++++++---- crates/organize/src/tests.rs | 78 ++++++++++++++++++++++++++++++++++-- 2 files changed, 89 insertions(+), 11 deletions(-) diff --git a/crates/organize/src/prune.rs b/crates/organize/src/prune.rs index 85ddb5f743..b771b07da9 100644 --- a/crates/organize/src/prune.rs +++ b/crates/organize/src/prune.rs @@ -40,12 +40,15 @@ enum Owner { struct Pruner<'a> { doc: &'a Document, used: HashMap>>, - /// Dictionaries whose drawn names can't be known for certain (undecodable content, nesting - /// past the depth limit, Type 3 glyphs drawing through them). They are left untouched: - /// pruning must never drop an XObject something still draws. + /// Dictionaries whose drawn names can't be known for certain (undecodable content, Type 3 + /// glyphs drawing through them). They are left untouched: pruning must never drop an XObject + /// something still draws. unsure: HashSet, /// An overlarge glyph-resource graph is not safe to prune at all. glyph_budget_exhausted: bool, + /// Nor is a file with forms nested past [`MAX_FORM_DEPTH`]: what the forms there draw isn't + /// read, and they may draw it through resources shared with content that was. + too_deep: bool, glyph_work_left: usize, glyph_seen: HashSet, /// An indirect font's explicit resources have one owner regardless of which page uses it. @@ -188,7 +191,7 @@ impl Pruner<'_> { return; }; if depth > MAX_FORM_DEPTH { - self.unsure.insert(key); + self.too_deep = true; return; } for op in pdfcraft_content::parse(data).ops { @@ -209,7 +212,11 @@ impl Pruner<'_> { self.protect_type3_resources(resources, owner); let Some((xobjects, key)) = self.xobjects(resources, owner) else { return }; self.unsure.insert(key); - if depth > MAX_FORM_DEPTH || !self.partly_read.insert(owner) { + if depth > MAX_FORM_DEPTH { + self.too_deep = true; + return; + } + if !self.partly_read.insert(owner) { return; } for (name, _) in xobjects.iter() { @@ -341,6 +348,7 @@ pub(crate) fn prune_unused_xobjects(doc: &mut Document) -> Result Result pruner.unread(&res_dict, owner, 0), } } - let Pruner { used, unsure, glyph_budget_exhausted, .. } = pruner; - if glyph_budget_exhausted { + let Pruner { used, unsure, glyph_budget_exhausted, too_deep, .. } = pruner; + if glyph_budget_exhausted || too_deep { return Ok(0); } let mut removed = 0; diff --git a/crates/organize/src/tests.rs b/crates/organize/src/tests.rs index dfd9eecf35..7eb677a84c 100644 --- a/crates/organize/src/tests.rs +++ b/crates/organize/src/tests.rs @@ -2733,8 +2733,8 @@ fn extract_keeps_images_forms_drawn_around_damage_draw_through_shared_resources( } /// Forms are read 8 deep (`prune.rs`'s `MAX_FORM_DEPTH`). A damaged form that deep still has -/// every form it names read as drawn, and one drawing through resources another page shares keeps -/// what it draws from them. +/// every form it names read as drawn: here `/Nested`, which draws through resources another page +/// shares. Read 9 deep, it leaves the file unpruned, so what it draws from them stays. #[test] fn a_damaged_form_at_the_depth_limit_has_its_forms_read() { use pdfcraft_cos::ObjRef; @@ -2760,8 +2760,78 @@ fn a_damaged_form_at_the_depth_limit_has_its_forms_read() { assert!(names.iter().any(|n| n.as_slice() == b"ImA"), "{names:?}"); } +/// Past the depth limit a form isn't read, nor what the forms it draws draw: one of those may draw +/// through resources another page shares, and keep images there that page doesn't draw, so nothing +/// is pruned. Here `/Fm` is 9 deep, whole or damaged partway, and `/Nested`, which it draws, draws +/// `/ImA` through the resources the second page draws `/ImB` through. +#[test] +fn nothing_is_pruned_past_the_depth_limit() { + use pdfcraft_cos::ObjRef; + let whole = |content: &[u8]| pdfcraft_cos::Stream::from_raw(Dict::new(), content.to_vec()); + for (what, fm) in [("whole", whole(b"/Nested Do")), ("damaged", damaged_flate(b"0 g\n", b"/Nested Do"))] { + let mut src = doc_shared_form_resources(whole(b"/C Do"), fm, whole(b"/ImA Do")); + // Eight forms between the page and `/Fm`, each drawing the next as `/C`: `/Fm` is 9 deep. + let mut next = ObjRef::new(7, 0); + for _ in 0..8 { + let mut form = whole(b"/C Do"); + form.dict.set(b"Subtype".to_vec(), Object::name("Form")); + form.dict.set(b"BBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 1.into(), 1.into()])); + let names: Dict = [(b"C".to_vec(), Object::Ref(next))].into_iter().collect(); + form.dict.set(b"Resources".to_vec(), Object::Dict([(b"XObject".to_vec(), Object::Dict(names))].into_iter().collect())); + next = src.add(Object::Stream(form)); + } + let names: Dict = [(b"C".to_vec(), Object::Ref(next))].into_iter().collect(); + src.update_dict(ObjRef::new(3, 0), |p| { + p.set(b"Resources".to_vec(), Object::Dict([(b"XObject".to_vec(), Object::Dict(names))].into_iter().collect())) + }) + .unwrap(); + let part = full_roundtrip(&extract_pages(&src, &[0, 1]).unwrap()); + let names = xobject_names(&part, 1); + assert!(names.iter().any(|n| n.as_slice() == b"ImA"), "{what}: {names:?}"); + } +} + +/// Down to the depth limit pruning goes on: under a page whole or damaged partway, a form 8 deep, +/// whole or damaged, is read, and the image nothing draws, `/ImA`, still goes from the resources +/// the second page draws `/ImB` through. +#[test] +fn pruning_goes_on_down_to_the_depth_limit() { + use pdfcraft_cos::ObjRef; + let whole = |content: &[u8]| pdfcraft_cos::Stream::from_raw(Dict::new(), content.to_vec()); + let cases = [ + ("whole", whole(b"/C Do"), whole(b"0 g")), + ("damaged form", whole(b"/C Do"), damaged_flate(b"0 g\n", b"0 g")), + ("damaged page", damaged_flate(b"/C Do\n", b"0 g"), whole(b"0 g")), + ]; + for (what, page, fm) in cases { + let mut src = doc_shared_form_resources(page, fm, whole(b"")); + // `/Fm` names an image, not `/Nested`: nothing is read 9 deep. + let image: Dict = [(b"Img".to_vec(), Object::Ref(ObjRef::new(9, 0)))].into_iter().collect(); + let resources = Object::Dict([(b"XObject".to_vec(), Object::Dict(image))].into_iter().collect()); + src.update_dict(ObjRef::new(7, 0), |f| f.set(b"Resources".to_vec(), resources)).unwrap(); + // Seven forms between the page and `/Fm`, each drawing the next as `/C`: `/Fm` is 8 deep. + let mut next = ObjRef::new(7, 0); + for _ in 1..8 { + let mut form = whole(b"/C Do"); + form.dict.set(b"Subtype".to_vec(), Object::name("Form")); + form.dict.set(b"BBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 1.into(), 1.into()])); + let names: Dict = [(b"C".to_vec(), Object::Ref(next))].into_iter().collect(); + form.dict.set(b"Resources".to_vec(), Object::Dict([(b"XObject".to_vec(), Object::Dict(names))].into_iter().collect())); + next = src.add(Object::Stream(form)); + } + let names: Dict = [(b"C".to_vec(), Object::Ref(next))].into_iter().collect(); + src.update_dict(ObjRef::new(3, 0), |p| { + p.set(b"Resources".to_vec(), Object::Dict([(b"XObject".to_vec(), Object::Dict(names))].into_iter().collect())) + }) + .unwrap(); + let part = full_roundtrip(&extract_pages(&src, &[0, 1]).unwrap()); + assert_eq!(xobject_names(&part, 1), [b"ImB".to_vec()], "{what}"); + } +} + /// A chain of forms nested far past the depth limit, whole or each damaged partway, is read only -/// down to it: the walk's depth stays bounded however long the chain is, and pruning goes on. +/// down to it: the walk's depth stays bounded however long the chain is, and the run ends without +/// an error, having pruned nothing. #[test] fn a_long_chain_of_forms_is_read_only_to_the_depth_limit() { use pdfcraft_cos::ObjRef; @@ -2779,7 +2849,7 @@ fn a_long_chain_of_forms_is_read_only_to_the_depth_limit() { names = [(b"C".to_vec(), Object::Ref(next))].into_iter().collect(); } doc.update_dict(ObjRef::new(3, 0), |p| p.set(b"Resources".to_vec(), xobjects(names))).unwrap(); - assert!(crate::prune::prune_unused_xobjects(&mut doc).is_ok(), "damaged: {damaged}"); + assert!(matches!(crate::prune::prune_unused_xobjects(&mut doc), Ok(0)), "damaged: {damaged}"); } } From f19ecc4635a0a9e99d22dd032d8b24ec3438fdde Mon Sep 17 00:00:00 2001 From: vangel76 Date: Sat, 7 Nov 2026 18:49:31 +0100 Subject: [PATCH 184/198] Merge upstream/main; drop the object-stream pre-check from reconstruct Main now reports unreadable objects from the parse of every object in Document::reconstruct, so the pre-check that skipped that parse is gone. The single-pass /XRef positions and the rarest-byte find stay. Co-Authored-By: Claude Sonnet 5.5 --- crates/cos/src/document/reader.rs | 131 +---------------------------- crates/cos/src/document/tests.rs | 119 ++++++-------------------- crates/cos/tests/it/main.rs | 1 - crates/cos/tests/it/reconstruct.rs | 62 -------------- 4 files changed, 27 insertions(+), 286 deletions(-) delete mode 100644 crates/cos/tests/it/reconstruct.rs diff --git a/crates/cos/src/document/reader.rs b/crates/cos/src/document/reader.rs index 4b2da4fc8b..18f5ce848a 100644 --- a/crates/cos/src/document/reader.rs +++ b/crates/cos/src/document/reader.rs @@ -635,13 +635,7 @@ impl Document { // Offsets of objects that may be cross-reference streams: their dictionaries are the // trailers of files that have no `trailer` keyword (§7.5.8). let mut xref_streams = Vec::new(); - // Objects whose bytes (the last object of a number wins, as in `entries`) could hold an - // `/ObjStm` type; the others are not parsed for the object-stream check below. - let mut candidates: HashSet = HashSet::new(); let headers = object_headers(&data); - // What the dictionary pre-check may read in all, so overlapping unterminated dictionaries - // cost no more than one more pass over the file. - let mut scan_budget = data.len(); // Where `/XRef` occurs in the file, found in one pass. let mut xref_marks = Vec::new(); let mut mark_from = 0; @@ -652,11 +646,6 @@ impl Document { for &(num, offset, generation, body) in &headers { entries.insert(num, XrefEntry::InFile { offset, generation }); objstms.push(num); - if may_be_object_stream(&data, body, &mut scan_budget) { - candidates.insert(num); - } else { - candidates.remove(&num); - } // `/XRef` within the 1024 bytes after the header (the first one that is there). let first = xref_marks.partition_point(|&p| p < body); if xref_marks.get(first).is_some_and(|&p| p + 5 <= body.saturating_add(1024).min(data.len())) @@ -673,8 +662,7 @@ impl Document { *self.unreadable.lock().map_err(|_| CosError::Poisoned)? = Unreadable::default(); // Objects inside object streams. for num in objstms { - if candidates.contains(&num) - && let Ok(o) = self.try_get(num) + if let Ok(o) = self.try_get(num) && let Object::Stream(s) = &*o && s.dict.name(b"Type") == Some(b"ObjStm") && let Ok(stm) = self.objstm(num) @@ -773,123 +761,6 @@ fn shifted_subsection_end(data: &[u8], at: usize) -> Option { } } -/// Longest dictionary [`may_be_object_stream`] reads before it gives up and says "maybe". -const DICTIONARY_SCAN_LIMIT: usize = 1 << 20; - -/// Where the dictionary that starts at `data[from..]` (after `N G obj`) ends, reading it as the -/// parser does: strings, hex strings, comments, names and nested dictionaries. `Some(None)`: the -/// object is not a dictionary, so it is never a stream. `None`: the end is not certain (no end -/// within the limit, a hex string with a bad digit, a `>>` inside an array); the caller must -/// then treat the object as a possible object stream. `limit` ends the read; the second value is -/// how far the read got, for the caller's budget. -fn dictionary_end(data: &[u8], from: usize, limit: usize) -> (Option>, usize) { - let mut i = from; - let result = 'scan: { - while i < limit && (is_whitespace(data[i]) || data[i] == b'%') { - if data[i] == b'%' { - while i < limit && !matches!(data[i], b'\n' | b'\r') { - i += 1; - } - } else { - i += 1; - } - } - if data.get(i..i + 2) != Some(b"<<") { - break 'scan if i < limit { Some(None) } else { None }; - } - i += 2; - let (mut dicts, mut arrays) = (1usize, 0usize); - while i < limit { - match data[i] { - b'(' => { - let mut depth = 1usize; - i += 1; - while i < limit && depth > 0 { - match data[i] { - b'\\' => i += 1, - b'(' => depth += 1, - b')' => depth -= 1, - _ => {} - } - i += 1; - } - if depth > 0 { - break 'scan None; - } - } - b'<' if data.get(i + 1) == Some(&b'<') => { - dicts += 1; - i += 2; - } - b'<' => { - i += 1; - while i < limit && data[i] != b'>' { - if !data[i].is_ascii_hexdigit() && !is_whitespace(data[i]) { - break 'scan None; - } - i += 1; - } - i += 1; - } - b'>' if data.get(i + 1) == Some(&b'>') => { - if arrays > 0 { - break 'scan None; - } - dicts -= 1; - i += 2; - if dicts == 0 { - break 'scan Some(Some(i)); - } - } - b'%' => { - while i < limit && !matches!(data[i], b'\n' | b'\r') { - i += 1; - } - } - b'[' => { - arrays += 1; - i += 1; - } - b']' => { - arrays = arrays.saturating_sub(1); - i += 1; - } - _ => i += 1, - } - } - None - }; - (result, i.min(limit)) -} - -/// Whether the object whose body starts at `data[from..]` could be an object stream: its -/// dictionary holds `ObjStm`, or a `#` (a name may hide its letters as `#xx`, as in -/// `/Obj#53tm`). Only the dictionary is searched, not the data of a stream. Where the -/// dictionary ends is read by [`dictionary_end`], so text inside a string (an object header, a -/// `stream` keyword) is not taken for syntax; when that is not certain the object counts as a -/// candidate, so a doubtful object is parsed as it always was. -/// -/// `budget` is the bytes all the calls of one rebuild may still read. A call reads at most -/// [`DICTIONARY_SCAN_LIMIT`] bytes or what is left, takes what it read off the budget on every -/// path, and says "maybe" once the budget cannot cover it. Dictionaries that do not overlap -/// add up to no more than the file, so only unterminated ones can use the budget up. -pub(super) fn may_be_object_stream(data: &[u8], from: usize, budget: &mut usize) -> bool { - let limit = data.len().min(from.saturating_add(DICTIONARY_SCAN_LIMIT.min(*budget))); - if *budget == 0 { - return true; - } - let (end, read) = dictionary_end(data, from, limit); - *budget = budget.saturating_sub(read.saturating_sub(from).max(1)); - match end { - Some(None) => false, - None => true, - Some(Some(end)) => { - let dict = &data[from..end]; - dict.contains(&b'#') || find(dict, b"ObjStm", 0, dict.len()).is_some() - } - } -} - /// Every `N G obj` header in `data`, in file order, as (number, offset, generation, position /// after `obj`). fn object_headers(data: &[u8]) -> Vec<(u32, u64, u16, usize)> { diff --git a/crates/cos/src/document/tests.rs b/crates/cos/src/document/tests.rs index 59221dc0ce..f69ffae042 100644 --- a/crates/cos/src/document/tests.rs +++ b/crates/cos/src/document/tests.rs @@ -1,4 +1,3 @@ -use super::reader::may_be_object_stream; use super::*; std::thread_local! { @@ -1920,98 +1919,6 @@ fn a_prev_reference_to_an_object_holding_the_offset_reads_that_offset() { /// cross-reference data, as in #2365), is rebuilt from its object headers. The rebuild used /// to look only for the `trailer` keyword, so /Info and /ID were lost, and a full save after /// editing one Info field dropped every other. -/// Whether parsing `body` as an object gives a stream whose /Type is /ObjStm. -fn parses_as_object_stream(file: &[u8]) -> bool { - matches!(parse_indirect(file, 0, &|_| None), Ok((_, Object::Stream(s))) if s.dict.name(b"Type") == Some(b"ObjStm")) -} - -/// A scan budget that never runs out. -fn unlimited() -> usize { - usize::MAX -} - -#[test] -fn the_object_stream_pre_check_never_rejects_what_parses_as_one() { - // Two inputs from review: header-shaped text, and `>>` + `stream` + newline, inside a - // string that comes before the /Type. - for dict in [ - "<< /Note (9 0 obj\n) /Type /ObjStm /N 0 /First 0 >>", - "<< /Note (>>stream\n) /Type /ObjStm /N 0 /First 0 >>", - "<< /Note (a (nested\n9 0 obj\nstream) \\) )) /Type /ObjStm >>", - "<< /Note <3e3e> /Other << /A (>>\nstream\n) >> /Type /ObjStm >>", - ] { - let file = format!("1 0 obj\n{dict}\nstream\nxx\nendstream\nendobj\n").into_bytes(); - assert!(parses_as_object_stream(&file), "test input must be an object stream: {dict}"); - assert!(may_be_object_stream(&file, 7, &mut unlimited()), "{dict}"); - } - // Ordinary streams stay out; their data is never read. - let plain = b"1 0 obj\n<< /Length 9 >>\nstream\nObjStm # \nendstream\nendobj\n"; - assert!(!may_be_object_stream(plain, 7, &mut unlimited())); - assert!(!may_be_object_stream(b"1 0 obj\n[ /ObjStm ]\nendobj\n", 7, &mut unlimited())); - // Unterminated: not certain, so a candidate. - assert!(may_be_object_stream(b"1 0 obj\n<< /Note (open ", 7, &mut unlimited())); - // The scan is charged to the shared budget on every path, and a spent budget says "maybe" - // without reading. - let open = b"1 0 obj\n<< /Note (open "; - let mut budget = 1000; - assert!(may_be_object_stream(open, 7, &mut budget)); - assert_eq!(budget, 1000 - (open.len() - 7)); - let mut budget = 3; - assert!(may_be_object_stream(plain, 7, &mut budget), "a budget too small for the dictionary is a candidate"); - assert_eq!(budget, 0); - let mut budget = 0; - assert!(may_be_object_stream(plain, 7, &mut budget)); - assert_eq!(budget, 0); - // Random dictionaries built from tricky pieces: whatever parses as an object stream is - // a candidate. - let pieces: [&str; 22] = [ - "/Type /ObjStm", - "/Type/ObjStm", - "/Type /Obj#53tm", - "/T#79pe /ObjStm", - "(", - ")", - "\\", - "(>>stream\n)", - "(9 0 obj\n)", - "<3e3e>", - "<", - ">", - ">>", - "<<", - "[", - "]", - "% c\n", - "% >>\n", - "/N 0", - " ", - "\n", - "", - ]; - let mut seed = 0x2545_F491_4F6C_DD1Du64; - let mut next = move || { - seed ^= seed << 13; - seed ^= seed >> 7; - seed ^= seed << 17; - seed - }; - let mut hits = 0; - for _ in 0..200_000 { - let mut dict = String::from("<<"); - for _ in 0..(next() % 9) { - dict.push_str(pieces[(next() % pieces.len() as u64) as usize]); - dict.push(' '); - } - dict.push_str(if next() % 8 == 0 { "" } else { ">>" }); - let file = format!("1 0 obj\n{dict}\nstream\nxx\nendstream\nendobj\n").into_bytes(); - if parses_as_object_stream(&file) { - hits += 1; - assert!(may_be_object_stream(&file, 7, &mut unlimited()), "rejected an object stream: {dict:?}"); - } - } - assert!(hits > 1000, "the generator must produce object streams ({hits})"); -} - #[test] fn reconstruction_takes_the_trailer_from_a_cross_reference_stream() { let table = build( @@ -2282,3 +2189,29 @@ fn the_editor_cache_charges_a_dictionary_for_the_values_it_holds() { let sum = cache.values().fold(0usize, |n, o| n.saturating_add(held_bytes(o, &doc.data, 0))); assert_eq!(cache.owned(), sum, "the kept total is the cache's own"); } + +#[test] +fn find_agrees_with_a_plain_window_search() { + let naive = |hay: &[u8], needle: &[u8], from: usize, to: usize| { + if from >= to || needle.is_empty() || to - from < needle.len() { + return None; + } + hay[from..to].windows(needle.len()).position(|w| w == needle).map(|p| p + from) + }; + let mut seed = 0x9E37_79B9_7F4A_7C15u64; + let mut next = move || { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + seed + }; + let alphabet = b"/XRef t\n<>0"; + for _ in 0..20_000 { + let hay: Vec = (0..next() % 200).map(|_| alphabet[(next() % alphabet.len() as u64) as usize]).collect(); + let needles: [&[u8]; 5] = [b"/XRef", b"trailer", b"R", b"ff", b"<<"]; + let needle = needles[(next() % 5) as usize]; + let from = (next() % (hay.len() as u64 + 1)) as usize; + let to = from + (next() % (hay.len() as u64 + 1 - from as u64)) as usize; + assert_eq!(find(&hay, needle, from, to), naive(&hay, needle, from, to), "{hay:?} {needle:?} {from} {to}"); + } +} diff --git a/crates/cos/tests/it/main.rs b/crates/cos/tests/it/main.rs index d0b73b16d4..1bd7ee1fd6 100644 --- a/crates/cos/tests/it/main.rs +++ b/crates/cos/tests/it/main.rs @@ -5,6 +5,5 @@ mod corpus; mod encryption; mod generations; mod proptests; -mod reconstruct; mod shared_streams; mod writer; diff --git a/crates/cos/tests/it/reconstruct.rs b/crates/cos/tests/it/reconstruct.rs deleted file mode 100644 index 4611c0d17e..0000000000 --- a/crates/cos/tests/it/reconstruct.rs +++ /dev/null @@ -1,62 +0,0 @@ -//! Rebuilding the cross-reference index of a damaged file finds the objects inside object -//! streams however the stream's type is written. - -use std::sync::Arc; - -use pdfcraft_cos::{Document, Object}; - -/// A file whose `startxref` points nowhere, with three object streams (type written plainly, as -/// `/Obj#53tm`, and after a string that contains the word `stream`), and one ordinary stream -/// whose data mentions `ObjStm` and `#`. -fn damaged() -> Vec { - let mut out = b"%PDF-1.5\n".to_vec(); - let mut add = |n: u32, body: &[u8]| { - out.extend_from_slice(format!("{n} 0 obj\n").as_bytes()); - out.extend_from_slice(body); - out.extend_from_slice(b"\nendobj\n"); - }; - let objstm = |dict_prefix: &str, members: &[(u32, &str)]| { - let mut header = String::new(); - let mut body = String::new(); - for (n, text) in members { - header.push_str(&format!("{n} {} ", body.len())); - body.push_str(text); - body.push('\n'); - } - let mut b = - format!("<< {dict_prefix} /N {} /First {} /Length {} >>\nstream\n", members.len(), header.len(), header.len() + body.len()).into_bytes(); - b.extend_from_slice(header.as_bytes()); - b.extend_from_slice(body.as_bytes()); - b.extend_from_slice(b"\nendstream"); - b - }; - add(1, b"<< /Type /Catalog /Pages 2 0 R >>"); - add(2, b"<< /Type /Pages /Kids [] /Count 0 >>"); - add(3, &objstm("/Type /ObjStm", &[(10, "<< /From 3 >>"), (11, "[3]")])); - add(4, &objstm("/Type /Obj#53tm", &[(12, "<< /From 4 >>")])); - add(5, &objstm("/Note (a stream) /Type /ObjStm", &[(13, "<< /From 5 >>")])); - let data = b"/Type /ObjStm # 14 0 << /From 6 >>"; - let mut plain = format!("<< /Length {} >>\nstream\n", data.len()).into_bytes(); - plain.extend_from_slice(data); - plain.extend_from_slice(b"\nendstream"); - add(6, &plain); - // Strings before the type that look like an object header, or like the end of the - // dictionary and the `stream` keyword, and a hex string and a comment. - add(7, &objstm("/Note (\n99 0 obj\n) /Type /ObjStm", &[(15, "<< /From 7 >>")])); - add(8, &objstm("/Note (>>stream\n) /Type /ObjStm", &[(16, "<< /From 8 >>")])); - add(9, &objstm("/Note <3e3e> /Other << /A (a (b) \\) >>\nstream\n) >> % >>\nstream\n\n/Type /ObjStm", &[(17, "<< /From 9 >>")])); - out.extend_from_slice(b"trailer\n<< /Size 20 /Root 1 0 R >>\nstartxref\n99999999\n%%EOF\n"); - out -} - -#[test] -fn objects_in_object_streams_are_found_however_the_type_is_written() { - let doc = Document::open(Arc::new(damaged())).expect("opens"); - assert!(doc.reconstructed()); - for n in [10, 11, 12, 13, 15, 16, 17] { - let o = doc.try_get(n).unwrap_or_else(|e| panic!("object {n}: {e}")); - assert!(matches!(&*o, Object::Dict(_) | Object::Array(_)), "object {n} is {o:?}"); - } - // The data of an ordinary stream is not an object stream, whatever it says. - assert!(!doc.object_numbers().contains(&14)); -} From 569f631f2b074784bf479020fce8fb862347fda9 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Sat, 7 Nov 2026 20:18:07 +0100 Subject: [PATCH 185/198] M2.6: an image's move changes no state a stream not read whole draws in MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Moving or resizing an image in a stream before one damaged partway, or one that doesn't decode, goes on, as the edit changes no stream past it. But the image's `q cm … Q` can close past that stream, and what the stream draws inside the scope is unknown. When the image looked alone there, the move changed the scope's `cm` in place, and anything the stream draws in it moved too: a square drawn there in the image's `cm` moved with the image, as on `main`. A scope still open after such a stream now counts as one that paints something else (`first_op_after_damage`, read by `image_scopes` and `enclosing_transform`). An operator counts by its keyword there: a page may split its content between an operator's operands and its keyword, so a `Q` whose operands come before the hidden stream still closes after it. The move gives the `Do` a `cm` of its own and leaves the scope's as it was; with a clip on the image, the clip stays, as one the stream may share. A crop goes around the `Do` alone rather than at the scope's start, where it would clip what the stream draws. A crop PdfCraft put in such a scope isn't removed, and the refusal says why rather than that there is no crop. Arrange is unchanged: it takes out the scope it finds, which changes the stream past it and is refused. `an_image_whose_scope_closes_past_a_stream_not_read_whole_moves_alone`: a stream that doesn't decode and one damaged, each also with the square's operands before what is hidden (ending the stream before, or in the damaged stream before its damage); the image with no clip, a clip, and a PdfCraft crop; moved, cropped, and its crop removed. A scope that closes in a damaged stream before its damage still takes the move in its `cm`. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/edit/src/images.rs | 57 ++++++++++++++++++-------- crates/edit/src/tests.rs | 77 +++++++++++++++++++++++++++++++++++ crates/edit/src/text.rs | 4 +- crates/edit/src/text/lines.rs | 9 ++++ 4 files changed, 128 insertions(+), 19 deletions(-) diff --git a/crates/edit/src/images.rs b/crates/edit/src/images.rs index 5e5c44fab7..fbb37ec1f0 100644 --- a/crates/edit/src/images.rs +++ b/crates/edit/src/images.rs @@ -527,12 +527,17 @@ fn exclusive_clip(ops: &[Op], op: usize) -> Option<(usize, Matrix)> { /// The `q cm … Q` whose only content after its `cm` is `start..=end` (a `Do`, or a balanced /// `q … Q` scope), with the index and value of that `cm`. Another transform of the same /// placement is folded into it, so repeated moves and resizes don't nest one more `q` each. -fn enclosing_transform(ops: &[Op], start: usize, end: usize) -> Option<(usize, Matrix)> { +/// None when its `Q` is at or past `hidden` ([`crate::text::first_op_after_damage`]): a stream +/// between that wasn't read may draw in that `cm` too. +fn enclosing_transform(ops: &[Op], start: usize, end: usize, hidden: usize) -> Option<(usize, Matrix)> { let at = start.checked_sub(1)?; let cm = ops.get(at)?; if !ops.get(at.checked_sub(1)?)?.is("q") || !ops.get(end.checked_add(1)?)?.is("Q") || !cm.is("cm") || cm.operands.len() != 6 { return None; } + if end.checked_add(1)? >= hidden { + return None; + } let m = Matrix(cm.nums::<6>()?); m.0.iter().all(|v| v.is_finite()).then_some((at, m)) } @@ -1052,7 +1057,9 @@ fn determine_arrange_spans( has_other_paint: bool, marked_unit: bool, ) -> Result<(std::ops::Range, usize, usize, usize), EditError> { - let scope = image_scopes(ops, target, "arranged") + // Nothing is hidden for Arrange: it takes out the scope it finds, so a scope that closes past a + // stream not read whole changes a stream past it, which `rewritten_contents` refuses. + let scope = image_scopes(ops, target, "arranged", ops.len()) .ok() .and_then(|s| s.isolated) .filter(|&(start, end, _)| start >= target_unit.start && end < target_unit.end); @@ -1491,8 +1498,10 @@ impl ImageScopes { } } -/// [`ImageScopes`] of the `Do` at `target`. An image inside a text object cannot be `done`. -fn image_scopes(ops: &[Op], target: usize, done: &str) -> Result { +/// [`ImageScopes`] of the `Do` at `target`. An image inside a text object cannot be `done`. A scope +/// still open at op `hidden` ([`crate::text::first_op_after_damage`]) holds a stream that wasn't +/// read, which may paint there: it counts as painting something else. +fn image_scopes(ops: &[Op], target: usize, done: &str, hidden: usize) -> Result { struct Scope { start: usize, matrix: Matrix, @@ -1511,6 +1520,11 @@ fn image_scopes(ops: &[Op], target: usize, done: &str) -> Result