From 7cea6dfeadf31b512fd791b2ca0b78c3d694aaad Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Tue, 10 Nov 2026 06:19:08 +0100 Subject: [PATCH 01/60] M7: a font's metrics are past their allowance only when it refused a read `Reads::decode` and the Type 3 glyph-box read took any failed read, with nothing left of the allowance after it, for one the allowance refused. The Thai edit refuses on what the allowance itself counts (`DecodeBudget::refused`), so the two could disagree: once the edit's allowance was spent to the byte, an empty stream behind a filter nothing decodes, which is unread with any allowance, made a font's metrics read past it with no refusal counted. The edit then skipped that font of the face and went through without it, which its refusal is there to prevent. Both now take a read as refused only when `DecodeBudget::refused` grew during it, so `Metrics::past_allowance` is set exactly when the allowance counted a refusal, as its doc now says. Tests: - `an_empty_stream_reads_with_nothing_left` adds such a stream: it isn't past the allowance, and the flag agrees with the count in each case. - `a_type3_description_unread_on_its_own_is_not_past_the_allowance`: the same for a glyph description after one that spends the allowance to the byte; a description with data after it is still refused. - `a_font_of_the_face_unread_on_its_own_once_the_allowance_is_spent_still_refuses_the_edit`: the edit above is refused, where it went through. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/edit/src/tests.rs | 32 ++++++++++++++++++++ crates/fonts/src/pdf.rs | 17 ++++++++--- crates/fonts/src/pdf/tests.rs | 57 +++++++++++++++++++++++++++++++---- 3 files changed, 96 insertions(+), 10 deletions(-) diff --git a/crates/edit/src/tests.rs b/crates/edit/src/tests.rs index f2033a64de..1f2a6c6a9a 100644 --- a/crates/edit/src/tests.rs +++ b/crates/edit/src/tests.rs @@ -14337,6 +14337,38 @@ fn a_thai_edit_that_spends_its_allowance_to_the_byte_goes_through() { assert!(error.contains("THSarabunPSK") && error.contains("MiB of font data"), "{error}"); } +/// Only a read the allowance refuses leaves a font of the face unread for want of it. With the +/// allowance spent to the byte as above, another font of the face, whose ToUnicode is an empty +/// stream behind a filter nothing decodes (unread with any allowance), still has its program read, +/// which the allowance refuses: the edit is refused, rather than going through without that font. +#[test] +fn a_font_of_the_face_unread_on_its_own_once_the_allowance_is_spent_still_refuses_the_edit() { + let mut program = legacy_thai_program(); + program.resize((16 << 20) - 192, 0); + let mut unicode = LEGACY_THAI_UNICODE.to_vec(); + unicode.resize(256, b' '); + let mut doc = text_page("BT /F1 16 Tf 50 650 Td <00d6> Tj ET"); + let font = legacy_thai_font(&mut doc, "ABCDEF+THSarabunPSK", &program, &unicode); + doc.set(pdfcraft_cos::ObjRef::new(5, 0), Object::Dict(font)); + // Not a subset, so its metrics read no program, only the ToUnicode. + let mut other = legacy_thai_font(&mut doc, "THSarabunPSK", &legacy_thai_program(), LEGACY_THAI_UNICODE); + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::name("NoSuchFilter")); + other.set(b"ToUnicode".to_vec(), Object::Ref(doc.add(Object::Stream(Stream::from_raw(dict, Vec::new()))))); + let other = doc.add(Object::Dict(other)); + doc.update_dict(pdfcraft_cos::ObjRef::new(3, 0), |page| { + let Some(Object::Dict(mut resources)) = page.get(b"Resources").cloned() else { panic!("fixture") }; + let Some(Object::Dict(mut fonts)) = resources.get(b"Font").cloned() else { panic!("fixture") }; + fonts.set(b"F2".to_vec(), Object::Ref(other)); + resources.set(b"Font".to_vec(), Object::Dict(fonts)); + page.set(b"Resources".to_vec(), Object::Dict(resources)); + }) + .unwrap(); + let style = BlockStyle { width: Some(400.0), ..Default::default() }; + let error = text::rewrite_block(&mut doc, 0, 0, Some("กขค"), &style).unwrap_err().to_string(); + assert!(error.contains("THSarabunPSK") && error.contains("MiB of font data"), "{error}"); +} + /// A THSarabunPSK-named subset that has OpenType Thai mark positioning is a modern face: it is /// shaped with its own layout like any source font, not by the legacy shaper (which would ask /// for Windows PUA alternates such a face doesn't need). diff --git a/crates/fonts/src/pdf.rs b/crates/fonts/src/pdf.rs index f68761e02e..00ace3bf44 100644 --- a/crates/fonts/src/pdf.rs +++ b/crates/fonts/src/pdf.rs @@ -70,7 +70,9 @@ pub struct Metrics { /// A stream these metrics come from (a CMap, the program, `/CIDToGIDMap`, a Type 3 glyph /// description) was left unread because the decode allowance [`Metrics::from_dict_within`] /// was given was spent: what it would have decided (code lengths, CIDs, widths, meanings, a - /// Type 3 font's height) is a guess. + /// Type 3 font's height) is a guess. It is set exactly when the allowance counted a refusal + /// during the read ([`DecodeBudget::refused`]), which callers that share it may refuse on; a + /// stream that fails on its own data doesn't set it, with or without anything left. pub past_allowance: bool, } @@ -420,12 +422,16 @@ impl<'a> Reads<'a> { /// stage past `max` is unread. When what is left is more than what the stages before that one /// made and `max`, it is charged those two and isn't refused; otherwise, with more left than /// the stages' total (behind one filter, `max`), it is charged that total and isn't refused; - /// with no more left than that total, it spends all of it and counts as refused. + /// with no more left than that total, it spends all of it and counts as refused. Only what + /// the allowance counts as refused ([`DecodeBudget::refused`]) is: a stream that fails on its + /// own data fails the same with more left, even when it spends the last of it or nothing was + /// left, and callers that refuse on that count must agree with [`Metrics::past_allowance`]. fn decode(&mut self, stream: &Stream, max: usize) -> Option> { + let refused = self.budget.refused(); match self.budget.decode_stages_within(self.doc, stream, max) { Ok(data) => Some(data), Err(_) => { - self.refused |= self.budget.remaining() == 0; + self.refused |= self.budget.refused() > refused; None } } @@ -491,10 +497,13 @@ fn type3_glyph_box(doc: &Document, font: &Dict, reads: &mut Reads<'_>) -> Option break; } let before = reads.budget.remaining(); + let refused = reads.budget.refused(); let read = reads.budget.decode_with_damage(doc, s, MAX_TYPE3_GLYPH_BYTES.min(left)).map(|(data, _)| data); left = left.saturating_sub(before.saturating_sub(reads.budget.remaining())); let Ok(data) = read else { - if reads.budget.remaining() == 0 { + // As in `Reads::decode`, only a refusal the allowance counts: a description that fails + // on its own data, spending the last of it, leaves the rest to be refused above. + if reads.budget.refused() > refused { // The rest go unread too, for want of the reading's allowance. reads.refused = true; complete = false; diff --git a/crates/fonts/src/pdf/tests.rs b/crates/fonts/src/pdf/tests.rs index 975b71289e..7a925c0b78 100644 --- a/crates/fonts/src/pdf/tests.rs +++ b/crates/fonts/src/pdf/tests.rs @@ -309,17 +309,27 @@ fn a_fonts_streams_are_paid_for_from_its_allowance() { } /// With nothing left, an empty stream still reads, as the allowance reads one, and is no reason to -/// call the font read past it; a stream with data is left unread, and is. +/// call the font read past it; a stream with data is left unread, and is. An empty stream behind a +/// filter nothing decodes is unread with any allowance, so it isn't either: the font is read past +/// its allowance exactly when the allowance counts a refusal, which is what callers refuse on. #[test] fn an_empty_stream_reads_with_nothing_left() { - let past = |data: &[u8]| { + let past = |filter: Option<&str>, data: &[u8]| { let mut doc = Document::new_empty(); - let unicode = Object::Ref(doc.add(Object::Stream(Stream::from_raw(Dict::new(), data.to_vec())))); + let mut dict = Dict::new(); + if let Some(filter) = filter { + dict.set(b"Filter".to_vec(), Object::name(filter)); + } + let unicode = Object::Ref(doc.add(Object::Stream(Stream::from_raw(dict, data.to_vec())))); let f = font(&mut doc, vec![("Subtype", Object::name("TrueType")), ("ToUnicode", unicode)]); - Metrics::from_dict_within(&doc, &f, &mut DecodeBudget::new(0, 0)).past_allowance + let mut budget = DecodeBudget::new(0, 0); + let past = Metrics::from_dict_within(&doc, &f, &mut budget).past_allowance; + assert_eq!(past, budget.refused() > 0, "{filter:?}"); + past }; - assert!(!past(b"")); - assert!(past(b"1 beginbfchar <41> <0058> endbfchar")); + assert!(!past(None, b"")); + assert!(!past(Some("NoSuchFilter"), b"")); + assert!(past(None, b"1 beginbfchar <41> <0058> endbfchar")); } /// A stream behind one filter past its own limit, with more left than its limit, is left unread as @@ -969,6 +979,41 @@ fn type3_glyph_boxes_are_paid_for_from_the_allowance() { assert!(m.past_allowance, "a byte short, the second description is left unread"); } +/// A glyph description that fails on its own once the allowance is spent to the byte, an empty one +/// behind a filter nothing decodes, fails the same with more left: it doesn't make the metrics read +/// past the allowance, and the descriptions after it are read, or refused by the allowance. +#[test] +fn a_type3_description_unread_on_its_own_is_not_past_the_allowance() { + let mut doc = Document::new_empty(); + let procs = ["1 0 0 -20 1 300 d1", "", "1 0 0 0 1 400 d1"]; + let mut f = type3(&mut doc, [1.0, 0.0, 0.0, 1.0, 0.0, 0.0], [0, 0, 0, 0], &procs); + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::name("NoSuchFilter")); + let unread = Object::Ref(doc.add(Object::Stream(Stream::from_raw(dict, Vec::new())))); + let Some(Object::Dict(mut charprocs)) = f.get(b"CharProcs").cloned() else { panic!("CharProcs") }; + charprocs.set(b"g1".to_vec(), unread); + f.set(b"CharProcs".to_vec(), Object::Dict(charprocs)); + with_differences(&mut f, &["g0", "g1", "g2"].map(String::from)); + let first = procs[0].len(); + let whole = Metrics::from_dict_within(&doc, &f, &mut DecodeBudget::new(usize::MAX, 0)); + assert!(!whole.past_allowance); + assert_eq!((whole.ascent, whole.descent), (400.0, -20.0)); + // The first description spends it to the byte; the third, which has data, is refused. + let mut budget = DecodeBudget::new(first, 0); + let m = Metrics::from_dict_within(&doc, &f, &mut budget); + assert_eq!(budget.refused(), 1); + assert!(m.past_allowance); + // Without the third, nothing is refused, and the metrics aren't past the allowance. + let Some(Object::Dict(mut charprocs)) = f.get(b"CharProcs").cloned() else { panic!("CharProcs") }; + charprocs.remove(b"g2"); + f.set(b"CharProcs".to_vec(), Object::Dict(charprocs)); + let mut budget = DecodeBudget::new(first, 0); + let m = Metrics::from_dict_within(&doc, &f, &mut budget); + assert_eq!((budget.refused(), budget.remaining()), (0, 0)); + assert!(!m.past_allowance); + assert_eq!((m.ascent, m.descent), (300.0, -20.0)); +} + /// A Type 3 font measured from its glyphs' `d1` boxes keeps them over a descriptor's `/Ascent` /// and `/Descent`, which are optional for Type 3 (ISO 32000-2 Table 120). #[test] From 973ddc17e9f5c17726e86d5ecfae7e9a21b7a540 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Tue, 10 Nov 2026 06:46:04 +0100 Subject: [PATCH 02/60] M7: a test pins the editor's program list against a stream unread on its own The editor's program list (`embedded_sfnt_within`) reads a legacy Thai font's metrics after its program, and passes over the program when they are past the allowance. With the program spending the allowance to the byte, an empty ToUnicode behind a filter nothing decodes no longer passes it over; a ToUnicode with data is still refused, and the program isn't listed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/fonts/src/pdf/tests.rs | 39 +++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/crates/fonts/src/pdf/tests.rs b/crates/fonts/src/pdf/tests.rs index 7a925c0b78..7c1d5ca0c0 100644 --- a/crates/fonts/src/pdf/tests.rs +++ b/crates/fonts/src/pdf/tests.rs @@ -434,6 +434,45 @@ fn each_stage_of_a_metrics_stream_is_held_to_its_limit() { assert_eq!(read(MAX_CID_TO_GID / 4), (false, false)); } +/// The editor's program list reads a legacy Thai font's metrics after its program. With the program +/// spending the allowance to the byte, a ToUnicode that is an empty stream behind a filter nothing +/// decodes, unread with any allowance, doesn't put the metrics past the allowance: the program is +/// listed, as with plenty. A ToUnicode with data is refused by the allowance, and the program isn't. +#[test] +fn an_editors_program_is_listed_past_a_stream_unread_on_its_own() { + let bytes = include_bytes!("../../../../assets/fonts/Inter-Regular.ttf"); + let listed = |filter: Option<&str>, data: &[u8]| { + let mut doc = Document::new_empty(); + let mut dict = Dict::new(); + if let Some(filter) = filter { + dict.set(b"Filter".to_vec(), Object::name(filter)); + } + let unicode = Object::Ref(doc.add(Object::Stream(Stream::from_raw(dict, data.to_vec())))); + let program = Object::Ref(doc.add(Object::Stream(Stream::from_raw(Dict::new(), bytes.to_vec())))); + let mut descriptor = Dict::new(); + descriptor.set(b"FontFile2".to_vec(), program); + let f = font( + &mut doc, + vec![ + ("Subtype", Object::name("TrueType")), + ("BaseFont", Object::name("THSarabunPSK")), + ("FirstChar", Object::Int(65)), + ("LastChar", Object::Int(65)), + ("Widths", Object::Array(vec![Object::Int(500)])), + ("Encoding", Object::name("WinAnsiEncoding")), + ("FontDescriptor", Object::Dict(descriptor)), + ("ToUnicode", unicode), + ], + ); + let mut budget = DecodeBudget::new(bytes.len(), 0); + let listed = embedded_sfnt_within(&doc, &f, &mut budget).is_some(); + assert_eq!(budget.remaining(), 0, "the program spends the allowance to the byte"); + (listed, budget.refused()) + }; + assert_eq!(listed(Some("NoSuchFilter"), b""), (true, 0)); + assert_eq!(listed(None, b"1 beginbfchar <41> <0058> endbfchar"), (false, 1)); +} + /// The editor's program is held to its limit as the metrics reads' streams are: a 12 MiB program /// whose Flate data is about 8 MiB reads behind ASCIIHex and Flate as behind Flate alone, though /// the two stages make about 20 MiB together. A program of the limit reads behind either, and one a From 5de168955bd285ac3b5370cb1fbbfa6009e45247 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Tue, 10 Nov 2026 06:59:28 +0100 Subject: [PATCH 03/60] M7: the docs say which failed reads the allowance doesn't count `Metrics::past_allowance`, `Reads::decode` and the Type 3 glyph-box read said a stream that fails on its own data isn't counted as refused even when it spends the last of the allowance. A stream past its own limit fails with any allowance, yet with no more left than its stages' total it is counted, as `Reads::decode` itself says just above. They now name the case that isn't: a read that fails without the allowance refusing it, such as an empty stream behind a filter nothing decodes. The program-list test's doc no longer says more than it checks. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/fonts/src/pdf.rs | 12 +++++++----- crates/fonts/src/pdf/tests.rs | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/crates/fonts/src/pdf.rs b/crates/fonts/src/pdf.rs index 00ace3bf44..37bbc52819 100644 --- a/crates/fonts/src/pdf.rs +++ b/crates/fonts/src/pdf.rs @@ -72,7 +72,8 @@ pub struct Metrics { /// was given was spent: what it would have decided (code lengths, CIDs, widths, meanings, a /// Type 3 font's height) is a guess. It is set exactly when the allowance counted a refusal /// during the read ([`DecodeBudget::refused`]), which callers that share it may refuse on; a - /// stream that fails on its own data doesn't set it, with or without anything left. + /// read that fails without one, such as an empty stream behind a filter nothing decodes, + /// doesn't set it, with or without anything left. pub past_allowance: bool, } @@ -423,9 +424,9 @@ impl<'a> Reads<'a> { /// made and `max`, it is charged those two and isn't refused; otherwise, with more left than /// the stages' total (behind one filter, `max`), it is charged that total and isn't refused; /// with no more left than that total, it spends all of it and counts as refused. Only what - /// the allowance counts as refused ([`DecodeBudget::refused`]) is: a stream that fails on its - /// own data fails the same with more left, even when it spends the last of it or nothing was - /// left, and callers that refuse on that count must agree with [`Metrics::past_allowance`]. + /// the allowance counts as refused ([`DecodeBudget::refused`]) is noted: a read that fails + /// without one, such as an empty stream behind a filter nothing decodes, fails the same with + /// more left, and callers that refuse on that count must agree with [`Metrics::past_allowance`]. fn decode(&mut self, stream: &Stream, max: usize) -> Option> { let refused = self.budget.refused(); match self.budget.decode_stages_within(self.doc, stream, max) { @@ -502,7 +503,8 @@ fn type3_glyph_box(doc: &Document, font: &Dict, reads: &mut Reads<'_>) -> Option left = left.saturating_sub(before.saturating_sub(reads.budget.remaining())); let Ok(data) = read else { // As in `Reads::decode`, only a refusal the allowance counts: a description that fails - // on its own data, spending the last of it, leaves the rest to be refused above. + // without one (an empty one nothing decodes, once the allowance is spent) leaves the + // rest to be read, or refused above. if reads.budget.refused() > refused { // The rest go unread too, for want of the reading's allowance. reads.refused = true; diff --git a/crates/fonts/src/pdf/tests.rs b/crates/fonts/src/pdf/tests.rs index 7c1d5ca0c0..4c5f90044f 100644 --- a/crates/fonts/src/pdf/tests.rs +++ b/crates/fonts/src/pdf/tests.rs @@ -437,7 +437,7 @@ fn each_stage_of_a_metrics_stream_is_held_to_its_limit() { /// The editor's program list reads a legacy Thai font's metrics after its program. With the program /// spending the allowance to the byte, a ToUnicode that is an empty stream behind a filter nothing /// decodes, unread with any allowance, doesn't put the metrics past the allowance: the program is -/// listed, as with plenty. A ToUnicode with data is refused by the allowance, and the program isn't. +/// listed. A ToUnicode with data is refused by the allowance, and the program isn't. #[test] fn an_editors_program_is_listed_past_a_stream_unread_on_its_own() { let bytes = include_bytes!("../../../../assets/fonts/Inter-Regular.ttf"); From e5d7184558e60d56cf2fd38691d083fb5244010d Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Tue, 10 Nov 2026 07:36:25 +0100 Subject: [PATCH 04/60] M11: Optimize stores a picture's one-bit masks as monochrome images The image pass visits the images pages draw by name. A picture's stencil `/Mask` and its `/SMask` are images of their own that pages don't name, so the monochrome step never reached them (the colour and grey settings resample an eight-bit soft mask with its picture, but store no mask as Group 4 or JBIG2). In a mixed-raster scan, a colour layer drawn through a one-bit mask that holds the text, most of the black-and-white data stayed as it was written (Flate, say). After the pictures, each picture's stencil mask and one-bit soft mask, once however many pictures name it, now goes through the monochrome step at its size (`decide_mask`): stored again as the compression asks (Group 4, JBIG2) when that is smaller, its bits as they were, so it masks as it did. It isn't resampled (its resolution isn't measured) and shares no JBIG2 dictionary (only images named directly do, which redaction can store again). A stencil pages also draw by name stays one of the pictures; a soft mask they draw is still stored as a mask. A soft mask of more bits stays, to be resampled with its picture, which wants it at eight. The report counts masks as images; Retain leaves them as they are. The pictures and the masks go through one loop (`work_through`), so masks are worked out on threads and from the run's allowance as pictures are. Tests: `a_pictures_one_bit_masks_are_stored_again`, `a_pictures_mask_stored_as_jbig2_shares_no_dictionary` and, through `doc_optimize`, `doc_optimize_stores_a_pictures_one_bit_soft_mask_as_ccitt_group_4`. `a_mask_past_the_runs_allowance_stays_and_is_counted` holds a mask past the run's allowance to staying, counted as refused, and the progress to counting masks with pictures. `a_picture_with_a_stencil_mask_is_downsampled_under_its_mask` now checks the mask's size and bits rather than its bytes. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/automation/src/tools.rs | 2 +- crates/automation/tests/it/automation.rs | 55 +++++ crates/optimize/README.md | 19 +- crates/optimize/src/images.rs | 251 ++++++++++++++++++++--- crates/optimize/src/lib.rs | 9 +- crates/optimize/tests/optimize.rs | 104 +++++++++- parity/acrobat-features.toml | 4 +- 7 files changed, 401 insertions(+), 43 deletions(-) diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index 4342e162f8..38bd692b61 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -1726,7 +1726,7 @@ pub fn tools() -> Vec { .with(schema(json!({ "doc": doc() }), &["doc"])), t("doc_open_revision", "Open a revision", "Open saved revision `revision` (1 = the oldest) of a document as a new, unsaved document, to see the file as it was then.") .with(schema(json!({ "doc": doc(), "revision": { "type": "integer", "minimum": 1 } }), &["doc", "revision"])), - t("doc_optimize", "Optimize PDF", "Write an optimized copy to `path` (Acrobat's PDF Optimizer). preset: standard (default; Reduce File Size's choices) or mobile (images to 96 ppi above 144, JPEG high quality); the options below change it. compatibility: retain (default; keeps the document's version, or 1.5 when object streams are written), 1.4, 1.5, 1.6, 1.7 or 2.0. merge_fragments (default true) first puts a page picture drawn as many stacked image strips (as some PDF printers write it) back together into one image, so it can be resampled (merged_images, fragments_merged). color / gray: { downsample, ppi, above_ppi, compression: jpeg|zip|retain, quality 1–100 } (defaults: downsample to 150 ppi above 225, JPEG 60). mono: { downsample, ppi, above_ppi, compression: ccitt_g4|jbig2|retain } for black-and-white images and image masks (default: averaged down to 300 ppi above 450 ppi, counted in images_resampled; stored losslessly as CCITT Group 4 or JBIG2 (or 1-bit Flate, once resampled or packed from more bits) when that is smaller, up to 16 JBIG2 images sharing one symbol dictionary (/JBIG2Globals) where that is smaller still (image masks and DeviceGray images named directly, of at most 64 Mi pixels, which redaction can store again without it), Group 3 fax images, JBIG2 images, black-and-white RGB and indexed images using only palette entries 0 and 1 among them, counted in images_recompressed; retain resamples only, a CCITT image staying CCITT, a JBIG2 image JBIG2 and any other becoming Flate). Images past what one run may decode stay as they are, counted in images_refused; streams it only stores with Flate past that stay LZW, uncounted. Images are measured where pages draw them and replaced only when smaller. discard_*: thumbnails (default true), alternate_images (true), tags, search_index (removes the catalog's /PieceInfo entries whose application name contains \"index\" in any letter case; search_indexes), print_settings (entries removed: print_settings), form_actions (submit, import and reset actions removed: form_actions; the fields and links stay, and the actions chained after one still run); flate_unencoded (true); object_streams (default true; always off for 1.4); lzw_to_flate (true) decodes LZW streams and stores them with Flate instead, leaving any it cannot decode unchanged (lzw_streams counts the streams converted); remove_invalid_links and remove_unreferenced_dests (true); discard_external_refs takes GoToR, GoToE, Launch and ImportData actions, and Thread actions with /F, out of links and bookmarks, removes a link left with nothing to do, and keeps links within the document and web links (external_links, external_bookmarks); images and forms the pages list but never draw are always dropped (unused_xobjects). discard: Remove Hidden Information categories (metadata, attachments, comments, form-fields, hidden-text, hidden-layers, bookmarks, links-actions-scripts, private-data). Signed documents are refused. target_bytes (optional): if the copy is larger, retry with up to 5 progressively stronger image settings (downsample to at most 110, 96, 72, 54, 36 ppi, applied to colour and grey images; images set to JPEG also get JPEG quality at most 45, 35, 28, 20, 12, while zip and retain images keep their encoding and only get the lower ppi) and stop at the first copy that fits; reports target_reached, attempts and target_settings, and writes the smallest copy with target_reached false when none fits. Without it nothing changes. The open document is unchanged.") + t("doc_optimize", "Optimize PDF", "Write an optimized copy to `path` (Acrobat's PDF Optimizer). preset: standard (default; Reduce File Size's choices) or mobile (images to 96 ppi above 144, JPEG high quality); the options below change it. compatibility: retain (default; keeps the document's version, or 1.5 when object streams are written), 1.4, 1.5, 1.6, 1.7 or 2.0. merge_fragments (default true) first puts a page picture drawn as many stacked image strips (as some PDF printers write it) back together into one image, so it can be resampled (merged_images, fragments_merged). color / gray: { downsample, ppi, above_ppi, compression: jpeg|zip|retain, quality 1–100 } (defaults: downsample to 150 ppi above 225, JPEG 60). mono: { downsample, ppi, above_ppi, compression: ccitt_g4|jbig2|retain } for black-and-white images and image masks (default: averaged down to 300 ppi above 450 ppi, counted in images_resampled; stored losslessly as CCITT Group 4 or JBIG2 (or 1-bit Flate, once resampled or packed from more bits) when that is smaller, up to 16 JBIG2 images sharing one symbol dictionary (/JBIG2Globals) where that is smaller still (image masks and DeviceGray images named directly, of at most 64 Mi pixels, which redaction can store again without it), Group 3 fax images, JBIG2 images, black-and-white RGB and indexed images using only palette entries 0 and 1 among them, counted in images_recompressed; a picture's stencil /Mask and one-bit /SMask are stored again the same way, at their size, not resampled and sharing no dictionary, counted as images; retain resamples only, a CCITT image staying CCITT, a JBIG2 image JBIG2 and any other becoming Flate). Images past what one run may decode stay as they are, counted in images_refused; streams it only stores with Flate past that stay LZW, uncounted. Images are measured where pages draw them and replaced only when smaller. discard_*: thumbnails (default true), alternate_images (true), tags, search_index (removes the catalog's /PieceInfo entries whose application name contains \"index\" in any letter case; search_indexes), print_settings (entries removed: print_settings), form_actions (submit, import and reset actions removed: form_actions; the fields and links stay, and the actions chained after one still run); flate_unencoded (true); object_streams (default true; always off for 1.4); lzw_to_flate (true) decodes LZW streams and stores them with Flate instead, leaving any it cannot decode unchanged (lzw_streams counts the streams converted); remove_invalid_links and remove_unreferenced_dests (true); discard_external_refs takes GoToR, GoToE, Launch and ImportData actions, and Thread actions with /F, out of links and bookmarks, removes a link left with nothing to do, and keeps links within the document and web links (external_links, external_bookmarks); images and forms the pages list but never draw are always dropped (unused_xobjects). discard: Remove Hidden Information categories (metadata, attachments, comments, form-fields, hidden-text, hidden-layers, bookmarks, links-actions-scripts, private-data). Signed documents are refused. target_bytes (optional): if the copy is larger, retry with up to 5 progressively stronger image settings (downsample to at most 110, 96, 72, 54, 36 ppi, applied to colour and grey images; images set to JPEG also get JPEG quality at most 45, 35, 28, 20, 12, while zip and retain images keep their encoding and only get the lower ppi) and stop at the first copy that fits; reports target_reached, attempts and target_settings, and writes the smallest copy with target_reached false when none fits. Without it nothing changes. The open document is unchanged.") .cmd("optimize.advanced") .with(schema( json!({ diff --git a/crates/automation/tests/it/automation.rs b/crates/automation/tests/it/automation.rs index e0ceba48c0..6be92da1c3 100644 --- a/crates/automation/tests/it/automation.rs +++ b/crates/automation/tests/it/automation.rs @@ -8790,6 +8790,61 @@ fn doc_optimize_stores_black_and_white_images_as_ccitt_group_4() { } } +/// A mixed-raster page: a flat colour layer drawn through a one-bit soft mask, [`scanned_page`]'s +/// scan, which the page no longer draws by name. `doc_optimize` stores the mask as Group 4, at its +/// size, still the picture's soft mask, and counts it as an image; with `mono: retain` it stays. +#[test] +fn doc_optimize_stores_a_pictures_one_bit_soft_mask_as_ccitt_group_4() { + use pdfcraft_cos::{Dict, Document, ObjRef, Object, SaveOptions, Stream, write_full}; + let dir = workdir("optimize-mono-mask"); + let mut doc = Document::open(std::sync::Arc::new(scanned_page())).unwrap(); + let find = |doc: &Document, kind: &[u8], key: &[u8]| { + doc.object_numbers().into_iter().map(|num| ObjRef { num, generation: doc.generation(num) }).find(|r| match &*doc.get(*r) { + Object::Stream(s) => s.dict.name(key) == Some(kind), + Object::Dict(d) => d.name(key) == Some(kind), + _ => false, + }) + }; + let scan = find(&doc, b"Image", b"Subtype").unwrap(); + let page = find(&doc, b"Page", b"Type").unwrap(); + let mut d = Dict::new(); + d.set(b"Type".to_vec(), Object::name("XObject")); + d.set(b"Subtype".to_vec(), Object::name("Image")); + d.set(b"Width".to_vec(), Object::Int(250)); + d.set(b"Height".to_vec(), Object::Int(100)); + d.set(b"BitsPerComponent".to_vec(), Object::Int(8)); + d.set(b"ColorSpace".to_vec(), Object::name("DeviceRGB")); + d.set(b"SMask".to_vec(), Object::Ref(scan)); + let picture = doc.add(Object::Stream(Stream::flate(d, &[0, 102, 204].repeat(250 * 100)))); + doc.update_dict(page, |p| { + let mut x = Dict::new(); + x.set(b"Scan".to_vec(), Object::Ref(picture)); + let mut res = Dict::new(); + res.set(b"XObject".to_vec(), Object::Dict(x)); + p.set(b"Resources".to_vec(), Object::Dict(res)); + }) + .unwrap(); + std::fs::write(dir.join("masked.pdf"), write_full(&doc, &SaveOptions::default()).unwrap()).unwrap(); + let mut a = auto(&dir); + let doc = ok(&mut a, "doc_open", json!({ "path": "masked.pdf" }))["doc"].as_u64().unwrap(); + let kept = ok(&mut a, "doc_optimize", json!({ "doc": doc, "path": "kept.pdf", "mono": { "compression": "retain" } })); + assert_eq!(kept["images_recompressed"].as_u64(), Some(0), "{kept}"); + let r = ok(&mut a, "doc_optimize", json!({ "doc": doc, "path": "small.pdf" })); + assert_eq!((r["images"].as_u64(), r["images_recompressed"].as_u64()), (Some(2), Some(1)), "the picture and its mask: {r}"); + // In the copy written, the picture's soft mask is Group 4, at its size. + let small = Document::open(std::sync::Arc::new(std::fs::read(dir.join("small.pdf")).unwrap())).unwrap(); + let picture = small + .object_numbers() + .into_iter() + .map(|num| small.get(ObjRef { num, generation: small.generation(num) })) + .find(|o| matches!(&**o, Object::Stream(s) if s.dict.contains(b"SMask"))) + .unwrap(); + let Object::Stream(picture) = &*picture else { unreachable!() }; + let Object::Stream(mask) = &*small.get(picture.dict.reference(b"SMask").unwrap()) else { panic!("the soft mask is a stream") }; + assert_eq!(mask.dict.name(b"Filter"), Some(&b"CCITTFaxDecode"[..])); + assert_eq!((mask.dict.int(b"Width"), mask.dict.int(b"Height")), (Some(1000), Some(400))); +} + /// A scan drawn at 600 ppi is averaged down to 300 ppi by default. `mono: { downsample: false }` /// keeps its size, `ppi` and `above_ppi` choose others, and values of the wrong type are refused. #[test] diff --git a/crates/optimize/README.md b/crates/optimize/README.md index 3207b69cab..a6dfd9caed 100644 --- a/crates/optimize/README.md +++ b/crates/optimize/README.md @@ -32,7 +32,8 @@ let report = optimize_with_progress(&mut doc, &settings, &mut |stage| keep_going damaged partway, which may draw an image larger past the damage), no image is resampled. Colour and grayscale images above a threshold are resampled with a bicubic (Catmull-Rom) filter to a target resolution and stored as JPEG (Acrobat's quality steps) or ZIP; soft masks are resampled with their image. A - stencil `/Mask` has its own resolution, so the picture changes under it and it stays as it is. + stencil `/Mask` has its own resolution, so the picture changes under it and it keeps its size and + its bits (stored again as a picture's mask, below). An image is replaced only if the result is smaller. Images over 80 million pixels are left alone by the image steps, here and below, whatever their settings (memory); Clean Up may still Flate an unfiltered or LZW one. Left alone by these colour and grey @@ -108,10 +109,18 @@ let report = optimize_with_progress(&mut doc, &settings, &mut |stage| keep_going - Apart from the downsampling it is lossless: the samples keep their meaning, so the colour space, decode array and `/Mask` stay, but for packed RGB, which becomes grey with one decode pair. Group 4 wins on scans, whose ragged edges defeat Flate; on clean computer-drawn 1-bit - art Flate can be smaller. Not yet: inline images, an image's own `/SMask` or `/Mask` - image (only images drawn on a page are visited), Group 4 for rows wider than 32,767 - samples, and JBIG2 past 65,535 samples on a side, where the decoder stops (its 2^28 in all - is past the 80 million pixels above). + art Flate can be smaller. + - **A picture's masks:** after the pictures, each picture's stencil `/Mask` and one-bit + `/SMask` (a picture's own images, which the pages don't draw by name), once however many + pictures name it, is stored again as the compression asks when that is smaller, at its size: + its bits stay as they are, so it masks as it did. A stencil the pages also draw by name is one + of the images above; a soft mask they draw is still stored as a mask. It isn't resampled, as its resolution isn't measured (the walk measures the + picture's), and it shares no JBIG2 dictionary, as only images named directly do. A soft mask of + more bits stays as it is, to be resampled with its picture, which wants it at eight. The report + counts masks as images. + - Not yet: inline images, a picture's mask resampled, or packed to one bit from more, Group 4 + for rows wider than 32,767 samples, and JBIG2 past 65,535 samples on a side, where the decoder + stops (its 2^28 in all is past the 80 million pixels above). - Lossless JBIG2 uses bounded generic-region encoding or exact-copy symbol dictionaries when smaller, a dictionary shared by up to 16 images when that is smaller still. No lossy symbol substitution is used. Export, redaction and this step read JBIG2 diff --git a/crates/optimize/src/images.rs b/crates/optimize/src/images.rs index 7bfa44dafd..24fba3ef7a 100644 --- a/crates/optimize/src/images.rs +++ b/crates/optimize/src/images.rs @@ -1364,9 +1364,9 @@ pub(crate) fn run( .collect(); let mut refs: Vec<(ObjRef, f64)> = ppi.into_iter().filter(|(r, _)| !masks.contains(r)).collect(); refs.sort_by_key(|(r, _)| (r.num, r.generation)); - let total = refs.len(); - let mut jbig2 = false; - let mut group: Vec = Vec::new(); + // Then the pictures' masks of one bit a sample, each once ([`decide_mask`]). + let mask_refs = picture_masks(doc, &refs); + let total = refs.len() + mask_refs.len(); // By number: a repaired object answers to two generations. let mut smask_uses: HashMap = HashMap::new(); for (r, _) in &refs { @@ -1374,23 +1374,96 @@ pub(crate) fn run( *smask_uses.entry(m.num).or_default() += 1; } } + let holds = settings.mono.compression == MonoCompression::Jbig2; + let threads = threads_for_items(doc, &refs, settings); + let mut out = Out { doc, report, budget, jbig2: false, group: Vec::new() }; + let pictures = Items { items: &refs, threads, first: 0, total }; + work_through( + &mut out, + &pictures, + &|doc, rest| batch_len(doc, rest, &smask_uses, holds), + &|doc, r, ppi, budget| decide(doc, r, ppi, defaults.contains(&r.num), settings, budget), + progress, + )?; + share(out.doc, &mut out.group, out.report); + let threads = threads_for_items(out.doc, &mask_refs, settings); + let masks = Items { items: &mask_refs, threads, first: refs.len(), total }; + // Each mask is an object of its own (by number), which only its own decision changes. + work_through( + &mut out, + &masks, + &|_, rest| rest.len().min(PICTURES_AT_ONCE), + &|doc, r, _, budget| decide_mask(doc, r, settings, budget), + progress, + )?; + if !progress(Stage::Images { done: total, total }) { + return Err(OptimizeError::Cancelled); + } + if out.jbig2 { + at_least_pdf_1_4(out.doc)?; + } + Ok(()) +} + +/// What the image pass changes and counts as it goes: the document, the report, the run's +/// allowance, whether it stored an image as JBIG2, and the JBIG2 images held to share a dictionary +/// ([`share`]). +struct Out<'a> { + doc: &'a mut Document, + report: &'a mut Report, + budget: &'a mut ImageBudget, + jbig2: bool, + group: Vec, +} + +/// Images the pass works through, each with its effective resolution, on up to `threads` threads; +/// for progress, the first of them is number `first` of the pass's `total`. +struct Items<'a> { + items: &'a [(ObjRef, f64)], + threads: usize, + first: usize, + total: usize, +} + +/// How many of `items` to work on at once: one below [`MIN_FOR_THREADS`], else as many as the +/// workers, and as [`PICTURE_MEMORY`] holds of the largest. +fn threads_for_items(doc: &Document, items: &[(ObjRef, f64)], settings: &Settings) -> usize { // Too few to be worth the threads, which keeps a small file's progress in step too. - let threads = if total < MIN_FOR_THREADS { - 1 - } else { - let largest = refs.iter().filter_map(|(r, _)| picture_pixels(doc, *r)).max().unwrap_or(0); - threads_for(crate::pool::workers(settings.workers), largest) - }; + if items.len() < MIN_FOR_THREADS { + return 1; + } + let largest = items.iter().filter_map(|(r, _)| picture_pixels(doc, *r)).max().unwrap_or(0); + threads_for(crate::pool::workers(settings.workers), largest) +} + +/// How [`work_through`] works out what's to become of an image, at its effective resolution, within +/// an allowance; threads share it. +type Decide<'a> = dyn Fn(&Document, ObjRef, f64, &mut ImageBudget) -> Decision + Sync + 'a; + +/// How many of the images left, from the first, [`work_through`] may work out beside each other. +type Batch<'a> = dyn Fn(&Document, &[(ObjRef, f64)]) -> usize + 'a; + +/// Works out what's to become of each of `items` by `decide`, and does it, in their order: a batch +/// of them beside each other where `batch` says they can be, one at a time otherwise. +fn work_through( + out: &mut Out<'_>, + items: &Items<'_>, + batch: &Batch<'_>, + decide: &Decide<'_>, + progress: &mut dyn FnMut(Stage) -> bool, +) -> Result<(), OptimizeError> { + let (threads, total) = (items.threads, items.total); let mut done = 0; - while let Some(rest) = refs.get(done..).filter(|rest| !rest.is_empty()) { - let run = if threads > 1 { batch_len(doc, rest, &smask_uses, settings.mono.compression == MonoCompression::Jbig2) } else { 0 }; + while let Some(rest) = items.items.get(done..).filter(|rest| !rest.is_empty()) { + let at = items.first.saturating_add(done); + let run = if threads > 1 { batch(out.doc, rest) } else { 0 }; let Some(batch) = rest.get(..run).filter(|batch| batch.len() > 1) else { - if !progress(Stage::Images { done, total }) { + if !progress(Stage::Images { done: at, total }) { return Err(OptimizeError::Cancelled); } if let Some((r, ppi)) = rest.first() { - let decision = decide(doc, *r, *ppi, defaults.contains(&r.num), settings, budget); - apply(doc, decision, report, &mut jbig2, &mut group); + let decision = decide(out.doc, *r, *ppi, out.budget); + apply(out.doc, decision, out.report, &mut out.jbig2, &mut out.group); } done += 1; continue; @@ -1399,10 +1472,10 @@ pub(crate) fn run( // order below: a picture the order can't pay for is done again as if there were no threads, // so the same pictures are refused with any number of them. Once more than what's left has // been used, no more are started. - let allowed = budget.work_left(); + let allowed = out.budget.work_left(); let used = AtomicU64::new(0); let tried = { - let (doc, defaults) = (&*doc, &defaults); + let doc = &*out.doc; crate::pool::map( batch, threads, @@ -1411,36 +1484,74 @@ pub(crate) fn run( return None; } let mut own = ImageBudget::with_work(allowed); - let decision = decide(doc, *r, *ppi, defaults.contains(&r.num), settings, &mut own); + let decision = decide(doc, *r, *ppi, &mut own); let cost = allowed.saturating_sub(own.work_left()); used.fetch_add(cost, Ordering::Relaxed); (own.refused() == 0).then_some((decision, cost)) }, - &mut |finished| progress(Stage::Images { done: done.saturating_add(finished), total }), + &mut |finished| progress(Stage::Images { done: at.saturating_add(finished), total }), )? }; for ((r, ppi), tried) in batch.iter().zip(tried) { let decision = match tried.flatten() { - Some((decision, cost)) if cost <= budget.work_left() => { - budget.charge(cost); + Some((decision, cost)) if cost <= out.budget.work_left() => { + out.budget.charge(cost); decision } - _ => decide(doc, *r, *ppi, defaults.contains(&r.num), settings, budget), + _ => decide(out.doc, *r, *ppi, out.budget), }; - apply(doc, decision, report, &mut jbig2, &mut group); + apply(out.doc, decision, out.report, &mut out.jbig2, &mut out.group); } done += batch.len(); } - share(doc, &mut group, report); - if !progress(Stage::Images { done: total, total }) { - return Err(OptimizeError::Cancelled); - } - if jbig2 { - at_least_pdf_1_4(doc)?; - } Ok(()) } +/// The masks of `pictures` the pass stores again after them, each once (by number): their soft +/// masks and their stencil masks, unless one is drawn as a picture of its own, and so is one of +/// them. A colour-key `/Mask` is an array, not an image. Their resolution isn't measured. +fn picture_masks(doc: &Document, pictures: &[(ObjRef, f64)]) -> Vec<(ObjRef, f64)> { + let drawn: HashSet = pictures.iter().map(|(r, _)| r.num).collect(); + let mut seen = HashSet::new(); + let mut masks = Vec::new(); + for (r, _) in pictures { + let object = doc.get(*r); + let Object::Stream(s) = &*object else { continue }; + for key in [&b"SMask"[..], b"Mask"] { + if let Some(m) = s.dict.reference(key) + && !drawn.contains(&m.num) + && seen.insert(m.num) + { + masks.push((m, f64::NAN)); + } + } + } + masks.sort_by_key(|(r, _)| (r.num, r.generation)); + masks +} + +/// Work out what's to become of mask `r` of a picture, changing nothing yet: stored again as the +/// monochrome settings ask ([`mono`]), at its size, when it has one bit a sample (a stencil mask, +/// or a soft mask of one bit) and that is smaller. The bits stay as they are, so it masks as it did. +/// It isn't resampled: its resolution isn't measured, and its edges are the picture's. A soft mask +/// of more bits is left to be resampled with its picture ([`process`]), which wants it at eight. It +/// shares no JBIG2 dictionary: those are for images named directly, which redaction can store again. +fn decide_mask(doc: &Document, r: ObjRef, settings: &Settings, budget: &mut ImageBudget) -> Decision { + let obj = doc.get(r); + let Object::Stream(s) = &*obj else { return Decision::NotAnImage }; + let stencil = matches!(s.dict.get(b"ImageMask"), Some(Object::Bool(true))); + if !stencil && s.dict.int(b"BitsPerComponent") != Some(1) { + return Decision::NotAnImage; + } + let refused = budget.refused(); + // Its resolution unmeasured, it isn't resampled. As where a default colour space is in force: + // at one bit a sample that changes nothing, and nothing is packed. + match mono(doc, s, f64::NAN, true, &settings.mono, budget) { + Some((new, _, _)) => Decision::Replace { image: r, before: s.raw.len(), new, resampled: false, smask: None, bits: None }, + None => Decision::Stays { refused: budget.refused() > refused }, + } +} + /// A picture's soft mask, if it names one. fn smask_of(doc: &Document, r: ObjRef) -> Option { match &*doc.get(r) { @@ -1511,7 +1622,7 @@ fn threads_for(workers: usize, largest: u64) -> usize { /// What's to become of one picture. enum Decision { - /// Not an image stream after all. + /// Not an image stream after all, or a picture's mask the pass doesn't store again. NotAnImage, /// An image that stays as it is, `refused` if the allowance couldn't pay for it. Stays { refused: bool }, @@ -1994,6 +2105,86 @@ mod dimension_tests { } } + /// A picture's mask past the run's allowance stays as it is, counted as refused, as an image + /// is; one the allowance paid for and that stays (its noise is no smaller as Group 4) isn't. The + /// progress counts the masks with the pictures. + #[test] + fn a_mask_past_the_runs_allowance_stays_and_is_counted() { + let mut seed = 0x2545_F491_4F6C_DD1Du64; + let noise: Vec = (0..32 * 256) + .map(|_| { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + seed as u8 + }) + .collect(); + let page_of = |pictures: usize, masked: bool| { + let mut doc = Document::new_empty(); + let mut xobjects = Dict::new(); + let mut drawn = String::from("q 10 0 0 10 0 0 cm "); + let mut masks = Vec::new(); + for i in 0..pictures { + let mut dict = Dict::new(); + dict.set(b"Subtype".to_vec(), Object::name("Image")); + dict.set(b"Width".to_vec(), Object::Int(16)); + dict.set(b"Height".to_vec(), Object::Int(16)); + dict.set(b"BitsPerComponent".to_vec(), Object::Int(8)); + dict.set(b"ColorSpace".to_vec(), Object::name("DeviceCMYK")); + if masked { + let mut mask = Dict::new(); + mask.set(b"Subtype".to_vec(), Object::name("Image")); + mask.set(b"ImageMask".to_vec(), Object::Bool(true)); + mask.set(b"Width".to_vec(), Object::Int(256)); + mask.set(b"Height".to_vec(), Object::Int(256)); + let mask = doc.add(Object::Stream(Stream::flate(mask, &noise))); + dict.set(b"Mask".to_vec(), Object::Ref(mask)); + masks.push(mask); + } + let picture = doc.add(Object::Stream(Stream::flate(dict, &[0; 16 * 16 * 4]))); + xobjects.set(format!("I{i}").into_bytes(), Object::Ref(picture)); + drawn.push_str(&format!("/I{i} Do ")); + } + drawn.push('Q'); + let mut resources = Dict::new(); + resources.set(b"XObject".to_vec(), Object::Dict(xobjects)); + let content = doc.add(Object::Stream(Stream::flate(Dict::new(), drawn.as_bytes()))); + let mut page = Dict::new(); + page.set(b"Type".to_vec(), Object::name("Page")); + page.set(b"Resources".to_vec(), Object::Dict(resources)); + page.set(b"Contents".to_vec(), Object::Ref(content)); + let page = doc.add(Object::Dict(page)); + (doc, page, masks) + }; + let keep = crate::ImageSettings { downsample: false, compression: crate::Compression::Retain, ..crate::ImageSettings::REDUCE }; + let settings = Settings { color: keep, gray: keep, ..Settings::default() }; + let cost = |pictures: usize, masked: bool| { + let (mut doc, page, _) = page_of(pictures, masked); + let mut budget = ImageBudget::with_work(u64::MAX); + let mut report = Report::default(); + run(&mut doc, &[page], &settings, &mut report, &mut |_| true, &mut budget).unwrap(); + assert_eq!(report.images_refused, 0); + u64::MAX - budget.work_left() + }; + let mask = cost(1, true) - cost(1, false); + assert!(mask > 0); + let (mut doc, page, masks) = page_of(3, true); + let before: Vec<_> = masks.iter().map(|m| doc.get(*m)).collect(); + let mut budget = ImageBudget::with_work(cost(3, false) + 2 * mask); + let mut report = Report::default(); + let mut last = None; + let mut progress = |stage: Stage| { + if let Stage::Images { done, total } = stage { + last = Some((done, total)); + } + true + }; + run(&mut doc, &[page], &settings, &mut report, &mut progress, &mut budget).unwrap(); + assert_eq!((report.images, report.images_refused), (6, 1), "{report:?}"); + assert_eq!(last, Some((6, 6)), "three pictures and their three masks"); + assert!(masks.iter().zip(&before).all(|(m, b)| doc.get(*m) == *b), "every mask stays as it was"); + } + /// An image the allowance ran out on between being packed and being resampled is rewritten, /// as the packed one, so it isn't one that stayed as it was. #[test] diff --git a/crates/optimize/src/lib.rs b/crates/optimize/src/lib.rs index 97296844a2..0f80698de4 100644 --- a/crates/optimize/src/lib.rs +++ b/crates/optimize/src/lib.rs @@ -15,7 +15,8 @@ //! that use only palette entries 0 and 1, packed to one bit) above a threshold are averaged down to a //! target resolution when grey, image masks or indexed, halftones and dithers aside (see the README), and //! are stored as CCITT Group 4 or lossless JBIG2, or 1-bit -//! Flate once resampled or packed, when that is smaller. +//! Flate once resampled or packed, when that is smaller. So are pictures' stencil and one-bit soft +//! masks, at their size. //! - **Discard objects:** page thumbnails, alternate images, document tags (structure tree), //! print settings, form submission, import and reset actions. //! - **Clean up:** drop images and forms that pages list but never draw (such as a deleted @@ -154,7 +155,8 @@ pub enum MonoCompression { /// How black-and-white images (1-bit, CCITT fax among them; and eight-bit grey and RGB of only black /// and white, and indexed images that use only palette entries 0 and 1, packed to one bit), and -/// image masks, are resampled and stored. +/// image masks, are resampled and stored. Pictures' stencil and one-bit soft masks are stored the +/// same way, at their size. #[derive(Clone, Copy, Debug, PartialEq)] pub struct MonoSettings { /// Average images whose effective resolution is above `above_ppi` down to `target_ppi`. @@ -201,7 +203,7 @@ pub struct Settings { pub merge_fragments: bool, pub color: ImageSettings, pub gray: ImageSettings, - /// Black-and-white images, and image masks. + /// Black-and-white images, image masks, and pictures' one-bit masks. pub mono: MonoSettings, pub discard_thumbnails: bool, pub discard_alternate_images: bool, @@ -309,6 +311,7 @@ pub struct Report { /// Images merged from fragments, and how many fragments went into them. pub merged_images: usize, pub fragments_merged: usize, + /// Images the image step read: those the pages draw, and their pictures' one-bit masks. pub images: usize, pub images_resampled: usize, pub images_recompressed: usize, diff --git a/crates/optimize/tests/optimize.rs b/crates/optimize/tests/optimize.rs index 7ab25262f8..7c3e76f9b9 100644 --- a/crates/optimize/tests/optimize.rs +++ b/crates/optimize/tests/optimize.rs @@ -5363,7 +5363,8 @@ fn stencil(doc: &mut Document, w: u32, h: u32) -> ObjRef { } /// A picture with a stencil mask is made smaller like any other: the mask has a resolution of its -/// own, so it stays exactly as it was and the hole it cuts stays where it was. +/// own, so it keeps its size and its bits (stored again as Group 4, as a picture's mask is) and the +/// hole it cuts stays where it was. #[test] fn a_picture_with_a_stencil_mask_is_downsampled_under_its_mask() { let mut doc = Document::new_empty(); @@ -5381,7 +5382,9 @@ fn a_picture_with_a_stencil_mask_is_downsampled_under_its_mask() { let after = stream(&doc, img); assert_eq!(after.dict.int(b"Width"), Some(300), "150 ppi"); assert_eq!(after.dict.reference(b"Mask"), Some(mask), "the mask stays on it"); - assert_eq!(stream(&doc, mask), mask_before, "and doesn't change"); + let mask_after = stream(&doc, mask); + assert_eq!((mask_after.dict.int(b"Width"), mask_after.dict.int(b"Height")), (Some(1200), Some(1200)), "the mask isn't resampled"); + assert_eq!(decoded_image(&doc, &mask_after).unwrap(), decoded_image(&doc, &mask_before).unwrap(), "and keeps its bits"); let rendered = render(&doc); assert_eq!(rendered.len(), before.len()); let changed = before.iter().zip(&rendered).filter(|(a, b)| a.abs_diff(**b) > 96).count(); @@ -5396,6 +5399,103 @@ fn a_picture_with_a_stencil_mask_is_downsampled_under_its_mask() { } } +/// A picture's masks of one bit a sample, which the page doesn't draw by name, are stored again as +/// the monochrome settings ask, losslessly: a stencil `/Mask` (with a decode array that inverts it) +/// and a one-bit `/SMask` become Group 4, keep their size and bits, and stay the pictures' masks; +/// the page looks the same to the pixel. A mask two pictures share is stored once, and a stencil +/// the page also draws by name is stored as that image. An eight-bit soft mask of only black and +/// white stays as it is, to be resampled with its picture. With `mono: Retain` nothing changes. +#[test] +fn a_pictures_one_bit_masks_are_stored_again() { + let mut doc = Document::new_empty(); + let (w, h) = (1200, 480); + let text = bilevel(w, h, false); + let decode = Object::Array(vec![1.into(), 0.into()]); + let stencil_mask = mono_image(&mut doc, w, h, &text, None, &[("ImageMask", Object::Bool(true)), ("Decode", decode.clone())]); + let one_bit = mono_image(&mut doc, w, h, &text, None, &[("BitsPerComponent", 1.into()), ("ColorSpace", Object::name("DeviceGray"))]); + let wide: Vec = text.iter().flat_map(|b| (0..8).rev().map(move |i| if b >> i & 1 == 1 { 255 } else { 0 })).collect(); + let eight_bit = mono_image(&mut doc, w, h, &wide, None, &[("BitsPerComponent", 8.into()), ("ColorSpace", Object::name("DeviceGray"))]); + let drawn = mono_image(&mut doc, w, h, &text, None, &[("ImageMask", Object::Bool(true))]); + let masked = |doc: &mut Document, key: &str, mask: ObjRef| { + let picture = image(doc, 200, 80, 3, false, None); + let mut s = stream(doc, picture); + s.dict.set(key.as_bytes().to_vec(), Object::Ref(mask)); + doc.set(picture, Object::Stream(s)); + picture + }; + let a = masked(&mut doc, "Mask", stencil_mask); + let b = masked(&mut doc, "Mask", stencil_mask); + let c = masked(&mut doc, "SMask", one_bit); + let d = masked(&mut doc, "SMask", eight_bit); + let e = masked(&mut doc, "Mask", drawn); + page( + &mut doc, + &[("A", a), ("B", b), ("C", c), ("D", d), ("E", e), ("M", drawn)], + "q 300 0 0 120 20 660 cm /A Do Q q 300 0 0 120 320 660 cm /B Do Q q 300 0 0 120 20 520 cm /C Do Q \ + q 300 0 0 120 320 520 cm /D Do Q q 300 0 0 120 20 380 cm /E Do Q q 0 0.4 0.8 rg 300 0 0 120 320 380 cm /M Do Q", + ); + // Pictures in colour and grey are left as they are, so only the masks can change. + let keep = ImageSettings { downsample: false, compression: Compression::Retain, ..ImageSettings::REDUCE }; + let settings = Settings { color: keep, gray: keep, ..Settings::default() }; + let before = render(&doc); + let masks = [stencil_mask, one_bit, eight_bit, drawn].map(|r| stream(&doc, r)); + + let mut kept = doc.clone(); + let r = optimize(&mut kept, &Settings { mono: RETAIN, ..settings.clone() }).unwrap(); + assert_eq!(r.images_recompressed, 0, "Retain keeps every mask as it is"); + assert_eq!([stencil_mask, one_bit, eight_bit, drawn].map(|r| stream(&kept, r)), masks); + + let r = optimize(&mut doc, &settings).unwrap(); + assert_eq!(r.images_recompressed, 3, "the shared stencil once, the soft mask, the stencil drawn by name once: {r:?}"); + for (mask, old) in [(stencil_mask, &masks[0]), (one_bit, &masks[1]), (drawn, &masks[3])] { + let new = stream(&doc, mask); + assert_eq!(new.dict.name(b"Filter"), Some(&b"CCITTFaxDecode"[..]), "{mask:?}"); + assert_eq!((new.dict.int(b"Width"), new.dict.int(b"Height")), (Some(w as i64), Some(h as i64))); + assert_eq!(decoded_image(&doc, &new).unwrap(), decoded_image(&doc, old).unwrap(), "{mask:?} keeps its bits"); + } + assert_eq!(stream(&doc, stencil_mask).dict.get(b"Decode"), Some(&decode), "the decode array stays"); + assert_eq!(stream(&doc, eight_bit), masks[2], "an eight-bit soft mask stays"); + for (picture, key, mask) in + [(a, "Mask", stencil_mask), (b, "Mask", stencil_mask), (c, "SMask", one_bit), (d, "SMask", eight_bit), (e, "Mask", drawn)] + { + assert_eq!(stream(&doc, picture).dict.reference(key.as_bytes()), Some(mask), "{picture:?} keeps its mask"); + } + assert!(render(&doc) == before, "the page looks the same"); +} + +/// Stored as JBIG2, a picture's mask is coded on its own: it shares no symbol dictionary with the +/// images the page draws by name, which do share one. +#[test] +fn a_pictures_mask_stored_as_jbig2_shares_no_dictionary() { + let mut doc = Document::new_empty(); + let (w, h) = (1200, 480); + let text = bilevel(w, h, false); + let mask = mono_image(&mut doc, w, h, &text, None, &[("ImageMask", Object::Bool(true))]); + let picture = image(&mut doc, 200, 80, 3, false, None); + let mut s = stream(&doc, picture); + s.dict.set(b"Mask".to_vec(), Object::Ref(mask)); + doc.set(picture, Object::Stream(s)); + let named: Vec = (0..3) + .map(|_| mono_image(&mut doc, w, h, &text, None, &[("BitsPerComponent", 1.into()), ("ColorSpace", Object::name("DeviceGray"))])) + .collect(); + page( + &mut doc, + &[("P", picture), ("G0", named[0]), ("G1", named[1]), ("G2", named[2])], + "q 300 0 0 120 20 660 cm /P Do Q q 300 0 0 120 320 660 cm /G0 Do Q q 300 0 0 120 20 520 cm /G1 Do Q q 300 0 0 120 320 520 cm /G2 Do Q", + ); + let keep = ImageSettings { downsample: false, compression: Compression::Retain, ..ImageSettings::REDUCE }; + let jbig2 = MonoSettings { compression: MonoCompression::Jbig2, ..MonoSettings::REDUCE }; + let before = render(&doc); + let old = stream(&doc, mask); + optimize(&mut doc, &Settings { color: keep, gray: keep, mono: jbig2, ..Settings::default() }).unwrap(); + let new = stream(&doc, mask); + assert_eq!(new.dict.name(b"Filter"), Some(&b"JBIG2Decode"[..])); + assert!(!new.dict.contains(b"DecodeParms"), "no /JBIG2Globals"); + assert_eq!(decoded_image(&doc, &new).unwrap(), decoded_image(&doc, &old).unwrap()); + assert!(named.iter().any(|r| stream(&doc, *r).dict.get(b"DecodeParms").is_some()), "the images drawn by name share one"); + assert!(render(&doc) == before, "the page looks the same"); +} + /// A colour-key mask names sample values: resampling or JPEG would make holes where there were none /// and fill those there were, so the picture stays as it is, however the array is written. A mask /// that can't be read as a stencil is no reason to change the picture either. diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index ae3261876d..046c080c62 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -7549,8 +7549,8 @@ milestone = "M11" status = "partial" commands = ["optimize.advanced", "optimize.reduce"] tools = ["doc_optimize", "doc_reduce"] -evidence = ["crates/optimize/src/images.rs::flate_images_past_the_runs_allowance_stay_as_they_are", "crates/optimize/tests/optimize.rs::eight_bit_black_and_white_images_become_one_bit", "crates/filters/src/ccitt.rs::bitmaps_round_trip", "crates/filters/src/ccitt.rs::a_page_of_text_like_marks_round_trips_and_shrinks", "crates/filters/src/ccitt.rs::what_it_cannot_encode_is_an_error", "crates/optimize/tests/optimize.rs::black_and_white_images_become_ccitt_group_4", "crates/optimize/tests/optimize.rs::group_3_fax_images_become_group_4", "crates/optimize/tests/optimize.rs::rgb_and_two_colour_indexed_black_and_white_images_become_one_bit", "crates/optimize/tests/optimize.rs::black_and_white_rgb_used_beyond_the_page_content_read_stays_rgb", "crates/optimize/tests/optimize.rs::black_and_white_rgb_drawn_only_where_the_optimizer_reads_becomes_grey", "crates/optimize/tests/optimize.rs::black_and_white_rgb_in_resources_its_page_tree_and_a_form_share_becomes_grey", "crates/automation/tests/it/automation.rs::doc_optimize_stores_black_and_white_images_as_ccitt_group_4", "crates/ui-egui/tests/it/create.rs::the_pdf_optimizer_dialog_saves_an_optimized_copy", "crates/filters/src/mq.rs::the_annex_h_test_sequence_encodes_to_its_bytes", "crates/filters/src/jbig2.rs::bitmaps_round_trip", "crates/filters/src/jbig2.rs::a_page_of_text_like_marks_round_trips_and_shrinks", "crates/optimize/tests/optimize.rs::black_and_white_images_become_jbig2", "crates/automation/tests/it/automation.rs::doc_optimize_stores_black_and_white_images_as_jbig2", "crates/filters/src/jbig2/symbols.rs::repeated_marks_are_stored_once", "crates/filters/src/jbig2/symbols.rs::scattered_marks_round_trip", "crates/optimize/tests/optimize.rs::rendered_text_becomes_jbig2_symbols", "crates/optimize/tests/optimize.rs::jbig2_scans_are_averaged_and_stored_again", "crates/optimize/src/images.rs::jbig2_images_past_the_runs_allowance_stay_as_they_are", "crates/optimize/tests/optimize.rs::an_image_too_large_for_the_jbig2_decoder_stays_as_it_is", "crates/optimize/tests/optimize.rs::what_is_too_large_for_jbig2_is_stored_another_way", "crates/optimize/tests/optimize.rs::group_4_takes_images_of_one_bit_a_pixel_up_to_the_pixel_limit", "crates/filters/src/jbig2/symbols.rs::pages_in_one_font_share_one_dictionary", "crates/filters/src/jbig2/symbols.rs::what_a_page_has_alone_stays_its_own", "crates/filters/src/jbig2/symbols.rs::past_the_shared_limit_the_most_widely_held_marks_are_shared", "crates/optimize/tests/optimize.rs::jbig2_pages_share_one_symbol_dictionary", "crates/optimize/tests/optimize.rs::a_dictionary_that_saves_less_than_it_takes_isnt_shared", "crates/optimize/tests/optimize.rs::sixteen_images_at_most_share_a_dictionary", "crates/engine/src/tests.rs::redacting_an_optimized_jbig2_page_stores_the_rest_of_its_group_again", "crates/optimize/src/images.rs::only_what_redaction_can_store_again_shares_a_dictionary"] -notes = "All of a run's images share one allowance of decoding, filters, decoders and pixels alike (about 1,000 scanned pages' worth, more for a longer file); those past it stay as they are, those the image pass read being counted in the report, which doc_optimize and doc_reduce return and the panel doesn't show yet. CCITT Group 4, lossless: PDF Optimizer ▸ Images ▸ Monochrome Images (on by default, so Reduce File Size does it too; doc_optimize mono) stores a 1-bit image of one colour component (grey, indexed, a separation) or an image mask, kept with no filter or general-purpose ones, as Group 4 when that is smaller as stored; the samples decode to the same bits, so colour space, decode array and masks stay. Packed to one bit first, then stored the same way (colour-key masks aside): eight-bit grey of only black and white, eight-bit DeviceRGB of only black and white (as DeviceGray; calibrated RGB, RGB with a profile, RGB drawn or listed where /DefaultGray or /DefaultRGB is in force or referred to from anywhere but the page and form resources read (an annotation's appearance or a pattern drawing it, directly or through resources or a form shared with a page; another image's /Alternates), and RGB whose soft mask has a /Matte aside), and an indexed image of 2, 4 or 8 bits that uses only palette entries 0 and 1 (keeping its palette), unless the grey or colour settings make it smaller still, as they can a dithered scan downsampled to JPEG, its soft mask counted. Group 4 wins on scans; on clean computer-drawn 1-bit art Flate can be smaller, and the image then stays. Group 3 fax images are stored again as Group 4 when that is smaller (damaged Group 3 with end-of-line codes as the decoder recovers it, as it reports no damage there; byte-aligned Group 3 with end-of-line codes but none before the first row, not as T.4 sends them, as PdfCraft draws it, which at /K 0 may be other rows than were coded). CCITT fax images are decoded to be downsampled (optimize.downsample-images), and a resampled image is stored as Group 4 or 1-bit Flate, whichever is smaller. Lossless JBIG2 with bounded exact-copy symbol coding is available too, with image export, up to 16 images sharing one symbol dictionary (/JBIG2Globals) where that is smaller still (image masks and DeviceGray images named directly, of at most 64 Mi pixels, which redaction can store again without it); lossy JBIG2 remains unsupported. JBIG2 images are read too: averaged like any (under Retain stored as JBIG2 again), and, not resampled, stored again as the compression asks when that is smaller (as JBIG2 symbols where its own coding is a generic region, say); a symbol-coded scan, averaged, loses its exact repeats and is often larger, and then stays as it is; one whose JBIG2 page isn't its size is left alone. Not yet: images over 80 million pixels (left alone by the image steps, whatever their settings; Clean Up may still Flate an unfiltered or LZW one), inline images, soft masks and /Mask images, Group 4 for rows wider than 32,767 samples, JBIG2 past 65,535 samples on a side, where the decoder stops: such an image isn't stored as JBIG2 (the image step leaves a 1-bit one as it is unless it is averaged, and may store one packed from more bits as 1-bit Flate)." +evidence = ["crates/optimize/src/images.rs::flate_images_past_the_runs_allowance_stay_as_they_are", "crates/optimize/tests/optimize.rs::eight_bit_black_and_white_images_become_one_bit", "crates/filters/src/ccitt.rs::bitmaps_round_trip", "crates/filters/src/ccitt.rs::a_page_of_text_like_marks_round_trips_and_shrinks", "crates/filters/src/ccitt.rs::what_it_cannot_encode_is_an_error", "crates/optimize/tests/optimize.rs::black_and_white_images_become_ccitt_group_4", "crates/optimize/tests/optimize.rs::group_3_fax_images_become_group_4", "crates/optimize/tests/optimize.rs::rgb_and_two_colour_indexed_black_and_white_images_become_one_bit", "crates/optimize/tests/optimize.rs::black_and_white_rgb_used_beyond_the_page_content_read_stays_rgb", "crates/optimize/tests/optimize.rs::black_and_white_rgb_drawn_only_where_the_optimizer_reads_becomes_grey", "crates/optimize/tests/optimize.rs::black_and_white_rgb_in_resources_its_page_tree_and_a_form_share_becomes_grey", "crates/automation/tests/it/automation.rs::doc_optimize_stores_black_and_white_images_as_ccitt_group_4", "crates/ui-egui/tests/it/create.rs::the_pdf_optimizer_dialog_saves_an_optimized_copy", "crates/filters/src/mq.rs::the_annex_h_test_sequence_encodes_to_its_bytes", "crates/filters/src/jbig2.rs::bitmaps_round_trip", "crates/filters/src/jbig2.rs::a_page_of_text_like_marks_round_trips_and_shrinks", "crates/optimize/tests/optimize.rs::black_and_white_images_become_jbig2", "crates/automation/tests/it/automation.rs::doc_optimize_stores_black_and_white_images_as_jbig2", "crates/filters/src/jbig2/symbols.rs::repeated_marks_are_stored_once", "crates/filters/src/jbig2/symbols.rs::scattered_marks_round_trip", "crates/optimize/tests/optimize.rs::rendered_text_becomes_jbig2_symbols", "crates/optimize/tests/optimize.rs::jbig2_scans_are_averaged_and_stored_again", "crates/optimize/src/images.rs::jbig2_images_past_the_runs_allowance_stay_as_they_are", "crates/optimize/tests/optimize.rs::an_image_too_large_for_the_jbig2_decoder_stays_as_it_is", "crates/optimize/tests/optimize.rs::what_is_too_large_for_jbig2_is_stored_another_way", "crates/optimize/tests/optimize.rs::group_4_takes_images_of_one_bit_a_pixel_up_to_the_pixel_limit", "crates/filters/src/jbig2/symbols.rs::pages_in_one_font_share_one_dictionary", "crates/filters/src/jbig2/symbols.rs::what_a_page_has_alone_stays_its_own", "crates/filters/src/jbig2/symbols.rs::past_the_shared_limit_the_most_widely_held_marks_are_shared", "crates/optimize/tests/optimize.rs::jbig2_pages_share_one_symbol_dictionary", "crates/optimize/tests/optimize.rs::a_dictionary_that_saves_less_than_it_takes_isnt_shared", "crates/optimize/tests/optimize.rs::sixteen_images_at_most_share_a_dictionary", "crates/engine/src/tests.rs::redacting_an_optimized_jbig2_page_stores_the_rest_of_its_group_again", "crates/optimize/src/images.rs::only_what_redaction_can_store_again_shares_a_dictionary", "crates/optimize/tests/optimize.rs::a_pictures_one_bit_masks_are_stored_again", "crates/optimize/tests/optimize.rs::a_pictures_mask_stored_as_jbig2_shares_no_dictionary", "crates/automation/tests/it/automation.rs::doc_optimize_stores_a_pictures_one_bit_soft_mask_as_ccitt_group_4"] +notes = "All of a run's images share one allowance of decoding, filters, decoders and pixels alike (about 1,000 scanned pages' worth, more for a longer file); those past it stay as they are, those the image pass read being counted in the report, which doc_optimize and doc_reduce return and the panel doesn't show yet. CCITT Group 4, lossless: PDF Optimizer ▸ Images ▸ Monochrome Images (on by default, so Reduce File Size does it too; doc_optimize mono) stores a 1-bit image of one colour component (grey, indexed, a separation) or an image mask, kept with no filter or general-purpose ones, as Group 4 when that is smaller as stored; the samples decode to the same bits, so colour space, decode array and masks stay. Packed to one bit first, then stored the same way (colour-key masks aside): eight-bit grey of only black and white, eight-bit DeviceRGB of only black and white (as DeviceGray; calibrated RGB, RGB with a profile, RGB drawn or listed where /DefaultGray or /DefaultRGB is in force or referred to from anywhere but the page and form resources read (an annotation's appearance or a pattern drawing it, directly or through resources or a form shared with a page; another image's /Alternates), and RGB whose soft mask has a /Matte aside), and an indexed image of 2, 4 or 8 bits that uses only palette entries 0 and 1 (keeping its palette), unless the grey or colour settings make it smaller still, as they can a dithered scan downsampled to JPEG, its soft mask counted. Group 4 wins on scans; on clean computer-drawn 1-bit art Flate can be smaller, and the image then stays. Group 3 fax images are stored again as Group 4 when that is smaller (damaged Group 3 with end-of-line codes as the decoder recovers it, as it reports no damage there; byte-aligned Group 3 with end-of-line codes but none before the first row, not as T.4 sends them, as PdfCraft draws it, which at /K 0 may be other rows than were coded). CCITT fax images are decoded to be downsampled (optimize.downsample-images), and a resampled image is stored as Group 4 or 1-bit Flate, whichever is smaller. Lossless JBIG2 with bounded exact-copy symbol coding is available too, with image export, up to 16 images sharing one symbol dictionary (/JBIG2Globals) where that is smaller still (image masks and DeviceGray images named directly, of at most 64 Mi pixels, which redaction can store again without it); lossy JBIG2 remains unsupported. JBIG2 images are read too: averaged like any (under Retain stored as JBIG2 again), and, not resampled, stored again as the compression asks when that is smaller (as JBIG2 symbols where its own coding is a generic region, say); a symbol-coded scan, averaged, loses its exact repeats and is often larger, and then stays as it is; one whose JBIG2 page isn't its size is left alone. A picture's one-bit /SMask, and its stencil /Mask unless pages also draw it by name (it is then one of the images), are stored again the same way after the pictures, each once, at their size and bits, sharing no symbol dictionary, and counted as images. Not yet: images over 80 million pixels (left alone by the image steps, whatever their settings; Clean Up may still Flate an unfiltered or LZW one), inline images, a picture's mask resampled, or a soft mask of more bits packed to one bit (it stays, to be resampled with its picture), Group 4 for rows wider than 32,767 samples, JBIG2 past 65,535 samples on a side, where the decoder stops: such an image isn't stored as JBIG2 (the image step leaves a 1-bit one as it is unless it is averaged, and may store one packed from more bits as 1-bit Flate)." [[feature]] id = "optimize.only-if-smaller" From 506502386e009e0721a14a9149852326e91ece49 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Tue, 10 Nov 2026 13:59:52 +0100 Subject: [PATCH 05/60] M8: the Type 3 cleanup charges the page only when there is a Type 3 font CodeRabbit found this on #6805, which landed #6341 on main. `prune_type3_from_resources` reads what the content draws through its resources (`resource_ops`) whenever the resources list any font, and only then looks for a Type 3 font, the only kind it prunes. Since #6341 that read is charged to what the content walk left of the page's allowance. So a page with only TrueType fonts and a large form without resources of its own could be refused, by Apply Redactions or Remove Hidden Information, for a read that prunes nothing: a 100 MiB form crossing the mark is read by the walk, the page's check and this read, 300 MiB against the page's 256. Without a Type 3 font, the read is now charged to an allowance of a page of its own; with one, to the page's, as before. The read is still made with any font listed, as CodeRabbit's suggested early return would not: it refuses bytes the parser can't read in the streams and forms it reads, and an early return would drop that refusal on every page without a Type 3 font. The walk reads past such bytes, so where nothing else reads the same content, text a stray `)` hides in a `TJ` array under the mark would stay while Apply Redactions and Remove Hidden Information reported success (found in review). The refusal covers only what the cleanups read, so such bytes still go by elsewhere, here as on main. The read and the pruning tell a Type 3 font the same way (`is_type3`). Tests: - `the_cleanups_after_the_content_walk_spend_what_it_left_of_the_pages_allowance` now also redacts the 100 MiB page with no Type 3 font, which goes on; it and the two tests like it (a rewritten form's cleanups, Remove Hidden Information) list an unused Type 3 font where their font cleanup runs, so that it still charges the page's allowance (`list_type3`). - `text_beside_bytes_the_parser_cannot_read_refuses_the_redaction`: four such `TJ` arrays in a page's second stream, one in a form without resources, and hidden text beside one, on pages listing only a TrueType font, are refused by Apply Redactions and Remove Hidden Information. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/redact/src/tests.rs | 96 ++++++++++++++++++++++++++++++++++---- crates/redact/src/type3.rs | 35 +++++++++----- 2 files changed, 109 insertions(+), 22 deletions(-) diff --git a/crates/redact/src/tests.rs b/crates/redact/src/tests.rs index 46ac131450..465294dc9a 100644 --- a/crates/redact/src/tests.rs +++ b/crates/redact/src/tests.rs @@ -3571,14 +3571,43 @@ fn what_content_draws_through_its_resources_is_read_within_its_allowance() { } } +/// An unused Type 3 font, listed as /T3 beside /F1 in the `/Font` of `holder`'s resources (a page's +/// or a form's), so that the font cleanup, which prunes only Type 3 fonts, charges its read of what +/// they draw to the page's allowance (without one it has an allowance of its own). +fn list_type3(doc: &mut Document, holder: ObjRef) { + let font = Object::Dict(Dict::from_iter([ + (b"Type".to_vec(), Object::name("Font")), + (b"Subtype".to_vec(), Object::name("Type3")), + (b"FontBBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 1000.into(), 1000.into()])), + (b"FontMatrix".to_vec(), Object::Array(vec![Object::Real(0.001), 0.into(), 0.into(), Object::Real(0.001), 0.into(), 0.into()])), + (b"CharProcs".to_vec(), Object::Dict(Dict::new())), + ])); + let font = Object::Ref(doc.add(font)); + let list = |d: &mut Dict| { + let Some(Object::Dict(res)) = d.get_mut(b"Resources") else { panic!("{holder:?} has resources") }; + let Some(Object::Dict(fonts)) = res.get_mut(b"Font") else { panic!("{holder:?} lists fonts") }; + fonts.set(b"T3".to_vec(), font); + }; + match &*doc.get(holder) { + Object::Stream(s) => { + let mut s = s.clone(); + list(&mut s.dict); + doc.set(holder, Object::Stream(s)); + } + _ => doc.update_dict(holder, list).unwrap(), + } +} + /// The cleanups after the content walk spend what the walk left of the page's allowance, one /// allowance between them. A form without resources of its own that crosses the mark is read by -/// the walk, then again by the check of what the page still draws and by the font cleanup: at -/// 100 MiB, three reads are more than the page's 256 MiB, and the redaction is refused; at -/// 50 MiB it goes on. +/// the walk, then again by the check of what the page still draws and by the font cleanup, which +/// charges the page's allowance when the page lists a Type 3 font: at 100 MiB, three reads are more +/// than the page's 256 MiB, and the redaction is refused; at 50 MiB it goes on. With no Type 3 font +/// the font cleanup's read has an allowance of its own, so the page's pays for two reads of +/// 100 MiB, and the redaction goes on. #[test] fn the_cleanups_after_the_content_walk_spend_what_it_left_of_the_pages_allowance() { - for (mib, refused) in [(100, true), (50, false)] { + for (mib, type3, refused) in [(100, true, true), (50, true, false), (100, false, false)] { let blank = pdfcraft_filters::encode_flate(&vec![b' '; mib << 20]); let mut doc = one_page( b"q 50 0 0 50 20 230 cm /Im1 Do Q /FmBare Do", @@ -3588,16 +3617,61 @@ fn the_cleanups_after_the_content_walk_spend_what_it_left_of_the_pages_allowance stream("/Type /XObject /Subtype /Image /Width 1 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8", b"o"), ], ); + if type3 { + list_type3(&mut doc, ObjRef::new(3, 0)); + } mark(&mut doc, 0, &[[0.0, 200.0, 300.0, 300.0]], ""); - assert_eq!(apply(&mut doc, None).is_err(), refused, "{mib} MiB"); + assert_eq!(apply(&mut doc, None).is_err(), refused, "{mib} MiB, Type 3: {type3}"); } } +/// Text under the mark in a content stream that also holds bytes the parser can't read, inside a +/// `TJ` array, is read past by the walk and would stay. The font cleanup's read of what the content +/// draws refuses such bytes, and makes that read with any font listed, a Type 3 font or not (with +/// none, on an allowance of its own): Apply Redactions refuses the page, whether the stream is the +/// page's or a form's without resources of its own, and so does Remove Hidden Information. +#[test] +fn text_beside_bytes_the_parser_cannot_read_refuses_the_redaction() { + use crate::sanitize::{Hidden, remove_hidden}; + let page_of = |second: &[u8]| { + pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents [4 0 R 5 0 R] /Resources << /Font << /F1 6 0 R >> >> >>".to_vec(), + stream("", b"BT /F1 10 Tf 20 230 Td (SECRET) Tj ET"), + stream("", second), + FONT.replace("95 0 R", "7 0 R").into_bytes(), + widths(), + ]) + }; + for tail in [ + &b"BT /F1 10 Tf 20 260 Td [(TAIL) ) (SECRET)] TJ ET"[..], + b"BT /F1 10 Tf 20 260 Td [(TAIL) } (SECRET)] TJ ET", + b"BT /F1 10 Tf 20 260 Td [(TAILSECRET) )] TJ ET", + b"BT /F1 10 Tf 20 260 Td [) (TAILSECRET)] TJ ET", + ] { + let mut doc = page_of(tail); + mark(&mut doc, 0, &[[0.0, 200.0, 300.0, 300.0]], ""); + assert!(apply(&mut doc, None).is_err(), "{}", String::from_utf8_lossy(tail)); + } + let mut doc = one_page( + b"BT /F1 10 Tf 20 230 Td (SECRET) Tj ET /FmBare Do", + "/XObject << /FmBare 7 0 R >>", + vec![stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300]", b"BT /F1 10 Tf 20 260 Td [(TAIL) ) (SECRET)] TJ ET")], + ); + mark(&mut doc, 0, &[[0.0, 200.0, 300.0, 300.0]], ""); + assert!(apply(&mut doc, None).is_err(), "a form's text"); + let mut doc = page_of(b"BT /F1 10 Tf 3 Tr 20 260 Td [(TAIL) ) (HIDDEN)] TJ ET"); + doc.set(ObjRef::new(4, 0), Object::Stream(Stream::from_raw(Dict::new(), b"BT /F1 10 Tf 3 Tr 20 230 Td (HIDDEN) Tj ET".to_vec()))); + assert!(remove_hidden(&mut doc, &[Hidden::HiddenText]).is_err(), "hidden text"); +} + /// The cleanups of a form the redaction rewrites spend the page's allowance too. The rewritten /// form has resources of its own, so the page's cleanups don't follow it, and draws through them a /// form without any that crosses the mark: the walk reads that form, then the check of what the -/// rewritten form still draws and its font cleanup read it again. At 100 MiB, three reads are more -/// than the page's 256 MiB, and the redaction is refused; at 50 MiB it goes on. +/// rewritten form still draws and its font cleanup (it lists a Type 3 font) read it again. At +/// 100 MiB, three reads are more than the page's 256 MiB, and the redaction is refused; at 50 MiB +/// it goes on. #[test] fn the_cleanups_of_a_rewritten_form_spend_the_pages_allowance() { for (mib, refused) in [(100, true), (50, false)] { @@ -3613,6 +3687,7 @@ fn the_cleanups_of_a_rewritten_form_spend_the_pages_allowance() { stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Filter /FlateDecode", &blank), ], ); + list_type3(&mut doc, ObjRef::new(7, 0)); mark(&mut doc, 0, &[[0.0, 200.0, 300.0, 300.0]], ""); assert_eq!(apply(&mut doc, None).is_err(), refused, "{mib} MiB"); } @@ -3620,9 +3695,9 @@ fn the_cleanups_of_a_rewritten_form_spend_the_pages_allowance() { /// Removing hidden text spends what its content walk left of the page's allowance too. The page /// draws a form holding hidden text, which is rewritten, and a form without resources of its own: -/// the walk reads the second, then the check of what the page still draws and the font cleanup -/// read it again. At 100 MiB, three reads are more than the page's 256 MiB, and the removal is -/// refused; at 50 MiB it goes on. +/// the walk reads the second, then the check of what the page still draws and the font cleanup (the +/// page lists a Type 3 font) read it again. At 100 MiB, three reads are more than the page's +/// 256 MiB, and the removal is refused; at 50 MiB it goes on. #[test] fn removing_hidden_text_spends_what_the_content_walk_left_of_the_pages_allowance() { use crate::sanitize::{Hidden, remove_hidden}; @@ -3636,6 +3711,7 @@ fn removing_hidden_text_spends_what_the_content_walk_left_of_the_pages_allowance stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Filter /FlateDecode", &blank), ], ); + list_type3(&mut doc, ObjRef::new(3, 0)); assert_eq!(remove_hidden(&mut doc, &[Hidden::HiddenText]).is_err(), refused, "{mib} MiB"); } } diff --git a/crates/redact/src/type3.rs b/crates/redact/src/type3.rs index 4c3f2211e8..759f66000b 100644 --- a/crates/redact/src/type3.rs +++ b/crates/redact/src/type3.rs @@ -539,6 +539,14 @@ fn gs_font_object(doc: &Document, state_obj: &Object) -> Option { font_entry.into_iter().next() } +/// Whether font `f` (a reference, or a font dictionary) is a Type 3 font, the only kind the font +/// cleanup prunes: its `/Subtype` is read directly, as the cleanup's other readers read it, so one +/// given by reference isn't recognised by any of them. Which allowance the cleanup's read is charged +/// to, and what it prunes, both go by this. +fn is_type3(doc: &Document, f: &Object) -> bool { + doc.resolve(f).as_dict().is_some_and(|d| d.name(b"Subtype") == Some(b"Type3")) +} + /// Prune Type 3 fonts from a rewritten page or form's `/Resources` dictionary if the rewritten /// content no longer draws any text using that font name. `page` is the page's dictionary when /// `res` are a page's resources. What it reads is charged to `budget`, what is left of the page's @@ -556,12 +564,18 @@ pub(crate) fn prune_type3_from_resources( } // The operators of forms that draw through these resources count too: dropping a font a child - // form still shows would leave its text without one. - let mut ops = crate::interp::resource_ops(doc, budget, res, content)?; + // form still shows would leave its text without one. Without a Type 3 font nothing is pruned, + // but the read is still made: it refuses bytes the parser can't read in what the content draws, + // which for a text redaction is the only check that does (the walk reads past them, so text + // they hide would stay under the mark). It is then charged to an allowance of a page of its own, + // not to what the walk left of the page's, since it decides nothing about the fonts. + let type3 = fonts.iter().any(|(_, f)| is_type3(doc, f)); + let mut own = pdfcraft_cos::DecodeBudget::page(); + let mut ops = crate::interp::resource_ops(doc, if type3 { &mut *budget } else { &mut own }, res, content)?; // So do those of a page's annotation appearances, and of soft mask groups, without resources of // their own. They are read only when there is a Type 3 font to keep, so other files are not // refused over them. - if fonts.iter().any(|(_, f)| doc.resolve(f).as_dict().is_some_and(|d| d.name(b"Subtype") == Some(b"Type3"))) { + if type3 { if let Some(page) = page { // One total for all of them, as for the content: each appearance under the cap could // still add up to far more than it. Their decoded bytes, and their forms', are charged to @@ -605,15 +619,12 @@ pub(crate) fn prune_type3_from_resources( let mut changed = false; let font_names: Vec> = fonts.iter().map(|(n, _)| n.to_vec()).collect(); for name in font_names { - if let Some(font_obj) = fonts.get(&name) { - let resolved = doc.resolve(font_obj); - if let Some(d) = resolved.as_dict() - && d.name(b"Subtype") == Some(b"Type3") - && !drawn_font_names.contains(&name) - { - fonts.remove(&name); - changed = true; - } + if let Some(font_obj) = fonts.get(&name) + && is_type3(doc, font_obj) + && !drawn_font_names.contains(&name) + { + fonts.remove(&name); + changed = true; } } From ac1eba10927457983b992a768845ca872a822cbc Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Tue, 10 Nov 2026 13:59:52 +0100 Subject: [PATCH 06/60] M8: the form and hidden-text callers are tested without a Type 3 font too Review of the allowance change. Only Apply's page caller had a row with no Type 3 font listed. `the_cleanups_of_a_rewritten_form_spend_the_pages_allowance` and `removing_hidden_text_spends_what_the_content_walk_left_of_the_pages_allowance` now have one too: at 100 MiB the redaction and the removal go on. Doc comments that said every cleanup of a page spends the page's allowance (`prune_type3_from_resources`, `resource_ops`, `prune`) now say the font cleanup has one of its own when no Type 3 font is listed. The doc of `text_beside_bytes_the_parser_cannot_read_refuses_the_redaction` says the cleanups run only when the walk rewrites some of the page, and read what it rewrote as rewritten. The comment in `prune_type3_from_resources` no longer calls its read the only check for text: `prune_form_resources` (for a rewritten form), `prune` (when an image or form is replaced) and `prune_properties` make the same read when they run, and `unlink_type3` reads every page when the document still lists a Type 3 font. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/redact/src/interp/prune.rs | 8 +++--- crates/redact/src/tests.rs | 44 ++++++++++++++++++++----------- crates/redact/src/type3.rs | 11 +++++--- 3 files changed, 40 insertions(+), 23 deletions(-) diff --git a/crates/redact/src/interp/prune.rs b/crates/redact/src/interp/prune.rs index 39089601ec..43cc886826 100644 --- a/crates/redact/src/interp/prune.rs +++ b/crates/redact/src/interp/prune.rs @@ -9,8 +9,9 @@ use super::{ /// by a form without /Resources of its own, which draws through `xo` too. Such forms are read /// within `budget`, what is left of the page's allowance ([`resource_ops`]). Fails, as the cleanup /// of the fonts and properties these forms draw with does, when that can't be told: such a form -/// doesn't decode (an image codec or a /Filter it can't run, say) or runs past `budget`, the walk -/// passes 2,000,000 operators, or the parser can't read all of such a form or of `content`. +/// doesn't decode (an image codec or a /Filter it can't run, say) or runs past `budget` (for the +/// fonts' cleanup, an allowance of its own when no Type 3 font is listed), the walk passes +/// 2,000,000 operators, or the parser can't read all of such a form or of `content`. /// Corrupt data in such a form is read, tolerantly, up to the damage, which may be nothing. pub(crate) fn prune(doc: &Document, budget: &mut DecodeBudget, xo: &mut Dict, replaced: &[Vec], content: &[Vec]) -> Result<(), CosError> { if replaced.is_empty() { @@ -473,7 +474,8 @@ pub(super) fn clip_out(rects: &[[f64; 4]], ctm: &Matrix, around: [f64; 4]) -> Op /// The operators of `content`, and of every form without /Resources of its own that it draws /// through `res`, nested ones too: what draws with these resources. Each such form is read once, /// with its `/DecodeParms` given by reference, its decoded bytes charged to `budget`, so the -/// cleanups of one page that walk the same forms spend one allowance between them. An error +/// cleanups of one page that walk the same forms spend one allowance between them (all but the +/// fonts' cleanup when no Type 3 font is listed, which has one of its own). An error /// when that can't all be read: the parser can't read all of `content` or of such a form, the walk /// passes 2,000,000 operators, or such a form doesn't decode or runs past `budget`. Operators are /// read one at a time, so the walk stops at one past the limit rather than holding them all. diff --git a/crates/redact/src/tests.rs b/crates/redact/src/tests.rs index 465294dc9a..99e808139b 100644 --- a/crates/redact/src/tests.rs +++ b/crates/redact/src/tests.rs @@ -3625,11 +3625,16 @@ fn the_cleanups_after_the_content_walk_spend_what_it_left_of_the_pages_allowance } } -/// Text under the mark in a content stream that also holds bytes the parser can't read, inside a -/// `TJ` array, is read past by the walk and would stay. The font cleanup's read of what the content -/// draws refuses such bytes, and makes that read with any font listed, a Type 3 font or not (with -/// none, on an allowance of its own): Apply Redactions refuses the page, whether the stream is the -/// page's or a form's without resources of its own, and so does Remove Hidden Information. +/// Text under the mark in a content stream that holds bytes the parser can't read, inside a `TJ` +/// array, is read past by the walk and would stay. The font cleanup's read of what the content +/// draws refuses such bytes in the streams and forms the walk leaves as they were, and makes that +/// read with any font listed, a Type 3 font or not (with none, on an allowance of its own). Here +/// the walk rewrites the text in the page's first stream and leaves the broken `TJ` as it was: +/// Apply Redactions refuses the page, whether that `TJ` is in the page's second stream or in a +/// form without resources of its own, and so does Remove Hidden Information. The cleanups run only +/// when the walk rewrites some of the page, and read what it rewrote as rewritten, so such bytes in +/// a stream it rewrites still go by, and so do those on a page it rewrites nothing of, unless the +/// document still lists a Type 3 font (`unlink_type3` then reads every page). #[test] fn text_beside_bytes_the_parser_cannot_read_refuses_the_redaction() { use crate::sanitize::{Hidden, remove_hidden}; @@ -3669,12 +3674,13 @@ fn text_beside_bytes_the_parser_cannot_read_refuses_the_redaction() { /// The cleanups of a form the redaction rewrites spend the page's allowance too. The rewritten /// form has resources of its own, so the page's cleanups don't follow it, and draws through them a /// form without any that crosses the mark: the walk reads that form, then the check of what the -/// rewritten form still draws and its font cleanup (it lists a Type 3 font) read it again. At +/// rewritten form still draws and its font cleanup (when it lists a Type 3 font) read it again. At /// 100 MiB, three reads are more than the page's 256 MiB, and the redaction is refused; at 50 MiB -/// it goes on. +/// it goes on. With no Type 3 font the font cleanup's read has an allowance of its own, so the +/// page's pays for two reads of 100 MiB, and the redaction goes on. #[test] fn the_cleanups_of_a_rewritten_form_spend_the_pages_allowance() { - for (mib, refused) in [(100, true), (50, false)] { + for (mib, type3, refused) in [(100, true, true), (50, true, false), (100, false, false)] { let blank = pdfcraft_filters::encode_flate(&vec![b' '; mib << 20]); let mut doc = one_page( b"/FmText Do", @@ -3687,21 +3693,25 @@ fn the_cleanups_of_a_rewritten_form_spend_the_pages_allowance() { stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Filter /FlateDecode", &blank), ], ); - list_type3(&mut doc, ObjRef::new(7, 0)); + if type3 { + list_type3(&mut doc, ObjRef::new(7, 0)); + } mark(&mut doc, 0, &[[0.0, 200.0, 300.0, 300.0]], ""); - assert_eq!(apply(&mut doc, None).is_err(), refused, "{mib} MiB"); + assert_eq!(apply(&mut doc, None).is_err(), refused, "{mib} MiB, Type 3: {type3}"); } } /// Removing hidden text spends what its content walk left of the page's allowance too. The page /// draws a form holding hidden text, which is rewritten, and a form without resources of its own: -/// the walk reads the second, then the check of what the page still draws and the font cleanup (the -/// page lists a Type 3 font) read it again. At 100 MiB, three reads are more than the page's -/// 256 MiB, and the removal is refused; at 50 MiB it goes on. +/// the walk reads the second, then the check of what the page still draws and the font cleanup +/// (when the page lists a Type 3 font) read it again. At 100 MiB, three reads are more than the +/// page's 256 MiB, and the removal is refused; at 50 MiB it goes on. With no Type 3 font the font +/// cleanup's read has an allowance of its own, so the page's pays for two reads of 100 MiB, and the +/// removal goes on. #[test] fn removing_hidden_text_spends_what_the_content_walk_left_of_the_pages_allowance() { use crate::sanitize::{Hidden, remove_hidden}; - for (mib, refused) in [(100, true), (50, false)] { + for (mib, type3, refused) in [(100, true, true), (50, true, false), (100, false, false)] { let blank = pdfcraft_filters::encode_flate(&vec![b' '; mib << 20]); let mut doc = one_page( b"/FmHidden Do /FmBare Do", @@ -3711,8 +3721,10 @@ fn removing_hidden_text_spends_what_the_content_walk_left_of_the_pages_allowance stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Filter /FlateDecode", &blank), ], ); - list_type3(&mut doc, ObjRef::new(3, 0)); - assert_eq!(remove_hidden(&mut doc, &[Hidden::HiddenText]).is_err(), refused, "{mib} MiB"); + if type3 { + list_type3(&mut doc, ObjRef::new(3, 0)); + } + assert_eq!(remove_hidden(&mut doc, &[Hidden::HiddenText]).is_err(), refused, "{mib} MiB, Type 3: {type3}"); } } diff --git a/crates/redact/src/type3.rs b/crates/redact/src/type3.rs index 759f66000b..8f8579a7f4 100644 --- a/crates/redact/src/type3.rs +++ b/crates/redact/src/type3.rs @@ -550,7 +550,8 @@ fn is_type3(doc: &Document, f: &Object) -> bool { /// Prune Type 3 fonts from a rewritten page or form's `/Resources` dictionary if the rewritten /// content no longer draws any text using that font name. `page` is the page's dictionary when /// `res` are a page's resources. What it reads is charged to `budget`, what is left of the page's -/// allowance. +/// allowance, when the resources list a Type 3 font, and otherwise to an allowance of a page of its +/// own (below). pub(crate) fn prune_type3_from_resources( doc: &Document, budget: &mut pdfcraft_cos::DecodeBudget, @@ -566,9 +567,11 @@ pub(crate) fn prune_type3_from_resources( // The operators of forms that draw through these resources count too: dropping a font a child // form still shows would leave its text without one. Without a Type 3 font nothing is pruned, // but the read is still made: it refuses bytes the parser can't read in what the content draws, - // which for a text redaction is the only check that does (the walk reads past them, so text - // they hide would stay under the mark). It is then charged to an allowance of a page of its own, - // not to what the walk left of the page's, since it decides nothing about the fonts. + // and on a page with nothing replaced and no properties listed no other check does, unless the + // document still lists a Type 3 font (`unlink_type3` then reads every page). The walk reads + // past them, so text they hide would stay under the mark. The read is then charged to an + // allowance of a page of its own, not to what the walk left of the page's, since it decides + // nothing about the fonts. let type3 = fonts.iter().any(|(_, f)| is_type3(doc, f)); let mut own = pdfcraft_cos::DecodeBudget::page(); let mut ops = crate::interp::resource_ops(doc, if type3 { &mut *budget } else { &mut own }, res, content)?; From f2d64ca1367412dbf998de27fa9777d419089f9b Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Tue, 10 Nov 2026 15:53:40 +0100 Subject: [PATCH 07/60] M8: redaction refuses a page whose content it can't fully parse The redaction walk reads past bytes the content parser can't read: a stray `)` or `}` in a `TJ` array, say. Text they hide is neither found nor removed, while other readers draw it: Poppler's `pdftotext` extracts `[(TAIL) ) (SECRET)] TJ` as TAILSECRET. The only refusal was a side effect of the reads after the walk, which run only when the walk rewrote something, read what it rewrote as rewritten, and in the font cleanup's case only when the resources list a font. So Apply Redactions reported success and left the text where the walk rewrites nothing (the only text under the mark in such a `TJ`, or one in a form with resources of its own), in a stream it rewrites (it wrote the array out as `(TAIL) (SECRET) TJ`, without the stray byte or the brackets), and on a page whose font is set only through an ExtGState. Remove Hidden Information never counted hidden text in such a `TJ`: it found nothing to remove, or left it beside what it removed. The walk now records that it met such bytes (`Output::unparsed`, from `Parsed::skipped`, nested forms included), and Apply Redactions refuses the page with `RedactError::Unreadable`, as it does an inline image whose end is a guess. Remove Hidden Information counts such a page in its scan, so the removal runs and refuses it, as it does content that doesn't decode; it refuses an inline image whose end is a guess the same way, which it took on trust before. Apply walks only the forms that reach a mark, so bytes in a form away from the marks don't make the walk refuse the page; in a form without resources of its own, the check after the walk still refuses them when it runs, as before. Tests (`crates/redact/src/tests/unparsed.rs`): text under a mark in four broken `TJ` arrays, alone and beside other text; the same in a stream the walk rewrites; in a form under the mark with resources of its own; on a page without font resources; hidden text in such a `TJ`, alone and beside other hidden text, through the scan, Remove Hidden Information and Sanitize Document; hidden text beside an inline image the renderer can't end; and a form away from the marks, which is still redacted around. Two tests in `tests.rs` now expect the walk's refusal; the check after the walk keeps a case of its own, a form without resources away from the mark. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/redact/README.md | 5 +- crates/redact/src/interp.rs | 7 ++ crates/redact/src/lib.rs | 6 +- crates/redact/src/sanitize.rs | 11 +++ crates/redact/src/tests.rs | 48 +++++++----- crates/redact/src/tests/unparsed.rs | 114 ++++++++++++++++++++++++++++ 6 files changed, 170 insertions(+), 21 deletions(-) create mode 100644 crates/redact/src/tests/unparsed.rs diff --git a/crates/redact/README.md b/crates/redact/README.md index e47df98f17..d8f4b46e94 100644 --- a/crates/redact/README.md +++ b/crates/redact/README.md @@ -130,7 +130,10 @@ Text is placed with the font and size set by `Tf` or by an ExtGState's `/Font` t form XObject starts from the text state of the content that draws it. Content that can't be decoded makes the operation fail rather than leave it unredacted, and so does an inline image the renderer's tokenizer reads no image for (`RedactError::Unreadable`): where the page's operators -resume after it would be a guess. A Redact annotation with neither a usable `/QuadPoints` array nor a usable +resume after it would be a guess. So do bytes the content parser can't read, in the page's content +or in a form under an area (a stray `)` or `}` in a `TJ` array, say): the interpreter reads past +them, so text they hide, which other readers may draw, would be neither found nor removed. Remove +hidden information refuses a page for both too, in whatever content it walks. A Redact annotation with neither a usable `/QuadPoints` array nor a usable `/Rect` makes it fail with `RedactError::UnreadableMark` before anything changes, instead of skipping that mark. The count covers the marks on the pages being applied. diff --git a/crates/redact/src/interp.rs b/crates/redact/src/interp.rs index 1e69cb32e2..ae525989a1 100644 --- a/crates/redact/src/interp.rs +++ b/crates/redact/src/interp.rs @@ -143,6 +143,11 @@ pub(crate) struct Output { /// ([`pdfcraft_content::Parsed::ambiguous_images`]): what this scope placed after it may not /// be what the page shows, so a redaction must not trust it. pub ambiguous: bool, + /// The content holds bytes the parser can't read ([`pdfcraft_content::Parsed::skipped`]), in + /// this scope or a nested form: the walk reads past them, so text they hide (a stray `)` in a + /// `TJ` array, say), which other readers may still draw, is neither found nor removed, and a + /// rewrite would drop the bytes but keep the strings. A redaction must not trust it. + pub unparsed: bool, pub tag_changes: Vec<(ObjRef, ObjRef, HashSet, HashSet)>, pub error: Option, } @@ -407,6 +412,7 @@ fn process_from(doc: &mut Document, scope: &mut Scope<'_>, streams: &[Vec], let joined = Pieces::join(streams); let mut parsed = joined.parse(); walk.out.ambiguous = parsed.ambiguous_images > 0; + walk.out.unparsed = parsed.skipped > 0; // A rewritten piece loses its comments, so keep where they began and ended: a comment that // ran on into the next piece still hides that piece's first line. let comment_edges = joined.comment_edges(&parsed); @@ -1150,6 +1156,7 @@ fn xobject( out.residue += inner.residue; out.vertical |= inner.vertical; out.ambiguous |= inner.ambiguous; + out.unparsed |= inner.unparsed; out.doomed.extend_from_slice(&inner.doomed); out.retired_properties.extend(inner.retired_properties); // A Type 3 glyph without resources shown inside the form may paint with the page's diff --git a/crates/redact/src/lib.rs b/crates/redact/src/lib.rs index 7c1a65f55a..5938f20a68 100644 --- a/crates/redact/src/lib.rs +++ b/crates/redact/src/lib.rs @@ -791,8 +791,10 @@ fn redact_page_content( } // An inline image the renderer reads no image for (no `EI` it accepts, or parameters it // can't read): where the page's operators resume after it is a guess, so neither removal - // nor the verify pass can be trusted on this page. - if out.ambiguous { + // nor the verify pass can be trusted on this page. Likewise bytes the parser can't read in + // what the walk read, the page's content or a form under a mark: text they hide is neither + // found nor removed, and other readers may draw it. + if out.ambiguous || out.unparsed { return Err(RedactError::Unreadable(pi + 1)); } doomed_xobjects.extend_from_slice(&out.doomed); diff --git a/crates/redact/src/sanitize.rs b/crates/redact/src/sanitize.rs index 0d4ee2e18a..3da224e8ac 100644 --- a/crates/redact/src/sanitize.rs +++ b/crates/redact/src/sanitize.rs @@ -659,6 +659,17 @@ fn content_pass(doc: &mut Document, text: bool, layers: bool, write: bool) -> Re if let Some(error) = out.error { return Err(error.into()); } + // Bytes the parser can't read, or an inline image whose end is a guess: the walk reads past + // them, so hidden text or a hidden layer's content they hold, which other readers may draw, + // is neither found nor removed. As for content that doesn't decode, count the page so the + // removal runs and refuses it. + if out.unparsed || out.ambiguous { + if write { + return Err(RedactError::Unreadable(pi + 1)); + } + total += 1; + continue; + } let blocks = scope.layer_blocks; let glyphs = scope.report.glyphs; let retained_for_page = std::mem::take(&mut scope.retained_off_layers); diff --git a/crates/redact/src/tests.rs b/crates/redact/src/tests.rs index 46ac131450..4e132f20c6 100644 --- a/crates/redact/src/tests.rs +++ b/crates/redact/src/tests.rs @@ -3297,31 +3297,42 @@ fn image_and_a_tail(catalog: &str, image: &str, image_dict: &str, tail: &[u8]) - ]) } -/// The refusal of the check of what content still draws through its resources (`resource_ops`). -fn unread_content_refusal(result: &Result) -> bool { - matches!(result, Err(RedactError::Cos(pdfcraft_cos::CosError::Syntax { detail, .. })) - if detail == "redaction could not fully read the content that uses these resources") -} - -/// What the page draws can't be told when a content stream the redaction leaves as it was holds -/// bytes the parser can't read: the redaction of an image is refused, as it was already on a page -/// with fonts or properties to clean up. +/// What the page draws can't be told when content the redaction leaves as it was holds bytes the +/// parser can't read: the redaction of an image is refused. In a content stream of the page, the +/// walk reads past them and refuses the page itself (`Unreadable`). In a form without resources of +/// its own that lies away from the mark, which the walk doesn't enter, the check of what the page +/// still draws through its resources refuses it. #[test] fn unreadable_bytes_in_a_stream_left_as_it_was_refuse_the_redaction_of_an_image() { for (tail, refused) in [(&b"q Q )"[..], true), (b"q Q", false)] { let mut doc = image_and_a_tail("", "Im1", "", tail); mark(&mut doc, 0, &[[0.0, 200.0, 300.0, 300.0]], ""); - let result = apply(&mut doc, None); - if refused { - assert!(unread_content_refusal(&result), "{result:?}"); - } else { - assert_eq!(result.map(|r| r.images_removed).ok(), Some(1)); - } + let result = apply(&mut doc, None).map(|r| r.images_removed); + assert_eq!(result, if refused { Err(RedactError::Unreadable(1)) } else { Ok(1) }); } + let mut doc = image_and_a_tail("", "Im1", "", b"/Fm Do"); + let bbox = Object::Array(vec![0.into(), 0.into(), 300.into(), 100.into()]); + let form = Dict::from_iter([(b"Type".to_vec(), Object::name("XObject")), (b"Subtype".to_vec(), Object::name("Form")), (b"BBox".to_vec(), bbox)]); + let form = doc.add(Object::Stream(Stream::from_raw(form, b"q Q )".to_vec()))); + doc.update_dict(ObjRef::new(3, 0), |d| { + if let Some(Object::Dict(res)) = d.get_mut(b"Resources") + && let Some(Object::Dict(xo)) = res.get_mut(b"XObject") + { + xo.set(b"Fm".to_vec(), Object::Ref(form)); + } + }) + .unwrap(); + mark(&mut doc, 0, &[[0.0, 200.0, 300.0, 300.0]], ""); + let result = apply(&mut doc, None); + assert!( + matches!(&result, Err(RedactError::Cos(pdfcraft_cos::CosError::Syntax { detail, .. })) if detail == "redaction could not fully inspect inherited form resources"), + "{result:?}" + ); } -/// Remove Hidden Information makes the same check: removing an off layer's image is refused on a -/// page whose drawing can't be told. +/// Remove Hidden Information refuses the same page: removing an off layer's image is refused on a +/// page whose drawing can't be told. Its walk reads every content stream, and refuses the page +/// itself. #[test] fn unreadable_bytes_in_a_stream_left_as_it_was_refuse_removing_a_hidden_image() { use crate::sanitize::{Hidden, remove_hidden}; @@ -3329,7 +3340,7 @@ fn unreadable_bytes_in_a_stream_left_as_it_was_refuse_removing_a_hidden_image() let mut doc = image_and_a_tail("/OCProperties << /OCGs [7 0 R] /D << /OFF [7 0 R] >> >>", "ImOC", "/OC 7 0 R", tail); let result = remove_hidden(&mut doc, &[Hidden::HiddenLayers]); if refused { - assert!(unread_content_refusal(&result), "{result:?}"); + assert_eq!(result, Err(RedactError::Unreadable(1))); } else { assert!(result.unwrap().iter().any(|(h, n)| *h == Hidden::HiddenLayers && *n > 0)); } @@ -6152,6 +6163,7 @@ mod metadata_matching; mod property_retirement; mod surviving_references; mod text_state; +mod unparsed; /// `[(FIRSTARR)] [(LASTARR)] TJ` is malformed (a surplus array operand), but the operator still /// draws only the one immediately before it (LASTARR, 7 glyphs, 35 pt wide from x = 10) -- not diff --git a/crates/redact/src/tests/unparsed.rs b/crates/redact/src/tests/unparsed.rs new file mode 100644 index 0000000000..4fcc9fcb75 --- /dev/null +++ b/crates/redact/src/tests/unparsed.rs @@ -0,0 +1,114 @@ +//! Bytes the content parser can't read are read past by the redaction walk, so text they hide (a +//! stray `)` or `}` in a `TJ` array) is neither found nor removed, while other readers still draw +//! it. A page whose walk meets such bytes is refused, by Apply Redactions and by Remove Hidden +//! Information. + +use super::*; +use crate::sanitize::{Hidden, remove_hidden, sanitize, scan}; + +/// `TJ` arrays each holding a byte the parser can't read beside the text. +const BROKEN: [&[u8]; 4] = [b"[(TAIL) ) (SECRET)] TJ", b"[(TAIL) } (SECRET)] TJ", b"[(TAILSECRET) )] TJ", b"[) (TAILSECRET)] TJ"]; + +/// Over the top third of the 300 × 300 page. +const MARK: [f64; 4] = [0.0, 200.0, 300.0, 300.0]; + +/// `tj` shown in /F1 at (20, 260), under [`MARK`]. +fn shown(tj: &[u8]) -> Vec { + [b"BT /F1 10 Tf 20 260 Td ".as_slice(), tj, b" ET"].concat() +} + +/// The only text under the mark sits in such a `TJ`: the walk finds nothing there and rewrites +/// nothing, so it went on, and the text stayed in the file. +#[test] +fn text_under_a_mark_beside_bytes_the_parser_cannot_read_is_refused() { + for tj in BROKEN { + let mut doc = one_page(&shown(tj), "", vec![]); + mark(&mut doc, 0, &[MARK], ""); + assert_eq!(apply(&mut doc, None), Err(RedactError::Unreadable(1)), "{}", String::from_utf8_lossy(tj)); + } + // With other text on the page, outside the mark. + let mut doc = one_page(&[b"BT /F1 10 Tf 20 20 Td (PUBLIC) Tj ET ".as_slice(), &shown(BROKEN[0])].concat(), "", vec![]); + mark(&mut doc, 0, &[MARK], ""); + assert_eq!(apply(&mut doc, None), Err(RedactError::Unreadable(1)), "beside other text"); +} + +/// In a stream the walk rewrites, it removed the text it could read and wrote the broken `TJ` out +/// as `(TAIL) (SECRET) TJ`: the stray byte and the brackets went, the strings stayed. +#[test] +fn bytes_the_parser_cannot_read_in_a_stream_the_walk_rewrites_are_refused() { + for tj in BROKEN { + let mut doc = one_page(&[b"BT /F1 10 Tf 20 230 Td (SECRET) Tj ET ".as_slice(), &shown(tj)].concat(), "", vec![]); + mark(&mut doc, 0, &[MARK], ""); + assert_eq!(apply(&mut doc, None), Err(RedactError::Unreadable(1)), "{}", String::from_utf8_lossy(tj)); + } +} + +/// A form under the mark with resources of its own, which the walk enters but doesn't rewrite: +/// no later read of the page follows it. +#[test] +fn bytes_the_parser_cannot_read_in_a_form_under_a_mark_are_refused() { + let form = stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Resources << /Font << /F1 5 0 R >> >>", shown(BROKEN[0])); + let mut doc = one_page(b"/Fm Do", "/XObject << /Fm 7 0 R >>", vec![form]); + mark(&mut doc, 0, &[MARK], ""); + assert_eq!(apply(&mut doc, None), Err(RedactError::Unreadable(1))); +} + +/// A form the mark doesn't reach isn't walked: bytes in it can't hide anything under the mark, so +/// the page is redacted. +#[test] +fn bytes_the_parser_cannot_read_in_a_form_away_from_the_marks_are_left() { + let form = stream( + "/Type /XObject /Subtype /Form /BBox [0 0 300 100] /Resources << /Font << /F1 5 0 R >> >>", + [b"BT /F1 10 Tf 20 20 Td ".as_slice(), BROKEN[0], b" ET"].concat(), + ); + let mut doc = one_page(b"BT /F1 10 Tf 20 230 Td (SECRET) Tj ET /Fm Do", "/XObject << /Fm 7 0 R >>", vec![form]); + mark(&mut doc, 0, &[MARK], ""); + assert_eq!(apply(&mut doc, None).unwrap().glyphs, 6); + assert!(!content(&doc, 0).contains("(SECRET)")); +} + +/// The page sets its font only through an ExtGState `/Font` and lists no `/Font` resources, so the +/// cleanups after the walk, which look at the listed fonts, read nothing. +#[test] +fn bytes_the_parser_cannot_read_on_a_page_without_font_resources_are_refused() { + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents [4 0 R 5 0 R] /Resources << /ExtGState << /GS1 << /Font [6 0 R 10] >> >> >> >>" + .to_vec(), + stream("", b"BT /GS1 gs 20 230 Td (SECRET) Tj ET"), + stream("", b"BT /GS1 gs 20 260 Td [(TAIL) ) (SECRET)] TJ ET"), + FONT.replace("95 0 R", "7 0 R").into_bytes(), + widths(), + ]); + mark(&mut doc, 0, &[MARK], ""); + assert_eq!(apply(&mut doc, None), Err(RedactError::Unreadable(1))); +} + +/// Hidden text inside a broken `TJ` is never counted: alone, Remove Hidden Information found +/// nothing to remove and Sanitize Document went on; beside other hidden text, the removal went on +/// and left it. The scan now counts the page, and removing hidden text refuses it. +#[test] +fn hidden_text_beside_bytes_the_parser_cannot_read_is_refused() { + let alone = b"BT /F1 10 Tf 3 Tr 20 260 Td [(TAIL) ) (HIDDEN)] TJ ET".as_slice(); + let beside = [b"BT /F1 10 Tf 3 Tr 20 230 Td (HIDDEN) Tj ET ".as_slice(), alone].concat(); + for content in [alone, &beside] { + let label = String::from_utf8_lossy(content); + let doc = one_page(content, "", vec![]); + let counts: std::collections::HashMap = scan(&doc).into_iter().collect(); + assert!(counts[&Hidden::HiddenText] > 0, "{label}"); + let mut doc = one_page(content, "", vec![]); + assert_eq!(remove_hidden(&mut doc, &[Hidden::HiddenText]), Err(RedactError::Unreadable(1)), "{label}"); + let mut doc = one_page(content, "", vec![]); + assert_eq!(sanitize(&mut doc), Err(RedactError::Unreadable(1)), "{label}"); + } +} + +/// An inline image whose end the parser guesses, as Apply Redactions already refuses it: what +/// follows it may not be what the page draws, so Remove Hidden Information refuses it too. +#[test] +fn hidden_text_beside_an_inline_image_the_renderer_cannot_end_is_refused() { + let page = b"BT /F1 10 Tf 3 Tr 10 100 Td (HIDDEN) Tj ET BI /W 1 /H 1 /CS /G /BPC 8 /F /AHx ID 7F".as_slice(); + let mut doc = one_page(page, "", vec![]); + assert_eq!(remove_hidden(&mut doc, &[Hidden::HiddenText]), Err(RedactError::Unreadable(1))); +} From a7f0074381c5aa0e3d4d2f85212aaa876b3491e5 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Tue, 10 Nov 2026 15:53:40 +0100 Subject: [PATCH 08/60] M8: doc_hidden_info and the parity note say what the refusal covers Review of the refusal. `doc_hidden_info` said it lists what Remove Hidden Information would remove. A page that can't be read in full is counted as one item of hidden text and of hidden layers, so that the removal runs and refuses it: since the refusal before this, that includes a page holding bytes the parser can't read or an inline image whose end is a guess, not only content streams that don't decode. The description now says so, and that with form fields removed too, a field's appearance is read the same way though it isn't counted. The parity note on Apply Redactions lists the new refusal beside the inline image's. Removing hidden layers alone refuses such a page in a document without `/OCProperties` too, which a review asked to narrow: a membership dictionary there (`/AllOff` of a layer that is on) still hides what it marks, for the walk and for the app's renderer. The comment at the refusal in `sanitize.rs` says only what the bytes are shown to hide: text in a broken `TJ`. Tests: `bytes_the_parser_cannot_read_refuse_removing_hidden_layers_alone`; `hidden_text_beside_bytes_the_parser_cannot_read_in_a_field_appearance_is_refused` (removing form fields with hidden text, and Sanitize Document, which went on and left it); and `hidden_text_beside_bytes_the_parser_cannot_read_is_refused` gains hidden text in a broken `TJ` inside a form. The form-away test's doc says the check after the walk would still refuse a form without resources. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/automation/src/tools.rs | 2 +- crates/redact/src/sanitize.rs | 6 +-- crates/redact/src/tests/unparsed.rs | 60 ++++++++++++++++++++++++++++- parity/acrobat-features.toml | 2 +- 4 files changed, 63 insertions(+), 7 deletions(-) diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index 1cd876d0b1..a37fe49ffe 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -726,7 +726,7 @@ pub fn tools() -> Vec { t( "doc_hidden_info", "Find hidden information", - "What Remove Hidden Information would remove, by category (metadata, attachments, comments, form-fields, hidden-text, hidden-layers, bookmarks, links-actions-scripts, private-data) with counts.", + "What Remove Hidden Information would remove, by category (metadata, attachments, comments, form-fields, hidden-text, hidden-layers, bookmarks, links-actions-scripts, private-data) with counts. A page that can't be read in full (its content streams don't decode, or they or a form they draw hold bytes the parser can't read or an inline image whose end is a guess) counts as one item of hidden-text and of hidden-layers: removing either category then fails on that page. With form-fields removed too, a field's appearance becomes page content and is read the same way, though it isn't counted here.", ) .ro() .with(schema(json!({ "doc": doc() }), &["doc"])), diff --git a/crates/redact/src/sanitize.rs b/crates/redact/src/sanitize.rs index 3da224e8ac..cd38b9eb19 100644 --- a/crates/redact/src/sanitize.rs +++ b/crates/redact/src/sanitize.rs @@ -660,9 +660,9 @@ fn content_pass(doc: &mut Document, text: bool, layers: bool, write: bool) -> Re return Err(error.into()); } // Bytes the parser can't read, or an inline image whose end is a guess: the walk reads past - // them, so hidden text or a hidden layer's content they hold, which other readers may draw, - // is neither found nor removed. As for content that doesn't decode, count the page so the - // removal runs and refuses it. + // them, so what it reads around them may not be what other readers draw (hidden text in a + // broken `TJ` is neither found nor removed). As for content that doesn't decode, count the + // page so the removal runs and refuses it. if out.unparsed || out.ambiguous { if write { return Err(RedactError::Unreadable(pi + 1)); diff --git a/crates/redact/src/tests/unparsed.rs b/crates/redact/src/tests/unparsed.rs index 4fcc9fcb75..90dcb1ae47 100644 --- a/crates/redact/src/tests/unparsed.rs +++ b/crates/redact/src/tests/unparsed.rs @@ -53,8 +53,9 @@ fn bytes_the_parser_cannot_read_in_a_form_under_a_mark_are_refused() { assert_eq!(apply(&mut doc, None), Err(RedactError::Unreadable(1))); } -/// A form the mark doesn't reach isn't walked: bytes in it can't hide anything under the mark, so -/// the page is redacted. +/// A form the mark doesn't reach isn't walked: bytes in it can't hide anything under the mark, and +/// the walk doesn't refuse the page over them. This one has resources of its own, so the check after +/// the walk doesn't read it either (it would refuse one without), and the page is redacted. #[test] fn bytes_the_parser_cannot_read_in_a_form_away_from_the_marks_are_left() { let form = stream( @@ -102,6 +103,61 @@ fn hidden_text_beside_bytes_the_parser_cannot_read_is_refused() { let mut doc = one_page(content, "", vec![]); assert_eq!(sanitize(&mut doc), Err(RedactError::Unreadable(1)), "{label}"); } + // In a form with resources of its own, which the walk enters on every page. + let form = stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Resources << /Font << /F1 5 0 R >> >>", alone); + let mut doc = one_page(b"/Fm Do", "/XObject << /Fm 7 0 R >>", vec![form]); + assert_eq!(remove_hidden(&mut doc, &[Hidden::HiddenText]), Err(RedactError::Unreadable(1)), "in a form"); +} + +/// Removing hidden layers alone refuses such bytes too, even in a document without +/// `/OCProperties`: a membership dictionary (`/AllOff` of a layer that is on) still hides what it +/// marks, for the walk as for the app's renderer. With nothing in front of `/OC`, the marked text +/// is removed. Here a string beside a stray `}` is left in front of it, which keeps the block from +/// the walk (it reads a `BDC`'s first operands; a string alone does so too, a matter of its own), +/// and the stray byte has the page refused rather than the text kept. +#[test] +fn bytes_the_parser_cannot_read_refuse_removing_hidden_layers_alone() { + let marked = |before: &[u8]| { + let content = [before, b" /OC /MC0 BDC BT /F1 10 Tf 20 260 Td (LAYERSECRET) Tj ET EMC BT /F1 10 Tf 20 20 Td (PUBLIC) Tj ET"].concat(); + one_page( + &content, + "/Properties << /MC0 8 0 R >>", + vec![b"<< /Type /OCG /Name (Layer) >>".to_vec(), b"<< /Type /OCMD /OCGs [7 0 R] /P /AllOff >>".to_vec()], + ) + }; + let mut doc = marked(b""); + let removed = remove_hidden(&mut doc, &[Hidden::HiddenLayers]); + assert!(removed.as_ref().is_ok_and(|r| r.iter().any(|(h, n)| *h == Hidden::HiddenLayers && *n > 0)), "{removed:?}"); + assert!(!content(&doc, 0).contains("LAYERSECRET")); + let mut doc = marked(b"(x) }"); + assert_eq!(remove_hidden(&mut doc, &[Hidden::HiddenLayers]), Err(RedactError::Unreadable(1))); +} + +/// With form fields removed too, a field's appearance is flattened into page content and read the +/// same way, though the scan, which runs before flattening, doesn't count it: hidden text in a +/// broken `TJ` there refuses removing hidden text, and Sanitize Document, which went on and left it. +/// Removing form fields alone doesn't read the content, and goes on. +#[test] +fn hidden_text_beside_bytes_the_parser_cannot_read_in_a_field_appearance_is_refused() { + let fresh = || { + pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R /AcroForm << /Fields [6 0 R] >> >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> /Annots [6 0 R] >>" + .to_vec(), + stream("", b"BT /F1 10 Tf 10 230 Td (Name:) Tj ET"), + FONT.replace("95 0 R", "8 0 R").into_bytes(), + b"<< /Type /Annot /Subtype /Widget /FT /Tx /T (name) /P 3 0 R /Rect [50 225 200 240] /AP << /N 7 0 R >> >>".to_vec(), + stream( + "/Type /XObject /Subtype /Form /BBox [0 0 150 15] /Resources << /Font << /F1 5 0 R >> >>", + b"BT /F1 10 Tf 3 Tr 2 3 Td [(TAIL) ) (HIDDEN)] TJ ET", + ), + widths(), + ]) + }; + assert!(remove_hidden(&mut fresh(), &[Hidden::FormFields]).is_ok()); + assert_eq!(remove_hidden(&mut fresh(), &[Hidden::FormFields, Hidden::HiddenText]), Err(RedactError::Unreadable(1))); + assert_eq!(sanitize(&mut fresh()), Err(RedactError::Unreadable(1))); } /// An inline image whose end the parser guesses, as Apply Redactions already refuses it: what diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 5f3dc08f35..bbffa91408 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -5920,7 +5920,7 @@ status = "shipped" commands = ["redact.apply"] tools = ["redact_apply"] evidence = ["crates/ui-egui/tests/it/redact.rs::marking_applying_and_clearing", "crates/engine/src/tests.rs::redaction_marks_apply_for_good_and_undo", "crates/automation/tests/it/automation.rs::redacting_through_tools", "crates/engine/src/tests.rs::the_renderer_check_refuses_text_still_shown_under_a_mark", "crates/redact/src/tests.rs::redacted_xobjects_do_not_survive_through_inherited_or_shared_resources", "crates/redact/src/tests.rs::redacted_text_does_not_survive_in_info_or_xmp", "crates/redact/src/tests.rs::metadata_is_cleared_when_the_removed_text_was_cut_short", "crates/redact/src/tests/metadata_fail_closed.rs::an_xmp_stream_whose_filter_data_is_corrupt_goes", "crates/redact/src/tests/elsewhere.rs::places_apply_leaves_that_still_hold_the_removed_text_are_counted_and_kept", "crates/engine/src/tests.rs::applied_redactions_report_a_bookmark_that_still_holds_the_removed_text", "crates/automation/tests/it/automation.rs::redact_apply_reports_places_that_still_hold_the_removed_text", "crates/ui-egui/tests/it/redact.rs::applying_says_where_the_removed_text_is_still_held", "crates/redact/src/tests/metadata_matching.rs::a_heading_tracked_with_tj_numbers_matches_the_metadata", "crates/redact/src/tests/metadata_matching.rs::unicode_forms_of_the_removed_text_match", "crates/redact/src/tests/metadata_matching.rs::a_utf16le_string_is_searched_as_utf16le", "crates/redact/src/tests/identification.rs::the_identification_of_a_removed_packet_is_written_back", "crates/automation/tests/it/automation.rs::redact_apply_reports_the_identification_written_back", "crates/ui-egui/tests/it/redact.rs::applying_says_the_pdfa_identification_was_written_back"] -notes = "confirmation; one undo step; next save is a full rewrite. Document information entries that contain the removed text, and the document's and pages' XMP streams that do, are removed (#2448; matched without regard to case or spacing, by the whole run of removed text and by its words of three letters or digits or more, and a value wholly part of a run as whole words, each also compared by letters and digits alone (so spacing, hyphens and TJ tracking don't hide it), after NFKC and lower case, with FF FE strings read as UTF-16LE; fail-closed: all of it goes when the removed text can't be known in full (too much to collect or search for, a glyph whose text isn't known, a form removed unread), and an XMP packet goes when it can't be decoded strictly, is over 16 MiB or isn't UTF-8 or UTF-16). When the document's XMP goes, its PDF/A and PDF/UA identification is written back in a packet holding only that, value by value when each is a valid code that doesn't hold the removed text, else reported lost (Report::identification, the notification, redact_apply's identification); /Info entries that remain aren't mirrored into that packet. Bookmark titles, attachments (names, file names, descriptions), the comments that remain and other objects' XMP are not changed: applying counts those that still hold the removed text, and the notification and redact_apply's still_holding say where (Remove hidden information removes them); when the removed text couldn't be searched for, every such place holding any text is counted. Refused, with nothing changed, when the page still shows text under a mark afterwards: text in some predefined CMaps and rotated Type 3 fonts is not removed yet, so those documents cannot be redacted. Also refused on a page with an inline image the renderer reads no image for, since the operators after it can't be known. Removed images and forms are unlinked from page-tree resources (own, inherited or shared) and from surviving forms' resources, not yet from annotation appearances or patterns" +notes = "confirmation; one undo step; next save is a full rewrite. Document information entries that contain the removed text, and the document's and pages' XMP streams that do, are removed (#2448; matched without regard to case or spacing, by the whole run of removed text and by its words of three letters or digits or more, and a value wholly part of a run as whole words, each also compared by letters and digits alone (so spacing, hyphens and TJ tracking don't hide it), after NFKC and lower case, with FF FE strings read as UTF-16LE; fail-closed: all of it goes when the removed text can't be known in full (too much to collect or search for, a glyph whose text isn't known, a form removed unread), and an XMP packet goes when it can't be decoded strictly, is over 16 MiB or isn't UTF-8 or UTF-16). When the document's XMP goes, its PDF/A and PDF/UA identification is written back in a packet holding only that, value by value when each is a valid code that doesn't hold the removed text, else reported lost (Report::identification, the notification, redact_apply's identification); /Info entries that remain aren't mirrored into that packet. Bookmark titles, attachments (names, file names, descriptions), the comments that remain and other objects' XMP are not changed: applying counts those that still hold the removed text, and the notification and redact_apply's still_holding say where (Remove hidden information removes them); when the removed text couldn't be searched for, every such place holding any text is counted. Refused, with nothing changed, when the page still shows text under a mark afterwards: text in some predefined CMaps and rotated Type 3 fonts is not removed yet, so those documents cannot be redacted. Also refused on a page with an inline image the renderer reads no image for, since the operators after it can't be known, or with bytes the content parser can't read in its content or in a form under a mark, which can hide text other readers draw. Removed images and forms are unlinked from page-tree resources (own, inherited or shared) and from surviving forms' resources, not yet from annotation appearances or patterns" [[feature]] id = "protect.redact-glyph-removal" From d2c87ae78930a9b7a6627196e334573fc24a7305 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Tue, 10 Nov 2026 16:45:34 +0100 Subject: [PATCH 09/60] M8: the redaction walk reads a form's /Matrix entries by reference MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The walk read a form's `/Matrix` with `Matrix::from_operands` on the array as written, so an entry given by reference made it the identity. That is legal: an array's entries may each be an indirect reference (ISO 32000-2 §7.3.10), and Arlington's `XObjectFormType1` allows the matrix itself to be one. A form moved onto a mark by such a matrix was then tested against the wrong place, never walked, and Apply Redactions reported success with its text still in the file: `/Matrix [1 0 0 1 0 8 0 R]` with `8 0 obj 200` left SECRET, which Poppler's `pdftotext` places under the mark. Remove Hidden Information missed text such a matrix moves off the page the same way. Found in review of the unparsed-bytes refusal. The walk now reads it with the proof's `matrix_of` (`verify/extract.rs`), which resolves the array and each entry. A `/Matrix` that isn't six numbers was the identity too; now where the form draws can't be told, so Apply refuses the page and the scans count it, as for a form past the page's budget. `matrix_of` now reads a null `/Matrix`, given directly or by reference, as the identity: a null entry is no entry (ISO 32000-2 §7.3.9). Tests (`crates/redact/src/tests/form_matrix.rs`): a form moved onto the mark by a matrix written direct, as a reference, and with an entry by reference, redacted; a null matrix, the identity; a matrix that isn't six numbers, refused, and counted by the verify pass and Remove Hidden Information's scan; and hidden text a matrix entry by reference moves off the page, removed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/redact/src/interp.rs | 19 ++++- crates/redact/src/tests.rs | 1 + crates/redact/src/tests/form_matrix.rs | 96 ++++++++++++++++++++++++++ crates/redact/src/verify/extract.rs | 11 ++- crates/redact/src/verify/mod.rs | 1 + 5 files changed, 124 insertions(+), 4 deletions(-) create mode 100644 crates/redact/src/tests/form_matrix.rs diff --git a/crates/redact/src/interp.rs b/crates/redact/src/interp.rs index 1e69cb32e2..04fafd283d 100644 --- a/crates/redact/src/interp.rs +++ b/crates/redact/src/interp.rs @@ -1071,7 +1071,24 @@ fn xobject( } } Some(b"Form") => { - let m = s.dict.get(b"Matrix").and_then(|m| Matrix::from_operands(doc.resolve(m).as_array()?)).unwrap_or_default(); + // Read as the proof reads it, entries by reference included: read as the identity, a + // form moved onto a mark would be tested against the wrong place and never walked. + let Ok(m) = crate::verify::matrix_of(doc, &s.dict) else { + // Not six numbers: where the form draws, and so whether it reaches a mark, can't be + // told. Apply refuses the page, and the scans count it, as for a form past the + // page's budget below. + if scope.mode == Mode::Apply { + out.error = Some(CosError::Syntax { + offset: 0, + detail: "a form XObject's /Matrix is not six numbers, so where it draws can't be told".into(), + }); + } else if scope.everywhere() { + scope.layer_blocks += 1; + } else { + out.residue += 1; + } + return None; + }; let fctm = m.then(&gs.ctm); let bbox = s.dict.get(b"BBox").and_then(|b| { let v: Vec = doc.resolve(b).as_array()?.iter().filter_map(Object::as_f64).collect(); diff --git a/crates/redact/src/tests.rs b/crates/redact/src/tests.rs index 46ac131450..47951343fb 100644 --- a/crates/redact/src/tests.rs +++ b/crates/redact/src/tests.rs @@ -6143,6 +6143,7 @@ fn unused_descriptions_in_shared_or_inherited_resources_are_retired() { mod descriptions; mod elsewhere; +mod form_matrix; mod hidden_scope; mod identification; mod inline_images; diff --git a/crates/redact/src/tests/form_matrix.rs b/crates/redact/src/tests/form_matrix.rs new file mode 100644 index 0000000000..c3e386f47d --- /dev/null +++ b/crates/redact/src/tests/form_matrix.rs @@ -0,0 +1,96 @@ +//! A form's `/Matrix` places what it draws. The redaction walk reads it as the proof does, the +//! array and each entry direct or by reference: read as the identity, a form moved onto a mark +//! was tested against the wrong place, never walked, and its text stayed. + +use super::*; + +/// One page drawing a form with resources of its own, whose content shows SECRET at (20, 60) in a +/// `/BBox` of [0 0 300 100], with `matrix` in its dictionary; object 8 is `extra`. +fn moved_form(matrix: &str, extra: &[u8]) -> Document { + one_page( + b"/Fm Do", + "/XObject << /Fm 7 0 R >>", + vec![ + stream( + &format!("/Type /XObject /Subtype /Form /BBox [0 0 300 100] {matrix} /Resources << /Font << /F1 5 0 R >> >>"), + b"BT /F1 10 Tf 20 60 Td (SECRET) Tj ET", + ), + extra.to_vec(), + ], + ) +} + +/// Over the top third of the 300 × 300 page, where a move of 200 puts the text. +const MARK: [f64; 4] = [0.0, 200.0, 300.0, 300.0]; + +#[test] +fn a_form_moved_onto_a_mark_is_redacted_however_its_matrix_is_given() { + for (matrix, extra) in [ + ("/Matrix [1 0 0 1 0 200]", &b"null"[..]), + ("/Matrix 8 0 R", b"[1 0 0 1 0 200]"), + ("/Matrix [1 0 0 1 0 8 0 R]", b"200"), + ("/Matrix [8 0 R 0 0 1 0 200]", b"1"), + ] { + let mut doc = moved_form(matrix, extra); + mark(&mut doc, 0, &[MARK], ""); + assert_eq!(apply(&mut doc, None).map(|r| r.glyphs), Ok(6), "{matrix} with 8 0 obj {}", String::from_utf8_lossy(extra)); + } +} + +/// A `/Matrix` of null, given directly or by a reference to null, is no `/Matrix` (ISO 32000-2 +/// §7.3.9): the identity, under which the text lies below the mark and stays. +#[test] +fn a_form_whose_matrix_is_null_draws_where_it_stands() { + for (matrix, extra) in [("/Matrix null", &b"null"[..]), ("/Matrix 8 0 R", b"null")] { + let mut doc = moved_form(matrix, extra); + mark(&mut doc, 0, &[MARK], ""); + assert_eq!(apply(&mut doc, None).map(|r| r.glyphs), Ok(0), "{matrix}"); + let mut doc = moved_form(matrix, extra); + mark(&mut doc, 0, &[[0.0, 0.0, 300.0, 100.0]], ""); + assert_eq!(apply(&mut doc, None).map(|r| r.glyphs), Ok(6), "{matrix}, marked where it stands"); + } +} + +/// A `/Matrix` that isn't six numbers: where the form draws can't be told, so neither can whether +/// it reaches the mark, and the page is refused. +#[test] +fn a_form_whose_matrix_is_not_six_numbers_is_refused() { + for (matrix, extra) in [ + ("/Matrix [1 0 0 1 0 (up)]", &b"null"[..]), + ("/Matrix [1 0 0 1 0]", b"null"), + ("/Matrix [1 0 0 1 0 8 0 R]", b"(up)"), + ("/Matrix 8 0 R", b"(up)"), + ] { + let mut doc = moved_form(matrix, extra); + mark(&mut doc, 0, &[MARK], ""); + let result = apply(&mut doc, None); + assert!( + matches!(&result, Err(RedactError::Cos(pdfcraft_cos::CosError::Syntax { detail, .. })) if detail.contains("/Matrix")), + "{matrix} with 8 0 obj {}: {result:?}", + String::from_utf8_lossy(extra) + ); + } +} + +/// The scans count such a form as content that may remain: the verify pass under a mark, and +/// Remove Hidden Information's scan, whose removal then refuses the page. +#[test] +fn a_form_whose_matrix_is_not_six_numbers_counts_in_the_scans() { + use crate::sanitize::{Hidden, remove_hidden, scan}; + let mut doc = moved_form("/Matrix [1 0 0 1 0 (up)]", b"null"); + assert_eq!(under(&mut doc, 0, &[MARK]), 1); + let counts: std::collections::HashMap = scan(&doc).into_iter().collect(); + assert_eq!(counts[&Hidden::HiddenText], 1); + let removed = remove_hidden(&mut doc, &[Hidden::HiddenText]); + assert!(matches!(&removed, Err(RedactError::Cos(pdfcraft_cos::CosError::Syntax { detail, .. })) if detail.contains("/Matrix")), "{removed:?}"); +} + +/// Remove Hidden Information reads it the same way: text a matrix entry by reference moves off the +/// page is hidden text, and goes. +#[test] +fn text_a_matrix_entry_by_reference_moves_off_the_page_is_hidden_text() { + use crate::sanitize::{Hidden, remove_hidden}; + let mut doc = moved_form("/Matrix [1 0 0 1 0 8 0 R]", b"1000"); + let removed = remove_hidden(&mut doc, &[Hidden::HiddenText]); + assert!(removed.as_ref().is_ok_and(|r| r.iter().any(|(h, n)| *h == Hidden::HiddenText && *n > 0)), "{removed:?}"); +} diff --git a/crates/redact/src/verify/extract.rs b/crates/redact/src/verify/extract.rs index fe92e4e78d..8ef3d406cd 100644 --- a/crates/redact/src/verify/extract.rs +++ b/crates/redact/src/verify/extract.rs @@ -396,13 +396,18 @@ pub(super) fn push_squashed(into: &mut String, s: &str) { into.extend(s.chars().filter(|c| !c.is_whitespace() && *c != '\u{feff}')); } -/// A form's or pattern's `/Matrix`: absent is the identity, present but malformed is an error -/// (guessing the identity would put its content in the wrong place). -pub(super) fn matrix_of(doc: &Document, dict: &Dict) -> Result { +/// A form's or pattern's `/Matrix`, the array and each entry direct or by reference: absent or null +/// is the identity, present but malformed is an error (guessing the identity would put its content in the +/// wrong place). The redaction walk reads a form's with it too. +pub(crate) fn matrix_of(doc: &Document, dict: &Dict) -> Result { match dict.get(b"Matrix") { None => Ok(Matrix::IDENTITY), Some(m) => { let m = doc.resolve(m); + // A null value is no entry (ISO 32000-2 §7.3.9), whether given directly or by reference. + if matches!(*m, Object::Null) { + return Ok(Matrix::IDENTITY); + } let items: Vec = m.as_array().ok_or(())?.iter().map(|x| (*doc.resolve(x)).clone()).collect(); Matrix::from_operands(&items).filter(|m| m.0.iter().all(|v| v.is_finite())).ok_or(()) } diff --git a/crates/redact/src/verify/mod.rs b/crates/redact/src/verify/mod.rs index 3a36c9e08e..d58d4c307c 100644 --- a/crates/redact/src/verify/mod.rs +++ b/crates/redact/src/verify/mod.rs @@ -70,6 +70,7 @@ mod snapshot; mod sweep; mod util; +pub(crate) use extract::matrix_of; pub use snapshot::Snapshot; /// What the proof established for one region. #[derive(Clone, Copy, Debug, PartialEq, Eq)] From cf26980471a668b52eaa527a3b4212f16c4c187d Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Tue, 10 Nov 2026 16:45:34 +0100 Subject: [PATCH 10/60] M8: a long form /Matrix is read cheaply, an unreadable one leaves layer removal alone Review of the walk's read of a form's `/Matrix`. Removing hidden layers alone never asks where content lands, but a form whose `/Matrix` isn't six numbers was counted in its scan and refused, in a document without layers too. Such a form is now walked as it stands there; Apply Redactions, and Remove Hidden Information with hidden text, which tells hidden text by where it lands, still refuse it. `matrix_of` resolved and copied every entry before checking there were six, and the walk reads it at every `Do`. Measured in review (debug build), 20,000 draws of a form whose `/Matrix` is 20,000 entries long took 9 s by reference (16 ms before the walk read entries) and 12 s written directly (2.6 s before, the array being copied by `resolve`). It now reads an array written directly in place, and checks the length before resolving any entry. Tests: `a_form_whose_matrix_is_not_six_numbers_leaves_removing_hidden_layers_alone`, and `tests/matrix_work.rs` (a binary of its own), whose `a_long_form_matrix_costs_no_more_than_a_short_one` times 4,000 draws of a form with a 4,000-entry `/Matrix` against a 7-entry one, written directly and by reference. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/redact/src/interp.rs | 34 +++++++++------ crates/redact/src/tests/form_matrix.rs | 11 +++++ crates/redact/src/verify/extract.rs | 26 +++++++----- crates/redact/tests/matrix_work.rs | 59 ++++++++++++++++++++++++++ 4 files changed, 106 insertions(+), 24 deletions(-) create mode 100644 crates/redact/tests/matrix_work.rs diff --git a/crates/redact/src/interp.rs b/crates/redact/src/interp.rs index 04fafd283d..f08ab2da8d 100644 --- a/crates/redact/src/interp.rs +++ b/crates/redact/src/interp.rs @@ -1073,21 +1073,27 @@ fn xobject( Some(b"Form") => { // Read as the proof reads it, entries by reference included: read as the identity, a // form moved onto a mark would be tested against the wrong place and never walked. - let Ok(m) = crate::verify::matrix_of(doc, &s.dict) else { - // Not six numbers: where the form draws, and so whether it reaches a mark, can't be - // told. Apply refuses the page, and the scans count it, as for a form past the - // page's budget below. - if scope.mode == Mode::Apply { - out.error = Some(CosError::Syntax { - offset: 0, - detail: "a form XObject's /Matrix is not six numbers, so where it draws can't be told".into(), - }); - } else if scope.everywhere() { - scope.layer_blocks += 1; - } else { - out.residue += 1; + let m = match crate::verify::matrix_of(doc, &s.dict) { + Ok(m) => m, + // Not six numbers. Removing hidden layers alone never asks where content lands, so + // the form is walked as it stands. + Err(()) if scope.everywhere() && scope.hidden_text.is_none() => Matrix::IDENTITY, + // Otherwise where the form draws, and so whether it reaches a mark or leaves the + // page, can't be told. Apply refuses the page, and the scans count it, as for a + // form past the page's budget below. + Err(()) => { + if scope.mode == Mode::Apply { + out.error = Some(CosError::Syntax { + offset: 0, + detail: "a form XObject's /Matrix is not six numbers, so where it draws can't be told".into(), + }); + } else if scope.everywhere() { + scope.layer_blocks += 1; + } else { + out.residue += 1; + } + return None; } - return None; }; let fctm = m.then(&gs.ctm); let bbox = s.dict.get(b"BBox").and_then(|b| { diff --git a/crates/redact/src/tests/form_matrix.rs b/crates/redact/src/tests/form_matrix.rs index c3e386f47d..300c2933a1 100644 --- a/crates/redact/src/tests/form_matrix.rs +++ b/crates/redact/src/tests/form_matrix.rs @@ -85,6 +85,17 @@ fn a_form_whose_matrix_is_not_six_numbers_counts_in_the_scans() { assert!(matches!(&removed, Err(RedactError::Cos(pdfcraft_cos::CosError::Syntax { detail, .. })) if detail.contains("/Matrix")), "{removed:?}"); } +/// Removing hidden layers alone never asks where content lands, so such a form is walked as it +/// stands: nothing is counted or refused over it in a document without layers. +#[test] +fn a_form_whose_matrix_is_not_six_numbers_leaves_removing_hidden_layers_alone() { + use crate::sanitize::{Hidden, remove_hidden, scan}; + let mut doc = moved_form("/Matrix [1 0 0 1 0 (up)]", b"null"); + let counts: std::collections::HashMap = scan(&doc).into_iter().collect(); + assert_eq!(counts[&Hidden::HiddenLayers], 0); + assert_eq!(remove_hidden(&mut doc, &[Hidden::HiddenLayers]), Err(RedactError::NothingToApply)); +} + /// Remove Hidden Information reads it the same way: text a matrix entry by reference moves off the /// page is hidden text, and goes. #[test] diff --git a/crates/redact/src/verify/extract.rs b/crates/redact/src/verify/extract.rs index 8ef3d406cd..bc46348715 100644 --- a/crates/redact/src/verify/extract.rs +++ b/crates/redact/src/verify/extract.rs @@ -400,18 +400,24 @@ pub(super) fn push_squashed(into: &mut String, s: &str) { /// is the identity, present but malformed is an error (guessing the identity would put its content in the /// wrong place). The redaction walk reads a form's with it too. pub(crate) fn matrix_of(doc: &Document, dict: &Dict) -> Result { - match dict.get(b"Matrix") { - None => Ok(Matrix::IDENTITY), - Some(m) => { - let m = doc.resolve(m); - // A null value is no entry (ISO 32000-2 §7.3.9), whether given directly or by reference. - if matches!(*m, Object::Null) { - return Ok(Matrix::IDENTITY); - } - let items: Vec = m.as_array().ok_or(())?.iter().map(|x| (*doc.resolve(x)).clone()).collect(); - Matrix::from_operands(&items).filter(|m| m.0.iter().all(|v| v.is_finite())).ok_or(()) + let Some(m) = dict.get(b"Matrix") else { return Ok(Matrix::IDENTITY) }; + // Read in place when written directly, and the length checked before any entry is resolved: + // the walk reads it at every `Do`, and an array of any length would be copied each time. + let held; + let m = match m { + Object::Ref(_) => { + held = doc.resolve(m); + &*held } + direct => direct, + }; + // A null value is no entry (ISO 32000-2 §7.3.9), whether given directly or by reference. + if matches!(m, Object::Null) { + return Ok(Matrix::IDENTITY); } + let items = m.as_array().filter(|a| a.len() == 6).ok_or(())?; + let items: Vec = items.iter().map(|x| (*doc.resolve(x)).clone()).collect(); + Matrix::from_operands(&items).filter(|m| m.0.iter().all(|v| v.is_finite())).ok_or(()) } /// A form's `/BBox` as `[x0 y0 x1 y1]`: absent is `None`, present but malformed is an error. diff --git a/crates/redact/tests/matrix_work.rs b/crates/redact/tests/matrix_work.rs new file mode 100644 index 0000000000..d96447ff71 --- /dev/null +++ b/crates/redact/tests/matrix_work.rs @@ -0,0 +1,59 @@ +//! Work regression for a form's `/Matrix`, which the redaction walk reads at every `Do` of the +//! form: an array of any length costs no more than one of seven entries, written directly or by +//! reference. A test binary of its own, so that none of the crate's other tests shares its process. +//! The test compares two timings of its own, the least of three runs each, not a fixed time. + +use std::sync::Arc; +use std::time::{Duration, Instant}; + +use pdfcraft_annot::{Meta, NewAnnotation, Shape, Style, add_annotation, rect_quad}; +use pdfcraft_cos::Document; +use pdfcraft_redact::apply; +use pdfcraft_testkit::{Pdf, stream_obj}; + +/// One page drawing a form `draws` times under a mark, the form's `/Matrix` an array of `entries` +/// zeros, written in its dictionary or, `by_reference`, as object 6. +fn document(draws: usize, entries: usize, by_reference: bool) -> Document { + let zeros = format!("[{}]", vec!["0"; entries].join(" ")); + let matrix = if by_reference { "6 0 R".to_string() } else { zeros.clone() }; + let objs = vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /XObject << /Fm 5 0 R >> >> >>".to_vec(), + stream_obj("", "/Fm Do ".repeat(draws).as_bytes()), + stream_obj(&format!("/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Matrix {matrix}"), b"0 0 10 10 re f"), + zeros.into_bytes(), + ]; + let mut doc = Document::open(Arc::new(Pdf::new().objs(objs).build())).unwrap(); + let shape = Shape::Redact { quads: vec![rect_quad([0.0, 0.0, 300.0, 300.0])], overlay: String::new(), look: Default::default() }; + let style = Style::default_for(&shape); + add_annotation(&mut doc, &NewAnnotation { page: 0, shape, style, contents: String::new(), author: "Synthetic test".into() }, &Meta::default()) + .unwrap(); + doc +} + +/// How long redacting `document(draws, entries, by_reference)` takes, the least of three runs. A +/// matrix that isn't six numbers is refused. +fn redaction(draws: usize, entries: usize, by_reference: bool) -> Duration { + (0..3) + .map(|_| { + let mut doc = document(draws, entries, by_reference); + let start = Instant::now(); + let refused = apply(&mut doc, None).is_err(); + let took = start.elapsed(); + assert!(refused, "a /Matrix of {entries} entries is refused"); + took + }) + .min() + .unwrap_or_default() +} + +/// Written when the walk resolved and copied every entry before checking there were six: 4,000 +/// draws of a form whose `/Matrix` is 4,000 entries long read sixteen million entries. +#[test] +fn a_long_form_matrix_costs_no_more_than_a_short_one() { + for by_reference in [false, true] { + let (short, long) = (redaction(4_000, 7, by_reference), redaction(4_000, 4_000, by_reference)); + assert!(long < short * 3 + Duration::from_millis(50), "by reference: {by_reference}: long {long:?}, short {short:?}"); + } +} From 4f107a6f4065e0d2fabbf4ea8318ec60f2f20dee Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Tue, 10 Nov 2026 16:45:34 +0100 Subject: [PATCH 11/60] M8: a form /Matrix through a reference to a reference is refused Review of the walk's read of a form's `/Matrix`. Through `Document::resolve` it followed a reference to a reference, for the matrix and for each entry, as the app's renderer does; Poppler doesn't, and reads such a matrix as the identity and such an entry as 0. With `/Matrix [1 0 0 1 0 8 0 R]`, `8 0 obj 9 0 R` and `9 0 obj 200`, a mark where Poppler shows the text applied with success and left it in the file, where `main`, reading any entry by reference as the identity, removed it. `matrix_of` now follows each reference one hop, and a reference to a reference fails as a malformed matrix: where the content draws can't be told, so Apply refuses the page, as for a matrix that isn't six numbers. The refusal now says "isn't six numbers given directly or through one reference", and `matrix_of`'s doc names the appearances it reads, not patterns. The proof, which reads the matrix of forms and appearances with it, treats such a matrix as malformed too. Test: `a_form_whose_matrix_goes_through_a_reference_to_a_reference_is_refused` (an entry and the whole matrix, with a mark where either reader draws the text). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/redact/src/interp.rs | 6 ++++-- crates/redact/src/tests/form_matrix.rs | 24 ++++++++++++++++++++++++ crates/redact/src/verify/extract.rs | 26 ++++++++++++++++++-------- 3 files changed, 46 insertions(+), 10 deletions(-) diff --git a/crates/redact/src/interp.rs b/crates/redact/src/interp.rs index f08ab2da8d..9cfc981381 100644 --- a/crates/redact/src/interp.rs +++ b/crates/redact/src/interp.rs @@ -1075,7 +1075,7 @@ fn xobject( // form moved onto a mark would be tested against the wrong place and never walked. let m = match crate::verify::matrix_of(doc, &s.dict) { Ok(m) => m, - // Not six numbers. Removing hidden layers alone never asks where content lands, so + // Not six numbers given directly or through one reference. Removing hidden layers alone never asks where content lands, so // the form is walked as it stands. Err(()) if scope.everywhere() && scope.hidden_text.is_none() => Matrix::IDENTITY, // Otherwise where the form draws, and so whether it reaches a mark or leaves the @@ -1085,7 +1085,9 @@ fn xobject( if scope.mode == Mode::Apply { out.error = Some(CosError::Syntax { offset: 0, - detail: "a form XObject's /Matrix is not six numbers, so where it draws can't be told".into(), + detail: + "a form XObject's /Matrix isn't six numbers given directly or through one reference, so where it draws can't be told" + .into(), }); } else if scope.everywhere() { scope.layer_blocks += 1; diff --git a/crates/redact/src/tests/form_matrix.rs b/crates/redact/src/tests/form_matrix.rs index 300c2933a1..e4bb33608e 100644 --- a/crates/redact/src/tests/form_matrix.rs +++ b/crates/redact/src/tests/form_matrix.rs @@ -72,6 +72,30 @@ fn a_form_whose_matrix_is_not_six_numbers_is_refused() { } } +/// A reference to a reference, for the matrix or one of its entries: Poppler doesn't follow it (it +/// reads such a matrix as the identity, such an entry as 0), the app's renderer follows it and +/// reads the move, so where the form draws can't be told, and the page is refused, as for a matrix that isn't six numbers. Read as either, +/// a mark where the other reader shows the text would leave it in the file. +#[test] +fn a_form_whose_matrix_goes_through_a_reference_to_a_reference_is_refused() { + for (matrix, target) in [("/Matrix [1 0 0 1 0 8 0 R]", &b"200"[..]), ("/Matrix 8 0 R", b"[1 0 0 1 0 200]")] { + for mark_at in [MARK, [0.0, 0.0, 300.0, 100.0]] { + let form = stream( + &format!("/Type /XObject /Subtype /Form /BBox [0 0 300 100] {matrix} /Resources << /Font << /F1 5 0 R >> >>"), + b"BT /F1 10 Tf 20 60 Td (SECRET) Tj ET", + ); + let mut doc = one_page(b"/Fm Do", "/XObject << /Fm 7 0 R >>", vec![form, b"9 0 R".to_vec(), target.to_vec()]); + mark(&mut doc, 0, &[mark_at], ""); + let result = apply(&mut doc, None); + assert!( + matches!(&result, Err(RedactError::Cos(pdfcraft_cos::CosError::Syntax { detail, .. })) if detail.contains("/Matrix")), + "{matrix}, 8 0 obj 9 0 R, 9 0 obj {}, mark at {mark_at:?}: {result:?}", + String::from_utf8_lossy(target) + ); + } + } +} + /// The scans count such a form as content that may remain: the verify pass under a mark, and /// Remove Hidden Information's scan, whose removal then refuses the page. #[test] diff --git a/crates/redact/src/verify/extract.rs b/crates/redact/src/verify/extract.rs index bc46348715..63dbeff342 100644 --- a/crates/redact/src/verify/extract.rs +++ b/crates/redact/src/verify/extract.rs @@ -396,17 +396,20 @@ pub(super) fn push_squashed(into: &mut String, s: &str) { into.extend(s.chars().filter(|c| !c.is_whitespace() && *c != '\u{feff}')); } -/// A form's or pattern's `/Matrix`, the array and each entry direct or by reference: absent or null +/// A form's or an annotation appearance's `/Matrix`, the array and each entry direct or by reference: absent or null /// is the identity, present but malformed is an error (guessing the identity would put its content in the -/// wrong place). The redaction walk reads a form's with it too. +/// wrong place). A reference to a reference is malformed here: readers disagree on following it +/// (Poppler doesn't, the app's renderer does), so where the content draws can't be told. The +/// redaction walk reads a form's with it too. pub(crate) fn matrix_of(doc: &Document, dict: &Dict) -> Result { let Some(m) = dict.get(b"Matrix") else { return Ok(Matrix::IDENTITY) }; - // Read in place when written directly, and the length checked before any entry is resolved: - // the walk reads it at every `Do`, and an array of any length would be copied each time. + // Each reference is followed one hop. The array is read in place when written directly, and + // its length checked before any entry is read: the walk reads it at every `Do`, and an array of + // any length would be copied each time. let held; let m = match m { - Object::Ref(_) => { - held = doc.resolve(m); + Object::Ref(r) => { + held = doc.get(*r); &*held } direct => direct, @@ -416,8 +419,15 @@ pub(crate) fn matrix_of(doc: &Document, dict: &Dict) -> Result { return Ok(Matrix::IDENTITY); } let items = m.as_array().filter(|a| a.len() == 6).ok_or(())?; - let items: Vec = items.iter().map(|x| (*doc.resolve(x)).clone()).collect(); - Matrix::from_operands(&items).filter(|m| m.0.iter().all(|v| v.is_finite())).ok_or(()) + let mut values = [0.0; 6]; + for (value, item) in values.iter_mut().zip(items) { + let n = match item { + Object::Ref(r) => doc.get(*r).as_f64(), + direct => direct.as_f64(), + }; + *value = n.filter(|n| n.is_finite()).ok_or(())?; + } + Ok(Matrix(values)) } /// A form's `/BBox` as `[x0 y0 x1 y1]`: absent is `None`, present but malformed is an error. From b16f0c0a7729019a73f5c128da329b384300df8d Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Tue, 10 Nov 2026 17:25:07 +0100 Subject: [PATCH 12/60] M8: redaction reads an operator's operands from the end, as the renderer does An operator takes its operands from the end, and the renderer ignores a surplus before them: `(x) /OC /MC0 BDC` marks a layer's block, and `(x) /GS1 gs` selects `/GS1`. Redaction read some of them from the front, so a surplus stood in for them and what they named went unseen. Found in review of #6882, where Remove Hidden Information left the text of a hidden layer in `(x) /OC /MC0 BDC (LAYERSECRET) Tj EMC`, which both the app's renderer and Poppler hide. In what Apply Redactions and Remove Hidden Information decide: - the layers walk read a block's tag and property list from the front, and kept a hidden layer's block; - the cleanup of `/ActualText`, `/Alt`, `/E` and `/T` in a block whose paint was removed read (and wrote) the property list from the front, and kept the name that still read the removed word; - the tags cleanup read a block's `MCID` from the front, and left the removed text in its structure element's `/ActualText` and `/Alt`; and a `TJ`'s array, so a block that still drew was taken for empty and its structure element lost the content the page still shows; - the property cleanups read the name of a `BDC` or `DP` property list from the front, and retired a property list the page still draws with; - the walk read the ExtGState a `gs` selects from the front, so a font set only that way was not set, and `cs`/`CS` the same for a pattern space; - the Type 3 cleanup's reads of `cs`, `gs`, `Tf`, `BDC` and `DP`, which decide which resources and fonts are still used. The redaction proof read `Tc`, `Tw`, `Tz`, `TL`, `Ts`, `Tf`, `"`, `TJ` and `Do` from the front too. `pdfcraft_content::Op` gains `back`, `name_back` and `num_back`: operand `k` counted from the end. Tests (`crates/redact/src/tests/operands_from_the_end.rs`): a hidden layer's block, `/ActualText` on removed paint, a tag's `MCID`, a tagged `TJ` that still draws, a property list still drawn with, a font from an ExtGState, the pattern a `/Pattern cs` keeps, and a Type 3 font with the ExtGState, colour space, property list and font its glyph uses, each after a surplus operand. In `tests_verify.rs`, `the_proof_reads_operands_from_the_end` (`Tf`, `TJ`, and `Do` of a form and of images), `the_proof_reads_the_text_states_operands_from_the_end` (`Tc`, `Tw`, `Tz`, `TL`, `Ts` and `"`) and `an_image_a_type3_glyph_draws_after_a_surplus_operand_is_found`. And `operands_are_counted_from_the_end` in the content crate. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/content/src/lib.rs | 17 ++ crates/content/src/tests.rs | 11 ++ crates/redact/src/interp.rs | 11 +- crates/redact/src/interp/layers.rs | 6 +- crates/redact/src/interp/prune.rs | 4 +- crates/redact/src/tags.rs | 7 +- crates/redact/src/tests.rs | 1 + .../redact/src/tests/operands_from_the_end.rs | 170 ++++++++++++++++++ crates/redact/src/tests_verify.rs | 72 ++++++++ crates/redact/src/type3.rs | 20 +-- crates/redact/src/verify/extract.rs | 23 +-- crates/redact/src/verify/geometry.rs | 4 +- 12 files changed, 310 insertions(+), 36 deletions(-) create mode 100644 crates/redact/src/tests/operands_from_the_end.rs diff --git a/crates/content/src/lib.rs b/crates/content/src/lib.rs index fb17a903d1..f77c54e1dc 100644 --- a/crates/content/src/lib.rs +++ b/crates/content/src/lib.rs @@ -58,6 +58,23 @@ impl Op { pub fn last_name(&self) -> Option<&[u8]> { self.operands.last().and_then(Object::as_name) } + + /// Operand `k` counted from the end (0 is the last), as the renderer reads an operator's + /// operands: `(x) /OC /MC0 BDC` marks with `/OC /MC0`, and `(x) /F1 10 Tf` selects `/F1`. + pub fn back(&self, k: usize) -> Option<&Object> { + let i = self.operands.len().checked_sub(k)?.checked_sub(1)?; + self.operands.get(i) + } + + /// [`Op::back`] as a name. + pub fn name_back(&self, k: usize) -> Option<&[u8]> { + self.back(k).and_then(Object::as_name) + } + + /// [`Op::back`] as a number. + pub fn num_back(&self, k: usize) -> Option { + self.back(k).and_then(Object::as_f64) + } } /// A parsed stream. diff --git a/crates/content/src/tests.rs b/crates/content/src/tests.rs index 9ca12c4536..89f258853d 100644 --- a/crates/content/src/tests.rs +++ b/crates/content/src/tests.rs @@ -10,6 +10,17 @@ thread_local! { pub(crate) static PIECE_VISITS: std::cell::Cell = const { std::cell::Cell::new(0) }; } +#[test] +fn operands_are_counted_from_the_end() { + let op = parse(b"(x) /OC /MC0 BDC").ops.remove(0); + assert_eq!((op.name_back(1), op.name_back(0)), (Some(&b"OC"[..]), Some(&b"MC0"[..]))); + assert_eq!(op.back(2).and_then(Object::as_string).map(|s| s.bytes.clone()), Some(b"x".to_vec())); + assert_eq!((op.back(3), op.back(usize::MAX)), (None, None)); + let op = parse(b"/F1 10 Tf").ops.remove(0); + assert_eq!((op.name_back(1), op.num_back(0), op.num_back(1)), (Some(&b"F1"[..]), Some(10.0), None)); + assert_eq!(Op::new("Q", vec![]).back(0), None); +} + #[test] fn splicing_visits_each_piece_a_few_times() { let n = 5_000; diff --git a/crates/redact/src/interp.rs b/crates/redact/src/interp.rs index 1e69cb32e2..b470d7984e 100644 --- a/crates/redact/src/interp.rs +++ b/crates/redact/src/interp.rs @@ -571,7 +571,7 @@ impl Walk { Some(op) } b"cs" | b"CS" => { - if op.name(0) != Some(b"Pattern") { + if op.last_name() != Some(b"Pattern") { if o == b"cs" { self.gs.fill_pattern = None; } else { @@ -656,10 +656,10 @@ impl Walk { } b"Tf" => self.select_font(doc, scope, op), _ => { - if let Some((font, vertical, size)) = op.name(0).and_then(|n| extgstate_font(doc, scope, &self.ext_gstates, n, &self.fallback)) { + if let Some((font, vertical, size)) = op.last_name().and_then(|n| extgstate_font(doc, scope, &self.ext_gstates, n, &self.fallback)) { (self.gs.font, self.gs.vertical, self.gs.size) = (font, vertical, size); // As for `Tf`: a Type 3 font without /Resources paints through this scope's. - self.gs.type3_inherits = op.name(0).is_some_and(|n| extgstate_type3_inherits(doc, &self.ext_gstates, n)); + self.gs.type3_inherits = op.last_name().is_some_and(|n| extgstate_type3_inherits(doc, &self.ext_gstates, n)); } } } @@ -864,7 +864,8 @@ fn clean_replacements( if !op.is("BDC") || !affected.contains(&op.span.start) { continue; } - let property = match op.operands.get(1) { + // The property list is the last operand, as the renderer reads it. + let property = match op.back(0) { Some(Object::Name(name)) => properties.get(name).map(|p| doc.resolve(p)), Some(p) => Some(doc.resolve(p)), None => None, @@ -874,7 +875,7 @@ fn clean_replacements( for key in [&b"ActualText"[..], b"Alt", b"E", b"T"] { stripped |= dict.remove(key).is_some(); } - if stripped && let Some(property) = op.operands.get_mut(1) { + if stripped && let Some(property) = op.operands.last_mut() { *property = Object::Dict(dict); *changed = true; report.replacement_text += 1; diff --git a/crates/redact/src/interp/layers.rs b/crates/redact/src/interp/layers.rs index 88d1c4e677..362cb36620 100644 --- a/crates/redact/src/interp/layers.rs +++ b/crates/redact/src/interp/layers.rs @@ -225,8 +225,10 @@ pub(super) fn strip_hidden_layers(doc: &Document, scope: &mut Scope<'_>, ops: Ve match op.op.as_slice() { b"BDC" | b"BMC" => { let this = op.is("BDC") - && op.name(0) == Some(b"OC") - && match op.operands.get(1) { + // The tag and the property list are the last two operands, as the renderer + // reads them: `(x) /OC /MC0 BDC` is a layer's block too. + && op.name_back(1) == Some(b"OC") + && match op.back(0) { Some(Object::Name(n)) => props.get(n).is_some_and(|o| scope.layer_hidden(doc, o)), Some(o) => scope.layer_hidden(doc, o), None => false, diff --git a/crates/redact/src/interp/prune.rs b/crates/redact/src/interp/prune.rs index 39089601ec..6b0be5d977 100644 --- a/crates/redact/src/interp/prune.rs +++ b/crates/redact/src/interp/prune.rs @@ -357,7 +357,7 @@ fn prune_retired_properties(doc: &Document, res: &mut Dict, names: &[Vec], c // Each is one page's content or one form's, with the allowance of a page. for op in resource_ops(doc, &mut DecodeBudget::page(), res, content)? { if matches!(op.op.as_slice(), b"BDC" | b"DP") - && let Some(name) = op.name(1) + && let Some(name) = op.last_name() { used.insert(name.to_vec()); } @@ -520,7 +520,7 @@ pub(crate) fn prune_properties(doc: &Document, budget: &mut DecodeBudget, res: & let ops = resource_ops(doc, budget, res, content)?; let names: Vec> = properties.iter().map(|(n, _)| n.to_vec()).collect(); for name in names { - if !ops.iter().any(|op| matches!(op.op.as_slice(), b"BDC" | b"DP") && op.name(1) == Some(&name)) + if !ops.iter().any(|op| matches!(op.op.as_slice(), b"BDC" | b"DP") && op.last_name() == Some(&name)) && let Some(value) = properties.remove(&name) { retired.push(value); diff --git a/crates/redact/src/tags.rs b/crates/redact/src/tags.rs index 648c0c29e5..9cccb93665 100644 --- a/crates/redact/src/tags.rs +++ b/crates/redact/src/tags.rs @@ -52,7 +52,8 @@ fn by_mcid(doc: &Document, resources: &Dict, data: &[u8]) -> Marked { for op in pdfcraft_content::parse(data).ops { match op.op.as_slice() { b"BDC" => { - let id = match op.operands.get(1) { + // The property list is the last operand, as the renderer reads it. + let id = match op.back(0) { Some(Object::Name(n)) => resources.get(b"Properties").and_then(|p| { let props = doc.resolve(p); let prop = doc.resolve(props.as_dict()?.get(n)?); @@ -86,9 +87,7 @@ fn by_mcid(doc: &Document, resources: &Dict, data: &[u8]) -> Marked { } let draws = match op.op.as_slice() { b"Tj" | b"'" | b"\"" => op.operands.last().and_then(Object::as_string).is_some_and(|s| !s.bytes.is_empty()), - b"TJ" => { - op.operands.first().and_then(Object::as_array).is_some_and(|a| a.iter().any(|x| x.as_string().is_some_and(|s| !s.bytes.is_empty()))) - } + b"TJ" => op.back(0).and_then(Object::as_array).is_some_and(|a| a.iter().any(|x| x.as_string().is_some_and(|s| !s.bytes.is_empty()))), b"f" | b"F" | b"f*" | b"B" | b"B*" | b"b" | b"b*" | b"S" | b"s" | b"sh" | b"Do" | b"BI" => true, _ => false, }; diff --git a/crates/redact/src/tests.rs b/crates/redact/src/tests.rs index 46ac131450..015a574d4a 100644 --- a/crates/redact/src/tests.rs +++ b/crates/redact/src/tests.rs @@ -6149,6 +6149,7 @@ mod inline_images; mod limits; mod metadata_fail_closed; mod metadata_matching; +mod operands_from_the_end; mod property_retirement; mod surviving_references; mod text_state; diff --git a/crates/redact/src/tests/operands_from_the_end.rs b/crates/redact/src/tests/operands_from_the_end.rs new file mode 100644 index 0000000000..d2d6640121 --- /dev/null +++ b/crates/redact/src/tests/operands_from_the_end.rs @@ -0,0 +1,170 @@ +//! An operator takes its operands from the end, as the renderer reads them, and a surplus before +//! them is ignored: `(x) /OC /MC0 BDC` marks a layer's block, `(x) /GS1 gs` selects `/GS1`. Read +//! from the front, the surplus stood in for the operands, and what they named went unseen. + +use super::*; + +/// Hidden-layer text in a block whose tag and property list follow a string: Remove Hidden +/// Information found nothing to remove, and the text stayed. +#[test] +fn a_hidden_layers_block_after_a_surplus_operand_is_removed() { + use crate::sanitize::{Hidden, remove_hidden}; + let mut doc = one_page( + b"(x) /OC /MC0 BDC BT /F1 10 Tf 20 260 Td (LAYERSECRET) Tj ET EMC BT /F1 10 Tf 20 20 Td (PUBLIC) Tj ET", + "/Properties << /MC0 7 0 R >>", + vec![b"<< /Type /OCG /Name (Hidden) >>".to_vec()], + ); + doc.update_dict(ObjRef::new(1, 0), |d| { + d.set( + b"OCProperties".to_vec(), + Object::Dict(Dict::from_iter([ + (b"OCGs".to_vec(), Object::Array(vec![Object::Ref(ObjRef::new(7, 0))])), + (b"D".to_vec(), Object::Dict(Dict::from_iter([(b"OFF".to_vec(), Object::Array(vec![Object::Ref(ObjRef::new(7, 0))]))]))), + ])), + ); + }) + .unwrap(); + let removed = remove_hidden(&mut doc, &[Hidden::HiddenLayers]); + assert!(removed.as_ref().is_ok_and(|r| r.iter().any(|(h, n)| *h == Hidden::HiddenLayers && *n > 0)), "{removed:?}"); + let c = content(&doc, 0); + assert!(!c.contains("LAYERSECRET") && c.contains("PUBLIC"), "{c}"); +} + +/// Paint under a mark named by `/ActualText` in a block after a surplus operand: the name, which +/// would still read the word, kept it. +#[test] +fn actual_text_after_a_surplus_operand_goes_with_the_paint_it_names() { + let mut doc = one_page(b"(x) /Span <> BDC 100 100 m 160 100 l 160 120 l 100 120 l h f EMC", "", vec![]); + mark(&mut doc, 0, &[[90.0, 95.0, 130.0, 125.0]], ""); + let r = apply(&mut doc, None).unwrap(); + assert_eq!(r.paths_removed, 1); + let c = content(&doc, 0); + assert!(!c.contains("SECRET"), "{c}"); +} + +/// A tagged paragraph whose `MCID` follows a surplus operand: its structure element kept the +/// removed text in `/ActualText` and `/Alt`. +#[test] +fn a_tag_after_a_surplus_operand_loses_what_redaction_removed() { + let content = b"(x) /P <> BDC BT /F1 10 Tf 10 200 Td (SECRET) Tj ET EMC"; + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R /StructTreeRoot 7 0 R /MarkInfo << /Marked true >> >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /StructParents 0 /Resources << /Font << /F1 5 0 R >> >> >>".to_vec(), + stream("", content), + FONT.replace("95 0 R", "6 0 R").into_bytes(), + widths(), + b"<< /Type /StructTreeRoot /K 8 0 R >>".to_vec(), + b"<< /S /Document /P 7 0 R /K [9 0 R] >>".to_vec(), + b"<< /S /P /P 8 0 R /Pg 3 0 R /ActualText (SECRET) /Alt (the secret) /K 0 >>".to_vec(), + ]); + mark(&mut doc, 0, &[[5.0, 195.0, 60.0, 212.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().tags, 1); + let secret = doc.get(ObjRef::new(9, 0)).as_dict().cloned().unwrap(); + assert!(secret.get(b"ActualText").is_none() && secret.get(b"Alt").is_none(), "{secret:?}"); +} + +/// A property list still drawn with, named after a surplus operand, stays in the page's resources +/// when the redaction cleans them; read from the front, it was taken for unused and retired. +#[test] +fn a_property_list_named_after_a_surplus_operand_is_kept() { + let mut doc = one_page( + b"BT /F1 10 Tf 20 260 Td (SECRET) Tj ET (x) /Span /Pr1 BDC BT /F1 10 Tf 20 20 Td (KEPT) Tj ET EMC", + "/Properties << /Pr1 << /Lang (en-GB) >> >>", + vec![], + ); + mark(&mut doc, 0, &[[0.0, 200.0, 300.0, 300.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().glyphs, 6); + let page = pdfcraft_model::pages(&doc).swap_remove(0); + let res = page.dict.get(b"Resources").map(|r| doc.resolve(r)).and_then(|r| r.as_dict().cloned()).unwrap_or_default(); + let props = res.get(b"Properties").map(|p| doc.resolve(p)).and_then(|p| p.as_dict().cloned()).unwrap_or_default(); + assert!(props.contains(b"Pr1"), "{res:?}"); +} + +/// A font set only by an ExtGState selected after a surplus operand: read from the front, no font +/// was set, every glyph stood at the start of the line with no size, and a mark over the end of +/// the word removed nothing. +#[test] +fn a_font_from_an_extgstate_selected_after_a_surplus_operand_places_the_text() { + let mut doc = one_page(b"BT (x) /GS1 gs 20 260 Td (SECRET) Tj ET", "/ExtGState << /GS1 << /Font [5 0 R 10] >> >>", vec![]); + // Over the last two glyphs, E and T at x = 40 to 50. + mark(&mut doc, 0, &[[42.0, 255.0, 55.0, 275.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().glyphs, 2); +} + +/// `/Pattern cs` after a surplus operand, once the pattern's paint under the mark went: it selects +/// the Pattern space and keeps the pattern, so the fill after it still counts as painting with it +/// and the pattern stays. Read from the front, the surplus cleared the pattern, which was pruned. +/// A grey space clears it. +#[test] +fn a_pattern_space_selected_after_a_surplus_operand_keeps_the_pattern() { + for (space, pruned) in [("(x) /Pattern", 0), ("/Pattern", 0), ("(x) /DeviceGray", 1)] { + let mut doc = one_page( + format!("/Pattern cs /P0 scn 10 260 20 20 re f {space} cs 200 20 50 50 re f").as_bytes(), + "/Pattern << /P0 7 0 R >>", + vec![stream("/PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 10 10] /XStep 10 /YStep 10 /Resources << >>", b"0 0 5 5 re f")], + ); + mark(&mut doc, 0, &[[0.0, 250.0, 40.0, 290.0]], ""); + let r = apply(&mut doc, None).unwrap(); + assert_eq!((r.paths_removed, r.patterns_removed), (1, pruned), "{space}"); + } +} + +/// A Type 3 font selected after a surplus operand, outside the mark, whose glyph selects an +/// ExtGState, a colour space, a property list and a font after one too: the cleanups after the +/// walk took the font for undrawn and those for unused. All stay, with the glyph. +#[test] +fn a_type3_font_and_its_glyphs_resources_named_after_a_surplus_operand_are_kept() { + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /Font << /F1 5 0 R /T3 7 0 R >> >> >>".to_vec(), + stream("", b"BT /F1 10 Tf 20 260 Td (SECRET) Tj ET BT (x) /T3 12 Tf 20 20 Td (a) Tj ET"), + FONT.replace("95 0 R", "6 0 R").into_bytes(), + widths(), + b"<< /Type /Font /Subtype /Type3 /FontBBox [0 0 10 10] /FontMatrix [0.1 0 0 0.1 0 0] /CharProcs << /a 8 0 R >> \ + /Encoding << /Differences [97 /a] >> /FirstChar 97 /LastChar 97 /Widths [10] /Resources << /ExtGState << /GS0 << /CA 0.5 >> >> \ + /ColorSpace << /CS0 /DeviceGray >> /Properties << /P0 << >> >> /Font << /F0 5 0 R >> >> >>" + .to_vec(), + stream("", b"10 0 d0 (x) /GS0 gs (x) /CS0 cs (x) /Span /P0 BDC 0 0 10 10 re f EMC BT (x) /F0 1 Tf ET"), + ]); + mark(&mut doc, 0, &[[0.0, 200.0, 300.0, 300.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().glyphs, 6); + let page = pdfcraft_model::pages(&doc).swap_remove(0); + let res = page.dict.get(b"Resources").map(|r| doc.resolve(r)).and_then(|r| r.as_dict().cloned()).unwrap_or_default(); + let fonts = res.get(b"Font").map(|f| doc.resolve(f)).and_then(|f| f.as_dict().cloned()).unwrap_or_default(); + let font = fonts.get(b"T3").map(|f| doc.resolve(f)).and_then(|f| f.as_dict().cloned()); + let font = font.unwrap_or_else(|| panic!("the drawn Type 3 font stays: {fonts:?}")); + let procs = font.get(b"CharProcs").map(|p| doc.resolve(p)).and_then(|p| p.as_dict().cloned()).unwrap_or_default(); + assert!(procs.contains(b"a"), "its glyph stays: {font:?}"); + let font_res = font.get(b"Resources").map(|r| doc.resolve(r)).and_then(|r| r.as_dict().cloned()).unwrap_or_default(); + let states = font_res.get(b"ExtGState").map(|s| doc.resolve(s)).and_then(|s| s.as_dict().cloned()).unwrap_or_default(); + assert!(states.contains(b"GS0"), "the ExtGState its glyph selects stays: {font_res:?}"); + for (kind, name) in [(&b"ColorSpace"[..], &b"CS0"[..]), (b"Properties", b"P0"), (b"Font", b"F0")] { + let held = font_res.get(kind).map(|d| doc.resolve(d)).and_then(|d| d.as_dict().cloned()).unwrap_or_default(); + assert!(held.contains(name), "the {} its glyph uses stays: {font_res:?}", String::from_utf8_lossy(name)); + } +} + +/// A tagged paragraph whose `TJ` after a surplus operand still draws once the mark has removed the +/// word beside it: read from the front, that `TJ` drew nothing, the paragraph was taken for empty, +/// and its structure element lost the content the page still shows. +#[test] +fn a_tagged_tj_after_a_surplus_operand_still_draws() { + let content = b"/P <> BDC BT /F1 10 Tf 10 200 Td (SECRET) Tj 10 -100 Td (x) [(KEPT)] TJ ET EMC"; + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R /StructTreeRoot 7 0 R /MarkInfo << /Marked true >> >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /StructParents 0 /Resources << /Font << /F1 5 0 R >> >> >>".to_vec(), + stream("", content), + FONT.replace("95 0 R", "6 0 R").into_bytes(), + widths(), + b"<< /Type /StructTreeRoot /K 8 0 R >>".to_vec(), + b"<< /S /Document /P 7 0 R /K [9 0 R] >>".to_vec(), + b"<< /S /P /P 8 0 R /Pg 3 0 R /K 0 >>".to_vec(), + ]); + mark(&mut doc, 0, &[[5.0, 195.0, 60.0, 212.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().glyphs, 6); + let paragraph = doc.get(ObjRef::new(9, 0)).as_dict().cloned().unwrap(); + assert_eq!(paragraph.get(b"K").and_then(Object::as_int), Some(0), "it still tags what the page shows: {paragraph:?}"); +} diff --git a/crates/redact/src/tests_verify.rs b/crates/redact/src/tests_verify.rs index 136cb2b207..3937fcda4f 100644 --- a/crates/redact/src/tests_verify.rs +++ b/crates/redact/src/tests_verify.rs @@ -897,6 +897,78 @@ fn glyphs_are_placed_with_the_text_matrix_and_the_form_matrix() { assert!(proof.entries[0].detail.contains("glyph(s) still show")); } +/// The proof reads an operator's operands from the end, as the renderer does: a surplus before +/// them doesn't hide the text a `Tf`, `TJ` or `Do` after it draws, or the image a `Do` draws. +#[test] +fn the_proof_reads_operands_from_the_end() { + let blank = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()); + let snap = Snapshot::capture(&blank, None).unwrap(); + let form = stream( + "/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Resources << /Font << /F1 5 0 R >> >>", + b"BT /F1 10 Tf 10 100 Td (SECRETVALUE) Tj ET", + ); + let image = stream("/Type /XObject /Subtype /Image /Width 1 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8", b"o"); + let image_form = + stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Resources << /XObject << /Im1 8 0 R >> >>", b"q 50 0 0 12 10 98 cm (x) /Im1 Do Q"); + for (content, spec) in [ + (&b"BT (x) /F1 10 Tf 10 100 Td (SECRETVALUE) Tj ET"[..], Spec::default()), + (b"BT /F1 10 Tf 10 100 Td (x) [(SECRETVALUE)] TJ ET", Spec::default()), + (b"(x) /Fm1 Do", Spec { resources: "/XObject << /Fm1 7 0 R >>", extra: vec![form.clone()], ..Spec::default() }), + // An image under the mark, and one in a form under it. + (b"q 50 0 0 12 10 98 cm (x) /Im1 Do Q", Spec { resources: "/XObject << /Im1 7 0 R >>", extra: vec![image.clone()], ..Spec::default() }), + (b"(x) /Fm1 Do", Spec { resources: "/XObject << /Fm1 7 0 R >>", extra: vec![image_form.clone(), image.clone()], ..Spec::default() }), + ] { + let mut doc = marked_content(content, spec); + doc.require_full_save(); + let proof = snap.prove(&doc, None, &ProofOptions::default()); + assert_eq!(proof.entries[0].status, Status::Failed, "{}: {}", String::from_utf8_lossy(content), proof.to_text()); + } +} + +/// The proof reads the text state's operands from the end too. In each case the text starts outside +/// the region and only the operand after the surplus brings a glyph into it: char spacing, word +/// spacing, horizontal scaling, leading with `T*`, rise, and the spacing `"` sets. +#[test] +fn the_proof_reads_the_text_states_operands_from_the_end() { + let blank = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()); + let snap = Snapshot::capture(&blank, None).unwrap(); + for content in [ + &b"BT /F1 10 Tf -60 100 Td (x) 70 Tc (AB) Tj ET"[..], + b"BT /F1 10 Tf -60 100 Td (x) 70 Tw (A B) Tj ET", + b"BT /F1 10 Tf -60 100 Td (x) 1500 Tz (AB) Tj ET", + b"BT /F1 10 Tf 10 200 Td (x) 100 TL T* (A) Tj ET", + b"BT /F1 10 Tf 10 200 Td (x) -100 Ts (A) Tj ET", + b"BT /F1 10 Tf -60 100 Td (x) 0 70 (AB) \" ET", + ] { + let mut doc = marked_content(content, Spec::default()); + doc.require_full_save(); + let proof = snap.prove(&doc, None, &ProofOptions::default()); + assert_eq!(proof.entries[0].status, Status::Failed, "{}: {}", String::from_utf8_lossy(content), proof.to_text()); + } +} + +/// An image a Type 3 glyph (one without resources of its own, drawing through the page's) draws +/// after a surplus operand can't be placed, as one it draws without one can't: the region is +/// unverifiable. +#[test] +fn an_image_a_type3_glyph_draws_after_a_surplus_operand_is_found() { + let spec = Spec { + resources: "/Font << /F1 5 0 R /T3 7 0 R >> /XObject << /Im0 9 0 R >>", + extra: vec![ + b"<< /Type /Font /Subtype /Type3 /FontBBox [0 0 10 10] /FontMatrix [0.1 0 0 0.1 0 0] /CharProcs << /a 8 0 R >> \ + /Encoding << /Differences [97 /a] >> /FirstChar 97 /LastChar 97 /Widths [10] >>" + .to_vec(), + stream("", b"10 0 d0 (x) /Im0 Do"), + stream("/Type /XObject /Subtype /Image /Width 1 /Height 1 /ColorSpace /DeviceGray /BitsPerComponent 8", b"o"), + ], + ..Spec::default() + }; + let mut doc = open(build_with(b"BT /F1 10 Tf 10 100 Td (SECRETVALUE) Tj ET BT /T3 12 Tf 150 20 Td (a) Tj ET", spec)); + mark(&mut doc, &[REGION]); + let (_, proof) = run(&mut doc); + assert_eq!(proof.entries[0].status, Status::Unverifiable, "{}", proof.to_text()); +} + #[test] fn an_ambiguous_needle_is_still_swept_outside_page_content() { // The word also stays on the page legitimately (so page text, the raw file and decoded diff --git a/crates/redact/src/type3.rs b/crates/redact/src/type3.rs index 4c3f2211e8..0d0ee00559 100644 --- a/crates/redact/src/type3.rs +++ b/crates/redact/src/type3.rs @@ -151,7 +151,7 @@ fn record_used(op: &pdfcraft_content::Op, used: &mut UsedInGlyphs) { } } b"cs" | b"CS" => { - if let Some(n) = op.name(0) { + if let Some(n) = op.last_name() { used.colorspaces.insert(n.to_vec()); } } @@ -166,17 +166,17 @@ fn record_used(op: &pdfcraft_content::Op, used: &mut UsedInGlyphs) { } } b"gs" => { - if let Some(n) = op.name(0) { + if let Some(n) = op.last_name() { used.extgstates.insert(n.to_vec()); } } b"Tf" => { - if let Some(n) = op.name(0) { + if let Some(n) = op.name_back(1) { used.fonts.insert(n.to_vec()); } } b"BDC" | b"DP" => { - if let Some(n) = op.name(1) { + if let Some(n) = op.last_name() { used.properties.insert(n.to_vec()); } } @@ -315,7 +315,7 @@ fn scan_content_stream( } b"gs" => { apply_text_op(ctx, res, &op, &mut current_font); - if let Some(name) = op.name(0) { + if let Some(name) = op.last_name() { scan_soft_mask(ctx, res, holder_ref, name, depth)?; } } @@ -338,12 +338,12 @@ fn scan_content_stream( fn apply_text_op(ctx: &mut ScanContext<'_>, res: &Dict, op: &pdfcraft_content::Op, current_font: &mut Option) { match op.op.as_slice() { b"Tf" => { - if let Some(name) = op.name(0) { + if let Some(name) = op.name_back(1) { *current_font = resolve_font_in_res(ctx.doc, res, name).map(|(k, _)| k); } } b"gs" => { - if let Some(name) = op.name(0) + if let Some(name) = op.last_name() && let Some((k, _)) = resolve_extgstate_font(ctx.doc, res, name) { *current_font = Some(k); @@ -477,7 +477,7 @@ fn add_soft_mask_ops(doc: &Document, res: &Dict, ops: &mut Vec op - .name(0) + .last_name() .and_then(|n| states.get(n)) .and_then(|s| doc.resolve(s).as_dict().and_then(|s| s.get(b"SMask").cloned())) .and_then(|m| doc.resolve(&m).as_dict().and_then(|m| m.get(b"G").and_then(Object::as_ref))), @@ -584,11 +584,11 @@ pub(crate) fn prune_type3_from_resources( let mut drawn_font_names = HashSet::new(); for op in &ops { if op.is("Tf") { - if let Some(name) = op.name(0) { + if let Some(name) = op.name_back(1) { drawn_font_names.insert(name.to_vec()); } } else if op.is("gs") - && let Some(name) = op.name(0) + && let Some(name) = op.last_name() { let extg = res_dict(doc, res, b"ExtGState"); if let Some(font_obj) = extg.get(name).and_then(|state_obj| gs_font_object(doc, state_obj)) { diff --git a/crates/redact/src/verify/extract.rs b/crates/redact/src/verify/extract.rs index fe92e4e78d..294fd92dda 100644 --- a/crates/redact/src/verify/extract.rs +++ b/crates/redact/src/verify/extract.rs @@ -16,7 +16,7 @@ pub(super) fn carries_glyphs(op: &pdfcraft_content::Op) -> bool { let nonempty = |o: &Object| o.as_string().is_some_and(|s| !s.bytes.is_empty()); match op.op.as_slice() { b"Tj" | b"'" | b"\"" => op.operands.last().is_some_and(nonempty), - b"TJ" => op.operands.first().and_then(Object::as_array).is_some_and(|a| a.iter().any(nonempty)), + b"TJ" => op.back(0).and_then(Object::as_array).is_some_and(|a| a.iter().any(nonempty)), _ => false, } } @@ -552,7 +552,7 @@ impl Extractor<'_> { let o = op.op.as_slice(); // Strings and TJ adjustments, in order. let items: Vec<&Object> = match o { - b"TJ" => op.operands.first().and_then(Object::as_array).map(|a| a.iter().collect()).unwrap_or_default(), + b"TJ" => op.back(0).and_then(Object::as_array).map(|a| a.iter().collect()).unwrap_or_default(), _ => op.operands.last().into_iter().collect(), }; let mut any_text = false; @@ -637,14 +637,15 @@ impl Extractor<'_> { tm = Matrix::IDENTITY; tlm = Matrix::IDENTITY; } - b"Tc" => gs.tc = op.num(0).unwrap_or(0.0), - b"Tw" => gs.tw = op.num(0).unwrap_or(0.0), - b"Tz" => gs.th = op.num(0).unwrap_or(100.0) / 100.0, - b"TL" => gs.tl = op.num(0).unwrap_or(0.0), - b"Ts" => gs.rise = op.num(0).unwrap_or(0.0), + // Operands are read from the end, as the renderer reads them. + b"Tc" => gs.tc = op.num_back(0).unwrap_or(0.0), + b"Tw" => gs.tw = op.num_back(0).unwrap_or(0.0), + b"Tz" => gs.th = op.num_back(0).unwrap_or(100.0) / 100.0, + b"TL" => gs.tl = op.num_back(0).unwrap_or(0.0), + b"Ts" => gs.rise = op.num_back(0).unwrap_or(0.0), b"Tf" => { - gs.font = op.name(0).map(<[u8]>::to_vec).unwrap_or_default(); - gs.size = op.num(1).unwrap_or(0.0); + gs.font = op.name_back(1).map(<[u8]>::to_vec).unwrap_or_default(); + gs.size = op.num_back(0).unwrap_or(0.0); } b"Td" | b"TD" => { if let Some([x, y]) = op.nums::<2>() { @@ -671,7 +672,7 @@ impl Extractor<'_> { tm = tlm; } if o == b"\"" - && let (Some(aw), Some(ac)) = (op.num(0), op.num(1)) + && let (Some(aw), Some(ac)) = (op.num_back(2), op.num_back(1)) { (gs.tw, gs.tc) = (aw, ac); } @@ -686,7 +687,7 @@ impl Extractor<'_> { self.show_text(font, &op, &gs, &mut tm, placed); } b"Do" => { - let Some(name) = op.name(0) else { continue }; + let Some(name) = op.last_name() else { continue }; let Some(entry) = xobjects.get(name) else { continue }; if let Object::Stream(s) = &*doc.resolve(entry) && s.dict.name(b"Subtype") == Some(b"Form") diff --git a/crates/redact/src/verify/geometry.rs b/crates/redact/src/verify/geometry.rs index 7ea0de5699..0d3ef179fc 100644 --- a/crates/redact/src/verify/geometry.rs +++ b/crates/redact/src/verify/geometry.rs @@ -188,7 +188,7 @@ impl Geo<'_> { } } b"Do" => { - let Some(entry) = op.name(0).and_then(|n| xobjects.get(n)) else { continue }; + let Some(entry) = op.last_name().and_then(|n| xobjects.get(n)) else { continue }; let obj = self.doc.resolve(entry); let Object::Stream(s) = &*obj else { continue }; match s.dict.name(b"Subtype") { @@ -344,7 +344,7 @@ impl Geo<'_> { match op.op.as_slice() { b"BI" => return Ok(true), b"Do" => { - let Some(entry) = op.name(0).and_then(|n| xobjects.get(n)) else { continue }; + let Some(entry) = op.last_name().and_then(|n| xobjects.get(n)) else { continue }; let Object::Stream(x) = &*self.doc.resolve(entry) else { continue }; match x.dict.name(b"Subtype") { Some(b"Image") => return Ok(true), From 7c78222612d18d065b54a9319b65f248d3df273a Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Tue, 10 Nov 2026 18:54:42 +0100 Subject: [PATCH 13/60] M8: path points and an MCID by reference are read as the renderer does Review of the operand reads. Path construction was still read from the front, in the walk (`construct_path`) and in the proof (`extend_path`): after a surplus number, `9 100 100 m 9 140 100 l ...` placed the square at x = 9, so a mark over it removed nothing, and an even surplus placed a square beside a mark under it, which was removed. `m` and `l` now take their last two numbers, `c` six, `v` and `y` four, and the proof's `re` four, as the walk's already did. The tags cleanup read an `MCID` with `Dict::int`, so one given by reference in a named property list, or in a structure element's `/MCR`, was missed and the element kept the removed text in `/ActualText` and `/Alt`. It is read through the reference now. Tests: `a_paths_points_after_a_surplus_number_are_where_the_renderer_draws_them` (lines, and curves by `c`, `v` and `y`), `an_mcid_given_by_reference_loses_what_redaction_removed` (and an `/MCR` of a form's stream that stays), and `the_proof_reads_a_paths_points_from_the_end` (lines, `re`, and each of `c`, `v` and `y` from a point outside the region). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/redact/src/interp.rs | 8 ++- crates/redact/src/tags.rs | 5 +- .../redact/src/tests/operands_from_the_end.rs | 68 +++++++++++++++++++ crates/redact/src/tests_verify.rs | 21 ++++++ crates/redact/src/verify/geometry.rs | 17 +++-- 5 files changed, 110 insertions(+), 9 deletions(-) diff --git a/crates/redact/src/interp.rs b/crates/redact/src/interp.rs index b470d7984e..b567735d2d 100644 --- a/crates/redact/src/interp.rs +++ b/crates/redact/src/interp.rs @@ -598,8 +598,12 @@ impl Walk { let o = op.op.as_slice(); let pts: Vec<(f64, f64)> = match o { b"re" => op.nums::<4>().map(|[x, y, w, h]| vec![(x, y), (x + w, y), (x, y + h), (x + w, y + h)]).unwrap_or_default(), - b"h" | b"W" | b"W*" => Vec::new(), - _ => op.operands.iter().filter_map(Object::as_f64).collect::>().as_chunks::<2>().0.iter().map(|p| (p[0], p[1])).collect(), + // Each takes its points from the end, as the renderer reads them: a surplus before + // them is ignored, not read as a point. + b"m" | b"l" => op.nums::<2>().map(|[x, y]| vec![(x, y)]).unwrap_or_default(), + b"c" => op.nums::<6>().map(|[x1, y1, x2, y2, x3, y3]| vec![(x1, y1), (x2, y2), (x3, y3)]).unwrap_or_default(), + b"v" | b"y" => op.nums::<4>().map(|[x1, y1, x2, y2]| vec![(x1, y1), (x2, y2)]).unwrap_or_default(), + _ => Vec::new(), }; for (x, y) in pts { let (u, v) = self.gs.ctm.apply(x, y); diff --git a/crates/redact/src/tags.rs b/crates/redact/src/tags.rs index 9cccb93665..15096433d5 100644 --- a/crates/redact/src/tags.rs +++ b/crates/redact/src/tags.rs @@ -57,7 +57,8 @@ fn by_mcid(doc: &Document, resources: &Dict, data: &[u8]) -> Marked { Some(Object::Name(n)) => resources.get(b"Properties").and_then(|p| { let props = doc.resolve(p); let prop = doc.resolve(props.as_dict()?.get(n)?); - prop.as_dict()?.int(b"MCID") + // A named property list may give its MCID by reference. + prop.as_dict()?.get(b"MCID").and_then(|id| doc.resolve(id).as_int()) }), Some(p) => doc.resolve(p).as_dict().and_then(|d| d.int(b"MCID")), _ => None, @@ -138,7 +139,7 @@ fn mcid_of(doc: &Document, o: &Object, page: Option, target: ObjRef, str Object::Dict(m) if m.name(b"Type") == Some(b"MCR") => { let pg = m.get(b"Pg").and_then(Object::as_ref).or(page); let stm = m.get(b"Stm").and_then(Object::as_ref); - (pg == Some(target) && stm == stream.map(|s| s.0)).then(|| m.int(b"MCID")).flatten() + (pg == Some(target) && stm == stream.map(|s| s.0)).then(|| m.get(b"MCID").and_then(|id| doc.resolve(id).as_int())).flatten() } _ => None, } diff --git a/crates/redact/src/tests/operands_from_the_end.rs b/crates/redact/src/tests/operands_from_the_end.rs index d2d6640121..c18daa4dd8 100644 --- a/crates/redact/src/tests/operands_from_the_end.rs +++ b/crates/redact/src/tests/operands_from_the_end.rs @@ -168,3 +168,71 @@ fn a_tagged_tj_after_a_surplus_operand_still_draws() { let paragraph = doc.get(ObjRef::new(9, 0)).as_dict().cloned().unwrap(); assert_eq!(paragraph.get(b"K").and_then(Object::as_int), Some(0), "it still tags what the page shows: {paragraph:?}"); } + +/// A path's points follow a surplus number: read from the front, each point was shifted by one, +/// so a square under the mark was placed beside it and kept, and one beside the mark was placed +/// under it and removed. +#[test] +fn a_paths_points_after_a_surplus_number_are_where_the_renderer_draws_them() { + let under = b"0 0 1 rg 9 100 100 m 9 140 100 l 9 140 140 l 9 100 140 l h f 9 100 100 m 9 140 140 l S".as_slice(); + let mut doc = one_page(under, "", vec![]); + mark(&mut doc, 0, &[[90.0, 90.0, 150.0, 150.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().paths_removed, 2, "both paths under the mark go"); + let beside = + b"0 0 1 rg 0 200 200 m 250 200 l 250 250 l 200 250 l h f 0 200 200 m 210 210 220 220 230 230 c 240 240 250 250 v 200 250 200 260 y h f" + .as_slice(); + let mut doc = one_page(beside, "", vec![]); + mark(&mut doc, 0, &[[0.0, 0.0, 20.0, 20.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().paths_removed, 0, "the paths beside the mark stay"); + // Curves whose points, after a surplus, reach under the mark. + for curve in [&b"200 200 m 9 10 10 12 12 15 15 c f"[..], b"200 200 m 9 10 10 15 15 v f", b"200 200 m 9 10 10 15 15 y f"] { + let mut doc = one_page(curve, "", vec![]); + mark(&mut doc, 0, &[[0.0, 0.0, 20.0, 20.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().paths_removed, 1, "{}", String::from_utf8_lossy(curve)); + } +} + +/// An `MCID` given by reference: in a named property list, and in a structure element's `/MCR`. +/// Read directly, it was missed, and the element kept the removed text. +#[test] +fn an_mcid_given_by_reference_loses_what_redaction_removed() { + for (content, properties, kid) in [ + (b"/P /MC0 BDC BT /F1 10 Tf 10 200 Td (SECRET) Tj ET EMC".as_slice(), "/Properties << /MC0 << /MCID 10 0 R >> >>", "0"), + (b"/P <> BDC BT /F1 10 Tf 10 200 Td (SECRET) Tj ET EMC", "", "<< /Type /MCR /Pg 3 0 R /MCID 10 0 R >>"), + ] { + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R /StructTreeRoot 7 0 R /MarkInfo << /Marked true >> >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /StructParents 0 /Resources << /Font << /F1 5 0 R >> {properties} >> >>").into_bytes(), + stream("", content), + FONT.replace("95 0 R", "6 0 R").into_bytes(), + widths(), + b"<< /Type /StructTreeRoot /K 8 0 R >>".to_vec(), + b"<< /S /Document /P 7 0 R /K [9 0 R] >>".to_vec(), + format!("<< /S /P /P 8 0 R /Pg 3 0 R /ActualText (SECRET) /Alt (the SECRET) /K {kid} >>").into_bytes(), + b"0".to_vec(), + ]); + mark(&mut doc, 0, &[[5.0, 195.0, 60.0, 212.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().tags, 1, "{properties}{kid}"); + let secret = doc.get(ObjRef::new(9, 0)).as_dict().cloned().unwrap(); + assert!(secret.get(b"ActualText").is_none() && secret.get(b"Alt").is_none(), "{secret:?}"); + } + // An `/MCR` naming the same MCID in a form's stream (`/Stm`) is other content: it stays. + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R /StructTreeRoot 7 0 R /MarkInfo << /Marked true >> >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /StructParents 0 /Resources << /Font << /F1 5 0 R >> >> >>".to_vec(), + stream("", b"/P <> BDC BT /F1 10 Tf 10 200 Td (SECRET) Tj ET EMC"), + FONT.replace("95 0 R", "6 0 R").into_bytes(), + widths(), + b"<< /Type /StructTreeRoot /K 8 0 R >>".to_vec(), + b"<< /S /Document /P 7 0 R /K [9 0 R] >>".to_vec(), + b"<< /S /P /P 8 0 R /Pg 3 0 R /ActualText (FORMTEXT) /K << /Type /MCR /Pg 3 0 R /Stm 11 0 R /MCID 10 0 R >> >>".to_vec(), + b"0".to_vec(), + stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300]", b"/P <> BDC EMC"), + ]); + mark(&mut doc, 0, &[[5.0, 195.0, 60.0, 212.0]], ""); + apply(&mut doc, None).unwrap(); + let other = doc.get(ObjRef::new(9, 0)).as_dict().cloned().unwrap(); + assert!(other.get(b"ActualText").is_some(), "{other:?}"); +} diff --git a/crates/redact/src/tests_verify.rs b/crates/redact/src/tests_verify.rs index 3937fcda4f..bd9cb909de 100644 --- a/crates/redact/src/tests_verify.rs +++ b/crates/redact/src/tests_verify.rs @@ -947,6 +947,27 @@ fn the_proof_reads_the_text_states_operands_from_the_end() { } } +/// The proof reads a path's points from the end too: a square under the region, its points after +/// a surplus number, still paints there. +#[test] +fn the_proof_reads_a_paths_points_from_the_end() { + let blank = marked_content(b"BT /F1 10 Tf 10 200 Td (Public) Tj ET", Spec::default()); + let snap = Snapshot::capture(&blank, None).unwrap(); + for content in [ + &b"0 0 1 rg 9 20 98 m 9 60 98 l 9 60 108 l 9 20 108 l h f"[..], + b"0 0 1 rg 9 20 98 40 10 re f", + // Curves from a point outside the region whose own points reach into it. + b"0 0 1 rg 9 200 200 m 9 20 98 40 100 60 108 c h f", + b"0 0 1 rg 9 200 200 m 9 20 98 60 108 v h f", + b"0 0 1 rg 9 200 200 m 9 20 98 60 108 y h f", + ] { + let mut doc = marked_content(content, Spec::default()); + doc.require_full_save(); + let proof = snap.prove(&doc, None, &ProofOptions::default()); + assert_eq!(proof.entries[0].status, Status::Failed, "{}: {}", String::from_utf8_lossy(content), proof.to_text()); + } +} + /// An image a Type 3 glyph (one without resources of its own, drawing through the page's) draws /// after a surplus operand can't be placed, as one it draws without one can't: the region is /// unverifiable. diff --git a/crates/redact/src/verify/geometry.rs b/crates/redact/src/verify/geometry.rs index 0d3ef179fc..7b6833edef 100644 --- a/crates/redact/src/verify/geometry.rs +++ b/crates/redact/src/verify/geometry.rs @@ -372,21 +372,28 @@ impl Geo<'_> { /// carries) by one construction operator. fn extend_path(op: &pdfcraft_content::Op, ctm: &Matrix, sub: &mut Vec>, clip: &mut Option) { let o = op.op.as_slice(); - let nums: Vec = op.operands.iter().filter_map(Object::as_f64).collect(); let at = |x: f64, y: f64| ctm.apply(x, y); + // Each takes its numbers from the end, as the renderer reads them: a surplus before them is + // ignored, not read as a point. + let points: Option> = match o { + b"l" => op.nums::<2>().map(|[x, y]| vec![at(x, y)]), + b"c" => op.nums::<6>().map(|[x1, y1, x2, y2, x3, y3]| vec![at(x1, y1), at(x2, y2), at(x3, y3)]), + b"v" | b"y" => op.nums::<4>().map(|[x1, y1, x2, y2]| vec![at(x1, y1), at(x2, y2)]), + _ => None, + }; match o { b"m" => { - if let [x, y] = nums[..] { + if let Some([x, y]) = op.nums::<2>() { sub.push(vec![at(x, y)]); } } b"l" | b"c" | b"v" | b"y" => { - if let Some(cur) = sub.last_mut() { - cur.extend(nums.as_chunks::<2>().0.iter().map(|p| at(p[0], p[1]))); + if let (Some(cur), Some(points)) = (sub.last_mut(), points) { + cur.extend(points); } } b"re" => { - if let [x, y, w, h] = nums[..] { + if let Some([x, y, w, h]) = op.nums::<4>() { sub.push(vec![at(x, y), at(x + w, y), at(x + w, y + h), at(x, y + h)]); } } From 5e2552dcb0e7cef8a24f5e930f4b13a3ddd14a9a Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Tue, 10 Nov 2026 22:18:19 +0100 Subject: [PATCH 14/60] M8: named property lists and their MCIDs are read once for a stream The tags cleanup resolved the page's `/Properties` at every named `BDC`, on main too: 20,000 of them with 5,000 entries took 28 s, as measured in review (optimized), against 0.07 s with one entry. And the second commit's `MCID` by reference was read again at every `BDC`: the document keeps an object larger than its cache only until it reads another, so reading the `MCID` evicted a property list that large, which was parsed again, all of it, at the next `BDC` (50 `BDC` took 12.3 s against 0.44 s with the `MCID` given directly). Now the named property lists are read once for the stream, in place when they are direct, and each name's `MCID` once. Tests (`crates/redact/tests/operator_work.rs`, a binary of its own whose tests run one at a time), each the least of three runs against one entry or operator, allowing ten times as long plus a second: `named_property_lists_are_read_once_for_a_page` (5,000 `BDC`, 2,000 entries) and `an_oversized_property_list_is_read_once_for_a_stream` (100 `BDC` naming a list too large to keep). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/redact/src/tags.rs | 22 +++++-- crates/redact/tests/operator_work.rs | 97 ++++++++++++++++++++++++++++ 2 files changed, 115 insertions(+), 4 deletions(-) create mode 100644 crates/redact/tests/operator_work.rs diff --git a/crates/redact/src/tags.rs b/crates/redact/src/tags.rs index 15096433d5..03573a9743 100644 --- a/crates/redact/src/tags.rs +++ b/crates/redact/src/tags.rs @@ -35,6 +35,14 @@ impl Marked { /// left are taken as changed (their descriptions go) rather than compared. const MAX_COMPARED: usize = 1 << 28; +/// `f` of `o`, or of the object it refers to, without copying a direct one. +fn with_held(doc: &Document, o: &Object, f: impl FnOnce(&Object) -> R) -> R { + match o { + Object::Ref(_) => f(&doc.resolve(o)), + o => f(o), + } +} + fn by_mcid(doc: &Document, resources: &Dict, data: &[u8]) -> Marked { let mut out = Marked::default(); // Open sequences: the MCID with where its content starts in `ops` and the drawing operators @@ -49,16 +57,22 @@ fn by_mcid(doc: &Document, resources: &Dict, data: &[u8]) -> Marked { e.1 |= drawn > drawn_before; } }; + // The named property lists, read once, and each name's MCID, read once: a stream can name one + // at every `BDC` of thousands, and reading an object again can parse it again, all of it (the + // document keeps an object larger than its cache only until it reads another). A direct one + // is read where it is. + let held = resources.get(b"Properties").filter(|p| matches!(p, Object::Ref(_))).map(|p| doc.resolve(p)); + let named = held.as_deref().or_else(|| resources.get(b"Properties")).and_then(Object::as_dict); + let mut mcids: HashMap, Option> = HashMap::new(); for op in pdfcraft_content::parse(data).ops { match op.op.as_slice() { b"BDC" => { // The property list is the last operand, as the renderer reads it. let id = match op.back(0) { - Some(Object::Name(n)) => resources.get(b"Properties").and_then(|p| { - let props = doc.resolve(p); - let prop = doc.resolve(props.as_dict()?.get(n)?); + Some(Object::Name(n)) => *mcids.entry(n.to_vec()).or_insert_with(|| { + let prop = named?.get(n)?; // A named property list may give its MCID by reference. - prop.as_dict()?.get(b"MCID").and_then(|id| doc.resolve(id).as_int()) + with_held(doc, prop, |prop| prop.as_dict()?.get(b"MCID").and_then(|id| with_held(doc, id, Object::as_int))) }), Some(p) => doc.resolve(p).as_dict().and_then(|d| d.int(b"MCID")), _ => None, diff --git a/crates/redact/tests/operator_work.rs b/crates/redact/tests/operator_work.rs new file mode 100644 index 0000000000..95bf8e33d5 --- /dev/null +++ b/crates/redact/tests/operator_work.rs @@ -0,0 +1,97 @@ +//! Work regressions for what a redaction looks up at operators a stream can hold thousands of: the +//! tags cleanup's named property lists at each `BDC` are not read whole each time. A test binary of +//! its own, so that none of the crate's other tests shares its process, and its tests run one at a +//! time. Each test compares two timings of its own, the least of three runs each, with many +//! entries or operators and with one, not a fixed time. The bound leaves room for slower machines: +//! each regression took seconds against milliseconds. + +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::{Duration, Instant}; + +use pdfcraft_annot::{Meta, NewAnnotation, Shape, Style, add_annotation, rect_quad}; +use pdfcraft_cos::Document; +use pdfcraft_redact::apply; +use pdfcraft_testkit::{Pdf, stream_obj}; + +/// One page with `resources` and `extra` objects (from object 5 on), drawing a word under a mark +/// (so the redaction rewrites the page) and then `ops`. +fn page(resources: &str, ops: &[u8], extra: Vec>) -> Document { + let content = [b"BT /F1 10 Tf 20 260 Td (SECRET) Tj ET ".as_slice(), ops].concat(); + let mut objs = vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + format!( + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /Font << /F1 << /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> >> {resources} >> >>" + ) + .into_bytes(), + stream_obj("", content), + ]; + objs.extend(extra); + let mut doc = Document::open(Arc::new(Pdf::new().objs(objs).build())).unwrap(); + let shape = Shape::Redact { quads: vec![rect_quad([0.0, 200.0, 300.0, 300.0])], overlay: String::new(), look: Default::default() }; + let style = Style::default_for(&shape); + add_annotation(&mut doc, &NewAnnotation { page: 0, shape, style, contents: String::new(), author: "Synthetic test".into() }, &Meta::default()) + .unwrap(); + doc +} + +/// One page whose `/Properties` holds `entries` entries, drawing `ops`. +fn properties(entries: usize, ops: &[u8]) -> Document { + let lists: String = (0..entries).map(|i| format!("/M{i} << /Lang (en) >> ")).collect(); + page(&format!("/Properties << {lists}>>"), ops, Vec::new()) +} + +/// One page whose property list `/M0` is object 5, which gives its `MCID` as object 6 and holds +/// 400,000 more entries: larger than the document keeps, so that reading it again parses it again, +/// all of it, once another object has been read since. The page draws `ops`. +fn oversized(ops: &[u8]) -> Document { + let more: String = (0..400_000).map(|i| format!("/J{i} /DeviceGray ")).collect(); + page("/Properties << /M0 5 0 R >>", ops, vec![format!("<< /MCID 6 0 R {more}>>").into_bytes(), b"0".to_vec()]) +} + +/// The tests' lock: they run one at a time, so that none is timed while another works. +static SERIAL: Mutex<()> = Mutex::new(()); + +fn one_at_a_time() -> std::sync::MutexGuard<'static, ()> { + SERIAL.lock().unwrap_or_else(PoisonError::into_inner) +} + +/// How long redacting a document `make` builds takes, the least of three runs. +fn redaction(make: impl Fn() -> Document) -> Duration { + (0..3) + .map(|_| { + let mut doc = make(); + let start = Instant::now(); + let report = apply(&mut doc, None).unwrap(); + let took = start.elapsed(); + assert_eq!(report.glyphs, 6, "the word under the mark goes"); + took + }) + .min() + .unwrap_or_default() +} + +/// Whether `many` took about as long as `one`. +fn about_as_long(many: Duration, one: Duration) -> bool { + many < one * 10 + Duration::from_secs(1) +} + +/// Written when the tags cleanup resolved the page's `/Properties` at every named `BDC`: 20,000 of +/// them with 5,000 entries took 28 s against 0.07 s with one entry (optimized). +#[test] +fn named_property_lists_are_read_once_for_a_page() { + let _serial = one_at_a_time(); + let ops = b"/P /M0 BDC EMC ".repeat(5_000); + let (one, many) = (redaction(|| properties(1, &ops)), redaction(|| properties(2_000, &ops))); + assert!(about_as_long(many, one), "2,000 entries {many:?}, one {one:?}"); +} + +/// Written when the tags cleanup read a named property list's `MCID` by reference at each `BDC`: a +/// list larger than the document keeps was parsed again at each, 50 of them taking 12.3 s against +/// 0.44 s with the `MCID` given directly (optimized). +#[test] +fn an_oversized_property_list_is_read_once_for_a_stream() { + let _serial = one_at_a_time(); + let (one, many) = (redaction(|| oversized(b"/P /M0 BDC EMC ")), redaction(|| oversized(&b"/P /M0 BDC EMC ".repeat(100)))); + assert!(about_as_long(many, one), "100 operators {many:?}, one {one:?}"); +} From 042371bda9aa12c8e99e83b637342f5342aef85e Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Tue, 10 Nov 2026 22:51:42 +0100 Subject: [PATCH 15/60] M8: a test of its own for each operand read the fix changes Review of the whole change found reads it fixes that no test failed without: each was put back to main's read alone, and the tests passed. Each now has a case that fails without it: - the proof's `Tf` size (text starting outside the region reaches into it only at its size) and font (an undecodable font's text leaves the region unverifiable), and the strings a `TJ` carries; - the proof's `l`, `c`, `v` and `y` points after a surplus, from a point outside the region, and the walk's `m`, `l`, `c`, `v` and `y` points beside a mark, each alone; - the property list a second page sharing the resources still names; - a Type 3 font an ExtGState's `/Font` sets, kept by the page and its glyph by the font, and one without resources of its own, which keeps the page's pattern its glyph may paint with; - a Type 3 font only a soft mask's group selects, the mask's ExtGState named after a surplus; - the named property lists read once, with each `BDC` naming another entry, so that it is the lists, not each name's answer, that are kept. And the walk's `cs` says why it tells a Pattern space by the name `/Pattern` alone. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/redact/src/interp.rs | 5 + .../redact/src/tests/operands_from_the_end.rs | 102 ++++++++++++++++++ .../redact/src/tests/surviving_references.rs | 31 ++++++ crates/redact/src/tests_verify.rs | 20 ++++ crates/redact/tests/operator_work.rs | 9 +- 5 files changed, 163 insertions(+), 4 deletions(-) diff --git a/crates/redact/src/interp.rs b/crates/redact/src/interp.rs index b567735d2d..40cf5bc87e 100644 --- a/crates/redact/src/interp.rs +++ b/crates/redact/src/interp.rs @@ -571,6 +571,11 @@ impl Walk { Some(op) } b"cs" | b"CS" => { + // A Pattern space is told by the name `/Pattern` alone: one named through the + // resources' `/ColorSpace` clears the pattern, which the app's renderer keeps and + // Poppler doesn't (`cs` resets the colour, ISO 32000-2 §8.6.8). Looking the name up + // at each `cs` wants a cache for the page: a `/ColorSpace` too large for the + // document to keep is parsed again at each read. if op.last_name() != Some(b"Pattern") { if o == b"cs" { self.gs.fill_pattern = None; diff --git a/crates/redact/src/tests/operands_from_the_end.rs b/crates/redact/src/tests/operands_from_the_end.rs index c18daa4dd8..0c802763e9 100644 --- a/crates/redact/src/tests/operands_from_the_end.rs +++ b/crates/redact/src/tests/operands_from_the_end.rs @@ -190,6 +190,19 @@ fn a_paths_points_after_a_surplus_number_are_where_the_renderer_draws_them() { mark(&mut doc, 0, &[[0.0, 0.0, 20.0, 20.0]], ""); assert_eq!(apply(&mut doc, None).unwrap().paths_removed, 1, "{}", String::from_utf8_lossy(curve)); } + // A line, and curves, from a point beside the mark whose points, after a surplus, stay beside + // it: read from the front, the surplus was a point under it. + for curve in [ + &b"100 100 500 m 510 510 l S"[..], + b"500 500 m 100 100 510 l S", + b"100 500 m 100 100 500 110 500 120 500 c f", + b"100 500 m 100 110 500 120 500 v f", + b"100 500 m 100 110 500 120 500 y f", + ] { + let mut doc = one_page(curve, "", vec![]); + mark(&mut doc, 0, &[[90.0, 90.0, 150.0, 150.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().paths_removed, 0, "{}", String::from_utf8_lossy(curve)); + } } /// An `MCID` given by reference: in a named property list, and in a structure element's `/MCR`. @@ -236,3 +249,92 @@ fn an_mcid_given_by_reference_loses_what_redaction_removed() { let other = doc.get(ObjRef::new(9, 0)).as_dict().cloned().unwrap(); assert!(other.get(b"ActualText").is_some(), "{other:?}"); } + +/// The resources of page `page`, resolved. +fn resources_of(doc: &Document, page: usize) -> Dict { + let page = pdfcraft_model::pages(doc).swap_remove(page); + page.dict.get(b"Resources").map(|r| doc.resolve(r)).and_then(|r| r.as_dict().cloned()).unwrap_or_default() +} + +/// `dict`'s dictionary `key`, resolved. +fn dict_in(doc: &Document, dict: &Dict, key: &[u8]) -> Dict { + dict.get(key).map(|d| doc.resolve(d)).and_then(|d| d.as_dict().cloned()).unwrap_or_default() +} + +/// A property list a second page sharing the resources names after a surplus operand, once the +/// first page's block that named it was redacted: read from the front, the second page's name went +/// unseen and the list was retired from under it. +#[test] +fn a_property_list_another_page_names_after_a_surplus_operand_is_kept() { + use pdfcraft_cos::Stream; + let mut doc = one_page( + b"/Span /P0 BDC BT /F1 10 Tf 10 200 Td (SECRET) Tj ET EMC", + "/Properties << /P0 7 0 R >>", + vec![b"<< /ActualText (SECRET) >>".to_vec()], + ); + let shared = doc.add(Object::Dict(resources_of(&doc, 0))); + doc.update_dict(ObjRef::new(3, 0), |d| d.set(b"Resources".to_vec(), Object::Ref(shared))).unwrap(); + let mut second = doc.get(ObjRef::new(3, 0)).as_dict().cloned().unwrap(); + let contents = doc.add(Object::Stream(Stream::from_raw(Dict::new(), b"(x) /Span /P0 BDC BT /F1 10 Tf 10 100 Td (Other) Tj ET EMC".to_vec()))); + second.set(b"Contents".to_vec(), Object::Ref(contents)); + let second = doc.add(Object::Dict(second)); + doc.update_dict(ObjRef::new(2, 0), |d| { + d.set(b"Kids".to_vec(), Object::Array(vec![Object::Ref(ObjRef::new(3, 0)), Object::Ref(second)])); + d.set(b"Count".to_vec(), Object::Int(2)); + }) + .unwrap(); + mark(&mut doc, 0, &[[5.0, 195.0, 60.0, 212.0]], ""); + apply(&mut doc, Some(&[0])).unwrap(); + let properties = dict_in(&doc, &resources_of(&doc, 1), b"Properties"); + assert!(properties.contains(b"P0"), "the second page's property list stays: {properties:?}"); +} + +/// A Type 3 font set only by an ExtGState's `/Font`, selected after a surplus operand: read from the +/// front, the cleanups took it for undrawn, dropped it from the page and unlinked its glyph. +#[test] +fn a_type3_font_an_extgstate_selects_after_a_surplus_operand_is_kept() { + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /Font << /F1 5 0 R /T3 7 0 R >> /ExtGState << /GS1 << /Font [7 0 R 12] >> >> >> >>".to_vec(), + stream("", b"BT /F1 10 Tf 20 260 Td (SECRET) Tj ET BT (x) /GS1 gs 20 20 Td (a) Tj ET"), + FONT.replace("95 0 R", "6 0 R").into_bytes(), + widths(), + b"<< /Type /Font /Subtype /Type3 /FontBBox [0 0 10 10] /FontMatrix [0.1 0 0 0.1 0 0] /CharProcs << /a 8 0 R /b 9 0 R >> \ + /Encoding << /Differences [97 /a /b] >> /FirstChar 97 /LastChar 98 /Widths [10 10] >>" + .to_vec(), + stream("", b"10 0 d0 0 0 10 10 re f"), + stream("", b"10 0 d0 0 0 5 5 re f"), + ]); + mark(&mut doc, 0, &[[0.0, 200.0, 300.0, 300.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().glyphs, 6); + let fonts = dict_in(&doc, &resources_of(&doc, 0), b"Font"); + assert!(fonts.contains(b"T3"), "the page still lists the font: {fonts:?}"); + let font = doc.get(ObjRef::new(7, 0)).as_dict().cloned().unwrap_or_default(); + let procs = dict_in(&doc, &font, b"CharProcs"); + assert!(procs.contains(b"a"), "the glyph the page draws stays: {procs:?}"); +} + +/// A Type 3 font without resources of its own, set by an ExtGState selected after a surplus operand: +/// its glyph paints through the page's resources, so it may paint with the page's pattern, which +/// stays once the page's own fill with it was redacted (the text itself fills with grey). Read from the front, the font was taken +/// for one with resources, and the pattern was pruned. +#[test] +fn a_type3_font_without_resources_an_extgstate_selects_after_a_surplus_operand_keeps_the_pattern() { + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> /Pattern << /P0 9 0 R >> /ExtGState << /GS1 << /Font [7 0 R 12] >> >> >> >>".to_vec(), + stream("", b"/Pattern cs /P0 scn 10 260 20 20 re f 0 g BT (x) /GS1 gs 150 20 Td (a) Tj ET"), + FONT.replace("95 0 R", "6 0 R").into_bytes(), + widths(), + b"<< /Type /Font /Subtype /Type3 /FontBBox [0 0 10 10] /FontMatrix [0.1 0 0 0.1 0 0] /CharProcs << /a 8 0 R >> \ + /Encoding << /Differences [97 /a] >> /FirstChar 97 /LastChar 97 /Widths [10] >>" + .to_vec(), + stream("", b"10 0 d0 /Pattern cs /P0 scn 0 0 10 10 re f"), + stream("/PatternType 1 /PaintType 1 /TilingType 1 /BBox [0 0 10 10] /XStep 10 /YStep 10 /Resources << >>", b"0 0 5 5 re f"), + ]); + mark(&mut doc, 0, &[[0.0, 250.0, 40.0, 290.0]], ""); + let r = apply(&mut doc, None).unwrap(); + assert_eq!((r.paths_removed, r.patterns_removed), (1, 0)); +} diff --git a/crates/redact/src/tests/surviving_references.rs b/crates/redact/src/tests/surviving_references.rs index 0d171b4008..d02e76832e 100644 --- a/crates/redact/src/tests/surviving_references.rs +++ b/crates/redact/src/tests/surviving_references.rs @@ -783,6 +783,37 @@ fn a_type3_font_only_a_soft_mask_without_resources_selects_stays_on_the_page() { assert!(saved_first_page_lists_font(&doc, b"T4"), "the page no longer lists the font the soft mask selects"); } +/// The same soft mask, its ExtGState selected after a surplus operand (`(x) /GS1 gs`), which the +/// renderer ignores: read from the front, the cleanups never read the group, and dropped the font +/// it selects and its glyph. +#[test] +fn a_type3_font_only_a_soft_mask_selected_after_a_surplus_operand_selects_stays_on_the_page() { + let mut doc = pdf(vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources 5 0 R >>".to_vec(), + stream("", b"BT /T3 10 Tf 10 100 Td (AC) Tj ET (x) /GS1 gs 150 150 100 100 re f"), + b"<< /Font << /T3 6 0 R /T4 7 0 R >> /ExtGState << /GS1 << /Type /ExtGState /SMask << /Type /Mask /S /Luminosity /G 8 0 R >> >> >> >>" + .to_vec(), + type3_abc(9), + type3_abc(12), + stream( + "/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Group << /S /Transparency /CS /DeviceGray >>", + b"1 g BT /T4 40 Tf 160 180 Td (B) Tj ET", + ), + stream("", b"% REDACTED_GLYPH_A"), + stream("", b"% UNUSED_GLYPH_B"), + stream("", b"% PAGE_GLYPH_C"), + stream("", b"% UNUSED_T4_GLYPH_A"), + stream("", b"% MASK_GLYPH_B"), + stream("", b"% UNUSED_T4_GLYPH_C"), + ]); + mark(&mut doc, 0, &[[9.0, 95.0, 14.9, 110.0]], ""); + assert_eq!(apply(&mut doc, None).unwrap().glyphs, 1); + assert!(full_save_holds(&doc, b"MASK_GLYPH_B"), "the glyph the soft mask shows was unlinked"); + assert!(saved_first_page_lists_font(&doc, b"T4"), "the page no longer lists the font the soft mask selects"); +} + /// The same group, its content behind a predictor whose `/DecodeParms` are given by reference: /// the page's cleanup reads it through them, and keeps the font it selects. #[test] diff --git a/crates/redact/src/tests_verify.rs b/crates/redact/src/tests_verify.rs index bd9cb909de..6e34fba352 100644 --- a/crates/redact/src/tests_verify.rs +++ b/crates/redact/src/tests_verify.rs @@ -912,6 +912,8 @@ fn the_proof_reads_operands_from_the_end() { stream("/Type /XObject /Subtype /Form /BBox [0 0 300 300] /Resources << /XObject << /Im1 8 0 R >> >>", b"q 50 0 0 12 10 98 cm (x) /Im1 Do Q"); for (content, spec) in [ (&b"BT (x) /F1 10 Tf 10 100 Td (SECRETVALUE) Tj ET"[..], Spec::default()), + // The size: text starting outside the region reaches into it only at its size. + (b"BT (x) /F1 10 Tf -60 100 Td (AAAAAAAAAAAAAAAAAAAAAAAAAAAA) Tj ET", Spec::default()), (b"BT /F1 10 Tf 10 100 Td (x) [(SECRETVALUE)] TJ ET", Spec::default()), (b"(x) /Fm1 Do", Spec { resources: "/XObject << /Fm1 7 0 R >>", extra: vec![form.clone()], ..Spec::default() }), // An image under the mark, and one in a form under it. @@ -960,6 +962,11 @@ fn the_proof_reads_a_paths_points_from_the_end() { b"0 0 1 rg 9 200 200 m 9 20 98 40 100 60 108 c h f", b"0 0 1 rg 9 200 200 m 9 20 98 60 108 v h f", b"0 0 1 rg 9 200 200 m 9 20 98 60 108 y h f", + // A segment after a surplus, from a point outside the region. + b"0 0 1 rg 200 200 m 200 20 100 l h f", + b"0 0 1 rg 200 200 m 200 20 100 30 100 40 100 c h f", + b"0 0 1 rg 200 200 m 200 20 100 40 100 v h f", + b"0 0 1 rg 200 200 m 200 20 100 40 100 y h f", ] { let mut doc = marked_content(content, Spec::default()); doc.require_full_save(); @@ -968,6 +975,19 @@ fn the_proof_reads_a_paths_points_from_the_end() { } } +/// Which font a `Tf` selects, and which strings a `TJ` shows, are read from the end too: text an +/// undecodable font shows after a surplus operand, away from the mark, leaves the region +/// unverifiable. Read from the front, no font was set, or no string was shown, and it passed. +#[test] +fn the_proof_reads_the_font_and_the_strings_from_the_end() { + for content in [&b"BT (x) /F2 10 Tf 10 200 Td <00410042> Tj ET"[..], b"BT /F2 10 Tf 10 200 Td (x) [<00410042>] TJ ET"] { + let mut doc = open(build_with(content, opaque_spec())); + mark(&mut doc, &[[150.0, 20.0, 200.0, 40.0]]); + let (_, proof) = run(&mut doc); + assert_eq!(proof.entries[0].status, Status::Unverifiable, "{}: {}", String::from_utf8_lossy(content), proof.to_text()); + } +} + /// An image a Type 3 glyph (one without resources of its own, drawing through the page's) draws /// after a surplus operand can't be placed, as one it draws without one can't: the region is /// unverifiable. diff --git a/crates/redact/tests/operator_work.rs b/crates/redact/tests/operator_work.rs index 95bf8e33d5..e4340fc021 100644 --- a/crates/redact/tests/operator_work.rs +++ b/crates/redact/tests/operator_work.rs @@ -77,13 +77,14 @@ fn about_as_long(many: Duration, one: Duration) -> bool { } /// Written when the tags cleanup resolved the page's `/Properties` at every named `BDC`: 20,000 of -/// them with 5,000 entries took 28 s against 0.07 s with one entry (optimized). +/// them with 5,000 entries took 28 s against 0.07 s with one entry (optimized). Each `BDC` here +/// names another entry, so that it is the lists, not each name's answer, that are read once. #[test] fn named_property_lists_are_read_once_for_a_page() { let _serial = one_at_a_time(); - let ops = b"/P /M0 BDC EMC ".repeat(5_000); - let (one, many) = (redaction(|| properties(1, &ops)), redaction(|| properties(2_000, &ops))); - assert!(about_as_long(many, one), "2,000 entries {many:?}, one {one:?}"); + let ops: Vec = (0..5_000).map(|i| format!("/P /M{i} BDC EMC ")).collect::().into_bytes(); + let (one, many) = (redaction(|| properties(1, &ops)), redaction(|| properties(5_000, &ops))); + assert!(about_as_long(many, one), "5,000 entries {many:?}, one {one:?}"); } /// Written when the tags cleanup read a named property list's `MCID` by reference at each `BDC`: a From 3ad707d07baf5df9b9f1a3b4f62144c285a3c1a4 Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Tue, 10 Nov 2026 23:14:40 +0100 Subject: [PATCH 16/60] =?UTF-8?q?Test:=20wait=20for=20the=20paper=20row=20?= =?UTF-8?q?before=20clicking=20Custom=E2=80=A6=20while=20discovery=20is=20?= =?UTF-8?q?pending=20(#6925)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit adding_custom_paper_overrides_pending_driver_default clicked Custom… as soon as the background paper read started; on hosted macOS the dialog hadn't drawn its paper row by then and the test failed (no node "Custom…"). It now waits for the button, up to 10 s, before clicking. Found by the first nightly macOS run. --- crates/ui-egui/tests/it/print.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crates/ui-egui/tests/it/print.rs b/crates/ui-egui/tests/it/print.rs index 87d77a7eaa..cf382e095b 100644 --- a/crates/ui-egui/tests/it/print.rs +++ b/crates/ui-egui/tests/it/print.rs @@ -1602,6 +1602,14 @@ fn adding_custom_paper_overrides_pending_driver_default() { assert!(std::time::Instant::now() < deadline, "paper discovery started"); std::thread::sleep(std::time::Duration::from_millis(5)); } + // The read starts as the dialog opens; its paper row may take a few more frames to draw (on + // hosted macOS it hadn't when the read started). + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + while h.query_by_label("Custom…").is_none() { + assert!(std::time::Instant::now() < deadline, "the paper row is drawn"); + h.run_steps(1); + std::thread::sleep(std::time::Duration::from_millis(5)); + } h.get_by_label("Custom…").click(); h.run_steps(2); let units = h.state().print_draft.units.per_point(); From d0b17ef95fd9434197b4a06207c423b3e4a78c6d Mon Sep 17 00:00:00 2001 From: Harrison Booth <73434116+Harrison-Booth@users.noreply.github.com> Date: Tue, 10 Nov 2026 16:21:48 -0600 Subject: [PATCH 17/60] Add JavaScript push-button caption methods (#5505) * Add push-button caption script methods * Apply scripted caption with other field changes * Complete caption field property initializers * Apply scripted captions to every button widget * Format caption regression test * Cover caption getter errors and redo state * Assert button caption undo state * Reject button caption writes during format events * Test empty push-button captions * Validate push-button caption updates * Keep no-widget caption test data alive * Check caption clipping marker --------- Co-authored-by: FredApps <153173854+FredApps@users.noreply.github.com> --- ROADMAP.md | 1 + crates/automation/src/forms.rs | 1 + crates/automation/tests/it/automation.rs | 18 +++++++++ crates/automation/tests/it/button_caption.rs | 9 ++++- crates/engine/src/js.rs | 5 ++- crates/engine/src/tests.rs | 37 +++++++++++++++++++ crates/forms/src/author.rs | 2 + crates/forms/src/author/edit.rs | 16 ++++++++ crates/forms/src/scripting.rs | 35 +++++++++++------- crates/forms/src/tests.rs | 39 ++++++++++++++++++++ crates/js/src/document.rs | 38 ++++++++++++++++++- crates/js/src/host.rs | 1 + crates/js/src/model.rs | 6 +++ crates/js/src/run.rs | 1 + crates/js/src/tests.rs | 35 ++++++++++++++++++ crates/ui-egui/src/canvas/area.rs | 1 + crates/ui-egui/src/tools/prepare/draft.rs | 1 + docs/roadmap.md | 3 ++ parity/acrobat-features.toml | 4 +- 19 files changed, 234 insertions(+), 19 deletions(-) diff --git a/ROADMAP.md b/ROADMAP.md index cf33d5cbf7..8889a8f368 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -313,6 +313,7 @@ Ranked; detail and the full milestone table in [docs/roadmap.md](docs/roadmap.md - 2026-11-01: PR #2155 maintainer edits remove redundant image/text helpers and add regressions for crop ownership, arrange scope/state fidelity, clipboard precedence, crop-drag bounds, and font scanning. Transform and Unicode-script tools complete the headless paths. - **2026-10-31 (M3/M4, #5330):** The document quick toolbar offers Crop pages directly through the existing page.crop command, with the command’s assembly-permission check and an active-tool highlight. The existing crop, undo and headless page_set_box paths are unchanged. No toolbar customization or new shortcut. Estimates unchanged. +- **2026-10-30 (M6):** Button scripts can read and change the normal push-button caption with `buttonGetCaption` and `buttonSetCaption`. The caption is drawn and retained through undo/redo and save/reopen; unsupported face indices are refused. Estimates unchanged. - **2026-10-18 (M3, #1354, local candidate):** Integrated --two-paged with current main, retaining upstream modal/picker ownership and single-tab reuse. Native launch/opening/password regressions, negative checks against the old wrapper, visual inspection, formatting, strict scoped Clippy, layers, assets, parity and scoped WASM compilation pass. Three existing socket tests are unavailable in this workspace; one existing network test remains ignored. No push or GitHub merge; estimates unchanged. - **2026-10-31 (M2, PR #4172 review):** Added same-machine pixel oracles for GBK Song, Hei, Kai and FangSong, including all four subset-prefixed Founder aliases. Reference PDFs use explicit Unicode mappings and independently selected installed faces; font data and rasters remain in memory. All eight cases ran; wrong-glyph and wrong-family production mutations each failed the new assertion and were restored. Validation: 422 renderer unit tests, renderer all-target Clippy with -D warnings, formatting, layers/assets/parity and renderer WASM checks passed; existing optional-patch/WASM warnings and parity notes remain. Estimates unchanged. diff --git a/crates/automation/src/forms.rs b/crates/automation/src/forms.rs index 37f1ed2cbb..76eef490a2 100644 --- a/crates/automation/src/forms.rs +++ b/crates/automation/src/forms.rs @@ -645,6 +645,7 @@ impl Automation { Ok((wi, caption.to_owned())) }) .transpose()?, + caption_all: None, read_only: a.opt_bool("read_only")?, required: a.opt_bool("required")?, multiline: a.opt_bool("multiline")?, diff --git a/crates/automation/tests/it/automation.rs b/crates/automation/tests/it/automation.rs index 9516a76542..ea883f499e 100644 --- a/crates/automation/tests/it/automation.rs +++ b/crates/automation/tests/it/automation.rs @@ -9501,6 +9501,24 @@ fn javascript_through_tools() { let r = ok(&mut a, "js_run", json!({ "doc": doc, "script": "money(5)" })); assert_eq!((r["error"].clone(), r["result"].clone()), (json!(null), json!("EUR 5,00"))); assert!(r["document_errors"][0].as_str().unwrap().starts_with("document script 2: TypeError"), "{r}"); + ok( + &mut a, + "form_add_field", + json!({ "doc": doc, "page": 1, "type": "button", "rect": [20, 150, 120, 180], "name": "CaptionButton", "caption": "Before" }), + ); + let r = ok( + &mut a, + "js_run", + json!({ "doc": doc, "field": "CaptionButton", "script": "getField('CaptionButton').buttonSetCaption('Continue'); getField('CaptionButton').buttonGetCaption();" }), + ); + assert_eq!(r["result"], "Continue"); + assert_eq!( + ok(&mut a, "form_fields", json!({ "doc": doc }))["fields"].as_array().unwrap().iter().find(|f| f["name"] == "CaptionButton").unwrap()["caption"], + "Continue" + ); + let bad_face = + ok(&mut a, "js_run", json!({ "doc": doc, "field": "CaptionButton", "script": "getField('CaptionButton').buttonSetCaption('Ignored', 1);" })); + assert!(bad_face["error"].as_str().is_some_and(|e| e.contains("normal button face"))); assert_eq!(ok(&mut a, "js_enabled", json!({ "enabled": false }))["enabled"], false); assert!(a.call("js_run", &json!({ "doc": doc, "script": "1" })).is_err()); } diff --git a/crates/automation/tests/it/button_caption.rs b/crates/automation/tests/it/button_caption.rs index cb1a22a026..bb53504a43 100644 --- a/crates/automation/tests/it/button_caption.rs +++ b/crates/automation/tests/it/button_caption.rs @@ -34,11 +34,16 @@ fn button_caption_can_be_changed_after_creation() { let id = tool(&mut a, "doc_open", json!({"path": "buttons.pdf"}))["doc"].as_u64().unwrap(); let before = a.session().save_bytes(DocId(id)).unwrap(); let (_, _, empty_pixels) = a.render_pixels(id, 1, 1.0).unwrap(); - tool(&mut a, "form_set_props", json!({"doc": id, "field": "clear_button", "caption": "Clear Form"})); + let result = tool( + &mut a, + "js_run", + json!({ "doc": id, "field": "clear_button", "script": "getField('clear_button').value='ignored'; getField('clear_button').buttonSetCaption('Clear Form'); getField('clear_button').buttonGetCaption();" }), + ); + assert_eq!(result["result"], "Clear Form"); let fields = tool(&mut a, "form_fields", json!({"doc": id})); assert_eq!(fields["fields"][0]["caption"], "Clear Form"); assert_eq!(fields["fields"][0]["captions"], json!([{"widget": 1, "caption": "Clear Form"}])); - assert_eq!(a.session().get(DocId(id)).unwrap().can_undo(), Some("Change field properties")); + assert_eq!(a.session().get(DocId(id)).unwrap().can_undo(), Some("Run JavaScript")); let (_, _, labelled_pixels) = a.render_pixels(id, 1, 1.0).unwrap(); assert_ne!(empty_pixels, labelled_pixels, "the label must reach the actual page appearance"); tool(&mut a, "edit_undo", json!({"doc": id})); diff --git a/crates/engine/src/js.rs b/crates/engine/src/js.rs index 13939aad54..eaa4677890 100644 --- a/crates/engine/src/js.rs +++ b/crates/engine/src/js.rs @@ -223,6 +223,7 @@ fn field_state_with_display(f: &Field, display: i32) -> pdfcraft_js::FieldState FieldKind::CheckBox | FieldKind::Radio => f.widgets.iter().filter_map(|w| w.on_state.clone()).map(|o| (o.clone(), o)).collect(), _ => f.options.clone(), }; + s.caption = f.widgets.first().and_then(|w| w.caption.clone()); s } @@ -252,9 +253,11 @@ fn changes( read_only: (a.readonly != b.readonly).then_some(a.readonly), required: (a.required != b.required).then_some(a.required), display: (a.display != b.display).then_some(a.display), + caption: a.caption_was_set.then(|| a.caption.clone()).flatten(), rect: (a.rect != b.rect).then_some(a.rect).flatten(), }; - (c.value.is_some() || c.read_only.is_some() || c.required.is_some() || c.display.is_some() || c.rect.is_some()).then_some(c) + (c.value.is_some() || c.read_only.is_some() || c.required.is_some() || c.display.is_some() || c.caption.is_some() || c.rect.is_some()) + .then_some(c) }) .collect() } diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 56972abaa6..db7fca7559 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -6625,6 +6625,43 @@ fn form_javascript_validates_calculates_and_formats() { assert!(s.get(id).unwrap().form.iter().find(|f| f.name == "total").unwrap().read_only()); assert_eq!(s.get(id).unwrap().can_undo(), Some("Run JavaScript")); + s.apply(id, Edit::InsertBlankPage { at: 1, width: 200.0, height: 300.0 }).unwrap(); + s.apply(id, Edit::DuplicateField { name: "go".into(), pages: vec![1] }).unwrap(); + let props = Box::new(FieldProps { caption: Some((0, "Continue".into())), ..Default::default() }); + s.apply(id, Edit::SetFieldProps { name: "go".into(), props }).unwrap(); + s.apply(id, Edit::SetFieldProps { name: "go".into(), props: Box::new(FieldProps { caption: Some((1, "Old".into())), ..Default::default() }) }) + .unwrap(); + let caption = + s.run_javascript(id, "getField('go').buttonSetCaption('Continue'); event.value = getField('go').buttonGetCaption();", Some("go")).unwrap(); + assert_eq!(caption.error, None); + assert_eq!(caption.result.as_deref(), Some("Continue")); + assert_eq!( + s.get(id).unwrap().form.iter().find(|f| f.name == "go").map(|f| f.widgets.iter().map(|w| w.caption.as_deref()).collect::>()), + Some(vec![Some("Continue"), Some("Continue")]), + "setting a caption already shown by the first widget updates every widget", + ); + assert_eq!(s.get(id).unwrap().can_undo(), Some("Run JavaScript")); + s.undo(id).unwrap(); + assert_eq!( + s.get(id).unwrap().form.iter().find(|f| f.name == "go").and_then(|f| f.widgets.first()).and_then(|w| w.caption.as_deref()), + Some("Continue"), + "undo keeps the first widget's original caption", + ); + assert_eq!( + s.get(id).unwrap().form.iter().find(|f| f.name == "go").and_then(|f| f.widgets.get(1)).and_then(|w| w.caption.as_deref()), + Some("Old"), + "undo restores the second widget's original caption", + ); + s.redo(id).unwrap(); + assert_eq!( + s.get(id).unwrap().form.iter().find(|f| f.name == "go").and_then(|f| f.widgets.first()).and_then(|w| w.caption.as_deref()), + Some("Continue") + ); + assert_eq!( + s.get(id).unwrap().form.iter().find(|f| f.name == "go").and_then(|f| f.widgets.get(1)).and_then(|w| w.caption.as_deref()), + Some("Continue") + ); + // JavaScript off: scripts don't run. s.set_javascript(false); s.apply(id, Edit::SetFieldValue { name: "qty".into(), value: FieldValue::Text("500".into()) }).unwrap(); diff --git a/crates/forms/src/author.rs b/crates/forms/src/author.rs index ee6eef518b..c8aba5b057 100644 --- a/crates/forms/src/author.rs +++ b/crates/forms/src/author.rs @@ -82,6 +82,8 @@ pub struct FieldProps { /// Normal push-button label (`/MK /CA`) of one widget (0-based index). /// An empty string clears the label; other widgets and icon layouts are preserved. pub caption: Option<(usize, String)>, + /// Normal push-button label for every widget in the field. + pub caption_all: Option, pub read_only: Option, pub required: Option, pub multiline: Option, diff --git a/crates/forms/src/author/edit.rs b/crates/forms/src/author/edit.rs index ea718369bf..4836ce4138 100644 --- a/crates/forms/src/author/edit.rs +++ b/crates/forms/src/author/edit.rs @@ -141,6 +141,15 @@ pub fn set_props(doc: &mut Document, name: &str, props: &FieldProps) -> Result, /// `display.visible` 0, `hidden` 1, `noPrint` 2, `noView` 3. pub display: Option, + pub caption: Option, /// Move/resize the first widget, in PDF user space. pub rect: Option<[f64; 4]>, } @@ -127,8 +128,8 @@ pub(crate) fn formatted_in_english(doc: &Document, f: &Field, value: &str, scrip } /// Apply script changes to other fields (values written without re-running their scripts; -/// read-only, required and visibility as flags). `except` is the event's own field, whose value -/// the event sets. +/// read-only, required and visibility as flags; button captions). `except` is the event's own +/// field, whose value the event sets. pub(crate) fn apply_changes(doc: &mut Document, changes: &[FieldChange], except: &str, scripts: &mut dyn Scripts) -> Result<(), FormError> { if changes.is_empty() { return Ok(()); @@ -141,18 +142,20 @@ pub(crate) fn apply_changes(doc: &mut Document, changes: &[FieldChange], except: && *v != f.value { let value = match f.kind { - FieldKind::Text => FieldValue::Text(v.first().cloned().unwrap_or_default()), - FieldKind::CheckBox => f.check_box_value(v.first().map(String::as_str)), - FieldKind::Radio => FieldValue::Radio(v.first().cloned()), - FieldKind::Combo | FieldKind::List => FieldValue::Choice(v.clone()), - _ => continue, + FieldKind::Text => Some(FieldValue::Text(v.first().cloned().unwrap_or_default())), + FieldKind::CheckBox => Some(f.check_box_value(v.first().map(String::as_str))), + FieldKind::Radio => Some(FieldValue::Radio(v.first().cloned())), + FieldKind::Combo | FieldKind::List => Some(FieldValue::Choice(v.clone())), + _ => None, }; - let mut tmp = f.clone(); - tmp.flags &= !flags::NO_TOGGLE_TO_OFF; - tmp.actions = Default::default(); - crate::write_value(doc, &tmp, &value, &mut NoScripts)?; - if c.formatted { - crate::redraw_calculated_value(doc, f, v, scripts)?; + if let Some(value) = value { + let mut tmp = f.clone(); + tmp.flags &= !flags::NO_TOGGLE_TO_OFF; + tmp.actions = Default::default(); + crate::write_value(doc, &tmp, &value, &mut NoScripts)?; + if c.formatted { + crate::redraw_calculated_value(doc, f, v, scripts)?; + } } } let mut ff = f.flags; @@ -182,6 +185,12 @@ pub(crate) fn apply_changes(doc: &mut Document, changes: &[FieldChange], except: } } } + if let Some(caption) = &c.caption { + if f.kind != FieldKind::PushButton { + return Err(FormError::Invalid(format!("{} is not a push button", f.name))); + } + crate::set_props(doc, &f.name, &crate::FieldProps { caption_all: Some(caption.clone()), ..Default::default() })?; + } if let Some(rect) = c.rect { crate::set_props(doc, &c.name, &crate::FieldProps { rect: Some((0, rect)), ..Default::default() })?; } diff --git a/crates/forms/src/tests.rs b/crates/forms/src/tests.rs index 55cb97c887..1dd0524272 100644 --- a/crates/forms/src/tests.rs +++ b/crates/forms/src/tests.rs @@ -221,6 +221,24 @@ fn button_caption_preserves_other_widgets_and_shared_dictionaries() { assert!(!apd.contains(b"R") && !apd.contains(b"D"), "stale hover/pressed text must not survive"); assert!(ap(&doc, &f.widgets[1]).contains("(Clear Form) Tj")); } + let mut doc = original.clone(); + set_props(&mut doc, "clear_button", &FieldProps { caption_all: Some("Continue".into()), ..Default::default() }).unwrap(); + let doc = Document::open(Arc::new(pdfcraft_cos::write_full(&doc, &SaveOptions::default()).unwrap())).unwrap(); + let f = field(&fields(&doc), "clear_button").clone(); + assert_eq!(f.widgets.iter().map(|w| w.caption.as_deref()).collect::>(), [Some("Continue"), Some("Continue")]); + assert!(f.widgets.iter().all(|w| ap(&doc, w).contains("(Continue) Tj"))); + + let mut doc = original.clone(); + apply_script_changes( + &mut doc, + &[FieldChange { name: "clear_button".into(), caption: Some("Proceed".into()), ..Default::default() }], + &mut NoScripts, + ) + .unwrap(); + let doc = Document::open(Arc::new(pdfcraft_cos::write_full(&doc, &SaveOptions::default()).unwrap())).unwrap(); + let f = field(&fields(&doc), "clear_button").clone(); + assert_eq!(f.widgets.iter().map(|w| w.caption.as_deref()).collect::>(), [Some("Proceed"), Some("Proceed")]); + assert!(f.widgets.iter().all(|w| ap(&doc, w).contains("(Proceed) Tj"))); } } @@ -242,6 +260,26 @@ fn button_caption_rejects_wrong_type_and_index_before_mutation() { assert!(!ap(&doc, &field(&fields(&doc), "go").widgets[0]).contains(" Tj")); } +#[test] +fn field_wide_button_caption_rejects_a_field_without_widgets() { + let objs = vec![ + "<< /Type /Catalog /Pages 2 0 R /AcroForm 4 0 R >>".into(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 600 800] >>".into(), + "<< /Type /Page /Parent 2 0 R /Annots [] >>".into(), + "<< /Fields [5 0 R] >>".into(), + "<< /FT /Btn /T (empty) /Ff 65536 /Kids [] >>".into(), + ]; + let mut doc = document(&objs); + let all_fields = fields(&doc); + let button = field(&all_fields, "empty"); + assert_eq!(button.kind, FieldKind::PushButton); + assert!(button.widgets.is_empty()); + let before = pdfcraft_cos::write_full(&doc, &SaveOptions::default()).unwrap(); + let result = set_props(&mut doc, "empty", &FieldProps { caption_all: Some("Caption".into()), ..Default::default() }); + assert!(result.is_err()); + assert_eq!(pdfcraft_cos::write_full(&doc, &SaveOptions::default()).unwrap(), before); +} + /// Read the drawn strings together with the appearance's actual font after saving and reopening. fn appearance_text(doc: &Document, w: &Widget, font_name: &[u8]) -> (Object, Vec>) { let widget = doc.get(w.obj); @@ -3619,6 +3657,7 @@ fn recalc_chain_where_script_changes_checkbox() { read_only: None, required: None, display: None, + caption: None, rect: None, }], message: None, diff --git a/crates/js/src/document.rs b/crates/js/src/document.rs index 54274bb140..26b5a5ab73 100644 --- a/crates/js/src/document.rs +++ b/crates/js/src/document.rs @@ -1,4 +1,4 @@ -use super::host::{arg, colour, error, function, host, read, s, set_value, strings, text, value_of, write}; +use super::host::{arg, colour, error, function, host, read, s, set_value, strings, text, text_line, value_of, write}; use super::model::{DISPLAY_HIDDEN, DISPLAY_VISIBLE, FieldType, Request}; use super::{Attribute, Context, JsArray, JsObject, JsResult, JsString, JsValue, NativeFunction, ObjectInitializer, js_string}; @@ -125,6 +125,38 @@ pub(crate) fn field_object(ctx: &mut Context, name: &str) -> JsObject { function(ctx, NativeFunction::from_copy_closure_with_captures(set, n.clone())) }; + fn button_get_caption(_: &JsValue, args: &[JsValue], name: &JsString, c: &mut Context) -> JsResult { + let f = read(c, name)?; + if f.kind != FieldType::Button { + return Err(error(format!("{} is not a push button", f.name))); + } + let face = if args.is_empty() { 0.0 } else { arg(args, 0).to_number(c)? }; + if !face.is_finite() || face != 0.0 { + return Err(error("only the normal button face (0) is supported")); + } + Ok(s(f.caption.as_deref().unwrap_or(""))) + } + fn button_set_caption(_: &JsValue, args: &[JsValue], name: &JsString, c: &mut Context) -> JsResult { + let f = read(c, name)?; + if f.kind != FieldType::Button { + return Err(error("buttonSetCaption is only available on push buttons")); + } + let caption_arg = args.first().ok_or_else(|| error("buttonSetCaption requires a caption argument"))?; + let caption = text_line(caption_arg, c)?; + let face = if args.get(1).is_none() { 0.0 } else { arg(args, 1).to_number(c)? }; + if !face.is_finite() || face != 0.0 { + return Err(error("only the normal button face (0) is supported")); + } + if host(c)?.borrow().event_name == "Format" { + return Err(error("buttonSetCaption is not available in a Format event")); + } + write(c, name, |f| { + f.caption = Some(caption); + f.caption_was_set = true; + })?; + Ok(JsValue::undefined()) + } + fn get_item_at(_: &JsValue, args: &[JsValue], name: &JsString, c: &mut Context) -> JsResult { let f = read(c, name)?; let i = arg(args, 0).to_number(c)?; @@ -164,6 +196,8 @@ pub(crate) fn field_object(ctx: &mut Context, name: &str) -> JsObject { let is_box_checked = NativeFunction::from_copy_closure_with_captures(is_box_checked, n.clone()); let check_this_box = NativeFunction::from_copy_closure_with_captures(check_this_box, n.clone()); let set_focus = NativeFunction::from_copy_closure_with_captures(set_focus, n.clone()); + let button_get_caption = NativeFunction::from_copy_closure_with_captures(button_get_caption, n.clone()); + let button_set_caption = NativeFunction::from_copy_closure_with_captures(button_set_caption, n.clone()); ObjectInitializer::new(ctx) .accessor(js_string!("value"), Some(value_get), Some(value_set), rw) .accessor(js_string!("valueAsString"), Some(as_string), None, rw) @@ -185,6 +219,8 @@ pub(crate) fn field_object(ctx: &mut Context, name: &str) -> JsObject { .function(is_box_checked, js_string!("isBoxChecked"), 1) .function(check_this_box, js_string!("checkThisBox"), 2) .function(set_focus, js_string!("setFocus"), 0) + .function(button_get_caption, js_string!("buttonGetCaption"), 1) + .function(button_set_caption, js_string!("buttonSetCaption"), 2) .build() } diff --git a/crates/js/src/host.rs b/crates/js/src/host.rs index d57cccbe74..828a0726b0 100644 --- a/crates/js/src/host.rs +++ b/crates/js/src/host.rs @@ -6,6 +6,7 @@ use super::{ }; pub(crate) struct Host { + pub(crate) event_name: String, pub(crate) fields: Arc>, pub(crate) changed: BTreeSet, pub(crate) formatted: BTreeSet, diff --git a/crates/js/src/model.rs b/crates/js/src/model.rs index 6314adbfa2..eabe3ecd57 100644 --- a/crates/js/src/model.rs +++ b/crates/js/src/model.rs @@ -52,6 +52,10 @@ pub struct FieldState { /// `textColor` / `fillColor` as Acrobat colour arrays (`["RGB", r, g, b]` etc.), when set. pub text_color: Option>, pub fill_color: Option>, + /// Normal push-button caption, when this is a button. + pub caption: Option, + /// `buttonSetCaption` was called, even when it set the current text again. + pub caption_was_set: bool, /// Rectangle of the first widget, in PDF user space. pub rect: Option<[f64; 4]>, } @@ -70,6 +74,8 @@ impl FieldState { char_limit: None, text_color: None, fill_color: None, + caption: None, + caption_was_set: false, rect: None, } } diff --git a/crates/js/src/run.rs b/crates/js/src/run.rs index 8f760f8130..edc6332ca8 100644 --- a/crates/js/src/run.rs +++ b/crates/js/src/run.rs @@ -233,6 +233,7 @@ fn run_here(script: &str, event: &Event, doc: &DocInfo, fields: Arc **Last reviewed:** 2026-10-28 | **Last updated:** 2026-10-28 | **Change:** minor (M10, #4119 partial: Enhance scanned file cleans scanned pages in place: background removal, edge shadow removal, despeckle; estimates unchanged) > **Last reviewed:** 2026-11-03 | **Last updated:** 2026-11-03 | **Change:** patch (M5 #5850 review: indirect typewriter intent and composer edits preserve outer whitespace; movement regression commits by outside click; estimates unchanged) + > **Last reviewed:** 2026-11-03 | **Last updated:** 2026-11-03 | **Change:** patch (M5 #5850: Enter adds Fill & Sign line breaks; outside clicks commit, Escape cancels, and saved multiline appearances retain their text and height; estimates unchanged) > **2026-11-03, M7 / #5835 review:** active image crops retain secondary clicks over overlapping images; crop controls share measured layout with hit detection. Rotation, handle and page-edge regressions added; focused tests, scoped Clippy, formatting, layering and asset checks passed locally. Estimates unchanged. @@ -32,6 +33,8 @@ > **Last reviewed:** 2026-10-31 | **Last updated:** 2026-10-31 | **Change:** patch (M2, PR #4172: GBK rasters compared with explicit Unicode and independently selected installed faces; wrong-glyph and wrong-family negative controls fail; estimates unchanged) > **Last reviewed:** 2026-10-31 | **Last updated:** 2026-10-31 | **Change:** patch (M2, PR #4172: integrate main fa5f9f8c; retain CJK viewer and upstream raster regressions, restore explicit display_config imports; estimates unchanged) +> **Last reviewed:** 2026-10-30 | **Last updated:** 2026-10-30 | **Change:** minor (M6: scripts can read and change a push-button's normal caption with undo/redo and save/reopen coverage; estimates unchanged) + > **Last reviewed:** 2026-10-28 · **Last updated:** 2026-10-28 · **Change:** patch (M2, PR #4172: integrate main 5af10276; retain both roadmap histories and viewer-only CJK font policy; estimates unchanged) > **Last reviewed:** 2026-10-27 · **Last updated:** 2026-10-27 · **Change:** patch (M2, PR #4172: integrate main 1dd2737b; preserve viewer-only CJK fallback and regression coverage across engine, renderer and Combine module splits; estimates unchanged) > **Last reviewed:** 2026-10-27 · **Last updated:** 2026-10-27 · **Change:** patch (M2, PR #4172: integrate main 9a9813bf and migrate viewer-only CJK configuration to the split engine modules; estimates unchanged) diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 3ea69e1bf7..0d70edf70c 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -5363,8 +5363,8 @@ milestone = "M6" status = "partial" commands = ["tools.js_console"] tools = ["js_run"] -evidence = ["crates/js/src/tests.rs::calculate_scripts_read_fields_and_set_the_value", "crates/engine/src/tests.rs::form_javascript_validates_calculates_and_formats", "crates/automation/tests/it/automation.rs::javascript_through_tools", "crates/js/src/tests.rs::an_xfa_form_sees_the_xfa_plugin_installed", "crates/js/src/tests.rs::a_plain_form_sees_the_xfa_version_but_not_installed", "crates/engine/src/tests.rs::scripts_in_an_xfa_form_see_the_xfa_plugin_installed", "crates/js/src/tests.rs::scand_allows_separators_between_fields_but_nothing_before_or_after_them", "crates/ui-egui/tests/it/forms.rs::a_form_button_opens_its_attached_instructions_in_a_new_tab", "crates/js/src/tests.rs::app_alert_keeps_its_title_icon_and_buttons", "crates/js/src/tests.rs::app_alert_reads_the_object_form", "crates/js/src/tests.rs::app_alert_falls_back_on_bad_icon_and_button_numbers", "crates/js/src/tests.rs::app_alert_titles_are_clipped_and_charged", "crates/ui-egui/tests/it/javascript.rs::every_message_a_script_shows_takes_the_toast_in_turn", "crates/js/src/tests.rs::widget_rect_reads_and_writes_a_four_number_array", "crates/engine/src/tests.rs::javascript_field_rect_moves_first_widget_and_undoes", "crates/engine/src/tests.rs::js_get_page_box_reads_effective_boxes_without_editing_the_document", "crates/js/src/tests.rs::get_page_box_returns_supported_boxes_in_rotated_user_space"] -notes = "boa (pure Rust) with the forms object model: event, getField/Field (value, readonly, display, colours, items, check boxes, rect), this.getPageBox(cBox, nPage) for effective Media, Crop (default), Trim, Bleed and Art boxes in rotated user space, app (with execMenuItem for the side panels and Save As, from a button only, and Print where this.print() works), util.printf/printd/printx/scand (scand reads dates as another viewer was observed to, in AcroForm and XFA scripts), console, display, color, exportDataObject (from a button only: an attached PDF opens in a new tab, other files are offered to save, also as event.target in XFA scripts); xfa_version (3.5) in every document and xfa_installed in XFA forms, so their version check passes. field.rect reads and writes the first widget in PDF user space. no annotations, layers or dialogs (app.response returns null) yet, and no app.findComponent. app.alert keeps its title, icon (error, warning, question, status) and buttons (OK, OK/Cancel, Yes/No, Yes/No/Cancel), given in order or as one object; js_run reports them with each message, and the app shows every message in turn in the toast with its title and icon. Still missing: answers. Every app.alert returns 1 (OK), so a script is never told No or Cancel (#4291)." +evidence = ["crates/js/src/tests.rs::calculate_scripts_read_fields_and_set_the_value", "crates/js/src/tests.rs::push_button_caption_methods_read_write_and_reject_other_faces", "crates/engine/src/tests.rs::form_javascript_validates_calculates_and_formats", "crates/automation/tests/it/automation.rs::javascript_through_tools", "crates/js/src/tests.rs::an_xfa_form_sees_the_xfa_plugin_installed", "crates/js/src/tests.rs::a_plain_form_sees_the_xfa_version_but_not_installed", "crates/engine/src/tests.rs::scripts_in_an_xfa_form_see_the_xfa_plugin_installed", "crates/js/src/tests.rs::scand_allows_separators_between_fields_but_nothing_before_or_after_them", "crates/ui-egui/tests/it/forms.rs::a_form_button_opens_its_attached_instructions_in_a_new_tab", "crates/js/src/tests.rs::app_alert_keeps_its_title_icon_and_buttons", "crates/js/src/tests.rs::app_alert_reads_the_object_form", "crates/js/src/tests.rs::app_alert_falls_back_on_bad_icon_and_button_numbers", "crates/js/src/tests.rs::app_alert_titles_are_clipped_and_charged", "crates/ui-egui/tests/it/javascript.rs::every_message_a_script_shows_takes_the_toast_in_turn", "crates/js/src/tests.rs::widget_rect_reads_and_writes_a_four_number_array", "crates/engine/src/tests.rs::javascript_field_rect_moves_first_widget_and_undoes", "crates/engine/src/tests.rs::js_get_page_box_reads_effective_boxes_without_editing_the_document", "crates/js/src/tests.rs::get_page_box_returns_supported_boxes_in_rotated_user_space", "crates/automation/tests/it/button_caption.rs::button_caption_can_be_changed_after_creation"] +notes = "boa (pure Rust) with the forms object model: event, getField/Field (value, readonly, display, colours, items, check boxes, rect, and normal push-button captions through buttonGetCaption/buttonSetCaption), this.getPageBox(cBox, nPage) for effective Media, Crop (default), Trim, Bleed and Art boxes in rotated user space, app (with execMenuItem for the side panels and Save As, from a button only, and Print where this.print() works), util.printf/printd/printx/scand (scand reads dates as another viewer was observed to, in AcroForm and XFA scripts), console, display, color, exportDataObject (from a button only: an attached PDF opens in a new tab, other files are offered to save, also as event.target in XFA scripts); xfa_version (3.5) in every document and xfa_installed in XFA forms, so their version check passes. field.rect reads and writes the first widget in PDF user space. no annotations, layers or dialogs (app.response returns null) yet, and no app.findComponent. app.alert keeps its title, icon (error, warning, question, status) and buttons (OK, OK/Cancel, Yes/No, Yes/No/Cancel), given in order or as one object; js_run reports them with each message, and the app shows every message in turn in the toast with its title and icon. Still missing: answers. Every app.alert returns 1 (OK), so a script is never told No or Cancel (#4291)." [[feature]] id = "form.js-console" From 832a51e6d71d960a08726cd422455211018f44d0 Mon Sep 17 00:00:00 2001 From: Harrison Booth <73434116+Harrison-Booth@users.noreply.github.com> Date: Tue, 10 Nov 2026 16:43:44 -0600 Subject: [PATCH 18/60] Fix #5211: persist JavaScript field colours (#5499) * Fix #5211: persist JavaScript field colours * Read current document state in field scripts * Redraw form fields after validation changes * Format validation redraw regression test * Keep field list alive in redraw test * Refresh fields only after same-field scripts * Format conditional field refresh * Borrow accepted field value for storage * Redraw calculated fields from current appearance * Format JavaScript field state merge * Restore JavaScript runner document state * Group JavaScript runner form inputs * Read current field display state in scripts --------- Co-authored-by: FredApps <153173854+FredApps@users.noreply.github.com> --- crates/engine/src/edit.rs | 4 +- crates/engine/src/js.rs | 74 +++++++++++++-- crates/engine/src/tests.rs | 144 ++++++++++++++++++++++++++++++ crates/forms/src/scripting.rs | 53 ++++++++++- crates/forms/src/tests.rs | 62 ++++++++++++- crates/forms/src/values.rs | 11 ++- crates/ui-egui/src/canvas/area.rs | 3 +- parity/acrobat-features.toml | 4 +- 8 files changed, 335 insertions(+), 20 deletions(-) diff --git a/crates/engine/src/edit.rs b/crates/engine/src/edit.rs index 1d5181bf65..a6b1f0b2bf 100644 --- a/crates/engine/src/edit.rs +++ b/crates/engine/src/edit.rs @@ -452,8 +452,8 @@ pub enum Edit { RemoveStandardId { standard: Standard, }, - /// What a script (button or console) changed in form fields: values, read-only, required - /// and visibility. + /// What a script (button or console) changed in form fields: values, appearance colours, + /// read-only, required and visibility. ApplyScriptChanges { changes: Vec, }, diff --git a/crates/engine/src/js.rs b/crates/engine/src/js.rs index eaa4677890..f73be5ec03 100644 --- a/crates/engine/src/js.rs +++ b/crates/engine/src/js.rs @@ -208,9 +208,22 @@ fn display(doc: &pdfcraft_cos::Document, f: &Field) -> i32 { /// A form field as scripts see it. pub fn field_state(doc: &pdfcraft_cos::Document, f: &Field) -> pdfcraft_js::FieldState { - field_state_with_display(f, display(doc, f)) + let mut s = field_state_with_display(f, display(doc, f)); + (s.text_color, s.fill_color) = colours(doc, f); + s } +/// `field.textColor` and `field.fillColor` as Acrobat colour arrays, from the field's look +/// (`None` for a transparent fill, which scripts read as `["T"]`). +fn colours(doc: &pdfcraft_cos::Document, f: &Field) -> Colours { + let rgb = |c: [f64; 3]| std::iter::once("RGB".to_string()).chain(c.map(|v| v.to_string())).collect::>(); + let look = pdfcraft_forms::look(doc, f); + (Some(rgb(look.text)), look.fill.map(rgb)) +} + +/// A field's text and fill colours as scripts read them. +type Colours = (Option>, Option>); + fn field_state_with_display(f: &Field, display: i32) -> pdfcraft_js::FieldState { let mut s = pdfcraft_js::FieldState::new(f.name.clone(), kind(f.kind), f.value.clone()); s.default = f.default.clone(); @@ -255,9 +268,18 @@ fn changes( display: (a.display != b.display).then_some(a.display), caption: a.caption_was_set.then(|| a.caption.clone()).flatten(), rect: (a.rect != b.rect).then_some(a.rect).flatten(), + text_color: (a.text_color != b.text_color).then(|| a.text_color.clone()).flatten(), + fill_color: (a.fill_color != b.fill_color).then(|| a.fill_color.clone()).flatten(), }; - (c.value.is_some() || c.read_only.is_some() || c.required.is_some() || c.display.is_some() || c.caption.is_some() || c.rect.is_some()) - .then_some(c) + (c.value.is_some() + || c.read_only.is_some() + || c.required.is_some() + || c.display.is_some() + || c.caption.is_some() + || c.rect.is_some() + || c.text_color.is_some() + || c.fill_color.is_some()) + .then_some(c) }) .collect() } @@ -274,6 +296,14 @@ pub struct JsRunner { // otherwise reparse every widget for every calculation. Values/flags still come from // the live `fields` snapshot on each call. A new edit gets a new runner and cache. displays: std::collections::HashMap, + /// Field colours read from `cos`, cached for the same reason as `displays`. + colours: std::collections::HashMap, + /// Colours this runner's scripts wrote, by field name: they reach the document only once + /// the edit applies each script's changes, after `cos` was taken, so a later script in the + /// same edit reads them from here. + written_colours: std::collections::HashMap, + /// Whether a document-level script names a colour property (`fillColor`, `textColor`, …). + doc_scripts_use_colours: bool, calculations: Option>>, } @@ -300,7 +330,18 @@ impl JsRunner { // Packets alone are not enough: static XFA and an unlaid-out shell keep their scripts. let xfa_active = xfa && pdfcraft_xfa::existing_layout(cos).is_some(); let doc_scripts = if xfa_active { Vec::new() } else { pdfcraft_forms::document_scripts(cos) }; - JsRunner { doc, doc_scripts, output: JsOutput::default(), cos: cos.clone(), displays: std::collections::HashMap::new(), calculations: None } + let doc_scripts_use_colours = doc_scripts.iter().any(|source| source.contains("Color")); + JsRunner { + doc, + doc_scripts, + output: JsOutput::default(), + cos: cos.clone(), + displays: std::collections::HashMap::new(), + colours: std::collections::HashMap::new(), + written_colours: std::collections::HashMap::new(), + doc_scripts_use_colours, + calculations: None, + } } fn calculations(&mut self, fields: &[Field]) -> std::sync::Arc> { @@ -315,6 +356,8 @@ impl JsRunner { pub(crate) fn invalidate_calculations(&mut self, cos: &pdfcraft_cos::Document) { self.cos = cos.clone(); self.displays.clear(); + self.colours.clear(); + self.written_colours.clear(); self.calculations = None; } } @@ -399,6 +442,12 @@ impl JsRunner { /// [`Scripts::run`], keeping what the script showed and asked for only when `log`; `None` /// when the script failed. fn run_logged(&mut self, event: FieldEvent, script: &str, target: &Field, value: &str, fields: &[Field], log: bool) -> Option { + // Field colours are read (and their arrays built for every field) only for a script + // that can reach them: one that names a colour property, or any script when a document + // script does. A large form's calculations then cost what they did before colours were + // readable. A property name built at run time (`f["fill" + "Color"]`) reads as + // transparent in a script that names no colour. + let reads_colours = self.doc_scripts_use_colours || script.contains("Color"); let states: Vec<_> = fields .iter() .map(|f| { @@ -406,7 +455,14 @@ impl JsRunner { Some(w) => *self.displays.entry(w.obj).or_insert_with(|| display(&self.cos, f)), None => pdfcraft_js::DISPLAY_VISIBLE, }; - field_state_with_display(f, display) + let mut state = field_state_with_display(f, display); + if reads_colours { + (state.text_color, state.fill_color) = match self.written_colours.get(&f.name) { + Some(written) => written.clone(), + None => self.colours.entry(f.obj).or_insert_with(|| colours(&self.cos, f)).clone(), + }; + } + state }) .collect(); let states = std::sync::Arc::new(states); @@ -431,11 +487,17 @@ impl JsRunner { if o.error.is_some() { return None; } + let changes = changes(&states, &o.changed, &o.formatted); + for c in changes.iter().filter(|c| c.text_color.is_some() || c.fill_color.is_some()) { + if let Some(state) = o.changed.iter().find(|s| s.name == c.name) { + self.written_colours.insert(c.name.clone(), (state.text_color.clone(), state.fill_color.clone())); + } + } Some(ScriptResult { rc: o.rc, value: o.value.clone(), value_set: o.value_set, - changes: changes(&states, &o.changed, &o.formatted), + changes, message: (!o.rc).then(|| o.alerts.last().map(|m| m.text.clone())).flatten(), }) } diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index db7fca7559..5952efab88 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -6669,6 +6669,89 @@ fn form_javascript_validates_calculates_and_formats() { assert!(s.run_javascript(id, "1", None).is_err()); } +#[test] +fn javascript_field_colours_persist_and_undo_without_changing_other_appearance_properties() { + let mut s = Session::new().with_clock(|| 1_700_000_000); + let id = s.open("order.pdf", None, Arc::new(scripted_form()), None).unwrap(); + s.apply( + id, + Edit::SetFieldProps { + name: "price".into(), + props: Box::new(FieldProps { + appearance: Some(pdfcraft_forms::LookPatch { + border: Some(Some([0.1, 0.2, 0.3])), + font: Some(pdfcraft_forms::FieldFont::Courier), + ..Default::default() + }), + ..Default::default() + }), + }, + ) + .unwrap(); + let before = s.get(id).unwrap().form.iter().find(|f| f.name == "price").unwrap().value.clone(); + let out = s.run_javascript(id, "var f = getField('price'); f.fillColor = color.red; f.textColor = color.blue;", None).unwrap(); + assert_eq!(out.error, None); + assert_eq!(s.get(id).unwrap().can_undo(), Some("Run JavaScript")); + + let inspect = |bytes: Arc>| { + let doc = pdfcraft_cos::Document::open(bytes).unwrap(); + let f = pdfcraft_forms::fields(&doc).into_iter().find(|f| f.name == "price").unwrap(); + let look = pdfcraft_forms::look(&doc, &f); + assert_eq!(look.fill, Some([1.0, 0.0, 0.0])); + assert_eq!(look.text, [0.0, 0.0, 1.0]); + assert_eq!(look.border, Some([0.1, 0.2, 0.3])); + assert_eq!(look.font, pdfcraft_forms::FieldFont::Courier); + assert_eq!(f.value, before); + }; + inspect(s.save_bytes(id).unwrap()); + s.undo(id).unwrap(); + let undone = pdfcraft_cos::Document::open(s.save_bytes(id).unwrap()).unwrap(); + let f = pdfcraft_forms::fields(&undone).into_iter().find(|f| f.name == "price").unwrap(); + assert_eq!(pdfcraft_forms::look(&undone, &f).fill, None); + s.redo(id).unwrap(); + inspect(s.save_bytes(id).unwrap()); + + let reopened = s.open("saved.pdf", None, s.save_bytes(id).unwrap(), None).unwrap(); + let reopened_doc = pdfcraft_cos::Document::open(s.save_bytes(reopened).unwrap()).unwrap(); + let f = pdfcraft_forms::fields(&reopened_doc).into_iter().find(|f| f.name == "price").unwrap(); + assert_eq!(pdfcraft_forms::look(&reopened_doc, &f).fill, Some([1.0, 0.0, 0.0])); + assert_eq!(pdfcraft_forms::look(&reopened_doc, &f).text, [0.0, 0.0, 1.0]); +} + +#[test] +fn javascript_invalid_field_colour_array_is_refused_without_partial_changes() { + let mut s = Session::new().with_clock(|| 1_700_000_000); + let id = s.open("order.pdf", None, Arc::new(scripted_form()), None).unwrap(); + let before = s.save_bytes(id).unwrap(); + let err = s.run_javascript(id, "var f = getField('price'); f.value = '9'; f.fillColor = ['RGB', 2, 0, 0];", None).unwrap_err(); + assert!(err.to_string().contains("finite values between 0 and 1"), "{err}"); + assert_eq!(s.save_bytes(id).unwrap(), before, "a refused script changes nothing"); +} + +#[test] +fn javascript_field_colours_respect_document_fill_permissions() { + let mut plain = pdfcraft_cos::Document::open(Arc::new(scripted_form())).unwrap(); + plain + .set_encryption(&pdfcraft_cos::NewEncryption { + algorithm: pdfcraft_cos::Algorithm::Aes256, + user_password: "", + owner_password: "owner", + permissions: 0, + encrypt_metadata: true, + seed: [7; 32], + }) + .unwrap(); + let bytes = Arc::new(pdfcraft_cos::write_full(&plain, &Default::default()).unwrap()); + let mut s = Session::new().with_clock(|| 1_700_000_000); + let id = s.open("locked.pdf", None, bytes, None).unwrap(); + assert!(!s.get(id).unwrap().allows_form_filling()); + assert_eq!( + s.run_javascript(id, "getField('price').fillColor = color.red;", None).unwrap_err(), + EditError::NotPermitted("filling in form fields") + ); + assert!(!s.get(id).unwrap().dirty); +} + #[test] fn calculate_now_updates_live_fields_in_one_undo_step() { let mut s = Session::new().with_clock(|| 1_700_000_000); @@ -13859,6 +13942,67 @@ fn a_document_encrypting_only_its_embedded_files_opens_and_saves_without_a_passw assert!(reopened.embedded_files_locked(), "the attachment stays encrypted"); } +#[test] +fn javascript_non_rgb_field_colours_convert_and_keep_the_scripts_other_changes() { + let fill_after = |colour: &str| { + let mut s = Session::new().with_clock(|| 1_700_000_000); + let id = s.open("order.pdf", None, Arc::new(scripted_form()), None).unwrap(); + s.run_javascript(id, "getField('price').fillColor = color.red;", None).unwrap(); + let out = s.run_javascript(id, &format!("var f = getField('price'); f.value = '9'; f.fillColor = {colour}; f.textColor = {colour};"), None); + assert!(out.is_ok(), "{colour}: {out:?}"); + let f = s.get(id).unwrap().form.iter().find(|f| f.name == "price").unwrap().clone(); + assert_eq!(f.value, ["9"], "{colour}: the value write is kept"); + let doc = pdfcraft_cos::Document::open(s.save_bytes(id).unwrap()).unwrap(); + let f = pdfcraft_forms::fields(&doc).into_iter().find(|f| f.name == "price").unwrap(); + let look = pdfcraft_forms::look(&doc, &f); + (look.fill, look.text) + }; + assert_eq!(fill_after("color.white"), (Some([1.0, 1.0, 1.0]), [1.0, 1.0, 1.0])); + assert_eq!(fill_after("color.gray"), (Some([0.5, 0.5, 0.5]), [0.5, 0.5, 0.5])); + assert_eq!(fill_after("color.cyan"), (Some([0.0, 1.0, 1.0]), [0.0, 1.0, 1.0])); + assert_eq!(fill_after("['CMYK', 0, 0, 0, 1]"), (Some([0.0, 0.0, 0.0]), [0.0, 0.0, 0.0])); + // Transparent clears the fill; text can't be transparent, so its colour stays. + let (fill, text) = fill_after("color.transparent"); + assert_eq!(fill, None); + assert_eq!(text, [0.0, 0.0, 0.0]); + + let mut s = Session::new().with_clock(|| 1_700_000_000); + let id = s.open("order.pdf", None, Arc::new(scripted_form()), None).unwrap(); + let before = s.save_bytes(id).unwrap(); + for bad in ["['CMYK', 1]", "['G']", "['T', 0]", "['Lab', 0, 0, 0]", "['G', 'x']"] { + let err = s.run_javascript(id, &format!("var f = getField('price'); f.value = '9'; f.fillColor = {bad};"), None).unwrap_err(); + assert!(err.to_string().contains("T, G, RGB or CMYK"), "{bad}: {err}"); + assert_eq!(s.save_bytes(id).unwrap(), before, "{bad}: a malformed colour still refuses the script"); + } +} + +#[test] +fn a_later_field_script_in_the_same_edit_reads_a_colour_an_earlier_one_wrote() { + let mut s = Session::new().with_clock(|| 1_700_000_000); + let id = s.open("order.pdf", None, Arc::new(scripted_form()), None).unwrap(); + for (name, script) in + [("qty", "getField('price').fillColor = color.green; event.value = 1;"), ("total", "event.value = getField('price').fillColor.join(',');")] + { + s.apply(id, Edit::SetFieldScript { name: name.into(), event: "calculate".into(), script: Some(script.into()) }).unwrap(); + } + s.apply(id, Edit::SetFieldScript { name: "total".into(), event: "format".into(), script: None }).unwrap(); + s.apply(id, Edit::SetCalculationOrder { names: vec!["qty".into(), "total".into()] }).unwrap(); + s.apply(id, Edit::SetFieldValue { name: "price".into(), value: FieldValue::Text("3".into()) }).unwrap(); + let total = s.get(id).unwrap().form.iter().find(|f| f.name == "total").unwrap().value.clone(); + assert_eq!(total, ["RGB,0,1,0"]); +} + +#[test] +fn a_document_script_helper_can_set_field_colours() { + let helpers = "function mark\\(name\\) { getField\\(name\\).fillColor = color.red; }"; + let mut s = Session::new().with_clock(|| 1_700_000_000); + let id = s.open("helpers.pdf", None, Arc::new(scripted_form_with(helpers)), None).unwrap(); + s.run_javascript(id, "mark('price');", None).unwrap(); + let doc = pdfcraft_cos::Document::open(s.save_bytes(id).unwrap()).unwrap(); + let f = pdfcraft_forms::fields(&doc).into_iter().find(|f| f.name == "price").unwrap(); + assert_eq!(pdfcraft_forms::look(&doc, &f).fill, Some([1.0, 0.0, 0.0])); +} + #[test] fn calculate_now_simple_calculation_reads_formatted_values_and_groups_as_the_native_path_does() { // amount (4), group b (5) with kids b.c (6) and b.d (7), total (8) = SUM(amount, b). diff --git a/crates/forms/src/scripting.rs b/crates/forms/src/scripting.rs index 4a8c28fae8..c01b1f8842 100644 --- a/crates/forms/src/scripting.rs +++ b/crates/forms/src/scripting.rs @@ -44,6 +44,10 @@ pub struct FieldChange { pub required: Option, /// `display.visible` 0, `hidden` 1, `noPrint` 2, `noView` 3. pub display: Option, + /// Acrobat `textColor` / `fillColor` colour arrays: transparent `["T"]`, gray `["G", g]`, + /// `["RGB", r, g, b]` or `["CMYK", c, m, y, k]`, components between 0 and 1. + pub text_color: Option>, + pub fill_color: Option>, pub caption: Option, /// Move/resize the first widget, in PDF user space. pub rect: Option<[f64; 4]>, @@ -95,8 +99,9 @@ impl Scripts for NoScripts { /// Run the Keystroke (commit) and Validate scripts for a new text value. Returns the value to /// store, or why it is rejected. -pub(crate) fn accept(doc: &mut Document, f: &Field, value: &str, scripts: &mut dyn Scripts) -> Result { +pub(crate) fn accept(doc: &mut Document, f: &Field, value: &str, scripts: &mut dyn Scripts) -> Result<(String, bool), FormError> { let mut v = value.to_string(); + let mut refresh_field = false; for (event, script, default) in [ (FieldEvent::Keystroke, &f.actions.scripts.keystroke, "The value entered does not match the format of the field"), (FieldEvent::Validate, &f.actions.scripts.validate, "The value entered is not valid for the field"), @@ -104,13 +109,14 @@ pub(crate) fn accept(doc: &mut Document, f: &Field, value: &str, scripts: &mut d let Some(js) = script else { continue }; let all = fields(doc); let r = scripts.run(event, js, f, &v, &all); + refresh_field |= r.changes.iter().any(|change| change.name == f.name); apply_changes(doc, &r.changes, &f.name, scripts)?; if !r.rc { return Err(FormError::Invalid(r.message.unwrap_or_else(|| format!("{default} [ {} ]", f.name)))); } v = r.value; } - Ok(v) + Ok((v, refresh_field)) } /// The text a custom Format script shows for `value`, if the field has one. @@ -134,6 +140,12 @@ pub(crate) fn apply_changes(doc: &mut Document, changes: &[FieldChange], except: if changes.is_empty() { return Ok(()); } + // Validate every colour before applying any part of the script's changes. + for c in changes { + for color in [&c.text_color, &c.fill_color].into_iter().flatten() { + js_color(color)?; + } + } let all = fields(doc); for c in changes { let Some(f) = all.iter().find(|f| f.name == c.name) else { continue }; @@ -185,6 +197,17 @@ pub(crate) fn apply_changes(doc: &mut Document, changes: &[FieldChange], except: } } } + // Text can't be transparent: `["T"]` leaves its colour as it is. A transparent fill + // clears the background. + let text = c.text_color.as_deref().map(js_color).transpose()?.flatten(); + let fill = c.fill_color.as_deref().map(js_color).transpose()?; + if text.is_some() || fill.is_some() { + crate::set_props( + doc, + &c.name, + &crate::FieldProps { appearance: Some(crate::LookPatch { text, fill, ..Default::default() }), ..Default::default() }, + )?; + } if let Some(caption) = &c.caption { if f.kind != FieldKind::PushButton { return Err(FormError::Invalid(format!("{} is not a push button", f.name))); @@ -198,6 +221,32 @@ pub(crate) fn apply_changes(doc: &mut Document, changes: &[FieldChange], except: Ok(()) } +/// An Acrobat colour array as the RGB the look model stores, `None` for transparent (`["T"]`). +/// Gray and CMYK convert as Acrobat's `color.convert` does. Anything else (an unknown colour +/// space, the wrong number of components, or a component that is not a finite number between +/// 0 and 1) is refused, and with it the script's changes. +fn js_color(values: &[String]) -> Result, FormError> { + let invalid = || FormError::Invalid("JavaScript field colours must be T, G, RGB or CMYK arrays with finite values between 0 and 1".into()); + let (space, parts) = values.split_first().ok_or_else(invalid)?; + let mut numbers = [0.0; 4]; + if parts.len() > numbers.len() { + return Err(invalid()); + } + for (slot, part) in numbers.iter_mut().zip(parts) { + *slot = part.trim().parse::().ok().filter(|v| v.is_finite() && (0.0..=1.0).contains(v)).ok_or_else(invalid)?; + } + match (space.as_str(), parts.len()) { + ("T", 0) => Ok(None), + ("G", 1) => Ok(Some([numbers[0]; 3])), + ("RGB", 3) => Ok(Some([numbers[0], numbers[1], numbers[2]])), + ("CMYK", 4) => { + let [c, m, y, k] = numbers; + Ok(Some([1.0 - (c + k).min(1.0), 1.0 - (m + k).min(1.0), 1.0 - (y + k).min(1.0)])) + } + _ => Err(invalid()), + } +} + /// Apply what a button or console script changed (values, read-only, required, visibility), /// then recalculate. pub fn apply_script_changes(doc: &mut Document, changes: &[FieldChange], scripts: &mut dyn Scripts) -> Result<(), FormError> { diff --git a/crates/forms/src/tests.rs b/crates/forms/src/tests.rs index 1dd0524272..859467744e 100644 --- a/crates/forms/src/tests.rs +++ b/crates/forms/src/tests.rs @@ -3657,8 +3657,7 @@ fn recalc_chain_where_script_changes_checkbox() { read_only: None, required: None, display: None, - caption: None, - rect: None, + ..Default::default() }], message: None, } @@ -3742,6 +3741,65 @@ fn an_export_value_wins_over_an_earlier_options_display_text() { } } +#[test] +fn validation_script_colour_is_used_when_the_field_redraws() { + struct TestScripts; + impl Scripts for TestScripts { + fn run(&mut self, _event: FieldEvent, _script: &str, _target: &Field, value: &str, _fields: &[Field]) -> ScriptResult { + ScriptResult { + rc: true, + value: value.into(), + value_set: false, + changes: vec![FieldChange { + name: "name".into(), + text_color: Some(vec!["RGB".into(), "1".into(), "0".into(), "0".into()]), + ..Default::default() + }], + message: None, + } + } + } + + let mut doc = fixture(); + set_field_script(&mut doc, "name", "validate", Some("set_red")).unwrap(); + set_value_with(&mut doc, "name", &FieldValue::Text("Grace".into()), &mut TestScripts).unwrap(); + + let all = fields(&doc); + let f = field(&all, "name"); + assert!(f.da.contains("1 0 0 rg")); + assert!(ap(&doc, &f.widgets[0]).contains("1 0 0 rg"), "redraw uses the Validate script's updated appearance"); +} + +#[test] +fn calculation_script_colour_is_used_when_the_field_redraws() { + struct TestScripts; + impl Scripts for TestScripts { + fn run(&mut self, _event: FieldEvent, _script: &str, _target: &Field, _value: &str, _fields: &[Field]) -> ScriptResult { + ScriptResult { + rc: true, + value: "Grace".into(), + value_set: true, + changes: vec![FieldChange { + name: "name".into(), + text_color: Some(vec!["RGB".into(), "1".into(), "0".into(), "0".into()]), + ..Default::default() + }], + message: None, + } + } + } + + let mut doc = fixture(); + set_field_script(&mut doc, "name", "calculate", Some("set_red")).unwrap(); + recalculate_with(&mut doc, &mut TestScripts).unwrap(); + + let all = fields(&doc); + let f = field(&all, "name"); + assert_eq!(f.value, ["Grace"]); + assert!(f.da.contains("1 0 0 rg")); + assert!(ap(&doc, &f.widgets[0]).contains("1 0 0 rg"), "redraw uses the Calculate script's updated appearance"); +} + #[test] fn partial_appearance_keeps_custom_default_appearance_and_font_resources() { let mut doc = fixture(); diff --git a/crates/forms/src/values.rs b/crates/forms/src/values.rs index b1d50a87ae..bbf1467415 100644 --- a/crates/forms/src/values.rs +++ b/crates/forms/src/values.rs @@ -262,7 +262,8 @@ pub fn recalculate_with(doc: &mut Document, scripts: &mut dyn Scripts) -> Result _ => {} } })?; - redraw(doc, &f, std::slice::from_ref(&v), scripts)?; + let fresh = all.iter().find(|field| field.obj == obj).cloned().unwrap_or_else(|| f.clone()); + redraw(doc, &fresh, std::slice::from_ref(&v), scripts)?; if let Some(target) = all.iter_mut().find(|x| x.obj == obj) { target.value = vec![v]; } @@ -305,17 +306,19 @@ pub(crate) fn write_value(doc: &mut Document, f: &Field, value: &FieldValue, scr // Keystroke (on commit) and Validate, as Acrobat runs them before accepting a value. let t = &f.actions.keystroke(&f.name, t).map_err(FormError::Format)?; af::validate(&f.actions.validate, t).map_err(FormError::Invalid)?; - let t = &scripting::accept(doc, f, t, scripts)?; + let (t, refresh_field) = scripting::accept(doc, f, t, scripts)?; let shadow = t.is_empty() && crate::author::ancestor_sets(doc, f.obj, b"V"); doc.update_dict(f.obj, |d| { if t.is_empty() && !shadow { d.remove(b"V"); } else { - d.set(b"V".to_vec(), PdfString::text(t)); + d.set(b"V".to_vec(), PdfString::text(&t)); } d.remove(b"RV"); })?; - redraw(doc, f, std::slice::from_ref(t), scripts) + let current = + if refresh_field { fields(doc).into_iter().find(|current| current.obj == f.obj).unwrap_or_else(|| f.clone()) } else { f.clone() }; + redraw(doc, ¤t, std::slice::from_ref(&t), scripts) } (FieldKind::CheckBox | FieldKind::Radio, FieldValue::Widget(i)) => { // A button can be read-only on its own (its own `/Ff`), though the field isn't. diff --git a/crates/ui-egui/src/canvas/area.rs b/crates/ui-egui/src/canvas/area.rs index 5ec07a1fb1..724e5270a5 100644 --- a/crates/ui-egui/src/canvas/area.rs +++ b/crates/ui-egui/src/canvas/area.rs @@ -205,8 +205,7 @@ pub(super) fn run_button(app: &mut PdfCraftApp, index: usize, name: &str, action read_only: None, required: None, display: Some(if hide { 1 } else { 0 }), - caption: None, - rect: None, + ..Default::default() }) .collect(); if !changes.is_empty() { diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 0d70edf70c..79423e76cb 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -5375,8 +5375,8 @@ milestone = "M6" status = "shipped" commands = ["tools.js_console"] tools = ["js_run"] -evidence = ["crates/ui-egui/tests/it/javascript.rs::console_runs_scripts_against_the_document", "crates/automation/tests/it/automation.rs::javascript_through_tools", "crates/engine/src/tests.rs::calculate_now_updates_live_fields_in_one_undo_step"] -notes = "\u2318J: run JavaScript against the document, with console output, results and errors. calculateNow() runs calculated fields synchronously in /CO order against writes already made by the script." +evidence = ["crates/ui-egui/tests/it/javascript.rs::console_runs_scripts_against_the_document", "crates/automation/tests/it/automation.rs::javascript_through_tools", "crates/engine/src/tests.rs::javascript_field_colours_persist_and_undo_without_changing_other_appearance_properties", "crates/engine/src/tests.rs::javascript_invalid_field_colour_array_is_refused_without_partial_changes", "crates/engine/src/tests.rs::calculate_now_updates_live_fields_in_one_undo_step"] +notes = "\u2318J: run JavaScript against the document, with console output, results and errors. `fillColor` and `textColor` changes (transparent, gray, RGB or CMYK) persist through save/reopen and participate in undo/redo; malformed or out-of-range colour arrays are refused without applying partial script changes. calculateNow() runs calculated fields synchronously in /CO order against writes already made by the script." [[feature]] id = "form.js-debugger" From ad3e6d4b85cd8d16eaad652f1704bfe6ad1c7481 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Tue, 10 Nov 2026 17:53:58 -0500 Subject: [PATCH 19/60] fix(optimize): bound each unused-resource decode by the remaining shared budget (#6046) --- crates/cos/src/object.rs | 13 +++++++++++++ crates/optimize/src/unused.rs | 8 +++++--- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/crates/cos/src/object.rs b/crates/cos/src/object.rs index f2c9760c12..48a23c4af2 100644 --- a/crates/cos/src/object.rs +++ b/crates/cos/src/object.rs @@ -453,6 +453,12 @@ impl Stream { self.decode_chain(max, decode).map(|(v, _)| v).map_err(|(e, _)| CosError::Filter(e.to_string())) } + /// [`Stream::decoded_strict`] for a caller with less than [`MAX_DECODED`] left to spend: + /// decoding past `max` bytes (in any filter stage) is an error. + pub fn decoded_strict_within(&self, max: usize) -> Result, CosError> { + self.decoded_strict_within_counted(max).map(|(v, _)| v).map_err(|(e, _)| CosError::Filter(e.to_string())) + } + /// [`Stream::decoded_strict`] bounded by `max` and counted like /// [`Stream::decoded_within_counted`]. pub(crate) fn decoded_strict_within_counted(&self, max: usize) -> Result<(Vec, usize), (pdfcraft_filters::FilterError, usize)> { @@ -746,6 +752,13 @@ mod tests { } } + #[test] + fn strict_decoding_is_bounded_by_the_callers_remaining_allowance() { + let stream = Stream::flate(Dict::new(), &vec![0u8; 1 << 20]); + assert!(matches!(stream.decoded_strict_within(1000), Err(CosError::Filter(_)))); + assert_eq!(stream.decoded_strict_within(1 << 20).map(|v| v.len()).ok(), Some(1 << 20)); + } + #[test] fn empty_filter_arrays_refuse_decoding_past_a_caller_limit() { let mut dict = Dict::new(); diff --git a/crates/optimize/src/unused.rs b/crates/optimize/src/unused.rs index b8c8a62b23..002d50263f 100644 --- a/crates/optimize/src/unused.rs +++ b/crates/optimize/src/unused.rs @@ -168,7 +168,9 @@ fn plain_metadata(doc: &Document) -> bool { .is_some_and(|e| matches!(e.as_dict().and_then(|e| e.get(b"EncryptMetadata")), Some(Object::Bool(false)))) } -fn data(s: &Stream, num: u32, g: &Graph, encrypted: bool, plain_metadata: bool) -> Data { +/// `left` is what the caller may still decode: a stream that would decode past it (in any filter +/// stage) is `Unknown`, so no stage is allowed to grow beyond the shared budget first. +fn data(s: &Stream, num: u32, g: &Graph, encrypted: bool, plain_metadata: bool, left: usize) -> Data { let d = &s.dict; // Streams the security handler leaves as stored, which a viewer may still decrypt. let ty = d.name(b"Type"); @@ -219,7 +221,7 @@ fn data(s: &Stream, num: u32, g: &Graph, encrypted: bool, plain_metadata: bool) if !parms_line_up { return Data::Unknown; } - s.decoded_strict().map_or(Data::Unknown, Data::Decoded) + s.decoded_strict_within(left).map_or(Data::Unknown, Data::Decoded) } /// Drop the image and form XObjects that `pages` list in their own resources but no stream @@ -254,7 +256,7 @@ pub(crate) fn remove_unused(doc: &mut Document, pages: &[ObjRef], progress: &mut } let obj = doc.get(ObjRef::new(*num, doc.generation(*num))); let Object::Stream(s) = &*obj else { continue }; - match data(s, *num, &g, encrypted, plain_metadata) { + match data(s, *num, &g, encrypted, plain_metadata, DECODE_BUDGET.saturating_sub(decoded)) { Data::Picture => {} Data::Decoded(bytes) => { decoded = decoded.saturating_add(bytes.len()); From 6b53325f286207fa43b22b4e11fc5761febef618 Mon Sep 17 00:00:00 2001 From: Grzegorz Martens Date: Tue, 10 Nov 2026 19:09:11 +0100 Subject: [PATCH 20/60] M11: Optimize stores one-bit images and masks past 80 million pixels as monochrome images The image steps leave an image over 80 million pixels alone, for memory: decoded at eight bits a pixel or more, it would hold 80 MB a component and more. An image already of one bit a pixel holds a byte for each 8 samples of a row, so a large black-and-white scan (an A0 page at 600 ppi has about 560 million pixels) was left as stored though its samples would take less than 80 MB. The monochrome step now takes an image of one bit a pixel as stored (an image mask, or one bit in one colour component), a picture's one-bit mask included, while its rows hold no more bytes than 80 million pixels of eight bits (`one_bit_bytes`): up to 640 million pixels when it is wide, and a strip one sample wide no taller than before. One it would pack from eight bits keeps the 80 million limit, as it is decoded at eight bits first. A fax image's code is allowed no more than 80 MB for its pixels however large the page, and four bytes a row, where it was allowed about a byte a pixel (`fax_code_allowance`). When planning how many pictures run at once, a one-bit image counts as the bytes its rows hold and one more a row, as decoding allows each row's end a few bytes besides its samples. The optimize README and the parity note say which images the monochrome step now takes, and the note states JBIG2's limit of 2^28 pixels in all, as the README does. `group_4_takes_images_of_one_bit_a_pixel_up_to_the_pixel_limit` now expects its one-bit image of 10,000 by 8,001 stored again, where it was left alone. The packing path counts what it decodes, for the tests, as the one-bit path does. Tests: `one_bit_images_past_80_million_pixels_are_stored_as_group_4` (a one-bit image, a picture's stencil mask and a picture's one-bit soft mask of 9,000 by 8,900), `a_one_bit_image_past_80_million_pixels_is_averaged_with_the_default_settings`, `an_image_too_large_for_the_jbig2_decoder_stays_as_it_is` (2^28 pixels stored as JBIG2, a row more left alone, and averaged), and the unit tests `one_bit_images_are_decoded_up_to_their_own_limit` and `a_one_bit_picture_counts_as_the_bytes_of_its_rows` (each with a strip one sample wide, at its limit and past it) and `a_fax_images_code_is_allowed_80_million_bytes_and_four_a_row` and `a_zero_wide_image_is_not_decoded`. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01AmsdeUCdTaD72jkDw2pJKY --- crates/optimize/README.md | 15 ++-- crates/optimize/src/images.rs | 129 ++++++++++++++++++++++++++++-- crates/optimize/tests/optimize.rs | 95 +++++++++++++++++++--- parity/acrobat-features.toml | 4 +- 4 files changed, 217 insertions(+), 26 deletions(-) diff --git a/crates/optimize/README.md b/crates/optimize/README.md index a6dfd9caed..a2436676bd 100644 --- a/crates/optimize/README.md +++ b/crates/optimize/README.md @@ -34,9 +34,13 @@ let report = optimize_with_progress(&mut doc, &settings, &mut |stage| keep_going stored as JPEG (Acrobat's quality steps) or ZIP; soft masks are resampled with their image. A stencil `/Mask` has its own resolution, so the picture changes under it and it keeps its size and its bits (stored again as a picture's mask, below). - An image is replaced only if the result is smaller. Images over 80 million pixels are left - alone by the image steps, here and below, whatever their settings (memory); Clean Up may still - Flate an unfiltered or LZW one. Left alone by these colour and grey + An image is replaced only if the result is smaller. Images over 80 million pixels are left alone + by the image steps, here and below, whatever their settings (memory), except that the monochrome + step below takes an image already of one bit a pixel, a mask included, while its rows hold no more + bytes than 80 million pixels of eight bits (each row starts on a new byte: a byte for each 8 + samples and one for any left over): up to 640 million pixels when it is wide, and a strip one + sample wide no taller than one of eight bits. Clean Up may still Flate an unfiltered or LZW one. + Left alone by these colour and grey settings: CMYK and other colour spaces, image masks, colour-key masks, decode arrays, more than 8 bits, JPEG 2000, JBIG2 and CCITT (black-and-white images, CCITT and JBIG2 among them, are below). A JPEG is read with its colour transform (an Adobe marker's, else `/ColorTransform`, ISO 32000-2 §7.4.8), so one @@ -119,8 +123,9 @@ let report = optimize_with_progress(&mut doc, &settings, &mut |stage| keep_going more bits stays as it is, to be resampled with its picture, which wants it at eight. The report counts masks as images. - Not yet: inline images, a picture's mask resampled, or packed to one bit from more, Group 4 - for rows wider than 32,767 samples, and JBIG2 past 65,535 samples on a side, where the decoder - stops (its 2^28 in all is past the 80 million pixels above). + for rows wider than 32,767 samples, and JBIG2 past 65,535 samples on a side or 2^28 in all, + where the decoder stops: with JBIG2 chosen, a one-bit image larger than that stays unless + it is averaged. - Lossless JBIG2 uses bounded generic-region encoding or exact-copy symbol dictionaries when smaller, a dictionary shared by up to 16 images when that is smaller still. No lossy symbol substitution is used. Export, redaction and this step read JBIG2 diff --git a/crates/optimize/src/images.rs b/crates/optimize/src/images.rs index 24fba3ef7a..4037de82f3 100644 --- a/crates/optimize/src/images.rs +++ b/crates/optimize/src/images.rs @@ -13,6 +13,15 @@ use crate::{Compression, ImageSettings, MonoCompression, MonoSettings, OptimizeE /// Images with more pixels than this are left alone (memory). pub(crate) const MAX_PIXELS: u64 = 80_000_000; +/// The bytes an image of one bit a pixel holds as stored: each row starts on a new byte, a byte for +/// each 8 samples and one for any left over. The monochrome step takes such an image while these +/// are no more than [`MAX_PIXELS`], as many as that many pixels of eight bits hold: a wide one up +/// to 640 million pixels (an A0 page at 600 ppi is about 560 million), where at eight bits a pixel +/// it would be left alone, and a strip one sample wide no taller than one of eight bits. +fn one_bit_bytes(w: u64, h: u64) -> u64 { + w.div_ceil(8).saturating_mul(h) +} + /// The resources of a page, inherited from the page tree if the page has none. pub(crate) fn page_resources(doc: &Document, page: ObjRef) -> Option { held_page_resources(doc, page).map(|(resources, _)| resources) @@ -829,6 +838,21 @@ fn one_component(doc: &Document, cs: &Object) -> bool { } } +/// What the filters before a fax image's decoder may produce, its code: at most about a byte a +/// pixel and each row's end, as redaction allows, but no more than `MAX_PIXELS` bytes for its +/// pixels (and four a row), so a large page's code can't cost a byte a pixel. +fn fax_code_allowance(w: usize, h: usize) -> Option { + w.checked_mul(h)?.min(usize::try_from(MAX_PIXELS).ok()?).checked_add(h.checked_mul(4)?)?.checked_add(4096) +} + +/// Whether `d` is an image of one bit a pixel as stored: an image mask, or one bit in one colour +/// component. +fn one_bit_as_stored(doc: &Document, d: &Dict) -> bool { + let bits = d.int(b"BitsPerComponent"); + let mask = matches!(d.get(b"ImageMask"), Some(Object::Bool(true))); + (mask && bits.is_none_or(|b| b == 1)) || (bits == Some(1) && d.get(b"ColorSpace").is_some_and(|cs| one_component(doc, cs))) +} + /// A pixel's samples as a 1 or a 0, or `None` for a colour that is neither. type Pixel = fn(&[u8]) -> Option; @@ -854,19 +878,23 @@ fn one_bit(doc: &Document, s: &Stream, pack: bool, defaults: bool, budget: &mut let d = &s.dict; // Through u32 first, as everywhere here, so the product can't overflow. let (w, h) = (u32::try_from(d.int(b"Width")?).ok()?, u32::try_from(d.int(b"Height")?).ok()?); - if w == 0 || h == 0 || u64::from(w) * u64::from(h) > MAX_PIXELS { + // An image already of one bit a pixel is taken while its rows hold no more than `MAX_PIXELS` + // bytes (`one_bit_bytes`), one packed from more bits (2, 4 or 8) up to `MAX_PIXELS` pixels, as + // before: its samples are decoded as stored first, eight-bit RGB at three bytes a pixel. + let as_stored = one_bit_as_stored(doc, d); + let held = if as_stored { one_bit_bytes(u64::from(w), u64::from(h)) } else { u64::from(w) * u64::from(h) }; + if w == 0 || h == 0 || held > MAX_PIXELS { return None; } let (wu, hu) = (usize::try_from(w).ok()?, usize::try_from(h).ok()?); let stride = wu.div_ceil(8); let bits = d.int(b"BitsPerComponent"); let mask = matches!(d.get(b"ImageMask"), Some(Object::Bool(true))); - if (mask && bits.is_none_or(|b| b == 1)) || (bits == Some(1) && d.get(b"ColorSpace").is_some_and(|cs| one_component(doc, cs))) { - // No further than its rows can need: a fax image's code (the filters before its decoder - // produce it) takes at most about a byte a pixel and each row's end, as redaction allows. + if as_stored { + // No further than its rows can need. let len = stride.checked_mul(hu)?; let max = match s.filters().last() { - Some((pdfcraft_filters::Filter::CcittFax, _)) => wu.checked_mul(hu)?.checked_add(hu.checked_mul(4)?)?.checked_add(4096)?, + Some((pdfcraft_filters::Filter::CcittFax, _)) => fax_code_allowance(wu, hu)?, // A JBIG2 page, which has to be the image's size, is `len` bytes as rows of a bit. _ => allowance(len, h)?, }; @@ -937,6 +965,7 @@ fn one_bit(doc: &Document, s: &Stream, pack: bool, defaults: bool, budget: &mut _ => return None, }; let row_len = wu.checked_mul(n)?.checked_mul(bpc)?.div_ceil(8); + count_decoded(u64::from(w) * u64::from(h)); // Through its parameters given by reference too: without a PNG predictor's tag bytes taken // off, rows of 0 and 255 (or palette entries 0 and 1) would still pass below, each shifted. let samples = budget.decode_filters(doc, s, allowance(row_len.checked_mul(hu)?, h)?).ok()?; @@ -1605,12 +1634,21 @@ const PICTURE_MEMORY: u64 = 1 << 30; /// What a picture holds while it is worked out, for each pixel: decoded, and resampled or encoded. const PICTURE_BYTES_PER_PIXEL: u64 = 8; -/// A picture's pixels, if it is small enough to be decoded at all. +/// A picture's pixels, if it is small enough to be decoded at all. One of one bit a pixel as +/// stored is taken while the bytes its rows hold ([`one_bit_bytes`]) are no more than +/// `MAX_PIXELS`, and counts as those and one more a row: decoding allows each row's end a few +/// bytes besides its samples ([`allowance`], [`fax_code_allowance`]), which for an image a few +/// samples wide is as much again as its samples. fn picture_pixels(doc: &Document, r: ObjRef) -> Option { let object = doc.get(r); let Object::Stream(s) = &*object else { return None }; let (w, h) = (u64::try_from(s.dict.int(b"Width")?).ok()?, u64::try_from(s.dict.int(b"Height")?).ok()?); - w.checked_mul(h).filter(|pixels| *pixels <= MAX_PIXELS) + if one_bit_as_stored(doc, &s.dict) { + let bytes = one_bit_bytes(w, h); + return (bytes <= MAX_PIXELS).then(|| bytes.saturating_add(h)); + } + let pixels = w.checked_mul(h)?; + (pixels <= MAX_PIXELS).then_some(pixels) } /// How many pictures of `largest` pixels to work on at once: `workers`, or as many as @@ -1861,6 +1899,83 @@ mod dimension_tests { assert_eq!(threads_for(8, 0), 8); } + /// A one-bit image counts as the bytes its rows hold and one more a row when planning how many + /// pictures run at once, while those it holds are up to `MAX_PIXELS`: about an eighth of its + /// pixels when wide, two bytes a row when a few samples wide. One of eight bits counts whole, + /// up to `MAX_PIXELS`. + #[test] + fn a_one_bit_picture_counts_as_the_bytes_of_its_rows() { + let mut doc = Document::new_empty(); + let mut add = |w: i64, h: i64, bits: i64| { + let mut dict = Dict::new(); + dict.set(b"Width".to_vec(), Object::Int(w)); + dict.set(b"Height".to_vec(), Object::Int(h)); + dict.set(b"BitsPerComponent".to_vec(), Object::Int(bits)); + dict.set(b"ColorSpace".to_vec(), Object::name("DeviceGray")); + doc.add(Object::Stream(Stream::from_raw(dict, Vec::new()))) + }; + let (one_bit, past_one_bit, eight_bit) = (add(20_000, 32_000, 1), add(20_000, 32_001, 1), add(10_000, 8_001, 8)); + let (strip, past_strip, narrow) = (add(1, 80_000_000, 1), add(1, 80_000_001, 1), add(8, 10_000_000, 1)); + assert_eq!(picture_pixels(&doc, one_bit), Some(80_000_000 + 32_000)); + assert_eq!(picture_pixels(&doc, past_one_bit), None); + assert_eq!(picture_pixels(&doc, strip), Some(160_000_000), "a byte a row, and one more"); + assert_eq!(picture_pixels(&doc, past_strip), None); + assert_eq!(picture_pixels(&doc, narrow), Some(20_000_000)); + assert_eq!(threads_for(8, 20_000_000), 6); + assert_eq!(threads_for(8, 160_000_000), 1); + assert_eq!(picture_pixels(&doc, eight_bit), None); + assert_eq!(threads_for(8, 80_000_000), 1); + } + + /// A fax image's code may take about a byte a pixel and its rows' ends, but no more than + /// `MAX_PIXELS` bytes for its pixels however large the page, and four a row. + #[test] + fn a_fax_images_code_is_allowed_80_million_bytes_and_four_a_row() { + assert_eq!(fax_code_allowance(1_000, 1_000), Some(1_000_000 + 4_000 + 4_096)); + assert_eq!(fax_code_allowance(20_000, 32_000), Some(80_000_000 + 128_000 + 4_096)); + assert_eq!(fax_code_allowance(1, 80_000_000), Some(80_000_000 + 320_000_000 + 4_096)); + } + + /// The monochrome step decodes an image already of one bit a pixel while its rows hold no more + /// than `MAX_PIXELS` bytes, and one it would pack from eight bits up to `MAX_PIXELS` pixels. + /// Past those, nothing is decoded. + #[test] + fn one_bit_images_are_decoded_up_to_their_own_limit() { + let image = |w: i64, h: i64, bits: i64| { + let mut dict = Dict::new(); + dict.set(b"Width".to_vec(), Object::Int(w)); + dict.set(b"Height".to_vec(), Object::Int(h)); + dict.set(b"BitsPerComponent".to_vec(), Object::Int(bits)); + dict.set(b"ColorSpace".to_vec(), Object::name("DeviceGray")); + Stream::from_raw(dict, Vec::new()) + }; + let decoded = |s: &Stream| { + DECODED_PIXELS.with(|c| c.set(0)); + one_bit(&Document::new_empty(), s, true, false, &mut ImageBudget::with_work(u64::MAX)); + DECODED_PIXELS.with(|c| c.get()) + }; + assert_eq!(decoded(&image(20_000, 32_000, 1)), 640_000_000, "one bit, at the limit"); + assert_eq!(decoded(&image(20_000, 32_001, 1)), 0, "one bit, past it"); + assert_eq!(decoded(&image(1, 80_000_000, 1)), 80_000_000, "one bit a sample wide, a byte a row, at the limit"); + assert_eq!(decoded(&image(1, 80_000_001, 1)), 0, "one bit a sample wide, past it"); + assert_eq!(decoded(&image(10_000, 8_000, 8)), 80_000_000, "eight bits, at theirs"); + assert_eq!(decoded(&image(10_000, 8_001, 8)), 0, "eight bits, past theirs"); + } + + /// A zero-wide image isn't decoded: its rows hold no bytes, but its filters would be allowed + /// three a row. + #[test] + fn a_zero_wide_image_is_not_decoded() { + let mut dict = Dict::new(); + dict.set(b"Width".to_vec(), Object::Int(0)); + dict.set(b"Height".to_vec(), Object::Int(1_000_000)); + dict.set(b"ImageMask".to_vec(), Object::Bool(true)); + let s = Stream::flate(dict, &vec![0u8; 1 << 20]); + let mut budget = ImageBudget::with_work(1 << 40); + assert!(one_bit(&Document::new_empty(), &s, true, false, &mut budget).is_none()); + assert_eq!(budget.work_left(), 1 << 40); + } + /// Only what redaction can store again without the dictionary shares one: an image mask, or /// DeviceGray named directly, of at most 2^26 pixels. #[test] diff --git a/crates/optimize/tests/optimize.rs b/crates/optimize/tests/optimize.rs index 7c3e76f9b9..7da6151deb 100644 --- a/crates/optimize/tests/optimize.rs +++ b/crates/optimize/tests/optimize.rs @@ -423,8 +423,10 @@ fn black_and_white_images_become_ccitt_group_4() { /// Which 1-bit images qualify for Group 4: those of one colour component, an ICC profile of one /// component and a DeviceN of one colourant among them, but not one of three components or two /// colourants (more than a bit a pixel); image masks of one bit, as `/BitsPerComponent 1` says or -/// by default, but not one that says 8; and images of up to 80 million pixels, not one row more. -/// The byte totals count exactly the images stored again. +/// by default, but not one that says 8; and images past 80 million pixels too, where an image of +/// eight bits would be left alone (`one_bit_images_are_decoded_up_to_their_own_limit` checks the +/// bytes of rows a one-bit image may hold without decoding so much). The byte totals count exactly +/// the images stored again. #[test] fn group_4_takes_images_of_one_bit_a_pixel_up_to_the_pixel_limit() { let mut doc = Document::new_empty(); @@ -457,7 +459,7 @@ fn group_4_takes_images_of_one_bit_a_pixel_up_to_the_pixel_limit() { let mask = |bits: i64| [("ImageMask", Object::Bool(true)), ("BitsPerComponent", bits.into())]; let mask_1 = mono_image(&mut doc, w, h, &text, None, &mask(1)); let mask_8 = mono_image(&mut doc, w, h, &bilevel(8 * w, h, false), None, &mask(8)); - // 10,000 × 8,000 is the limit itself. + // 10,000 × 8,000 is the limit of an image of eight bits; one of one bit goes past it. let grey = one_bit(Object::name("DeviceGray")); let at_limit = mono_image(&mut doc, 10_000, 8_000, &bilevel(10_000, 8_000, false), None, &grey); let past_limit = mono_image(&mut doc, 10_000, 8_001, &bilevel(10_000, 8_001, false), None, &grey); @@ -474,8 +476,8 @@ fn group_4_takes_images_of_one_bit_a_pixel_up_to_the_pixel_limit() { }) .collect(); page(&mut doc, &names.iter().map(String::as_str).zip(all).collect::>(), &content); - let converted = [icc_1, spot, mask_1, at_limit]; - let left = [icc_3, two_spots, mask_8, past_limit]; + let converted = [icc_1, spot, mask_1, at_limit, past_limit]; + let left = [icc_3, two_spots, mask_8]; let before = |doc: &Document, refs: &[ObjRef]| refs.iter().map(|r| stream(doc, *r).raw).collect::>(); let (old, untouched) = (before(&doc, &converted), before(&doc, &left)); let r = optimize(&mut doc, &Settings::default()).unwrap(); @@ -4925,15 +4927,22 @@ fn indexed_thin_rules_survive_averaging_for_both_palette_orders_and_decodes() { } } -/// An image past what the JBIG2 decoder takes on a side stays as it is: JBIG2 can state the size, -/// but PdfCraft's decoder refuses such a stream, so stored as JBIG2 it drew nothing here. At 65,535 -/// on a side, the same bars do become JBIG2, so the fixture does reach that choice, and the coding -/// stored (symbols, here) reads back through that decoder. +/// An image past what the JBIG2 decoder takes on a side, or past 2^28 pixels in all, stays as it +/// is: JBIG2 can state the size, but PdfCraft's decoder refuses such a stream, so stored as JBIG2 it +/// drew nothing here. At 65,535 on a side, and at 2^28 in all (past 80 million, which the +/// monochrome step takes of a one-bit image), the same bars do become JBIG2, so the fixture does +/// reach that choice, and the coding stored reads back through that decoder. Averaged, an image past +/// 2^28 becomes small enough and is stored as JBIG2 too. #[test] fn an_image_too_large_for_the_jbig2_decoder_stays_as_it_is() { - for (w, h, stored) in - [(65_535u32, 16u32, &b"JBIG2Decode"[..]), (16, 65_535, b"JBIG2Decode"), (65_536, 16, b"FlateDecode"), (16, 65_536, b"FlateDecode")] - { + for (w, h, stored) in [ + (65_535u32, 16u32, &b"JBIG2Decode"[..]), + (16, 65_535, b"JBIG2Decode"), + (65_536, 16, b"FlateDecode"), + (16, 65_536, b"FlateDecode"), + (16_384, 16_384, b"JBIG2Decode"), + (16_384, 16_385, b"FlateDecode"), + ] { let mut doc = Document::new_empty(); let image = mono_image(&mut doc, w, h, &bars(w, h), None, &[("BitsPerComponent", 1.into()), ("ColorSpace", Object::name("DeviceGray"))]); page(&mut doc, &[("Im", image)], &format!("q {w} 0 0 {h} 0 0 cm /Im Do Q")); @@ -4958,6 +4967,15 @@ fn an_image_too_large_for_the_jbig2_decoder_stays_as_it_is() { assert!(decoded_image(&doc, &s).unwrap() == bits, "{w} × {h} reads back"); } } + let (w, h) = (16_384u32, 16_385u32); + let mut doc = Document::new_empty(); + let image = mono_image(&mut doc, w, h, &bars(w, h), None, &[("BitsPerComponent", 1.into()), ("ColorSpace", Object::name("DeviceGray"))]); + let (pw, ph) = (f64::from(w) * 72.0 / 600.0, f64::from(h) * 72.0 / 600.0); + page(&mut doc, &[("Im", image)], &format!("q {pw} 0 0 {ph} 0 0 cm /Im Do Q")); + let settings = Settings { mono: MonoSettings { compression: MonoCompression::Jbig2, ..MonoSettings::REDUCE }, ..Settings::default() }; + let r = optimize(&mut doc, &settings).unwrap(); + assert_eq!(r.images_resampled, 1, "{r:?}"); + assert_eq!(stream(&doc, image).dict.name(b"Filter"), Some(&b"JBIG2Decode"[..]), "averaged, it is stored as JBIG2"); } /// The colour and grey steps leave an image over 80 million pixels as it is, whatever their settings @@ -5405,6 +5423,59 @@ fn a_picture_with_a_stencil_mask_is_downsampled_under_its_mask() { /// the page looks the same to the pixel. A mask two pictures share is stored once, and a stencil /// the page also draws by name is stored as that image. An eight-bit soft mask of only black and /// white stays as it is, to be resampled with its picture. With `mono: Retain` nothing changes. +/// One-bit images past 80 million pixels are stored again, as images of eight bits that large are +/// not: their rows hold a byte for each 8 samples. A picture's stencil mask and one-bit soft mask +/// too. +#[test] +fn one_bit_images_past_80_million_pixels_are_stored_as_group_4() { + let mut doc = Document::new_empty(); + let (w, h) = (9_000, 8_900); + let text = bilevel(w, h, false); + let one_bit = mono_image(&mut doc, w, h, &text, None, &[("BitsPerComponent", 1.into()), ("ColorSpace", Object::name("DeviceGray"))]); + let stencil = mono_image(&mut doc, w, h, &text, None, &[("ImageMask", Object::Bool(true))]); + let soft = mono_image(&mut doc, w, h, &text, None, &[("BitsPerComponent", 1.into()), ("ColorSpace", Object::name("DeviceGray"))]); + let masked = |doc: &mut Document, key: &str, mask: ObjRef| { + let picture = image(doc, 200, 80, 3, false, None); + let mut s = stream(doc, picture); + s.dict.set(key.as_bytes().to_vec(), Object::Ref(mask)); + doc.set(picture, Object::Stream(s)); + picture + }; + let (picture, shaded) = (masked(&mut doc, "Mask", stencil), masked(&mut doc, "SMask", soft)); + page( + &mut doc, + &[("I", one_bit), ("P", picture), ("S", shaded)], + "q 300 0 0 300 20 400 cm /I Do Q q 300 0 0 120 20 100 cm /P Do Q q 300 0 0 120 20 250 cm /S Do Q", + ); + let keep = ImageSettings { downsample: false, compression: Compression::Retain, ..ImageSettings::REDUCE }; + let mono = MonoSettings { downsample: false, ..MonoSettings::REDUCE }; + let before = [one_bit, stencil, soft].map(|r| stream(&doc, r)); + let r = optimize(&mut doc, &Settings { color: keep, gray: keep, mono, ..Settings::default() }).unwrap(); + assert_eq!(r.images_recompressed, 3, "{r:?}"); + for (r, old) in [one_bit, stencil, soft].into_iter().zip(&before) { + let new = stream(&doc, r); + assert_eq!(new.dict.name(b"Filter"), Some(&b"CCITTFaxDecode"[..]), "{r:?}"); + assert!(new.raw.len() < old.raw.len(), "{r:?}: {} against {}", new.raw.len(), old.raw.len()); + assert_eq!(decoded_image(&doc, &new).unwrap(), text, "{r:?} keeps its bits"); + } +} + +/// With the default settings (Reduce File Size), a one-bit image past 80 million pixels drawn at +/// 600 ppi is averaged to 300 ppi, as a smaller one is: vertical stripes four samples wide. +#[test] +fn a_one_bit_image_past_80_million_pixels_is_averaged_with_the_default_settings() { + let (w, h) = (9_000u32, 8_900u32); + let mut doc = Document::new_empty(); + let stripes = vec![0x0F; (w as usize).div_ceil(8) * h as usize]; + let one_bit = mono_image(&mut doc, w, h, &stripes, None, &[("BitsPerComponent", 1.into()), ("ColorSpace", Object::name("DeviceGray"))]); + let (pw, ph) = (f64::from(w) * 72.0 / 600.0, f64::from(h) * 72.0 / 600.0); + page(&mut doc, &[("I", one_bit)], &format!("q {pw} 0 0 {ph} 0 0 cm /I Do Q")); + let r = optimize(&mut doc, &Settings::default()).unwrap(); + assert_eq!(r.images_resampled, 1, "{r:?}"); + let s = stream(&doc, one_bit); + assert_eq!((s.dict.int(b"Width"), s.dict.int(b"Height")), (Some(4_500), Some(4_450))); +} + #[test] fn a_pictures_one_bit_masks_are_stored_again() { let mut doc = Document::new_empty(); diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 046c080c62..7d0864676b 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -7549,8 +7549,8 @@ milestone = "M11" status = "partial" commands = ["optimize.advanced", "optimize.reduce"] tools = ["doc_optimize", "doc_reduce"] -evidence = ["crates/optimize/src/images.rs::flate_images_past_the_runs_allowance_stay_as_they_are", "crates/optimize/tests/optimize.rs::eight_bit_black_and_white_images_become_one_bit", "crates/filters/src/ccitt.rs::bitmaps_round_trip", "crates/filters/src/ccitt.rs::a_page_of_text_like_marks_round_trips_and_shrinks", "crates/filters/src/ccitt.rs::what_it_cannot_encode_is_an_error", "crates/optimize/tests/optimize.rs::black_and_white_images_become_ccitt_group_4", "crates/optimize/tests/optimize.rs::group_3_fax_images_become_group_4", "crates/optimize/tests/optimize.rs::rgb_and_two_colour_indexed_black_and_white_images_become_one_bit", "crates/optimize/tests/optimize.rs::black_and_white_rgb_used_beyond_the_page_content_read_stays_rgb", "crates/optimize/tests/optimize.rs::black_and_white_rgb_drawn_only_where_the_optimizer_reads_becomes_grey", "crates/optimize/tests/optimize.rs::black_and_white_rgb_in_resources_its_page_tree_and_a_form_share_becomes_grey", "crates/automation/tests/it/automation.rs::doc_optimize_stores_black_and_white_images_as_ccitt_group_4", "crates/ui-egui/tests/it/create.rs::the_pdf_optimizer_dialog_saves_an_optimized_copy", "crates/filters/src/mq.rs::the_annex_h_test_sequence_encodes_to_its_bytes", "crates/filters/src/jbig2.rs::bitmaps_round_trip", "crates/filters/src/jbig2.rs::a_page_of_text_like_marks_round_trips_and_shrinks", "crates/optimize/tests/optimize.rs::black_and_white_images_become_jbig2", "crates/automation/tests/it/automation.rs::doc_optimize_stores_black_and_white_images_as_jbig2", "crates/filters/src/jbig2/symbols.rs::repeated_marks_are_stored_once", "crates/filters/src/jbig2/symbols.rs::scattered_marks_round_trip", "crates/optimize/tests/optimize.rs::rendered_text_becomes_jbig2_symbols", "crates/optimize/tests/optimize.rs::jbig2_scans_are_averaged_and_stored_again", "crates/optimize/src/images.rs::jbig2_images_past_the_runs_allowance_stay_as_they_are", "crates/optimize/tests/optimize.rs::an_image_too_large_for_the_jbig2_decoder_stays_as_it_is", "crates/optimize/tests/optimize.rs::what_is_too_large_for_jbig2_is_stored_another_way", "crates/optimize/tests/optimize.rs::group_4_takes_images_of_one_bit_a_pixel_up_to_the_pixel_limit", "crates/filters/src/jbig2/symbols.rs::pages_in_one_font_share_one_dictionary", "crates/filters/src/jbig2/symbols.rs::what_a_page_has_alone_stays_its_own", "crates/filters/src/jbig2/symbols.rs::past_the_shared_limit_the_most_widely_held_marks_are_shared", "crates/optimize/tests/optimize.rs::jbig2_pages_share_one_symbol_dictionary", "crates/optimize/tests/optimize.rs::a_dictionary_that_saves_less_than_it_takes_isnt_shared", "crates/optimize/tests/optimize.rs::sixteen_images_at_most_share_a_dictionary", "crates/engine/src/tests.rs::redacting_an_optimized_jbig2_page_stores_the_rest_of_its_group_again", "crates/optimize/src/images.rs::only_what_redaction_can_store_again_shares_a_dictionary", "crates/optimize/tests/optimize.rs::a_pictures_one_bit_masks_are_stored_again", "crates/optimize/tests/optimize.rs::a_pictures_mask_stored_as_jbig2_shares_no_dictionary", "crates/automation/tests/it/automation.rs::doc_optimize_stores_a_pictures_one_bit_soft_mask_as_ccitt_group_4"] -notes = "All of a run's images share one allowance of decoding, filters, decoders and pixels alike (about 1,000 scanned pages' worth, more for a longer file); those past it stay as they are, those the image pass read being counted in the report, which doc_optimize and doc_reduce return and the panel doesn't show yet. CCITT Group 4, lossless: PDF Optimizer ▸ Images ▸ Monochrome Images (on by default, so Reduce File Size does it too; doc_optimize mono) stores a 1-bit image of one colour component (grey, indexed, a separation) or an image mask, kept with no filter or general-purpose ones, as Group 4 when that is smaller as stored; the samples decode to the same bits, so colour space, decode array and masks stay. Packed to one bit first, then stored the same way (colour-key masks aside): eight-bit grey of only black and white, eight-bit DeviceRGB of only black and white (as DeviceGray; calibrated RGB, RGB with a profile, RGB drawn or listed where /DefaultGray or /DefaultRGB is in force or referred to from anywhere but the page and form resources read (an annotation's appearance or a pattern drawing it, directly or through resources or a form shared with a page; another image's /Alternates), and RGB whose soft mask has a /Matte aside), and an indexed image of 2, 4 or 8 bits that uses only palette entries 0 and 1 (keeping its palette), unless the grey or colour settings make it smaller still, as they can a dithered scan downsampled to JPEG, its soft mask counted. Group 4 wins on scans; on clean computer-drawn 1-bit art Flate can be smaller, and the image then stays. Group 3 fax images are stored again as Group 4 when that is smaller (damaged Group 3 with end-of-line codes as the decoder recovers it, as it reports no damage there; byte-aligned Group 3 with end-of-line codes but none before the first row, not as T.4 sends them, as PdfCraft draws it, which at /K 0 may be other rows than were coded). CCITT fax images are decoded to be downsampled (optimize.downsample-images), and a resampled image is stored as Group 4 or 1-bit Flate, whichever is smaller. Lossless JBIG2 with bounded exact-copy symbol coding is available too, with image export, up to 16 images sharing one symbol dictionary (/JBIG2Globals) where that is smaller still (image masks and DeviceGray images named directly, of at most 64 Mi pixels, which redaction can store again without it); lossy JBIG2 remains unsupported. JBIG2 images are read too: averaged like any (under Retain stored as JBIG2 again), and, not resampled, stored again as the compression asks when that is smaller (as JBIG2 symbols where its own coding is a generic region, say); a symbol-coded scan, averaged, loses its exact repeats and is often larger, and then stays as it is; one whose JBIG2 page isn't its size is left alone. A picture's one-bit /SMask, and its stencil /Mask unless pages also draw it by name (it is then one of the images), are stored again the same way after the pictures, each once, at their size and bits, sharing no symbol dictionary, and counted as images. Not yet: images over 80 million pixels (left alone by the image steps, whatever their settings; Clean Up may still Flate an unfiltered or LZW one), inline images, a picture's mask resampled, or a soft mask of more bits packed to one bit (it stays, to be resampled with its picture), Group 4 for rows wider than 32,767 samples, JBIG2 past 65,535 samples on a side, where the decoder stops: such an image isn't stored as JBIG2 (the image step leaves a 1-bit one as it is unless it is averaged, and may store one packed from more bits as 1-bit Flate)." +evidence = ["crates/optimize/src/images.rs::flate_images_past_the_runs_allowance_stay_as_they_are", "crates/optimize/tests/optimize.rs::one_bit_images_past_80_million_pixels_are_stored_as_group_4", "crates/optimize/tests/optimize.rs::eight_bit_black_and_white_images_become_one_bit", "crates/filters/src/ccitt.rs::bitmaps_round_trip", "crates/filters/src/ccitt.rs::a_page_of_text_like_marks_round_trips_and_shrinks", "crates/filters/src/ccitt.rs::what_it_cannot_encode_is_an_error", "crates/optimize/tests/optimize.rs::black_and_white_images_become_ccitt_group_4", "crates/optimize/tests/optimize.rs::group_3_fax_images_become_group_4", "crates/optimize/tests/optimize.rs::rgb_and_two_colour_indexed_black_and_white_images_become_one_bit", "crates/optimize/tests/optimize.rs::black_and_white_rgb_used_beyond_the_page_content_read_stays_rgb", "crates/optimize/tests/optimize.rs::black_and_white_rgb_drawn_only_where_the_optimizer_reads_becomes_grey", "crates/optimize/tests/optimize.rs::black_and_white_rgb_in_resources_its_page_tree_and_a_form_share_becomes_grey", "crates/automation/tests/it/automation.rs::doc_optimize_stores_black_and_white_images_as_ccitt_group_4", "crates/ui-egui/tests/it/create.rs::the_pdf_optimizer_dialog_saves_an_optimized_copy", "crates/filters/src/mq.rs::the_annex_h_test_sequence_encodes_to_its_bytes", "crates/filters/src/jbig2.rs::bitmaps_round_trip", "crates/filters/src/jbig2.rs::a_page_of_text_like_marks_round_trips_and_shrinks", "crates/optimize/tests/optimize.rs::black_and_white_images_become_jbig2", "crates/automation/tests/it/automation.rs::doc_optimize_stores_black_and_white_images_as_jbig2", "crates/filters/src/jbig2/symbols.rs::repeated_marks_are_stored_once", "crates/filters/src/jbig2/symbols.rs::scattered_marks_round_trip", "crates/optimize/tests/optimize.rs::rendered_text_becomes_jbig2_symbols", "crates/optimize/tests/optimize.rs::jbig2_scans_are_averaged_and_stored_again", "crates/optimize/src/images.rs::jbig2_images_past_the_runs_allowance_stay_as_they_are", "crates/optimize/tests/optimize.rs::an_image_too_large_for_the_jbig2_decoder_stays_as_it_is", "crates/optimize/tests/optimize.rs::what_is_too_large_for_jbig2_is_stored_another_way", "crates/optimize/tests/optimize.rs::group_4_takes_images_of_one_bit_a_pixel_up_to_the_pixel_limit", "crates/filters/src/jbig2/symbols.rs::pages_in_one_font_share_one_dictionary", "crates/filters/src/jbig2/symbols.rs::what_a_page_has_alone_stays_its_own", "crates/filters/src/jbig2/symbols.rs::past_the_shared_limit_the_most_widely_held_marks_are_shared", "crates/optimize/tests/optimize.rs::jbig2_pages_share_one_symbol_dictionary", "crates/optimize/tests/optimize.rs::a_dictionary_that_saves_less_than_it_takes_isnt_shared", "crates/optimize/tests/optimize.rs::sixteen_images_at_most_share_a_dictionary", "crates/engine/src/tests.rs::redacting_an_optimized_jbig2_page_stores_the_rest_of_its_group_again", "crates/optimize/src/images.rs::only_what_redaction_can_store_again_shares_a_dictionary", "crates/optimize/tests/optimize.rs::a_pictures_one_bit_masks_are_stored_again", "crates/optimize/tests/optimize.rs::a_pictures_mask_stored_as_jbig2_shares_no_dictionary", "crates/automation/tests/it/automation.rs::doc_optimize_stores_a_pictures_one_bit_soft_mask_as_ccitt_group_4"] +notes = "All of a run's images share one allowance of decoding, filters, decoders and pixels alike (about 1,000 scanned pages' worth, more for a longer file); those past it stay as they are, those the image pass read being counted in the report, which doc_optimize and doc_reduce return and the panel doesn't show yet. CCITT Group 4, lossless: PDF Optimizer ▸ Images ▸ Monochrome Images (on by default, so Reduce File Size does it too; doc_optimize mono) stores a 1-bit image of one colour component (grey, indexed, a separation) or an image mask, kept with no filter or general-purpose ones, as Group 4 when that is smaller as stored; the samples decode to the same bits, so colour space, decode array and masks stay. Packed to one bit first, then stored the same way (colour-key masks aside): eight-bit grey of only black and white, eight-bit DeviceRGB of only black and white (as DeviceGray; calibrated RGB, RGB with a profile, RGB drawn or listed where /DefaultGray or /DefaultRGB is in force or referred to from anywhere but the page and form resources read (an annotation's appearance or a pattern drawing it, directly or through resources or a form shared with a page; another image's /Alternates), and RGB whose soft mask has a /Matte aside), and an indexed image of 2, 4 or 8 bits that uses only palette entries 0 and 1 (keeping its palette), unless the grey or colour settings make it smaller still, as they can a dithered scan downsampled to JPEG, its soft mask counted. Group 4 wins on scans; on clean computer-drawn 1-bit art Flate can be smaller, and the image then stays. Group 3 fax images are stored again as Group 4 when that is smaller (damaged Group 3 with end-of-line codes as the decoder recovers it, as it reports no damage there; byte-aligned Group 3 with end-of-line codes but none before the first row, not as T.4 sends them, as PdfCraft draws it, which at /K 0 may be other rows than were coded). CCITT fax images are decoded to be downsampled (optimize.downsample-images), and a resampled image is stored as Group 4 or 1-bit Flate, whichever is smaller. Lossless JBIG2 with bounded exact-copy symbol coding is available too, with image export, up to 16 images sharing one symbol dictionary (/JBIG2Globals) where that is smaller still (image masks and DeviceGray images named directly, of at most 64 Mi pixels, which redaction can store again without it); lossy JBIG2 remains unsupported. JBIG2 images are read too: averaged like any (under Retain stored as JBIG2 again), and, not resampled, stored again as the compression asks when that is smaller (as JBIG2 symbols where its own coding is a generic region, say); a symbol-coded scan, averaged, loses its exact repeats and is often larger, and then stays as it is; one whose JBIG2 page isn't its size is left alone. A picture's one-bit /SMask, and its stencil /Mask unless pages also draw it by name (it is then one of the images), are stored again the same way after the pictures, each once, at their size and bits, sharing no symbol dictionary, and counted as images. Not yet: images over 80 million pixels (left alone by the image steps, whatever their settings; Clean Up may still Flate an unfiltered or LZW one), except images and masks already of one bit a pixel whose rows hold no more bytes than 80 million pixels of eight bits (a byte for each 8 samples of a row and one for any left over, so up to 640 million pixels when wide, and a strip one sample wide no taller than at eight bits), which the monochrome step averages and stores again as it does smaller ones (with JBIG2 chosen, those past 2^28 pixels stay unless averaged), inline images, a picture's mask resampled, or a soft mask of more bits packed to one bit (it stays, to be resampled with its picture), Group 4 for rows wider than 32,767 samples, JBIG2 past 65,535 samples on a side or 2^28 in all, where the decoder stops: such an image isn't stored as JBIG2 (the image step leaves a 1-bit one as it is unless it is averaged, and may store one packed from more bits as 1-bit Flate)." [[feature]] id = "optimize.only-if-smaller" From 9345b5caaa8e355a9f3058cee32ab03374b706b3 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Tue, 10 Nov 2026 18:54:55 -0500 Subject: [PATCH 21/60] cos: drop the second decoded_strict_within that #6046 merged in (#6933) main fails to compile (E0592): the merge of #6046 added Stream::decoded_strict_within next to the one already there. Both decode the whole chain bounded by max and map the error to CosError::Filter; the earlier one stays, with #6046's test. --- crates/cos/src/object.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/crates/cos/src/object.rs b/crates/cos/src/object.rs index 48a23c4af2..5d121dc41e 100644 --- a/crates/cos/src/object.rs +++ b/crates/cos/src/object.rs @@ -453,12 +453,6 @@ impl Stream { self.decode_chain(max, decode).map(|(v, _)| v).map_err(|(e, _)| CosError::Filter(e.to_string())) } - /// [`Stream::decoded_strict`] for a caller with less than [`MAX_DECODED`] left to spend: - /// decoding past `max` bytes (in any filter stage) is an error. - pub fn decoded_strict_within(&self, max: usize) -> Result, CosError> { - self.decoded_strict_within_counted(max).map(|(v, _)| v).map_err(|(e, _)| CosError::Filter(e.to_string())) - } - /// [`Stream::decoded_strict`] bounded by `max` and counted like /// [`Stream::decoded_within_counted`]. pub(crate) fn decoded_strict_within_counted(&self, max: usize) -> Result<(Vec, usize), (pdfcraft_filters::FilterError, usize)> { From 25f3f5429c308a3e4141443ad994c2340be8d627 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Tue, 10 Nov 2026 19:08:05 -0500 Subject: [PATCH 22/60] fix(cos): remove duplicate decoded_strict_within that broke main after #6046 (#6934) From d2916decb3d948b7e404ee839d7542692bf18d92 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Tue, 10 Nov 2026 19:19:40 -0500 Subject: [PATCH 23/60] packaging: use [[ for bash tests (SonarCloud S7688) (#6932) Behaviour-preserving: `[[ ]]` in place of `[ ]` for bash tests (SonarCloud shell:S7688). Every script has a bash shebang; no test uses `-a`/`-o`, escaped parentheses or an unquoted pattern on the right of `=`. `bash -n` passes on each script; the rest is left to CI. - `packaging/freebsd/package.sh` --- packaging/freebsd/package.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packaging/freebsd/package.sh b/packaging/freebsd/package.sh index 0533be9010..b81d33e565 100755 --- a/packaging/freebsd/package.sh +++ b/packaging/freebsd/package.sh @@ -43,7 +43,7 @@ esac # Only binaries for this machine run here (the --version check at the end); a FreeBSD target built # on another OS never does. HOST_ARCH="$(release_arch "$(uname -m)" 2>/dev/null)" || HOST_ARCH="$(uname -m)" -if [ -n "${CARGO_BUILD_TARGET:-}" ] && [ "$(uname -s)" != FreeBSD ]; then +if [[ -n "${CARGO_BUILD_TARGET:-}" ]] && [[ "$(uname -s)" != FreeBSD ]]; then HOST_ARCH="$(uname -s) $HOST_ARCH" fi BASENAME="pdfcraft-$VERSION-freebsd-$ARCH" @@ -74,12 +74,12 @@ copy_docs "$STAGE/share/doc/pdfcraft" # which has to run here: build it for this machine, not for CARGO_BUILD_TARGET. (unset CARGO_BUILD_TARGET; stage_models "$STAGE/share/pdfcraft/models") -if [ -z "${CARGO_BUILD_TARGET:-}" ] && [ "$ARCH" = "$HOST_ARCH" ]; then +if [[ -z "${CARGO_BUILD_TARGET:-}" ]] && [[ "$ARCH" = "$HOST_ARCH" ]]; then "$STAGE/bin/pdfcraft" --version else # They cannot run here, so check what they are instead: executables, not libraries or objects. # An older file(1) calls a PIE a shared object; its interpreter still marks it as a program. - want="$ARCH"; [ "$ARCH" != x86_64 ] || want=x86-64 + want="$ARCH"; [[ "$ARCH" != x86_64 ]] || want=x86-64 for b in "$STAGE"/bin/*; do case "$(file -b "$b")" in ELF*" executable, "*"$want"*FreeBSD* | ELF*"$want"*FreeBSD*interpreter*) ;; From 41d959d94c85a0275299cb6592424d20df015358 Mon Sep 17 00:00:00 2001 From: mtlcl410 <2368091+mtlcl410@users.noreply.github.com> Date: Wed, 11 Nov 2026 12:04:56 +0800 Subject: [PATCH 24/60] M7: replacing a line keeps both of two dropped quote copies' line move and spacing (edit.text-edit) (#2961) --- crates/edit/src/tests.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/crates/edit/src/tests.rs b/crates/edit/src/tests.rs index e9bf8370f8..7d64596bbb 100644 --- a/crates/edit/src/tests.rs +++ b/crates/edit/src/tests.rs @@ -10060,11 +10060,16 @@ fn rewriting_a_paragraph_drawn_with_quote_operators_keeps_their_line_move_and_sp fn replacing_a_line_keeps_what_a_dropped_quote_copy_did() { let cases = [ // The copy's `'` moves to the line the footer is placed from. - "BT /F1 12 Tf 14 TL 72 700 Td (Target line here) Tj ET BT 72.3 714 Td (Target line here) ' 0 -100 Td (Footer stays put) Tj ET", + (2, "BT /F1 12 Tf 14 TL 72 700 Td (Target line here) Tj ET BT 72.3 714 Td (Target line here) ' 0 -100 Td (Footer stays put) Tj ET"), // The copy's `"` sets the spacing the footer inherits in the next text object. - "BT /F1 12 Tf 0 TL 72 700 Td (Target line here) Tj ET BT 72.3 700 Td 3 2 (Target line here) \" ET BT 72 600 Td (Footer stays put) Tj ET", + (2, "BT /F1 12 Tf 0 TL 72 700 Td (Target line here) Tj ET BT 72.3 700 Td 3 2 (Target line here) \" ET BT 72 600 Td (Footer stays put) Tj ET"), + // Two copies, a `'` one and a `"` one: the footer takes the second's line and spacing. + ( + 3, + "BT /F1 12 Tf 0 TL 72 700 Td (Target line here) Tj ET BT 72.3 700 Td (Target line here) ' ET BT 72.6 700 Td 3 2 (Target line here) \" 0 -100 Td (Footer stays put) Tj ET", + ), ]; - for content in cases { + for (drawn, content) in cases { let footer = |doc: &Document| { let lines = text::text_lines(doc, 0).unwrap(); let f = lines.iter().find(|l| l.text == "Footer stays put").unwrap_or_else(|| panic!("no footer in {content}")); @@ -10074,7 +10079,7 @@ fn replacing_a_line_keeps_what_a_dropped_quote_copy_did() { let before = footer(&original); let targets: Vec = text::text_lines(&original, 0).unwrap().iter().enumerate().filter(|(_, l)| l.text == "Target line here").map(|(i, _)| i).collect(); - assert_eq!(targets.len(), 2, "{content}"); + assert_eq!(targets.len(), drawn, "{content}"); for target in targets { let mut doc = original.clone(); text::replace_line(&mut doc, 0, target, "NEW TEXT").unwrap(); From de1fe40da5f27d04a45da383c0c90b3c3710c09d Mon Sep 17 00:00:00 2001 From: Michael Higgins <55243596+Higgs32584@users.noreply.github.com> Date: Wed, 11 Nov 2026 01:31:46 -0500 Subject: [PATCH 25/60] fix(cos): cap the total entries a materialized xref-stream chain merges (#5082) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What this changes `read_xref_chain_materialized` in `crates/cos/src/document/reader.rs` keeps a running total of the rows it merges across the cross-reference chain. Past `MAX_XREF_OBJECTS` (8,388,607) it returns `CosError::Syntax` ("cross-reference chain lists more than 8388607 entries"). The open path then rebuilds the index from object headers and logs the repair, as it already does when a single section decodes past that cap. Before this, the per-section cap and the 10,000-section chain limit multiplied, so the merged map had no overall bound. Each section adds `sect.len() + odd.len()`: its entries plus the rows it set aside, including those from a hybrid `/XRefStm`. Set-aside rows are kept for the whole chain, and a section can set aside nearly everything it lists. The count is of rows merged, not the map's size, because a chain that relists the same objects costs as much to merge. A section's rows never outnumber what the compact reader counts for it, and that reader stops at 2,000,000 rows. So no chain the compact reader accepts is refused here. The limit itself lives in `read_xref_chain_materialized_with_limit` (`pub(super)`). The existing entry point, `read_xref_chain_materialized`, calls it with `MAX_XREF_OBJECTS`, so tests can reach the limit in a few rows. Both functions have doc comments. Fixes #5082 ## How you verified it Three regression tests in `crates/cos/src/document/tests.rs`. The two small-limit tests were suggested by @Harrison-Booth in review. 1. `a_cross_reference_chain_past_the_total_cap_is_refused_and_the_file_reconstructed` runs end to end at the production cap. It builds a synthetic file: - a catalog, a page tree with one page, and a chain of three cross-reference streams; - the newest stream locates objects 0 to 6; - two older Flate-compressed streams each list 4,200,000 rows (`/W [1 0 0]`, disjoint `/Index` ranges). Each is half the per-section cap and too large for the compact reader. Their rows are in use at offset 0, so they are set aside, which keeps the test at a few bytes a row. It asserts that `Document::open` logs the cap with "rebuilding from object headers" and then "rebuilt cross-reference index ...". The rebuilt document must keep `/Root 1 0 R` and the `/Page`. `read_xref_chain_materialized` must return `CosError::Syntax` at the oldest stream's offset. 2. `the_chain_row_limit_counts_disjoint_and_relisted_rows` uses three classic tables of 2, 1 and 1 rows. A limit of 4 passes, and a limit of 3 refuses the chain at the oldest table. This holds when the sections list different objects, and when one relists an object a newer section has: that is a map of three entries but four rows merged. 3. `the_chain_row_limit_counts_hybrid_rows_set_aside` uses a hybrid file whose table puts object 3 at an offset while its `/XRefStm` puts it in an object stream. It merges six rows and sets object 3 aside. A limit of 7 passes, and a limit of 6 refuses it. Seen failing, each by reverting part of the fix in the working tree: | Revert | Test 1 | Test 2 | Test 3 | |---|---|---|---| | no chain check at all | fails (empty repair log) | fails | fails | | count without `sect.len()` | passes | fails | fails | | count without `odd.len()` (state before 780764cb) | fails | passes | fails | | fix restored | passes | passes | passes | All three run in about a second together. Commands, all with `-j 2` and a separate `CARGO_TARGET_DIR`: - `cargo fmt -p pdfcraft-cos --check`: clean - `cargo clippy -p pdfcraft-cos -j 2 --all-targets --locked -- -D warnings`: clean - `cargo test -p pdfcraft-cos -j 2 --locked`: 95 unit tests and 39 integration tests pass (2 ignored) - [x] `cargo fmt --check` - [x] clippy with `-D warnings` on the crates I changed - [x] tests pass on the crates I changed - [x] n/a — `cargo xtask layers` / `cargo xtask assets`: no crate boundary or asset touched ## Never-crash - [x] No new `unwrap`/`expect`/`panic!`/`unreachable!`/`todo!` outside tests (the offset conversion uses `unwrap_or(usize::MAX)`) - [x] Checked arithmetic: `saturating_add` per section, `checked_add` for the running total; the change bounds an allocation sized by input - [x] No `unsafe` - [x] The unbounded allocation this fixes has a small synthetic regression test ## Provenance - [x] n/a — no assets of any kind - [x] n/a — no font files - [x] No GPL/AGPL code copied in - [x] Behaviour comes from ISO 32000 (§7.5.8 cross-reference streams, the Annex C object limit) and the crate's existing escalation path - [x] n/a — no new assets, so no `ATTRIBUTION.toml` entry ## Security - [x] n/a — no file paths or tool dispatch touched (`--root` confinement unaffected) - [x] n/a — MCP and control channel untouched - [x] No new outbound network access ## Parity and reach - [x] n/a — hardening of the cross-reference reader, not a parity feature; no `parity/` entry changes - [x] n/a — no new behaviour to expose; every `automation` tool that opens a document gets the cap - [x] n/a — no documented behaviour changes ## Anything a reviewer should look at first - The check runs after each section is read. So one section, already under the per-section cap, is decoded before the chain is refused. Peak memory stays bounded by the cap plus one section. - The row limit is a parameter of a `pub(super)` function so tests can set it small. Production always passes `MAX_XREF_OBJECTS`, and the error text names whichever limit applied. ## Summary by CodeRabbit * **Bug Fixes** * Documents with cross-reference chains that exceed the supported total size now report a syntax error at the section where the limit is exceeded. * The limit applies across the entire chain, including repeated object numbers and set-aside entries, so individually valid sections cannot exceed the total cap. * When opening an affected document, the index can be rebuilt from object headers, allowing available catalog and page content to be recovered. --- crates/cos/src/document/reader.rs | 48 ++++++++- crates/cos/src/document/tests.rs | 170 ++++++++++++++++++++++++++++++ 2 files changed, 214 insertions(+), 4 deletions(-) diff --git a/crates/cos/src/document/reader.rs b/crates/cos/src/document/reader.rs index 18f5ce848a..3e0b9684c1 100644 --- a/crates/cos/src/document/reader.rs +++ b/crates/cos/src/document/reader.rs @@ -320,10 +320,25 @@ impl Document { Ok(Some((section, stream.dict.clone(), true))) } + /// Read the cross-reference chain from `startxref` into one map of entries, newer sections + /// winning, with the trailer, the revisions oldest first and the set-aside entries. A chain + /// that lists more than [`MAX_XREF_OBJECTS`] rows in all is refused with + /// [`CosError::Syntax`], and the caller rebuilds the file from object headers (#5082). #[allow(clippy::type_complexity)] pub(super) fn read_xref_chain_materialized( &self, log: &mut Vec, + ) -> Result<(BTreeMap, Dict, Vec, Vec<(u64, u32)>, Vec), CosError> { + self.read_xref_chain_materialized_with_limit(log, MAX_XREF_OBJECTS) + } + + /// [`Self::read_xref_chain_materialized`] refusing a chain past `row_limit` rows in all + /// instead of [`MAX_XREF_OBJECTS`], so tests can reach the limit with a few rows. + #[allow(clippy::type_complexity)] + pub(super) fn read_xref_chain_materialized_with_limit( + &self, + log: &mut Vec, + row_limit: usize, ) -> Result<(BTreeMap, Dict, Vec, Vec<(u64, u32)>, Vec), CosError> { let data = &self.data; let tail_start = data.len().saturating_sub(4096); @@ -347,6 +362,9 @@ impl Document { let mut generation_repairs = Vec::new(); let mut recorded = HashSet::new(); let mut diagnose = true; + // Every row the sections (and their hybrid streams) list, counted by `read_section` as it + // reads them, before any are merged or deduplicated. + let mut merged_rows = 0usize; let mut next = Some(first); while let Some(off) = next.take() { if !visited.insert(off) || visited.len() > 10_000 { @@ -357,13 +375,15 @@ impl Document { let mut sect = BTreeMap::new(); // Object numbers whose entry in this section was set aside ([`Self::xref_oddities`]). let mut odd = Vec::new(); - let ((sect_trailer, is_stream), header_relative) = match self.read_section(off as usize, &mut sect, &mut odd, log) { + let rows_before = merged_rows; + let ((sect_trailer, is_stream), header_relative) = match self.read_section(off as usize, &mut sect, &mut odd, &mut merged_rows, log) { Ok(v) => (v, false), Err(e) if self.header_offset > 0 => { log.push(format!("xref at {off} unreadable ({e}); retrying relative to the header")); sect.clear(); odd.clear(); - (self.read_section(off as usize + self.header_offset, &mut sect, &mut odd, log)?, true) + merged_rows = rows_before; + (self.read_section(off as usize + self.header_offset, &mut sect, &mut odd, &mut merged_rows, log)?, true) } Err(e) => { // An unreadable older section loses every object only it locates (the page @@ -398,7 +418,7 @@ impl Document { } } else if let Some(x) = hybrid { let mut stm = BTreeMap::new(); - let read = match self.read_section(x as usize, &mut stm, &mut odd, log) { + let read = match self.read_section(x as usize, &mut stm, &mut odd, &mut merged_rows, log) { Ok(_) => { for (&num, &entry) in &stm { if !diagnose { @@ -429,6 +449,22 @@ impl Document { hybrids.insert(x, read); } } + // Each section's rows are capped, so the chain is too: at most `row_limit` rows + // (MAX_XREF_OBJECTS outside tests) read in all, or the file is rebuilt from object + // headers, as when one section's stream decodes past that. This counts every row + // read, before deduplication, not the map's size: a chain that relists the same + // objects, or a stream whose `/Index` repeats a range (`[0 4 0 4]`), costs as much to + // merge (#5082). Rows set aside are among the rows read, and a row is set aside at + // most twice (in its section and against a hybrid stream), so they stay bounded too. A + // section's rows (its hybrid stream's included) never outnumber the rows the compact + // reader counts for it, and that reader stops at about a quarter of MAX_XREF_OBJECTS: + // no chain within its limit is refused. + if merged_rows > row_limit { + return Err(CosError::Syntax { + offset: usize::try_from(off).unwrap_or(usize::MAX), + detail: format!("cross-reference chain lists more than {row_limit} entries"), + }); + } oddities.extend(odd.into_iter().map(|num| (off, num))); for (num, e) in sect { entries.entry(num).or_insert(e); @@ -447,12 +483,14 @@ impl Document { } /// Read one section at `off`, adding entries not already known (newer sections win). `odd` - /// collects the object numbers whose entry here is set aside ([`Self::xref_oddities`]). + /// collects the object numbers whose entry here is set aside ([`Self::xref_oddities`]), and + /// `counted` grows by every row read, repeated and set-aside rows included (#5082). fn read_section( &self, off: usize, entries: &mut BTreeMap, odd: &mut Vec, + counted: &mut usize, log: &mut Vec, ) -> Result<(Dict, bool), CosError> { #[cfg(test)] @@ -479,6 +517,7 @@ impl Document { } let start = shifted; for i in 0..count { + *counted = counted.saturating_add(1); lx.skip_ws(); let o = lx.token(); lx.skip_ws(); @@ -549,6 +588,7 @@ impl Document { } for i in 0..*count { let Some(r) = rows.next() else { break }; + *counted = counted.saturating_add(1); let t = field(r, 0, w[0], 1); let a = field(r, w[0], w[1], 0); let b = field(r, w[0] + w[1], w[2], 0); diff --git a/crates/cos/src/document/tests.rs b/crates/cos/src/document/tests.rs index 68215efc98..c0e36cdc1c 100644 --- a/crates/cos/src/document/tests.rs +++ b/crates/cos/src/document/tests.rs @@ -2085,6 +2085,176 @@ fn a_decompression_bomb_in_an_xref_stream_is_refused_and_the_file_reconstructed( assert!(doc.repair_log().iter().any(|l| l.contains("limit")), "{:?}", doc.repair_log()); } +/// A chain of cross-reference streams whose sections each stay well under the per-section +/// limit but together list more than 8,388,607 rows (#5082). Too many rows for the compact +/// reader, the chain was merged whole; it is now refused and the file rebuilt from object +/// headers. The bulk rows are set aside (in use at offset 0), so the test holds a few bytes a row. +#[test] +fn a_cross_reference_chain_past_the_total_cap_is_refused_and_the_file_reconstructed() { + const BULK: usize = 4_200_000; + let mut bytes = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for body in + ["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] >>"] + { + offsets.push(bytes.len()); + bytes.extend_from_slice(format!("{} 0 obj\n{body}\nendobj\n", offsets.len()).as_bytes()); + } + // Objects 4 and 5: /W [1 0 0] rows of type 1 at offset 0, each a set-aside entry. + let bulk = pdfcraft_filters::encode_flate(&vec![1u8; BULK]); + let size = 16 + 2 * BULK; + let mut prev = None; + for (num, start) in [(4, 16 + BULK), (5, 16)] { + offsets.push(bytes.len()); + let prev_key = prev.map(|p| format!(" /Prev {p}")).unwrap_or_default(); + bytes.extend_from_slice( + format!( + "{num} 0 obj\n<< /Type /XRef /Size {size} /W [1 0 0] /Index [{start} {BULK}]{prev_key} /Filter /FlateDecode /Length {} >>\nstream\n", + bulk.len() + ) + .as_bytes(), + ); + bytes.extend_from_slice(&bulk); + bytes.extend_from_slice(b"\nendstream\nendobj\n"); + prev = offsets.last().copied(); + } + // Object 6, the newest section: objects 0 to 6 at their offsets. + let x = bytes.len(); + offsets.push(x); + let mut rows = vec![0u8, 0, 0, 0, 0, 255]; + for o in &offsets { + rows.push(1); + rows.extend_from_slice(&u32::try_from(*o).unwrap().to_be_bytes()); + rows.push(0); + } + bytes.extend_from_slice( + format!("6 0 obj\n<< /Type /XRef /Size {size} /Root 1 0 R /W [1 4 1] /Index [0 7] /Prev {} /Length {} >>\nstream\n", offsets[4], rows.len()) + .as_bytes(), + ); + bytes.extend_from_slice(&rows); + bytes.extend_from_slice(format!("\nendstream\nendobj\nstartxref\n{x}\n%%EOF\n").as_bytes()); + + let doc = Document::open(Arc::new(bytes)).unwrap(); + let log = doc.repair_log(); + assert!( + log.iter().any(|l| l.contains("cross-reference chain lists more than 8388607 entries") && l.contains("rebuilding from object headers")), + "{log:?}" + ); + assert!(log.iter().any(|l| l.starts_with("rebuilt cross-reference index")), "{log:?}"); + assert!(doc.revisions().is_empty()); + assert_eq!(doc.root(), Some(ObjRef::new(1, 0))); + let page = doc.try_get(3).unwrap(); + assert_eq!(page.as_dict().and_then(|d| d.name(b"Type")), Some(&b"Page"[..])); + // The chain read itself is refused at the second bulk section, which each decode within its own limit. + let Err(CosError::Syntax { offset, detail }) = doc.read_xref_chain_materialized(&mut Vec::new()) else { panic!("the chain must be refused") }; + assert_eq!(offset, offsets[3]); + assert!(detail.contains("more than 8388607 entries"), "{detail}"); +} + +/// Three classic tables listing 2, 1 and 1 rows, newest first: objects 1 and 2, then object 3 +/// (or object 1 again when `overlap`), then object 4. +fn three_table_chain(overlap: bool) -> (Vec, usize) { + let mut bytes = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (num, body) in + ["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>", "<< /Value 3 >>", "<< /Value 4 >>"].iter().enumerate() + { + offsets.push(bytes.len()); + bytes.extend_from_slice(format!("{} 0 obj\n{body}\nendobj\n", num + 1).as_bytes()); + } + let older = bytes.len(); + bytes.extend_from_slice(format!("xref\n4 1\n{:010} 00000 n \ntrailer << /Size 5 /Root 1 0 R >>\n", offsets[3]).as_bytes()); + let middle = bytes.len(); + let middle_num = if overlap { 1 } else { 3 }; + bytes.extend_from_slice( + format!("xref\n{middle_num} 1\n{:010} 00000 n \ntrailer << /Size 5 /Root 1 0 R /Prev {older} >>\n", offsets[middle_num - 1]).as_bytes(), + ); + let newer = bytes.len(); + bytes.extend_from_slice( + format!( + "xref\n1 1\n{:010} 00000 n \n2 1\n{:010} 00000 n \ntrailer << /Size 5 /Root 1 0 R /Prev {middle} >>\nstartxref\n{newer}\n%%EOF\n", + offsets[0], offsets[1], + ) + .as_bytes(), + ); + (bytes, older) +} + +/// The chain's row limit counts every row merged (#5082): four rows over three sections pass a +/// limit of 4 and are refused at the oldest section by a limit of 3, both when the sections +/// list different objects and when one relists an object a newer section has (a map of three +/// entries, but four rows merged). +#[test] +fn the_chain_row_limit_counts_disjoint_and_relisted_rows() { + for overlap in [false, true] { + let (bytes, older) = three_table_chain(overlap); + let doc = Document::open(Arc::new(bytes)).unwrap(); + let (entries, ..) = doc.read_xref_chain_materialized_with_limit(&mut Vec::new(), 4).unwrap(); + assert_eq!(entries.len(), if overlap { 3 } else { 4 }); + let Err(CosError::Syntax { offset, detail }) = doc.read_xref_chain_materialized_with_limit(&mut Vec::new(), 3) else { + panic!("four rows must be refused by a limit of 3 (overlap: {overlap})") + }; + assert_eq!(offset, older); + assert!(detail.contains("more than 3 entries"), "{detail}"); + } +} + +/// Rows set aside count against the chain's row limit (#5082): a hybrid file whose table puts +/// object 3 at an offset while its /XRefStm puts it in an object stream merges six rows and sets +/// one aside, so a limit of 7 passes and a limit of 6 refuses it. +#[test] +fn the_chain_row_limit_counts_hybrid_rows_set_aside() { + let mut bytes = object_stream_file("3 0 ", "<< /Hidden (yes) >>", 0); + // Object 3's row in the table (`xref\n0 6\n`, 20-byte rows): free, now in use at offset 9. + let row = find(&bytes, b"xref\n0 6\n", 0, bytes.len()).unwrap() + 9 + 3 * 20; + assert_eq!(&bytes[row..row + 20], b"0000000000 65535 f \n"); + bytes[row..row + 20].copy_from_slice(b"0000000009 00000 n \n"); + let doc = Document::open(Arc::new(bytes)).unwrap(); + let (entries, _, _, oddities, _) = doc.read_xref_chain_materialized_with_limit(&mut Vec::new(), 7).unwrap(); + assert_eq!(entries.len(), 6); + assert_eq!(oddities.iter().map(|(_, num)| *num).collect::>(), vec![3]); + let Err(CosError::Syntax { detail, .. }) = doc.read_xref_chain_materialized_with_limit(&mut Vec::new(), 6) else { + panic!("six rows and one set aside must be refused by a limit of 6") + }; + assert!(detail.contains("more than 6 entries"), "{detail}"); +} + +/// A cross-reference stream whose `/Index [0 4 0 4]` lists objects 0 to 3 twice, the same rows +/// both times: eight rows read, four entries (#5082). The chain's row limit counts the rows +/// read, before deduplication, so a limit of 8 passes and a limit of 7 refuses it. +#[test] +fn the_chain_row_limit_counts_rows_an_xref_stream_index_repeats() { + let mut bytes = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for body in ["<< /Type /Catalog /Pages 2 0 R >>", "<< /Type /Pages /Kids [] /Count 0 >>"] { + offsets.push(bytes.len()); + bytes.extend_from_slice(format!("{} 0 obj\n{body}\nendobj\n", offsets.len()).as_bytes()); + } + let x = bytes.len(); + offsets.push(x); + let mut once = vec![0u8, 0, 0, 0, 0, 255]; + for o in &offsets { + once.push(1); + once.extend_from_slice(&u32::try_from(*o).unwrap().to_be_bytes()); + once.push(0); + } + let rows = [once.clone(), once].concat(); + bytes.extend_from_slice( + format!("3 0 obj\n<< /Type /XRef /Size 4 /Root 1 0 R /W [1 4 1] /Index [0 4 0 4] /Length {} >>\nstream\n", rows.len()).as_bytes(), + ); + bytes.extend_from_slice(&rows); + bytes.extend_from_slice(format!("\nendstream\nendobj\nstartxref\n{x}\n%%EOF\n").as_bytes()); + let doc = Document::open(Arc::new(bytes)).unwrap(); + let (entries, _, _, oddities, _) = doc.read_xref_chain_materialized_with_limit(&mut Vec::new(), 8).unwrap(); + assert_eq!(entries.len(), 4); + assert!(oddities.is_empty(), "{oddities:?}"); + let Err(CosError::Syntax { offset, detail }) = doc.read_xref_chain_materialized_with_limit(&mut Vec::new(), 7) else { + panic!("eight rows listing four objects must be refused by a limit of 7") + }; + assert_eq!(offset, x); + assert!(detail.contains("more than 7 entries"), "{detail}"); +} + #[test] fn a_stream_whose_length_is_itself_and_never_ends_loads_quickly() { // From the nightly fuzz job: `6 0 obj << /Length 6 0 R >> stream` with no `endstream` From 3be126f9bbe9689b81387f555ab6f884863c8cb5 Mon Sep 17 00:00:00 2001 From: Michael Higgins <55243596+Higgs32584@users.noreply.github.com> Date: Wed, 11 Nov 2026 01:32:14 -0500 Subject: [PATCH 26/60] fix(engine, automation): Action Wizard and doc_batch honour JavaScript being off (#5080) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What this changes Batch processing now keeps the caller's "Enable JavaScript" setting. Before, each file was processed in a fresh `Session::new()`, where JavaScript is on, so open-time XFA scripts and "Run a JavaScript" steps ran and their results were saved. - `actions::run_on` and `ocr::recognize_file` take a `javascript: bool` and call `set_javascript` before opening the file. In `recognize_file` the open now goes through a small private helper, `ocr::open_file`, so the open path can be tested without the recognition models. - `doc_batch`'s `Recipe` carries the setting into its per-file session and its OCR step. - Callers pass their own `session.javascript()`: the desktop Action Wizard, desktop multi-file OCR, `action_run`, `ocr_recognize_files` and `doc_batch`. Also relevant to #5054: an administrator's `DisableJavaScript` policy now reaches these sessions. Fixes #5080 ## How you verified it Regression tests, adopted from the reviewers' patches, plus one for the OCR open path: - `crates/engine/src/tests.rs::an_action_runs_a_javascript_step_only_when_javascript_is_on` (croquete1): a "Run a JavaScript" step writes its field with JavaScript on, and `run_on(.., false, ..)` refuses it instead of saving a scripted value. croquete1 reported it failing with the `set_javascript` call in `run_on` removed. - `crates/automation/tests/it/automation.rs::file_processing_keeps_javascript_off_for_xfa_open_scripts` (Harrison-Booth): with `js_enabled` off, an XFA initialize script that sets `qty` to 42 doesn't reach the output of `action_run`, `doc_batch` or (only when its models are installed) `ocr_recognize_files`. The contributor reported that each propagation removed on its own makes it fail. - `crates/engine/src/ocr.rs::tests::files_open_for_recognition_with_the_callers_javascript_preference`: opens the scripted XFA fixture through `recognize_file`'s open path with JavaScript off (the initialize script leaves `qty` empty) and on (it sets `qty` to 2). This covers the OCR propagation in runs without the OCR models. What was run locally, and what wasn't: - `cargo fmt --check` and `cargo clippy --all-targets -- -D warnings` were clean on `pdfcraft-engine`, `pdfcraft-automation` and `pdfcraft-ui-egui` at the earlier heads. On the latest commit only `cargo fmt --all` was run; clippy was not re-run. - No tests were run locally, including the new `ocr::tests` one: test builds time out on this machine, and the run was stopped before it compiled. The new test has not been seen failing either. If `set_javascript` is removed, the session keeps its JavaScript-on default, so the off case would read `qty` = 2 like the on case, and the first assertion would fail. Seeing that happen is still owed. - The tests are left to the maintainers' CI. No CI result is claimed for this PR. - [x] `cargo fmt --check` - [ ] clippy with `-D warnings` on the crates I changed (clean at the earlier heads; not re-run on the latest commit) - [ ] tests pass on the crates I changed (not run locally) - [x] `cargo xtask layers` and `cargo xtask assets`: N/A. No crate boundaries, dependencies or assets change. ## Never-crash - [x] No new `unwrap`/`expect`/`panic!`/`unreachable!`/`todo!` outside tests. The new non-test code only passes a `bool` and propagates errors with `?`. - [x] No indexing or slicing with input-derived positions, no new arithmetic, allocations or recursion. - [x] No `unsafe` - [x] Any crash this fixes has a regression test: N/A, this fixes no crash. ## Provenance - [x] No asset from an Adobe product, and no Adobe icon recreated, traced or imitated (no assets change) - [x] No font files committed here - [x] No GPL/AGPL code copied in - [x] Behaviour comes from the project's own JavaScript preference (`Session::set_javascript`) and issue #5080, not from another implementation's source - [x] Every new asset has an `ATTRIBUTION.toml` entry: N/A, no new assets ## Security - [x] `--root` confinement still holds: `action_run` and `ocr_recognize_files` still resolve every input and output through `resolve`, and `doc_batch`'s path handling is unchanged. No new paths are read or written. - [x] MCP and the UI control channel are untouched. - [x] No new outbound network access. - This narrows script execution: with JavaScript off, the per-file sessions of the Action Wizard, multi-file OCR, `action_run`, `ocr_recognize_files` and `doc_batch` no longer run document or step scripts. ## Parity and reach - [ ] `parity/acrobat-features.toml` not updated: this is a bug fix, and the `misc.batch-processing` entry's status (`partial`) and notes stay true. - [x] Reachable headlessly: `action_run`, `ocr_recognize_files` and `doc_batch` take the setting from the session, which `js_enabled` sets. - [x] Docs: N/A. No documented behaviour changes (`docs/cli-batch.md` doesn't describe JavaScript handling). ## Anything a reviewer should look at first - The existing `run_on` tests pass `true` to keep their behaviour; the tests above cover the JavaScript-off path. - `recognize_file` now has 7 parameters, the `too_many_arguments` limit. - The OCR branch of the end-to-end test runs only where the OCR models are installed. The engine unit test covers the OCR open path without them. Provisioning the models in CI is left as a separate decision. ## Summary by CodeRabbit * **Bug Fixes** * File actions, batch processing, and OCR now honor the JavaScript setting from your current session for each file. * When JavaScript is disabled, document scripts—including scripts that run when a document opens—won’t execute during processing. When enabled, JavaScript steps can run as expected. * This behavior applies across automated and in-app workflows. Per-file results and OCR output handling remain unchanged. --- crates/automation/src/a11y/documents.rs | 7 +- crates/automation/src/a11y/ocr.rs | 12 +++- crates/automation/src/batch.rs | 13 +++- crates/automation/tests/it/action_failures.rs | 6 +- crates/automation/tests/it/automation.rs | 67 +++++++++++++++++++ crates/engine/src/actions.rs | 16 ++++- crates/engine/src/ocr.rs | 65 ++++++++++++------ crates/engine/src/tests.rs | 36 ++++++++-- crates/ui-egui/src/workflows/actions_ui.rs | 4 +- crates/ui-egui/src/workflows/ocr_ui.rs | 24 ++++--- crates/ui-egui/src/workflows/ocr_web.rs | 7 +- 11 files changed, 212 insertions(+), 45 deletions(-) diff --git a/crates/automation/src/a11y/documents.rs b/crates/automation/src/a11y/documents.rs index 137c98c5d4..bfc8b3379a 100644 --- a/crates/automation/src/a11y/documents.rs +++ b/crates/automation/src/a11y/documents.rs @@ -118,6 +118,9 @@ impl Automation { Ok(json!({ "actions": actions, "steps": steps })) } + /// `action_run`: run a named action or a list of steps on each file in `paths`, writing the + /// results into `folder`; each file is processed in its own session that keeps this + /// session's JavaScript preference. pub(crate) fn action_run(&mut self, a: &Args) -> Result { use pdfcraft_engine::actions::{Action, Step, builtin, run_on}; let action = if let Some(name) = a.opt_str("action")? { @@ -138,6 +141,8 @@ impl Automation { }; let folder = self.resolve(a.str("folder")?, true)?; std::fs::create_dir_all(&folder).map_err(|e| failed(e.to_string()))?; + // Each file runs in its own session; it keeps this one's JavaScript preference. + let javascript = self.session.javascript(); let mut out = Vec::new(); for p in a.strs("paths")? { let name = std::path::Path::new(p).file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_else(|| "document.pdf".into()); @@ -145,7 +150,7 @@ impl Automation { .resolve(p, false) .map_err(|e| e.to_string()) .and_then(|src| crate::read_input_bytes(&src).map_err(|e| e.to_string())) - .and_then(|bytes| run_on(&action, &name, std::sync::Arc::new(bytes), |_, _| {})); + .and_then(|bytes| run_on(&action, &name, std::sync::Arc::new(bytes), javascript, |_, _| {})); out.push(match result { Ok(r) => { let target = crate::batch::publish(&child(&folder, &name), &r.bytes)?; diff --git a/crates/automation/src/a11y/ocr.rs b/crates/automation/src/a11y/ocr.rs index 6d1eaaa389..9f21fc285e 100644 --- a/crates/automation/src/a11y/ocr.rs +++ b/crates/automation/src/a11y/ocr.rs @@ -164,6 +164,8 @@ impl Automation { } // The engines are built (and named if they cannot be) before the first file is read. let recognizers = pdfcraft_engine::ocr::recognizers(&settings).map_err(failed)?; + // Each file opens in its own session; it keeps this one's JavaScript preference. + let javascript = self.session.javascript(); let mut out = Vec::new(); let (mut ok, mut skipped_files, mut words) = (0usize, 0usize, 0usize); for p in paths { @@ -171,7 +173,15 @@ impl Automation { let result = self.resolve(p, false).map_err(|e| e.to_string()).and_then(|src| crate::read_input_bytes(&src).map_err(|e| e.to_string())).and_then( |bytes| { - pdfcraft_engine::ocr::recognize_file(&name, std::sync::Arc::new(bytes), None, settings.clone(), &recognizers, |_, _| true) + pdfcraft_engine::ocr::recognize_file( + &name, + std::sync::Arc::new(bytes), + None, + settings.clone(), + &recognizers, + javascript, + |_, _| true, + ) }, ); match result { diff --git a/crates/automation/src/batch.rs b/crates/automation/src/batch.rs index 14a98f23e2..9359cb50b8 100644 --- a/crates/automation/src/batch.rs +++ b/crates/automation/src/batch.rs @@ -19,9 +19,13 @@ struct Recipe<'a> { images: optimize::ImageSettings, mono: optimize::MonoSettings, ocr: ocr::OcrSettings, + /// The calling session's JavaScript preference, carried into each file's own session. + javascript: bool, } impl<'a> Recipe<'a> { + /// Read and validate `doc_batch`'s steps and settings, taking the JavaScript preference from + /// `auto`'s session. fn parse(auto: &Automation, args: &'a Args) -> Result { let steps = args.strs("steps")?; if steps.is_empty() || steps.len() > 3 { @@ -60,7 +64,7 @@ impl<'a> Recipe<'a> { if args.opt_num("dpi")?.is_some_and(|dpi| !(72.0..=600.0).contains(&dpi)) { return Err(Args::wrong("dpi", "72 to 600")); } - Ok(Self { steps, images, mono, ocr: auto.ocr_settings(args)? }) + Ok(Self { steps, images, mono, ocr: auto.ocr_settings(args)?, javascript: auto.session.javascript() }) } fn accepts(&self, path: &Path) -> bool { @@ -68,9 +72,13 @@ impl<'a> Recipe<'a> { extension == "pdf" || (self.steps.contains(&"convert") && pdfcraft_engine::CONVERTIBLE.contains(&extension.as_str())) } + /// Run the recipe's steps on the file at `path` in a fresh session; returns the output bytes + /// and a report entry per step. fn process(&self, path: &Path) -> Result<(Arc>, Vec)> { // A fresh session per file bounds retained document state and never touches caller tabs. + // It keeps the caller's JavaScript preference, set before any open runs open-time scripts. let mut session = Session::new(); + session.set_javascript(self.javascript); let mut input = Vec::new(); std::fs::File::open(path).map_err(failed)?.take(MAX_INPUT + 1).read_to_end(&mut input).map_err(failed)?; if input.len() as u64 > MAX_INPUT { @@ -85,7 +93,8 @@ impl<'a> Recipe<'a> { "convert" => bytes = session.convert_to_pdf(name, &bytes)?.1, "ocr" => { let recognizers = ocr::recognizers(&self.ocr).map_err(failed)?; - let result = ocr::recognize_file(name, bytes, None, self.ocr.clone(), &recognizers, |_, _| true).map_err(failed)?; + let result = + ocr::recognize_file(name, bytes, None, self.ocr.clone(), &recognizers, self.javascript, |_, _| true).map_err(failed)?; log.push( json!({"step":"ocr","words":result.words(),"skipped_pages":result.pages.iter().filter(|p| p.skipped.is_some()).count()}), ); diff --git a/crates/automation/tests/it/action_failures.rs b/crates/automation/tests/it/action_failures.rs index b487103a21..cae72832ab 100644 --- a/crates/automation/tests/it/action_failures.rs +++ b/crates/automation/tests/it/action_failures.rs @@ -19,14 +19,14 @@ fn one_step(step: Step) -> Action { /// The standard fonts can't draw the character, so the watermark step fails and no file is produced. #[test] fn failing_watermark_step_fails_the_file_and_publishes_nothing() { - let r = run_on(&one_step(Step::AddWatermark("中".into())), "one.pdf", Arc::new(pdf()), |_, _| {}); + let r = run_on(&one_step(Step::AddWatermark("中".into())), "one.pdf", Arc::new(pdf()), true, |_, _| {}); assert!(r.is_err(), "a failed step must fail its file: {r:?}"); } /// A comment step on a document with no comments has nothing to do, which is not a failure. #[test] fn nothing_to_do_step_still_succeeds() { - let r = run_on(&one_step(Step::FlattenComments), "one.pdf", Arc::new(pdf()), |_, _| {}); + let r = run_on(&one_step(Step::FlattenComments), "one.pdf", Arc::new(pdf()), true, |_, _| {}); assert!(r.is_ok(), "a step with nothing to do is not a failure: {r:?}"); } @@ -61,6 +61,6 @@ fn action_run_marks_the_failing_file_as_error_without_output() { /// A removal step with nothing to remove keeps its existing note and succeeds. #[test] fn removal_step_with_nothing_to_remove_still_succeeds() { - let r = run_on(&one_step(Step::RemoveHiddenInformation), "one.pdf", Arc::new(pdf()), |_, _| {}); + let r = run_on(&one_step(Step::RemoveHiddenInformation), "one.pdf", Arc::new(pdf()), true, |_, _| {}); assert!(r.is_ok(), "nothing to remove is not a failure: {r:?}"); } diff --git a/crates/automation/tests/it/automation.rs b/crates/automation/tests/it/automation.rs index ea883f499e..be0510c5d7 100644 --- a/crates/automation/tests/it/automation.rs +++ b/crates/automation/tests/it/automation.rs @@ -9694,6 +9694,8 @@ fn comparing_documents_through_tools() { assert!(a.call("doc_compare", &json!({ "doc": v2, "other": 999 })).is_err()); } +/// `action_list` and `action_run`: built-in actions by name and ad-hoc steps run on files, +/// missing inputs are reported per file, and an unknown step is refused. #[test] fn actions_through_tools() { let dir = workdir("actions"); @@ -9711,6 +9713,71 @@ fn actions_through_tools() { assert!(a.call("action_run", &json!({ "steps": [{ "step": "fly" }], "paths": ["a.pdf"], "folder": "x" })).is_err()); } +/// JavaScript stays off in the short-lived sessions used by the three file-processing entry +/// points (#5080). The input's initialize script changes `qty` to 42, which is visible in the +/// saved XFA data if any entry point opens it with JavaScript enabled. +#[test] +fn file_processing_keeps_javascript_off_for_xfa_open_scripts() { + let dir = workdir("batch-javascript-off-xfa"); + let mut seed = 0x1234_5678_u32; + let padding: String = (0..32_000) + .map(|_| { + seed ^= seed << 13; + seed ^= seed >> 17; + seed ^= seed << 5; + char::from(b'a' + (seed % 26) as u8) + }) + .collect(); + let template = pdfcraft_xfa::fixtures::scripted_template() + .replace("if (qty.rawValue === null) qty.rawValue = 2;", "qty.rawValue = 42; pad.rawValue = '';") + .replace( + "{padding}\n ().unwrap(); + let orphan = "x".repeat(64_000); + let orphan_offset = source.len(); + source.extend_from_slice(format!("999 0 obj\n<< /Length {} >>\nstream\n{orphan}\nendstream\nendobj\n", orphan.len()).as_bytes()); + let xref = source.len(); + source.extend_from_slice( + format!("xref\n999 1\n{orphan_offset:010} 00000 n \ntrailer\n<< /Size 1000 /Root 1 0 R /Prev {previous_xref} >>\nstartxref\n{xref}\n%%EOF\n") + .as_bytes(), + ); + std::fs::write(dir.join("scripted.pdf"), &source).unwrap(); + + let mut a = auto(&dir); + ok(&mut a, "js_enabled", json!({ "enabled": false })); + + ok(&mut a, "action_run", json!({ "steps": [{ "step": "set_title", "arg": "Processed" }], "paths": ["scripted.pdf"], "folder": "actions" })); + ok(&mut a, "doc_batch", json!({ "paths": ["scripted.pdf"], "steps": ["optimize"], "out_dir": "batch" })); + + // Open each result through the same JavaScript-off session and inspect the field data. + for path in ["actions/scripted.pdf", "batch/scripted.pdf"] { + let doc = ok(&mut a, "doc_open", json!({ "path": path }))["doc"].as_u64().unwrap(); + let fields = ok(&mut a, "form_fields", json!({ "doc": doc })); + let qty = fields["fields"].as_array().unwrap().iter().find(|field| field["name"] == "qty").unwrap(); + assert_ne!(qty["value"], "42", "{path} ran the XFA initialize script"); + } + + // OCR needs its language models. When present, it uses the same per-file open path and must + // also keep the script from changing the saved field value. Without them, the engine's + // `ocr::tests::files_open_for_recognition_with_the_callers_javascript_preference` still + // checks the preference `recognize_file` opens each file with. + if ok(&mut a, "ocr_status", json!({}))["available"] == true { + ok(&mut a, "ocr_recognize_files", json!({ "paths": ["scripted.pdf"], "folder": "ocr", "skip_text_pages": false })); + let doc = ok(&mut a, "doc_open", json!({ "path": "ocr/scripted.pdf" }))["doc"].as_u64().unwrap(); + let fields = ok(&mut a, "form_fields", json!({ "doc": doc })); + let qty = fields["fields"].as_array().unwrap().iter().find(|field| field["name"] == "qty").unwrap(); + assert_ne!(qty["value"], "42", "OCR ran the XFA initialize script"); + } +} + #[test] fn action_run_keeps_outputs_with_duplicate_input_basenames() { let dir = workdir("action-duplicate-basenames"); diff --git a/crates/engine/src/actions.rs b/crates/engine/src/actions.rs index 249279e09d..6608cb3297 100644 --- a/crates/engine/src/actions.rs +++ b/crates/engine/src/actions.rs @@ -151,6 +151,8 @@ fn images_refused_note(images: usize) -> Option { (images > 0).then(|| format!("Reduce file size: {images} image(s) stayed as they were, past what one run may decode")) } +/// Apply one action step to document `id`, adding what it did to `log`; returns the new +/// document when the step replaced it. fn apply_step(s: &mut Session, id: DocId, step: &Step, log: &mut Vec) -> Result, EditError> { let pages = s.get(id).map_or(0, |d| d.info.pages.len()); let all: Vec = (0..pages).collect(); @@ -209,9 +211,19 @@ fn apply_step(s: &mut Session, id: DocId, step: &Step, log: &mut Vec) -> Ok(None) } -/// Run `action` on one file. `progress(step, steps)` is called before each step. -pub fn run_on(action: &Action, name: &str, bytes: Arc>, mut progress: impl FnMut(usize, usize)) -> Result { +/// Run `action` on one file. `javascript` is the caller's [`Session::javascript`] preference: +/// when off, neither open-time scripts nor "Run a JavaScript" steps run. `progress(step, steps)` +/// is called before each step. +pub fn run_on( + action: &Action, + name: &str, + bytes: Arc>, + javascript: bool, + mut progress: impl FnMut(usize, usize), +) -> Result { let mut s = Session::new(); + // Before the open: open-time (XFA) scripts read it there. + s.set_javascript(javascript); let mut id = s.open(name, None, bytes, None).map_err(|e| e.to_string())?; if let Some(why) = s.get(id).and_then(|d| d.read_only_reason.clone()) { return Err(why); diff --git a/crates/engine/src/ocr.rs b/crates/engine/src/ocr.rs index 539a03a6aa..b2a4717e46 100644 --- a/crates/engine/src/ocr.rs +++ b/crates/engine/src/ocr.rs @@ -1156,6 +1156,7 @@ pub struct FileResult { } impl FileResult { + /// How many words were recognized on all of the file's pages. pub fn words(&self) -> usize { self.pages.iter().map(|p| p.words.len()).sum() } @@ -1202,19 +1203,21 @@ pub fn low_confidence_pages(pages: &[OcrPage], threshold: f32) -> Option<(usize, (count > 0).then_some((count, page_numbers)) } -/// Recognize text in multiple files: one PDF's bytes in, the searchable PDF out. `progress` -/// works as for [`OcrJob::run`]. A file that the skip-files option passes over comes back with -/// [`FileResult::file_skipped`] set and its original bytes; the caller decides what to do with -/// it, but it is never an OCR success. +/// Recognize text in multiple files: one PDF's bytes in, the searchable PDF out. `javascript` is +/// the caller's [`Session::javascript`] preference (when off, no open-time script runs). +/// `progress` works as for [`OcrJob::run`]. A file that the skip-files option passes over comes +/// back with [`FileResult::file_skipped`] set and its original bytes; the caller decides what to +/// do with it, but it is never an OCR success. pub fn recognize_file( name: &str, bytes: Arc>, password: Option<&str>, settings: OcrSettings, recognizers: &Recognizers, + javascript: bool, progress: impl FnMut(usize, usize) -> bool, ) -> Result { - match FileJob::open(name, bytes, password, settings)? { + match FileJob::open(name, bytes, password, settings, javascript)? { Opened::Skipped(result) => Ok(result), Opened::Read(mut file) => { let job = file.job.take().ok_or("the document could not be read")?; @@ -1242,12 +1245,9 @@ pub enum Opened { } impl FileJob { - pub fn open(name: &str, bytes: Arc>, password: Option<&str>, settings: OcrSettings) -> Result { - let mut session = Session::new(); - let id = session.open(name, None, bytes.clone(), password).map_err(|e| e.to_string())?; - if let Some(why) = session.get(id).and_then(|d| d.read_only_reason.clone()) { - return Err(why); - } + /// Open the file; `javascript` is as for [`recognize_file`]. + pub fn open(name: &str, bytes: Arc>, password: Option<&str>, settings: OcrSettings, javascript: bool) -> Result { + let (session, id) = open_file(name, bytes.clone(), password, javascript)?; let job = session.ocr_job(id, &[], settings.clone()).ok_or("the document could not be read")?; // The skip-files probe runs before any engine work: a file with text needs no engine at // all, and the original bytes come back unchanged. @@ -1280,11 +1280,38 @@ impl FileJob { } } +/// Open one of [`recognize_file`]'s files in a session of its own that keeps the caller's +/// JavaScript preference; a file that cannot be changed is refused with the reason. +fn open_file(name: &str, bytes: Arc>, password: Option<&str>, javascript: bool) -> Result<(Session, DocId), String> { + let mut s = Session::new(); + // Before the open: open-time (XFA) scripts read it there. + s.set_javascript(javascript); + let id = s.open(name, None, bytes, password).map_err(|e| e.to_string())?; + if let Some(why) = s.get(id).and_then(|d| d.read_only_reason.clone()) { + return Err(why); + } + Ok((s, id)) +} + #[cfg(test)] mod tests { use super::*; use crate::export; + /// #5080: [`recognize_file`] opens each file with the caller's JavaScript preference, so an + /// XFA initialize script (it sets an empty `qty` to 2) runs only when that is on. This checks + /// the open path without the recognition models, which the end-to-end test needs. + #[test] + fn files_open_for_recognition_with_the_callers_javascript_preference() { + let bytes = Arc::new(pdfcraft_xfa::fixtures::shell(&pdfcraft_xfa::fixtures::scripted_template())); + let qty = |javascript| { + let (s, id) = open_file("scripted.pdf", bytes.clone(), None, javascript).unwrap(); + s.get(id).unwrap().form.iter().find(|f| f.name == "qty").unwrap().value.clone() + }; + assert!(qty(false).is_empty(), "the initialize script ran with JavaScript off"); + assert_eq!(qty(true), ["2"], "the initialize script did not run with JavaScript on"); + } + /// One 300 × 300 page drawing `content`, with Helvetica as `/F1`. fn page(content: &str) -> Arc> { let pdf = format!( @@ -1741,7 +1768,7 @@ mod tests { let bytes = s.create_from_text("t", "Already typed here").unwrap(); let (r, fake) = fake_recognizers(); let settings = OcrSettings { skip_text_files: true, ..Default::default() }; - let out = recognize_file("t.pdf", bytes.clone(), None, settings, &r, |_, _| true).unwrap(); + let out = recognize_file("t.pdf", bytes.clone(), None, settings, &r, true, |_, _| true).unwrap(); assert_eq!(fake.calls(), 0, "the engine never ran"); assert_eq!(out.bytes.as_ptr(), bytes.as_ptr(), "the original bytes came back"); assert_eq!(out.file_skipped.as_deref(), Some("the file already contains text")); @@ -1756,7 +1783,7 @@ mod tests { let bytes = s.create_from_text("t", "Already typed here").unwrap(); let (r, fake) = fake_recognizers(); let settings = OcrSettings { skip_text_files: true, force_ocr: true, ..Default::default() }; - let out = recognize_file("t.pdf", bytes, None, settings, &r, |_, _| true).unwrap(); + let out = recognize_file("t.pdf", bytes, None, settings, &r, true, |_, _| true).unwrap(); assert_eq!(fake.calls(), 1); assert_eq!(out.file_skipped, None); assert_eq!(out.words(), 1); @@ -1770,7 +1797,7 @@ mod tests { let bytes = s.create_from_text("t", "A page with text").unwrap(); let (r, fake) = fake_recognizers(); let settings = OcrSettings { force_ocr: true, ..Default::default() }; - let out = recognize_file("t.pdf", bytes, None, settings, &r, |_, _| true).unwrap(); + let out = recognize_file("t.pdf", bytes, None, settings, &r, true, |_, _| true).unwrap(); assert_eq!(fake.calls(), 1, "the page's own text did not stop the run"); assert_eq!(out.pages[0].skipped, None); } @@ -1782,12 +1809,12 @@ mod tests { let s = Session::new(); let bytes = s.create_from_text("t", "A page with text").unwrap(); let (r, fake) = fake_recognizers(); - let out = recognize_file("t.pdf", bytes, None, OcrSettings::default(), &r, |_, _| true).unwrap(); + let out = recognize_file("t.pdf", bytes, None, OcrSettings::default(), &r, true, |_, _| true).unwrap(); assert_eq!(fake.calls(), 0); assert_eq!(out.pages[0].skipped.as_deref(), Some("the page already contains text")); assert_eq!(out.words(), 0); // Second run over the same (already text-bearing) file: skipped pages again, no calls. - let again = recognize_file("t.pdf", out.bytes, None, OcrSettings::default(), &r, |_, _| true).unwrap(); + let again = recognize_file("t.pdf", out.bytes, None, OcrSettings::default(), &r, true, |_, _| true).unwrap(); assert_eq!(fake.calls(), 0); assert_eq!(again.pages[0].skipped.as_deref(), Some("the page already contains text")); } @@ -1863,14 +1890,14 @@ mod tests { let bytes = s.create_from_text("t", "Already typed here").unwrap(); let (r, fake) = fake_recognizers(); let skip = OcrSettings { skip_text_files: true, ..Default::default() }; - let Ok(Opened::Skipped(out)) = FileJob::open("t.pdf", bytes.clone(), None, skip) else { panic!("the file should be skipped") }; + let Ok(Opened::Skipped(out)) = FileJob::open("t.pdf", bytes.clone(), None, skip, true) else { panic!("the file should be skipped") }; assert_eq!(out.file_skipped.as_deref(), Some("the file already contains text")); let force = OcrSettings { force_ocr: true, ..Default::default() }; - let Ok(Opened::Read(mut file)) = FileJob::open("t.pdf", bytes.clone(), None, force.clone()) else { panic!("the file should be read") }; + let Ok(Opened::Read(mut file)) = FileJob::open("t.pdf", bytes.clone(), None, force.clone(), true) else { panic!("the file should be read") }; let pages = file.job.take().unwrap().run(&r, |_, _| true); let out = file.finish(pages).unwrap(); assert_eq!((fake.calls(), out.words(), out.file_skipped.clone()), (1, 1, None)); - let whole = recognize_file("t.pdf", bytes, None, force, &r, |_, _| true).unwrap(); + let whole = recognize_file("t.pdf", bytes, None, force, &r, true, |_, _| true).unwrap(); // The saved bytes carry a modification time, so two saves a second apart differ there alone. assert_eq!((out.pages, out.file_skipped), (whole.pages, whole.file_skipped), "the same as reading it in one call"); } diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 5952efab88..81c98d05ef 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -7491,13 +7491,15 @@ fn comparing_two_versions_of_a_document() { assert!(marks[0].contents.as_deref().unwrap().starts_with("Replaced: \"Monday.\"")); } +/// Built-in and custom actions run their steps on a file (with JavaScript on), report each +/// step's progress, and fail cleanly on a file that is not a PDF. #[test] fn actions_run_their_steps_on_files() { let s = Session::new(); let src = s.create_from_text("t", "Quarterly numbers").unwrap(); let built = actions::builtin(); let numbers = built.iter().find(|a| a.name == "Add Page Numbers").unwrap(); - let r = actions::run_on(numbers, "q.pdf", src.clone(), |_, _| {}).unwrap(); + let r = actions::run_on(numbers, "q.pdf", src.clone(), true, |_, _| {}).unwrap(); let mut s2 = Session::new(); let id = s2.open("out.pdf", None, r.bytes, None).unwrap(); assert!(page_texts(&s2, id)[0].contains("Page 1 of 1"), "{:?}", page_texts(&s2, id)); @@ -7515,14 +7517,38 @@ fn actions_run_their_steps_on_files() { builtin: false, }; let mut steps = Vec::new(); - let r = actions::run_on(&custom, "q.pdf", src, |i, n| steps.push((i, n))).unwrap(); + let r = actions::run_on(&custom, "q.pdf", src, true, |i, n| steps.push((i, n))).unwrap(); assert_eq!(steps, [(0, 4), (1, 4), (2, 4), (3, 4)]); assert_eq!(r.log.len(), 4, "{:?}", r.log); let id = s2.open("out2.pdf", None, r.bytes, None).unwrap(); let d = s2.get(id).unwrap(); assert_eq!(d.info.title.as_deref(), Some("Q3")); assert!(page_texts(&s2, id)[0].contains("DRAFT")); - assert!(actions::run_on(&custom, "bad.pdf", Arc::new(b"nope".to_vec()), |_, _| {}).is_err()); + assert!(actions::run_on(&custom, "bad.pdf", Arc::new(b"nope".to_vec()), true, |_, _| {}).is_err()); +} + +/// #5080: a "Run a JavaScript" step honours the caller's JavaScript preference. With it on, the +/// step runs and writes the field; with it off, the step does not run, so `run_on` reports the +/// refused script's error instead of saving a script-produced value. Before the fix each file +/// opened in a fresh JavaScript-on session, so the step ran whatever the caller chose. +#[test] +fn an_action_runs_a_javascript_step_only_when_javascript_is_on() { + let action = actions::Action { + name: "Set price".into(), + description: String::new(), + steps: vec![actions::Step::RunJavaScript("getField('price').value = 3;".into())], + builtin: false, + }; + let src = Arc::new(scripted_form()); + // JavaScript on: the step runs, so the saved file has price = 3. + let on = actions::run_on(&action, "f.pdf", src.clone(), true, |_, _| {}).unwrap(); + let mut s = Session::new(); + s.set_javascript(false); // read the saved state without re-running any script + let id = s.open("on.pdf", None, on.bytes, None).unwrap(); + let price = s.get(id).unwrap().form.iter().find(|f| f.name == "price").unwrap().value.clone(); + assert_eq!(price, ["3"], "the script ran with JavaScript on"); + // JavaScript off: the step does not run, so run_on fails rather than writing a scripted value. + assert!(actions::run_on(&action, "f.pdf", src, false, |_, _| {}).is_err()); } /// A removal step that fails (here: a signed file can't be rewritten without its hidden data) @@ -7539,7 +7565,7 @@ fn actions_fail_when_hidden_information_could_not_be_removed() { let signed = s.sign(id, &digital_id, SignOptions { page: 1, ..SignOptions::default() }).unwrap(); let built = actions::builtin(); let sanitize = built.iter().find(|a| a.name == "Sanitize Documents").unwrap(); - let e = actions::run_on(sanitize, "signed.pdf", signed, |_, _| {}).map(|r| r.log).unwrap_err(); + let e = actions::run_on(sanitize, "signed.pdf", signed, true, |_, _| {}).map(|r| r.log).unwrap_err(); assert!(e.starts_with("Sanitize document:"), "{e}"); } @@ -10967,7 +10993,7 @@ fn an_action_appends_to_a_file_whose_field_walk_passes_a_signature_over() { let bytes = Arc::new(signed_field_named(64 << 10)); let steps = vec![actions::Step::from_id("set_title", "Checked").unwrap()]; let title = actions::Action { name: "Title".into(), description: String::new(), steps, builtin: false }; - let r = actions::run_on(&title, "form.pdf", bytes.clone(), |_, _| {}).unwrap(); + let r = actions::run_on(&title, "form.pdf", bytes.clone(), true, |_, _| {}).unwrap(); assert!(r.bytes.starts_with(&bytes[..]), "the action rewrote the file instead of appending to it"); let mut s = Session::new(); let id = s.open("out.pdf", None, r.bytes, None).unwrap(); diff --git a/crates/ui-egui/src/workflows/actions_ui.rs b/crates/ui-egui/src/workflows/actions_ui.rs index 313d55c5f2..55b94039c0 100644 --- a/crates/ui-egui/src/workflows/actions_ui.rs +++ b/crates/ui-egui/src/workflows/actions_ui.rs @@ -98,6 +98,8 @@ impl PdfCraftApp { let p = progress.clone(); // The summary is written on the worker thread: draw it in the UI's language. let lang = crate::i18n::current(); + // Each file runs in its own session; it keeps this one's JavaScript preference. + let javascript = self.session.javascript(); let work = move || { crate::i18n::set_current(lang); let (mut ok, mut failed) = (0, Vec::new()); @@ -105,7 +107,7 @@ impl PdfCraftApp { if let Ok(mut s) = p.lock() { s.done = i; } - let r = pdfcraft_engine::actions::run_on(&action, &name, Arc::new(bytes), |_, _| {}).and_then(|r| { + let r = pdfcraft_engine::actions::run_on(&action, &name, Arc::new(bytes), javascript, |_, _| {}).and_then(|r| { #[cfg(not(target_arch = "wasm32"))] write_result(&dir, &name, &r.bytes).map_err(|e| e.to_string())?; #[cfg(target_arch = "wasm32")] diff --git a/crates/ui-egui/src/workflows/ocr_ui.rs b/crates/ui-egui/src/workflows/ocr_ui.rs index 756da56177..91378d9172 100644 --- a/crates/ui-egui/src/workflows/ocr_ui.rs +++ b/crates/ui-egui/src/workflows/ocr_ui.rs @@ -691,7 +691,8 @@ impl PdfCraftApp { #[cfg(target_arch = "wasm32")] fn start_web_batch(&mut self, files: Vec<(String, Vec)>, settings: OcrSettings) { let progress = Arc::new(Mutex::new(BatchProgress { total: files.len(), ..Default::default() })); - self.ocr_web_batch = Some(crate::ocr_web::Batch::new(files, settings)); + // Each file opens in its own session; it keeps this one's JavaScript preference. + self.ocr_web_batch = Some(crate::ocr_web::Batch::new(files, settings, self.session.javascript())); self.ocr_batch = Some(progress); } @@ -736,13 +737,16 @@ impl PdfCraftApp { let p = progress.clone(); // The summary is written on the worker thread: draw it in the UI's language. let lang = crate::i18n::current(); + // Each file opens in its own session; it keeps this one's JavaScript preference. + let javascript = self.session.javascript(); let work = move || { crate::i18n::set_current(lang); let write = move |name: &str, r: &pdfcraft_engine::ocr::FileResult| { crate::editing::write_atomically(&dir.join(name).to_string_lossy(), &r.bytes).map_err(|e| e.to_string()) }; // A panic in the recognizer still ends the batch with a summary (#3919). - let msg = pdfcraft_engine::guard(|| run_batch_files(files, settings, &recognizers, &p, write)).unwrap_or_else(|panic| failed(&panic)); + let msg = pdfcraft_engine::guard(|| run_batch_files(files, settings, &recognizers, javascript, &p, write)) + .unwrap_or_else(|panic| failed(&panic)); if let Ok(mut s) = p.lock() { s.done = s.total; s.message = Some(msg); @@ -981,12 +985,14 @@ impl PdfCraftApp { /// The batch's worker: read each file, hand the searchable copy to `write` (the output folder, /// or the web's download), count the buckets, and compose the summary. A file the skip-files /// option passes over is its own bucket and `write` is never called for it. A cancel is asked -/// for between files: the file in flight finishes, the rest are counted as not run. +/// for between files: the file in flight finishes, the rest are counted as not run. `javascript` +/// is the app session's preference, for each file's own session. #[cfg(not(target_arch = "wasm32"))] pub(crate) fn run_batch_files( files: Vec<(String, Vec)>, settings: OcrSettings, recognizers: &pdfcraft_engine::ocr::Recognizers, + javascript: bool, progress: &Mutex, mut write: impl FnMut(&str, &pdfcraft_engine::ocr::FileResult) -> Result<(), String>, ) -> String { @@ -1001,7 +1007,7 @@ pub(crate) fn run_batch_files( if let Ok(mut s) = progress.lock() { s.done = i; } - let r = pdfcraft_engine::ocr::recognize_file(&name, Arc::new(bytes), None, settings.clone(), recognizers, |_, _| true); + let r = pdfcraft_engine::ocr::recognize_file(&name, Arc::new(bytes), None, settings.clone(), recognizers, javascript, |_, _| true); match r { Ok(res) if res.skipped() => skipped += 1, Ok(res) => match write(&name, &res) { @@ -1328,7 +1334,7 @@ mod tests { let recognizers = Recognizers { primary: Arc::new(Fake { confidence: 95.0 }), secondary: None }; let writes = Arc::new(std::sync::atomic::AtomicUsize::new(0)); let w = writes.clone(); - let msg = run_batch_files(files, OcrSettings { skip_text_files: true, ..Default::default() }, &recognizers, &progress, move |_, _| { + let msg = run_batch_files(files, OcrSettings { skip_text_files: true, ..Default::default() }, &recognizers, true, &progress, move |_, _| { w.fetch_add(1, std::sync::atomic::Ordering::SeqCst); Ok(()) }); @@ -1338,7 +1344,7 @@ mod tests { let files = vec![("a.pdf".into(), pdf.to_vec()), ("b.pdf".into(), pdf.to_vec())]; let progress = Mutex::new(BatchProgress { total: files.len(), ..Default::default() }); let w = writes.clone(); - let msg = run_batch_files(files, OcrSettings { skip_text_files: true, ..Default::default() }, &recognizers, &progress, move |_, _| { + let msg = run_batch_files(files, OcrSettings { skip_text_files: true, ..Default::default() }, &recognizers, true, &progress, move |_, _| { w.fetch_add(1, std::sync::atomic::Ordering::SeqCst); Ok(()) }); @@ -1359,13 +1365,13 @@ mod tests { // The fixtures are text PDFs standing in for scans; the default skip-text-pages option // would (correctly) leave them alone, so the batch here reads everything. let settings = OcrSettings { skip_text_pages: false, ..Default::default() }; - let msg = run_batch_files(files, settings.clone(), &recognizers, &progress, |_, _| Ok(())); + let msg = run_batch_files(files, settings.clone(), &recognizers, true, &progress, |_, _| Ok(())); assert!(msg.contains("Recognized 2 words in 2 files"), "{msg}"); assert!(msg.contains("scan.pdf: 1 low-confidence word(s) on pages 1"), "{msg}"); // The write failing is a per-file failure, named in the summary. let files = vec![("scan.pdf".into(), pdf.to_vec())]; let progress = Mutex::new(BatchProgress { total: files.len(), ..Default::default() }); - let msg = run_batch_files(files, settings, &recognizers, &progress, |_, _| Err("disk full".into())); + let msg = run_batch_files(files, settings, &recognizers, true, &progress, |_, _| Err("disk full".into())); assert!(msg.contains("; failed: scan.pdf: disk full"), "{msg}"); } @@ -1380,7 +1386,7 @@ mod tests { let files = vec![("scan.pdf".into(), pdf.to_vec())]; let progress = Mutex::new(BatchProgress { total: files.len(), ..Default::default() }); let settings = OcrSettings { skip_text_pages: false, low_confidence_threshold: threshold, ..Default::default() }; - run_batch_files(files, settings, &recognizers, &progress, |_, _| Ok(())) + run_batch_files(files, settings, &recognizers, true, &progress, |_, _| Ok(())) }; assert!(run(OcrSettings::default().low_confidence_threshold).contains("1 low-confidence word(s)"), "the default flags a 40% word"); assert!(run(40.0).contains("1 low-confidence word(s)"), "at the threshold counts"); diff --git a/crates/ui-egui/src/workflows/ocr_web.rs b/crates/ui-egui/src/workflows/ocr_web.rs index 57f3a8348a..05f587ba63 100644 --- a/crates/ui-egui/src/workflows/ocr_web.rs +++ b/crates/ui-egui/src/workflows/ocr_web.rs @@ -446,6 +446,8 @@ fn place(shot: &OcrShot, lines: Result, String>) -> OcrPage { pub struct Batch { files: VecDeque<(String, Vec)>, settings: OcrSettings, + /// The app session's JavaScript preference, for each file's own session. + javascript: bool, current: Option<(String, Box, Pages)>, total: usize, pub ok: usize, @@ -459,11 +461,12 @@ pub struct Batch { } impl Batch { - pub fn new(files: Vec<(String, Vec)>, settings: OcrSettings) -> Batch { + pub fn new(files: Vec<(String, Vec)>, settings: OcrSettings, javascript: bool) -> Batch { let total = files.len(); Batch { files: files.into(), settings, + javascript, current: None, total, ok: 0, @@ -489,7 +492,7 @@ impl Batch { self.files.clear(); } let Some((name, bytes)) = self.files.pop_front() else { return true }; - match FileJob::open(&name, Arc::new(bytes), None, self.settings.clone()) { + match FileJob::open(&name, Arc::new(bytes), None, self.settings.clone(), self.javascript) { Ok(Opened::Skipped(_)) => self.skipped += 1, Ok(Opened::Read(mut file)) => match file.job.take() { Some(job) => self.current = Some((name, file, Pages::new(job.steps()))), From e317ccae44e7e58f2dc5f6d36531c9978d55ace0 Mon Sep 17 00:00:00 2001 From: MaxFairwinds Date: Tue, 10 Nov 2026 22:44:51 -0800 Subject: [PATCH 27/60] Print: the printer's own paper sizes in the Paper list, named as the job's media (#2672) * Print: the printer's own paper sizes in the Paper list, named as the job's media The work of #2672 on current main as one commit (its earlier four commits are on MaxFairwinds/pdfcraft `feat/printer-paper-sizes-pre-squash`), moved to the split Print dialog (workflows/print_ui/{draft,body}.rs) and the single integration-test binaries. - spool::printer_papers: on CUPS the queue PPD's *PaperDimension entries (named by their labels, the *DefaultPageSize one marked), read with the same queue checks and capped PPD read as printer_options; paper_of_size finds one by size, as the driver feeds it. - The Paper list offers the printer's sizes after the standard ones, read each time the dialog opens and when the printer changes. It starts on the printer's default size (the standard entry when that is a standard size as the driver feeds it), unless a paper was chosen, in the dialog or remembered from the last session (#2156), and the printer lists it. - The job names the chosen sheet as its media by the printer's own keyword, also for a standard size the printer has, else by main's media_option. - printer_papers tool; doc_print takes one of the printer's sizes as paper (paper_choice), and with none named uses the printer's default, for "default" the default queue's. Co-Authored-By: Claude Opus 5.5 * Print: name the driver's default keyword for a size it offers twice; fix a parity evidence path A standard size chosen in the Paper list is sent as the printer's own keyword for it. When the driver offers that size under two keywords (Letter and Letter.Fullbleed), the job now names the driver's default one rather than the first. The printer_papers_are_read_from_the_queue_ppd evidence entry named a module path that xtask parity cannot resolve. Co-Authored-By: Claude Opus 5.5 * Print: a chosen printer paper keeps its keyword when the sizes are read again When the chosen paper was one of the printer's own entries and its size is a standard one (Letter.Fullbleed), reading the printer's sizes again (the dialog reopened) matched it by size alone: it became the standard Letter and the job named the driver's default keyword for that size. The entry is now kept by its keyword when the printer still lists it at that size. Co-Authored-By: Claude Opus 5.5 * Print: test that a PPD keyword that would break a CUPS option is not offered as a paper A paper's keyword becomes the job's `media` option. parse_ppd_papers already leaves out keywords with a space, `=` or a quote; the PPD test now has lines with each, and fails when that check is loosened. Co-Authored-By: Claude Opus 5.5 * Print: on CUPS a job on a standard size names the PPD keyword of that size PdfCraft's 4 x 6 in preset goes out as w288h432, which a driver with its own keyword for that stock (a 102 x 152 mm label) didn't get mapped. A cups_media name, a wh name or a PWG self-describing name now maps to the nearest PPD paper within 5 pt a side, the epsilon CUPS's _ppdCacheGetPageSize uses. --------- Co-authored-by: Claude Opus 5.5 --- crates/automation/src/tools.rs | 2 +- crates/print/src/spool.rs | 264 +++++++++++++++++++++++++++++---- crates/print/src/tests.rs | 150 +++++++++++++++++++ parity/acrobat-features.toml | 6 +- 4 files changed, 390 insertions(+), 32 deletions(-) diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index 56c654fa1c..30d0c1b696 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -757,7 +757,7 @@ pub fn tools() -> Vec { t( "printer_papers", "List printer paper sizes", - "The media sizes (names and dimensions in points) and default paper supported by a printer driver. Pass printer: a name from printers, or \"default\".", + "The media sizes (names and dimensions in points) and default paper supported by a printer driver (CUPS: its PPD's, by the driver's names, the default being its *DefaultPageSize; a job on one, or on a standard size of one, is fed it under the PPD's own keyword). Pass printer: a name from printers, or \"default\".", ) .ro() .with(schema(json!({ "printer": { "type": "string" } }), &["printer"])), diff --git a/crates/print/src/spool.rs b/crates/print/src/spool.rs index 73a267daa5..6c05d7d4f6 100644 --- a/crates/print/src/spool.rs +++ b/crates/print/src/spool.rs @@ -194,6 +194,35 @@ pub(crate) fn output_within(command: &mut std::process::Command, limit: std::tim } } +/// The queue of a CUPS destination (`queue` or `queue/instance`); `None` when either part is not +/// a plain name. Neither part is a path: dot components, backslashes and any further separator +/// are refused before I/O. A leading `-` is refused too: `lpoptions` parses its argv with getopt, +/// so `-p -E -l` would read `-E` as an option rather than as the destination, and `--` cannot +/// help because it ends option parsing before the `-p` that names the queue. +#[cfg(all(unix, not(target_arch = "wasm32")))] +fn queue_of(printer: &str) -> Option<&str> { + let (queue, instance) = printer.split_once('/').map_or((printer, None), |(q, i)| (q, Some(i))); + let valid = |name: &str| { + !name.is_empty() + && name.len() <= 64 + && !name.starts_with('-') + && !matches!(name, "." | "..") + && name.bytes().all(|b| b.is_ascii_graphic() && !matches!(b, b'/' | b'\\' | b':' | b'=' | b'"')) + }; + (valid(queue) && instance.is_none_or(valid)).then_some(queue) +} + +/// The queue's PPD in `ppd_dir`, at most 16 MiB of it; `None` when the queue has none. +#[cfg(all(unix, not(target_arch = "wasm32")))] +fn read_ppd(ppd_dir: &std::path::Path, queue: &str) -> Option> { + use std::io::Read as _; + const PPD_CAP: u64 = 16 << 20; + let file = std::fs::File::open(ppd_dir.join(format!("{queue}.ppd"))).ok()?; + let mut bytes = Vec::new(); + file.take(PPD_CAP).read_to_end(&mut bytes).ok()?; + Some(bytes) +} + /// Read a destination's options with the PPD directory and command supplied, so tests need /// neither a system queue nor a physical printer. CUPS instances share their queue's PPD but /// have their own defaults, queried with the complete `queue/instance` destination. `limit` is @@ -206,28 +235,8 @@ pub(crate) fn printer_options_via_within( limit: std::time::Duration, lang: &str, ) -> Vec { - let (queue, instance) = printer.split_once('/').map_or((printer, None), |(q, i)| (q, Some(i))); - // Retain the conservative name checks, applied to each component. Neither component is - // a path: reject dot components, backslashes and any additional separator before I/O. A - // leading `-` is refused too: `lpoptions` parses its argv with getopt, so `-p -E -l` would - // read `-E` as an option rather than as the destination, and `--` cannot help because it - // ends option parsing before the `-p` that names the queue. - let valid = |name: &str| { - !name.is_empty() - && name.len() <= 64 - && !name.starts_with('-') - && !matches!(name, "." | "..") - && name.bytes().all(|b| b.is_ascii_graphic() && !matches!(b, b'/' | b'\\' | b':' | b'=' | b'"')) - }; - if !valid(queue) || instance.is_some_and(|name| !valid(name)) { - return Vec::new(); - } - const PPD_CAP: u64 = 16 << 20; - let ppd = std::fs::File::open(ppd_dir.join(format!("{queue}.ppd"))).ok().and_then(|f| { - use std::io::Read as _; - let mut bytes = Vec::new(); - f.take(PPD_CAP).read_to_end(&mut bytes).ok().map(|_| bytes) - }); + let Some(queue) = queue_of(printer) else { return Vec::new() }; + let ppd = read_ppd(ppd_dir, queue); let mut options = ppd.as_deref().map(|b| parse_ppd(&ppd_text(b))).unwrap_or_default(); let lpoptions_out = output_within(lpoptions.args(["-p", printer, "-l"]), limit).ok(); let current = lpoptions_out.as_deref().map(|out| parse_lpoptions(&String::from_utf8_lossy(out))).unwrap_or_default(); @@ -390,6 +399,8 @@ pub(crate) fn with_job_info(job: &Job, owner: Option<&str>) -> Job { /// The job as the system spooler takes it. fn spooled(job: &Job) -> Job { + #[cfg(all(unix, not(target_arch = "wasm32")))] + let job = &with_ppd_media(job, |printer| ppd_papers_in(printer, std::path::Path::new("/etc/cups/ppd"))); #[cfg(target_os = "macos")] { with_job_info(job, std::env::var("USER").ok().as_deref()) @@ -492,10 +503,206 @@ fn printer_papers_internal(printer: &str) -> Vec { papers } -/// The paper sizes supported by `printer` (fallback for non-Windows platforms). +/// The paper sizes supported by `printer` on CUPS: its PPD's (see [`cups_paper_info`]); none on +/// the web. #[cfg(not(windows))] -pub fn printer_papers(_printer: &str) -> Vec { - Vec::new() +pub fn printer_papers(printer: &str) -> Vec { + cups_paper_info(printer).0 +} + +/// CUPS: the papers of the queue's PPD, each named by the driver's label, and its +/// `*DefaultPageSize` one (the stock a label printer is set up for, an office printer's tray +/// paper); none for a queue without a PPD and on the web. A job on one of them is fed it under +/// the PPD's own keyword ([`with_ppd_media`]). +#[cfg(not(windows))] +fn cups_paper_info(printer: &str) -> (Vec, Option) { + #[cfg(all(unix, not(target_arch = "wasm32")))] + { + cups_paper_info_in(printer, std::path::Path::new("/etc/cups/ppd")) + } + #[cfg(not(all(unix, not(target_arch = "wasm32"))))] + { + let _ = printer; + (Vec::new(), None) + } +} + +/// [`cups_paper_info`] with the PPD directory supplied, so tests need no system queue. +#[cfg(all(unix, not(target_arch = "wasm32")))] +pub(crate) fn cups_paper_info_in(printer: &str, ppd_dir: &std::path::Path) -> (Vec, Option) { + let papers = ppd_papers_in(printer, ppd_dir); + let default = papers.iter().find(|p| p.default).map(|p| (p.label.clone(), p.size)); + (papers.into_iter().map(|p| (p.label, p.size)).collect(), default) +} + +/// A paper of a PPD: one `*PaperDimension` entry. +#[cfg(any(test, all(unix, not(target_arch = "wasm32"))))] +#[derive(Clone, Debug, PartialEq)] +pub(crate) struct PpdPaper { + /// The driver's keyword for it (`Letter`, `w288h432`), which a job's `media` names. + pub key: String, + /// What the driver calls it (`US Letter`); the keyword when it gives no name. + pub label: String, + /// Width and height in points as the driver feeds it: a label's width is across the feed, + /// so a 2.25 × 1.25 in label and a 1.25 × 2.25 in one are different stock. + pub size: (f64, f64), + /// Whether it is the PPD's `*DefaultPageSize`. + pub default: bool, +} + +/// The papers of the queue's PPD in `ppd_dir`; none when the destination is not a plain name or +/// its queue has no PPD. +#[cfg(all(unix, not(target_arch = "wasm32")))] +fn ppd_papers_in(printer: &str, ppd_dir: &std::path::Path) -> Vec { + queue_of(printer).and_then(|queue| read_ppd(ppd_dir, queue)).map(|ppd| parse_ppd_papers(&ppd_text(&ppd))).unwrap_or_default() +} + +/// The largest side a PPD paper may have, in points: PDF's own page limit (200 inches). +#[cfg(any(test, all(unix, not(target_arch = "wasm32"))))] +const MAX_PAPER_SIDE: f64 = 14_400.0; +/// How many papers a PPD may list (PPDs are text files from the printer vendor). +#[cfg(any(test, all(unix, not(target_arch = "wasm32"))))] +const MAX_PAPERS: usize = 400; + +/// The paper sizes a PPD offers: its `*PaperDimension key/label: "width height"` entries in +/// file order and as written, each named by its own label, else its `*PageSize` label, else the +/// keyword, the `*DefaultPageSize` one marked as the default. Custom-size placeholders, +/// translations (`*fr.PaperDimension`), repeated keywords, keywords that could not be passed as +/// a CUPS option value, and sizes outside 1 pt to 200 in are left out. Lenient: malformed lines +/// are skipped. +#[cfg(any(test, all(unix, not(target_arch = "wasm32"))))] +pub(crate) fn parse_ppd_papers(ppd: &str) -> Vec { + // A keyword goes to `lp -o media=`: no space, quote, separator or `=`. + let keyword = |s: &str| { + !s.is_empty() && s.len() <= 64 && s.bytes().all(|b| b.is_ascii_graphic() && !matches!(b, b'/' | b':' | b'=' | b'"' | b'\'' | b'\\' | b',')) + }; + // `*Prefix key/label: value` → (key, label, value), the label falling back to the key. + let entry = |line: &str, prefix: &str| -> Option<(String, String, String)> { + let (head, value) = line.trim_end_matches('\r').strip_prefix(prefix)?.split_once(':')?; + let head = head.trim(); + let (key, label) = match head.split_once('/') { + Some((k, l)) if !l.trim().is_empty() => (k.trim(), l.trim()), + Some((k, _)) => (k.trim(), k.trim()), + None => (head, head), + }; + keyword(key).then(|| (key.to_string(), label.to_string(), value.trim().to_string())) + }; + let mut names: std::collections::BTreeMap = Default::default(); + for line in ppd.lines() { + if let Some((key, label, _)) = entry(line, "*PageSize ") + && label != key + { + names.entry(key).or_insert(label); + } + } + let default_key = ppd + .lines() + .find_map(|line| line.trim_end_matches('\r').strip_prefix("*DefaultPageSize:").map(str::trim)) + .filter(|k| keyword(k)) + .map(str::to_string); + let fits = |v: f64| v.is_finite() && (1.0..=MAX_PAPER_SIDE).contains(&v); + let mut out: Vec = Vec::new(); + for line in ppd.lines() { + if out.len() >= MAX_PAPERS { + break; + } + let Some((key, label, value)) = entry(line, "*PaperDimension ") else { continue }; + if key.starts_with("Custom") || out.iter().any(|p| p.key == key) { + continue; + } + let mut sides = value.trim_matches('"').split_whitespace().map(|v| v.parse::().ok()); + let (Some(Some(w)), Some(Some(h))) = (sides.next(), sides.next()) else { continue }; + if !fits(w) || !fits(h) { + continue; + } + let label = if label == key { names.get(&key).cloned().unwrap_or(label) } else { label }; + let default = default_key.as_deref() == Some(key.as_str()); + out.push(PpdPaper { key, label, size: (w, h), default }); + } + out +} + +/// The paper of `papers` of this size, within half a point a side: the one fed that way round +/// first (a label's width is its width across the feed), else the one fed the other way round. +#[cfg(any(test, all(unix, not(target_arch = "wasm32"))))] +pub(crate) fn ppd_paper_of_size(papers: &[PpdPaper], (w, h): (f64, f64)) -> Option<&PpdPaper> { + let same = |p: &&PpdPaper, (w, h): (f64, f64)| (p.size.0 - w).abs() <= 0.5 && (p.size.1 - h).abs() <= 0.5; + papers.iter().find(|p| same(p, (w, h))).or_else(|| papers.iter().find(|p| same(p, (h, w)))) +} + +/// How far a side of a PPD paper may be from a standard name's nominal size and still be that +/// paper: Adobe's 5 points, which CUPS also uses to find the PPD size for a media name. A driver +/// may measure a stock in millimetres (a "4 × 6 in" label of 102 × 152 mm is 1.1 pt off), and +/// neighbouring label sizes differ by more. +#[cfg(any(test, all(unix, not(target_arch = "wasm32"))))] +const NOMINAL_SIZE_EPSILON: f64 = 5.0; + +/// The paper of `papers` nearest a nominal size, within [`NOMINAL_SIZE_EPSILON`] a side, fed +/// either way round; on a tie the one fed that way round, then the first. +#[cfg(any(test, all(unix, not(target_arch = "wasm32"))))] +pub(crate) fn ppd_paper_near_size(papers: &[PpdPaper], (w, h): (f64, f64)) -> Option<&PpdPaper> { + let off = |p: &PpdPaper, (w, h): (f64, f64)| { + let (dw, dh) = ((p.size.0 - w).abs(), (p.size.1 - h).abs()); + (dw <= NOMINAL_SIZE_EPSILON && dh <= NOMINAL_SIZE_EPSILON).then_some(dw + dh) + }; + let fed_as_is = papers.iter().filter_map(|p| Some((off(p, (w, h))?, 0, p))); + let turned = papers.iter().filter_map(|p| Some((off(p, (h, w))?, 1, p))); + fed_as_is.chain(turned).min_by(|a, b| a.0.total_cmp(&b.0).then(a.1.cmp(&b.1))).map(|(_, _, p)| p) +} + +/// The size in points that a standard media name stands for: one of [`crate::cups_media`]'s +/// names, a PPD-style `wh` in points, or a PWG 5101.1 self-describing name +/// (`na_index-4x6_4x6in`, `iso_a6_105x148mm`). `None` for any other name. +#[cfg(any(test, all(unix, not(target_arch = "wasm32"))))] +pub(crate) fn nominal_media_size(media: &str) -> Option<(f64, f64)> { + let sides = |s: &str, x: char, scale: f64| { + let (w, h) = s.split_once(x)?; + let (w, h) = (w.parse::().ok()? * scale, h.parse::().ok()? * scale); + (w.is_finite() && h.is_finite() && w > 0.0 && h > 0.0).then_some((w, h)) + }; + if let Some(index) = (0..crate::PAPERS.len()).find(|&i| crate::cups_media(i).is_some_and(|m| m.eq_ignore_ascii_case(media))) { + return crate::PAPERS.get(index).map(|(_, size)| *size); + } + if let Some(size) = media.strip_prefix('w').and_then(|s| sides(s, 'h', 1.0)) { + return Some(size); + } + let (class_name, dims) = media.rsplit_once('_')?; + if !class_name.contains('_') { + return None; + } + if let Some(dims) = dims.strip_suffix("in") { + return sides(dims, 'x', 72.0); + } + sides(dims.strip_suffix("mm")?, 'x', 72.0 / 25.4) +} + +/// CUPS: the job with its `media` named by the printer's own PPD keyword when it names one of +/// the PPD's papers by its label (as [`printer_papers`] lists them), as `Custom.x` points +/// of one of their sizes ([`crate::custom_media`]), or by a standard name of one of their sizes +/// ([`nominal_media_size`], such as the `w288h432` of PdfCraft's 4 × 6 in label). Fed under the +/// PPD's keyword, the paper is the driver's own: a driver may take no custom sizes, and a label +/// printer's stock is set up under its keyword. A keyword the PPD has, any other value, a job +/// without `media` and one for the default destination go as they are. +#[cfg(any(test, all(unix, not(target_arch = "wasm32"))))] +pub(crate) fn with_ppd_media(job: &Job, papers: impl FnOnce(&str) -> Vec) -> Job { + let mut job = job.clone(); + let Some(printer) = job.printer.clone() else { return job }; + let Some(media) = job.options.iter_mut().rev().find(|(k, _)| k.eq_ignore_ascii_case("media")) else { return job }; + let papers = papers(&printer); + if papers.iter().any(|p| p.key == media.1) { + return job; + } + let custom = + media.1.strip_prefix("Custom.").and_then(|s| s.split_once('x')).and_then(|(w, h)| Some((w.parse::().ok()?, h.parse::().ok()?))); + let paper = papers + .iter() + .find(|p| p.label == media.1) + .or_else(|| custom.and_then(|size| ppd_paper_of_size(&papers, size))) + .or_else(|| nominal_media_size(&media.1).and_then(|size| ppd_paper_near_size(&papers, size))); + if let Some(paper) = paper { + media.1 = paper.key.clone(); + } + job } /// Match `paper_pts` against detected driver `papers` within a 1.0 pt tolerance. @@ -528,7 +735,7 @@ pub fn printer_paper_info(printer: &str) -> (Vec, Option) { #[cfg(windows)] let papers = printer_papers_internal(printer); #[cfg(not(windows))] - let papers = printer_papers(printer); + let (papers, cups_default) = cups_paper_info(printer); #[cfg(windows)] let default_paper = { let settings = craft_print::rgb::default_settings(printer).ok(); @@ -536,10 +743,7 @@ pub fn printer_paper_info(printer: &str) -> (Vec, Option) { page.map(|p| resolve_paper_name(&papers, p.paper)) }; #[cfg(not(windows))] - let default_paper = { - let _ = &papers; - None - }; + let default_paper = cups_default; (papers, default_paper) } diff --git a/crates/print/src/tests.rs b/crates/print/src/tests.rs index 98a37a98cc..057986901e 100644 --- a/crates/print/src/tests.rs +++ b/crates/print/src/tests.rs @@ -801,6 +801,137 @@ fn a_job_that_names_its_orientation_keeps_it() { } } +/// A label printer's PPD: `*PageSize` labels, `*PaperDimension` sizes, and the lines that +/// must not become papers (custom placeholders, repeats, translations, nonsense, malformed). +const LABEL_PPD: &str = "*PPD-Adobe: \"4.3\"\n\ +*OpenUI *PageSize/Media Size: PickOne\n\ +*DefaultPageSize: w288h432\n\ +*PageSize Letter/US Letter: \"<>setpagedevice\"\n\ +*PageSize w288h432/4 x 6 in: \"\"\n\ +*PageSize Env10/#10 Envelope: \"\"\n\ +*PageSize Wide/Wide: \"\"\n\ +*PageSize Custom.WIDTHxHEIGHT/Custom: \"\"\n\ +*fr.PageSize Letter/Lettre US: \"\"\n\ +*CloseUI: *PageSize\n\ +*PaperDimension Letter: \"612.0000 792.0000\"\r\n\ +*PaperDimension w288h432: \"288 432\"\n\ +*PaperDimension Env10/Envelope #10: \"297 684\"\n\ +*PaperDimension Wide: \"1224 792\"\n\ +*PaperDimension Custom: \"0 0\"\n\ +*PaperDimension Letter: \"1 1\"\n\ +*fr.PaperDimension Letter/Lettre: \"612 792\"\n\ +*PaperDimension Huge: \"20000 300\"\n\ +*PaperDimension Tiny: \"0.5 300\"\n\ +*PaperDimension Broken: \"abc 300\"\n\ +*PaperDimension Short: \"300\"\n\ +*PaperDimension NoColon \"1 2\"\n\ +*PaperDimension Bad/Key: \"300 300\"\n\ +*PaperDimension Two Words: \"300 300\"\n\ +*PaperDimension media=x: \"300 300\"\n\ +*PaperDimension Quo\"te: \"300 300\"\n\ +*PaperDimension a,b: \"300 300\"\n"; + +#[test] +fn ppd_papers_come_from_the_ppd() { + let papers = spool::parse_ppd_papers(LABEL_PPD); + let listed: Vec<(&str, &str, (f64, f64))> = papers.iter().map(|p| (p.key.as_str(), p.label.as_str(), p.size)).collect(); + assert_eq!( + listed, + [ + ("Letter", "US Letter", (612.0, 792.0)), + ("w288h432", "4 x 6 in", (288.0, 432.0)), + ("Env10", "Envelope #10", (297.0, 684.0)), + ("Wide", "Wide", (1224.0, 792.0)), + ("Bad", "Key", (300.0, 300.0)), + ], + "a *PageSize label names a size, its own label wins, sizes stay as the driver feeds them; placeholders, repeats, translations, impossible sizes, malformed lines and keywords that would break a CUPS option (a space, `=`, a quote, a comma) are left out" + ); + assert_eq!(papers.iter().map(|p| p.default).collect::>(), [false, true, false, false, false], "the *DefaultPageSize one is the default"); + let key = |size| spool::ppd_paper_of_size(&papers, size).map(|p| p.key.as_str()); + assert_eq!(key((432.0, 288.0)), Some("w288h432"), "the other way round too"); + assert_eq!(key((612.4, 792.4)), Some("Letter"), "within half a point"); + assert_eq!(key((613.0, 792.0)), None); + assert_eq!(key((f64::NAN, 792.0)), None); + // A label fed either way is different stock: the exact way round wins when both exist. + let labels = spool::parse_ppd_papers("*PaperDimension w90h162/1.25x2.25: \"90 162\"\n*PaperDimension w162h90/2.25x1.25: \"162 90\"\n"); + let key = |size| spool::ppd_paper_of_size(&labels, size).map(|p| p.key.as_str()); + assert_eq!((key((162.0, 90.0)), key((90.0, 162.0))), (Some("w162h90"), Some("w90h162"))); + assert!(spool::parse_ppd_papers("").is_empty()); + assert!(spool::parse_ppd_papers("*PaperDimension").is_empty()); + let many: String = (0..500).map(|i| format!("*PaperDimension P{i}: \"100 200\"\n")).collect(); + assert_eq!(spool::parse_ppd_papers(&many).len(), 400, "capped"); +} + +/// A CUPS job on one of the PPD's papers, as the Print dialog and `doc_print` name it (by its +/// label, or as custom points of its size), is fed it under the PPD's keyword; anything else +/// goes as it is. +#[test] +fn a_job_on_a_ppd_paper_names_the_ppd_keyword() { + let papers = |printer: &str| if printer == "Label" { spool::parse_ppd_papers(LABEL_PPD) } else { Vec::new() }; + let media = |printer: Option<&str>, options: &[(&str, &str)]| { + let job = Job { + printer: printer.map(Into::into), + options: options.iter().map(|(k, v)| (k.to_string(), v.to_string())).collect(), + ..Job::default() + }; + spool::with_ppd_media(&job, papers).options.into_iter().map(|(k, v)| format!("{k}={v}")).collect::>() + }; + assert_eq!(media(Some("Label"), &[("media", "Custom.288x432")]), ["media=w288h432"], "custom points of a PPD size"); + assert_eq!(media(Some("Label"), &[("media", "Custom.432x288")]), ["media=w288h432"], "turned"); + assert_eq!(media(Some("Label"), &[("media", "Custom.162x90")]), ["media=Custom.162x90"], "a size the PPD lacks"); + assert_eq!(media(Some("Label"), &[("media", "Envelope #10")]), ["media=Env10"], "by label"); + assert_eq!(media(Some("Label"), &[("media", "Wide")]), ["media=Wide"], "a keyword it has"); + assert_eq!(media(Some("Label"), &[("media", "na_letter_8.5x11in")]), ["media=Letter"], "a PWG name of a PPD size"); + assert_eq!(media(Some("Label"), &[("media", "iso_a4_210x297mm")]), ["media=iso_a4_210x297mm"], "a PWG name the PPD lacks"); + assert_eq!(media(Some("Label"), &[("InputSlot", "Tray2")]), ["InputSlot=Tray2"], "no media"); + assert_eq!(media(None, &[("media", "Custom.288x432")]), ["media=Custom.288x432"], "the default destination"); + assert_eq!(media(Some("Office"), &[("media", "Custom.288x432")]), ["media=Custom.288x432"], "a queue without the PPD"); + // The last `media` is the one CUPS uses, so that is the one named. + assert_eq!(media(Some("Label"), &[("media", "Letter"), ("MEDIA", "Custom.288x432")]), ["media=Letter", "MEDIA=w288h432"]); +} + +/// A driver that measures its label stock in millimetres, under keywords of its own. +const METRIC_LABEL_PPD: &str = "*PPD-Adobe: \"4.3\"\n\ +*PageSize 102x152/4\" x 6\": \"\"\n\ +*PageSize 57x32/57 x 32 mm: \"\"\n\ +*PageSize 58x32/58 x 32 mm: \"\"\n\ +*PaperDimension 102x152: \"289.13 430.87\"\n\ +*PaperDimension 57x32: \"161.57 90.71\"\n\ +*PaperDimension 58x32: \"164.41 90.71\"\n"; + +#[test] +fn a_job_on_a_standard_size_names_the_ppd_keyword_of_that_size() { + let media = |value: &str| { + let job = Job { printer: Some("Label".into()), options: vec![("media".into(), value.into())], ..Job::default() }; + let job = spool::with_ppd_media(&job, |_| spool::parse_ppd_papers(METRIC_LABEL_PPD)); + job.options.into_iter().map(|(_, v)| v).collect::>() + }; + let label = cups_media(LABEL_PAPER).unwrap_or_default(); + assert_eq!(media(label), ["102x152"], "PdfCraft's 4 × 6 in label on the driver's 102 × 152 mm stock"); + assert_eq!(media("na_index-4x6_4x6in"), ["102x152"], "a PWG name in inches"); + assert_eq!(media("oe_4x6-label_4x6in"), ["102x152"]); + assert_eq!(media("w432h288"), ["102x152"], "turned"); + assert_eq!(media("w162h90"), ["57x32"], "the nearest of two sizes in reach"); + assert_eq!(media("om_small-label_57x32mm"), ["57x32"], "a PWG name in millimetres"); + assert_eq!(media("w288h440"), ["w288h440"], "more than 5 pt off"); + assert_eq!(media("Custom.288x432"), ["Custom.288x432"], "custom points match within half a point or not at all"); + for name in ["w288", "wxh432", "w-288h432", "na_x_4x6", "na_x_axbin", "x_4x6in", "winfhinf"] { + assert_eq!(media(name), [name], "{name} names no size"); + } +} + +#[test] +fn nominal_media_sizes() { + assert_eq!(spool::nominal_media_size("Letter"), Some((612.0, 792.0))); + assert_eq!(spool::nominal_media_size("a4"), Some((595.28, 841.89)), "cups_media's names, in any case"); + assert_eq!(spool::nominal_media_size("w288h432"), Some((288.0, 432.0))); + assert_eq!(spool::nominal_media_size("na_index-4x6_4x6in"), Some((288.0, 432.0))); + let a6 = spool::nominal_media_size("iso_a6_105x148mm").unwrap_or_default(); + assert!((a6.0 - 297.64).abs() < 0.01 && (a6.1 - 419.53).abs() < 0.01, "{a6:?}"); + assert_eq!(spool::nominal_media_size("Roll80mm"), None); + assert_eq!(spool::nominal_media_size("w0h432"), None); +} + #[cfg(all(unix, not(target_arch = "wasm32")))] mod cups_options { use crate::spool; @@ -917,6 +1048,25 @@ mod cups_options { std::fs::remove_dir_all(dir).unwrap(); } + /// `printer_papers` and `printer_default_paper` on CUPS: the queue's PPD papers by label and + /// its `*DefaultPageSize`, instances sharing their queue's PPD; nothing for a queue without + /// one or a destination that isn't a plain name. + #[test] + fn printer_papers_are_read_from_the_queue_ppd() { + let dir = folder(); + std::fs::write(dir.join("Label.ppd"), super::LABEL_PPD).unwrap(); + for destination in ["Label", "Label/draft"] { + let (papers, default) = spool::cups_paper_info_in(destination, &dir); + let names: Vec<&str> = papers.iter().map(|(name, _)| name.as_str()).collect(); + assert_eq!(names, ["US Letter", "4 x 6 in", "Envelope #10", "Wide", "Key"], "{destination}"); + assert_eq!(default, Some(("4 x 6 in".to_string(), (288.0, 432.0))), "{destination}"); + } + for destination in ["Missing", "../Label", "Label/a/b", "Label\\draft", "", ".", "-Label"] { + assert_eq!(spool::cups_paper_info_in(destination, &dir), (Vec::new(), None), "{destination:?}"); + } + std::fs::remove_dir_all(dir).unwrap(); + } + #[test] fn invalid_destinations_are_rejected_before_running_lpoptions() { let dir = folder(); diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 79423e76cb..a25e77d178 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -8409,6 +8409,10 @@ evidence = [ "crates/ui-egui/tests/it/print.rs::printer_papers_and_properties_selection", "crates/automation/tests/it/automation.rs::printer_papers_through_tools", "crates/print/src/tests.rs::printer_papers_and_default_paper_api", + "crates/print/src/tests.rs::ppd_papers_come_from_the_ppd", + "crates/print/src/tests.rs::a_job_on_a_ppd_paper_names_the_ppd_keyword", + "crates/print/src/tests.rs::a_job_on_a_standard_size_names_the_ppd_keyword_of_that_size", + "crates/print/src/tests.rs::printer_papers_are_read_from_the_queue_ppd", "crates/print/src/tests.rs::overlapping_paper_discovery_two_sided_and_queue_queries", "crates/ui-egui/tests/it/print.rs::a_custom_paper_prints_at_its_size_and_its_printer_keeps_it", "crates/ui-egui/src/workflows/print_paper.rs::the_paper_list_is_the_standard_sizes_then_the_custom_ones", @@ -8417,7 +8421,7 @@ evidence = [ "crates/ui-egui/tests/it/print_custom_paper_names.rs::a_custom_paper_cannot_take_a_standard_name", "crates/automation/tests/it/automation.rs::doc_print_takes_a_paper_size_of_its_own", ] -notes = "In the Print dialog: Custom… beside Paper adds a named size in the preferences' unit (1 to 200 inches a side), kept with the settings and removable. A printer printed to on a custom paper starts on it next time (#2001). The Page Setup button opens the driver's page setup to choose paper size and orientation. printer_papers enumerates driver-reported papers and default paper headlessly. doc_print takes sizes such as 8.5x14in or 210x297mm or driver papers. Missing: margins." +notes = "In the Print dialog: Custom… beside Paper adds a named size in the preferences' unit (1 to 200 inches a side), kept with the settings and removable. A printer printed to on a custom paper starts on it next time (#2001). The Page Setup button opens the driver's page setup to choose paper size and orientation. printer_papers enumerates driver-reported papers and default paper headlessly. On CUPS they are the queue's PPD papers by the driver's names and its *DefaultPageSize, and a job on one, or on a standard size of one (such as the 4 x 6 in label), is fed it under the PPD's keyword. doc_print takes sizes such as 8.5x14in or 210x297mm or driver papers. Missing: margins." [[feature]] id = "print.print-ready-pdf" From f51ce3cfccddef0bb44e836bc8abd00f289fd53e Mon Sep 17 00:00:00 2001 From: Michael Higgins <55243596+Higgs32584@users.noreply.github.com> Date: Wed, 11 Nov 2026 01:48:14 -0500 Subject: [PATCH 28/60] fix(render): bound the objects lopdf's fallback parses from object streams and arrays (#5083) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What this changes lopdf's fallback loader (used by `inspect` when the lazy reader fails, and always by `attachment_data`) parsed every object stream eagerly, capping only decoded bytes. A stream of `0 0 0 …` inside the 256 MiB limit built ~120M `Object`s (10+ GB) and aborted. This bounds what it builds: - **Per top-level object:** at most 4,194,304 objects (`parser::MAX_OBJECT_NODES`, the same as cos's `MAX_NODES`), counting every array element and dictionary value at any depth. One budget is threaded through `array` / `_dictionary` / `_direct_object`. An object is charged once it is built, so failed `alt` branches cost nothing, and an exhausted budget is a nom `Failure`. - **Per object stream:** at most 1,000,000 index entries (cos's `/N` clamp), and the index list is no longer allocated from the whole block. - **Per load:** at most 8,388,608 objects across all object streams, in an atomic `NodeBudget` shared by rayon workers. A member reserves up to its own byte length (every object takes at least one byte) and refunds what it doesn't use, so parallel members can't hold the budget between them. Once the budget is spent, the remaining members are skipped with a warning. - **`convert` in `crates/render/src/structure.rs`:** the depth limit is now 128, one level per array or dictionary, matching cos's parser. Before, it was 100 with two levels per dictionary, so ~51 nested dictionaries alone forced the lopdf fallback. Logged as lopdf patch (7) in `vendor/README.md`. Fixes #5083 ## How you verified it `cargo fmt --check` on `pdfcraft-render` and `cargo clippy -p pdfcraft-render --all-targets -- -D warnings` (which compiles the vendored lopdf). Regression tests in `crates/render/src/inspect/tests.rs`, where lopdf patches are tested in-tree (the vendored lopdf's own tests need its missing `assets/example.pdf`): - `object_stream_members_over_the_node_cap_are_skipped` (croquete1): a member that is one array of 4.3M elements is skipped at the per-object cap; a small member is kept. - `object_stream_index_entries_past_the_clamp_are_ignored`: of 1,000,001 distinct members, the one past the 1,000,000-entry clamp is never built. - `object_streams_of_one_load_share_their_node_budget`, `object_streams_loaded_in_parallel_share_their_node_budget` and `a_malformed_object_stream_member_refunds_its_reservation` (Harrison-Booth): a small shared `NodeBudget` bounds the members of several streams, serially and across threads, and a malformed member refunds what it didn't build. `NodeBudget` and `ObjectStream::new_within` are now public for this. - [x] `cargo fmt --check` - [x] clippy with `-D warnings` on the crates I changed - [x] tests pass on the crates I changed (`cargo test -p pdfcraft-render --lib object_stream`: 9 passed, including the five new tests) ## Never-crash - [x] No new `unwrap`/`expect`/`panic!`/`unreachable!`/`todo!` outside tests - [x] No indexing or slicing with input-derived positions; the budget uses `checked_sub` and a `min`-bounded reservation; index entries and objects built are capped; nesting stays bounded (128 in `convert`) - [x] No `unsafe` - [x] The abort this fixes has synthetic regression tests (above) ## Provenance - [x] No assets, fonts or copied code; the limits follow `pdfcraft_cos`'s own caps ## Security - [x] No change to `--root`, MCP, the control channel or network access ## Parity and reach - Not applicable: an internal robustness fix to the fallback loader, with no user-facing feature or parity entry. The vendored-patch log (`vendor/README.md`, lopdf (7)) is updated. ## Anything a reviewer should look at first - `vendor/README.md` now lists the tests above for patch (7). - Ordinary (not compressed) indirect objects are capped at 4M objects each but aren't charged to the per-load budget; their total is bounded only by file size. - `get_compressed_object` still decodes and re-parses the whole stream per lookup (pre-existing); it now gets one object's budget. - Objects nested 101–128 deep now reach the Inspector through the lazy reader rather than being skipped by the fallback. The Inspector's own walkers cap depth at 32 or less. ## Summary by CodeRabbit * **Bug Fixes** * Added limits to PDF object parsing and object-stream processing. Loading now parses only selected members, while lookups parse only the requested member. Members that exceed available limits are skipped in a deterministic order. * Increased the supported nesting depth for dictionary and object conversion, allowing more deeply nested content to be processed. * Documented that conversion failures can trigger eager loading as a fallback. --- crates/render/src/inspect/tests.rs | 166 ++++++++++++++++++++++++++ crates/render/src/structure.rs | 19 ++- vendor/README.md | 2 +- vendor/lopdf/src/lib.rs | 2 +- vendor/lopdf/src/object_stream.rs | 185 +++++++++++++++++++++++++---- vendor/lopdf/src/parser/mod.rs | 102 ++++++++++++---- vendor/lopdf/src/reader.rs | 125 ++++++++++++------- 7 files changed, 511 insertions(+), 90 deletions(-) diff --git a/crates/render/src/inspect/tests.rs b/crates/render/src/inspect/tests.rs index e89fab5c99..68486d8097 100644 --- a/crates/render/src/inspect/tests.rs +++ b/crates/render/src/inspect/tests.rs @@ -249,6 +249,8 @@ fn object_stream_member_extents_bound_owned_strings() { assert!(!parsed.objects.contains_key(&(201, 0))); } +/// Members are read at their indexed offsets, in any order and with any whitespace around +/// them; a number listed twice keeps its last entry. #[test] fn object_stream_member_extents_preserve_index_order_and_whitespace() { let members = b" \n42\n /Name \n[true 7]\n << /Key (value) >>\n (last)"; @@ -262,6 +264,170 @@ fn object_stream_member_extents_preserve_index_order_and_whitespace() { assert_eq!(parsed.objects.get(&(12, 0)).unwrap().as_dict().unwrap().get(b"Key").unwrap().as_str().unwrap(), b"value"); } +/// lopdf patch (7), #5083: a decoded object-stream member could build any number of objects +/// (`0 0 0 ...` inside the decompression limit built ~120M and aborted). A member now builds at +/// most the per-object node cap (1 << 22), so one array with more elements than that is skipped +/// while a small member is kept. Without the node charge in the vendored parser the oversize +/// array builds in full, so object 11 is present and the last assertion fails. +#[test] +fn object_stream_members_over_the_node_cap_are_skipped() { + let small = b"[ 1 2 3 ]"; + let mut big = vec![b'[']; + for _ in 0..4_300_000 { + big.extend_from_slice(b"0 "); + } + big.push(b']'); + let mut members = small.to_vec(); + let big_off = members.len(); + members.extend_from_slice(&big); + let stream = indexed_object_stream(&[(10, 0), (11, big_off)], &members); + let parsed = lopdf::ObjectStream::new_with_limit(&stream, Some(16 << 20)).unwrap(); + assert!(parsed.objects.contains_key(&(10, 0)), "the small member is kept"); + assert!(!parsed.objects.contains_key(&(11, 0)), "the oversize member is skipped"); +} + +/// lopdf patch (7), #5083: an object stream reads at most 1,000,000 index entries (cos's `/N` +/// clamp), so a huge index can't allocate an entry list or parse a member per entry. Of +/// 1,000,001 distinct one-byte members, the last is never built. +#[test] +fn object_stream_index_entries_past_the_clamp_are_ignored() { + let count = 1_000_001_u32; + let mut members = Vec::with_capacity(2 * count as usize); + let pairs: Vec<(u32, usize)> = (1..=count) + .map(|id| { + let at = members.len(); + members.extend_from_slice(b"0 "); + (id, at) + }) + .collect(); + let parsed = lopdf::ObjectStream::new_with_limit(&indexed_object_stream(&pairs, &members), None).unwrap(); + assert_eq!(parsed.objects.len(), 1_000_000); + assert!(parsed.objects.contains_key(&(1_000_000, 0))); + assert!(!parsed.objects.contains_key(&(count, 0)), "the entry past the clamp is ignored"); +} + +/// Six object streams, each holding one nested member of seven objects. +fn nested_object_streams() -> Vec { + (0..6).map(|i| indexed_object_stream(&[(100 + i, 0)], b"<< /A [1 [2]] /B << /C 3 >> >>")).collect() +} + +/// lopdf patch (7), #5083: one load's object streams share a single node budget (the load-wide +/// `MAX_LOAD_NODES`, here 12 nodes so the test stays small), so later streams are skipped once +/// earlier ones spent it. A stream given a budget of its own would load all six. +#[test] +fn object_streams_of_one_load_share_their_node_budget() { + let streams = nested_object_streams(); + let budget = lopdf::NodeBudget::new(12); + let loaded: usize = streams.iter().map(|s| lopdf::ObjectStream::new_within(s, None, &budget).unwrap().objects.len()).sum(); + assert!(loaded > 0); + assert!(loaded < streams.len(), "the budget is shared by all streams"); + assert!(budget.spent()); +} + +/// The shared budget holds across threads too, as rayon's workers load object streams in +/// parallel: members reserve their share before parsing, so together they can't overshoot. +#[test] +fn object_streams_loaded_in_parallel_share_their_node_budget() { + let streams = nested_object_streams(); + let budget = &lopdf::NodeBudget::new(12); + let loaded: usize = std::thread::scope(|scope| { + let workers: Vec<_> = + streams.iter().map(|s| scope.spawn(move || lopdf::ObjectStream::new_within(s, None, budget).unwrap().objects.len())).collect(); + workers.into_iter().map(|w| w.join().unwrap()).sum() + }); + assert!(loaded > 0); + assert!(loaded < streams.len(), "the budget is shared by all streams"); +} + +/// A member that fails to parse gives back the nodes it reserved and did not build, so a +/// malformed member doesn't use up the budget of the members after it. +#[test] +fn a_malformed_object_stream_member_refunds_its_reservation() { + let stream = indexed_object_stream(&[(10, 0), (11, 3)], b"[1 [2]"); + let budget = lopdf::NodeBudget::new(5); + let parsed = lopdf::ObjectStream::new_within(&stream, None, &budget).unwrap(); + assert!(!parsed.objects.contains_key(&(10, 0))); + assert!(parsed.objects.contains_key(&(11, 0))); + assert!(!budget.spent(), "unused reservation from the malformed member is refunded"); +} + +/// A budget too small for every member builds them in index order, so the members a spent +/// budget skips are the same on every load. Here the first member takes four of the five +/// nodes, the second the last one, and the third is skipped. Parsed in parallel from the shared +/// budget, the second and third could find it held by the first and be skipped as well. +#[test] +fn a_tight_budget_builds_object_stream_members_in_index_order() { + let stream = indexed_object_stream(&[(10, 0), (11, 8), (12, 10)], b"[1 2 3] 7 8"); + let budget = lopdf::NodeBudget::new(5); + let parsed = lopdf::ObjectStream::new_within(&stream, None, &budget).unwrap(); + assert!(parsed.objects.contains_key(&(10, 0))); + assert!(parsed.objects.contains_key(&(11, 0)), "the second member is built from what the first left"); + assert!(!parsed.objects.contains_key(&(12, 0)), "the third member finds the budget spent"); + assert!(budget.spent()); +} + +/// A PDF with a cross-reference stream whose Info dictionary (object 4) sits in object stream 3 +/// after `first`, a member listed before it. +fn info_in_object_stream_pdf(first: &[u8]) -> Vec { + let info = b"<< /Title (Kept) >>"; + let index = format!("5 0 4 {} ", first.len() + 1); + let mut data = index.clone().into_bytes(); + data.extend_from_slice(first); + data.push(b' '); + data.extend_from_slice(info); + let mut pdf = b"%PDF-1.5\n".to_vec(); + let mut offsets = [0_u32; 7]; + let mut append = |pdf: &mut Vec, id: usize, head: &[u8], body: &[u8], tail: &[u8]| { + offsets[id] = pdf.len() as u32; + pdf.extend_from_slice(format!("{id} 0 obj\n").as_bytes()); + pdf.extend_from_slice(head); + pdf.extend_from_slice(body); + pdf.extend_from_slice(tail); + pdf.extend_from_slice(b"\nendobj\n"); + }; + append(&mut pdf, 1, b"<< /Type /Catalog /Pages 2 0 R >>", b"", b""); + append(&mut pdf, 2, b"<< /Type /Pages /Kids [] /Count 0 >>", b"", b""); + let head = format!("<< /Type /ObjStm /N 2 /First {} /Length {} >>\nstream\n", index.len(), data.len()); + append(&mut pdf, 3, head.as_bytes(), &data, b"\nendstream"); + let xref_offset = pdf.len() as u32; + offsets[6] = xref_offset; + let mut records = Vec::new(); + for (id, &offset) in offsets.iter().enumerate() { + let (kind, field, index) = match id { + 0 => (0_u8, 0_u32, 0xffff_u16), + 4 => (2, 3, 1), + 5 => (2, 3, 0), + _ => (1, offset, 0), + }; + records.push(kind); + records.extend_from_slice(&field.to_be_bytes()); + records.extend_from_slice(&index.to_be_bytes()); + } + let head = format!("<< /Type /XRef /Size 7 /Root 1 0 R /Info 4 0 R /W [1 4 2] /Length {} >>\nstream\n", records.len()); + pdf.extend_from_slice(format!("6 0 obj\n{head}").as_bytes()); + pdf.extend_from_slice(&records); + pdf.extend_from_slice(format!("\nendstream\nendobj\nstartxref\n{xref_offset}\n%%EOF\n").as_bytes()); + pdf +} + +/// lopdf patch (7), #5083: a lookup in an object stream (lopdf's metadata reader, or an indirect +/// `/Length` read while loading) parses only the member it wants. It used to parse every member +/// from one object's budget, so a member before it of more than `MAX_OBJECT_NODES` objects spent +/// that budget and the wanted member was skipped: the Info dictionary read as missing. +#[test] +fn a_lookup_in_an_object_stream_parses_only_the_member_it_wants() { + let mut big = vec![b'[']; + for _ in 0..4_300_000 { + big.extend_from_slice(b"0 "); + } + big.push(b']'); + let metadata = Document::load_metadata_mem(&info_in_object_stream_pdf(&big)).unwrap(); + assert_eq!(metadata.title.as_deref(), Some("Kept")); + // The fixture itself reads: with a small member before it, the Info dictionary was read before too. + let metadata = Document::load_metadata_mem(&info_in_object_stream_pdf(b"[1 2 3]")).unwrap(); + assert_eq!(metadata.title.as_deref(), Some("Kept")); +} + #[test] fn view_and_user_space_round_trip_for_every_rotation() { for rotation in [0u16, 90, 180, 270] { diff --git a/crates/render/src/structure.rs b/crates/render/src/structure.rs index 441f48998c..914bab731f 100644 --- a/crates/render/src/structure.rs +++ b/crates/render/src/structure.rs @@ -166,6 +166,8 @@ impl Structure for LazyStructure { fn encrypted(&self) -> bool { self.source.security().is_some() || self.trailer.has(b"Encrypt") } + /// The object `id`, read from the lazy source and converted once, then kept; a failed + /// conversion marks the structure so the caller can fall back to the eager loader. fn get_object(&self, id: ObjectId) -> Result<&Object> { let missing = || Error::ObjectNotFound(id); let slot = self.objects.binary_search_by_key(&id.0, |(n, _)| *n).ok().and_then(|i| self.objects.get(i)).ok_or_else(missing)?; @@ -185,8 +187,15 @@ impl Structure for LazyStructure { } } +/// Nesting `convert` accepts: the same as `pdfcraft_cos`'s parser (each array or dictionary +/// one level), so every object cos parsed converts. A lower limit would turn deep but valid +/// nesting into a lazy-reader failure and force the eager lopdf fallback. +const CONVERT_DEPTH: usize = 128; + +/// Convert a cos dictionary to lopdf's, without `/PieceInfo`. `depth` is the nesting level of +/// the dictionary itself; its values sit one level deeper. fn convert_dict(dict: &pdfcraft_cos::Dict, depth: usize) -> Result { - if depth > 100 { + if depth > CONVERT_DEPTH { return Err(Error::RecursionLimit); } dict.iter() @@ -203,8 +212,10 @@ fn convert_dict(dict: &pdfcraft_cos::Dict, depth: usize) -> Result { /// untouched; this is the read-only view inspection walks. const PRIVATE_DATA: &[u8] = b"PieceInfo"; +/// Convert a cos object to lopdf's, failing past [`CONVERT_DEPTH`] levels of arrays and +/// dictionaries. fn convert(object: &CosObject, depth: usize) -> Result { - if depth > 100 { + if depth > CONVERT_DEPTH { return Err(Error::RecursionLimit); } Ok(match object { @@ -218,8 +229,8 @@ fn convert(object: &CosObject, depth: usize) -> Result { } CosObject::Ref(value) => Object::Reference((value.num, value.generation)), CosObject::Array(value) => Object::Array(value.iter().map(|o| convert(o, depth + 1)).collect::>()?), - CosObject::Dict(value) => Object::Dictionary(convert_dict(value, depth + 1)?), - CosObject::Stream(value) => Object::Stream(lopdf::Stream::new(convert_dict(&value.dict, depth + 1)?, value.raw.to_vec())), + CosObject::Dict(value) => Object::Dictionary(convert_dict(value, depth)?), + CosObject::Stream(value) => Object::Stream(lopdf::Stream::new(convert_dict(&value.dict, depth)?, value.raw.to_vec())), }) } diff --git a/vendor/README.md b/vendor/README.md index 391dab88e2..558dbbbc12 100644 --- a/vendor/README.md +++ b/vendor/README.md @@ -58,7 +58,7 @@ Other resource categories retain their existing parent-chain lookup. | hayro-interpret | 0.7.0 | Apache-2.0 OR MIT | (1) select `/AP /N` appearance states via `/AS` (checkboxes, radios, stateful icons) and honour the NoView flag; (2) Type3 Unicode fallback to `/Encoding` glyph names, then printable ASCII; (3) `Type3Glyph::advance_width`; (4) `InterpreterSettings::ocg_overrides` for viewer layer toggles; (5) soft masks whose group lacks `/CS` were dropped (Chrome gradient text); (6) paint nesting cap `MAX_PAINT_NESTING` for tiling patterns and Type 3 glyphs (self-referencing content overflowed the stack); (7) CID width ranges in `/W` and `/W2` clamped to `MAX_CID` (a `0 4294967295 w` range hung); (8) `InterpreterSettings::hide_comments` skips markup annotations but keeps widgets and links (View ▸ Hide all comments); (9) `image_size_ok`: images and masks over 2^28 pixels are not decoded (a fuzzed `/W 4294967295` stencil mask allocated 4 GiB); (10) `MAX_NESTED_PAINTS`: nested paints (forms, Type 3 glyphs and tiling patterns inside one another) share a per-page budget of 50,000 (a glyph showing eight glyphs of its own font, sixteen deep, hung); (11) depends on skrifa 0.47 instead of 0.42 (upstream read-fonts fixed a Type 1 tokenizer that recursed through runs of numbers: a fuzzed font hung or overflowed the stack), with the one API change (`Charset::glyph_id` returns `Option`); (12) annotation appearances whose `/BBox` or `/Matrix` collapses to a line or a point are skipped (mapping them onto `/Rect` divided by zero and handed the device infinite or NaN matrices); (13) a Highlight annotation without a usable normal appearance is drawn from its `/QuadPoints`, `/C` and `/CA` with Multiply blending, as pdfcraft-annot draws highlights (it was not drawn at all); (14) simple-font `/Widths` (`read_widths`): entries only for codes 0–255 (one entry per code below `/FirstChar` was allocated: `4294967295` asked for 34 GB), an entry that isn't a number leaves its code missing instead of ending the array, and a `/LastChar` below `/FirstChar` gives no widths instead of discarding the font for a standard one; (15) the appearance box is mapped onto `/Rect` corner to corner: the translation now accounts for the scale (an appearance whose box didn't start at the origin and differed in size from `/Rect` was drawn shifted); (16) a clip set by `W`/`W*` takes effect after whichever path-painting operator ends the path (`S`, `f`, `B`, … as well as `n`; ISO 32000-2 §8.5.4): it was applied only on `n`, so `re W* S` clipped nothing and the clip stayed pending for a later, unrelated `n`; (17) `InterpreterSettings::cancelled`: once set, interpretation stops before the next content operator (a render pool sets it for every worker when dropped, after an edit or on closing, and for one worker when the watchdog gives up on its render, so renders whose answers nobody can receive stop instead of running to the end); (18) `MAX_PAGE_CONTENT`: the content a page holds at once (its contents, and the forms, tiling patterns and Type 3 glyph procedures being painted) stays within 512 MiB (`decode_content`; once it runs out, `InterpreterWarning::ContentTruncated` is reported (once per page by pdfcraft-render) so a truncated page is not a clean render; each level gives its bytes back when it finishes): forms nest fifty deep and each level kept its content while the next ran, so a form painting itself around a stream inflating to 256 MiB would have held over 12 GB; and Type 0 sampled functions take at most `MAX_SAMPLE_ENTRIES` (2^21) table points and `MAX_SAMPLE_VALUES` (2^23) samples, read no further than `/Size` asks (256 MiB of 1-bit samples asked for over 8 GB); (19) mesh shadings (types 4–7) are sampled only inside the area being drawn (`ShadingPattern::encode_within`, which the vendored hayro renderer calls with that area plus a pixel), within 8 visits per pixel of it plus 4 per triangle (`TriangleSampler`), and patches are triangulated one at a time: points far off the page were sampled over their whole bounding box, up to 65536 × 65536 entries (a fuzzed 147-byte patch stream; a synthetic page passed 6 GB in six seconds), ten thousand overlapping triangles ran for minutes, and every patch became 722 triangles before any limit; meshes hold at most `MAX_MESH_PATCHES` (2^16) patches and `MAX_MESH_TRIANGLES` (2^20) triangles, a lattice needs two to 2^19 vertices per row (`/VerticesPerRow 0` pushed empty rows for ever; one huge row read the whole stream), and a page samples at most `MAX_PAGE_MESH_TRIANGLES` (2^25) mesh triangles in all (a mesh painted a thousand times was triangulated a thousand times); each cut logs a warning; (20) PDF function construction is limited to 64 levels and 10,000 total nodes with cycle detection; calculator parsing and evaluation share procedure depth/work budgets, integer division/bitshift/rotation/index are panic-free, and fixed-size stack errors propagate; (21) a Type 0 font drawn with a substitute that isn't CID-keyed (it isn't embedded, or its program doesn't load) draws `.notdef` for a CID of an Adobe character collection (Japan1, GB1, CNS1, Korea1) that it can't find by Unicode: hayro took the CID for a glyph index of the substitute, which numbers its glyphs its own way, so the character was drawn as an unrelated one (𠮷 as "Ｑ" in BIZ UDPGothic); (22) a Type 0 font drawn with a substitute finds a character given with a variation selector (`/ToUnicode`, or Adobe-Japan1-UCS2, which maps the JIS X 0208 forms of the kanji JIS X 0213:2004 redrew that way: CID 1247 is 噂 U+E0100) through the substitute's variation sequences, else as the character alone: such a CID found no glyph and was drawn as an unrelated one; (23) backport of hayro 0.8's `gradient` module: `EncodedShadingPattern::as_svg_gradient` turns an axial or radial shading into stops of an sRGB gradient (within 0.01 per channel, a stop at every stitching-function bound via `Function::stitching_bounds`) when a native gradient draws it exactly (extended past the box drawn in; a radial one with no background colour); (24) `DecodedImageCache`: decoded image pixels shared across renders through the interpreter cache within a 64 MiB budget (an image-heavy page decoded its thousands of images again per tile and per scale), keyed by every decode input (`DecodedImageKey`: the stream's place and dictionary bytes, its object, mask/stencil/raster output, interpolation, decode target size, preview or final); images under a transfer function and inline images are not cached, an image over the budget is never kept, and a poisoned lock decodes uncached; (25) `Device::actual_text` hands the device a marked content sequence's `/ActualText` (inline or named in `/Properties`), so text extraction keeps the Thai, Indic and Arabic clusters Chrome marks this way (their shaped tone marks and vowels have no Unicode of their own); (26) non-embedded simple fonts drawn with a standard-font stand-in build the Latin Extended-A letters the stand-in lacks (`/Gbreve`, `/Idotaccent`, `/Scedilla`, `/aogonek`, `/ccaron`…) from the base letter and accent it has (`StandardKind::composite_glyph`, `ACCENTED_LETTERS`): PDFium's Foxit stand-ins hold only the Standard/WinAnsi/Mac Roman glyphs, so iText letters with Times New Roman unembedded showed blanks for Ğ, İ, Ş, ğ and ş; the comma-below letters (`/scommaaccent`, `/Tcommaaccent`, `/kcommaaccent`…, found by character, so `/uni0136` and the glyph list's `/Kcedilla` too) are the letter and the font's comma, smaller and under the baseline, since the stand-ins have no `commaaccent` (Romanian ș drew nothing and ț a cedilla in forms laid out from XFA; ģ, whose comma is turned and above, is not built); (27) an embedded TrueType font without a base encoding reads its codes as WinAnsi for text extraction (`EmbeddedKind::code_to_unicode_name`), matching pdftotext's output, instead of Standard: OCR text layers of scans write WinAnsi codes with no `/Encoding`, and Standard has no ü, ö or ß and reads é as Ø (glyph selection unchanged); (28) an image whose pixels hold at most 16 bits of samples (an 8-bit Separation or DeviceN image, a 1-, 2- or 4-bit gray or indexed one) is converted once per possible pixel value through a lookup table instead of once per pixel (`rgb_via_lookup`): a spot-colour tint transform is often a PostScript calculator program, run for every pixel, so a fillable character sheet's 3304 × 4105 DeviceN background took four seconds and 430 MB to draw a page; each entry is converted on its own (`convert_each`), so a tint transform giving the wrong number of components for a value the image never uses cannot shift the colours of the values it does; (29) `Device::begin_annotations`/`end_annotations` bracket the page's annotations, so a device can tell comment paints from document paints (reading colours recolour the document, never a comment, #1548); (30) the Annex D.2 alias codes read as the ink they draw: WinAnsiEncoding 0xA0 and 0xAD, and MacRomanEncoding 0xCA, carry the glyph names `space` and `hyphen` -- the names those tables also give 0x20 and 0x2D -- not `nbspace` and `sfthyphen`, whose glyph list values U+00A0 and U+00AD are invisible formatting characters. Extraction hid drawn ink behind them, so search and redaction matched neither (`XiaoBiaoSong.pdf` in the pdf.js corpus drew 104 soft hyphens; across the 989 files the soft hyphens we extract fall from 117 to 13 and the non-breaking spaces from 30 to 26, what remains coming from `/ToUnicode` maps, which say so explicitly). The base table only, in the text direction only: a `/Differences` name keeps its glyph list value and glyph lookup is untouched, both as pdftotext does; (31) a Type 0 sampled function keeps its samples in one flat array indexed by strides (`Type0::entry`) instead of a hash map keyed by each grid point: the map was built again, one hashed insert per sample, every time a shading using the function was painted (a 2,097,152-sample ramp painted 30 times took 38 s; `bug1721218` in the pdf.js corpus spent most of a 1.5 s page on it); the interpolation is unchanged, so results are bit for bit the same, a sample missing from a short stream still gives no value, and the maximum sample value is computed without overflowing at 32 bits per sample; (32) the decoded-image cache (24) keys a decode by the size the image is drawn at only when a JPX stream decodes it or its soft mask or mask (`decode_reads_target`): JPX is the only filter that decodes a reduced resolution for a size, so a JPEG or Flate scan was decoded again at every zoom step and kept once per zoom level; one decode now serves every size; `InterpreterCache::decoded_image_decodes` counts the cache's decodes; (33) Coons and tensor-product patches copy their vertices' colours less: each point of a patch's 20 × 20 triangle grid is interpolated once (`generate_patch_triangles`; each cell interpolated its four corners itself, 1,444 interpolations a patch for 400 points), a cell's second triangle takes the vertices its first copied, and the mesh sampler moves a patch's triangles into device space instead of copying each vertex (`TriangleSampler::sample_owned`); the same values, 10–16% faster on mesh-gradient pages; (34) `color::tint_transform_runs`: how many colours the calling thread has run through a Separation or DeviceN tint transform (counted once per batch in `convert_f32`), so a test counts patch (28)'s once-per-sample-value runs instead of timing them (its 10 s deadline failed on a busy FreeBSD VM in CI); (35) `EncodedShadingPattern::extent`: where a mesh shading (types 4–7) can paint at all, the pixels its triangles touched when it was encoded (`None` for the types that paint everywhere, and when the shading has a `/Background`), so the renderer samples a texture there alone and skips a mesh with nothing on the canvas; (36) a mesh shading's pixel whose corner no triangle holds strictly (on an edge two triangles share, or on the mesh's side) takes the triangle that covers its centre, edges included (`Triangle::covers`): testing only the corner left a white seam along every shared edge, and the colour is still taken at the corner; (38) the `/DefaultGray`, `/DefaultRGB` and `/DefaultCMYK` colour spaces of a resource dictionary (ISO 32000-2 §8.6.5.6) replace the device space content selects: with `g`/`G`, `rg`/`RG`, `k`/`K` and `cs`/`CS` (by name, or through a `/ColorSpace` resource that is a device space), and as the `/ColorSpace` of an image XObject, an inline image or a shading painted by `sh` or as a shading pattern (`DefaultColorSpaces`, looked up once per content window in the resources and those they inherit, so a form XObject without a default of its own draws in its page's, as Poppler and MuPDF draw it); a default that is a Lab, Indexed, Pattern or device space, has another number of components or is no colour space is ignored and device colour is drawn as before; device spaces inside other spaces (an Indexed base, a Separation or DeviceN alternate, a Pattern's underlying space), shading dictionaries, masks and transparency group spaces are unchanged, and the decoded-image cache (24) keys a decode by the default it was drawn in. hayro drew device colour as it was, so pages that calibrate it this way (common in PDF/A) came out in the wrong colours; (39) knockout groups (#2971): `Device::set_knockout` right after a `/K true` group is pushed; `Device::knockout_image` for an unmasked image before its group is pushed, when `Device::in_knockout_group`: the device clears its placement rectangle only if it will draw the image (on the canvas, not a preview's tiny image, and it decodes), decoded at its draw size so the draw reuses that decode; and, when `Device::knockout_erase_wanted`, a nested group's content drawn once between `begin_knockout_erase` and `end_knockout_erase` before the group itself, which then always gets a layer of its own; (40) an optional content membership dictionary's visibility expression (`/VE`: `/And`, `/Or`, `/Not`, nested up to 32 deep) decides its visibility ahead of `/OCGs` and `/P` (ISO 32000-2 §8.11.2.2); it was ignored, so content under it stayed visible with its layers hidden; a malformed expression, including one naming an object the document doesn't list in `/OCProperties /OCGs`, falls back to `/OCGs` and `/P` (#2986); (41) a transparency group that is neither isolated nor knockout, entered at full opacity with no soft mask and Normal blending, is drawn straight onto the page instead of into a layer of its own (§11.4.7: its elements composite with the backdrop; the layer made a `/BM /Multiply` highlight appearance an opaque block, #1968); (42) text in a Type 3 font is drawn filled whatever the text rendering mode, except mode 3 (invisible): only that mode has any effect on Type 3 fonts (ISO 32000-1 §9.3.6). Stroking ran the glyph procedure a second time, so `2 Tr` made bitmap fonts look bold, `1 Tr` painted an uncoloured glyph in the stroking colour, and `7 Tr` hid it (`7 Tr` since follows 64); (43) Lab colour spaces honour their own `/WhitePoint`: L\*a\*b\* goes to XYZ relative to that white, is Bradford-adapted to D65 and converted to sRGB (ISO 32000-2 §8.6.5.4) instead of through `LAB.icc` as if every white were D50 (a D65 blue came out 45 levels too red; the profile is removed); components normalised to [0, 1] (`manual_scale`, an `/Indexed` palette over Lab) map onto L\* 0..100 and the a\*/b\* `/Range`; (44) a Type 0 font whose `/ToUnicode` is the name `/Identity-H` or `/Identity-V` (not a CMap stream; WSCAD writes it) reads each code as its Unicode value, as Acrobat shows it (`Type0Font::identity_to_unicode`); the identity CMap the name resolved to maps codes to CIDs only, so such pages had no text to select or find; (46) an image XObject's group applies the fill opacity, so the fill alpha is reset to 1 before a stencil mask takes the fill colour as its paint, and a pattern paint drops the opacity it captured at `scn` (`Pattern::without_opacity`) (it was applied twice: a mask at `ca 0.5` was drawn at a quarter, #2975); (47) `path_as_rect`: a five-element path is a rectangle only when its four points are distinct, each step from one to the next runs along a single axis, and the last returns to the first. The length test admits one MoveTo and four LineTo as well as one MoveTo, three LineTo and a ClosePath, and asking only that every corner of the bounding box was visited says nothing about the order or the steps between, so a five-point polyline whose ends reached the corners was painted as that box -- a fourth side the path does not have -- and a clip of that shape was discarded; the coordinates are compared exactly, since the matrix is applied after this check and a `cm` of 10,000 turns `is_nearly_equal`'s 256th of a unit into tens of device units (#3327); (48) an `/ICCBased` colour space is cached by what its profile is read from (`IccProfileKey`): the stream object it names, looked up without reading the stream, and then the stream's dictionary and its own bytes (decrypted, not decoded), so a byte-for-byte copy of a profile in another object is read once, and a profile stream written directly in its array, which reports the object holding the array as every other stream written there does, is told apart by its bytes. hayro keyed it by the stream dictionary's bytes alone, so two different profiles with the same `/N` and `/Length` shared whichever was read first, and colours in the second were drawn through the first. `InterpreterCache::icc_profile_reads` counts the profiles read; (49) `OpenTypeFontBlob::new` also accepts a `post` table version 1.0, 2.0 or 3.0 written with its 16-bit halves swapped (`00 00 00 03`), as ScanSoft PDF Create! embeds every font; such fonts were refused and their text drawn with a substitute (Hebrew came out garbled or missing, #2946); `glyph_names` reads such a table from a copy with the version written correctly, so a 2.0 table's names serve `/Differences`. Any other unknown version is still refused (pdf.js 9462); (50) `CffFontBlob::glyph_index_by_name` looks a glyph name up in a name to glyph map built on first use (shared between clones) instead of searching the whole charset, with a string lookup per glyph, for every character code of a font; the first glyph with a name still wins; (51) at most 4096 graphics states saved at once, holding at most 64 MiB together (each `q` copies the whole state, dash array and clip list included, so short operators multiplied one large state); a `q` past either limit is counted but not copied, and its `Q` still removes the clips added since (from storytold/craft-libs#239); (52) a `/W2` run whose first CID is past `MAX_CID` stops there, as in `/W` (7), checked before the run's metrics are read: a first CID of 4294967295 inserted one entry and then overflowed, discarding the whole `/W2` table (from storytold/craft-libs#254); (55) a Separation or DeviceN tint transform that cannot be read keeps its colour space, and one that cannot be read or evaluated paints the alternate space's initial colour, as evaluation failures already did; that fallback, and a `cs`/`CS` colour space that cannot be read (drawn in DeviceGray as before), are now reported once per content stream as `InterpreterWarning::ColorFallback` (`RenderWarning::ColourFallback` in `pdfcraft-render`) instead of silently (#4260); (56) an annotation whose Invisible flag (bit 1) is set and whose `/Subtype` is none of the standard types of ISO 32000-2 Table 171 (`is_standard_annotation_type`; no `/Subtype` counts as unknown) is not drawn (Table 167), and a `/Popup` annotation's appearance is never painted on the page (§12.5.6.14: a popup has no appearance stream of its own; the viewer shows it as a window for its parent annotation). Both were drawn from their `/AP`; (57) a simple font drawn with a standard-font stand-in because the document names a font it doesn't embed (`StandardKind` with `fallback`) narrows a glyph to a narrower `/Widths` advance but no longer widens it to a wider one: the advance already places the next glyph (ISO 32000-2 §9.2.4; /Widths in Table 109), and widening drew a non-embedded `Arial,Bold` with 600-unit widths as a heavy fixed-pitch face whose `i` and `l` were solid blocks twice their width; an exact standard-14 font still matches its standard metrics both ways; (58) the fallback request for a Type 0 font that is not embedded (`FallbackFontQuery::with_descriptor`) takes its serif, italic and small-cap flags, `/FontWeight`, `/FontFamily` and `/FontStretch` from the descendant CIDFont's font descriptor (ISO 32000-2 §9.7.4.1, §9.7.6.1; §9.8): it read them from the Type 0 dictionary, which has none, so every such request asked for a regular sans-serif face and no resolver could tell a Song or Mincho font from a Hei or Gothic one, or see `/FontWeight 700`; (59) the NoRotate flag (`/F` bit 5) is honoured: the appearance, fitted to `/Rect`, is turned against the page's `/Rotate` about the rectangle's upper-left corner, which stays where it is, so it reads upright as the page is displayed (ISO 32000-2 §12.5.3). Sticky notes, which are saved NoRotate, were drawn turned with the page (#2583). Form fields and links stay in `/Rect`, where PdfCraft takes their click targets from it; (61) stroke geometry by ISO 32000-2 §8.5.3.2 and §8.4.3: a degenerate subpath (a single-point closed path, or points that all coincide) is stroked as a dot of the line width with round caps and not at all with butt or projecting square caps (upstream nudged a zero-length `l` so square caps drew a square, and `x y m h` lost its `h` because kurbo's `is_empty` counts only segments); a segment after `h` begins a new subpath with a `MoveTo` to the point `h` reached (§8.5.2.1, Table 58), so `h` then an `l` back to that point is a degenerate subpath of its own; a dash array that is all zero or has a negative or non-finite entry is kept as `[]`, a solid line (`[0 0]` became hundredth-of-a-point dashes, a faint half-covered line); a negative or non-finite line width (`w`, `/LW`) is kept as 0, so no device sees one below zero; (63) a luminosity soft mask whose group colour space is `/DeviceCMYK` (by name, not an ICC-based or calibrated space) reads a `/DeviceCMYK` fill or stroke colour drawn in it, and its `/BC`, as their gray `1 - min(1, 0.3 c + 0.59 m + 0.11 y + k)` (ISO 32000-2 §10.4.2.3, §11.5.3: device colour converts to gray with no colour calibration; `Context::luminosity_in_device_cmyk`, `color::device_cmyk_to_gray`): it went through the CMYK display profile, where 100% K is a dark grey, so a 100% K mask let about an eighth of the masked paint through. CMYK images, shadings and tiling patterns inside such a mask, and CMYK on the page, still use the profile; (64) text in a Type 3 font follows ISO 32000-2 §9.3.6 in the clipping modes: under `7 Tr` it is not rendered (reported to the device as clipping text, `GlyphDrawMode::Clip`, so extraction keeps it), and in modes 4 to 7 a Type 3 glyph adds nothing to the clipping path (`clip_glyph`), so text shown only in a Type 3 font leaves no clip at `ET`. Since (42) `7 Tr` painted the glyphs in the fill colour (black by default), so a fill meant to follow showed them wherever it didn't cover them; 4 to 6 still render filled; (65) a fill or clip whose path has no width or no height (but not both) marks the device pixels the line touches (ISO 32000-2 §10.7.4: "A zero-width or zero-height rectangle paints a line 1 pixel wide", and a clip is "the set of pixels that would be included by a fill operation"): whole pixels when the line stays within one device row or column, a band one device pixel wide when a rotated or skewed CTM carries it across rows and columns, built per subpath so the gap between two subpaths on one line stays unpainted (`degenerate_fill_area`). A zero-height clip such as `10 10 100 0 re W n` hid everything painted after it, and the fill fell back to a thinnest-line stroke that half-covered two rows on a pixel boundary. A point paints as before, and a path with area, however thin, is filled as the shape it is (the thinnest-line fallback took a side under 1/256 unit as zero, so a thin rectangle that the CTM scales up to whole pixels was drawn as a hairline across its bounding box); (66) a Separation colour space whose colourant is the special name `All` paints every colourant at once (ISO 32000-2 §8.6.6.4): on screen and in an RGB print raster its tint t is the gray 1 − t, complemented because the device is additive, whatever its alternate space and tint transform say (hayro drew it through the tint transform, so a `/DeviceCMYK` transform giving cyan drew registration marks light cyan); `/None` still paints nothing, any other name still goes through its alternate, and a DeviceN space naming `All` keeps its alternate, since the clause gives the name that meaning in a Separation space; (71) a lookup table in an indirect string reads: it was read as a stream first, which uses up a reference (ISO 32000-2 §8.6.6.3); (72) an uncoloured pattern's underlying space given as a name or a reference is used: it was skipped, so a CMYK cyan drew red (ISO 32000-2 §8.7.3.3); (73) `InterpreterSettings::jbig2_work`, set for each page in `interpret_page` and ended with it, and a JBIG2 image or an image with a JBIG2 mask past it left out with `ContentTruncated`; so is an image whose JBIG2 soft mask or `/Mask` one of the decoder's limits refuses, or PdfCraft's bound on the pixels of an image (`image_size_ok`), where a mask the file damaged still leaves its image whole (#6722), and an image drawn on its own, a stencil mask included, that a limit refuses is still a failed image (`ImageXObject::masks_picture`): `decode_raster` leaves the image out when its mask reported `ContentTruncated`, decoded or kept by the cache (91), and a size or count the file damaged can read as past a limit and is taken for one, the safe side; (74) colour spaces nest at most 32 levels, through every way one names another, so a chain of them ends instead of overflowing the stack; an ICC profile's `/Alternate` is read where it's met and isn't kept with the profile, or a page could build a chain past the limit on spaces already read; (75) a soft hyphen (U+00AD) that a CFF substitute for a CID font has no glyph for draws its hyphen, as Annex D.2 reads WinAnsi 0xAD in (30), instead of the CID taken as a glyph index (#5888); (76) a CID of an Arial the PDF doesn't embed (`CIDFontType2`, not of an Adobe character collection; `ArialMT`, `Arial-BoldMT`, `Arial,Bold` and the other two faces, not Arial Narrow, Black or Unicode MS) that no Unicode route draws names, through its `/CIDToGIDMap`, the Arial glyph of the standard Macintosh order (read-fonts' `post::DEFAULT_GLYPH_NAMES`, its first 258 glyphs), and a CFF substitute draws its glyph of that name (.notdef when it has none) instead of the CID taken as its own glyph index (#5889); (77) a standard 14 font drawn with its bundled stand-in (`StandardFontBlob::Cff`) reads each glyph's own advance from its charstring (`CffFontBlob::advance_width`), so a stand-in glyph wider or narrower than the standard metrics is fitted to them as the OpenType stand-ins already were (ISO 32000-2 §9.6.2.2): with no advance the outline was drawn as it is, and the stand-ins' € (WinAnsiEncoding code 128, Annex D Table D.2), 667 to 722 units wide against the metrics' 556 (Helvetica) and 500 (Times), overlapped the next glyph. Symbol and ZapfDingbats (Annex D.5, D.6) are fitted the same way, the check mark `4` a check box draws for its `/MK /CA` caption among them; (78) `TilingPattern::instance` numbers each setting of a tiling pattern (`scn`/`SCN`), its copies keeping the number, and `TilingPattern::is_coloured` reports `/PaintType 1`, so a renderer can draw a coloured cell once for every fill of one setting (hayro patch (30)); (79) text extraction from a symbolic embedded TrueType font with no `/Encoding` (and no `/ToUnicode`) reads a code through the `post` table name of the glyph its `cmap` selects (`EmbeddedKind::symbolic_text`), when that name has a Unicode value by the glyph-name rules of `glyph_name_text`, which are the Adobe Glyph List Specification's (ISO 32000-2 §9.10.2) and the editor's in pdfcraft-fonts (a `.suffix` is dropped, `_` separates the characters of a ligature, `uni` with one or more groups of exactly four uppercase hex digits and `u` with four to six are code points; lowercase digits, other lengths and surrogate or out-of-range values name nothing, so `uni20ac` reads as WinAnsi), instead of reading the code as WinAnsi: FrameMaker files distilled with Distiller 3 number their codes in the Macintosh glyph order, 29 below ASCII, so `Solid` was extracted, edited and exported to Word as `6ROLG` (#3962). Codes with no glyph, or whose glyph has no such name, still read as WinAnsi; glyph selection is unchanged; (80) an `/Indexed` colour space that breaks ISO 32000-2 §8.6.6.3 is drawn as most readers draw it: a lookup table shorter than m × (hival + 1) bytes keeps the entries it holds in full and an index past them is drawn black after conversion, in any base space (zero bytes would be white in DeviceCMYK, Separation and DeviceN, and blue in Lab); a `hival` past 255 is taken as 255, and a real `hival` is truncated toward zero. An image drawn so reports `InterpreterWarning::MalformedIndexedImage`, surfaced as `RenderWarning::MalformedIndexedImage`, and a fill or stroke colour in such a space reports `ColorFallback`. An image whose lookup is missing, not a string or a stream, or a stream that does not decode (or decodes to nothing although it holds data) is not drawn, with the same warning; a colour set in such a space keeps the upstream DeviceGray fallback. Upstream treated all of these as an unreadable space and fell back to DeviceGray, painting the indices near-black, even ones whose entries were present. A `hival` below zero or not a number has no reading: the space falls back as upstream, and an image in it reports the warning. A lookup string given by an indirect reference is read (upstream consumed the reference trying it as a stream and found no table). The warning is taken from the space as built, so no table is read twice: inline images naming the space through a resource read its table once however many there are (`InterpreterCache::indexed_lookup_reads`); (82) a soft mask (or `/Mask` image) larger than its image and more than twice as large on a side as the device pixels its image is drawn at is averaged down to that size, rounded up to a power of two, in `ImageXObject::decoded_soft_mask`, and cached there (`DecodedImageKey::shrunk_to`, with the image's raster keyed by the same rounded size); every size is made from one cached intermediate mask at most 4096 on a side, so the full mask is decoded once however many sizes it is drawn at: it bypassed the decoded-image cache and was decoded, copied and staged at full size each time it was drawn. A 2 × 2 image with a 34,862 × 4,332 mask drawn 40 times on one slide (pdf.js `issue16263.pdf`) took about 3.6 s a draw and minutes a page; it now decodes the mask once and draws the page in about 2.5 s. A mask no larger than twice its drawn size keeps its resolution, which the renderer filters as before; `InterpreterCache::decoded_mask_decodes` counts the masks among the decodes; (83) an embedded TrueType program whose `head.indexToLocFormat` holds neither 0 nor 1 is read with the `loca` format its table's length gives (`with_loca_format_fixed`): the glyph reader took anything but 1 as short offsets, so a font marked 256 had its long offsets misread and drew boxes (pdf.js `issue2537r.pdf`); (84) a Type 0 font of an Adobe character collection (Japan1, GB1, CNS1, Korea1) reads a code its `/ToUnicode` has nothing for, or every code when it has none, through the collection's UCS2 CMap, code to CID to Unicode, embedded or not (ISO 32000-2 §9.10.2; `Type0Font::ucs2`, parsed the first time a code needs it): only fonts that are not embedded took that route, so Japanese and Chinese pages whose fonts are embedded without a `/ToUnicode`, or with one that maps nothing, had no text to select, find or copy; (85) a Type 0 font's `/ToUnicode` written as a CID CMap (`begincidchar` / `begincidrange`, destinations as numbers, as Word writes it) gives each number as the code's Unicode value for text, as glyph lookup (`map_via_unicode`) already read it and as pdftotext and PDFium extract it; ISO 32000-2 §9.10.3 has `beginbfchar` / `beginbfrange` there, and such text extracted nothing (pdf.js `issue9367`); (86) form fields' widgets are drawn after the page's other annotations, each group in `/Annots` order (`interpret_page`), as MuPDF and PDFium draw them; ISO 32000-2 doesn't order the two: a check box listed before a filled square comment that covers it was hidden under the comment (pdf.js `issue13003.pdf`); (87) a Type 3 font keeps each glyph procedure it decoded (`Type3::decoded_proc`), so a glyph shown again isn't inflated again: a font used as a fill pattern drew one glyph about 650,000 times on a page and decoded its procedure for each; a procedure over 64 KiB isn't kept, and the fonts of one `InterpreterCache` keep at most 32 MiB together (`MAX_CACHED_TYPE3_PROCS`), each kept procedure counting its bytes, its name and 64 bytes besides, as fonts live as long as the cache; a procedure holds its bytes within the page's content budget while the glyph is painted, kept or not (`context::hold_content`), so a page skips the same glyphs once the budget is spent, and one decoded with no more than 64 KiB of budget left, which the budget may have cut short, isn't kept; `InterpreterCache::type3_proc_decodes` counts the procedures decoded; (88) the rule of (76) also reads Times New Roman (`TimesNewRomanPSMT`, `TimesNewRomanPS-BoldMT`, `TimesNewRoman,Bold` and the other faces, not Times or Times New Roman MT Std), whose first 258 glyphs follow that order too: in a public form whose embedded Times New Roman subsets keep their glyph indexes, 941 of 942 `/ToUnicode` entries below 258 name that order's character, and its non-embedded `TimesNewRoman,Bold` drew "Salarié(e) du régime général" as "Salari²*g+f w't²i lo g'ì²p²tcn" (#6449); (89) a form XObject already being drawn is not drawn inside itself (`Context::enter_form`, reported as `ContentTruncated`, §8.10.1): a name in its content that reached it again, through its own resources or the page resources a form with its own still falls back on, drew it until the page's nested-paint budget was spent (PDFBOX-5782, 38 s); the forms being drawn are kept per page and thread, so the tiling patterns, Type 3 glyphs, soft masks and groups painted inside a form, which are interpreted in contexts of their own, see it too; (90) a Type 3 font without `/Widths` (or `/FirstChar`/`/LastChar`) advances each glyph by the width its glyph procedure sets with `d0` or `d1` (`Type3::proc_width`, ISO 32000-2 9.6.4) instead of 0, read once per code; a font with `/Widths` is unchanged, a code outside `/FirstChar`..`/LastChar` keeping width 0 (Table 110) (PDFBOX-2943: a misspelt `/Widths` key drew every glyph on one spot), and selecting such a font reports `InterpreterWarning::Type3WidthsMissing` (`Font::type3_widths_missing`, checked in `Context::resolve_font`); (91) the decoded-image cache keeps a failed decode for its key, as the warnings it reported, so a picture or stencil mask that doesn't decode (damaged data, a JPX or JBIG2 stream its decoder refuses) isn't decoded again at every draw, tile and render that shares the cache, and each draw still reports what the decode did (`DecodedImageCache::get_or_insert_with`, `ImageXObject::reporting_to`); a failure counts 128 bytes towards the cache's budget, and one the page's JBIG2 allowance may have caused (73) isn't kept (the hayro-syntax comment on `charge_page_reading` says so), nor one a JBIG2 mask's decoding work refused, while a JBIG2 mask refused by a limit its stream reaches by itself is kept, with the image it takes out (`note_stream_limited`), so a page drawing that image a hundred times decodes it once; (92) the text of a glyph whose name is not in the glyph list reads `uniXXXX` and `uXXXX` to `uXXXXXX` by the Adobe Glyph List Specification (`agl_unicode_from_name`: `uni` and exactly four uppercase hex digits, `u` and four to six, no surrogate, nothing past U+10FFFF), so `uni65`, `u65`, `uni20ac` and `uni+065` read as nothing instead of `e` and `€`; upstream's lenient `unicode_from_name` stays for finding a glyph in a font (TrueType `cmap` lookup, accented standard-font letters), so drawing is unchanged; (93) a CID-keyed CFF font keeps the font dicts it can read: a font dict without a Private DICT (one a subsetter emptied) leaves only the glyphs FDSelect gives it missing, where one such dict dropped the whole font and its text drew with a fallback (ISO 32000-2 §9.9 Table 124, Adobe TN #5176); (94) a font dictionary whose `/Subtype` is missing or the empty name (`/Subtype /`) is read as Type 1, so its `/BaseFont` picks the face (ISO 32000-2 Table 109 requires `/Subtype`; other readers fall back this way): a `/Courier-Bold` was drawn in the Helvetica fallback (PDFBOX-1988). Any other unknown subtype is still refused; (98) a non-embedded font named `SymbolMT`, or `Symbol`/`SymbolMT` with a `,` or `-` style suffix (`Symbol,Bold`, `Symbol-Bold`; `SYMBOL_STYLES`, so `Symbol-Helvetica` stays another font), is the standard Symbol font and draws through its built-in encoding, so a TrueType `SymbolMT` declaring `/WinAnsiEncoding` draws `mu` for code 0x6D, not `m` (ISO 32000-2 §9.6.2.2); (99) the mesh sampler walks each row of a triangle only across the columns where that row's sample points can lie in it, not its whole box (a row with none still costs a visit): thin triangles of the fixed 19 × 19 patch grid lying across the pixel grid (a narrow curved band of tensor patches) have boxes many times their area, and a mesh painted through a path the size of its own box spent the visit budget before its last patches and left them unpainted; (100) a Type 1 or Type 3 font glyph name that only numbers its glyph (`C39`, `MT72`, `G4E`: one to three letters, then decimal or hexadecimal digits) and names the very code it is shown with reads as that code's Latin-1 character (controls excepted; in a Type 3 font, which already reads codes 32-126 by the code, above 126), in a font where every such name names its own code (`numbered_names_name_their_codes`, worked out once per font); a font with any numbered name at another code (`/C1` at 233, review on #3239; `/Differences [1 /C70 …]`) or at a C0 control code (a subset counting its glyphs from code 1, `/Differences [1 /C1 /C2 …]`, where `/C65` is the 65th glyph used) keeps reading them as nothing (#3170); (101) a symbolic TrueType font with no encoding reads a code as Mac OS Roman in text extraction when its Macintosh Roman (1, 0) `cmap` subtable selects the glyph the code draws (ISO 32000-2 §9.6.5.4) and that glyph sits where the standard Macintosh glyph order (`post` version 1.0) puts the code's Mac OS Roman character, for a font whose `post` table names no glyphs, that names no base encoding (an unknown name included) and that isn't Symbol or ZapfDingbats (§9.6.6.1), after `/Differences` and before the WinAnsi last resort; Windows subsets in the same order keep their WinAnsi reading, and glyph selection is unchanged; (102) `OutlineGlyph::vertical_glyph_width`: a vertical-writing Type 0 glyph's width w0 from `/W` or `/DW`, its extent across the column (ISO 32000-2 §9.7.4.3), which `advance_width` (the horizontal advance, 0 in vertical writing) does not give; (81) a reply of type R (a markup annotation with an `/IRT` and an `/RT` other than `/Group`; R is the default, so a missing, unknown or malformed `/RT` reads as R; a widget, link or pop-up with `/IRT` still draws) is not drawn, whatever its appearance or flags: ISO 32000-2 Table 172 says such replies "shall not" be displayed individually but in their comment's thread (files from editors that give every reply a note-icon appearance showed a stack of icons, #5631); a `/RT /Group` member still draws; (103) `InterpreterSettings::view_auto_state` (on by default): the default configuration's auto state for the View event (`/AS` usage applications with `/Event /View` and `/View` in `/Category`) sets each managed group from its `/Usage /View /ViewState` before `ocg_overrides`, ON only if every recommendation is ON (ISO 32000-2 §8.11.4.4); `/Zoom`, `/User` and `/Language` give no recommendation; at most 256 applications, 4096 `/OCGs` entries and 16 categories each are read; off for print renders | (1) `raster::tests::widget_appearance_states`; (2, 3) `xtask text-oracle` on corpus/pdfjs issue918; (4) `raster::tests::layer_override_hides_content`; (5) `raster::tests::soft_mask_without_group_colour_space_is_applied`; (6) `raster::tests::self_referencing_tiling_pattern_terminates`, `raster::tests::self_referencing_type3_glyph_terminates`; (7) `raster::tests::huge_cid_width_ranges_terminate`; (8) `raster::tests::hiding_comments_keeps_fields`; (9) `raster::tests::absurd_mask_and_fax_sizes_are_refused_before_decoding`; (10) `raster::tests::type3_glyphs_that_fan_out_into_themselves_finish`; (11) `raster::tests::type1_font_programs_with_long_runs_of_numbers_load_quickly`; (12) `raster::tests::degenerate_annotation_appearances_are_skipped`; (13) `raster::tests::highlights_without_an_appearance_are_drawn`; (14) `raster::tests::malformed_simple_font_widths_keep_later_widths_and_the_font`, `raster::tests::huge_simple_font_first_char_terminates`; (15) `raster::tests::annotation_appearance_box_maps_onto_rect`; (16) `raster::tests::clipping_path_applies_after_any_painting_operator`; (17) `raster::tests::dropping_the_pool_cancels_its_renders`, `raster::tests::the_watchdog_stops_the_render_it_gives_up_on`; (18) `raster::tests::page_contents_and_nested_forms_share_one_budget`, `raster::tests::content_painted_again_and_again_keeps_drawing`, `raster::tests::sampled_functions_with_billions_of_samples_are_refused`; (19) `raster::tests::mesh_shadings_far_off_the_page_render`, `raster::tests::many_overlapping_mesh_triangles_finish`, `raster::tests::mesh_shadings_with_many_patches_or_no_rows_finish`; (20) `raster::tests::function_limits_calculator_arithmetic_and_depth`, `raster::tests::function_limits_public_construction_work_and_depth`, `raster::tests::function_limits_stitching_cycles_are_refused_and_shared_children_work`, `raster::tests::function_limits_invalid_calculator_keeps_the_page_renderable`; (21) `raster::tests::cids_a_substitute_lacks_are_not_drawn_as_other_glyphs`; (22) `raster::tests::variation_sequences_draw_their_kanji_in_a_substitute`; (23) `raster::tests::axial_and_radial_shadings_draw_as_native_gradients`; (24) `raster::tests::tiles_match_full_render`, `raster::tests::skipping_paths_off_the_canvas_changes_no_pixel` (both decode through the cache), `raster::tests::decoded_images_with_identical_dictionaries_keep_their_own_pixels`, `raster::tests::decoded_images_under_a_transfer_function_are_not_shared`; (25) `text::tests::actual_text_replaces_the_glyphs_it_marks_in_every_writing_direction`, `text::tests::actual_text_follows_the_pdf_rules_for_marked_content`; (26) `raster::tests::accented_letters_the_standard_stand_ins_lack_are_built_from_their_parts`, `raster::tests::comma_below_letters_in_standard_fonts_draw_a_comma_under_the_letter`; (27) `text::tests::truetype_without_an_encoding_reads_its_codes_as_win_ansi`; (28) `raster::tests::spot_colour_images_run_their_tint_transform_once_per_sample_value`, `raster::tests::images_converted_through_a_table_keep_every_samples_colour`, `raster::tests::a_malformed_tint_for_an_unused_sample_keeps_the_used_colours`; (29) `raster::tests::reading_colors_keep_highlight_annotations_readable`; (30) `text::tests::the_annex_d2_alias_codes_read_as_an_ordinary_hyphen_and_space`; (31) `raster::tests::shadings_with_large_sampled_functions_paint_quickly`, `raster::tests::sampled_functions_interpolate_their_grid_as_before`; (32) `raster::tests::decoded_images_serve_every_size_they_are_drawn_at`; (33) `raster::tests::patch_meshes_keep_their_corner_colours`; (34) `raster::tests::spot_colour_images_run_their_tint_transform_once_per_sample_value`; (35) `mesh_extent_tests::{a_mesh_paints_its_triangles_alone, a_mesh_keeps_the_graphics_state_alpha, a_mesh_off_the_canvas_paints_nothing}` (`cargo test -p hayro`); (36) `raster::tests::mesh_triangles_sharing_an_edge_leave_no_seam`; (38) `raster::tests::default_colour_spaces::device_rgb_operators_draw_in_default_rgb`, `raster::tests::default_colour_spaces::device_cmyk_operators_draw_in_default_cmyk`, `raster::tests::default_colour_spaces::device_gray_operators_draw_in_default_gray`, `raster::tests::default_colour_spaces::device_images_draw_in_the_default_colour_spaces`, `raster::tests::default_colour_spaces::device_shadings_draw_in_the_default_colour_spaces`, `raster::tests::default_colour_spaces::an_image_drawn_with_and_without_a_default_keeps_both_colours`, `raster::tests::default_colour_spaces::forms_inherit_the_default_colour_spaces_they_do_not_override`, `raster::tests::default_colour_spaces::unusable_defaults_leave_device_colour_alone`; (39) `raster::tests::knockout_groups_replace_what_their_elements_cover`; (40) `raster::tests::visibility_expressions_decide_what_a_membership_dictionary_shows`; (41) `raster::tests::a_multiply_highlight_inside_a_transparency_group_keeps_the_text`; (42) `raster::tests::rendering_modes_but_3_and_7_draw_type3_glyphs_as_filled`; (43) `raster::tests::lab_honours_the_space_white_point`, `raster::tests::indexed_palettes_over_lab_keep_their_colours`; (44) `text::tests::identity_to_unicode_names_give_text`; (46) `raster::tests::stencil_masks_are_drawn_at_the_fill_opacity_once`; (47) `raster::tests::a_five_point_polyline_is_not_painted_as_its_bounding_box`, `raster::tests::a_path_that_visits_every_corner_out_of_order_is_not_a_rectangle`, `raster::tests::a_rectangle_is_still_painted_as_a_rectangle`; (48) `raster::tests::icc_profile_cache::icc_profiles_with_identical_dictionaries_draw_their_own_colours`, `raster::tests::icc_profile_cache::icc_profiles_used_again_are_read_once`; (49) `raster::tests::a_font_whose_post_version_has_swapped_halves_is_still_used`; (50) `font::blob::tests::glyph_names_resolve_to_the_first_glyph_with_that_name`; (51) `context::saved_state_tests::repeated_saves_stay_bounded_and_balanced`; (52) `font::cid::tests::w2_run_past_the_largest_cid_keeps_the_rest_of_the_table` (hayro-interpret); (55) `raster::tests::devicen_inks::a_tint_transform_that_cannot_be_evaluated_draws_the_rest_of_the_page_and_warns`, `raster::tests::devicen_inks::a_tint_transform_that_cannot_be_read_falls_back_and_warns`; (56) `raster::tests::invisible_unknown_annotations_and_popups_paint_nothing`; (57) `substitute_widths::a_substitute_keeps_its_glyph_shapes_at_wider_advances`, `substitute_widths::a_substitute_still_narrows_glyphs_to_narrower_advances` (pdfcraft-render `tests/it`); (58) `cid_fallback_query::a_cid_font_fallback_request_carries_its_descendants_style`, `cid_fallback_query::a_cid_font_without_a_descriptor_asks_for_the_default_style` (pdfcraft-render integration tests); (59) `raster::tests::no_rotate_notes_stay_upright_at_their_corner_on_rotated_pages` (hayro-interpret); (61) `stroke_geometry::{degenerate_subpaths_draw_a_round_dot_or_nothing, degenerate_subpaths_leave_the_rest_of_the_path_alone, a_segment_after_h_begins_a_new_subpath, all_zero_dash_arrays_draw_a_solid_line, invalid_dash_arrays_reach_a_device_as_solid, valid_dash_patterns_still_dash, negative_line_widths_reach_a_device_as_zero, negative_line_widths_draw_the_thinnest_line_in_the_stroke_colour}` (pdfcraft-render integration tests); (63) `color::device_cmyk_gray_tests::{each_ink_darkens_by_its_iso_weight, ink_past_full_coverage_is_black_and_bad_components_read_as_zero}` (hayro-interpret), `it::luminosity_group_space::{black_ink_in_a_device_cmyk_group_masks_fully_and_no_ink_shows_fully, each_process_ink_in_a_device_cmyk_group_masks_by_its_weight, a_device_cmyk_backdrop_colour_masks_by_its_gray, gray_and_rgb_groups_mask_as_before, device_cmyk_on_the_page_keeps_its_display_colour}` (pdfcraft-render); (64) `type3_text_clip::{clip_only_type3_text_is_not_rendered_and_leaves_no_clip, invisible_type3_text_is_not_rendered, other_modes_render_type3_glyphs_filled_and_never_clip, only_outline_glyphs_shape_a_clip_shared_with_type3_text, self_nesting_type3_text_in_a_clip_mode_stays_bounded}` (`cargo test -p pdfcraft-render --test it`), `raster::tests::rendering_modes_but_3_and_7_draw_type3_glyphs_as_filled`, `text::tests::clipping_mode_type3_text_is_kept_by_the_ocr_filter`; (65) `it::zero_area_fill_clip` (`crates/render/tests/it/zero_area_fill_clip.rs`); (66) `separation_all::{separation_all_fills_paint_the_complemented_tint_as_gray, separation_all_strokes_paint_the_complemented_tint_as_gray, separation_all_images_paint_each_sample_as_the_complemented_gray, separation_all_shadings_paint_the_complemented_gray, separation_none_paints_nothing, a_named_separation_still_paints_through_its_alternate, a_device_n_space_naming_all_still_paints_through_its_alternate}` (`cargo test -p pdfcraft-render --test it`); (71) `raster::tests::an_indexed_lookup_table_is_read_in_every_legal_form` (pdfcraft-render); (72) `raster::tests::an_uncoloured_pattern_uses_its_underlying_space_in_every_form` (pdfcraft-render); (73) `raster::tests::the_jbig2_images_of_a_page_share_one_allowance`, `a_picture_whose_jbig2_mask_is_past_the_pages_allowance_is_not_drawn`, `a_picture_whose_jbig2_mask_a_decoder_limit_refuses_is_not_drawn`, `a_jbig2_image_drawn_on_its_own_that_a_limit_refuses_is_a_failed_image`, `a_limit_refusal_says_nothing_of_the_next_image`, `a_jpx_picture_whose_jbig2_mask_a_limit_refuses_stays_out_at_every_size` and `a_pages_jbig2_allowance_ends_with_the_page` (pdfcraft-render); (74) `raster::tests::{a_chain_of_colour_spaces_ends, colour_spaces_are_read_to_the_limit_of_their_nesting, a_chain_built_on_spaces_already_read_ends, a_profile_read_at_the_limit_keeps_its_alternate_for_a_read_nearer_the_top}` (pdfcraft-render); (75) `raster::tests::a_soft_hyphen_a_substitute_lacks_draws_its_hyphen` (pdfcraft-render); (76) `raster::tests::non_embedded_arial::{an_arial_the_pdf_does_not_embed_draws_the_glyphs_its_codes_name, codes_of_other_fonts_are_not_read_in_the_macintosh_order}` (pdfcraft-render); (77) `euro_width::the_euro_fits_its_standard_advance`, `euro_width::two_euros_stay_apart`, `euro_width::a_slanted_euro_fits_its_standard_advance`, `symbol_dingbats_width::{symbol_glyphs_keep_their_standard_advances, zapf_dingbats_glyphs_keep_their_standard_advances}` (pdfcraft-render `tests/it`), `dingbat_check_box::a_check_box_caption_check_is_drawn_at_its_standard_advance` (pdfcraft-engine `tests/it`); (78) `raster::tests::{fills_of_one_pattern_setting_share_its_cell, fills_of_interleaved_pattern_settings_each_show_their_own_cell, an_uncoloured_pattern_shows_each_settings_colour}` (pdfcraft-render); (79) `raster::tests::a_symbolic_truetype_font_without_an_encoding_reads_its_glyph_names`, `pdf::tests::a_symbolic_truetype_font_without_an_encoding_reads_its_glyph_names` (pdfcraft-fonts, the editor's reading by the same rules) ; (80) `crates/render/tests/it/indexed_lookup.rs` (`cargo test -p pdfcraft-render --test it indexed_lookup`), `raster::tests::indexed_lookup_cache`; (82) `raster::tests::a_soft_mask_far_larger_than_where_it_is_drawn_is_shrunk_once`, `raster::tests::an_averaged_down_soft_mask_follows_its_samples_and_its_drawn_size`, `raster::tests::decoded_images_serve_every_size_they_are_drawn_at` (pdfcraft-render); (83) `raster::tests::truetype_loca_format::a_truetype_font_with_an_out_of_range_loca_format_draws_its_glyphs` (pdfcraft-render); (84) `text::tests::an_embedded_adobe_collection_font_reads_through_its_ucs2_cmap`, `every_adobe_collection_reads_through_its_own_ucs2_cmap` (GB1, CNS1, Korea1, Identity-V, `/H` over CFF) and `a_font_outside_the_adobe_collections_gets_no_text_from_a_ucs2_cmap` (pdfcraft-render); (85) `text::tests::{a_tounicode_written_as_cid_ranges_gives_text, a_tounicode_bf_entry_wins_over_a_cid_entry_for_the_same_code}` (pdfcraft-render); (86) `widgets_above_annotations::a_widget_is_drawn_above_the_comments_on_its_page` (pdfcraft-render `tests/it`); (87) `raster::tests::type3_proc_cache::{a_type3_glyph_shown_again_decodes_its_procedure_once, a_procedure_over_64_kib_is_decoded_for_each_copy, the_procedures_one_cache_keeps_stop_at_32_mib, fonts_with_the_same_glyph_name_keep_their_own_procedures}` (pdfcraft-render); (88) `raster::tests::non_embedded_arial::{a_times_new_roman_the_pdf_does_not_embed_draws_the_glyphs_its_codes_name, codes_of_other_fonts_are_not_read_in_the_macintosh_order}` (pdfcraft-render); (89) `raster::tests::a_form_that_draws_itself_is_drawn_once`, `raster::tests::a_form_that_reaches_itself_through_a_pattern_or_a_glyph_is_drawn_once` (pdfcraft-render); (90) `raster::tests::type3_proc_widths::{a_type3_font_without_widths_advances_by_its_glyph_procedures, a_code_outside_first_char_to_last_char_still_has_width_zero, widths_the_font_gives_are_kept, a_code_without_a_usable_procedure_width_still_advances_nothing, a_type3_font_without_widths_is_reported}` (pdfcraft-render); (91) `raster::tests::failed_image_decodes::{a_picture_that_does_not_decode_is_decoded_once_however_often_it_is_drawn, a_picture_that_does_not_decode_is_decoded_once_at_each_size, a_stencil_mask_that_does_not_decode_is_decoded_once_however_often_it_is_drawn, a_picture_whose_soft_mask_does_not_decode_draws_unmasked_decoded_once, a_picture_that_decodes_is_still_decoded_once_and_drawn_every_time, a_picture_left_out_for_the_pages_allowance_draws_on_the_next_page, a_picture_whose_jbig2_mask_a_limit_refuses_is_decoded_once_however_often_it_is_drawn, a_picture_whose_jbig2_mask_is_refused_for_its_work_is_decoded_at_every_draw}` (pdfcraft-render); (92) `text::tests::agl_unicode_from_name_rule_reads_text` (pdfcraft-render); (93) `cid_cff_unusable_font_dict::{glyphs_of_a_usable_font_dict_draw_beside_an_unusable_one, a_program_with_no_usable_font_dict_draws_no_square}` (pdfcraft-render integration tests); (94) `raster::tests::a_font_without_a_subtype_is_drawn_by_its_base_font` (pdfcraft-render); (98) `symbol_mt_builtin_encoding::{symbol_mt_draws_symbols_through_the_built_in_encoding, a_name_that_only_starts_with_symbol_is_not_symbol, symbol_mt_applies_its_differences, an_embedded_symbol_mt_draws_its_own_program}` (pdfcraft-render integration tests); (99) `raster::tests::thin_patches_of_a_mesh_in_a_tight_path_are_all_painted` (pdfcraft-render); (100) `text::tests::numbered_glyph_names_read_as_their_own_code`, `text::tests::numbered_type3_glyph_names_read_as_their_own_code`, `text::tests::glyph_names_outside_the_glyph_list_still_read_as_text` (pdfcraft-render); (101) `raster::tests::symbolic_truetype_mac_roman_text::{a_symbolic_truetype_font_in_the_macintosh_order_reads_its_codes_as_mac_roman, winansi_codes_of_a_font_in_the_macintosh_order_read_as_before, a_symbolic_truetype_font_without_a_1_0_subtable_reads_as_before, a_1_0_subtable_selecting_other_glyphs_than_the_drawn_ones_is_not_read, a_font_that_names_its_glyphs_is_not_read_by_their_order, a_symbol_or_zapfdingbats_font_in_the_macintosh_order_gets_no_mac_roman_reading, differences_naming_notdef_clear_the_mac_roman_reading_for_its_code, a_base_encoding_name_this_reader_does_not_know_keeps_the_mac_roman_reading_off}` (pdfcraft-render), `pdf::tests::a_symbolic_truetype_font_in_the_macintosh_order_reads_its_codes_as_mac_roman` (pdfcraft-fonts); (102) `text::tests::tracked_vertical_latin_stays_one_line` (pdfcraft-render); (81) `raster::tests::replies_are_not_drawn_but_their_comment_and_group_members_are`, `raster::tests::an_unknown_or_malformed_rt_is_a_reply_and_a_widget_with_irt_still_draws` (hayro-interpret); (103) `raster::tests::the_view_auto_state_sets_a_groups_state_on_screen`, `raster::tests::layer_toggles_and_print_renders_are_not_the_view_auto_state` and `raster::tests::malformed_view_auto_state_is_ignored` (pdfcraft-render) | | hayro | 0.7.1 | Apache-2.0 OR MIT | (1) `RenderSettings::{x_offset, y_offset}` to render one tile of a large page; (2) images over `MAX_IMAGE_PIXELS` (2^28) are skipped (a fuzzed `/W 4294967295` hung in resampling); (3) `tiling_cell_scale` keeps a tiling pattern's cell pixmap within 3000 px a side whatever its `/XStep`/`/YStep` (a fuzzed step far beyond `/BBox` allocated 2 GB); (4) stroke widths clamped to 4 × (canvas width + height) device pixels (a fuzzed `/LW 9223372036854775807` allocated 10 GB in stroke expansion); (5) `usable_dash_pattern`: a stroke whose dash pattern could cut it into more than `MAX_DASHES_PER_STROKE` (250,000) pieces, counting kurbo's restart of the pattern at every subpath and `ClosePath`, is drawn solid (a fuzzed `/D [[11] 0]` on a line from x = 92234775807 allocated over 5 GB, `[0.000001] 0 d` on a 20 pt line over 10 GB); (6) a dash pattern with a negative or non-finite entry, or a period that isn't positive, is drawn solid (`[-1 -1] 0 d` looped for ever in kurbo); (7) `render_size` and `render_into`: render into a buffer the caller allocates (PdfCraft renders into pixel-aligned memory that the GUI takes over as texture data); `render` is `render_into` on a new pixmap; (8) checked image source/target/RGBA, alpha-premultiplication scratch and intermediate sizes before resampling, finite transforms and exact source/mask buffers; sources wider than u16 may shrink before backend limits, with a 2^20 source-axis cap and conservative 128 MiB filter-planning estimate; an over-limit or failed resize keeps the original image resolution and geometry when the source fits the backend, otherwise skips it, with a logged warning; staging, resampling output/scratch, RGBA conversion and padding buffers use fallible reserves; (9) `may_paint_canvas`: a fill or stroke whose control-point box, grown by how far its paint reaches past the path (half the stroke width times the larger of the miter limit and √2) plus 0.5 user units and 2 pixels, misses the canvas is not drawn, so a tile no longer strokes and fills every path of its page (item 5 of #623); `set_skip_offscreen_paths` turns this off for the regression test; (10) `/Interpolate` governs the pre-resize too (#624): a minified image that does not request interpolation is pre-resized with nearest-neighbour (since (21), only inside Type 3 glyphs), planned without the Catmull-Rom filter limits (`image_resampling_size` takes the flag); a Type 3 glyph image is sampled bicubically only when interpolated; a minified image with a matching soft mask is nearest-sampled straight into premultiplied RGBA; (11) `may_paint_rect`: an image whose stream-size box, mapped to the device, misses the canvas is skipped before decoding (decoded data can only shrink against that box), so a tile of an image-heavy page no longer decodes, resamples and rasterizes every image on it; (12) preview renders (`InterpreterSettings::preview`: backdrops, thumbnails, grid cells, signature-drag backgrounds) skip images under `PREVIEW_MIN_IMAGE_AREA` device pixels, while final renders always draw them; (13) reading colours (#897): `RenderSettings::color_replacement` takes a `ColorReplacement` (text colour, background, `only_black`, `line_art`) that remaps solid text and line-art paint while rendering, glyph outlines counting as text and stencil-mask ink as line art; images, image alpha masks, soft masks and pattern cells keep their colours; `render_pdf` gains a `color_replacement: Option` argument (pass `None` for true colours); the upstream-only `[dev-dependencies.log]` entry is removed from `Cargo.toml` (log is a normal dependency since the logging patch), so `cargo test -p hayro` runs this crate's unit tests; (14) a shrunk image that is interpolated (or, since (21), area-averaged) and whose axes lie along the device's (upright, mirrored or a quarter turn, as on a page under `/Rotate`) is snapped to whole device pixels: shrunk once to its rounded device rectangle (halves round up, so a tile snaps an edge where the whole page does) and drawn 1:1 from the rounded corner, instead of a ceil-sized raster resampled again at a fractional scale and offset (`snap_minified_image`; scans looked soft next to Acrobat, and until #5530 a turned one still did, up to 200 grey levels off the same page drawn upright). An image counts as axis-aligned when the terms the snap drops move its far edges by at most a hundredth of a device pixel (`SNAP_SKEW`), not by a tolerance on the coefficients, which let a slight skew of a very wide image through; and the snapped 1:1 transform is used only when the shrink produced the snapped size, so a failed resize keeps the full transform instead of drawing the source raster at its source size; (15) a soft mask of another size than its image is filtered when shrunk, whatever its `/Interpolate` (#1649): such a mask carries the shape on its own (Word saves text as a tiny black image with a glyph-bitmap mask), and nearest-neighbour dropped every stroke thinner than the shrink factor; (16) under a soft mask or a blend mode, axial and radial shadings are sampled instead of drawn as native gradients (#2128): vello_cpu 0.0.8 copies an opaque gradient over fully covered tiles without the mask or blend mode, so PowerPoint's faded gradient fills came out solid; (17) the renderer keeps annotation paints out of the reading-colours mapping between `begin_annotations` and `end_annotations` (#1548); (18) the page renders as one transparency group; under reading colours its background is painted inside it as the initial backdrop, so a Multiply highlight blends with the dark background instead of staying an opaque block, and otherwise the background stays outside, so a blend mode over an empty part of the page keeps its colour instead of blending with the white paper (#2970); (19) `set_paint` samples a mesh shading's texture over its triangles alone (`EncodedShadingPattern::extent`, padded rather than repeated) and skips a mesh with none on the canvas: a BricsCAD plan's 310 one-triangle `sh` each covered the whole tile, 2 s per 1024 px tile at any zoom (the watchdog gave the page up on a busy machine), now about 70 ms; (20) a tiling pattern whose step is too large for a cell pixmap, where what is painted (a fill, or a stroke with its width) can only show the first cell (PowerPoint's `/XStep 2147480100` for a picture fill drawn once), has its `/BBox` drawn at full scale with a transparent border and padded instead of tiled (`shows_one_cell`; a cell whose scale isn't finite and positive, from a `/BBox` too thin to divide by, keeps the old path: `usable_cell_scale`); shrinking the cell to fit the 3000 px limit of (3) left the picture under a pixel, so network diagrams lost their pictures (#3349); (21) an image that does not request interpolation and is drawn below its resolution, outside a Type 3 glyph, is shrunk by area averaging (`area_shrink`: each device pixel the coverage-weighted mean of the samples it covers, in 16.16 fixed point whose weights sum to exactly one, premultiplied when the image has a mask; a soft mask no larger than the image is first sampled onto the image's grid by `mask_on_image_grid` (nearest-neighbour, or bilinearly when the mask asks for `/Interpolate`, so its edge stays as smooth as an enlarged mask's), so its colour and mask are averaged together instead of the averaged colour of transparent samples showing under a mask layer at the mask's edges) to its snapped device rectangle, instead of nearest-neighbour: `/Interpolate` asks for smoothing only when an image is enlarged (ISO 32000-2 §8.9.5.3), and nearest-neighbour kept one sample in N per axis, so the text of a 300 dpi scan at 100% lost most of its strokes (sampling them again at a fractional offset dropped more). PicScale's `Box` and `Area` filters were not used: shrinking, both average a window four target pixels wide. On a 13-page 300 dpi scan every rendered sample at 96, 120 and 192 dpi is within 1 of the exact area average (it was up to 255 off, 17-23% of samples by more than 1). A snapped image whose shrink failed keeps the full transform, so it is not drawn at its source size. The shrink never enlarges: a side drawn at or above its resolution (an image shrunk across and stretched down, as a 204 × 98 dpi fax is) keeps its samples for the sampler. Each target row averages the whole source rows it covers (contiguous, cheap) and is then shrunk across with taps worked out once per image, which made the shrink about three times faster (a 300 dpi page at 96 dpi: 25 ms, nearest-neighbour 11 ms); working a row at a time (`area_shrink_rows`) keeps its work memory to a few rows. An image with a soft mask, a stencil's ink included, is premultiplied a source row at a time as it is averaged (`area_shrink_premultiplied`), never as a full-resolution RGBA copy: a 10718 × 13701 stencil shown at page size peaked at 1.16 GB with that copy and peaks at 1.03 GB without it, as with nearest-neighbour. A source with a side over `MAX_RESAMPLING_SOURCE_SIDE` (2^20) keeps (10)'s nearest-neighbour shrink, and `area_shrink` refuses it, so its accumulator row and tap table stay a few times one source row (a 2^27 × 2 image of zeros is a tiny file); (22) knockout groups: while one is innermost, fills, strokes, glyph outlines and shadings are drawn with `Compose::Copy` (weighted by coverage), so each element replaces what earlier elements of the group drew instead of compositing over it; an image or nested group, drawn into a layer of its own, first has what is under its shape cleared (`knockout_image` for an unmasked image's rectangle, only when `draw_image` will draw it, decoded at the size it is drawn; a knockout erase pass that draws a nested group's content with every paint clearing instead of painting, a shading fill's clear clipped to the shading's `/BBox` as its paint is) and then composites over it (#2971); (23) backport of hayro 0.8: shadings that `as_svg_gradient` can represent are painted as a `vello_cpu` linear or two-point radial gradient instead of a texture sampled pixel by pixel through the shading function (InDesign fades are luminosity soft masks of one full-page axial `sh`: 40 on a report page took 15 s at 144 dpi, now under 1 s); without upstream's half-pixel shift, which suits vello_cpu 0.3 but moves 0.0.8's gradients off pixel centres; (25) a tiling cell is drawn at the scale that makes one `/XStep` (`/YStep`) exactly the cell pixmap's whole number of pixels (`tiling_cell_pixels`, at least 1 and at most the cap), so the repeated pixmap puts cells at whole multiples of the step at any device scale; drawing at the unrounded scale repeated the cell up to half a pixel off at every step, and the error grew with distance from the pattern's origin (ISO 32000-2 8.7.3.1); (28) an image of page-scan size (`SMOOTHED_IMAGE_SIDE`: 512 samples or more on its shorter side) that does not request interpolation is drawn with `ImageQuality::Medium` (bilinear, as interpolated images are) when enlarged by more than 1% on either side (`SMOOTHED_IMAGE_SCALE`: a scan drawn at its resolution can come out a few ulps above 1, and blurred by half a pixel at a half-pixel offset), also when the other side is shrunk (the area shrink leaves that side at 1:1), outside a Type 3 glyph: scans rarely set `/Interpolate`, and drawn nearest-neighbour every sample of a zoomed scan became a block, so its text had staircase edges; icons, QR codes and pixel art keep their hard pixels. Bicubic (`High`) looked the same on a zoomed 300 dpi scan and drew four 1024 px tiles at 400% in 267 ms against 50 ms; (30) a stencil mask painted with a pattern is drawn under a clip of its device rectangle, padded by two pixels: the pattern in an isolated layer, cut to the stencil by a layer holding it composited with `Compose::DestIn` (drawn without a Type 3 glyph's image rules, as the sub-render it replaces was), instead of through a canvas-sized luminance mask rendered for every draw; and a coloured tiling pattern's cell pixmap, repeated or shown once under patch (20), is drawn once per setting of the pattern and cell scale (`Renderer::tiling_cell`, `TilingCells`, eight entries per renderer, keyed by hayro-interpret patch (78)'s `TilingPattern::instance`) instead of once per fill, where every cell was kept until the page was composited. pdf.js issue 8078 paints 73,047 16 × 16 stencils with one pattern whose cell is 3.5 MB at 36 dpi: it ran out of memory after 5 minutes and now renders in about 5 s. For its tests, `offscreen_draws` counts on each thread the tiling cells drawn and the pixels of every pixmap drawn offscreen (all made by `offscreen_pixmap`), so a page's cost is checked by count, not by time; (31) `RenderSettings::max_coverage` keeps the rasterizer to that many coverage values for the page (vello_cpu patch 1), and a page whose paths it left out says so with `InterpreterWarning::ContentTruncated` | (1) `raster::tests::tiles_match_full_render`; (2) `raster::tests::absurd_image_dimensions_are_skipped`; (3) `raster::tests::tiling_cells_stay_small_whatever_the_step`; (4) `raster::tests::absurd_predictor_columns_and_line_widths_render`; (5) `raster::tests::dash_patterns_that_cut_a_stroke_into_billions_of_pieces_render`; (6) `raster::tests::invalid_dash_patterns_terminate_and_are_drawn_solid`; (7) `raster::tests::pixels_are_a_plain_renders_bytes_in_a_buffer_a_gui_can_take_over`; (8) `raster::tests::image_resampling_rejects_anisotropic_target_growth`, `raster::tests::image_resampling_bounds_intermediate_before_planning`, `raster::tests::image_resampling_rejects_invalid_sources_and_scales`, `raster::tests::image_resampling_keeps_valid_sizes_and_exact_limits`, `raster::tests::resampling_fallback_preserves_original_geometry_and_pixels`, `raster::tests::image_resampling_wide_sources_shrink_before_backend_side_limits`, `raster::tests::image_resampling_wide_transparent_sources_still_shrink`, `raster::tests::image_resampling_padded_backend_dimensions_stay_checked`, `raster::tests::image_resampling_mismatched_alpha_mask_keeps_pixels_and_placement`; (9) `raster::tests::skipping_paths_off_the_canvas_changes_no_pixel`; (10) `raster::tests::non_interpolated_images_are_not_smoothed_when_minified`, `raster::tests::non_interpolated_gray_images_are_not_smoothed_when_minified`, `raster::tests::non_integer_minification_averages_the_columns_each_pixel_covers`, `raster::tests::non_integer_minification_averages_colour_and_mask_together`, `raster::tests::non_interpolated_alpha_images_keep_their_colours_and_mask_when_minified`, `raster::tests::non_interpolated_type3_glyph_images_are_not_smoothed_when_minified`, `raster::tests::image_resampling_nearest_accepts_wide_sources_the_filter_refuses`, `raster::tests::wide_non_interpolated_sources_draw_after_shrinking`; (11) `raster::tests::skipping_paths_off_the_canvas_changes_no_pixel`; (12) `raster::tests::preview_renders_skip_tiny_images_but_finals_draw_everything`; (13) `color_replacement_tests::{black_maps_to_text_colour, white_maps_to_background, midgrey_interpolates, alpha_copied_verbatim, only_black_threshold, line_art_false_skips_paths_but_not_text}`, `color_replacement_render_tests::{color_replacement_maps_text_and_line_art, color_replacement_maps_stencil_mask_ink}`; (14) `raster::tests::minified_image_edges_snap_to_whole_pixels`, `raster::tests::a_tile_snaps_half_pixel_image_edges_as_the_whole_page_does` (`cargo test -p hayro`), `raster::tests::a_turned_or_mirrored_minified_image_snaps_to_whole_pixels`, `rotated_image_snap::a_shrunk_image_on_a_turned_page_is_the_upright_render_turned` (`cargo test -p pdfcraft-render --test it`), `renderer::snap_tests::a_skew_that_moves_a_far_edge_is_not_snapped` (`cargo test -p hayro`); (15) `raster::tests::shrunk_soft_mask_larger_than_its_image_keeps_thin_strokes`; (16) `raster::tests::soft_mask_applies_to_a_shading_pattern_fill`; (17), (18) `raster::tests::reading_colors_keep_highlight_annotations_readable`; (19) `mesh_extent_tests::{a_mesh_paints_its_triangles_alone, a_mesh_keeps_the_graphics_state_alpha, a_mesh_off_the_canvas_paints_nothing}` (`cargo test -p hayro`); (20) `raster::tests::a_tiling_pattern_drawn_once_shows_its_cell`, `raster::tests::a_large_step_still_tiles_where_the_next_cell_shows`, `raster::tests::a_tiling_cell_too_thin_to_scale_keeps_the_old_path`; (21) `raster::tests::non_interpolated_minification_keeps_thin_strokes`, `raster::tests::non_integer_minification_averages_the_columns_each_pixel_covers`, `raster::tests::non_integer_minification_averages_colour_and_mask_together`, `raster::tests::non_interpolated_minification_does_not_bleed_transparent_colour`, `raster::tests::a_smaller_soft_mask_does_not_bleed_transparent_colour_when_minified`, `raster::tests::an_interpolated_smaller_soft_mask_stays_smooth_when_its_image_is_minified`, `raster::tests::a_shrunk_stencil_mask_keeps_thin_strokes`, `raster::tests::minified_image_edges_snap_to_whole_pixels`, `raster::tests::an_image_shrunk_one_way_and_stretched_the_other_is_still_averaged`, `raster::tests::wide_non_interpolated_sources_draw_after_shrinking`, `renderer::area_shrink_tests` (`cargo test -p hayro`); (22) `raster::tests::knockout_groups_replace_what_their_elements_cover`; (23) `raster::tests::axial_and_radial_shadings_draw_as_native_gradients`; (25) `tiling_cell_placement::{far_cells_of_a_fill_sit_at_whole_steps, far_cells_sit_at_whole_steps_under_a_turned_scaled_matrix, far_cells_of_a_stroke_sit_at_whole_steps, a_cell_pixmap_spans_exactly_one_step}` (`cargo test -p pdfcraft-render --test it`); (28) `raster::tests::large_non_interpolated_images_are_smoothed_when_enlarged`, `raster::tests::small_non_interpolated_images_keep_hard_pixels_when_enlarged`, `raster::tests::a_scan_shrunk_one_way_is_smoothed_the_way_it_is_stretched`, `raster::tests::a_scan_drawn_a_rounding_error_above_its_resolution_stays_sharp`; (30) `raster::tests::{a_patterned_stencil_mask_paints_only_where_it_inks, a_patterned_stencil_mask_under_a_soft_mask_is_masked_once, a_patterned_stencil_mask_in_a_type3_glyph_keeps_thin_strokes, many_patterned_stencil_masks_draw_no_page_sized_masks, fills_of_one_pattern_setting_share_its_cell, fills_of_interleaved_pattern_settings_each_show_their_own_cell, an_uncoloured_pattern_shows_each_settings_colour}`; (31) `raster::tests::coverage_budget::a_page_past_its_coverage_leaves_the_rest_out_and_says_so` | | hayro-jbig2 | 0.3.0 | Apache-2.0 OR MIT | (1) at most 2^28 pixels per bitmap and page bitmap, and 2^25 cells per halftone grid and grayscale image (each side could be 65535: 4.3 billion pixels decoded one by one, and a 4-byte value per cell); (2) at most as many symbol pixels per symbol dictionary as the largest page bitmap holds, 2^28 (a fuzzed dictionary decoded for minutes; each symbol now pays for its pixels out of the stream's decoding work and is held in the retained bitmaps, and at the earlier 2^26 the soft mask of a scanned drawing, one dictionary of 7,559 symbols, was refused: #6722), and a refusal by one of the decoder's limits is told apart from damaged data (`DecodeError::is_limit`, `OverflowError::BitmapLimit`); (3) a symbol dictionary's height classes are capped at one per new symbol and its export runs at `2 × symbols + 1`, each plus `EXTRA_EMPTY_STEPS` (1024): an empty height class or a zero-length export run doesn't advance, and past the end of its data the arithmetic decoder can decode them for ever (one changed byte hung the renderer); (4) a segment header's long-form referred-to segment count is capped at `u16::MAX` and a larger count is refused before its retention or referred-to bytes are read: the 29-bit count went straight to `Vec::with_capacity`, a reservation of about 2 GiB whose failure aborts the process (from storytold/craft-libs#176); (5) what a page's segments hold for later ones (intermediate regions, pattern and symbol dictionaries with their headers and retained contexts, and Huffman tables) is at most 64 MiB, two square bitmaps of 16,384 a side (`MAX_RETAINED_WORDS`), and a dictionary's new and exported symbols are counted against it as they are made: each was bounded on its own, but a few dozen bytes an intermediate region held 33 MB each, and 8 MiB symbols exported again and again killed the process, so a few kilobytes could exhaust memory; (6) a segment may refer to at most 2^20 symbols, checked before any are collected: references may repeat, and 65,535 of them to one dictionary, in a 78 KB stream, asked for 17 GB; (7) a pattern dictionary keeps at most 16 MiB of shifted copies, which only speed up drawing, and none that wouldn't fit what the page has left (`MAX_SHIFTED_WORDS`); (8) a custom Huffman table has at most 2^16 lines, and a stream with its globals at most 2^20 segments and 2^22 references, checked as they are read: both grew before anything counted them, and PDFs of a kilobyte or so reached 5 GB and more behind nested Flate; (9) a page past the bounds of (1) is refused when the stream is opened: a caller sizes its output from the header before the page bitmap is made, and an Indexed image in a 743-byte PDF asked for 137 GB, which aborts; (10) a stream may ask for at most 2^30 units of decoding work, four of the largest pages, about 6 s to 17 s (`MAX_WORK`): decoding a bitmap costs a pixel each and 4 a row, copying one a word each and 1 a row, and each integer an item reads costs 38, what one takes at worst, so one with no pixels pays too, before it is decoded: each bitmap was bounded, but nothing bounded how many there were, and a 795-byte PDF took 265 s; (11) `Image::decode_within` spends from what a run of images may still do, starting with its page's pixels, and `charge_reading` charges reading a stream and its globals before anything is parsed, so an operation over a document can bound what `MAX_WORK` bounds for one image, and images that fail or decode nothing add up too: a file of thousands of images made to reach that limit took as many times as long; (12) adding to a location, a reference offset, a symbol's corner or a Huffman range line near the end of an `i32`, and negating a decoded integer of 2^31, overflowed: a panic wherever overflow is checked. The refinement context, a text region's corners, the integer decoder and the Huffman table wrap, as a release build did, and `Bitmap::combine` saturates, which comes to the same, as a location that overflows is past the end of any page. What is drawn doesn't change; (13) a bitmap filled with a default pixel of 1 (a page, or an intermediate text or halftone region) had 1s past its width in the last word of each row, and a refinement, which reads its reference a word at a time, read them as pixels where the standard has 0 (6.3.5.2): refining such a reference not a multiple of 32 wide decoded to other pixels than were coded, release builds too. `Bitmap::clear_padding` clears those bits when a bitmap is filled with 1s, as cutting symbols and patterns from a collective bitmap already did. Ghostscript reads the test's page and text streams as they decode now; (14) a region's location is unsigned (5.4.4), so one past `i32::MAX` is off the page, where the decoder took it for a signed one: a region there was drawn back onto the page from the left or the top (at 0xFFFFFFFC, four pixels left of it), and a refinement there, of the page or of a region at the same place (7.4.7.5), was refused as an overflow, and the page with it. `Bitmap::combine_region` keeps such a location past the page. A refinement of another region refines it where it is, as Table 35 has it (GRREFERENCEDX and GRREFERENCEDY 0), whatever the two locations say; a refinement of the page, see (16); (15) a refinement whose reference is offset right of it (GRREFERENCEDX above 0, which a refined text instance has from a wider bitmap, Table 12, and a refined symbol from an RDX above 0, Table 18) read the reference's first 32 columns as 0: `RefinementContextGatherer::maybe_reload_buffers` emptied its buffers left of the reference and didn't load them again at its first column. Such a refinement decoded to other pixels than were coded, or failed and lost the page. Left of the reference the buffers now keep the words of its first column each row starts with; with an offset of 0 or less nothing changes; (16) a refinement of the page refines "the current contents of the page buffer, restricted to the area of the page buffer specified by this segment's region segment information field" (7.4.7.4), at an offset of 0 (Table 35): `Bitmap::area` copies that area a word at a time, 0 off the page and past its width whatever the padding of its rows holds, and every refinement region reads its reference at offset 0. The page around the area was read too, so a refinement of the page decoded to other pixels than were coded, and one wider or taller than the page was refused. And `RefinementContextGatherer::start_row` loads the reference's words a column left of where a row starts in it, as the reload would, not from column 0: with the reference offset 32 or more left of the refinement (GRREFERENCEDX below −31, as a refined text instance has from a narrower bitmap or an RDX below −31, and a refined symbol from such an RDX), each row's first context read 0 for the reference column it starts at, and from 33 left for the one before it too, and the refinement decoded to other pixels or failed; (17) `Image::new_embedded` reads data that starts with a JBIG2 file header (the ID string `0x97 4A 42 32 0D 0A 1A 0A`, flags and an optional page count, D.4) past it, in the organization the header declares (sequential or random access), its segments counted together with the globals' as (8) has them: ISO 32000-1 7.4.7 has `JBIG2Decode` data in the embedded organization (D.3), but some producers write the file header too, and Acrobat and PDFBox draw such streams. The header's ID string was read as a segment header and the image failed (`unknown or reserved segment type`), so the page drew white | (1) `raster::tests::jbig2_regions_of_billions_of_pixels_are_refused`, `raster::tests::jbig2_halftone_grids_past_their_bound_are_refused`; (2) the nightly `cargo xtask fuzz` finding (about a second at the earlier 2^26 bound; not run at 2^28), `raster::tests::a_symbol_dictionary_holds_as_many_pixels_as_a_page`, and for `is_limit` and `BitmapLimit` `error::pdfcraft_limit_tests`, `bitmap::pdfcraft_limit_tests` and `gray_scale::pdfcraft_limit_tests` (hayro-jbig2); (3) `raster::tests::jbig2_symbol_dictionaries_that_never_advance_terminate`; (4) `segment::tests::test_referred_to_count_is_capped` (`cargo test --manifest-path vendor/hayro-jbig2/Cargo.toml`); (5) `raster::tests::jbig2_regions_held_past_a_pages_bound_are_refused`, `jbig2_symbols_without_pixels_count_towards_a_pages_bound`, `jbig2_retained_contexts_count_towards_a_pages_bound`, `jbig2_huffman_tables_count_towards_a_pages_bound`, `jbig2_new_symbols_count_towards_a_pages_bound`, `jbig2_new_symbols_are_held_to_what_a_page_has_left`, `jbig2_pattern_dictionaries_count_towards_a_pages_bound`, and in the crate (`cargo test --manifest-path vendor/hayro-jbig2/Cargo.toml`) `decode::symbol::tests::exported_symbols_are_counted_before_they_are_copied`, `tests::the_room_left_shrinks_as_segments_are_stored`; (6) `raster::tests::jbig2_segments_referring_to_too_many_symbols_are_refused`; (7) `raster::tests::jbig2_pattern_dictionaries_too_large_to_shift_still_draw`, `jbig2_shifted_patterns_never_refuse_an_image`; (8) `raster::tests::jbig2_huffman_tables_of_too_many_lines_are_refused`, `jbig2_streams_of_too_many_segments_are_refused`, `jbig2_streams_of_too_many_references_are_refused`, and in the crate `tests::the_segments_of_a_stream_and_its_globals_count_together`; (9) `raster::tests::jbig2_pages_past_the_bounds_are_refused_before_anything_is_sized_from_them`, `tests::a_page_past_the_bounds_is_refused_when_opened`; (10) `raster::tests::jbig2_decoding_work_is_counted_item_by_item`; (11) `jbig2::decode::tests::a_run_of_images_spends_its_allowance_down_to_the_step` (pdfcraft-filters); (12) `raster::tests::jbig2_placements_at_the_end_of_an_i32_draw_nothing`; (13) `jbig2::decode::tests::a_refinement_reads_nothing_past_its_references_width` (pdfcraft-filters), and in the crate `tests::a_bitmap_of_1s_has_nothing_past_its_width`; (14) `jbig2::decode::tests::a_region_located_past_i32_max_is_off_the_page` (pdfcraft-filters), and in the crate `tests::a_region_located_past_i32_max_is_off_the_page` and `tests::a_refinement_of_a_region_decodes_the_same_wherever_they_are`; (15) `jbig2::decode::tests::a_refinement_reads_a_reference_right_of_it_from_its_first_column` (pdfcraft-filters), and in the crate `tests::a_refinement_reads_a_reference_right_of_it_from_its_first_column`; (16) `jbig2::decode::tests::a_refinement_refines_the_page_in_its_area_and_any_reference_where_it_is` (pdfcraft-filters), and in the crate `tests::a_refinement_refines_the_page_in_its_area_and_any_reference_where_it_is` and `tests::a_page_area_is_the_page_inside_it_and_0_outside` ; (17) `jbig2::decode::tests::a_stream_with_a_jbig2_file_header_decodes_past_it`, `jbig2::decode::tests::a_truncated_jbig2_file_header_is_an_error`, `jbig2::decode::tests::segments_after_a_jbig2_file_header_count_with_the_globals` (pdfcraft-filters), `raster::tests::a_jbig2_image_with_a_file_header_draws` (pdfcraft-render) | -| lopdf | 0.45.0 | MIT | (1) PNG predictor rows longer than the data are refused before their buffers are allocated (a fuzzed `/Columns 4294967295` allocated two 4 GiB rows), and empty frames return without row buffers; (2) predictor row widths use saturating arithmetic or reject overflow; (3) sub-byte TIFF Predictor 2 scratch is bounded by the samples present in the input instead of the declared `/Colors`, aligned with [lopdf#589](https://github.com/J-F-Liu/lopdf/pull/589); (4) the object lookup keeps its cycle history through compressed objects, requires an ordinary in-file container, and caps the chain of indirect lookups at 16 (a chain of new objects, each a `/Length` compressed in a container whose own `/Length` names the next, or a stream whose `/Length` names the next stream, overflowed the stack: a dev build's 2 MiB thread within 800 hops); (5) object-stream members are parsed within their own extent (up to the next distinct offset), an entry whose offset repeats an earlier entry's is skipped (so a repeated object number can resolve to its earlier entry), and offsets are processed in index order; (6) `find_latest_trailer`: a rebuild takes a `/Type /XRef` stream dictionary as the trailer when the file has no `trailer` keyword (the latest in the file wins, as a later `trailer` would), so a concatenated or truncated file keeps its /Info and /ID instead of failing to load; a candidate whose /Root is the catalog beats one whose /Root is another dictionary, and an in-file /Root must match its generation (#2365) | (1) `inspect::tests::png_predictor_rows_longer_than_the_data_are_refused_before_allocating`, `inspect::tests::empty_png_predictor_frames_do_not_allocate_rows`; (2) `inspect::tests::tiff_subbyte_predictor_row_width_overflow_is_refused`; (3) `inspect::tests::tiff_subbyte_predictor_scratch_is_bounded_by_available_samples`, `inspect::tests::tiff_subbyte_predictors_keep_components_rows_and_padding`; (4) `inspect::tests::compressed_object_self_container_does_not_recurse`, `inspect::tests::compressed_object_mutual_containers_do_not_recurse`, `inspect::tests::object_stream_length_cycle_does_not_recurse`, `inspect::tests::compressed_object_container_chain_does_not_recurse`, `inspect::tests::compressed_length_chain_does_not_recurse`, `inspect::tests::plain_stream_length_chain_does_not_recurse`, `inspect::tests::compressed_objects_and_indirect_stream_length_still_load`, `inspect::tests::compressed_length_in_a_container_with_an_indirect_length_still_loads`; (5) `inspect::tests::object_stream_member_extents_bound_owned_strings`, `inspect::tests::object_stream_member_extents_preserve_index_order_and_whitespace`; (6) `inspect::tests::rebuilt_file_keeps_the_info_of_its_cross_reference_stream`, `inspect::tests::rebuilt_file_prefers_a_trailer_whose_root_is_the_catalog` | +| lopdf | 0.45.0 | MIT | (1) PNG predictor rows longer than the data are refused before their buffers are allocated (a fuzzed `/Columns 4294967295` allocated two 4 GiB rows), and empty frames return without row buffers; (2) predictor row widths use saturating arithmetic or reject overflow; (3) sub-byte TIFF Predictor 2 scratch is bounded by the samples present in the input instead of the declared `/Colors`, aligned with [lopdf#589](https://github.com/J-F-Liu/lopdf/pull/589); (4) the object lookup keeps its cycle history through compressed objects, requires an ordinary in-file container, and caps the chain of indirect lookups at 16 (a chain of new objects, each a `/Length` compressed in a container whose own `/Length` names the next, or a stream whose `/Length` names the next stream, overflowed the stack: a dev build's 2 MiB thread within 800 hops); (5) object-stream members are parsed within their own extent (up to the next distinct offset), an entry whose offset repeats an earlier entry's is skipped (so a repeated object number can resolve to its earlier entry), and offsets are processed in index order; (6) `find_latest_trailer`: a rebuild takes a `/Type /XRef` stream dictionary as the trailer when the file has no `trailer` keyword (the latest in the file wins, as a later `trailer` would), so a concatenated or truncated file keeps its /Info and /ID instead of failing to load; a candidate whose /Root is the catalog beats one whose /Root is another dictionary, and an in-file /Root must match its generation (#2365); (7) object parsing builds at most 4,194,304 objects (every array element and dictionary value at any depth, `parser::MAX_OBJECT_NODES`, the same cap as `pdfcraft_cos`) per top-level object, an object stream reads at most 1,000,000 index entries (cos's `/N` clamp), and one load builds at most 8,388,608 objects from all of its object streams together, shared across rayon workers (a decoded stream of `0 0 0 ...` inside the 256 MiB decompression limit built ~120M objects, 10+ GB, and aborted). A lookup parses only the member it wants, so it isn't skipped behind a large member, and a load parses only the members it would add: none its cross-reference table places in another stream (stale copies) and none already loaded as an ordinary object or from an earlier stream. Members the table doesn't list at all are still parsed (a rebuilt table lists no compressed objects) and share the budget, so they can spend it before listed members of the same stream; a load expands its object streams one at a time in object-number order, a stream's members in parallel only when the budget holds all of them and otherwise in index order, so a spent budget skips the same members on every load (#5083) | (1) `inspect::tests::png_predictor_rows_longer_than_the_data_are_refused_before_allocating`, `inspect::tests::empty_png_predictor_frames_do_not_allocate_rows`; (2) `inspect::tests::tiff_subbyte_predictor_row_width_overflow_is_refused`; (3) `inspect::tests::tiff_subbyte_predictor_scratch_is_bounded_by_available_samples`, `inspect::tests::tiff_subbyte_predictors_keep_components_rows_and_padding`; (4) `inspect::tests::compressed_object_self_container_does_not_recurse`, `inspect::tests::compressed_object_mutual_containers_do_not_recurse`, `inspect::tests::object_stream_length_cycle_does_not_recurse`, `inspect::tests::compressed_object_container_chain_does_not_recurse`, `inspect::tests::compressed_length_chain_does_not_recurse`, `inspect::tests::plain_stream_length_chain_does_not_recurse`, `inspect::tests::compressed_objects_and_indirect_stream_length_still_load`, `inspect::tests::compressed_length_in_a_container_with_an_indirect_length_still_loads`; (5) `inspect::tests::object_stream_member_extents_bound_owned_strings`, `inspect::tests::object_stream_member_extents_preserve_index_order_and_whitespace`; (6) `inspect::tests::rebuilt_file_keeps_the_info_of_its_cross_reference_stream`, `inspect::tests::rebuilt_file_prefers_a_trailer_whose_root_is_the_catalog`; (7) `inspect::tests::{object_stream_members_over_the_node_cap_are_skipped, object_stream_index_entries_past_the_clamp_are_ignored, object_streams_of_one_load_share_their_node_budget, object_streams_loaded_in_parallel_share_their_node_budget, a_malformed_object_stream_member_refunds_its_reservation, a_tight_budget_builds_object_stream_members_in_index_order, a_lookup_in_an_object_stream_parses_only_the_member_it_wants}` (pdfcraft-render; `NodeBudget` and `ObjectStream::new_within` are public so these can use a small budget) | | hayro-syntax | 0.7.2 | Apache-2.0 OR MIT | (1) Page-tree cycle guard in `resolve_pages` (visited set + depth cap): a `/Kids` loop among compressed objects overflowed the stack and aborted the process; (2) `ccitt_size_ok`: CCITT images wider than 2^20 columns or over 2^28 pixels are not decoded (a fuzzed `/Columns 4294967295` allocated 4 GiB); (3) Flate/LZW predictor row widths are saturating `usize`, a zero width is refused and a row longer than the data is not allocated (a fuzzed `/Columns 9223372036854775807` wrapped to a 2^61-byte allocation and aborted); (4) the inline-image end-of-data check (`EI`) doesn't nest inside its own look-ahead and examines at most 256 candidates and steps (50 images with "EI" in their data ran for over ten minutes); an EI followed by a delimiter (EI%..., EI[...) ends the image like one followed by white space; only a regular character after EI skips the candidate; (5) a stream whose `/Length` is wrong is recovered only when `stream` is the next token — scanning ahead to a later object's `stream` made an indirect appearance-state dictionary (Quartz checkboxes: `/AP /N` → `<< /Yes … /Off … >>`) render as a non-form, so the widget was skipped; (6) JPX rescaling uses an exact integer sample range for 1–32-bit components (32 shifted past u32, and floating-point maxima at 25–31 bits wrapped to zero), and refuses zero-sized rows; (7) `decode_limit`: Flate (through flate2 and the fallback inflater), LZW, RunLength and ASCII85 never write past `MAX_DECODED_STREAM` (256 MiB), or, for the last filter of an image of known size, past twice its pixel bytes (a predictor tag byte per byte), at most 1 GiB; `Stream::decoded_within` for budgeted callers; a page's `/Contents` array shares one `MAX_DECODED_STREAM` (a 2 MB page whose content stream or 8 × 8 image inflated to 2 GB took the renderer past 3 GB in under a second; naming the stream twenty times in `/Contents`, 5 GB); (8) high-precision numbers (jsPDF MediaBox coordinates with over 19 digits) parse without overflowing the u64 mantissa: fractional digits past it are dropped, integer digits past it scale the value (a real, kept finite); (9) large dictionary indexes are built on first use and shared across repeated reads within each PDF (#307); the validated-byte cache is bounded, reader contexts remain separate, repairs and inline/content dictionaries bypass sharing, and small dictionaries stay eager; (10) decoded object-stream slots publish once under contention, cold loading rejects recursive dependencies, and xref repair serializes without holding recursive read guards; this permits one parser per native render pool with private worker caches; (11) `sync.rs`: a poisoned lock is recovered instead of unwrapped (a worker panic no longer cascades into every reader of the same PDF), and xref repair takes a blocking write lock instead of asserting a non-blocking one; (12) `SegmentList` (`util.rs`) returns `None` for an out-of-range index instead of panicking; (13) a page's `/UserUnit` scales `render_dimensions` and `initial_transform`, so the page and everything on it are sized in points, as Acrobat shows them; it is read as Acrobat reads it (the page's own entry, not inherited; a number from 1 up, at most 75,000; anything else counts as 1), and a page without it keeps exactly the transform it had; (14) revision 2–4 passwords are tried as given and then in PDFDocEncoding (`legacy_passwords` in `crypto/mod.rs`, which resolves the upstream `TODO: Convert to PDFDocEncoding`): passwords arrive as UTF-8 text, so a password with characters outside ASCII (`Çok güzel`, `ılık€`) written in PDFDocEncoding, as ISO 32000-2 §7.6.4.3.2 asks and qpdf and Acrobat write it, was always wrong; (15) page content is read a piece at a time (`page::ContentReader`, `Page::content_reader`): plain streams and Flate streams without a predictor are decoded as they are read, with a budget of 64 times their compressed size and never less than `MAX_DECODED_STREAM` (a stream that expands past it is cut and reported); other streams are decoded whole, as before, and share `MAX_DECODED_STREAM`. The content tokenizer reads a window at a time (`UntypedIter::new_window`, `TypedIter::needs_more`, `TypedIter::committed`): an instruction that runs past the window is not decided until more data follows, which `Reader` records by flagging reads and `at_end` past the end of the data, including through the clones the parser makes. The Flate decoder keeps its own 32 KiB history, so it pauses at any output size. `ContentReader::new` reuses raw/decrypted stream bytes for budget accounting and piece creation, and `decoded_raw_within` applies the existing Whole-stream filters and limits to those bytes without decrypting twice; (16) revision 5–6 (AES-256) passwords are tried in their SASLprep form (RFC 4013, then UTF-8, at most 127 bytes, as ISO 32000-2 §7.6.4.3.3 Algorithm 2.A asks and pdfcraft-crypt writes them) and then, if that differs, as given (`aes256_passwords` in `crypto/mod.rs`, which resolves the upstream `TODO: Do the above.`; `stringprep`, already in the tree through pdfcraft-crypt, is an optional dependency enabled by `std`, and without `std` passwords are used as given): the renderer used the password as typed, so one that SASLprep changes (a decomposed `café`, a no-break space, the fullwidth `ａｂ` a Japanese IME types in full-width mode) was wrong, although pdfcraft-cos accepted it: protecting a document with such an open password was rolled back ("the password is incorrect"), and a file with such a password did not open with it.; (17) CCITT `/Rows 0` means the height is not predetermined (ISO 32000-2 Table 11), not zero rows: scanned Group 4 pages written that way rendered blank; (18) The streaming Flate decoder (`lzw_flate::flate::fallback`) copies a back-reference in runs of up to its distance (`extend_from_within`), not byte by byte. The copy reads only bytes already in the output, and a copy whose source does not exist still writes nothing, so the bytes are those a byte-by-byte copy writes. Inflating the A1 sample's content stream (278.6 MB) takes about 14% less time (0.66 s to 0.56 s); the output is identical; (19) CCITT `/Rows` below the image's `/Height` decodes to the height: a plotter driver wrote `/Rows` equal to `/Columns`, so only the first rows were decoded and the rest drawn black (#3308) (20) `stream` followed by spaces or tabs and then the line break (`stream \r\n`, as Microsoft Reporting Services writes it) starts the data after the line break; the content stream was taken to start at the space and the page rendered blank.; (21) a JP2 colour specification naming an enumerated colour space the decoder refuses (CIEJab in a veraPDF fixture, YCbCr, PhotoYCC, YPbPr, bilevel…) no longer fails the image: it is decoded again from a copy whose `colr` names the space the decoder assumes without one, greyscale, sRGB or CMYK by component count, as MuPDF and Poppler draw it (`without_refused_enumerated_colour`; the page drew blank, #3306); (22) an object stream's index of `objnum offset` pairs is parsed once per stream and kept beside its decoded data (`ObjectStreamIndexes`, `XRef::with_object_stream_indexes`), and an offset past `usize` is malformed instead of wrapping: `XRef::get_with` parsed the whole index again for every object it fetched from the stream, so reading all of a stream of N objects took N² steps (a Firefox page whose 65,542 objects share one stream took 37 s to draw and 40 s to extract its text); (23) the inline-image end-of-data look-ahead no longer rejects a candidate `EI` because a byte above 0x7F lies between it and the next one: when the candidate is the end, those bytes are the content stream that follows the image, and its strings carry such bytes (WinAnsi `(Grüße) Tj`, the two-byte codes of any Type 0 font), so the operator check that follows decides alone, bounded as in (4) (a scanned page whose OCR text held an accented word and a later capital "EI" — EINGANG, BEI, KLEIN — drew and extracted nothing after the image: 40 of 275 pages across five real documents); (24) `xref::prev_offset`: a trailer `/Prev` written as a reference `N G R` is read as the offset that object holds when the section's own entries place it (iText's `indirectPrev.pdf`; each section's entries are read before its `Prev` is followed), and otherwise as offset N (tiff2pdf; JHOVE's PDF-HUL-157 files, #2366), as `pdfcraft-cos` and other readers take it, so the earlier sections' pages are seen instead of one; (25) direct arrays and dictionaries share a reader-local nesting cap of 128, counted while skipping and while parsing a dictionary (its lenient repair reads, rather than skips, an object found where a key belongs, so `<< << …` recursed through that read); deeper objects return `None` instead of overflowing the stack (including unused trailer values, trailer entries, content operands and marked-content properties); a chain of 65 direct stitching functions now stops at this cap, before the 64-function guard; (26) an xref stream subsection must be backed by its decoded data: all-zero `/W` widths (an entry reads no bytes) and a declared count longer than the data are refused before any entry is inserted, where a tiny stream with a large `/Size` or `/Index` count inserted that many entries (from storytold/craft-libs#239); a subsection running past object number `i32::MAX` is refused instead of overflowing or turning negative; (27) an xref stream's type field is read big-endian at its full `/W[0]` width, where only its first byte was read and wider fields shifted every later entry, and `/W[0]` over 8 bytes is refused like `/W[1]` (from storytold/craft-libs#241); (28) a content instruction keeps up to 256 operands (`MAX_OPERANDS`) instead of 10, and past that drops the oldest instead of ending the content stream: the eleventh operand of a twelve-ink DeviceN `scn` (ISO 32000-2 §8.6.6.5 allows any number of components) blanked the rest of the page (#4260); (29) a `/JBIG2Globals` stream that is itself JBIG2 isn't decoded: globals are raw segments, and a chain of 16,000 such streams overflowed the stack, which aborts the process; (30) a stream's own crypt filter: `/Crypt` in its chain read as the filter it names, `/Identity` without one, instead of the document's filter and then a filter that can't be decoded, which left such a stream unreadable in any spelling (ISO 32000-2 §7.4.10); (31) `set_page_jbig2_work`: the JBIG2 images decoded on a thread spend one allowance, set per page, 2^30 steps (`MAX_PAGE_JBIG2_WORK`), so a page of many images made to reach their own limit stops at one, and what they cost to read counts, whether they decode or not, and `jbig2_limited` says whether the image decoded last on the thread was refused by one of the decoder's limits rather than its data, and `jbig2_limited_by_stream` whether that limit is one the stream reaches by itself rather than its decoding work, which may be the page's; `clear_jbig2_limited` and each new page forget it; unbounded until a page sets it; (32) a group of array items read together (a tuple: an image's `/Decode`, `/W`, `/W2`, and a function's `/Domain`, `/Range`, `/Encode` and `/Decode`) can start with a reference: it was taken for the whole group and lost, which ended the array, so an image with `/Decode [7 0 R 0]` didn't draw and CID widths fell back to `/DW`; (33) an object that holds only a reference stands for what that names, stored on its own or in an object stream, up to 32 references: read as a number it was the reference's own object number, so a `/Decode` or `/W` entry naming such an object took a wrong value, and as anything else it was lost. Past 32, as any cycle goes, it reads as a missing object. An object stream isn't reached that way, as its checks are of the object named; reading its index counts against the 64 nested loads and reads its dictionary alone, as a chain of them, each with a member of the next for its `/N`, `/First` or `/Length`, ran out of stack; and a comment inside a reference counts as white space; (34) a table rebuilt by scanning the file (`fallback_xref_map`) finds an object by its number when its header names another generation, including one no generation may be (`3 18446744073709551616 obj` in `poppler-85140-0.pdf`, #5580): the header is read leniently only in that scan (`read_object_header`), the object is also filed under generation 0 unless a real `N 0 obj` holds that key (a later header for the number moves the stand-in), `XRef::get_with` matches by object number only in a repaired table, and `XRef::objects` yields each location once; (35) PNG predictors (`lzw_flate::unfilter_png`) are undone a byte row at a time on slices, as the PNG filters are defined, instead of through the bit-level readers TIFF predictors need: a 993 × 936 RGB image took about 63 ms to unfilter whatever its row filters, and now takes about 3 ms (Up) to 16 ms (Paeth) in a release build (#5430); a row ending part-way through a pixel (three 4-bit samples, an odd width) keeps its last byte, where the bit-level readers dropped it and each later row started a byte early, shearing the picture; (36) a stream with no `endstream` left after it is the bytes its `/Length` gives, "how many bytes of the PDF file are used for the stream's data" (ISO 32000-2 7.3.8.2), when the file holds that many (`parse_declared`: first, before the search for `endstream`, when the `/Length` ends before a whole `endstream` token, or a whole `endstrea` or `endobj` token after which the object ends too (the end of the data, another object's `N G obj` header, `xref`, `trailer` or `startxref`: not payload such as `endobject` or `endobj more`, ISO 32000-2 7.2.3), or at the end of the data, so the search can't run on into a later object's stream; then after the search finds none; `parse_proper` and the search for `endstream` take only a whole token, not `endstreamer`): a producer that writes `endstrea` after every stream lost every font and page content, and its pages drew blank while other readers draw them; (37) a document that encrypts only its embedded files, authorised when one is opened (`/AuthEvent /EFOpen`, ISO 32000-2 §7.6.6, Table 25: every encrypting crypt filter is EFOpen, `/EFF` names one, `/StmF` and `/StrF` don't), opens without a password with the pass-through decryptor: its strings and streams are in the clear and nothing drawn reads an embedded file; (38) a page box that encloses no area is read as a missing one: a `/MediaBox` without area (`[0 0 0 0]`) as missing (an inherited one with area, else the default page, US Letter since patch 44), and a `/CropBox` without area, or with none in common with the media box, as missing (an inherited one, else the media box; ISO 32000-2 §14.11.2, Table 33); kept, the visible region had no area, so nothing was drawn and no text extracted (pdf.js `boundingBox_invalid`); (40) the last Flate filter of an image whose PNG predictor (`/Predictor` 10–15) its writer declared but never applied gives the image's samples as they are: when the inflated data is exactly `/Height` rows of the predictor's row length with no filter-type byte before each (where predicted rows would need one more byte each, ISO 32000-2 §7.4.4.4), `untagged_png_rows` takes it for unpredicted samples and reports it (`warn!`), as PDFium, MuPDF and Poppler read it; read with the predictor, each row's first sample was taken for a filter type and the image came out a row short and shifted, drawn in fragments (PDFBOX-3932). `Filter::apply` is given the image's rows for its last filter only; any other length is predicted as before; (41) CCITT `/Rows` above the image's `/Height` decodes to the height too, so the decoder is never asked for more rows than the image has (ISO 32000-2 Table 87): a scan with `/Rows 1084975104` on 2,592 columns had the size check refuse every page, and all 17 drew blank (PDFBOX-771); `/Rows` still counts for a stream decoded without an image's height; (42) TIFF predictor 2 on 8- and 16-bit samples (`lzw_flate::undo_tiff_bytes`) is undone a byte row at a time on slices, each sample added to the same component of the pixel to its left, wrapping at the sample's width (16-bit samples big-endian), instead of through the bit-level readers 1-, 2- and 4-bit samples still need; the bytes are those the bit-level path wrote, and a partial last row is still dropped. A page of 45 full-page 2445 × 4724 8-bit grey images so predicted (PDFBOX-1350) took about 23 s to render at 96 dpi and now takes about 4 s, with the same pixels; (43) a last update cut off before its trailer (the last `startxref` more than 4 KiB from the end of the file, as `pdfcraft-cos` judges the footer, with `N G obj` headers after it: `cut_off_update_follows`) is read by rebuilding the table from object headers (`Pdf::new_with_password` calls `fallback` first, after the table the last `startxref` names has refused a wrong password, and takes the rebuilt table only when it is encrypted exactly when that one is), the latest definition of each object winning, as `pdfcraft-cos` reads such a file; the older table the last `startxref` names is read only when the rebuilt table has no page (a file cut short inside the update, whose new page tree names pages that are not there). The renderer had shown the older revision while the document model read the newer one (PDFBOX-3934: 12 pages against 13; ISO 32000-2 §7.5.6). A `startxref` near the end, or one followed only by data that holds no object header, is read as before; (44) a page with no `/MediaBox`, its own or inherited, or one without area (`[0 0 0 0]`, which now counts as missing, so the page is placed and clipped by the region it is sized by and its content shows), is US Letter (`LETTER`, 612 × 792 pt), not A4, and so is the `base_dimensions` fallback for a visible region without area: ISO 32000-2 Table 31 requires the box and gives no default, other readers use Letter, and PdfCraft's page model and inspector already did (PDFBOX-4001); (45) `root_xref`: when the newest cross-reference section names no earlier one (no `/Prev`, ISO 32000-2 §7.5.6, §7.5.8.2; a hybrid section's `/XRefStm` stream belongs to the same update and doesn't count, §7.5.8.4), the file holds another `startxref`, and the catalog's `/Pages` can't be read, the table is rebuilt by scanning the file (`XRef::repair`) before the page tree is read, so the cut-off page tree is found instead of the page dictionaries `Pages::new_brute_force` turns up (the signed files of PDFBOX-2720 and PDFBOX-2942 showed 2 blank pages for 1, 10, 13 and 1754); `Pdf::rebuilt_unlinked_update` says so, and `pdfcraft-cos` rebuilds under the same conditions | (1) `cos/tests/writer.rs::page_tree_loops_in_object_streams_do_not_crash_readers`; (2) `raster::tests::absurd_mask_and_fax_sizes_are_refused_before_decoding`; (3) `raster::tests::absurd_predictor_columns_and_line_widths_render`; (4) `raster::tests::inline_images_with_ei_in_their_data_parse_in_linear_time`, `content::tests::inline_image_ei_may_be_followed_by_a_delimiter`; (5) `raster::tests::indirect_appearance_state_dict_is_not_another_stream`; (6) `raster::tests::jpx_stream_bit_depth_boundaries_do_not_panic`, `raster::tests::jpx_32_bit_image_does_not_prevent_other_page_content_from_rendering`, `raster::tests::jpx_gray_rgb_and_alpha_keep_eight_bit_samples`, `raster::tests::jpx_streams_reject_out_of_range_bit_depths`; (7) `raster::tests::decompression_bombs_stop_at_the_stream_limit`, `raster::tests::page_contents_and_nested_forms_share_one_budget`; (8) `inspect::tests::jspdf_high_precision_mediabox_does_not_overflow_dimensions`, `object::number::tests::high_precision_float_does_not_overflow_mantissa` (hayro-syntax); (9) `object::dict::lazy_index_tests`, `raster::tests::large_resource_indexes_preserve_pixels_and_text`; (10) `data::tests`, `xref::tests::concurrent_cold_object_stream_reads_all_succeed`, `raster::tests::shared_parser_is_lazy_and_initializes_once_under_contention`; (11) `data::tests::panicking_initializer_wakes_waiters_and_prevents_retries`, `xref::tests::concurrent_cold_object_stream_reads_all_succeed`; (12) `util::tests::segment_list_rejects_out_of_range_indices`; (13) `raster::tests::user_unit_scales_the_page_and_everything_on_it`; (14) `crypto::pdfcraft_password_tests`, `pdfcraft-engine` `tests::legacy_passwords_with_turkish_characters_open`; (15) `content::tests::windows_keep_every_instruction_at_every_split`, `content::tests::windows_keep_every_instruction_in_pieces_of_any_size`, `content::tests::windows_match_the_slice_on_random_content`, `content::tests::windows_drop_blanks_and_keep_a_comment_that_may_go_on`, `page::content_reader_tests::content_past_the_cap_is_read_to_the_end`, `page::content_reader_tests::a_stream_that_expands_past_the_budget_is_cut_there`, `page::content_reader_tests::a_small_expansion_under_the_cap_is_not_cut`, `raster::tests::page_content_past_the_old_cap_renders`, `raster::tests::content_reader_returns_the_same_bytes_in_any_pieces`, `raster::tests::content_bombs_stop_at_the_budget_and_say_so`, `page::content_reader_tests::plain_and_whole_pieces_keep_their_bytes`, `page::content_reader_tests::repeated_content_reference_counts_compressed_bytes_once`, `page::content_reader_tests::whole_cap_still_counts_later_unique_streams`; (16) `crypto::pdfcraft_password_tests::aes256_passwords_are_tried_in_their_saslprep_form_then_as_given`, `crypto::pdfcraft_password_tests::aes256_passwords_open_as_typed_and_in_their_saslprep_form`, `pdfcraft-engine` `tests::aes256_open_passwords_that_saslprep_changes_protect_and_reopen_as_typed`, `tests::aes256_files_whose_passwords_saslprep_changes_open_as_typed`; (17) `raster::tests::group4_fax_with_rows_zero_decodes_to_the_image_height`; (18) `filter::lzw_flate::tests::back_reference_longer_than_its_distance_repeats_its_source`, `back_reference_cut_by_the_limit_resumes_in_the_next_call` and `back_reference_reads_bytes_written_by_an_earlier_call`; each fails when the copy returns the wrong remaining count; (19) `raster::tests::group4_fax_with_rows_below_its_height_decodes_to_the_image_height`; (20) `raster::tests::space_before_the_line_break_after_stream_still_renders`; (21) `raster::tests::a_jp2_image_in_a_refused_enumerated_colour_space_is_drawn`, `filter::jpx::tests::refused_enumerated_colour_spaces_become_the_component_counts_default`, `filter::jpx::tests::what_the_decoder_handles_or_cant_be_read_is_left_alone` (`cargo test --manifest-path vendor/hayro-syntax/Cargo.toml`); (22) `raster::tests::objects_in_one_large_object_stream_are_read_in_linear_time`; (23) `text::tests::text_after_an_inline_image_survives_a_non_ascii_byte_before_a_later_ei`, `content::tests::an_inline_image_ends_at_its_own_ei_when_later_text_holds_a_non_ascii_byte` (hayro-syntax); (24) `xref::tests::prev_as_reference_reaches_the_earlier_section`, `xref::tests::prev_as_reference_to_an_object_holding_the_offset_reads_that_offset`, `pdfcraft-cos` `document::tests::a_prev_written_as_a_reference_still_reaches_the_earlier_sections`, `document::tests::a_prev_reference_to_an_object_holding_the_offset_reads_that_offset`; (25) `raster::tests::deeply_nested_direct_objects_do_not_overflow_the_stack`, `raster::tests::dictionaries_nested_without_keys_in_content_do_not_overflow_the_stack`, `raster::tests::dictionaries_nested_without_keys_in_the_trailer_do_not_overflow_the_stack`, `raster::tests::dictionaries_nested_without_keys_as_marked_content_properties_do_not_overflow_the_stack`, `raster::tests::dictionaries_nested_without_keys_are_read_up_to_the_nesting_cap`, `raster::tests::nesting_at_the_cap_fits_a_512_kib_stack` (native only), `raster::tests::function_limits_depth_guard_refuses_a_chain_of_references`; (26) `xref::tests::xref_stream_entries_need_backing_bytes`; (27) `xref::tests::xref_stream_type_field_uses_the_full_w0_width`, `xref::tests::xref_stream_loader_rejects_oversized_w0` (`cargo test --manifest-path vendor/hayro-syntax/Cargo.toml`); (28) `content::ops::tests::scn_with_twelve_components_keeps_them_all`, `content::ops::tests::operands_past_the_cap_keep_the_latest_in_order` (hayro-syntax), `raster::tests::devicen_inks::a_twelve_ink_devicen_colour_paints`, `raster::tests::devicen_inks::a_four_ink_devicen_colour_paints`, `raster::tests::devicen_inks::an_operator_with_hundreds_of_operands_does_not_end_the_content`; (29) `raster::tests::jbig2_globals_that_are_jbig2_themselves_are_not_decoded`, `raster::tests::jbig2_globals_are_read`, `raster::tests::the_jbig2_images_of_a_page_share_one_allowance`, `raster::tests::an_image_that_fails_spends_the_pages_allowance_too`; (31) `jbig2_limited`: `raster::tests::a_picture_whose_jbig2_mask_a_decoder_limit_refuses_is_not_drawn`, `raster::tests::a_limit_refusal_says_nothing_of_the_next_image`, `raster::tests::failed_image_decodes::a_picture_whose_jbig2_mask_is_refused_for_its_work_is_decoded_at_every_draw`; (30) `raster::tests::a_stream_with_a_crypt_filter_of_its_own_is_drawn`; (32) `raster::tests::a_decode_array_with_references_in_it_is_read`, `raster::tests::a_functions_arrays_can_start_with_a_reference` (pdfcraft-render), `font::cid::width_tests::a_width_group_can_start_with_a_reference` (hayro-interpret); (33) `raster::tests::a_reference_to_a_reference_is_what_it_names`, `raster::tests::a_decode_array_with_references_in_it_is_read` (pdfcraft-render), `font::cid::width_tests::a_width_group_can_start_with_a_reference` (hayro-interpret), `xref::tests::an_object_stream_isnt_reached_through_a_reference`, `xref::tests::a_chain_of_object_streams_ends_at_the_limit_of_nested_loads`, `xref::tests::a_chain_of_object_streams_takes_no_more_work_for_two_ways_on`, `xref::tests::a_reference_held_in_either_storage_is_followed`, `xref::tests::a_page_a_reference_is_held_to_is_found_once` (hayro-syntax); (34) `raster::tests::a_rebuilt_table_finds_an_object_whose_header_names_another_generation`, `raster::tests::a_rebuilt_tables_generation_0_stand_in_is_not_a_second_object` (pdfcraft-render), `xref::tests::an_object_stream_and_its_first_member_are_both_enumerated` (hayro-syntax); (35) `filter::lzw_flate::pdfcraft_png_predictor_tests` (hayro-syntax: every filter type at 1–16 bits and 1–4 colours against the PNG specification's byte-by-byte definition, a row ending part-way through a pixel, Average and Paeth over 6-byte pixels), `raster::tests::png_predicted_images::a_four_bit_rgb_image_of_odd_width_with_up_predicted_rows_is_not_sheared`; (36) `raster::tests::a_content_stream_without_endstream_is_drawn_to_its_length`, `raster::tests::a_content_stream_without_endstream_is_not_read_into_a_later_stream`, `raster::tests::hayro_reads_a_stream_without_endstream_to_its_length_before_a_later_one`, `raster::tests::hayro_does_not_end_a_stream_inside_a_longer_word` (pdfcraft-render); (37) `inspect::tests::a_document_encrypting_only_its_embedded_files_opens_without_a_password`; (38) `text::tests::text_on_a_page_whose_box_has_no_area_is_extracted` (pdfcraft-render); (40) `filter::lzw_flate::pdfcraft_png_predictor_tests::an_image_whose_rows_carry_no_png_tags_is_read_as_its_samples`, `an_image_whose_rows_carry_png_tags_still_has_its_predictor_undone`, `data_of_neither_length_decodes_as_it_did`, `rows_of_no_bytes_or_of_more_than_memory_holds_do_not_panic` (hayro-syntax), `raster::tests::png_predicted_images::an_image_and_soft_mask_whose_rows_carry_no_png_tags_draw_their_samples`, `an_image_one_byte_short_of_untagged_rows_still_goes_through_the_predictor`; (41) `raster::tests::group4_fax_with_rows_past_its_height_decodes_to_the_image_height`; (42) `filter::lzw_flate::pdfcraft_tiff_predictor_tests` (hayro-syntax: random 8- and 16-bit rows at 1–7 colours and 1–50 columns against the bit-level path, partial and empty rows, and a row counter showing whole-byte samples skip the bit-level path), `raster::tests::tiff_predicted_images`; (43) `raster::tests::an_update_cut_off_before_its_trailer_shows_its_pages_as_the_model_reads_them`, `raster::tests::a_complete_update_reads_as_before`, `raster::tests::trailing_data_without_objects_reads_as_before`, `raster::tests::a_short_cut_off_update_reads_as_before`, `raster::tests::a_cut_off_update_with_obj_followed_by_a_delimiter_is_rebuilt`, `raster::tests::a_cut_off_update_with_obj_followed_by_a_comment_is_rebuilt`, `raster::tests::trailing_data_with_an_object_header_reads_as_the_model_reads_it`, `raster::tests::an_update_cut_off_before_its_new_pages_keeps_the_older_table`, `raster::tests::an_encrypted_file_with_a_cut_off_update_still_needs_its_password`, `raster::tests::a_rebuild_whose_trailer_lacks_the_encryption_dictionary_is_not_taken` (pdfcraft-render); (44) `raster::tests::a_page_without_a_media_box_is_letter`, `raster::tests::raw_crop_unit_floor_preserves_nearly_zero_fallback` (pdfcraft-render), `pdfcraft-model::tests::{a_page_without_a_media_box_is_letter, a_page_whose_media_box_has_no_area_is_letter}`; (45) `inspect::core::unlinked_update_tests::an_update_without_prev_shows_the_page_tree_and_says_it_was_rebuilt`, `inspect::core::unlinked_update_tests::a_hybrid_update_without_prev_is_rebuilt_as_the_document_model_rebuilds_it` (pdfcraft-render) | | winit | 0.30.13 | Apache-2.0 | (1) on Windows 11 (build 22000 and later), `WM_DPICHANGED` applies the rect Windows suggests. It used to nudge the window onto the monitor `MonitorFromWindow` names, which during a drag is still the monitor being left; the nudge sent another `WM_DPICHANGED`, so a window dragged to a monitor with another scale factor grew and stayed on the wrong monitor at the wrong scale (#324). Backport of winit master 488c036a ([winit#4341](https://github.com/rust-windowing/winit/pull/4341)), which no 0.30 release has; Windows 10 keeps the old adjustment, as on winit master. (2) on Wayland, a seat's text input is destroyed when the seat goes away, not whenever it loses a capability: losing any capability (a touch screen or tablet unplugged, a remote-desktop session ending) destroyed it, and only an added capability recreated it, so IME stopped working until restart. Port of [winit#4747](https://github.com/rust-windowing/winit/pull/4747); (3) on Windows, an IME composition reports its cursor from `GCS_CURSORPOS` only when `WM_IME_COMPOSITION` says it is set, and at the end of the composition otherwise: the Microsoft Korean IME never sets it, so the cursor read as 0 (egui then drew no caret while composing; it now draws it after the syllable). Port of [winit#4746](https://github.com/rust-windowing/winit/pull/4746); (4) on Windows, a hidden window is not maximized until it is shown, and a hidden window shown while maximized gets one `ShowWindow(SW_MAXIMIZE)`: `SW_MAXIMIZE` shows a hidden window, so PdfCraft, which eframe keeps hidden until its first frame is drawn, appeared unpainted (white) when it restored a maximized window, and could lose its focus. Port of [winit#4587](https://github.com/rust-windowing/winit/pull/4587) (the later, focus-safe form of [winit#4575](https://github.com/rust-windowing/winit/pull/4575)); (5) on X11, when allowing or disallowing IME replaces the focused window's input context (egui does it each time a text field gains or loses focus), the new context is focused: no focus event follows, so it stayed unfocused until the window lost and regained focus. Port of [winit#4727](https://github.com/rust-windowing/winit/pull/4727); (6) on macOS, `insertText` commits through IME when a composition was started (`pending_commit`, set by `setMarkedText`) rather than when marked text is left: input methods that end a composition with `unmarkText` before `insertText` lost the text, and Apple Korean dropped the key typed after a syllable (`한` then `5` or `;`); other `insertText` calls stay `KeyboardInput`; a commit outside a key press (a candidate clicked with the mouse, dictation) clears the marked text and leaves no key to swallow, as in [winit#4748](https://github.com/rust-windowing/winit/pull/4748) (without it the next key, such as Backspace, was lost). `validAttributesForMarkedText` lists the underline and marked-clause attributes, without which some input methods don't compose. Port of [winit#4650](https://github.com/rust-windowing/winit/pull/4650), the 0.30 backport of [winit#4651](https://github.com/rust-windowing/winit/pull/4651) (merged on winit master), with the underline attribute spelled as AppKit's `NSUnderlineStyleAttributeName` (`NSUnderline`); (7) on macOS, disallowing IME with a composition in progress (focus leaving a text field) tells the input method to discard its marked text (`discardMarkedText`): it kept composing it once IME was allowed again, so the abandoned syllable came back (`하`, then `가` typed in the next field, became `학`, then `하` + `가`; Japanese `か` then `さ` became `傘`). Port of [winit#4745](https://github.com/rust-windowing/winit/pull/4745); (8) on macOS, text inserted outside a key press without a composition (the emoji picker) is reported as `Ime::Commit`; it was dropped. Port of the emoji-picker part of [winit#4749](https://github.com/rust-windowing/winit/pull/4749) (0.30 backport of [winit#4748](https://github.com/rust-windowing/winit/pull/4748)); its `selectedRange` change for dictation is left out because it switches on press-and-hold for every held key (an accent menu instead of key repeat, and the chosen accent added after the repeated letters); (9) on macOS, a key whose text was committed through IME isn't also sent as `KeyboardInput` unless AppKit forwards it as a command (Enter): Apple Korean answers Space with no syllable in progress with `setMarkedText(" ")`, `insertText(" ")`, `setMarkedText("")`, and the last call reset winit's state, so the space arrived twice. Same goal as [winit#4478](https://github.com/rust-windowing/winit/pull/4478), whose changes don't reach this call order (macOS 14); to be proposed upstream there; (10) on macOS, when the first key in a fresh app reaches Apple Korean's `insertText` as a raw compatibility jamo instead of starting a composition, that key is interpreted once more, synchronously, before it is sent as `KeyboardInput` (g k s typed ㅎㅏㄴ instead of 한); only an unmodified, non-repeated first key whose one callback inserts that same jamo, with IME enabled and nothing composed, focus and input source unchanged, is retried (`ime_startup.rs`). Port of [winit#4744](https://github.com/rust-windowing/winit/pull/4744), with its policy tests.; (11) on X11 without `Xft.dpi`, the scale factor comes from the X screen's configured DPI (its size in pixels and millimetres) before the output's physical density from RandR: Xorg can deliberately run a dense panel at 96 DPI, and only winit applications then looked zoomed; `Xft.dpi` still wins, `WINIT_X11_SCALE_FACTOR` still overrides, and a screen reporting no size keeps the RandR fallback. PdfCraft patch (#1150, replacing the launcher pin proposed in #908). (12) on macOS, `WinitView` answers `mouseDownCanMoveWindow` with the window's `movable_by_window_background` attribute (NO unless asked): NSView answers YES for a view that isn't opaque, so with the transparent title bar and full-size content view PdfCraft uses, AppKit moved the window for any drag that began in the 28 pt title-bar band before the app saw it, and document tabs (which live in that band) could not be dragged to reorder them. The empty strip still moves the window through `ViewportCommand::StartDrag`. PdfCraft patch. The published crate's asset files (`docs/res/*.svg`, `examples/data/*.png`) are left out, so its `window` example doesn't build; (13) on Wayland, pixel-only wheel, wheel-tilt and continuous-source scroll events report `Moved` rather than starting an unterminated finger gesture, so egui follows modifier release instead of keeping Ctrl zoom latched (#991). Finger gestures and unknown sources retain their existing phase policy | (1) `pdfcraft` `tests::winit_carries_the_windows_11_monitor_scale_fix` keeps the patch in the build; the drag itself needs two monitors with different scale factors on Windows 11; (2) `pdfcraft` `tests/winit_wayland_text_input.rs`; the capability change itself needs a Wayland session and an input method; (3) `pdfcraft` `tests/winit_windows_ime_cursor.rs`; composing itself needs the Microsoft Korean IME; (4) `pdfcraft` `tests/winit_windows_hidden_maximized.rs`; the start-up itself needs Windows and a window that was closed maximized; (5) `pdfcraft` `tests/winit_x11_ime_focus.rs`; switching fields itself needs an X11 session and an input method (IBus, fcitx5); (6) `pdfcraft` `tests/winit_macos_ime_commit.rs`; composing itself needs macOS and an input method; (7) `pdfcraft` `tests/winit_macos_ime_discard.rs`; composing itself needs macOS and an input method; (8) `pdfcraft` `tests/winit_macos_emoji_picker.rs`; the picker itself needs macOS; (9) `pdfcraft` `tests/winit_macos_korean_space.rs`; composing itself needs macOS and Apple Korean; (10) `pdfcraft` `tests/winit_macos_korean_first_key.rs` and `ime_startup.rs`'s own tests (run them with `rustc --test`); typing itself needs macOS and Apple Korean; (11) `pdfcraft` `tests/winit_x11_dpi.rs` keeps the patch in the build, and `util::randr` tests the policy (run them in the vendored crate); the scale itself needs an X11 session; (12) `pdfcraft` `tests/winit_macos_title_bar_drag.rs` keeps the patch in the build; the drag itself needs macOS and a pointer; (13) `pdfcraft` `tests/winit_wayland_scroll.rs` compiles the production phase policy and exercises modifier release through egui; live COSMIC confirmation remains open | | egui-wgpu | 0.36.2 | MIT OR Apache-2.0 | (1) `winit::surface_fit`: a window's surface is configured within the device's `max_texture_dimension_2d`, both sides shrunk by one factor that egui then draws at (`pixels_per_point × render_scale`), so the whole window is drawn into the surface (on DX12 it is stretched over the window and lines up with the pointer; wgpu's GL backend copies it unscaled into a corner); the longer side becomes exactly the limit; the depth and MSAA textures follow, and a recreated surface starts again from the window's own size. egui-wgpu configured the window's size as is, and `Surface::configure` panics beyond the limit: a window restored at another monitor's scale (opening an Outlook attachment on a mixed-DPI laptop asked an 8192-pixel device for 8600 × 3423, #577) or stretched across monitors closed PdfCraft, before its window appeared when it happened at start-up ([emilk/egui#8361](https://github.com/emilk/egui/issues/8361)). The published crate leaves out egui's licence files; `LICENSE-APACHE` and `LICENSE-MIT` are from the 0.36.2 tag; (2) `winit::catch_errors`: a new window's surface is configured inside wgpu error scopes (validation, out of memory, internal), and a failure is returned as `WgpuError::ConfigureSurface` (for the root window at start-up eframe returns it as `Error::Wgpu`, which PdfCraft retries with OpenGL); nothing is kept for the window. wgpu's default handler panicked instead, and eframe configures the root surface before the app is created, so a driver that can't configure one closed PdfCraft at launch (a 2015 Intel GPU through wgpu's GL backend timed out waiting for the GPU to go idle, `GpuWaitTimeout`, #519). Unchanged, so still a panic: an error in a later reconfigure (resize, a lost surface), a device lost during the first configure (wgpu reports it only to a callback), and a window shown at zero size (first configured on resize) | (1) `pdfcraft` `tests::a_window_larger_than_the_gpu_limit_gets_a_surface_within_it`, `tests::surface_fit_never_empties_or_overflows_a_surface`, and `tests::vendor_provenance_egui_wgpu_surface_fit_identity`, which keeps the patch in the build and in the paths that size a surface; (2) `pdfcraft` `tests::a_gpu_error_while_a_window_is_set_up_is_returned_not_a_panic` (a real device: an error inside `catch_errors` is returned, not a panic) and `tests::vendor_provenance_egui_wgpu_catch_errors_identity`, which keeps it around `add_surface`'s configure | diff --git a/vendor/lopdf/src/lib.rs b/vendor/lopdf/src/lib.rs index 4c1d845316..ca6d11d753 100644 --- a/vendor/lopdf/src/lib.rs +++ b/vendor/lopdf/src/lib.rs @@ -47,7 +47,7 @@ pub use encryption::{EncryptionState, EncryptionVersion, Permissions}; pub use error::{DecompressError, Error, ParseError, Result}; pub use incremental_document::IncrementalDocument; pub use load_options::{FilterFunc, LoadOptions}; -pub use object_stream::{ObjectStream, ObjectStreamBuilder, ObjectStreamConfig}; +pub use object_stream::{NodeBudget, ObjectStream, ObjectStreamBuilder, ObjectStreamConfig}; pub use outlines::Outline; pub use reader::{PdfMetadata, Reader}; pub use save_options::{SaveOptions, SaveOptionsBuilder}; diff --git a/vendor/lopdf/src/object_stream.rs b/vendor/lopdf/src/object_stream.rs index 5a921b4bab..e60a9f5e09 100644 --- a/vendor/lopdf/src/object_stream.rs +++ b/vendor/lopdf/src/object_stream.rs @@ -1,13 +1,78 @@ use crate::parser; use crate::{Document, Error, Object, ObjectId, Result, Stream}; +use std::cell::Cell; use std::collections::BTreeMap; use std::num::TryFromIntError; use std::str::FromStr; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use log::warn; #[cfg(feature = "rayon")] use rayon::prelude::*; +/// PdfCraft patch: index entries read from one object stream (the same clamp `pdfcraft_cos` +/// applies to `/N`). Entries past it are ignored, so a huge index block cannot allocate an entry +/// list or parse a member per entry. +pub(crate) const MAX_OBJECT_STREAM_MEMBERS: usize = 1_000_000; + +/// PdfCraft patch: objects (every member and everything nested in it) one load may build from +/// all of its object streams together, about one per object of the largest cross-reference +/// table `pdfcraft_cos` accepts (8,388,607). Each stream is bounded by the decompression limit, +/// but a file may hold many, and every two decoded bytes can be an object of a hundred bytes or +/// more; this keeps the worst case near a gigabyte instead of tens. +pub(crate) const MAX_LOAD_NODES: usize = 1 << 23; + +/// PdfCraft patch: the objects parsing may still build, shared by every object stream one load +/// parses (and by rayon workers, hence atomic). Each member reserves its share before parsing +/// and refunds what it did not use, so concurrent members cannot overshoot the total. Public so +/// PdfCraft's tests can load object streams from a small budget. +/// +/// The loaders parse their object streams one after another in object-number order, and a +/// stream whose members could run the budget out parses them in index order (see +/// [`ObjectStream::new_within`]), so the members skipped once it is spent are the same on every +/// load of a file. +pub struct NodeBudget(AtomicUsize); + +impl NodeBudget { + /// A budget of `nodes` objects. + pub fn new(nodes: usize) -> Self { + Self(AtomicUsize::new(nodes)) + } + + /// Take up to `want` nodes and return how many were granted (0 once the budget is spent). + fn reserve(&self, want: usize) -> usize { + // A compare-exchange loop rather than `fetch_update`, which newer toolchains deprecate + // in favour of a `try_update` older than this crate's minimum Rust version. + let mut left = self.0.load(Ordering::Relaxed); + loop { + let take = left.min(want); + match self + .0 + .compare_exchange_weak(left, left - take, Ordering::Relaxed, Ordering::Relaxed) + { + Ok(_) => return take, + Err(now) => left = now, + } + } + } + + /// Give back nodes [`Self::reserve`] granted and parsing did not use. Refunds never exceed + /// what was taken, so the count stays within the starting budget and cannot overflow. + fn refund(&self, nodes: usize) { + self.0.fetch_add(nodes, Ordering::Relaxed); + } + + /// The nodes not yet reserved. + fn left(&self) -> usize { + self.0.load(Ordering::Relaxed) + } + + /// Whether nothing is left to build from. + pub fn spent(&self) -> bool { + self.left() == 0 + } +} + #[derive(Debug)] pub struct ObjectStream { pub objects: BTreeMap, @@ -50,7 +115,33 @@ impl ObjectStream { /// Parse an existing object stream without modifying it, rejecting the /// decoded content if it would exceed `max_decompressed_size` bytes. `None` /// means no limit (the behavior of [`ObjectStream::new`]). + /// + /// PdfCraft patch: members are built from a budget of `MAX_LOAD_NODES` objects of their own. pub fn new_with_limit(stream: &Stream, max_decompressed_size: Option) -> Result { + Self::new_within(stream, max_decompressed_size, &NodeBudget::new(MAX_LOAD_NODES)) + } + + /// PdfCraft patch: [`ObjectStream::new_with_limit`], building members from `budget`, which a + /// load shares between all of its object streams. A member that would exceed its share, or + /// any member once the budget is spent, is skipped with a warning. + /// + /// When the budget holds enough for every member, they are parsed in parallel from a share + /// reserved for this stream alone, which no member can run out; otherwise they are parsed in + /// index order from `budget`. Either way which members are built doesn't depend on the order + /// in which rayon's workers reach them. + pub fn new_within( + stream: &Stream, max_decompressed_size: Option, budget: &NodeBudget, + ) -> Result { + Self::new_members_within(stream, max_decompressed_size, budget, |_| true) + } + + /// PdfCraft patch: [`ObjectStream::new_within`], parsing only the members whose object + /// number `keep` accepts. The others are never parsed, so they spend nothing from `budget`: + /// a loader passes the members it would add (not placed in another stream by the + /// cross-reference table, and not already loaded), and a lookup the one member it wants. + pub(crate) fn new_members_within( + stream: &Stream, max_decompressed_size: Option, budget: &NodeBudget, keep: impl Fn(u32) -> bool, + ) -> Result { let content = match max_decompressed_size { // Object streams are decoded while the document is loaded, so // enforcing the limit here bounds the memory a single stream can use. @@ -75,8 +166,10 @@ impl ObjectStream { let index_block = content.get(..first_offset).ok_or(Error::InvalidOffset(first_offset))?; let numbers_str = std::str::from_utf8(index_block).map_err(|e| Error::InvalidObjectStream(e.to_string()))?; + // PdfCraft patch: read at most `MAX_OBJECT_STREAM_MEMBERS` index entries. let numbers: Vec<_> = numbers_str .split_whitespace() + .take(2 * MAX_OBJECT_STREAM_MEMBERS) .map(|number| u32::from_str(number).ok()) .collect(); let len = numbers.len() / 2 * 2; // Ensure only pairs. @@ -85,6 +178,9 @@ impl ObjectStream { if numbers.len().try_into().ok() != n.checked_mul(2) { warn!("object stream: the object stream dictionary specifies a wrong number of objects") } + if usize::try_from(n).is_ok_and(|n| n > MAX_OBJECT_STREAM_MEMBERS) { + warn!("object stream: /N {n} exceeds {MAX_OBJECT_STREAM_MEMBERS}; later members are ignored") + } // PdfCraft patch: bound members by the next distinct offset and parse each offset // only once, so overlapping or duplicate entries cannot amplify decoded content. @@ -103,44 +199,89 @@ impl ObjectStream { end = offset; } - let chunks_filter_map = |(index, chunk): (usize, &[_])| { - let id = chunk[0]?; - let offset = first_offset.saturating_add(chunk[1]? as usize); + // PdfCraft patch: find each member's bytes first, in index order, so the members can be + // charged to the budget before any is parsed. + let mut members = Vec::new(); + for (index, chunk) in numbers[..len].chunks(2).enumerate() { + let &[Some(id), Some(offset)] = chunk else { continue }; + let offset = first_offset.saturating_add(offset as usize); if offset >= content.len() { warn!("out-of-bounds offset in object stream"); - return None; + continue; } - let &(first_index, extent_end) = extents.get(&offset)?; + let Some(&(first_index, extent_end)) = extents.get(&offset) else { + continue; + }; if index != first_index { warn!("duplicate offset in object stream"); - return None; + continue; + } + if !keep(id) { + continue; } // Skip leading whitespace — some PDFs emit newlines before objects in ObjStm let mut start = offset; - while start < extent_end && content[start].is_ascii_whitespace() { + while start < extent_end && content.get(start).is_some_and(u8::is_ascii_whitespace) { start += 1; } if start >= extent_end { warn!("only whitespace after offset in object stream"); + continue; + } + members.push((id, start, extent_end)); + } + + // PdfCraft patch: each member builds at most `MAX_OBJECT_NODES` objects, reserved from + // `from`; once that is spent the remaining members are skipped unparsed. Every object + // takes at least one byte, so a member reserves no more than its length. + let share = |&(_, start, end): &(u32, usize, usize)| parser::MAX_OBJECT_NODES.min(end - start); + let skipped = AtomicBool::new(false); + let parse_member = |from: &NodeBudget, member: &(u32, usize, usize)| -> Option<(ObjectId, Object)> { + let &(id, start, end) = member; + let granted = from.reserve(share(member)); + if granted == 0 { + skipped.store(true, Ordering::Relaxed); return None; } - let object = parser::direct_object(&content[start..extent_end])?; - - Some(((id, 0), object)) + let nodes = Cell::new(granted); + let object = content + .get(start..end) + .and_then(|bytes| parser::direct_object_within(bytes, &nodes)); + from.refund(nodes.get()); + if object.is_none() && nodes.get() == 0 { + warn!("object stream: member {id} exceeds the object budget and was skipped"); + } + Some(((id, 0), object?)) }; - #[cfg(feature = "rayon")] - let objects = numbers[..len] - .par_chunks(2) - .enumerate() - .filter_map(chunks_filter_map) - .collect(); - #[cfg(not(feature = "rayon"))] - let objects = numbers[..len] - .chunks(2) - .enumerate() - .filter_map(chunks_filter_map) - .collect(); + + let want = members.iter().map(&share).fold(0, usize::saturating_add); + let granted = budget.reserve(want); + let objects: BTreeMap = if granted == want { + // Every member fits, so they are parsed in parallel from this stream's own share: + // its members' reservations add up to it, so none is refused whatever the order. + let own = NodeBudget::new(granted); + #[cfg(feature = "rayon")] + let objects = members + .par_iter() + .filter_map(|member| parse_member(&own, member)) + .collect(); + #[cfg(not(feature = "rayon"))] + let objects = members.iter().filter_map(|member| parse_member(&own, member)).collect(); + budget.refund(own.left()); + objects + } else { + // The budget may run out part-way, so the members are parsed in index order and the + // ones skipped are the same on every load. + budget.refund(granted); + members + .iter() + .filter_map(|member| parse_member(budget, member)) + .collect() + }; + if skipped.load(Ordering::Relaxed) { + warn!("object stream: the object budget is spent; remaining members were skipped"); + } Ok(ObjectStream { objects, diff --git a/vendor/lopdf/src/parser/mod.rs b/vendor/lopdf/src/parser/mod.rs index f0944be526..2cc191419b 100644 --- a/vendor/lopdf/src/parser/mod.rs +++ b/vendor/lopdf/src/parser/mod.rs @@ -3,6 +3,7 @@ use crate::Error; use crate::content::*; use crate::error; use crate::xref::*; +use std::cell::Cell; use std::collections::HashSet; use std::str::{self, FromStr}; @@ -27,6 +28,19 @@ pub(crate) type NomError<'a> = nom::error::Error>; pub(crate) type NomResult<'a, O, E = NomError<'a>> = IResult, O, E>; +/// PdfCraft patch: objects one top-level parse may build, counting every array element and +/// dictionary value at any depth (the same cap as `pdfcraft_cos`'s `MAX_NODES`). Each object +/// costs a hundred bytes or more, so a decoded stream of `0 0 0 ...` otherwise built one per +/// two input bytes: gigabytes from a stream inside the decompression limit. Real arrays (page +/// trees, parent trees, widths) stay far below. +pub(crate) const MAX_OBJECT_NODES: usize = 1 << 22; + +/// PdfCraft patch: the budget of a container parsed outside `_direct_object` (an indirect +/// object's body, a content-stream operand, a CMap or inline-image dictionary). Its contents are +/// charged but the container itself isn't, so it starts one short and the whole object stays +/// within `MAX_OBJECT_NODES`. (`MAX_OBJECT_NODES` is a nonzero constant, so this can't underflow.) +const MAX_NESTED_NODES: usize = MAX_OBJECT_NODES - 1; + #[inline] fn strip_nom(r: NomResult) -> Option { r.ok().map(|(_, o)| o) @@ -270,33 +284,48 @@ fn boolean(input: ParserInput) -> NomResult { .parse(input) } +/// `null`. fn null(input: ParserInput) -> NomResult { map(tag(&b"null"[..]), |_| Object::Null).parse(input) } -fn array(depth: usize) -> impl Fn(ParserInput) -> NomResult> { +/// An array, nested at most `depth` levels. PdfCraft patch: every element is charged to `nodes` +/// (see `_direct_object`), which bounds the element count of one array and of all arrays nested +/// in it. +fn array(depth: usize, nodes: &Cell) -> impl Fn(ParserInput) -> NomResult> { move |input| { delimited( pair(tag(&b"["[..]), space), - many0(_direct_object(depth)), + many0(_direct_object(depth, nodes)), tag(&b"]"[..]), ) .parse(input) } } +/// A dictionary, from a node budget of its own (`MAX_NESTED_NODES`). pub(crate) fn dictionary(input: ParserInput) -> NomResult { - _dictionary(crate::reader::MAX_NESTING_DEPTH)(input) + let nodes = Cell::new(MAX_NESTED_NODES); + _dictionary(crate::reader::MAX_NESTING_DEPTH, &nodes)(input) } -fn _dictionary(depth: usize) -> impl Fn(ParserInput) -> NomResult { - move |input| delimited(pair(tag(&b"<<"[..]), space), inner_dictionary(depth), tag(&b">>"[..])).parse(input) +/// `<< ... >>`, nested at most `depth` levels, its values charged to `nodes`. +fn _dictionary(depth: usize, nodes: &Cell) -> impl Fn(ParserInput) -> NomResult { + move |input| { + delimited( + pair(tag(&b"<<"[..]), space), + inner_dictionary(depth, nodes), + tag(&b">>"[..]), + ) + .parse(input) + } } -fn inner_dictionary(depth: usize) -> impl Fn(ParserInput) -> NomResult { +/// A dictionary's key-value pairs, without its delimiters; each value is charged to `nodes`. +fn inner_dictionary(depth: usize, nodes: &Cell) -> impl Fn(ParserInput) -> NomResult { move |input| { fold_many0( - pair(terminated(name, space), _direct_object(depth)), + pair(terminated(name, space), _direct_object(depth, nodes)), Dictionary::new, |mut dict, (key, value)| { dict.set(key, value); @@ -307,7 +336,9 @@ fn inner_dictionary(depth: usize) -> impl Fn(ParserInput) -> NomResult NomResult { + let nodes = Cell::new(MAX_NESTED_NODES); delimited( ( digit1, @@ -323,7 +354,7 @@ pub(crate) fn dict_dup(input: ParserInput) -> NomResult { terminated( pair( terminated(name, space), - _direct_object(crate::reader::MAX_NESTING_DEPTH), + _direct_object(crate::reader::MAX_NESTING_DEPTH, &nodes), ), pair(tag(&b"def"[..]), multispace1), ), @@ -434,7 +465,8 @@ fn reference(input: ParserInput) -> NomResult { map(terminated(object_id, tag(&b"R"[..])), Object::Reference).parse(input) } -fn _direct_objects(depth: usize) -> impl Fn(ParserInput) -> NomResult { +/// Any direct object, its nested arrays and dictionaries at most `depth` levels deep. +fn _direct_objects(depth: usize, nodes: &Cell) -> impl Fn(ParserInput) -> NomResult { move |input| { alt(( null, @@ -445,34 +477,55 @@ fn _direct_objects(depth: usize) -> impl Fn(ParserInput) -> NomResult { map(name, Object::Name), map(literal_string, Object::string_literal), hexadecimal_string, - map(array(depth), Object::Array), - map(_dictionary(depth), Object::Dictionary), + map(array(depth, nodes), Object::Array), + map(_dictionary(depth, nodes), Object::Dictionary), )) .parse(input) } } -fn _direct_object(depth: usize) -> impl Fn(ParserInput) -> NomResult { +/// A direct object and the whitespace after it, charged one node once built; a spent budget +/// or a depth of 0 is a nom `Failure`. +fn _direct_object(depth: usize, nodes: &Cell) -> impl Fn(ParserInput) -> NomResult { move |input| { if depth == 0 { return Err(nom::Err::Failure(NomError::from_error_kind(input, ErrorKind::TooLarge))); } - terminated(_direct_objects(depth - 1), space).parse(input) + let (rest, object) = terminated(_direct_objects(depth - 1, nodes), space).parse(input)?; + // PdfCraft patch: charge each object once it is built, so a failed alternative costs + // nothing; its children were charged as they were built. A spent budget is a Failure, + // which ends the whole parse instead of backtracking into another alternative. + let Some(left) = nodes.get().checked_sub(1) else { + return Err(nom::Err::Failure(NomError::from_error_kind(input, ErrorKind::TooLarge))); + }; + nodes.set(left); + Ok((rest, object)) } } +/// One direct object from a fresh node budget. PdfCraft patch: object streams now parse through +/// `direct_object_within`; only tests use this. +#[cfg(test)] pub fn direct_object(input: ParserInput) -> Option { - strip_nom(_direct_object(crate::reader::MAX_NESTING_DEPTH)(input)) + direct_object_within(input, &Cell::new(MAX_OBJECT_NODES)) } +/// PdfCraft patch: parse one direct object, building at most `nodes` objects (the object itself +/// and everything nested in it) and leaving the unspent count in `nodes`. +pub(crate) fn direct_object_within(input: ParserInput, nodes: &Cell) -> Option { + strip_nom(_direct_object(crate::reader::MAX_NESTING_DEPTH, nodes)(input)) +} + +/// An indirect object's body (a stream or a direct object), from a node budget of its own. fn object<'a>( input: ParserInput<'a>, reader: &Reader, already_seen: &mut HashSet, recover_stream_length: bool, recovery_bound: Option, ) -> NomResult<'a, Object> { + let nodes = Cell::new(MAX_NESTED_NODES); terminated( alt(( |input| stream(input, reader, already_seen, recover_stream_length, recovery_bound), - _direct_objects(crate::reader::MAX_NESTING_DEPTH), + _direct_objects(crate::reader::MAX_NESTING_DEPTH, &nodes), )), space, ) @@ -689,7 +742,9 @@ fn operator(input: ParserInput) -> NomResult { .parse(input) } +/// One content-stream operand, from a node budget of its own. fn operand(input: ParserInput) -> NomResult { + let nodes = Cell::new(MAX_NESTED_NODES); terminated( alt(( null, @@ -699,7 +754,7 @@ fn operand(input: ParserInput) -> NomResult { map(name, Object::Name), map(literal_string, Object::string_literal), hexadecimal_string, - map(array(crate::reader::MAX_NESTING_DEPTH), Object::Array), + map(array(crate::reader::MAX_NESTING_DEPTH, &nodes), Object::Array), map(dictionary, Object::Dictionary), )), content_space, @@ -724,8 +779,10 @@ fn inline_image(input: ParserInput) -> NomResult<(Vec, String)> { preceded(pair(tag(&b"BI"[..]), content_space), cut(inline_image_impl)).parse(input) } +/// An inline image after `BI`: its dictionary (from a node budget of its own) and data. fn inline_image_impl(input: ParserInput) -> NomResult<(Vec, String)> { - let (input, stream_dict) = inner_dictionary(crate::reader::MAX_NESTING_DEPTH).parse(input)?; + let nodes = Cell::new(MAX_NESTED_NODES); + let (input, stream_dict) = inner_dictionary(crate::reader::MAX_NESTING_DEPTH, &nodes).parse(input)?; let (input, _) = pair(tag(&b"ID"[..]), content_space).parse(input)?; match image_data_stream(input, stream_dict) { Ok((input, stream)) => { @@ -1299,6 +1356,7 @@ EI"; .unwrap(); } + /// Arrays nested past the depth limit are an error, not a stack overflow. #[test] fn deeply_nested_array_is_rejected() { on_big_stack(|| { @@ -1306,11 +1364,13 @@ EI"; let mut input = Vec::with_capacity(depth * 2); input.extend(std::iter::repeat_n(b'[', depth)); input.extend(std::iter::repeat_n(b']', depth)); - let result = _direct_object(crate::reader::MAX_NESTING_DEPTH)(input.as_slice()); + let result = + _direct_object(crate::reader::MAX_NESTING_DEPTH, &Cell::new(MAX_OBJECT_NODES))(input.as_slice()); assert!(result.is_err()); }); } + /// Dictionaries nested past the depth limit are an error, not a stack overflow. #[test] fn deeply_nested_dictionary_is_rejected() { on_big_stack(|| { @@ -1322,15 +1382,17 @@ EI"; for _ in 0..depth { input.extend_from_slice(b">>"); } - let result = _direct_object(crate::reader::MAX_NESTING_DEPTH)(input.as_slice()); + let result = + _direct_object(crate::reader::MAX_NESTING_DEPTH, &Cell::new(MAX_OBJECT_NODES))(input.as_slice()); assert!(result.is_err()); }); } + /// Nesting well within the limit still parses. #[test] fn modestly_nested_array_still_parses() { let input = b"[[[[[1]]]]]"; - let obj = _direct_object(crate::reader::MAX_NESTING_DEPTH)(test_span(input)); + let obj = _direct_object(crate::reader::MAX_NESTING_DEPTH, &Cell::new(MAX_OBJECT_NODES))(test_span(input)); assert!(obj.is_ok()); } } diff --git a/vendor/lopdf/src/reader.rs b/vendor/lopdf/src/reader.rs index d7e33b7dc4..f4da53e3c1 100644 --- a/vendor/lopdf/src/reader.rs +++ b/vendor/lopdf/src/reader.rs @@ -1,13 +1,13 @@ use log::{error, warn}; use std::cmp; -use std::collections::{BTreeMap, HashMap, HashSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::convert::TryInto; #[cfg(not(feature = "async"))] use std::fs::File; #[cfg(not(feature = "async"))] use std::io::Read; use std::path::Path; -use std::sync::Mutex; +use std::sync::{Mutex, PoisonError}; #[cfg(feature = "rayon")] use rayon::prelude::*; @@ -22,7 +22,7 @@ use crate::common_data_structures; use crate::encryption::{self, EncryptionState}; use crate::error::{ParseError, XrefError}; use crate::load_options::{FilterFunc, LoadOptions}; -use crate::object_stream::ObjectStream; +use crate::object_stream::{MAX_LOAD_NODES, NodeBudget, ObjectStream}; use crate::parser; use crate::xref::{Xref, XrefEntry, XrefType}; use crate::{Dictionary, Document, Error, IncrementalDocument, Object, ObjectId, Result}; @@ -838,6 +838,8 @@ impl Reader<'_> { Ok(self.document) } + /// Load an encrypted document: read every object's raw bytes first, then decrypt and parse + /// them, object streams from one shared node budget. fn load_encrypted_document(&mut self, _filter_func: Option) -> Result<()> { // First, extract all raw object bytes without parsing let entries: Vec<_> = self @@ -849,6 +851,8 @@ impl Reader<'_> { .collect(); let mut object_streams = Vec::new(); + // PdfCraft patch: one object budget for every object stream this load parses. + let node_budget = NodeBudget::new(MAX_LOAD_NODES); for (obj_num, entry) in entries { match entry { @@ -894,8 +898,9 @@ impl Reader<'_> { } } - let mut streams_to_process: std::collections::HashMap> = - std::collections::HashMap::new(); + // PdfCraft patch: object streams in object-number order, so the members a spent node + // budget skips are the same on every load. + let mut streams_to_process: BTreeMap> = BTreeMap::new(); for (obj_num, container_id, index) in object_streams { streams_to_process .entry(container_id) @@ -907,7 +912,11 @@ impl Reader<'_> { if let Some(container_obj) = self.document.objects.get(&(container_id, 0)) && let Ok(stream) = container_obj.as_stream() { - match ObjectStream::new_with_limit(stream, self.max_decompressed_size) { + // PdfCraft patch: parse only the members the table places in this stream, so + // the others spend nothing from the budget. + let listed: BTreeSet = objects_in_stream.iter().map(|&(obj_num, _)| obj_num).collect(); + let keep = |obj_num: u32| listed.contains(&obj_num); + match ObjectStream::new_members_within(stream, self.max_decompressed_size, &node_budget, keep) { Ok(object_stream) => { for (obj_num, _index) in objects_in_stream { let obj_id = (obj_num, 0); @@ -960,10 +969,12 @@ impl Reader<'_> { self.document.reference_table = xref; } + /// Load every object in the cross-reference table, expanding object streams from one node + /// budget shared by the whole load. fn load_objects_raw(&mut self, filter_func: Option) -> Result<()> { let is_encrypted = self.document.trailer.get(b"Encrypt").is_ok(); let zero_length_streams = Mutex::new(vec![]); - let object_streams = Mutex::new(vec![]); + let object_stream_ids = Mutex::new(vec![]); // Build a map of which container each compressed object belongs to // according to the xref. This prevents stale ObjStm copies (e.g., from @@ -1014,36 +1025,10 @@ impl Reader<'_> { if let Ok(stream) = object.as_stream() { if stream.dict.has_type(b"ObjStm") && !is_encrypted { - let obj_stream = match ObjectStream::new_with_limit(stream, self.max_decompressed_size) { - Ok(obj_stream) => obj_stream, - // Not an encryption-related loss, so it obeys the same - // strict-mode policy as any other load error. - Err(e) => { - error!("Object stream load error for {object_id:?}: {e:?}"); - return if self.strict { Err(e) } else { Ok(None) }; - } - }; - let container_id = object_id.0; - let mut object_streams = object_streams.lock().unwrap(); - if let Some(filter_func) = filter_func { - let objects: BTreeMap<(u32, u16), Object> = obj_stream - .objects - .into_iter() - .filter(|((obj_num, _), _)| { - compressed_obj_containers - .get(obj_num) - .is_none_or(|&c| c == container_id) - }) - .filter_map(|(object_id, mut object)| filter_func(object_id, &mut object)) - .collect(); - object_streams.extend(objects); - } else { - object_streams.extend(obj_stream.objects.into_iter().filter(|((obj_num, _), _)| { - compressed_obj_containers - .get(obj_num) - .is_none_or(|&c| c == container_id) - })); - } + // PdfCraft patch: object streams are expanded after this parallel pass, + // one at a time in object-number order (below). + let mut object_stream_ids = object_stream_ids.lock().unwrap_or_else(PoisonError::into_inner); + object_stream_ids.push(object_id); } else if stream.content.is_empty() { let mut zero_length_streams = zero_length_streams.lock().unwrap(); zero_length_streams.push(object_id); @@ -1079,9 +1064,59 @@ impl Reader<'_> { .collect::>>()?; } - // Only add entries, but never replace entries - for (id, entry) in object_streams.into_inner().unwrap() { - self.document.objects.entry(id).or_insert(entry); + // PdfCraft patch: expand the object streams one at a time in object-number order, from one + // node budget for the whole load, so many streams, each inside the decompression limit, + // cannot together build billions of objects, and the members a spent budget skips are the + // same on every load. A stream parses only the members it would add: none the table + // places in another stream (stale copies), and none already loaded, as an ordinary object + // or from an earlier stream, since a member never replaces an object. Those spend nothing. + // Members the table doesn't list at all are still parsed (a rebuilt table lists no + // compressed objects) and share the budget. + let node_budget = NodeBudget::new(MAX_LOAD_NODES); + let mut object_stream_ids = object_stream_ids.into_inner().unwrap_or_else(PoisonError::into_inner); + object_stream_ids.sort_unstable(); + object_stream_ids.dedup(); + for object_id in object_stream_ids { + let container_id = object_id.0; + let loaded = &self.document.objects; + let keep = |obj_num: u32| { + compressed_obj_containers + .get(&obj_num) + .is_none_or(|&c| c == container_id) + && !loaded.contains_key(&(obj_num, 0)) + }; + let parsed = match loaded.get(&object_id).map(Object::as_stream) { + Some(Ok(stream)) => { + ObjectStream::new_members_within(stream, self.max_decompressed_size, &node_budget, keep) + } + _ => continue, + }; + match parsed { + Ok(obj_stream) => { + let members: BTreeMap<_, _> = match filter_func { + Some(filter_func) => obj_stream + .objects + .into_iter() + .filter_map(|(object_id, mut object)| filter_func(object_id, &mut object)) + .collect(), + None => obj_stream.objects, + }; + // Only add entries, but never replace entries + for (id, entry) in members { + self.document.objects.entry(id).or_insert(entry); + } + } + // Not an encryption-related loss, so it obeys the same strict-mode policy as + // any other load error; a lenient load drops the stream, as it drops any object + // that fails to load. + Err(e) => { + error!("Object stream load error for {object_id:?}: {e:?}"); + if self.strict { + return Err(e); + } + self.document.objects.remove(&object_id); + } + } } for object_id in zero_length_streams.into_inner().unwrap() { @@ -1174,8 +1209,14 @@ impl Reader<'_> { let container_id = (container_id, 0); let container_obj = self.get_object(container_id, already_seen)?; let container_stream = container_obj.as_stream()?; - let object_stream = ObjectStream::new_with_limit(container_stream, self.max_decompressed_size)?; - object_stream.objects.get(&id).cloned().ok_or(Error::MissingXrefEntry) + // PdfCraft patch: a lookup parses only the member it wants, from one object's budget + // rather than a load's, so the members before it can't spend that budget first. + let budget = NodeBudget::new(parser::MAX_OBJECT_NODES); + let mut object_stream = + ObjectStream::new_members_within(container_stream, self.max_decompressed_size, &budget, |obj_num| { + obj_num == id.0 + })?; + object_stream.objects.remove(&id).ok_or(Error::MissingXrefEntry) } pub fn get_object(&self, id: ObjectId, already_seen: &mut HashSet) -> Result { From e51654bc836689679c35356c09246880f270ad29 Mon Sep 17 00:00:00 2001 From: Sidd Date: Wed, 11 Nov 2026 12:18:32 +0530 Subject: [PATCH 29/60] fix: anchor comments to highlighted text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Fixed selected-text comments that were previously shown as separate sticky notes, often covering text and making their target unclear. Comments are now attached directly to highlight annotations, with a small marker and connector beside the highlighted word. Hovering shows only the comment text, while normal highlights and other annotation types keep their existing behavior. Fixes #3759 ## What this changes Fixes the confusing behavior when adding comments to selected text. - Selected-text comments are stored as highlight annotations. - Commented highlights display a small message marker and connector. - Hovering a commented highlight shows only the comment text. - Normal highlights and other annotation types retain their existing display. - Cancelling a selected-text comment clears its pending selection state. ### Screenshots ### Before Screenshot from 2026-10-21 04-18-37 ### After image ## How you verified it - [x] `git diff --check` - [x] `cargo fmt --check` - [x] clippy with `-D warnings` on the changed crate - [x] tests pass on the changed crate - [x] `cargo xtask layers` and `cargo xtask assets` — not applicable Manual verification was performed with a PDF containing selectable text: - Added a comment to a selected word. - Confirmed the word remains highlighted. - Confirmed the comment marker appears near the highlighted word. - Confirmed the popup displays the comment text. - Confirmed ordinary highlights remain visually distinct. Cargo tests could not be run in the agent environment. ## Never-crash - [x] No new `unwrap`/`expect`/`panic!`/`unsafe` - [x] No new input-derived indexing or unsafe arithmetic - [x] No crash fix is included in this PR ## Provenance - [x] No assets added - [x] No fonts added - [x] No GPL/AGPL code copied - [x] Behavior was implemented from the existing PDF annotation model and project behavior - [x] No new asset requires attribution ## Security - [x] No path or automation behavior changed - [x] No MCP or UI control-channel behavior changed - [x] No outbound network access added ## Parity and reach - [ ] `parity/acrobat-features.toml` updated — not applicable - [ ] New automation tool — not applicable - [ ] Documentation update — not required for this UI bug fix ## Anything a reviewer should look at first The main behavior change is in the selected-text comment path: it now creates a highlighted annotation carrying the comment instead of an independent sticky note. This preserves the selected text as the comment’s visual anchor while keeping normal sticky-note placement unchanged. ## Summary by CodeRabbit * **New Features** * Comments on selected text are saved as highlights, preserving the text location. * Choosing the sticky-note tool and clicking elsewhere replaces the selection-anchored composer and places a comment at the new location. * Comments with text markup display a marker on the page, including near page edges. Hovering over markers shows the author and comment, and markers remain selectable across page rotations and zoom levels. * **Bug Fixes** * Markers take priority when selecting or hovering over overlapping annotations. Markers are not shown for comments without text or outside the page. --- crates/ui-egui/src/canvas/selection.rs | 15 +- crates/ui-egui/src/tools/comments/composer.rs | 9 +- crates/ui-egui/src/tools/comments/input.rs | 13 +- crates/ui-egui/src/tools/comments/paint.rs | 15 +- crates/ui-egui/src/tools/comments/state.rs | 47 +++++- crates/ui-egui/src/tools/comments/tests.rs | 136 ++++++++++++++++++ crates/ui-egui/tests/it/comments.rs | 96 ++++++++++++- 7 files changed, 309 insertions(+), 22 deletions(-) diff --git a/crates/ui-egui/src/canvas/selection.rs b/crates/ui-egui/src/canvas/selection.rs index f50bbcfa70..3d465a0c8e 100644 --- a/crates/ui-egui/src/canvas/selection.rs +++ b/crates/ui-egui/src/canvas/selection.rs @@ -250,16 +250,13 @@ pub(super) fn selection_actions( view.clear_selection(); match action { SelectionAction::Note | SelectionAction::Replace => { - if matches!(action, SelectionAction::Replace) { + let kind = if matches!(action, SelectionAction::Replace) { view.comments.replace = Some(quads); - } - view.comments.composer = Some(comments::Composer { - page, - at, - kind: if matches!(action, SelectionAction::Replace) { comments::ComposerKind::Replace } else { comments::ComposerKind::Note }, - text: String::new(), - focus: true, - }); + comments::ComposerKind::Replace + } else { + comments::ComposerKind::TextNote { quads } + }; + view.comments.composer = Some(comments::Composer { page, at, kind, text: String::new(), focus: true }); } SelectionAction::Markup(tool) => { // Every markup button carries a markup tool; anything else adds nothing. diff --git a/crates/ui-egui/src/tools/comments/composer.rs b/crates/ui-egui/src/tools/comments/composer.rs index 5c765f7f41..c4fb1d7cbc 100644 --- a/crates/ui-egui/src/tools/comments/composer.rs +++ b/crates/ui-egui/src/tools/comments/composer.rs @@ -2,8 +2,8 @@ use super::geometry::{drawn_text_box_rect, page_rotation, text_box_rect}; use super::paint::{edited_in_view, new_comment}; use super::state::{ComposerKind, PageCx}; use super::{ - Color32, CommentPrefs, CommentTool, CornerRadius, DocInfo, DocView, Edit, LineEnding, NoteIcon, PagePointer, QuickTool, Shape, Tokens, pos2, - theme, + Color32, CommentPrefs, CommentTool, CornerRadius, DocInfo, DocView, Edit, LineEnding, Markup, NoteIcon, PagePointer, QuickTool, Shape, Tokens, + pos2, theme, }; use crate::page_keys::PageKeys; @@ -38,6 +38,7 @@ pub(crate) fn composer(ctx: &egui::Context, keys: PageKeys, view: &mut DocView, let c = view.comments.composer.as_mut()?; let title = match c.kind { ComposerKind::Note => "Sticky note", + ComposerKind::TextNote { .. } => "Add comment", ComposerKind::TextBox { .. } => "Text box", ComposerKind::Callout { .. } => "Callout", ComposerKind::Caret { .. } => "Inserted text", @@ -125,6 +126,10 @@ pub(crate) fn composer(ctx: &egui::Context, keys: PageKeys, view: &mut DocView, view.comments.tool_done = true; Some(new_comment(&cx, CommentTool::Note, Shape::Note { at: c.at, icon: NoteIcon::Comment }, text)) } + ComposerKind::TextNote { quads } => { + view.comments.tool_done = true; + Some(new_comment(&cx, CommentTool::Highlight, Shape::TextMarkup { kind: Markup::Highlight, quads }, text)) + } ComposerKind::TextBox { size } => { view.comments.tool_done = true; let rotation = page_rotation(info, page); diff --git a/crates/ui-egui/src/tools/comments/input.rs b/crates/ui-egui/src/tools/comments/input.rs index 469ff5a262..79b69a6eba 100644 --- a/crates/ui-egui/src/tools/comments/input.rs +++ b/crates/ui-egui/src/tools/comments/input.rs @@ -211,10 +211,15 @@ pub(crate) fn describe(cx: &PageCx<'_>, view: &mut DocView) { .comments() // An open pop-up already shows the comment: no tooltip over it. .filter(|a| !a.popup.is_some_and(|w| w.open)) - .filter(|a| cx.screen_rects(a).any(|r| r.contains(p))) - .filter_map(|a| hover_tip(a, cx.prefs).map(|text| (a.index, text))) - .max_by_key(|(index, _)| *index); - if let Some((_, text)) = top { + .filter_map(|a| { + let marker = cx.comment_marker_rect(a).is_some_and(|r| r.contains(p)); + if !marker && !cx.screen_rects(a).any(|r| r.contains(p)) { + return None; + } + hover_tip(a, cx.prefs).map(|text| (marker, a.index, text)) + }) + .max_by_key(|(marker, index, _)| (*marker, *index)); + if let Some((_, _, text)) = top { view.cursor.describe(Subject::Comment, text); } } diff --git a/crates/ui-egui/src/tools/comments/paint.rs b/crates/ui-egui/src/tools/comments/paint.rs index 8cdcb03c81..cb85b1c9a0 100644 --- a/crates/ui-egui/src/tools/comments/paint.rs +++ b/crates/ui-egui/src/tools/comments/paint.rs @@ -3,7 +3,7 @@ use super::popups::paint_connectors; use super::state::{CommentView, Gesture, PageCx, SETTLE_SECS}; use super::{ Annotation, Color32, CommentTool, CornerRadius, DocInfo, DocView, Edit, LineEnding, NewAnnotation, Pos2, QuickTool, Rect, SELECT_BLUE, Shape, - Stroke, color32, pos2, vec2, + Stroke, color32, icons, pos2, vec2, }; use crate::page_router::{CommentPart, Hit, Owner}; @@ -21,6 +21,19 @@ pub(super) fn hover_outline<'c>(cx: &'c PageCx<'_>, cv: &CommentView, hover: Opt /// pointer. pub(crate) fn paint_page(ui: &egui::Ui, painter: &egui::Painter, cx: &PageCx<'_>, view: &DocView, hover: Option<&Hit>) { let cv = &view.comments; + // All commented text markup, including imported markup, gets a viewer-only + // marker. Plain markup has none; comment_marker_rect owns that distinction. + for a in cx.comments() { + if let Some(marker) = cx.comment_marker_rect(a) + && let Some(target) = cx.screen_rects(a).last() + { + painter.line_segment( + [marker.left_bottom(), pos2(target.right(), target.top())], + Stroke::new(1.0, Color32::from_rgba_unmultiplied(0xD4, 0x8A, 0x00, 150)), + ); + icons::paint(ui, marker, "message-square", 12.0, Color32::from_rgb(0xD4, 0x8A, 0x00)); + } + } let pointer = cx.ptr.pos; // A redaction mark shows its fill while the pointer is over it, as Acrobat's does, so the // marked area reads before anything is applied. diff --git a/crates/ui-egui/src/tools/comments/state.rs b/crates/ui-egui/src/tools/comments/state.rs index e6f9ae75b8..de3265dbd5 100644 --- a/crates/ui-egui/src/tools/comments/state.rs +++ b/crates/ui-egui/src/tools/comments/state.rs @@ -1,7 +1,7 @@ use std::collections::HashMap; use super::geometry::{corner_ratio, frame, is_markup, resized}; -use super::{Annotation, CommentPrefs, CommentTool, DocInfo, Edit, PagePointer, PageXform, Pos2, QuickTool, Rect}; +use super::{Annotation, CommentPrefs, CommentTool, DocInfo, Edit, PagePointer, PageXform, Pos2, QuickTool, Rect, pos2, vec2}; use crate::page_keys::PageKeys; /// The user's login name, which Acrobat uses as the default comment author. @@ -36,9 +36,13 @@ pub enum Gesture { Place { xfdf: String, bounds: [f64; 4], count: usize, outlines: Vec }, } -#[derive(Clone, Copy, Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq)] pub enum ComposerKind { Note, + /// A comment anchored to selected text; replacing the composer drops its quads. + TextNote { + quads: Vec<[f64; 8]>, + }, /// A text box; `size` is the box dragged out (displayed width and height, in user space /// units), `None` after a click: then the box fits its text. TextBox { @@ -602,7 +606,7 @@ impl PageCx<'_> { self.xf.user_rect(self.info, self.page, frame(a, self.rotation())) } - /// Top-level comments on this page, in paint order. + /// Top-level comments on this page, in inspection order. pub(super) fn comments(&self) -> impl Iterator { self.info.annotations.iter().filter(move |a| !self.hidden && a.page == self.page && a.is_comment() && a.subtype != "Popup") } @@ -623,6 +627,32 @@ impl PageCx<'_> { .chain(fallback) } + /// A viewer-only marker for commented text markup intersecting the page. + pub(crate) fn comment_marker_rect(&self, a: &Annotation) -> Option { + if !is_markup(&a.subtype) || a.contents.as_deref().is_none_or(|s| s.trim().is_empty()) { + return None; + } + let r = self.screen_rects(a).last()?; + let size = vec2(12.0, 12.0); + let page = self.xf.rect; + let marker_max = page.max - size; + let desired = pos2(r.right() + 3.0, r.top() - 14.0); + // Keep the fixed-size marker visible at page edges. At extremely small zooms, + // omit it rather than covering a page that is smaller than the marker itself. + if !page.is_finite() + || !r.is_finite() + || !desired.is_finite() + || !page.intersects(r) + || page.width() < size.x + || page.height() < size.y + || marker_max.x < page.min.x + || marker_max.y < page.min.y + { + return None; + } + Some(Rect::from_min_size(desired.clamp(page.min, marker_max), size)) + } + /// A redaction mark's areas: one per marked quad, or its rectangle. pub(super) fn redact_rects(&self, a: &Annotation) -> Vec { if a.quads.is_empty() { @@ -655,9 +685,16 @@ impl PageCx<'_> { .filter_map(|s| { let (page, index) = s.parent?; let caret = self.get(index).filter(|c| page == self.page && c.subtype == "Caret")?; - Some((s.index, caret)) + Some((false, s.index, caret)) }); - self.comments().filter(under).map(|a| (a.index, a)).chain(strikes).max_by_key(|(i, _)| *i).map(|(_, a)| a) + self.comments() + .filter_map(|a| { + let marker = self.comment_marker_rect(a).is_some_and(|r| r.expand(3.0).contains(p)); + (marker || under(&a)).then_some((marker, a.index, a)) + }) + .chain(strikes) + .max_by_key(|(marker, index, _)| (*marker, *index)) + .map(|(_, _, a)| a) } pub(crate) fn get(&self, index: usize) -> Option<&Annotation> { diff --git a/crates/ui-egui/src/tools/comments/tests.rs b/crates/ui-egui/src/tools/comments/tests.rs index 01086e788e..18c511debb 100644 --- a/crates/ui-egui/src/tools/comments/tests.rs +++ b/crates/ui-egui/src/tools/comments/tests.rs @@ -561,6 +561,107 @@ fn text_boxes_fit_their_text() { assert!(long[3] - long[1] > 50.0); } +#[test] +fn markup_comment_markers_skip_rounded_inverted_clamp_bounds() { + let prefs = CommentPrefs::default(); + let ptr = PagePointer::default(); + let info = DocInfo { + pages: vec![PageInfo { width: 12.0, height: 12.0, label: String::new(), crop: [0.0, 0.0, 12.0, 12.0], rotation: 0 }], + ..Default::default() + }; + let a = Annotation { rect: [0.0, 0.0, 12.0, 12.0], contents: Some("Review".into()), ..annotation("Highlight", None, false) }; + for min in [pos2(1.0e-7, 0.0), pos2(0.0, 1.0e-7)] { + let page = Rect::from_min_max(min, pos2(12.0, 12.0)); + assert_eq!(page.size(), vec2(12.0, 12.0)); + let marker_max = page.max - vec2(12.0, 12.0); + assert!(marker_max.x < page.min.x || marker_max.y < page.min.y); + let xf = PageXform { rect: page, rot: 0, pw: 12.0, ph: 12.0 }; + let cx = PageCx { + page: 0, + xf: &xf, + ptr: &ptr, + info: &info, + tool: QuickTool::Select, + prefs: &prefs, + allowed: true, + hidden: false, + redact_mark: ([0.0; 3], 1.0), + keys: crate::page_keys::PageKeys::new(false), + }; + assert!(page.intersects(cx.screen_rects(&a).last().unwrap())); + assert!(cx.comment_marker_rect(&a).is_none()); + } +} + +#[test] +fn markup_comment_markers_stay_inside_turned_and_zoomed_pages() { + let prefs = CommentPrefs::default(); + for (page_rotation, rotation) in [(0, 0), (90, 0), (180, 0), (270, 0), (0, 90), (0, 180), (0, 270), (90, 270)] { + for zoom in [0.25, 1.0, 4.0] { + let (pw, ph) = if page_rotation % 180 == 0 { (300.0, 200.0) } else { (200.0, 300.0) }; + let (w, h) = if rotation % 180 == 0 { (pw, ph) } else { (ph, pw) }; + let xf = PageXform { rect: Rect::from_min_size(pos2(40.0, 60.0), vec2(w, h) * zoom), rot: rotation, pw, ph }; + let info = DocInfo { + pages: vec![PageInfo { width: pw, height: ph, label: String::new(), crop: [0.0, 0.0, 300.0, 200.0], rotation: page_rotation }], + ..Default::default() + }; + let cx = PageCx { + page: 0, + xf: &xf, + ptr: &PagePointer::default(), + info: &info, + tool: QuickTool::Select, + prefs: &prefs, + allowed: true, + hidden: false, + redact_mark: ([0.0; 3], 1.0), + keys: crate::page_keys::PageKeys::new(false), + }; + for rect in [[0.0, 180.0, 30.0, 200.0], [270.0, 180.0, 300.0, 200.0], [270.0, 0.0, 300.0, 20.0], [0.0, 0.0, 30.0, 20.0]] { + let [x0, y0, x1, y1] = rect; + let mut a = Annotation { + page: 0, + subtype: "Highlight".into(), + author: Some("Ada".into()), + contents: Some("Review".into()), + modified: None, + name: None, + in_reply_to: None, + parent: None, + grouped: false, + rect, + color: None, + index: 0, + state: None, + quads: vec![[x0, y1, x1, y1, x0, y0, x1, y0]], + locked: false, + intent: None, + fill_sign: false, + no_rotate: false, + points: Vec::new(), + text_box: None, + popup: None, + }; + let marker = cx.comment_marker_rect(&a).expect("commented markup has a marker"); + assert!((marker.width() - 12.0).abs() < 0.001 && (marker.height() - 12.0).abs() < 0.001, "{marker:?}"); + assert!(xf.rect.contains_rect(marker), "page/view rotation {page_rotation}/{rotation}, zoom {zoom}, anchor {rect:?}: {marker:?}"); + let mut top = a.clone(); + top.index = 1; + // Inspection order can differ from /Annots order. Picking the marker + // must select the same topmost comment as hovering its marked text. + let hit_info = DocInfo { annotations: vec![top, a.clone()], ..info.clone() }; + let hit_cx = PageCx { info: &hit_info, ..cx }; + assert_eq!(hit_cx.comment_at(marker.center()).map(|a| a.index), Some(1)); + a.contents = None; + assert!(cx.comment_marker_rect(&a).is_none(), "plain markup has no marker"); + a.contents = Some("Review".into()); + a.quads = vec![[1000.0, 1010.0, 1010.0, 1010.0, 1000.0, 1000.0, 1010.0, 1000.0]]; + assert!(cx.comment_marker_rect(&a).is_none(), "off-page markup stays off the page"); + } + } + } +} + #[test] fn stale_comment_state_is_cleared_and_delete_never_hits_non_comment_slots() { let annot = |index: usize, name: &str| Annotation { @@ -762,3 +863,38 @@ fn the_hover_outline_goes_round_what_a_press_would_take() { let selected = CommentView { selected: Some((0, 0)), ..CommentView::default() }; assert!(hover_outline(&cx, &selected, Some(&comment)).is_none(), "the selected comment has its own frame"); } + +#[test] +fn comment_marker_wins_over_an_overlapping_body_for_pick_and_hover() { + let page = PageInfo { width: 200.0, height: 200.0, label: String::new(), crop: [0.0, 0.0, 200.0, 200.0], rotation: 0 }; + let xf = PageXform { rect: Rect::from_min_size(Pos2::ZERO, vec2(200.0, 200.0)), rot: 0, pw: 200.0, ph: 200.0 }; + let markup = Annotation { rect: [50.0, 50.0, 80.0, 65.0], index: 0, ..annotation("Highlight", Some("Anchor"), false) }; + let body = Annotation { rect: [0.0, 0.0, 200.0, 200.0], index: 1, ..annotation("Square", Some("Body"), false) }; + let info = DocInfo { pages: vec![page], annotations: vec![body, markup], ..Default::default() }; + let prefs = CommentPrefs::default(); + let ptr = PagePointer::default(); + let cx = PageCx { + page: 0, + xf: &xf, + ptr: &ptr, + info: &info, + tool: QuickTool::Select, + prefs: &prefs, + allowed: true, + hidden: false, + redact_mark: ([0.0; 3], 1.0), + keys: crate::page_keys::PageKeys::new(false), + }; + let marker = cx.comment_marker_rect(&info.annotations[1]).unwrap(); + assert_eq!(cx.comment_at(marker.center()).map(|a| a.index), Some(0)); + let ptr = PagePointer { pos: Some(marker.center()), ..PagePointer::default() }; + let cx = PageCx { ptr: &ptr, ..cx }; + let mut view = DocView::new(pdfcraft_engine::DocId(1), &info, crate::canvas::ViewDefaults::default()); + super::input::describe(&cx, &mut view); + let ctx = egui::Context::default(); + assert_eq!(view.cursor.resolve(&ctx, QuickTool::Select).as_deref(), Some("Highlight\nAnchor")); + let hidden = PageCx { hidden: true, ..cx }; + assert!(hidden.comment_at(marker.center()).is_none()); + super::input::describe(&hidden, &mut view); + assert!(view.cursor.resolve(&ctx, QuickTool::Select).is_none()); +} diff --git a/crates/ui-egui/tests/it/comments.rs b/crates/ui-egui/tests/it/comments.rs index 076f08413c..655e25b50c 100644 --- a/crates/ui-egui/tests/it/comments.rs +++ b/crates/ui-egui/tests/it/comments.rs @@ -614,10 +614,74 @@ fn selected_text_comment_action_opens_note_composer() { h.run_steps(3); h.get_by_label("Add comment").click(); h.run_steps(2); - assert!(matches!(h.state().views[0].comments.composer.as_ref().map(|c| &c.kind), Some(pdfcraft_ui_egui::comments::ComposerKind::Note))); + assert!(matches!( + h.state().views[0].comments.composer.as_ref().map(|c| &c.kind), + Some(pdfcraft_ui_egui::comments::ComposerKind::TextNote { .. }) + )); + h.get_by_label("Add comment"); + assert!(h.query_by_label("Sticky note").is_none()); assert!(h.state().views[0].selected_text().is_none()); } +#[test] +fn selection_note_replaced_by_sticky_note_posts_at_the_new_anchor() { + let mut h = harness(|_| {}); + h.state_mut().views[0].select_text(0, 4, 8); + h.run_steps(3); + h.get_by_label("Add comment").click(); + h.run_steps(2); + h.state_mut().set_option("quick", "note").unwrap(); + click_pt(&mut h, (250.0, 180.0)); + let anchor = h.state().views[0].comments.composer.as_ref().unwrap().at; + field(&h, "Add a comment").type_text("New sticky note"); + h.run_steps(2); + h.get_by_label("Post").click(); + h.run_steps(3); + let c = comments(&h); + assert_eq!(c.len(), 1); + assert_eq!(c[0].subtype, "Text"); + assert_eq!(c[0].contents.as_deref(), Some("New sticky note")); + assert!(c[0].quads.is_empty()); + assert!((anchor[0] - 250.0).abs() < 1.0 && (anchor[1] - 180.0).abs() < 1.0); + assert!((f64::from(c[0].rect[0]) - anchor[0]).abs() < 1.0 && (f64::from(c[0].rect[3]) - anchor[1]).abs() < 1.0, "{:?}", c[0].rect); +} + +#[test] +fn selection_note_posts_highlight_and_hovers_author_and_comment() { + let _gpu = std::env::var_os("PDFCRAFT_SHOTS").map(|_| crate::gpu()); + let mut h = harness(|_| {}); + h.state_mut().views[0].select_text(0, 4, 8); + let selected = h.state().views[0].selected_text().unwrap(); + h.run_steps(3); + h.get_by_label("Add comment").click(); + h.run_steps(2); + field(&h, "Add a comment").type_text("Check these words"); + h.run_steps(2); + h.get_by_label("Post").click(); + h.run_steps(3); + crate::support::settle(&mut h); + let c = comments(&h); + assert_eq!(c.len(), 1); + assert_eq!(c[0].subtype, "Highlight"); + assert_eq!(c[0].contents.as_deref(), Some("Check these words")); + assert!(!c[0].quads.is_empty()); + let hover = format!("{}\nCheck these words", c[0].author.as_deref().expect("comment has an author")); + h.state_mut().set_option("panel", "none").unwrap(); + h.state_mut().views[0].comments.selected = None; + h.hover_at(pos2(0.0, 0.0)); + h.run_steps(3); + assert!(h.query_by_label(&hover).is_none()); + let r = c[0].rect; + let p = at(&h, (r[0] + r[2]) / 2.0, (r[1] + r[3]) / 2.0); + h.hover_at(p); + h.run_steps(3); + h.get_by_label(&hover); + assert!(h.query_by_label_contains(&selected).is_none(), "hover omits the selected text"); + if let Ok(dir) = std::env::var("PDFCRAFT_SHOTS") { + h.render().unwrap().save(format!("{dir}/selection-comment-hover.png")).unwrap(); + } +} + #[test] fn shapes_and_ink_are_drawn_by_dragging() { let mut h = harness(|app| app.set_option("quick", "square").unwrap()); @@ -3099,6 +3163,36 @@ fn hover_markup_at(h: &mut Harness<'static, PdfCraftApp>, x: f32, y: f32) { h.run_steps(3); } +#[test] +fn imported_commented_markup_keeps_author_at_rotated_zoomed_page_edges() { + let _gpu = std::env::var_os("PDFCRAFT_SHOTS").map(|_| crate::gpu()); + for subtype in ["Highlight", "Underline", "StrikeOut", "Squiggly"] { + for (page_rotation, view_rotation) in [(0, 0), (90, 0), (0, 90), (90, 90)] { + for zoom in ["50", "125"] { + let pdf = String::from_utf8(markup_hover_fixture(subtype, page_rotation, "", "")) + .unwrap() + .replace("/Rect [20 105 250 165]", "/Rect [270 180 300 200]") + .replace( + "/QuadPoints [120 165 250 165 120 145 250 145 20 125 100 125 20 105 100 105]", + "/QuadPoints [270 200 300 200 270 180 300 180]", + ); + let mut h = markup_hover_harness(pdf.into_bytes(), view_rotation); + h.state_mut().set_option("zoom", zoom).unwrap(); + h.run_steps(4); + support::settle(&mut h); + hover_markup_at(&mut h, 285.0, 190.0); + h.get_by_label("Ada\nFirst multiline comment"); + if subtype == "Highlight" + && matches!((page_rotation, view_rotation, zoom), (0, 0, "50") | (90, 90, "125")) + && let Ok(dir) = std::env::var("PDFCRAFT_SHOTS") + { + h.render().unwrap().save(format!("{dir}/edge-comment-{page_rotation}-{view_rotation}-{zoom}.png")).unwrap(); + } + } + } + } +} + #[test] fn multiline_markup_hover_stays_on_marked_lines() { // #3541: /Rect encloses both lines, including text before/after the selection and the gap. From 495cba8efcba96d2a499be486d8e982e25ce8b01 Mon Sep 17 00:00:00 2001 From: Michael Higgins <55243596+Higgs32584@users.noreply.github.com> Date: Wed, 11 Nov 2026 01:52:34 -0500 Subject: [PATCH 30/60] fix(js): follow every reading of a script in the nesting guard, so a regex can't put it out of phase (#5079) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What this changes The nesting guard that keeps hostile scripts from overflowing the stack in boa's parser could be put out of phase, and this replaces it with one that can't be. boa parses by recursive descent with no depth limit of its own (boa_parser 0.22 has none to turn on), and a stack overflow aborts the process, since `catch_unwind` can't catch it. **The problem.** `refuse()` scanned bytes and guessed where strings begin. A quote inside a regular expression put it out of phase with the real strings. The first version of this PR counted opening brackets inside skipped text, which closed the reported `/'/` reproduction. It still let closing brackets inside *real* strings count as code. `x=` followed by 5,000 lines of `(/"/,"))))",` nests 5,000 deep in 65 KB (under the 256 KiB limit), while that scan never went past depth 1. The same works with `'`, and with template literals, which the scan didn't read at all. **The fix.** The guard now lives in `crates/js/src/nesting.rs`. - **It doesn't guess.** Whether a `/` is a division or starts a regular expression depends on where the parser is: boa's parser, not its lexer, sets that. So the scan follows every reading at once. At each `/` that isn't a comment, it splits into a division reading and a regular-expression reading, and counts nesting on each. boa's reading is always one of them, so the deepest count is at least boa's real depth. That makes it a sound bound, not a mitigation for one reproduction. - **It drops readings boa rejects,** at the byte where boa stops with a syntax error: a string or regular expression running into a line break, or a regular expression whose groups don't match. boa runs `regress::try_parse` on each literal as it lexes it. This keeps ordinary divisions such as `Math.round(a / b * 100) / 100` from leaving stray readings that add up to false depth. - **It reads the rest of the lexical grammar that changes what's code:** strings with escapes and line continuations, comments, template text and `${}` substitutions (which count as a level), the Annex B `` comments (followed both ways), and a leading `#!`. - **It caps the work.** At most 16 readings are followed at once. Readings in the same place are merged first, keeping the larger counts, and past that the script is refused. The limits are unchanged: 64 brackets, a run of 64 prefix operators, 4,096 conditionals. Every path that hands source to boa goes through `refuse()`: AcroForm scripts, XFA JavaScript, and `CheckedCodeFromStrings`. Brackets inside strings and comments no longer count at all. That drops the trade-off in this PR's first version, which refused a script whose strings held more than 64 opening brackets in total. Fixes #5079 ## How you verified it Locally, only formatting was run. I didn't build or test this commit, and I'm relying on CI for the build, clippy and the tests. - [x] `cargo fmt --check -p pdfcraft-js` passes. - [ ] clippy with `-D warnings` on `pdfcraft-js`: not run locally. - [ ] `cargo test -p pdfcraft-js`: not run locally. A local test build was stopped before it compiled anything, so I have no local results for this commit. What the tests check: - `a_quote_in_a_regex_cannot_put_the_guard_out_of_phase`: the 5,000-line bypass above, and the same with `'` and backticks. It also runs a script nested exactly to the limit, closed properly. - `nesting_guard_reads_templates_regexes_and_comments` and `divisions_do_not_add_up_to_a_refusal`: controls that must run. - `boa_rejects_what_the_nesting_guard_drops`: the boa behaviour the guard relies on. - The XFA-open test (`session::a_quote_in_a_regex_cannot_hide_nesting_from_an_initialize_script`) now runs the 5,000-line bypass too. - `ordinary_nesting_still_runs` again expects 1,000 brackets inside strings and comments to run. Not yet shown: that these tests fail on the previous head (439f86220). By reading the previous scan, the `(/"/,"))))",` script never goes past depth 1 there, but I haven't run that. ## Anything a reviewer should look at first - `boa_rejects_what_the_nesting_guard_drops` pins the assumptions behind dropping a reading. If a boa upgrade starts accepting one of those inputs, that test fails, and the guard has to keep that reading. - Out of scope: recursion that isn't bracket nesting, prefix runs or conditionals, such as long `a=a=a=…` chains or `if(1)if(1)…` statements. The guard didn't count those before this PR either. They're worth a separate issue once someone checks whether any of them overflows the stack within 256 KiB. ## Summary by CodeRabbit * **Bug Fixes** * Scripts that exceed supported nesting limits are rejected with a “nested too deeply” error, including excessive bracket nesting, prefix operators, and conditional expressions. * Nesting checks now account for strings, comments, templates, regular expressions, and other JavaScript syntax, avoiding incorrect results from bracket-like text or ambiguous slash characters. * After a nesting-related refusal during initialization, the session remains usable. --- crates/js/src/lib.rs | 1 + crates/js/src/nesting.rs | 319 +++++++++++++++++++++++++++++++++++++++ crates/js/src/run.rs | 75 +-------- crates/js/src/tests.rs | 175 ++++++++++++++++++++- 4 files changed, 496 insertions(+), 74 deletions(-) create mode 100644 crates/js/src/nesting.rs diff --git a/crates/js/src/lib.rs b/crates/js/src/lib.rs index 1c0fe24f7b..2e9dba0c6d 100644 --- a/crates/js/src/lib.rs +++ b/crates/js/src/lib.rs @@ -39,6 +39,7 @@ mod format; mod globals; mod host; mod model; +mod nesting; mod run; #[cfg(test)] diff --git a/crates/js/src/nesting.rs b/crates/js/src/nesting.rs new file mode 100644 index 0000000000..9f99c5351a --- /dev/null +++ b/crates/js/src/nesting.rs @@ -0,0 +1,319 @@ +//! The nesting guard: whether a script nests deeper than boa's parser can take. +//! +//! boa parses by recursive descent with no depth limit of its own, and a stack overflow aborts +//! the process (it is not a panic, so `catch_unwind` can't stop it). So every script is scanned +//! before boa sees it, and one that nests too deeply is refused. +//! +//! A scan has to know which brackets are code, and that depends on where strings, templates, +//! comments and regular expressions begin and end. Most of that follows from the bytes, but a +//! `/` that isn't a comment is a division or the start of a regular expression depending on +//! where the parser is, which a scan can't know. A scan that guesses can be put out of phase: +//! after `/'/`, the quote in the regular expression made the real nesting that followed look +//! like string text (#5079). So the scan doesn't guess. It follows every reading at once, +//! splitting at each such `/` into a division and a regular expression, and counts nesting on +//! each reading. boa's reading is always one of them, so the deepest is at least as deep as +//! boa's. A reading boa would reject is dropped at the byte where boa stops with a syntax error +//! (a string or regular expression running into a line break, a regular expression's groups not +//! matching), which keeps divisions from leaving stray readings behind. + +/// Deepest nesting of `(`, `[`, `{` and template substitutions (`${`). +pub(crate) const MAX_BRACKET_DEPTH: usize = 64; + +/// Longest run of prefix operators (`!`, `~`, `+`, `-`). +pub(crate) const MAX_PREFIX_RUN: usize = 64; + +/// Most `?` (conditional expressions nest without brackets). +pub(crate) const MAX_CONDITIONALS: usize = 4096; + +/// Most readings followed at once. Readings that differ only in their counts are merged first, +/// keeping the larger counts; past this, the script is refused. +const MAX_READINGS: usize = 16; + +/// Where a reading is in the script. +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug)] +enum Mode { + Code, + /// In a string that this quote closes. + Str(u8), + /// In a template's text: after its backtick, or after a substitution's closing `}`. + Template, + /// In a regular expression literal: in a class (`[...]`), and the groups open outside one. + Regex { + class: bool, + groups: usize, + }, + /// In a `//`, `` comment, to the end of the line. + LineComment, + /// In a `/* */` comment. + BlockComment, +} + +/// One way of reading the script, and what it has counted so far. +#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Debug)] +struct Reading { + mode: Mode, + /// Bytes to pass over: the rest of a two-byte token, or what a backslash escapes. + skip: usize, + /// For each open template substitution, the braces open inside it, so the `}` that closes + /// the substitution is told from one that closes a block. + substitutions: Vec, + depth: usize, + prefix: usize, + conditionals: usize, +} + +impl Reading { + fn in_code() -> Self { + Self { mode: Mode::Code, skip: 0, substitutions: Vec::new(), depth: 0, prefix: 0, conditionals: 0 } + } + + fn with_mode(&self, mode: Mode) -> Self { + Self { mode, ..self.clone() } + } + + fn over_limit(&self) -> bool { + self.depth > MAX_BRACKET_DEPTH || self.prefix > MAX_PREFIX_RUN || self.conditionals > MAX_CONDITIONALS + } + + fn open(&mut self) { + self.depth = self.depth.saturating_add(1); + self.prefix = 0; + } + + fn close(&mut self) { + self.depth = self.depth.saturating_sub(1); + self.prefix = 0; + } + + /// Read byte `i` of `b`. A second reading that splits off here goes to `forks`. Returns + /// false when boa would stop with a syntax error at this byte, ending this reading. + fn advance(&mut self, b: &[u8], i: usize, forks: &mut Vec) -> bool { + if self.skip > 0 { + self.skip -= 1; + return true; + } + let Some(&c) = b.get(i) else { return true }; + match self.mode { + Mode::Code => { + self.code(c, b, i, forks); + true + } + Mode::Str(quote) => self.string(quote, c, b, i), + Mode::Template => { + self.template(c, b, i); + true + } + Mode::Regex { class, groups } => self.regex(class, groups, c, b, i), + Mode::LineComment => { + let n = line_break(b, i); + if n > 0 { + self.mode = Mode::Code; + self.skip = n - 1; + } + true + } + Mode::BlockComment => { + if c == b'*' && at(b, i + 1, b"/") { + self.mode = Mode::Code; + self.skip = 1; + } + true + } + } + } + + fn code(&mut self, c: u8, b: &[u8], i: usize, forks: &mut Vec) { + match c { + b'\'' | b'"' => { + self.mode = Mode::Str(c); + self.prefix = 0; + } + b'`' => { + self.mode = Mode::Template; + self.prefix = 0; + } + b'/' if at(b, i + 1, b"/") => self.mode = Mode::LineComment, + b'/' if at(b, i + 1, b"*") => { + self.mode = Mode::BlockComment; + self.skip = 1; + } + b'/' => { + // A division, or the start of a regular expression: the parser decides, so + // follow both. + forks.push(self.with_mode(Mode::Regex { class: false, groups: 0 })); + self.prefix = 0; + } + b'(' | b'[' => self.open(), + b'{' => { + if let Some(open) = self.substitutions.last_mut() { + *open = open.saturating_add(1); + } + self.open(); + } + b')' | b']' => self.close(), + b'}' => { + self.close(); + if self.substitutions.last() == Some(&0) { + self.substitutions.pop(); + self.mode = Mode::Template; + } else if let Some(open) = self.substitutions.last_mut() { + *open = open.saturating_sub(1); + } + } + b'!' | b'~' | b'+' | b'-' => { + // `-->` begins a comment at the start of a line (Annex B); followed anywhere. + if c == b'-' && at(b, i + 1, b"->") { + forks.push(self.with_mode(Mode::LineComment)); + } + self.prefix = self.prefix.saturating_add(1); + } + b'?' => { + self.conditionals = self.conditionals.saturating_add(1); + self.prefix = 0; + } + // ` 0; ## Summary by CodeRabbit - **New Features** - Page extraction and splitting now follow a dedicated permission check, distinct from page assembly. - Pages can be inserted from certain protected PDFs when their protection settings match the destination document. - **Bug Fixes** - Separate-page extraction now cleans up files created during a failed operation and avoids creating an output directory when the first page cannot be extracted. - Security details now report whether page extraction is permitted. --- crates/automation/src/assemble.rs | 53 ++++-- crates/automation/src/json.rs | 2 +- crates/automation/tests/it/automation.rs | 44 +++++ crates/crypt/src/lib.rs | 7 + crates/crypt/src/tests.rs | 17 ++ crates/engine/src/commands.rs | 9 +- crates/engine/src/document.rs | 8 +- crates/engine/src/edit.rs | 4 +- crates/engine/src/edit/run.rs | 6 +- crates/engine/src/session/files.rs | 26 ++- crates/engine/src/sources.rs | 24 ++- crates/engine/src/tests.rs | 200 +++++++++++++++++++++++ crates/ui-egui/src/commands.rs | 4 +- crates/ui-egui/src/workflows/files.rs | 8 +- crates/ui-egui/tests/it/editing.rs | 6 +- parity/acrobat-features.toml | 2 +- 16 files changed, 387 insertions(+), 33 deletions(-) diff --git a/crates/automation/src/assemble.rs b/crates/automation/src/assemble.rs index 5fd9d9b0be..b091e54d19 100644 --- a/crates/automation/src/assemble.rs +++ b/crates/automation/src/assemble.rs @@ -239,6 +239,8 @@ impl Automation { Ok(json!({ "files": out })) } + /// `page_extract`: the chosen pages as one new PDF, or with `separate` one file per page in + /// `out_dir`; with `delete` the pages are then removed from the original. pub(crate) fn page_extract(&mut self, a: &Args) -> Result { let doc = self.doc(a)?; let (id, name) = (doc.id, format!("{} (extract)", doc.name)); @@ -246,16 +248,8 @@ impl Automation { let pages = self.pages(a, "pages")?; let mut out = if a.opt_bool("separate")?.unwrap_or(false) { // Each page as its own file. - let dir = self.resolve(a.str("out_dir").map_err(|_| ToolError::InvalidArgs("separate files need out_dir".into()))?, true)?; - std::fs::create_dir_all(&dir).map_err(|e| failed(format!("{}: {e}", dir.display())))?; - let mut files = Vec::new(); - for &p in &pages { - let bytes = self.session.extract(id, &[p]).map_err(failed)?; - let path = child(&dir, &format!("{stem}-page{}.pdf", p + 1)); - write_atomic(&path, &bytes)?; - files.push(path.to_string_lossy().into_owned()); - } - json!({ "files": files }) + let out_dir = a.str("out_dir").map_err(|_| ToolError::InvalidArgs("separate files need out_dir".into()))?; + json!({ "files": self.extract_separate(id, &pages, out_dir, &stem)? }) } else { let bytes = self.session.extract(id, &pages).map_err(failed)?; self.deliver(a, &name, bytes)? @@ -267,6 +261,45 @@ impl Automation { Ok(out) } + /// Each of `pages` as its own `-page.pdf` in `out_dir`, extracted and written one at a + /// time, so only one page's PDF is held at once. The first page is extracted before `out_dir` is + /// resolved and made, so a refusal (the permissions refuse every page alike, #5081) leaves nothing + /// behind; if a later page fails, the files this call created, and the directory if it made it, + /// are removed. + fn extract_separate(&self, id: pdfcraft_engine::DocId, pages: &[usize], out_dir: &str, stem: &str) -> Result> { + let Some(&first) = pages.first() else { return Ok(Vec::new()) }; + let mut first_bytes = Some(self.session.extract(id, &[first]).map_err(failed)?); + let dir = self.resolve(out_dir, true)?; + let made_dir = !dir.exists(); + std::fs::create_dir_all(&dir).map_err(|e| failed(format!("{}: {e}", dir.display())))?; + let (mut files, mut created) = (Vec::new(), Vec::new()); + let outcome = pages.iter().try_for_each(|&p| -> Result<()> { + let bytes = match first_bytes.take() { + Some(bytes) => bytes, + None => self.session.extract(id, &[p]).map_err(failed)?, + }; + let path = child(&dir, &format!("{stem}-page{}.pdf", p.saturating_add(1))); + let existed = path.exists(); + write_atomic(&path, &bytes)?; + if !existed { + created.push(path.clone()); + } + files.push(path.to_string_lossy().into_owned()); + Ok(()) + }); + if let Err(e) = outcome { + // Best effort: whatever can't be removed stays, and the error reported is the extraction's. + for path in &created { + let _ = std::fs::remove_file(path); + } + if made_dir { + let _ = std::fs::remove_dir(&dir); + } + return Err(e); + } + Ok(files) + } + pub(crate) fn doc_combine(&mut self, a: &Args) -> Result { let paths = a.strs("paths")?; if paths.len() < 2 { diff --git a/crates/automation/src/json.rs b/crates/automation/src/json.rs index e6ca95e905..f612c3050d 100644 --- a/crates/automation/src/json.rs +++ b/crates/automation/src/json.rs @@ -256,7 +256,7 @@ pub(crate) fn security(d: &Document) -> Value { "method": s.method, "pending": s.pending, "printing": if !p.print() { "none" } else if p.print_high_quality() { "high" } else { "low" }, - "modify": p.modify(), "assemble": p.assemble(), "copy": p.copy(), "annotate": p.annotate(), + "modify": p.modify(), "assemble": p.assemble(), "extract_pages": p.extract_pages(), "copy": p.copy(), "annotate": p.annotate(), "fill_forms": p.fill_forms(), "accessibility": p.extract_for_accessibility(), }) } diff --git a/crates/automation/tests/it/automation.rs b/crates/automation/tests/it/automation.rs index 39f8ab7a16..3a87607373 100644 --- a/crates/automation/tests/it/automation.rs +++ b/crates/automation/tests/it/automation.rs @@ -3040,6 +3040,8 @@ fn replace_text_lists_its_replacement() { } } +/// Protect through the tools, then reopen: the open password is required, the restrictions +/// (including page extraction) hold for the user, and the permissions password lifts them. #[test] fn protecting_through_tools() { let dir = workdir("protect"); @@ -3057,12 +3059,14 @@ fn protecting_through_tools() { (r["security"]["printing"].as_str(), r["security"]["annotate"].as_bool(), r["security"]["copy"].as_bool()), (Some("low"), Some(true), Some(false)) ); + assert_eq!(r["security"]["extract_pages"], false); assert!(!r.to_string().contains("boss"), "passwords are never echoed"); ok(&mut a, "doc_save", json!({ "doc": doc, "path": "locked.pdf" })); // Another session: the open password is required, and the restrictions hold. let mut b = auto(&dir); assert!(matches!(b.call("doc_open", &json!({ "path": "locked.pdf" })), Err(ToolError::Failed(_)))); let re = ok(&mut b, "doc_open", json!({ "path": "locked.pdf", "password": "open" }))["doc"].as_u64().unwrap(); + assert_eq!(ok(&mut b, "doc_info", json!({ "doc": re }))["security"]["extract_pages"], false); assert!(matches!(b.call("page_delete", &json!({ "doc": re, "pages": [1] })), Err(ToolError::Failed(_)))); assert!(matches!(b.call("doc_unprotect", &json!({ "doc": re })), Err(ToolError::Failed(_)))); ok(&mut b, "comment_add", json!({ "doc": re, "page": 1, "type": "note", "at": [10, 10], "contents": "allowed" })); @@ -3086,6 +3090,7 @@ fn protecting_through_tools() { assert_eq!(ok(&mut b, "text_find", json!({ "doc": re, "query": "Page" }))["count"], 3); // With the permissions password everything is possible, including removing security. let owner = ok(&mut b, "doc_open", json!({ "path": "locked.pdf", "password": "boss" }))["doc"].as_u64().unwrap(); + assert_eq!(ok(&mut b, "doc_info", json!({ "doc": owner }))["security"]["extract_pages"], true); assert!(ok(&mut b, "text_extract", json!({ "doc": owner }))["pages"][0]["text"].as_str().is_some_and(|t| t.contains("Page"))); assert_eq!(ok(&mut b, "doc_unprotect", json!({ "doc": owner }))["security"]["protected"], false); ok(&mut b, "doc_save", json!({ "doc": owner, "path": "open.pdf" })); @@ -3093,6 +3098,45 @@ fn protecting_through_tools() { ok(&mut c, "doc_open", json!({ "path": "open.pdf" })); } +/// A refused `doc_split` or separate-file `page_extract` leaves no output directory behind +/// (#5081), whether the protection allows no page changes ("Any except extracting pages") or +/// page changes without copying; with copying allowed as well, the user may extract. +#[test] +fn refused_extraction_writes_no_split_or_page_output() { + let dir = workdir("refused-extraction-output"); + // (changes, copy, assemble, extract_pages) as the user sees them. + let cases = [("any-except-extract", false, false, false), ("pages", false, true, false), ("pages", true, true, true)]; + for (i, (changes, copy, assemble, extract)) in cases.into_iter().enumerate() { + let mut a = auto(&dir); + let doc = ok(&mut a, "doc_open", json!({ "path": "a.pdf" }))["doc"].as_u64().unwrap(); + let protect = json!({ "doc": doc, "open_password": "open", "permissions_password": "boss", "changes": changes, "copy": copy }); + ok(&mut a, "doc_protect", protect); + let path = format!("restricted-{i}.pdf"); + ok(&mut a, "doc_save", json!({ "doc": doc, "path": path })); + + let mut b = auto(&dir); + let restricted = ok(&mut b, "doc_open", json!({ "path": path, "password": "open" }))["doc"].as_u64().unwrap(); + let security = ok(&mut b, "doc_info", json!({ "doc": restricted }))["security"].clone(); + assert_eq!((security["assemble"].as_bool(), security["extract_pages"].as_bool()), (Some(assemble), Some(extract)), "{changes}, copy {copy}"); + let (split_dir, extract_dir) = (format!("split-{i}"), format!("extract-{i}")); + let split = b.call("doc_split", &json!({ "doc": restricted, "every": 1, "out_dir": split_dir })); + let pages = b.call("page_extract", &json!({ "doc": restricted, "pages": [1, 2], "separate": true, "out_dir": extract_dir })); + if extract { + assert!(split.is_ok() && dir.join(&split_dir).exists(), "{changes}, copy {copy}"); + let files = match pages.as_deref() { + Ok([Content::Json(v), ..]) => v["files"].as_array().map(Vec::len), + other => panic!("{changes}, copy {copy}: page_extract: {other:?}"), + }; + assert_eq!(files, Some(2), "{changes}, copy {copy}"); + } else { + assert!(matches!(split, Err(ToolError::Failed(_))), "{changes}, copy {copy}"); + assert!(!dir.join(&split_dir).exists(), "refused split creates no output directory"); + assert!(matches!(pages, Err(ToolError::Failed(_))), "{changes}, copy {copy}"); + assert!(!dir.join(&extract_dir).exists(), "refused extraction creates no output directory"); + } + } +} + /// A document that allows changes but not copying: the tools that return a page's text refuse, /// or leave the text out where the tool also changes the page (a restyled paragraph, recognized /// words). diff --git a/crates/crypt/src/lib.rs b/crates/crypt/src/lib.rs index 8b520d281d..31c4963ae5 100644 --- a/crates/crypt/src/lib.rs +++ b/crates/crypt/src/lib.rs @@ -127,6 +127,13 @@ impl Permissions { pub fn assemble(&self) -> bool { if self.is_revision_3_or_later() { self.bit(11) } else { self.bit(4) } } + /// Take pages out into another document (Extract, Split, a source of Combine or Insert + /// Pages): [`Self::assemble`] and copying (bit 5), since the pages carry their text and + /// graphics into a new file without the document's protection (#5081). "Any except + /// extracting pages" sets bit 4 but not bit 11, so at revision 3 and later it grants neither. + pub fn extract_pages(&self) -> bool { + self.assemble() && self.copy() + } /// Revision 3 and later need bit 3 as well as bit 12. Table 22 defines bit 12 for those /// revisions only: at revision 2 the print flag (bit 3) is all there is. pub fn print_high_quality(&self) -> bool { diff --git a/crates/crypt/src/tests.rs b/crates/crypt/src/tests.rs index f7bbb874bb..39853f33ad 100644 --- a/crates/crypt/src/tests.rs +++ b/crates/crypt/src/tests.rs @@ -248,6 +248,23 @@ fn permission_bits_follow_table_22() { assert!(p.modify(), "owners ignore /P"); } +/// Page extraction (#5081) is assembly as the revision reads it, and copying (bit 5). +#[test] +fn page_extraction_needs_assembly_and_copying() { + let bit = |n: u32| 1i32 << (n - 1); + let user = |bits: i32, revision: i64| Permissions { bits, owner: false, revision }; + // "Any except extracting pages" (bits 4, 6, 9; copying off) at revision 3 and later. + assert!(!user(bit(4) | bit(6) | bit(9), 4).extract_pages()); + // Pages may change in place, but copying is withheld. + assert!(user(bit(11), 6).assemble() && !user(bit(11), 6).extract_pages()); + assert!(user(bit(11) | bit(5), 6).extract_pages()); + // Revision 2: bit 4 covers assembly, and its reserved bit 11 grants nothing. + assert!(user(bit(4), 2).assemble() && !user(bit(4) | bit(11), 2).extract_pages(), "copying withheld"); + assert!(user(bit(4) | bit(5), 2).extract_pages()); + assert!(!user(bit(5) | bit(11), 2).extract_pages(), "no assembly without bit 4 at revision 2"); + assert!(Permissions { bits: 0, owner: true, revision: 2 }.extract_pages(), "owners ignore /P"); +} + #[test] fn perms_limits_what_an_edited_p_grants() { let made = new(Algorithm::Aes256, "user pw", "owner pw", 0b0100); // print only diff --git a/crates/engine/src/commands.rs b/crates/engine/src/commands.rs index dfbc33d658..55a5d840ba 100644 --- a/crates/engine/src/commands.rs +++ b/crates/engine/src/commands.rs @@ -18,8 +18,10 @@ pub enum Needs { Nothing, /// A document is open. Document, - /// The open document's security allows page changes (insert, delete, rotate, extract…). + /// The open document's security allows page changes (insert, delete, rotate…). Assembly, + /// The open document's security allows taking pages out into new files (extract, split). + PageExtraction, /// The open document's security allows content and metadata changes. Modification, /// The open document's security allows adding and changing comments. @@ -416,8 +418,8 @@ pub const COMMANDS: &[CommandSpec] = &[ c("page.boxes", "Set page boxes…", PAGES, None, Assembly, "square-dashed-mouse-pointer"), c("page.insert", "Insert pages from file…", PAGES, None, Assembly, "file-input"), c("page.replace", "Replace pages…", PAGES, None, Assembly, "replace"), - c("page.extract", "Extract pages…", PAGES, None, Assembly, "file-output"), - c("page.split", "Split document…", PAGES, None, Assembly, "scissors"), + c("page.extract", "Extract pages…", PAGES, None, PageExtraction, "file-output"), + c("page.split", "Split document…", PAGES, None, PageExtraction, "scissors"), c("page.number", "Number pages…", PAGES, None, Assembly, "hash"), c("prepress.marks", "Add printer marks…", None, None, Modification, "ruler"), c("prepress.marks.remove", "Remove printer marks", None, None, Marks(crate::MarkKind::PrinterMarks), "ruler"), @@ -449,6 +451,7 @@ pub fn is_enabled(spec: &CommandSpec, session: &Session, active: Option) Nothing => true, Document | TwoPageView => doc.is_some(), Assembly => doc.is_some_and(|d| d.allows_assembly()), + PageExtraction => doc.is_some_and(|d| d.allows_page_extraction()), Modification => doc.is_some_and(|d| d.allows_modification()), Annotate => doc.is_some_and(|d| d.allows_annotation()), Copying => doc.is_some_and(|d| d.allows_copying()), diff --git a/crates/engine/src/document.rs b/crates/engine/src/document.rs index 937fd84bf9..602410922f 100644 --- a/crates/engine/src/document.rs +++ b/crates/engine/src/document.rs @@ -554,11 +554,17 @@ impl Document { if self.allows_copying() { Ok(()) } else { Err(EditError::NotPermitted("copying text and images")) } } - /// Page changes (insert, delete, rotate, move, extract) are allowed. + /// Page changes (insert, delete, rotate, move) are allowed. pub fn allows_assembly(&self) -> bool { self.editable() && self.permissions().is_none_or(|p| p.assemble()) } + /// Pages may be taken out into new files (Extract, Split): the page changes above and copying, + /// since the new files don't keep the document's protection (#5081). + pub fn allows_page_extraction(&self) -> bool { + self.editable() && self.permissions().is_none_or(|p| p.extract_pages()) + } + /// Redaction marks waiting to be applied. pub fn redaction_marks(&self) -> usize { self.info.annotations.iter().filter(|a| a.subtype == "Redact").count() diff --git a/crates/engine/src/edit.rs b/crates/engine/src/edit.rs index b00acfb8a8..80c51658e4 100644 --- a/crates/engine/src/edit.rs +++ b/crates/engine/src/edit.rs @@ -6,8 +6,8 @@ use crate::{ AddedContent, AddedText, Background, BorderShape, BoxSpec, EditError, FieldProps, FieldValue, FillMark, Hairlines, HeaderFooter, Hidden, ImageEdit, InitialView, Label, LineEnding, LinkAction, LinkStyle, MarkFile, MarkKind, Markup, NewAnnotation, NewField, NoteIcon, ObjectTarget, PageBox, PrinterMarks, Protection, ReviewState, Rgb, Shape, Standard, Style, TabOrder, TransitionSpec, Watermark, a11y, copy_image, - decode_data_file, embed_image, field_table, fit_picture, image_content_type, is_xfa_data_file, js, mark_source, measure, model_poster, - open_source, strip_xfa, unrestricted, xfa, xfa_after_data_change, + decode_data_file, embed_image, field_table, fit_picture, image_content_type, is_xfa_data_file, js, mark_source, measure, model_poster, strip_xfa, + unrestricted, xfa, xfa_after_data_change, }; mod labels; diff --git a/crates/engine/src/edit/run.rs b/crates/engine/src/edit/run.rs index 067f64929d..57e5729fc8 100644 --- a/crates/engine/src/edit/run.rs +++ b/crates/engine/src/edit/run.rs @@ -1,6 +1,6 @@ use super::{ AddedContent, Arc, Edit, EditError, FieldValue, ImageEdit, NewAnnotation, Shape, a11y, copy_image, decode_data_file, embed_image, fit_picture, - image_content_type, is_xfa_data_file, js, mark_source, measure, model_poster, open_source, strip_xfa, unrestricted, xfa, xfa_after_data_change, + image_content_type, is_xfa_data_file, js, mark_source, measure, model_poster, strip_xfa, unrestricted, xfa, xfa_after_data_change, }; use crate::SourceProblem; use crate::sources::{comment_source, import_pdf_comments}; @@ -531,13 +531,13 @@ pub(crate) fn run_edit(doc: &mut pdfcraft_cos::Document, edit: &Edit, cx: &mut E Edit::SetInfo { key, value } => pdfcraft_organize::set_info(doc, key, value)?, Edit::DuplicatePages { pages, at } => pdfcraft_organize::duplicate_pages(doc, pages, *at)?, Edit::ReplacePages { pages, name, bytes, src_pages } => { - let src = open_source(name, bytes)?; + let src = crate::sources::open_source_into(name, bytes, doc)?; pdfcraft_organize::replace_pages(doc, pages, &src, src_pages)?; } Edit::SetPageBox { pages, which, spec } => pdfcraft_organize::set_page_box(doc, pages, *which, *spec)?, Edit::SetTransition { pages, spec } => pdfcraft_organize::set_transition(doc, pages, spec)?, Edit::InsertPagesFrom { name, bytes, pages, at, field_conflicts } => { - let src = open_source(name, bytes)?; + let src = crate::sources::open_source_into(name, bytes, doc)?; let pages = match pages { Some(p) => p.clone(), None => (0..pdfcraft_organize::page_count(&src)?).collect(), diff --git a/crates/engine/src/session/files.rs b/crates/engine/src/session/files.rs index 4177d1705f..cbdf178e9d 100644 --- a/crates/engine/src/session/files.rs +++ b/crates/engine/src/session/files.rs @@ -58,6 +58,26 @@ impl Session { Ok((kind, created?)) } + /// A file for Insert Pages into `dest`, as PDF bytes and its page count: an image or text + /// file is converted, and a PDF is checked as the insert itself checks it, so a file under + /// `dest`'s own protection that withholds only page extraction is accepted, as pasted pages + /// are (#5081). + pub fn insert_source(&self, dest: DocId, name: &str, bytes: &Arc>) -> Result<(usize, Arc>), EditError> { + if !matches!(source_kind(name, bytes), Some(SourceKind::Pdf)) { + let (_, pdf) = self.convert_to_pdf(name, bytes)?; + return Ok((self.page_count_of(name, &pdf)?, pdf)); + } + let src = crate::sources::open_source_into(name, bytes, self.cos(dest)?)?; + Ok((pdfcraft_organize::page_count(&src)?, bytes.clone())) + } + + /// The page count of a PDF for Replace Pages in `dest`, checked as the replace itself checks + /// it: as with [`Self::insert_source`], a file under `dest`'s own protection that withholds + /// only page extraction is accepted (#5081). + pub fn replace_source(&self, dest: DocId, name: &str, bytes: &Arc>) -> Result { + Ok(pdfcraft_organize::page_count(&crate::sources::open_source_into(name, bytes, self.cos(dest)?)?)?) + } + /// A file for Combine as PDF bytes: an image or text file is converted; a PDF is returned as /// it is, unchecked (Combine reads it itself, with its password), as is a file named .pdf /// without a `%PDF-` header, which the reader may still open. @@ -397,7 +417,7 @@ impl Session { /// New PDF bytes containing copies of `pages` of the document (Extract Pages). pub fn extract(&self, id: DocId, pages: &[usize]) -> Result>, EditError> { let src = self.cos(id)?; - if src.permissions().is_some_and(|p| !p.assemble()) { + if src.permissions().is_some_and(|p| !p.extract_pages()) { return Err(EditError::NotPermitted("extracting pages")); } let out = pdfcraft_organize::extract_pages(src, pages)?; @@ -407,7 +427,7 @@ impl Session { /// Split the document into several new PDFs. pub fn split(&self, id: DocId, by: &pdfcraft_organize::SplitBy) -> Result, EditError> { let src = self.cos(id)?; - if src.permissions().is_some_and(|p| !p.assemble()) { + if src.permissions().is_some_and(|p| !p.extract_pages()) { return Err(EditError::NotPermitted("splitting the document")); } let n = pdfcraft_organize::page_count(src)?; @@ -425,7 +445,7 @@ impl Session { /// images count once per part) fits; a page larger than the limit is a part of its own. pub fn split_by_size(&self, id: DocId, max_bytes: usize) -> Result, EditError> { let src = self.cos(id)?; - if src.permissions().is_some_and(|p| !p.assemble()) { + if src.permissions().is_some_and(|p| !p.extract_pages()) { return Err(EditError::NotPermitted("splitting the document")); } let n = pdfcraft_organize::page_count(src)?; diff --git a/crates/engine/src/sources.rs b/crates/engine/src/sources.rs index 91d6efa3ee..e7646e223c 100644 --- a/crates/engine/src/sources.rs +++ b/crates/engine/src/sources.rs @@ -25,6 +25,19 @@ pub(crate) fn open_source_with(name: &str, bytes: &Arc>, password: Optio source_document(bytes, password).map_err(|p| EditError::Source(format!("{name}: {p}"))) } +/// [`open_source`] for pages inserted into `dest`. A source that withholds page extraction but +/// allows assembly is accepted when `dest` will be saved under the very same protection (file +/// key and `/Encrypt` values), as when pages cut or copied from a document are pasted back into +/// it: they never leave its protection (#5081). +pub(crate) fn open_source_into(name: &str, bytes: &Arc>, dest: &pdfcraft_cos::Document) -> Result { + let same_protection = |src: &pdfcraft_cos::Document| match (src.security(), dest.output_handler()) { + (Some(s), Some(d)) => s.file_key() == d.file_key() && s.dict() == d.dict(), + _ => false, + }; + let permitted = |src: &pdfcraft_cos::Document| src.permissions().is_none_or(|p| p.extract_pages() || (p.assemble() && same_protection(src))); + source_document_if(bytes, None, permitted).map_err(|p| EditError::Source(format!("{name}: {p}"))) +} + /// Import the comments of `src` onto the pages of `doc` with the same numbers. pub(crate) fn import_pdf_comments(doc: &mut pdfcraft_cos::Document, src: &pdfcraft_cos::Document) -> Result<(), EditError> { if pdfcraft_organize::import_comments(doc, src)? == 0 { @@ -44,8 +57,17 @@ pub(crate) fn comment_source(bytes: &Arc>) -> Result>, password: Option<&str>) -> Result { + source_document_if(bytes, password, |d| d.permissions().is_none_or(|p| p.extract_pages())) +} + +/// Open `bytes` as a page source, refused as [`SourceProblem::NotPermitted`] unless `permitted`. +fn source_document_if( + bytes: &Arc>, + password: Option<&str>, + permitted: impl Fn(&pdfcraft_cos::Document) -> bool, +) -> Result { match std::panic::catch_unwind(|| pdfcraft_cos::Document::open_with_password(bytes.clone(), password)) { - Ok(Ok(d)) if d.permissions().is_some_and(|p| !p.assemble()) => Err(SourceProblem::NotPermitted), + Ok(Ok(d)) if !permitted(&d) => Err(SourceProblem::NotPermitted), Ok(Ok(d)) => Ok(d), Ok(Err(pdfcraft_cos::CosError::NeedsPassword)) => Err(SourceProblem::Password), Ok(Err(pdfcraft_cos::CosError::WrongPassword)) => Err(SourceProblem::WrongPassword), diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 8bad98eea4..7b82a9b8e7 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -1615,6 +1615,206 @@ fn protected(user: &str, owner: &str, permissions: i32) -> Arc> { Arc::new(pdfcraft_cos::write_full(&doc, &Default::default()).unwrap()) } +/// Two pages protected with `algorithm` under the permissions password "boss" (no open +/// password), restricting changes to `changes`. +fn protected_with_changes(algorithm: pdfcraft_cos::Algorithm, changes: Changes) -> Arc> { + protected_pages(2, algorithm, changes, 8) +} + +/// [`protected_with_changes`] with `pages` pages; `seed` gives the file key and salts, so two +/// files with the same arguments but `pages` share their protection exactly. +fn protected_pages(pages: usize, algorithm: pdfcraft_cos::Algorithm, changes: Changes, seed: u8) -> Arc> { + let mut doc = pdfcraft_cos::Document::open(Arc::new(fixture(pages))).unwrap(); + let protection = Protection { permissions_password: Some("boss".into()), changes, algorithm, ..Protection::default() }; + doc.set_encryption(&pdfcraft_cos::NewEncryption { + algorithm, + user_password: "", + owner_password: "boss", + permissions: protection.permission_bits(), + encrypt_metadata: true, + seed: [seed; 32], + }) + .unwrap(); + Arc::new(pdfcraft_cos::write_full(&doc, &Default::default()).unwrap()) +} + +/// "Any except extracting pages" (#5081) withholds page extraction on every handler revision. At +/// revision 3 and later the preset (bit 4 without bit 11) allows no page changes either; at +/// revision 2, where bit 4 covers assembly, extraction, splitting and being a page source are +/// still refused because copying is withheld. +#[test] +fn extraction_permissions_refuse_any_except_extract_on_every_revision() { + use pdfcraft_cos::Algorithm::{Aes128, Aes256, Rc4_40, Rc4_128}; + + for algorithm in [Rc4_128, Aes128, Aes256] { + let bytes = protected_with_changes(algorithm, Changes::AnyExceptExtract); + let doc = pdfcraft_cos::Document::open_with_password(bytes.clone(), None).unwrap(); + let permissions = doc.permissions().unwrap(); + assert!(!permissions.extract_pages(), "{algorithm:?}, /P={:#x}", permissions.bits); + assert!(!permissions.assemble(), "{algorithm:?}: bit 4 without bit 11 allows no page changes"); + } + let r2 = protected_with_changes(Rc4_40, Changes::AnyExceptExtract); + let mut session = Session::new(); + let r2_id = session.open("r2.pdf", None, r2.clone(), None).unwrap(); + let permissions = session.get(r2_id).unwrap().permissions().unwrap(); + assert!(permissions.assemble() && !permissions.extract_pages(), "revision 2, /P={:#x}", permissions.bits); + assert_eq!(session.extract(r2_id, &[0]), Err(EditError::NotPermitted("extracting pages"))); + assert_eq!(session.split(r2_id, &pdfcraft_organize::SplitBy::PageCount(1)), Err(EditError::NotPermitted("splitting the document"))); + assert_eq!(crate::combine_source_check(&r2, None), Err(crate::SourceProblem::NotPermitted)); +} + +/// "Any except extracting pages" (#5081): Extract, Split and Combine refuse the file without the +/// permissions password and leave it unchanged; Combine takes it with that password. +#[test] +fn extraction_permissions_cover_revision_three_to_six_and_leave_no_partial_output() { + let bytes = protected_with_changes(pdfcraft_cos::Algorithm::Aes256, Changes::AnyExceptExtract); + assert_eq!(crate::combine_source_check(&bytes, None), Err(crate::SourceProblem::NotPermitted)); + assert_eq!(crate::combine_source_check(&bytes, Some("boss")), Ok(())); + let mut restricted = Session::new(); + let id = restricted.open("restricted.pdf", None, bytes.clone(), None).unwrap(); + let before = restricted.get(id).unwrap().bytes.clone(); + assert_eq!(restricted.extract(id, &[0]), Err(EditError::NotPermitted("extracting pages"))); + assert_eq!(restricted.split(id, &pdfcraft_organize::SplitBy::PageCount(1)), Err(EditError::NotPermitted("splitting the document"))); + assert_eq!(restricted.split_by_size(id, 1), Err(EditError::NotPermitted("splitting the document"))); + let doc = restricted.get(id).unwrap(); + assert_eq!(doc.info.pages.len(), 2); + assert!(!doc.dirty); + assert_eq!(doc.bytes, before, "a refused extraction leaves the source untouched"); + + let sources = vec![("restricted.pdf".into(), bytes.clone(), None)]; + assert!(matches!(restricted.combine_unlocked(&sources, &[None]), Err(EditError::Source(_)))); + let combined = restricted.combine_unlocked(&sources, &[Some("boss")]).unwrap(); + assert_eq!(pdfcraft_organize::page_count(&pdfcraft_cos::Document::open(combined).unwrap()).unwrap(), 2); +} + +/// A file under "Any except extracting pages" (#5081) is refused as an Insert Pages or Replace +/// Pages source for an unprotected document, which is left as it was, with nothing to undo. +#[test] +fn extraction_permissions_refuse_a_restricted_page_source() { + let bytes = protected_with_changes(pdfcraft_cos::Algorithm::Aes256, Changes::AnyExceptExtract); + let plain = Arc::new(fixture(1)); + let mut destination = Session::new(); + let dest = destination.open("destination.pdf", None, plain.clone(), None).unwrap(); + let insert = Edit::InsertPagesFrom { + name: "restricted.pdf".into(), + bytes: bytes.clone(), + pages: None, + at: 1, + field_conflicts: pdfcraft_organize::FieldConflicts::Rename, + }; + assert!(matches!(destination.apply(dest, insert), Err(EditError::Source(_)))); + assert_eq!(destination.get(dest).unwrap().bytes, plain, "a refused insert leaves the destination untouched"); + assert_eq!(destination.get(dest).unwrap().info.pages.len(), 1); + assert_eq!(destination.get(dest).unwrap().can_undo(), None); + assert!(matches!( + destination.apply(dest, Edit::ReplacePages { pages: vec![0], name: "restricted.pdf".into(), bytes: bytes.clone(), src_pages: vec![0] }), + Err(EditError::Source(_)) + )); + assert_eq!(destination.get(dest).unwrap().bytes, plain, "a refused replace leaves the destination untouched"); +} + +/// Opened with the permissions password, a file under "Any except extracting pages" (#5081) can +/// be extracted from and split. +#[test] +fn extraction_permissions_yield_to_the_owner_password() { + let bytes = protected_with_changes(pdfcraft_cos::Algorithm::Aes256, Changes::AnyExceptExtract); + let mut owner = Session::new(); + let owner_id = owner.open("restricted.pdf", None, bytes, Some("boss")).unwrap(); + assert!(owner.get(owner_id).unwrap().permissions().unwrap().extract_pages()); + assert!(owner.extract(owner_id, &[0]).is_ok()); + assert_eq!(owner.split(owner_id, &pdfcraft_organize::SplitBy::PageCount(1)).unwrap().len(), 2); + assert_eq!(owner.split_by_size(owner_id, 1).unwrap().len(), 2); +} + +/// A document that allows page changes but not copying, so not page extraction (#5081), takes +/// pages back from a file under exactly its own protection (a cut and paste, or another file +/// made with the same key and `/Encrypt` values), through Insert Pages' file check as well, and +/// through Replace Pages and its check, and keeps the restriction; the same file is refused as a +/// source for an unprotected or a differently protected document. +#[test] +fn pages_under_the_same_protection_are_inserted_without_page_extraction() { + use pdfcraft_cos::Algorithm::Aes256; + + let bytes = protected_pages(2, Aes256, Changes::Pages, 8); + let mut same = Session::new(); + let same_id = same.open("same.pdf", None, bytes.clone(), None).unwrap(); + let permissions = same.get(same_id).unwrap().permissions().unwrap(); + assert!(permissions.assemble() && !permissions.extract_pages(), "/P={:#x}", permissions.bits); + let insert = |name: &str, bytes: &Arc>, at| Edit::InsertPagesFrom { + name: name.into(), + bytes: bytes.clone(), + pages: None, + at, + field_conflicts: pdfcraft_organize::FieldConflicts::Rename, + }; + // Its own bytes, as Paste inserts them after a cut or copy. + same.apply(same_id, insert("same.pdf", &bytes, 2)).unwrap(); + assert_eq!(same.get(same_id).unwrap().info.pages.len(), 4); + assert!(same.get(same_id).unwrap().dirty); + + // A separate file made with the same key and /Encrypt values, and other pages. + let other = protected_pages(3, Aes256, Changes::Pages, 8); + let (a, b) = (pdfcraft_cos::Document::open(bytes.clone()).unwrap(), pdfcraft_cos::Document::open(other.clone()).unwrap()); + let (a, b) = (a.security().unwrap(), b.security().unwrap()); + assert!(a.file_key() == b.file_key() && a.dict() == b.dict(), "the two files share their protection"); + assert!(matches!(same.page_count_of("other.pdf", &other), Err(EditError::Source(_))), "a plain page source must allow extraction"); + assert_eq!(same.insert_source(same_id, "other.pdf", &other).map(|(n, _)| n), Ok(3)); + same.apply(same_id, insert("other.pdf", &other, 4)).unwrap(); + // Replace Pages takes them on the same terms. + let replace = |bytes: &Arc>| Edit::ReplacePages { pages: vec![0], name: "other.pdf".into(), bytes: bytes.clone(), src_pages: vec![2] }; + assert_eq!(same.replace_source(same_id, "other.pdf", &other), Ok(3)); + same.apply(same_id, replace(&other)).unwrap(); + assert_eq!(same.get(same_id).unwrap().info.pages.len(), 7); + let saved = same.save_bytes(same_id).unwrap(); + let reopened = pdfcraft_cos::Document::open_with_password(saved, None).unwrap(); + assert_eq!(pdfcraft_organize::page_count(&reopened).unwrap(), 7); + assert!(!reopened.permissions().unwrap().extract_pages(), "same-protection assembly keeps the extraction restriction"); + + // Neither an unprotected document nor one under other protection takes them. + for destination in [Arc::new(fixture(1)), protected_pages(1, Aes256, Changes::Pages, 9)] { + let mut session = Session::new(); + let dest = session.open("destination.pdf", None, destination, None).unwrap(); + let before = session.get(dest).unwrap().bytes.clone(); + assert!(matches!(session.insert_source(dest, "same.pdf", &bytes), Err(EditError::Source(_)))); + assert!(matches!(session.apply(dest, insert("same.pdf", &bytes, 1)), Err(EditError::Source(_)))); + assert!(matches!(session.replace_source(dest, "same.pdf", &bytes), Err(EditError::Source(_)))); + assert!(matches!(session.apply(dest, replace(&bytes)), Err(EditError::Source(_)))); + assert_eq!(session.get(dest).unwrap().bytes, before, "a refused insert or replace leaves the destination untouched"); + assert_eq!(session.get(dest).unwrap().info.pages.len(), 1); + } +} + +/// Taking pages out also needs copying (bit 5, #5081): a document that withholds only copying +/// refuses Extract, Split and being a page source, on a revision-2 handler too, whose reserved +/// bit 11 is always set. The owner password still lifts it. +#[test] +fn page_extraction_needs_the_copy_permission_on_every_revision() { + for algorithm in [pdfcraft_cos::Algorithm::Rc4_40, pdfcraft_cos::Algorithm::Aes256] { + let mut doc = pdfcraft_cos::Document::open(Arc::new(fixture(2))).unwrap(); + doc.set_encryption(&pdfcraft_cos::NewEncryption { + algorithm, + user_password: "", + owner_password: "boss", + permissions: -1 ^ 0b1_0000, // everything but copying + encrypt_metadata: true, + seed: [9; 32], + }) + .unwrap(); + let bytes = Arc::new(pdfcraft_cos::write_full(&doc, &Default::default()).unwrap()); + let mut s = Session::new(); + let id = s.open("no-copy.pdf", None, bytes.clone(), None).unwrap(); + let permissions = s.get(id).unwrap().permissions().unwrap(); + assert!(permissions.assemble() && !permissions.extract_pages(), "{algorithm:?}"); + assert!(s.get(id).unwrap().allows_assembly() && !s.get(id).unwrap().allows_page_extraction(), "{algorithm:?}"); + assert_eq!(s.extract(id, &[0]), Err(EditError::NotPermitted("extracting pages")), "{algorithm:?}"); + assert_eq!(s.split(id, &pdfcraft_organize::SplitBy::PageCount(1)), Err(EditError::NotPermitted("splitting the document"))); + assert_eq!(crate::combine_source_check(&bytes, None), Err(crate::SourceProblem::NotPermitted), "{algorithm:?}"); + assert_eq!(crate::combine_source_check(&bytes, Some("boss")), Ok(()), "{algorithm:?}"); + let owner = s.open("no-copy.pdf", None, bytes, Some("boss")).unwrap(); + assert!(s.extract(owner, &[0]).is_ok(), "{algorithm:?}"); + } +} + #[test] fn encrypted_documents_open_edit_and_save_encrypted() { let bytes = protected("pw", "owner", -1); diff --git a/crates/ui-egui/src/commands.rs b/crates/ui-egui/src/commands.rs index 549a877172..3b422181a9 100644 --- a/crates/ui-egui/src/commands.rs +++ b/crates/ui-egui/src/commands.rs @@ -156,7 +156,9 @@ impl PdfCraftApp { } } commands::Needs::Unlock if self.active.is_some() => tl!("This document has no restrictions to lift").to_string(), - commands::Needs::Assembly | commands::Needs::Modification | commands::Needs::Annotate if self.active.is_some() => { + commands::Needs::Assembly | commands::Needs::PageExtraction | commands::Needs::Modification | commands::Needs::Annotate + if self.active.is_some() => + { tl!("The document's security settings don't allow this change").to_string() } commands::Needs::Copying if self.active.is_some() => crate::copy_refused().to_string(), diff --git a/crates/ui-egui/src/workflows/files.rs b/crates/ui-egui/src/workflows/files.rs index 6a7a0434b3..18c1c24e0f 100644 --- a/crates/ui-egui/src/workflows/files.rs +++ b/crates/ui-egui/src/workflows/files.rs @@ -619,9 +619,9 @@ impl PdfCraftApp { /// Open the Replace Pages dialog for `bytes`, replacing the selection (or the current page). pub fn start_replace(&mut self, name: String, bytes: Vec) { - let Some(i) = self.active else { return }; + let Some((i, id)) = self.active_ids() else { return }; let bytes = Arc::new(bytes); - let src_pages = match self.session.page_count_of(&name, &bytes) { + let src_pages = match self.session.replace_source(id, &name, &bytes) { Ok(n) => n, Err(e) => { self.notify_fmt("Couldn't use {name}: {e}", &[("name", &name), ("e", &e.to_string())]); @@ -666,9 +666,7 @@ impl PdfCraftApp { continue; } }; - let converted = - self.session.convert_to_pdf(&name, &Arc::new(bytes)).and_then(|(_, pdf)| Ok((self.session.page_count_of(&name, &pdf)?, pdf))); - match converted { + match self.session.insert_source(id, &name, &Arc::new(bytes)) { Ok((pages, bytes)) => { if self.apply_edit(Edit::InsertPagesFrom { name, diff --git a/crates/ui-egui/tests/it/editing.rs b/crates/ui-egui/tests/it/editing.rs index 173e64bdea..719f871f48 100644 --- a/crates/ui-egui/tests/it/editing.rs +++ b/crates/ui-egui/tests/it/editing.rs @@ -1723,18 +1723,20 @@ fn undo_and_redo_restore_the_combine_list() { assert!(h.state().session.get(h.state().views[0].id).unwrap().can_undo().is_none()); } +/// A protected file that still allows page extraction (only printing is withheld) is listed +/// for Combine with a warning that the result won't keep its security, not as a problem. #[test] fn protected_files_that_can_be_combined_are_flagged() { let mut h = harness(1, |app| { app.set_option("combine-view", "list").unwrap(); app.use_files( pdfcraft_ui_egui::FilePurpose::Combine, - vec![("one.pdf".into(), fixture(1)), ("no-copy.pdf".into(), protected("", "owner", -1 ^ 16))], + vec![("one.pdf".into(), fixture(1)), ("no-print.pdf".into(), protected("", "owner", -1 ^ 4))], ); }); h.run_steps(3); let app = h.state(); - assert!(app.combine_draft[1].problem.is_none(), "copying text is withheld, assembling pages isn't"); + assert!(app.combine_draft[1].problem.is_none(), "printing is withheld, extracting pages isn't"); h.get_by_label("Protected: the combined file won't keep its security settings"); assert!(!h.get_by_label("Combine").accesskit_node().is_disabled(), "a warning, not a blocker"); } diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 1019a4a811..a5e052341a 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -3047,7 +3047,7 @@ status = "shipped" commands = ["page.combine"] tools = ["doc_combine"] evidence = ["crates/organize/src/tests.rs::combine_concatenates_with_a_bookmark_per_file", "crates/ui-egui/tests/it/editing.rs::combining_files_opens_a_new_unsaved_tab", "crates/ui-egui/tests/it/ui.rs::files_to_combine_from_the_operating_system", "apps/pdfcraft/src/single_instance.rs::a_second_launch_hands_its_files_to_the_first", "crates/ui-egui/tests/it/editing.rs::combine_files_opens_in_its_own_tab", "crates/ui-egui/tests/it/editing.rs::combine_lists_size_and_warns_before_combining", "crates/ui-egui/tests/it/editing.rs::a_password_protected_file_is_unlocked_with_its_password_and_combined", "crates/ui-egui/tests/it/editing.rs::a_folder_adds_its_pdfs_images_and_text_in_order_and_optionally_its_subfolders", "crates/ui-egui/tests/it/editing.rs::images_and_text_files_join_the_combine_list_as_pdfs", "crates/ui-egui/tests/it/editing.rs::files_that_cannot_be_read_are_listed_and_left_out_once_the_user_agrees", "crates/ui-egui/tests/it/ui.rs::pdfs_dropped_on_the_combine_tab_join_its_list", "crates/ui-egui/tests/it/ui.rs::files_dropped_on_the_combine_tab_join_the_end_of_its_list_in_the_order_dropped", "crates/ui-egui/tests/it/ui.rs::a_file_combine_does_not_take_is_left_out_of_a_drop_and_named", "crates/ui-egui/tests/it/ui.rs::a_pdf_dropped_on_a_document_opens_while_the_combine_tab_waits_behind_it", "crates/automation/tests/it/automation.rs::combine_extract_and_split", "crates/automation/tests/it/automation.rs::combine_opens_protected_files_with_their_passwords", "crates/automation/tests/it/automation.rs::combine_takes_images_and_text_files", "crates/engine/src/tests.rs::files_for_combine_are_made_into_pdfs_and_pdfs_pass_through", "crates/organize/src/tests.rs::combine_keeps_pdfx_when_every_source_agrees", "crates/organize/src/tests.rs::combine_claims_no_pdfx_when_the_sources_disagree"] -notes = "Combine files is its own tab: a table of the files (name, pages, size, modified, warnings) with column sorting, resizing and moving; files, folders (with subfolders) and open documents can be added, or dropped on the tab (desktop and web): a drop joins the end of the list in the order dropped, as one undo step, not at the place under the pointer; files of a kind Combine does not take are left out and named. Dropped elsewhere, files open as before. File Explorer's \"Combine files in PdfCraft\" and `pdfcraft --combine files…` add files from outside, to the running PdfCraft's tab (#585). Dropping adds no engine behaviour, so it has no tool of its own (doc_combine takes the files). Images (PNG, JPEG, TIFF, GIF, BMP, JPEG 2000) and .txt files become PDFs as they are added (#569; also in doc_combine). A file that can't be read stays in the list with why, and Combine asks before leaving it out; Office documents, HTML and other formats are not converted yet. Problems show before Combine (password, security settings that withhold page assembly, bad page ranges, duplicates); protected files are unlocked with their open or permissions password (also doc_combine passwords). The result is never encrypted, and is not yet offered for protection in the same step. A PDF/X result (output intent, GTS_PDFXVersion, Trapped, XMP pdfxid) only when every source declares the same version and output intent (#263). When they differ the result claims none, and nothing reports that yet; Acrobat behaviour not observed." +notes = "Combine files is its own tab: a table of the files (name, pages, size, modified, warnings) with column sorting, resizing and moving; files, folders (with subfolders) and open documents can be added, or dropped on the tab (desktop and web): a drop joins the end of the list in the order dropped, as one undo step, not at the place under the pointer; files of a kind Combine does not take are left out and named. Dropped elsewhere, files open as before. File Explorer's \"Combine files in PdfCraft\" and `pdfcraft --combine files…` add files from outside, to the running PdfCraft's tab (#585). Dropping adds no engine behaviour, so it has no tool of its own (doc_combine takes the files). Images (PNG, JPEG, TIFF, GIF, BMP, JPEG 2000) and .txt files become PDFs as they are added (#569; also in doc_combine). A file that can't be read stays in the list with why, and Combine asks before leaving it out; Office documents, HTML and other formats are not converted yet. Problems show before Combine (password, security settings that withhold page extraction, bad page ranges, duplicates); protected files are unlocked with their open or permissions password (also doc_combine passwords). The result is never encrypted, and is not yet offered for protection in the same step. A PDF/X result (output intent, GTS_PDFXVersion, Trapped, XMP pdfxid) only when every source declares the same version and output intent (#263). When they differ the result claims none, and nothing reports that yet; Acrobat behaviour not observed." [[feature]] id = "organize.combine-bookmarks" From 3a117c3c0e361c04a525e6d1f144e52428b2bea6 Mon Sep 17 00:00:00 2001 From: Max Abramowitz Date: Wed, 11 Nov 2026 05:06:01 -0500 Subject: [PATCH 37/60] M10: open Word, Excel and PowerPoint files through LibreOffice MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What this changes **File ▸ Open opens Word, Excel, PowerPoint, OpenDocument and RTF files** as new, unsaved PDFs (`Report.docx` → `Report.pdf`), the "LibreOffice sidecar" that `docs/gaps.md` (gap 8) and `docs/file-format-parity.md` planned for `create.from-office`. - **Conversion** (`crates/create/src/office.rs`): an installed LibreOffice runs headless as a separate process, in a private temporary folder (the input is always `document.`, so the user's file name never reaches its command line). It gets a fresh profile with `DisableMacrosExecution`, `MacroSecurityLevel` 3 and `BlockUntrustedRefererLinks` set, and Writer's and Calc's `Content/Update/Link` set to *Never* (2 for Writer and 1 for Calc: the two number their values differently), no console window on Windows (`CREATE_NO_WINDOW`), and is killed after 120 s. The output is size-capped (512 MB), must start `%PDF-`, and must open as a PDF before it is used. LibreOffice is found through `PDFCRAFT_SOFFICE`, the `PATH`, then the usual install folders per OS. Without it, the user is told it's needed rather than "not a readable PDF". - **Off the frame thread:** File ▸ Open converts on the existing open worker (`ResultKind::Converted`); the frame thread only opens the result. - **Detection:** `SourceKind::Office` requires an Office extension *and* an OLE, ZIP or RTF signature, so `photos.zip`, or a PDF saved with a `.docx` name, are not misread. - **Open dialog:** PDF (still the default), Images, Office documents, Text, All files. All files is left out on macOS, where rfd merges every filter into one allowed-types list, and on the web; Office documents are left out on the web. The extension lists are now shared constants in `pdfcraft-create` (`IMAGE_EXTENSIONS`, `TEXT_EXTENSIONS`, `OFFICE_EXTENSIONS`), used by both detection and the picker. The duplicate filter list in `open_dialog` is gone. The two new labels are in all 22 catalogues, with their `ATTRIBUTION.toml` hashes updated. - **Automation:** `doc_create_multiple`, `doc_combine` and batch `convert` go through `convert_to_pdf`, so they take Office documents too; their descriptions say so. **Deliberately not in this PR:** Insert pages, Combine files and Create ▸ Multiple files convert on the frame thread, so they now refuse Office documents with a pointer to File ▸ Open (`convert_to_pdf_inline` / `as_pdf_inline`) rather than freezing the window for seconds. Moving them onto a worker is the follow-up. `create.from-office` goes planned → **partial**, with that and the rest listed in its notes. ## How you verified it On Windows 11 Pro with LibreOffice 26.8.1 installed, per crate: - [x] `cargo fmt --check` - [x] `cargo clippy -p pdfcraft-create -p pdfcraft-engine -p pdfcraft-ui-egui -p pdfcraft-automation -j 2 --all-targets -- -D warnings` - [x] `cargo test -p pdfcraft-create -p pdfcraft-engine --lib` (42 and 276 pass); `-p pdfcraft-automation --test automation` (the combine/create tests); `-p pdfcraft-ui-egui --lib open_worker` and `--test editing --test create` - [x] `cargo xtask layers`, `cargo xtask assets`, `cargo xtask parity --partial` - [ ] wasm check: the `wasm32-unknown-unknown` target isn't installed here, so I'm leaving it to CI. The web paths are `cfg`'d and were reviewed by eye. **Tests seen failing before the fix**, by reverting each part: - Detection off → `office_documents_are_their_own_kind`, `source_kind_tells_pdfs_images_and_text_apart`, and both engine Office tests fail. - Worker conversion off → `file_open_converts_an_office_document_on_the_worker` fails with the old user-facing message, *"the file is not a readable PDF: no readable page tree or cross-reference data was found"*. New tests also cover a converter that hangs (it's stopped; the test binary stands in for LibreOffice), one that writes no PDF, a missing converter, temp-folder cleanup, `file://` URL escaping (spaces, drive letters, UTF-8), and the search order. **Real conversions, end to end:** `crates/automation/tests/it/office.rs` builds a minimal DOCX, an XLSX (with a formula, `=6*7`), a PPTX and an RTF inside the test, so no binary fixtures are committed. It runs them through `doc_create_multiple` in `separate` mode (each output reopened and its text checked, including `42`) and in `combine` mode (page order and bookmarks), and through `doc_combine` beside a PDF, with a page range on one file. These tests, `libreoffice_converts_a_document_when_installed` and the process-tree cleanup test (5 out of 5 reruns) pass with LibreOffice 26.8.1. The Office tests skip where LibreOffice isn't installed, which includes CI. Three existing tests used a 4-byte `report.docx` as their "file that can't be converted" example. That's no longer true, so they now use `archive.zip`, which keeps what they test. ## Never-crash - [x] No new `unwrap`/`expect`/`panic!`/`unreachable!`/`todo!` outside tests - [x] No input-derived indexing or slicing (`rsplit_once` for the extension); the PDF read back is size-capped - [x] No `unsafe`. The Windows flag uses `std::os::windows::process::CommandExt` - [x] A crashing, hanging or misbehaving LibreOffice is an error, never a panic or a stuck app: the worker's `catch_unwind`, a 120 s kill, and output checks ## Provenance - [x] No Adobe assets; no fonts; no copied code - [x] LibreOffice (MPL-2.0) is only *run* as an external process, as the docs planned; nothing of it is linked or copied. Its command-line options and the profile keys come from LibreOffice's public documentation. - [x] The translations are contributor-original, written from the plain meaning of "Office documents" and "All files" ("Office" kept as a product name); hashes updated, `cargo xtask assets` passes ## Security - [x] `--root` confinement unchanged: automation still resolves and reads paths before converting; only the private temp folder is written outside it - [x] MCP and the control channel untouched - [ ] **No new outbound network: please decide.** PdfCraft itself opens no connection. The profile blocks untrusted referer links and sets Writer and Calc never to update linked content on load. I converted eight synthetic files that link to a listener on 127.0.0.1: RTF `INCLUDETEXT` and `INCLUDEPICTURE`, a DOCX external picture, an ODT linked section and linked picture, an ODS table source and an external cell reference, and an ODP linked picture. None made a connection, with or without the link settings. Two things remain open: Impress has no link-update setting, and the process's network access isn't sandboxed (documented in `crates/create/README.md` and `docs/gaps.md`). ## Parity and reach - [x] `create.from-office` planned → partial, with the evidence tests and what's still missing - [x] Reachable headlessly (`doc_create_multiple`, `doc_combine`, batch `convert`) - [x] Docs: `docs/file-format-parity.md` (Word/Excel/PowerPoint and RTF rows), `docs/gaps.md` gap 8, `docs/target-app-parity.md` row J (numbers not re-measured), `docs/roadmap.md` M10, the `ROADMAP.md` log, and the `crates/create` README ## Anything a reviewer should look at first 1. **The network question** above. It's the owner's call. 2. **The frame-thread restraint:** Insert, Combine and Create Multiple refuse Office documents rather than block. Is a worker for those wanted in this PR, or as a follow-up? --- Cargo.lock | 61 +- ROADMAP.md | 2 + crates/automation/src/tools.rs | 4 +- crates/automation/tests/it/automation.rs | 12 +- crates/automation/tests/it/main.rs | 1 + crates/automation/tests/it/office.rs | 277 ++++++++ crates/create/Cargo.toml | 5 + crates/create/README.md | 24 + crates/create/src/lib.rs | 27 +- crates/create/src/office.rs | 599 ++++++++++++++++++ crates/create/src/tests.rs | 18 +- crates/engine/src/lib.rs | 6 +- crates/engine/src/session.rs | 2 +- crates/engine/src/session/files.rs | 79 ++- crates/engine/src/sources.rs | 10 + crates/engine/src/tests.rs | 58 +- crates/ui-egui/src/app/files.rs | 44 +- crates/ui-egui/src/platform/open_worker.rs | 14 +- .../ui-egui/src/platform/open_worker/tests.rs | 61 +- .../ui-egui/src/workflows/combine_ui/parts.rs | 10 +- .../src/workflows/combine_ui/session.rs | 2 +- .../src/workflows/create_multiple_ui.rs | 4 +- crates/ui-egui/src/workflows/create_ui.rs | 19 +- crates/ui-egui/src/workflows/files.rs | 2 +- crates/ui-egui/tests/it/editing.rs | 8 +- docs/file-format-parity.md | 7 +- docs/gaps.md | 27 +- docs/roadmap.md | 2 +- docs/target-app-parity.md | 2 +- parity/acrobat-features.toml | 24 +- 30 files changed, 1290 insertions(+), 121 deletions(-) create mode 100644 crates/automation/tests/it/office.rs create mode 100644 crates/create/src/office.rs diff --git a/Cargo.lock b/Cargo.lock index 7458e0af6f..9cf008f02e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -238,7 +238,7 @@ dependencies = [ "objc2-foundation 0.3.2", "parking_lot", "percent-encoding", - "windows-sys 0.61.2", + "windows-sys 0.52.0", "wl-clipboard-rs", "x11rb", ] @@ -321,7 +321,7 @@ dependencies = [ "futures-lite", "parking", "polling", - "rustix 1.1.5", + "rustix 1.1.6", "slab", "windows-sys 0.61.2", ] @@ -352,7 +352,7 @@ dependencies = [ "cfg-if", "event-listener", "futures-lite", - "rustix 1.1.5", + "rustix 1.1.6", ] [[package]] @@ -378,7 +378,7 @@ dependencies = [ "cfg-if", "futures-core", "futures-io", - "rustix 1.1.5", + "rustix 1.1.6", "signal-hook-registry", "slab", "windows-sys 0.61.2", @@ -878,7 +878,7 @@ checksum = "4dbf9978365bac10f54d1d4b04f7ce4427e51f71d61f2fe15e3fed5166474df7" dependencies = [ "bitflags 2.13.2", "polling", - "rustix 1.1.5", + "rustix 1.1.6", "slab", "tracing", ] @@ -902,7 +902,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "138efcf0940a02ebf0cc8d1eff41a1682a46b431630f4c52450d6265876021fa" dependencies = [ "calloop 0.14.4", - "rustix 1.1.5", + "rustix 1.1.6", "wayland-backend", "wayland-client", ] @@ -2050,7 +2050,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2427,7 +2427,7 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1bd49230192a3797a9a4d6abe9b3eed6f7fa4c8a8a4947977c6f80025f92cbd8" dependencies = [ - "rustix 1.1.5", + "rustix 1.1.6", "windows-link", ] @@ -4172,7 +4172,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d8fae84b431384b68627d0f9b3b1245fcf9f46f6c0e3dc902e9dce64edd1967" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -4338,7 +4338,7 @@ dependencies = [ "pdfcraft-render", "pdfcraft-xfa", "png 0.18.1", - "rustix 1.1.5", + "rustix 1.1.6", "serde_json", "sha2 0.11.1", "tempfile", @@ -4357,7 +4357,7 @@ dependencies = [ "pdfcraft-preflight", "pdfcraft-render", "png 0.18.1", - "rustix 1.1.5", + "rustix 1.1.6", "serde_json", "winresource", ] @@ -4405,6 +4405,7 @@ dependencies = [ "pdfcraft-model", "pdfcraft-xfa", "png 0.18.1", + "rustix 1.1.6", "thiserror 2.0.21", "tiff", "zune-jpeg", @@ -4625,7 +4626,7 @@ dependencies = [ "async-channel", "futures-lite", "log", - "rustix 1.1.5", + "rustix 1.1.6", "zbus", ] @@ -4807,7 +4808,7 @@ dependencies = [ "pollster 1.0.1", "proptest", "rfd", - "rustix 1.1.5", + "rustix 1.1.6", "serde", "serde_json", "skrifa 0.44.0", @@ -5143,7 +5144,7 @@ dependencies = [ "concurrent-queue", "hermit-abi", "pin-project-lite", - "rustix 1.1.5", + "rustix 1.1.6", "windows-sys 0.61.2", ] @@ -5401,7 +5402,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -5953,15 +5954,15 @@ dependencies = [ [[package]] name = "rustix" -version = "1.1.5" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +checksum = "e5e76742e5294153d4dde1207dbec5bdd81eef6fabdc0d2118babadabfd858aa" dependencies = [ "bitflags 2.13.2", "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -6031,7 +6032,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -6655,7 +6656,7 @@ dependencies = [ "libc", "log", "memmap2", - "rustix 1.1.5", + "rustix 1.1.6", "thiserror 2.0.21", "wayland-backend", "wayland-client", @@ -6920,8 +6921,8 @@ dependencies = [ "fastrand", "getrandom 0.4.3", "once_cell", - "rustix 1.1.5", - "windows-sys 0.61.2", + "rustix 1.1.6", + "windows-sys 0.52.0", ] [[package]] @@ -7673,7 +7674,7 @@ checksum = "38a91b4eaddff87b1cd1074985e3713da4af2c49742d1b356b2c01670a67a078" dependencies = [ "cc", "downcast-rs", - "rustix 1.1.5", + "rustix 1.1.6", "scoped-tls", "smallvec", "wayland-sys", @@ -7686,7 +7687,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3c36a0f861ad76d0901f2800b46321410d9f73f2ea88aac0650d86c32688073" dependencies = [ "bitflags 2.13.2", - "rustix 1.1.5", + "rustix 1.1.6", "wayland-backend", "wayland-scanner", ] @@ -7708,7 +7709,7 @@ version = "0.31.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a52d18780be9b1314328a3de5f930b73d2200112e3849ca6cb11822793fb34d" dependencies = [ - "rustix 1.1.5", + "rustix 1.1.6", "wayland-client", "xcursor", ] @@ -8046,7 +8047,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -8404,7 +8405,7 @@ dependencies = [ "libc", "log", "os_pipe", - "rustix 1.1.5", + "rustix 1.1.6", "thiserror 2.0.21", "tree_magic_mini", "wayland-backend", @@ -8462,7 +8463,7 @@ dependencies = [ "libc", "libloading", "once_cell", - "rustix 1.1.5", + "rustix 1.1.6", "x11rb-protocol", ] @@ -8504,7 +8505,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" dependencies = [ "libc", - "rustix 1.1.5", + "rustix 1.1.6", ] [[package]] @@ -8611,7 +8612,7 @@ dependencies = [ "hex", "libc", "ordered-stream", - "rustix 1.1.5", + "rustix 1.1.6", "serde", "serde_repr", "tracing", diff --git a/ROADMAP.md b/ROADMAP.md index 7838ac2ff4..86ee190fa6 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -87,6 +87,7 @@ > **Last reviewed:** 2026-10-18 · **Last updated:** 2026-10-18 · **Change:** minor (M8, #1350: named replacement descriptions retired during Sanitize and across shared resources; estimates unchanged) > **Last reviewed:** 2026-10-17 · **Last updated:** 2026-10-17 · **Change:** minor (M4 #2997: Ctrl-drag (Option on macOS) in the Organize grid copies the pages into the drop gap; `page_duplicate` takes `at`; estimates unchanged) · **Target:** Adobe Acrobat Pro (Acrobat DC, continuous track 26.002.21931, macOS) > **Last reviewed:** 2026-10-15 · **Last updated:** 2026-10-15 · **Change:** minor (M6: AcroForm XML text; estimates unchanged) +> **Last reviewed:** 2026-10-17 · **Last updated:** 2026-10-17 · **Change:** minor (M10, gap 8: Office documents open through an installed LibreOffice; estimates unchanged) · **Target:** Adobe Acrobat Pro (Acrobat DC, continuous track 26.002.21931, macOS) > **Last reviewed:** 2026-10-15 · **Last updated:** 2026-10-15 · **Change:** minor (M2/gap 12: the Annex D.2 alias codes read as the ink they draw in both tables; the text oracle scores reading order; estimates unchanged) · **Target:** Adobe Acrobat Pro (Acrobat DC, continuous track 26.002.21931, macOS) > **Last reviewed:** 2026-10-15 · **Last updated:** 2026-10-15 · **Change:** minor (M14/#2383 GPU-limit candidate; estimates unchanged) · **Target:** Adobe Acrobat Pro (Acrobat DC, continuous track 26.002.21931, macOS) > **Last reviewed:** 2026-10-12 · **Last updated:** 2026-10-13 · **Change:** minor (M4: indirect page rotations; M13: CLI check visits each directory once through aliases and cycles; estimates unchanged) · **Target:** Adobe Acrobat Pro (Acrobat DC, continuous track 26.002.21931, macOS) @@ -494,6 +495,7 @@ Ranked; detail and the full milestone table in [docs/roadmap.md](docs/roadmap.md - **2026-10-16 (M14, localization):** Czech goes from menus-and-dialogs to the whole surface: 2,674 entries after adding the ~220 keys the other catalogs had -- the sentences the engine words (signature details, accessibility findings and report, PDF/A problems, pages to print, history labels), the date picker's weekday heads, the certificate viewer's fields and shared buttons. "Tag" is *tag* throughout (the Tags panel said *značky*, which is also a redaction mark), and "{n} tags" no longer reads "1 značek". Two new catalog tests. Estimates unchanged. - **2026-10-16 (M5, #2583, #1869):** A sticky note on a page with `/Rotate` is drawn upright, with its top-left where it was placed. Notes are saved with the NoRotate flag, and the renderer now honours it: the appearance is turned against the page's rotation about the upper-left corner of `/Rect`, which stays put (ISO 32000-2 §12.5.3; vendored hayro-interpret patch 37). Attachment icons follow the same rule. Selection, hover, the edit composer, the Comments panel's flash and the `comment_list` and `doc_info` rectangles use the drawn box, and `comment_add` anchors an icon's displayed top-left at `at`. Estimates unchanged. - **2026-10-16 (M11, #1201):** Optimize decodes a JPEG with its colour transform: an Adobe APP14 marker's flag, else the filter's `/ColorTransform` (ISO 32000-2 §7.4.8). An RGB JPEG stored untransformed (`/ColorTransform 0`, no marker) used to be converted from YCbCr anyway and came out in other colours, recompressed or merged from strips; it now keeps them. A JPEG whose marker and `/ColorTransform` disagree, or whose components are named R, G and B without `/ColorTransform 0`, is left as it is, since readers differ on it. Estimates unchanged. +- **2026-10-17 (M10, gap 8):** File ▸ Open opens Word, Excel, PowerPoint, OpenDocument and RTF files as new, unsaved PDFs, converted on the open worker by an installed LibreOffice (headless, a private temporary folder and fresh profile with macros off, stopped after 120 s); without LibreOffice the user is told it is needed. The Open dialog lists PDF (the default), Images, Office documents, Text and All files. Insert, Combine and Create ▸ Multiple files refuse Office documents for now, since they convert on the frame thread. `create.from-office` planned → partial. On 2026-10-26 the profile also stopped Writer and Calc from updating linked content (Impress has no such setting), and tests that skip without LibreOffice convert generated DOCX, XLSX, PPTX and RTF files through `doc_create_multiple` and `doc_combine` and check their text. Estimates unchanged. - **2026-10-15 (M2, gap 12):** `cargo xtask text-oracle` scores reading order as well as word F1: the longest common subsequence of the two word sequences over the longer one. F1 is a bag of words and cannot see order at all -- measuring #2431, a change that moves text and nothing else, it called 26 of the 29 files whose output changed identical, where the order score separated 18 improvements from 6 regressions. Reports, does not gate; a disagreement with Poppler, not a verdict. Estimates unchanged. - **2026-10-15 (M2, #2305 follow-up):** WinAnsiEncoding 0xA0 and 0xAD and MacRomanEncoding 0xCA extract as the ordinary space and hyphen Annex D.2 names them, in the vendored hayro tables (patch 30) as well as `crates/fonts`, so search, copy and redaction see characters that were hidden behind U+00A0 and U+00AD. Across the pdf.js corpus the soft hyphens we extract fall 117 to 13 and the non-breaking spaces 30 to 26; the rest come from `/ToUnicode`. An explicit `/Differences` name keeps its glyph list value, as pdftotext does. Estimates unchanged. - **2026-10-15 (M14, gap 16, #1683):** Split view. View ▸ Split (⌘\\ / Ctrl+\\) and a tab's Split right / Split down show two panes, each with its own tab strip, scroll and zoom; a tab dropped on the right half of the page area splits right, and the divider drags. One document in both panes is one document with one Undo; closing one of its views keeps it open. The pane last clicked is the one menus, keys and panels act on. Nothing about the panes is kept across restarts. `view.split-window` planned → shipped. Multiple windows stay planned. Estimates unchanged. diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index 83362aa019..7d948858cd 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -347,7 +347,7 @@ pub fn tools() -> Vec { json!({ "doc": doc(), "pages": pages("to extract"), "out": save_out.clone(), "open": open.clone(), "separate": { "type": "boolean" }, "out_dir": { "type": "string" }, "delete": { "type": "boolean" } }), &["doc", "pages"], )), - t("doc_combine", "Combine files", "Combine files, in order, into one PDF (bookmarks are kept under one entry per file). Images (PNG, JPEG, TIFF, GIF, BMP, JPEG 2000) and .txt files are made into PDFs first. pages optionally chooses each file's pages, in step with paths: a range such as \"1-3, 6\" or null for all pages. passwords, also in step with paths, opens encrypted files (the open password, or the permissions password where a file's security doesn't allow copying pages; null for none). The result is not encrypted. Passwords are never echoed back. groups, also in step with paths, splits a file around others: entries with the same number (they must name the same path) are one file, copied once, so it keeps one bookmark at its first page, its links and its form fields; null for a file of its own. A group uses its first entry's password. When two files' forms name the same field, field_conflicts says what happens: \"rename\" (the default) keeps the fields separate by renaming the later one (Text1 becomes Text1-1) and the result lists renamed_fields as [[old, new], …] — scripts that refer to the old name are not rewritten; \"merge\" makes them one field, as before.") + t("doc_combine", "Combine files", "Combine files, in order, into one PDF (bookmarks are kept under one entry per file). Images (PNG, JPEG, TIFF, GIF, BMP, JPEG 2000), .txt files and Office documents (Word, Excel, PowerPoint, OpenDocument, RTF; through an installed LibreOffice) are made into PDFs first. pages optionally chooses each file's pages, in step with paths: a range such as \"1-3, 6\" or null for all pages. passwords, also in step with paths, opens encrypted files (the open password, or the permissions password where a file's security doesn't allow copying pages; null for none). The result is not encrypted. Passwords are never echoed back. groups, also in step with paths, splits a file around others: entries with the same number (they must name the same path) are one file, copied once, so it keeps one bookmark at its first page, its links and its form fields; null for a file of its own. A group uses its first entry's password. When two files' forms name the same field, field_conflicts says what happens: \"rename\" (the default) keeps the fields separate by renaming the later one (Text1 becomes Text1-1) and the result lists renamed_fields as [[old, new], …] — scripts that refer to the old name are not rewritten; \"merge\" makes them one field, as before.") .cmd("page.combine") .with(schema( json!({ @@ -361,7 +361,7 @@ pub fn tools() -> Vec { }), &["paths"], )), - t("doc_create_multiple", "Create PDF from multiple files", "Convert several files (PDFs, PNG/JPEG/JPEG 2000/TIFF/GIF/BMP images, .txt files) to PDF in one run. mode \"combine\" (default) joins them, in order, into one PDF with a bookmark per file; pages optionally chooses each file's pages, in step with paths (a range such as \"1-3, 6\" or null). mode \"separate\" writes one PDF per file into out_dir (files that are already PDFs are skipped; existing files are never overwritten) and reports each file's result.") + t("doc_create_multiple", "Create PDF from multiple files", "Convert several files (PDFs, PNG/JPEG/JPEG 2000/TIFF/GIF/BMP images, .txt files, Office documents through an installed LibreOffice) to PDF in one run. mode \"combine\" (default) joins them, in order, into one PDF with a bookmark per file; pages optionally chooses each file's pages, in step with paths (a range such as \"1-3, 6\" or null). mode \"separate\" writes one PDF per file into out_dir (files that are already PDFs are skipped; existing files are never overwritten) and reports each file's result.") .cmd("create.multiple") .with(schema( json!({ diff --git a/crates/automation/tests/it/automation.rs b/crates/automation/tests/it/automation.rs index 3a87607373..54e10a035e 100644 --- a/crates/automation/tests/it/automation.rs +++ b/crates/automation/tests/it/automation.rs @@ -1226,7 +1226,7 @@ fn combine_takes_images_and_text_files() { std::fs::write(dir.join("scan.png"), png).unwrap(); std::fs::write(dir.join("notes.txt"), "Some notes").unwrap(); std::fs::write(dir.join("broken.png"), b"\x89PNG\r\n\x1a\nbroken").unwrap(); - std::fs::write(dir.join("report.docx"), b"PK\x03\x04").unwrap(); + std::fs::write(dir.join("archive.zip"), b"PK\x03\x04").unwrap(); let mut a = auto(&dir); let combined = ok(&mut a, "doc_combine", json!({ "paths": ["a.pdf", "scan.png", "notes.txt"], "pages": [null, null, "1"], "open": true })); let doc = combined["document"]["doc"].as_u64().unwrap(); @@ -1236,7 +1236,7 @@ fn combine_takes_images_and_text_files() { let titles: Vec<&str> = marks.as_array().unwrap().iter().map(|b| b["title"].as_str().unwrap()).collect(); assert_eq!(titles, ["a", "scan", "notes"]); // A file that can't be made into a PDF stops the combine, and says which. - for bad in ["broken.png", "report.docx"] { + for bad in ["broken.png", "archive.zip"] { let refused = a.call("doc_combine", &json!({ "paths": ["a.pdf", bad] })); assert!(matches!(&refused, Err(ToolError::Failed(e)) if e.contains(bad)), "{bad}: {refused:?}"); } @@ -5452,7 +5452,7 @@ fn creating_from_multiple_files_through_tools() { } std::fs::write(dir.join("scan.png"), png).unwrap(); std::fs::write(dir.join("notes.txt"), "hello").unwrap(); - std::fs::write(dir.join("report.docx"), b"PK\x03\x04").unwrap(); + std::fs::write(dir.join("archive.zip"), b"PK\x03\x04").unwrap(); let mut a = auto(&dir); // Combine: every file converted, in the order given, with a bookmark per file. @@ -5472,8 +5472,8 @@ fn creating_from_multiple_files_through_tools() { assert_eq!(titles, ["notes", "a", "scan"]); assert!(dir.join("all.pdf").is_file()); // A file that can't be converted fails the whole combine, naming it. - let err = a.call("doc_create_multiple", &json!({ "paths": ["a.pdf", "report.docx"] })).unwrap_err(); - assert!(matches!(&err, ToolError::Failed(m) if m.contains("report.docx") && m.contains("can't be converted")), "{err}"); + let err = a.call("doc_create_multiple", &json!({ "paths": ["a.pdf", "archive.zip"] })).unwrap_err(); + assert!(matches!(&err, ToolError::Failed(m) if m.contains("archive.zip") && m.contains("can't be converted")), "{err}"); assert!(matches!(a.call("doc_create_multiple", &json!({ "paths": [] })), Err(ToolError::InvalidArgs(_)))); assert!(matches!(a.call("doc_create_multiple", &json!({ "paths": ["a.pdf"], "mode": "zip" })), Err(ToolError::InvalidArgs(_)))); assert!(matches!(a.call("doc_create_multiple", &json!({ "paths": ["a.pdf"], "mode": "separate" })), Err(ToolError::InvalidArgs(_)))); @@ -5482,7 +5482,7 @@ fn creating_from_multiple_files_through_tools() { // existing file is never overwritten. std::fs::create_dir_all(dir.join("out")).unwrap(); std::fs::write(dir.join("out/notes.pdf"), b"mine").unwrap(); - let args = json!({ "paths": ["notes.txt", "report.docx", "a.pdf", "scan.png", "missing.txt"], "mode": "separate", "out_dir": "out" }); + let args = json!({ "paths": ["notes.txt", "archive.zip", "a.pdf", "scan.png", "missing.txt"], "mode": "separate", "out_dir": "out" }); let made = ok(&mut a, "doc_create_multiple", args); let files = made["files"].as_array().unwrap(); assert!(files[0]["output"].as_str().unwrap().ends_with("notes (2).pdf"), "{files:?}"); diff --git a/crates/automation/tests/it/main.rs b/crates/automation/tests/it/main.rs index 7a5a6c1bae..35b9d988f7 100644 --- a/crates/automation/tests/it/main.rs +++ b/crates/automation/tests/it/main.rs @@ -15,6 +15,7 @@ mod hostile_args; mod list_selection; mod model_3d; mod model_render; +mod office; mod optimize_content_boundaries; mod portfolio_info; mod redact_after_stray_q_form; diff --git a/crates/automation/tests/it/office.rs b/crates/automation/tests/it/office.rs new file mode 100644 index 0000000000..e406b19ad8 --- /dev/null +++ b/crates/automation/tests/it/office.rs @@ -0,0 +1,277 @@ +//! Office documents converted by a real LibreOffice through the tools that take them +//! (`doc_create_multiple`, `doc_combine`), with the converted text checked. Every test here +//! skips when LibreOffice isn't installed, which is the case on CI; run them on a machine that +//! has it. The Word, Excel and PowerPoint files are written here, as the smallest packages +//! LibreOffice reads, so no binary fixtures are committed. + +use std::path::Path; + +use pdfcraft_automation::{Automation, Content, ToolError}; +use serde_json::{Value, json}; + +fn auto(dir: &Path) -> Automation { + Automation::new().with_root(dir).unwrap().with_clock(|| 1_700_000_000) +} + +fn ok(a: &mut Automation, tool: &str, args: Value) -> Value { + match a.call(tool, &args) { + Ok(mut c) => match c.remove(0) { + Content::Json(v) => v, + other => panic!("{tool}: expected JSON, got {other:?}"), + }, + Err(e) => panic!("{tool} {args}: {e}"), + } +} + +/// Each page's text, with runs of whitespace made single spaces. +fn page_text(a: &mut Automation, doc: u64) -> Vec { + ok(a, "text_extract", json!({ "doc": doc }))["pages"] + .as_array() + .unwrap() + .iter() + .map(|p| p["text"].as_str().unwrap().split_whitespace().collect::>().join(" ")) + .collect() +} + +/// A ZIP of `entries`, stored without compression (Office packages may be). +fn zip(entries: &[(&str, &str)]) -> Vec { + fn crc32(data: &[u8]) -> u32 { + let mut crc = !0u32; + for &b in data { + crc ^= u32::from(b); + for _ in 0..8 { + crc = if crc & 1 == 1 { (crc >> 1) ^ 0xEDB8_8320 } else { crc >> 1 }; + } + } + !crc + } + let (mut out, mut central) = (Vec::new(), Vec::new()); + for (name, data) in entries { + let (crc, size, offset) = (crc32(data.as_bytes()), data.len() as u32, out.len() as u32); + // version 2.0, no flags, stored, 1980-01-01 00:00 + let common = [&20u16.to_le_bytes()[..], &0u16.to_le_bytes(), &0u16.to_le_bytes(), &0u16.to_le_bytes(), &0x21u16.to_le_bytes()].concat(); + let sizes = [crc.to_le_bytes(), size.to_le_bytes(), size.to_le_bytes()].concat(); + out.extend_from_slice(&0x0403_4b50u32.to_le_bytes()); + out.extend_from_slice(&common); + out.extend_from_slice(&sizes); + out.extend_from_slice(&(name.len() as u16).to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); + out.extend_from_slice(name.as_bytes()); + out.extend_from_slice(data.as_bytes()); + central.extend_from_slice(&0x0201_4b50u32.to_le_bytes()); + central.extend_from_slice(&20u16.to_le_bytes()); + central.extend_from_slice(&common); + central.extend_from_slice(&sizes); + central.extend_from_slice(&(name.len() as u16).to_le_bytes()); + // extra, comment, disk, internal and external attributes + central.extend_from_slice(&[0u8; 12]); + central.extend_from_slice(&offset.to_le_bytes()); + central.extend_from_slice(name.as_bytes()); + } + let (at, count) = (out.len() as u32, entries.len() as u16); + out.extend_from_slice(¢ral); + out.extend_from_slice(&0x0605_4b50u32.to_le_bytes()); + out.extend_from_slice(&[0u8; 4]); + out.extend_from_slice(&count.to_le_bytes()); + out.extend_from_slice(&count.to_le_bytes()); + out.extend_from_slice(&(central.len() as u32).to_le_bytes()); + out.extend_from_slice(&at.to_le_bytes()); + out.extend_from_slice(&0u16.to_le_bytes()); + out +} + +const XML: &str = r#""#; +const PACKAGE_RELS: &str = "http://schemas.openxmlformats.org/package/2006/relationships"; +const DOC_RELS: &str = "http://schemas.openxmlformats.org/officeDocument/2006/relationships"; + +/// `[Content_Types].xml` with one override per (part, type). +fn content_types(parts: &[(&str, &str)]) -> String { + let overrides: String = parts.iter().map(|(part, kind)| format!(r#""#)).collect(); + format!( + r#"{XML}{overrides}"# + ) +} + +/// A relationships part with one relationship per (id, type under officeDocument, target). +fn rels(list: &[(&str, &str, &str)]) -> String { + let items: String = + list.iter().map(|(id, kind, target)| format!(r#""#)).collect(); + format!(r#"{XML}{items}"#) +} + +/// A Word document of one paragraph. +fn docx(text: &str) -> Vec { + let document = format!( + r#"{XML}{text}"# + ); + zip(&[ + ( + "[Content_Types].xml", + &content_types(&[("word/document.xml", "application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml")]), + ), + ("_rels/.rels", &rels(&[("rId1", "officeDocument", "word/document.xml")])), + ("word/document.xml", &document), + ]) +} + +/// An Excel workbook whose one sheet holds `text` in A1 and a formula, `=6*7`, in A2. +fn xlsx(text: &str) -> Vec { + let workbook = format!( + r#"{XML}"# + ); + let sheet = format!( + r#"{XML}{text}6*7"# + ); + zip(&[ + ( + "[Content_Types].xml", + &content_types(&[ + ("xl/workbook.xml", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml"), + ("xl/worksheets/sheet1.xml", "application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml"), + ]), + ), + ("_rels/.rels", &rels(&[("rId1", "officeDocument", "xl/workbook.xml")])), + ("xl/workbook.xml", &workbook), + ("xl/_rels/workbook.xml.rels", &rels(&[("rId1", "worksheet", "worksheets/sheet1.xml")])), + ("xl/worksheets/sheet1.xml", &sheet), + ]) +} + +/// A PowerPoint presentation of one slide with one text box. +fn pptx(text: &str) -> Vec { + const P: &str = + r#"xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main" xmlns:p="http://schemas.openxmlformats.org/presentationml/2006/main""#; + let presentation = format!( + r#"{XML}"# + ); + let tree = r#""#; + let master = format!( + r#"{XML}{tree}"# + ); + let layout = format!(r#"{XML}{tree}"#); + let slide = format!( + r#"{XML}{tree}{text}"# + ); + let theme = format!( + r#"{XML}"# + ); + const PML: &str = "application/vnd.openxmlformats-officedocument.presentationml"; + let types = content_types(&[ + ("ppt/presentation.xml", &format!("{PML}.presentation.main+xml")), + ("ppt/slideMasters/slideMaster1.xml", &format!("{PML}.slideMaster+xml")), + ("ppt/slideLayouts/slideLayout1.xml", &format!("{PML}.slideLayout+xml")), + ("ppt/slides/slide1.xml", &format!("{PML}.slide+xml")), + ("ppt/theme/theme1.xml", "application/vnd.openxmlformats-officedocument.theme+xml"), + ]); + zip(&[ + ("[Content_Types].xml", &types), + ("_rels/.rels", &rels(&[("rId1", "officeDocument", "ppt/presentation.xml")])), + ("ppt/presentation.xml", &presentation), + ( + "ppt/_rels/presentation.xml.rels", + &rels(&[ + ("rId1", "slideMaster", "slideMasters/slideMaster1.xml"), + ("rId2", "slide", "slides/slide1.xml"), + ("rId3", "theme", "theme/theme1.xml"), + ]), + ), + ("ppt/slideMasters/slideMaster1.xml", &master), + ( + "ppt/slideMasters/_rels/slideMaster1.xml.rels", + &rels(&[("rId1", "slideLayout", "../slideLayouts/slideLayout1.xml"), ("rId2", "theme", "../theme/theme1.xml")]), + ), + ("ppt/slideLayouts/slideLayout1.xml", &layout), + ("ppt/slideLayouts/_rels/slideLayout1.xml.rels", &rels(&[("rId1", "slideMaster", "../slideMasters/slideMaster1.xml")])), + ("ppt/slides/slide1.xml", &slide), + ("ppt/slides/_rels/slide1.xml.rels", &rels(&[("rId1", "slideLayout", "../slideLayouts/slideLayout1.xml")])), + ("ppt/theme/theme1.xml", &theme), + ]) +} + +/// An RTF document of one paragraph. +fn rtf(text: &str) -> Vec { + format!("{{\\rtf1\\ansi{{\\fonttbl\\f0\\fswiss Helvetica;}}\\f0\\pard {text}\\par}}").into_bytes() +} + +/// A folder with one Word, Excel, PowerPoint and RTF file, each carrying its own word, and the +/// tools rooted there; `None` when LibreOffice isn't installed. +fn office_files() -> Option<(tempfile::TempDir, Automation)> { + let dir = tempfile::tempdir().unwrap(); + std::fs::write(dir.path().join("letter.docx"), docx("Quokka letter")).unwrap(); + std::fs::write(dir.path().join("budget.xlsx"), xlsx("Narwhal budget")).unwrap(); + std::fs::write(dir.path().join("deck.pptx"), pptx("Axolotl deck")).unwrap(); + std::fs::write(dir.path().join("notes.rtf"), rtf("Pangolin notes")).unwrap(); + let mut a = auto(dir.path()); + match a.call("doc_create_multiple", &json!({ "paths": ["notes.rtf"] })) { + Err(ToolError::Failed(e)) if e.contains("needs LibreOffice") => { + eprintln!("skipped: LibreOffice isn't installed"); + None + } + _ => Some((dir, a)), + } +} + +/// Each Office format converts to a PDF of its own that holds the document's text (and, for +/// the spreadsheet, the formula's value). +#[test] +fn office_documents_convert_separately_with_their_text() { + let Some((dir, mut a)) = office_files() else { return }; + let paths = ["letter.docx", "budget.xlsx", "deck.pptx", "notes.rtf"]; + let made = ok(&mut a, "doc_create_multiple", json!({ "paths": paths, "mode": "separate", "out_dir": "out" })); + let files = made["files"].as_array().unwrap(); + assert_eq!(files.len(), 4, "{made}"); + let expected = [ + ("letter.pdf", &["Quokka letter"][..]), + ("budget.pdf", &["Narwhal budget", "42"]), + ("deck.pdf", &["Axolotl deck"]), + ("notes.pdf", &["Pangolin notes"]), + ]; + for (file, (name, words)) in files.iter().zip(expected) { + let output = file["output"].as_str().unwrap_or_else(|| panic!("{file}")); + // The tool reports the folder in its canonical form (`\\?\C:\…` on Windows). + assert_eq!(Path::new(output).file_name(), Some(std::ffi::OsStr::new(name)), "{output}"); + assert!(dir.path().join("out").join(name).is_file(), "{name}"); + let doc = ok(&mut a, "doc_open", json!({ "path": output }))["doc"].as_u64().unwrap(); + let text = page_text(&mut a, doc).join("\n"); + for word in words { + assert!(text.contains(word), "{output}: {word:?} not in {text:?}"); + } + } +} + +/// Office documents combine, in the order given, with a PDF made here, one bookmark each. +#[test] +fn office_documents_combine_in_order_with_a_bookmark_each() { + let Some((dir, mut a)) = office_files() else { return }; + let made = ok( + &mut a, + "doc_create_multiple", + json!({ "paths": ["deck.pptx", "letter.docx", "budget.xlsx", "notes.rtf"], "out": "all.pdf", "open": true }), + ); + let doc = made["document"]["doc"].as_u64().unwrap(); + let pages = page_text(&mut a, doc); + assert_eq!(pages.len(), 4, "{pages:?}"); + for (page, word) in pages.iter().zip(["Axolotl deck", "Quokka letter", "Narwhal budget", "Pangolin notes"]) { + assert!(page.contains(word), "{word:?} not in {page:?}"); + } + let titles: Vec = ok(&mut a, "bookmark_list", json!({ "doc": doc }))["bookmarks"] + .as_array() + .unwrap() + .iter() + .map(|b| b["title"].as_str().unwrap().to_string()) + .collect(); + assert_eq!(titles, ["deck", "letter", "budget", "notes"]); + assert!(dir.path().join("all.pdf").is_file()); + + // doc_combine takes them beside a PDF, with a page range for one of them. + std::fs::write(dir.path().join("cover.txt"), "Cover page").unwrap(); + ok(&mut a, "doc_create_multiple", json!({ "paths": ["cover.txt"], "out": "cover.pdf" })); + assert!(dir.path().join("cover.pdf").is_file()); + let combined = ok(&mut a, "doc_combine", json!({ "paths": ["cover.pdf", "notes.rtf", "letter.docx"], "pages": [null, "1", null], "open": true })); + let doc = combined["document"]["doc"].as_u64().unwrap(); + let pages = page_text(&mut a, doc); + assert_eq!(pages.len(), 3, "{pages:?}"); + for (page, word) in pages.iter().zip(["Cover page", "Pangolin notes", "Quokka letter"]) { + assert!(page.contains(word), "{word:?} not in {page:?}"); + } +} diff --git a/crates/create/Cargo.toml b/crates/create/Cargo.toml index 51eca668d0..79c7a4f276 100644 --- a/crates/create/Cargo.toml +++ b/crates/create/Cargo.toml @@ -24,6 +24,11 @@ thiserror = { workspace = true } # Already in the graph through lopdf; used only to read plain-text files (decode_text). encoding_rs = "0.8" +[target.'cfg(unix)'.dependencies] +# Already in the graph (pdfcraft-platform, ui-egui); stops a timed-out LibreOffice's whole process +# group, soffice.bin included, without unsafe code. +rustix = { version = "1", features = ["process"] } + [dev-dependencies] # An independent CCITT decoder, to check Create PDF's Group 4 images. hayro-ccitt.workspace = true diff --git a/crates/create/README.md b/crates/create/README.md index 95d47d27a9..4c9d1a8823 100644 --- a/crates/create/README.md +++ b/crates/create/README.md @@ -28,6 +28,30 @@ Layer L4. Create a PDF (Acrobat's Create a PDF tool), execution plan M10.2: Every function returns a `pdfcraft_cos::Document`; the caller writes it. +Office documents (`office.rs`) are the exception: PdfCraft doesn't read Word, Excel, PowerPoint, +OpenDocument or RTF itself. `is_office(name, bytes)` recognises them (an `OFFICE_EXTENSIONS` +name and an OLE, ZIP or RTF signature; `source_kind` reports `SourceKind::Office`), and +`office_to_pdf(name, bytes)` returns PDF bytes made by an installed LibreOffice, run headless as +a separate process: a private temporary folder (owner-only on Unix; the document is always +`document.` there, never its own name), a fresh profile with macros disabled, untrusted +links blocked and Writer and Calc set never to update linked content (sections, files, external +data) on load, no console window on Windows, and a stop after `TIMEOUT` (120 s) or as soon as the +output grows past 512 MB (the whole process tree is stopped, not only `soffice`). LibreOffice is +found through `PDFCRAFT_SOFFICE`, the `PATH`, then the usual install folders (`libreoffice()`). +A snap-installed LibreOffice is not among those folders: its private `/tmp` cannot see the work +folder. Where `/snap/bin` is on the `PATH` it can still be found there, so point +`PDFCRAFT_SOFFICE` at a non-snap install if conversions fail. It takes seconds, so callers keep it +off the frame thread. On the web it returns an error. + +Impress has no setting for linked content, and the profile settings are LibreOffice's own: +PdfCraft doesn't sandbox the process's network access. Converting test documents whose linked +sections, pictures and external cell references point at a local listener made no connection, +with or without the Writer and Calc settings (LibreOffice 26.8.1, Windows, 2026-10-26), so the settings are a +second line of defence rather than a measured fix. + +`IMAGE_EXTENSIONS`, `TEXT_EXTENSIONS` and `OFFICE_EXTENSIONS` are what the app's file pickers +offer, so detection and pickers agree. + Image export and Save Image As apply each lossless image's Decode pair before converting its color components. JBIG2 images and masks are decoded through `pdfcraft_cos::ImageBudget` (`pdfcraft_filters::decode_jbig2_within`), behind other filters too, with the `/JBIG2Globals` of their own `/DecodeParms` entry, as `JBIG2Decode` gives them: black as 0. An image diff --git a/crates/create/src/lib.rs b/crates/create/src/lib.rs index 389173d848..d6c2208a83 100644 --- a/crates/create/src/lib.rs +++ b/crates/create/src/lib.rs @@ -1,4 +1,4 @@ -//! pdfcraft-create — create PDFs from nothing, images or text (L4). See the README. +//! pdfcraft-create — create PDFs from nothing, images, text or (through LibreOffice) Office documents (L4). See the README. #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] @@ -8,6 +8,10 @@ mod extract; mod text_font; mod upright_image; pub use extract::{ExtractedImage, ImageExport, extract_images, image_file, image_file_within}; +mod office; +#[cfg(not(target_arch = "wasm32"))] +pub use office::libreoffice; +pub use office::{OFFICE_EXTENSIONS, is_office, office_to_pdf}; use pdfcraft_fonts::wrap_with; use text_font::TextFont; @@ -17,6 +21,9 @@ pub enum CreateError { Image(String, String), #[error("{0}")] Invalid(String), + /// An Office document LibreOffice couldn't convert, or LibreOffice isn't installed. + #[error("{0}")] + Office(String), } /// US Letter in points. @@ -757,9 +764,18 @@ pub enum SourceKind { Pdf, Image, Text, + /// A Word, Excel, PowerPoint or OpenDocument file, converted through LibreOffice. + Office, } -/// File extensions Create converts to PDF (images and plain text). +/// File extensions of the images Create converts to PDF. +pub const IMAGE_EXTENSIONS: [&str; 11] = ["png", "jpg", "jpeg", "tif", "tiff", "gif", "bmp", "jp2", "j2k", "jpx", "pcx"]; + +/// File extensions of the plain-text files Create converts to PDF. +pub const TEXT_EXTENSIONS: [&str; 2] = ["txt", "text"]; + +/// File extensions Create converts to PDF itself: [`IMAGE_EXTENSIONS`] and [`TEXT_EXTENSIONS`] +/// (Office documents go through LibreOffice, [`OFFICE_EXTENSIONS`]). pub const CONVERTIBLE: [&str; 13] = ["png", "jpg", "jpeg", "tif", "tiff", "gif", "bmp", "jp2", "j2k", "jpx", "pcx", "txt", "text"]; /// A PCX file: its manufacturer byte, a version (1 was never used), run-length coding and 1 or 8 @@ -792,8 +808,8 @@ fn is_image(bytes: &[u8]) -> bool { && bytes.get(14..18).and_then(|h| <[u8; 4]>::try_from(h).ok()).is_some_and(|h| matches!(u32::from_le_bytes(h), 12 | 40 | 52 | 56 | 108 | 124))) } -/// Whether `bytes` named `name` is a PDF, an image Create can embed, or plain text (a `.txt` or -/// `.text` file). `None` for anything else. +/// Whether `bytes` named `name` is a PDF, an image Create can embed, plain text (a `.txt` or +/// `.text` file) or an Office document ([`is_office`]). `None` for anything else. pub fn source_kind(name: &str, bytes: &[u8]) -> Option { let head = bytes.get(..bytes.len().min(1024)).unwrap_or_default(); if head.windows(5).any(|w| w == b"%PDF-") { @@ -802,6 +818,9 @@ pub fn source_kind(name: &str, bytes: &[u8]) -> Option { if is_image(bytes) { return Some(SourceKind::Image); } + if is_office(name, bytes) { + return Some(SourceKind::Office); + } let lower = name.to_ascii_lowercase(); (lower.ends_with(".txt") || lower.ends_with(".text")).then_some(SourceKind::Text) } diff --git a/crates/create/src/office.rs b/crates/create/src/office.rs new file mode 100644 index 0000000000..3d8332c149 --- /dev/null +++ b/crates/create/src/office.rs @@ -0,0 +1,599 @@ +//! Office documents (Word, Excel, PowerPoint, OpenDocument, RTF) to PDF through an installed +//! LibreOffice, run as a separate process: the "LibreOffice sidecar" of +//! `docs/file-format-parity.md`. PdfCraft never parses these formats itself. +//! +//! Detection ([`is_office`]) is pure and works everywhere. Conversion ([`office_to_pdf`]) runs +//! only on desktop builds: each run gets its own temporary folder and a fresh LibreOffice +//! profile with macros and untrusted links turned off, and is stopped after [`TIMEOUT`]. + +use crate::CreateError; + +/// File extensions of the Office documents converted through LibreOffice. +pub const OFFICE_EXTENSIONS: &[&str] = &[ + // Word and Writer + "doc", "docx", "docm", "dot", "dotx", "dotm", "odt", "ott", "rtf", // + // Excel and Calc + "xls", "xlsx", "xlsm", "xlt", "xltx", "xltm", "ods", "ots", // + // PowerPoint and Impress + "ppt", "pptx", "pptm", "pot", "potx", "potm", "pps", "ppsx", "ppsm", "odp", "otp", +]; + +/// The OLE compound-file signature of Office 97–2003 files (`.doc`, `.xls`, `.ppt`). +const OLE: &[u8] = &[0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1]; + +/// The longest a conversion may run before LibreOffice is stopped. A fresh profile costs a +/// few seconds on its own; large spreadsheets take longer. +#[cfg_attr(target_arch = "wasm32", allow(dead_code))] +pub const TIMEOUT: std::time::Duration = std::time::Duration::from_secs(120); + +/// The largest PDF read back from LibreOffice. +#[cfg_attr(target_arch = "wasm32", allow(dead_code))] +const MAX_PDF: u64 = 512 * 1024 * 1024; + +/// `name`'s extension as it appears in [`OFFICE_EXTENSIONS`], if it is one. +fn office_extension(name: &str) -> Option<&'static str> { + let (_, ext) = name.rsplit_once('.')?; + OFFICE_EXTENSIONS.iter().copied().find(|e| ext.eq_ignore_ascii_case(e)) +} + +/// Whether `bytes` named `name` is an Office document: an Office extension, and the bytes start +/// like one (an OLE compound file, a ZIP package as OOXML and OpenDocument are, or RTF). +pub fn is_office(name: &str, bytes: &[u8]) -> bool { + office_extension(name).is_some() && (bytes.starts_with(OLE) || bytes.starts_with(b"PK\x03\x04") || bytes.starts_with(b"{\\rtf")) +} + +/// Convert the Office document `bytes` named `name` to PDF bytes with LibreOffice. +#[cfg(target_arch = "wasm32")] +pub fn office_to_pdf(name: &str, _bytes: &[u8]) -> Result, CreateError> { + Err(CreateError::Office(format!("{name}: Office documents open only in the desktop app, which converts them with LibreOffice"))) +} + +/// Convert the Office document `bytes` named `name` to PDF bytes with LibreOffice. +#[cfg(not(target_arch = "wasm32"))] +pub fn office_to_pdf(name: &str, bytes: &[u8]) -> Result, CreateError> { + let program = native::libreoffice().ok_or_else(|| { + CreateError::Office(format!( + "{name}: opening Word, Excel and PowerPoint files needs LibreOffice, which isn't installed (get it free from libreoffice.org)" + )) + })?; + native::convert_with(&program, name, bytes, TIMEOUT) +} + +/// Where LibreOffice is installed, if it is: `PDFCRAFT_SOFFICE`, then the `PATH`, then the +/// usual install folders. +#[cfg(not(target_arch = "wasm32"))] +pub fn libreoffice() -> Option { + native::libreoffice() +} + +#[cfg(not(target_arch = "wasm32"))] +mod native { + use super::{CreateError, MAX_PDF, office_extension}; + use std::ffi::OsString; + use std::path::{Path, PathBuf}; + use std::process::{Child, Command, ExitStatus, Stdio}; + use std::sync::atomic::{AtomicU64, Ordering}; + use std::time::{Duration, Instant}; + + /// The program names LibreOffice installs on the `PATH`. + #[cfg(windows)] + const PATH_NAMES: &[&str] = &["soffice.exe"]; + #[cfg(not(windows))] + const PATH_NAMES: &[&str] = &["soffice", "libreoffice"]; + + pub(super) fn libreoffice() -> Option { + let defaults = default_locations(&|key| std::env::var_os(key)); + candidates(std::env::var_os("PDFCRAFT_SOFFICE"), std::env::var_os("PATH"), defaults).into_iter().find(|p| p.is_file()) + } + + /// Where to look, in order: the override, each `PATH` folder, then `defaults`. + pub(super) fn candidates(override_path: Option, path: Option, defaults: Vec) -> Vec { + let mut out: Vec = override_path.filter(|p| !p.is_empty()).map(PathBuf::from).into_iter().collect(); + if let Some(path) = path { + for dir in std::env::split_paths(&path) { + out.extend(PATH_NAMES.iter().map(|name| dir.join(name))); + } + } + out.extend(defaults); + out + } + + /// The usual install locations, given a way to read environment variables. + #[cfg(windows)] + pub(super) fn default_locations(env: &dyn Fn(&str) -> Option) -> Vec { + ["ProgramFiles", "ProgramW6432", "ProgramFiles(x86)"] + .iter() + .filter_map(|key| env(key)) + .map(|dir| PathBuf::from(dir).join("LibreOffice").join("program").join("soffice.exe")) + .collect() + } + + /// The usual install locations, given a way to read environment variables. + #[cfg(target_os = "macos")] + pub(super) fn default_locations(env: &dyn Fn(&str) -> Option) -> Vec { + let app = Path::new("LibreOffice.app/Contents/MacOS/soffice"); + let mut out = vec![Path::new("/Applications").join(app)]; + out.extend(env("HOME").map(|home| PathBuf::from(home).join("Applications").join(app))); + out + } + + /// The usual install locations, given a way to read environment variables. + #[cfg(not(any(windows, target_os = "macos")))] + pub(super) fn default_locations(_env: &dyn Fn(&str) -> Option) -> Vec { + [ + "/usr/bin/soffice", + "/usr/bin/libreoffice", + "/usr/local/bin/soffice", + "/usr/local/bin/libreoffice", + "/usr/lib/libreoffice/program/soffice", + "/opt/libreoffice/program/soffice", + ] + .iter() + .map(PathBuf::from) + .collect() + } + + /// The fresh profile's settings: no macros at all, no links to untrusted locations, and + /// linked content in Writer and Calc documents (sections, files, external data) never + /// updated on load, so a document can't make LibreOffice fetch from the network or a share. + /// The two `Link` settings number "Never" differently (Writer.xcs: 0 Always, 1 On request, + /// 2 Never; Calc.xcs: 0 Always, 1 Never, 2 On request). Impress has no such setting. + pub(super) const PROFILE_SETTINGS: &str = r#" + +true +3 +true +2 +1 + +"#; + + /// A private temporary folder, removed (with everything in it) when dropped. + pub(super) struct WorkDir(pub(super) PathBuf); + + impl WorkDir { + pub(super) fn new() -> std::io::Result { + static NEXT: AtomicU64 = AtomicU64::new(0); + let n = NEXT.fetch_add(1, Ordering::Relaxed); + let nanos = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map_or(0, |d| d.subsec_nanos()); + let dir = std::env::temp_dir().join(format!("pdfcraft-office-{}-{n}-{nanos}", std::process::id())); + // Not recursive: a folder that already exists is not ours. Owner-only on Unix, where the + // temporary folder is shared: the document and its PDF are not for other users. + #[cfg_attr(not(unix), allow(unused_mut))] + let mut builder = std::fs::DirBuilder::new(); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o700); + } + builder.create(&dir)?; + Ok(Self(dir)) + } + } + + impl Drop for WorkDir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + + /// `path` as a `file://` URL, which `-env:UserInstallation` takes. + pub(super) fn file_url(path: &Path) -> String { + let path = path.to_string_lossy().replace('\\', "/"); + let mut url = String::from("file://"); + // A Windows path (`C:/…`) needs the third slash a Unix path already starts with. + if !path.starts_with('/') { + url.push('/'); + } + for b in path.bytes() { + if b.is_ascii_alphanumeric() || b"-._~/:".contains(&b) { + url.push(char::from(b)); + } else { + url.push_str(&format!("%{b:02X}")); + } + } + url + } + + pub(super) enum Run { + Exited(ExitStatus), + TimedOut, + /// The watched folder grew past its limit while the command ran. + TooLarge, + } + + /// The bytes in the files directly inside `dir` (LibreOffice writes its one output there). + /// Anything unreadable counts as nothing, and the sum saturates. Each size is read from the + /// file itself: on Windows a directory listing keeps the old size while a process still + /// has the file open for writing. + pub(super) fn folder_size(dir: &Path) -> u64 { + let Ok(entries) = std::fs::read_dir(dir) else { return 0 }; + entries + .flatten() + .filter_map(|entry| std::fs::symlink_metadata(entry.path()).ok()) + .filter(|meta| meta.is_file()) + .fold(0u64, |sum, meta| sum.saturating_add(meta.len())) + } + + /// Run `command` to completion, killing it and every process it started once `timeout` + /// passes, or once the files in `watch`'s folder add up to more than its limit (so a + /// document that converts to an enormous PDF can't fill the disk before it is checked). + /// `soffice` hands the conversion to its own child, `soffice.bin`, which would + /// otherwise outlive the kill, holding the profile folder and its memory. + pub(super) fn run_limited(mut command: Command, timeout: Duration, watch: Option<(&Path, u64)>) -> std::io::Result { + #[cfg(unix)] + { + use std::os::unix::process::CommandExt; + // Its own process group, so the whole group can be stopped together. + command.process_group(0); + } + let mut child = command.spawn()?; + let start = Instant::now(); + loop { + match child.try_wait() { + Ok(Some(status)) => return Ok(Run::Exited(status)), + Ok(None) => {} + Err(error) => { + kill_tree(&mut child); + return Err(error); + } + } + if watch.is_some_and(|(dir, limit)| folder_size(dir) > limit) { + kill_tree(&mut child); + return Ok(Run::TooLarge); + } + if start.elapsed() >= timeout { + kill_tree(&mut child); + return Ok(Run::TimedOut); + } + std::thread::sleep(Duration::from_millis(50)); + } + } + + /// Kill `child` and every process it started, then reap it. + fn kill_tree(child: &mut Child) { + #[cfg(unix)] + { + use rustix::process::{Pid, Signal, kill_process_group}; + // The group `process_group(0)` made has the child's id. + if let Some(group) = i32::try_from(child.id()).ok().and_then(Pid::from_raw) { + let _ = kill_process_group(group, Signal::KILL); + } + } + #[cfg(windows)] + { + use std::os::windows::process::CommandExt; + // `/T` walks the tree from the child's id, so it runs while the child is still alive. + // Windows' own copy, not whatever `taskkill` the PATH finds first. + let taskkill = std::env::var_os("SystemRoot") + .map_or_else(|| PathBuf::from("taskkill.exe"), |root| PathBuf::from(root).join("System32").join("taskkill.exe")); + let _ = Command::new(taskkill) + .args(["/T", "/F", "/PID", &child.id().to_string()]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + // CREATE_NO_WINDOW, as for LibreOffice itself. + .creation_flags(0x0800_0000) + .status(); + } + // The child itself, if the tree kill couldn't reach it. + let _ = child.kill(); + let _ = child.wait(); + } + + /// Convert with the LibreOffice at `program`, giving up after `timeout`. + pub(super) fn convert_with(program: &Path, name: &str, bytes: &[u8], timeout: Duration) -> Result, CreateError> { + convert_capped(program, name, bytes, timeout, MAX_PDF) + } + + /// [`convert_with`] with the converted PDF's size limit given: the output folder is watched + /// while LibreOffice runs, and the finished file is checked again. + pub(super) fn convert_capped(program: &Path, name: &str, bytes: &[u8], timeout: Duration, max_pdf: u64) -> Result, CreateError> { + let too_large = || format!("the converted PDF is larger than {} MB", max_pdf / (1024 * 1024)); + let failed = |why: String| CreateError::Office(format!("{name}: {why}")); + let ext = office_extension(name).ok_or_else(|| failed("this isn't a Word, Excel or PowerPoint file".into()))?; + let work = WorkDir::new().map_err(|e| failed(format!("couldn't make a temporary folder: {e}")))?; + // A fixed input name: the document's own name never reaches LibreOffice's command line. + let input = work.0.join(format!("document.{ext}")); + let profile = work.0.join("profile"); + let out = work.0.join("out"); + let prepared = std::fs::write(&input, bytes) + .and_then(|()| std::fs::create_dir_all(profile.join("user"))) + .and_then(|()| std::fs::write(profile.join("user").join("registrymodifications.xcu"), PROFILE_SETTINGS)) + .and_then(|()| std::fs::create_dir(&out)); + prepared.map_err(|e| failed(format!("couldn't prepare the conversion: {e}")))?; + + let mut command = Command::new(program); + command + .arg(format!("-env:UserInstallation={}", file_url(&profile))) + .args(["--headless", "--invisible", "--nologo", "--norestore", "--nodefault", "--nolockcheck", "--convert-to", "pdf", "--outdir"]) + .arg(&out) + .arg(&input) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()); + #[cfg(windows)] + { + use std::os::windows::process::CommandExt; + // CREATE_NO_WINDOW: no console window flashes up during the conversion. + command.creation_flags(0x0800_0000); + } + let status = match run_limited(command, timeout, Some((&out, max_pdf))) { + Ok(Run::Exited(status)) => status, + Ok(Run::TooLarge) => return Err(failed(format!("{}; LibreOffice was stopped", too_large()))), + Ok(Run::TimedOut) => return Err(failed(format!("LibreOffice took longer than {} seconds and was stopped", timeout.as_secs()))), + Err(e) => return Err(failed(format!("couldn't start LibreOffice ({}): {e}", program.display()))), + }; + let pdf = out.join("document.pdf"); + let size = match std::fs::metadata(&pdf) { + Ok(meta) => meta.len(), + Err(_) if !status.success() => return Err(failed(format!("LibreOffice couldn't convert it ({status})"))), + Err(_) => return Err(failed("LibreOffice didn't produce a PDF; the file may be damaged or password-protected".into())), + }; + if size > max_pdf { + return Err(failed(too_large())); + } + let bytes = std::fs::read(&pdf).map_err(|e| failed(format!("couldn't read the converted PDF: {e}")))?; + if !bytes.starts_with(b"%PDF-") { + return Err(failed("LibreOffice's output isn't a PDF".into())); + } + Ok(bytes) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const DOCX_HEAD: &[u8] = b"PK\x03\x04\x14\x00\x06\x00"; + + #[test] + fn office_files_are_recognised_by_extension_and_signature() { + assert!(is_office("Report.docx", DOCX_HEAD)); + assert!(is_office("SHEET.XLSX", DOCX_HEAD)); + assert!(is_office("deck.odp", DOCX_HEAD)); + assert!(is_office("old.doc", OLE)); + assert!(is_office("old.ppt", &[OLE, b"rest"].concat())); + assert!(is_office("letter.rtf", b"{\\rtf1\\ansi")); + // Word saves RTF under .doc too. + assert!(is_office("letter.doc", b"{\\rtf1\\ansi")); + } + + #[test] + fn other_files_are_not_office_documents() { + // The right signature with another extension, or an Office name with other bytes. + assert!(!is_office("archive.zip", DOCX_HEAD)); + assert!(!is_office("Report.docx", b"%PDF-1.7")); + assert!(!is_office("Report.docx", b"plain text")); + assert!(!is_office("Report.docx", b"")); + assert!(!is_office("docx", DOCX_HEAD)); + assert!(!is_office("", DOCX_HEAD)); + // A multi-byte name never splits a character. + assert!(!is_office("résumé.dócx", DOCX_HEAD)); + assert!(is_office("résumé.docx", DOCX_HEAD)); + } + + #[cfg(not(target_arch = "wasm32"))] + mod native { + use super::super::native::*; + use super::DOCX_HEAD; + use std::ffi::OsString; + use std::path::{Path, PathBuf}; + use std::time::Duration; + + #[test] + fn file_urls_escape_spaces_and_keep_drive_letters() { + assert_eq!(file_url(Path::new("/tmp/a b/profile")), "file:///tmp/a%20b/profile"); + assert_eq!(file_url(Path::new(r"C:\Users\Jo Ann\profile")), "file:///C:/Users/Jo%20Ann/profile"); + assert_eq!(file_url(Path::new("/tmp/é")), "file:///tmp/%C3%A9"); + } + + #[test] + fn the_override_comes_first_then_the_path_then_the_defaults() { + let path = std::env::join_paths([Path::new("first"), Path::new("second")]).unwrap(); + let found = candidates(Some(OsString::from("custom/soffice")), Some(path), vec![PathBuf::from("default/soffice")]); + assert_eq!(found.first(), Some(&PathBuf::from("custom/soffice"))); + assert!(found.get(1).is_some_and(|p| p.starts_with("first"))); + assert_eq!(found.last(), Some(&PathBuf::from("default/soffice"))); + // An empty override is no override. + assert_eq!(candidates(Some(OsString::new()), None, Vec::new()), Vec::::new()); + } + + #[test] + fn default_locations_read_only_the_given_environment() { + let found = default_locations(&|_| None); + assert!(found.iter().all(|p| p.is_absolute()), "{found:?}"); + } + + /// Each setting the fresh profile makes, as (path, property, value). + fn profile_settings() -> Vec<(&'static str, &'static str, &'static str)> { + let between = |s: &'static str, start: &str, end: &str| s.split_once(start).and_then(|(_, rest)| rest.split_once(end)).map(|(v, _)| v); + PROFILE_SETTINGS + .split("", "")?)) + }) + .collect() + } + + #[test] + fn the_profile_turns_off_macros_and_never_updates_links() { + let settings = profile_settings(); + let scripting = "/org.openoffice.Office.Common/Security/Scripting"; + assert!(settings.contains(&(scripting, "DisableMacrosExecution", "true")), "{settings:?}"); + assert!(settings.contains(&(scripting, "MacroSecurityLevel", "3")), "{settings:?}"); + assert!(settings.contains(&(scripting, "BlockUntrustedRefererLinks", "true")), "{settings:?}"); + // "Never" is 2 for Writer but 1 for Calc (Writer.xcs and Calc.xcs); Writer's 1 is + // its default, "On request". + assert!(settings.contains(&("/org.openoffice.Office.Writer/Content/Update", "Link", "2")), "{settings:?}"); + assert!(settings.contains(&("/org.openoffice.Office.Calc/Content/Update", "Link", "1")), "{settings:?}"); + assert_eq!(settings.len(), 5, "{settings:?}"); + } + + #[test] + fn a_missing_converter_is_an_error_not_a_crash() { + let missing = std::env::temp_dir().join("pdfcraft-no-such-soffice").join("soffice"); + let error = convert_with(&missing, "Report.docx", DOCX_HEAD, Duration::from_secs(5)).unwrap_err(); + assert!(error.to_string().contains("couldn't start LibreOffice"), "{error}"); + } + + #[test] + fn a_converter_that_makes_no_pdf_is_reported() { + // This test binary stands in for LibreOffice: it rejects the arguments and exits + // without writing anything. + let program = std::env::current_exe().unwrap(); + let error = convert_with(&program, "Report.docx", DOCX_HEAD, Duration::from_secs(60)).unwrap_err(); + assert!(error.to_string().contains("LibreOffice"), "{error}"); + } + + #[test] + fn a_converter_that_hangs_is_stopped() { + let mut command = std::process::Command::new(std::env::current_exe().unwrap()); + command.args(["--ignored", "--exact", "office::tests::native::sleeps_for_a_minute", "--nocapture"]); + command.stdout(std::process::Stdio::null()).stderr(std::process::Stdio::null()); + let started = std::time::Instant::now(); + assert!(matches!(run_limited(command, Duration::from_millis(500), None), Ok(Run::TimedOut))); + assert!(started.elapsed() < Duration::from_secs(30)); + } + + /// The stand-in for a LibreOffice that never finishes (run only by the test above). + #[test] + #[ignore] + fn sleeps_for_a_minute() { + std::thread::sleep(Duration::from_secs(60)); + } + + /// Where the stand-ins below write their heartbeat. + const HEARTBEAT: &str = "PDFCRAFT_TEST_OFFICE_HEARTBEAT"; + + /// This test binary run as one of its own ignored tests, with the heartbeat file set. + fn stand_in(test: &str, heartbeat: &Path) -> std::process::Command { + let mut command = std::process::Command::new(std::env::current_exe().unwrap()); + command.args(["--ignored", "--exact", test, "--nocapture"]).env(HEARTBEAT, heartbeat); + command.stdin(std::process::Stdio::null()).stdout(std::process::Stdio::null()).stderr(std::process::Stdio::null()); + command + } + + /// `soffice` starts `soffice.bin` as its own child: a timeout must stop both, or the + /// child keeps running, holding the profile folder. + #[test] + fn a_timeout_stops_the_processes_the_converter_started() { + let work = WorkDir::new().unwrap(); + let heartbeat = work.0.join("heartbeat"); + let parent = stand_in("office::tests::native::starts_a_child_that_keeps_beating", &heartbeat); + assert!(matches!(run_limited(parent, Duration::from_secs(5), None), Ok(Run::TimedOut))); + let beats = || std::fs::metadata(&heartbeat).map_or(0, |m| m.len()); + // A killed process may take a moment to go. + std::thread::sleep(Duration::from_millis(300)); + let after_kill = beats(); + assert!(after_kill > 0, "the grandchild started and was beating before the timeout"); + std::thread::sleep(Duration::from_secs(1)); + assert_eq!(beats(), after_kill, "the grandchild was stopped with its parent"); + } + + /// The stand-in for `soffice`: starts [`beats_until_stopped`] and waits (run only by the + /// test above). + #[test] + #[ignore] + fn starts_a_child_that_keeps_beating() { + let Some(heartbeat) = std::env::var_os(HEARTBEAT) else { return }; + let mut child = stand_in("office::tests::native::beats_until_stopped", Path::new(&heartbeat)).spawn().unwrap(); + std::thread::sleep(Duration::from_secs(60)); + let _ = child.kill(); + let _ = child.wait(); + } + + /// The stand-in for `soffice.bin`: appends to the heartbeat file every 50 ms for a minute. + #[test] + #[ignore] + fn beats_until_stopped() { + use std::io::Write; + let Some(heartbeat) = std::env::var_os(HEARTBEAT) else { return }; + let mut file = std::fs::OpenOptions::new().create(true).append(true).open(heartbeat).unwrap(); + for _ in 0..1200 { + file.write_all(b".").unwrap(); + std::thread::sleep(Duration::from_millis(50)); + } + } + + /// Where the stand-in below writes. + const FILL_DIR: &str = "PDFCRAFT_TEST_OFFICE_FILL_DIR"; + + /// A converter that writes without end is stopped as soon as its output passes the + /// limit, not after it has run out its time. + #[test] + fn a_converter_whose_output_grows_past_the_limit_is_stopped() { + let work = WorkDir::new().unwrap(); + let mut command = std::process::Command::new(std::env::current_exe().unwrap()); + command.args(["--ignored", "--exact", "office::tests::native::writes_without_end", "--nocapture"]).env(FILL_DIR, &work.0); + command.stdin(std::process::Stdio::null()).stdout(std::process::Stdio::null()).stderr(std::process::Stdio::null()); + let limit = 256 * 1024; + let started = std::time::Instant::now(); + let run = run_limited(command, Duration::from_secs(60), Some((&work.0, limit))); + assert!(matches!(run, Ok(Run::TooLarge)), "stopped for size, not for time"); + assert!(started.elapsed() < Duration::from_secs(30), "stopped long before the timeout"); + // A killed process may take a moment to go; after that nothing is written. + std::thread::sleep(Duration::from_millis(300)); + let after_kill = folder_size(&work.0); + assert!(after_kill > limit, "the stand-in had written past the limit"); + std::thread::sleep(Duration::from_secs(1)); + assert_eq!(folder_size(&work.0), after_kill, "the converter was stopped"); + } + + /// The stand-in for a LibreOffice that fills the disk: 64 KiB every 10 ms for a minute + /// (run only by the test above). + #[test] + #[ignore] + fn writes_without_end() { + use std::io::Write; + let Some(dir) = std::env::var_os(FILL_DIR) else { return }; + let mut file = std::fs::File::create(Path::new(&dir).join("document.pdf")).unwrap(); + let chunk = vec![0u8; 64 * 1024]; + for _ in 0..6000 { + file.write_all(&chunk).unwrap(); + std::thread::sleep(Duration::from_millis(10)); + } + } + + #[test] + fn a_folder_that_cannot_be_read_counts_as_empty() { + assert_eq!(folder_size(&std::env::temp_dir().join("pdfcraft-no-such-folder")), 0); + } + + /// The temporary folder holds a confidential document and its PDF: on Unix only the owner + /// may enter it. + #[cfg(unix)] + #[test] + fn the_temporary_folder_is_private_to_its_owner() { + use std::os::unix::fs::PermissionsExt; + let work = WorkDir::new().unwrap(); + let mode = std::fs::metadata(&work.0).unwrap().permissions().mode(); + assert_eq!(mode & 0o777, 0o700, "{mode:o}"); + } + + #[test] + fn temporary_folders_are_removed_with_their_contents() { + let work = WorkDir::new().unwrap(); + let other = WorkDir::new().unwrap(); + assert_ne!(work.0, other.0); + let dir = work.0.clone(); + std::fs::create_dir_all(dir.join("profile").join("user")).unwrap(); + std::fs::write(dir.join("document.docx"), DOCX_HEAD).unwrap(); + drop(work); + assert!(!dir.exists()); + } + + /// A real conversion, when LibreOffice is installed (skipped otherwise). + #[test] + fn libreoffice_converts_a_document_when_installed() { + let Some(program) = super::super::libreoffice() else { + eprintln!("skipped: LibreOffice isn't installed"); + return; + }; + let rtf = b"{\\rtf1\\ansi{\\fonttbl\\f0\\fswiss Helvetica;}\\f0\\pard Hello from PdfCraft.\\par}"; + let pdf = convert_with(&program, "hello.rtf", rtf, super::super::TIMEOUT).unwrap(); + assert!(pdf.starts_with(b"%PDF-")); + } + } +} diff --git a/crates/create/src/tests.rs b/crates/create/src/tests.rs index d2a8c143e7..f8b870832b 100644 --- a/crates/create/src/tests.rs +++ b/crates/create/src/tests.rs @@ -559,7 +559,7 @@ fn source_kind_tells_pdfs_images_and_text_apart() { // Text that merely starts like a BMP stays text; a truncated BMP header is not an image. assert_eq!(source_kind("b.txt", b"BMW drivers"), Some(SourceKind::Text)); assert_eq!(source_kind("b.bmp", b"BM"), None); - assert_eq!(source_kind("a.docx", b"PK\x03\x04"), None); + assert_eq!(source_kind("a.docx", b"PK\x03\x04"), Some(SourceKind::Office), "Office documents convert through LibreOffice"); assert_eq!(source_kind("", b""), None); // Only the GIF versions Create decodes count as images, so detection never promises a // conversion that then fails. @@ -2952,3 +2952,19 @@ fn stripping_extended_xmp_leaves_the_rest_of_the_jpeg() { // The helper puts a stray byte before the chunks, which the decoder skips: it stays. assert_eq!(crate::upright_image::without_extended_xmp(&flooded), [&plain[..2], b"\0".as_slice(), &plain[2..]].concat()); } + +#[test] +fn convertible_is_the_image_and_text_extensions() { + let both: Vec<&str> = IMAGE_EXTENSIONS.iter().chain(TEXT_EXTENSIONS.iter()).copied().collect(); + assert_eq!(both, CONVERTIBLE.to_vec()); + assert!(OFFICE_EXTENSIONS.iter().all(|e| !CONVERTIBLE.contains(e) && *e != "pdf")); +} + +#[test] +fn office_documents_are_their_own_kind() { + assert_eq!(source_kind("Report.docx", b"PK\x03\x04\x14\x00"), Some(SourceKind::Office)); + assert_eq!(source_kind("Budget.xls", &[0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0x1A, 0xE1]), Some(SourceKind::Office)); + // A PDF saved with a Word name is still a PDF; a ZIP that isn't named like Office is nothing. + assert_eq!(source_kind("Report.docx", b"%PDF-1.7\n"), Some(SourceKind::Pdf)); + assert_eq!(source_kind("photos.zip", b"PK\x03\x04\x14\x00"), None); +} diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 161b2c1d34..62ea42cd54 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -92,7 +92,9 @@ pub use pdfcraft_organize::check_custom_key; pub use pdfcraft_organize::view::{InitialView, Layout as InitialLayout, Magnification, Navigation}; pub use pdfcraft_cos::{Algorithm, civil_from_days}; -pub use pdfcraft_create::{CONVERTIBLE, CreateError, ImageResolution, SourceKind, decode_text, source_kind}; +pub use pdfcraft_create::{ + CONVERTIBLE, CreateError, IMAGE_EXTENSIONS, ImageResolution, OFFICE_EXTENSIONS, SourceKind, TEXT_EXTENSIONS, decode_text, source_kind, +}; pub use pdfcraft_edit::added::{ CHAR_SPACING_LIMIT as ADDED_CHAR_SPACING_LIMIT, paste_place as added_paste_place, paste_rect as added_paste_rect, text_rect as added_text_rect, text_width as added_text_width, @@ -166,7 +168,7 @@ use pictures::{embed_image, fit_picture, mark_source, model_poster}; use protection::unrestricted; pub use protection::{Changes, Printing, Protection, SecuritySummary}; pub use session::Session; -pub use sources::{CombineSource, Combined, combine_source_check, source_page_count}; +pub use sources::{CombineSource, Combined, combine_source_check, office_to_pdf, source_page_count}; use sources::{open_source, open_source_with}; #[cfg(test)] use test_hooks::{ diff --git a/crates/engine/src/session.rs b/crates/engine/src/session.rs index bd83242682..187cc217d8 100644 --- a/crates/engine/src/session.rs +++ b/crates/engine/src/session.rs @@ -13,7 +13,7 @@ use crate::{ BookmarkPage, BookmarkRow, CombineSource, Combined, DataFormat, DocId, Document, Edit, EditError, Hairlines, HairlinesFixed, Hidden, ImageResolution, LOW_RESOLUTION_PRINT_DPI, Label, ObjectKind, OptimizeReport, RecoverySnapshot, SignOptions, SourceKind, SplitPart, StatusCaptions, SummarySort, TrustStore, XFA_REMOVED_PREFIX, XFA_SCRIPTS_OFF, apply_layer_state, carry_verdict_marks, comment_list, - comment_summary, decode_text, display_bytes, display_config, guard, info_fields, js, local_utc_offset, note_warnings, open_source, + comment_summary, decode_text, display_bytes, display_config, guard, info_fields, js, local_utc_offset, note_warnings, office_to_pdf, open_source, open_source_with, optimize, outline_rows, parse_ymd, print, print_layer_states, print_raster, render_threads, scan_colours, set_default_layer_state, shown_under_redactions, signatures_of, source_kind, sync_xfa_state, undrawn_notices, use_layer_choices, writable, xfa, xfa_sync_datasets, diff --git a/crates/engine/src/session/files.rs b/crates/engine/src/session/files.rs index cbdf178e9d..2f2774ed22 100644 --- a/crates/engine/src/session/files.rs +++ b/crates/engine/src/session/files.rs @@ -1,9 +1,17 @@ use super::{ Arc, CombineSource, Combined, DataFormat, DocId, EditError, Hairlines, HairlinesFixed, Hidden, ImageResolution, LOW_RESOLUTION_PRINT_DPI, - OptimizeReport, SaveOptions, Session, SourceKind, SplitPart, decode_text, guard, open_source, open_source_with, optimize, print, + OptimizeReport, SaveOptions, Session, SourceKind, SplitPart, decode_text, guard, office_to_pdf, open_source, open_source_with, optimize, print, print_layer_states, print_raster, set_default_layer_state, source_kind, writable, write_full, }; +/// Whether a conversion may run LibreOffice for an Office document (seconds, in another +/// process) or must refuse it because the caller is on the frame thread. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Office { + Convert, + Refuse, +} + impl Session { /// The page count of another PDF (Replace Pages, Insert Pages dialogs). pub fn page_count_of(&self, name: &str, bytes: &Arc>) -> Result { @@ -41,30 +49,63 @@ impl Session { self.write_new(&doc) } - /// Convert a file Create understands (an image or plain text) to PDF bytes; a PDF is checked - /// (it must open and allow copying pages) and returned as it is. + /// Convert a file Create understands (an image, plain text or an Office document) to PDF + /// bytes; a PDF is checked (it must open and allow copying pages) and returned as it is. + /// An Office document runs LibreOffice ([`office_to_pdf`]), which takes seconds: keep it off + /// the frame thread ([`Self::convert_to_pdf_inline`] there). pub fn convert_to_pdf(&self, name: &str, bytes: &Arc>) -> Result<(SourceKind, Arc>), EditError> { + self.convert(name, bytes, Office::Convert) + } + + /// [`Self::convert_to_pdf`] for a caller on the frame thread, which must not wait seconds for + /// LibreOffice: an Office document is refused, pointing to File ▸ Open, which converts it in + /// the background. + pub fn convert_to_pdf_inline(&self, name: &str, bytes: &Arc>) -> Result<(SourceKind, Arc>), EditError> { + self.convert(name, bytes, Office::Refuse) + } + + fn convert(&self, name: &str, bytes: &Arc>, office: Office) -> Result<(SourceKind, Arc>), EditError> { let Some(kind) = source_kind(name, bytes) else { - return Err(EditError::Source(format!("{name}: this file type can't be converted; use a PDF, an image or a .txt file"))); + return Err(EditError::Source(format!( + "{name}: this file type can't be converted; use a PDF, an image, a .txt file or a Word, Excel or PowerPoint file" + ))); }; + if kind == SourceKind::Office && office == Office::Refuse { + return Err(EditError::Source(format!( + "{name}: Word, Excel and PowerPoint files can't be added here yet; open it with File ▸ Open, which converts it to a PDF" + ))); + } let title = name.rsplit_once('.').map_or(name, |(s, _)| s); // Image decoders read untrusted bytes: a panic in one must not take the app down. let created = guard(|| match kind { SourceKind::Pdf => open_source(name, bytes).map(|_| bytes.clone()), SourceKind::Image => self.create_from_images(&[(name.to_string(), bytes.to_vec())]), SourceKind::Text => self.create_from_text(title, &decode_text(bytes)), + SourceKind::Office => office_to_pdf(name, bytes), }) .map_err(|_| EditError::Source(format!("{name}: the file could not be read")))?; Ok((kind, created?)) } - /// A file for Insert Pages into `dest`, as PDF bytes and its page count: an image or text - /// file is converted, and a PDF is checked as the insert itself checks it, so a file under - /// `dest`'s own protection that withholds only page extraction is accepted, as pasted pages - /// are (#5081). + /// A file for Insert Pages into `dest`, as PDF bytes and its page count: an image, text or + /// Office file is converted, and a PDF is checked as the insert itself checks it, so a file + /// under `dest`'s own protection that withholds only page extraction is accepted, as pasted + /// pages are (#5081). An Office document runs LibreOffice: keep it off the frame thread + /// ([`Self::insert_source_inline`] there). pub fn insert_source(&self, dest: DocId, name: &str, bytes: &Arc>) -> Result<(usize, Arc>), EditError> { + self.insert_source_with(dest, name, bytes, Office::Convert) + } + + /// [`Self::insert_source`] on the frame thread: Office documents are refused, as + /// [`Self::convert_to_pdf_inline`] refuses them. PDFs are still checked as the insert itself + /// checks them (#5081). + pub fn insert_source_inline(&self, dest: DocId, name: &str, bytes: &Arc>) -> Result<(usize, Arc>), EditError> { + self.insert_source_with(dest, name, bytes, Office::Refuse) + } + + fn insert_source_with(&self, dest: DocId, name: &str, bytes: &Arc>, office: Office) -> Result<(usize, Arc>), EditError> { if !matches!(source_kind(name, bytes), Some(SourceKind::Pdf)) { - let (_, pdf) = self.convert_to_pdf(name, bytes)?; + let (_, pdf) = self.convert(name, bytes, office)?; return Ok((self.page_count_of(name, &pdf)?, pdf)); } let src = crate::sources::open_source_into(name, bytes, self.cos(dest)?)?; @@ -78,16 +119,26 @@ impl Session { Ok(pdfcraft_organize::page_count(&crate::sources::open_source_into(name, bytes, self.cos(dest)?)?)?) } - /// A file for Combine as PDF bytes: an image or text file is converted; a PDF is returned as - /// it is, unchecked (Combine reads it itself, with its password), as is a file named .pdf - /// without a `%PDF-` header, which the reader may still open. + /// A file for Combine as PDF bytes: an image, text or Office file is converted; a PDF is + /// returned as it is, unchecked (Combine reads it itself, with its password), as is a file + /// named .pdf without a `%PDF-` header, which the reader may still open. pub fn as_pdf(&self, name: &str, bytes: &Arc>) -> Result>, EditError> { + self.as_pdf_with(name, bytes, Office::Convert) + } + + /// [`Self::as_pdf`] on the frame thread: Office documents are refused, as + /// [`Self::convert_to_pdf_inline`] refuses them. + pub fn as_pdf_inline(&self, name: &str, bytes: &Arc>) -> Result>, EditError> { + self.as_pdf_with(name, bytes, Office::Refuse) + } + + fn as_pdf_with(&self, name: &str, bytes: &Arc>, office: Office) -> Result>, EditError> { let pdf = match source_kind(name, bytes) { Some(SourceKind::Pdf) => true, - Some(SourceKind::Image | SourceKind::Text) => false, + Some(SourceKind::Image | SourceKind::Text | SourceKind::Office) => false, None => name.to_ascii_lowercase().ends_with(".pdf"), }; - if pdf { Ok(bytes.clone()) } else { self.convert_to_pdf(name, bytes).map(|(_, pdf)| pdf) } + if pdf { Ok(bytes.clone()) } else { self.convert(name, bytes, office).map(|(_, pdf)| pdf) } } /// Reduce File Size: Acrobat's defaults (colour and grey images above 225 ppi to 150 ppi, diff --git a/crates/engine/src/sources.rs b/crates/engine/src/sources.rs index e7646e223c..4d1b5ab133 100644 --- a/crates/engine/src/sources.rs +++ b/crates/engine/src/sources.rs @@ -15,6 +15,16 @@ pub struct Combined { pub renamed_fields: Vec<(String, String)>, } +/// Convert the Office document `bytes` named `name` (Word, Excel, PowerPoint, OpenDocument, +/// RTF) to PDF with an installed LibreOffice, and check the result opens. It runs a separate +/// process for seconds, so callers keep it off the frame thread. Desktop only: on the web it +/// returns an error. +pub fn office_to_pdf(name: &str, bytes: &Arc>) -> Result>, EditError> { + let pdf = Arc::new(pdfcraft_create::office_to_pdf(name, bytes)?); + open_source(name, &pdf)?; + Ok(pdf) +} + /// Parse another PDF to copy pages from. pub(crate) fn open_source(name: &str, bytes: &Arc>) -> Result { open_source_with(name, bytes, None) diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 7b82a9b8e7..4e24a4f608 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -1575,8 +1575,13 @@ fn a_page_with_a_form_field_shown_twice_keeps_one_field() { #[test] fn files_that_cannot_be_converted_are_refused_clearly() { let s = Session::new(); - let err = s.convert_to_pdf("report.docx", &Arc::new(b"PK\x03\x04".to_vec())).unwrap_err(); - assert_eq!(err, EditError::Source("report.docx: this file type can't be converted; use a PDF, an image or a .txt file".into())); + let err = s.convert_to_pdf("archive.zip", &Arc::new(b"PK\x03\x04".to_vec())).unwrap_err(); + assert_eq!( + err, + EditError::Source( + "archive.zip: this file type can't be converted; use a PDF, an image, a .txt file or a Word, Excel or PowerPoint file".into() + ) + ); assert!(matches!(s.convert_to_pdf("empty", &Arc::new(Vec::new())), Err(EditError::Source(_)))); // Damaged inputs of a known type fail with an error, not a panic. assert!(s.convert_to_pdf("bad.png", &Arc::new(b"\x89PNG\r\n\x1a\nnope".to_vec())).is_err()); @@ -1597,8 +1602,8 @@ fn files_for_combine_are_made_into_pdfs_and_pdfs_pass_through() { assert!(Arc::ptr_eq(&s.as_pdf("old.PDF", &headless).unwrap(), &headless)); assert!(s.as_pdf("scan.jpg", &Arc::new(jpeg_bytes())).unwrap().starts_with(b"%PDF-")); assert!(s.as_pdf("notes.txt", &Arc::new(b"hello".to_vec())).unwrap().starts_with(b"%PDF-")); - let err = s.as_pdf("report.docx", &Arc::new(b"PK\x03\x04".to_vec())).unwrap_err(); - assert!(matches!(err, EditError::Source(m) if m.starts_with("report.docx: this file type can't be converted"))); + let err = s.as_pdf("archive.zip", &Arc::new(b"PK\x03\x04".to_vec())).unwrap_err(); + assert!(matches!(err, EditError::Source(m) if m.starts_with("archive.zip: this file type can't be converted"))); } fn protected(user: &str, owner: &str, permissions: i32) -> Arc> { @@ -15359,3 +15364,48 @@ fn calculate_now_simple_calculation_reads_formatted_values_and_groups_as_the_nat assert!(out.console.iter().any(|line| line == "3244.75"), "calculateNow agrees with the native path: {out:?}"); assert_eq!(total(&s), ["3244.75"]); } + +/// A Word document converts through LibreOffice when it is installed; without it, the error +/// says what is missing rather than calling the file a broken PDF. +#[test] +fn office_documents_convert_through_libreoffice_or_explain_why_not() { + let mut s = Session::new(); + let text = s.create_from_text("t", "Quarterly report").unwrap(); + let id = s.open("t.pdf", None, text, None).unwrap(); + let docx = Arc::new(s.get(id).unwrap().export_office(compare::OfficeFormat::Docx)); + assert_eq!(source_kind("Report.docx", &docx), Some(SourceKind::Office)); + match s.convert_to_pdf("Report.docx", &docx) { + Ok((kind, pdf)) => { + assert_eq!(kind, SourceKind::Office); + let converted = s.open_new("Report.pdf", pdf).unwrap(); + assert!(page_texts(&s, converted).concat().contains("Quarterly report")); + } + // Only "not installed" is an acceptable failure: a conversion that fails, times out or + // makes no PDF must fail this test. + Err(e) => assert!(e.to_string().contains("LibreOffice, which isn't installed"), "{e}"), + } + // Combine treats it as a file to convert, not as a PDF to pass through. + assert!( + s.as_pdf("Report.docx", &docx).map_or_else(|e| e.to_string().contains("LibreOffice, which isn't installed"), |pdf| pdf.starts_with(b"%PDF-")) + ); +} + +/// The frame thread never waits for LibreOffice: Insert, Combine and Create Multiple refuse an +/// Office document and point to File ▸ Open, while images and text still convert there. +#[test] +fn inline_conversions_refuse_office_documents_without_running_libreoffice() { + let s = Session::new(); + let docx = Arc::new(b"PK\x03\x04\x14\x00".to_vec()); + for err in + [s.convert_to_pdf_inline("Report.docx", &docx).map(|_| ()).unwrap_err(), s.as_pdf_inline("Report.docx", &docx).map(|_| ()).unwrap_err()] + { + assert_eq!( + err, + EditError::Source( + "Report.docx: Word, Excel and PowerPoint files can't be added here yet; open it with File ▸ Open, which converts it to a PDF".into() + ) + ); + } + assert_eq!(s.convert_to_pdf_inline("notes.txt", &Arc::new(b"hello".to_vec())).unwrap().0, SourceKind::Text); + assert!(s.as_pdf_inline("scan.jpg", &Arc::new(jpeg_bytes())).unwrap().starts_with(b"%PDF-")); +} diff --git a/crates/ui-egui/src/app/files.rs b/crates/ui-egui/src/app/files.rs index 8c96de4529..81c65a2b03 100644 --- a/crates/ui-egui/src/app/files.rs +++ b/crates/ui-egui/src/app/files.rs @@ -257,6 +257,14 @@ impl PdfCraftApp { Some(Err(error)) => self.notify_fmt("Couldn't open {name}: {e}", &[("name", &pending.name), ("e", &error)]), None => {} }, + open_worker::ResultKind::Converted(created) => match self.open_converted_pdf(&pending.name, created) { + Ok(()) => { + if let Some(command) = pending.then { + self.execute(command); + } + } + Err(error) => self.notify_fmt("Couldn't open {name}: {e}", &[("name", &pending.name), ("e", &error)]), + }, open_worker::ResultKind::Prepared { bytes, result } => match *result { Ok(prepared) => match self.finish_open(&pending.name, pending.path.clone(), bytes.len(), prepared, pending.activate) { Ok(id) => { @@ -1009,24 +1017,29 @@ impl PdfCraftApp { self.open_path_request(path, None, Some((layout, continuous))); } - /// The File ▸ Open picker: PDFs, and images and text that become PDFs. - #[cfg(not(target_arch = "wasm32"))] + /// The File ▸ Open picker's file types, in order; the first, PDF, is the default. Images, text + /// and (on the desktop, through LibreOffice) Office documents open as new, unsaved PDFs. + /// No "All files" in the browser or on macOS, whose panel merges every type into one list + /// rather than offering a choice. + pub(crate) fn open_filters() -> Vec<(String, Vec<&'static str>)> { + let mut filters = vec![("PDF".to_string(), vec!["pdf"]), (tl!("Images").to_string(), pdfcraft_engine::IMAGE_EXTENSIONS.to_vec())]; + #[cfg(not(target_arch = "wasm32"))] + filters.push((tl!("Office documents").to_string(), pdfcraft_engine::OFFICE_EXTENSIONS.to_vec())); + filters.push((tl!("Text").to_string(), pdfcraft_engine::TEXT_EXTENSIONS.to_vec())); + #[cfg(not(any(target_arch = "wasm32", target_os = "macos")))] + filters.push((tl!("All files").to_string(), vec!["*"])); + filters + } + + /// The File ▸ Open picker ([`Self::open_filters`]), each type matched in any letter case. pub(super) fn open_picker() -> rfd::AsyncFileDialog { - rfd::AsyncFileDialog::new() - .add_filter_any_case("PDF", &["pdf"]) - .add_filter_any_case(tl!("Images and text (converted to PDF)"), &create_ui::CONVERTIBLE) + Self::open_filters().into_iter().fold(rfd::AsyncFileDialog::new(), |dialog, (name, extensions)| dialog.add_filter_any_case(name, &extensions)) } pub fn open_dialog(&mut self) { self.startup_superseded = true; #[cfg(not(target_arch = "wasm32"))] - self.pick( - pickers::PickFor::Open, - rfd::AsyncFileDialog::new() - .add_filter_any_case("PDF", &["pdf"]) - .add_filter_any_case(tl!("Images and text (converted to PDF)"), &create_ui::CONVERTIBLE), - true, - ); + self.pick(pickers::PickFor::Open, Self::open_picker(), true); // Browsers pick files asynchronously; the bytes arrive through `inbox`. #[cfg(target_arch = "wasm32")] { @@ -1035,12 +1048,7 @@ impl PdfCraftApp { let read = super::input::WebRead::start(&self.web_reads); wasm_bindgen_futures::spawn_local(async move { let _read = read; - if let Some(h) = rfd::AsyncFileDialog::new() - .add_filter_any_case("PDF", &["pdf"]) - .add_filter_any_case(tl!("Images and text"), &create_ui::CONVERTIBLE) - .pick_file() - .await - { + if let Some(h) = Self::open_picker().pick_file().await { let bytes = h.read().await; if let Ok(mut q) = inbox.lock() { q.push((h.file_name(), bytes)); diff --git a/crates/ui-egui/src/platform/open_worker.rs b/crates/ui-egui/src/platform/open_worker.rs index 8f7b0c5d3f..b9c3735e4a 100644 --- a/crates/ui-egui/src/platform/open_worker.rs +++ b/crates/ui-egui/src/platform/open_worker.rs @@ -20,7 +20,12 @@ pub(crate) enum Input { pub(crate) enum ResultKind { Conversion(Arc>), - Prepared { bytes: Arc>, result: Box> }, + /// An Office document LibreOffice converted here, off the frame thread, or why it couldn't. + Converted(Result>, String>), + Prepared { + bytes: Arc>, + result: Box>, + }, ReadFailed(String), RemoteFailed(String), RecoveryFailed(String), @@ -213,8 +218,11 @@ fn prepare(job: Job) -> ResultKind { }, Input::Bytes(bytes) => bytes.clone(), }; - if matches!(pdfcraft_engine::source_kind(&job.name, &bytes), Some(SourceKind::Image | SourceKind::Text)) { - return ResultKind::Conversion(bytes); + match pdfcraft_engine::source_kind(&job.name, &bytes) { + Some(SourceKind::Image | SourceKind::Text) => return ResultKind::Conversion(bytes), + // LibreOffice takes seconds: it runs here, and the frame thread only opens the result. + Some(SourceKind::Office) => return ResultKind::Converted(pdfcraft_engine::office_to_pdf(&job.name, &bytes).map_err(|e| e.to_string())), + Some(SourceKind::Pdf) | None => {} } let result = job.context.prepare_open(job.name, job.path, bytes.clone(), job.password.as_deref().map(String::as_str)); ResultKind::Prepared { bytes, result: Box::new(result) } diff --git a/crates/ui-egui/src/platform/open_worker/tests.rs b/crates/ui-egui/src/platform/open_worker/tests.rs index 62520f3edf..7f8d73fa8d 100644 --- a/crates/ui-egui/src/platform/open_worker/tests.rs +++ b/crates/ui-egui/src/platform/open_worker/tests.rs @@ -130,10 +130,15 @@ impl HeldToolOpen { self.started.recv_timeout(Duration::from_secs(5)).expect("the native open reached its preparation barrier"); } - #[allow(clippy::expect_used)] fn complete(&self, harness: &mut Harness<'static, PdfCraftApp>) { + self.complete_within(harness, Duration::from_secs(5)); + } + + /// [`Self::complete`] for a preparation that may take up to `timeout`. + #[allow(clippy::expect_used)] + fn complete_within(&self, harness: &mut Harness<'static, PdfCraftApp>, timeout: Duration) { self.release.send(()).expect("release the native preparation"); - self.finished.recv_timeout(Duration::from_secs(5)).expect("the native open returned its result"); + self.finished.recv_timeout(timeout).expect("the native open returned its result"); harness.run_steps(2); } } @@ -997,3 +1002,55 @@ fn a_password_prompt_keeps_a_place_in_the_open_queue_for_its_retry() { assert!(app.password_prompt.is_none(), "the retry is queued: {:?}", app.password_prompt.as_ref().and_then(|p| p.error.clone())); assert!(app.password_retry.is_some()); } + +/// A Word document built from the synthetic fixture (PdfCraft's own Word export). +#[allow(clippy::expect_used)] +fn synthetic_docx() -> Vec { + let mut session = pdfcraft_engine::Session::new(); + let id = session.open("source.pdf", None, Arc::new(fixture(1)), None).expect("open the fixture"); + session.get(id).expect("the fixture").export_office(pdfcraft_engine::compare::OfficeFormat::Docx) +} + +/// File ▸ Open of a Word document: LibreOffice runs on the open worker, never on the frame +/// thread, and the result opens as a new, unsaved PDF. Without LibreOffice the user is +/// told it is needed, rather than that the file is a broken PDF. +#[test] +#[allow(clippy::expect_used)] +fn file_open_converts_an_office_document_on_the_worker() { + let mut harness = tool_open_harness(); + let held = HeldToolOpen::install(&mut harness); + let file = tool_open_file("office", "docx", &synthetic_docx()); + harness.state_mut().pick_override = Some(vec![file.0.to_string_lossy().into_owned()]); + harness.state_mut().run_command("file.open"); + harness.run_steps(2); + held.wait_started(); + assert!(harness.state().views.is_empty(), "nothing opens before the conversion finishes"); + // An installed LibreOffice takes seconds; it is stopped after 120 s. + held.complete_within(&mut harness, Duration::from_secs(150)); + let app = harness.state(); + assert!(app.pending_opens.is_empty()); + if let Some(view) = app.views.first() { + let doc = app.session.get(view.id).expect("the converted document"); + // Named after the Word file: `.docx` opens as `.pdf`. + assert_eq!(doc.name, format!("pdfcraft-open-tool-{}-office.pdf", std::process::id())); + assert!(doc.path.is_none(), "the converted PDF is new and unsaved, not the .docx on disk"); + } else { + let message = app.toast.as_ref().map(|(message, _)| message.clone()).unwrap_or_default(); + assert!(message.contains("LibreOffice, which isn't installed"), "the failure names the missing dependency: {message}"); + } +} + +#[test] +fn the_open_picker_defaults_to_pdf_and_offers_images_office_text_and_all_files() { + let filters = PdfCraftApp::open_filters(); + let names: Vec<&str> = filters.iter().map(|(name, _)| name.as_str()).collect(); + #[cfg(not(target_os = "macos"))] + assert_eq!(names, ["PDF", "Images", "Office documents", "Text", "All files"]); + #[cfg(target_os = "macos")] + assert_eq!(names, ["PDF", "Images", "Office documents", "Text"]); + let extensions = |name: &str| filters.iter().find(|(n, _)| n == name).map(|(_, e)| e.clone()).unwrap_or_default(); + assert_eq!(extensions("PDF"), ["pdf"]); + assert_eq!(extensions("Images"), pdfcraft_engine::IMAGE_EXTENSIONS); + assert!(["doc", "docx", "xls", "xlsx", "ppt", "pptx", "odt", "rtf"].iter().all(|e| extensions("Office documents").contains(e))); + assert_eq!(extensions("Text"), pdfcraft_engine::TEXT_EXTENSIONS); +} diff --git a/crates/ui-egui/src/workflows/combine_ui/parts.rs b/crates/ui-egui/src/workflows/combine_ui/parts.rs index 4527ec061d..cc05552ba1 100644 --- a/crates/ui-egui/src/workflows/combine_ui/parts.rs +++ b/crates/ui-egui/src/workflows/combine_ui/parts.rs @@ -39,9 +39,15 @@ fn combinable(ext: &str) -> bool { } /// Whether Combine takes a file by what it is: a PDF (by its header or its name) or an image or -/// text file it converts, as `Session::as_pdf` tells them apart. +/// text file it converts, as `Session::as_pdf_inline` tells them apart. Not an Office document: +/// converting one runs LibreOffice for seconds, which Combine can't do on the frame thread, so +/// it is left out and named rather than listed as a file that can't be read. pub(super) fn takes(name: &str, bytes: &[u8]) -> bool { - pdfcraft_engine::source_kind(name, bytes).is_some() || name.to_ascii_lowercase().ends_with(".pdf") + match pdfcraft_engine::source_kind(name, bytes) { + Some(pdfcraft_engine::SourceKind::Office) => false, + Some(_) => true, + None => name.to_ascii_lowercase().ends_with(".pdf"), + } } /// A file's bookmark in the combined PDF: its name without ".pdf", or without the extension of diff --git a/crates/ui-egui/src/workflows/combine_ui/session.rs b/crates/ui-egui/src/workflows/combine_ui/session.rs index 2297e70933..124260031c 100644 --- a/crates/ui-egui/src/workflows/combine_ui/session.rs +++ b/crates/ui-egui/src/workflows/combine_ui/session.rs @@ -279,7 +279,7 @@ impl PdfCraftApp { notes: Vec::new(), }; let source = bytes.clone(); - let (bytes, assessed) = match self.session.as_pdf(&name, &bytes) { + let (bytes, assessed) = match self.session.as_pdf_inline(&name, &bytes) { Ok(pdf) => { let assessed = assess(&pdf, None); (pdf, assessed) diff --git a/crates/ui-egui/src/workflows/create_multiple_ui.rs b/crates/ui-egui/src/workflows/create_multiple_ui.rs index 54d2181d0a..03fd3e1eaa 100644 --- a/crates/ui-egui/src/workflows/create_multiple_ui.rs +++ b/crates/ui-egui/src/workflows/create_multiple_ui.rs @@ -43,7 +43,7 @@ impl PdfCraftApp { continue; } }; - let (kind, pdf) = match self.session.convert_to_pdf(&name, &Arc::new(bytes)) { + let (kind, pdf) = match self.session.convert_to_pdf_inline(&name, &Arc::new(bytes)) { Ok(converted) => converted, Err(error) => { refused.push(format!("{name} ({error})")); @@ -111,7 +111,7 @@ impl PdfCraftApp { let sources = std::iter::from_fn(|| { loop { match files.next()? { - Ok((name, bytes)) => match session.convert_to_pdf(&name, &Arc::new(bytes)) { + Ok((name, bytes)) => match session.convert_to_pdf_inline(&name, &Arc::new(bytes)) { Ok((_, pdf)) => { converted += 1; return Some(Ok((stem(&name).to_string(), pdf, None))); diff --git a/crates/ui-egui/src/workflows/create_ui.rs b/crates/ui-egui/src/workflows/create_ui.rs index f74ed10535..5d3d6f4cc6 100644 --- a/crates/ui-egui/src/workflows/create_ui.rs +++ b/crates/ui-egui/src/workflows/create_ui.rs @@ -43,9 +43,6 @@ pub(crate) fn image_import_body(ui: &mut egui::Ui, app: &mut PdfCraftApp) -> (bo (go, cancel) } -/// File types Open accepts besides PDF (converted on open). -pub use pdfcraft_engine::CONVERTIBLE; - /// What Create ▸ Clipboard found on the clipboard. #[derive(Clone, Debug, PartialEq)] pub enum Clip { @@ -105,15 +102,23 @@ fn stem(name: &str) -> &str { } impl PdfCraftApp { - /// Convert a non-PDF file (image, text) into a new tab. Returns `None` when `bytes` is not - /// something Create understands (the caller then tries to open it as a PDF). + /// Convert a non-PDF file (image, text, Office document) into a new tab. Returns `None` when + /// `bytes` is not something Create understands (the caller then tries to open it as a PDF). + /// An Office document blocks while LibreOffice runs: File ▸ Open converts it on the open + /// worker instead ([`Self::open_converted_pdf`]), leaving this for startup and inline opens. pub(crate) fn open_converted(&mut self, name: &str, bytes: &[u8]) -> Option> { use pdfcraft_engine::SourceKind; - if !matches!(pdfcraft_engine::source_kind(name, bytes), Some(SourceKind::Image | SourceKind::Text)) { + if !matches!(pdfcraft_engine::source_kind(name, bytes), Some(SourceKind::Image | SourceKind::Text | SourceKind::Office)) { return None; } let created = self.session.convert_to_pdf(name, &Arc::new(bytes.to_vec())).map(|(_, pdf)| pdf); - Some(self.open_created_bytes(&format!("{}.pdf", stem(name)), created.map_err(|e| e.to_string()))) + Some(self.open_converted_pdf(name, created.map_err(|e| e.to_string()))) + } + + /// Open `created`, the PDF converted from the file `name`, as a new, unsaved tab named after + /// it (`Report.docx` → `Report.pdf`). + pub(crate) fn open_converted_pdf(&mut self, name: &str, created: Result>, String>) -> Result<(), String> { + self.open_created_bytes(&format!("{}.pdf", stem(name)), created) } fn open_created_bytes(&mut self, name: &str, created: Result>, String>) -> Result<(), String> { diff --git a/crates/ui-egui/src/workflows/files.rs b/crates/ui-egui/src/workflows/files.rs index 18c1c24e0f..82057ecc55 100644 --- a/crates/ui-egui/src/workflows/files.rs +++ b/crates/ui-egui/src/workflows/files.rs @@ -666,7 +666,7 @@ impl PdfCraftApp { continue; } }; - match self.session.insert_source(id, &name, &Arc::new(bytes)) { + match self.session.insert_source_inline(id, &name, &Arc::new(bytes)) { Ok((pages, bytes)) => { if self.apply_edit(Edit::InsertPagesFrom { name, diff --git a/crates/ui-egui/tests/it/editing.rs b/crates/ui-egui/tests/it/editing.rs index 719f871f48..60c44bbfd5 100644 --- a/crates/ui-egui/tests/it/editing.rs +++ b/crates/ui-egui/tests/it/editing.rs @@ -1839,10 +1839,10 @@ fn images_and_text_files_join_the_combine_list_as_pdfs() { #[test] fn files_that_cannot_be_read_are_listed_and_left_out_once_the_user_agrees() { let mut h = combine_of(&[("one.pdf", 2)]); - let bad = vec![("broken.jpg".to_string(), b"\xff\xd8\xff\xe0 not a JPEG".to_vec()), ("report.docx".to_string(), b"PK\x03\x04".to_vec())]; + let bad = vec![("broken.jpg".to_string(), b"\xff\xd8\xff\xe0 not a JPEG".to_vec()), ("archive.zip".to_string(), b"PK\x03\x04".to_vec())]; h.state_mut().use_files(pdfcraft_ui_egui::FilePurpose::Combine, bad); h.run_steps(3); - assert_eq!(combine_names(&h), ["one.pdf", "broken.jpg", "report.docx"]); + assert_eq!(combine_names(&h), ["one.pdf", "broken.jpg", "archive.zip"]); for f in &h.state().combine_draft[1..] { let why = f.problem.as_deref().expect("why it can't be combined"); assert!(why.starts_with("Can't be read: ") && !why.contains(&f.name), "{why}"); @@ -1859,7 +1859,7 @@ fn files_that_cannot_be_read_are_listed_and_left_out_once_the_user_agrees() { h.run_steps(3); h.get_by_label("Leave out files that can't be read?"); h.get_by_label("• broken.jpg"); - h.get_by_label("• report.docx"); + h.get_by_label("• archive.zip"); h.get_by_label("Cancel").click(); h.run_steps(3); assert!(h.query_by_label("Leave out files that can't be read?").is_none()); @@ -2136,7 +2136,7 @@ fn plus_between_pages_inserts_picked_files_there_in_order() { assert_eq!(texts_of(h.state(), 0), ["a note", "Page 1", "a note", "Page 2", "Page 3", "a note"]); // A file that can't be converted changes nothing. let before = texts_of(h.state(), 0); - h.state_mut().use_files(pdfcraft_ui_egui::FilePurpose::InsertPages, vec![("report.docx".into(), b"PK\x03\x04".to_vec())]); + h.state_mut().use_files(pdfcraft_ui_egui::FilePurpose::InsertPages, vec![("archive.zip".into(), b"PK\x03\x04".to_vec())]); h.run_steps(2); assert_eq!(texts_of(h.state(), 0), before); } diff --git a/docs/file-format-parity.md b/docs/file-format-parity.md index 288b7ca0cf..eb38bc65cf 100644 --- a/docs/file-format-parity.md +++ b/docs/file-format-parity.md @@ -1,6 +1,6 @@ # File format parity -> **Last reviewed:** 2026-10-12 · **Last updated:** 2026-10-19 · **Change:** patch (`.p7b`/`.p7c` bundles read for trust import) · **Target:** Adobe Acrobat Pro (Acrobat DC, continuous track 26.002.21931, macOS) +> **Last reviewed:** 2026-10-12 · **Last updated:** 2026-10-22 · **Change:** minor (Word, Excel, PowerPoint, OpenDocument and RTF open through an installed LibreOffice; `.p7b`/`.p7c` bundles read for trust import) · **Target:** Adobe Acrobat Pro (Acrobat DC, continuous track 26.002.21931, macOS) Every format Acrobat Pro reads or writes, and PdfCraft's support for it. Acrobat's list comes from its bundle's `Info.plist` `CFBundleDocumentTypes` (what it opens) and its public documentation @@ -68,9 +68,9 @@ features total). | Plain text | yes | yes | good | `create.from-text` | | Clipboard (image or text) | yes | yes | good | `create.from-clipboard` | | Screenshot | yes | no | none | `create.from-screenshot` planned | -| Word, Excel, PowerPoint (and Visio, Publisher on Windows) | yes | no | none | `create.from-office` planned (LibreOffice sidecar) | +| Word, Excel, PowerPoint (and Visio, Publisher on Windows) | yes | yes, desktop only, through an installed LibreOffice (File ▸ Open); not Visio or Publisher | good where LibreOffice is (its layout, not measured against Acrobat) | `create.from-office` partial | | HTML file, web page URL | yes | no | none | `create.from-html`, `create.from-web-page` planned | -| RTF | yes | no | none | | +| RTF | yes | yes, through LibreOffice (as Office documents) | good where LibreOffice is | `create.from-office` partial | | PostScript / EPS (Distiller) | yes | no | none | `create.distiller` planned (P3) | | Markdown | no | no | none | `create.from-markdown` planned (P2; not an Acrobat feature) | | Scanner | yes | no | none | `ocr.scanner-acquire` planned | @@ -110,6 +110,7 @@ features total). | Date | Change | Summary | |---|---|---| +| 2026-10-22 | minor | Office documents (Word, Excel, PowerPoint, OpenDocument, RTF) open from File ▸ Open as new PDFs, converted by an installed LibreOffice; `create.from-office` partial. | | 2026-10-19 | patch | Certificates row: `.p7b`/`.p7c` bundles (DER or PEM `PKCS7`/`CMS`) are read for trust import. | | 2026-10-12 | minor | XDP split into data (read and write for XFA forms, already shipped under `form.xfa-data`) and opening `.xdp` as a document (none). Create PDF reads BigTIFF; malformed GIF headers are no longer detected as images. | | 2026-10-11 | minor | #773: inferred tables have no invented borders; rotated or overlapping invoice pages use floating shapes and page sections. Full Office layout fidelity remains partial. | diff --git a/docs/gaps.md b/docs/gaps.md index 8d472eb2d5..1a1c04a9da 100644 --- a/docs/gaps.md +++ b/docs/gaps.md @@ -1,5 +1,6 @@ # Where PdfCraft falls short of Acrobat Pro +> **Last reviewed:** 2026-11-11 | **Last updated:** 2026-11-11 | **Change:** minor (gap 8: known residual risk for the LibreOffice sidecar, sandbox tracked in #6953; estimates unchanged) > **Last reviewed:** 2026-10-28 | **Last updated:** 2026-10-28 | **Change:** minor (M10, #4119 partial: Enhance scanned file cleans scanned pages in place: background removal, edge shadow removal, despeckle; estimates unchanged) > **Last reviewed:** 2026-10-25 · **Last updated:** 2026-10-25 · **Change:** minor (gaps 5 and 14 current state: native release Hans face via LXGW WenKai Lite; Arabic logical catalog, explicit RTL direction and mirrored stock chrome, with native/browser/IME workflows unqualified; no readiness re-estimate) > **Last reviewed:** 2026-10-15 · **Last updated:** 2026-10-15 · **Change:** minor (gap 3: #2517 a blank line stays a paragraph break; estimates unchanged) · **Target:** Adobe Acrobat Pro (Acrobat DC, continuous track 26.002.21931, macOS) @@ -45,7 +46,7 @@ update the numbers. | 5 | Chinese UI lacks a web glyph face | localization | 6–12 | [localization-parity.md](localization-parity.md) | | 6 | Open-issue backlog of wrong results in shipped features | stability | 45–80 | this file | | 7 | OCR reads only unaccented Latin | feature | 40–75 | [file-format-parity.md](file-format-parity.md) | -| 8 | Office export weak, Office import absent | file format | 55–100 | [file-format-parity.md](file-format-parity.md) | +| 8 | Office export weak, Office import basic | file format | 55–100 | [file-format-parity.md](file-format-parity.md) | | 9 | No Preflight; PDF/A partial; PDF/X and PDF/UA absent | standards | 65–115 | [pdf-spec-parity.md](pdf-spec-parity.md) | | 10 | Signatures: no PKCS #11, timestamp servers or online revocation | feature, hardware | 40–70 | [pdf-spec-parity.md](pdf-spec-parity.md) | | 11 | Accessibility: checker only, no tagging tools | feature | 40–70 | [target-app-parity.md](target-app-parity.md) | @@ -189,19 +190,29 @@ update the numbers. - **Estimate:** 40–75 h, partly blocked on openly licensed models. **Doc:** [file-format-parity.md](file-format-parity.md). -### 8. Office export weak, Office import absent +### 8. Office export weak, Office import basic -- **Missing:** Word export that keeps layout; Excel and PowerPoint export; creating PDFs from Office, - HTML, web pages and PostScript; SVG/XML/EPS/PS/JPEG 2000 export. +- **Missing:** Word export that keeps layout; Excel and PowerPoint export; creating PDFs from HTML, + web pages and PostScript; SVG/XML/EPS/PS/JPEG 2000 export. Office documents open from File ▸ Open + through an installed LibreOffice, but Insert pages, Combine files and Create ▸ Multiple files + don't take them yet. - **Evidence:** `create.export-docx` partial; the synthetic invoice regression for [#773](https://github.com/storytold/pdfcraft/issues/773) now covers absent inferred borders, floating watermark text and per-page section sizes. Exact font matching, vector rules and broader layout fidelity remain incomplete; - `create.export-xlsx`, `create.export-pptx`, `create.from-office`, `create.from-html` planned. + `create.from-office` partial (LibreOffice sidecar; Writer and Calc never update linked content, + but Impress has no such setting and the process's network access isn't sandboxed); + `create.export-xlsx`, `create.export-pptx`, + `create.from-html` planned. +- **Known residual risk:** conversion runs the user's installed LibreOffice. PdfCraft + opens no connection itself. Macros are disabled, untrusted referer links are blocked, + and Writer and Calc never update links. Impress has no link-update setting, and the + `soffice` process isn't network-sandboxed, so linked content in presentations could be + fetched. Sandboxing the converter is [#6953](https://github.com/storytold/pdfcraft/issues/6953). - **Impact:** "Export PDF" and "Create PDF" are two of Acrobat's most used tools. - **Gate:** lossy or missing sub-cases of the convert-and-share workflow (partial, not blocking). - First beta items: Create PDF from Office (LibreOffice sidecar, 10–20 h) and a Word export that - keeps layout (20–35 h). + First beta items: the rest of Create PDF from Office (Insert, Combine and Create ▸ Multiple files + on a worker, a Create ▸ From Office command; 5–10 h) and a Word export that keeps layout (20–35 h). - **Estimate:** 55–100 h. **Doc:** [file-format-parity.md](file-format-parity.md). ### 9. No Preflight; PDF/A partial; PDF/X and PDF/UA absent @@ -422,6 +433,8 @@ update the numbers. | Date | Change | Summary | |---|---|---| +| 2026-11-11 | minor | Gap 8: known residual risk for the LibreOffice sidecar (Impress has no link-update setting; the soffice process isn't network-sandboxed). Sandboxing tracked in #6953. Estimates unchanged. | +| 2026-10-26 | minor | Gap 8: the LibreOffice profile never updates Writer or Calc linked content on load; Impress has no such setting and the process's network access isn't sandboxed. Estimates unchanged. | | 2026-10-25 | minor | Gaps 5 and 14 current state: native release builds embed LXGW WenKai Lite for Simplified Chinese, while the web build has no Hans face; Arabic catalogs stay logical with explicit RTL label direction and mirrored stock chrome, while native/browser RTL input, IME and complex workflows remain unqualified. Estimates unchanged. | | 2026-10-18 | minor | Gap 3: #2517 a partial rewrite keeps a kept source row in the paragraph and leaves the next marked region closed around its text. Estimates unchanged. | | 2026-10-15 | minor | Gap 3: #2517 a blank line at ordinary leading stays a paragraph break. Estimates unchanged. | diff --git a/docs/roadmap.md b/docs/roadmap.md index e276a51f44..7ca83dedd9 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -288,7 +288,7 @@ total matches the parity estimate. | M11 | Optimize, preflight, PDF/A/X/UA, print production | 200–350 | 18% | 80–140 | Done: new `optimize` crate: Reduce File Size and the PDF Optimizer (images measured where drawn, bicubic downsampling, JPEG/ZIP recompression only when smaller, discard objects and user data, Flate clean-up, resource merging, object streams). Target-version compatibility is partial: newer content is not detected or converted. Content stream boundaries preserve comments, split operands and graphics state when image fragments merge. Image downsampling includes page-local `/UserUnit` scaling when measuring physical resolution. Done since: reject invalid image and soft-mask dimensions with checked sizing (#2178). Missing: fonts and transparency panels, space audit, preflight, PDF/A/X/UA | | M8 | Security + redaction | 100–180 | 68% | 15–30 | Done: opening protected documents, permissions, Protect Using Password, Remove security; redaction (mark text/areas/pages, Search & Redact with patterns, apply removing glyphs, image pixels, vectors, XObject content, annotations and fields, verification, full rewrite on save); Remove hidden information and Sanitize.
Done since: redaction codes (U.S. FOIA and Privacy Act sets as overlay text, in Redaction properties and `redact_mark`). Redaction cleans indirect and form structure references, preserving surviving form tags. Redaction retires inherited resource dictionaries, so a parent `/Pages` can't keep a removed XObject reachable; unredacted pages are left untouched. Image redaction removes alternate images and unsupported masks only on affected images. Changed marked-content spans lose replacement descriptions; untouched spans keep theirs. Hidden-text cleanup retires unused named descriptions across inherited/shared page and form resources (64 focused tests and scoped Rust 1.96 Clippy/wasm checks pass; publication CI pending). Retained text/image-bearing or unreadable tiling-pattern cells now make Apply refuse atomically after a bounded check of rewritten resources; fully removed cells still apply, and cross-page retention is checked. The redaction audit counts a filled cover that hides half of any one word of a long line (#2662), applied or not. Phone-pattern redaction also matches labelled Italian geographic numbers with `+39` and a slash after the area code (#3205 follow-up); unlabelled international slash numbers remain unsupported. Missing: certificate security, custom redaction code sets and pattern locales, DCT re-encoding | | M9 | Signatures (PAdES, validation) | 160–280 | 60% | 40–70 | Done: new `sign` crate (DER, X.509, CMS, PKCS #12 on RustCrypto; aws-lc-rs for RSA private keys); PAdES B-B signing (visible, invisible, existing fields, certification with DocMDP); validation with trust store and changes-after-signing classification; self-signed digital IDs; Signatures panel, message bar, sign dialogs; agent tools. Checked with pdfsig and OpenSSL.
Done since: macOS Keychain and Windows Current User Personal store signing (software-backed CNG keys), certificate viewer, signed documents protected from rewrites, page validation overlays separate from original signature artwork, placed in the widget's bottom-right corner as it lies on the page, at a quarter of its shorter side and 48 screen points at most, so they annotate the artwork from a corner instead of covering the middle of it and turn with the page and the view (a field falling under 32 screen points, as a page does when zoomed out to fit, keeps its outline), snapshot tabs that reuse instead of multiplying, named-layer signature appearances redrawn for their real state (#79; display, the CLI's render, exports and printing, never saves; captions in the interface language, Chinese in English; the badge stays). Missing: timestamps (B-T), LTV (DSS, OCSP, CRL), FieldMDP, smart cards, PKCS #11 | -| M10 | OCR, create, export, print | 200–350 | 33% | 120–215 | Done: Windows Printer default sends native two-sided bindings only when automatic duplex support is confirmed, preserving manual driver defaults (#5130/#5772); physical output remains unqualified. DocumentDuplex-only bindings use the supported scope through craft-print 9fe50c94. create from blank/text/PNG/JPEG/TIFF (multi-page)/GIF/BMP; export PNG/JPEG/TIFF and text; Print (Acrobat's sizing, n-up, booklet, poster, comments & forms, preview, the CUPS and Windows spoolers, print-ready PDF).
Done since: embedded/72/custom DPI choices for image imports; export all images; OCR (searchable image for pages, ranges and multiple files); single-sided cut-and-stack imposition with cut marks, through Print and doc_print; architectural/ANSI/large ISO sheet presets, Match page at 100% and PDF-scale preview (#599 follow-up); Create PDF from multiple files (PDFs, images and text, edited as pages in the grid); OCR can also read pictures on pages that have text, without adding that text twice (#1028). Model staging fetches SHA-256-verified ONNX detection/recognition files after the upstream RTEN URL brownout; runtime discovery prefers complete ONNX pairs per directory, then complete legacy RTEN pairs, without mixing formats or changing directory precedence. Done since: Word invoice export uses borderless inferred tables, floating positioned text/artwork only where text overlaps an image or is rotated, and per-page sizes, covered by synthetic invoice automation regressions (#773); printing on the web through the browser's print dialog, sheets drawn by PdfCraft (gap 4). Lossless image export applies per-component Decode, including Indexed mapping (#1201 partial). Existing 16-bit precision and color-management limits remain. #3532 partial: vector printing retains existing appearances with absent Subtype through a print-owned Form copy; source artwork stays unchanged. #3120: Office export reads form XObjects 50 levels deep, as the renderer does, and up to 50 000 a page; pages past that are listed in `truncated_pages`. Word table cells use the grid columns they span, including cells after a merged cell. Done since: text in a table slot covered by another cell's span remains as paragraph text in Word, HTML, RTF and PowerPoint. Web OCR downloads its models on first use and reads in Web Workers with SIMD (`ocr.ocr-web`, shipped). Missing: OCR languages beyond Latin, web OCR near native speed, editable-text OCR output, full Office layout fidelity/import. Done since: dotted-leader contents entries stay in separate complete paragraphs (#1757) #2156 candidate: last selected print paper persists across sessions; other print preferences unchanged. Image export applies soft-mask Decode opacity intervals before resizing, including indirect entries and inverted ranges. Vector printing gives annotation appearances unoccupied XObject names so existing page artwork survives. Done since: Enhance scanned file cleans scanned pages in place (background removal, edge shadow removal, despeckle; the OCR text layer and comments stay; #4119 partial). | +| M10 | OCR, create, export, print | 200–350 | 33% | 120–215 | Done: Windows Printer default sends native two-sided bindings only when automatic duplex support is confirmed, preserving manual driver defaults (#5130/#5772); physical output remains unqualified. DocumentDuplex-only bindings use the supported scope through craft-print 9fe50c94. create from blank/text/PNG/JPEG/TIFF (multi-page)/GIF/BMP; export PNG/JPEG/TIFF and text; Print (Acrobat's sizing, n-up, booklet, poster, comments & forms, preview, the CUPS and Windows spoolers, print-ready PDF).
Done since: Word, Excel, PowerPoint, OpenDocument and RTF files open from File ▸ Open as new PDFs through an installed LibreOffice (`create.from-office` partial); embedded/72/custom DPI choices for image imports; export all images; OCR (searchable image for pages, ranges and multiple files); single-sided cut-and-stack imposition with cut marks, through Print and doc_print; architectural/ANSI/large ISO sheet presets, Match page at 100% and PDF-scale preview (#599 follow-up); Create PDF from multiple files (PDFs, images and text, edited as pages in the grid); OCR can also read pictures on pages that have text, without adding that text twice (#1028). Model staging fetches SHA-256-verified ONNX detection/recognition files after the upstream RTEN URL brownout; runtime discovery prefers complete ONNX pairs per directory, then complete legacy RTEN pairs, without mixing formats or changing directory precedence. Done since: Word invoice export uses borderless inferred tables, floating positioned text/artwork only where text overlaps an image or is rotated, and per-page sizes, covered by synthetic invoice automation regressions (#773); printing on the web through the browser's print dialog, sheets drawn by PdfCraft (gap 4). Lossless image export applies per-component Decode, including Indexed mapping (#1201 partial). Existing 16-bit precision and color-management limits remain. #3532 partial: vector printing retains existing appearances with absent Subtype through a print-owned Form copy; source artwork stays unchanged. #3120: Office export reads form XObjects 50 levels deep, as the renderer does, and up to 50 000 a page; pages past that are listed in `truncated_pages`. Word table cells use the grid columns they span, including cells after a merged cell. Done since: text in a table slot covered by another cell's span remains as paragraph text in Word, HTML, RTF and PowerPoint. Web OCR downloads its models on first use and reads in Web Workers with SIMD (`ocr.ocr-web`, shipped). Missing: OCR languages beyond Latin, web OCR near native speed, editable-text OCR output, full Office layout fidelity/import. Done since: dotted-leader contents entries stay in separate complete paragraphs (#1757) #2156 candidate: last selected print paper persists across sessions; other print preferences unchanged. Image export applies soft-mask Decode opacity intervals before resizing, including indirect entries and inverted ranges. Vector printing gives annotation appearances unoccupied XObject names so existing page artwork survives. Done since: Enhance scanned file cleans scanned pages in place (background removal, edge shadow removal, despeckle; the OCR text layer and comments stay; #4119 partial). | | M11 | Optimize, preflight, PDF/A/X/UA, print production | 200–350 | 18% | 80–140 | Done: new `optimize` crate: Reduce File Size and the PDF Optimizer (images measured where drawn, bicubic downsampling, JPEG/ZIP recompression only when smaller, discard objects and user data, Flate clean-up, resource merging, object streams). Target-version compatibility is partial: newer content is not detected or converted. Content stream boundaries preserve comments, split operands and graphics state when image fragments merge. Image downsampling includes page-local `/UserUnit` scaling when measuring physical resolution. Done since: reject invalid image and soft-mask dimensions with checked sizing (#2178); black-and-white images stored losslessly as CCITT Group 4 when smaller, averaged down to 300 ppi above 450 (halftones aside), Group 3 stored again as Group 4, and black-and-white RGB and indexed images that use only palette entries 0 and 1 packed to one bit. Missing: fonts and transparency panels, space audit, preflight, PDF/A/X/UA | | M8 | Security + redaction | 100–180 | 68% | 15–30 | Done: opening protected documents, permissions, Protect Using Password, Remove security; redaction (mark text/areas/pages, Search & Redact with patterns, apply removing glyphs, image pixels, vectors, XObject content, annotations and fields, verification, full rewrite on save); Remove hidden information and Sanitize.
Done since: redaction codes (U.S. FOIA and Privacy Act sets as overlay text, in Redaction properties and `redact_mark`). Redaction cleans indirect and form structure references, preserving surviving form tags. Redaction retires inherited resource dictionaries, so a parent `/Pages` can't keep a removed XObject reachable; unredacted pages are left untouched. Image redaction removes alternate images and unsupported masks only on affected images. Changed marked-content spans lose replacement descriptions; untouched spans keep theirs. Hidden-text cleanup retires unused named descriptions across inherited/shared page and form resources (64 focused tests and scoped Rust 1.96 Clippy/wasm checks pass; publication CI pending). The redaction audit counts a filled cover that hides half of any one word of a long line (#2662), applied or not. Phone-pattern redaction also matches labelled Italian geographic numbers with `+39` and a slash after the area code (#3205 follow-up); unlabelled international slash numbers remain unsupported. Missing: certificate security, custom redaction code sets and pattern locales, DCT re-encoding | | M9 | Signatures (PAdES, validation) | 160–280 | 60% | 40–70 | Done: new `sign` crate (DER, X.509, CMS, PKCS #12 on RustCrypto; aws-lc-rs for RSA private keys); PAdES B-B signing (visible, invisible, existing fields, certification with DocMDP); validation with trust store and changes-after-signing classification; self-signed digital IDs; Signatures panel, message bar, sign dialogs; agent tools. Checked with pdfsig and OpenSSL.
Done since: macOS Keychain and Windows Current User Personal store signing (software-backed CNG keys), certificate viewer, signed documents protected from rewrites, page validation overlays separate from original signature artwork (baked Not Verified captions remain; #79 open), snapshot tabs that reuse instead of multiplying. Missing: timestamps (B-T), LTV (DSS, OCSP, CRL), FieldMDP, smart cards, PKCS #11 | diff --git a/docs/target-app-parity.md b/docs/target-app-parity.md index 126496547d..2d219b096a 100644 --- a/docs/target-app-parity.md +++ b/docs/target-app-parity.md @@ -212,7 +212,7 @@ full parity in that area. | G Protect, redact, sanitize | 41 | 23 | 2 | 58.5% | 55% | 15–30 | Passwords and permissions, Remove security, redaction that removes glyphs, pixels, vectors, annotations and tags, with verification; word lists, patterns, FOIA/Privacy Act codes; sanitize. Missing: certificate security, pattern locales, custom code sets, partial-word redaction, folder redaction, protected view | | H Digital signatures | 45 | 23 | 10 | 62.2% | 45% | 40–70 | PAdES B-B signing and validation (PKCS #12, macOS Keychain, Windows store with PIN prompts), certification, encrypted documents, certificate usage checks; B-T, LTV, OCSP/CRL with caller-supplied evidence. Missing: PKCS #11 and smart cards outside Windows, timestamp-server preferences, online revocation fetching, FieldMDP and field locking, OS trust store, EU Trusted Lists, compare signed version | | I Scan and OCR | 26 | 5 | 2 | 23.1% | 15% | 40–75 | Searchable image (ocrs, Latin script without accents) on pages, ranges and many files. Missing: every other language (#744 Dutch; accents, CJK, Cyrillic, Arabic), editable-text output, deskew, rotation, camera images beyond straightening (found or placed corners): orientation, clean-up, MRC compression, scanners. Background removal, edge shadow removal and despeckle work on scanned pages (Enhance scanned file) but not yet as options of Recognize text, nor on CMYK, indexed, JBIG2 or CCITT scans | -| J Create and export | 34 | 12 | 3 | 39.7% | 25% | 55–100 | Create from images, multipage TIFF, text, clipboard, blank, many files at once; export PNG/JPEG/TIFF, text, RTF; Word and HTML partial. Missing: Office/HTML/web/PostScript to PDF, Excel and PowerPoint export, Word layout fidelity (#773), SVG/XML/EPS/PS/JPEG 2000 export, accessible text | +| J Create and export | 34 | 12 | 3 | 39.7% | 25% | 55–100 | Create from images, multipage TIFF, text, clipboard, blank, many files at once; export PNG/JPEG/TIFF, text, RTF; Word and HTML partial. Office documents open through an installed LibreOffice (File ▸ Open only). Missing: HTML/web/PostScript to PDF, Excel and PowerPoint export, Word layout fidelity (#773), SVG/XML/EPS/PS/JPEG 2000 export, accessible text | | K Optimize and standards | 58 | 15 | 11 | 35.3% | 20% | 65–115 | Reduce File Size and the Optimizer (images, CCITT and lossless JBIG2 monochrome compression, discards, clean-up, space audit); PDF/A-2b/3b verify and convert (partial). Missing: Preflight (profiles, fixups, reports), PDF/A-1/4 and a/u levels, PDF/X, PDF/UA, PDF/E, PDF/VT, transparency flattening, font unembedding, lossy/refinement JBIG2 | | L Print | 39 | 15 | 1 | 39.7% | 35% | 30–55 | Acrobat's sizing, n-up, booklet, poster, comments & forms, preview, print-ready PDF, CUPS on macOS/Linux, the Windows spooler (#756) and the browser's print dialog on the web. Missing: print as image, PostScript, all print-production tools (output preview, separations, ink manager, printer marks, colour conversion) | | M Accessibility | 61 | 38 | 1 | 63.1% | 30% | 40–70 | Accessibility Checker (all 32 rules, report, fixes), alternate text, title and language. Missing: autotag, Tags/Order/Content panels, Reading Order tool, artifact marking, table editor, Read Out Loud, full keyboard operation of the app, screen-reader audit | diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index a5e052341a..6af3b0b471 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -7140,11 +7140,25 @@ title = "Create PDF from Office documents (LibreOffice sidecar)" area = "J" tier = "P2" milestone = "M10" -status = "planned" -commands = [] -tools = [] -evidence = [] -notes = "" +status = "partial" +commands = ["file.open"] +tools = ["doc_create_multiple", "doc_combine"] +evidence = [ + "crates/create/src/office.rs::office_files_are_recognised_by_extension_and_signature", + "crates/create/src/office.rs::other_files_are_not_office_documents", + "crates/create/src/office.rs::a_converter_that_hangs_is_stopped", + "crates/create/src/office.rs::a_converter_that_makes_no_pdf_is_reported", + "crates/create/src/office.rs::libreoffice_converts_a_document_when_installed", + "crates/create/src/office.rs::the_profile_turns_off_macros_and_never_updates_links", + "crates/create/src/office.rs::a_timeout_stops_the_processes_the_converter_started", + "crates/automation/tests/it/office.rs::office_documents_convert_separately_with_their_text", + "crates/automation/tests/it/office.rs::office_documents_combine_in_order_with_a_bookmark_each", + "crates/engine/src/tests.rs::office_documents_convert_through_libreoffice_or_explain_why_not", + "crates/engine/src/tests.rs::inline_conversions_refuse_office_documents_without_running_libreoffice", + "crates/ui-egui/src/platform/open_worker/tests.rs::file_open_converts_an_office_document_on_the_worker", + "crates/ui-egui/src/platform/open_worker/tests.rs::the_open_picker_defaults_to_pdf_and_offers_images_office_text_and_all_files", +] +notes = "File ▸ Open (desktop) opens Word, Excel, PowerPoint, OpenDocument and RTF files (Office documents filter; PDF stays the default) as new, unsaved PDFs, converted on the open worker by an installed LibreOffice run headless in a private temporary folder with a fresh profile (macros disabled, untrusted links blocked, Writer and Calc linked content never updated) and stopped after 120 s. Found through PDFCRAFT_SOFFICE, the PATH, or the usual install folders; without it the user is told LibreOffice is needed. The automation tools that convert files (doc_create_multiple, doc_combine, batch convert) take Office documents too. Not yet: Insert pages, Combine files and Create ▸ Multiple files in the app refuse Office documents (they convert on the frame thread) and point to File ▸ Open; no Create ▸ From Office command or conversion settings; Visio and Publisher; the web build; layout fidelity is LibreOffice's, not measured against Acrobat; Impress has no link-update setting and the process's network access isn't sandboxed (synthetic linked sections, pictures and cell references made no connection with LibreOffice 26.8.1); the real-conversion tests skip where LibreOffice isn't installed, CI included." [[feature]] id = "create.create-multiple" From 059dbd6e00507cdfedd755bf6b712f883507fce0 Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Wed, 11 Nov 2026 13:09:37 +0100 Subject: [PATCH 38/60] feat(forms): open, Will Save and Will Print actions, widget pointer/focus scripts, setItems/setAction/toSource (port of #344) (#6969) * feat(js, forms): field setItems, setAction and a bounded toSource (#269) Ported from #344 by @DWAA1660 onto the per-document script session. - `field.setItems(items)` replaces a combo or list box's items (text, or [text, export] pairs). The value stays when an item still has it, and otherwise becomes the first item's. Written as the field's /Opt (pairs only where the export value differs), /I dropped, field redrawn. - `field.setAction(trigger, script)` installs a JavaScript action for one trigger (MouseUp/Down/Enter/Exit, OnFocus/OnBlur, Keystroke, Validate, Calculate, Format), replacing only that trigger's action. - `Object.prototype.toSource()` writes plain data (strings, numbers, booleans, null, undefined, arrays, objects) that eval reads back. It refuses functions, cycles, more than 64 levels, 10,000 values or 1 MiB with an error the script can catch. Co-authored-by: DWAA1660 * feat(engine): run a form's open, Will Save and Will Print actions and its widgets' pointer and focus scripts (#269) Ported from #344 by @DWAA1660 onto the per-document script session: each runs in the document's retained realm, so state the document scripts and earlier events left is kept. - The catalog /OpenAction's JavaScript (and its /Next chain, bounded and cycle-safe; other action types skipped) runs once on opening, after the document scripts, for background opens and reloads too, and again on Revert in a fresh realm. Event Doc/Open. - `Session::run_document_actions` runs /AA /WS (Will Save) or /AA /WP (Will Print). A failing action is reported and never stops the save or print. doc_save and doc_print run them first and report what they said (`will_save`, `will_print`). - Scripts that run unasked (open, Will Save, Will Print, pointer over, focus) may only ask for highlight, zoom, focus or a page; a click's Mouse Down / Mouse Up keep their requests. - `Session::field_trigger` runs a widget's Mouse Enter, Mouse Exit, Mouse Down, Mouse Up, Focus or Blur JavaScript (one undoable edit), reading only that widget's dictionary. `field_focus` now runs an AcroForm widget's On Focus / On Blur as well as XFA enter/exit. - New tool `js_event`; form_focus's description says AcroForm focus scripts run too. Co-authored-by: DWAA1660 * feat(ui): Will Save before saving, Will Print before printing, and widget pointer scripts in the app (#269, #247) Ported from #344 by @DWAA1660. - Save (both paths) runs the document's Will Save actions first, and Print its Will Print actions; what they change is saved or printed, and what they say is shown. - The pointer moving onto or off a form widget queues its Mouse Enter / Mouse Exit script, a press its Mouse Down. A click on a check box or radio button queues its Mouse Up after the toggle, so the script reads the value the click stored (#247). The queue runs after the frame's edit. Co-authored-by: DWAA1660 * docs(parity): document actions and widget triggers are partly supported (#269) Co-authored-by: DWAA1660 * test: check toSource's text (scripts can't eval it) and the trigger event names (#269) Co-authored-by: DWAA1660 * test: a check box's Mouse Up reads its new value; keep the fixtures on screen and the realm across an in-place save (#247, #269) Co-authored-by: DWAA1660 * fix(engine, forms, js, ui): review fixes for document actions and field triggers - widget trigger actions follow /Next, each action its own edit; errors don't stop the chain - setAction installs on the field the object stands for when names repeat - toSource refuses non-plain objects and a __proto__ key - doc_print checks the destination before Will Print; js_event returns only its own output - combo/list clicks run Mouse Up; a refused click's Mouse Up doesn't run - label crops are re-read when Will Print changed the document Co-authored-by: DWAA1660 * forms: run a pointer trigger on the field it hit and refresh the edited field's draft A trigger now carries the field's object, so of fields sharing a name the one under the pointer runs its own script (Session::field_trigger_target); js_event passes the field it resolved. A trigger script changing the field being edited shows in its editor unless the user already typed there, as focus scripts do. Co-authored-by: DWAA1660 * forms: carry the field's identity through focus scripts and js_event Focus transitions now track the field object, so moving between fields sharing a name is a move and the field that gained or lost focus runs its own script (Session::field_focus_target). js_event takes an optional field_id from form_fields to pick one of fields sharing a name. Co-authored-by: DWAA1660 --------- Co-authored-by: DWAA1660 --- crates/automation/src/a11y/form_tools.rs | 65 ++- crates/automation/src/dispatch.rs | 1 + crates/automation/src/documents.rs | 10 +- crates/automation/src/printing.rs | 35 +- crates/automation/src/tools.rs | 17 +- .../automation/tests/it/document_actions.rs | 125 ++++++ crates/automation/tests/it/main.rs | 1 + crates/engine/src/document_actions_tests.rs | 379 ++++++++++++++++++ crates/engine/src/js.rs | 192 ++++++++- crates/engine/src/lib.rs | 2 + crates/engine/src/open.rs | 30 +- crates/engine/src/session/save.rs | 8 +- crates/forms/src/actions.rs | 15 + crates/forms/src/lib.rs | 6 +- crates/forms/src/scripting.rs | 165 +++++++- crates/js/src/document.rs | 74 ++++ crates/js/src/globals.rs | 1 + crates/js/src/lib.rs | 1 + crates/js/src/model.rs | 6 + crates/js/src/source.rs | 145 +++++++ crates/js/src/tests.rs | 73 ++++ crates/ui-egui/src/canvas.rs | 5 + crates/ui-egui/src/canvas/page_input.rs | 3 + crates/ui-egui/src/editing.rs | 92 ++++- crates/ui-egui/src/editing/save.rs | 2 + crates/ui-egui/src/tools/forms_ui.rs | 54 ++- crates/ui-egui/src/workflows/js_ui.rs | 17 + crates/ui-egui/src/workflows/print_ui.rs | 10 + crates/ui-egui/tests/it/field_triggers.rs | 104 +++++ crates/ui-egui/tests/it/main.rs | 1 + docs/acroform-script-workflows.md | 16 +- parity/acrobat-features.toml | 16 +- 32 files changed, 1597 insertions(+), 74 deletions(-) create mode 100644 crates/automation/tests/it/document_actions.rs create mode 100644 crates/engine/src/document_actions_tests.rs create mode 100644 crates/js/src/source.rs create mode 100644 crates/ui-egui/tests/it/field_triggers.rs diff --git a/crates/automation/src/a11y/form_tools.rs b/crates/automation/src/a11y/form_tools.rs index 31234a28ed..0ef6b92821 100644 --- a/crates/automation/src/a11y/form_tools.rs +++ b/crates/automation/src/a11y/form_tools.rs @@ -21,12 +21,35 @@ fn request_json(r: &pdfcraft_engine::js::Request) -> Value { } } +/// The optional `field_id` argument: a field's identity from form_fields, selecting one of +/// fields sharing a name. +fn field_id(a: &Args) -> Result> { + a.0.get("field_id") + .map(|v| { + v.as_u64() + .filter(|id| *id < 1 << 48) + .ok_or_else(|| ToolError::InvalidArgs("field_id must be an integer between 0 and 281474976710655".into())) + }) + .transpose() +} + /// A message a script showed, as `js_run` reports it: its text, title (null when it gave none), /// icon and buttons. pub(crate) fn alert_json(m: &pdfcraft_engine::js::Message) -> Value { json!({ "message": m.text, "title": m.title, "icon": m.icon.name(), "buttons": m.buttons.name() }) } +/// What scripts that ran unasked (a document's Will Save or Will Print actions) showed, said +/// or asked for, as tools report it. +pub(crate) fn js_output_json(out: &pdfcraft_engine::js::JsOutput) -> Value { + json!({ + "alerts": out.alerts.iter().map(alert_json).collect::>(), + "console": out.console, + "requests": out.requests.iter().map(request_json).collect::>(), + "errors": out.errors, + }) +} + impl Automation { /// `js_run`: run `script` in the document, or as button `field`'s script, and report its /// messages, console lines, requests, error, the failing document scripts' errors and result. @@ -44,14 +67,7 @@ impl Automation { .collect::>>()?, _ => return Err(ToolError::InvalidArgs("answers must be an array of at most 32 strings or null".into())), }; - let identity = - a.0.get("field_id") - .map(|v| { - v.as_u64() - .filter(|id| *id < 1 << 48) - .ok_or_else(|| ToolError::InvalidArgs("field_id must be an integer between 0 and 281474976710655".into())) - }) - .transpose()?; + let identity = field_id(a)?; let (script, field) = (a.str("script")?, a.opt_str("field")?); let (mut o, edit) = self.session.with_own_js_output(id, |s| s.run_javascript_answers_target(id, script, field, identity, &answers)).map_err(failed)?; @@ -122,6 +138,39 @@ impl Automation { ) } + /// `js_event`: run the JavaScript widget `widget` (1-based) of field `field` has for `event` + /// (a trigger id: mouse_enter, mouse_exit, mouse_down, mouse_up, on_focus, on_blur); returns + /// whether the form changed and what the script showed or asked for. + pub(crate) fn js_event(&mut self, a: &Args) -> Result { + let event = a.str("event")?; + let trigger = pdfcraft_engine::FieldTrigger::from_id(event).ok_or_else(|| { + ToolError::InvalidArgs(format!("event must be mouse_enter, mouse_exit, mouse_down, mouse_up, on_focus or on_blur, not {event:?}")) + })?; + let name = a.str("field")?; + let identity = field_id(a)?; + let doc = self.doc(a)?; + let field = + doc.form.iter().find(|f| f.name == name && identity.is_none_or(|i| pdfcraft_engine::js::field_object_id(f.obj) == i)).ok_or_else( + || match identity { + Some(i) => failed(format!("there is no field named {name:?} with field_id {i} (see form_fields)")), + None => failed(format!("there is no field named {name:?} (see form_fields)")), + }, + )?; + let widgets = field.widgets.len(); + let widget = a.opt_int("widget")?.unwrap_or(1); + let widget = usize::try_from(widget).ok().and_then(|w| w.checked_sub(1)).filter(|w| *w < widgets).ok_or_else(|| match widgets { + 0 => ToolError::InvalidArgs(format!("{name:?} has no widget")), + n => ToolError::InvalidArgs(format!("widget must be 1 to {n} for {name:?}")), + })?; + let (id, object) = (doc.id, pdfcraft_engine::js::field_object_id(field.obj)); + // Only this event's output: what opening or an earlier tool left waiting stays. + let (changed, out) = + self.session.with_own_js_output(id, |s| s.field_trigger_target(id, name, Some(object), widget, trigger)).map_err(failed)?; + let mut result = js_output_json(&out); + result["changed"] = json!(changed); + Ok(result) + } + /// `form_merge_data`: merge the form data files at `paths` into one CSV at `path`, a row per /// file; returns the path and the row and column counts. pub(crate) fn form_merge_data(&mut self, a: &Args) -> Result { diff --git a/crates/automation/src/dispatch.rs b/crates/automation/src/dispatch.rs index 9644684772..4a91942fe4 100644 --- a/crates/automation/src/dispatch.rs +++ b/crates/automation/src/dispatch.rs @@ -835,6 +835,7 @@ impl Automation { "accessibility_autotag_form_fields" => self.accessibility_autotag_form_fields(&a)?, "ocr_recognize" => self.ocr_recognize(&a)?, "js_run" => self.js_run(&a)?, + "js_event" => self.js_event(&a)?, "js_document_scripts" => self.js_document_scripts(&a)?, "js_set_document_script" => self.js_set_document_script(&a)?, "js_enabled" => self.js_enabled(&a)?, diff --git a/crates/automation/src/documents.rs b/crates/automation/src/documents.rs index c00034521c..7dd78cff98 100644 --- a/crates/automation/src/documents.rs +++ b/crates/automation/src/documents.rs @@ -168,11 +168,19 @@ impl Automation { if flatten_fill_sign { self.apply(a, Edit::FlattenFillSign)?; } + // The document's Will Save actions run first: what they change is saved (#269). + let scripts = self.session.run_document_actions(id, pdfcraft_engine::js::DocumentScriptEvent::WillSave); let bytes = if full { self.session.save_full_bytes(id) } else { self.session.save_bytes(id) }.map_err(failed)?; // A save that adds or removes encryption, or must not keep the old revision (Sanitize, // redaction), is rewritten in full even when an incremental save was asked for. let incremental = !full && bytes.starts_with(&self.doc(a)?.saved_bytes()); - self.publish_save(id, &target, bytes, incremental, write) + let mut out = self.publish_save(id, &target, bytes, incremental, write)?; + if !scripts.is_empty() + && let Some(o) = out.as_object_mut() + { + o.insert("will_save".into(), crate::a11y::form_tools::js_output_json(&scripts)); + } + Ok(out) } /// Validate the prepared bytes before publishing them or marking the document saved. diff --git a/crates/automation/src/printing.rs b/crates/automation/src/printing.rs index 5d03304db1..3d7a76865a 100644 --- a/crates/automation/src/printing.rs +++ b/crates/automation/src/printing.rs @@ -361,29 +361,50 @@ impl Automation { let settings = print::Settings { pages, paper, orientation, layout, content, raster, crops: Vec::new(), area }; let laid_out = print::layout(&sizes, &settings).map_err(|e| bad(e.to_string()))?; let sheets = laid_out.len(); + // Where it goes is checked before the Will Print actions run, so a call that can't print + // doesn't change the document. + let destination = match (a.opt_str("path")?, printer) { + (Some(p), None) => Destination::File(self.resolve(p, true)?), + (None, Some(printer)) => { + let actual_printer = resolved_printer.as_deref().unwrap_or(printer); + Destination::Printer(spool_job(a, actual_printer, self.doc(a)?.name.clone(), duplex, media.as_deref(), settings.raster, &laid_out)?) + } + _ => return Err(bad("pass either path (save the print-ready PDF) or printer (a name from printers, or \"default\")")), + }; + // The document's Will Print actions run first: what they change is printed (#269). + let scripts = if self.doc(a)?.allows_printing() { + self.session.run_document_actions(id, pdfcraft_engine::js::DocumentScriptEvent::WillPrint) + } else { + Default::default() + }; let bytes = self.session.print_pdf(id, &settings).map_err(failed)?; let mut out = json!({ "sheets": sheets, "pages": settings.pages.len(), "bytes": bytes.len() }); - match (a.opt_str("path")?, printer) { - (Some(p), None) => { - let path = self.resolve(p, true)?; + if !scripts.is_empty() { + out["will_print"] = crate::a11y::form_tools::js_output_json(&scripts); + } + match destination { + Destination::File(path) => { write_atomic(&path, &bytes)?; out["path"] = json!(path.to_string_lossy()); } - (None, Some(printer)) => { - let actual_printer = resolved_printer.as_deref().unwrap_or(printer); - let mut job = spool_job(a, actual_printer, self.doc(a)?.name.clone(), duplex, media.as_deref(), settings.raster, &laid_out)?; + Destination::Printer(mut job) => { // A GDI (v3) driver gets transparent sheets flattened (#6155, #5838, #5940); XPS // (v4) drivers print them as they are. let flatten = job.printer.as_deref().is_some_and(spool::gdi_driver); let bytes = pdfcraft_engine::spooler::spooler_copy(bytes, &mut job, flatten); out["job"] = json!(spool::submit(&bytes, &job).map_err(|e| failed(e.to_string()))?); } - _ => return Err(bad("pass either path (save the print-ready PDF) or printer (a name from printers, or \"default\")")), } Ok(out) } } +/// Where `doc_print` sends the sheets: a print-ready PDF file, or a printer's queue. +enum Destination { + File(std::path::PathBuf), + Printer(spool::Job), +} + /// "WxH" with a unit (in, mm, cm or pt) as a portrait paper size in points; each side 1 to 200 /// inches (PDF's own page limit). fn paper_size(spec: &str) -> Option<(f64, f64)> { diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index 7d948858cd..4f1342605b 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -543,7 +543,7 @@ pub fn tools() -> Vec { t( "form_focus", "Move the focus into or out of a field", - "A field gains (event enter) or loses (event exit) focus, as when someone clicks or tabs into or out of it, and the dynamic XFA form's enter or exit scripts for it run (other forms have none, and nothing happens). widget picks the radio button or copy of the field (1-based, default 1; see form_fields). What the scripts change is one undoable edit, and none when they change nothing. Changing a field's value (form_fill) runs its exit scripts already, so the next exit of that field doesn't run them again. Returns changed, and the scripts' alerts, console output, requests and errors.", + "A field gains (event enter) or loses (event exit) focus, as when someone clicks or tabs into or out of it, and the dynamic XFA form's enter or exit scripts for it run; on another form, the widget's own On Focus or On Blur JavaScript runs, if it has one (see also js_event). widget picks the radio button or copy of the field (1-based, default 1; see form_fields). What the scripts change is one undoable edit, and none when they change nothing. Changing a field's value (form_fill) runs its exit scripts already, so the next exit of that field doesn't run them again. Returns changed, and the scripts' alerts, console output, requests and errors.", ) .with(schema( json!({ @@ -1421,6 +1421,21 @@ pub fn tools() -> Vec { json!({ "doc": doc(), "script": { "type": "string" }, "field": { "type": "string", "description": "Run as this button's Mouse Up event." }, "field_id": {"type":"integer","minimum":0,"maximum":281474976710655u64,"description":"Optional exact field identity from form_fields; requires matching field name and is scoped to this document."}, "answers": {"type":"array","maxItems":32,"items":{"type":["string","null"],"maxLength":4096},"description":"Ordered popup, response and alert answers; strings are limited to 4096 UTF-8 bytes at runtime; null cancels a popup or response, and selects an alert default (OK, Cancel or No, by button set)."} }), &["doc", "script"], )), + t( + "js_event", + "Run a field's pointer or focus script", + "Run the JavaScript action widget `widget` of AcroForm field `field` has for `event`, as when the pointer enters (mouse_enter), leaves (mouse_exit), presses (mouse_down) or releases (mouse_up) it, or it gains (on_focus) or loses (on_blur) the focus, in the document's script realm (variables document scripts and earlier scripts left are kept), with event.name Mouse Enter, Mouse Exit, Mouse Down, Mouse Up, Focus or Blur. Nothing runs when the widget has no JavaScript for that event, JavaScript is off, or the form is a laid-out XFA form (use form_focus and js_run there). What it changes is one undoable edit. Pointer-over and focus scripts run unasked in a viewer, so their requests to print, open Save As or show a panel are dropped (noted on the console); mouse_down and mouse_up keep them. Returns changed, and the script's alerts, console output, requests and errors.", + ) + .with(schema( + json!({ + "doc": doc(), + "field": { "type": "string", "description": "Field name (from form_fields)." }, + "event": { "type": "string", "enum": ["mouse_enter", "mouse_exit", "mouse_down", "mouse_up", "on_focus", "on_blur"] }, + "field_id": {"type":"integer","minimum":0,"maximum":281474976710655u64,"description":"Optional exact field identity from form_fields: of fields sharing the name, the one to run."}, + "widget": { "type": "integer", "minimum": 1, "description": "Widget of the field (1-based, default 1): a radio button of the group, or a copy of the field." }, + }), + &["doc", "field", "event"], + )), t("js_document_scripts", "Document JavaScripts", "List the document-level JavaScripts (name and source), which define functions field scripts use.") .ro() .cmd("tools.document_js") diff --git a/crates/automation/tests/it/document_actions.rs b/crates/automation/tests/it/document_actions.rs new file mode 100644 index 0000000000..a8646a4437 --- /dev/null +++ b/crates/automation/tests/it/document_actions.rs @@ -0,0 +1,125 @@ +//! A form's document actions and pointer/focus scripts through the tool table (#269, ported +//! from #344): the open action runs on doc_open, Will Save on doc_save, Will Print on +//! doc_print, and js_event runs a widget's pointer and focus scripts, all in one realm. + +use pdfcraft_automation::{Automation, Content}; +use serde_json::{Value, json}; + +fn tool(a: &mut Automation, name: &str, args: Value) -> Value { + match a.call(name, &args).unwrap().remove(0) { + Content::Json(value) => value, + other => panic!("expected JSON: {other:?}"), + } +} + +fn pdf_of(objs: &[String]) -> Vec { + let mut out = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (i, o) in objs.iter().enumerate() { + offsets.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n{o}\nendobj\n", i + 1).as_bytes()); + } + let xref = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes()); + for o in offsets { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n", objs.len() + 1).as_bytes()); + out +} + +/// `log` records what ran; `hot` has Mouse Enter and On Blur scripts. The open action counts +/// in a variable the Will Save and Will Print actions and `hot`'s scripts read. +fn form() -> Vec { + pdf_of(&[ + "<< /Type /Catalog /Pages 2 0 R /OpenAction 6 0 R /AA << /WS 7 0 R /WP 8 0 R >> /AcroForm << /Fields [4 0 R 9 0 R] /DA (/Helv 0 Tf 0 g) /DR << /Font << /Helv 5 0 R >> >> >> >>".into(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 200 300] >>".into(), + "<< /Type /Page /Parent 2 0 R /Annots [4 0 R 9 0 R] >>".into(), + "<< /Type /Annot /Subtype /Widget /FT /Tx /T (log) /V () /Rect [20 250 180 270] /P 3 0 R /F 4 >>".into(), + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>".into(), + "<< /S /JavaScript /JS (var n = 1; getField\\('log'\\).value = 'open' + n;) >>".into(), + "<< /S /JavaScript /JS (n++; getField\\('log'\\).value += ' save' + n; console.println\\('saving'\\);) >>".into(), + "<< /S /JavaScript /JS (n++; getField\\('log'\\).value += ' print' + n;) >>".into(), + "<< /Type /Annot /Subtype /Widget /FT /Tx /T (hot) /V () /Rect [20 200 180 220] /P 3 0 R /F 4 /AA << /E << /S /JavaScript /JS (n++; getField\\('log'\\).value += ' ' + event.name + n;) >> /Bl << /S /JavaScript /JS (this.print\\(\\); getField\\('log'\\).value += ' ' + event.name;) >> >> >>".into(), + ]) +} + +fn log(a: &mut Automation, doc: u64) -> Value { + let f = tool(a, "form_fields", json!({ "doc": doc })); + f["fields"].as_array().unwrap().iter().find(|f| f["name"] == "log").map(|f| f["value"].clone()).unwrap() +} + +#[test] +fn open_will_save_will_print_and_field_events_share_the_document_realm() { + let dir = std::env::temp_dir().join(format!("pdfcraft-document-actions-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("form.pdf"), form()).unwrap(); + let mut a = Automation::new().with_root(&dir).unwrap().with_clock(|| 1_700_000_000); + let doc = tool(&mut a, "doc_open", json!({ "path": "form.pdf" }))["doc"].as_u64().unwrap(); + assert_eq!(log(&mut a, doc), "open1"); + + let entered = tool(&mut a, "js_event", json!({ "doc": doc, "field": "hot", "event": "mouse_enter" })); + assert_eq!(entered["changed"], true, "{entered}"); + assert_eq!(log(&mut a, doc), "open1 Mouse Enter2"); + // Losing the focus happens unasked: its print request is dropped and noted. + let blurred = tool(&mut a, "js_event", json!({ "doc": doc, "field": "hot", "event": "on_blur" })); + assert_eq!(blurred["requests"], json!([]), "{blurred}"); + assert_eq!(log(&mut a, doc), "open1 Mouse Enter2 Blur"); + let none = tool(&mut a, "js_event", json!({ "doc": doc, "field": "log", "event": "mouse_down" })); + assert_eq!(none["changed"], false, "no script: {none}"); + assert!(a.call("js_event", &json!({ "doc": doc, "field": "hot", "event": "keystroke" })).is_err()); + assert!(a.call("js_event", &json!({ "doc": doc, "field": "hot", "event": "mouse_up", "widget": 2 })).is_err()); + + // Saved in place (incrementally): object numbers stay, so the realm is kept. A full + // rewrite starts a fresh realm for the next event. + let saved = tool(&mut a, "doc_save", json!({ "doc": doc })); + assert_eq!(saved["incremental"], true, "{saved}"); + std::fs::copy(dir.join("form.pdf"), dir.join("saved.pdf")).unwrap(); + assert_eq!(saved["will_save"]["console"], json!(["saving"]), "{saved}"); + assert_eq!(log(&mut a, doc), "open1 Mouse Enter2 Blur save3"); + + let printed = tool(&mut a, "doc_print", json!({ "doc": doc, "path": "printed.pdf" })); + assert!(printed.get("will_print").is_none(), "nothing to report: {printed}"); + assert_eq!(log(&mut a, doc), "open1 Mouse Enter2 Blur save3 print4"); + // A print with nowhere to go fails before Will Print runs. + assert!(a.call("doc_print", &json!({ "doc": doc })).is_err()); + assert_eq!(log(&mut a, doc), "open1 Mouse Enter2 Blur save3 print4"); + + // What Will Save changed is in the saved file. + tool(&mut a, "js_enabled", json!({ "enabled": false })); + let reopened = tool(&mut a, "doc_open", json!({ "path": "saved.pdf" }))["doc"].as_u64().unwrap(); + assert_eq!(log(&mut a, reopened), "open1 Mouse Enter2 Blur save3"); +} + +/// Of two fields sharing a name, `field_id` (from form_fields) picks the one js_event runs; by +/// name alone it is the first. +#[test] +fn js_event_runs_the_field_its_field_id_names() { + let enter = |text: &str| format!("/AA << /E << /S /JavaScript /JS (getField\\('log'\\).value += '{text}';) >> >>"); + let pdf = pdf_of(&[ + "<< /Type /Catalog /Pages 2 0 R /AcroForm << /Fields [4 0 R 6 0 R 7 0 R] /DA (/Helv 0 Tf 0 g) /DR << /Font << /Helv 5 0 R >> >> >> >>".into(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 200 300] >>".into(), + "<< /Type /Page /Parent 2 0 R /Annots [4 0 R 6 0 R 7 0 R] >>".into(), + "<< /Type /Annot /Subtype /Widget /FT /Tx /T (log) /V () /Rect [20 250 180 270] /P 3 0 R /F 4 >>".into(), + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>".into(), + format!("<< /Type /Annot /Subtype /Widget /FT /Tx /T (dup) /V () /Rect [20 200 180 220] /P 3 0 R /F 4 {} >>", enter("first;")), + format!("<< /Type /Annot /Subtype /Widget /FT /Tx /T (dup) /V () /Rect [20 150 180 170] /P 3 0 R /F 4 {} >>", enter("second;")), + ]); + let dir = std::env::temp_dir().join(format!("pdfcraft-js-event-dup-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("dup.pdf"), pdf).unwrap(); + let mut a = Automation::new().with_root(&dir).unwrap(); + let doc = tool(&mut a, "doc_open", json!({ "path": "dup.pdf" }))["doc"].as_u64().unwrap(); + let fields = tool(&mut a, "form_fields", json!({ "doc": doc })); + let ids: Vec = fields["fields"].as_array().unwrap().iter().filter(|f| f["name"] == "dup").map(|f| f["field_id"].as_u64().unwrap()).collect(); + assert_eq!(ids.len(), 2, "{fields}"); + tool(&mut a, "js_event", json!({ "doc": doc, "field": "dup", "field_id": ids[1], "event": "mouse_enter" })); + assert_eq!(log(&mut a, doc), "second;"); + tool(&mut a, "js_event", json!({ "doc": doc, "field": "dup", "event": "mouse_enter" })); + assert_eq!(log(&mut a, doc), "second;first;"); + assert!( + a.call("js_event", &json!({ "doc": doc, "field": "log", "field_id": ids[1], "event": "mouse_enter" })).is_err(), + "the id is another field's" + ); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/crates/automation/tests/it/main.rs b/crates/automation/tests/it/main.rs index 35b9d988f7..77c3d3eff3 100644 --- a/crates/automation/tests/it/main.rs +++ b/crates/automation/tests/it/main.rs @@ -9,6 +9,7 @@ mod batch; mod batch_failures; mod batch_secrets; mod button_caption; +mod document_actions; mod find_text; mod form_bulk; mod hostile_args; diff --git a/crates/engine/src/document_actions_tests.rs b/crates/engine/src/document_actions_tests.rs new file mode 100644 index 0000000000..8511163eff --- /dev/null +++ b/crates/engine/src/document_actions_tests.rs @@ -0,0 +1,379 @@ +//! Catalog JavaScript actions (#269, ported from #344): the `/OpenAction` runs once on opening +//! and again on Revert, `/AA /WS` before a save and `/AA /WP` before printing, all in the +//! document's retained realm. + +use std::sync::Arc; + +use crate::js::DocumentScriptEvent; +use crate::{DocId, Session}; + +/// A PDF of `objs`, numbered from 1, with a cross-reference table. +fn pdf_of(objs: &[String]) -> Vec { + let mut out = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (i, o) in objs.iter().enumerate() { + offsets.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n{o}\nendobj\n", i + 1).as_bytes()); + } + let xref = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes()); + for o in offsets { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n", objs.len() + 1).as_bytes()); + out +} + +/// A one-field form (`log`, a text field) whose catalog carries `catalog` (PDF syntax, e.g. +/// `/OpenAction 6 0 R`), with `extra` objects numbered from 6. +fn form(catalog: &str, extra: &[&str]) -> Vec { + let mut objs = vec![ + format!("<< /Type /Catalog /Pages 2 0 R {catalog} /AcroForm << /Fields [4 0 R] /DA (/Helv 0 Tf 0 g) /DR << /Font << /Helv 5 0 R >> >> >> >>"), + "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 200 300] >>".into(), + "<< /Type /Page /Parent 2 0 R /Annots [4 0 R] >>".into(), + "<< /Type /Annot /Subtype /Widget /FT /Tx /T (log) /V () /Rect [20 250 180 270] /P 3 0 R /F 4 >>".into(), + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>".into(), + ]; + objs.extend(extra.iter().map(|s| s.to_string())); + pdf_of(&objs) +} + +fn log(s: &Session, id: DocId) -> String { + s.get(id).unwrap().form.iter().find(|f| f.name == "log").unwrap().value.join(",") +} + +/// The open action runs once on opening, after the document scripts and in their realm, and +/// sees `event.name` "Open"; its field change is applied. +#[test] +fn the_open_action_runs_once_on_opening_in_the_document_realm() { + let bytes = form( + "/OpenAction 6 0 R /Names << /JavaScript << /Names [(init) 7 0 R] >> >>", + &[ + "<< /S /JavaScript /JS (opened++; getField\\('log'\\).value = event.type + '/' + event.name + '/' + opened;) >>", + "<< /S /JavaScript /JS (var opened = 0;) >>", + ], + ); + let mut s = Session::new(); + let id = s.open("open.pdf", None, Arc::new(bytes), None).unwrap(); + assert_eq!(log(&s, id), "Doc/Open/1"); + assert!(s.take_js_output(id).errors.is_empty()); + // The realm is kept: a later script sees the count the open action left. + assert_eq!(s.run_javascript(id, "opened;", None).unwrap().result.as_deref(), Some("1")); +} + +/// An open action chain runs in order through `/Next` (a dictionary or an array), skipping +/// actions of other types, and a cycle in it runs each action once. +#[test] +fn an_open_action_chain_runs_in_order_and_a_cycle_runs_each_action_once() { + let bytes = form( + "/OpenAction 6 0 R", + &[ + "<< /S /JavaScript /JS (var seen = ['a'];) /Next [7 0 R 8 0 R] >>", + "<< /S /GoTo /D [3 0 R /Fit] /Next 9 0 R >>", + "<< /S /JavaScript /JS (seen.push\\('c'\\); getField\\('log'\\).value = seen.join\\(''\\);) /Next 6 0 R >>", + "<< /S /JavaScript /JS (seen.push\\('b'\\);) /Next 7 0 R >>", + ], + ); + let mut s = Session::new(); + let id = s.open("chain.pdf", None, Arc::new(bytes), None).unwrap(); + assert_eq!(log(&s, id), "abc"); +} + +/// What an open action asks for that needs a click (a URL, printing) is dropped; a page +/// change it asks for is kept. +#[test] +fn an_open_action_may_not_open_a_url_or_print() { + let bytes = form("/OpenAction << /S /JavaScript /JS (app.launchURL\\('https://example.com'\\); this.print\\(\\); this.pageNum = 0;) >>", &[]); + let mut s = Session::new(); + let id = s.open("asks.pdf", None, Arc::new(bytes), None).unwrap(); + let out = s.take_js_output(id); + assert_eq!(out.requests, [crate::js::Request::GoToPage(0)], "{out:?}"); +} + +/// With JavaScript off the open action doesn't run. +#[test] +fn the_open_action_does_not_run_with_javascript_off() { + let bytes = form("/OpenAction << /S /JavaScript /JS (getField\\('log'\\).value = 'ran';) >>", &[]); + let mut s = Session::new(); + s.set_javascript(false); + let id = s.open("off.pdf", None, Arc::new(bytes), None).unwrap(); + assert_eq!(log(&s, id), ""); + assert!(s.run_document_actions(id, DocumentScriptEvent::CatalogOpen).is_empty()); +} + +/// Revert reopens the saved file in a fresh realm, so the open action runs again from scratch. +#[test] +fn revert_runs_the_open_action_again_in_a_fresh_realm() { + let bytes = form( + "/OpenAction << /S /JavaScript /JS (if \\(typeof runs == 'undefined'\\) var runs = 0; runs++; getField\\('log'\\).value = 'run ' + runs;) >>", + &[], + ); + let mut s = Session::new(); + let id = s.open("revert.pdf", None, Arc::new(bytes), None).unwrap(); + assert_eq!(log(&s, id), "run 1"); + s.run_javascript(id, "getField('log').value = 'typed';", None).unwrap(); + s.revert(id).unwrap(); + assert_eq!(log(&s, id), "run 1", "a fresh realm counts from zero again"); +} + +/// Will Save runs in the document realm before the bytes are written, and what it changes is in +/// the saved file; a failing Will Save is reported and the save still has bytes to write. +#[test] +fn will_save_changes_what_is_saved_and_a_failing_one_does_not_stop_the_save() { + let bytes = form( + "/Names << /JavaScript << /Names [(init) 6 0 R] >> >> /AA << /WS << /S /JavaScript /JS (saves++; getField\\('log'\\).value = event.name + ' ' + saves;) >> >>", + &["<< /S /JavaScript /JS (var saves = 0;) >>"], + ); + let mut s = Session::new(); + let id = s.open("save.pdf", None, Arc::new(bytes), None).unwrap(); + let out = s.run_document_actions(id, DocumentScriptEvent::WillSave); + assert!(out.errors.is_empty(), "{out:?}"); + assert_eq!(log(&s, id), "WillSave 1"); + let saved = s.save_bytes(id).unwrap(); + let mut reopened = Session::new(); + reopened.set_javascript(false); + let again = reopened.open("saved.pdf", None, saved, None).unwrap(); + assert_eq!(log(&reopened, again), "WillSave 1"); + + let failing = form("/AA << /WS << /S /JavaScript /JS (throw new Error\\('no'\\);) >> >>", &[]); + let id = s.open("fail.pdf", None, Arc::new(failing), None).unwrap(); + let out = s.run_document_actions(id, DocumentScriptEvent::WillSave); + assert!(out.errors.iter().any(|e| e.contains("no")), "{out:?}"); + assert!(s.save_bytes(id).is_ok()); +} + +/// Will Print runs before printing, in the same realm, and sees `event.name` "WillPrint". +#[test] +fn will_print_runs_in_the_document_realm() { + let bytes = form("/AA << /WP << /S /JavaScript /JS (getField\\('log'\\).value = event.name;) >> >>", &[]); + let mut s = Session::new(); + let id = s.open("print.pdf", None, Arc::new(bytes), None).unwrap(); + assert!(s.run_document_actions(id, DocumentScriptEvent::WillSave).is_empty(), "no Will Save action: nothing runs"); + assert_eq!(log(&s, id), ""); + s.run_document_actions(id, DocumentScriptEvent::WillPrint); + assert_eq!(log(&s, id), "WillPrint"); +} + +/// A form with a text field `log` and a text field `hot` whose widget has JavaScript for every +/// pointer and focus trigger, each appending `event.name` to `log`. +fn triggers_form() -> Vec { + let add = + || "<< /S /JavaScript /JS (var l = getField\\('log'\\); l.value = l.value + event.name + ':' + event.target.name + ';';) >>".to_string(); + let objs = vec![ + "<< /Type /Catalog /Pages 2 0 R /AcroForm << /Fields [4 0 R 6 0 R] /DA (/Helv 0 Tf 0 g) /DR << /Font << /Helv 5 0 R >> >> >> >>".to_string(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 200 300] >>".into(), + "<< /Type /Page /Parent 2 0 R /Annots [4 0 R 6 0 R] >>".into(), + "<< /Type /Annot /Subtype /Widget /FT /Tx /T (log) /V () /Rect [20 250 180 270] /P 3 0 R /F 4 >>".into(), + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>".into(), + format!( + "<< /Type /Annot /Subtype /Widget /FT /Tx /T (hot) /V () /Rect [20 200 180 220] /P 3 0 R /F 4 /AA << /E {} /X {} /D {} /U {} /Fo {} /Bl {} >> >>", + add(), + add(), + "<< /S /JavaScript /JS (getField\\('log'\\).value += 'D;'; this.print\\(\\);) >>", + add(), + add(), + "<< /S /JavaScript /JS (getField\\('log'\\).value += 'Bl;'; this.print\\(\\);) >>", + ), + ]; + pdf_of(&objs) +} + +/// Pointer and focus triggers run the widget's JavaScript with the event names Acrobat uses, +/// each as one undoable edit; a field without such a script runs nothing. +#[test] +fn pointer_and_focus_triggers_run_the_widget_scripts() { + use crate::js::FieldFocus; + use pdfcraft_forms::Trigger; + let mut s = Session::new(); + let id = s.open("hot.pdf", None, Arc::new(triggers_form()), None).unwrap(); + assert!(s.field_trigger(id, "hot", 0, Trigger::MouseEnter).unwrap()); + assert!(s.field_focus(id, "hot", 0, FieldFocus::Enter).unwrap()); + assert!(s.field_trigger(id, "hot", 0, Trigger::MouseUp).unwrap()); + assert!(s.field_trigger(id, "hot", 0, Trigger::MouseExit).unwrap()); + assert_eq!(log(&s, id), "Mouse Enter:hot;Focus:hot;Mouse Up:hot;Mouse Exit:hot;"); + assert_eq!(s.get(id).unwrap().can_undo(), Some("Run JavaScript")); + assert!(!s.field_trigger(id, "log", 0, Trigger::MouseEnter).unwrap(), "no script: nothing runs"); + assert!(!s.field_trigger(id, "hot", 5, Trigger::MouseEnter).unwrap(), "no such widget: nothing runs"); + assert!(s.field_trigger(id, "missing", 0, Trigger::MouseEnter).is_err()); + s.set_javascript(false); + assert!(!s.field_trigger(id, "hot", 0, Trigger::MouseEnter).unwrap(), "JavaScript off"); +} + +/// A click's Mouse Down may ask to print; losing the focus, which happens unasked, may not. +#[test] +fn only_a_click_trigger_may_print() { + use crate::js::{FieldFocus, Request}; + use pdfcraft_forms::Trigger; + let mut s = Session::new(); + let id = s.open("hot.pdf", None, Arc::new(triggers_form()), None).unwrap(); + s.field_trigger(id, "hot", 0, Trigger::MouseDown).unwrap(); + assert_eq!(s.take_js_output(id).requests, [Request::Print]); + s.field_focus(id, "hot", 0, FieldFocus::Exit).unwrap(); + let out = s.take_js_output(id); + assert!(out.requests.is_empty(), "{out:?}"); + assert_eq!(log(&s, id), "D;Bl;"); +} + +/// `setItems` and `setAction` are written to the document: the combo box gets its new items +/// (pairs where text and export value differ), the installed scripts run on their triggers, and +/// both survive a save (#269, from #344). +#[test] +fn set_items_and_set_action_are_written_to_the_document() { + use pdfcraft_forms::Trigger; + let objs = vec![ + "<< /Type /Catalog /Pages 2 0 R /AcroForm << /Fields [4 0 R 6 0 R] /DA (/Helv 0 Tf 0 g) /DR << /Font << /Helv 5 0 R >> >> >> >>".to_string(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 200 300] >>".into(), + "<< /Type /Page /Parent 2 0 R /Annots [4 0 R 6 0 R] >>".into(), + "<< /Type /Annot /Subtype /Widget /FT /Tx /T (log) /V () /Rect [20 250 180 270] /P 3 0 R /F 4 >>".into(), + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>".into(), + "<< /Type /Annot /Subtype /Widget /FT /Ch /Ff 131072 /T (size) /Opt [(Small) (Large)] /V (Large) /Rect [20 200 180 220] /P 3 0 R /F 4 >>" + .into(), + ]; + let mut s = Session::new(); + let id = s.open("items.pdf", None, Arc::new(pdf_of(&objs)), None).unwrap(); + let out = s + .run_javascript( + id, + "var f = getField('size'); f.setItems([['Medium', 'm'], 'Large']); \ + f.setAction('MouseEnter', \"getField('log').value = 'entered ' + event.target.value;\");", + None, + ) + .unwrap(); + assert_eq!(out.error, None, "{out:?}"); + let size = s.get(id).unwrap().form.iter().find(|f| f.name == "size").unwrap().clone(); + assert_eq!(size.options, [("m".to_string(), "Medium".to_string()), ("Large".into(), "Large".into())]); + assert_eq!(size.value, ["Large"], "the value is still an item: it stays"); + assert!(s.field_trigger(id, "size", 0, Trigger::MouseEnter).unwrap()); + assert_eq!(log(&s, id), "entered Large"); + s.undo(id).unwrap(); + s.undo(id).unwrap(); + assert_eq!(s.get(id).unwrap().form.iter().find(|f| f.name == "size").unwrap().options[0].1, "Small", "undone"); + s.redo(id).unwrap(); + let saved = s.save_bytes(id).unwrap(); + let mut again = Session::new(); + let reopened = again.open("saved.pdf", None, saved, None).unwrap(); + let size = again.get(reopened).unwrap().form.iter().find(|f| f.name == "size").unwrap().clone(); + assert_eq!(size.options[0], ("m".to_string(), "Medium".to_string())); + assert!(again.field_trigger(reopened, "size", 0, Trigger::MouseEnter).unwrap(), "the installed script is in the file"); +} + +/// A check box's Mouse Up, run after the click stored the new value, reads that value and can +/// show or hide another field by it (#247). +#[test] +fn a_check_box_mouse_up_reads_the_value_its_click_stored() { + use crate::{Edit, FieldValue}; + use pdfcraft_forms::Trigger; + let objs = vec![ + "<< /Type /Catalog /Pages 2 0 R /AcroForm << /Fields [4 0 R 6 0 R 7 0 R] /DA (/Helv 0 Tf 0 g) /DR << /Font << /Helv 5 0 R >> >> >> >>".to_string(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 200 300] >>".into(), + "<< /Type /Page /Parent 2 0 R /Annots [4 0 R 6 0 R 7 0 R] >>".into(), + "<< /Type /Annot /Subtype /Widget /FT /Tx /T (log) /V () /Rect [20 250 180 270] /P 3 0 R /F 4 >>".into(), + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>".into(), + "<< /Type /Annot /Subtype /Widget /FT /Btn /T (strike) /V /Off /AS /Off /Rect [20 150 50 180] /P 3 0 R /F 4 /MK << /CA (4) >> /AP << /N << /Yes 8 0 R /Off 8 0 R >> >> /A << /S /JavaScript /JS (getField\\('note'\\).display = event.target.value == 'Off' ? display.hidden : display.visible; getField\\('log'\\).value += 'up:' + event.target.value + ';';) >> >>".into(), + "<< /Type /Annot /Subtype /Widget /FT /Tx /T (note) /V (struck) /Rect [60 150 180 180] /P 3 0 R /F 6 >>".into(), + "<< /Type /XObject /Subtype /Form /BBox [0 0 30 30] /Length 0 >>\nstream\n\nendstream".into(), + ]; + let mut s = Session::new(); + let id = s.open("strike.pdf", None, Arc::new(pdf_of(&objs)), None).unwrap(); + s.apply(id, Edit::SetFieldValue { name: "strike".into(), value: FieldValue::Text("Yes".into()) }).unwrap(); + let ran = s.field_trigger(id, "strike", 0, Trigger::MouseUp); + let out = s.take_js_output(id); + assert!(ran.unwrap(), "{out:?}"); + assert_eq!(log(&s, id), "up:Yes;"); + let note = |s: &Session| s.get(id).unwrap().form.iter().find(|f| f.name == "note").unwrap().widgets[0].hidden; + assert!(!note(&s), "shown by the script"); + s.apply(id, Edit::SetFieldValue { name: "strike".into(), value: FieldValue::Check(false) }).unwrap(); + assert!(s.field_trigger(id, "strike", 0, Trigger::MouseUp).unwrap()); + assert_eq!(log(&s, id), "up:Yes;up:Off;"); + assert!(note(&s), "hidden again"); +} + +/// A widget action's `/Next` chain runs too, each JavaScript action in order (depth first), past +/// an action of another type at its head. +#[test] +fn a_widget_actions_next_chain_runs_each_script_in_order() { + use pdfcraft_forms::Trigger; + let js = |text: &str, next: &str| format!("<< /S /JavaScript /JS (getField\\('log'\\).value += '{text}';) {next} >>"); + let chain = format!("<< /S /URI /URI (https://example.com/) /Next [{} {}] >>", js("a;", &format!("/Next {}", js("b;", ""))), js("c;", "")); + let objs = vec![ + "<< /Type /Catalog /Pages 2 0 R /AcroForm << /Fields [4 0 R 6 0 R] /DA (/Helv 0 Tf 0 g) /DR << /Font << /Helv 5 0 R >> >> >> >>".to_string(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 200 300] >>".into(), + "<< /Type /Page /Parent 2 0 R /Annots [4 0 R 6 0 R] >>".into(), + "<< /Type /Annot /Subtype /Widget /FT /Tx /T (log) /V () /Rect [20 250 180 270] /P 3 0 R /F 4 >>".into(), + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>".into(), + format!("<< /Type /Annot /Subtype /Widget /FT /Tx /T (hot) /V () /Rect [20 200 180 220] /P 3 0 R /F 4 /AA << /E {chain} >> >>"), + ]; + let mut s = Session::new(); + let id = s.open("chain.pdf", None, Arc::new(pdf_of(&objs)), None).unwrap(); + assert!(s.field_trigger(id, "hot", 0, Trigger::MouseEnter).unwrap()); + assert_eq!(log(&s, id), "a;b;c;"); +} + +/// `setAction` on a field that shares its name with another installs the script on that field, +/// the one the script's object stands for, not the first of the name. +#[test] +fn set_action_installs_on_the_field_the_object_stands_for() { + let objs = vec![ + "<< /Type /Catalog /Pages 2 0 R /AcroForm << /Fields [4 0 R 6 0 R] /DA (/Helv 0 Tf 0 g) /DR << /Font << /Helv 5 0 R >> >> >> >>".to_string(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 200 300] >>".into(), + "<< /Type /Page /Parent 2 0 R /Annots [4 0 R 6 0 R] >>".into(), + "<< /Type /Annot /Subtype /Widget /FT /Tx /T (dup) /V () /Rect [20 250 180 270] /P 3 0 R /F 4 >>".into(), + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>".into(), + "<< /Type /Annot /Subtype /Widget /FT /Tx /T (dup) /V () /Rect [20 200 180 220] /P 3 0 R /F 4 >>".into(), + ]; + let mut s = Session::new(); + let id = s.open("dup.pdf", None, Arc::new(pdf_of(&objs)), None).unwrap(); + let second = crate::js::field_object_id(pdfcraft_cos::ObjRef::new(6, 0)); + let out = + s.run_javascript_answers_target(id, "event.target.setAction('Format', \"event.value = 'x';\");", Some("dup"), Some(second), &[]).unwrap(); + assert_eq!(out.error, None, "{out:?}"); + let cos = s.get(id).unwrap().editor.as_ref().unwrap().cos.clone(); + let formats: Vec<(u32, bool)> = pdfcraft_forms::fields(&cos).iter().map(|f| (f.obj.num, f.actions.scripts.format.is_some())).collect(); + assert_eq!(formats, [(4, false), (6, true)]); +} + +/// Of two fields sharing a name, the one the pointer is on runs its own widget script. +#[test] +fn a_trigger_on_a_field_sharing_a_name_runs_that_fields_script() { + use pdfcraft_forms::Trigger; + let enter = |text: &str| format!("/AA << /E << /S /JavaScript /JS (getField\\('log'\\).value += '{text}';) >> >>"); + let objs = vec![ + "<< /Type /Catalog /Pages 2 0 R /AcroForm << /Fields [4 0 R 6 0 R 7 0 R] /DA (/Helv 0 Tf 0 g) /DR << /Font << /Helv 5 0 R >> >> >> >>" + .to_string(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 200 300] >>".into(), + "<< /Type /Page /Parent 2 0 R /Annots [4 0 R 6 0 R 7 0 R] >>".into(), + "<< /Type /Annot /Subtype /Widget /FT /Tx /T (log) /V () /Rect [20 250 180 270] /P 3 0 R /F 4 >>".into(), + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>".into(), + format!("<< /Type /Annot /Subtype /Widget /FT /Tx /T (dup) /V () /Rect [20 200 180 220] /P 3 0 R /F 4 {} >>", enter("first;")), + format!("<< /Type /Annot /Subtype /Widget /FT /Tx /T (dup) /V () /Rect [20 150 180 170] /P 3 0 R /F 4 {} >>", enter("second;")), + ]; + let mut s = Session::new(); + let id = s.open("dup.pdf", None, Arc::new(pdf_of(&objs)), None).unwrap(); + let second = crate::js::field_object_id(pdfcraft_cos::ObjRef::new(7, 0)); + assert!(s.field_trigger_target(id, "dup", Some(second), 0, Trigger::MouseEnter).unwrap()); + assert_eq!(log(&s, id), "second;"); + assert!(s.field_trigger(id, "dup", 0, Trigger::MouseEnter).unwrap(), "by name alone, the first"); + assert_eq!(log(&s, id), "second;first;"); +} + +/// Focus too: of two fields sharing a name, the one that gained focus runs its On Focus script. +#[test] +fn focus_on_a_field_sharing_a_name_runs_that_fields_script() { + use crate::js::FieldFocus; + let focus = |text: &str| format!("/AA << /Fo << /S /JavaScript /JS (getField\\('log'\\).value += '{text}';) >> >>"); + let objs = vec![ + "<< /Type /Catalog /Pages 2 0 R /AcroForm << /Fields [4 0 R 6 0 R 7 0 R] /DA (/Helv 0 Tf 0 g) /DR << /Font << /Helv 5 0 R >> >> >> >>" + .to_string(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 200 300] >>".into(), + "<< /Type /Page /Parent 2 0 R /Annots [4 0 R 6 0 R 7 0 R] >>".into(), + "<< /Type /Annot /Subtype /Widget /FT /Tx /T (log) /V () /Rect [20 250 180 270] /P 3 0 R /F 4 >>".into(), + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>".into(), + format!("<< /Type /Annot /Subtype /Widget /FT /Tx /T (dup) /V () /Rect [20 200 180 220] /P 3 0 R /F 4 {} >>", focus("first;")), + format!("<< /Type /Annot /Subtype /Widget /FT /Tx /T (dup) /V () /Rect [20 150 180 170] /P 3 0 R /F 4 {} >>", focus("second;")), + ]; + let mut s = Session::new(); + let id = s.open("dup.pdf", None, Arc::new(pdf_of(&objs)), None).unwrap(); + let second = crate::js::field_object_id(pdfcraft_cos::ObjRef::new(7, 0)); + assert!(s.field_focus_target(id, "dup", Some(second), 0, FieldFocus::Enter).unwrap()); + assert_eq!(log(&s, id), "second;"); +} diff --git a/crates/engine/src/js.rs b/crates/engine/src/js.rs index cb4c0caf65..bde0d5a02c 100644 --- a/crates/engine/src/js.rs +++ b/crates/engine/src/js.rs @@ -408,6 +408,8 @@ fn changes( text_color: a.text_color.as_deref().filter(|_| a.text_color != b.text_color).map(<[_]>::to_vec), fill_color: a.fill_color.as_deref().filter(|_| a.fill_color != b.fill_color).map(<[_]>::to_vec), caption: (a.caption != b.caption || (a.caption_written && b.caption_mixed)).then(|| a.caption.clone()).flatten(), + options: (a.options != b.options).then(|| a.options.clone()), + actions: a.actions.clone(), }; (c.value.is_some() || c.read_only.is_some() @@ -422,7 +424,9 @@ fn changes( || c.line_width.is_some() || c.border_style.is_some() || c.user_name.is_some() - || c.icon.is_some()) + || c.icon.is_some() + || c.options.is_some() + || !c.actions.is_empty()) .then_some(c) }) .collect() @@ -827,6 +831,23 @@ impl DocumentScriptEvent { Self::WillPrint => "WillPrint", } } + + /// Where the catalog keeps this event's actions. + fn trigger(self) -> pdfcraft_forms::DocumentTrigger { + match self { + Self::CatalogOpen => pdfcraft_forms::DocumentTrigger::Open, + Self::WillSave => pdfcraft_forms::DocumentTrigger::WillSave, + Self::WillPrint => pdfcraft_forms::DocumentTrigger::WillPrint, + } + } +} + +/// Keep only what a script that ran unasked (on opening, before saving or printing) may ask +/// the viewer for: highlighting, zoom, focus and the page shown. Opening a URL, a file, mail or +/// a dialog needs a click. +fn unasked_requests(requests: &mut Vec) { + requests + .retain(|r| matches!(r, Request::Highlight(_) | Request::Zoom(_) | Request::Focus(_) | Request::FocusField { .. } | Request::GoToPage(_))); } impl Session { @@ -878,13 +899,33 @@ impl Session { /// change runs it) isn't run a second time. Returns whether the document changed; what the /// scripts showed or asked for is in [`Session::take_js_output`]. pub fn field_focus(&mut self, id: DocId, name: &str, widget: usize, focus: FieldFocus) -> Result { + self.field_focus_target(id, name, None, widget, focus) + } + + /// [`Session::field_focus`] for the field object `target` ([`field_object_id`]) when given: + /// of fields sharing `name`, the one that gained or lost focus runs its own script. + pub fn field_focus_target(&mut self, id: DocId, name: &str, target: Option, widget: usize, focus: FieldFocus) -> Result { if self.js_off { return Ok(false); } let doc = self.doc_mut(id)?; - let Some(tpl) = doc.xfa_template.clone().filter(|t| pdfcraft_xfa::has_focus_scripts(t)) else { return Ok(false) }; + let Some(tpl) = doc.xfa_template.clone().filter(|t| pdfcraft_xfa::has_focus_scripts(t)) else { + // An AcroForm widget's own On Focus / On Blur JavaScript (#269). + let trigger = match focus { + FieldFocus::Enter => pdfcraft_forms::Trigger::OnFocus, + FieldFocus::Exit => pdfcraft_forms::Trigger::OnBlur, + }; + return self.field_trigger_target(id, name, target, widget, trigger); + }; let Some(cos) = doc.editor.as_ref().map(|e| &e.cos) else { return Ok(false) }; - let Some(som) = doc.form.iter().find(|f| f.name == name).and_then(|f| crate::xfa::widget_som(cos, f, widget)) else { return Ok(false) }; + let Some(som) = doc + .form + .iter() + .find(|f| f.name == name && target.is_none_or(|t| field_object_id(f.obj) == t)) + .and_then(|f| crate::xfa::widget_som(cos, f, widget)) + else { + return Ok(false); + }; let activity = match focus { FieldFocus::Enter => "enter", FieldFocus::Exit => "exit", @@ -902,6 +943,85 @@ impl Session { Ok(self.get(id).is_some_and(|d| d.generation != before)) } + /// Widget `widget` (0-based) of AcroForm field `name` in document `id` saw `trigger`: the + /// pointer entered, left, pressed or released it, or it gained or lost the focus. Run that + /// widget's JavaScript action for the trigger, if it has one, in the document's realm, with + /// `event.type` "Field" and `event.name` "Mouse Enter", "Mouse Exit", "Mouse Down", "Mouse + /// Up", "Focus" or "Blur" (#269, ported from #344), then those of the actions chained after + /// it (`/Next`). What each action changes is one undoable edit. + /// Returns whether the document changed; what the script showed or asked for is in + /// [`Session::take_js_output`]. Pointer and focus scripts run unasked, so they may not + /// print, open Save As or show a panel; Mouse Down and Mouse Up come from a click and may. + /// Nothing runs on an XFA form (its own events run instead), with JavaScript off, or for a + /// widget with no JavaScript for the trigger. + pub fn field_trigger(&mut self, id: DocId, name: &str, widget: usize, trigger: pdfcraft_forms::Trigger) -> Result { + self.field_trigger_target(id, name, None, widget, trigger) + } + + /// [`Session::field_trigger`] for the field object `target` ([`field_object_id`]) when given: + /// of fields sharing `name`, the one the pointer is on runs its own script. + pub fn field_trigger_target( + &mut self, + id: DocId, + name: &str, + target: Option, + widget: usize, + trigger: pdfcraft_forms::Trigger, + ) -> Result { + use pdfcraft_forms::Trigger; + if self.js_off { + return Ok(false); + } + let doc = self.get(id).ok_or(EditError::NoDocument)?; + if doc.info.xfa.is_some() { + return Ok(false); + } + let Some(cos) = doc.editor.as_ref().map(|e| &e.cos) else { return Ok(false) }; + let field = doc + .form + .iter() + .find(|f| f.name == name && target.is_none_or(|t| field_object_id(f.obj) == t)) + .ok_or_else(|| EditError::Invalid(format!("there is no field named {name:?}")))?; + let Some(w) = field.widgets.get(widget) else { return Ok(false) }; + let scripts = pdfcraft_forms::widget_scripts(cos, w.obj, trigger); + if scripts.is_empty() { + return Ok(false); + } + let target_id = Some(field_object_id(field.obj)); + let event = match trigger { + Trigger::OnFocus => "Focus", + Trigger::OnBlur => "Blur", + other => other.label(), + }; + let before = doc.generation; + let mut out = JsOutput::default(); + // Each action of a /Next chain runs on its own, as Acrobat runs them: one that fails + // doesn't stop the next. + for script in scripts { + let o = match self.run_javascript_job(id, &script, (Some(name), target_id), &[], Some(event), false) { + Ok(o) => o, + Err(e) => { + out.errors.push(e.to_string()); + continue; + } + }; + if matches!(trigger, Trigger::MouseUp | Trigger::MouseDown) { + out.append(JsOutput { + alerts: o.alerts, + console: o.console, + requests: o.requests, + errors: o.error.into_iter().chain(o.doc_errors.iter().cloned()).collect(), + doc_errors: o.doc_errors, + }); + } else { + out.absorb(&o); + } + } + let doc = self.doc_mut(id)?; + doc.js_output.append(out); + Ok(doc.generation != before) + } + /// Run signature field `field`'s XFA signing scripts for `event` in document `id`: `preSign` /// before it is signed, `postSign` after, as one undoable step, as a form's signing events run /// (DA 31's dates the signature and locks the date). `Ok(false)`, with nothing changed, when @@ -970,9 +1090,66 @@ impl Session { self.run_javascript_job(id, "", (None, None), &[], None, true) } + /// Opening document `id`: its document-level scripts run once, then the catalog + /// `/OpenAction`'s JavaScript, in the document's retained realm (#269). Not on an XFA form + /// (its own opening scripts run instead) or with JavaScript off. What they showed or asked + /// for waits in [`Session::take_js_output`]. + pub(crate) fn run_open_scripts(&mut self, id: DocId) { + if self.js_off { + return; + } + let Some(doc) = self.get(id).filter(|d| d.info.xfa.is_none()) else { return }; + let has_document_scripts = !doc.document_scripts().is_empty(); + let opens = + doc.editor.as_ref().map(|e| pdfcraft_forms::document_action_scripts(&e.cos, pdfcraft_forms::DocumentTrigger::Open)).unwrap_or_default(); + if !has_document_scripts && opens.is_empty() { + return; + } + let mut out = JsOutput::default(); + if has_document_scripts && opens.is_empty() { + match self.run_opening_javascript(id) { + Ok(o) => out.absorb(&o), + Err(e) => out.errors.push(e.to_string()), + } + } + // The first open action initializes the realm with the document scripts too. + out.append(self.run_actions(id, DocumentScriptEvent::CatalogOpen, &opens)); + unasked_requests(&mut out.requests); + if let Some(doc) = self.docs.iter_mut().find(|d| d.id == id) { + doc.js_output.append(out); + } + } + + /// Run the catalog's JavaScript actions for document event `event` (`/AA /WS` before a save, + /// `/AA /WP` before printing, `/OpenAction` on opening) in document `id`'s retained realm. + /// Each action's field changes are one undoable edit. A failing action is reported in the + /// output and the next one still runs; it never stops the save or print, which must not + /// lose the user's work. Requests are limited to what an unasked script may make. With + /// JavaScript off, on an XFA form, or with no such action, nothing runs. + pub fn run_document_actions(&mut self, id: DocId, event: DocumentScriptEvent) -> JsOutput { + if self.js_off { + return JsOutput::default(); + } + let Some(doc) = self.get(id).filter(|d| d.info.xfa.is_none()) else { return JsOutput::default() }; + let scripts = doc.editor.as_ref().map(|e| pdfcraft_forms::document_action_scripts(&e.cos, event.trigger())).unwrap_or_default(); + self.run_actions(id, event, &scripts) + } + + fn run_actions(&mut self, id: DocId, event: DocumentScriptEvent, scripts: &[String]) -> JsOutput { + let mut out = JsOutput::default(); + for script in scripts { + match self.run_javascript_job(id, script, (None, None), &[], Some(event.name()), event == DocumentScriptEvent::CatalogOpen) { + Ok(o) => out.absorb(&o), + Err(e) => out.errors.push(format!("{} script: {e}", event.name())), + } + } + unasked_requests(&mut out.requests); + out + } + /// Run an explicitly dispatched PDF document action in the document's retained runtime. pub fn run_document_script_event(&mut self, id: DocId, event: DocumentScriptEvent, script: &str) -> Result { - self.run_javascript_job(id, script, (None, None), &[], Some(event), false) + self.run_javascript_job(id, script, (None, None), &[], Some(event.name()), false) } /// Route generated XFA clicks to the XFA engine, or run a synchronous AcroForm job and apply its outcome. @@ -982,7 +1159,7 @@ impl Session { script: &str, (target, target_id): (Option<&str>, Option), answers: &[Option], - document_event: Option, + event_name: Option<&'static str>, opening: bool, ) -> Result { if self.js_off { @@ -993,6 +1170,7 @@ impl Session { validate_script_target(target, target_id, &doc.form)?; // A button PdfCraft generated from an XFA template runs its XFA click script. if let Some(t) = target + && event_name.is_none_or(|n| n == "Mouse Up") && let Some(tpl) = doc.xfa_template.clone() && let Some(field) = doc.form.iter().find(|f| f.name == t && target_id.is_none_or(|id| field_object_id(f.obj) == id)) && field.kind == crate::FormFieldKind::PushButton @@ -1015,8 +1193,8 @@ impl Session { let fields = pdfcraft_forms::fields(&cos); let states = std::sync::Arc::new(fields.iter().map(|f| field_state(&cos, f, &runner.doc)).collect::>()); let event = match target { - Some(t) => pdfcraft_js::Event { will_commit: false, target_id, ..pdfcraft_js::Event::field("Mouse Up", t, "") }, - None => pdfcraft_js::Event::doc(document_event.map_or("Console", DocumentScriptEvent::name)), + Some(t) => pdfcraft_js::Event { will_commit: false, target_id, ..pdfcraft_js::Event::field(event_name.unwrap_or("Mouse Up"), t, "") }, + None => pdfcraft_js::Event::doc(event_name.unwrap_or("Console")), }; let calculations = calculation_scripts(&cos, &fields); let runtime = doc.js_runtime.clone(); diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index 62ea42cd54..adb69cec46 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -200,5 +200,7 @@ fn display_config(config: &pdfcraft_render::RenderConfig) -> pdfcraft_render::Re /// How long a custom stamp's text line may be, in characters (#1741). pub const MAX_STAMP_LINE: usize = 200; +#[cfg(test)] +mod document_actions_tests; #[cfg(test)] mod tests; diff --git a/crates/engine/src/open.rs b/crates/engine/src/open.rs index 8fd5cd992e..685ac50432 100644 --- a/crates/engine/src/open.rs +++ b/crates/engine/src/open.rs @@ -101,35 +101,7 @@ impl Session { Self::draw_captions(&mut document, self.captions.clone()); } self.docs.push(document); - if !self.js_off && self.get(id).is_some_and(|d| d.info.xfa.is_none() && !d.document_scripts().is_empty()) { - match self.run_opening_javascript(id) { - Ok(mut out) => { - out.requests.retain(|r| { - matches!( - r, - js::Request::Highlight(_) - | js::Request::Zoom(_) - | js::Request::Focus(_) - | js::Request::FocusField { .. } - | js::Request::GoToPage(_) - ) - }); - if let Some(doc) = self.docs.iter_mut().find(|d| d.id == id) { - doc.js_output.requests.extend(out.requests); - doc.js_output.alerts.extend(out.alerts); - doc.js_output.console.extend(out.console); - doc.js_output.errors.extend(out.error); - doc.js_output.doc_errors.extend(out.doc_errors.iter().cloned()); - doc.js_output.errors.extend(out.doc_errors); - } - } - Err(e) => { - if let Some(doc) = self.docs.iter_mut().find(|d| d.id == id) { - doc.js_output.errors.push(e.to_string()); - } - } - } - } + self.run_open_scripts(id); id } diff --git a/crates/engine/src/session/save.rs b/crates/engine/src/session/save.rs index d8193ba084..ec0033d318 100644 --- a/crates/engine/src/session/save.rs +++ b/crates/engine/src/session/save.rs @@ -132,7 +132,7 @@ impl Session { } /// File ▸ Revert: back to the last saved version (or the file as opened). Undo history is - /// cleared, as in Acrobat. + /// cleared, as in Acrobat, and the document's opening scripts run again. pub fn revert(&mut self, id: DocId) -> Result<(), EditError> { self.invalidate_javascript(id); let doc = self.doc_mut(id)?; @@ -156,7 +156,11 @@ impl Session { // The changes whose exits ran are gone with the rest: skip no exit. doc.xfa_exited.clear(); doc.generation += 1; - Self::refresh(doc).inspect_err(|_| Self::show_released(doc)) + Self::refresh(doc).inspect_err(|_| Self::show_released(doc))?; + // Reverting reopens the saved file, in a fresh realm: its opening scripts run again, + // as they do on opening it (#269). + self.run_open_scripts(id); + Ok(()) } /// The document's file changed on disk (a LaTeX or Typst build rewrote it, #431): open diff --git a/crates/forms/src/actions.rs b/crates/forms/src/actions.rs index 6a91220713..96d03e1869 100644 --- a/crates/forms/src/actions.rs +++ b/crates/forms/src/actions.rs @@ -128,6 +128,21 @@ fn read(doc: &Document, o: &Object, pages: &[pdfcraft_cos::ObjRef], scripts: &mu }) } +/// The JavaScript widget annotation `widget` runs for `trigger` (ISO 32000-2 Table 197: `/A` +/// or `/AA /U` for Mouse Up, the other `/AA` entries for the rest): that action's, when it is a +/// JavaScript action, then those of the actions chained after it with `/Next`, in the order +/// they run (bounded as [`crate::document_action_scripts`] is). Reads only the widget's own +/// dictionary, so a pointer moving over a large form costs nothing per field. +pub fn widget_scripts(doc: &Document, widget: pdfcraft_cos::ObjRef, trigger: Trigger) -> Vec { + let w = doc.get(widget); + let Some(wd) = w.as_dict() else { return Vec::new() }; + let action = match trigger.key() { + None => wd.get(b"A").cloned().or_else(|| wd.get(b"AA").map(|a| doc.resolve(a)).and_then(|a| a.as_dict()?.get(b"U").cloned())), + Some(k) => wd.get(b"AA").map(|a| doc.resolve(a)).and_then(|a| a.as_dict()?.get(k).cloned()), + }; + crate::scripting::chained_scripts(doc, action) +} + /// The actions of field `name` (its first widget), by trigger, in the tab's order. pub fn field_actions(doc: &Document, name: &str) -> Result, FormError> { let all = fields(doc); diff --git a/crates/forms/src/lib.rs b/crates/forms/src/lib.rs index 89d7e2998f..414132b42b 100644 --- a/crates/forms/src/lib.rs +++ b/crates/forms/src/lib.rs @@ -24,15 +24,15 @@ mod calc_order; pub mod detect; mod scripting; pub mod table; -pub use actions::{FieldAction, Trigger, field_action_chains, field_actions, set_field_actions}; +pub use actions::{FieldAction, Trigger, field_action_chains, field_actions, set_field_actions, widget_scripts}; pub use author::{ BorderStyle, CheckStyle, FieldFont, FieldProps, Look, LookPatch, MIN_FIELD_SIDE, NewField, add_field, check_style, delete_field, delete_field_object, delete_widget, duplicate_field, look, redraw_field, redraw_fields, set_button_icon, set_button_icon_target, set_props, }; pub use calc_order::{CalculatedField, CalculationIssue, CalculationOrder, calculation_order, set_calculation_order}; pub use scripting::{ - FieldChange, FieldEvent, NoScripts, ScriptResult, Scripts, apply_script_changes, document_scripts, document_scripts_named, set_document_script, - set_field_script, + DocumentTrigger, FieldChange, FieldEvent, NoScripts, ScriptResult, Scripts, apply_script_changes, document_action_scripts, document_scripts, + document_scripts_named, set_document_script, set_field_script, }; mod appearances; diff --git a/crates/forms/src/scripting.rs b/crates/forms/src/scripting.rs index 3d66dd0918..8ce3f1f68a 100644 --- a/crates/forms/src/scripting.rs +++ b/crates/forms/src/scripting.rs @@ -57,6 +57,11 @@ pub struct FieldChange { pub border_style: Option, pub user_name: Option, pub icon: Option>, + /// A combo box's or list box's new items (export value, text), from `setItems`. + pub options: Option>, + /// JavaScript actions the script installed with `setAction`, by trigger name (`MouseUp`, + /// `OnFocus`, `Keystroke`, …), each replacing what the field ran for that trigger. + pub actions: Vec<(String, String)>, } /// What a script did. @@ -171,6 +176,73 @@ fn protect_other_mk_owners(doc: &mut Document, field: &Field, owners: &std::coll Ok(()) } +/// Most items a script may give a choice field ([`FieldChange::options`]). +const MAX_SCRIPT_OPTIONS: usize = 10_000; + +/// Write a combo box's or list box's items (export value, text) as its `/Opt`: a plain text +/// where the two are the same, a pair where they differ (ISO 32000-2 Table 231). The selected +/// indices (`/I`) no longer apply and go; the value is written after, and the field redrawn. +fn write_options(doc: &mut Document, f: &Field, options: &[(String, String)]) -> Result<(), FormError> { + if !matches!(f.kind, FieldKind::Combo | FieldKind::List) { + return Err(FormError::Invalid(format!("{} is not a combo box or list box: it has no items to set", f.name))); + } + if options.len() > MAX_SCRIPT_OPTIONS { + return Err(FormError::Invalid(format!("a script may give a field at most {MAX_SCRIPT_OPTIONS} items"))); + } + let opt = options + .iter() + .map(|(export, text)| { + if export == text { + Object::String(PdfString::text(text)) + } else { + Object::Array(vec![Object::String(PdfString::text(export)), Object::String(PdfString::text(text))]) + } + }) + .collect(); + doc.update_dict(f.obj, |d| { + d.set(b"Opt".to_vec(), Object::Array(opt)); + d.remove(b"I"); + })?; + crate::author::redraw_field_object(doc, &f.name, f.obj) +} + +/// Install `script` as field `f`'s JavaScript for `trigger` (a `setAction` trigger name), +/// replacing the action it had there: the field's `/AA` for Keystroke, Validate, Calculate and +/// Format, each widget's `/A` (Mouse Up) or `/AA` for the rest. +fn set_action(doc: &mut Document, f: &Field, trigger: &str, script: &str) -> Result<(), FormError> { + let event = match trigger { + "Keystroke" => Some("keystroke"), + "Validate" => Some("validate"), + "Calculate" => Some("calculate"), + "Format" => Some("format"), + "MouseUp" => Some("mouse_up"), + _ => None, + }; + if let Some(event) = event { + // The field the script's object stands for, even when another shares its name. + return set_script_of(doc, f, event, Some(script)); + } + let key: &[u8] = match trigger { + "MouseDown" => b"D", + "MouseEnter" => b"E", + "MouseExit" => b"X", + "OnFocus" => b"Fo", + "OnBlur" => b"Bl", + other => return Err(FormError::Invalid(format!("unknown action trigger {other:?}"))), + }; + let mut action = pdfcraft_cos::Dict::new(); + action.set(b"S".to_vec(), Object::name("JavaScript")); + action.set(b"JS".to_vec(), PdfString::text(script)); + let action = doc.add(Object::Dict(action)); + for w in &f.widgets { + let mut aa = + doc.get(w.obj).as_dict().and_then(|d| d.get(b"AA").map(|a| doc.resolve(a))).and_then(|a| a.as_dict().cloned()).unwrap_or_default(); + aa.set(key.to_vec(), Object::Ref(action)); + doc.update_dict(w.obj, |d| d.set(b"AA".to_vec(), Object::Dict(aa)))?; + } + Ok(()) +} + /// [`formatted`] with dates in English names, for widgets whose fonts can't draw the date /// language's; `None` when the field has no Format script or the English run didn't happen. pub(crate) fn formatted_in_english(doc: &Document, f: &Field, value: &str, scripts: &mut dyn Scripts) -> Option { @@ -211,6 +283,19 @@ pub(crate) fn apply_changes( for c in changes { let Some(f) = all.iter().find(|f| f.name == c.name && c.object.is_none_or(|obj| f.obj == obj)) else { continue }; + // New items go first, so a value among them is written against them. + let reread; + let f = match &c.options { + Some(options) => { + write_options(doc, f, options)?; + reread = fields(doc).into_iter().find(|x| x.obj == f.obj).unwrap_or_else(|| f.clone()); + &reread + } + None => f, + }; + for (trigger, script) in &c.actions { + set_action(doc, f, trigger, script)?; + } if let Some(v) = &c.value && !except_object.map_or(c.name == except, |obj| f.obj == obj) && *v != f.value @@ -454,6 +539,79 @@ pub fn document_scripts_named(doc: &Document) -> Vec<(String, String)> { out } +/// A document-level action trigger whose JavaScript the viewer runs (ISO 32000-2 §12.6.3, +/// Table 199): the catalog's `/OpenAction`, or its `/AA` entries. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum DocumentTrigger { + /// The catalog `/OpenAction`, once the document has opened. + Open, + /// `/AA /WS`, before the document is saved. + WillSave, + /// `/AA /WP`, before the document is printed. + WillPrint, +} + +/// Most actions read for one document trigger, along its `/Next` chain. A chain can be cyclic +/// or arbitrarily long; the rest are ignored. +const MAX_CHAINED_ACTIONS: usize = 256; + +/// The JavaScript of the catalog actions for `trigger`, in the order a viewer runs them: the +/// action, then its `/Next` actions depth first (§12.6.2). Actions of other types (a `/GoTo` +/// open action, say) are skipped; their `/Next` chains are still followed. Bounded: each object +/// is read once and at most [`MAX_CHAINED_ACTIONS`] actions are visited. +pub fn document_action_scripts(doc: &Document, trigger: DocumentTrigger) -> Vec { + let Some(root) = doc.root() else { return Vec::new() }; + let cat = doc.get(root); + let Some(cat) = cat.as_dict() else { return Vec::new() }; + let first = match trigger { + DocumentTrigger::Open => cat.get(b"OpenAction").cloned(), + DocumentTrigger::WillSave | DocumentTrigger::WillPrint => { + let key: &[u8] = if trigger == DocumentTrigger::WillSave { b"WS" } else { b"WP" }; + cat.get(b"AA").map(|a| doc.resolve(a)).and_then(|a| a.as_dict().and_then(|d| d.get(key)).cloned()) + } + }; + chained_scripts(doc, first) +} + +/// The JavaScript of action `first` and of the actions chained after it, in the order they run: +/// the action, then its `/Next` actions depth first (ISO 32000-2 §12.6.2). Actions of other +/// types are skipped; their `/Next` chains are still followed. Bounded: each object is read +/// once and at most [`MAX_CHAINED_ACTIONS`] actions are visited. +pub(crate) fn chained_scripts(doc: &Document, first: Option) -> Vec { + let mut budget = crate::script_budget(); + let mut seen: HashSet = HashSet::new(); + let mut pending: Vec = first.into_iter().collect(); + let mut visited = 0usize; + let mut out = Vec::new(); + while let Some(raw) = pending.pop() { + visited += 1; + if visited > MAX_CHAINED_ACTIONS { + break; + } + if let Some(r) = raw.as_ref() + && !seen.insert(r.num) + { + continue; + } + let action = doc.resolve(&raw); + if let Some(list) = action.as_array() { + // Pushed in reverse so they run in the order listed. + pending.extend(list.iter().take(MAX_CHAINED_ACTIONS).rev().cloned()); + continue; + } + let Some(d) = action.as_dict() else { continue }; + if d.name(b"S") == Some(b"JavaScript".as_slice()) + && let Some(js) = crate::script(doc, &raw, &mut budget) + { + out.push(js); + } + if let Some(next) = d.get(b"Next") { + pending.push(next.clone()); + } + } + out +} + /// Call `f` on the array `o` is, or refers to, read in place: resolving a direct array would /// copy it. fn in_array(doc: &Document, o: &Object, f: impl FnOnce(&[Object])) { @@ -511,7 +669,12 @@ pub fn set_document_script(doc: &mut Document, name: &str, js: Option<&str>) -> /// (its widgets' `/A`). Calculated fields join the calculation order (`/CO`). pub fn set_field_script(doc: &mut Document, name: &str, event: &str, js: Option<&str>) -> Result<(), FormError> { let all = fields(doc); - let f = all.iter().find(|f| f.name == name).ok_or_else(|| FormError::NoSuchField(name.into()))?.clone(); + let f = all.iter().find(|f| f.name == name).ok_or_else(|| FormError::NoSuchField(name.into()))?; + set_script_of(doc, f, event, js) +} + +/// [`set_field_script`] for the field `f` itself, which may share its name with another. +fn set_script_of(doc: &mut Document, f: &Field, event: &str, js: Option<&str>) -> Result<(), FormError> { let action = |doc: &mut Document, js: &str| { let mut a = pdfcraft_cos::Dict::new(); a.set(b"S".to_vec(), Object::name("JavaScript")); diff --git a/crates/js/src/document.rs b/crates/js/src/document.rs index 3bcd371419..f13c9434e9 100644 --- a/crates/js/src/document.rs +++ b/crates/js/src/document.rs @@ -279,7 +279,71 @@ fn field_object_new(ctx: &mut Context, name: &str, id: Option, token: u64) Ok(JsValue::undefined()) } + /// `setAction(cTrigger, cScript)`: install `cScript` as the field's JavaScript for + /// `cTrigger`, replacing what it ran before; saved with the document (#269). + fn set_action(_: &JsValue, args: &[JsValue], name: &FieldKey, c: &mut Context) -> JsResult { + let trigger = text(&arg(args, 0), c)?; + if !SET_ACTION_TRIGGERS.contains(&trigger.as_str()) { + return Err(error(format!("setAction: unknown trigger {trigger:?}; use one of {}", SET_ACTION_TRIGGERS.join(", ")))); + } + let script = text(&arg(args, 1), c)?; + if let Some(why) = crate::run::refuse(&script) { + return Err(error(format!("setAction: {why}"))); + } + write(c, name, |f| { + f.actions.retain(|(t, _)| *t != trigger); + f.actions.push((trigger, script)); + })?; + Ok(JsValue::undefined()) + } + /// `setItems(oArray)`: replace a combo box's or list box's items. Each item is its text, or + /// a `[text, exportValue]` pair. The value stays when an item still has it, and otherwise + /// becomes the first item's (#269). + fn set_items(_: &JsValue, args: &[JsValue], name: &FieldKey, c: &mut Context) -> JsResult { + let f = read(c, name)?; + if !matches!(f.kind, FieldType::ComboBox | FieldType::ListBox) { + return Err(error(format!("setItems: {} is not a combo box or list box", f.name))); + } + let list = arg(args, 0).as_object().filter(JsObject::is_array).ok_or_else(|| error("setItems needs an array of items"))?; + let list = JsArray::from_object(list)?; + let len = list.length(c)?; + if len > MAX_ITEMS as u64 { + return Err(error(format!("setItems: at most {MAX_ITEMS} items"))); + } + let mut items = Vec::new(); + let mut bytes = 0usize; + for i in 0..len { + let item = list.get(i, c)?; + let (label, export) = match item.as_object().filter(JsObject::is_array) { + Some(pair) => { + let pair = JsArray::from_object(pair.clone())?; + let label = text(&pair.get(0, c)?, c)?; + let export = pair.get(1, c)?; + let export = if export.is_undefined() { label.clone() } else { text(&export, c)? }; + (label, export) + } + None => { + let label = text(&item, c)?; + (label.clone(), label) + } + }; + bytes = bytes.saturating_add(label.len()).saturating_add(export.len()); + if bytes > MAX_ITEM_BYTES { + return Err(error("setItems: the items are too long")); + } + items.push((export, label)); + } + write(c, name, |f| { + let kept: Vec = f.value.iter().filter(|v| items.iter().any(|(export, _)| export == *v)).cloned().collect(); + f.value = if kept.is_empty() { items.first().map(|(export, _)| vec![export.clone()]).unwrap_or_default() } else { kept }; + f.options = items; + })?; + Ok(JsValue::undefined()) + } + let rw = Attribute::CONFIGURABLE | Attribute::ENUMERABLE; + let set_action = NativeFunction::from_copy_closure_with_captures(set_action, n.clone()); + let set_items = NativeFunction::from_copy_closure_with_captures(set_items, n.clone()); let get_item_at = NativeFunction::from_copy_closure_with_captures(get_item_at, n.clone()); let is_box_checked = NativeFunction::from_copy_closure_with_captures(is_box_checked, n.clone()); let check_this_box = NativeFunction::from_copy_closure_with_captures(check_this_box, n.clone()); @@ -314,6 +378,8 @@ fn field_object_new(ctx: &mut Context, name: &str, id: Option, token: u64) .function(is_box_checked, js_string!("isBoxChecked"), 1) .function(check_this_box, js_string!("checkThisBox"), 2) .function(set_focus, js_string!("setFocus"), 0) + .function(set_action, js_string!("setAction"), 2) + .function(set_items, js_string!("setItems"), 1) .function(button_get_icon, js_string!("buttonGetIcon"), 1) .function(button_set_icon, js_string!("buttonSetIcon"), 2) .function(button_get_caption, js_string!("buttonGetCaption"), 1) @@ -321,6 +387,14 @@ fn field_object_new(ctx: &mut Context, name: &str, id: Option, token: u64) .build() } +/// The triggers `field.setAction` takes (Acrobat JavaScript reference, Field.setAction). +const SET_ACTION_TRIGGERS: [&str; 10] = + ["MouseUp", "MouseDown", "MouseEnter", "MouseExit", "OnFocus", "OnBlur", "Keystroke", "Validate", "Calculate", "Format"]; +/// Most items one `setItems` call takes, and most bytes of their texts and export values: a +/// script must not grow a field without bound. +const MAX_ITEMS: usize = 10_000; +const MAX_ITEM_BYTES: usize = 1 << 20; + // ── the document (global object) ──────────────────────────────────────────────────────────── pub(crate) fn get_field(_: &JsValue, args: &[JsValue], ctx: &mut Context) -> JsResult { diff --git a/crates/js/src/globals.rs b/crates/js/src/globals.rs index 3dfbf3924e..ef7b95c8c7 100644 --- a/crates/js/src/globals.rs +++ b/crates/js/src/globals.rs @@ -591,6 +591,7 @@ pub(crate) fn install(ctx: &mut Context) -> JsResult<()> { ctx.register_global_property(js_string!("display"), display, rw)?; let color = color_object(ctx); ctx.register_global_property(js_string!("color"), color, rw)?; + crate::source::install(ctx)?; Ok(()) } diff --git a/crates/js/src/lib.rs b/crates/js/src/lib.rs index 6d722c00ae..87bb881e13 100644 --- a/crates/js/src/lib.rs +++ b/crates/js/src/lib.rs @@ -44,6 +44,7 @@ mod host; mod model; mod nesting; mod run; +mod source; #[cfg(test)] pub(crate) use format::PRINTD_SCANNED; diff --git a/crates/js/src/model.rs b/crates/js/src/model.rs index 70bb677d8a..b162007420 100644 --- a/crates/js/src/model.rs +++ b/crates/js/src/model.rs @@ -74,6 +74,11 @@ pub struct FieldState { pub caption_mixed: bool, pub icon_mixed: bool, pub icon_written: bool, + /// JavaScript actions a script installed with `setAction` during this run, by trigger + /// (`MouseUp`, `MouseDown`, `MouseEnter`, `MouseExit`, `OnFocus`, `OnBlur`, `Keystroke`, + /// `Validate`, `Calculate`, `Format`), the last one for each trigger. Empty in a snapshot: + /// the host reads the document's own actions from the PDF. + pub actions: Vec<(String, String)>, } impl FieldState { @@ -108,6 +113,7 @@ impl FieldState { caption_mixed: false, icon_mixed: false, icon_written: false, + actions: Vec::new(), } } diff --git a/crates/js/src/source.rs b/crates/js/src/source.rs new file mode 100644 index 0000000000..deafcb8f18 --- /dev/null +++ b/crates/js/src/source.rs @@ -0,0 +1,145 @@ +//! `toSource()` on objects and arrays, which older forms use to keep state in a field or a +//! global as text (#269, ported from #344). It writes a JavaScript literal of the value; scripts +//! here can't `eval` it back, since strings never compile. Plain data only: strings, numbers, +//! booleans, null, undefined, arrays and objects of them. Functions and cycles are refused, and +//! so is a value past the depth, size or length bounds, rather than writing something that +//! reads as another value. + +use super::host::{error, function, s, text}; +use super::{Context, JsArray, JsObject, JsResult, JsValue, NativeFunction, js_string}; +use boa_engine::property::{PropertyDescriptor, PropertyKey}; + +/// Deepest nesting written. +const MAX_DEPTH: usize = 64; +/// Most values (each array item, property and the value itself) written. +const MAX_VALUES: usize = 10_000; +/// Longest source written, in bytes. +const MAX_BYTES: usize = 1 << 20; + +/// Install `toSource` on `Object.prototype` (arrays inherit it), not enumerable, as in the +/// engines that had it. +pub(crate) fn install(ctx: &mut Context) -> JsResult<()> { + let to_source = function(ctx, NativeFunction::from_fn_ptr(to_source)); + let prototype = ctx.intrinsics().constructors().object().prototype(); + prototype + .insert_property(js_string!("toSource"), PropertyDescriptor::builder().value(to_source).writable(true).configurable(true).enumerable(false)); + Ok(()) +} + +fn to_source(this: &JsValue, _: &[JsValue], ctx: &mut Context) -> JsResult { + let mut writer = Writer { values: MAX_VALUES, bytes: MAX_BYTES, open: Vec::new() }; + let out = writer.value(this, ctx)?; + Ok(s(&out)) +} + +/// What is left of the bounds, and the objects being written (to refuse a cycle). +struct Writer { + values: usize, + bytes: usize, + open: Vec, +} + +impl Writer { + fn value(&mut self, v: &JsValue, ctx: &mut Context) -> JsResult { + self.values = self.values.checked_sub(1).ok_or_else(|| error("toSource: the value has too many parts"))?; + let out = if v.is_callable() { + return Err(error("toSource: functions can't be written")); + } else if let Some(o) = v.as_object() { + self.object(&o, ctx)? + } else if let Some(text) = v.as_string() { + quote(&text.to_std_string_escaped()) + } else if v.is_null() { + "null".into() + } else if v.is_undefined() { + "undefined".into() + } else { + text(v, ctx)? + }; + self.charge(out.len())?; + Ok(out) + } + + fn object(&mut self, o: &JsObject, ctx: &mut Context) -> JsResult { + if self.open.iter().any(|open| open == o) { + return Err(error("toSource: the value refers to itself")); + } + if self.open.len() >= MAX_DEPTH { + return Err(error("toSource: the value is nested too deeply")); + } + if !o.is_array() && !plain(o, ctx) { + return Err(error("toSource: only plain objects, arrays, strings, numbers, booleans, null and undefined can be written")); + } + self.open.push(o.clone()); + let out = if o.is_array() { self.array(o, ctx) } else { self.fields(o, ctx) }; + self.open.pop(); + out + } + + fn array(&mut self, o: &JsObject, ctx: &mut Context) -> JsResult { + let array = JsArray::from_object(o.clone())?; + let len = array.length(ctx)?; + if len > self.values as u64 { + return Err(error("toSource: the array is too long")); + } + let mut items = Vec::new(); + for i in 0..len { + items.push(self.value(&array.get(i, ctx)?, ctx)?); + } + Ok(format!("[{}]", items.join(", "))) + } + + fn fields(&mut self, o: &JsObject, ctx: &mut Context) -> JsResult { + let mut items = Vec::new(); + for key in o.own_property_keys(ctx)? { + let label = match &key { + PropertyKey::Symbol(_) => continue, + PropertyKey::String(k) => k.to_std_string_escaped(), + PropertyKey::Index(i) => i.get().to_string(), + }; + // `"__proto__": v` in a literal sets the prototype rather than an own property. + if label == "__proto__" { + return Err(error("toSource: a property named __proto__ can't be written")); + } + self.charge(label.len())?; + let value = self.value(&o.get(key, ctx)?, ctx)?; + items.push(format!("{}:{value}", quote(&label))); + } + Ok(format!("({{{}}})", items.join(", "))) + } + + /// Count `n` bytes against the length bound. Every part counts, including the parts of what + /// is joined later, so a value repeated many times can't build a long result first. + fn charge(&mut self, n: usize) -> JsResult<()> { + self.bytes = self.bytes.checked_sub(n).ok_or_else(|| error("toSource: the result would be too long"))?; + Ok(()) + } +} + +/// Whether `o` is a plain object: an ordinary one whose prototype is `Object.prototype` (or +/// none), so its own properties are all there is to it. A Date, a RegExp, a boxed value, a +/// proxy or a class instance holds more than a literal of its properties would give back. +fn plain(o: &JsObject, ctx: &Context) -> bool { + let object_prototype = ctx.intrinsics().constructors().object().prototype(); + o.is_ordinary() && o.prototype().is_none_or(|p| p == object_prototype) +} + +/// `text` as a double-quoted JavaScript string literal. +fn quote(text: &str) -> String { + let mut out = String::with_capacity(text.len() + 2); + out.push('"'); + for c in text.chars() { + match c { + '\\' => out.push_str("\\\\"), + '"' => out.push_str("\\\""), + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + '\u{2028}' => out.push_str("\\u2028"), + '\u{2029}' => out.push_str("\\u2029"), + c if c.is_control() => out.push_str(&format!("\\u{:04x}", c as u32)), + c => out.push(c), + } + } + out.push('"'); + out +} diff --git a/crates/js/src/tests.rs b/crates/js/src/tests.rs index c79e609967..1a41afeea8 100644 --- a/crates/js/src/tests.rs +++ b/crates/js/src/tests.rs @@ -2702,3 +2702,76 @@ fn af_simple_calculate_bounds_selector_work_before_expansion() { run("AFSimple_Calculate('SUM',new Array(4096).fill('n'));", &Event::doc("Console"), &Default::default(), &fields, &[], Default::default()); assert!(out.error.as_deref().is_some_and(|e| e.contains("calculation field lookup exceeds"))); } + +/// `toSource` writes plain state as a JavaScript literal of the same value (#269, from #344). +/// Scripts here can't `eval` it back (strings never compile), so the exact text is checked. +#[test] +fn to_source_writes_plain_state_as_a_literal() { + let event = Event::doc("Console"); + let out = go( + "var v = {tab: 2, name: 'a \"q\"\\n', on: true, none: null, list: ['qty', null, undefined, 1.5, [3]]}; \ + [v.toSource(), [1, 'x'].toSource(), ({}).toSource()].join('|');", + &event, + ); + assert_eq!(out.error, None, "{out:?}"); + assert_eq!( + out.result.as_deref(), + Some(r#"({"tab":2, "name":"a \"q\"\n", "on":true, "none":null, "list":["qty", null, undefined, 1.5, [3]]})|[1, "x"]|({})"#) + ); + assert!(go("Object.keys({}).indexOf('toSource') < 0 && !({}).propertyIsEnumerable('toSource');", &event).result.as_deref() == Some("true")); +} + +/// `toSource` refuses what it can't write faithfully or within its bounds, with an error the +/// script can catch. +#[test] +fn to_source_refuses_cycles_functions_and_oversized_state() { + let event = Event::doc("Console"); + for (script, expected) in [ + ("var v = {}; v.v = v; v.toSource();", "refers to itself"), + ("({f: function () {}}).toSource();", "functions"), + ("var v = []; v.length = 1000000; v.toSource();", "too long"), + ("var v = []; for (var i = 0; i < 100; i++) v = [v]; v.toSource();", "nested too deeply"), + ("var s = new Array(4097).join('x'); var v = []; for (var i = 0; i < 300; i++) v.push(s); v.toSource();", "too long"), + ("({when: new Date(0)}).toSource();", "only plain objects"), + ("[/x/].toSource();", "only plain objects"), + ("function C() { this.a = 1; } new C().toSource();", "only plain objects"), + ("var v = JSON.parse('{\"__proto__\": 1}'); v.toSource();", "__proto__"), + ] { + let out = go(script, &event); + assert!(out.error.as_deref().is_some_and(|e| e.contains(expected)), "{script}: {out:?}"); + } + let out = go("try { var v = {}; v.v = v; v.toSource(); 'no'; } catch (e) { 'caught'; }", &event); + assert_eq!(out.result.as_deref(), Some("caught")); +} + +/// `setItems` replaces a choice field's items (text, or [text, export] pairs), keeping the value +/// when an item still has it; `setAction` records the script for its trigger, the last one +/// winning. Both refuse what they can't do. +#[test] +fn set_items_and_set_action_change_the_field() { + let event = Event::doc("Console"); + let out = go( + "var f = getField('size'); f.setItems([['Medium', 'm'], 'Huge', ['Tiny']]); \ + f.setAction('OnFocus', 'console.println(1)'); f.setAction('OnFocus', 'console.println(2)'); f.setAction('Keystroke', 'event.rc = true;'); \ + [f.numItems, f.getItemAt(1, true), f.getItemAt(2, false), f.value].join('|');", + &event, + ); + assert_eq!(out.error, None, "{out:?}"); + assert_eq!(out.result.as_deref(), Some("3|Huge|Tiny|m")); + let size = out.changed.iter().find(|f| f.name == "size").unwrap(); + assert_eq!(size.options, [("m".into(), "Medium".into()), ("Huge".into(), "Huge".into()), ("Tiny".into(), "Tiny".into())]); + assert_eq!(size.actions, [("OnFocus".into(), "console.println(2)".into()), ("Keystroke".into(), "event.rc = true;".into())]); + + let out = go("var f = getField('size'); f.setItems(['a', 'b']); f.value;", &event); + assert_eq!(out.result.as_deref(), Some("a"), "the old value is gone: the first item's"); + + for (script, expected) in [ + ("getField('price').setItems(['a']);", "not a combo box or list box"), + ("getField('size').setItems('a');", "needs an array"), + ("var a = []; a.length = 20000; getField('size').setItems(a);", "at most"), + ("getField('size').setAction('Explode', 'x');", "unknown trigger"), + ] { + let out = go(script, &event); + assert!(out.error.as_deref().is_some_and(|e| e.contains(expected)), "{script}: {out:?}"); + } +} diff --git a/crates/ui-egui/src/canvas.rs b/crates/ui-egui/src/canvas.rs index 90840225ef..79924f7128 100644 --- a/crates/ui-egui/src/canvas.rs +++ b/crates/ui-egui/src/canvas.rs @@ -1448,6 +1448,8 @@ struct PageFrame<'a> { current_height: f32, pressed_page: Option, calc_hover: Option, + /// The form field widget under the pointer on any page, for its Mouse Enter and Exit scripts. + field_hover: Option, /// A page took this frame's right-click. right_clicked: bool, } @@ -1930,6 +1932,7 @@ fn draw_pages( current_height: f32::INFINITY, pressed_page: None, calc_hover: None, + field_hover: None, right_clicked: false, }; for &i in visible_pages { @@ -1944,10 +1947,12 @@ fn draw_pages( current_height, pressed_page, calc_hover, + field_hover, out: page_out, right_clicked, .. } = frame; + crate::forms_ui::hover_moved(view, field_hover); // A right-click no page took deselects the comment, so the page's menu opens, not the comment's, // and with Edit text the picture (one on a page went to that page, `page_input`). if resp.secondary_clicked() && !right_clicked { diff --git a/crates/ui-egui/src/canvas/page_input.rs b/crates/ui-egui/src/canvas/page_input.rs index b39c16fde0..1f6666933d 100644 --- a/crates/ui-egui/src/canvas/page_input.rs +++ b/crates/ui-egui/src/canvas/page_input.rs @@ -966,6 +966,9 @@ pub(super) fn page_overlays(ui: &mut egui::Ui, view: &mut DocView, frame: &mut P } else { crate::forms_ui::paint_page(painter, &xf, i, info, form, view, input.hover.as_ref()); crate::forms_ui::tooltip(resp, form, input.hover.as_ref()); + if let Some(hovered) = crate::forms_ui::hovered(form, input.hover.as_ref()) { + frame.field_hover = Some(hovered); + } } // Form-field highlight (Acrobat's "Highlight existing fields"); required fields get a diff --git a/crates/ui-egui/src/editing.rs b/crates/ui-egui/src/editing.rs index df8f77741c..f6144ec143 100644 --- a/crates/ui-egui/src/editing.rs +++ b/crates/ui-egui/src/editing.rs @@ -524,10 +524,22 @@ impl PdfCraftApp { // click that brought this pane forward took the focus). It goes in first, as it came first. let (other_applied, stay) = self.apply_other_pane_edit(i); let modal = self.modal_open(); + let generation = |app: &Self| app.views.get(i).and_then(|v| app.session.get(v.id)).map(|d| d.edit_generation()); + let before = generation(self); + let queued = self.views.get(i).is_some_and(|v| v.pending_edit.is_some()); let applied = self.apply_queued_edit(i) && other_applied; + // A click's Mouse Up waits on the edit it queued (a check box's toggle, a list's pick): + // when that was refused, the click did nothing and its Mouse Up doesn't run either. + if queued + && generation(self) == before + && let Some(view) = self.views.get_mut(i) + { + view.forms.triggers.retain(|(_, t)| *t != pdfcraft_engine::FieldTrigger::MouseUp); + } // After the queued edit: a value typed into the field the focus left has been taken (and // its change has run the field's exit) before the focus scripts run. self.run_focus_scripts(i); + self.run_field_triggers(i); if !applied { // Nor may a shortcut pressed with it (⌘S right after Enter) run behind the refusal. self.deferred_commands.clear(); @@ -818,7 +830,8 @@ impl PdfCraftApp { /// ([`crate::forms_ui::refresh_draft`]), so leaving the field doesn't write the old value back. fn run_focus_scripts(&mut self, i: usize) { let Some(view) = self.views.get_mut(i) else { return }; - let now = view.forms.focus.as_ref().map(|f| (f.name.clone(), f.widget)); + // The field's object too: moving between fields sharing a name is a move (#344 review). + let now = view.forms.focus.as_ref().map(|f| (f.name.clone(), f.obj, f.widget)); if view.forms.focused == now { return; } @@ -826,13 +839,13 @@ impl PdfCraftApp { let id = view.id; let edited = view.forms.focus.clone(); use pdfcraft_engine::js::FieldFocus; - for ((name, widget), focus) in [left.map(|l| (l, FieldFocus::Exit)), now.map(|n| (n, FieldFocus::Enter))].into_iter().flatten() { + for ((name, obj, widget), focus) in [left.map(|l| (l, FieldFocus::Exit)), now.map(|n| (n, FieldFocus::Enter))].into_iter().flatten() { // The edited field's value before this script, to tell its untouched draft from typing: // the exit of the field left may have changed it already, and the draft with it. let before: Option<(String, Vec)> = edited.as_ref().and_then(|e| { crate::forms_ui::focused_field(e, &self.session.get(id)?.form).map(|f| (crate::forms_ui::draft_text(f), f.value.clone())) }); - match self.session.field_focus(id, &name, widget, focus) { + match self.session.field_focus_target(id, &name, obj.map(pdfcraft_engine::js::field_object_id), widget, focus) { Ok(changed) => { if changed && let Some(info) = self.session.get(id).map(|d| d.info.clone()) @@ -854,6 +867,42 @@ impl PdfCraftApp { } } + /// The pointer triggers tab `i` queued (Mouse Enter, Exit, Down, and a check box's Mouse Up + /// after its toggle), each running its widget's JavaScript in the document's realm (#269). + fn run_field_triggers(&mut self, i: usize) { + let Some(view) = self.views.get_mut(i) else { return }; + let triggers = std::mem::take(&mut view.forms.triggers); + let id = view.id; + for ((name, object, widget), trigger) in triggers { + // The field being edited as it was, so a script changing it shows in its editor + // while a draft the user already typed into is kept (as `run_focus_scripts` does). + let before: Option<(String, Vec)> = self.views.get(i).and_then(|v| v.forms.focus.as_ref()).and_then(|e| { + crate::forms_ui::focused_field(e, &self.session.get(id)?.form).map(|f| (crate::forms_ui::draft_text(f), f.value.clone())) + }); + match self.session.field_trigger_target(id, &name, Some(object), widget, trigger) { + Ok(changed) => { + if changed + && let Some(info) = self.session.get(id).map(|d| d.info.clone()) + && let Some(view) = self.views.get_mut(i) + { + view.document_changed(&info); + if let Some(before) = &before + && let Some(draft) = view.forms.focus.as_mut() + && let Some(f) = self.session.get(id).and_then(|d| crate::forms_ui::focused_field(draft, &d.form)) + { + crate::forms_ui::refresh_draft(draft, (&before.0, &before.1), (crate::forms_ui::draft_text(f), &f.value)); + } + } + let out = self.session.take_js_output(id); + if !out.is_empty() { + self.handle_js(id, out); + } + } + Err(e) => self.notify_error(e), + } + } + } + /// Apply the edit queued on tab `i` (the active one). Returns false only when it was a /// field's typing and the field refused it (#166): the editor reopens with the text, and the /// user has been told why, so the typing isn't lost and the caller can stop. @@ -1578,6 +1627,43 @@ mod revert_draft_tests { assert_eq!(hint(&app), "", "hint's enter ran before the document is read"); assert_eq!(app.views[0].forms.focus.as_ref().map(|f| f.text.as_str()), Some(""), "and its editor shows it"); } + /// A pointer trigger whose script changes the field being edited shows the change in its + /// editor, unless the user already typed there: then the typed draft is kept. + #[test] + fn a_pointer_trigger_changing_the_edited_field_refreshes_an_untouched_draft() { + let objs = [ + "<< /Type /Catalog /Pages 2 0 R /AcroForm << /Fields [4 0 R 6 0 R] /DA (/Helv 0 Tf 0 g) /DR << /Font << /Helv 5 0 R >> >> >> >>".to_string(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 200 300] >>".into(), + "<< /Type /Page /Parent 2 0 R /Annots [4 0 R 6 0 R] >>".into(), + "<< /Type /Annot /Subtype /Widget /FT /Tx /T (name) /V (old) /Rect [20 250 180 270] /P 3 0 R /F 4 >>".into(), + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>".into(), + "<< /Type /Annot /Subtype /Widget /FT /Tx /T (hot) /V () /Rect [20 200 180 220] /P 3 0 R /F 4 /AA << /E << /S /JavaScript /JS (getField\\('name'\\).value = 'new';) >> >> >>".into(), + ]; + let mut pdf = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (i, o) in objs.iter().enumerate() { + offsets.push(pdf.len()); + pdf.extend_from_slice(format!("{} 0 obj\n{o}\nendobj\n", i + 1).as_bytes()); + } + let xref = pdf.len(); + pdf.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes()); + for o in offsets { + pdf.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + pdf.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n", objs.len() + 1).as_bytes()); + for (typed, shown) in [(false, "new"), (true, "typed")] { + let mut app = PdfCraftApp::new(); + app.open_bytes("hot.pdf", None, pdf.clone()).unwrap(); + let hot = app.session.get(app.views[0].id).unwrap().form.iter().find(|f| f.name == "hot").unwrap().obj; + let text = if typed { "typed" } else { "old" }; + app.views[0].forms.focus = Some(Focus { text: text.into(), picked: vec!["old".into()], ..focus("") }); + let hot = ("hot".to_string(), pdfcraft_engine::js::field_object_id(hot), 0); + app.views[0].forms.triggers.push((hot, pdfcraft_engine::FieldTrigger::MouseEnter)); + app.run_field_triggers(0); + assert_eq!(app.views[0].forms.focus.as_ref().map(|f| f.text.as_str()), Some(shown), "typed: {typed}"); + } + } + #[test] fn successful_revert_clears_focused_or_queued_drafts_only_on_its_document() { for queued in [false, true] { diff --git a/crates/ui-egui/src/editing/save.rs b/crates/ui-egui/src/editing/save.rs index ea29f7fe46..77eb9c2b58 100644 --- a/crates/ui-egui/src/editing/save.rs +++ b/crates/ui-egui/src/editing/save.rs @@ -128,6 +128,7 @@ impl PdfCraftApp { if !self.flatten_fill_sign_for_save(id) { return false; } + self.run_document_actions(id, pdfcraft_engine::js::DocumentScriptEvent::WillSave); let bytes = match self.session.save_bytes(id) { Ok(b) => b, Err(e) => { @@ -293,6 +294,7 @@ impl PdfCraftApp { self.notify_tr("The document was closed before it could be saved."); return false; }; + self.run_document_actions(id, pdfcraft_engine::js::DocumentScriptEvent::WillSave); let bytes = if full { self.session.save_full_bytes(id) } else { self.session.save_bytes(id) }; let bytes = match bytes.and_then(|b| self.session.check_save(id, &b).map(|()| b)) { Ok(b) => b, diff --git a/crates/ui-egui/src/tools/forms_ui.rs b/crates/ui-egui/src/tools/forms_ui.rs index 50d4cba0bc..94aafda3f7 100644 --- a/crates/ui-egui/src/tools/forms_ui.rs +++ b/crates/ui-egui/src/tools/forms_ui.rs @@ -101,11 +101,18 @@ pub struct FormView { /// The field and widget that had focus when it was last looked at: when the focus moves, the /// one it left runs its XFA exit scripts and the one it reached its enter scripts /// (`PdfCraftApp::run_focus_scripts`). - pub(crate) focused: Option<(String, usize)>, + pub(crate) focused: Option<(String, Option, usize)>, /// The list box row clicked last, without Shift: (its field name, the row). A Shift-click picks /// the rows from it to the one clicked; the field's value can't say which row came last when /// rows share an export value, and `/I` lists them in ascending order. pub(crate) list_anchor: Option<(String, usize)>, + /// The field and widget under the pointer when it was last looked at: when it changes, the + /// one left runs its Mouse Exit JavaScript and the one reached its Mouse Enter (#269). + pub(crate) hovered: Option, + /// Pointer triggers waiting to run their widget's JavaScript, in order (field name, widget, + /// trigger): `PdfCraftApp::run_field_triggers` runs them after the frame's edit, so a check + /// box's Mouse Up sees the value its click just stored (#247). + pub(crate) triggers: Vec<(WidgetRef, pdfcraft_engine::FieldTrigger)>, } /// A form message for the app to show, with document data kept separate so the visible @@ -458,6 +465,38 @@ pub(crate) fn hit(xf: &PageXform, info: &DocInfo, page: usize, form: &[FormField Some(Hit::new(Owner::Field { field, widget }, Layer::Field, icon)) } +/// A widget a pointer trigger is for: its field's name and object (so a field sharing another's +/// name runs its own script), and the widget's index in the field. +pub(crate) type WidgetRef = (String, u64, usize); + +/// `f`'s widget `widget` as a trigger names it. +fn widget_ref(f: &FormField, widget: usize) -> WidgetRef { + (f.name.clone(), pdfcraft_engine::js::field_object_id(f.obj), widget) +} + +/// The field and widget under the pointer on a page, when a press there would take it. +pub(crate) fn hovered(form: &[FormField], hover: Option<&Hit>) -> Option { + match hover.map(|h| h.owner) { + Some(Owner::Field { field, widget }) => form.get(field).map(|f| widget_ref(f, widget)), + _ => None, + } +} + +/// Queue Mouse Exit for the widget the pointer left and Mouse Enter for the one it reached, when +/// they differ from the last frame's (`now`: the widget under the pointer on any page). +pub(crate) fn hover_moved(view: &mut DocView, now: Option) { + if view.forms.hovered == now { + return; + } + let left = std::mem::replace(&mut view.forms.hovered, now.clone()); + if let Some(w) = left { + view.forms.triggers.push((w, pdfcraft_engine::FieldTrigger::MouseExit)); + } + if let Some(w) = now { + view.forms.triggers.push((w, pdfcraft_engine::FieldTrigger::MouseEnter)); + } +} + /// The tooltip (`/TU`) of the field under the pointer, as Acrobat shows it on hover (#3854): only /// where the field is what a press would take (`hover`, the top hit there). ISO 32000-2 Table 226 /// makes it the name the user interface identifies the field by. @@ -484,6 +523,10 @@ pub(crate) fn page_input( let PagePress::Mine(Owner::Field { field, widget: wi }) = press else { return false }; let Some((f, w)) = form.get(field).and_then(|f| f.widgets.get(wi).map(|w| (f, w))) else { return false }; let usable = usable(f, w, allowed); + // The press runs the widget's Mouse Down JavaScript, as in Acrobat (#269). + if usable && ptr.pressed().is_some() { + view.forms.triggers.push((widget_ref(f, wi), pdfcraft_engine::FieldTrigger::MouseDown)); + } // A text field opens on the press rather than the click, so a drag that starts on it // selects its text (#2213). if f.kind == FormFieldKind::Text @@ -544,6 +587,10 @@ pub(crate) fn page_input( commit(view, form); view.forms.focus = Some(open_focus(f, wi)); } + // A combo box's click runs its Mouse Up JavaScript (#269). + if f.kind == FormFieldKind::Combo { + view.forms.triggers.push((widget_ref(f, wi), pdfcraft_engine::FieldTrigger::MouseUp)); + } } FormFieldKind::List => { let Some(p) = ptr.within(xf.rect) else { return true }; @@ -589,6 +636,8 @@ pub(crate) fn page_input( view.pending_edit = Some(list_pick_edit(form, f, picked)); } } + // A list box's click runs its Mouse Up JavaScript after the pick is stored (#269). + view.forms.triggers.push((widget_ref(f, wi), pdfcraft_engine::FieldTrigger::MouseUp)); } } true @@ -1376,6 +1425,9 @@ pub(crate) fn overlay( view.forms.toggle = true; } else if toggle { edit = toggle_edit(f, focus.widget); + // The click's Mouse Up JavaScript runs once the toggle is stored, so it reads the + // new value (#247). + view.forms.triggers.push((widget_ref(f, focus.widget), pdfcraft_engine::FieldTrigger::MouseUp)); } else if space { // Space toggles a check box and selects a radio button, as in browsers: it never // turns the selected button off. diff --git a/crates/ui-egui/src/workflows/js_ui.rs b/crates/ui-egui/src/workflows/js_ui.rs index 3547c511f2..1f3721c511 100644 --- a/crates/ui-egui/src/workflows/js_ui.rs +++ b/crates/ui-egui/src/workflows/js_ui.rs @@ -200,6 +200,23 @@ impl PdfCraftApp { true } + /// Run document `id`'s catalog JavaScript for `event` (Will Save before a save, Will Print + /// before printing) and show what it said. What it changed is part of what is saved or + /// printed. A failing script is reported and the save or print still goes ahead. + pub(crate) fn run_document_actions(&mut self, id: DocId, event: pdfcraft_engine::js::DocumentScriptEvent) { + let before = self.session.get(id).map(|d| d.edit_generation()); + let out = self.session.run_document_actions(id, event); + if self.session.get(id).map(|d| d.edit_generation()) != before + && let Some(i) = self.views.iter().position(|v| v.id == id) + && let Some(info) = self.session.get(id).map(|d| d.info.clone()) + { + self.views[i].document_changed(&info); + } + if !out.is_empty() { + self.handle_js(id, out); + } + } + /// Run a push button's JavaScript (its Mouse Up action). pub fn run_button_script(&mut self, id: DocId, field: &str, script: &str) { self.run_button_script_target(id, field, None, script); diff --git a/crates/ui-egui/src/workflows/print_ui.rs b/crates/ui-egui/src/workflows/print_ui.rs index aa02635110..3ea2099a96 100644 --- a/crates/ui-egui/src/workflows/print_ui.rs +++ b/crates/ui-egui/src/workflows/print_ui.rs @@ -633,6 +633,16 @@ impl PdfCraftApp { return false; } let Some((_, id)) = self.active_ids() else { return false }; + // A form's Will Print script runs first: what it changes is printed. + if self.session.get(id).is_some_and(|d| d.allows_printing()) { + let before = self.session.get(id).map(|d| d.edit_generation()); + self.run_document_actions(id, pdfcraft_engine::js::DocumentScriptEvent::WillPrint); + // What it showed may lie outside the printed areas Label printing found: find them again. + if self.session.get(id).map(|d| d.edit_generation()) != before && self.print_draft.handling == Handling::Label { + self.print_draft.crop_pages = None; + self.read_label_crops(id); + } + } let Some(doc) = self.session.get(id) else { return false }; let labels: Vec = doc.info.pages.iter().map(|p| p.label.clone()).collect(); let settings = match self.print_draft.settings(doc.info.pages.len(), &labels) { diff --git a/crates/ui-egui/tests/it/field_triggers.rs b/crates/ui-egui/tests/it/field_triggers.rs new file mode 100644 index 0000000000..f648b7a445 --- /dev/null +++ b/crates/ui-egui/tests/it/field_triggers.rs @@ -0,0 +1,104 @@ +//! A form field widget's pointer scripts in the app (#269, ported from #344): the pointer +//! moving onto and off a widget runs its Mouse Enter and Mouse Exit JavaScript, and a click on a +//! check box runs its Mouse Up JavaScript after the click stored the new value (#247). + +use egui_kittest::Harness; +use pdfcraft_ui_egui::PdfCraftApp; + +/// `log` records what ran; `hot` (a text field) logs the pointer entering and leaving it; +/// `strike` (a check box) shows or hides `note` from its Mouse Up script by its own new value. +fn form() -> Vec { + let objs = [ + "<< /Type /Catalog /Pages 2 0 R /AcroForm << /Fields [4 0 R 5 0 R 6 0 R 8 0 R] /DA (/Helv 0 Tf 0 g) /DR << /Font << /Helv 7 0 R >> >> >> >>", + "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 300 400] >>", + "<< /Type /Page /Parent 2 0 R /Annots [4 0 R 5 0 R 6 0 R 8 0 R] >>", + "<< /Type /Annot /Subtype /Widget /FT /Tx /T (log) /V () /Rect [20 350 280 370] /P 3 0 R /F 4 >>", + "<< /Type /Annot /Subtype /Widget /FT /Tx /T (hot) /V () /Rect [20 250 280 290] /P 3 0 R /F 4 /AA << /E << /S /JavaScript /JS (getField\\('log'\\).value += 'in;';) >> /X << /S /JavaScript /JS (getField\\('log'\\).value += 'out;';) >> >> >>", + "<< /Type /Annot /Subtype /Widget /FT /Btn /T (strike) /V /Off /AS /Off /Rect [20 300 50 330] /P 3 0 R /F 4 /MK << /CA (4) >> /AP << /N << /Yes 9 0 R /Off 9 0 R >> >> /A << /S /JavaScript /JS (getField\\('note'\\).display = event.target.value == 'Off' ? display.hidden : display.visible; getField\\('log'\\).value += 'up:' + event.target.value + ';';) >> >>", + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>", + "<< /Type /Annot /Subtype /Widget /FT /Tx /T (note) /V (struck) /Rect [60 300 280 330] /P 3 0 R /F 6 >>", + "<< /Type /XObject /Subtype /Form /BBox [0 0 30 30] /Length 0 >>\nstream\n\nendstream", + ]; + let mut out = b"%PDF-1.7\n".to_vec(); + let mut offsets = Vec::new(); + for (i, o) in objs.iter().enumerate() { + offsets.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n{o}\nendobj\n", i + 1).as_bytes()); + } + let xref = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes()); + for o in offsets { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{xref}\n%%EOF\n", objs.len() + 1).as_bytes()); + out +} + +fn harness() -> Harness<'static, PdfCraftApp> { + let mut h = Harness::builder().with_size(egui::vec2(1400.0, 900.0)).build_eframe(|_cc| { + let mut app = PdfCraftApp::new(); + app.set_option("language", "en").unwrap(); + app.set_option("left", "closed").unwrap(); + app.open_bytes("triggers.pdf", None, form()).unwrap(); + app + }); + crate::support::settle(&mut h); + h +} + +fn field<'a>(app: &'a PdfCraftApp, name: &str) -> &'a pdfcraft_engine::FormField { + let id = app.active_ids().unwrap().1; + app.session.get(id).unwrap().form.iter().find(|f| f.name == name).unwrap() +} + +fn centre(h: &Harness<'static, PdfCraftApp>, name: &str) -> egui::Pos2 { + let app = h.state(); + let (index, id) = app.active_ids().unwrap(); + let doc = app.session.get(id).unwrap(); + pdfcraft_ui_egui::forms_ui::field_screen_rect(&app.views[index], &doc.info, field(app, name), 0).unwrap().center() +} + +fn log(h: &Harness<'static, PdfCraftApp>) -> String { + field(h.state(), "log").value.join(",") +} + +#[test] +fn the_pointer_entering_and_leaving_a_widget_runs_its_scripts() { + let mut h = harness(); + let hot = centre(&h, "hot"); + let away = hot + egui::vec2(0.0, 200.0); + h.hover_at(away); + h.run_steps(2); + h.hover_at(hot); + crate::support::wait_until(&mut h, "Mouse Enter ran", |app| field(app, "log").value.join(",") == "in;"); + // Moving within the widget runs nothing more. + h.hover_at(hot + egui::vec2(10.0, 0.0)); + h.run_steps(3); + assert_eq!(log(&h), "in;"); + h.hover_at(away); + crate::support::wait_until(&mut h, "Mouse Exit ran", |app| field(app, "log").value.join(",") == "in;out;"); +} + +/// The check box's Mouse Up reads the value its click stored: checking it shows the note, +/// unchecking hides it again (#247). +#[test] +fn a_check_box_mouse_up_script_sees_the_value_its_click_stored() { + let mut h = harness(); + let strike = centre(&h, "strike"); + let click = |h: &mut Harness<'static, PdfCraftApp>| { + h.hover_at(strike); + h.run_steps(1); + h.drag_at(strike); + h.run_steps(1); + h.drop_at(strike); + h.run_steps(1); + }; + click(&mut h); + crate::support::wait_until(&mut h, "checked and its Mouse Up ran", |app| field(app, "log").value.join(",") == "up:Yes;"); + assert_eq!(field(h.state(), "strike").value, ["Yes"]); + assert!(!field(h.state(), "note").widgets[0].hidden, "shown by the script"); + h.run_steps(6); + click(&mut h); + crate::support::wait_until(&mut h, "unchecked and its Mouse Up ran", |app| field(app, "log").value.join(",") == "up:Yes;up:Off;"); + assert!(field(h.state(), "note").widgets[0].hidden, "hidden again"); +} diff --git a/crates/ui-egui/tests/it/main.rs b/crates/ui-egui/tests/it/main.rs index ea5dec6e61..0aae004df6 100644 --- a/crates/ui-egui/tests/it/main.rs +++ b/crates/ui-egui/tests/it/main.rs @@ -43,6 +43,7 @@ mod download_mime; mod editing; mod editor_taps; mod export; +mod field_triggers; mod fill_sign; mod find_replace; mod folders; diff --git a/docs/acroform-script-workflows.md b/docs/acroform-script-workflows.md index 200a59966a..ae08396dc6 100644 --- a/docs/acroform-script-workflows.md +++ b/docs/acroform-script-workflows.md @@ -115,9 +115,19 @@ supply an anchored renderer through the UI menu callback when it adds link actio the shared parser lands upstream, this integration can use it directly. `ScriptSession::run_host_task` gives future timer and page actions (#4751) access to the -retained realm on its own worker. `DocumentScriptEvent` and `run_document_script_event` -provide explicit entry points for catalog OpenAction, WillSave and WillPrint (#344). -The callers for those document actions are part of #344. +retained realm on its own worker. + +Document actions and widget triggers (ported from #344 by @DWAA1660) run through the same +realm. On opening, the document scripts run, then the catalog `/OpenAction`'s JavaScript +and its `/Next` chain (event `Doc/Open`); Revert runs them again in a fresh realm. +`Session::run_document_actions` runs `/AA /WS` before a save and `/AA /WP` before printing: +the app's Save and Print and the `doc_save` and `doc_print` tools call it first. A failing +action is reported and never stops the save or print. `Session::field_trigger` runs a +widget's Mouse Enter, Mouse Exit, Mouse Down, Mouse Up, Focus or Blur JavaScript; the app +queues them from the pointer and the focus, and `js_event` runs them headlessly. A check +box's or radio button's Mouse Up runs after its click has stored the new value (#247). +Scripts that run unasked (open, Will Save, Will Print, pointer over, focus) may only ask for +highlighting, zoom, focus or a page. The lexical preflight recognizes unambiguous top-level and block-body ASI boundaries before identifiers, including declarations and assignments separated by line breaks or comments. It preserves counters after operators and member access, and across binary keyword operators. It distinguishes control-statement parentheses for regexp literals; inside expression parentheses, array/object literals and template interpolations it remains conservative; this is not a complete ECMAScript parser or ASI implementation. diff --git a/parity/acrobat-features.toml b/parity/acrobat-features.toml index 6af3b0b471..2f3cda2051 100644 --- a/parity/acrobat-features.toml +++ b/parity/acrobat-features.toml @@ -5348,11 +5348,11 @@ title = "Document actions (will close, will save, did save, will print, did prin area = "F" tier = "P1" milestone = "M6" -status = "planned" -commands = [] -tools = [] -evidence = [] -notes = "" +status = "partial" +commands = ["file.save", "print.dialog"] +tools = ["doc_open", "doc_save", "doc_print"] +evidence = ["crates/engine/src/document_actions_tests.rs::the_open_action_runs_once_on_opening_in_the_document_realm", "crates/engine/src/document_actions_tests.rs::an_open_action_chain_runs_in_order_and_a_cycle_runs_each_action_once", "crates/engine/src/document_actions_tests.rs::an_open_action_may_not_open_a_url_or_print", "crates/engine/src/document_actions_tests.rs::the_open_action_does_not_run_with_javascript_off", "crates/engine/src/document_actions_tests.rs::revert_runs_the_open_action_again_in_a_fresh_realm", "crates/engine/src/document_actions_tests.rs::will_save_changes_what_is_saved_and_a_failing_one_does_not_stop_the_save", "crates/engine/src/document_actions_tests.rs::will_print_runs_in_the_document_realm", "crates/automation/tests/it/document_actions.rs::open_will_save_will_print_and_field_events_share_the_document_realm"] +notes = "The catalog /OpenAction's JavaScript (with its /Next chain, bounded and cycle-safe) runs once on opening after the document scripts, and again on Revert in a fresh realm; /AA /WS runs before a save (the app's Save and Save As, doc_save) and /AA /WP before printing (the app's Print, doc_print), in the document's realm, so what they change is saved or printed. A failing action is reported and never stops the save or print. These scripts run unasked, so they may only ask for highlighting, zoom, focus or a page. Missing: Will Close (/WC), Did Save (/DS), Did Print (/DP) and page open/close actions (#269, ported from #344)." [[feature]] id = "form.js-host" @@ -5362,9 +5362,9 @@ tier = "P0" milestone = "M6" status = "partial" commands = ["tools.js_console"] -tools = ["js_run"] -evidence = ["crates/automation/tests/it/timesheet_compat.rs::timesheet_keypad_geometry_icons_calculation_and_persistence", "crates/automation/tests/it/timesheet_compat.rs::popup_and_manual_response_resume_the_same_script_and_validate_choices", "crates/js/src/session.rs::document_realm_keeps_keypad_state_and_isolates_documents", "crates/js/src/tests.rs::calculate_scripts_read_fields_and_set_the_value", "crates/engine/src/tests.rs::form_javascript_validates_calculates_and_formats", "crates/automation/tests/it/automation.rs::javascript_through_tools", "crates/js/src/tests.rs::an_xfa_form_sees_the_xfa_plugin_installed", "crates/js/src/tests.rs::a_plain_form_sees_the_xfa_version_but_not_installed", "crates/engine/src/tests.rs::scripts_in_an_xfa_form_see_the_xfa_plugin_installed", "crates/js/src/tests.rs::scand_allows_separators_between_fields_but_nothing_before_or_after_them", "crates/js/src/tests.rs::app_alert_keeps_its_title_icon_and_buttons", "crates/js/src/tests.rs::app_alert_reads_the_object_form", "crates/js/src/tests.rs::app_alert_falls_back_on_bad_icon_and_button_numbers", "crates/js/src/tests.rs::app_alert_titles_are_clipped_and_charged", "crates/ui-egui/tests/it/javascript.rs::every_message_a_script_shows_takes_the_toast_in_turn", "crates/ui-egui/tests/it/forms.rs::a_form_button_opens_its_attached_instructions_in_a_new_tab", "crates/js/src/tests.rs::widget_rect_reads_and_writes_a_four_number_array", "crates/engine/src/tests.rs::javascript_field_rect_moves_first_widget_and_undoes"] -notes = "Document-scoped native Boa realms shared by button, console and field events; synchronous calculateNow in CO order; normal captions/icons; rect and appearance/tooltip writeback; effective page boxes; native explicit-button popup/response/alert continuation and viewer requests, including opening or saving attachments from form buttons. Headless js_run has deterministic ordered answers. Partial: web prompt UI, timers, annotation/layer APIs, non-normal button faces, widget-indexed Field access and remaining document action triggers are not implemented. See docs/acroform-script-workflows.md for lifecycle, limits and validation scope." +tools = ["js_run", "js_event"] +evidence = ["crates/js/src/tests.rs::to_source_writes_plain_state_as_a_literal", "crates/js/src/tests.rs::to_source_refuses_cycles_functions_and_oversized_state", "crates/js/src/tests.rs::set_items_and_set_action_change_the_field", "crates/engine/src/document_actions_tests.rs::set_items_and_set_action_are_written_to_the_document", "crates/engine/src/document_actions_tests.rs::pointer_and_focus_triggers_run_the_widget_scripts", "crates/engine/src/document_actions_tests.rs::only_a_click_trigger_may_print", "crates/engine/src/document_actions_tests.rs::a_check_box_mouse_up_reads_the_value_its_click_stored", "crates/ui-egui/tests/it/field_triggers.rs::the_pointer_entering_and_leaving_a_widget_runs_its_scripts", "crates/ui-egui/tests/it/field_triggers.rs::a_check_box_mouse_up_script_sees_the_value_its_click_stored", "crates/automation/tests/it/timesheet_compat.rs::timesheet_keypad_geometry_icons_calculation_and_persistence", "crates/automation/tests/it/timesheet_compat.rs::popup_and_manual_response_resume_the_same_script_and_validate_choices", "crates/js/src/session.rs::document_realm_keeps_keypad_state_and_isolates_documents", "crates/js/src/tests.rs::calculate_scripts_read_fields_and_set_the_value", "crates/engine/src/tests.rs::form_javascript_validates_calculates_and_formats", "crates/automation/tests/it/automation.rs::javascript_through_tools", "crates/js/src/tests.rs::an_xfa_form_sees_the_xfa_plugin_installed", "crates/js/src/tests.rs::a_plain_form_sees_the_xfa_version_but_not_installed", "crates/engine/src/tests.rs::scripts_in_an_xfa_form_see_the_xfa_plugin_installed", "crates/js/src/tests.rs::scand_allows_separators_between_fields_but_nothing_before_or_after_them", "crates/js/src/tests.rs::app_alert_keeps_its_title_icon_and_buttons", "crates/js/src/tests.rs::app_alert_reads_the_object_form", "crates/js/src/tests.rs::app_alert_falls_back_on_bad_icon_and_button_numbers", "crates/js/src/tests.rs::app_alert_titles_are_clipped_and_charged", "crates/ui-egui/tests/it/javascript.rs::every_message_a_script_shows_takes_the_toast_in_turn", "crates/ui-egui/tests/it/forms.rs::a_form_button_opens_its_attached_instructions_in_a_new_tab", "crates/js/src/tests.rs::widget_rect_reads_and_writes_a_four_number_array", "crates/engine/src/tests.rs::javascript_field_rect_moves_first_widget_and_undoes"] +notes = "Document-scoped native Boa realms shared by button, console and field events; synchronous calculateNow in CO order; normal captions/icons; rect and appearance/tooltip writeback; effective page boxes; native explicit-button popup/response/alert continuation and viewer requests, including opening or saving attachments from form buttons. Headless js_run has deterministic ordered answers. A widget's Mouse Enter, Mouse Exit, Mouse Down, Focus and Blur JavaScript run in the app and through js_event (and form_focus for focus), and a check box's or radio button's Mouse Up runs after its click stored the new value (#247); field.setItems, field.setAction and a bounded Object.prototype.toSource (plain data only) are written back to the document (#269, ported from #344). Partial: web prompt UI, timers, annotation/layer APIs, non-normal button faces, widget-indexed Field access, and Will Close / Did Save / Did Print are not implemented. See docs/acroform-script-workflows.md for lifecycle, limits and validation scope." [[feature]] id = "form.js-console" From a23b7c7f738e130f05ad8daa4c57f4ef56a356ab Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 09:17:32 -0500 Subject: [PATCH 39/60] fix(redact): apply marks with an indirect /Subtype and judge inline annotations (#5849) * fix(redact): apply marks with an indirect /Subtype and judge inline annotations A Redact annotation whose /Subtype is an indirect reference was passed over by apply although the audit, which resolves it, reports it as a mark: the other marks applied and the content under that one stayed. Annotation dictionaries written in place in /Annots were kept whatever they covered, and an inline reply or pop-up kept a removed comment reachable through /IRT or /Parent. /Subtype is now resolved wherever redaction classifies an annotation. Inline entries are judged by their rectangle and relationships like referenced ones, on the marked page and across pages, and the pop-up an annotation names (/Popup) goes with it. * address review: an inline widget's /Parent field goes even when no referenced widget names it * fmt --- crates/redact/src/lib.rs | 227 ++++++++++++++++++++++++++++--------- crates/redact/src/tests.rs | 115 +++++++++++++++++++ 2 files changed, 289 insertions(+), 53 deletions(-) diff --git a/crates/redact/src/lib.rs b/crates/redact/src/lib.rs index 5938f20a68..7fd34dae12 100644 --- a/crates/redact/src/lib.rs +++ b/crates/redact/src/lib.rs @@ -247,6 +247,19 @@ pub(crate) fn annots_of(doc: &Document, page: &Dict) -> Vec { page.get(b"Annots").map(|a| doc.resolve(a)).and_then(|a| a.as_array().cloned()).unwrap_or_default() } +/// An annotation's `/Subtype`, resolved: it may be an indirect reference (ISO 32000-2 §7.3.10), +/// and the audit reads it that way, so a mark it reports must not be one [`apply`] passes over. +pub(crate) fn subtype(doc: &Document, d: &Dict) -> Option> { + d.get(b"Subtype").and_then(|s| doc.resolve(s).as_name().map(<[u8]>::to_vec)) +} + +/// The pop-up `d` names (`/Popup`), when it is one: it goes with `d`. Anything else it might name +/// (a mark on another page, say) is left alone. +fn named_popup(doc: &Document, d: &Dict) -> Option { + let r = d.reference(b"Popup")?; + doc.get(r).as_dict().is_some_and(|p| subtype(doc, p).as_deref() == Some(b"Popup")).then_some(r) +} + /// Every annotation the pages list. Compared before and after a removal, what is no longer listed /// leaves the structure tree too ([`tags::drop_object_refs`]). pub(crate) fn listed_annotations(doc: &Document) -> HashSet { @@ -267,14 +280,14 @@ pub(crate) fn marks_checked(doc: &Document) -> (Vec, Vec) { let Some(r) = a.as_ref() else { // A mark written in place has no object to find, apply or remove: refuse it // rather than apply the others as if it were not there. - if doc.resolve(&a).as_dict().is_some_and(|d| d.name(b"Subtype") == Some(b"Redact")) { + if doc.resolve(&a).as_dict().is_some_and(|d| subtype(doc, d).as_deref() == Some(b"Redact")) { unreadable.push(pi); } continue; }; let obj = doc.get(r); let Some(d) = obj.as_dict() else { continue }; - if d.name(b"Subtype") != Some(b"Redact") { + if subtype(doc, d).as_deref() != Some(b"Redact") { continue; } let mut rects: Vec<[f64; 4]> = quad_points(doc, d) @@ -580,18 +593,41 @@ fn apply_within(doc: &mut Document, pages: Option<&[usize]>, images: pdfcraft_co let annots = annots_of(doc, &p.dict); let mut kept = Vec::with_capacity(annots.len()); for a in &annots { - let Some(r) = a.as_ref() else { - kept.push(a.clone()); - continue; - }; - let obj = doc.get(r); + // An entry written in place is read as one listed by reference: its `/IRT` or + // `/Parent` keeps a removed comment reachable all the same, and it has its own text. + let obj = doc.resolve(a); let Some(d) = obj.as_dict() else { kept.push(a.clone()); continue; }; - let is_reply = - d.get(b"IRT").and_then(Object::as_ref).is_some_and(|x| removed_everywhere.contains(&x)) && d.name(b"Subtype") != Some(b"Widget"); - let is_popup = d.get(b"Parent").and_then(Object::as_ref).is_some_and(|x| removed_everywhere.contains(&x)); + let is_reply = d.reference(b"IRT").is_some_and(|x| removed_everywhere.contains(&x)) && subtype(doc, d).as_deref() != Some(b"Widget"); + let is_popup = d.reference(b"Parent").is_some_and(|x| removed_everywhere.contains(&x)); + let Some(r) = a.as_ref() else { + if is_reply || is_popup { + grew = true; + if is_reply { + report.annotations += 1; + } + // Its appearance can be shared with an annotation that stays elsewhere. + if !appearance_scan_limit { + match appearance_refs(doc, d) { + Some(references) => removed_appearances.extend(references), + None => appearance_scan_limit = true, + } + if removed_appearances.len() > 16_384 { + appearance_scan_limit = true; + removed_appearances.clear(); + } + } + // It has no reference to remove elsewhere; the pop-up it names has. + if let Some(p) = named_popup(doc, d) { + removed_everywhere.insert(p); + } + } else { + kept.push(a.clone()); + } + continue; + }; if removed_everywhere.contains(&r) || is_reply || is_popup { // A reply or pop-up removed here can share its appearance with an annotation // that stays on an unselected page, like one removed under a mark: the @@ -638,6 +674,30 @@ fn apply_within(doc: &mut Document, pages: Option<&[usize]>, images: pdfcraft_co report.fields += 1; } } + // A widget written in place on another page names its deleted field through `/Parent`, which + // would keep the field and its value in the file: it goes with the field. + if !doomed_fields.is_empty() { + for p in pdfcraft_model::pages(doc) { + let annots = annots_of(doc, &p.dict); + let kept: Vec = annots + .iter() + .filter(|a| { + a.as_ref().is_some() || !matches!(&**a, Object::Dict(d) if d.reference(b"Parent").is_some_and(|f| doomed_fields.contains(&f))) + }) + .cloned() + .collect(); + if kept.len() != annots.len() { + report.annotations += annots.len() - kept.len(); + doc.update_dict(p.obj, |d| { + if kept.is_empty() { + d.remove(b"Annots"); + } else { + d.set(b"Annots".to_vec(), Object::Array(kept)); + } + })?; + } + } + } // 3a. A tagged PDF's structure tree still points at the removed comments and widgets // (`/OBJR`, #2874): drop those references, or a full rewrite keeps them reachable, the @@ -885,30 +945,66 @@ fn remove_page_annotations( page_marks: &[&Mark], ) -> Result<(), RedactError> { let Applying { report, doomed_fields, widget_owner, removed_everywhere, removed_appearances, appearance_scan_limit, .. } = applying; - // 2. Annotations: the marks, and whatever lies under them (with their pop-ups). + // 2. Annotations: the marks, and whatever lies under them (with their pop-ups). An entry + // written in place (a dictionary in `/Annots`, not a reference) is judged like any other: its + // `/Contents` and `/IRT` stay in the file for as long as the page lists it. let annots = annots_of(doc, &doc.get(page.obj).as_dict().cloned().unwrap_or_default()); + /// What decides an entry's fate, read once. + struct Entry { + r: Option, + widget: bool, + irt: Option, + parent: Option, + popup: Option, + } + let entries: Vec> = annots + .iter() + .map(|a| { + let obj = doc.resolve(a); + let d = obj.as_dict()?; + Some(Entry { + r: a.as_ref(), + widget: subtype(doc, d).as_deref() == Some(b"Widget"), + irt: d.reference(b"IRT"), + parent: d.reference(b"Parent"), + popup: named_popup(doc, d), + }) + }) + .collect(); let mut removed: Vec = Vec::new(); - for a in &annots { - let Some(r) = a.as_ref() else { continue }; - let obj = doc.get(r); + // Entries written in place that go, by position in `/Annots`. + let mut inline_gone = vec![false; annots.len()]; + for (i, a) in annots.iter().enumerate() { + let obj = doc.resolve(a); let Some(d) = obj.as_dict() else { continue }; - let subtype = d.name(b"Subtype").unwrap_or(b""); - if subtype == b"Redact" { - if page_marks.iter().any(|m| m.obj == r) { + let r = a.as_ref(); + let kind = subtype(doc, d).unwrap_or_default(); + if kind == b"Redact" { + // A mark written in place was refused before anything changed. + if let Some(r) = r + && page_marks.iter().any(|m| m.obj == r) + { removed.push(r); } continue; } - if subtype == b"Popup" { + if kind == b"Popup" { continue; } if rect_of(doc, d.get(b"Rect")).is_some_and(|b| rects.iter().any(|x| pdfcraft_content::overlaps(*x, b, 0.0))) { - removed.push(r); - if subtype == b"Widget" { - if let Some((_, field)) = widget_owner.iter().find(|(w, _)| *w == r) - && !doomed_fields.contains(field) + match r { + Some(r) => removed.push(r), + None => inline_gone[i] = true, + } + if kind == b"Widget" { + // A widget written in place has no reference of its own; its `/Parent` is the field, + // even one no referenced widget belongs to (it then has no entry in `widget_owner`). + // Step 3 deletes only what the form lists as a field, so any `/Parent` is safe here. + let owner = r.and_then(|r| widget_owner.iter().find(|(w, _)| *w == r).map(|(_, f)| *f)).or_else(|| d.reference(b"Parent")); + if let Some(field) = owner + && !doomed_fields.contains(&field) { - doomed_fields.push(*field); + doomed_fields.push(field); } } else { report.annotations += 1; @@ -919,29 +1015,25 @@ fn remove_page_annotations( // those: a reply placed elsewhere would keep its parent — the comment and the QuadPoints // of the redacted words — in the file through its `/IRT` (#2659). Each pass removes at // least one more annotation of this page or stops, so it ends. - let replies: Vec<(ObjRef, ObjRef)> = annots - .iter() - .filter_map(Object::as_ref) - .filter_map(|r| { - let obj = doc.get(r); - let d = obj.as_dict()?; - (d.name(b"Subtype") != Some(b"Widget")).then_some(())?; - Some((r, d.get(b"IRT").and_then(Object::as_ref)?)) - }) - .collect(); loop { - let before = removed.len(); - for (r, parent) in &replies { - if !removed.contains(r) && removed.contains(parent) { - removed.push(*r); - report.annotations += 1; + let mut grew = false; + for (i, e) in entries.iter().enumerate() { + let Some(e) = e else { continue }; + let gone = e.r.map_or(inline_gone[i], |r| removed.contains(&r)); + if gone || e.widget || !e.irt.is_some_and(|p| removed.contains(&p)) { + continue; } + match e.r { + Some(r) => removed.push(r), + None => inline_gone[i] = true, + } + report.annotations += 1; + grew = true; } - if removed.len() == before { + if !grew { break; } } - removed_everywhere.extend(removed.iter().copied()); // A removed annotation's appearance streams can be shared with a kept one on another page, // which would keep a pruned pattern's cell reachable: remember them for the pattern guard. for reference in &removed { @@ -961,24 +1053,53 @@ fn remove_page_annotations( } } } - // Pop-ups dropped with their parent below count too: a reply on another page may answer one. - for a in &annots { - let Some(r) = a.as_ref() else { continue }; - if doc.get(r).as_dict().and_then(|d| d.get(b"Parent")).and_then(Object::as_ref).is_some_and(|p| removed.contains(&p)) { - removed_everywhere.insert(r); + // An entry written in place that goes keeps no reference, but its appearances count as well. + for (i, a) in annots.iter().enumerate() { + if *appearance_scan_limit { + break; + } + if let (true, Some(annotation)) = (inline_gone[i], doc.resolve(a).as_dict()) { + match appearance_refs(doc, annotation) { + Some(references) => removed_appearances.extend(references), + None => *appearance_scan_limit = true, + } + if removed_appearances.len() > 16_384 { + *appearance_scan_limit = true; + removed_appearances.clear(); + } } } + // Pop-ups go with their annotation: those that name it (`/Parent`), and those it names + // (`/Popup`), the only link an annotation written in place can have to its pop-up. + let mut going: HashSet = removed.iter().copied().collect(); + for (i, e) in entries.iter().enumerate() { + let Some(e) = e else { continue }; + let gone = e.r.map_or(inline_gone[i], |r| removed.contains(&r)); + if gone { + going.extend(e.popup); + } + } + for (i, e) in entries.iter().enumerate() { + let Some(e) = e else { continue }; + if e.parent.is_some_and(|p| removed.contains(&p)) { + match e.r { + Some(r) => { + going.insert(r); + } + None => inline_gone[i] = true, + } + } + } + // Pop-ups dropped with their parent count too: a reply on another page may answer one. + removed_everywhere.extend(going.iter().copied()); let kept: Vec = annots .into_iter() - .filter(|a| { - let Some(r) = a.as_ref() else { return true }; - if removed.contains(&r) { - return false; - } - // Pop-ups of removed annotations go too. - let obj = doc.get(r); - !obj.as_dict().and_then(|d| d.get(b"Parent")).and_then(Object::as_ref).is_some_and(|p| removed.contains(&p)) + .enumerate() + .filter(|(i, a)| match a.as_ref() { + Some(r) => !going.contains(&r), + None => !inline_gone[*i], }) + .map(|(_, a)| a) .collect(); doc.update_dict(page.obj, |d| { if kept.is_empty() { diff --git a/crates/redact/src/tests.rs b/crates/redact/src/tests.rs index e8ff3cd788..4ad4103e10 100644 --- a/crates/redact/src/tests.rs +++ b/crates/redact/src/tests.rs @@ -6691,6 +6691,78 @@ fn a_type3_text_layer_is_redacted_like_any_other() { assert_eq!(under(&mut doc, 0, &[[98.0, 96.0, 112.0, 110.0]]), 0, "nothing is left under the mark"); } +#[test] +fn a_mark_whose_subtype_is_an_indirect_name_is_applied() { + // `/Subtype` may be an indirect reference (ISO 32000-2 §7.3.10); the audit reads it that way. + // Passing over the mark would leave the text under it while the others report success. + let objs: Vec> = vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> /Annots [7 0 R] >>".to_vec(), + stream("", b"BT /F1 10 Tf 20 20 Td (SECRET) Tj ET"), + FONT.replace("95 0 R", "6 0 R").into_bytes(), + widths(), + b"<< /Type /Annot /Subtype 8 0 R /Rect [10 10 90 40] >>".to_vec(), + b"/Redact".to_vec(), + ]; + let mut doc = pdf(objs); + assert_eq!(marks(&doc).len(), 1, "the mark is found"); + let r = apply(&mut doc, None).unwrap(); + assert_eq!(r.marks, 1); + assert!(r.glyphs >= 6, "the text under the mark went: {r:?}"); + assert!(marks(&doc).is_empty(), "the mark itself went"); +} + +#[test] +fn a_mark_written_in_place_with_an_indirect_subtype_is_refused() { + let objs: Vec> = vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Annots [<< /Type /Annot /Subtype 5 0 R /Rect [10 10 70 40] >>] >>" + .to_vec(), + stream("", b"BT ET"), + b"/Redact".to_vec(), + ]; + let mut doc = pdf(objs); + mark(&mut doc, 0, &[[100.0, 100.0, 120.0, 120.0]], ""); + assert!(matches!(apply(&mut doc, None), Err(RedactError::UnreadableMark(1)))); +} + +#[test] +fn annotations_written_in_place_go_when_covered_or_hanging_off_a_removed_one() { + // `/Annots` may hold dictionaries rather than references. One under the mark, a reply to a + // removed highlight on the same page and on another, and a reply to that inline comment's + // pop-up must all go; an inline comment elsewhere stays. + let objs: Vec> = vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R 4 0 R] /Count 2 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 5 0 R /Annots [6 0 R \ + << /Type /Annot /Subtype /Text /Rect [30 15 40 25] /Contents (INLINE-UNDER) /Popup 7 0 R >> \ + << /Type /Annot /Subtype /Text /Rect [250 250 270 270] /IRT 6 0 R /Contents (INLINE-REPLY) >> \ + << /Type /Annot /Subtype /Text /Rect [250 100 270 120] /Contents (INLINE-KEPT) >> 7 0 R] >>" + .to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 5 0 R /Annots [\ + << /Type /Annot /Subtype /Text /Rect [250 250 270 270] /IRT 6 0 R /Contents (OTHER-PAGE-INLINE-REPLY) >> \ + << /Type /Annot /Subtype /Text /Rect [200 200 220 220] /IRT 7 0 R /Contents (POPUP-INLINE-REPLY) >>] >>" + .to_vec(), + stream("", b"BT ET"), + b"<< /Type /Annot /Subtype /Highlight /Rect [20 20 60 30] /QuadPoints [20 30 60 30 20 20 60 20] /Contents (HIGHLIGHT-NOTE) >>".to_vec(), + b"<< /Type /Annot /Subtype /Popup /Rect [200 150 290 190] /Contents (INLINE-POPUP) >>".to_vec(), + ]; + let mut doc = pdf(objs); + mark(&mut doc, 0, &[[10.0, 10.0, 70.0, 40.0]], ""); + let r = apply(&mut doc, None).unwrap(); + assert_eq!(r.annotations, 5, "the highlight, the inline comment and the three replies: {r:?}"); + let pages = pdfcraft_model::pages(&doc); + assert_eq!(annots_of(&doc, &pages[0].dict).len(), 1, "only the inline comment elsewhere stays"); + assert!(annots_of(&doc, &pages[1].dict).is_empty(), "the replies leave page 2"); + let bytes = write_full(&doc, &SaveOptions { object_streams: false, ..SaveOptions::default() }).unwrap(); + for gone in [&b"HIGHLIGHT-NOTE"[..], b"INLINE-UNDER", b"INLINE-REPLY", b"OTHER-PAGE-INLINE-REPLY", b"POPUP-INLINE-REPLY", b"INLINE-POPUP"] { + assert!(!bytes.windows(gone.len()).any(|w| w == gone), "{} survived the save", String::from_utf8_lossy(gone)); + } + assert!(bytes.windows(11).any(|w| w == b"INLINE-KEPT")); +} + #[test] fn a_rewritten_form_xobject_drops_the_source_text_of_added_content() { // A form drawing invisible text whose dictionary repeats it in /PCAdded. @@ -6717,6 +6789,49 @@ fn a_redacted_form_xobject_drops_the_source_text_of_added_content() { assert!(!bytes.windows(4).any(|w| w == b"1234"), "redacted text left in the saved file"); } +#[test] +fn an_inline_widget_takes_its_field_even_with_no_referenced_widget() { + // The field's only widget is written in place in `/Annots`, so no referenced widget names it: + // the field (and its value) must still go with the widget under the mark. + let objs: Vec> = vec![ + b"<< /Type /Catalog /Pages 2 0 R /AcroForm << /Fields [6 0 R] >> >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Annots [<< /Type /Annot /Subtype /Widget /Rect [20 20 60 30] /Parent 6 0 R >>] >>" + .to_vec(), + stream("", b"BT ET"), + b"<< >>".to_vec(), + b"<< /FT /Tx /T (name) /V (FIELD-SECRET) >>".to_vec(), + ]; + let mut doc = pdf(objs); + mark(&mut doc, 0, &[[10.0, 10.0, 70.0, 40.0]], ""); + let r = apply(&mut doc, None).unwrap(); + assert_eq!(r.fields, 1, "{r:?}"); + let bytes = write_full(&doc, &SaveOptions { object_streams: false, ..SaveOptions::default() }).unwrap(); + assert!(!bytes.windows(12).any(|w| w == b"FIELD-SECRET"), "the field value survived the save"); +} + +#[test] +fn an_inline_widget_of_the_field_on_another_page_goes_with_it() { + // Two pages: the widget under the mark, and a sibling widget in place on the second page with the + // same `/Parent`. Deleting the field must not leave the sibling naming it. + let objs: Vec> = vec![ + b"<< /Type /Catalog /Pages 2 0 R /AcroForm << /Fields [7 0 R] >> >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R 5 0 R] /Count 2 >>".to_vec(), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Annots [<< /Type /Annot /Subtype /Widget /Rect [20 20 60 30] /Parent 7 0 R >>] >>" + .to_vec(), + stream("", b"BT ET"), + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Annots [<< /Type /Annot /Subtype /Widget /Rect [200 200 260 230] /Parent 7 0 R >>] >>" + .to_vec(), + b"<< >>".to_vec(), + b"<< /FT /Tx /T (name) /V (SIBLING-SECRET) >>".to_vec(), + ]; + let mut doc = pdf(objs); + mark(&mut doc, 0, &[[10.0, 10.0, 70.0, 40.0]], ""); + apply(&mut doc, None).unwrap(); + let bytes = write_full(&doc, &SaveOptions { object_streams: false, ..SaveOptions::default() }).unwrap(); + assert!(!bytes.windows(14).any(|w| w == b"SIBLING-SECRET"), "the field value survived through the sibling widget"); +} + /// Private data takes an image's prepress data too: its OPI link to the original file and its /// alternate images, which can be the unedited original. #[test] From 0b0519f5df2056b9a9430d72021b88bd7b779cd6 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 10:57:05 -0500 Subject: [PATCH 40/60] fix(forms): deleting a field removes ancestors it leaves without kids (#6997) * forms: deleting a field removes ancestors it leaves without kids * forms test: full-save check, widget only in the sibling case --- crates/forms/src/author/lifecycle.rs | 18 +++++++++++++++- crates/forms/src/tests.rs | 32 ++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/crates/forms/src/author/lifecycle.rs b/crates/forms/src/author/lifecycle.rs index 50d552ccdb..28b56749d9 100644 --- a/crates/forms/src/author/lifecycle.rs +++ b/crates/forms/src/author/lifecycle.rs @@ -262,10 +262,26 @@ pub fn delete_widget(doc: &mut Document, name: &str, widget: usize) -> Result<() fn delete_found_field(doc: &mut Document, f: Field) -> Result<(), FormError> { let widgets: std::collections::HashSet = f.widgets.iter().map(|w| w.obj).collect(); drop_from_pages(doc, &widgets)?; + let mut parent = doc.get(f.obj).as_dict().and_then(|d| d.reference(b"Parent")); detach(doc, f.obj)?; // Out of the calculation order too: a `/CO` entry would keep the field, its value and its // widgets reachable for a full save. - drop_from_calculation_order(doc, |r| r == f.obj || widgets.contains(&r)) + drop_from_calculation_order(doc, |r| r == f.obj || widgets.contains(&r))?; + // An ancestor left with no kids would become a field of its own, and keep the `/V` and `/DV` + // the deleted field inherited from it: it goes too, up the chain. + let mut gone = std::collections::HashSet::new(); + while let Some(p) = parent.filter(|p| gone.len() < 64 && !gone.contains(p)) { + let Some(d) = doc.get(p).as_dict().cloned() else { break }; + let empty = d.get(b"Kids").map(|o| doc.resolve(o)).and_then(|o| o.as_array().map(Vec::is_empty)) == Some(true); + if !empty { + break; + } + parent = d.reference(b"Parent"); + gone.insert(p); + drop_from_pages(doc, &std::iter::once(p).collect())?; + detach(doc, p)?; + } + drop_from_calculation_order(doc, |r| gone.contains(&r)) } /// Take the annotations `widgets` off every page's `/Annots`, held directly or by reference. diff --git a/crates/forms/src/tests.rs b/crates/forms/src/tests.rs index caeba9491d..75dd075889 100644 --- a/crates/forms/src/tests.rs +++ b/crates/forms/src/tests.rs @@ -1664,6 +1664,38 @@ fn properties_rename_and_delete() { assert_eq!(annots.as_array().unwrap().len(), 13 - 3, "the deleted widgets left the page"); } +/// A deleted field's ancestor left with no kids stays in `/Fields` as a field of its own, with the +/// `/V` the deleted field inherited from it: it leaves too. An ancestor a sibling still has stays. +#[test] +fn deleting_the_last_kid_removes_the_emptied_ancestors_and_their_values() { + let form = |kids: &str, annots: &str| { + document(&[ + "<< /Type /Catalog /Pages 2 0 R /AcroForm 4 0 R >>".into(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 600 800] >>".into(), + format!("<< /Type /Page /Parent 2 0 R /Annots [{annots}] >>"), + "<< /Fields [5 0 R] >>".into(), + "<< /FT /Tx /T (outer) /V (SECRET) /DV (SECRET) /Kids [6 0 R] >>".into(), + format!("<< /T (inner) /Parent 5 0 R /Kids [{kids}] >>"), + "<< /Type /Annot /Subtype /Widget /T (a) /Parent 6 0 R /Rect [50 700 250 720] /P 3 0 R >>".into(), + "<< /Type /Annot /Subtype /Widget /T (b) /Parent 6 0 R /Rect [50 650 250 670] /P 3 0 R >>".into(), + ]) + }; + let top = |doc: &Document| { + let af = doc.get(doc.root().unwrap()).as_dict().unwrap().get(b"AcroForm").map(|o| doc.resolve(o)).unwrap().as_dict().unwrap().clone(); + doc.resolve(af.get(b"Fields").unwrap()).as_array().unwrap().len() + }; + let mut doc = form("7 0 R", "7 0 R"); + delete_field(&mut doc, "outer.inner.a").unwrap(); + assert_eq!(top(&doc), 0, "the emptied ancestors left /Fields"); + let saved = pdfcraft_cos::write_full(&doc, &SaveOptions::default()).unwrap(); + assert!(!saved.windows(6).any(|w| w == b"SECRET"), "the inherited value is gone from a full save"); + + let mut doc = form("7 0 R 8 0 R", "7 0 R 8 0 R"); + delete_field(&mut doc, "outer.inner.a").unwrap(); + assert_eq!(top(&doc), 1, "an ancestor with a surviving kid stays"); + assert!(fields(&doc).iter().any(|f| f.name == "outer.inner.b")); +} + /// A parent `MASTER` with kids `a` and `b` (each a field merged with its widget) and `e` (a field /// with two widget kids), and the top-level fields `c` and `d`. fn dotted_form() -> Document { From f44ea8380d996351069835c09e86dd8cd3a8a708 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 10:57:36 -0500 Subject: [PATCH 41/60] redact: a covered clip-and-paint path leaves an empty clip, not its outline (#6996) --- crates/redact/src/interp.rs | 9 ++++++--- crates/redact/src/tests.rs | 14 ++++++++++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/crates/redact/src/interp.rs b/crates/redact/src/interp.rs index a12fa1c041..80117f451d 100644 --- a/crates/redact/src/interp.rs +++ b/crates/redact/src/interp.rs @@ -648,9 +648,12 @@ impl Walk { piece.changed = true; scope.report.paths_removed += 1; if scope.covered(b) && clips { - // Keep the clipping, drop the painting. - piece.ops.extend(body); - piece.ops.push(Op::new("n", vec![])); + // Everything this clip lets through is under a mark, so an empty clip hides the same + // content. The original outline is confidential and must not stay in the stream. + for o in ["re", "W", "n"] { + let operands = if o == "re" { vec![num(0.0); 4] } else { vec![] }; + piece.ops.push(Op::new(o, operands)); + } } // Partial vector redaction must not retain underlying drawing commands // behind clipping (F-008). Refuse partial clip retention and remove the path. diff --git a/crates/redact/src/tests.rs b/crates/redact/src/tests.rs index 4ad4103e10..dc94a2c5f5 100644 --- a/crates/redact/src/tests.rs +++ b/crates/redact/src/tests.rs @@ -1057,6 +1057,20 @@ fn partially_covered_vector_paths_do_not_leak_drawing_commands() { assert!(c.contains("300 300 m\n350 350 l\nS"), "unaffected vector path must be kept: {c}"); } +#[test] +fn a_covered_clip_and_paint_path_leaves_an_empty_clip_not_its_outline() { + // The outline of a fully covered path that clips and paints is confidential geometry: only + // an empty clip may stay, and what follows is hidden as the covered clip would have hidden it. + let src = b"q 100 100 m 150 180 l 200 100 l h W f 300 300 50 50 re f Q"; + let mut doc = one_page(src, "", vec![]); + mark(&mut doc, 0, &[[90.0, 90.0, 210.0, 190.0]], ""); + let r = apply(&mut doc, None).unwrap(); + assert_eq!(r.paths_removed, 1); + let c = content(&doc, 0); + assert!(!c.contains("150 180 l") && !c.contains("100 100 m"), "outline must not stay: {c}"); + assert!(c.contains("0 0 0 0 re\nW\nn"), "an empty clip stays: {c}"); +} + /// A chain of nested forms deeper than the pass can walk was removed from the page, taking /// whatever else it drew with it even when nothing in it was marked. The limit now reaches as /// deep as the renderer and Edit text read (50), so anything a reader can see can be walked; From 3650f01f233916fc9d14d94637c6a9ce8e228f03 Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Wed, 11 Nov 2026 17:58:30 +0100 Subject: [PATCH 42/60] Release: PdfCraft v0.21.0 (#6995) --- Cargo.lock | 78 +++++++++++++++++++++++++++--------------------------- Cargo.toml | 72 ++++++++++++++++++++++++------------------------- 2 files changed, 75 insertions(+), 75 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9cf008f02e..c4e08c1086 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4263,7 +4263,7 @@ dependencies = [ [[package]] name = "pdfcraft" -version = "0.18.1" +version = "0.21.0" dependencies = [ "chrono", "craft-update", @@ -4288,7 +4288,7 @@ dependencies = [ [[package]] name = "pdfcraft-a11y" -version = "0.18.1" +version = "0.21.0" dependencies = [ "pdfcraft-content", "pdfcraft-cos", @@ -4299,7 +4299,7 @@ dependencies = [ [[package]] name = "pdfcraft-annot" -version = "0.18.1" +version = "0.21.0" dependencies = [ "hayro-syntax", "pdfcraft-cos", @@ -4311,7 +4311,7 @@ dependencies = [ [[package]] name = "pdfcraft-audit" -version = "0.18.1" +version = "0.21.0" dependencies = [ "pdfcraft-content", "pdfcraft-cos", @@ -4322,7 +4322,7 @@ dependencies = [ [[package]] name = "pdfcraft-automation" -version = "0.18.1" +version = "0.21.0" dependencies = [ "base64 0.23.1", "flate2", @@ -4346,7 +4346,7 @@ dependencies = [ [[package]] name = "pdfcraft-cli" -version = "0.18.1" +version = "0.21.0" dependencies = [ "base64 0.23.1", "pdfcraft-audit", @@ -4364,11 +4364,11 @@ dependencies = [ [[package]] name = "pdfcraft-compare" -version = "0.18.1" +version = "0.21.0" [[package]] name = "pdfcraft-content" -version = "0.18.1" +version = "0.21.0" dependencies = [ "hayro-syntax", "pdfcraft-cos", @@ -4376,7 +4376,7 @@ dependencies = [ [[package]] name = "pdfcraft-cos" -version = "0.18.1" +version = "0.21.0" dependencies = [ "chrono", "hayro-syntax", @@ -4393,7 +4393,7 @@ dependencies = [ [[package]] name = "pdfcraft-create" -version = "0.18.1" +version = "0.21.0" dependencies = [ "encoding_rs", "hayro-ccitt", @@ -4413,7 +4413,7 @@ dependencies = [ [[package]] name = "pdfcraft-crypt" -version = "0.18.1" +version = "0.21.0" dependencies = [ "aes", "md-5", @@ -4426,7 +4426,7 @@ dependencies = [ [[package]] name = "pdfcraft-edit" -version = "0.18.1" +version = "0.21.0" dependencies = [ "hayro-syntax", "icu_segmenter", @@ -4445,7 +4445,7 @@ dependencies = [ [[package]] name = "pdfcraft-engine" -version = "0.18.1" +version = "0.21.0" dependencies = [ "chrono", "craft-docpath", @@ -4489,14 +4489,14 @@ dependencies = [ [[package]] name = "pdfcraft-export" -version = "0.18.1" +version = "0.21.0" dependencies = [ "flate2", ] [[package]] name = "pdfcraft-filters" -version = "0.18.1" +version = "0.21.0" dependencies = [ "fax", "flate2", @@ -4509,7 +4509,7 @@ dependencies = [ [[package]] name = "pdfcraft-fonts" -version = "0.18.1" +version = "0.21.0" dependencies = [ "harfrust", "pdfcraft-cos", @@ -4519,7 +4519,7 @@ dependencies = [ [[package]] name = "pdfcraft-forms" -version = "0.18.1" +version = "0.21.0" dependencies = [ "hayro-syntax", "pdfcraft-content", @@ -4532,11 +4532,11 @@ dependencies = [ [[package]] name = "pdfcraft-geom" -version = "0.18.1" +version = "0.21.0" [[package]] name = "pdfcraft-js" -version = "0.18.1" +version = "0.21.0" dependencies = [ "boa_engine", "chrono", @@ -4544,7 +4544,7 @@ dependencies = [ [[package]] name = "pdfcraft-measure" -version = "0.18.1" +version = "0.21.0" dependencies = [ "pdfcraft-annot", "pdfcraft-content", @@ -4557,14 +4557,14 @@ dependencies = [ [[package]] name = "pdfcraft-model" -version = "0.18.1" +version = "0.21.0" dependencies = [ "pdfcraft-cos", ] [[package]] name = "pdfcraft-ocr" -version = "0.18.1" +version = "0.21.0" dependencies = [ "ocrs", "pdfcraft-fonts", @@ -4579,7 +4579,7 @@ dependencies = [ [[package]] name = "pdfcraft-ocr-worker" -version = "0.18.1" +version = "0.21.0" dependencies = [ "js-sys", "pdfcraft-ocr", @@ -4589,7 +4589,7 @@ dependencies = [ [[package]] name = "pdfcraft-optimize" -version = "0.18.1" +version = "0.21.0" dependencies = [ "image", "pdfcraft-annot", @@ -4606,11 +4606,11 @@ dependencies = [ [[package]] name = "pdfcraft-oracle" -version = "0.18.1" +version = "0.21.0" [[package]] name = "pdfcraft-organize" -version = "0.18.1" +version = "0.21.0" dependencies = [ "hayro-syntax", "pdfcraft-content", @@ -4621,7 +4621,7 @@ dependencies = [ [[package]] name = "pdfcraft-platform" -version = "0.18.1" +version = "0.21.0" dependencies = [ "async-channel", "futures-lite", @@ -4632,7 +4632,7 @@ dependencies = [ [[package]] name = "pdfcraft-prc" -version = "0.18.1" +version = "0.21.0" dependencies = [ "flate2", "proptest", @@ -4641,7 +4641,7 @@ dependencies = [ [[package]] name = "pdfcraft-prc-render" -version = "0.18.1" +version = "0.21.0" dependencies = [ "flate2", "pdfcraft-prc", @@ -4650,7 +4650,7 @@ dependencies = [ [[package]] name = "pdfcraft-preflight" -version = "0.18.1" +version = "0.21.0" dependencies = [ "flate2", "pdfcraft-content", @@ -4661,7 +4661,7 @@ dependencies = [ [[package]] name = "pdfcraft-print" -version = "0.18.1" +version = "0.21.0" dependencies = [ "craft-print", "log", @@ -4676,7 +4676,7 @@ dependencies = [ [[package]] name = "pdfcraft-redact" -version = "0.18.1" +version = "0.21.0" dependencies = [ "aho-corasick", "pdfcraft-annot", @@ -4700,7 +4700,7 @@ dependencies = [ [[package]] name = "pdfcraft-render" -version = "0.18.1" +version = "0.21.0" dependencies = [ "bytemuck", "flate2", @@ -4724,7 +4724,7 @@ dependencies = [ [[package]] name = "pdfcraft-sign" -version = "0.18.1" +version = "0.21.0" dependencies = [ "aes", "aws-lc-rs", @@ -4763,11 +4763,11 @@ dependencies = [ [[package]] name = "pdfcraft-testkit" -version = "0.18.1" +version = "0.21.0" [[package]] name = "pdfcraft-ui-egui" -version = "0.18.1" +version = "0.21.0" dependencies = [ "arboard", "base64 0.23.1", @@ -4826,7 +4826,7 @@ dependencies = [ [[package]] name = "pdfcraft-web" -version = "0.18.1" +version = "0.21.0" dependencies = [ "craft-web-api", "craft-web-host", @@ -4841,7 +4841,7 @@ dependencies = [ [[package]] name = "pdfcraft-xfa" -version = "0.18.1" +version = "0.21.0" dependencies = [ "base64 0.23.1", "image", @@ -4855,7 +4855,7 @@ dependencies = [ [[package]] name = "pdfcraft-xfdf" -version = "0.18.1" +version = "0.21.0" dependencies = [ "base64 0.23.1", "pdfcraft-annot", diff --git a/Cargo.toml b/Cargo.toml index d566a69d1f..eaf28264e5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ exclude = ["vendor"] default-members = ["crates/*", "apps/pdfcraft", "apps/pdfcraft-cli", "xtask"] [workspace.package] -version = "0.18.1" +version = "0.21.0" edition = "2024" license = "MIT OR Apache-2.0" rust-version = "1.95" @@ -32,41 +32,41 @@ craft-docpath = { version = "0.1.0", git = "https://github.com/storytold/craft-l # craft-update: app-specific pretend versions apply to startup and manual checks. craft-update = { git = "https://github.com/storytold/craft-libs", version = "=0.1.0", rev = "1b34eba41e62138fc2438d2734523b302840b45d", features = ["egui"] } # internal -pdfcraft-geom = { path = "crates/geom", version = "0.18.1" } -pdfcraft-filters = { path = "crates/filters", version = "0.18.1" } -pdfcraft-cos = { path = "crates/cos", version = "0.18.1" } -pdfcraft-crypt = { path = "crates/crypt", version = "0.18.1" } -pdfcraft-organize = { path = "crates/organize", version = "0.18.1" } -pdfcraft-measure = { path = "crates/measure", version = "0.18.1" } -pdfcraft-annot = { path = "crates/annot", version = "0.18.1" } -pdfcraft-audit = { path = "crates/audit", version = "0.18.1" } -pdfcraft-fonts = { path = "crates/fonts", version = "0.18.1" } -pdfcraft-model = { path = "crates/model", version = "0.18.1" } -pdfcraft-content = { path = "crates/content", version = "0.18.1" } -pdfcraft-redact = { path = "crates/redact", version = "0.18.1" } -pdfcraft-print = { path = "crates/print", version = "0.18.1" } -pdfcraft-xfdf = { path = "crates/xfdf", version = "0.18.1" } -pdfcraft-sign = { path = "crates/sign", version = "0.18.1" } -pdfcraft-optimize = { path = "crates/optimize", version = "0.18.1" } -pdfcraft-a11y = { path = "crates/a11y", version = "0.18.1" } -pdfcraft-oracle = { path = "crates/oracle", version = "0.18.1" } -pdfcraft-testkit = { path = "crates/testkit", version = "0.18.1" } -pdfcraft-ocr = { path = "crates/ocr", version = "0.18.1" } -pdfcraft-js = { path = "crates/js", version = "0.18.1" } -pdfcraft-compare = { path = "crates/compare", version = "0.18.1" } -pdfcraft-preflight = { path = "crates/preflight", version = "0.18.1" } -pdfcraft-export = { path = "crates/export", version = "0.18.1" } -pdfcraft-edit = { path = "crates/edit", version = "0.18.1" } -pdfcraft-create = { path = "crates/create", version = "0.18.1" } -pdfcraft-forms = { path = "crates/forms", version = "0.18.1" } -pdfcraft-xfa = { path = "crates/xfa", version = "0.18.1" } -pdfcraft-render = { path = "crates/render", version = "0.18.1" } -pdfcraft-engine = { path = "crates/engine", version = "0.18.1" } -pdfcraft-platform = { path = "crates/platform", version = "0.18.1" } -pdfcraft-ui-egui = { path = "crates/ui-egui", version = "0.18.1" } -pdfcraft-automation = { path = "crates/automation", version = "0.18.1", default-features = false } -pdfcraft-prc = { path = "crates/prc", version = "0.18.1" } -pdfcraft-prc-render = { path = "crates/prc-render", version = "0.18.1" } +pdfcraft-geom = { path = "crates/geom", version = "0.21.0" } +pdfcraft-filters = { path = "crates/filters", version = "0.21.0" } +pdfcraft-cos = { path = "crates/cos", version = "0.21.0" } +pdfcraft-crypt = { path = "crates/crypt", version = "0.21.0" } +pdfcraft-organize = { path = "crates/organize", version = "0.21.0" } +pdfcraft-measure = { path = "crates/measure", version = "0.21.0" } +pdfcraft-annot = { path = "crates/annot", version = "0.21.0" } +pdfcraft-audit = { path = "crates/audit", version = "0.21.0" } +pdfcraft-fonts = { path = "crates/fonts", version = "0.21.0" } +pdfcraft-model = { path = "crates/model", version = "0.21.0" } +pdfcraft-content = { path = "crates/content", version = "0.21.0" } +pdfcraft-redact = { path = "crates/redact", version = "0.21.0" } +pdfcraft-print = { path = "crates/print", version = "0.21.0" } +pdfcraft-xfdf = { path = "crates/xfdf", version = "0.21.0" } +pdfcraft-sign = { path = "crates/sign", version = "0.21.0" } +pdfcraft-optimize = { path = "crates/optimize", version = "0.21.0" } +pdfcraft-a11y = { path = "crates/a11y", version = "0.21.0" } +pdfcraft-oracle = { path = "crates/oracle", version = "0.21.0" } +pdfcraft-testkit = { path = "crates/testkit", version = "0.21.0" } +pdfcraft-ocr = { path = "crates/ocr", version = "0.21.0" } +pdfcraft-js = { path = "crates/js", version = "0.21.0" } +pdfcraft-compare = { path = "crates/compare", version = "0.21.0" } +pdfcraft-preflight = { path = "crates/preflight", version = "0.21.0" } +pdfcraft-export = { path = "crates/export", version = "0.21.0" } +pdfcraft-edit = { path = "crates/edit", version = "0.21.0" } +pdfcraft-create = { path = "crates/create", version = "0.21.0" } +pdfcraft-forms = { path = "crates/forms", version = "0.21.0" } +pdfcraft-xfa = { path = "crates/xfa", version = "0.21.0" } +pdfcraft-render = { path = "crates/render", version = "0.21.0" } +pdfcraft-engine = { path = "crates/engine", version = "0.21.0" } +pdfcraft-platform = { path = "crates/platform", version = "0.21.0" } +pdfcraft-ui-egui = { path = "crates/ui-egui", version = "0.21.0" } +pdfcraft-automation = { path = "crates/automation", version = "0.21.0", default-features = false } +pdfcraft-prc = { path = "crates/prc", version = "0.21.0" } +pdfcraft-prc-render = { path = "crates/prc-render", version = "0.21.0" } # pdf hayro = "0.7.1" From 619a239cebe25054ba59486ce2b878b66613c0ce Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 12:12:27 -0500 Subject: [PATCH 43/60] ci: run Harden-Runner on every job (#7003) Adds step-security/harden-runner (audit, pinned to v2.22.4 like craft-common-actions) as the first step of each job that did not have it, so every workflow in the org's public repos reports its egress to StepSecurity. Audit, not block: block-mode allow lists come from each job's observed traffic. --- .github/workflows/ci.yml | 4 ++++ .github/workflows/fuzz.yml | 4 ++++ .github/workflows/packaging-lint.yml | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 94a4c545fe..38ee53caf3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,6 +29,10 @@ jobs: permissions: contents: read steps: + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 + with: + egress-policy: audit - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index b7b1d409c8..fe0a0074d7 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -24,6 +24,10 @@ jobs: name: fuzz (mutation, 15 min) runs-on: ubuntu-latest steps: + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 + with: + egress-policy: audit - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false diff --git a/.github/workflows/packaging-lint.yml b/.github/workflows/packaging-lint.yml index ea61d84303..1771a35fea 100644 --- a/.github/workflows/packaging-lint.yml +++ b/.github/workflows/packaging-lint.yml @@ -27,6 +27,10 @@ jobs: name: Windows installation policy (no app build) runs-on: windows-latest steps: + - name: Harden runner + uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4 + with: + egress-policy: audit - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false From d4608308591dc566dd31bd1486801cc48d14209f Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 12:44:55 -0500 Subject: [PATCH 44/60] Updates: take craft-update from craft-libs main at 6e03f65bf --- Cargo.lock | 3 ++- Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9cf008f02e..5d4d1fb2b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1289,7 +1289,7 @@ dependencies = [ [[package]] name = "craft-update" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=1b34eba41e62138fc2438d2734523b302840b45d#1b34eba41e62138fc2438d2734523b302840b45d" +source = "git+https://github.com/storytold/craft-libs?rev=6e03f65bf45181db77ba6229716e6624db9eb25b#6e03f65bf45181db77ba6229716e6624db9eb25b" dependencies = [ "chrono", "dunce", @@ -1299,6 +1299,7 @@ dependencies = [ "image", "log", "plist", + "rustix 1.1.6", "semver", "serde", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index d566a69d1f..d850df512f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,7 +30,7 @@ craft-print = { git = "https://github.com/storytold/craft-libs", version = "0.1. # Whether a path stored in a document may be opened (GoToR link targets). craft-docpath = { version = "0.1.0", git = "https://github.com/storytold/craft-libs", rev = "787a7a0f972fd1d7a7618fdbeca3a5267a0407ff" } # craft-update: app-specific pretend versions apply to startup and manual checks. -craft-update = { git = "https://github.com/storytold/craft-libs", version = "=0.1.0", rev = "1b34eba41e62138fc2438d2734523b302840b45d", features = ["egui"] } +craft-update = { git = "https://github.com/storytold/craft-libs", version = "=0.1.0", rev = "6e03f65bf45181db77ba6229716e6624db9eb25b", features = ["egui"] } # internal pdfcraft-geom = { path = "crates/geom", version = "0.18.1" } pdfcraft-filters = { path = "crates/filters", version = "0.18.1" } From 583ae295993ab0c944ebb87d50f7db6acb289d1c Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 14:46:34 -0500 Subject: [PATCH 45/60] fix(sign): a widget removed from a page's inline Annots is not a comment change (#6013) --- crates/sign/tests/it/pdf.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/crates/sign/tests/it/pdf.rs b/crates/sign/tests/it/pdf.rs index e765941c9a..fa45ab1ecb 100644 --- a/crates/sign/tests/it/pdf.rs +++ b/crates/sign/tests/it/pdf.rs @@ -325,6 +325,21 @@ fn an_indirect_annots_array_is_judged_by_the_annotations_it_gains() { assert_eq!((s.status, s.modification), (Status::Invalid, Modification::Disallowed(vec![kind::OTHER]))); } +/// A signature widget taken off the page's inline /Annots array is covered by no permission (the +/// indirect-array case is the test above), and stays a change whatever level the signature has. +#[test] +fn removing_a_widget_from_an_inline_annots_array_is_not_a_comment_change() { + let id = pkcs12::open(&data("rsa-aes.p12"), "test").unwrap(); + let trust = TrustStore { certs: vec![id.certificate.clone()] }; + let page = pdfcraft_cos::ObjRef { num: 3, generation: 0 }; + let signed = pdfcraft_sign::sign(&open(&fixture()), &id, &opts()).unwrap(); + let inline = open(&signed).get(page).as_dict().and_then(|d| d.get(b"Annots").cloned()).unwrap(); + assert!(matches!(inline, Object::Array(ref a) if !a.is_empty()), "the page keeps its widgets inline"); + let stripped = edit_after(&signed, |doc| doc.update_dict(page, |p| p.set(b"Annots".to_vec(), Object::Array(vec![]))).unwrap()); + let s = signatures(&open(&stripped), &stripped, &trust).into_iter().find(|s| s.signed).unwrap(); + assert_eq!((s.status, s.modification), (Status::Invalid, Modification::Disallowed(vec![kind::OTHER]))); +} + /// Issue #1328: a signature field whose /V is null, directly or through a reference, is unsigned: /// it is listed as unsigned and can be signed by name. #[test] From d46d9daea5b0b3bb316bb5889b1757df5e2512ab Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 14:46:47 -0500 Subject: [PATCH 46/60] fix(edit): editor font enumeration reads a face name without building font metrics (#6086) --- crates/edit/src/text/editor.rs | 4 ++-- crates/fonts/src/pdf.rs | 25 +++++++++++++++---------- crates/fonts/src/pdf/tests.rs | 10 ++++++++++ 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/crates/edit/src/text/editor.rs b/crates/edit/src/text/editor.rs index 4f4abf3a43..e6df1dbfee 100644 --- a/crates/edit/src/text/editor.rs +++ b/crates/edit/src/text/editor.rs @@ -2,7 +2,7 @@ use super::blocks::{BlockStyle, TextBlock}; use super::fallback::MAX_TYPE3_GLYPHS; use super::lines::{page_dict, text_lines}; use super::rewrite::rewrite_block_from; -use super::{DecodeBudget, Dict, Document, EditError, Metrics, Object, TextLine, TextSession}; +use super::{DecodeBudget, Dict, Document, EditError, Object, TextLine, TextSession}; /// An embedded source font or an approved bundled document-font choice. /// Source bytes stay in memory and are never installed or copied into another document. @@ -45,7 +45,7 @@ pub fn text_editor_fonts(doc: &Document, page: usize) -> Result Option { } } +/// The font's `/BaseFont` (a Type 3 font's descriptor `/FontName` when it has none), read without +/// building its metrics, so it decodes no CMap. +pub fn base_font_name(doc: &Document, font: &Dict) -> String { + font.name(b"BaseFont") + .map(|name| String::from_utf8_lossy(name).into_owned()) + .or_else(|| { + (font.name(b"Subtype") == Some(b"Type3")) + .then(|| dict(doc, font.get(b"FontDescriptor"))) + .flatten() + .and_then(|d| d.name(b"FontName").map(|name| String::from_utf8_lossy(name).into_owned())) + }) + .unwrap_or_default() +} + /// Codespace ranges and CID mappings from an embedded CMap stream, at most /// [`MAX_CMAP_RANGES`] of each (every glyph searches them). #[allow(clippy::type_complexity)] @@ -909,16 +923,7 @@ impl Metrics { pub fn from_dict_within(doc: &Document, font: &Dict, budget: &mut DecodeBudget) -> Self { let mut reads = Reads::new(doc, budget); let mut m = Self::read_metrics(doc, font, &mut reads); - let base = font - .name(b"BaseFont") - .map(|name| String::from_utf8_lossy(name).into_owned()) - .or_else(|| { - (font.name(b"Subtype") == Some(b"Type3")) - .then(|| dict(doc, font.get(b"FontDescriptor"))) - .flatten() - .and_then(|d| d.name(b"FontName").map(|name| String::from_utf8_lossy(name).into_owned())) - }) - .unwrap_or_default(); + let base = base_font_name(doc, font); m.subset = base.len() > 7 && base.as_bytes()[6] == b'+' && base[..6].bytes().all(|b| b.is_ascii_uppercase()); let (bold, italic) = style_from_name(&base); m.bold |= bold; diff --git a/crates/fonts/src/pdf/tests.rs b/crates/fonts/src/pdf/tests.rs index 4c5f90044f..74880c94a4 100644 --- a/crates/fonts/src/pdf/tests.rs +++ b/crates/fonts/src/pdf/tests.rs @@ -1823,6 +1823,16 @@ fn low_codes_resolve_to_the_covering_range() { assert_eq!((u.covering(0xFFFF).is_some(), u.covering(0x1_0001).is_some(), u.covering(0x1_0002).is_none()), (true, true, true)); } +#[test] +fn base_font_name_matches_the_metrics_name() { + let doc = Document::new_empty(); + let mut f = Dict::new(); + f.set(b"BaseFont".to_vec(), Object::name("ABCDEF+Body")); + assert_eq!(base_font_name(&doc, &f), "ABCDEF+Body"); + assert_eq!(base_font_name(&doc, &f), Metrics::from_dict(&doc, &f).base_font); + assert_eq!(base_font_name(&doc, &Dict::new()), ""); +} + #[test] fn refused_subset_reads_do_not_encode_unverified_glyphs() { let mut doc = Document::new_empty(); From 683ccc9fcca358568605e7226ae65a9447135fce Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 14:47:00 -0500 Subject: [PATCH 47/60] fix(optimize): walk shared resource dictionaries and name-tree kids once (#5956) * fix(optimize): walk shared resource dictionaries and name-tree kids once * test(optimize): restore the file around the appended tests * address review: import ObjRef in the optimize audit module --- crates/optimize/src/audit.rs | 31 ++++++++++----- crates/optimize/src/links.rs | 8 +++- crates/optimize/tests/optimize.rs | 64 +++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 12 deletions(-) diff --git a/crates/optimize/src/audit.rs b/crates/optimize/src/audit.rs index 789bf2e89d..b06d76d6c7 100644 --- a/crates/optimize/src/audit.rs +++ b/crates/optimize/src/audit.rs @@ -9,7 +9,7 @@ use std::collections::{HashMap, HashSet}; -use pdfcraft_cos::{Dict, Document, MAX_DECODED, Object, XrefEntry}; +use pdfcraft_cos::{Dict, Document, MAX_DECODED, ObjRef, Object, XrefEntry}; /// The categories, in Acrobat's order. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] @@ -91,6 +91,9 @@ const MAX_FORM_DEPTH: usize = 64; struct Audit<'a> { doc: &'a Document, owner: HashMap, + /// Resource dictionaries (by reference) already walked: one shared by many pages, forms or + /// appearances is walked once, as claiming is the same each time. + walked: HashSet, } impl Audit<'_> { @@ -124,6 +127,18 @@ impl Audit<'_> { o.map(|x| self.doc.resolve(x)).and_then(|x| x.as_dict().cloned()).unwrap_or_default() } + /// The resource dictionary `holder` points at (if it does), by kind: [`Self::resources`], + /// once per dictionary given by reference. + fn resources_of(&mut self, holder: Option<&Object>, depth: usize) { + if let Some(Object::Ref(r)) = holder + && !self.walked.insert(*r) + { + return; + } + let res = self.resolve_dict(holder); + self.resources(&res, depth); + } + /// A resource dictionary, by kind; `depth` counts the form XObjects it is nested in. fn resources(&mut self, res: &Dict, depth: usize) { let xobjects = self.resolve_dict(res.get(b"XObject")); @@ -135,8 +150,7 @@ impl Audit<'_> { // A form: its own resources first, by kind, then the rest of it. Owning it // first stops a form that draws itself. self.owner.insert(r.num, SpaceCategory::XObjectForms); - let inner = self.resolve_dict(s.dict.get(b"Resources")); - self.resources(&inner, depth + 1); + self.resources_of(s.dict.get(b"Resources"), depth + 1); self.claim(&Object::Dict(s.dict.clone()), SpaceCategory::XObjectForms); } _ => self.claim(x, SpaceCategory::XObjectForms), @@ -172,8 +186,7 @@ impl Audit<'_> { _ => Vec::new(), }; for dict in streams { - let res = self.resolve_dict(dict.get(b"Resources")); - self.resources(&res, 1); + self.resources_of(dict.get(b"Resources"), 1); } } } @@ -216,19 +229,17 @@ fn object_stream(doc: &Document, num: u32) -> Option { /// Audit the space a document uses; `file_len` is the size of the file on disk (the rest is /// overhead). pub fn audit_space(doc: &Document, file_len: u64) -> Vec { - let mut a = Audit { doc, owner: HashMap::new() }; + let mut a = Audit { doc, owner: HashMap::new(), walked: HashSet::new() }; let cat = doc.root().and_then(|r| doc.get(r).as_dict().cloned()).unwrap_or_default(); let pages = pdfcraft_model::pages(doc); // Order matters: the most specific owners first. Resources by kind before anything that // uses them: a content stream's dictionary may point at what it draws (PdfCraft's /PCAdded // records an added image), and form fields and comments draw with fonts and images too. for p in &pages { - let res = a.resolve_dict(p.dict.get(b"Resources")); - a.resources(&res, 0); + a.resources_of(p.dict.get(b"Resources"), 0); } let form = a.resolve_dict(cat.get(b"AcroForm")); - let form_resources = a.resolve_dict(form.get(b"DR")); - a.resources(&form_resources, 0); + a.resources_of(form.get(b"DR"), 0); for p in &pages { for x in annotations(doc, &p.dict) { a.appearance_resources(&x); diff --git a/crates/optimize/src/links.rs b/crates/optimize/src/links.rs index 5b74ad7023..bef69c43b7 100644 --- a/crates/optimize/src/links.rs +++ b/crates/optimize/src/links.rs @@ -52,8 +52,12 @@ fn tree_entries(doc: &Document, node: &Object) -> Vec<(Vec, Object)> { } } } - if let Some(kids) = d.get(b"Kids").map(|x| doc.resolve(x)).and_then(|x| x.as_array().cloned()) { - queue.extend(kids.into_iter().map(|k| (k, depth + 1))); + // A kids array shared by reference is read once, and only kids not yet seen are queued: + // nodes sharing one array can't multiply the queue. + if let Some(kids) = + d.get(b"Kids").filter(|k| !matches!(k, Object::Ref(r) if !seen.insert(*r))).map(|x| doc.resolve(x)).and_then(|x| x.as_array().cloned()) + { + queue.extend(kids.into_iter().filter(|k| !matches!(k, Object::Ref(r) if seen.contains(r))).map(|k| (k, depth + 1))); } } out diff --git a/crates/optimize/tests/optimize.rs b/crates/optimize/tests/optimize.rs index 7da6151deb..eee2e9570c 100644 --- a/crates/optimize/tests/optimize.rs +++ b/crates/optimize/tests/optimize.rs @@ -5613,6 +5613,70 @@ fn a_picture_whose_mask_is_not_a_stencil_is_left_alone() { } } +/// A resource dictionary shared by reference between thousands of pages is walked once by the +/// space audit, not once per page. +#[test] +fn space_audit_walks_a_shared_resource_dictionary_once() { + let mut doc = Document::new_empty(); + let img = image(&mut doc, 4, 4, 3, false, None); + let mut x = Dict::new(); + for i in 0..5_000 { + x.set(format!("Im{i}").into_bytes(), Object::Ref(img)); + } + let mut res = Dict::new(); + res.set(b"XObject".to_vec(), Object::Dict(x)); + let shared = doc.add(Object::Dict(res)); + let pages = doc.root().and_then(|r| doc.get(r).as_dict().and_then(|d| d.reference(b"Pages"))).unwrap(); + let mut kids = Vec::new(); + for _ in 0..2_000 { + let mut p = Dict::new(); + p.set(b"Type".to_vec(), Object::name("Page")); + p.set(b"Parent".to_vec(), Object::Ref(pages)); + p.set(b"MediaBox".to_vec(), Object::Array(vec![0.into(), 0.into(), 612.into(), 792.into()])); + p.set(b"Resources".to_vec(), Object::Ref(shared)); + kids.push(Object::Ref(doc.add(Object::Dict(p)))); + } + doc.update_dict(pages, |d| { + d.set(b"Count".to_vec(), Object::Int(kids.len() as i64)); + d.set(b"Kids".to_vec(), Object::Array(kids)); + }) + .unwrap(); + let started = std::time::Instant::now(); + let audit = audit_space(&doc, 1 << 20); + assert!(started.elapsed() < std::time::Duration::from_secs(5), "{:?}", started.elapsed()); + assert!(audit.iter().any(|u| u.category == SpaceCategory::Images && u.bytes > 0), "{audit:?}"); +} + +/// A name tree whose nodes all share one kids array (by reference) is read once per node, not +/// once per node per kid: the queue can't grow to the nodes times the kids. +#[test] +fn a_name_tree_sharing_one_kids_array_is_walked_once() { + let mut doc = Document::new_empty(); + let p = page(&mut doc, &[], "BT ET"); + let s = |t: &str| Object::String(pdfcraft_cos::PdfString::literal(t.as_bytes().to_vec())); + let fit = Object::Array(vec![Object::Ref(p), Object::name("Fit")]); + let kids = doc.add(Object::Array(Vec::new())); + let mut nodes = Vec::new(); + for i in 0..3_000 { + let mut node = Dict::new(); + node.set(b"Kids".to_vec(), Object::Ref(kids)); + node.set(b"Names".to_vec(), Object::Array(vec![s(&format!("d{i}")), fit.clone()])); + nodes.push(Object::Ref(doc.add(Object::Dict(node)))); + } + doc.set(kids, Object::Array(nodes)); + let mut tree = Dict::new(); + tree.set(b"Kids".to_vec(), Object::Ref(kids)); + let tree = doc.add(Object::Dict(tree)); + let mut names = Dict::new(); + names.set(b"Dests".to_vec(), Object::Ref(tree)); + let root = doc.root().unwrap(); + doc.update_dict(root, |c| c.set(b"Names".to_vec(), Object::Dict(names))).unwrap(); + let started = std::time::Instant::now(); + let report = optimize(&mut doc, &Settings::default()).unwrap(); + assert!(started.elapsed() < std::time::Duration::from_secs(10), "{:?}", started.elapsed()); + assert_eq!(report.unreferenced_dests, 3_000, "every node's destination was found, once"); +} + /// Pictures run on several threads: what comes out doesn't depend on how many. #[test] fn the_same_file_comes_out_with_any_number_of_threads() { From fba22fc73840986b7e7307d4844680868b29d667 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 14:47:15 -0500 Subject: [PATCH 48/60] fix(optimize): the fragment search charges every operator of a group it reads (#6095) * fix(optimize): the fragment search charges every operator of a group it reads * address review: conflict markers removed; the fragment search charges every operator it reads to the production allowance * address review: group reads are charged before a parse failure and count all four operators --- crates/optimize/src/fragments.rs | 98 +++++++++++++++++++++++++------- 1 file changed, 78 insertions(+), 20 deletions(-) diff --git a/crates/optimize/src/fragments.rs b/crates/optimize/src/fragments.rs index aa52c20750..9ccca4791f 100644 --- a/crates/optimize/src/fragments.rs +++ b/crates/optimize/src/fragments.rs @@ -30,17 +30,21 @@ pub const MIN_FRAGMENTS: usize = 4; /// Most pixels a merged image may have (memory). const MAX_PIXELS: u64 = 80_000_000; -/// Fragment groups a page's search for runs may read per content operator, and at least. A run -/// that is not one picture is searched again from each of its groups, so without a limit images -/// drawn over one another take time quadratic in their number; past it, the rest of the page is -/// left as it is. -const SCAN_PER_OP: usize = 16; +/// Content operators a page's search for runs may read per content operator of the page, and at +/// least. A run that is not one picture is searched again from each of its groups, so without a +/// limit images drawn over one another take time quadratic in their number; past it, the rest of +/// the page is left as it is. Every operator of a group counts (its `cm` operators are composed +/// each time), not just the group; a group of four operators costs four, so the allowance is 16 +/// groups per operator, as before, and groups with many `cm` operators pay for them. +const SCAN_PER_OP: usize = 64; const MIN_SCAN: usize = 1 << 16; #[cfg(test)] thread_local! { /// Fragment groups read, for the test that keeps a page's search linear in its content. static GROUP_READS: std::cell::Cell = const { std::cell::Cell::new(0) }; + /// Operators group reads walked, for the test that keeps the search linear in operators. + static OP_READS: std::cell::Cell = const { std::cell::Cell::new(0) }; } /// One fragment: its image, its box in the coordinates the run is drawn in, and its size. @@ -53,21 +57,44 @@ struct Fragment { /// The ops `q`, one or more `cm`, `Do`, `Q` starting at `i`: the index after them, the matrix /// and the name drawn. -fn group_at(ops: &[Op], i: usize) -> Option<(usize, Matrix, Vec)> { +fn group_at(ops: &[Op], i: usize, reads: &std::cell::Cell) -> Option<(usize, Matrix, Vec)> { #[cfg(test)] GROUP_READS.with(|c| c.set(c.get().saturating_add(1))); - if !ops.get(i)?.is("q") { + let mut looked = 0usize; + let found = group_scan(ops, i, &mut looked); + // Every operator looked at is charged to the page's search allowance, whether or not the + // group turns out to be one. + reads.set(reads.get().saturating_add(looked)); + #[cfg(test)] + OP_READS.with(|c| c.set(c.get().saturating_add(looked))); + found +} + +/// [`group_at`]'s search, counting in `looked` each operator it examines. +fn group_scan(ops: &[Op], i: usize, looked: &mut usize) -> Option<(usize, Matrix, Vec)> { + let q = ops.get(i)?; + *looked += 1; + if !q.is("q") { return None; } let mut j = i + 1; let mut m = Matrix([1.0, 0.0, 0.0, 1.0, 0.0, 0.0]); while let Some(op) = ops.get(j).filter(|o| o.is("cm")) { + *looked += 1; m = Matrix::from_operands(&op.operands)?.then(&m); j += 1; } - let draw = ops.get(j).filter(|o| o.is("Do"))?; + let draw = ops.get(j)?; + *looked += 1; + if !draw.is("Do") { + return None; + } let name = draw.name(0)?.to_vec(); - ops.get(j + 1).filter(|o| o.is("Q"))?; + let end = ops.get(j + 1)?; + *looked += 1; + if !end.is("Q") { + return None; + } (j > i + 1).then_some((j + 2, m, name)) } @@ -152,7 +179,7 @@ fn stacked(run: &mut [Fragment]) -> Option>> { /// The fragment groups from `i` on (at most `limit`) while they form a picture drawn row by row /// or strip by strip in one direction, up to a gap or a jump. -fn drawn_in_sequence(ops: &[Op], i: usize, limit: usize, xobjects: &Dict, doc: &Document) -> Vec { +fn drawn_in_sequence(ops: &[Op], i: usize, limit: usize, xobjects: &Dict, doc: &Document, reads: &std::cell::Cell) -> Vec { let mut run: Vec = Vec::new(); let mut row_x = [0.0f64, 0.0]; let mut cur_x = [0.0f64, 0.0]; @@ -163,7 +190,7 @@ fn drawn_in_sequence(ops: &[Op], i: usize, limit: usize, xobjects: &Dict, doc: & let mut complete = 0usize; let mut j = i; while run.len() < limit { - let Some((next, m, name)) = group_at(ops, j) else { break }; + let Some((next, m, name)) = group_at(ops, j, reads) else { break }; let Some((image, width, height, ..)) = fragment_image(doc, xobjects, &name) else { break }; let [a, _, _, d, e, f] = m.0; let f2 = Fragment { image, rect: [e, f, e + a, f + d], width, height }; @@ -240,10 +267,10 @@ fn drawn_in_sequence(ops: &[Op], i: usize, limit: usize, xobjects: &Dict, doc: & } /// The op index after the first `count` fragment groups from `i`. -fn run_end(ops: &[Op], i: usize, count: usize) -> usize { +fn run_end(ops: &[Op], i: usize, count: usize, reads: &std::cell::Cell) -> usize { let mut j = i; for _ in 0..count { - match group_at(ops, j) { + match group_at(ops, j, reads) { Some((next, ..)) => j = next, None => break, } @@ -386,9 +413,9 @@ fn merge_page( let mut out: Vec = Vec::with_capacity(ops.len()); let (mut i, mut merged, mut k) = (0, 0, 0); let budget = ops.len().saturating_mul(SCAN_PER_OP).max(MIN_SCAN); - let mut scanned = 0usize; + let reads = std::cell::Cell::new(0usize); while i < ops.len() { - if scanned > budget { + if reads.get() > budget { out.extend(ops.get(i..).unwrap_or_default().iter().cloned()); break; } @@ -396,7 +423,7 @@ fn merge_page( let mut run = Vec::new(); let mut kind: Option<(Object, usize)> = None; let mut j = i; - while let Some((next, m, name)) = group_at(&ops, j) { + while let Some((next, m, name)) = group_at(&ops, j, &reads) { let Some((image, width, height, cs, n)) = fragment_image(doc, &xobjects, &name) else { break }; let [a, b, c, d, e, f] = m.0; if b.abs() > 1e-9 || c.abs() > 1e-9 || !(a > 0.0 && d > 0.0) { @@ -410,13 +437,11 @@ fn merge_page( run.push(Fragment { image, rect: [e, f, e + a, f + d], width, height }); j = next; } - scanned = scanned.saturating_add(run.len()).saturating_add(1); // Not one picture as a whole: a prefix drawn row by row or strip by strip may be. let mut rows = if run.len() >= MIN_FRAGMENTS { stacked(&mut run) } else { None }; if run.len() >= MIN_FRAGMENTS && rows.is_none() { - run = drawn_in_sequence(&ops, i, run.len(), &xobjects, doc); - j = run_end(&ops, i, run.len()); - scanned = scanned.saturating_add(run.len().saturating_mul(2)).saturating_add(2); + run = drawn_in_sequence(&ops, i, run.len(), &xobjects, doc, &reads); + j = run_end(&ops, i, run.len(), &reads); rows = stacked(&mut run); } let Some(((cs, n), rows)) = kind.zip(rows).filter(|_| run.len() >= MIN_FRAGMENTS) else { @@ -773,4 +798,37 @@ mod tests { assert_eq!((merged, report.merged_images), (0, 0)); } } + + /// The same with many `cm` operators in each group: the search reads a bounded number of operators + /// for each one of the page, however the matrices are spread over the groups. + #[test] + fn groups_with_many_matrices_are_charged_for_each_operator() { + let mut doc = Document::new_empty(); + let mut image = Dict::new(); + for (k, v) in + [("Subtype", Object::name("Image")), ("Width", Object::Int(1)), ("Height", Object::Int(1)), ("BitsPerComponent", Object::Int(8))] + { + image.set(k.as_bytes().to_vec(), v); + } + image.set(b"ColorSpace".to_vec(), Object::name("DeviceGray")); + let image = doc.add(Object::Stream(Stream::from_raw(image, vec![0]))); + let groups = 400; + let content = doc.add(Object::Stream(Stream::flate( + Dict::new(), + format!("q 10 0 0 10 0 0 cm {}/Im Do Q\n", "1 0 0 1 0 0 cm ".repeat(49)).repeat(groups).as_bytes(), + ))); + let mut xobjects = Dict::new(); + xobjects.set(b"Im".to_vec(), Object::Ref(image)); + let mut resources = Dict::new(); + resources.set(b"XObject".to_vec(), Object::Dict(xobjects)); + let mut page = Dict::new(); + page.set(b"Contents".to_vec(), Object::Ref(content)); + page.set(b"Resources".to_vec(), Object::Dict(resources)); + let page = doc.add(Object::Dict(page)); + OP_READS.with(|c| c.set(0)); + let merged = merge_page(&mut doc, page, &mut Report::default(), &mut |_| true, &mut ImageBudget::with_work(u64::MAX)); + assert_eq!(merged.ok(), Some(0)); + let reads = OP_READS.with(|c| c.get()); + assert!(reads <= 80 * 53 * groups, "{reads} operator reads"); + } } From 87aebd1b24618fa822d994f32209bff08e974056 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 14:47:29 -0500 Subject: [PATCH 49/60] fix(preflight): follow every /Next action and bound colour and intent work (#5926) * fix(preflight): follow every /Next action and bound colour and intent work Action chains followed an indirect /Next only when it named an array, and skipped references inside a /Next array, so a forbidden action one indirect hop behind an allowed one survived both verification and conversion; an untyped dictionary with /P and /K was exempt from action checks whatever its /S. Both are now followed and checked, and arrays are borrowed instead of deep-copied per chain. Conversion treats a /Subtype /Widget without /Type as an annotation, and takes forbidden annotations off their pages in one pass. Device-colour inspection decoded each content stream with the 1 GiB per-stream default and no total; it now shares one 1 GiB DecodeBudget and reports unread content instead of treating it as clean. Type 3 CharProcs shared by many fonts are read once, and declared() reads each output intent once (at most 8). * address review: no sRGB intent when colour couldn't all be read; count array reads in action chains; declare every output intent * address review: cut only an over-long identifier; test with a filter that can't be read * review: leave untyped structure elements out of the forbidden-action check * merge main, keep both * address review: a wide direct /Next array is refused before it is queued; queued references are resolved through aliases --- crates/preflight/src/lib.rs | 190 ++++++++++++++++++++++++++-------- crates/preflight/src/tests.rs | 124 ++++++++++++++++++++++ 2 files changed, 269 insertions(+), 45 deletions(-) diff --git a/crates/preflight/src/lib.rs b/crates/preflight/src/lib.rs index 7b6e960006..90c6d9db83 100644 --- a/crates/preflight/src/lib.rs +++ b/crates/preflight/src/lib.rs @@ -203,6 +203,7 @@ pub const WORDINGS: &[&str] = &[ "A graphics state has the rendering intent {}, which isn't a standard one", "A graphics state has a rendering intent that isn't a name", "The form asks viewers to make appearances (NeedAppearances)", + "Some content could not be read for device colour", // Pages, actions, annotations and fields. "Page {} has additional actions", "A {} action isn't allowed", @@ -229,7 +230,8 @@ pub struct Declared { /// `pdfaid:part` and `pdfaid:conformance`, e.g. (2, "B"). pub pdfa: Option<(u8, String)>, pub pdfua: Option, - /// Output intents' condition identifiers. + /// Output intents' condition identifiers: every intent the document lists (each object + /// once), each identifier cut to [`MAX_IDENTIFIER_BYTES`]. pub output_intents: Vec, } @@ -276,10 +278,27 @@ pub fn declared(doc: &Document) -> Declared { d.pdfua = xmp::identifier(&x, xmp::PDFUA_ID, "part").and_then(|p| p.parse().ok()); } let cat = catalog(doc); - if let Some(list) = cat.get(b"OutputIntents").map(|o| doc.resolve(o)).and_then(|o| o.as_array().cloned()) { - for oi in list { - if let Some(od) = doc.resolve(&oi).as_dict() { - let id = od.get(b"OutputConditionIdentifier").and_then(|v| doc.resolve(v).as_string().map(|s| s.to_text())).unwrap_or_default(); + let list = cat.get(b"OutputIntents").map(|o| doc.resolve(o)); + if let Some(list) = list.as_deref().and_then(Object::as_array) { + // One intent object listed again is one declaration: its identifier is not copied once + // per listing. Every distinct one is kept (verification reads them all too), and only + // the start of an identifier longer than any real one. + let mut seen = std::collections::HashSet::new(); + for oi in list.iter().filter(|oi| oi.as_ref().is_none_or(|r| seen.insert(r))) { + if let Some(od) = doc.resolve(oi).as_dict() { + let id = od + .get(b"OutputConditionIdentifier") + .and_then(|v| { + doc.resolve(v).as_string().map(|s| { + if s.bytes.len() <= MAX_IDENTIFIER_BYTES { + s.to_text() + } else { + // An even cut keeps UTF-16 text whole code units. + pdfcraft_cos::PdfString { bytes: s.bytes[..MAX_IDENTIFIER_BYTES & !1].to_vec(), hex: s.hex }.to_text() + } + }) + }) + .unwrap_or_default(); d.output_intents.push(id); } } @@ -287,6 +306,14 @@ pub fn declared(doc: &Document) -> Declared { d } +/// The most bytes of an output condition identifier [`declared`] keeps (a real one is a few +/// dozen). +const MAX_IDENTIFIER_BYTES: usize = 1024; + +/// Decoded bytes one verification reads looking for device colour, all content streams +/// together. Past it, content is not inspected, and the report says so. +const MAX_COLOUR_DECODED: usize = 1 << 30; + /// Every indirect object, with its reference. fn objects(doc: &Document) -> Vec<(ObjRef, std::sync::Arc)> { doc.object_numbers() @@ -361,6 +388,9 @@ struct DeviceColour { /// space, which PDF/A accepts whatever the output intent (ISO 19005-2 6.2.4.3). default_rgb: bool, default_cmyk: bool, + /// Content that could not be read (undecodable, or past [`MAX_COLOUR_DECODED`]) may use + /// device colour that was not seen. + incomplete: bool, } impl DeviceColour { @@ -388,27 +418,35 @@ const MAX_DIRECT_DEPTH: u8 = 8; /// The glyph procedures of every Type 3 font in `d`: `d` itself, or a font held directly inside /// it (a font can sit inline in a resource dictionary). -fn type3_procs(doc: &Document, d: &Dict, depth: u8, out: &mut std::collections::HashSet) { +fn type3_procs(doc: &Document, d: &Dict, depth: u8, read: &mut std::collections::HashSet, out: &mut std::collections::HashSet) { if d.name(b"Subtype") == Some(b"Type3") - && let Some(procs) = d.get(b"CharProcs").map(|c| doc.resolve(c)) - && let Some(procs) = procs.as_dict() + && let Some(c) = d.get(b"CharProcs") + // One glyph dictionary shared by many fonts is read once: its entries are the same. + && c.as_ref().is_none_or(|r| read.insert(r)) + && let Some(procs) = doc.resolve(c).as_dict() { out.extend(procs.iter().filter_map(|(_, p)| p.as_ref())); } for (_, v) in d.iter() { - type3_procs_within(doc, v, depth.saturating_add(1), out); + type3_procs_within(doc, v, depth.saturating_add(1), read, out); } } -fn type3_procs_within(doc: &Document, o: &Object, depth: u8, out: &mut std::collections::HashSet) { +fn type3_procs_within( + doc: &Document, + o: &Object, + depth: u8, + read: &mut std::collections::HashSet, + out: &mut std::collections::HashSet, +) { if depth > MAX_DIRECT_DEPTH { return; } match o { - Object::Dict(d) => type3_procs(doc, d, depth, out), + Object::Dict(d) => type3_procs(doc, d, depth, read, out), Object::Array(a) => { for v in a { - type3_procs_within(doc, v, depth.saturating_add(1), out); + type3_procs_within(doc, v, depth.saturating_add(1), read, out); } } _ => {} @@ -430,9 +468,10 @@ fn referenced_content(doc: &Document, objs: &[(ObjRef, std::sync::Arc)]) }; out.extend(contents.iter().filter_map(Object::as_ref)); } + let mut read = std::collections::HashSet::new(); for (_, o) in objs { if let Some(d) = dict_of(o) { - type3_procs(doc, d, 0, &mut out); + type3_procs(doc, d, 0, &mut read, &mut out); } } out @@ -560,6 +599,7 @@ fn mark_cs_object(doc: &Document, cs: &Object, used: &mut DeviceColour, depth: u fn uses_device_colour(doc: &Document, objs: &[(ObjRef, std::sync::Arc)]) -> DeviceColour { let mut used = DeviceColour::default(); let referenced = referenced_content(doc, objs); + let mut budget = pdfcraft_cos::DecodeBudget::new(MAX_COLOUR_DECODED, usize::MAX); for (r, o) in objs { let Some(d) = dict_of(o) else { continue }; // A resource dictionary's /ColorSpace, whether it is this object or the page's inline @@ -586,9 +626,14 @@ fn uses_device_colour(doc: &Document, objs: &[(ObjRef, std::sync::Arc)]) if !(used.gray && used.rgb && used.cmyk) && let Object::Stream(s) = &**o && (referenced.contains(r) || is_content(&s.dict)) - && let Ok(data) = s.decoded() { - let _ = content_colour(&data, &mut used); + match budget.decode(s) { + Ok(data) => { + let _ = content_colour(&data, &mut used); + } + // Not read: it may draw in a device space nothing above saw. + Err(_) => used.incomplete = true, + } } } used @@ -830,6 +875,9 @@ fn check_output_intent( issue: &mut impl FnMut(&'static str, &'static str, Option, Option, bool), ) { let used = uses_device_colour(doc, objs); + if used.incomplete { + issue("6.2.3", "Some content could not be read for device colour", None, None, false); + } match pdfa_intent_profile(doc) { None if used.any() => { issue("6.2.3", "Device colour is used, but there is no PDF/A output intent", None, None, !used.cmyk); @@ -1215,22 +1263,56 @@ const MAX_ACTION_CHAIN: usize = 1024; fn chain_forbidden(doc: &Document, a: &Dict) -> Option { let mut read = 0; let mut seen = std::collections::HashSet::new(); - // Actions written in an indirect `/Next` array, copied out of it. - let mut held: Vec = Vec::new(); + // Objects `/Next` refers to (an action, or an array of them), each read once and borrowed: + // one array shared by many chains is not copied per chain. + let mut held: Vec> = Vec::new(); if let Some(s) = walk_chain(doc, a, &mut read, &mut seen, &mut held) { return Some(s); } - while let Some(d) = held.pop() { - if let Some(s) = walk_chain(doc, &d, &mut read, &mut seen, &mut held) { - return Some(s); + while let Some(o) = held.pop() { + let dicts: Vec<&Dict> = match &*o { + Object::Dict(d) => vec![d], + Object::Array(list) => { + // An array read is work too, with its entries: a chain of arrays alone must + // still reach the limit. + read = read.saturating_add(1 + list.len().min(MAX_ACTION_CHAIN)); + if read > MAX_ACTION_CHAIN { + return Some("too deeply chained".into()); + } + // More than the chain limit can't be read anyway. + hold_referenced(doc, list, &mut seen, &mut held); + list.iter().filter_map(Object::as_dict).take(MAX_ACTION_CHAIN).collect() + } + _ => continue, + }; + for d in dicts { + if let Some(s) = walk_chain(doc, d, &mut read, &mut seen, &mut held) { + return Some(s); + } } } None } -/// [`chain_forbidden`] for the actions reachable from `a` through direct `/Next` entries. The -/// actions of each indirect `/Next` array (each read once) go to `held`. -fn walk_chain(doc: &Document, a: &Dict, read: &mut usize, seen: &mut std::collections::HashSet, held: &mut Vec) -> Option { +/// Queue the objects the references in a `/Next` array name, each once. +fn hold_referenced(doc: &Document, list: &[Object], seen: &mut std::collections::HashSet, held: &mut Vec>) { + for r in list.iter().filter_map(Object::as_ref).take(MAX_ACTION_CHAIN) { + if seen.insert(r) { + // Resolved through any alias: a reference to a reference names its object's value. + held.push(doc.resolve(&Object::Ref(r))); + } + } +} + +/// [`chain_forbidden`] for the actions reachable from `a` through direct `/Next` entries. What +/// an indirect `/Next`, or a reference inside a `/Next` array, names goes to `held`. +fn walk_chain( + doc: &Document, + a: &Dict, + read: &mut usize, + seen: &mut std::collections::HashSet, + held: &mut Vec>, +) -> Option { let mut stack = vec![a]; while let Some(d) = stack.pop() { *read += 1; @@ -1242,13 +1324,16 @@ fn walk_chain(doc: &Document, a: &Dict, read: &mut usize, seen: &mut std::collec } match d.get(b"Next") { Some(Object::Dict(next)) => stack.push(next), - Some(Object::Array(list)) => stack.extend(list.iter().filter_map(Object::as_dict)), - Some(Object::Ref(r)) if seen.insert(*r) => { - if let Object::Array(list) = &*doc.get(*r) { - // More than the chain limit can't be read anyway. - held.extend(list.iter().filter_map(Object::as_dict).take(MAX_ACTION_CHAIN).cloned()); + Some(Object::Array(list)) => { + *read = read.saturating_add(list.len().min(MAX_ACTION_CHAIN)); + // A wide array is refused before any of it is queued. + if *read > MAX_ACTION_CHAIN { + return Some("too deeply chained".into()); } + hold_referenced(doc, list, seen, held); + stack.extend(list.iter().filter_map(Object::as_dict)); } + Some(Object::Ref(r)) if seen.insert(*r) => held.push(doc.resolve(&Object::Ref(*r))), _ => {} } } @@ -1291,6 +1376,26 @@ fn strip_actions(doc: &Document, d: &mut Dict) -> bool { changed } +/// Take the annotations `gone` off every page that lists them, each page's list rewritten once, and +/// replace their indirect objects with `Null`. +fn remove_from_pages(doc: &mut Document, gone: &std::collections::HashSet) -> Result<(), pdfcraft_cos::CosError> { + if gone.is_empty() { + return Ok(()); + } + for p in pdfcraft_model::pages(doc) { + let annots = p.dict.get(b"Annots").map(|a| doc.resolve(a)).and_then(|a| a.as_array().cloned()).unwrap_or_default(); + let listed = |a: &Object| a.as_ref().is_some_and(|r| gone.contains(&r)); + if annots.iter().any(listed) { + let kept: Vec = annots.into_iter().filter(|a| !listed(a)).collect(); + doc.update_dict(p.obj, |pd| pd.set(b"Annots".to_vec(), Object::Array(kept)))?; + } + } + for r in gone { + doc.set(*r, Object::Null); + } + Ok(()) +} + /// Collects every indirect annotation reference listed in any page's `/Annots` array. fn page_annot_refs(doc: &Document) -> std::collections::HashSet { let mut refs = std::collections::HashSet::new(); @@ -1304,19 +1409,6 @@ fn page_annot_refs(doc: &Document) -> std::collections::HashSet { refs } -/// Removes annotation `r` from every page's `/Annots` list and replaces its indirect object with `Null`. -fn remove_annot_from_pages(doc: &mut Document, r: ObjRef) -> Result<(), pdfcraft_cos::CosError> { - for p in pdfcraft_model::pages(doc) { - let annots = p.dict.get(b"Annots").map(|a| doc.resolve(a)).and_then(|a| a.as_array().cloned()).unwrap_or_default(); - if annots.iter().any(|a| a.as_ref() == Some(r)) { - let kept: Vec = annots.into_iter().filter(|a| a.as_ref() != Some(r)).collect(); - doc.update_dict(p.obj, |pd| pd.set(b"Annots".to_vec(), Object::Array(kept)))?; - } - } - doc.set(r, Object::Null); - Ok(()) -} - fn fix_annot_or_field(doc: &Document, nd: &mut Dict, is_annot: bool, sub: Option<&[u8]>, actions: &mut usize, flags: &mut usize) -> bool { let mut changed = strip_actions(doc, nd); *actions += changed as usize; @@ -1413,8 +1505,10 @@ pub fn convert(doc: &mut Document, level: Level) -> Result Result::to_vec); let sub = d.name(b"Subtype").map(<[u8]>::to_vec); - let is_annot = matches!(&*o, Object::Dict(_)) && is_annotation(doc, &d, ty.as_deref(), sub.as_deref(), page_annots.contains(&r)); + let is_annot = matches!(&*o, Object::Dict(_)) + // A widget is an annotation whether or not it says `/Type /Annot` (the form reader + // finds it by its field), so its actions go the same way. + && (is_annotation(doc, &d, ty.as_deref(), sub.as_deref(), page_annots.contains(&r)) || sub.as_deref() == Some(b"Widget")); if is_annot && sub.as_deref().is_some_and(|s| FORBIDDEN_ANNOTS.contains(&s)) { - remove_annot_from_pages(doc, r)?; + // Taken off its page below, all of them in one pass over the pages. + forbidden_annots.insert(r); removed += 1; continue; } @@ -1489,6 +1588,7 @@ pub fn convert(doc: &mut Document, level: Level) -> Result>", pages, page, &link(&format!("<< /S /GoTo /D [3 0 R /Fit] /Next {js} >>"))], ""), // `/Next` refers to an array that holds the action directly. build(&["<< /Type /Catalog /Pages 2 0 R >>", pages, page, &link("<< /S /GoTo /D [3 0 R /Fit] /Next 5 0 R >>"), &format!("[{js}]")], ""), + // `/Next` refers to the action itself, which carries `/P` and `/K` as a structure + // element would: those keys must not exempt it. + build( + &[ + "<< /Type /Catalog /Pages 2 0 R >>", + pages, + page, + &link("<< /S /GoTo /D [3 0 R /Fit] /Next 5 0 R >>"), + "<< /S /JavaScript /JS (app.alert(1)) /P 3 0 R /K 0 >>", + ], + "", + ), + // A `/Next` array refers to the action. + build(&["<< /Type /Catalog /Pages 2 0 R >>", pages, page, &link("<< /S /GoTo /D [3 0 R /Fit] /Next [5 0 R] >>"), js], ""), ]; for (i, mut doc) in cases.into_iter().enumerate() { let found = messages(&verify(&doc, Level::A2b)).contains(&"A JavaScript action isn't allowed"); @@ -495,6 +509,116 @@ fn an_output_intent_profile_header_is_read_within_a_size_cap() { } } +/// A widget the form reader finds by its field is an annotation whether or not it writes +/// `/Type /Annot`: conversion takes its additional actions off all the same. +#[test] +fn a_widget_without_a_type_loses_its_additional_actions() { + let pages = "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 200 200] >>"; + let page = "<< /Type /Page /Parent 2 0 R /Annots [4 0 R] >>"; + let widget = "<< /Subtype /Widget /Rect [0 0 10 10] /F 4 /T (f) /AA << /K << /S /JavaScript /JS (1) >> >> >>"; + let mut doc = build(&["<< /Type /Catalog /Pages 2 0 R >>", pages, page, widget], ""); + convert(&mut doc, Level::A2b).unwrap(); + let w = doc.get(pdfcraft_cos::ObjRef::new(4, 0)); + assert!(w.as_dict().is_some_and(|d| d.get(b"AA").is_none()), "{w:?}"); +} + +/// Forbidden annotations on one page are taken off it in one pass, all of them. +#[test] +fn forbidden_annotations_leave_their_page_together() { + let n = 300; + let annots: Vec = (0..n).map(|i| format!("{} 0 R", i + 4)).collect(); + let page = format!("<< /Type /Page /Parent 2 0 R /Annots [{} 2 0 R] >>", annots.join(" ")); + let mut objs = + vec!["<< /Type /Catalog /Pages 2 0 R >>".to_string(), "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 200 200] >>".into(), page]; + objs.extend((0..n).map(|_| "<< /Type /Annot /Subtype /Sound /Rect [0 0 0 0] /P 3 0 R >>".to_string())); + let refs: Vec<&str> = objs.iter().map(String::as_str).collect(); + let mut doc = build(&refs, ""); + let r = convert(&mut doc, Level::A2b).unwrap(); + assert!(r.fixed.iter().any(|f| f.starts_with(&format!("{n}: removed multimedia"))), "{:?}", r.fixed); + let left = pdfcraft_model::pages(&doc)[0].dict.get(b"Annots").and_then(|a| doc.resolve(a).as_array().map(Vec::len)); + assert_eq!(left, Some(1)); +} + +/// Content that can't be read may draw in DeviceCMYK: convert then adds no sRGB output intent. +#[test] +fn convert_adds_no_srgb_intent_when_colour_could_not_all_be_read() { + let mut doc = build( + &[ + "<< /Type /Catalog /Pages 2 0 R >>", + "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 200 200] >>", + "<< /Type /Page /Parent 2 0 R /Contents 4 0 R /Resources << >> >>", + // A /Filter that isn't a name can't be decoded: the content isn't read. + "<< /Length 9 /Filter 5 >>\nstream\nnot flate\nendstream", + ], + "", + ); + convert(&mut doc, Level::A2b).unwrap(); + assert!(declared(&doc).output_intents.is_empty()); +} + +/// A `/Next` chain of arrays alone (each array naming the next) still reaches the chain limit, +/// so the walk's work is bounded however the arrays are linked. +#[test] +fn an_array_only_next_chain_is_too_deeply_chained() { + let mut doc = Document::new_empty(); + let mut next = Object::Array(Vec::new()); + for _ in 0..2 * MAX_ACTION_CHAIN { + next = Object::Array(vec![Object::Ref(doc.add(next))]); + } + let mut action = Dict::new(); + action.set(b"S".to_vec(), Object::name("URI")); + action.set(b"Next".to_vec(), Object::Ref(doc.add(next))); + assert_eq!(chain_forbidden(&doc, &action).as_deref(), Some("too deeply chained")); + // A short chain of arrays is read through. + let mut short = Object::Array(Vec::new()); + for _ in 0..10 { + short = Object::Array(vec![Object::Ref(doc.add(short))]); + } + action.set(b"Next".to_vec(), Object::Ref(doc.add(short))); + assert_eq!(chain_forbidden(&doc, &action), None); +} + +/// A direct `/Next` array wider than the chain limit is refused as soon as it is charged, and a +/// `/Next` that reaches its array through a reference alias is read through it. +#[test] +fn a_wide_direct_next_array_is_refused_and_an_alias_is_followed() { + let uri = || { + let mut d = Dict::new(); + d.set(b"S".to_vec(), Object::name("URI")); + d + }; + let mut action = uri(); + action.set(b"Next".to_vec(), Object::Array((0..=MAX_ACTION_CHAIN).map(|_| Object::Dict(uri())).collect())); + let doc = Document::new_empty(); + assert_eq!(chain_forbidden(&doc, &action).as_deref(), Some("too deeply chained")); + // `/Next` -> 1 0 R -> 2 0 R -> [ << /S /JavaScript >> ] + let mut doc = Document::new_empty(); + let mut js = Dict::new(); + js.set(b"S".to_vec(), Object::name("JavaScript")); + let list = doc.add(Object::Array(vec![Object::Dict(js)])); + let alias = doc.add(Object::Ref(list)); + let mut action = uri(); + action.set(b"Next".to_vec(), Object::Ref(alias)); + assert_eq!(chain_forbidden(&doc, &action).as_deref(), Some("JavaScript")); +} + +/// Every distinct output intent is declared, past eight; an identifier longer than any real one +/// keeps its start. +#[test] +fn every_output_intent_is_declared_with_its_identifier_cut_to_the_bound() { + let long = "x".repeat(4 * MAX_IDENTIFIER_BYTES); + let intents: Vec = (0..9) + .map(|i| { + format!("<< /Type /OutputIntent /S /GTS_PDFA1 /OutputConditionIdentifier ({}) >>", if i == 8 { long.clone() } else { format!("c{i}") }) + }) + .collect(); + let catalog = format!("<< /Type /Catalog /Pages 2 0 R /OutputIntents [{}] >>", intents.join(" ")); + let doc = build(&[catalog.as_str(), "<< /Type /Pages /Kids [] /Count 0 >>"], ""); + let d = declared(&doc); + assert_eq!(d.output_intents.len(), 9); + assert_eq!(d.output_intents[8].len(), MAX_IDENTIFIER_BYTES); +} + #[test] fn typeless_annotations_indexed_cmyk_and_indirect_filter_elements_are_handled() { // Annotations in /Annots that omit optional /Type /Annot and /P (while keeping required /Subtype From 13d63673857d10138c0fa4302c3bcc75cf50171c Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 14:47:42 -0500 Subject: [PATCH 50/60] fix(redact): read each Type 3 glyph program once per cleanup (#5993) clean_type3_font decoded and parsed every surviving glyph program of a font, for every font. Many small font dictionaries sharing one compressed CharProc multiplied that work without multiplying the stored data, with a fresh allowance for each decode. The resource names each indirect glyph program uses are now cached by object across fonts (resource-name interpretation stays per font, since each font prunes its own /Resources), and all decoding spends one cleanup-wide DecodeBudget; past it the cleanup fails closed. --- .../redact/src/tests/surviving_references.rs | 29 ++++++++++++++ crates/redact/src/type3.rs | 39 +++++++++++++++++-- 2 files changed, 64 insertions(+), 4 deletions(-) diff --git a/crates/redact/src/tests/surviving_references.rs b/crates/redact/src/tests/surviving_references.rs index d02e76832e..bec850e302 100644 --- a/crates/redact/src/tests/surviving_references.rs +++ b/crates/redact/src/tests/surviving_references.rs @@ -390,6 +390,35 @@ fn type3_font_used_in_a_form_xobject_has_its_redacted_glyph_removed() { assert!(!full_save_holds(&doc, b"SECRET_FORM_GLYPH_PAYLOAD"), "redacted glyph program survived"); } +/// Type 3 cleanup decoded a glyph program again for every font listing it: 2 000 fonts sharing +/// one large program read it 2 000 times. It is read once now, and the cleanup still keeps the +/// glyph every font shows. +#[test] +fn a_glyph_program_many_type3_fonts_share_is_read_once() { + const FONTS: usize = 2_000; + let names: String = (0..FONTS).map(|i| format!("/T{i} {} 0 R ", 6 + i)).collect(); + let shows: String = (0..FONTS).map(|i| format!("/T{i} 10 Tf (A) Tj ")).collect(); + let mut objs = vec![ + b"<< /Type /Catalog /Pages 2 0 R >>".to_vec(), + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>".to_vec(), + format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 300] /Contents 4 0 R /Resources << /Font << {names}>> >> >>").into_bytes(), + stream("", format!("BT 10 100 Td {shows}ET").as_bytes()), + stream("", [b"SHARED_GLYPH ".as_slice(), &b"0 0 m 1 1 l S ".repeat(50_000)].concat()), + ]; + objs.extend((0..FONTS).map(|_| { + b"<< /Type /Font /Subtype /Type3 /FontBBox [0 0 1000 1000] /FontMatrix [0.001 0 0 0.001 0 0] /FirstChar 65 /LastChar 65 /Widths [0] /Encoding << /Differences [65 /A] >> /CharProcs << /A 5 0 R >> >>".to_vec() + })); + let mut doc = pdf(objs); + // The glyphs are drawn at x 10 with no width, so a mark away from them leaves them all. + mark(&mut doc, 0, &[[200.0, 200.0, 250.0, 250.0]], ""); + let started = std::time::Instant::now(); + apply(&mut doc, None).unwrap(); + assert!(started.elapsed() < std::time::Duration::from_secs(60), "{:?}", started.elapsed()); + let font = doc.get(ObjRef::new(6, 0)); + let procs = doc.resolve(font.as_dict().unwrap().get(b"CharProcs").unwrap()); + assert!(procs.as_dict().unwrap().contains(b"A"), "the glyph every font shows was unlinked"); +} + /// A Type 3 font with glyphs A (its program object 7) and B (object 8), at object 6. const TYPE3: &str = "<< /Type /Font /Subtype /Type3 /FontBBox [0 -200 1000 800] /FontMatrix [0.001 0 0 0.001 0 0] /FirstChar 65 /LastChar 66 /Widths [500 500] /Encoding << /Differences [65 /A /B] >> /CharProcs << /A 7 0 R /B 8 0 R >> >>"; diff --git a/crates/redact/src/type3.rs b/crates/redact/src/type3.rs index 11d5a76f6a..1817fb3c39 100644 --- a/crates/redact/src/type3.rs +++ b/crates/redact/src/type3.rs @@ -3,7 +3,7 @@ use std::collections::{HashMap, HashSet}; use pdfcraft_content::Pieces; -use pdfcraft_cos::{CosError, Dict, Document, ObjRef, Object}; +use pdfcraft_cos::{CosError, DecodeBudget, Dict, Document, ObjRef, Object}; use crate::{RedactError, page_streams}; @@ -130,6 +130,25 @@ struct UsedInGlyphs { properties: HashSet>, } +impl UsedInGlyphs { + fn merge(&mut self, other: &UsedInGlyphs) { + self.xobjects.extend(other.xobjects.iter().cloned()); + self.colorspaces.extend(other.colorspaces.iter().cloned()); + self.patterns.extend(other.patterns.iter().cloned()); + self.shadings.extend(other.shadings.iter().cloned()); + self.extgstates.extend(other.extgstates.iter().cloned()); + self.fonts.extend(other.fonts.iter().cloned()); + self.properties.extend(other.properties.iter().cloned()); + } +} + +/// What one Type 3 cleanup reads of glyph programs: the resource names each indirect program +/// uses, read once however many fonts share it, and one decoding allowance for all of them. +struct GlyphReads { + used: HashMap, + budget: DecodeBudget, +} + fn collect_used_resources_in_stream(data: &[u8], used: &mut UsedInGlyphs) -> Result<(), CosError> { // One operator at a time, so a glyph program of millions of them is never held whole. A byte // that can't be read anywhere in it still refuses, once the loop has read it all. @@ -731,9 +750,10 @@ pub(crate) fn unlink_type3(doc: &mut Document) -> Result<(), RedactError> { } // For each Type 3 font: unlink unused glyphs from CharProcs and prune exclusive resources. + let mut reads = GlyphReads { used: HashMap::new(), budget: DecodeBudget::page() }; for (font_key, font_dict) in &type3_fonts { let surviving = used_glyphs.get(font_key).cloned().unwrap_or_default(); - clean_type3_font(doc, &holders, &form_holders, font_key, font_dict, &surviving)?; + clean_type3_font(doc, &holders, &form_holders, font_key, font_dict, &surviving, &mut reads)?; } Ok(()) @@ -808,6 +828,7 @@ fn clean_type3_font( font_key: &FontKey, font_dict: &Dict, surviving: &HashSet>, + reads: &mut GlyphReads, ) -> Result<(), RedactError> { let Some(char_procs_val) = font_dict.get(b"CharProcs") else { return Err(CosError::Syntax { offset: 0, detail: "redaction could not establish safe Type 3 font cleanup: missing CharProcs".into() }.into()); @@ -851,6 +872,10 @@ fn clean_type3_font( let mut used_in_glyphs = UsedInGlyphs::default(); for (_, proc_val) in char_procs.iter() { + if let Some(used) = proc_val.as_ref().and_then(|r| reads.used.get(&r)) { + used_in_glyphs.merge(used); + continue; + } let resolved = doc.resolve(proc_val); let Object::Stream(s) = &*resolved else { return Err(CosError::Syntax { @@ -859,8 +884,14 @@ fn clean_type3_font( } .into()); }; - let data = doc.with_direct_parms(s).decoded()?; - collect_used_resources_in_stream(&data, &mut used_in_glyphs)?; + // Past the allowance the cleanup can't be shown complete, and the redaction is refused. + let data = reads.budget.decode(&doc.with_direct_parms(s))?; + let mut used = UsedInGlyphs::default(); + collect_used_resources_in_stream(&data, &mut used)?; + used_in_glyphs.merge(&used); + if let Some(r) = proc_val.as_ref() { + reads.used.insert(r, used); + } } if prune_type3_font_resources(doc, &mut f_res, &used_in_glyphs) { From b52bc303281b58eb0bd28134cc7a1dc764177007 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 14:47:56 -0500 Subject: [PATCH 51/60] fix(annot): count ink points as strokes resolve and cap the eraser's samples (#6001) * fix(annot): count ink points as strokes resolve and cap the eraser's samples Resizing and erasing a drawing resolved every /InkList entry without a total, so one stroke array listed many times expanded freely; the eraser densified every segment into up to 2,000 samples with no overall limit. * review: refuse an eraser result that exceeds the stored ink point cap * review: say the erasure would make the drawing too large * address review: the eraser counts each stroke's final point against its sample total --- crates/annot/src/modify.rs | 69 ++++++++++++++++++++++++++------------ crates/annot/src/tests.rs | 67 ++++++++++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+), 22 deletions(-) diff --git a/crates/annot/src/modify.rs b/crates/annot/src/modify.rs index ae2222b387..6566337ca1 100644 --- a/crates/annot/src/modify.rs +++ b/crates/annot/src/modify.rs @@ -857,19 +857,38 @@ pub fn set_points(doc: &mut Document, page: usize, index: usize, points: &[[f64; /// Scale a drawing's strokes into `rect`. The margin between the strokes and the old `/Rect` (half /// the line width and a little more) stays the same on each side, so the line isn't clipped. +/// Most stroke points a stored drawing may have for resizing or erasing it. Counted as each +/// stroke is resolved, so one stroke array referenced again and again can't expand past it. +const MAX_STORED_INK_POINTS: usize = 1_000_000; +/// Most points the eraser densifies a drawing into before testing them against its path. +const MAX_ERASE_SAMPLES: usize = 4_000_000; + +/// The strokes of an ink annotation's `/InkList`, at most [`MAX_STORED_INK_POINTS`] points. +fn ink_strokes(doc: &Document, d: &Dict) -> Result>, AnnotError> { + let too_many = || AnnotError::Invalid(format!("the drawing has more than {MAX_STORED_INK_POINTS} points")); + let list = d.get(b"InkList").map(|l| doc.resolve(l)); + let mut points = 0usize; + let mut strokes = Vec::new(); + for s in list.as_deref().and_then(Object::as_array).map(Vec::as_slice).unwrap_or_default() { + let stroke = doc.resolve(s); + let Some(a) = stroke.as_array() else { + strokes.push(Vec::new()); + continue; + }; + // Charged before the coordinates are read: a shared array costs each time it is listed. + points = points.saturating_add(a.len() / 2); + if points > MAX_STORED_INK_POINTS { + return Err(too_many()); + } + let v: Vec = a.iter().filter_map(|x| doc.resolve(x).as_f64()).collect(); + strokes.push(v.as_chunks::<2>().0.iter().map(|p| [p[0], p[1]]).collect()); + } + Ok(strokes) +} + fn resize_ink(doc: &mut Document, r: ObjRef, rect: [f64; 4], meta: &Meta) -> Result<(), AnnotError> { let d = annot_dict(doc, r); - let strokes: Vec> = d - .get(b"InkList") - .map(|l| doc.resolve(l)) - .and_then(|l| l.as_array().cloned()) - .unwrap_or_default() - .iter() - .map(|s| { - let v: Vec = doc.resolve(s).as_array().map(|a| a.iter().filter_map(|x| doc.resolve(x).as_f64()).collect()).unwrap_or_default(); - v.as_chunks::<2>().0.iter().map(|p| [p[0], p[1]]).collect() - }) - .collect(); + let strokes = ink_strokes(doc, &d)?; let Some(inner) = bounds(strokes.iter().flatten().copied()) else { return Err(AnnotError::Invalid("the drawing has no strokes".into())); }; @@ -1475,17 +1494,7 @@ pub fn erase_ink(doc: &mut Document, page: usize, index: usize, path: &[[f64; 2] if path.is_empty() || !path.iter().all(|p| finite(p)) || !radius.is_finite() || radius <= 0.0 { return Err(AnnotError::Invalid("invalid eraser path".into())); } - let strokes: Vec> = d - .get(b"InkList") - .map(|l| doc.resolve(l)) - .and_then(|l| l.as_array().cloned()) - .unwrap_or_default() - .iter() - .map(|s| { - let v: Vec = doc.resolve(s).as_array().map(|a| a.iter().filter_map(|x| doc.resolve(x).as_f64()).collect()).unwrap_or_default(); - v.as_chunks::<2>().0.iter().map(|p| [p[0], p[1]]).collect() - }) - .collect(); + let strokes = ink_strokes(doc, &d)?; // Distance from a point to the eraser's path (segments, or a single point). let near = |p: [f64; 2]| -> bool { if path.len() == 1 { @@ -1501,17 +1510,27 @@ pub fn erase_ink(doc: &mut Document, page: usize, index: usize, path: &[[f64; 2] }; let mut kept: Vec> = Vec::new(); let mut erased = false; + let mut samples = 0usize; for s in &strokes { // Densify so long segments are cut where the eraser crosses them. let mut pts: Vec<[f64; 2]> = Vec::new(); for w in s.windows(2) { let n = ((w[1][0] - w[0][0]).hypot(w[1][1] - w[0][1]) / (radius / 2.0)).ceil().clamp(1.0, 2000.0) as usize; + samples = samples.saturating_add(n); + if samples > MAX_ERASE_SAMPLES { + return Err(AnnotError::Invalid("the drawing is too large to erase at this eraser size".into())); + } for k in 0..n { let t = k as f64 / n as f64; pts.push([w[0][0] + (w[1][0] - w[0][0]) * t, w[0][1] + (w[1][1] - w[0][1]) * t]); } } if let Some(last) = s.last() { + // The final point is a sample too. + samples = samples.saturating_add(1); + if samples > MAX_ERASE_SAMPLES { + return Err(AnnotError::Invalid("the drawing is too large to erase at this eraser size".into())); + } pts.push(*last); } let mut run: Vec<[f64; 2]> = Vec::new(); @@ -1537,6 +1556,12 @@ pub fn erase_ink(doc: &mut Document, page: usize, index: usize, path: &[[f64; 2] delete_annotation(doc, page, index)?; return Ok(true); } + // The retained runs are stored as the new `/InkList`: past the cap, writing them would leave a + // drawing that later `resize_ink` and `erase_ink` refuse to read. + let points: usize = kept.iter().map(Vec::len).sum(); + if points > MAX_STORED_INK_POINTS { + return Err(AnnotError::Invalid(format!("erasing would leave a drawing with more than {MAX_STORED_INK_POINTS} points"))); + } let width = d.get(b"BS").and_then(|b| b.as_dict()).and_then(|b| b.get(b"W")).and_then(|w| w.as_f64()).unwrap_or(1.0); let rect = grow(bounds(kept.iter().flatten().copied()).unwrap_or_default(), width / 2.0 + 1.0); doc.update_dict(r, |d| { diff --git a/crates/annot/src/tests.rs b/crates/annot/src/tests.rs index ce5a8c1604..980959d4d9 100644 --- a/crates/annot/src/tests.rs +++ b/crates/annot/src/tests.rs @@ -4797,6 +4797,73 @@ fn freehand_highlight_is_compatible_ink_with_marker_blending() { } } +/// A drawing whose /InkList lists one stroke array over and over is refused once its points +/// pass the cap, counted as each stroke is resolved, before they are collected or scaled. +#[test] +fn ink_points_are_counted_as_strokes_resolve() { + let numbers: String = (0..2000).map(|k| format!(" {}", k % 100)).collect(); + let refs = " 5 0 R".repeat(501); + let doc_with = |refs: &str| { + doc_of(&[ + "<< /Type /Catalog /Pages 2 0 R >>".into(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 612 792] >>".into(), + "<< /Type /Page /Parent 2 0 R /Annots [4 0 R] >>".into(), + format!("<< /Type /Annot /Subtype /Ink /Rect [0 0 100 100] /InkList [{refs}] >>"), + format!("[{numbers}]"), + ]) + }; + // 501 × 1,000 points is under the cap: the points are read (this bare drawing then has no + // appearance it can be redrawn with, which is another matter). + let mut doc = doc_with(&refs); + let under = set_rect(&mut doc, 0, 0, [0.0, 0.0, 50.0, 50.0], &meta("")); + assert!(!matches!(&under, Err(AnnotError::Invalid(m)) if m.contains("points")), "{under:?}"); + // 1,001 × 1,000 points is over it. + let mut doc = doc_with(&" 5 0 R".repeat(1001)); + match set_rect(&mut doc, 0, 0, [0.0, 0.0, 50.0, 50.0], &meta("")) { + Err(AnnotError::Invalid(m)) => assert!(m.contains("more than 1000000 points"), "{m}"), + other => panic!("refused: {other:?}"), + } + match erase_ink(&mut doc, 0, 0, &[[1.0, 1.0]], 1.0, &meta("")) { + Err(AnnotError::Invalid(m)) => assert!(m.contains("more than 1000000 points"), "{m}"), + other => panic!("refused: {other:?}"), + } +} + +/// The eraser's densified samples have a total: a tiny eraser over long strokes is refused +/// instead of interpolating millions of points. +#[test] +fn the_eraser_densifies_within_a_total() { + let zigzag: String = (0..2100).map(|k| format!(" {} {}", if k % 2 == 0 { 0 } else { 5000 }, k)).collect(); + let mut doc = doc_of(&[ + "<< /Type /Catalog /Pages 2 0 R >>".into(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 612 792] >>".into(), + "<< /Type /Page /Parent 2 0 R /Annots [4 0 R] >>".into(), + format!("<< /Type /Annot /Subtype /Ink /Rect [0 0 5000 2100] /InkList [[{zigzag}]] >>"), + ]); + match erase_ink(&mut doc, 0, 0, &[[1.0, 1.0]], 0.01, &meta("")) { + Err(AnnotError::Invalid(m)) => assert!(m.contains("too large to erase"), "{m}"), + other => panic!("refused: {other:?}"), + } +} + +/// Each stroke's final point is a sample too: 500,000 strokes of two points at 8 samples a segment +/// are 4,000,000 densified samples and as many final points again, past the total. +#[test] +fn the_eraser_counts_each_strokes_final_point() { + let refs = " 5 0 R".repeat(500_000); + let mut doc = doc_of(&[ + "<< /Type /Catalog /Pages 2 0 R >>".into(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 /MediaBox [0 0 612 792] >>".into(), + "<< /Type /Page /Parent 2 0 R /Annots [4 0 R] >>".into(), + format!("<< /Type /Annot /Subtype /Ink /Rect [0 0 100 100] /InkList [{refs}] >>"), + "[0 0 8 0]".into(), + ]); + match erase_ink(&mut doc, 0, 0, &[[1.0, 1.0]], 2.0, &meta("")) { + Err(AnnotError::Invalid(m)) => assert!(m.contains("too large to erase"), "{m}"), + other => panic!("refused: {other:?}"), + } +} + /// A 3D annotation is PDF 1.6 and a PRC model 1.7 (ISO 32000-2 §13.6): an older document's /// catalog says so, and a newer one is left as it is. #[test] From ee367a4a1c6087c07003cdf9d3999bc4281a997a Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 14:48:10 -0500 Subject: [PATCH 52/60] fix(xfa): bound the name and path bytes data nodes and queued instances copy (#6032) * fix(xfa): bound the name and path bytes data nodes and queued instances copy * review: bound the name bytes of a whole data path and report each cap separately * address review: the queued-instance allowance counts path segment buffers and the dedup copy * address review: the queued path's spare capacity is counted --- crates/xfa/src/data.rs | 16 +++++++++++++--- crates/xfa/src/data/build.rs | 2 +- crates/xfa/src/data/write.rs | 24 +++++++++++++++++++++++- crates/xfa/src/tests.rs | 25 +++++++++++++++++++++++++ 4 files changed, 62 insertions(+), 5 deletions(-) diff --git a/crates/xfa/src/data.rs b/crates/xfa/src/data.rs index 34be028d89..4e7de4af74 100644 --- a/crates/xfa/src/data.rs +++ b/crates/xfa/src/data.rs @@ -148,13 +148,21 @@ pub enum DataOp { impl DataNode { /// The node at `path`, made (with empty siblings before it) where missing; `None` past /// the index cap. - fn ensure(&mut self, path: &[(String, usize)]) -> Option<&mut DataNode> { + fn ensure(&mut self, path: &[(String, usize)], budget: &mut usize) -> Option<&mut DataNode> { let mut node = self; for (name, idx) in path { if *idx >= MAX_INDEX { return None; } let have = node.children.iter().filter(|c| &c.name == name).count(); + // Every skipped sibling owns a copy of the name: bounded as `Build::at` bounds a write. + // The budget is the whole path's, not each level's: a deep path that is affordable at + // every level taken alone would otherwise allocate its way past the bound. + let need = (*idx + 1).saturating_sub(have).saturating_mul(name.len()); + if need > *budget { + return None; + } + *budget -= need; for _ in have..=*idx { node.children.push(DataNode { name: name.clone(), ..Default::default() }); } @@ -166,9 +174,11 @@ impl DataNode { /// Apply `op` to this tree (the root is the `xfa:data` element), as [`write_data_ops`] /// will to the packet, so scripts that run before it is written see the change. pub fn apply(&mut self, op: &DataOp) { + // One budget for the op's whole path, as `Build::at` bounds a write. + let mut budget = build::MAX_NEW_NAME_BYTES; match op { DataOp::Value { path, text } => { - if let Some(n) = self.ensure(path) { + if let Some(n) = self.ensure(path, &mut budget) { // A multi-select list's picks, one `value` each (see `value_at`), told apart // as the packet writer does: `value` leaves only, or a group holding none. let picks = if n.children.is_empty() { n.group } else { n.children.iter().all(|c| c.name == "value" && c.children.is_empty()) }; @@ -189,7 +199,7 @@ impl DataNode { if let Some(&last) = count.checked_sub(1).as_ref() { let mut path = parent.clone(); path.push((name.clone(), last.min(MAX_INDEX.saturating_sub(1)))); - let _ = self.ensure(&path); + let _ = self.ensure(&path, &mut budget); } } DataOp::Remove(path) => { diff --git a/crates/xfa/src/data/build.rs b/crates/xfa/src/data/build.rs index 98e34f0f49..e90904d60c 100644 --- a/crates/xfa/src/data/build.rs +++ b/crates/xfa/src/data/build.rs @@ -38,7 +38,7 @@ pub(super) const MAX_INDEX: usize = 10_000; /// Most bytes of names the nodes one write creates hold together: every sibling an index /// skips over gets its own copy of the name. -const MAX_NEW_NAME_BYTES: usize = 4 << 20; +pub(super) const MAX_NEW_NAME_BYTES: usize = 4 << 20; /// What one write may still create: nodes and bytes of their names. pub(super) struct NewNodes { diff --git a/crates/xfa/src/data/write.rs b/crates/xfa/src/data/write.rs index 4667036fe5..554685e1ad 100644 --- a/crates/xfa/src/data/write.rs +++ b/crates/xfa/src/data/write.rs @@ -82,6 +82,11 @@ pub fn remove_data_instance(doc: &mut Document, path: &DataPath) -> Result) let mut run: Vec = Vec::new(); // Instances queued in this run, by parent and name: asking for more only queues the rest. let mut queued: HashMap<(&DataPath, &str), usize> = HashMap::new(); - let (mut ensured, mut capped) = (0usize, false); + let (mut ensured, mut ensured_bytes, mut capped, mut capped_bytes) = (0usize, 0usize, false, false); let flush = |run: &mut Vec, steps: &mut Vec>, removes: &mut Vec>| { if !run.is_empty() { let mut seen = std::collections::HashSet::new(); @@ -117,12 +122,24 @@ pub fn write_data_ops(doc: &mut Document, ops: &[DataOp], reuse: Option) } DataOp::Instances { parent, name, count } => { let done = queued.entry((parent, name.as_str())).or_insert(0); + // The path (its segments' buffers and their text) is held by the queued step, which a + // clone then `push` can leave with up to twice the room, and cloned again into + // `flush`'s dedup set, beside the label and the step itself. + let segment = std::mem::size_of::<(String, usize)>(); + let path_bytes = + parent.iter().fold(segment.saturating_add(name.len()), |n, (p, _)| n.saturating_add(segment).saturating_add(p.len())); + let per_instance = path_bytes.saturating_mul(3).saturating_add(name.len()).saturating_add(16 + std::mem::size_of::()); while *done < (*count).min(MAX_INDEX) { if ensured >= MAX_ENSURED { capped = true; break; } + if ensured_bytes.saturating_add(per_instance) > MAX_ENSURED_BYTES { + capped_bytes = true; + break; + } ensured += 1; + ensured_bytes += per_instance; #[cfg(test)] ENSURED.with(|n| n.set(n.get() + 1)); let k = *done; @@ -154,6 +171,11 @@ pub fn write_data_ops(doc: &mut Document, ops: &[DataOp], reuse: Option) if capped { written.warnings.push(format!("XFA data instances not written past {MAX_ENSURED}: the rows were added and removed too often")); } + if capped_bytes { + written + .warnings + .push(format!("XFA data instances not written past {MAX_ENSURED_BYTES} bytes of names: the rows were added and removed too often")); + } Ok(written) } diff --git a/crates/xfa/src/tests.rs b/crates/xfa/src/tests.rs index 682887e673..27e6522f26 100644 --- a/crates/xfa/src/tests.rs +++ b/crates/xfa/src/tests.rs @@ -7114,6 +7114,31 @@ fn a_container_is_as_wide_as_a_turned_draw_stands() { } } +/// Every sibling a SOM index skips owns a copy of the element name: the in-memory builder refuses +/// a path whose copies would pass the name-byte bound the writer holds itself to, and creates +/// nothing for it. +#[test] +fn the_in_memory_data_builder_bounds_copied_name_bytes() { + let mut root = DataNode::default(); + // 1 KiB x 9,999 skipped siblings is about 10 MB, over the 4 MiB bound. + let long = "n".repeat(1 << 10); + root.apply(&DataOp::Value { path: vec![(long.clone(), 9_999)], text: "x".into() }); + assert!(root.children.is_empty()); + // Within the bound, the same shape still builds (and keeps indexes). + root.apply(&DataOp::Value { path: vec![(long, 100)], text: "x".into() }); + assert_eq!(root.children.len(), 101); +} + +/// Each queued instance owns a copy of its parent path: a long ancestor stops the queue at the +/// byte allowance, reported as capped, instead of being multiplied by the instance count. +#[test] +fn queued_instances_are_bounded_by_the_path_bytes_they_copy() { + let mut doc = Document::new_empty(); + let parent: DataPath = vec![("a".repeat(16 << 20), 0), ("b".repeat(16 << 20), 0)]; + let written = crate::data::write_data_ops(&mut doc, &[DataOp::Instances { parent, name: "row".into(), count: 100 }], None).unwrap(); + assert!(written.warnings.iter().any(|w| w.contains("not written past")), "{:?}", written.warnings); +} + /// A page break splits an empty multi-line field that grows with its text (XFA 3.3 pp. 291–292: /// text splits between lines, and a field's `keep intact` is `none`): it starts where it is, its /// widget runs to the bottom of the content area, and the rest of its box, closed on every side, From 1e91e8a93ec8b8107f2c1f4f654c0a7629b07119 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 14:48:24 -0500 Subject: [PATCH 53/60] fix(forms): a read-only radio button is not picked or left through set_value (#6038) * fix(forms): a read-only radio button is not picked or left through set_value * address review: use /V as the selection in the read-only radio check, keep reset going * address review: radio read-only checks judge each button by the state it would get, and cover FieldValue::Widget --- crates/forms/src/tests.rs | 62 ++++++++++++++++++++++++++++++++++++ crates/forms/src/values.rs | 65 ++++++++++++++++++++++++++++++++------ 2 files changed, 117 insertions(+), 10 deletions(-) diff --git a/crates/forms/src/tests.rs b/crates/forms/src/tests.rs index 75dd075889..a2a87f3a73 100644 --- a/crates/forms/src/tests.rs +++ b/crates/forms/src/tests.rs @@ -6802,6 +6802,48 @@ fn locked_button_export_edits_leave_the_document_unchanged() { } } +/// A radio button that is read-only on its own (its own `/Ff`) can't be picked, and a selected one +/// can't be switched away from, though the group as a whole is writable. +#[test] +fn a_read_only_radio_button_is_neither_picked_nor_left() { + let lock = |doc: &mut Document, num: u32| doc.update_dict(pdfcraft_cos::ObjRef::new(num, 0), |d| d.set(b"Ff".to_vec(), Object::Int(1))).unwrap(); + let mut doc = radio_group_form(); + lock(&mut doc, 7); // Medium + assert!(matches!(set_value(&mut doc, "size", &FieldValue::Radio(Some("M".into()))), Err(FormError::ReadOnly(_)))); + set_value(&mut doc, "size", &FieldValue::Radio(Some("L".into()))).expect("an unlocked button is picked"); + // Small (selected) locked: the group stays on it. + let mut doc = radio_group_form(); + lock(&mut doc, 6); + assert!(matches!(set_value(&mut doc, "size", &FieldValue::Radio(Some("L".into()))), Err(FormError::ReadOnly(_)))); + assert_eq!(field(&fields(&doc), "size").value, ["S"]); +} + +/// The group's `/V` names its selection even when the button's `/AS` says nothing: a read-only +/// button that `/V` selects is not switched away from either. +#[test] +fn a_read_only_radio_button_selected_by_the_group_value_is_not_left() { + let mut doc = radio_group_form(); + doc.update_dict(pdfcraft_cos::ObjRef::new(6, 0), |d| { + d.set(b"Ff".to_vec(), Object::Int(1)); + d.set(b"AS".to_vec(), Object::name("Off")); + }) + .unwrap(); + assert!(matches!(set_value(&mut doc, "size", &FieldValue::Radio(Some("L".into()))), Err(FormError::ReadOnly(_)))); + assert_eq!(field(&fields(&doc), "size").value, ["S"]); +} + +/// Clearing the form skips a group held by a read-only button and still clears the rest. +#[test] +fn clearing_the_form_goes_on_past_a_group_held_by_a_read_only_button() { + let mut doc = radio_group_form(); + doc.update_dict(pdfcraft_cos::ObjRef::new(6, 0), |d| d.set(b"Ff".to_vec(), Object::Int(1))).unwrap(); + doc.update_dict(pdfcraft_cos::ObjRef::new(9, 0), |d| d.set(b"DV".to_vec(), pdfcraft_cos::PdfString::text("Bob"))).unwrap(); + assert_eq!(reset(&mut doc, None).unwrap(), 1); + let all = fields(&doc); + assert_eq!(field(&all, "size").value, ["S"]); + assert_eq!(field(&all, "name").value, ["Bob"]); +} + #[test] /// Cosmetic field updates skip calculations, while a value write still triggers the chain. fn script_cosmetic_mutations_do_not_run_the_calculation_chain() { @@ -6937,6 +6979,26 @@ fn radio_buttons_without_as_sharing_the_value_show_one_on() { assert_eq!(on_widgets(&doc, "r").0, [false, true, false]); } +/// Two buttons share the on state `Yes`; the first is on by its `/AS` and the second is read-only +/// and off. `/V` is `Yes` for both, but only the first is selected: picking another option switches +/// the first off, which is allowed, and the read-only one never changes. +#[test] +fn a_read_only_button_sharing_the_on_state_of_the_selected_one_does_not_block() { + let mut doc = radio_without_as(49152, "/Ff 1 /AS /Off"); + doc.update_dict(pdfcraft_cos::ObjRef::new(6, 0), |d| d.set(b"AS".to_vec(), Object::name("Yes"))).unwrap(); + set_value(&mut doc, "r", &FieldValue::Radio(Some("No".into()))).expect("the writable selected button may be switched off"); + assert_eq!(on_widgets(&doc, "r").0, [false, false, true]); +} + +/// Selecting another button through `FieldValue::Widget` can't switch a read-only selected one off. +#[test] +fn a_read_only_selected_radio_button_is_not_left_by_picking_a_widget() { + let mut doc = radio_group_form(); + doc.update_dict(pdfcraft_cos::ObjRef::new(6, 0), |d| d.set(b"Ff".to_vec(), Object::Int(1))).unwrap(); + assert!(matches!(set_value(&mut doc, "size", &FieldValue::Widget(2)), Err(FormError::ReadOnly(_)))); + assert_eq!(field(&fields(&doc), "size").value, ["S"]); +} + /// Table 227, ReadOnly: a processor "shall not allow a user to change the value". A button whose own /// dictionary sets it can't be turned on by number. #[test] diff --git a/crates/forms/src/values.rs b/crates/forms/src/values.rs index af2c5e1aca..b60d8aa140 100644 --- a/crates/forms/src/values.rs +++ b/crates/forms/src/values.rs @@ -356,6 +356,9 @@ fn write_value_impl(doc: &mut Document, f: &Field, value: &FieldValue, scripts: let Some(state) = f.widgets.get(*i).and_then(|w| w.on_state.clone()) else { return invalid(format!("{:?} has no button {} with an on state", f.name, i.saturating_add(1))); }; + if f.kind == FieldKind::Radio && radio_change_blocked(f, Some(&state), Some(*i)) { + return Err(FormError::ReadOnly(f.name.clone())); + } set_states(doc, f, Some(&state), Some(*i)) } (FieldKind::CheckBox, v) => { @@ -406,6 +409,9 @@ fn write_value_impl(doc: &mut Document, f: &Field, value: &FieldValue, scripts: if choice.is_none() && f.has(flags::NO_TOGGLE_TO_OFF) && !f.value.is_empty() { return invalid(format!("{:?} must keep one option selected", f.name)); } + if radio_change_blocked(f, choice.as_deref(), None) { + return Err(FormError::ReadOnly(f.name.clone())); + } set_states(doc, f, choice.as_deref(), None) } (FieldKind::Combo | FieldKind::List, FieldValue::Choice(_) | FieldValue::Text(_) | FieldValue::ChoiceIndices(_)) => { @@ -541,17 +547,49 @@ pub(crate) fn scroll_widgets(doc: &mut Document, f: &Field, picked: &[usize], co Ok(()) } +/// The one widget of an exclusive radio group that turns on when `on` is chosen (`pick` when it has +/// that state, else the one already on, else the first); `None` when `on` is `None` or the group +/// turns every widget with that state on together. +fn chosen_widget(f: &Field, on: Option<&str>, pick: Option) -> Option { + let exclusive = f.kind == FieldKind::Radio && !f.has(flags::RADIOS_IN_UNISON); + on.filter(|_| exclusive).and_then(|c| { + let has = |i: &usize| f.widgets.get(*i).is_some_and(|w| w.on_state.as_deref() == Some(c)); + let n = f.widgets.len(); + pick.filter(has).or_else(|| (0..n).filter(has).find(|&i| f.widget_on(i))).or_else(|| (0..n).find(has)) + }) +} + +/// Whether setting a radio group to `on` (through widget `pick`, when the caller chose a widget) +/// would turn a read-only button on or off: such a button can't be picked, and a selected one can't +/// be switched off by picking another. Judged per widget, by the state `set_states` would give it, +/// so buttons sharing an on-state name are told apart by their `/AS`; the group's `/V` stands for +/// the selection only when no button's `/AS` says which is on. +fn radio_change_blocked(f: &Field, on: Option<&str>, pick: Option) -> bool { + let chosen = chosen_widget(f, on, pick); + let before = |i: usize, w: &Widget| { + let Some(s) = w.on_state.as_deref() else { return false }; + if f.widget_on(i) { + return true; + } + // No button sharing this on state is on by its `/AS` and the group's `/V` names it: the + // first of them holds the selection, whatever its own `/AS` says (an `Off` among them). + let sharing = |x: &Widget| x.on_state.as_deref() == Some(s); + f.value.first().map(String::as_str) == Some(s) + && !f.widgets.iter().any(|x| sharing(x) && x.state.as_deref() == Some(s)) + && f.widgets.iter().position(sharing) == Some(i) + }; + f.widgets.iter().enumerate().any(|(i, w)| { + let after = on.is_some() && w.on_state.as_deref() == on && chosen.is_none_or(|k| k == i); + w.read_only && before(i, w) != after + }) +} + /// Check boxes and radio buttons: `/V` on the field, `/AS` on each widget. The widgets whose on /// state is `on` turn on together in a check box (ISO 32000-2 §12.7.5.2.3, EXAMPLE 2) and in a /// radio group with RadiosInUnison. Without it, at most one radio button is on (Table 229, /// §12.7.5.2.3): widget `pick` when it has that state, else the one already on, else the first. fn set_states(doc: &mut Document, f: &Field, on: Option<&str>, pick: Option) -> Result<(), FormError> { - let exclusive = f.kind == FieldKind::Radio && !f.has(flags::RADIOS_IN_UNISON); - let chosen = on.filter(|_| exclusive).and_then(|c| { - let has = |i: &usize| f.widgets.get(*i).is_some_and(|w| w.on_state.as_deref() == Some(c)); - let n = f.widgets.len(); - pick.filter(has).or_else(|| (0..n).filter(has).find(|&i| f.widget_on(i))).or_else(|| (0..n).find(has)) - }); + let chosen = chosen_widget(f, on, pick); // `/V` names the on state, the same for every widget that has it: which of them is on is // their `/AS` alone (§12.7.5.2.4). let v = on.unwrap_or("Off"); @@ -919,10 +957,17 @@ pub fn reset_with_formatting( // A reset is not blocked by NoToggleToOff: go through the writer directly. let mut tmp = f.clone(); tmp.flags &= !flags::NO_TOGGLE_TO_OFF; - write_value_impl(doc, &tmp, &v, scripts, true)?; - changed += 1; - if matches!(f.kind, FieldKind::Text | FieldKind::Combo | FieldKind::List) { - redraw_fields.insert(f.obj); + match write_value_impl(doc, &tmp, &v, scripts, true) { + Ok(()) => { + changed += 1; + if matches!(f.kind, FieldKind::Text | FieldKind::Combo | FieldKind::List) { + redraw_fields.insert(f.obj); + } + } + // a read-only button of the group holds the value: this field is left as it is, and the + // rest of the form is still cleared + Err(FormError::ReadOnly(_)) => {} + Err(e) => return Err(e), } } let calculation_runner: &mut dyn Scripts = if calculate_with_scripts { scripts } else { &mut NoScripts }; From 2b65ab000bfe0997dec5fbed2769d3da3799355b Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 15:20:18 -0500 Subject: [PATCH 54/60] ci: use StepSecurity's maintained actions (#7026) Each third-party action that StepSecurity publishes a secure drop-in replacement for now uses that replacement, pinned to a commit with its version as a comment. Versions follow what we used: the newest StepSecurity release in the same major, or the closest major they publish. dtolnay/rust-toolchain refs that named a toolchain (stable, nightly, 1.76) become the replacement's `toolchain` input. --- .github/workflows/fuzz.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index fe0a0074d7..78dc1b614b 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -31,10 +31,10 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: dtolnay/rust-toolchain@686976e191b89faba57d3206551f0f330d8cb249 # stable + - uses: step-security/dtolnay-rust-toolchain@8dd607977d06adf4ed49b190381e74c17a9dd25f # v1.0.0 with: toolchain: stable - - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 + - uses: step-security/rust-cache@8871c8ba063416da7c32246932ea6ae08700453e # v2.9.2 with: key: fuzz - name: Fetch the pdf.js corpus (seeds) From c73cccfb33a9146470060bf692578fbabb543607 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 15:37:44 -0500 Subject: [PATCH 55/60] fix(redact): hidden-layer removal reads the default state as viewers do (#5867) * fix(redact): hidden-layer removal reads the default state as viewers do hidden_layers ignored an indirect /BaseState and, with /BaseState /OFF, let /ON override /OFF, so a group both lists named survived as visible. The OCMD and OCG classifier also read /Type directly, so a membership dictionary with an indirect /Type was never evaluated and its content kept. The default state is now /BaseState (resolved), then /ON, then /OFF, and /Type is resolved before classifying a group or membership dictionary. * address review: follow chained /Type and /BaseState references, hash-set ON/OFF lists * address review: a visibility expression naming a group through an alias reference fails closed --- crates/redact/src/interp/layers.rs | 22 +++++++-- crates/redact/src/lib.rs | 13 +++++ crates/redact/src/sanitize.rs | 13 ++--- crates/redact/src/tests.rs | 76 ++++++++++++++++++++++++++++++ 4 files changed, 114 insertions(+), 10 deletions(-) diff --git a/crates/redact/src/interp/layers.rs b/crates/redact/src/interp/layers.rs index 362cb36620..7b8ef82120 100644 --- a/crates/redact/src/interp/layers.rs +++ b/crates/redact/src/interp/layers.rs @@ -20,7 +20,7 @@ impl<'a> Scope<'a> { // groups. A membership dictionary's default policy is AnyOn. let d = doc.resolve(o); let Some(d) = d.as_dict() else { return false }; - if d.name(b"Type") != Some(b"OCMD") { + if !has_type(doc, d, b"OCMD") { return false; } let groups: Vec = match d.get(b"OCGs").map(|g| (*doc.resolve(g)).clone()) { @@ -102,7 +102,7 @@ impl<'a> Scope<'a> { } let resolved = doc.resolve(expression); let complete = match &*resolved { - Object::Dict(d) if d.name(b"Type") == Some(b"OCG") => { + Object::Dict(d) if has_type(doc, d, b"OCG") => { if let Some(r) = reference && hidden_layers.contains(&r) { @@ -144,6 +144,11 @@ impl<'a> Scope<'a> { } *budget -= 1; let reference = expression.as_ref(); + // A reference to a reference is no listed group: the renderer rejects the alias, so it + // is not taken as visible because the object it names isn't hidden. + if reference.is_some_and(|r| matches!(*doc.get(r), Object::Ref(_))) { + return None; + } if let Some(r) = reference && !path.insert(r) { @@ -151,7 +156,7 @@ impl<'a> Scope<'a> { } let resolved = doc.resolve(expression); let result = match &*resolved { - Object::Dict(d) if d.name(b"Type") == Some(b"OCG") => reference.map(|r| !self.hidden_layers.contains(&r)), + Object::Dict(d) if has_type(doc, d, b"OCG") => reference.map(|r| !self.hidden_layers.contains(&r)), Object::Array(items) if (2..=4097).contains(&items.len()) => { let operator = match items.first() { Some(Object::Name(n)) => n.as_slice(), @@ -195,6 +200,9 @@ impl<'a> Scope<'a> { if self.hidden_layers.contains(&r) { return true; } + if matches!(*doc.get(r), Object::Ref(_)) { + return true; + } if !path.insert(r) { return true; } @@ -202,7 +210,7 @@ impl<'a> Scope<'a> { let resolved = doc.resolve(expression); let found = match &*resolved { Object::Array(items) => items.iter().any(|item| self.expression_mentions_hidden(doc, item, depth + 1, path, budget)), - Object::Dict(d) if d.name(b"Type") == Some(b"OCG") => expression.as_ref().is_none_or(|r| self.hidden_layers.contains(&r)), + Object::Dict(d) if has_type(doc, d, b"OCG") => expression.as_ref().is_none_or(|r| self.hidden_layers.contains(&r)), Object::Dict(_) => true, _ => false, }; @@ -256,3 +264,9 @@ pub(super) fn strip_hidden_layers(doc: &Document, scope: &mut Scope<'_>, ops: Ve } (out, removed) } + +/// Is this dictionary's `/Type` the given name? An indirect `/Type` is followed, as a viewer +/// follows it: a group or membership dictionary must not escape classification through one. +fn has_type(doc: &Document, d: &Dict, name: &[u8]) -> bool { + d.get(b"Type").is_some_and(|t| crate::resolve_chain(doc, t).as_name() == Some(name)) +} diff --git a/crates/redact/src/lib.rs b/crates/redact/src/lib.rs index 7fd34dae12..8f54cefdc6 100644 --- a/crates/redact/src/lib.rs +++ b/crates/redact/src/lib.rs @@ -1137,6 +1137,19 @@ pub fn clear_marks(doc: &mut Document, pages: Option<&[usize]>) -> Result std::sync::Arc { + let mut o = doc.resolve(o); + for _ in 0..16 { + if !matches!(*o, pdfcraft_cos::Object::Ref(_)) { + break; + } + o = doc.resolve(&o); + } + o +} + /// The name to save a document under after [`apply`]: `_Redacted` before the extension, as /// Acrobat appends it ("contract.pdf" gives "contract_Redacted.pdf"). The `.pdf` extension is /// matched in any letter case. A name that already ends in `_Redacted` keeps it once. diff --git a/crates/redact/src/sanitize.rs b/crates/redact/src/sanitize.rs index cd38b9eb19..61c7cedda4 100644 --- a/crates/redact/src/sanitize.rs +++ b/crates/redact/src/sanitize.rs @@ -479,12 +479,13 @@ fn hidden_layers(doc: &Document) -> Vec { o.and_then(|g| doc.resolve(g).as_array().map(|a| a.iter().filter_map(Object::as_ref).collect())).unwrap_or_default() }; let all = refs(oc.get(b"OCGs")); - let (on, off) = (refs(d.get(b"ON")), refs(d.get(b"OFF"))); - let mut hidden: Vec = if d.name(b"BaseState") == Some(b"OFF") { - all.iter().copied().filter(|g| !on.contains(g)).collect() - } else { - off.into_iter().filter(|g| all.contains(g)).collect() - }; + // sets: the lists are the document's, and a filter over every group would scan them each time + let (on, off): (std::collections::HashSet, std::collections::HashSet) = + (refs(d.get(b"ON")).into_iter().collect(), refs(d.get(b"OFF")).into_iter().collect()); + // The default state is /BaseState, then /ON turns groups on and /OFF turns them off, so a + // group both lists name ends up off (§8.11.4.3, Table 101) — the viewer reads it the same way. + let base_off = d.get(b"BaseState").map(|b| crate::resolve_chain(doc, b)).is_some_and(|b| b.as_name() == Some(b"OFF".as_slice())); + let mut hidden: Vec = all.iter().copied().filter(|g| off.contains(g) || (base_off && !on.contains(g))).collect(); // Each rule, layer list and layer is read once, however often the rules name them: many rules // can share one long list. let known: std::collections::HashSet = all.iter().copied().collect(); diff --git a/crates/redact/src/tests.rs b/crates/redact/src/tests.rs index dc94a2c5f5..08125165b5 100644 --- a/crates/redact/src/tests.rs +++ b/crates/redact/src/tests.rs @@ -4151,6 +4151,8 @@ fn hidden_ocmd_policy_matches_the_default_configuration() { (Some("AllOn"), false, Some("not_off"), false), (None, false, Some("malformed_not"), true), (None, false, Some("malformed_without_ocgs"), true), + // `/VE` names the hidden group through an alias object (10 0 R is a reference to 8 0 R). + (None, false, Some("alias"), true), ]; for (policy, malformed_policy, visibility, should_remove) in cases { let mut doc = one_page( @@ -4188,9 +4190,11 @@ fn hidden_ocmd_policy_matches_the_default_configuration() { Some("malformed_without_ocgs") => { membership.set(b"VE".to_vec(), Object::Array(vec![Object::name("UnknownOperator"), Object::Ref(ObjRef::new(8, 0))])) } + Some("alias") => membership.set(b"VE".to_vec(), Object::Array(vec![Object::name("Or"), Object::Ref(ObjRef::new(10, 0))])), _ => {} } doc.set(ObjRef::new(9, 0), Object::Dict(membership)); + doc.set(ObjRef::new(10, 0), Object::Ref(ObjRef::new(8, 0))); let root = doc.root().unwrap(); doc.update_dict(root, |d| { d.set( @@ -6670,6 +6674,78 @@ fn a_reply_on_another_page_to_a_removed_popup_goes_too() { } } +/// One page drawing `SECRET` in the `/OC /L` block, with layer 7 0 R in the default +/// configuration `config` (objects 8 and up are `extra`). +fn layer_state_page(config: Vec<(&str, Object)>, resource: &str, extra: Vec>) -> Document { + let mut objects = vec![b"<< /Type /OCG /Name (Secret) >>".to_vec()]; + objects.extend(extra); + let mut doc = one_page(b"/OC /L BDC BT /F1 10 Tf 10 100 Td (SECRET) Tj ET EMC", &format!("/Properties << /L {resource} >>"), objects); + let config = Object::Dict(Dict::from_iter(config.into_iter().map(|(k, v)| (k.as_bytes().to_vec(), v)))); + let root = doc.root().unwrap(); + doc.update_dict(root, |d| { + d.set( + b"OCProperties".to_vec(), + Object::Dict(Dict::from_iter([(b"OCGs".to_vec(), Object::Array(vec![Object::Ref(ObjRef::new(7, 0))])), (b"D".to_vec(), config)])), + ); + }) + .unwrap(); + doc +} + +fn assert_layer_secret_removed(mut doc: Document) { + use crate::sanitize::{Hidden, remove_hidden, scan}; + let counts = scan(&doc); + assert!(counts.iter().any(|(h, n)| *h == Hidden::HiddenLayers && *n > 0), "{counts:?}"); + remove_hidden(&mut doc, &[Hidden::HiddenLayers]).unwrap(); + let saved = Document::open(Arc::new(write_full(&doc, &SaveOptions::default()).unwrap())).unwrap(); + assert!(!pdfcraft_edit::text_lines(&saved, 0).unwrap().iter().any(|line| line.text.contains("SECRET"))); +} + +#[test] +fn an_indirect_base_state_off_hides_the_layer() { + assert_layer_secret_removed(layer_state_page(vec![("BaseState", Object::Ref(ObjRef::new(8, 0)))], "7 0 R", vec![b"/OFF".to_vec()])); +} + +#[test] +fn a_chained_base_state_off_hides_the_layer() { + assert_layer_secret_removed(layer_state_page( + vec![("BaseState", Object::Ref(ObjRef::new(8, 0)))], + "7 0 R", + vec![b"9 0 R".to_vec(), b"/OFF".to_vec()], + )); +} + +#[test] +fn a_chained_membership_type_is_evaluated() { + assert_layer_secret_removed(layer_state_page( + vec![("BaseState", Object::name("ON"))], + "8 0 R", + vec![b"<< /Type 9 0 R /OCGs [7 0 R] /P /AllOff >>".to_vec(), b"10 0 R".to_vec(), b"/OCMD".to_vec()], + )); +} + +#[test] +fn off_wins_over_on_when_the_base_state_is_off() { + assert_layer_secret_removed(layer_state_page( + vec![ + ("BaseState", Object::name("OFF")), + ("ON", Object::Array(vec![Object::Ref(ObjRef::new(7, 0))])), + ("OFF", Object::Array(vec![Object::Ref(ObjRef::new(7, 0))])), + ], + "7 0 R", + Vec::new(), + )); +} + +#[test] +fn a_membership_dictionary_with_an_indirect_type_is_evaluated() { + assert_layer_secret_removed(layer_state_page( + vec![("BaseState", Object::name("ON"))], + "8 0 R", + vec![b"<< /Type 9 0 R /OCGs [7 0 R] /P /AllOff >>".to_vec(), b"/OCMD".to_vec()], + )); +} + /// An invisible OCR-style text layer drawn in a Type3 font (what the OCR output writes for /// characters beyond WinAnsi) is redacted like any other text: its glyphs are hit-tested by /// their real advances, removed, and the verifier passes. From daec898d7c56fc9f283777c99e2f1ca1f52d53c2 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 15:49:35 -0500 Subject: [PATCH 56/60] fix(sign): judge a shared signature value once per listing; dedupe DSS evidence by object number (#6041) * fix(sign): judge a shared signature value once per listing; dedupe DSS evidence by object number * address review: shared signature values are judged by output index without copying the dictionary; a dead evidence reference does not hide a live one * address review: repeated evidence references are skipped before they are resolved * address review: each full evidence reference is tried once --- crates/sign/src/pdf/listing.rs | 16 ++++++++++++- crates/sign/src/pdf/validate.rs | 20 +++++++++++++--- crates/sign/src/pdf_discovery_tests.rs | 13 +++++++++++ crates/sign/tests/it/pdf.rs | 32 ++++++++++++++++++++++++++ 4 files changed, 77 insertions(+), 4 deletions(-) diff --git a/crates/sign/src/pdf/listing.rs b/crates/sign/src/pdf/listing.rs index c2a7183db1..2d41477127 100644 --- a/crates/sign/src/pdf/listing.rs +++ b/crates/sign/src/pdf/listing.rs @@ -431,8 +431,12 @@ pub fn list(doc: &Document, bytes: &[u8], trust: &TrustStore) -> Vec Vec { let pages = annot_pages(doc); - let mut out = Vec::new(); + let mut out: Vec = Vec::new(); let mut field_values: HashSet = HashSet::new(); + // Fields that name one signature value are judged once: validation does not depend on the + // field, and repeating it per field would repeat the digest, chain and evidence work. + // The row of `out` that holds each value's verdict: a later field copies it, not a map of copies. + let mut judged: HashMap = HashMap::new(); let (fields, whole) = walk_sig_fields(doc); for f in fields { if let Some(v_ref) = f.dict.get(b"V").and_then(Object::as_ref) { @@ -445,6 +449,13 @@ pub fn list_cached(doc: &Document, bytes: &[u8], trust: &TrustStore, cache: &Dig .filter(|r| r.len() == 4) .map(|r| [r[0].min(r[2]), r[1].min(r[3]), r[0].max(r[2]), r[1].max(r[3])]); let visible = widget.is_some_and(|(_, w)| widget_shown(doc, w)); + // A repeat of a judged value reads nothing of it: the signature dictionary is not copied. + let shared = f.dict.get(b"V").and_then(Object::as_ref).filter(|_| f.dict.get(b"V").is_some_and(|v| doc.resolve(v).as_dict().is_some())); + if let Some(first) = shared.and_then(|r| judged.get(&r)).and_then(|&i| out.get(i)) { + let own = SignatureInfo { field: f.name.clone(), signed: true, page, rect, visible, ..first.clone() }; + out.push(own); + continue; + } let v = f.signature(doc); let mut info = SignatureInfo { field: f.name.clone(), @@ -477,6 +488,9 @@ pub fn list_cached(doc: &Document, bytes: &[u8], trust: &TrustStore, cache: &Dig if let Some(v) = v { validate_into(doc, bytes, trust, &v, &mut info, cache); } + if let Some(r) = shared { + judged.insert(r, out.len()); + } out.push(info); } // Standalone document timestamps (ISO 32000-2 §12.8.2.2) live outside AcroForm fields. diff --git a/crates/sign/src/pdf/validate.rs b/crates/sign/src/pdf/validate.rs index 52bf79351f..d3ad411a62 100644 --- a/crates/sign/src/pdf/validate.rs +++ b/crates/sign/src/pdf/validate.rs @@ -371,7 +371,11 @@ pub(crate) const MAX_DSS_BYTES: usize = 64 << 20; pub(crate) struct DssEvidence<'a> { doc: &'a Document, items: std::vec::IntoIter, - seen: HashSet, + /// Object numbers read: `Document::get` resolves a reference by number, so `5 0 R` and `5 1 R` + /// are one stream. + seen: HashSet, + /// Full references already tried, resolved or not. + tried: HashSet, taken: usize, left: usize, /// A cap stopped the array being read in full. @@ -381,7 +385,7 @@ pub(crate) struct DssEvidence<'a> { impl<'a> DssEvidence<'a> { pub(crate) fn new(doc: &'a Document, dss: &Dict, key: &[u8]) -> Self { let items = dss.get(key).map(|o| doc.resolve(o)).and_then(|o| o.as_array().cloned()).unwrap_or_default(); - DssEvidence { doc, items: items.into_iter(), seen: HashSet::new(), taken: 0, left: MAX_DSS_BYTES, cut: false } + DssEvidence { doc, items: items.into_iter(), seen: HashSet::new(), tried: HashSet::new(), taken: 0, left: MAX_DSS_BYTES, cut: false } } } @@ -395,11 +399,21 @@ impl Iterator for DssEvidence<'_> { // Repeats are skipped before the caps count anything, so repeating one stream cannot use // them up; the scan of the array itself is bounded by the parser's cap on array length. for item in self.items.by_ref() { - if item.as_ref().is_some_and(|r| !self.seen.insert(r)) { + // Resolved first: a reference to a generation the slot doesn't hold reads as null and + // must not make a later, valid reference to the same object number a repeat. + // A repeat is skipped before anything is resolved: a long chain of repeats costs no + // lookups. The number is recorded only once the reference gave a stream. + // Each full reference is tried once, so one that never reaches a stream (a cycle) costs + // its lookups once, however often it is listed; `seen` keeps object numbers apart so a + // dead generation can't hide a live one. + if item.as_ref().is_some_and(|r| self.seen.contains(&r.num) || !self.tried.insert(r)) { continue; } let o = self.doc.resolve(&item); let Object::Stream(s) = o.as_ref() else { continue }; + if let Some(r) = item.as_ref() { + self.seen.insert(r.num); + } if self.taken >= MAX_DSS_ENTRIES { self.cut = true; return None; diff --git a/crates/sign/src/pdf_discovery_tests.rs b/crates/sign/src/pdf_discovery_tests.rs index 29f115cb07..d3250d62ed 100644 --- a/crates/sign/src/pdf_discovery_tests.rs +++ b/crates/sign/src/pdf_discovery_tests.rs @@ -373,6 +373,19 @@ fn only_grew_compares_huge_arrays_without_quadratic_work() { assert!(start.elapsed() < std::time::Duration::from_secs(10), "{:?}", start.elapsed()); } +/// A reference to a generation the object doesn't have reads as null: it is no repeat of the valid +/// reference to the same object number that follows it. +#[test] +fn dss_evidence_skips_a_dead_reference_without_hiding_the_live_one() { + let mut doc = fixture(&[]); + let live = doc.add(Object::Stream(Stream::flate(Dict::new(), b"evidence"))); + let dead = ObjRef::new(live.num, live.generation + 1); + let mut dss = Dict::new(); + dss.set(b"CRLs".to_vec(), Object::Array(vec![Object::Ref(dead), Object::Ref(live)])); + let read: Vec> = DssEvidence::new(&doc, &dss, b"CRLs").collect(); + assert_eq!(read, vec![b"evidence".to_vec()]); +} + #[test] fn dss_evidence_decodes_each_stream_once_within_its_caps() { // One small stream listed 1000 times, then streams that inflate past the evidence cap. diff --git a/crates/sign/tests/it/pdf.rs b/crates/sign/tests/it/pdf.rs index fa45ab1ecb..5d24d05dce 100644 --- a/crates/sign/tests/it/pdf.rs +++ b/crates/sign/tests/it/pdf.rs @@ -3369,6 +3369,38 @@ fn signing_a_field_with_a_null_value_after_another_signature_keeps_the_first_val ); } +/// Fields that name one signature value are listed each, under their own name, with the verdict +/// the value earned once. +#[test] +fn fields_sharing_a_signature_value_are_listed_with_one_verdict() { + let id = pkcs12::open(&data("rsa-aes.p12"), "test").unwrap(); + let trust = TrustStore { certs: vec![id.certificate.clone()] }; + let signed = pdfcraft_sign::sign(&open(&fixture()), &id, &SignOptions { field: Some("Approval".into()), ..opts() }).unwrap(); + let v = open(&signed).get(pdfcraft_cos::ObjRef { num: 6, generation: 0 }).as_dict().and_then(|d| d.get(b"V").cloned()).unwrap(); + let shared = edit_after(&signed, |doc| { + let mut w = pdfcraft_cos::Dict::new(); + w.set(b"Type".to_vec(), Object::name("Annot")); + w.set(b"Subtype".to_vec(), Object::name("Widget")); + w.set(b"FT".to_vec(), Object::name("Sig")); + w.set(b"T".to_vec(), PdfString::text("Twin")); + w.set(b"V".to_vec(), v.clone()); + w.set(b"Rect".to_vec(), Object::Array(vec![Object::Int(0); 4])); + let w = doc.add(Object::Dict(w)); + let root = doc.root().unwrap(); + let af = doc.get(root).as_dict().and_then(|c| c.get(b"AcroForm").cloned()).unwrap(); + let af = doc.resolve(&af).as_dict().cloned().unwrap(); + let mut fields = af.get(b"Fields").map(|f| doc.resolve(f)).and_then(|f| f.as_array().cloned()).unwrap(); + fields.push(Object::Ref(w)); + let mut af = af; + af.set(b"Fields".to_vec(), Object::Array(fields)); + doc.update_dict(root, |c| c.set(b"AcroForm".to_vec(), Object::Dict(af))).unwrap(); + }); + let list = signatures(&open(&shared), &shared, &trust); + let (a, b) = (list.iter().find(|s| s.field == "Approval").unwrap(), list.iter().find(|s| s.field == "Twin").unwrap()); + assert!(a.signed && b.signed); + assert_eq!((a.status, &a.modification, &a.signer, a.signed_len), (b.status, &b.modification, &b.signer, b.signed_len)); +} + #[test] fn the_catalogs_perms_do_not_change_under_no_changes() { // /Perms names the certification signature (DocMDP). Under P=1 nothing may change, so a later From 77078864f41f07818624ea98c3d1ddee1201e547 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 15:50:02 -0500 Subject: [PATCH 57/60] fix(render): charge inspection text (fields, tooltips, layers, bookmarks, blank comments) against shared allowances (#6015) * fix(render): charge field, tooltip, layer, bookmark and blank-comment text against shared allowances * address review: charge comment text once, keep JS and field-action flags past the budgets * review: count an unlisted field's actions through its whole subtree * review: mark multi-select values the allowance dropped * review: count only real actions, walk each field once, keep name selections * address review: revisit shared field descendants from shallower paths, charge value separators, check every widget for actions * merge main, keep both --- crates/render/src/inspect/annotations.rs | 169 +++++++++++++++++++---- crates/render/src/inspect/core.rs | 68 ++++++++- crates/render/src/inspect/navigation.rs | 2 +- crates/render/src/inspect/resources.rs | 7 +- crates/render/src/inspect/tests.rs | 80 +++++++++++ 5 files changed, 289 insertions(+), 37 deletions(-) diff --git a/crates/render/src/inspect/annotations.rs b/crates/render/src/inspect/annotations.rs index 7313716a2e..673d930743 100644 --- a/crates/render/src/inspect/annotations.rs +++ b/crates/render/src/inspect/annotations.rs @@ -144,12 +144,7 @@ impl<'a> Inspector<'a> { info.attachments.push(self.attachment(fs, AttachmentSource::Annotation { page, index }, text_budget, fallback)); } if subtype == "Link" { - if let Some(link) = self.link(page, index, d, info.uri_base.as_deref(), text_budget, &mut unfit_quads) { - if matches!(&link.target, LinkTarget::Other(s) if s == "JavaScript") { - info.has_javascript = true; - } - info.links.push(link); - } + self.collect_link(info, (page, index), d, text_budget, &mut unfit_quads); continue; } // Not comments: form widgets, pop-ups and non-markup annotations such as the @@ -161,6 +156,20 @@ impl<'a> Inspector<'a> { if comments_full { continue; } + // Text that trims to nothing is not kept, so `retained` never sees it: charge what is + // read, so many comments sharing one huge blank string stop being decoded over and over. + let raw: usize = [&b"T"[..], b"Contents", b"M", b"NM", b"State", b"DA"] + .iter() + .filter_map(|k| match d.get(k).map(|o| self.resolve(o)) { + Ok(Object::String(b, _) | Object::Name(b)) => Some(b.len()), + _ => None, + }) + .fold(0, usize::saturating_add); + // (charged once: below, by the larger of these raw bytes and what the annotation keeps) + if kept.saturating_add(raw) > MAX_ANNOTATION_BYTES { + comments_full = true; + continue; + } let rect = rect4(self.resolve(d.get(b"Rect").unwrap_or(&Object::Null))); let c = match d.get(b"C").map(|o| self.resolve(o)) { Ok(Object::Array(c)) => Some(c), @@ -262,7 +271,7 @@ impl<'a> Inspector<'a> { text_box, popup, }; - kept = kept.saturating_add(retained(&annotation)); + kept = kept.saturating_add(retained(&annotation).max(raw)); if kept > MAX_ANNOTATION_BYTES { comments_full = true; continue; @@ -356,6 +365,23 @@ impl<'a> Inspector<'a> { Some(Link { page, index, rect, quads, target }) } + /// Lists the link annotation `d`, and notes the JavaScript it runs. JavaScript is read from the + /// action, not the listed link: once the text budget is spent the action's name is not kept. + fn collect_link(&self, info: &mut DocInfo, at: (usize, usize), d: &Dictionary, text_budget: &mut usize, unfit_quads: &mut HashSet) { + info.has_javascript |= self.runs_javascript(d); + if let Some(link) = self.link(at.0, at.1, d, info.uri_base.as_deref(), text_budget, unfit_quads) { + if matches!(&link.target, LinkTarget::Other(s) if s == "JavaScript") { + info.has_javascript = true; + } + info.links.push(link); + } + } + + /// Whether the link's own action is a JavaScript action. + fn runs_javascript(&self, d: &Dictionary) -> bool { + d.get(b"A").ok().and_then(|a| self.dict(a)).is_some_and(|a| self.name(a, b"S").as_deref() == Some("JavaScript")) + } + /// What an action does when a link or a bookmark runs it. Only a go-to's `/D` is a place in /// this document; a remote go-to's names a page or destination of another file (ISO 32000-2 /// §12.6.4.3). `None` for an action without a type, or a go-to whose destination is not here. @@ -376,7 +402,7 @@ impl<'a> Inspector<'a> { "SetOCGState" => self.layer_state(a), "GoToR" => self.remote_go_to(a).unwrap_or_else(|| LinkTarget::Other("GoToR".into())), "Launch" => self.launch_pdf(a).unwrap_or_else(|| LinkTarget::Other("Launch".into())), - other => LinkTarget::Other(other.to_string()), + other => LinkTarget::Other(spend(other.as_bytes(), text_budget, |b| String::from_utf8_lossy(b).into_owned()).unwrap_or_default()), }) } @@ -492,6 +518,53 @@ impl<'a> Inspector<'a> { // ── form fields ───────────────────────────────────────────────────────────────────────── + /// Whether `o` or one of its descendants under `/Kids` carries an action (`/A` or `/AA`). It + /// walks the same tree `field` does, bounded the same way: a field left out of the listing for + /// want of text allowance can still have its action on a widget several levels down. + pub(super) fn has_action(&self, o: &Object, depth: u32) -> bool { + self.has_action_seen(o, depth, &mut HashMap::new()) + } + + /// The walk behind [`Self::has_action`]. Each indirect descendant is read again only from a + /// shallower depth than before (the depth limit may have cut its subtree): a field tree whose + /// `/Kids` repeat the same objects would otherwise be walked exponentially, and this reads each + /// object at most once per depth. + fn has_action_seen(&self, o: &Object, depth: u32, seen: &mut HashMap) -> bool { + if depth > 32 { + return false; + } + if let Object::Reference(id) = o { + if seen.get(id).is_some_and(|&first| first <= depth) { + return false; + } + seen.insert(*id, depth); + } + let Some(d) = self.dict(o) else { return false }; + if self.carries_action(d) { + return true; + } + let kids = match d.get(b"Kids").map(|k| self.resolve(k)) { + Ok(Object::Array(k)) => k, + _ => return false, + }; + kids.iter().any(|k| self.has_action_seen(k, depth + 1, seen)) + } + + /// Whether `/A` or `/AA` here holds an action. An editor can leave an empty dictionary (or + /// array) behind, and that is no action: a document with none must not report JavaScript. An + /// `/A` is an action when it carries `/S`; an `/AA` is one when an event maps to such an action. + fn carries_action(&self, d: &Dictionary) -> bool { + let action = |o: &Object| matches!(self.resolve(o), Object::Dictionary(a) if a.get(b"S").is_ok()); + [&b"A"[..], &b"AA"[..]].iter().any(|k| match d.get(k) { + Ok(o) => match self.resolve(o) { + Object::Dictionary(x) => x.get(b"S").is_ok() || x.iter().any(|(_, v)| action(v)), + Object::Array(a) => a.iter().any(action), + _ => false, + }, + Err(_) => false, + }) + } + pub(super) fn field( &self, o: &'a Object, @@ -510,19 +583,35 @@ impl<'a> Inspector<'a> { } let Some(d) = self.dict(o) else { return }; let (parent_name, parent_ft, parent_ff) = parent.unwrap_or(("", None, None)); - let partial = self.text(d, b"T"); - let full = match (&partial, parent_name.is_empty()) { - (Some(t), true) => t.clone(), - (Some(t), false) => format!("{parent_name}.{t}"), - (None, _) => parent_name.to_string(), - }; - let ft = self.name(d, b"FT").or(parent_ft); - let ff = d.get(b"Ff").ok().and_then(|x| x.as_i64().ok()).or(parent_ff); // Non-terminal: has kids that are fields (they carry /T). Widgets-only kids are terminal. let mut kids: Vec = match d.get(b"Kids").map(|k| self.resolve(k)) { Ok(Object::Array(k)) => k.clone(), _ => Vec::new(), }; + // A field left unlisted for want of text allowance still has its actions counted, through + // its whole subtree: an action can sit on a descendant widget, not only on the field. + let unlisted = || { + if self.has_action(o, 0) { + self.field_actions.set(true); + } + }; + // A /T that does not fit the allowance lists no field (not one under its parent's name). + let Ok(partial) = self.field_text_within(d, b"T") else { + unlisted(); + return; + }; + let full = match (&partial, parent_name.is_empty()) { + (Some(t), true) => t.clone(), + (Some(t), false) => format!("{parent_name}.{t}"), + (None, _) => parent_name.to_string(), + }; + // Every field copies its parent's full name: charge it, and list no more once it is spent. + if !self.field_charge(full.len()) { + unlisted(); + return; + } + let ft = self.name(d, b"FT").or(parent_ft); + let ff = d.get(b"Ff").ok().and_then(|x| x.as_i64().ok()).or(parent_ff); let (field_kids, widget_kids): (Vec, Vec) = kids.iter().cloned().partition(|k| self.dict(k).is_some_and(|kd| kd.get(b"T").is_ok())); let field_kids: Vec<_> = field_kids.iter().filter(|k| !self.not_a_child(walk, o, k)).collect(); @@ -570,17 +659,37 @@ impl<'a> Inspector<'a> { _ => FieldKind::Unknown, }; let value = match d.get(b"V").map(|v| self.resolve(v)) { - Ok(Object::Name(n)) => Some(String::from_utf8_lossy(n).into_owned()), - Ok(Object::String(bytes, _)) => Some(text_string(bytes)), - Ok(Object::Array(a)) => Some( - a.iter() - .filter_map(|x| match self.resolve(x) { - Object::String(bytes, _) => Some(text_string(bytes)), - _ => None, - }) - .collect::>() - .join(", "), - ), + Ok(Object::Name(n)) => self.field_spend(n, |n| String::from_utf8_lossy(n).into_owned()), + Ok(Object::String(bytes, _)) => self.field_spend(bytes, text_string), + Ok(Object::Array(a)) => { + let mut shown = Vec::new(); + let mut dropped = false; + for x in a { + let text = match self.resolve(x) { + Object::String(bytes, _) => self.field_spend(bytes, text_string), + // A name entry (`/V [/Opt1 (b)]`) is a selection too, as the forms reader + // takes it; leaving it out reported a partial selection as complete. + Object::Name(n) => self.field_spend(n, |n| String::from_utf8_lossy(n).into_owned()), + _ => continue, + }; + // The ", " between selections is charged to the allowance like the text. + match text { + Some(text) if shown.is_empty() || self.field_charge(2) => shown.push(text), + _ => dropped = true, + } + } + // A selected value the allowance could not hold shows as an omission rather than + // as part of a complete selection (its few bytes are charged, fit or not). + let mut value = shown.join(", "); + if dropped { + self.field_charge(5); + if !value.is_empty() { + value.push_str(", "); + } + value.push('…'); + } + Some(value) + } Ok(Object::Dictionary(_)) if kind == FieldKind::Signature => Some("signed".into()), _ => None, }; @@ -602,9 +711,11 @@ impl<'a> Inspector<'a> { .and_then(|p| self.page_of(p)) .or_else(|| self.find_widget_page(o, &kids)) .or_else(|| named.and_then(|id| self.annotation_pages.get_or_init(|| self.annotation_pages()).get(&id).copied())); - let has_actions = d.get(b"AA").is_ok() || widget.is_some_and(|w| w.get(b"AA").is_ok() || w.get(b"A").is_ok()); + let has_actions = self.carries_action(d) + || widget.is_some_and(|w| self.carries_action(w)) + || kids.iter().any(|k| self.dict(k).is_some_and(|w| self.carries_action(w))); let rect = widget.and_then(|w| w.get(b"Rect").ok()).map(|r| rect4(self.resolve(r))); - out.push(Field { name: full, kind, value, page, tooltip: self.text(d, b"TU"), has_actions, rect }); + out.push(Field { name: full, kind, value, page, tooltip: self.field_text(d, b"TU"), has_actions, rect }); } /// Whether `k`, listed in the `/Kids` of field `o`, is not a child of `o` after all: `o` itself diff --git a/crates/render/src/inspect/core.rs b/crates/render/src/inspect/core.rs index 88dc13c34a..adf3edffbd 100644 --- a/crates/render/src/inspect/core.rs +++ b/crates/render/src/inspect/core.rs @@ -155,16 +155,34 @@ pub(super) struct Inspector<'a> { /// has no `/P`. Searching every page's `/Annots` for each such field was O(fields × /// annotations): a second on a form of 8,000 fields. pub(super) annotation_pages: OnceCell>, + /// What is left of [`MAX_FIELD_TEXT_BYTES`]: names, values and tooltips of form fields and + /// layer names, which many entries can share one string of. + pub(super) field_text: std::cell::Cell, + /// Set when a field with an action went unlisted because the text allowance was spent, so the + /// document still reports JavaScript. + pub(super) field_actions: std::cell::Cell, /// The first page widget naming each field as its `/Parent`, built when a field has no widget /// among its `/Kids`. pub(super) parent_widgets: OnceCell>, } +/// Bytes of field and layer text one inspection reads and keeps together. +pub(super) const MAX_FIELD_TEXT_BYTES: usize = 64 << 20; + impl<'a> Inspector<'a> { pub(super) fn new(doc: &'a dyn Structure) -> Self { let page_ids = doc.page_ids(); let page_index = page_ids.iter().copied().enumerate().map(|(index, id)| (id, index)).collect(); - let mut me = Self { doc, page_ids, page_index, named: HashMap::new(), annotation_pages: OnceCell::new(), parent_widgets: OnceCell::new() }; + let mut me = Self { + doc, + page_ids, + page_index, + named: HashMap::new(), + annotation_pages: OnceCell::new(), + field_text: MAX_FIELD_TEXT_BYTES.into(), + field_actions: false.into(), + parent_widgets: OnceCell::new(), + }; let mut named = HashMap::new(); if let Some(tree) = doc.catalog().ok().and_then(|c| c.get(b"Names").ok()).and_then(|o| me.dict(o)).and_then(|n| n.get(b"Dests").ok()).and_then(|o| me.dict(o)) @@ -260,7 +278,7 @@ impl<'a> Inspector<'a> { } } } - info.has_javascript |= info.fields.iter().any(|f| f.has_actions); + info.has_javascript |= info.fields.iter().any(|f| f.has_actions) || self.field_actions.get(); let needs_rendering = catalog.get(b"NeedsRendering").ok().and_then(|o| self.resolve(o).as_bool().ok()).unwrap_or(false); if let Some(xfa) = catalog.get(b"AcroForm").ok().and_then(|o| self.dict(o)).and_then(|form| form.get(b"XFA").ok()) { // A form with no fields is taken to be laid out from its template, unless its @@ -351,11 +369,49 @@ impl<'a> Inspector<'a> { /// Whitespace-trimmed metadata, spent from `budget` as [`spend`] does, so text that trims /// short or to nothing still spends its raw length. + /// [`Self::budgeted_text`] against the shared field and layer text allowance. + pub(super) fn field_text(&self, d: &Dictionary, key: &[u8]) -> Option { + let mut left = self.field_text.get(); + let text = self.budgeted_text(d, key, &mut left); + self.field_text.set(left); + text + } + + /// Converts `bytes` against the field and layer text allowance, as [`spend`] does. + pub(super) fn field_spend(&self, bytes: &[u8], convert: impl FnOnce(&[u8]) -> String) -> Option { + let mut left = self.field_text.get(); + let text = spend(bytes, &mut left, convert); + self.field_text.set(left); + text + } + + /// Spends `n` bytes of the field and layer text allowance; `false` when too few are left. + pub(super) fn field_charge(&self, n: usize) -> bool { + let left = self.field_text.get(); + self.field_text.set(left.saturating_sub(n)); + n <= left + } + pub(super) fn budgeted_text(&self, d: &Dictionary, key: &[u8], budget: &mut usize) -> Option { - let o = self.resolve(d.get(key).ok()?); - let (Object::String(bytes, _) | Object::Name(bytes)) = o else { return None }; - let text = spend(bytes, budget, |_| trimmed_text(o).unwrap_or_default())?; - (!text.is_empty()).then_some(text) + self.budgeted_text_within(d, key, budget).ok().flatten() + } + + /// [`Self::budgeted_text`], but `Err` when the text is there and does not fit `budget` + /// (against `Ok(None)`: no text, or none that trims to anything). + pub(super) fn budgeted_text_within(&self, d: &Dictionary, key: &[u8], budget: &mut usize) -> Result, ()> { + let Ok(o) = d.get(key) else { return Ok(None) }; + let o = self.resolve(o); + let (Object::String(bytes, _) | Object::Name(bytes)) = o else { return Ok(None) }; + let text = spend(bytes, budget, |_| trimmed_text(o).unwrap_or_default()).ok_or(())?; + Ok((!text.is_empty()).then_some(text)) + } + + /// [`Self::field_text`], but `Err` when the text does not fit the allowance. + pub(super) fn field_text_within(&self, d: &Dictionary, key: &[u8]) -> Result, ()> { + let mut left = self.field_text.get(); + let text = self.budgeted_text_within(d, key, &mut left); + self.field_text.set(left); + text } pub(super) fn name(&self, d: &Dictionary, key: &[u8]) -> Option { diff --git a/crates/render/src/inspect/navigation.rs b/crates/render/src/inspect/navigation.rs index c5f2a1be86..f680b46246 100644 --- a/crates/render/src/inspect/navigation.rs +++ b/crates/render/src/inspect/navigation.rs @@ -250,7 +250,7 @@ impl<'a> Inspector<'a> { None => {} } let open = d.get(b"Count").ok().and_then(|c| c.as_i64().ok()).is_some_and(|c| c > 0); - let title = self.text(d, b"Title").unwrap_or_default(); + let title = self.field_text(d, b"Title").unwrap_or_default(); let color = match d.get(b"C").map(|o| self.resolve(o)) { Ok(Object::Array(c)) => outline_color(c.iter().map(|v| self.resolve(v).as_float().ok())), _ => None, diff --git a/crates/render/src/inspect/resources.rs b/crates/render/src/inspect/resources.rs index 3dbc1c665f..6d90f715e1 100644 --- a/crates/render/src/inspect/resources.rs +++ b/crates/render/src/inspect/resources.rs @@ -121,9 +121,14 @@ impl<'a> Inspector<'a> { .unwrap_or_default(); let auto = config.map(|c| self.view_auto_state(c)).unwrap_or_default(); let Ok(Object::Array(ocgs)) = props.get(b"OCGs").map(|o| self.resolve(o)) else { return }; + let mut seen = HashSet::new(); for g in ocgs { let Some(d) = self.dict(g) else { continue }; let id = g.as_reference().ok(); + // A group listed twice is one layer (and its name is read once). + if id.is_some_and(|id| !seen.insert(id)) { + continue; + } // The View auto state, when it manages the group, then the configuration. let visible = match id { Some(id) if let Some(&state) = auto.get(&id) => state, @@ -132,7 +137,7 @@ impl<'a> Inspector<'a> { _ => !base_off, }; let Some(id) = id else { continue }; - out.push(Layer { id, name: self.text(d, b"Name").unwrap_or_else(|| "Layer".into()), visible, initial: visible }); + out.push(Layer { id, name: self.field_text(d, b"Name").unwrap_or_else(|| "Layer".into()), visible, initial: visible }); } } diff --git a/crates/render/src/inspect/tests.rs b/crates/render/src/inspect/tests.rs index 68486d8097..c908b241cf 100644 --- a/crates/render/src/inspect/tests.rs +++ b/crates/render/src/inspect/tests.rs @@ -1161,6 +1161,50 @@ fn repeated_annotations_are_listed_once_within_a_byte_budget() { assert!(notes.iter().map(retained).sum::() <= MAX_ANNOTATION_BYTES); } +/// Blank text is not kept, so only reading it is charged: many comments sharing one large blank +/// string stop being built at the budget instead of decoding it once each. +#[test] +fn comments_sharing_a_blank_string_stop_at_the_read_budget() { + let small = "<< /Subtype /Text /Rect [0 0 1 1] /Contents 7 0 R >> ".repeat(1200); + let pdf = format!( + "%PDF-1.7\n1 0 obj << /Type /Catalog /Pages 2 0 R >> endobj\n\ + 2 0 obj << /Type /Pages /Kids [3 0 R] /Count 1 >> endobj\n\ + 3 0 obj << /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] /Annots [{small}] >> endobj\n\ + 7 0 obj ({}) endobj\ntrailer << /Root 1 0 R >>\n%%EOF", + " ".repeat(64 << 10), + ); + let doc = Document::load_mem(pdf.as_bytes()).expect("loads"); + let page = PageInfo { width: 200.0, height: 200.0, crop: [0.0, 0.0, 200.0, 200.0], rotation: 0, label: String::new() }; + let mut info = DocInfo { pages: vec![page], ..Default::default() }; + Inspector::new(&doc).fill(&mut info); + assert!(!info.annotations.is_empty() && info.annotations.len() < 1200, "{}", info.annotations.len()); +} + +/// Many fields sharing one large tooltip, and one layer listed many times, are read within a byte +/// allowance rather than once each. +#[test] +fn fields_sharing_a_tooltip_and_repeated_layers_stop_at_the_text_budget() { + let fields = (0..1300).map(|i| format!("<< /FT /Tx /T (f{i}) /TU 7 0 R /V 7 0 R >> ")).collect::(); + let pdf = format!( + "%PDF-1.7\n1 0 obj << /Type /Catalog /Pages 2 0 R /AcroForm << /Fields [{fields}] >> \ + /OCProperties << /OCGs [{}] >> >> endobj\n\ + 2 0 obj << /Type /Pages /Kids [3 0 R] /Count 1 >> endobj\n\ + 3 0 obj << /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] >> endobj\n\ + 6 0 obj << /Type /OCG /Name (Ink) >> endobj\n\ + 7 0 obj ({}) endobj\ntrailer << /Root 1 0 R >>\n%%EOF", + "6 0 R ".repeat(1000), + "x".repeat(32 << 10), + ); + let doc = Document::load_mem(pdf.as_bytes()).expect("loads"); + let page = PageInfo { width: 200.0, height: 200.0, crop: [0.0, 0.0, 200.0, 200.0], rotation: 0, label: String::new() }; + let mut info = DocInfo { pages: vec![page], ..Default::default() }; + Inspector::new(&doc).fill(&mut info); + let kept: usize = info.fields.iter().map(|f| f.tooltip.as_ref().map_or(0, String::len) + f.value.as_ref().map_or(0, String::len)).sum(); + assert!(kept > 0 && kept <= super::core::MAX_FIELD_TEXT_BYTES, "{kept}"); + assert!(info.fields.iter().any(|f| f.tooltip.is_none()), "text past the allowance is left out"); + assert_eq!(info.layers.len(), 1); +} + /// A link reached through an indirect object that holds only a reference to it is the same /// annotation as the link itself, and so is a page array reached the same way: each is listed once. #[test] @@ -2686,6 +2730,42 @@ fn inspection_keeps_no_private_page_data() { assert_eq!(report(with), report(without), "what inspection reports does not change"); } +/// A field with several widgets reports an action on any of them, not only the first, and so does +/// the document's JavaScript flag. +#[test] +fn a_field_reports_an_action_on_a_later_widget() { + let pdf = "%PDF-1.7\n1 0 obj << /Type /Catalog /Pages 2 0 R /AcroForm << /Fields [4 0 R] >> >> endobj\n\ + 2 0 obj << /Type /Pages /Kids [3 0 R] /Count 1 >> endobj\n\ + 3 0 obj << /Type /Page /Parent 2 0 R /MediaBox [0 0 200 200] >> endobj\n\ + 4 0 obj << /FT /Tx /T (f) /Kids [5 0 R 6 0 R] >> endobj\n\ + 5 0 obj << /Subtype /Widget /Rect [0 0 1 1] >> endobj\n\ + 6 0 obj << /Subtype /Widget /Rect [0 0 1 1] /A << /S /JavaScript /JS (app.alert(1)) >> >> endobj\n\ + trailer << /Root 1 0 R >>\n%%EOF"; + let doc = Document::load_mem(pdf.as_bytes()).expect("loads"); + let page = PageInfo { width: 200.0, height: 200.0, crop: [0.0, 0.0, 200.0, 200.0], rotation: 0, label: String::new() }; + let mut info = DocInfo { pages: vec![page], ..Default::default() }; + Inspector::new(&doc).fill(&mut info); + assert!(info.fields.iter().any(|f| f.name == "f" && f.has_actions), "{:?}", info.fields); + assert!(info.has_javascript); +} + +/// A shared descendant first reached at the depth limit (its action-bearing child cut) is read +/// again from a shallower path, so the action is still found. +#[test] +fn an_action_below_a_shared_field_first_met_deep_is_found_from_a_shallower_path() { + // 1 is the root, 2..=32 a chain of 31 fields, 33 the shared field, 34 holds the action, + // 35 a short path to 33. + let mut bodies = vec!["<< /Kids [2 0 R 35 0 R] >>".to_string()]; + for i in 2..=32 { + bodies.push(format!("<< /Kids [{} 0 R] >>", i + 1)); + } + bodies.push("<< /Kids [34 0 R] >>".into()); + bodies.push("<< /A << /S /JavaScript /JS (x) >> >>".into()); + bodies.push("<< /Kids [33 0 R] >>".into()); + let doc = Document::load_mem(&pdf_of(&bodies)).expect("loads"); + assert!(Inspector::new(&doc).has_action(&Object::Reference((1, 0)), 0)); +} + /// A `/Kids` entry that loops back, or names a root field that doesn't name this one as its /// `/Parent`, is not a child (ISO 32000-2 §12.7.3, Table 224; §12.7.4.1, Table 226): the field /// keeps its other kids and is listed with them. From abec4e3feea96d3566c7c026d5d137172fdf2596 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 15:50:29 -0500 Subject: [PATCH 58/60] fix(js): bound array text, month parsing, name registration and list work (#5883) * fix(js): bound array text, month parsing, name registration and list work - strings() (field value and colour setters, resetForm) copied every array entry into its own String with no total: one long string listed a million times was copied a million times. The copies together are now held to the run's MAX_PAYLOAD byte budget. - FormCalc parse_date copied and lowercased the rest of the text for every month token of the picture, quadratic in both; it reads at most the longest month name now. - XFA global registration checked each child name against a Vec of every name defined so far, quadratic in a wide form; it is a HashSet. - FormCalc gather charged one step for a whole collection and list_push charged nothing for a duplicate it dropped, so .all on same-named siblings did quadratic work for one step a pair; every node visited costs a step. - add_instance reserved only the new root's SOM before copying a subtree; the copy's nodes, names and SOMs are charged to the byte budget first. * address review: gather charges each call; check an array entry's length before copying it * Hold an array entry to its widest escaped size before copying it The pre-copy check compared the payload budget against a JS string's UTF-16 length, which is shorter than the escaped UTF-8 form it is copied to: a string of quotes or non-ASCII text could allocate up to six times the budget before the post-check rejected it. The check now uses the widest escaped width, so the copy cannot exceed the budget it is checked against. * review: measure an array entry's escaped width before refusing it * address review: count an array entry's escaped width without copying it * address review: charge retained field text and colours to a run-wide budget * fmt * fmt * merge main, keep both --- crates/js/src/document.rs | 11 +++++--- crates/js/src/formcalc/access.rs | 13 ++++++--- crates/js/src/formcalc/dates.rs | 8 ++++-- crates/js/src/host.rs | 40 ++++++++++++++++++++++++--- crates/js/src/run.rs | 1 + crates/js/src/tests.rs | 46 ++++++++++++++++++++++++++++++++ crates/js/src/xfa/globals.rs | 6 +++-- crates/js/src/xfa/instances.rs | 16 ++++++++++- 8 files changed, 126 insertions(+), 15 deletions(-) diff --git a/crates/js/src/document.rs b/crates/js/src/document.rs index f13c9434e9..cccf89a7a2 100644 --- a/crates/js/src/document.rs +++ b/crates/js/src/document.rs @@ -1,4 +1,6 @@ -use super::host::{FieldKey, arg, colour, error, function, host, read, s, set_value, strings, text, text_line, value_of, write}; +use super::host::{ + FieldKey, arg, charge, colour, error, function, host, kind_of, read, s, strings, text, text_bytes, text_line, value_for, value_of, write, +}; use super::model::{DISPLAY_HIDDEN, DISPLAY_VISIBLE, FieldType, Request}; use super::{Attribute, Context, JsArray, JsObject, JsResult, JsString, JsValue, NativeFunction, ObjectInitializer, js_string}; @@ -41,8 +43,9 @@ fn field_object_new(ctx: &mut Context, name: &str, id: Option, token: u64) let value_get = getter!(ctx, n, |f, c| value_of(&f, c)); let value_set = { fn set(_: &JsValue, args: &[JsValue], name: &FieldKey, c: &mut Context) -> JsResult { - let vals = strings(&arg(args, 0), c)?; - write(c, name, |f| set_value(f, vals))?; + let vals = value_for(kind_of(c, name)?, strings(&arg(args, 0), c)?); + charge(c, name, |f| text_bytes(&f.value), text_bytes(&vals))?; + write(c, name, |f| f.value = vals)?; Ok(JsValue::undefined()) } function(ctx, NativeFunction::from_copy_closure_with_captures(set, n.clone())) @@ -104,6 +107,7 @@ fn field_object_new(ctx: &mut Context, name: &str, id: Option, token: u64) let text_color_set = { fn set(_: &JsValue, args: &[JsValue], name: &FieldKey, c: &mut Context) -> JsResult { let v = strings(&arg(args, 0), c)?; + charge(c, name, |f| f.text_color.as_deref().map_or(0, text_bytes), text_bytes(&v))?; write(c, name, |f| f.text_color = Some(v.into()))?; Ok(JsValue::undefined()) } @@ -113,6 +117,7 @@ fn field_object_new(ctx: &mut Context, name: &str, id: Option, token: u64) let fill_color_set = { fn set(_: &JsValue, args: &[JsValue], name: &FieldKey, c: &mut Context) -> JsResult { let v = strings(&arg(args, 0), c)?; + charge(c, name, |f| f.fill_color.as_deref().map_or(0, text_bytes), text_bytes(&v))?; write(c, name, |f| f.fill_color = Some(v.into()))?; Ok(JsValue::undefined()) } diff --git a/crates/js/src/formcalc/access.rs b/crates/js/src/formcalc/access.rs index 26de7ba9eb..3c323ca088 100644 --- a/crates/js/src/formcalc/access.rs +++ b/crates/js/src/formcalc/access.rs @@ -33,9 +33,12 @@ impl Interp<'_> { Ok(()) } - /// `more` nodes into `out`, each once, never past [`MAX_LIST`]. + /// `more` nodes into `out`, each once, never past [`MAX_LIST`]. Every node visited costs a + /// step, kept or not, and the call itself one more: a search that matches nothing still + /// costs work when it is repeated over a long list. pub(super) fn gather(&mut self, out: &mut Vec, seen: &mut HashSet, more: impl IntoIterator) -> R<()> { for i in more { + self.step()?; if seen.insert(i) { out.push(i); if out.len() > MAX_LIST { @@ -43,7 +46,7 @@ impl Interp<'_> { } } } - self.charge(1) + self.step() } /// The nodes of `v`, each once. @@ -294,8 +297,10 @@ impl Interp<'_> { } } - /// `v` onto `out`: a node only once, never past [`MAX_LIST`]. + /// `v` onto `out`: a node only once, never past [`MAX_LIST`]. A duplicate costs its step + /// too: `all` on a list of same-named siblings makes one for every pair. fn list_push(&mut self, out: &mut Vec, seen: &mut HashSet, v: Val) -> R<()> { + self.step()?; if let Val::Node(i) = v && !seen.insert(i) { @@ -305,7 +310,7 @@ impl Interp<'_> { if out.len() > MAX_LIST { return Err(LIST_TOO_LONG.into()); } - self.charge(1) + Ok(()) } /// Property `name` of form object `i` as FormCalc reads it: its value, attributes, diff --git a/crates/js/src/formcalc/dates.rs b/crates/js/src/formcalc/dates.rs index 4d9befeab2..f201a23525 100644 --- a/crates/js/src/formcalc/dates.rs +++ b/crates/js/src/formcalc/dates.rs @@ -56,6 +56,9 @@ const CENTURY_SPLIT: i64 = 30; const MONTHS: [&str; 12] = ["January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December"]; +/// Characters of the longest month name: all a month token needs to read of the text. +const LONGEST_MONTH: usize = 9; + /// Parse `text` as a date by `picture` (`YYYY-MM-DD`, `MM/DD/YYYY`, `MMM D, YYYY`, …). pub(super) fn parse_date(text: &str, picture: &str) -> Option<(i64, u32, u32)> { let pic: Vec = picture.chars().collect(); @@ -71,8 +74,9 @@ pub(super) fn parse_date(text: &str, picture: &str) -> Option<(i64, u32, u32)> { } i += run; if c == 'M' && run >= 3 { - // Month name. - let rest: String = txt.get(j..).unwrap_or(&[]).iter().collect(); + // Month name: only as much of the text as a name can be, so a picture of many + // month tokens does not copy the rest of a long text for each of them. + let rest: String = txt.get(j..).unwrap_or(&[]).iter().take(LONGEST_MONTH).collect(); let lower = rest.to_ascii_lowercase(); let found = MONTHS .iter() diff --git a/crates/js/src/host.rs b/crates/js/src/host.rs index 5966e63545..d829041ce6 100644 --- a/crates/js/src/host.rs +++ b/crates/js/src/host.rs @@ -27,6 +27,10 @@ pub(crate) struct Host { pub(crate) reset_limit_exceeded: bool, /// Bytes of text kept in alerts, console lines and requests, against [`xfa::MAX_PAYLOAD`]. pub(crate) kept: usize, + /// Bytes of text the script has added to the fields' values and colours, net of what it + /// replaced, against [`xfa::MAX_PAYLOAD`]: each assignment alone is capped, so without this + /// many fields could each hold a maximal array. + pub(crate) retained: usize, /// Alerts, console lines and requests the limits left out. pub(crate) dropped: usize, } @@ -262,12 +266,42 @@ pub(crate) fn value_of(f: &FieldState, ctx: &mut Context) -> JsValue { } } -pub(crate) fn set_value(f: &mut FieldState, vals: Vec) { - f.value = match f.kind { +/// Bytes of text in `v`, counting each string's heap size and its handle. +pub(crate) fn text_bytes(v: &[String]) -> usize { + v.iter().fold(0usize, |n, t| n.saturating_add(t.len()).saturating_add(std::mem::size_of::())) +} + +/// Charge the replacement of the text `old` selects from field `name` with `new_bytes` of text to +/// the run's retained-text budget; an assignment the budget has no room for is an error and +/// charges nothing. +pub(crate) fn charge(ctx: &Context, name: &FieldKey, old: impl FnOnce(&FieldState) -> usize, new_bytes: usize) -> JsResult<()> { + let i = field_index(ctx, name)?; + let mut h = host(ctx)?.borrow_mut(); + let freed = old(&h.fields[i]); + let total = h.retained.saturating_sub(freed).saturating_add(new_bytes); + if total > xfa::MAX_PAYLOAD { + return Err(error(format!("the fields' values and colours hold more than {} bytes of text", xfa::MAX_PAYLOAD))); + } + h.retained = total; + Ok(()) +} + +pub(crate) fn kind_of(ctx: &Context, name: &FieldKey) -> JsResult { + let i = field_index(ctx, name)?; + Ok(host(ctx)?.borrow().fields[i].kind) +} + +/// What a field of `kind` keeps of the values assigned to it. +pub(crate) fn value_for(kind: FieldType, vals: Vec) -> Vec { + match kind { FieldType::CheckBox | FieldType::RadioButton => vals.into_iter().filter(|v| v != "Off" && !v.is_empty()).take(1).collect(), FieldType::ListBox => vals, _ => vals.into_iter().take(1).collect(), - }; + } +} + +pub(crate) fn set_value(f: &mut FieldState, vals: Vec) { + f.value = value_for(f.kind, vals); } #[cfg(test)] diff --git a/crates/js/src/run.rs b/crates/js/src/run.rs index ececa107a3..7019bd768c 100644 --- a/crates/js/src/run.rs +++ b/crates/js/src/run.rs @@ -270,6 +270,7 @@ pub(crate) fn run_in_context( reset_calls: setup.reset_calls, reset_limit_exceeded: false, kept: 0, + retained: 0, dropped: 0, })); let mut out = Outcome { rc: true, value: event.value.clone(), change: event.change.clone(), ..Default::default() }; diff --git a/crates/js/src/tests.rs b/crates/js/src/tests.rs index 1a41afeea8..ece4443b7d 100644 --- a/crates/js/src/tests.rs +++ b/crates/js/src/tests.rs @@ -87,6 +87,17 @@ fn calculate_scripts_read_fields_and_set_the_value() { assert!(o.rc); } +/// An array of field values copies each entry: one long string listed many times is held to +/// the run's byte budget, not copied once per listing. +#[test] +fn an_array_of_values_stops_at_the_byte_budget() { + let fill = "var s = 'x'; while (s.length < 65536) s += s; var a = []; for (var i = 0; i < N; i++) a.push(s); getField('qty').value = a;"; + let o = go(&fill.replace('N', "600"), &Event::field("Calculate", "total", "")); + assert!(o.error.as_deref().is_some_and(|e| e.contains("byte limit exceeded")), "{:?}", o.error); + let o = go(&fill.replace('N', "400"), &Event::field("Calculate", "total", "")); + assert_eq!(o.error, None); +} + /// `event.value` reads and writes as a plain value, numbers included, and the outcome says whether /// the script assigned it, even to the value it already had. #[test] @@ -528,6 +539,7 @@ fn app_alert_titles_are_clipped_and_charged() { reset_calls: 0, reset_limit_exceeded: false, kept: 0, + retained: 0, dropped: 0, }; h.alert(Message { title: Some("Order form".into()), ..Message::plain("Saved") }); @@ -1113,6 +1125,24 @@ mod xfa_model { assert_eq!(o.result.as_deref(), Some("7")); } + /// A copy of a subform costs its nodes, names and SOMs: instances of one with a long name + /// stop at the run's byte budget, well before the instance cap. + #[test] + fn instances_of_a_subform_with_long_names_stop_at_the_byte_budget() { + let mut f = form(); + let table = &mut f.children[0].children[4]; + assert_eq!(table.name, "table"); + for row in &mut table.children { + row.children.push(field(&"n".repeat(1 << 20), "", "", false)); + } + let ev = XfaEvent { activity: "click".into(), target: "form1[0].page1[0].go[0]".into(), ..Default::default() }; + let o = run_xfa("for (var i = 0; i < 100; i++) table._row.addInstance(1); table._row.count", &ev, &XfaDoc::default(), &f, Limits::default()); + assert_eq!(o.error, None, "{:?}", o.error); + let count: usize = o.result.as_deref().unwrap_or_default().parse().expect("a count"); + assert!((3..50).contains(&count), "{count} instances"); + assert_eq!(o.effects.iter().filter(|e| matches!(e, XfaEffect::AddInstance { .. })).count(), count - 2); + } + /// On screen every page is a sheet of its own: `xfa.layout.sheet` counts it from 0, as /// `absPage` does (forms show `sheet(this) + 1`), and `sheetCount` is the page count. #[test] @@ -1625,6 +1655,10 @@ mod formcalc { #[test] fn date2num_rejects_days_past_the_end_of_the_month() { + // A month name is read from the text's next few characters only, whatever follows. + let serial = calc(r#"Date2Num("2024-09-01")"#); + assert_eq!(calc(r#"Date2Num("September 1, 2024, and a note that runs on", "MMMM D, YYYY")"#), serial); + assert_eq!(calc(r#"Date2Num("Sep 1, 2024", "MMM D, YYYY")"#), serial); assert_eq!(calc(r#"Date2Num("2023-02-30")"#), "0"); assert_eq!(calc(r#"Date2Num("31/04/2024", "DD/MM/YYYY")"#), "0"); assert_eq!(calc(r#"Date2Num("2023-02-29")"#), "0"); @@ -2641,6 +2675,18 @@ fn a_plain_form_sees_the_xfa_version_but_not_installed() { assert_eq!(o.alerts, [Message::plain("old viewer")]); } +/// Arrays assigned to several fields are held to one run-wide budget, and replacing a field's +/// array frees its charge. +#[test] +fn retained_array_text_is_charged_across_fields() { + let fill = "var s = 'x'; while (s.length < 65536) s += s; var a = []; for (var i = 0; i < 320; i++) a.push(s);"; + let one = |tail: &str| go(&format!("{fill} {tail}"), &Event::field("Calculate", "total", "")); + let o = one("getField('qty').textColor = a; getField('qty').textColor = a; getField('zip').textColor = ['RGB'];"); + assert_eq!(o.error, None, "replacing an array frees its charge: {:?}", o.error); + let o = one("getField('qty').textColor = a; getField('zip').fillColor = a;"); + assert!(o.error.as_deref().is_some_and(|e| e.contains("hold more than")), "{:?}", o.error); +} + #[test] fn string_arrays_refuse_aggregate_text_before_request_clipping() { let o = go("resetForm(new Array(65).fill('x'.repeat(1 << 19)));", &Event::doc("Console")); diff --git a/crates/js/src/xfa/globals.rs b/crates/js/src/xfa/globals.rs index de57d7fd49..46dc8a713b 100644 --- a/crates/js/src/xfa/globals.rs +++ b/crates/js/src/xfa/globals.rs @@ -1,3 +1,5 @@ +use std::collections::HashSet; + use super::host::host; use super::instances::instance_manager; use super::model::{MAX_DEPTH, XfaDoc, XfaEffect, XfaEvent, clip, payload, reset_names}; @@ -379,7 +381,7 @@ pub(super) fn install(ctx: &mut Context, event: &XfaEvent, doc: &XfaDoc, current "DataView", "Intl", ]; - let mut defined: Vec = RESERVED.iter().map(ToString::to_string).collect(); + let mut defined: HashSet = RESERVED.iter().map(ToString::to_string).collect(); for scope in scopes { let kids: Vec<(usize, String, bool)> = { let h = host(ctx)?.borrow(); @@ -398,7 +400,7 @@ pub(super) fn install(ctx: &mut Context, event: &XfaEvent, doc: &XfaDoc, current let im = instance_manager(ctx, scope, &name)?; ctx.register_global_property(JsString::from(format!("_{name}").as_str()), im, rw)?; } - defined.push(name); + defined.insert(name); } } Ok(()) diff --git a/crates/js/src/xfa/instances.rs b/crates/js/src/xfa/instances.rs index 2985794dc3..c737ea9db9 100644 --- a/crates/js/src/xfa/instances.rs +++ b/crates/js/src/xfa/instances.rs @@ -137,6 +137,17 @@ fn child_som(parent_som: String, name: &str, index: usize) -> String { } } +/// Bytes a copy of `src`'s subtree keeps: a node, its name and its SOM (the copy's SOMs differ +/// in their indices only) for every node [`clone_subtree`] copies. +fn subtree_bytes(h: &XHost, src: usize, depth: usize) -> usize { + let Some(n) = h.nodes.get(src).filter(|_| depth <= MAX_DEPTH) else { return 0 }; + let own = std::mem::size_of::().saturating_add(n.name.len()).saturating_add(n.som.len()); + n.children + .iter() + .filter(|&&k| !h.nodes.get(k).is_some_and(|c| c.gone || c.repeatable && c.index > 0)) + .fold(own, |sum, &k| sum.saturating_add(subtree_bytes(h, k, depth + 1))) +} + /// A copy of `src`'s subtree (values cleared) under `parent`, returning the copy's index. fn clone_subtree(h: &mut XHost, src: usize, parent: Option, depth: usize) -> Option { if depth > MAX_DEPTH || h.nodes.len() >= MAX_NODES { @@ -214,7 +225,10 @@ impl XHost { let name = h.nodes.get(first).map(|n| n.name.to_string()).unwrap_or_default(); let parent_som = h.nodes.get(parent).map(|p| p.som.to_string()).unwrap_or_default(); let som = child_som(parent_som, &name, live.len()); - if !h.reserve(&XfaEffect::AddInstance { som: String::new() }, som.len()) { + // The copy's nodes, their names and SOMs, are charged to the run's byte budget before + // they are made: a subtree with long names must not be copied past it. + let copied = subtree_bytes(h, first, 0); + if !h.reserve(&XfaEffect::AddInstance { som: String::new() }, som.len().saturating_add(copied)) { return None; } let new = clone_subtree(h, first, Some(parent), 0)?; From 69936f1d1f2f589a907941fbc658ed3a75eea56b Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 15:50:56 -0500 Subject: [PATCH 59/60] fix(engine): run the renderer redaction check on redactions applied inside a batch (#5977) * fix(engine): run the renderer redaction check on redactions applied inside a batch * Resolve redaction marks by page object and measure the edited document The renderer check looked marks up by the page index they had when they were drawn, so a MovePages later in the same batch pointed them at another page's text, and it measured glyphs with page geometry from before the edit. Each mark now carries the page object it was placed on, and the geometry is read back from the document as the batch wrote it. * review: skip marks whose page the batch deleted * review: group marks by page once and refuse an ambiguous page tree * address review: refuse a batch that deletes a marked page while adding pages * merge main, keep both * address review: a batch that applies redactions and then copies a page is refused --- crates/engine/src/document.rs | 47 ++++++++++++++------ crates/engine/src/edit/run.rs | 27 +++++++++++- crates/engine/src/session/editing.rs | 20 ++++++--- crates/engine/src/tests.rs | 65 +++++++++++++++++++++++++++- crates/redact/src/lib.rs | 6 +++ 5 files changed, 142 insertions(+), 23 deletions(-) diff --git a/crates/engine/src/document.rs b/crates/engine/src/document.rs index 602410922f..805e089dd7 100644 --- a/crates/engine/src/document.rs +++ b/crates/engine/src/document.rs @@ -958,41 +958,60 @@ pub(crate) fn recovered_render_password(cos: &pdfcraft_cos::Document) -> Option< /// the document as it was. pub(crate) fn shown_under_redactions( after: &pdfcraft_cos::Document, - marks: &[pdfcraft_redact::Mark], + marks: &[(pdfcraft_cos::ObjRef, pdfcraft_redact::Mark)], overlays: &[pdfcraft_cos::ObjRef], - pages: &[pdfcraft_render::PageInfo], password: Option<&str>, ) -> Result<(), EditError> { use pdfcraft_cos::Object; use pdfcraft_redact::RedactError; - let mut marked: Vec = marks.iter().map(|m| m.page).collect(); + let mut shown = after.clone(); + let model = pdfcraft_model::pages(&shown); + // A batch can move pages after the marks were collected: a mark is found by the page object it + // was placed on, not by the index that page had when it was drawn. A page the batch deleted + // takes its marks with it, so they are not checked against whatever page now holds the index. + // One index, built once: each mark is resolved against it instead of scanning every page, + // and the marks are grouped by resolved page in the same pass. + let mut index: std::collections::HashMap = std::collections::HashMap::new(); + for (i, p) in model.iter().enumerate() { + index.entry(p.obj).or_insert(i); + } + let mut marked: Vec = Vec::new(); + let mut rects_of: std::collections::HashMap> = std::collections::HashMap::new(); + for (obj, m) in marks { + let Some(&pi) = index.get(obj) else { continue }; + if !rects_of.contains_key(&pi) { + marked.push(pi); + } + rects_of.entry(pi).or_default().extend(m.rects.iter().map(|r| [r[0].min(r[2]), r[1].min(r[3]), r[0].max(r[2]), r[1].max(r[3])])); + } marked.sort_unstable(); - marked.dedup(); if marked.is_empty() { return Ok(()); } - let mut shown = after.clone(); - let model = pdfcraft_model::pages(&shown); for &pi in &marked { let Some(page) = model.get(pi) else { continue }; let Some(Object::Array(list)) = page.dict.get(b"Contents").map(|c| (*shown.resolve(c)).clone()) else { continue }; let kept: Vec = list.into_iter().filter(|o| !o.as_ref().is_some_and(|r| overlays.contains(&r))).collect(); shown.update_dict(page.obj, |d| d.set(b"Contents".to_vec(), Object::Array(kept))).map_err(|e| EditError::Write(e.to_string()))?; } - let bytes = write_incremental(&shown, &SaveOptions::default()).map_err(|e| EditError::Write(e.to_string()))?; + let bytes = Arc::new(write_incremental(&shown, &SaveOptions::default()).map_err(|e| EditError::Write(e.to_string()))?); + // Page geometry comes from the document as it is now written: a page moved earlier in the batch + // brings its own size and rotation with it. + let pages = pdfcraft_render::inspect(bytes.clone(), password).map_err(|e| EditError::Write(e.to_string()))?.pages; + // The model counts one entry per `/Kids` entry; the renderer one per page it draws. If they + // disagree, a mark's index could land on another page's text, and the check cannot confirm + // the redaction. + if pages.len() != model.len() { + return Err(RedactError::AmbiguousPages.into()); + } let config = RenderConfig { password: password.map(Arc::from), ..Default::default() }; - let mut renderer = pdfcraft_render::PageRenderer::new(Arc::new(bytes), config); + let mut renderer = pdfcraft_render::PageRenderer::new(bytes, config); let mut residue = 0; for &pi in &marked { let request = pdfcraft_render::RenderRequest { page: pi, kind: pdfcraft_render::RequestKind::Text, scale: 1.0, ..Default::default() }; // A page that cannot be read back cannot be confirmed clean. let (Some(info), Some(text)) = (pages.get(pi), renderer.render(request).text) else { return Err(RedactError::Unreadable(pi + 1).into()) }; - let rects: Vec<[f64; 4]> = marks - .iter() - .filter(|m| m.page == pi) - .flat_map(|m| m.rects.iter()) - .map(|r| [r[0].min(r[2]), r[1].min(r[3]), r[0].max(r[2]), r[1].max(r[3])]) - .collect(); + let rects = rects_of.get(&pi).cloned().unwrap_or_default(); for glyph in text.glyphs.iter().filter(|g| !g.text.trim().is_empty()) { let [x0, y0, x1, y1] = info.view_rect_to_user(glyph.rect).map(f64::from); let area = (x1 - x0) * (y1 - y0); diff --git a/crates/engine/src/edit/run.rs b/crates/engine/src/edit/run.rs index 57e5729fc8..0a7a348fca 100644 --- a/crates/engine/src/edit/run.rs +++ b/crates/engine/src/edit/run.rs @@ -253,6 +253,11 @@ pub(crate) struct EditCtx { /// The overlay streams the edit's applied redactions added, which alone the renderer's check /// leaves out. pub(crate) redaction_overlays: Vec, + /// The marks the edit's applied redactions took out (a batch's included), as they were when + /// each apply ran, for the renderer's check. + /// Each applied mark with the page object it was drawn on, which a later page move changes the + /// index of but not the identity. + pub(crate) redaction_marks: Vec<(pdfcraft_cos::ObjRef, pdfcraft_redact::Mark)>, /// What the edit's applied redactions left as it was that still holds the removed text. pub(crate) redaction_elsewhere: Option, /// What became of the PDF/A and PDF/UA identification when the edit's applied redactions @@ -310,6 +315,7 @@ impl EditCtx { rng: getrandom::fill, renamed_fields: Vec::new(), redaction_overlays: Vec::new(), + redaction_marks: Vec::new(), redaction_elsewhere: None, redaction_identification: pdfcraft_redact::Identification::default(), prepared: None, @@ -529,7 +535,17 @@ pub(crate) fn run_edit(doc: &mut pdfcraft_cos::Document, edit: &Edit, cx: &mut E pdfcraft_organize::insert_blank_page(doc, *at, *width, *height)?; } Edit::SetInfo { key, value } => pdfcraft_organize::set_info(doc, key, value)?, - Edit::DuplicatePages { pages, at } => pdfcraft_organize::duplicate_pages(doc, pages, *at)?, + Edit::DuplicatePages { pages, at } => { + // A copy of a page cannot be tied back to the marks an earlier redaction of this edit + // applied: text put back under a mark would sit on the copy, unchecked. Fail closed. + if !cx.redaction_marks.is_empty() { + return Err(EditError::Invalid( + "a batch that applies redactions can't also copy pages: the copies can't be checked against the marks; the document was not changed" + .into(), + )); + } + pdfcraft_organize::duplicate_pages(doc, pages, *at)? + } Edit::ReplacePages { pages, name, bytes, src_pages } => { let src = crate::sources::open_source_into(name, bytes, doc)?; pdfcraft_organize::replace_pages(doc, pages, &src, src_pages)?; @@ -991,8 +1007,15 @@ pub(crate) fn run_edit(doc: &mut pdfcraft_cos::Document, edit: &Edit, cx: &mut E pdfcraft_edit::add_content(doc, *page, &content)?; } Edit::ApplyRedactions { pages } => { + let model = pdfcraft_model::pages(doc); + cx.redaction_marks.extend( + pdfcraft_redact::marks(doc) + .into_iter() + .filter(|m| pages.as_ref().is_none_or(|p| p.contains(&m.page))) + .filter_map(|m| model.get(m.page).map(|p| (p.obj, m))), + ); let report = pdfcraft_redact::apply(doc, pages.as_deref())?; - cx.redaction_overlays = report.overlays; + cx.redaction_overlays.extend(report.overlays); cx.redaction_elsewhere = Some(report.still_holding); cx.redaction_identification = report.identification; } diff --git a/crates/engine/src/session/editing.rs b/crates/engine/src/session/editing.rs index 4a6f372fb4..a039f56798 100644 --- a/crates/engine/src/session/editing.rs +++ b/crates/engine/src/session/editing.rs @@ -185,6 +185,7 @@ impl Session { } cx.renamed_fields.clear(); cx.redaction_overlays.clear(); + cx.redaction_marks.clear(); // `next` is a copy: if the edit fails or crashes, the document is unchanged. run_checked_edit(&mut next, &edit, &mut cx)?; // A field's enter or exit scripts that changed nothing leave no edit behind (no undo step, @@ -195,11 +196,20 @@ impl Session { return Ok(None); } // Redaction removed what its own interpreter places under the marks. The marks were drawn - // over the page as the renderer shows it, so the renderer has the last word. - if let Edit::ApplyRedactions { pages } = &edit { - let marks: Vec = - pdfcraft_redact::marks(&editor.cos).into_iter().filter(|m| pages.as_ref().is_none_or(|p| p.contains(&m.page))).collect(); - guard(|| shown_under_redactions(&next, &marks, &cx.redaction_overlays, &doc.info.pages, doc.password.as_deref())) + // over the page as the renderer shows it, so the renderer has the last word: on every + // apply the edit ran, a batch's included. + if !cx.redaction_marks.is_empty() { + // A batch that deletes a marked page and adds pages cannot be tied back to the marks: + // a copy of a page may carry the text the redaction removed from the original only. + // Fail closed. + let before: std::collections::HashSet<_> = pdfcraft_model::pages(&editor.cos).iter().map(|p| p.obj).collect(); + let after: std::collections::HashSet<_> = pdfcraft_model::pages(&next).iter().map(|p| p.obj).collect(); + if cx.redaction_marks.iter().any(|(o, _)| !after.contains(o)) && after.iter().any(|o| !before.contains(o)) { + return Err(EditError::Invalid( + "a batch that deletes a page with a redaction mark and adds pages can't be verified; the document was not changed".into(), + )); + } + guard(|| shown_under_redactions(&next, &cx.redaction_marks, &cx.redaction_overlays, doc.password.as_deref())) .unwrap_or_else(|m| Err(EditError::Invalid(format!("{} failed unexpectedly ({m}); the document was not changed", edit.label()))))?; } // Renamed fields are said, not slipped in: a repair-log note (with the caveat that diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 4e24a4f608..91b00df4cc 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -3542,9 +3542,10 @@ fn the_renderer_check_refuses_text_still_shown_under_a_mark() { mark_shown_text(&mut s, id, "SECRET42"); let doc = s.get(id).unwrap(); let mut cos = doc.editor.as_ref().unwrap().cos.clone(); - let marks = pdfcraft_redact::marks(&cos); + let model = pdfcraft_model::pages(&cos); + let marks: Vec<_> = pdfcraft_redact::marks(&cos).into_iter().filter_map(|m| model.get(m.page).map(|p| (p.obj, m))).collect(); let refused = |cos: &pdfcraft_cos::Document| { - let shown = shown_under_redactions(cos, &marks, &[], &doc.info.pages, None); + let shown = shown_under_redactions(cos, &marks, &[], None); matches!(shown, Err(EditError::Redact(pdfcraft_redact::RedactError::Residue(n))) if n > 0) }; assert!(refused(&cos)); @@ -13778,6 +13779,66 @@ fn image_transform_engine_refuses_singular_matrices_without_mutation() { } } +/// The renderer's check covers a redaction applied inside a batch too, with the marks as they +/// were when it ran: one the batch itself added is checked, and text a later edit of the batch +/// puts back under a mark refuses the whole batch. +#[test] +fn redactions_applied_in_a_batch_get_the_renderer_check() { + let mut s = Session::new().with_clock(|| 1_700_000_000); + let page = one_line_page_with("/Font << /F1 4 0 R >>", "", "BT /F1 16 Tf 20 150 Td (Public words then SECRET42) Tj ET"); + let id = s.open("batch.pdf", None, Arc::new(page), None).unwrap(); + mark_shown_text(&mut s, id, "SECRET42"); + let apply = Edit::ApplyRedactions { pages: None }; + // Text added where the mark was shows under it again: the renderer sees it and refuses. + let under = AddedText { rect: [150.0, 130.0, 300.0, 160.0], text: "SECRET42 SECRET42".into(), ..Default::default() }; + let refused = s.apply(id, Edit::Batch { label: "Redact then write".into(), edits: vec![apply.clone(), Edit::AddText { page: 0, text: under }] }); + assert!(matches!(refused, Err(EditError::Redact(pdfcraft_redact::RedactError::Residue(n))) if n > 0), "{refused:?}"); + assert!(page_texts(&s, id)[0].contains("SECRET42"), "nothing changed"); + assert_eq!(s.get(id).unwrap().redaction_marks(), 1); + // Applied alone in a batch, the redaction goes through, and the text is gone. + s.apply(id, Edit::Batch { label: "Redact".into(), edits: vec![apply] }).unwrap(); + assert!(!page_texts(&s, id)[0].contains("SECRET42"), "{:?}", page_texts(&s, id)); + assert_eq!(s.get(id).unwrap().redaction_marks(), 0); +} + +/// A batch that deletes a page holding a redaction mark and adds pages is refused whole: the +/// marks cannot be tied to copies, so redaction fails closed. Without added pages it goes through. +#[test] +fn a_batch_that_deletes_a_marked_page_and_adds_pages_is_refused() { + let mut s = Session::new().with_clock(|| 1_700_000_000); + let page = one_line_page_with("/Font << /F1 4 0 R >>", "", "BT /F1 16 Tf 20 150 Td (Public words then SECRET42) Tj ET"); + let id = s.open("batch.pdf", None, Arc::new(page), None).unwrap(); + mark_shown_text(&mut s, id, "SECRET42"); + let edits = vec![ + Edit::ApplyRedactions { pages: None }, + Edit::InsertBlankPage { at: 1, width: 612.0, height: 792.0 }, + Edit::DeletePages { pages: vec![0] }, + ]; + let refused = s.apply(id, Edit::Batch { label: "x".into(), edits }); + assert!(matches!(refused, Err(EditError::Invalid(_))), "{refused:?}"); + assert!(page_texts(&s, id)[0].contains("SECRET42"), "nothing changed"); + assert_eq!(s.get(id).unwrap().redaction_marks(), 1); +} + +/// A batch that applies redactions and then copies a page is refused whole: the copy can't be tied +/// to the marks, so text restored under one would be unchecked. Copying first, or alone, is fine. +#[test] +fn a_batch_that_applies_redactions_and_copies_a_page_is_refused() { + let mut s = Session::new().with_clock(|| 1_700_000_000); + let page = one_line_page_with("/Font << /F1 4 0 R >>", "", "BT /F1 16 Tf 20 150 Td (Public words then SECRET42) Tj ET"); + let id = s.open("batch.pdf", None, Arc::new(page), None).unwrap(); + mark_shown_text(&mut s, id, "SECRET42"); + let apply = Edit::ApplyRedactions { pages: None }; + let copy = Edit::DuplicatePages { pages: vec![0], at: None }; + let refused = s.apply(id, Edit::Batch { label: "x".into(), edits: vec![apply.clone(), copy.clone()] }); + assert!(matches!(&refused, Err(EditError::Invalid(m)) if m.contains("copy pages")), "{refused:?}"); + assert!(page_texts(&s, id)[0].contains("SECRET42"), "nothing changed"); + assert_eq!(s.get(id).unwrap().redaction_marks(), 1); + // Copying before the redaction is applied puts the mark on the copy too, and both are checked. + s.apply(id, Edit::Batch { label: "y".into(), edits: vec![copy, apply] }).unwrap(); + assert!(page_texts(&s, id).iter().all(|t| !t.contains("SECRET42")), "{:?}", page_texts(&s, id)); +} + /// A reset in an opening script reaches a field of a row an earlier opening script added, before /// the form is laid out again with the row's widgets: its data goes back to its default and stays /// so when the form is saved. diff --git a/crates/redact/src/lib.rs b/crates/redact/src/lib.rs index 8f54cefdc6..819cae99cd 100644 --- a/crates/redact/src/lib.rs +++ b/crates/redact/src/lib.rs @@ -58,6 +58,12 @@ pub enum RedactError { NothingToApply, #[error("page {0} has content that can't be read, so it can't be redacted safely")] Unreadable(usize), + /// The page tree and the renderer count a different number of pages (a page repeated in + /// `/Kids`, say), so a mark's page can't be matched to the page the renderer drew. + #[error( + "the document's page tree and its rendered pages disagree on how many pages there are, so the redaction could not be verified; nothing was changed" + )] + AmbiguousPages, /// Redact annotations with no readable area (neither usable `/QuadPoints` nor a usable `/Rect`). /// Skipping them would leave the text under them in the file while the others report success. #[error( From 1d7a9f9ffe45247dbc20a2047a4d2b151cb8385c Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Wed, 11 Nov 2026 15:51:22 -0500 Subject: [PATCH 60/60] fix(edit,a11y): font aliases share the metrics built for their font (#5998) * fix(edit,a11y): font aliases share the metrics built for their font Text extraction and heading extraction built and kept font metrics per resource name, so many names for one font object each held a copy of its Unicode and width tables. * fmt * address review: font metrics are cached by the object at the end of the reference chain --- crates/a11y/src/headings.rs | 29 ++++++++++++++++++++++++++--- crates/edit/src/tests.rs | 33 +++++++++++++++++++++++++++++++++ crates/edit/src/text.rs | 2 ++ crates/edit/src/text/interp.rs | 22 +++++++++++++++++++++- 4 files changed, 82 insertions(+), 4 deletions(-) diff --git a/crates/a11y/src/headings.rs b/crates/a11y/src/headings.rs index 235ba4b452..a9de970b65 100644 --- a/crates/a11y/src/headings.rs +++ b/crates/a11y/src/headings.rs @@ -206,7 +206,9 @@ fn sub(doc: &Document, d: &Dict, key: &[u8]) -> Dict { pub(crate) fn mcid_texts(doc: &Document, page: &Dict) -> HashMap { let res = sub(doc, page, b"Resources"); let (fonts, props) = (sub(doc, &res, b"Font"), sub(doc, &res, b"Properties")); - let mut metrics: HashMap, Metrics> = HashMap::new(); + let mut metrics: HashMap, std::rc::Rc> = HashMap::new(); + // Built once per font object, however many resource names alias it. + let mut by_font: HashMap> = HashMap::new(); // The id each open marked-content sequence is in: its own, else the enclosing one's. let mut stack: Vec> = Vec::new(); let mut out: HashMap = HashMap::new(); @@ -255,9 +257,30 @@ pub(crate) fn mcid_texts(doc: &Document, page: &Dict) -> HashMap { gs.font = op.name(0).map(<[u8]>::to_vec); if let Some(name) = op.name(0) && !metrics.contains_key(name) - && let Some(f) = fonts.get(name).map(|f| doc.resolve(f)).and_then(|f| f.as_dict().cloned()) { - metrics.insert(name.to_vec(), Metrics::from_dict(doc, &f)); + // Keyed by the object at the end of the reference chain, so names that reach one + // font through different chains share its metrics (the hop limit is resolve's). + let font = fonts.get(name).and_then(|o| { + let (mut cur, mut last) = (o.clone(), None); + for _ in 0..16 { + let Object::Ref(r) = cur else { break }; + last = Some(r); + cur = (*doc.get(r)).clone(); + } + last + }); + let built = match font.and_then(|r| by_font.get(&r)) { + Some(m) => Some(m.clone()), + None => { + fonts.get(name).map(|f| doc.resolve(f)).and_then(|f| f.as_dict().map(|d| std::rc::Rc::new(Metrics::from_dict(doc, d)))) + } + }; + if let Some(m) = built { + if let Some(r) = font { + by_font.insert(r, m.clone()); + } + metrics.insert(name.to_vec(), m); + } } } b"Tc" => gs.char_spacing = op.num(0).unwrap_or(0.0), diff --git a/crates/edit/src/tests.rs b/crates/edit/src/tests.rs index 1f2b925b46..b956cb7079 100644 --- a/crates/edit/src/tests.rs +++ b/crates/edit/src/tests.rs @@ -14042,6 +14042,39 @@ fn review_inside_scope_group_move_matches_direct_transform_under_outer_scale() { } } +/// A page that names one font under 300 resource names builds its metrics once: each alias +/// used to build (and keep) a set of its own. +#[test] +fn font_aliases_share_one_set_of_metrics() { + let fonts: String = (0..300).map(|i| format!(" /A{i} 5 0 R")).collect(); + let content: String = format!("BT 72 700 Td {} ET", (0..300).map(|i| format!("/A{i} 12 Tf (x) Tj ")).collect::()); + let objs: Vec = vec![ + "<< /Type /Catalog /Pages 2 0 R >>".into(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>".into(), + format!("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 600 800] /Contents 4 0 R /Resources << /Font <<{fonts} >> >> >>"), + format!("<< /Length {} >>\nstream\n{content}\nendstream", content.len()), + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>".into(), + ]; + let mut out = b"%PDF-1.7\n".to_vec(); + let mut offs = Vec::new(); + for (i, o) in objs.iter().enumerate() { + offs.push(out.len()); + out.extend_from_slice(format!("{} 0 obj\n{o}\nendobj\n", i + 1).as_bytes()); + } + let x = out.len(); + out.extend_from_slice(format!("xref\n0 {}\n0000000000 65535 f \n", objs.len() + 1).as_bytes()); + for o in offs { + out.extend_from_slice(format!("{o:010} 00000 n \n").as_bytes()); + } + out.extend_from_slice(format!("trailer\n<< /Size {} /Root 1 0 R >>\nstartxref\n{x}\n%%EOF\n", objs.len() + 1).as_bytes()); + let doc = Document::open(Arc::new(out)).unwrap(); + text::METRICS_BUILT.with(|n| n.set(0)); + let lines = text::text_lines(&doc, 0).unwrap(); + assert!(lines.iter().map(|l| l.text.matches('x').count()).sum::() >= 300, "{lines:?}"); + let built = text::METRICS_BUILT.with(std::cell::Cell::get); + assert!(built < 10, "metrics built {built} times for one font"); +} + /// Each `Tj`/`TJ` moves the text matrix by the width of its text. A run shown later in the same /// text object without a `Td` or `Tm` of its own (a raised run in another colour, a column after /// a `TJ` gap) is placed by those advances. Rewriting the paragraph before it must keep them, as diff --git a/crates/edit/src/text.rs b/crates/edit/src/text.rs index b58c41303f..eba8834c2b 100644 --- a/crates/edit/src/text.rs +++ b/crates/edit/src/text.rs @@ -54,6 +54,8 @@ pub(crate) use fallback::{SOURCE_SPACE_CALLS, take_source_character_measurements #[cfg(test)] use fallback::{add_type3_font, is_type3_fallback}; pub(crate) use fallback::{glyph_path, type3_font_dict, unique_font_name}; +#[cfg(test)] +pub(crate) use interp::METRICS_BUILT; use interp::{Origin, Paint, union}; #[cfg(test)] pub(crate) use lines::dropped_advances_within; diff --git a/crates/edit/src/text/interp.rs b/crates/edit/src/text/interp.rs index 0bc9373899..e628e017c4 100644 --- a/crates/edit/src/text/interp.rs +++ b/crates/edit/src/text/interp.rs @@ -664,6 +664,13 @@ impl Carry { /// The advance is an error message when the font's writing mode can't be read. type CachedFont = (Rc, Option, Result, String>); +#[cfg(test)] +thread_local! { + /// Font metrics built by [`interpret`] on this thread, for the test that keeps font aliases + /// sharing one set. + pub(crate) static METRICS_BUILT: std::cell::Cell = const { std::cell::Cell::new(0) }; +} + /// A `Do` met while interpreting: its operator index, XObject name and the state there. type Invocation = (usize, Vec, Ts); @@ -976,6 +983,7 @@ pub(super) fn interpret( let fonts_res = super::resource_dict(doc, resources, b"Font", &mut held_fonts).unwrap_or(&none); let states = super::resource_dict(doc, resources, b"ExtGState", &mut held_states).unwrap_or(&none); let mut out: Vec = Vec::new(); + let mut by_font: HashMap = HashMap::new(); let mut ts = carry.ts.clone(); let mut at_bt = (0usize, ts.clone()); let mut stack: Vec<(Ts, Matrix, Matrix)> = std::mem::take(&mut carry.stack); @@ -1105,11 +1113,23 @@ pub(super) fn interpret( ts.size = op.num(1).unwrap_or(ts.size); if let Some(name) = op.name(0) { let (m, face, vertical) = cache.entry(name.to_vec()).or_insert_with(|| { + // Names that refer to one font object share what was built for it: a + // thousand aliases of one font are one set of metrics, not a thousand. + let font = fonts_res.get(name).and_then(Object::as_ref); + if let Some(built) = font.and_then(|r| by_font.get(&r)) { + return built.clone(); + } + #[cfg(test)] + METRICS_BUILT.with(|n| n.set(n.get() + 1)); let dict = fonts_res.get(name).and_then(|f| doc.resolve(f).as_dict().cloned()); let metrics = dict.as_ref().map_or_else(Metrics::fallback, |d| Metrics::from_dict(doc, d)); let vertical = dict.as_ref().map_or(Ok(None), |d| vertical_advance(doc, d).map_err(|e| e.to_string())); let face = dict.filter(is_type3_fallback).map(|_| metrics.base_font.clone()); - (Rc::new(metrics), face, vertical) + let built: CachedFont = (Rc::new(metrics), face, vertical); + if let Some(r) = font { + by_font.insert(r, built.clone()); + } + built }); ts.font = Some((name.to_vec(), m.clone())); ts.face = face.clone();