diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9403a1e044..4b44d79b68 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -19,7 +19,14 @@ concurrency: cancel-in-progress: false jobs: + # Regenerates contributors/contributors.json for the release commit; every build bundles it. + credits: + permissions: + contents: read + pull-requests: read + uses: storytold/craft-common-actions/.github/workflows/refresh-contributors.yml@main release: + needs: credits # Intentionally tracked at @main (not SHA-pinned) so org-wide CI improvements, security fixes, and build optimizations propagate without PR churn across all craftapp repos. uses: storytold/craft-common-actions/.github/workflows/release.yml@main with: @@ -27,6 +34,7 @@ jobs: app-title: PdfCraft craft-fonts-ref: b076417b5880bd4154bd22f00695b1d28c6bea83 version: ${{ inputs.version }} + contributors-artifact: ${{ needs.credits.outputs.artifact-name }} secrets: APPLE_CERTIFICATE: ${{ secrets.APPLE_CERTIFICATE }} APPLE_CERTIFICATE_PASSWORD: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }} diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index 38c28a80ec..a8de85e07a 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -45,7 +45,7 @@ Every asset PdfCraft includes, bundles or uses to build its published material, | `assets/icons/chevron-up.svg` | Lucide icon "chevron-up" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/chevron-up.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/chevrons-left.svg` | Lucide icon "chevrons-left" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/chevrons-left.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/chevrons-right.svg` | Lucide icon "chevrons-right" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/chevrons-right.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | -| `assets/icons/circle-check.svg` | Lucide icon "circle-check" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/circle-check.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | +| `assets/icons/circle-check.svg` | Lucide icon "circle-check" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/circle-check.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs; drawn as PDF path data by crates/sign/src/status_layers.rs into displayed signature status layers | | `assets/icons/circle-dashed.svg` | Lucide icon "circle-dashed" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/circle-dashed.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/circle-dot.svg` | Lucide icon "circle-dot" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/circle-dot.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/circle-help.svg` | Lucide icon "circle-help" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/circle-help.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | @@ -174,7 +174,7 @@ Every asset PdfCraft includes, bundles or uses to build its published material, | `assets/icons/text-select.svg` | Lucide icon "text-select" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/text-select.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/trash-2.svg` | Lucide icon "trash-2" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/trash-2.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/trash.svg` | Lucide icon "trash" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/trash.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | -| `assets/icons/triangle-alert.svg` | Lucide icon "triangle-alert" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/triangle-alert.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | +| `assets/icons/triangle-alert.svg` | Lucide icon "triangle-alert" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/triangle-alert.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs; drawn as PDF path data by crates/sign/src/status_layers.rs into displayed signature status layers | | `assets/icons/type.svg` | Lucide icon "type" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/type.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/underline.svg` | Lucide icon "underline" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/underline.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/undo-2.svg` | Lucide icon "undo-2" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/undo-2.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | @@ -184,7 +184,7 @@ Every asset PdfCraft includes, bundles or uses to build its published material, | `assets/icons/x.svg` | Lucide icon "x" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/x.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/zoom-in.svg` | Lucide icon "zoom-in" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/zoom-in.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/zoom-out.svg` | Lucide icon "zoom-out" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/zoom-out.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | -| `assets/fonts/Inter-Regular.ttf` | Inter Regular | The Inter Project Authors (Rasmus Andersson) | OFL-1.1 | https://github.com/rsms/inter | UI font, embedded in the app; demo PDF labels; slanted fallback glyphs in Fill & Sign typed signatures for characters Dancing Script lacks (Cyrillic, Greek), drawn as filled paths | +| `assets/fonts/Inter-Regular.ttf` | Inter Regular | The Inter Project Authors (Rasmus Andersson) | OFL-1.1 | https://github.com/rsms/inter | UI font, embedded in the app; demo PDF labels; slanted fallback glyphs in Fill & Sign typed signatures for characters Dancing Script lacks (Cyrillic, Greek), drawn as filled paths; embedded into displayed signature status captions beyond WinAnsi by crates/sign/src/status_layers.rs | | `assets/fonts/Inter-Medium.ttf` | Inter Medium | The Inter Project Authors (Rasmus Andersson) | OFL-1.1 | https://github.com/rsms/inter | UI font, embedded in the app; demo PDF labels | | `assets/fonts/Inter-SemiBold.ttf` | Inter SemiBold | The Inter Project Authors (Rasmus Andersson) | OFL-1.1 | https://github.com/rsms/inter | UI font, embedded in the app; demo PDF labels | | `assets/fonts/JetBrainsMono-Regular.ttf` | JetBrains Mono Regular | The JetBrains Mono Project Authors | OFL-1.1 | https://github.com/JetBrains/JetBrainsMono | UI monospace font; demo PDF code | @@ -249,7 +249,7 @@ Every asset PdfCraft includes, bundles or uses to build its published material, | `assets/icons/thumbs-up.svg` | Lucide icon "thumbs-up" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/thumbs-up.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/thumbs-down.svg` | Lucide icon "thumbs-down" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/thumbs-down.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/message-square-plus.svg` | Lucide icon "message-square-plus" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/message-square-plus.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | -| `assets/icons/circle-x.svg` | Lucide icon "circle-x" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/circle-x.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | +| `assets/icons/circle-x.svg` | Lucide icon "circle-x" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/circle-x.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs; drawn as PDF path data by crates/sign/src/status_layers.rs into displayed signature status layers | | `crates/ui-egui/tests/data/form.pdf` | One-page AcroForm test fixture (text, check box, radio group, combo box) | PdfCraft contributors | MIT OR Apache-2.0 | Contributor-original: hand-written PDF objects, no third-party content | Form-filling UI tests (crates/ui-egui/tests/it/forms.rs, commands.rs) | | `assets/icons/pentagon.svg` | Lucide icon "pentagon" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/pentagon.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | | `assets/icons/message-square-quote.svg` | Lucide icon "message-square-quote" | Lucide Contributors (portions Cole Bemis, Feather) | ISC AND MIT | https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icons/message-square-quote.svg) | UI icon, embedded via crates/ui-egui/src/icon_data.rs | @@ -314,4 +314,4 @@ Not in this repository and never downloaded by it: compiled in only when the bui | Input | Option | Title | Author | Licence | Source | Attribution | Used for | |---|---|---|---|---|---|---|---| | `craft-i18n` | `CRAFT_I18N_DIR` | Interface translations: the craft-i18n catalogs (locales//common.tsv and pdfcraft.tsv) of the 34 languages PdfCraft offers besides English | The craft-i18n translators, including every translator of PdfCraft's former in-repo catalogs (crates/ui-egui/src/i18n/*.tsv, migrated at pdfcraft@ecdd1e6) | MIT OR Apache-2.0 | https://github.com/storytold/craft-i18n (pinned by commit in craft-i18n.rev, which CI, releases and the flatpak manifests read) | https://github.com/storytold/craft-i18n/blob/main/ATTRIBUTION.md | Interface text in languages other than English, embedded by crates/ui-egui/build.rs and looked up through craft-libs' craft-i18n crate; a build without CRAFT_I18N_DIR is English only. Clean-room translations from PdfCraft's English interface labels | -| `craft-fonts` | `CRAFT_FONTS_DIR` | Japanese fonts: BIZ UDPGothic (Regular, Bold), Shippori Mincho, BIZ UDMincho; native Simplified Chinese font: LXGW WenKai Lite; Arabic font: Noto Sans Arabic | The BIZ UDGothic and BIZ UDMincho Project Authors (Morisawa Inc.), The Shippori Mincho Project Authors (FONTDASU), LXGW and the original Klee Project Authors / Fontworks (LXGW WenKai Lite), The Noto Project Authors (Noto Sans Arabic) | OFL-1.1 | https://github.com/storytold/craft-fonts (pinned by commit in .github/workflows/release.yml and .github/workflows/ci.yml); LXGW WenKai Lite from https://github.com/lxgw/LxgwWenKai-Lite/releases/tag/v1.530 (commit b712c83d98acf977bacfb7fb010207802808bda9) | https://github.com/storytold/craft-fonts/blob/main/ATTRIBUTION.md | Japanese fallback in the interface (BIZ UDPGothic first) and Japanese glyphs in edited PDF text (Shippori Mincho); Simplified Chinese interface fallback on native builds only (LXGW WenKai Lite Regular, font SHA-256 f799a538a3355877e597a0ae04050e897932c9ae7689eb9227c2953c5dd549ea, OFL SHA-256 0333a85fe077b6849f8e417acc44e8bd879442ceb681e271acf93e2d57fd4e2e, read from fonts/simplified-chinese-manifest.txt and never embedded on wasm); Arabic interface text, file names and document titles (Noto Sans Arabic); the web build embeds BIZ UDPGothic Regular and the pinned small Arabic, Hebrew, Thai and Devanagari faces; the selected craft-fonts revision contains no Telugu face. Other non-Japanese CJK faces retain the general 4 MiB cap; the reviewed native-only LXGW fallback has a separate 16 MiB cap. Open-source OFL releases are permitted under current AGENTS.md §1.1 | +| `craft-fonts` | `CRAFT_FONTS_DIR` | Japanese fonts: BIZ UDPGothic (Regular, Bold), Shippori Mincho, BIZ UDMincho; native Simplified Chinese font: LXGW WenKai Lite; Arabic font: Noto Sans Arabic | The BIZ UDGothic and BIZ UDMincho Project Authors (Morisawa Inc.), The Shippori Mincho Project Authors (FONTDASU), LXGW and the original Klee Project Authors / Fontworks (LXGW WenKai Lite), The Noto Project Authors (Noto Sans Arabic) | OFL-1.1 | https://github.com/storytold/craft-fonts (pinned by commit in .github/workflows/release.yml and .github/workflows/ci.yml); LXGW WenKai Lite from https://github.com/lxgw/LxgwWenKai-Lite/releases/tag/v1.530 (commit b712c83d98acf977bacfb7fb010207802808bda9) | https://github.com/storytold/craft-fonts/blob/main/ATTRIBUTION.md | Japanese fallback in the interface (BIZ UDPGothic first) and Japanese glyphs in edited PDF text (Shippori Mincho); Simplified Chinese interface fallback on native builds only (LXGW WenKai Lite Regular, font SHA-256 f799a538a3355877e597a0ae04050e897932c9ae7689eb9227c2953c5dd549ea, OFL SHA-256 0333a85fe077b6849f8e417acc44e8bd879442ceb681e271acf93e2d57fd4e2e, read from fonts/simplified-chinese-manifest.txt and never embedded on wasm); Arabic interface text, file names and document titles (Noto Sans Arabic); status captions of named-layer signatures in Arabic, Japanese, Thai and Devanagari, drawn as filled outlines into the display copy by crates/sign/src/status_layers.rs; the web build embeds BIZ UDPGothic Regular and the pinned small Arabic, Hebrew, Thai and Devanagari faces; the selected craft-fonts revision contains no Telugu face. Other non-Japanese CJK faces retain the general 4 MiB cap; the reviewed native-only LXGW fallback has a separate 16 MiB cap. Open-source OFL releases are permitted under current AGENTS.md §1.1 | diff --git a/ATTRIBUTION.toml b/ATTRIBUTION.toml index 99073babb1..608bb601a5 100644 --- a/ATTRIBUTION.toml +++ b/ATTRIBUTION.toml @@ -425,7 +425,7 @@ source = "https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icon licence = "ISC AND MIT" licence_file = "assets/icons/LICENSE-lucide.txt" kind = "icon" -usage = "UI icon, embedded via crates/ui-egui/src/icon_data.rs" +usage = "UI icon, embedded via crates/ui-egui/src/icon_data.rs; drawn as PDF path data by crates/sign/src/status_layers.rs into displayed signature status layers" sha256 = "201e790f3a05c65b2c4b0f952c9b9e1f35cd1c5b21698b3f4be131883fe90bc8" [[asset]] @@ -1844,7 +1844,7 @@ source = "https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icon licence = "ISC AND MIT" licence_file = "assets/icons/LICENSE-lucide.txt" kind = "icon" -usage = "UI icon, embedded via crates/ui-egui/src/icon_data.rs" +usage = "UI icon, embedded via crates/ui-egui/src/icon_data.rs; drawn as PDF path data by crates/sign/src/status_layers.rs into displayed signature status layers" sha256 = "55aa37e0b73e7f8cd906952dbdeb96283c6595ca64aba240a17f3455cb989114" [[asset]] @@ -1954,7 +1954,7 @@ source = "https://github.com/rsms/inter" licence = "OFL-1.1" licence_file = "assets/fonts/OFL-Inter.txt" kind = "font" -usage = "UI font, embedded in the app; demo PDF labels; slanted fallback glyphs in Fill & Sign typed signatures for characters Dancing Script lacks (Cyrillic, Greek), drawn as filled paths" +usage = "UI font, embedded in the app; demo PDF labels; slanted fallback glyphs in Fill & Sign typed signatures for characters Dancing Script lacks (Cyrillic, Greek), drawn as filled paths; embedded into displayed signature status captions beyond WinAnsi by crates/sign/src/status_layers.rs" sha256 = "40d692fce188e4471e2b3cba937be967878f631ad3ebbbdcd587687c7ebe0c82" [[asset]] @@ -3004,7 +3004,7 @@ source = "https://github.com/lucide-icons/lucide (npm lucide-static 1.49.0, icon licence = "ISC AND MIT" licence_file = "assets/icons/LICENSE-lucide.txt" kind = "icon" -usage = "UI icon, embedded via crates/ui-egui/src/icon_data.rs" +usage = "UI icon, embedded via crates/ui-egui/src/icon_data.rs; drawn as PDF path data by crates/sign/src/status_layers.rs into displayed signature status layers" sha256 = "ef1fec17d88794ba6c34c5cf63bdadbd234b3d4dffbcc3d55468b47e3f398472" [[asset]] @@ -3142,7 +3142,7 @@ source = "https://github.com/storytold/craft-fonts (pinned by commit in .github/ licence = "OFL-1.1" attribution = "https://github.com/storytold/craft-fonts/blob/main/ATTRIBUTION.md" kind = "font" -usage = "Japanese fallback in the interface (BIZ UDPGothic first) and Japanese glyphs in edited PDF text (Shippori Mincho); Simplified Chinese interface fallback on native builds only (LXGW WenKai Lite Regular, font SHA-256 f799a538a3355877e597a0ae04050e897932c9ae7689eb9227c2953c5dd549ea, OFL SHA-256 0333a85fe077b6849f8e417acc44e8bd879442ceb681e271acf93e2d57fd4e2e, read from fonts/simplified-chinese-manifest.txt and never embedded on wasm); Arabic interface text, file names and document titles (Noto Sans Arabic); the web build embeds BIZ UDPGothic Regular and the pinned small Arabic, Hebrew, Thai and Devanagari faces; the selected craft-fonts revision contains no Telugu face. Other non-Japanese CJK faces retain the general 4 MiB cap; the reviewed native-only LXGW fallback has a separate 16 MiB cap. Open-source OFL releases are permitted under current AGENTS.md §1.1" +usage = "Japanese fallback in the interface (BIZ UDPGothic first) and Japanese glyphs in edited PDF text (Shippori Mincho); Simplified Chinese interface fallback on native builds only (LXGW WenKai Lite Regular, font SHA-256 f799a538a3355877e597a0ae04050e897932c9ae7689eb9227c2953c5dd549ea, OFL SHA-256 0333a85fe077b6849f8e417acc44e8bd879442ceb681e271acf93e2d57fd4e2e, read from fonts/simplified-chinese-manifest.txt and never embedded on wasm); Arabic interface text, file names and document titles (Noto Sans Arabic); status captions of named-layer signatures in Arabic, Japanese, Thai and Devanagari, drawn as filled outlines into the display copy by crates/sign/src/status_layers.rs; the web build embeds BIZ UDPGothic Regular and the pinned small Arabic, Hebrew, Thai and Devanagari faces; the selected craft-fonts revision contains no Telugu face. Other non-Japanese CJK faces retain the general 4 MiB cap; the reviewed native-only LXGW fallback has a separate 16 MiB cap. Open-source OFL releases are permitted under current AGENTS.md §1.1" [[asset]] path = "crates/sign/tests/data/x509-rsa-sha1.pdf" diff --git a/Cargo.lock b/Cargo.lock index 4e947a191c..08c6ef9ae8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4265,7 +4265,6 @@ version = "0.18.1" dependencies = [ "pdfcraft-content", "pdfcraft-cos", - "pdfcraft-filters", "pdfcraft-fonts", "pdfcraft-model", "thiserror 2.0.21", @@ -4642,6 +4641,7 @@ dependencies = [ "pdfcraft-render", "pdfcraft-testkit", "pdfcraft-xfa", + "regex", "sha2 0.11.1", "thiserror 2.0.21", ] @@ -4688,9 +4688,11 @@ dependencies = [ "p521", "pbkdf2", "pdfcraft-annot", + "pdfcraft-content", "pdfcraft-cos", "pdfcraft-fonts", "pdfcraft-model", + "pdfcraft-testkit", "rc2", "ripemd", "rsa", @@ -5521,6 +5523,29 @@ dependencies = [ "bitflags 2.13.2", ] +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + [[package]] name = "regex-syntax" version = "0.8.11" diff --git a/Cargo.toml b/Cargo.toml index bf6a4e3146..407e40040f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -70,7 +70,11 @@ pdfcraft-prc-render = { path = "crates/prc-render", version = "0.18.1" } hayro = "0.7.1" hayro-syntax = "0.7.2" lopdf = "0.45" -flate2 = { version = "1.1", default-features = false, features = ["rust_backend"] } +# zlib-rs: miniz_oxide took about 1.7 µs for every fixed-Huffman block, empty or not +# (`pdfcraft_filters::FLATE_READ_WORK`). Other dependencies still build flate2's default +# miniz_oxide backend; flate2 prefers zlib-rs when both are built, as the Windows build already did +# through craft-update. `flate::tests::a_copy_from_before_the_start_is_damage` shows which runs. +flate2 = { version = "1.1", default-features = false, features = ["zlib-rs"] } weezl = "0.2" # CCITT Group 4 encoding (pdfcraft-filters); already in the tree through tiff. fax = "0.2.7" @@ -80,6 +84,9 @@ hayro-jbig2 = "0.3" # foundation thiserror = "2" +# Linear-time matching (no backtracking) for caller-supplied redaction patterns. `perf-dfa` gives +# the lazy-DFA cache that `RegexBuilder::dfa_size_limit` bounds (the limit is inert without it). +regex = { version = "1", default-features = false, features = ["std", "unicode", "perf-dfa"] } serde = { version = "1", features = ["derive", "rc"] } serde_json = "1" log = "0.4" diff --git a/ROADMAP.md b/ROADMAP.md index 820ceb0179..3d075c415d 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -3,6 +3,8 @@ > **Last reviewed:** 2026-11-03 | **Last updated:** 2026-11-03 | **Change:** patch (M5 #5850 review: indirect typewriter intent and composer edits preserve outer whitespace; movement regression commits by outside click; estimates unchanged) > **Last reviewed:** 2026-11-03 | **Last updated:** 2026-11-03 | **Change:** patch (M5 #5850: Enter adds Fill & Sign line breaks; outside clicks commit, Escape cancels, and saved multiline appearances retain their text and height; estimates unchanged) +> **Last reviewed:** 2026-10-31 | **Last updated:** 2026-10-31 | **Change:** minor (M3/M4, #5330: Crop pages is directly available in the document quick toolbar; estimates unchanged) + > **Last reviewed:** 2026-11-01 | **Last updated:** 2026-11-01 | **Change:** patch (M7, PR #2155 further review: inherited drawing/text state, affine image gestures, paste/selection ownership, no-op scripts and menu permissions; estimates unchanged) @@ -229,6 +231,7 @@ Ranked; detail and the full milestone table in [docs/roadmap.md](docs/roadmap.md | 8 | 1.0 polish: performance, right-to-left, missing languages, keyboard-only (M14) | 55–95 | ## Progress log +- **2026-10-13 (M8, redact by regex):** `redact_mark` takes `regex` (caller expression, linear-time engine, length/size/nesting caps, empty-matching refused) with optional `validate` (luhn, mod97, iban). Headless only; issue #1886. - **2026-10-28 (M10, #4119 partial):** Scan & OCR ▸ Enhance scanned file cleans scanned pages in place: background removal (level 0–100, as the camera clean-up), edge shadow removal (book gutters, copier borders) and despeckle. Each scan image is decoded, cleaned and written back into its own object at the same size and colour space (JPEG stays JPEG, others lossless, 1-bit stays 1-bit), so the OCR text layer, comments and links stay; one undo step, shared images once, unsupported images (CMYK, indexed, JPEG 2000, JBIG2, CCITT, broken data) reported. `page_enhance_scan` drives it headlessly. Not yet an option of Recognize text. Estimates unchanged. - **2026-11-03 (M11, shared JBIG2 dictionary):** With Monochrome Images set to JBIG2, Optimize has up to 16 JBIG2 images share one symbol dictionary (`/JBIG2Globals`), the letters found on more than one page stored once, when that is smaller (`pdfcraft_filters::encode_jbig2_shared`). Eight pages of 10 pt text rendered at 300 dpi take 41% less in Helvetica and 42% in Times than coded alone, 10% in Courier; groups of 16 save 45%, of 32 only 47%. Only images redaction can store again without the dictionary share one (image masks and DeviceGray named directly, at most 64 Mi pixels), and redacting one stores the rest of its group so, which the group bounds to 15. @@ -256,6 +259,7 @@ Ranked; detail and the full milestone table in [docs/roadmap.md](docs/roadmap.md - 2026-11-01: PR #2155 review follow-up preserves editor crops through group moves and keeps arranged artwork outside added-item streams; unsafe nested-tag extraction is refused. Image handles and flips follow page orientation, added images accept general transforms, and clipboard/text/Home/font-discovery edge cases have focused regressions. Candidate only; estimates unchanged. - 2026-11-01: PR #2155 maintainer edits remove redundant image/text helpers and add regressions for crop ownership, arrange scope/state fidelity, clipboard precedence, crop-drag bounds, and font scanning. Transform and Unicode-script tools complete the headless paths. +- **2026-10-31 (M3/M4, #5330):** The document quick toolbar offers Crop pages directly through the existing page.crop command, with the command’s assembly-permission check and an active-tool highlight. The existing crop, undo and headless page_set_box paths are unchanged. No toolbar customization or new shortcut. Estimates unchanged. - **2026-10-18 (M3, #1354, local candidate):** Integrated --two-paged with current main, retaining upstream modal/picker ownership and single-tab reuse. Native launch/opening/password regressions, negative checks against the old wrapper, visual inspection, formatting, strict scoped Clippy, layers, assets, parity and scoped WASM compilation pass. Three existing socket tests are unavailable in this workspace; one existing network test remains ignored. No push or GitHub merge; estimates unchanged. - **2026-10-31 (M2, PR #4172 review):** Added same-machine pixel oracles for GBK Song, Hei, Kai and FangSong, including all four subset-prefixed Founder aliases. Reference PDFs use explicit Unicode mappings and independently selected installed faces; font data and rasters remain in memory. All eight cases ran; wrong-glyph and wrong-family production mutations each failed the new assertion and were restored. Validation: 422 renderer unit tests, renderer all-target Clippy with -D warnings, formatting, layers/assets/parity and renderer WASM checks passed; existing optional-patch/WASM warnings and parity notes remain. Estimates unchanged. @@ -315,6 +319,7 @@ Ranked; detail and the full milestone table in [docs/roadmap.md](docs/roadmap.md - **2026-10-22 (M14, macOS association recovery candidate):** Successful desktop open/save/sign can queue recovery of an existing per-file binding to the running PDFCraft bundle through NSWorkspace. Metadata-only watcher changes retain undo and avoid false overwrite prompts; real same-length/same-mtime edits remain protected. Rebased on 0.12.0 and reuses #3042's shared atomic metadata preservation; the bounded recovery parser does not narrow save limits. Prior native Preview TEST-signature and Finder workflows passed on macOS 27.0.1; other versions remain unqualified. Estimates unchanged. - **2026-10-13 (M7, free image rotation):** A selected page image has a rotate knob (Shift: 15 degree steps) and `image_edit` a `rotate_by` action; the angle is written as one undoable `cm`. Snapping to page edges while moving is in #1740; resize snapping is not done. Estimates unchanged. +- **2026-10-20 (M9 #79 review, M2, M8):** text extraction reads a marked-content /ActualText that marks no glyph where it paints (paths and images, cut to what is visible, along the direction they are drawn), so words drawn as outlines can be found, copied and exported; OCR and scan field detection still count such a page as having no text of its own. Redaction strips /ActualText, /Alt, /E and /T from inline marked content around paint it clips, as it already did around paint it removed, so a partly redacted outlined word leaves no name in the file. Named-layer recognition also needs the layer form to draw n1 and n3. Estimates unchanged. - **2026-10-20 (M4, initial view):** Window and interface options read indirect boolean ViewerPreferences values. Synthetic regressions cover true/false, direct/indirect preference dictionaries and save/reopen; malformed entries retain false defaults. Estimates unchanged. - **2026-10-20 (M3, #3535):** Escape with a menu open closes the menu and keeps the text selection; the next Escape clears it, as before. One keypress used to do both. A synthetic UI regression checks it. Estimates unchanged. - **2026-10-16 (M3, close prompt keyboard):** The "Save changes?" prompt opens with Save chosen (Enter saves, as before) and the arrow keys and Tab / Shift+Tab move the choice to Cancel or Don't save, shown with a focus ring; Enter or Space press the chosen button and Escape still cancels. Before, Enter always saved whatever was Tab-focused. Estimates unchanged. @@ -405,6 +410,8 @@ Ranked; detail and the full milestone table in [docs/roadmap.md](docs/roadmap.md - **2026-10-16 (M10, #1201 partial):** Lossless image export applies each component's Decode pair. Indexed values round and clip before palette lookup. Color-key masks keep raw samples. The existing 16-bit precision limit remains. JPEG and mask Decode, ICC transforms, and calibrated colors remain outside this fix. Estimates unchanged. - **2026-10-16 (M3, #2739):** Wrapped revision buttons stay above Binding in Document Properties. A synthetic regression checks control bounds and opens a saved revision at two viewport widths. Estimates unchanged. - **2026-10-19 (M7, tracked text):** Editable paragraphs and object labels distinguish repeated glyph tracking from word gaps in TJ arrays and separately positioned glyphs. Synthetic regressions cover character spacing, horizontal scaling, rotated baselines, larger word gaps, and rows of one-glyph words or cells (inferred tracking is capped at half an em, so `a b c` and digit or Y/N columns keep their spaces). Estimates unchanged. +- **2026-10-16 (M9, #79):** a signature appearance that follows the named-layer convention (its /N draws one Form that holds and draws n1 and n3) shows the signature's real state: n1, n3 and n4 are redrawn in the display copy with the panel's icons and a caption in the interface language (Helvetica; else shaped in embedded Inter, or in a craft-fonts face for Arabic, Japanese and Telugu; else English, as Chinese always is). Exports, both print modes and the CLI's render show the redrawn state; saves never do. Every signature keeps its badge, since a document can clip or cover its own layers. Estimates unchanged. +- **2026-10-16 (M9):** the page validation overlay moves to the widget's bottom-right corner as it lies on the page (so it turns with the page and the view) and takes a quarter of the widget's shorter side, at most 48 screen points, so it annotates the signer's artwork from a corner instead of covering the middle of it; a field too small for a legible mark keeps its outline, a rectangle the screen cannot place is not drawn at all, and the whole widget answers the pointer. Estimates unchanged. - **2026-10-16 (M5, #2669, item 5):** The Comments panel groups replies by name and counts all page roots once per frame. Estimates unchanged. - **2026-10-16 (M7, #2140):** Header & footer (and Bates numbering, which uses the same dialog) can shrink the page content so the header and footer don't cover it: the original content is scaled evenly into the band they leave free, centred across, also on rotated pages; Remove and Update take the scaling away, and margins that leave no room are refused. Dialog checkbox with a preview of the band, `shrink` on `doc_header_footer`, all catalogs. Annotations and added text and images are not moved yet. Estimates unchanged. - **2026-10-20 (M14, Spanish catalog gaps):** `es.tsv` gains the 143 `(context, English)` keys the other catalogs carried and it was missing, so Spanish covers the whole bundled key union like Lithuanian and Vietnamese: calendar months and weekdays, the `certificate` context, X.509 key usages, signature-revision and digital-ID lines, password validation, eight plural messages, the combine-from-folder strings, the rendering and stale-file notices and the shortcut-spelled tooltips. Counts in `localization-parity.md` re-measured against the current union (`vi` restated against it for the same reason). Status stays `partial`: no translated help, and OS details stay outside the catalog. Estimates unchanged. @@ -793,6 +800,7 @@ Newest first: the date, what moved, and the new overall estimate. | 2026-11-01 | patch | M7 #5657: header/footer, watermark and background operations commit all selected pages together; later-page errors preserve marks, content, shared/inherited resources and pending edits; estimates unchanged. | | 2026-10-28 | patch | M2, PR #4172: integrate main 5af10276 and retain both roadmap histories; viewer-only CJK and print/output font policies and estimates unchanged. | | 2026-10-27 | minor | M10: OCR stages SHA-256-verified ONNX models after the upstream RTEN URL brownout; discovery prefers a complete ONNX pair per directory, falls back to a complete RTEN pair, and never mixes formats or directories; language/web limits and estimates unchanged. | +| 2026-10-20 | minor | M9 #79 review, M2, M8: /ActualText that marks no glyph extracts where it paints, as far as it is visible; redaction strips /ActualText around paint it clips; named-layer recognition needs the layer form to draw n1 and n3. Estimates unchanged. | | 2026-10-23 | minor | M8, #3205 follow-up: phone-pattern redaction includes the +39 country code on labelled Italian geographic numbers with a slash; dates, protocol identifiers and whole-run guards remain. Estimates unchanged. | | 2026-10-23 | minor | M11: image downsampling measures physical resolution after page-local UserUnit scaling, including shared nested forms; synthetic measurement and optimization regressions. Estimates unchanged. | | 2026-10-23 | minor | M10: Word table cells use the grid columns they span, including cells after a merged cell; synthetic export regression and CLI reproduction. Estimates unchanged. | @@ -807,6 +815,8 @@ Newest first: the date, what moved, and the new overall estimate. | 2026-10-18 | minor | M1, #1143: a live reference may match the xref generation or a same-number header at its indexed offset; neither-match and free slots remain null. Bounded repair diagnostics and synthetic cache, free-slot, edit and classic/packed save regressions; estimates unchanged. | | 2026-10-18 | minor | M5, PR #1971: nested source `/Page` and `/Pages` dictionaries export as null without traversing page content or the page tree. Synthetic saved/reopened three-page regression retains unrelated metadata and aliases. FDF import semantics, copied-graph limits and estimates unchanged. | | 2026-10-15 | minor | M5: budget retained FDF entries before cloning; discovery keeps lazy array owners and annotation positions, with alias/cache-eviction stress fixtures. Limits cover the copied graph; source parsing/page discovery retain existing COS/parser limits, with no total source-memory bound. FDF import semantics and estimates unchanged. | +| 2026-10-16 | minor | M9 #79: named-layer signature appearances show the signature's real state in the display copy, exports and both print modes, never in saves; captions follow the interface language. Estimates unchanged. | +| 2026-10-16 | minor | M9: the page validation overlay moves to the widget's bottom-right corner as it lies on the page and stops growing past 48 screen points, so it annotates the signer's artwork from a corner instead of covering the middle of it; a field under 32 screen points keeps its outline. Estimates unchanged. | | 2026-10-15 | minor | M7: the Edit text caret is as tall as the row it is on in a region with several sizes; estimates unchanged. | | 2026-10-18 | minor | M4, PR #1919 review: repeated page indices apply each rotation to the latest page-local value, preserving indirect/inherited angles. Synthetic adjacent/interleaved duplicates retain siblings and page contents through incremental and full saves. Estimates unchanged. | | 2026-10-18 | minor | M3, #3149: page replacement cancels render demand collected for the previous geometry; all viewer raster upload paths reject empty or oversized textures before cache eviction. Synthetic regressions reproduce the reported 1880×2350 upload at a 2048 limit, retain cached images and unsaved edits, and resume bounded backdrops and tiles. Estimates unchanged. | diff --git a/apps/pdfcraft-cli/src/commands.rs b/apps/pdfcraft-cli/src/commands.rs index 9d0b2057f0..3118c901b4 100644 --- a/apps/pdfcraft-cli/src/commands.rs +++ b/apps/pdfcraft-cli/src/commands.rs @@ -366,8 +366,9 @@ pub(crate) fn render(args: &[String]) -> Result<(), CliError> { Some("pam") => None, _ => return Err(format!("render: --out {out}: use a .png, .jpg, .tif or .pam name").into()), }; - // The app renders a display copy that draws comments missing their /AP streams; the CLI - // must agree with it (and with independent viewers), while keeping the file it reads intact. + // The app renders a display copy that draws comments missing their /AP streams and the state + // of named-layer signatures (here with no trusted certificates); the CLI must agree with it + // (and with independent viewers), while keeping the file it reads intact. let bytes = read(path)?; let password = flag(args, "--password"); // The renderer takes only the user password of R2–R4 files; `Session::open` and the display diff --git a/apps/pdfcraft/src/main.rs b/apps/pdfcraft/src/main.rs index be2290d2f9..4de423db7f 100644 --- a/apps/pdfcraft/src/main.rs +++ b/apps/pdfcraft/src/main.rs @@ -333,7 +333,7 @@ fn native_options(integrated: bool, app_frame: bool, renderer: eframe::Renderer) let mut viewport = egui::ViewportBuilder::default() .with_title("PdfCraft") .with_inner_size([1440.0, 920.0]) - .with_min_inner_size([820.0, 520.0]) + .with_min_inner_size(pdfcraft_ui_egui::MIN_WINDOW_SIZE) .with_drag_and_drop(true) // Wayland app id: matches packaging/linux/ai.storyteller.pdfcraft.desktop. .with_app_id(APP_ID); diff --git a/contributors/people.toml b/contributors/people.toml index 8ab196abc9..a22c884ccb 100644 --- a/contributors/people.toml +++ b/contributors/people.toml @@ -186,3 +186,8 @@ sync_github_name = true [people.helenkwok] real_name = "Helen Kwok" + +[people.vangel76] +real_name = "Valentin Klein" +display_name = "vangel76" +sync_github_name = false diff --git a/crates/a11y/Cargo.toml b/crates/a11y/Cargo.toml index 42a86bbad2..adef6053a1 100644 --- a/crates/a11y/Cargo.toml +++ b/crates/a11y/Cargo.toml @@ -14,6 +14,5 @@ workspace = true pdfcraft-cos = { workspace = true } pdfcraft-model = { workspace = true } pdfcraft-content = { workspace = true } -pdfcraft-filters = { workspace = true } pdfcraft-fonts = { workspace = true } thiserror.workspace = true diff --git a/crates/a11y/src/alt.rs b/crates/a11y/src/alt.rs index fde1048e47..55c0857878 100644 --- a/crates/a11y/src/alt.rs +++ b/crates/a11y/src/alt.rs @@ -25,6 +25,8 @@ pub enum AltError { NotAFigure(u32), #[error("the page has more content than can be edited")] TooLarge, + #[error("the figure's content may be in a damaged or unreadable content stream, or after the damage, so nothing was changed")] + Damaged, #[error("{0}")] Cos(#[from] pdfcraft_cos::CosError), } @@ -52,6 +54,8 @@ pub enum ArtifactError { ParentTree(String), #[error("the page has more content than can be edited")] TooLarge, + #[error("the tag's marked content may be in a damaged or unreadable content stream, or after the damage, so nothing was changed")] + Damaged, #[error(transparent)] Cos(#[from] pdfcraft_cos::CosError), } @@ -372,7 +376,19 @@ fn content_entries(doc: &Document, page: &Dict) -> Vec { } } -type DecodedStream = (Object, Option<(Stream, Vec)>); +/// How much of a content stream decoded. +#[derive(Clone, Copy, Debug, PartialEq)] +enum Read { + Whole, + /// Damaged partway: the data is what decoded before the damage. What follows it is unknown, + /// the syntax the later streams start in too. + Damaged, + /// Not at all (a filter this reader lacks, say): the data is empty. + Unread, +} + +/// A `/Contents` entry and, for a stream, the stream, its decoded data and how much of it decoded. +type DecodedStream = (Object, Option<(Stream, Vec, Read)>); /// Decode the page's content streams within the page decode budget. `None` when a stream is /// rejected because the budget is exhausted. @@ -383,14 +399,14 @@ fn decode_page_streams(doc: &Document, page: &Dict) -> Option for entry in entries { let decoded = match &*doc.resolve(&entry) { Object::Stream(s) => { - let data = match budget.decode(s) { - Ok(data) => data, + let (data, read) = match budget.decode_with_damage(doc, s, usize::MAX) { + Ok((data, damaged)) => (data, if damaged { Read::Damaged } else { Read::Whole }), // Content past the budget may hold more of the tag: editing the rest // would leave that part marked as a tag the tree no longer has. Err(_) if budget.exhausted() => return None, - Err(_) => Vec::new(), + Err(_) => (Vec::new(), Read::Unread), }; - Some((s.clone(), data)) + Some((s.clone(), data, read)) } _ => None, }; @@ -403,6 +419,18 @@ fn marked_id(doc: &Document, props: &Dict, op: &pdfcraft_content::Op, ids: &Hash (op.op == b"BDC").then(|| bdc_mcid(doc, props, op)).flatten().filter(|n| ids.contains(n)) } +/// What [`artifact_marked_content`] did. +#[derive(Debug, PartialEq)] +enum Marked { + /// The streams holding the content were rewritten. + Done, + /// None of the content is on the page, or with `require_all` not all of it: nothing changed. + NotFound, + /// Some of it is, or may be, in a stream damaged partway or one that doesn't decode, neither of + /// which can be rewritten without losing what wasn't read, or after the damage: nothing changed. + Damaged, +} + /// Turn the tag's marked-content blocks into artifacts, rewriting only the streams that /// contain them. Other content streams, and unknown entries on a rewritten stream, stay. /// With `require_all`, every id in `ids` must appear on the page before any stream is rewritten. @@ -412,25 +440,45 @@ fn artifact_marked_content( streams: Vec, ids: &HashSet, require_all: bool, -) -> Result { +) -> Result { let props = page_properties(doc, &page.dict); - let decoded: Vec<&[u8]> = streams.iter().filter_map(|(_, s)| s.as_ref().map(|(_, bytes)| bytes.as_slice())).collect(); + // The pieces are the streams, in order. + let decoded: Vec<&[u8]> = streams.iter().filter_map(|(_, s)| s.as_ref().map(|(_, bytes, _)| bytes.as_slice())).collect(); + let read: Vec = streams.iter().filter_map(|(_, s)| s.as_ref().map(|(_, _, read)| *read)).collect(); + let damage = read.iter().position(|r| *r == Read::Damaged); let pieces = pdfcraft_content::Pieces::join(&decoded); let mut found = HashSet::new(); let mut edits = Vec::new(); for op in pieces.parse().ops { if let Some(id) = marked_id(doc, &props, &op, ids) { + // What follows the damage is unknown: a string or comment the damage left open runs + // on into the next stream, so what reads as an operator after it may not be one. An + // operator whose operands start before a stream not read whole and whose keyword comes + // after it takes them from around what wasn't read: that stream may end them itself. + let (first, last) = pieces.pieces_of(&op); + if damage.is_some_and(|d| last > d) || read.get(first..last).is_some_and(|r| r.iter().any(|r| *r != Read::Whole)) { + return Ok(Marked::Damaged); + } found.insert(id); edits.push((op.span, b"/Artifact BMC".to_vec())); } } - if if require_all { found != *ids } else { found.is_empty() } { - return Ok(false); + // Content not found may be past the damage, or in a stream that doesn't decode: whether it + // is on the page isn't known. + if found != *ids && read.iter().any(|r| *r != Read::Whole) { + return Ok(Marked::Damaged); } - let mut spliced = pieces.splice(edits).into_iter(); + if if require_all { found != *ids } else { found.is_empty() } { + return Ok(Marked::NotFound); + } + let spliced = pieces.splice(edits); + if read.iter().zip(&decoded).zip(&spliced).any(|((read, old), new)| *read != Read::Whole && *old != new.as_slice()) { + return Ok(Marked::Damaged); + } + let mut spliced = spliced.into_iter(); let mut next = Vec::with_capacity(streams.len()); for (entry, stream) in streams { - let Some((stream, old)) = stream else { + let Some((stream, old, _)) = stream else { next.push(entry); continue; }; @@ -456,7 +504,7 @@ fn artifact_marked_content( _ => Object::Array(next), }; doc.update_dict(page.obj, |p| p.set(b"Contents".to_vec(), contents))?; - Ok(true) + Ok(Marked::Done) } /// Decorative figure: its marked content on the page becomes an artifact, and the element @@ -468,11 +516,15 @@ pub fn mark_decorative(doc: &mut Document, obj: ObjRef) -> Result<(), AltError> let pg = d.get(b"Pg").and_then(Object::as_ref).or_else(|| elements(doc).into_iter().find(|e| e.object == obj).and_then(|e| e.page)); if let Some(page) = pg.and_then(|p| pages.iter().find(|x| x.obj == p)).filter(|_| !ids.is_empty()) { let streams = decode_page_streams(doc, &page.dict).ok_or(AltError::TooLarge)?; - // The parent tree no longer points those ids at the figure. - if artifact_marked_content(doc, page, streams, &ids, false)? - && let Some(key) = page.dict.get(b"StructParents").and_then(Object::as_int) - { - clear_parent_tree(doc, key, obj)?; + match artifact_marked_content(doc, page, streams, &ids, false)? { + // The parent tree no longer points those ids at the figure. + Marked::Done => { + if let Some(key) = page.dict.get(b"StructParents").and_then(|o| doc.resolve(o).as_int()) { + clear_parent_tree(doc, key, obj)?; + } + } + Marked::NotFound => {} + Marked::Damaged => return Err(AltError::Damaged), } } // Out of its parent's kids. @@ -513,10 +565,12 @@ pub fn mark_tag_artifact(doc: &mut Document, obj: ObjRef) -> Result<(), Artifact let parent = element.dict.get(b"P").and_then(Object::as_ref).ok_or(ArtifactError::NoParent)?; let replacement = replacement_kids(doc, parent, obj)?; let streams = decode_page_streams(doc, &page.dict).ok_or(ArtifactError::TooLarge)?; - if !artifact_marked_content(doc, page, streams, &ids, true)? { - return Err(ArtifactError::ContentNotFound); + match artifact_marked_content(doc, page, streams, &ids, true)? { + Marked::Done => {} + Marked::NotFound => return Err(ArtifactError::ContentNotFound), + Marked::Damaged => return Err(ArtifactError::Damaged), } - if let Some(key) = page.dict.get(b"StructParents").and_then(Object::as_int) { + if let Some(key) = page.dict.get(b"StructParents").and_then(|o| doc.resolve(o).as_int()) { clear_parent_tree(doc, key, obj).map_err(|e| ArtifactError::ParentTree(e.to_string()))?; } doc.update_dict(parent, |p| p.set(b"K".to_vec(), replacement))?; diff --git a/crates/a11y/src/autotag.rs b/crates/a11y/src/autotag.rs index 9b65f6972b..5c0dc2bd19 100644 --- a/crates/a11y/src/autotag.rs +++ b/crates/a11y/src/autotag.rs @@ -4,7 +4,7 @@ //! which can then be retagged with the existing tag-property tool after human review. use pdfcraft_content::{Op, parse, serialize_ops}; -use pdfcraft_cos::{Dict, Document, ObjRef, Object, Stream}; +use pdfcraft_cos::{DecodeBudget, Dict, Document, ObjRef, Object, Stream}; const MAX_PAGES: usize = 1_000; const MAX_PAGE_BYTES: usize = 16 * 1024 * 1024; @@ -57,18 +57,24 @@ fn content_bytes(doc: &Document, page: &pdfcraft_model::Page, index: usize) -> R let Object::Stream(stream) = &*value else { return Err(unsupported(index, "contents include a non-stream object")) }; let remaining = MAX_PAGE_BYTES.saturating_sub(all.len().saturating_add(1)); if remaining == 0 { - return Err(unsupported(index, "decoded content exceeds 16 MiB")); + return Err(unsupported(index, "decoding the content takes more than 16 MiB")); } - let chain = stream.filters(); - let decoded = if chain.is_empty() { - if stream.raw.len() > remaining { - return Err(unsupported(index, "decoded content exceeds 16 MiB")); + // Strictly, and with `/DecodeParms` given by reference: the content is written back from it. + let mut budget = DecodeBudget::new(remaining, 0); + let decoded = budget.decode_strict(doc, stream).map_err(|e| { + // The allowance counts what every filter stage decodes, as a page's does. + if budget.remaining() == 0 { + unsupported(index, "decoding the content takes more than 16 MiB") + } else { + // The decoder's own reason: `CosError::Filter`'s text would say again that the data + // couldn't be decoded. + let why = match e { + pdfcraft_cos::CosError::Filter(why) => why, + other => other.to_string(), + }; + unsupported(index, format!("content cannot be decoded strictly: {why}")) } - stream.raw.to_vec() - } else { - pdfcraft_filters::decode(&chain, &stream.raw, remaining) - .map_err(|e| unsupported(index, format!("content cannot be decoded strictly: {e}")))? - }; + })?; all.extend_from_slice(&decoded); all.push(b' '); } diff --git a/crates/a11y/src/tests.rs b/crates/a11y/src/tests.rs index 3024319ec3..d8566eddf1 100644 --- a/crates/a11y/src/tests.rs +++ b/crates/a11y/src/tests.rs @@ -79,6 +79,68 @@ fn autotag_text_writes_marked_content_and_parent_tree_across_pages() { assert_eq!(status(&check(&reopened, &Options::default()), Rule::TaggedContent), Status::Passed); } +/// Page content behind a predictor whose `/DecodeParms` are given by reference is read through +/// them: autotagging writes the text back as it was, marked, not with the predictor's tag bytes in +/// it (a `\0` every eight bytes, which split `24` into `2 4`). +#[test] +fn autotag_text_reads_content_with_parameters_given_by_reference() { + use pdfcraft_cos::{Dict, ObjRef}; + let mut content = b"BT /F1 24 Tf 72 700 Td (Hello World) Tj ET".to_vec(); + content.resize(content.len().div_ceil(8) * 8, b' '); + // PNG rows of eight bytes, each tagged 0 (no prediction). + let rows: Vec = content.chunks(8).flat_map(|row| std::iter::once(0).chain(row.iter().copied())).collect(); + let mut doc = pdf( + &[ + "<< /Type /Catalog /Pages 2 0 R >>".into(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>".into(), + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 600 800] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >>".into(), + stream(""), + FONT.into(), + ], + "", + ); + let parms: Dict = [(b"Predictor".to_vec(), Object::Int(12)), (b"Columns".to_vec(), Object::Int(8))].into_iter().collect(); + let parms = doc.add(Object::Dict(parms)); + let mut predicted = pdfcraft_cos::Stream::flate(Dict::new(), &rows); + predicted.dict.set(b"DecodeParms".to_vec(), Object::Ref(parms)); + doc.set(ObjRef::new(4, 0), predicted); + assert_eq!(autotag_text(&mut doc).unwrap(), AutoTagResult { pages: 1, paragraphs: 1 }); + let page = &pdfcraft_model::pages(&doc)[0]; + let written = doc.resolve(page.dict.get(b"Contents").unwrap()); + let Object::Stream(written) = &*written else { panic!("page content") }; + let text = String::from_utf8_lossy(&written.decoded_strict().unwrap()).into_owned(); + assert!(text.contains("/F1 24 Tf") && text.contains("(Hello World) Tj"), "{text}"); +} + +/// Autotagging refuses a page whose content it can't read whole, and says why: content past +/// 16 MiB, or content that doesn't decode strictly, with the decoder's reason right after. +#[test] +fn autotag_text_says_why_it_cannot_read_a_page() { + use pdfcraft_cos::{Dict, ObjRef, Stream}; + let reason = |content: Stream| { + let mut doc = pdf( + &[ + "<< /Type /Catalog /Pages 2 0 R >>".into(), + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>".into(), + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 600 800] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >>".into(), + stream(""), + FONT.into(), + ], + "", + ); + doc.set(ObjRef::new(4, 0), content); + match autotag_text(&mut doc) { + Err(AutoTagError::Unsupported { reason, .. }) => reason, + other => panic!("{other:?}"), + } + }; + assert_eq!(reason(Stream::from_raw(Dict::new(), vec![b' '; 17 << 20])), "decoding the content takes more than 16 MiB"); + let mut corrupt = Stream::from_raw(Dict::new(), b"not zlib data".to_vec()); + corrupt.dict.set(b"Filter".to_vec(), Object::name("FlateDecode")); + let why = reason(corrupt); + assert!(why.starts_with("content cannot be decoded strictly: FlateDecode: "), "{why}"); +} + #[test] fn autotag_text_refuses_paint_and_does_not_modify_the_document() { let mut doc = pdf( @@ -1087,6 +1149,185 @@ fn mark_tag_artifact_validates_parent_and_all_mcids_before_mutating_document() { assert_eq!(*absent.get(ObjRef::new(9, 0)), before_pt); } +/// zlib data that reads as `readable` and is then damaged, with more of the stream after the +/// damage: a stored block, then a block of the reserved type (RFC 1951 §3.2.3), then `rest`. +fn damaged_flate(readable: &[u8], rest: &[u8]) -> pdfcraft_cos::Stream { + let len = u16::try_from(readable.len()).unwrap(); + let mut raw = vec![0x78, 0x01, 0x00]; + raw.extend_from_slice(&len.to_le_bytes()); + raw.extend_from_slice(&(!len).to_le_bytes()); + raw.extend_from_slice(readable); + raw.push(0x07); + raw.extend_from_slice(rest); + let mut dict = pdfcraft_cos::Dict::new(); + dict.set(b"Filter".to_vec(), pdfcraft_cos::Object::name("FlateDecode")); + pdfcraft_cos::Stream::from_raw(dict, raw) +} + +/// A tag's content in a stream damaged partway stays tagged: rewriting the stream from what +/// decoded would lose what follows the damage. Nothing changes, the structure tree and the stream +/// included. Content in a whole stream before a damaged one is marked, and the damaged one kept +/// as stored. +#[test] +fn a_tags_content_in_a_stream_damaged_partway_is_not_rewritten() { + use pdfcraft_cos::{ObjRef, Object}; + let content = b"/Figure << /MCID 0 >> BDC q 40 0 0 20 10 150 cm /Im Do Q EMC /Photo << /MCID 1 >> BDC 100 100 50 30 re f EMC\n"; + let damaged = damaged_flate(content, b"0 0 1 rg 0 0 10 10 re f"); + let mut doc = figures_doc(); + doc.set(ObjRef::new(4, 0), damaged.clone()); + let contents = |doc: &Document| doc.get(ObjRef::new(3, 0)).as_dict().unwrap().get(b"Contents").cloned(); + let state = |doc: &Document| (contents(doc), doc.get(ObjRef::new(9, 0)), doc.get(ObjRef::new(6, 0)), doc.get(ObjRef::new(4, 0))); + let before = state(&doc); + assert_eq!(figures(&doc).len(), 2, "read up to the damage"); + assert_eq!(mark_tag_artifact(&mut doc, ObjRef::new(7, 0)), Err(ArtifactError::Damaged)); + assert_eq!(mark_decorative(&mut doc, ObjRef::new(7, 0)), Err(AltError::Damaged)); + assert_eq!(state(&doc), before); + + let mut doc = figures_doc(); + let other = doc.add(damaged_flate(b"0 g\n", b"0 0 1 rg 0 0 10 10 re f")); + doc.update_dict(ObjRef::new(3, 0), |p| p.set(b"Contents".to_vec(), Object::Array(vec![Object::Ref(ObjRef::new(4, 0)), Object::Ref(other)]))) + .unwrap(); + mark_tag_artifact(&mut doc, ObjRef::new(7, 0)).unwrap(); + let Some(Object::Array(list)) = contents(&doc) else { panic!("contents {:?}", contents(&doc)) }; + assert_ne!(list.first(), Some(&Object::Ref(ObjRef::new(4, 0))), "the tag's stream is rewritten"); + assert_eq!(list.get(1), Some(&Object::Ref(other)), "kept as stored"); +} + +/// What follows the damage is unknown, the syntax the next stream starts in too, so a tag's +/// content in a stream after a damaged one stays tagged. Here a comment the damage leaves open +/// runs on into the next stream, up to its first end of line, and a string starting there reads +/// as operators, among them what looks like the figure's content: rewritten, it would change the +/// string. Nothing changes, the structure tree included. +#[test] +fn a_tags_content_after_a_damaged_stream_is_not_rewritten() { + use pdfcraft_cos::{Dict, ObjRef, Object, Stream}; + let in_a_string = b"(a note\n/Figure << /MCID 0 >> BDC 0 g EMC\n) pop /Photo << /MCID 1 >> BDC 100 100 50 30 re f EMC"; + for (damaged, after) in [(b"0 g\n".as_slice(), None), (b"0 g % a comment".as_slice(), Some(in_a_string.as_slice()))] { + let mut doc = figures_doc(); + if let Some(after) = after { + doc.set(ObjRef::new(4, 0), Stream::from_raw(Dict::new(), after.to_vec())); + } + let first = doc.add(damaged_flate(damaged, b"\n0 0 1 rg 0 0 10 10 re f")); + doc.update_dict(ObjRef::new(3, 0), |p| p.set(b"Contents".to_vec(), Object::Array(vec![Object::Ref(first), Object::Ref(ObjRef::new(4, 0))]))) + .unwrap(); + let state = |doc: &Document| { + (doc.get(ObjRef::new(3, 0)).as_dict().unwrap().get(b"Contents").cloned(), doc.get(ObjRef::new(9, 0)), doc.get(ObjRef::new(6, 0))) + }; + let before = state(&doc); + assert_eq!(mark_tag_artifact(&mut doc, ObjRef::new(7, 0)), Err(ArtifactError::Damaged), "{damaged:?}"); + assert_eq!(mark_decorative(&mut doc, ObjRef::new(7, 0)), Err(AltError::Damaged), "{damaged:?}"); + assert_eq!(state(&doc), before, "{damaged:?}"); + } +} + +/// A page may split its content between an operator's operands and its keyword. When the operands +/// of the figure's `BDC` end the stream before one damaged from its start or one that doesn't +/// decode, the `BDC` the page reads is in what wasn't read: joined without it, those operands and +/// the next tag's read as one `BDC`, which reaches into the stream after it. Neither marking +/// changes anything, the structure tree included. +#[test] +fn a_tags_operands_before_a_stream_not_read_whole_are_not_rewritten() { + use pdfcraft_cos::{Dict, ObjRef, Object, Stream}; + let rest = b" BDC q 40 0 0 20 10 150 cm /Im Do Q EMC"; + for damaged in [true, false] { + let mut doc = figures_doc(); + doc.set(ObjRef::new(4, 0), Stream::from_raw(Dict::new(), b"/Figure << /MCID 0 >>".to_vec())); + let hidden = if damaged { + damaged_flate(b"", rest) + } else { + let filter = doc.add(Object::name("FlateDecode")); + let mut s = Stream::flate(Dict::new(), rest); + s.dict.set(b"Filter".to_vec(), Object::Ref(filter)); + s + }; + let hidden = doc.add(hidden); + let after = doc.add(Object::Stream(Stream::from_raw(Dict::new(), b"/Photo << /MCID 1 >> BDC 100 100 50 30 re f EMC".to_vec()))); + let listed = Object::Array(vec![Object::Ref(ObjRef::new(4, 0)), Object::Ref(hidden), Object::Ref(after)]); + doc.update_dict(ObjRef::new(3, 0), |p| p.set(b"Contents".to_vec(), listed)).unwrap(); + let state = |doc: &Document| { + (doc.get(ObjRef::new(3, 0)).as_dict().unwrap().get(b"Contents").cloned(), doc.get(ObjRef::new(9, 0)), doc.get(ObjRef::new(6, 0))) + }; + let before = state(&doc); + assert_eq!(mark_tag_artifact(&mut doc, ObjRef::new(7, 0)), Err(ArtifactError::Damaged), "damaged: {damaged}"); + assert_eq!(mark_decorative(&mut doc, ObjRef::new(7, 0)), Err(AltError::Damaged), "damaged: {damaged}"); + assert_eq!(state(&doc), before, "damaged: {damaged}"); + } +} + +/// A tag's content not found on the page may be in a stream that doesn't decode: it isn't known +/// to be off the page, so neither marking changes anything, the structure tree included. +#[test] +fn a_tags_content_not_found_beside_a_stream_that_does_not_decode_is_not_taken_for_absent() { + use pdfcraft_cos::{Dict, ObjRef, Object, Stream}; + let mut doc = figures_doc(); + doc.update_dict(ObjRef::new(7, 0), |d| d.set(b"K".to_vec(), Object::Int(2))).unwrap(); + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::name("NoSuchFilter")); + let other = doc.add(Object::Stream(Stream::from_raw(dict, b"/Figure << /MCID 2 >> BDC 0 g EMC".to_vec()))); + doc.update_dict(ObjRef::new(3, 0), |p| p.set(b"Contents".to_vec(), Object::Array(vec![Object::Ref(other), Object::Ref(ObjRef::new(4, 0))]))) + .unwrap(); + let state = |doc: &Document| { + (doc.get(ObjRef::new(3, 0)).as_dict().unwrap().get(b"Contents").cloned(), doc.get(ObjRef::new(9, 0)), doc.get(ObjRef::new(6, 0))) + }; + let before = state(&doc); + assert_eq!(mark_decorative(&mut doc, ObjRef::new(7, 0)), Err(AltError::Damaged)); + assert_eq!(mark_tag_artifact(&mut doc, ObjRef::new(7, 0)), Err(ArtifactError::Damaged)); + assert_eq!(state(&doc), before); +} + +/// A tag's content not found before the damage in a damaged stream may be past it: it isn't known +/// to be off the page, so neither marking changes anything. +#[test] +fn a_tags_content_past_the_damage_is_not_taken_for_absent() { + use pdfcraft_cos::ObjRef; + let damaged = + damaged_flate(b"/Photo << /MCID 1 >> BDC 100 100 50 30 re f EMC\n", b"/Figure << /MCID 0 >> BDC q 40 0 0 20 10 150 cm /Im Do Q EMC"); + let mut doc = figures_doc(); + doc.set(ObjRef::new(4, 0), damaged); + let contents = |doc: &Document| doc.get(ObjRef::new(3, 0)).as_dict().unwrap().get(b"Contents").cloned(); + let before = (contents(&doc), doc.get(ObjRef::new(9, 0)), doc.get(ObjRef::new(6, 0))); + assert_eq!(mark_decorative(&mut doc, ObjRef::new(7, 0)), Err(AltError::Damaged)); + assert_eq!(mark_tag_artifact(&mut doc, ObjRef::new(7, 0)), Err(ArtifactError::Damaged)); + assert_eq!((contents(&doc), doc.get(ObjRef::new(9, 0)), doc.get(ObjRef::new(6, 0))), before); +} + +/// A page's `/StructParents` given by reference is read through it: the parent tree no longer +/// points the marked content at the tag. +#[test] +fn struct_parents_given_by_reference_are_read() { + use pdfcraft_cos::{ObjRef, Object}; + let by_reference = || { + let mut doc = figures_doc(); + let key = doc.add(Object::Int(0)); + doc.update_dict(ObjRef::new(3, 0), |p| p.set(b"StructParents".to_vec(), Object::Ref(key))).unwrap(); + doc + }; + let nums = |doc: &Document| doc.get(ObjRef::new(9, 0)).as_dict().and_then(|d| d.get(b"Nums").cloned()); + let mut doc = by_reference(); + mark_decorative(&mut doc, ObjRef::new(7, 0)).unwrap(); + assert_eq!(nums(&doc), Some(Object::Array(vec![Object::Int(0), Object::Array(vec![Object::Null, Object::Ref(ObjRef::new(8, 0))])]))); + let mut doc = by_reference(); + mark_tag_artifact(&mut doc, ObjRef::new(8, 0)).unwrap(); + assert_eq!(nums(&doc), Some(Object::Array(vec![Object::Int(0), Object::Array(vec![Object::Ref(ObjRef::new(7, 0)), Object::Null])]))); +} + +/// A content stream that doesn't decode at all is kept as stored, and a tag in the page's other +/// streams is still marked: only running past the decode budget refuses. +#[test] +fn a_tag_beside_a_stream_that_does_not_decode_is_marked() { + use pdfcraft_cos::{Dict, ObjRef, Object, Stream}; + let mut doc = figures_doc(); + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::name("NoSuchFilter")); + let other = doc.add(Object::Stream(Stream::from_raw(dict, b"0 g".to_vec()))); + doc.update_dict(ObjRef::new(3, 0), |p| p.set(b"Contents".to_vec(), Object::Array(vec![Object::Ref(other), Object::Ref(ObjRef::new(4, 0))]))) + .unwrap(); + mark_tag_artifact(&mut doc, ObjRef::new(7, 0)).unwrap(); + let contents = doc.get(ObjRef::new(3, 0)).as_dict().unwrap().get(b"Contents").cloned(); + let Some(Object::Array(list)) = contents else { panic!("contents {contents:?}") }; + assert_eq!(list.first(), Some(&Object::Ref(other)), "kept as stored"); +} + fn ref_(n: u32) -> Option { Some(pdfcraft_cos::ObjRef::new(n, 0)) } diff --git a/crates/audit/src/cover.rs b/crates/audit/src/cover.rs index bb5adf7ff8..9f771f7040 100644 --- a/crates/audit/src/cover.rs +++ b/crates/audit/src/cover.rs @@ -283,7 +283,7 @@ impl<'a> Walker<'a> { return Err(AuditError::TooManyForms); } // A form stream that cannot be decoded fails the audit: it might hide a cover. - let data = self.budget.decode_strict(stream)?; + let data = self.budget.decode_strict(self.doc, stream)?; // The XObject matrix maps form space into the user space active at `Do`. let mut sub_gs = gs.clone(); sub_gs.ctm = matrix.then(&gs.ctm); @@ -326,7 +326,7 @@ pub fn filled_rects(doc: &Document, page: &Dict) -> Result, Audi let mut streams = Vec::with_capacity(contents.len()); for c in &contents { if let Object::Stream(s) = &*doc.resolve(c) { - streams.push(budget.decode_strict(s)?); + streams.push(budget.decode_strict(doc, s)?); } } let pieces = Pieces::join(&streams); diff --git a/crates/automation/src/batch.rs b/crates/automation/src/batch.rs index dd5ca6aec0..f650e565b7 100644 --- a/crates/automation/src/batch.rs +++ b/crates/automation/src/batch.rs @@ -185,6 +185,7 @@ impl Automation { let recipe = Recipe::parse(self, args)?; let recursive = args.opt_bool("recursive")?.unwrap_or(false); let stop = args.opt_bool("stop_on_error")?.unwrap_or(false); + let dry_run = args.opt_bool("dry_run")?.unwrap_or(false); let paths = args.strs("paths")?; if paths.is_empty() || paths.len() > 1000 { return Err(Args::wrong("paths", "1 to 1000 files or folders")); @@ -211,11 +212,41 @@ impl Automation { let mut files = discovery.skipped; let (mut completed, mut failed_count) = (0, 0); let mut stopped = false; + let mut planned_outputs: HashSet = HashSet::new(); for (path, relative) in discovery.files { if stopped || !recipe.accepts(&path) { files.push(json!({"path":path.to_string_lossy(),"status":"skipped","reason":if stopped {"stopped after error"} else {"unsupported input extension"}})); continue; } + if dry_run { + let planned: Result = (|| { + let target = out.join(&relative).with_extension("pdf"); + let target = self.resolve(target.to_str().ok_or_else(|| failed("output path is not UTF-8"))?, true)?; + if !target.starts_with(&out) { + return Err(failed("output path leaves out_dir")); + } + let dir = target.parent().ok_or_else(|| failed("output has no parent"))?; + let stem = target.file_stem().ok_or_else(|| failed("output has no stem"))?.to_string_lossy().into_owned(); + for n in 1..=10_000 { + let candidate = if n == 1 { target.clone() } else { dir.join(format!("{stem} ({n}).pdf")) }; + if candidate.symlink_metadata().is_ok() || planned_outputs.contains(&candidate) { + continue; + } + planned_outputs.insert(candidate.clone()); + return Ok(candidate); + } + Err(failed("all 10000 output names are occupied")) + })(); + match planned { + Ok(target) => files.push(json!({"path":path.to_string_lossy(),"status":"planned","output":target.to_string_lossy().replace('\\', "/"),"steps":recipe.steps})), + Err(e) => { + failed_count += 1; + files.push(json!({"path":path.to_string_lossy(),"status":"error","error":e.to_string()})); + stopped = stop; + } + } + continue; + } // Recheck confinement and links before reading a discovered file. let result: Result<_> = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { let current = self.resolve(path.to_str().ok_or_else(|| failed("input path is not UTF-8"))?, false)?; @@ -244,8 +275,8 @@ impl Automation { } } } - let skipped = files.len().saturating_sub(completed + failed_count); - Ok(json!({"completed":completed,"failed":failed_count,"skipped":skipped,"stopped":stopped,"files":files})) + let skipped = files.iter().filter(|file| file["status"] == "skipped").count(); + Ok(json!({"dry_run":dry_run,"completed":completed,"failed":failed_count,"skipped":skipped,"stopped":stopped,"files":files})) } } @@ -279,4 +310,20 @@ mod tests { discovery.visited = 0; assert!(discovery.walk(Path::new("absent"), Path::new(""), Path::new("output"), true, MAX_DEPTH + 1).unwrap_err().to_string().contains("64")); } + + #[test] + fn doc_batch_dry_run_lists_outputs_without_writing() { + let dir = tempfile::tempdir().unwrap(); + std::fs::create_dir(dir.path().join("input")).unwrap(); + let pdf = Session::new().create_from_text("Doc", "Hello").unwrap(); + std::fs::write(dir.path().join("input/a.pdf"), &*pdf).unwrap(); + let auto = Automation::new().with_root(dir.path()).unwrap(); + let report = auto.doc_batch(&Args(&json!({"paths":["input"],"steps":["optimize"],"out_dir":"out","dry_run":true}))).unwrap(); + assert_eq!(report["dry_run"], true, "{report}"); + assert_eq!(report["completed"], 0, "{report}"); + assert_eq!(report["failed"], 0, "{report}"); + assert_eq!(report["files"][0]["status"], "planned", "{report}"); + assert!(report["files"][0]["output"].as_str().unwrap().ends_with("out/a.pdf"), "{report}"); + assert!(!dir.path().join("out").exists()); + } } diff --git a/crates/automation/src/documents.rs b/crates/automation/src/documents.rs index f8c7988bbc..e4d0d722a0 100644 --- a/crates/automation/src/documents.rs +++ b/crates/automation/src/documents.rs @@ -6,6 +6,10 @@ use super::{Arc, Automation, BlockKey, DocId, Document, Edit, Path, PathBuf, Tex /// How far a watermark may be moved from the page centre, in points (a page is at most 14,400). const WATERMARK_MAX_OFFSET: f64 = 20_000.0; +/// `doc_optimize` with `target_bytes`: (max target ppi, max JPEG quality) per extra attempt, mildest first. +/// The quality applies only to images the caller left on JPEG; zip and retain keep their encoding. +const TARGET_LADDER: [(f64, u8); 5] = [(110.0, 45), (96.0, 35), (72.0, 28), (54.0, 20), (36.0, 12)]; + /// Bookmarks in one page of the bookmark list, by default and at most. const BOOKMARK_PAGE: usize = 100; @@ -380,7 +384,45 @@ impl Automation { }) .collect::>()?, }; - let (bytes, r) = self.session.optimized_bytes(id, &settings, &discard).map_err(failed)?; + let target = match a.opt_int("target_bytes")? { + None => None, + Some(t) if t >= 1 => Some(usize::try_from(t).map_err(|_| ToolError::InvalidArgs(format!("target_bytes {t} is too large")))?), + Some(t) => return Err(ToolError::InvalidArgs(format!("target_bytes must be at least 1, not {t}"))), + }; + let (mut bytes, mut r) = self.session.optimized_bytes(id, &settings, &discard).map_err(failed)?; + let mut attempts = 1; + let mut step_used = 0; + // Settings already optimized with; a rung that changes nothing is not run again. + let mut tried = vec![settings.clone()]; + if let Some(t) = target { + // Stronger image settings, one rung per attempt, until the copy fits. Every attempt + // starts from the open document, so a failed one changes nothing; the number of + // attempts is fixed. The smallest copy is kept if none fits. + for (k, &(ppi, quality)) in TARGET_LADDER.iter().enumerate() { + if bytes.len() <= t { + break; + } + let mut s = settings.clone(); + for img in [&mut s.color, &mut s.gray] { + img.downsample = true; + img.target_ppi = img.target_ppi.min(ppi); + img.above_ppi = img.above_ppi.min(ppi * 1.5).max(img.target_ppi); + // Only a caller who chose JPEG gets a lower quality; zip and retain stay as asked. + if let Compression::Jpeg(q) = img.compression { + img.compression = Compression::Jpeg(q.min(quality)); + } + } + if tried.contains(&s) { + continue; + } + let (b2, r2) = self.session.optimized_bytes(id, &s, &discard).map_err(failed)?; + tried.push(s); + attempts += 1; + if b2.len() < bytes.len() { + (bytes, r, step_used) = (b2, r2, k + 1); + } + } + } write_atomic(&path, &bytes)?; fn valid_version(v: &[u8]) -> Option<&str> { if matches!(v, [b'1'..=b'9', b'.', b'0'..=b'9']) { std::str::from_utf8(v).ok() } else { None } @@ -400,7 +442,19 @@ impl Automation { // entry). Both are "d.d", so text order is version order. let pdf_version = catalog_version.as_deref().into_iter().chain(header_version).max(); let o = &r.optimize; - Ok(json!({ + let mut extra = json!({}); + if let Some(t) = target { + let (ppi, quality) = step_used.checked_sub(1).and_then(|k| TARGET_LADDER.get(k)).copied().unzip(); + let lossy = matches!(settings.color.compression, Compression::Jpeg(_)) || matches!(settings.gray.compression, Compression::Jpeg(_)); + let quality = quality.filter(|_| lossy); + extra = json!({ + "target_bytes": t, + "target_reached": bytes.len() <= t, + "attempts": attempts, + "target_settings": { "step": step_used, "max_ppi": ppi, "max_jpeg_quality": quality }, + }); + } + let mut out = json!({ "path": path.to_string_lossy(), "pdf_version": pdf_version, "object_streams": settings.effective_object_streams(), @@ -425,7 +479,11 @@ impl Automation { "unused_xobjects": o.unused_xobjects, "merged_objects": r.merged, "discarded": r.discarded.iter().map(|(h, n)| json!({ "category": h.id(), "count": n })).collect::>(), - })) + }); + if let (Some(o), Some(e)) = (out.as_object_mut(), extra.as_object()) { + o.extend(e.iter().map(|(k, v)| (k.clone(), v.clone()))); + } + Ok(out) } /// A watermark or background picture: `file` (an image or a PDF) and `file_page` (1-based). diff --git a/crates/automation/src/redact.rs b/crates/automation/src/redact.rs index a146f3f8ff..6ec66ce680 100644 --- a/crates/automation/src/redact.rs +++ b/crates/automation/src/redact.rs @@ -2,8 +2,8 @@ //! (removing what they cover for good) or clear them. use pdfcraft_engine::{ - Edit, Hidden, NewAnnotation, REDACT_MAX_WORDS, REDACTION_CODE_SETS, RedactPattern, RedactionCodeSet, Shape, Style, find_pattern, rect_quad, - redact_word_list, + Edit, Hidden, NewAnnotation, REDACT_MAX_WORDS, REDACT_VALIDATORS, REDACTION_CODE_SETS, RedactPattern, RedactRegex, RedactValidator, + RedactionCodeSet, Shape, Style, find_pattern, rect_quad, redact_word_list, }; use serde_json::{Value, json}; @@ -59,16 +59,20 @@ impl Automation { a.get("find").is_some(), a.get("words").is_some(), a.get("pattern").is_some(), + a.get("regex").is_some(), a.opt_bool("whole_pages")?.unwrap_or(false), ]; if chosen.iter().filter(|c| **c).count() != 1 { - return Err(ToolError::InvalidArgs("pass exactly one of rect (with page), find, words, pattern or whole_pages: true".into())); + return Err(ToolError::InvalidArgs("pass exactly one of rect (with page), find, words, pattern, regex or whole_pages: true".into())); } let case_sensitive = a.opt_bool("case_sensitive")?.unwrap_or(false); let whole_words = a.opt_bool("whole_words")?.unwrap_or(false); if (case_sensitive || whole_words) && a.get("find").is_none() && a.get("words").is_none() { return Err(ToolError::InvalidArgs("case_sensitive and whole_words apply only to find or words".into())); } + if a.get("validate").is_some() && a.get("regex").is_none() { + return Err(ToolError::InvalidArgs("validate applies only to regex".into())); + } // Per searched word or phrase: how many marks it made. let mut matched_words: Vec<(String, usize)> = Vec::new(); if a.get("rect").is_some() { @@ -89,6 +93,19 @@ impl Automation { Some(p) => Some(RedactPattern::from_id(p).ok_or_else(|| ToolError::InvalidArgs(format!("unknown pattern {p:?}")))?), None => None, }; + let user_re = match a.opt_str("regex")? { + Some(src) => { + let validator = match a.opt_str("validate")? { + Some(v) => Some(RedactValidator::from_id(v).ok_or_else(|| { + let ids: Vec<_> = REDACT_VALIDATORS.iter().map(|v| v.id()).collect(); + ToolError::InvalidArgs(format!("unknown validate {v:?}; allowed: {}", ids.join(", "))) + })?), + None => None, + }; + Some(RedactRegex::new(src, validator).map_err(|e| ToolError::InvalidArgs(e.to_string()))?) + } + None => None, + }; let needles: Vec = match (a.opt_str("find")?, a.get("words")) { (Some(f), _) => vec![f.to_owned()], (None, Some(_)) => { @@ -108,9 +125,27 @@ impl Automation { let texts = self.page_texts(id, &pages)?; let info = self.doc(a)?.info.clone(); for (&p, text) in pages.iter().zip(&texts) { - let hits: Vec<(Option, _)> = match pattern { - Some(pat) => text.find_with(|chars| find_pattern(pat, chars)).into_iter().map(|h| (None, h)).collect(), - None => needles + let hits: Vec<(Option, _)> = match (&user_re, pattern) { + (Some(re), _) => { + // A page with too many matches fails the call: marking only the first ones + // would leave the rest readable after redact_apply. + let too_many = std::cell::Cell::new(false); + let found = text.find_with(|chars| { + re.find(chars).unwrap_or_else(|_| { + too_many.set(true); + Vec::new() + }) + }); + if too_many.get() { + return Err(ToolError::InvalidArgs(format!( + "regex has more than 10000 matches on page {}; nothing was marked. Narrow the expression so that every match can be marked", + p + 1 + ))); + } + found.into_iter().map(|h| (None, h)).collect() + } + (None, Some(pat)) => text.find_with(|chars| find_pattern(pat, chars)).into_iter().map(|h| (None, h)).collect(), + (None, None) => needles .iter() .enumerate() .flat_map(|(i, w)| text.find_opts(w, case_sensitive, whole_words).into_iter().map(move |h| (Some(i), h))) diff --git a/crates/automation/src/signing.rs b/crates/automation/src/signing.rs index 743a36ff56..df0fd643fb 100644 --- a/crates/automation/src/signing.rs +++ b/crates/automation/src/signing.rs @@ -119,6 +119,7 @@ impl Automation { "page": s.page.map(|p| p + 1), "rect": rect, "visible": s.visible, + "verdict_shown": s.verdict_shown, "revision": s.revision, "sub_filter": s.sub_filter, "algorithm": s.algorithm, diff --git a/crates/automation/src/tools.rs b/crates/automation/src/tools.rs index 84354f51d7..8b302e326e 100644 --- a/crates/automation/src/tools.rs +++ b/crates/automation/src/tools.rs @@ -148,7 +148,7 @@ pub fn tools() -> Vec { "paths": {"type":"array","items":{"type":"string"},"minItems":1,"maxItems":1000}, "out_dir": {"type":"string"}, "steps": {"type":"array","items":{"type":"string","enum":["convert","ocr","optimize"]},"minItems":1,"maxItems":3,"uniqueItems":true}, - "recursive": {"type":"boolean"}, "stop_on_error": {"type":"boolean"}, + "recursive": {"type":"boolean"}, "stop_on_error": {"type":"boolean"}, "dry_run": {"type":"boolean"}, "ppi": {"type":"number","minimum":9,"maximum":1600}, "quality": {"type":"integer","minimum":1,"maximum":100}, "dpi": {"type":"number","minimum":72,"maximum":600}, @@ -620,7 +620,7 @@ pub fn tools() -> Vec { t( "redact_mark", "Mark for redaction", - "Mark content for redaction (nothing is removed until redact_apply). One of: rect [x0, y0, x1, y1] (points from the top-left of the displayed page) with page; find (text, every match); words (a list of words or phrases, every match of each; the result's matched_words says how many marks each made); pattern (phone, email, credit-card, ssn, date, iban: every match, Acrobat's Search & Redact patterns); whole_pages: true. find and words accept case_sensitive and whole_words (both default false). Select one page with page or multiple with pages (default all); do not pass both. rect requires page. overlay: text shown on the box once applied, or code_set (foia: U.S. FOIA (b)(1)(A)…(b)(9); privacy-act: U.S. Privacy Act (d)(5), (j)(1)…(k)(7)) with codes, shown as \"(b)(6), (b)(7)(C)\"; fill: box colour (default black). Undoable.", + "Mark content for redaction (nothing is removed until redact_apply). One of: rect [x0, y0, x1, y1] (points from the top-left of the displayed page) with page; find (text, every match); words (a list of words or phrases, every match of each; the result's matched_words says how many marks each made); pattern (phone, email, credit-card, ssn, date, iban: every match, Acrobat's Search & Redact patterns); regex (your own regular expression, every match, on the page text with words and lines separated by one space; Rust regex syntax: linear-time, no look-around or back-references, case-sensitive unless (?i), at most 1024 bytes, more than 10000 matches on one page fail the call, matches of the empty string are refused) with optional validate (luhn, mod97, iban: keep a match only if its checksum holds); whole_pages: true. find and words accept case_sensitive and whole_words (both default false). Select one page with page or multiple with pages (default all); do not pass both. rect requires page. overlay: text shown on the box once applied, or code_set (foia: U.S. FOIA (b)(1)(A)…(b)(9); privacy-act: U.S. Privacy Act (d)(5), (j)(1)…(k)(7)) with codes, shown as \"(b)(6), (b)(7)(C)\"; fill: box colour (default black). Undoable.", ) .with(schema( json!({ @@ -632,6 +632,8 @@ pub fn tools() -> Vec { "case_sensitive": { "type": "boolean", "description": "Match letter case for find or words (default false)." }, "whole_words": { "type": "boolean", "description": "Match whole words for find or words (default false)." }, "pattern": pattern_enum(), + "regex": { "type": "string", "minLength": 1, "maxLength": 1024, "description": "Regular expression (Rust regex syntax), at most 1024 bytes (maxLength counts characters, the limit is bytes); every match is marked." }, + "validate": { "type": "string", "enum": ["luhn", "mod97", "iban"], "description": "With regex: keep a match only if its checksum is valid (spaces and hyphens ignored)." }, "whole_pages": { "type": "boolean" }, "pages": pages("to search or mark (default: all); mutually exclusive with page, and not valid with rect"), "overlay": { "type": "string" }, @@ -896,7 +898,7 @@ pub fn tools() -> Vec { comment_ref(json!({ "status": { "type": "string", "enum": ["none", "accepted", "rejected", "cancelled", "completed"] }, "author": { "type": "string" } })), &["doc", "status"], )), - t("sign_list", "List and validate signatures", "Every signature field with its validation: status (valid, unknown = intact but the signer isn't trusted, invalid, unsigned), signer and certificate, date, reason, location, certification level, page and rect, the revision it covers and changes made after signing (none, allowed, disallowed), with Acrobat-style explanations.") + t("sign_list", "List and validate signatures", "Every signature field with its validation: status (valid, unknown = intact but the signer isn't trusted, invalid, unsigned), signer and certificate, date, reason, location, certification level, page and rect, whether PdfCraft draws that state into the signature's own appearance as displayed (verdict_shown; never written to the file), the revision it covers and changes made after signing (none, allowed, disallowed), with Acrobat-style explanations.") .ro() .with(schema(json!({ "doc": doc() }), &["doc"])), t("sign_id_create", "Create a digital ID", "Create a self-signed digital ID and save it as a password-protected .p12 file (Acrobat: Configure a new digital ID ▸ Create a new digital ID ▸ Save to file). Default key: 2048-bit RSA, valid 5 years.").with(schema( @@ -1290,7 +1292,7 @@ pub fn tools() -> Vec { "property": { "type": "string", "enum": ["type", "title", "actual_text", "alternate_text", "language", "summary", "scope"] }, "value": { "type": "string" }, }), &["doc", "object", "generation", "property"])), - t("accessibility_mark_tag_artifact", "Mark tag content as artifact", "Mark a reachable leaf structure tag's direct marked content as an artifact and remove the tag from the reading structure. Use object and generation from accessibility_tags. Tags with child structure elements, no direct marked content, or content not found on its page are rejected. Undoable.") + t("accessibility_mark_tag_artifact", "Mark tag content as artifact", "Mark a reachable leaf structure tag's direct marked content as an artifact and remove the tag from the reading structure. Use object and generation from accessibility_tags. Tags with child structure elements, no direct marked content, content not found on its page, or content that a damaged or undecodable content stream may hide or change are rejected. Undoable.") .with(schema(json!({ "doc": doc(), "object": { "type": "integer", "minimum": 1 }, "generation": { "type": "integer", "minimum": 0, "maximum": 65535 }, @@ -1633,7 +1635,7 @@ pub fn tools() -> Vec { .with(schema(json!({ "doc": doc() }), &["doc"])), t("doc_open_revision", "Open a revision", "Open saved revision `revision` (1 = the oldest) of a document as a new, unsaved document, to see the file as it was then.") .with(schema(json!({ "doc": doc(), "revision": { "type": "integer", "minimum": 1 } }), &["doc", "revision"])), - t("doc_optimize", "Optimize PDF", "Write an optimized copy to `path` (Acrobat's PDF Optimizer). preset: standard (default; Reduce File Size's choices) or mobile (images to 96 ppi above 144, JPEG high quality); the options below change it. compatibility: retain (default; keeps the document's version, or 1.5 when object streams are written), 1.4, 1.5, 1.6, 1.7 or 2.0. merge_fragments (default true) first puts a page picture drawn as many stacked image strips (as some PDF printers write it) back together into one image, so it can be resampled (merged_images, fragments_merged). color / gray: { downsample, ppi, above_ppi, compression: jpeg|zip|retain, quality 1–100 } (defaults: downsample to 150 ppi above 225, JPEG 60). mono: { downsample, ppi, above_ppi, compression: ccitt_g4|jbig2|retain } for black-and-white images and image masks (default: averaged down to 300 ppi above 450 ppi, counted in images_resampled; stored losslessly as CCITT Group 4 or JBIG2 (or 1-bit Flate, once resampled or packed from more bits) when that is smaller, up to 16 JBIG2 images sharing one symbol dictionary (/JBIG2Globals) where that is smaller still (image masks and DeviceGray images named directly, of at most 64 Mi pixels, which redaction can store again without it), Group 3 fax images, JBIG2 images, black-and-white RGB and indexed images using only palette entries 0 and 1 among them, counted in images_recompressed; retain resamples only, a CCITT image staying CCITT, a JBIG2 image JBIG2 and any other becoming Flate). Images past what one run may decode stay as they are, counted in images_refused; streams it only stores with Flate past that stay LZW, uncounted. Images are measured where pages draw them and replaced only when smaller. discard_*: thumbnails (default true), alternate_images (true), tags, print_settings; flate_unencoded (true); object_streams (default true; always off for 1.4); lzw_to_flate (true) decodes LZW streams and stores them with Flate instead, leaving any it cannot decode unchanged (lzw_streams counts the streams converted); remove_invalid_links and remove_unreferenced_dests (true); images and forms the pages list but never draw are always dropped (unused_xobjects). discard: Remove Hidden Information categories (metadata, attachments, comments, form-fields, hidden-text, hidden-layers, bookmarks, links-actions-scripts, private-data). Signed documents are refused. The open document is unchanged.") + t("doc_optimize", "Optimize PDF", "Write an optimized copy to `path` (Acrobat's PDF Optimizer). preset: standard (default; Reduce File Size's choices) or mobile (images to 96 ppi above 144, JPEG high quality); the options below change it. compatibility: retain (default; keeps the document's version, or 1.5 when object streams are written), 1.4, 1.5, 1.6, 1.7 or 2.0. merge_fragments (default true) first puts a page picture drawn as many stacked image strips (as some PDF printers write it) back together into one image, so it can be resampled (merged_images, fragments_merged). color / gray: { downsample, ppi, above_ppi, compression: jpeg|zip|retain, quality 1–100 } (defaults: downsample to 150 ppi above 225, JPEG 60). mono: { downsample, ppi, above_ppi, compression: ccitt_g4|jbig2|retain } for black-and-white images and image masks (default: averaged down to 300 ppi above 450 ppi, counted in images_resampled; stored losslessly as CCITT Group 4 or JBIG2 (or 1-bit Flate, once resampled or packed from more bits) when that is smaller, up to 16 JBIG2 images sharing one symbol dictionary (/JBIG2Globals) where that is smaller still (image masks and DeviceGray images named directly, of at most 64 Mi pixels, which redaction can store again without it), Group 3 fax images, JBIG2 images, black-and-white RGB and indexed images using only palette entries 0 and 1 among them, counted in images_recompressed; retain resamples only, a CCITT image staying CCITT, a JBIG2 image JBIG2 and any other becoming Flate). Images past what one run may decode stay as they are, counted in images_refused; streams it only stores with Flate past that stay LZW, uncounted. Images are measured where pages draw them and replaced only when smaller. discard_*: thumbnails (default true), alternate_images (true), tags, print_settings; flate_unencoded (true); object_streams (default true; always off for 1.4); lzw_to_flate (true) decodes LZW streams and stores them with Flate instead, leaving any it cannot decode unchanged (lzw_streams counts the streams converted); remove_invalid_links and remove_unreferenced_dests (true); images and forms the pages list but never draw are always dropped (unused_xobjects). discard: Remove Hidden Information categories (metadata, attachments, comments, form-fields, hidden-text, hidden-layers, bookmarks, links-actions-scripts, private-data). Signed documents are refused. target_bytes (optional): if the copy is larger, retry with up to 5 progressively stronger image settings (downsample to at most 110, 96, 72, 54, 36 ppi, applied to colour and grey images; images set to JPEG also get JPEG quality at most 45, 35, 28, 20, 12, while zip and retain images keep their encoding and only get the lower ppi) and stop at the first copy that fits; reports target_reached, attempts and target_settings, and writes the smallest copy with target_reached false when none fits. Without it nothing changes. The open document is unchanged.") .cmd("optimize.advanced") .with(schema( json!({ @@ -1645,6 +1647,7 @@ pub fn tools() -> Vec { "color": { "type": "object" }, "gray": { "type": "object" }, "mono": { "type": "object" }, + "target_bytes": { "type": "integer", "minimum": 1 }, "discard_thumbnails": { "type": "boolean" }, "discard_alternate_images": { "type": "boolean" }, "discard_tags": { "type": "boolean" }, diff --git a/crates/automation/tests/it/automation.rs b/crates/automation/tests/it/automation.rs index dc5dec0521..5addfe5fbd 100644 --- a/crates/automation/tests/it/automation.rs +++ b/crates/automation/tests/it/automation.rs @@ -4818,6 +4818,34 @@ fn redacting_through_tools() { assert!(matches!(a.call("redact_mark", &json!({ "doc": doc })), Err(ToolError::InvalidArgs(_)))); } +#[test] +fn redacting_with_a_caller_regex_through_tools() { + let dir = workdir("redact-regex"); + std::fs::write( + dir.join("ids.txt"), + "Ref 1 85 05 78 006 084 36 end\nRef 2 91 12 34 567 890 12 end\nCard 4539 1488 0343 6467 and 4539 1488 0343 6468", + ) + .unwrap(); + let mut a = auto(&dir); + let doc = ok(&mut a, "doc_create", json!({ "from": "text", "path": "ids.txt" }))["doc"].as_u64().unwrap(); + let bad = |a: &mut Automation, args: Value| matches!(a.call("redact_mark", &args), Err(ToolError::InvalidArgs(_))); + assert!(bad(&mut a, json!({ "doc": doc, "regex": "(unclosed" }))); + assert!(bad(&mut a, json!({ "doc": doc, "regex": "a*" }))); + assert!(bad(&mut a, json!({ "doc": doc, "regex": "x".repeat(2000) }))); + assert!(bad(&mut a, json!({ "doc": doc, "regex": "x", "pattern": "ssn" }))); + assert!(bad(&mut a, json!({ "doc": doc, "regex": "x", "validate": "crc" }))); + assert!(bad(&mut a, json!({ "doc": doc, "find": "x", "validate": "luhn" }))); + assert!(matches!(a.call("redact_mark", &json!({ "doc": doc, "regex": "Z{3}" })), Err(ToolError::Failed(_)))); + let r = ok(&mut a, "redact_mark", json!({ "doc": doc, "regex": r"\d \d\d \d\d \d\d \d{3} \d{3} \d\d" })); + assert_eq!(r["marked"].as_u64(), Some(2)); + let r = ok(&mut a, "redact_mark", json!({ "doc": doc, "regex": r"\d{4}( \d{4}){3}", "validate": "luhn" })); + assert_eq!(r["marked"].as_u64(), Some(1), "only the number with a valid checksum"); + ok(&mut a, "redact_apply", json!({ "doc": doc })); + let text = page_text(&mut a, doc)[0].clone(); + assert!(!text.contains("85 05") && !text.contains("91 12") && !text.contains("6467"), "{text}"); + assert!(text.contains("6468"), "{text}"); +} + #[test] /// Mark, apply and save remove the complete international number while retaining labels and controls. fn redacting_italian_international_slash_numbers_through_tools() { @@ -6899,6 +6927,43 @@ fn a_dynamic_xfa_forms_signature_box_is_signed_under_its_som_path() { let _ = std::fs::remove_dir_all(&dir); } +#[test] +fn named_layer_verdicts_are_reported_and_rendered_through_tools() { + use pdfcraft_cos::{Dict, Document, Lexer, ObjRef, Object, SaveOptions, Stream}; + let dir = workdir("named-layer-verdict"); + let mut cos = Document::open(std::sync::Arc::new(fixture(1))).unwrap(); + let dict = |text: &str| Lexer::new(text.as_bytes(), 0).object().unwrap().as_dict().unwrap().clone(); + let form = |resources: Dict, ops: &str| { + let mut d = dict("<< /Type /XObject /Subtype /Form /BBox [0 0 100 100] >>"); + d.set(b"Resources".to_vec(), resources); + Stream::from_raw(d, ops.as_bytes().to_vec()) + }; + let blank = cos.add(form(Dict::new(), "")); + let layers = dict(&format!("<< /XObject << /n1 {blank} /n3 {blank} >> >>")); + let layer_form = cos.add(form(layers, "/n1 Do /n3 Do")); + let normal = cos.add(form(dict(&format!("<< /XObject << /FRM {layer_form} >> >>")), "/FRM Do")); + // Deliberately malformed CMS: an invalid signature still gets the redrawn invalid state. + let value = cos.add(dict("<< /Type /Sig /SubFilter /ETSI.CAdES.detached /ByteRange [0 0 0 0] /Contents <00> >>")); + let widget = cos + .add(dict(&format!("<< /FT /Sig /T (Signature1) /Subtype /Widget /F 4 /Rect [50 50 150 150] /P 4 0 R /V {value} /AP << /N {normal} >> >>"))); + cos.update_dict(ObjRef::new(4, 0), |p| p.set(b"Annots".to_vec(), vec![Object::Ref(widget)])).unwrap(); + cos.update_dict(cos.root().unwrap(), |root| root.set(b"AcroForm".to_vec(), dict(&format!("<< /Fields [{widget}] >>")))).unwrap(); + let original = pdfcraft_cos::write_full(&cos, &SaveOptions::default()).unwrap(); + std::fs::write(dir.join("named.pdf"), &original).unwrap(); + let mut a = auto(&dir); + let doc = ok(&mut a, "doc_open", json!({"path": "named.pdf"}))["doc"].as_u64().unwrap(); + let list = ok(&mut a, "sign_list", json!({"doc": doc})); + assert_eq!(list["signatures"][0]["status"], "invalid"); + assert_eq!(list["signatures"][0]["verdict_shown"], true); + let rendered = a.call("page_render", &json!({"doc": doc, "page": 1, "dpi": 72})).unwrap(); + let Content::Png { data, .. } = &rendered[0] else { panic!("expected a PNG") }; + let pixels = image::load_from_memory_with_format(data, image::ImageFormat::Png).unwrap().to_rgba8(); + let red = pixels.pixels().filter(|p| p.0.iter().zip([0xD7, 0x37, 0x3F]).all(|(a, b)| a.abs_diff(b) < 30)).count(); + assert!(red > 100, "the invalid state is drawn: {red} red pixels"); + ok(&mut a, "doc_save", json!({"doc": doc, "path": "saved.pdf", "full": false})); + assert_eq!(std::fs::read(dir.join("saved.pdf")).unwrap(), original, "the drawn state never enters saves"); +} + #[test] fn digital_ids_signing_and_validation_through_tools() { let dir = workdir("signing"); @@ -6957,6 +7022,8 @@ fn sign_document_and_verify_trust(a: &mut Automation, dir: &Path) -> u64 { assert_eq!(t["trusted"].as_array().unwrap().len(), 1); let list = ok(a, "sign_list", json!({ "doc": doc })); assert_eq!((list["all_valid"].as_bool(), list["signatures"][0]["status"].as_str()), (Some(true), Some("valid"))); + // PdfCraft's own appearance has no status layers to draw into: only the page badge shows it. + assert_eq!(list["signatures"][0]["verdict_shown"], false); ok(a, "comment_add", json!({ "doc": doc, "page": 1, "type": "note", "at": [20, 20], "contents": "ok" })); let s = &ok(a, "sign_list", json!({ "doc": doc }))["signatures"][0]; assert_eq!((s["status"].as_str(), s["modification"].as_str()), (Some("valid"), Some("allowed"))); @@ -7565,6 +7632,63 @@ fn a_page_printed_in_strips_gets_small_through_tools() { assert!(matches!(a.call("doc_optimize", &json!({ "doc": doc, "path": "x.pdf", "preset": "tiny" })), Err(ToolError::InvalidArgs(_)))); } +#[test] +fn optimizing_to_a_target_size() { + // Dropped at the end of the test, also when an assertion fails. + let tmp = tempfile::TempDir::new().unwrap(); + let dir = tmp.path().to_path_buf(); + let (w, h) = (2400u32, 1600u32); + let px: Vec = (0..w * h) + .flat_map(|i| { + let (x, y) = (i % w, i / w); + [(x * 255 / w) as u8 ^ ((x * y) & 15) as u8, (y * 255 / h) as u8, ((x + y) & 255) as u8] + }) + .collect(); + let mut jpeg = Vec::new(); + let mut enc = image::codecs::jpeg::JpegEncoder::new_with_quality(&mut jpeg, 95); + enc.set_pixel_density(image::codecs::jpeg::PixelDensity::dpi(600)); + enc.encode(&px, w, h, image::ExtendedColorType::Rgb8).unwrap(); + std::fs::write(dir.join("photo.jpg"), &jpeg).unwrap(); + let mut a = auto(&dir); + let doc = ok(&mut a, "doc_create", json!({ "from": "images", "paths": ["photo.jpg"] }))["doc"].as_u64().unwrap(); + // Without a target nothing changes: no target fields. + let base = ok(&mut a, "doc_optimize", json!({ "doc": doc, "path": "base.pdf" })); + assert!(base.get("target_reached").is_none(), "{base}"); + let base_len = base["bytes_after"].as_u64().unwrap(); + // A target below the default result is met by a stronger step. + let r = ok(&mut a, "doc_optimize", json!({ "doc": doc, "path": "fit.pdf", "target_bytes": base_len - 1 })); + assert_eq!(r["target_reached"], true, "{r}"); + assert!(r["bytes_after"].as_u64().unwrap() < base_len, "{r}"); + assert!(r["target_settings"]["step"].as_u64().unwrap() >= 1, "{r}"); + assert_eq!(std::fs::metadata(dir.join("fit.pdf")).unwrap().len(), r["bytes_after"].as_u64().unwrap()); + // A target already met by the first attempt tries nothing more. + let r = ok(&mut a, "doc_optimize", json!({ "doc": doc, "path": "easy.pdf", "target_bytes": base_len })); + assert_eq!((r["target_reached"].as_bool(), r["attempts"].as_u64()), (Some(true), Some(1)), "{r}"); + // An impossible target: bounded attempts, smallest copy written, said plainly. + let r = ok(&mut a, "doc_optimize", json!({ "doc": doc, "path": "tiny.pdf", "target_bytes": 1 })); + assert_eq!((r["target_reached"].as_bool(), r["attempts"].as_u64()), (Some(false), Some(6)), "{r}"); + assert!(r["bytes_after"].as_u64().unwrap() < base_len, "{r}"); + assert!(std::fs::metadata(dir.join("tiny.pdf")).is_ok()); + // The open document is untouched by the failed attempts, and bad targets are refused. + let again = ok(&mut a, "doc_optimize", json!({ "doc": doc, "path": "again.pdf" })); + assert_eq!(again["bytes_after"].as_u64().unwrap(), base_len); + assert!(matches!(a.call("doc_optimize", &json!({ "doc": doc, "path": "x.pdf", "target_bytes": 0 })), Err(ToolError::InvalidArgs(_)))); + // A caller who asked for lossless images keeps them: retries only tighten the ppi, no JPEG quality. + let lossless = json!({ "compression": "zip" }); + let r = ok(&mut a, "doc_optimize", json!({ "doc": doc, "path": "zip.pdf", "color": lossless, "gray": lossless, "target_bytes": 1 })); + assert_eq!(r["attempts"].as_u64(), Some(6), "{r}"); + assert!(r["target_settings"]["max_ppi"].is_number() && r["target_settings"]["max_jpeg_quality"].is_null(), "{r}"); + // A JPEG caller gets the quality step. + let r = ok(&mut a, "doc_optimize", json!({ "doc": doc, "path": "jpg.pdf", "target_bytes": 1 })); + assert!(r["target_settings"]["max_jpeg_quality"].is_number(), "{r}"); + // Settings already at the ladder's floor leave every rung equal to the first run: no retries. + let floor = json!({ "ppi": 9, "above_ppi": 9, "compression": "jpeg", "quality": 1 }); + let r = ok(&mut a, "doc_optimize", json!({ "doc": doc, "path": "floor.pdf", "color": floor, "gray": floor, "target_bytes": 1 })); + assert_eq!((r["target_reached"].as_bool(), r["attempts"].as_u64()), (Some(false), Some(1)), "{r}"); + // Confinement still applies. + assert!(a.call("doc_optimize", &json!({ "doc": doc, "path": "../out.pdf", "target_bytes": 1000 })).is_err()); +} + #[test] fn initial_view_through_tools() { let dir = workdir("initial-view"); @@ -8287,6 +8411,52 @@ BT /F1 11 Tf 20 20 Td (Total due: 150.00) Tj ET"; assert!(rendered.iter().any(|c| matches!(c, Content::Png { .. }))); } +/// #5149: a two-column, two-row table drawn with ruled lines reaches Word as a table with its +/// rules, not as four paragraphs one under another. +#[test] +fn exporting_a_ruled_two_by_two_table_to_word_keeps_the_table() { + let dir = workdir("ruled-table-word"); + let content = "0.5 w 20 270 m 300 270 l S 20 245 m 300 245 l S 20 220 m 300 220 l S \ +20 220 m 20 270 l S 160 220 m 160 270 l S 300 220 m 300 270 l S \ +BT /F1 11 Tf 25 253 Td (Name) Tj 140 0 Td (Value) Tj -140 -25 Td (Alpha) Tj 140 0 Td (One) Tj ET \ +BT /F1 11 Tf 20 150 Td (A paragraph after the table.) Tj ET"; + let stream = format!("<< /Length {} >>\nstream\n{content}\nendstream", content.len()); + let objs = [ + "<< /Type /Catalog /Pages 2 0 R >>", + "<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + "<< /Type /Page /Parent 2 0 R /MediaBox [0 0 320 300] /Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >>", + stream.as_str(), + "<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + ]; + std::fs::write(dir.join("table.pdf"), pdf_of(&objs)).unwrap(); + let mut a = auto(&dir); + let doc = ok(&mut a, "doc_open", json!({ "path": "table.pdf" }))["doc"].as_u64().unwrap(); + ok(&mut a, "doc_export_office", json!({ "doc": doc, "path": "table.docx" })); + let zip = std::fs::read(dir.join("table.docx")).unwrap(); + let mut at = 0; + let mut xml = String::new(); + while zip[at..].starts_with(b"PK\x03\x04") { + let size = u32::from_le_bytes(zip[at + 18..at + 22].try_into().unwrap()) as usize; + let len = u16::from_le_bytes(zip[at + 26..at + 28].try_into().unwrap()) as usize; + let data = at + 30 + len; + if &zip[at + 30..data] == b"word/document.xml" { + std::io::Read::read_to_string(&mut flate2::read::DeflateDecoder::new(&zip[data..data + size]), &mut xml).unwrap(); + break; + } + at = data + size; + } + assert_eq!(xml.matches("").count(), 1, "the ruled grid is a table: {xml}"); + let table = xml.split_once("").and_then(|(_, t)| t.split_once("")).map(|(t, _)| t).unwrap(); + assert_eq!(table.matches("").count(), 2, "{table}"); + for word in ["Name", "Value", "Alpha", "One"] { + assert_eq!(table.matches(&format!(">{word}")).count(), 1, "{word} is a cell: {xml}"); + assert_eq!(xml.matches(&format!(">{word}")).count(), 1, "{word} is written once"); + } + assert!(table.contains("w:val=\"single\""), "the drawn rules become borders: {table}"); + assert!(!table.contains("A paragraph after the table.")); + assert_eq!(xml.matches(">A paragraph after the table.").count(), 1); +} + /// #4158: a word set in italic inside a sentence ("Before *Chevron* after.") reaches Word as an /// italic run of the sentence's paragraph, not as a paragraph of its own between two halves. #[test] diff --git a/crates/automation/tests/it/batch.rs b/crates/automation/tests/it/batch.rs index af5e0c1272..8b7650fd3f 100644 --- a/crates/automation/tests/it/batch.rs +++ b/crates/automation/tests/it/batch.rs @@ -238,3 +238,61 @@ fn mcp_batch_errors_keep_the_structured_per_file_report() { assert_eq!(report["result"]["isError"], true, "{report}"); assert_eq!(report["result"]["structuredContent"]["failed"], 1); } + +#[test] +fn dry_run_planned_files_are_not_counted_as_skipped() { + let dir = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(dir.path().join("input/sub")).unwrap(); + std::fs::create_dir_all(dir.path().join("out")).unwrap(); + let original = pdf(); + std::fs::write(dir.path().join("input/a.pdf"), &original).unwrap(); + std::fs::write(dir.path().join("input/sub/a.pdf"), &original).unwrap(); + std::fs::write(dir.path().join("out/a.pdf"), b"precious").unwrap(); + let mut auto = Automation::new().with_root(dir.path()).unwrap(); + let report = result(&mut auto, json!({"paths":["input","input/sub"],"steps":["optimize"],"out_dir":"out","dry_run":true})); + assert_eq!(report["dry_run"], true, "{report}"); + assert_eq!(report["completed"], 0, "{report}"); + assert_eq!(report["failed"], 0, "{report}"); + assert_eq!(report["skipped"], 0, "planned files are not skipped: {report}"); + let files = report["files"].as_array().unwrap(); + assert_eq!(files.len(), 2, "{report}"); + assert!(files.iter().all(|file| file["status"] == "planned"), "{report}"); + assert!(files.iter().all(|file| file["steps"] == json!(["optimize"])), "planned files list the ordered steps: {report}"); + assert!(files[0]["output"].as_str().unwrap().ends_with("out/a (2).pdf"), "existing output gets the publish collision name: {report}"); + assert!(files[1]["output"].as_str().unwrap().ends_with("out/a (3).pdf"), "later planned files reserve their names: {report}"); + assert_eq!(std::fs::read(dir.path().join("out/a.pdf")).unwrap(), b"precious"); + assert!(!dir.path().join("out/a (2).pdf").exists(), "dry run writes nothing: {report}"); + #[cfg(unix)] + { + // An output subfolder that resolves outside out_dir is a per-file error: + // it counts as failed, honors stop_on_error, and later files are skipped. + for outside_root in [false, true] { + let dir = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(dir.path().join("input/sub")).unwrap(); + std::fs::create_dir_all(dir.path().join("out")).unwrap(); + std::fs::write(dir.path().join("input/a.pdf"), pdf()).unwrap(); + std::fs::write(dir.path().join("input/sub/page.pdf"), pdf()).unwrap(); + std::fs::write(dir.path().join("input/z.pdf"), pdf()).unwrap(); + let target = if outside_root { outside.path().to_path_buf() } else { dir.path().join("outside") }; + if !outside_root { + std::fs::create_dir(&target).unwrap(); + } + std::os::unix::fs::symlink(&target, dir.path().join("out/sub")).unwrap(); + let mut auto = Automation::new().with_root(dir.path()).unwrap(); + let report = result( + &mut auto, + json!({"paths":["input"],"steps":["optimize"],"out_dir":"out","recursive":true,"dry_run":true,"stop_on_error":true}), + ); + assert_eq!(report["failed"], 1, "preview target failure is per-file: {report}"); + assert_eq!(report["stopped"], true, "{report}"); + assert_eq!(report["skipped"], 1, "only the unprocessed file is skipped: {report}"); + let files = report["files"].as_array().unwrap(); + assert_eq!(files.len(), 3, "{report}"); + assert_eq!(files[0]["status"], "planned", "{report}"); + assert_eq!(files[1]["status"], "error", "{report}"); + assert_eq!(files[2]["status"], "skipped", "stop_on_error leaves later files unplanned: {report}"); + assert!(!dir.path().join("out/a.pdf").exists(), "dry run writes nothing: {report}"); + } + } +} diff --git a/crates/cos/src/budget.rs b/crates/cos/src/budget.rs index 65b750e7b3..1a99885944 100644 --- a/crates/cos/src/budget.rs +++ b/crates/cos/src/budget.rs @@ -11,7 +11,9 @@ use crate::{CosError, Document, ObjRef, Object, Stream}; /// 16 ns, so this is 2 to 5 minutes, and about as long again for what an operation does with the /// pixels. A page scanned at 300 dpi takes about 17 million, 8.4 for its pixels and that again to /// decode them, so this alone reads about 1,000 of them. A byte into or out of a filter is a step -/// too, and takes less: 3 ns at most, or up to 19 ns through a predictor on samples of a bit. +/// too, and takes less: 3 ns at most, or up to 19 ns through a predictor on samples of a bit. A +/// byte Flate reads is [`pdfcraft_filters::FLATE_READ_WORK`] steps besides, since inflating blocks +/// that make nothing takes up to 21 ns a byte: about 7 ns a step. const IMAGE_WORK: u64 = 1 << 34; /// The steps each byte of the file adds to [`IMAGE_WORK`], so that a long scan is read whole: such a /// page pays for itself with about 2 KB of the file, about the smallest a page of text is stored @@ -38,9 +40,9 @@ const JPEG_WORK: u64 = 2; /// and images can share one, so reading it again for each has to count. const JPEG_READ_WORK: u64 = 4; -/// A stream decode within a byte bound, with the bytes its filter stages decoded: the data or -/// the error, each with that count. -type Decoder = fn(&Stream, usize) -> Result<(Vec, usize), (FilterError, usize)>; +/// A stream decode within a byte bound, with the bytes its filter stages decoded: the data and +/// whether it stopped at damage, or the error, each with that count. +type Decoder = fn(&Stream, usize) -> Result<(Vec, bool, usize), (FilterError, usize)>; /// What one operation over a document's images may still spend decoding them: exporting them, /// optimizing them, and storing LZW streams with Flate as it does, clearing them for redaction. @@ -263,45 +265,57 @@ impl DecodeBudget { } /// `stream` decoded like [`Stream::decoded`], charged to the allowance. Fails, without - /// decoding, once the allowance is spent. + /// decoding, once the allowance is spent. A stream damaged partway gives what decoded before + /// the damage, so code that writes the stream back reads it with + /// [`DecodeBudget::decode_with_damage`] instead. pub fn decode(&mut self, stream: &Stream) -> Result, CosError> { - self.charge(stream, usize::MAX, Stream::decoded_within_counted) + self.charge(stream, usize::MAX, Stream::decoded_within_counted).map(|(data, _)| data) } /// Like [`DecodeBudget::decode`], for a stream with a tighter bound of its own: decoding /// past `max` bytes is an error that charges `max`, the work it took to find out. pub fn decode_within(&mut self, stream: &Stream, max: usize) -> Result, CosError> { - self.charge(stream, max, Stream::decoded_within_counted) + self.charge(stream, max, Stream::decoded_within_counted).map(|(data, _)| data) + } + + /// [`DecodeBudget::decode_within`] (`usize::MAX` for no bound of its own), with the stream's + /// `/DecodeParms` given by reference read in `doc`, also saying whether the stream is damaged: + /// `true` when a filter stopped at corrupt data, so the bytes are only what decoded before it. + /// A stream written back from them loses everything after the damage, which other readers may + /// still read. + pub fn decode_with_damage(&mut self, doc: &Document, stream: &Stream, max: usize) -> Result<(Vec, bool), CosError> { + self.charge(&doc.decoding_view(stream), max, Stream::decoded_within_counted) } /// `stream` decoded like [`Stream::decoded_strict`] (any corruption is an error), charged to - /// the allowance. - pub fn decode_strict(&mut self, stream: &Stream) -> Result, CosError> { - self.charge(stream, usize::MAX, Stream::decoded_strict_within_counted) + /// the allowance, with its `/DecodeParms` given by reference read in `doc`. + pub fn decode_strict(&mut self, doc: &Document, stream: &Stream) -> Result, CosError> { + let strict: Decoder = |s, max| s.decoded_strict_within_counted(max).map(|(data, spent)| (data, false, spent)); + self.charge(&doc.decoding_view(stream), usize::MAX, strict).map(|(data, _)| data) } /// Decodes `stream` within `max` and the allowance, charging what every filter stage /// decoded: a stage's output counts even when a later stage shrinks it, and a corrupt stream /// pays for the bytes decoded before the error, so referencing a stream again and again /// spends the allowance by the work each decode does. - fn charge(&mut self, stream: &Stream, max: usize, decode: Decoder) -> Result, CosError> { + fn charge(&mut self, stream: &Stream, max: usize, decode: Decoder) -> Result<(Vec, bool), CosError> { if self.bytes == 0 && !stream.raw.is_empty() { return Err(CosError::Filter(FilterError::LimitExceeded(self.limit).to_string())); } if max < self.bytes { let result = decode(stream, max); let spent = match &result { - Ok((_, spent)) => *spent, + Ok((_, _, spent)) => *spent, Err((FilterError::LimitExceeded(_), _)) => max, Err((_, spent)) => *spent, }; self.bytes = self.bytes.saturating_sub(spent); - return result.map(|(data, _)| data).map_err(|(e, _)| CosError::Filter(e.to_string())); + return result.map(|(data, damaged, _)| (data, damaged)).map_err(|(e, _)| CosError::Filter(e.to_string())); } match decode(stream, self.bytes) { - Ok((data, spent)) => { + Ok((data, damaged, spent)) => { self.bytes = self.bytes.saturating_sub(spent); - Ok(data) + Ok((data, damaged)) } Err((FilterError::LimitExceeded(_), _)) => { self.bytes = 0; @@ -332,17 +346,24 @@ impl DecodeBudget { self.seen.insert(r) } - /// Each stream `contents` names, decoded against the allowance, in order. `contents` is a - /// page's `/Contents`: one stream, or an array whose non-stream entries are left out. - /// A stream that fails to decode, or arrives after the allowance is spent, is an `Err` in its - /// place, so the caller chooses between skipping it and giving up. + /// Each stream `contents` names, decoded against the allowance, in order, its `/DecodeParms` + /// given by reference read in `doc`. `contents` is a page's `/Contents`: one stream, or an + /// array whose non-stream entries are left out. A stream that fails to decode, or arrives + /// after the allowance is spent, is an `Err` in its place, so the caller chooses between + /// skipping it and giving up. pub fn decode_contents(&mut self, doc: &Document, contents: &Object) -> Vec, CosError>> { + self.decode_contents_with_damage(doc, contents).into_iter().map(|read| read.map(|(data, _)| data)).collect() + } + + /// [`DecodeBudget::decode_contents`], each stream also saying whether it is damaged partway + /// ([`DecodeBudget::decode_with_damage`], which reads `/DecodeParms` given by reference). + pub fn decode_contents_with_damage(&mut self, doc: &Document, contents: &Object) -> Vec, bool), CosError>> { match &*doc.resolve(contents) { - Object::Stream(s) => vec![self.decode(s)], + Object::Stream(s) => vec![self.decode_with_damage(doc, s, usize::MAX)], Object::Array(list) => list .iter() .filter_map(|entry| match &*doc.resolve(entry) { - Object::Stream(s) => Some(self.decode(s)), + Object::Stream(s) => Some(self.decode_with_damage(doc, s, usize::MAX)), _ => None, }) .collect(), @@ -367,6 +388,11 @@ mod image_tests { Stream::from_raw(dict, pdfcraft_filters::encode_jbig2_generic(&[0xFF; 4], 16, 2).unwrap()) } + /// What a Flate stage pays to read `n` bytes, besides what it makes. + fn read(n: usize) -> u64 { + pdfcraft_filters::FLATE_READ_WORK * n as u64 + } + /// Each image spends what it took from one allowance, and those past it are refused and /// counted, whatever their filters. What can't be decoded here costs nothing, and fails /// without being one of these. @@ -413,7 +439,7 @@ mod image_tests { let mut dict = Dict::new(); dict.set(b"Filter".to_vec(), Object::Array(vec![Object::name("FlateDecode"), Object::name("FlateDecode")])); let bomb = Stream::from_raw(dict, pdfcraft_filters::encode_flate(&inner)); - let cost = (bomb.raw.len() + inner.len() + samples.len()) as u64; + let cost = (bomb.raw.len() + inner.len() + samples.len()) as u64 + read(bomb.raw.len()) + read(inner.len()); let mut budget = ImageBudget::with_work(2 * cost + 1); for left in [cost + 1, 1] { assert_eq!(budget.decode(&doc, &bomb, 1 << 20).unwrap(), samples); @@ -442,17 +468,17 @@ mod image_tests { let image = behind(pdfcraft_filters::encode_flate(&alone.raw)); let mut budget = ImageBudget::with_work(u64::MAX); assert_eq!(budget.decode(&doc, &image, 1 << 20).unwrap(), samples); - assert_eq!(u64::MAX - budget.work_left(), decoding + (image.raw.len() + alone.raw.len()) as u64); + assert_eq!(u64::MAX - budget.work_left(), decoding + (image.raw.len() + alone.raw.len()) as u64 + read(image.raw.len())); // Past this image's limit: an error, with what finding out took paid and the rest left. let bomb = behind(pdfcraft_filters::encode_flate(&vec![0; 1 << 16])); let mut budget = ImageBudget::with_work(u64::MAX); assert!(budget.decode(&doc, &bomb, 1000).is_err()); - assert_eq!(u64::MAX - budget.work_left(), bomb.raw.len() as u64 + 1000); + assert_eq!(u64::MAX - budget.work_left(), bomb.raw.len() as u64 + read(bomb.raw.len()) + 1000); assert_eq!(budget.refused(), 0, "that image's limit isn't the run's"); // A small allowance is a limit too: it stops there, and the image is refused. - let mut budget = ImageBudget::with_work(bomb.raw.len() as u64 + 10); + let mut budget = ImageBudget::with_work(bomb.raw.len() as u64 + read(bomb.raw.len()) + 10); assert!(budget.decode(&doc, &bomb, 1 << 20).is_err()); assert_eq!((budget.work_left(), budget.refused()), (0, 1)); } @@ -492,7 +518,7 @@ mod image_tests { let doc = Document::open(std::sync::Arc::new(file)).unwrap(); let image = doc.get(doc.root().unwrap()).as_dict().unwrap().reference(b"Image").unwrap(); let s = stream(&doc, image); - let filters = (raw.len() + skipped.len()) as u64; + let filters = (raw.len() + skipped.len()) as u64 + read(raw.len()); assert_eq!(spent(&doc, &s), first, "the first read undoes them"); assert_eq!(spent(&doc, &s), first - filters, "kept"); @@ -507,7 +533,7 @@ mod image_tests { let image = jbig2(Some(Object::Dict(parms))); let mut budget = ImageBudget::with_work(u64::MAX); assert!(budget.decode(&doc, &image, 1 << 20).unwrap_err().to_string().contains("globals")); - assert_eq!(u64::MAX - budget.work_left(), (over.len() + crate::MAX_JBIG2_GLOBALS) as u64); + assert_eq!(u64::MAX - budget.work_left(), (over.len() + crate::MAX_JBIG2_GLOBALS) as u64 + read(over.len())); assert_eq!(budget.refused(), 0); } @@ -528,7 +554,7 @@ mod image_tests { dict.set(b"DecodeParms".to_vec(), Object::Array(vec![Object::Null, Object::Dict(parms)])); let image = Stream::from_raw(dict, pdfcraft_filters::encode_flate(&coded)); let pixels = u64::from(w) * u64::from(h); - let cost = (image.raw.len() + coded.len() + 4 * coded.len() + 512 * h as usize) as u64 + pixels; + let cost = (image.raw.len() + coded.len() + 4 * coded.len() + 512 * h as usize) as u64 + read(image.raw.len()) + pixels; let mut budget = ImageBudget::with_work(cost); assert_eq!(budget.decode(&doc, &image, 1 << 20).unwrap().len(), 512 * h as usize); assert_eq!(budget.work_left(), 0, "exactly that"); @@ -553,7 +579,7 @@ mod image_tests { }; let rows = vec![0xFFu8; 512 * 1024]; let flate = blank(Some("FlateDecode"), pdfcraft_filters::encode_flate(&rows)); - let cost = (flate.raw.len() + rows.len()) as u64 + 4096 * 1024; + let cost = (flate.raw.len() + rows.len()) as u64 + read(flate.raw.len()) + 4096 * 1024; let mut budget = ImageBudget::with_work(cost); assert_eq!(budget.decode(&doc, &flate, 1 << 24).unwrap(), rows); assert_eq!((budget.work_left(), budget.refused()), (0, 0), "exactly that"); @@ -590,6 +616,36 @@ mod image_tests { assert_eq!((budget.work_left(), budget.refused()), (0, 1)); } + /// What Flate reads is priced for blocks that make nothing, and an image an encoder made pays + /// little for it: a scan-like picture stored in under half its size costs no more than half + /// again its bytes, samples and pixels. Redaction decodes a picture once for each page that draws it, so + /// a letterhead on hundreds of pages has to fit about as often as it did unpriced (here, at + /// most half again the cost). + #[test] + fn an_image_an_encoder_made_pays_little_for_its_reading() { + let doc = Document::new_empty(); + let mut seed = 0x2545_F491_4F6C_DD1Du64; + let samples: Vec = (0..256 * 256usize) + .map(|i| { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + ((i % 256) as u8 / 16) ^ (seed as u8 & 3) + }) + .collect(); + let mut dict = Dict::new(); + dict.set(b"Width".to_vec(), Object::Int(256)); + dict.set(b"Height".to_vec(), Object::Int(256)); + let image = Stream::flate(dict, &samples); + assert!(image.raw.len() * 2 < samples.len(), "stored smaller: {}", image.raw.len()); + let unpriced = (image.raw.len() + samples.len()) as u64 + 256 * 256; + let mut budget = ImageBudget::with_work(u64::MAX); + budget.decode(&doc, &image, 1 << 20).unwrap(); + let paid = u64::MAX - budget.work_left(); + assert_eq!(paid, unpriced + read(image.raw.len())); + assert!(2 * paid <= 3 * unpriced, "{paid} steps against {unpriced}"); + } + /// An image pays for the pixels its data holds, not those its size declares: one claiming /// 131072 by 131072 pixels with a few bytes of Flate costs what those bytes give, and the /// images after it are decoded as they would be without it. @@ -609,8 +665,8 @@ mod image_tests { budget.decode(&doc, s, 1 << 24).unwrap(); u64::MAX - budget.work_left() }; - assert_eq!(cost(&claims), (claims.raw.len() + 16 + 16 * 8) as u64, "its bytes, and a pixel a bit of them"); - assert_eq!(cost(&image), (image.raw.len() + 64 * 64) as u64 + 64 * 64, "its bytes, and its pixels"); + assert_eq!(cost(&claims), (claims.raw.len() + 16 + 16 * 8) as u64 + read(claims.raw.len()), "its bytes, and a pixel a bit of them"); + assert_eq!(cost(&image), (image.raw.len() + 64 * 64) as u64 + read(image.raw.len()) + 64 * 64, "its bytes, and its pixels"); for decode in [ImageBudget::decode, ImageBudget::decode_filters] { let mut budget = ImageBudget::with_work(cost(&claims) + cost(&image)); assert_eq!(decode(&mut budget, &doc, &claims, 1 << 24).unwrap(), [0; 16]); @@ -627,7 +683,7 @@ mod image_tests { let samples = vec![7u8; 1000]; let raw = pdfcraft_filters::encode_flate(&samples); let chain = [(Filter::Flate, Params::default())]; - let cost = (raw.len() + samples.len()) as u64; + let cost = (raw.len() + samples.len()) as u64 + read(raw.len()); let mut budget = ImageBudget::with_work(cost + 10); assert_eq!(budget.decode_chain(&chain, &raw, 1 << 20).unwrap(), samples); assert_eq!(budget.work_left(), 10); @@ -664,7 +720,7 @@ mod image_tests { assert_eq!(budget.decode_filters(&doc, &image, 1 << 20).unwrap(), rows); let cost = u64::MAX - budget.work_left(); // The tag byte of each of its 100 rows counts, which the predictor takes off. - assert_eq!(cost, (image.raw.len() + rows.len() + 100) as u64); + assert_eq!(cost, (image.raw.len() + rows.len() + 100) as u64 + read(image.raw.len())); let mut budget = ImageBudget::with_work(cost - 1); assert!(budget.decode_filters(&doc, &image, 1 << 20).is_err()); assert_eq!((budget.work_left(), budget.refused()), (0, 1)); @@ -769,7 +825,7 @@ mod tests { assert_eq!(budget.remaining(), 0); let mut budget = DecodeBudget::new((1 << 20) - 1, 1); let Object::Stream(stream) = &*doc.get(bomb) else { panic!("not a stream") }; - assert!(budget.decode_strict(stream).is_err()); + assert!(budget.decode_strict(&doc, stream).is_err()); assert_eq!(budget.remaining(), 0); let mut budget = DecodeBudget::new(10 << 20, 1); @@ -793,14 +849,111 @@ mod tests { let corrupt = Stream::from_raw(whole.dict.clone(), whole.raw.get(..cut).unwrap_or_default().to_vec()); let partial = corrupt.decoded().map(|d| d.len()).unwrap_or(0); assert!(partial > 0, "a cut flate stream decodes in part"); + let doc = Document::new_empty(); let mut budget = DecodeBudget::new(10 << 20, 1); - assert!(budget.decode_strict(&corrupt).is_err()); + assert!(budget.decode_strict(&doc, &corrupt).is_err()); assert_eq!(budget.remaining(), (10 << 20) - partial); let mut budget = DecodeBudget::new(10 << 20, 1); assert!(budget.decode_within(&corrupt, 1 << 30).is_ok()); assert_eq!(budget.remaining(), (10 << 20) - partial); } + /// A stream damaged partway decodes to what came before the damage, and says so, so that code + /// writing it back can leave it as it is. What every reader takes is no damage: a missing + /// checksum, no filter at all. + #[test] + fn a_stream_damaged_partway_says_so() { + let doc = Document::new_empty(); + let read = |budget: &mut DecodeBudget, s: &Stream, max| budget.decode_with_damage(&doc, s, max).map(|(d, damaged)| (d.len(), damaged)).ok(); + let whole = Stream::flate(Dict::new(), &vec![b' '; 1 << 20]); + let cut = Stream::from_raw(whole.dict.clone(), whole.raw.get(..whole.raw.len() / 2).unwrap_or_default().to_vec()); + let partial = cut.decoded().map(|d| d.len()).unwrap_or(0); + assert!(partial > 0, "a cut flate stream decodes in part"); + let mut budget = DecodeBudget::new(10 << 20, 1); + assert_eq!(read(&mut budget, &cut, usize::MAX), Some((partial, true))); + assert_eq!(budget.remaining(), (10 << 20) - partial); + // Within a bound of its own, under what is left. + assert_eq!(read(&mut budget, &cut, 4 << 20), Some((partial, true))); + assert_eq!(budget.decode(&cut).map(|d| d.len()).ok(), Some(partial)); + + let mut budget = DecodeBudget::new(10 << 20, 1); + assert_eq!(read(&mut budget, &whole, usize::MAX), Some((1 << 20, false))); + assert_eq!(read(&mut budget, &whole, 4 << 20), Some((1 << 20, false))); + let unchecked = Stream::from_raw(whole.dict.clone(), whole.raw.get(..whole.raw.len() - 4).unwrap_or_default().to_vec()); + assert_eq!(read(&mut budget, &unchecked, usize::MAX), Some((1 << 20, false))); + assert_eq!(read(&mut budget, &Stream::from_raw(Dict::new(), b"0 0 m".to_vec()), usize::MAX), Some((5, false))); + // A later stage's damage is the stream's. + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::Array(vec![Object::name("FlateDecode"), Object::name("ASCIIHexDecode")])); + let hex = Stream::from_raw(dict, Stream::flate(Dict::new(), b"4142zz4344>").raw.to_vec()); + assert_eq!(budget.decode_with_damage(&doc, &hex, usize::MAX).ok(), Some((b"AB".to_vec(), true))); + // So is an earlier stage's, though the stage after it reads all it was given: a stored + // block of whole hex, then a block of the reserved type (RFC 1951 §3.2.3). + let readable = b"4142>"; + let len = u16::try_from(readable.len()).unwrap_or_default(); + let mut raw = vec![0x78, 0x01, 0x00]; + raw.extend_from_slice(&len.to_le_bytes()); + raw.extend_from_slice(&(!len).to_le_bytes()); + raw.extend_from_slice(readable); + raw.push(0x07); + let flate_damaged = Stream::from_raw(hex.dict.clone(), raw); + assert_eq!(budget.decode_with_damage(&doc, &flate_damaged, usize::MAX).ok(), Some((b"AB".to_vec(), true))); + } + + /// `/DecodeParms` given by reference, or a parameter given by reference in them, is read: a + /// predicted stream decodes as with them direct. Of the stream, decoding copies no more than + /// its filters and their parameters. + #[test] + fn parameters_given_by_reference_are_read() { + let content = b"q 10 0 0 10 0 0 cm /Im Do Q"; + // PNG Up, one column: each byte a row of its own, tagged 2. + let mut rows = Vec::new(); + let mut above = 0u8; + for &b in content { + rows.extend([2, b.wrapping_sub(above)]); + above = b; + } + let raw = Stream::flate(Dict::new(), &rows).raw; + let mut doc = Document::new_empty(); + let direct: Dict = [(b"Predictor".to_vec(), Object::Int(12))].into_iter().collect(); + let by_reference = Object::Ref(doc.add(Object::Dict(direct.clone()))); + let predictor = doc.add(Object::Int(12)); + let inner = Object::Dict([(b"Predictor".to_vec(), Object::Ref(predictor))].into_iter().collect()); + for parms in [Object::Dict(direct), by_reference, inner] { + let mut dict = Dict::new(); + dict.set(b"Filter".to_vec(), Object::name("FlateDecode")); + dict.set(b"DecodeParms".to_vec(), parms.clone()); + dict.set(b"Other".to_vec(), Object::Array(vec![Object::Int(0); 1000])); + let s = Stream::from_raw(dict, raw.to_vec()); + assert_eq!(DecodeBudget::page().decode_with_damage(&doc, &s, usize::MAX).ok(), Some((content.to_vec(), false)), "{parms:?}"); + assert_eq!(DecodeBudget::page().decode_strict(&doc, &s).ok(), Some(content.to_vec()), "{parms:?}"); + let r = doc.add(Object::Stream(s.clone())); + let read = DecodeBudget::page().decode_contents_with_damage(&doc, &Object::Array(vec![Object::Ref(r)])); + assert_eq!(read, [Ok((content.to_vec(), false))], "{parms:?}"); + assert_eq!(DecodeBudget::page().decode_contents(&doc, &Object::Ref(r)), [Ok(content.to_vec())], "{parms:?}"); + let view = doc.decoding_view(&s); + assert!(matches!(view, std::borrow::Cow::Borrowed(_)) || view.dict.len() == 2, "{:?}", view.dict); + } + } + + /// A page's content streams each say whether they are damaged, in their places. + #[test] + fn a_pages_streams_each_say_whether_they_are_damaged() { + let mut doc = Document::new_empty(); + let whole = doc.add(Object::Stream(Stream::flate(Dict::new(), b"0 0 m 1 1 l S"))); + let Object::Stream(s) = &*doc.get(whole) else { panic!("not a stream") }; + let cut = Stream::from_raw(s.dict.clone(), s.raw.get(..s.raw.len() - 6).unwrap_or_default().to_vec()); + let cut = doc.add(Object::Stream(cut)); + let contents = Object::Array(vec![Object::Ref(whole), Object::Null, Object::Ref(cut)]); + let mut budget = DecodeBudget::page(); + let read: Vec<_> = budget.decode_contents_with_damage(&doc, &contents).into_iter().map(|r| r.map(|(_, damaged)| damaged).ok()).collect(); + assert_eq!(read, [Some(false), Some(true)]); + let one: Vec<_> = + budget.decode_contents_with_damage(&doc, &Object::Ref(cut)).into_iter().map(|r| r.map(|(_, damaged)| damaged).ok()).collect(); + assert_eq!(one, [Some(true)]); + assert_eq!(budget.decode_contents(&doc, &contents).iter().filter(|r| r.is_ok()).count(), 2, "read up to the damage"); + } + /// A stream that fails its reading as zlib, and then as raw deflate after decoding all of it, /// gives nothing back, and each reference still pays for what it decoded. #[test] @@ -832,7 +985,7 @@ mod tests { assert_eq!(budget.remaining(), (10 << 20) - charged); assert!(budget.decode_within(&chained, 2 << 20).is_ok()); assert_eq!(budget.remaining(), (10 << 20) - 2 * charged); - assert!(budget.decode_strict(&chained).is_ok()); + assert!(budget.decode_strict(&Document::new_empty(), &chained).is_ok()); assert_eq!(budget.remaining(), (10 << 20) - 3 * charged); } } diff --git a/crates/cos/src/document.rs b/crates/cos/src/document.rs index d36e952c9c..2bf4bce42b 100644 --- a/crates/cos/src/document.rs +++ b/crates/cos/src/document.rs @@ -24,11 +24,11 @@ pub(crate) use objects::Held; pub(crate) use reader::find; #[cfg(test)] use reader::rfind; -pub(crate) use state::ObjectReader; use state::{ObjStm, Slot, Unreadable}; +pub(crate) use state::{ObjectReader, held_bytes}; pub use state::{ReadState, SourceIdentity}; #[cfg(test)] -use state::{SECTION_READS, held_bytes, objstm_member}; +use state::{SECTION_READS, objstm_member}; use xref::XrefIndex; pub use xref::{Revision, XrefEntry}; diff --git a/crates/cos/src/document/objects.rs b/crates/cos/src/document/objects.rs index e916e9241b..5882a1c1ac 100644 --- a/crates/cos/src/document/objects.rs +++ b/crates/cos/src/document/objects.rs @@ -330,7 +330,29 @@ impl Document { /// ([`pdfcraft_filters::MAX_CHAIN`]). Only small values are copied, so a file can't have a /// large object copied once for each key and filter. pub fn with_direct_parms<'s>(&self, s: &'s Stream) -> Cow<'s, Stream> { - let Some(p) = s.dict.get(b"DecodeParms") else { return Cow::Borrowed(s) }; + let Some(made) = self.direct_parms(s) else { return Cow::Borrowed(s) }; + let mut s = s.clone(); + s.dict.set(b"DecodeParms".to_vec(), made); + Cow::Owned(s) + } + + /// [`Document::with_direct_parms`] for decoding alone: where that copies `s`, this makes a + /// stream of `s`'s data, shared, and a dictionary of its `/Filter` and the parameters made + /// direct, all its filters read. A stream named again and again, as a page's `/Contents` may + /// list one, isn't copied whole each time. + pub(crate) fn decoding_view<'s>(&self, s: &'s Stream) -> Cow<'s, Stream> { + let Some(made) = self.direct_parms(s) else { return Cow::Borrowed(s) }; + let mut dict = Dict::new(); + if let Some(f) = s.dict.get(b"Filter") { + dict.set(b"Filter".to_vec(), f.clone()); + } + dict.set(b"DecodeParms".to_vec(), made); + Cow::Owned(Stream { dict, raw: s.raw.clone() }) + } + + /// The `/DecodeParms` [`Document::with_direct_parms`] gives `s`, or `None` when it keeps `s`. + fn direct_parms(&self, s: &Stream) -> Option { + let p = s.dict.get(b"DecodeParms")?; let crypt = |f: &Object| matches!(f.as_name(), Some(b"Crypt")); let filters: Vec = match s.dict.get(b"Filter") { Some(Object::Array(a)) => a.iter().map(crypt).collect(), @@ -338,7 +360,7 @@ impl Document { None => Vec::new(), }; if filters.iter().filter(|c| !**c).count() > pdfcraft_filters::MAX_CHAIN { - return Cow::Borrowed(s); + return None; } let mut changed = matches!(p, Object::Ref(_)); // Its numbers and flags kept, resolved; anything else is no filter's parameters (null). @@ -365,12 +387,7 @@ impl Document { } _ => one(p, &mut changed), }; - if !changed { - return Cow::Borrowed(s); - } - let mut s = s.clone(); - s.dict.set(b"DecodeParms".to_vec(), made); - Cow::Owned(s) + changed.then_some(made) } /// `o`, or the object it refers to, without copying either. diff --git a/crates/cos/src/document/reader.rs b/crates/cos/src/document/reader.rs index 2797a04e9a..18f5ce848a 100644 --- a/crates/cos/src/document/reader.rs +++ b/crates/cos/src/document/reader.rs @@ -635,10 +635,20 @@ impl Document { // Offsets of objects that may be cross-reference streams: their dictionaries are the // trailers of files that have no `trailer` keyword (§7.5.8). let mut xref_streams = Vec::new(); - for (num, offset, generation, body) in object_headers(&data) { + let headers = object_headers(&data); + // Where `/XRef` occurs in the file, found in one pass. + let mut xref_marks = Vec::new(); + let mut mark_from = 0; + while let Some(p) = find(&data, b"/XRef", mark_from, data.len()) { + xref_marks.push(p); + mark_from = p + 1; + } + for &(num, offset, generation, body) in &headers { entries.insert(num, XrefEntry::InFile { offset, generation }); objstms.push(num); - if find(&data, b"/XRef", body, body.saturating_add(1024)).is_some() + // `/XRef` within the 1024 bytes after the header (the first one that is there). + let first = xref_marks.partition_point(|&p| p < body); + if xref_marks.get(first).is_some_and(|&p| p + 5 <= body.saturating_add(1024).min(data.len())) && let Ok(at) = usize::try_from(offset) { xref_streams.push(at); @@ -827,7 +837,33 @@ pub(crate) fn find(hay: &[u8], needle: &[u8], from: usize, to: usize) -> Option< if from >= to || needle.is_empty() || to - from < needle.len() { return None; } - hay[from..to].windows(needle.len()).position(|w| w == needle).map(|p| p + from) + let hay = &hay[from..to]; + // Look for the needle's rarest byte first, a plain byte scan, and compare the rest only there. + const COMMON: &[u8] = b" etaoinsrhldcumfpgwybvkxjqz"; + let (k, rare) = + needle.iter().copied().enumerate().max_by_key(|&(_, b)| COMMON.iter().position(|&c| c == b).unwrap_or(COMMON.len())).unwrap_or((0, 0)); + let mut i = k; + while let Some(p) = hay.get(i..).and_then(|rest| position_of(rest, rare)) { + let start = i + p - k; + if hay.get(start..start + needle.len()) == Some(needle) { + return Some(start + from); + } + i += p + 1; + } + None +} + +/// The first position of `byte` in `hay`. Whole 32-byte blocks are tested without branching, which +/// the compiler turns into vector compares; only a block that holds the byte is searched. +fn position_of(hay: &[u8], byte: u8) -> Option { + const BLOCK: usize = 32; + let (blocks, rest) = hay.as_chunks::(); + for (i, block) in blocks.iter().enumerate() { + if block.iter().fold(false, |any, &b| any | (b == byte)) { + return block.iter().position(|&b| b == byte).map(|p| i * BLOCK + p); + } + } + rest.iter().position(|&b| b == byte).map(|p| blocks.len() * BLOCK + p) } pub(super) fn rfind(hay: &[u8], needle: &[u8], from: usize) -> Option { diff --git a/crates/cos/src/document/state.rs b/crates/cos/src/document/state.rs index 2f35174824..af71f95359 100644 --- a/crates/cos/src/document/state.rs +++ b/crates/cos/src/document/state.rs @@ -335,7 +335,7 @@ const OWNED_BYTES_DEPTH: usize = 256; /// InDesign's `/PieceInfo` -- thousands of small numeric arrays, parsed into tens of thousands of /// values -- weighed nothing, so the caches kept a thousand of them: a 35 MB file held 1.5 GB /// (#5873). -pub(super) fn held_bytes(object: &Object, data: &Arc>, depth: usize) -> usize { +pub(crate) fn held_bytes(object: &Object, data: &Arc>, depth: usize) -> usize { if depth > OWNED_BYTES_DEPTH { return usize::MAX; } diff --git a/crates/cos/src/document/tests.rs b/crates/cos/src/document/tests.rs index 56edba4b91..6b67935e9c 100644 --- a/crates/cos/src/document/tests.rs +++ b/crates/cos/src/document/tests.rs @@ -2237,3 +2237,29 @@ fn a_hybrid_update_without_prev_is_rebuilt_too() { assert_eq!(page_tree_pages(&linked).len(), 3); assert!(!linked.rebuilt_unlinked_update() && linked.repair_log().is_empty(), "{:?}", linked.repair_log()); } + +#[test] +fn find_agrees_with_a_plain_window_search() { + let naive = |hay: &[u8], needle: &[u8], from: usize, to: usize| { + if from >= to || needle.is_empty() || to - from < needle.len() { + return None; + } + hay[from..to].windows(needle.len()).position(|w| w == needle).map(|p| p + from) + }; + let mut seed = 0x9E37_79B9_7F4A_7C15u64; + let mut next = move || { + seed ^= seed << 13; + seed ^= seed >> 7; + seed ^= seed << 17; + seed + }; + let alphabet = b"/XRef t\n<>0"; + for _ in 0..20_000 { + let hay: Vec = (0..next() % 200).map(|_| alphabet[(next() % alphabet.len() as u64) as usize]).collect(); + let needles: [&[u8]; 5] = [b"/XRef", b"trailer", b"R", b"ff", b"<<"]; + let needle = needles[(next() % 5) as usize]; + let from = (next() % (hay.len() as u64 + 1)) as usize; + let to = from + (next() % (hay.len() as u64 + 1 - from as u64)) as usize; + assert_eq!(find(&hay, needle, from, to), naive(&hay, needle, from, to), "{hay:?} {needle:?} {from} {to}"); + } +} diff --git a/crates/cos/src/object.rs b/crates/cos/src/object.rs index 5d437faa82..bcb188afb1 100644 --- a/crates/cos/src/object.rs +++ b/crates/cos/src/object.rs @@ -297,13 +297,18 @@ impl Stream { /// [`Stream::decoded_within`] for a caller charging an allowance: the filter error is kept /// typed, so a caller can tell a spent limit from corrupt data, `max` bounds all filter - /// stages together, and the result gives the bytes they decoded (the work done, on success - /// and on failure). - pub(crate) fn decoded_within_counted(&self, max: usize) -> Result<(Vec, usize), (pdfcraft_filters::FilterError, usize)> { + /// stages together, and the result gives whether the data stopped at damage (it is then what + /// decoded before it) and the bytes the stages decoded (the work done, on success and on + /// failure). + pub(crate) fn decoded_within_counted(&self, max: usize) -> Result<(Vec, bool, usize), (pdfcraft_filters::FilterError, usize)> { + let mut damaged = false; let decode = |chain: &[(Filter, Params)], data: &[u8], max| { - pdfcraft_filters::decode_tolerant_counted(chain, data, max).map(|(v, _, spent)| (v, spent)) + pdfcraft_filters::decode_tolerant_counted(chain, data, max).map(|(v, partial, spent)| { + damaged = partial; + (v, spent) + }) }; - self.decode_chain(max, decode) + self.decode_chain(max, decode).map(|(v, spent)| (v, damaged, spent)) } /// [`Stream::decoded_within`] spending from `work`, what an operation may still do across a diff --git a/crates/cos/src/writer.rs b/crates/cos/src/writer.rs index 0af74f7bab..75d49dbd76 100644 --- a/crates/cos/src/writer.rs +++ b/crates/cos/src/writer.rs @@ -15,7 +15,7 @@ use std::io::Write; use std::sync::Arc; use crate::CosError; -use crate::document::{Document, ObjectReader, XrefEntry}; +use crate::document::{Document, ObjectReader, XrefEntry, held_bytes}; use crate::object::{Dict, ObjRef, Object, PdfString, Stream}; /// Serialize one object. Deterministic: same object → same bytes. @@ -23,9 +23,7 @@ pub fn serialize(o: &Object, out: &mut Vec) { match o { Object::Null => out.extend_from_slice(b"null"), Object::Bool(b) => out.extend_from_slice(if *b { b"true" } else { b"false" }), - Object::Int(i) => { - let _ = write!(out, "{i}"); - } + Object::Int(i) => write_int(*i, out), Object::Real(r) => write_real(*r, out), Object::String(s) => write_string(s, out), Object::Name(n) => write_name(n, out), @@ -42,11 +40,46 @@ pub fn serialize(o: &Object, out: &mut Vec) { Object::Dict(d) => write_dict(d, out), Object::Stream(s) => write_stream(s, out), Object::Ref(r) => { - let _ = write!(out, "{} {} R", r.num, r.generation); + write_digits(u64::from(r.num), out); + out.push(b' '); + write_digits(u64::from(r.generation), out); + out.extend_from_slice(b" R"); } } } +/// Upper-case hexadecimal digits. +const HEX: &[u8; 16] = b"0123456789ABCDEF"; + +/// The decimal digits of `v`, the same bytes `write!(out, "{v}")` gives, without the formatter. +fn write_digits(mut v: u64, out: &mut Vec) { + // `u64::MAX` has 20 digits, so `at` never goes below 0. + let mut buf = [0u8; 20]; + let mut at = buf.len(); + loop { + at -= 1; + buf[at] = b'0' + (v % 10) as u8; + v /= 10; + if v == 0 { + break; + } + } + out.extend_from_slice(&buf[at..]); +} + +/// A signed integer in decimal, as `write!(out, "{i}")` writes it. +fn write_int(i: i64, out: &mut Vec) { + if i < 0 { + out.push(b'-'); + } + write_digits(i.unsigned_abs(), out); +} + +/// `b` as two upper-case hexadecimal digits. +fn hex_pair(b: u8) -> [u8; 2] { + [HEX[usize::from(b >> 4)], HEX[usize::from(b & 15)]] +} + fn write_real(r: f64, out: &mut Vec) { if !r.is_finite() { out.push(b'0'); @@ -55,7 +88,7 @@ fn write_real(r: f64, out: &mut Vec) { // A whole number is written as an integer only within the integer range Annex C gives // (±2³¹−1), so `4e9` or `1e30` stays a real token for readers with 32-bit integers. if r.fract() == 0.0 && r.abs() <= f64::from(i32::MAX) { - let _ = write!(out, "{}", r as i32); + write_int(i64::from(r as i32), out); return; } // Plain decimal notation (§7.3.3 has no exponents). Rust's `Display` for f64 prints the @@ -76,7 +109,8 @@ fn write_name(n: &[u8], out: &mut Vec) { if regular { out.push(b); } else { - let _ = write!(out, "#{b:02X}"); + out.push(b'#'); + out.extend_from_slice(&hex_pair(b)); } } } @@ -84,9 +118,10 @@ fn write_name(n: &[u8], out: &mut Vec) { fn write_string(s: &PdfString, out: &mut Vec) { let binary = s.bytes.iter().filter(|b| !(b' '..=b'~').contains(*b) && !matches!(b, b'\n' | b'\r' | b'\t')).count(); if s.hex || binary * 4 > s.bytes.len().max(1) { + out.reserve(s.bytes.len().saturating_mul(2).saturating_add(2)); out.push(b'<'); - for b in &s.bytes { - let _ = write!(out, "{b:02X}"); + for &b in &s.bytes { + out.extend_from_slice(&hex_pair(b)); } out.push(b'>'); return; @@ -105,7 +140,7 @@ fn write_string(s: &PdfString, out: &mut Vec) { 0x0C => out.extend_from_slice(b"\\f"), b' '..=b'~' => out.push(b), _ => { - let _ = write!(out, "\\{b:03o}"); + out.extend_from_slice(&[b'\\', b'0' + (b >> 6), b'0' + ((b >> 3) & 7), b'0' + (b & 7)]); } } } @@ -275,14 +310,22 @@ pub fn write_incremental(doc: &Document, opts: &SaveOptions) -> Result, /// The objects [`write_full`] writes: everything reachable from the trailer's /Root, /Info and /// /Encrypt and from its unknown entries, breadth-first, with the new number each gets -/// (aliases of one slot included) and, in visit order, whether it is a stream. -type SaveOrder = (HashMap, Vec<(ObjRef, bool)>); +/// (aliases of one slot included) and, in visit order, whether it is a stream and the parsed +/// object where the budget allowed keeping it. +type SaveOrder = (HashMap, Vec<(ObjRef, bool)>, Vec>>); -fn full_save_order(reader: &mut ObjectReader, trailer_in: &Dict) -> Result { +/// Number the objects reachable from the trailer. `retain` keeps parsed objects for the write +/// pass, up to the budget. A caller that needs only the numbers passes `false`, so the traversal +/// holds no parsed objects beyond the current one. +fn full_save_order(reader: &mut ObjectReader, trailer_in: &Dict, retain: bool) -> Result { // Breadth-first walk from the trailer, assigning new numbers in visit order. let mut map: HashMap = HashMap::new(); // Record the type while traversing so classifying packed objects needs no second parse. let mut order: Vec<(ObjRef, bool)> = Vec::new(); + // The parsed object of each entry in `order`, kept for the write pass while the budget + // allows it. Past the budget the write pass reads the object again. + let mut kept: Vec>> = Vec::new(); + let mut kept_budget = KeptObjects::default(); let mut queue: VecDeque = VecDeque::new(); let visit_refs = |o: &Object, queue: &mut VecDeque| collect_refs(o, &mut |r| queue.push_back(r)); for key in [&b"Root"[..], b"Info", b"Encrypt"] { @@ -321,8 +364,9 @@ fn full_save_order(reader: &mut ObjectReader, trailer_in: &Dict) -> Result Result Result, CosError> { let mut reader = doc.object_reader(); let trailer_in = reader.document.trailer().clone(); - let (_, order) = full_save_order(&mut reader, &trailer_in)?; + let (_, order, _) = full_save_order(&mut reader, &trailer_in, false)?; Ok(order.into_iter().map(|(r, _)| r.num).collect()) } @@ -345,7 +389,7 @@ pub fn write_full(doc: &Document, opts: &SaveOptions) -> Result, CosErro let mut reader = doc.object_reader(); stamp_mod_date(&mut reader.document, opts); let trailer_in = reader.document.trailer().clone(); - let (map, order) = full_save_order(&mut reader, &trailer_in)?; + let (map, order, mut kept) = full_save_order(&mut reader, &trailer_in, true)?; let version = reader.document.version().chars().take(3).collect::(); let version = if opts.object_streams && version.as_str() < "1.5" { "1.5".to_string() } else { version }; let mut out = Vec::new(); @@ -375,14 +419,14 @@ pub fn write_full(doc: &Document, opts: &SaveOptions) -> Result, CosErro let mut rows: BTreeMap = BTreeMap::new(); // Keep only references, not renamed copies of the entire object graph. Standalone // objects still precede object streams, preserving the established output order. - let mut packed: Vec<(u32, ObjRef)> = Vec::new(); + let mut packed: Vec<(u32, ObjRef, Option>)> = Vec::new(); for (i, (r, is_stream)) in order.iter().enumerate() { let num = output_number(i)?; let in_stream = opts.object_streams && !is_stream && !encryption_objects.contains(&num) && root_outside != Some(num); if in_stream { - packed.push((num, *r)); + packed.push((num, *r, kept[i].take())); } else { - let o = reread(&mut reader, *r, *is_stream)?; + let o = written_object(&mut reader, *r, *is_stream, kept[i].take())?; let o = renumber(&prepared(&reader.document, r.num, num, 0, &o), &map); rows.insert(num, Row::InFile(out.len() as u64, 0)); write_indirect(num, 0, &o, &mut out); @@ -392,9 +436,9 @@ pub fn write_full(doc: &Document, opts: &SaveOptions) -> Result, CosErro // individually: the whole stream is (§7.5.7, §7.6.2). let mut next = output_number(order.len())?; let mut batch = ObjectStreamBatch::default(); - for (num, reference) in packed { + for (num, reference, parsed) in packed { let mut encoded = Vec::new(); - serialize(&renumber(&*reread(&mut reader, reference, false)?, &map), &mut encoded); + serialize(&renumber(&*written_object(&mut reader, reference, false, parsed)?, &map), &mut encoded); if batch.count > 0 && batch.body.len().saturating_add(encoded.len()).saturating_add(1) > OBJSTM_BYTES { batch.flush(&reader.document, &mut next, &mut rows, &mut out)?; } @@ -428,6 +472,38 @@ pub fn write_full(doc: &Document, opts: &SaveOptions) -> Result, CosErro Ok(out) } +/// What `write_full` keeps of the objects it parsed while numbering them, measured with +/// [`held_bytes`] (the parsed tree and any data that is not a view into the file). Beyond the +/// budget objects are read again when written, which bounds the memory of a save of a huge +/// document. +#[derive(Default)] +struct KeptObjects { + bytes: usize, +} + +impl KeptObjects { + const BYTES: usize = 64 * 1024 * 1024; + + /// Whether `object` still fits the budget; if so it is counted against it. + fn admit(&mut self, object: &Object, data: &Arc>) -> bool { + let bytes = self.bytes.saturating_add(held_bytes(object, data, 0)); + if bytes > Self::BYTES { + return false; + } + self.bytes = bytes; + true + } +} + +/// The object to write for `r`: the one parsed while numbering when it was kept, else `r` read +/// again. +fn written_object(reader: &mut ObjectReader, r: ObjRef, is_stream: bool, kept: Option>) -> Result, CosError> { + match kept { + Some(o) => Ok(o), + None => reread(reader, r, is_stream), + } +} + /// Assemble the full rewrite's trailer and apply its identifier policy. fn full_save_trailer(doc: &Document, trailer_in: &Dict, map: &HashMap, opts: &SaveOptions, out: &[u8]) -> Dict { let mut trailer = Dict::new(); @@ -935,6 +1011,38 @@ mod tests { Lexer::new(&out, 0).object().expect("parses") } + #[test] + fn kept_objects_stop_at_their_byte_budget() { + let data = Arc::new(Vec::new()); + let string = Object::String(PdfString::literal(vec![b'x'; 1000])); + let array = Object::Array(vec![Object::Int(1), Object::Null]); + let mut kept = KeptObjects::default(); + assert!(kept.admit(&string, &data) && kept.admit(&array, &data)); + assert_eq!(kept.bytes, held_bytes(&string, &data, 0) + held_bytes(&array, &data, 0)); + let mut full = KeptObjects { bytes: KeptObjects::BYTES - held_bytes(&string, &data, 0) + 1 }; + let before = full.bytes; + assert!(!full.admit(&string, &data), "one byte over the budget"); + assert_eq!(full.bytes, before, "a refused object is not counted"); + let mut deep = Object::Null; + for _ in 0..300 { + deep = Object::Array(vec![deep]); + } + assert!(!KeptObjects::default().admit(&deep, &data), "nesting past the limit is never kept"); + } + + #[test] + fn kept_objects_count_names_and_dictionary_keys() { + let data = Arc::new(Vec::new()); + let name = Object::Name(vec![b'n'; KeptObjects::BYTES + 1]); + let mut kept = KeptObjects::default(); + assert!(!kept.admit(&name, &data), "a name over the byte budget is not kept"); + assert_eq!(kept.bytes, 0); + let mut dict = Dict::new(); + dict.set(vec![b'k'; KeptObjects::BYTES + 1], Object::Null); + assert!(!kept.admit(&Object::Dict(dict), &data), "nor a dictionary key"); + assert_eq!(kept.bytes, 0); + } + #[test] fn a_second_read_that_differs_fails_the_save_instead_of_writing_null() { let mut doc = Document::new_empty(); diff --git a/crates/cos/tests/it/main.rs b/crates/cos/tests/it/main.rs index 1bd7ee1fd6..f8e75ba804 100644 --- a/crates/cos/tests/it/main.rs +++ b/crates/cos/tests/it/main.rs @@ -5,5 +5,6 @@ mod corpus; mod encryption; mod generations; mod proptests; +mod serialize_format; mod shared_streams; mod writer; diff --git a/crates/cos/tests/it/serialize_format.rs b/crates/cos/tests/it/serialize_format.rs new file mode 100644 index 0000000000..8cfba2baf1 --- /dev/null +++ b/crates/cos/tests/it/serialize_format.rs @@ -0,0 +1,116 @@ +//! `serialize` writes integers, references, hexadecimal strings, octal escapes and `#xx` name +//! escapes with its own digit loops. They must give the bytes the standard formatter gives. + +use pdfcraft_cos::{ObjRef, Object, PdfString, serialize}; +use proptest::prelude::*; + +fn bytes_of(o: &Object) -> Vec { + let mut v = Vec::new(); + serialize(o, &mut v); + v +} + +#[test] +fn integer_extremes_and_digit_boundaries() { + let mut cases = vec![0i64, 1, -1, 9, 10, 99, 100, i64::MAX, i64::MIN, i64::MIN + 1, i64::from(i32::MAX), i64::from(i32::MIN)]; + let mut p = 1i64; + while let Some(next) = p.checked_mul(10) { + cases.extend([next - 1, next, next + 1, -next]); + p = next; + } + for i in cases { + assert_eq!(bytes_of(&Object::Int(i)), i.to_string().into_bytes(), "{i}"); + } +} + +#[test] +fn reference_extremes() { + for (n, g) in [(0, 0), (1, 0), (u32::MAX, u16::MAX), (10, 65535), (4_000_000_000, 7)] { + assert_eq!(bytes_of(&Object::Ref(ObjRef::new(n, g))), format!("{n} {g} R").into_bytes()); + } +} + +#[test] +fn every_byte_value_in_hex_strings_octal_escapes_and_names() { + let all: Vec = (0..=255).collect(); + let hex: String = all.iter().map(|b| format!("{b:02X}")).collect(); + assert_eq!(bytes_of(&Object::String(PdfString { bytes: all.clone(), hex: true })), format!("<{hex}>").into_bytes()); + // A literal string with a few control bytes escapes them in octal. + let lit = bytes_of(&Object::String(PdfString { bytes: b"abcdefghijkl\x01\x7f\xffz".to_vec(), hex: false })); + assert_eq!(lit, b"(abcdefghijkl\\001\\177\\377z)"); + let name = bytes_of(&Object::Name(all.clone())); + let mut want = b"/".to_vec(); + for &b in &all { + let regular = (b'!'..=b'~').contains(&b) && !matches!(b, b'(' | b')' | b'<' | b'>' | b'[' | b']' | b'{' | b'}' | b'/' | b'%' | b'#'); + if regular { + want.push(b); + } else { + want.extend_from_slice(format!("#{b:02X}").as_bytes()); + } + } + assert_eq!(name, want); +} + +#[test] +fn whole_real_at_i32_min_stays_a_real_token() { + // -2^31 is outside the Annex C integer range (±2^31-1), so it keeps its trailing point. + assert_eq!(bytes_of(&Object::Real(f64::from(i32::MIN))), b"-2147483648."); +} + +proptest! { + #[test] + fn integers_match_the_formatter(i in any::()) { + prop_assert_eq!(bytes_of(&Object::Int(i)), format!("{i}").into_bytes()); + } + + #[test] + fn whole_reals_match_the_formatter(i in -i32::MAX..=i32::MAX) { + prop_assert_eq!(bytes_of(&Object::Real(f64::from(i))), format!("{i}").into_bytes()); + } + + #[test] + fn references_match_the_formatter(n in any::(), g in any::()) { + prop_assert_eq!(bytes_of(&Object::Ref(ObjRef::new(n, g))), format!("{n} {g} R").into_bytes()); + } + + #[test] + fn hex_strings_match_the_formatter(bytes in proptest::collection::vec(any::(), 0..300)) { + let mut want = String::from("<"); + for b in &bytes { + want.push_str(&format!("{b:02X}")); + } + want.push('>'); + prop_assert_eq!(bytes_of(&Object::String(PdfString { bytes, hex: true })), want.into_bytes()); + } + + #[test] + fn binary_literal_strings_match_the_formatter(bytes in proptest::collection::vec(any::(), 0..300)) { + // Mostly-binary input is written as hex; mostly-text input with escapes is a literal. + let got = bytes_of(&Object::String(PdfString { bytes: bytes.clone(), hex: false })); + let binary = bytes.iter().filter(|b| !(b' '..=b'~').contains(*b) && !matches!(b, b'\n' | b'\r' | b'\t')).count(); + let mut want = Vec::new(); + if binary * 4 > bytes.len().max(1) { + want.push(b'<'); + for b in &bytes { + want.extend_from_slice(format!("{b:02X}").as_bytes()); + } + want.push(b'>'); + } else { + want.push(b'('); + for &b in &bytes { + match b { + b'(' | b')' | b'\\' => want.extend_from_slice(&[b'\\', b]), + b'\n' => want.extend_from_slice(b"\\n"), + b'\r' => want.extend_from_slice(b"\\r"), + b'\t' => want.extend_from_slice(b"\\t"), + 0x08 => want.extend_from_slice(b"\\b"), + 0x0C => want.extend_from_slice(b"\\f"), + b' '..=b'~' => want.push(b), + _ => want.extend_from_slice(format!("\\{b:03o}").as_bytes()), + } + } + want.push(b')'); + } + prop_assert_eq!(got, want); + } +} diff --git a/crates/create/src/extract.rs b/crates/create/src/extract.rs index d5103071e6..863ad2da0a 100644 --- a/crates/create/src/extract.rs +++ b/crates/create/src/extract.rs @@ -183,12 +183,17 @@ fn image(doc: &Document, s: &Stream, budget: &mut ImageBudget) -> Result<(&'stat .unwrap_or(false); // In a PDF a CMYK JPEG's samples are ink amounts unless /Decode inverts them, while image // viewers read a CMYK JPEG file as inverted (Adobe's convention): as is, it would show as - // a negative. - let as_is_negative = jpeg && !inverted && matches!(space(doc, s.dict.get(b"ColorSpace")), Ok(Space::Cmyk)); + // a negative. The space is read once: an indexed one's lookup table is paid for each time. + let read = (jpeg && !inverted).then(|| space(doc, s.dict.get(b"ColorSpace"), budget)); + let as_is_negative = matches!(read, Some(Ok(Space::Cmyk))); if jpeg && alpha.is_none() && key.is_none() && !as_is_negative { return as_is(); } - let mut space = if mask { Space::Gray } else { space(doc, s.dict.get(b"ColorSpace"))? }; + let mut space = match read { + _ if mask => Space::Gray, + Some(read) => read?, + None => space(doc, s.dict.get(b"ColorSpace"), budget)?, + }; let (samples, bpc) = if jpeg { if !budget.pay_jpeg((w as u64).saturating_mul(h as u64).saturating_mul(space.components() as u64), &s.raw) { return Err(FilterError::Spent.to_string()); @@ -436,7 +441,10 @@ impl Space { /// A lookup table holds at most 256 entries of 4 bytes; decoding stops well past that. const MAX_LOOKUP: usize = 64 << 10; -fn space(doc: &Document, cs: Option<&Object>) -> Result { +/// The colour space `cs` names; an indexed one's lookup table is read within `budget`, as the +/// image's samples are, since each image reads it again and its filters can do work that makes +/// nothing. +fn space(doc: &Document, cs: Option<&Object>, budget: &mut ImageBudget) -> Result { let Some(cs) = cs else { return Err("the image has no colour space".into()) }; let cs = doc.resolve(cs); let unsupported = |n: &[u8]| Err(format!("{} images can't be exported yet", String::from_utf8_lossy(n))); @@ -470,7 +478,7 @@ fn space(doc: &Document, cs: Option<&Object>) -> Result { if nested { return Err("nested indexed colour spaces aren't valid".into()); } - let base = space(doc, a.get(1))?; + let base = space(doc, a.get(1), budget)?; let hival = match a.get(2).map(|value| doc.resolve(value)).as_deref() { Some(Object::Int(value)) if (0..=255).contains(value) => *value as u8, _ => return Err("an indexed colour space with an invalid hival".into()), @@ -478,10 +486,14 @@ fn space(doc: &Document, cs: Option<&Object>) -> Result { let table = match a.get(3).map(|t| doc.resolve(t)).as_deref() { Some(Object::String(s)) => s.bytes.clone(), // Through its parameters given by reference too, as the image's samples. - Some(Object::Stream(t)) => doc - .with_direct_parms(t) - .decoded_within(MAX_LOOKUP) - .map_err(|_| "an indexed colour space with an oversized or damaged lookup table".to_string())?, + Some(Object::Stream(t)) => { + let refused = budget.refused(); + match budget.decode_filters(doc, t, MAX_LOOKUP) { + Ok(table) => table, + Err(_) if budget.refused() > refused => return Err(FilterError::Spent.to_string()), + Err(_) => return Err("an indexed colour space with an oversized or damaged lookup table".into()), + } + } _ => return Err("an indexed colour space without a lookup table".into()), }; Ok(Space::Indexed(Box::new(base), hival, table)) diff --git a/crates/create/src/tests.rs b/crates/create/src/tests.rs index bea96c887d..88a6e05ac9 100644 --- a/crates/create/src/tests.rs +++ b/crates/create/src/tests.rs @@ -2409,6 +2409,98 @@ fn flate_images_past_the_exports_allowance_are_left_out_with_the_reason() { assert_eq!(budget.refused(), 1); } +/// An indexed image's lookup table, given as a stream, is read within the export's allowance as +/// the image's samples are: each image reads it again, and its filters can do work that makes +/// nothing (Flate over empty blocks). Past the allowance the image is left out with the reason. +#[test] +fn a_lookup_table_is_read_within_the_exports_allowance() { + use pdfcraft_cos::{Dict, Document, ImageBudget, Object, PdfString, Stream}; + let table = [255, 0, 0, 0, 0, 255]; + let lookup = Stream::flate(Dict::new(), &table); + let exported = |as_stream: bool, work: u64| { + let mut doc = Document::new_empty(); + let table = if as_stream { Object::Ref(doc.add(Object::Stream(lookup.clone()))) } else { Object::String(PdfString::literal(table.to_vec())) }; + let mut dict = Dict::new(); + dict.set(b"Subtype".to_vec(), Object::name("Image")); + dict.set(b"Width".to_vec(), Object::Int(2)); + dict.set(b"Height".to_vec(), Object::Int(1)); + dict.set(b"BitsPerComponent".to_vec(), Object::Int(8)); + dict.set(b"ColorSpace".to_vec(), Object::Array(vec![Object::name("Indexed"), Object::name("DeviceRGB"), Object::Int(1), table])); + let image = doc.add(Object::Stream(Stream::flate(dict, &[0, 1]))); + let mut budget = ImageBudget::with_work(work); + let file = crate::extract::image_file_within(&doc, image, &mut budget); + (file, work - budget.work_left(), budget.refused()) + }; + let (direct, image_alone, _) = exported(false, u64::MAX); + let (streamed, both, _) = exported(true, u64::MAX); + assert_eq!(streamed.unwrap(), direct.unwrap(), "the same picture"); + let read = (lookup.raw.len() + table.len()) as u64 + pdfcraft_filters::FLATE_READ_WORK * lookup.raw.len() as u64; + assert_eq!(both, image_alone + read, "the table is paid for as it is read"); + let (refused, _, count) = exported(true, image_alone); + assert!(refused.unwrap_err().contains("more image decoding than")); + assert_eq!(count, 1); +} + +/// A JPEG's colour space is read once for an export, its lookup table paid for once: whether it is +/// CMYK (a negative as is) and what its samples are both come from that reading. An indexed JPEG +/// isn't valid, and is written as it is, but its soft mask makes it go on past the first. With the +/// mask, a table the allowance can't pay for leaves the image out, with the reason. +#[test] +fn a_jpegs_lookup_table_is_paid_for_once() { + use pdfcraft_cos::{Dict, Document, ImageBudget, Object, PdfString, Stream}; + let table = [255, 0, 0, 0, 0, 255]; + let lookup = Stream::flate(Dict::new(), &table); + let mut jpeg = Vec::new(); + image::GrayImage::from_pixel(2, 1, image::Luma([1])).write_to(&mut std::io::Cursor::new(&mut jpeg), image::ImageFormat::Jpeg).unwrap(); + // `given`: None leaves the colour space out, Some(true) gives the table as a stream. + let exported = |given: Option, work: u64| { + let mut doc = Document::new_empty(); + let table = given.map(|as_stream| { + if as_stream { Object::Ref(doc.add(Object::Stream(lookup.clone()))) } else { Object::String(PdfString::literal(table.to_vec())) } + }); + let mut mask = Dict::new(); + for (k, v) in + [("Subtype", Object::name("Image")), ("Width", Object::Int(2)), ("Height", Object::Int(1)), ("BitsPerComponent", Object::Int(8))] + { + mask.set(k.as_bytes().to_vec(), v); + } + mask.set(b"ColorSpace".to_vec(), Object::name("DeviceGray")); + let mask = doc.add(Object::Stream(Stream::flate(mask, &[255, 128]))); + let mut dict = Dict::new(); + for (k, v) in + [("Subtype", Object::name("Image")), ("Width", Object::Int(2)), ("Height", Object::Int(1)), ("BitsPerComponent", Object::Int(8))] + { + dict.set(k.as_bytes().to_vec(), v); + } + if let Some(table) = table { + dict.set(b"ColorSpace".to_vec(), Object::Array(vec![Object::name("Indexed"), Object::name("DeviceRGB"), Object::Int(1), table])); + } + dict.set(b"Filter".to_vec(), Object::name("DCTDecode")); + dict.set(b"SMask".to_vec(), Object::Ref(mask)); + let image = doc.add(Object::Stream(Stream::from_raw(dict, jpeg.clone()))); + let mut budget = ImageBudget::with_work(work); + let file = crate::extract::image_file_within(&doc, image, &mut budget); + (file, work - budget.work_left(), budget.refused()) + }; + let (direct, direct_cost, _) = exported(Some(false), u64::MAX); + let (streamed, streamed_cost, _) = exported(Some(true), u64::MAX); + assert_eq!(direct, Ok(("jpg", jpeg.clone())), "written as it is"); + assert_eq!(streamed, direct); + let read = (lookup.raw.len() + table.len()) as u64 + pdfcraft_filters::FLATE_READ_WORK * lookup.raw.len() as u64; + assert_eq!(streamed_cost, direct_cost + read, "one reading of the table"); + + // Without a colour space nothing is read after the soft mask, so this is what the mask costs. + let (none, mask, _) = exported(None, u64::MAX); + assert_eq!(none.unwrap_err(), "the image has no colour space"); + // Read first, the table would be refused a step short of the mask too, and the mask after it. + assert!(read >= mask, "the table costs at least what the mask does"); + assert_eq!(exported(Some(true), mask - 1).2, 1, "the mask is read first, and refused alone"); + let (refused, _, count) = exported(Some(true), mask + read - 1); + let refused = refused.unwrap_err(); + assert!(refused.contains("more image decoding than"), "{refused}"); + assert_eq!(count, 1, "refused at the table, and nothing after it asks to be paid for"); +} + /// A JPEG is written as it is, at no cost, unless something means decoding it, here a mask: that /// pays for its samples, and past the allowance the image goes, with the reason, as it would come out without /// its transparency otherwise. diff --git a/crates/edit/src/backdrop.rs b/crates/edit/src/backdrop.rs index 43e16b40ec..f6788bb51c 100644 --- a/crates/edit/src/backdrop.rs +++ b/crates/edit/src/backdrop.rs @@ -208,7 +208,7 @@ pub(crate) fn restore(doc: &mut Document, page: &pdfcraft_model::Page) -> Result let same = |w: &ObjRef| match &*doc.get(*w) { Object::Stream(s) => { [TAG, &b"PCMark"[..], b"Subtype"].iter().all(|k| s.dict.get(k).is_none()) - && DecodeBudget::new(SCAN_BYTES, 0).decode_strict(s).ok().as_deref() == Some(bytes.as_slice()) + && DecodeBudget::new(SCAN_BYTES, 0).decode_strict(doc, s).ok().as_deref() == Some(bytes.as_slice()) } _ => false, }; @@ -248,7 +248,7 @@ fn restored_copy(doc: &Document, page: &pdfcraft_model::Page, copy: ObjRef) -> O return None; } let hash = note.get(b"Before")?.as_string()?; - let bytes = DecodeBudget::new(SCAN_BYTES, 0).decode_strict(s).ok()?; + let bytes = DecodeBudget::new(SCAN_BYTES, 0).decode_strict(doc, s).ok()?; let parsed = pdfcraft_content::parse(head(&bytes)); // The `n` standing in for the fill, after exactly the operators that came before it. let fill = parsed.ops.get(index)?; @@ -384,11 +384,11 @@ fn scan(doc: &Document, p: &pdfcraft_model::Page, swap: Option<(ObjRef, &[u8])>) if b.len() > SCAN_BYTES.saturating_sub(total) { return None; } - budget.decode_strict(&Stream::from_raw(Dict::new(), b.to_vec())).ok()? + budget.decode_strict(doc, &Stream::from_raw(Dict::new(), b.to_vec())).ok()? } // Tolerant decoding can return an empty or partial stream after corruption. // That cannot prove which paint preceded the candidate backdrop. - _ => budget.decode_strict(s).ok()?, + _ => budget.decode_strict(doc, s).ok()?, }; if bytes.len() > SCAN_BYTES.saturating_sub(total).saturating_sub(1) { return None; diff --git a/crates/edit/src/background.rs b/crates/edit/src/background.rs index 376cb5a165..27073ddc09 100644 --- a/crates/edit/src/background.rs +++ b/crates/edit/src/background.rs @@ -1,7 +1,7 @@ //! A page's background colour: the colour the page is painted with before anything else, as //! Edit a PDF ▸ Background and many producers do it, for exports that keep a page colour (#2992). -use pdfcraft_content::{Matrix, Op, Pieces}; +use pdfcraft_content::{Matrix, Op}; use pdfcraft_cos::{Dict, Document}; use crate::{EditError, Rgb}; @@ -51,7 +51,7 @@ pub fn page_background(doc: &Document, page: usize) -> Result, EditE let resources = p.dict.get(b"Resources").map(|r| doc.resolve(r)).and_then(|r| r.as_dict().cloned()).unwrap_or_default(); let gstates = resources.get(b"ExtGState").map(|g| doc.resolve(g)).and_then(|g| g.as_dict().cloned()).unwrap_or_default(); let streams = crate::text::content_streams(doc, &p.dict)?; - let pieces = Pieces::join(&streams.iter().map(|(_, d)| d.as_slice()).collect::>()); + let pieces = crate::text::joined(&streams); let ops = pieces.parse().ops; let mut st = State { ctm: Matrix::IDENTITY, space: Space::Gray, fill: Some([0.0; 3]), alpha: 1.0, plain: true, clip: Some(Vec::new()) }; diff --git a/crates/edit/src/images.rs b/crates/edit/src/images.rs index 72656fa1bb..fbb37ec1f0 100644 --- a/crates/edit/src/images.rs +++ b/crates/edit/src/images.rs @@ -4,12 +4,12 @@ //! or delete. Only the stream that draws it is rewritten, as a new object. use kurbo::{BezPath, Point, Shape}; -use pdfcraft_content::{Matrix, Op, Pieces, parse, serialize_ops}; +use pdfcraft_content::{Matrix, Op, parse, serialize_ops}; use pdfcraft_cos::{Dict, Document, ObjRef, Object}; use std::sync::Arc; use crate::EditError; -use crate::text::content_streams; +use crate::text::{ContentStream, content_streams, joined}; /// The identity of a soft mask (`/SMask`) in the PDF graphics state. #[derive(Clone, Debug)] @@ -47,7 +47,7 @@ pub struct PageImage { pub width: u32, pub height: u32, /// The content stream its `Do` (its first token) is in, and the `Do`'s index among the - /// operators of the page's streams joined ([`Pieces`]). + /// operators of the page's streams joined ([`pdfcraft_content::Pieces`]). pub stream: usize, op: usize, /// CTM before `Do`, excluding a Form's own Matrix. @@ -103,11 +103,6 @@ fn too_many_images() -> EditError { EditError::Invalid(format!("the page draws more than {MAX_PAGE_IMAGES} images")) } -/// The page's streams joined into the one stream they are: an operator may span two of them. -fn joined(streams: &[(Object, Vec)]) -> Pieces { - Pieces::join(&streams.iter().map(|(_, data)| data.as_slice()).collect::>()) -} - fn page_of(doc: &Document, page: usize) -> Result { pdfcraft_model::pages(doc).into_iter().nth(page).ok_or(EditError::NoSuchPage(page)) } @@ -532,12 +527,17 @@ fn exclusive_clip(ops: &[Op], op: usize) -> Option<(usize, Matrix)> { /// The `q cm … Q` whose only content after its `cm` is `start..=end` (a `Do`, or a balanced /// `q … Q` scope), with the index and value of that `cm`. Another transform of the same /// placement is folded into it, so repeated moves and resizes don't nest one more `q` each. -fn enclosing_transform(ops: &[Op], start: usize, end: usize) -> Option<(usize, Matrix)> { +/// None when its `Q` is at or past `hidden` ([`crate::text::first_op_after_damage`]): a stream +/// between that wasn't read may draw in that `cm` too. +fn enclosing_transform(ops: &[Op], start: usize, end: usize, hidden: usize) -> Option<(usize, Matrix)> { let at = start.checked_sub(1)?; let cm = ops.get(at)?; if !ops.get(at.checked_sub(1)?)?.is("q") || !ops.get(end.checked_add(1)?)?.is("Q") || !cm.is("cm") || cm.operands.len() != 6 { return None; } + if end.checked_add(1)? >= hidden { + return None; + } let m = Matrix(cm.nums::<6>()?); m.0.iter().all(|v| v.is_finite()).then_some((at, m)) } @@ -1057,7 +1057,9 @@ fn determine_arrange_spans( has_other_paint: bool, marked_unit: bool, ) -> Result<(std::ops::Range, usize, usize, usize), EditError> { - let scope = image_scopes(ops, target, "arranged") + // Nothing is hidden for Arrange: it takes out the scope it finds, so a scope that closes past a + // stream not read whole changes a stream past it, which `rewritten_contents` refuses. + let scope = image_scopes(ops, target, "arranged", ops.len()) .ok() .and_then(|s| s.isolated) .filter(|&(start, end, _)| start >= target_unit.start && end < target_unit.end); @@ -1191,13 +1193,7 @@ fn build_relocated_unit_ops( } /// Change the stacking order of `img`, drawn on page `p` by its content streams `all`. -fn arrange_image( - doc: &mut Document, - p: &pdfcraft_model::Page, - all: &[(Object, Vec)], - img: &PageImage, - order: ArrangeOrder, -) -> Result<(), EditError> { +fn arrange_image(doc: &mut Document, p: &pdfcraft_model::Page, all: &[ContentStream], img: &PageImage, order: ArrangeOrder) -> Result<(), EditError> { let pieces = joined(all); let ops = pieces.parse().ops; let target = img.op; @@ -1264,7 +1260,7 @@ fn arrange_image( ops.get(dest_unit.start).map(|op| pieces.piece_of(op.span.start)) }; let owned_destination = destination_piece - .filter(|piece| all.get(*piece).is_some_and(|(obj, _)| doc.resolve(obj).as_dict().is_some_and(|d| d.get(b"PCAdded").is_some()))); + .filter(|piece| all.get(*piece).is_some_and(|s| doc.resolve(&s.entry).as_dict().is_some_and(|d| d.get(b"PCAdded").is_some()))); if let Some(piece) = owned_destination { let whole_unit = ops.iter().enumerate().all(|(index, op)| { let (first, last) = pieces.pieces_of(op); @@ -1439,13 +1435,14 @@ fn arrange_image( }; if let Some(piece) = owned_destination { + let insertion = piece.saturating_add(usize::from(insert_after)); + crate::text::drawn_before_damage(all, insertion)?; let changed = pieces.splice([(target_span, removal_replacement)]); - let contents = crate::text::rewritten_contents(doc, all, changed); + let contents = crate::text::rewritten_contents(doc, all, changed)?; let mut list = match contents { Object::Array(list) => list, one => vec![one], }; - let insertion = piece.saturating_add(usize::from(insert_after)); if insertion > list.len() { return Err(EditError::Invalid("the destination content stream is missing".into())); } @@ -1467,7 +1464,7 @@ fn arrange_image( } let changed = pieces.splice(edits); - let contents = crate::text::rewritten_contents(doc, all, changed); + let contents = crate::text::rewritten_contents(doc, all, changed)?; doc.update_dict(p.obj, |d| d.set(b"Contents".to_vec(), contents))?; Ok(()) } @@ -1501,8 +1498,10 @@ impl ImageScopes { } } -/// [`ImageScopes`] of the `Do` at `target`. An image inside a text object cannot be `done`. -fn image_scopes(ops: &[Op], target: usize, done: &str) -> Result { +/// [`ImageScopes`] of the `Do` at `target`. An image inside a text object cannot be `done`. A scope +/// still open at op `hidden` ([`crate::text::first_op_after_damage`]) holds a stream that wasn't +/// read, which may paint there: it counts as painting something else. +fn image_scopes(ops: &[Op], target: usize, done: &str, hidden: usize) -> Result { struct Scope { start: usize, matrix: Matrix, @@ -1521,6 +1520,11 @@ fn image_scopes(ops: &[Op], target: usize, done: &str) -> Result