diff --git a/.config/nextest.toml b/.config/nextest.toml index 1037b2310..0bc679e47 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -1,7 +1,6 @@ # cargo-nextest (https://nexte.st) runs every test in its own process, in parallel across test -# binaries. The Windows leg of the Workspace gate uses it (.github/workflows/workspace.yml); every -# other leg, and local runs, keep `cargo test`. Nextest does not run doctests, so the Windows leg -# also runs `cargo test --doc`. +# binaries. Every leg of the Workspace gate uses it (.github/workflows/workspace.yml); local runs may +# keep `cargo test`. Nextest does not run doctests, so the gate also runs `cargo test --doc`. # # cargo nextest run --workspace --profile ci # cargo test --workspace --doc @@ -22,6 +21,9 @@ video-fixtures = { max-threads = 1 } # craft-print: `PRINT_TICKET_PROVIDER` serializes the Windows PrintTicket provider, and the tests # share the one test print queue the gate installs. print = { max-threads = 1 } +# craft-video integration tests on Linux and macOS: the binaries generate and read the same fixture +# directories (see the `ci` overrides below). +video-integration = { max-threads = 1 } [[profile.default.overrides]] filter = 'package(craft-ui) & (binary(visual_regression) | test(/^combo::tests::|selected_tab_erases_a_baseline_centered_on_its_bottom_edge/))' @@ -42,3 +44,18 @@ test-group = 'print' fail-fast = false slow-timeout = { period = "60s" } failure-output = "immediate-final" + + +# Linux and macOS (the resource audit in craft-libs#2058): every GPU consumer shares the one `gpu` +# slot, and the craft-video integration binaries share their fixture directories. Conservative on +# purpose: the in-process mutexes these tests rely on do not reach across nextest's processes. +# Windows keeps the default groups above, which its leg has passed with since it moved to nextest. +[[profile.ci.overrides]] +platform = 'cfg(unix)' +filter = 'package(craft-ui) | package(craft-keymap) | package(craft-update)' +test-group = 'gpu' + +[[profile.ci.overrides]] +platform = 'cfg(unix)' +filter = 'package(craft-video) & kind(test)' +test-group = 'video-integration' diff --git a/.github/workflows/actions.lock b/.github/workflows/actions.lock index 02cdf71c8..46d7b8d3c 100644 --- a/.github/workflows/actions.lock +++ b/.github/workflows/actions.lock @@ -18,7 +18,6 @@ workflows: - 'actions/checkout@v7.0.1' - 'step-security/dtolnay-rust-toolchain@v1.0.0' - 'step-security/harden-runner@v2.22.4' - - 'step-security/rust-cache@v2.9.2' '.github/workflows/cms.yml': - 'actions/checkout@v4.4.0' - 'dtolnay/rust-toolchain@stable' @@ -27,7 +26,6 @@ workflows: - 'actions/checkout@v7.0.1' - 'step-security/dtolnay-rust-toolchain@v1.0.0' - 'step-security/harden-runner@v2.22.4' - - 'step-security/rust-cache@v2.9.2' '.github/workflows/dialogs.yml': - 'actions/checkout@v7.0.1' - 'step-security/dtolnay-rust-toolchain@v1.0.0' @@ -36,10 +34,6 @@ workflows: - 'actions/checkout@v7.0.1' - 'step-security/dtolnay-rust-toolchain@v1.0.0' - 'step-security/harden-runner@v2.22.4' - '.github/workflows/docpath.yml': - - 'actions/checkout@v7.0.1' - - 'step-security/dtolnay-rust-toolchain@v1.0.0' - - 'step-security/harden-runner@v2.22.4' '.github/workflows/ecproj-format.yml': - 'actions/checkout@v7.0.1' - 'step-security/dtolnay-rust-toolchain@v1.0.0' @@ -52,22 +46,15 @@ workflows: - 'actions/checkout@v7.0.1' - 'dtolnay/rust-toolchain@stable' - 'step-security/harden-runner@v2.22.4' - - 'swatinem/rust-cache@v2.9.3' '.github/workflows/house-rules.yml': [] - '.github/workflows/i18n.yml': - - 'actions/checkout@v7.0.1' - - 'step-security/dtolnay-rust-toolchain@v1.0.0' - - 'step-security/harden-runner@v2.22.4' '.github/workflows/jpeg.yml': - 'actions/checkout@v7.0.1' - 'step-security/dtolnay-rust-toolchain@v1.0.0' - 'step-security/harden-runner@v2.22.4' - - 'step-security/rust-cache@v2.9.2' '.github/workflows/keymap.yml': - 'actions/checkout@v7.0.1' - 'step-security/dtolnay-rust-toolchain@v1.0.0' - 'step-security/harden-runner@v2.22.4' - - 'step-security/rust-cache@v2.9.2' '.github/workflows/lockfile.yml': - 'actions/checkout@v7.0.1' - 'step-security/harden-runner@v2.22.4' @@ -75,11 +62,10 @@ workflows: - 'actions/checkout@v7.0.1' - 'step-security/dtolnay-rust-toolchain@v1.0.0' - 'step-security/harden-runner@v2.22.4' - - 'step-security/rust-cache@v2.9.2' '.github/workflows/menu.yml': - 'actions/checkout@v7.0.1' - 'dtolnay/rust-toolchain@stable' - - 'swatinem/rust-cache@v2.9.3' + '.github/workflows/nightly.yml': [] '.github/workflows/pcraft-format.yml': - 'actions/checkout@v7.0.1' - 'step-security/dtolnay-rust-toolchain@v1.0.0' @@ -89,23 +75,12 @@ workflows: - 'actions/setup-python@v7.1.0' - 'step-security/dtolnay-rust-toolchain@v1.0.0' - 'step-security/harden-runner@v2.22.4' - - 'step-security/rust-cache@v2.9.2' - 'step-security/taiki-e-install-action@v2.87.32' - '.github/workflows/print.yml': - - 'actions/checkout@v7.0.1' - - 'step-security/dtolnay-rust-toolchain@v1.0.0' - - 'step-security/harden-runner@v2.22.4' '.github/workflows/proof.yml': - 'actions/checkout@v7.0.1' - 'step-security/dtolnay-rust-toolchain@v1.0.0' - 'step-security/harden-runner@v2.22.4' - - 'step-security/rust-cache@v2.9.2' '.github/workflows/sonarcloud.yml': [] - '.github/workflows/ui.yml': - - 'actions/checkout@v7.0.1' - - 'step-security/dtolnay-rust-toolchain@v1.0.0' - - 'step-security/harden-runner@v2.22.4' - - 'step-security/rust-cache@v2.9.2' '.github/workflows/update.yml': - 'actions/checkout@v7.0.1' - 'step-security/dtolnay-rust-toolchain@v1.0.0' @@ -118,7 +93,6 @@ workflows: - 'actions/checkout@v7.0.1' - 'step-security/dtolnay-rust-toolchain@v1.0.0' - 'step-security/harden-runner@v2.22.4' - - 'step-security/rust-cache@v2.9.2' '.github/workflows/web-api.yml': - 'actions/checkout@v7.0.1' - 'step-security/dtolnay-rust-toolchain@v1.0.0' @@ -128,7 +102,6 @@ workflows: - 'actions/checkout@v7.0.1' - 'step-security/dtolnay-rust-toolchain@v1.0.0' - 'step-security/harden-runner@v2.22.4' - - 'step-security/rust-cache@v2.9.2' - 'step-security/taiki-e-install-action@v2.87.32' '.github/workflows/window.yml': - 'actions/checkout@v7.0.1' diff --git a/.github/workflows/ai-core.yml b/.github/workflows/ai-core.yml index 5471bf1ec..2be123b23 100644 --- a/.github/workflows/ai-core.yml +++ b/.github/workflows/ai-core.yml @@ -12,20 +12,17 @@ permissions: contents: read concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued. - group: ai-core-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + # One group per pull request and per branch: a newer push cancels the run it supersedes, on main + # too, since the newest run checks everything before it. + group: ai-core-${{ github.ref }} + cancel-in-progress: true +# fmt, clippy and the tests run in the Workspace gate on every OS; this checks what it does not. jobs: ai-core: + name: AI model WASM timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - # macOS runners are kept for releases: macOS runs only when started by hand. - os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }} - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest steps: - name: Harden runner uses: step-security/harden-runner@v2.22.4 @@ -35,10 +32,6 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: stable - components: rustfmt, clippy targets: wasm32-unknown-unknown - - run: cargo fmt --check - - run: cargo clippy -p craft-ai-core --all-targets --locked -- -D warnings - - run: cargo test -p craft-ai-core --locked # The model promises to compile for the web. - run: cargo check -p craft-ai-core --target wasm32-unknown-unknown --locked diff --git a/.github/workflows/atomic.yml b/.github/workflows/atomic.yml index f72a64f1e..23de7f9f4 100644 --- a/.github/workflows/atomic.yml +++ b/.github/workflows/atomic.yml @@ -12,20 +12,17 @@ permissions: contents: read concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued. - group: atomic-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + # One group per pull request and per branch: a newer push cancels the run it supersedes, on main + # too, since the newest run checks everything before it. + group: atomic-${{ github.ref }} + cancel-in-progress: true +# fmt, clippy and the tests run in the Workspace gate on every OS; this runs what it does not. jobs: atomic: + name: Atomic inherited Linux ACLs timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - # macOS runners are kept for releases: macOS runs only when started by hand. - os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }} - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest steps: - name: Harden runner uses: step-security/harden-runner@v2.22.4 @@ -35,15 +32,9 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: stable - components: rustfmt, clippy - name: Install ACL tools - if: runner.os == 'Linux' run: sudo apt-get update && sudo apt-get install -y acl - - run: cargo fmt --check - - run: cargo clippy -p craft-atomic -p craft-settings --all-targets --locked -- -D warnings - - run: cargo test -p craft-atomic -p craft-settings --locked -- --nocapture - name: Check inherited Linux ACLs - if: runner.os == 'Linux' run: cargo test -p craft-atomic --locked -- --ignored --nocapture wasm: diff --git a/.github/workflows/audio.yml b/.github/workflows/audio.yml index d6a7ff5e2..5fd971c62 100644 --- a/.github/workflows/audio.yml +++ b/.github/workflows/audio.yml @@ -11,17 +11,22 @@ on: permissions: contents: read +# A fraction of the fuzz-style cases, as in the Workspace gate; nightly.yml runs them in full. +env: + CRAFT_HOSTILE_DIVISOR: "8" + PROPTEST_CASES: "64" + concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it - # always gets to save the cache. - group: shared-audio-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + # One group per pull request and per branch: a newer push cancels the run it supersedes, on main + # too, since the newest run checks everything before it. + group: shared-audio-${{ github.ref }} + cancel-in-progress: true jobs: lint: timeout-minutes: 15 - name: Audio lint, feature sets, docs, WASM + # fmt, clippy with all features and the minimum Rust version run in the Workspace gate. + name: Audio feature sets, docs, WASM runs-on: ubuntu-latest steps: - name: Harden runner @@ -34,13 +39,8 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: stable - components: rustfmt, clippy + components: clippy targets: wasm32-unknown-unknown - - uses: step-security/rust-cache@v2.9.2 - with: - key: craft-audio-lint - save-if: ${{ github.ref == 'refs/heads/main' }} - - run: cargo fmt -p craft-audio --check # Each feature on its own, so code that needs another feature fails here. - name: Clippy, each feature alone run: | @@ -49,7 +49,6 @@ jobs: cargo clippy -p craft-audio --all-targets --locked --no-default-features ${f:+--features "$f"} -- -D warnings echo "::endgroup::" done - - run: cargo clippy -p craft-audio --all-targets --locked --all-features -- -D warnings # missing_docs is a warning in the crate; -D warnings makes it (and broken links) fail here. - run: cargo doc -p craft-audio --all-features --no-deps --locked env: @@ -63,25 +62,6 @@ jobs: RUSTDOCFLAGS: "-D warnings" - run: cargo check -p craft-audio --all-features --locked --target wasm32-unknown-unknown - run: cargo check -p craft-audio --no-default-features --features lossless-encode --locked --target wasm32-unknown-unknown - msrv: - timeout-minutes: 10 - name: Audio on the minimum Rust version - runs-on: ubuntu-latest - steps: - - name: Harden runner - uses: step-security/harden-runner@v2.22.4 - with: - egress-policy: audit - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - # The workspace's rust-version. - - run: rustup toolchain install 1.95.0 --profile minimal - - uses: step-security/rust-cache@v2.9.2 - with: - key: craft-audio-msrv - save-if: ${{ github.ref == 'refs/heads/main' }} - - run: cargo +1.95.0 check -p craft-audio -p craft-media --all-features --locked features: timeout-minutes: 10 name: Audio tests without symphonia and lossless-encode only @@ -97,10 +77,6 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: stable - - uses: step-security/rust-cache@v2.9.2 - with: - key: craft-audio-features - save-if: ${{ github.ref == 'refs/heads/main' }} - run: cargo test -p craft-audio --no-default-features --features lossless-encode --locked - run: cargo test -p craft-audio --no-default-features --features pcm-encode --locked --lib - run: cargo test -p craft-audio --no-default-features --features flac-encode --locked --lib @@ -122,10 +98,6 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: stable - - uses: step-security/rust-cache@v2.9.2 - with: - key: craft-audio-test - save-if: ${{ github.ref == 'refs/heads/main' }} - uses: actions/cache@v6.1.0 with: path: ${{ runner.temp }}/media-fixtures/opus/vectors diff --git a/.github/workflows/cms.yml b/.github/workflows/cms.yml index 74ce0815f..9220d8e09 100644 --- a/.github/workflows/cms.yml +++ b/.github/workflows/cms.yml @@ -11,6 +11,13 @@ on: permissions: contents: read +concurrency: + # One group per pull request and per branch: a newer push cancels the run it supersedes, on main + # too, since the newest run checks everything before it. + group: cms-${{ github.ref }} + cancel-in-progress: true + +# clippy and the debug tests run in the Workspace gate; this runs the release build of the tests. jobs: cms: timeout-minutes: 20 @@ -18,9 +25,6 @@ jobs: steps: - uses: actions/checkout@v4.4.0 - uses: dtolnay/rust-toolchain@stable - with: - components: clippy - - run: cargo clippy -p craft-cms --all-targets --all-features --locked -- -D warnings # Release: the FOGRA39 fit and ink-limit tests take about 35 s in debug and 4 s here. Tests that need # installed system profiles print "skipped:" when none are present. - run: cargo test -p craft-cms --all-features --release --locked -- --nocapture diff --git a/.github/workflows/crw.yml b/.github/workflows/crw.yml index 54be7ba3b..466b6c05d 100644 --- a/.github/workflows/crw.yml +++ b/.github/workflows/crw.yml @@ -12,41 +12,13 @@ permissions: contents: read concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it - # always gets to save the cache. - group: shared-crw-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + # One group per pull request and per branch: a newer push cancels the run it supersedes, on main + # too, since the newest run checks everything before it. + group: shared-crw-${{ github.ref }} + cancel-in-progress: true +# fmt, clippy and the tests run in the Workspace gate on every OS; this checks what it does not. jobs: - native: - timeout-minutes: 20 - name: CRW lint and tests (${{ matrix.os }}) - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, windows-latest] - runs-on: ${{ matrix.os }} - steps: - - name: Harden runner - uses: step-security/harden-runner@v2.22.4 - with: - egress-policy: audit - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - uses: step-security/dtolnay-rust-toolchain@v1.0.0 - with: - toolchain: stable - components: rustfmt, clippy - - uses: step-security/rust-cache@v2.9.2 - with: - key: craft-crw-${{ matrix.os }} - save-if: ${{ github.ref == 'refs/heads/main' }} - - run: cargo fmt -p craft-crw --check - - run: cargo clippy -p craft-crw --all-targets --locked -- -D warnings - - run: cargo test -p craft-crw --locked - wasm: timeout-minutes: 10 name: CRW WASM @@ -63,10 +35,6 @@ jobs: with: toolchain: stable targets: wasm32-unknown-unknown - - uses: step-security/rust-cache@v2.9.2 - with: - key: craft-crw-wasm - save-if: ${{ github.ref == 'refs/heads/main' }} - run: cargo check -p craft-crw --locked --target wasm32-unknown-unknown samples: @@ -84,10 +52,6 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: stable - - uses: step-security/rust-cache@v2.9.2 - with: - key: craft-crw-samples - save-if: ${{ github.ref == 'refs/heads/main' }} - name: Cache the CC0 sample files uses: actions/cache@v6.1.0 with: diff --git a/.github/workflows/dialogs.yml b/.github/workflows/dialogs.yml index c8d43d725..23def9feb 100644 --- a/.github/workflows/dialogs.yml +++ b/.github/workflows/dialogs.yml @@ -12,10 +12,10 @@ permissions: contents: read concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued. - group: dialogs-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + # One group per pull request and per branch: a newer push cancels the run it supersedes, on main + # too, since the newest run checks everything before it. + group: dialogs-${{ github.ref }} + cancel-in-progress: true jobs: dialogs: @@ -37,6 +37,5 @@ jobs: components: clippy # The macOS code is linted on Linux: it compiles for the target without the SDK. targets: aarch64-apple-darwin - - run: cargo clippy -p craft-dialogs --all-targets --locked -- -D warnings + # Native clippy and the tests run in the Workspace gate. - run: cargo clippy -p craft-dialogs --target aarch64-apple-darwin --locked -- -D warnings - - run: cargo test -p craft-dialogs --locked diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 6caa4f0ee..627543381 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -12,20 +12,17 @@ permissions: contents: read concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued. - group: doc-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + # One group per pull request and per branch: a newer push cancels the run it supersedes, on main + # too, since the newest run checks everything before it. + group: doc-${{ github.ref }} + cancel-in-progress: true +# fmt, clippy and the tests run in the Workspace gate on every OS; this checks what it does not. jobs: doc: + name: Craft documents WASM timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - # macOS runners are kept for releases: macOS runs only when started by hand. - os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }} - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest steps: - name: Harden runner uses: step-security/harden-runner@v2.22.4 @@ -37,11 +34,5 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: stable - components: rustfmt, clippy targets: wasm32-unknown-unknown - - run: cargo fmt --check - - run: cargo clippy -p craft-doc --all-targets --locked -- -D warnings - - run: cargo clippy -p craft-doc --all-targets --all-features --locked -- -D warnings - - run: cargo test -p craft-doc --locked -- --nocapture - - run: cargo test -p craft-doc --all-features --locked -- --nocapture - run: cargo check -p craft-doc --all-features --locked --target wasm32-unknown-unknown diff --git a/.github/workflows/docpath.yml b/.github/workflows/docpath.yml deleted file mode 100644 index e0115f6f5..000000000 --- a/.github/workflows/docpath.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: Document paths - -on: - workflow_dispatch: - push: - branches: [main] - paths: ['crates/docpath/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/docpath.yml'] - pull_request: - paths: ['crates/docpath/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/docpath.yml'] - -permissions: - contents: read - -concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued. - group: docpath-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -jobs: - docpath: - timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - # macOS runners are kept for releases: macOS runs only when started by hand. - os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }} - runs-on: ${{ matrix.os }} - steps: - - name: Harden runner - uses: step-security/harden-runner@v2.22.4 - with: - egress-policy: audit - - uses: actions/checkout@v7.0.1 - - uses: step-security/dtolnay-rust-toolchain@v1.0.0 - with: - toolchain: stable - components: rustfmt, clippy - - run: cargo fmt --check - - run: cargo clippy -p craft-docpath --all-targets --locked -- -D warnings - - run: cargo test -p craft-docpath --locked -- --nocapture diff --git a/.github/workflows/ecproj-format.yml b/.github/workflows/ecproj-format.yml index 24a3350af..048c897b3 100644 --- a/.github/workflows/ecproj-format.yml +++ b/.github/workflows/ecproj-format.yml @@ -12,20 +12,17 @@ permissions: contents: read concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued. - group: ecproj-format-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + # One group per pull request and per branch: a newer push cancels the run it supersedes, on main + # too, since the newest run checks everything before it. + group: ecproj-format-${{ github.ref }} + cancel-in-progress: true +# fmt, clippy and the tests run in the Workspace gate on every OS; this checks what it does not. jobs: ecproj-format: + name: EffectCraft project format WASM timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - # macOS runners are kept for releases: macOS runs only when started by hand. - os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }} - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest steps: - name: Harden runner uses: step-security/harden-runner@v2.22.4 @@ -37,9 +34,5 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: stable - components: rustfmt, clippy targets: wasm32-unknown-unknown - - run: cargo fmt --check - - run: cargo clippy -p craft-ecproj-format --all-targets --locked -- -D warnings - - run: cargo test -p craft-ecproj-format --locked -- --nocapture - run: cargo check -p craft-ecproj-format --locked --target wasm32-unknown-unknown diff --git a/.github/workflows/font-store.yml b/.github/workflows/font-store.yml index 0578f18ec..e655697b3 100644 --- a/.github/workflows/font-store.yml +++ b/.github/workflows/font-store.yml @@ -12,20 +12,17 @@ permissions: contents: read concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued. - group: font-store-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + # One group per pull request and per branch: a newer push cancels the run it supersedes, on main + # too, since the newest run checks everything before it. + group: font-store-${{ github.ref }} + cancel-in-progress: true +# fmt, clippy and the tests run in the Workspace gate on every OS; this checks what it does not. jobs: font-store: + name: Font store WASM timeout-minutes: 20 - strategy: - fail-fast: false - matrix: - # macOS runners are kept for releases: macOS runs only when started by hand. - os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }} - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest steps: - name: Harden runner uses: step-security/harden-runner@v2.22.4 @@ -35,13 +32,6 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: stable - components: rustfmt, clippy targets: wasm32-unknown-unknown - - run: cargo fmt --check - - run: cargo clippy -p craft-font-store --all-targets --locked -- -D warnings - - run: cargo test -p craft-font-store --locked - - run: cargo clippy -p craft-font-catalog --all-targets --locked -- -D warnings - - run: cargo test -p craft-font-catalog --locked - name: Check the web build - if: runner.os == 'Linux' run: cargo check -p craft-font-store --target wasm32-unknown-unknown --locked diff --git a/.github/workflows/formats.yml b/.github/workflows/formats.yml index 84d2c0d7e..12478d2c9 100644 --- a/.github/workflows/formats.yml +++ b/.github/workflows/formats.yml @@ -42,11 +42,10 @@ permissions: contents: read concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it - # always gets to save the cache. - group: shared-formats-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + # One group per pull request and per branch: a newer push cancels the run it supersedes, on main + # too, since the newest run checks everything before it. + group: shared-formats-${{ github.ref }} + cancel-in-progress: true jobs: check: @@ -68,7 +67,9 @@ jobs: - { crate: craft-tiff, features: "" } - { crate: craft-heif, features: "testdata" } - { crate: craft-cms, features: "" } - name: ${{ matrix.crate }} lint, tests, WASM + # fmt, clippy with default and with all features, and the tests run in the Workspace gate on + # every OS; this lints each feature alone and checks the web build. + name: ${{ matrix.crate }} feature sets, WASM runs-on: ubuntu-latest steps: - name: Harden runner @@ -80,32 +81,19 @@ jobs: persist-credentials: false - uses: dtolnay/rust-toolchain@stable with: - components: rustfmt, clippy + components: clippy targets: wasm32-unknown-unknown - - uses: Swatinem/rust-cache@v2.9.3 - with: - key: ${{ matrix.crate }} - save-if: ${{ github.ref == 'refs/heads/main' }} - - run: cargo fmt -p "$CRATE" --check - env: - CRATE: ${{ matrix.crate }} - - name: Clippy, default features and each feature alone + - name: Clippy, each feature alone + if: matrix.features != '' env: CRATE: ${{ matrix.crate }} FEATURES: ${{ matrix.features }} run: | - cargo clippy -p "$CRATE" --all-targets --locked -- -D warnings for f in $FEATURES; do echo "::group::feature: $f" cargo clippy -p "$CRATE" --all-targets --locked --no-default-features --features "$f" -- -D warnings echo "::endgroup::" done - - run: cargo clippy -p "$CRATE" --all-targets --locked --all-features -- -D warnings - env: - CRATE: ${{ matrix.crate }} - - run: cargo test -p "$CRATE" --all-features --locked - env: - CRATE: ${{ matrix.crate }} - run: cargo check -p "$CRATE" --all-features --locked --target wasm32-unknown-unknown env: CRATE: ${{ matrix.crate }} diff --git a/.github/workflows/i18n.yml b/.github/workflows/i18n.yml deleted file mode 100644 index 7e6903188..000000000 --- a/.github/workflows/i18n.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: Translations - -on: - workflow_dispatch: - push: - branches: [main] - paths: ['crates/i18n/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/i18n.yml'] - pull_request: - paths: ['crates/i18n/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/i18n.yml'] - -permissions: - contents: read - -concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued. - group: i18n-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -jobs: - i18n: - timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - # macOS runners are kept for releases: macOS runs only when started by hand. - os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }} - runs-on: ${{ matrix.os }} - steps: - - name: Harden runner - uses: step-security/harden-runner@v2.22.4 - with: - egress-policy: audit - - uses: actions/checkout@v7.0.1 - - uses: step-security/dtolnay-rust-toolchain@v1.0.0 - with: - toolchain: stable - components: rustfmt, clippy - - run: cargo fmt --check - - run: cargo clippy -p craft-i18n --all-targets --features collate --locked -- -D warnings - - run: cargo test -p craft-i18n --features collate --locked -- --nocapture diff --git a/.github/workflows/jpeg.yml b/.github/workflows/jpeg.yml index 21db0ab79..495af8b23 100644 --- a/.github/workflows/jpeg.yml +++ b/.github/workflows/jpeg.yml @@ -12,16 +12,17 @@ permissions: contents: read concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it - # always gets to save the cache. - group: shared-jpeg-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + # One group per pull request and per branch: a newer push cancels the run it supersedes, on main + # too, since the newest run checks everything before it. + group: shared-jpeg-${{ github.ref }} + cancel-in-progress: true jobs: check: - timeout-minutes: 10 - name: JPEG lint, tests, WASM + # Uncached: the wasm check builds its dependencies on every run (about 10 minutes with setup). + timeout-minutes: 20 + # fmt, clippy and the tests run in the Workspace gate on every OS. + name: JPEG WASM runs-on: ubuntu-latest steps: - name: Harden runner @@ -34,13 +35,5 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: stable - components: rustfmt, clippy targets: wasm32-unknown-unknown - - uses: step-security/rust-cache@v2.9.2 - with: - key: craft-jpeg - save-if: ${{ github.ref == 'refs/heads/main' }} - - run: cargo fmt -p craft-jpeg --check - - run: cargo clippy -p craft-jpeg --all-targets --locked -- -D warnings - - run: cargo test -p craft-jpeg --locked - run: cargo check -p craft-jpeg --locked --target wasm32-unknown-unknown diff --git a/.github/workflows/keymap.yml b/.github/workflows/keymap.yml index 41fcc0502..bd4ef280a 100644 --- a/.github/workflows/keymap.yml +++ b/.github/workflows/keymap.yml @@ -12,16 +12,17 @@ permissions: contents: read concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it - # always gets to save the cache. - group: keymap-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + # One group per pull request and per branch: a newer push cancels the run it supersedes, on main + # too, since the newest run checks everything before it. + group: keymap-${{ github.ref }} + cancel-in-progress: true jobs: linux: timeout-minutes: 15 - name: Keymap (Linux) + # fmt, and clippy and the tests with default and with all features, run in the Workspace gate + # on every OS; this runs the feature subsets. + name: Keymap feature sets runs-on: ubuntu-latest env: CRAFT_FONTS_DIR: ${{ github.workspace }}/.test-inputs/craft-fonts @@ -43,12 +44,7 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: stable - components: rustfmt, clippy - - uses: step-security/rust-cache@v2.9.2 - with: - key: craft-keymap-linux - save-if: ${{ github.ref == 'refs/heads/main' }} - - run: cargo fmt -p craft-keymap --check + components: clippy - run: cargo clippy -p craft-keymap --all-targets --no-default-features --locked -- -D warnings # `snapshot` covers `ui` and `egui` and compiles the dialog screenshots, which need a GPU # to run, so the test steps leave them out. @@ -58,40 +54,6 @@ jobs: run: cargo test -p craft-keymap --no-default-features --lib --tests --locked - name: Test egui dispatch and the shortcuts dialog run: cargo test -p craft-keymap --features ui --locked - native: - timeout-minutes: 15 - name: Keymap (${{ matrix.os }}) - strategy: - fail-fast: false - matrix: - # macOS runners are kept for releases: macOS runs only when started by hand. - os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["macos-latest","windows-latest"]') || fromJSON('["windows-latest"]') }} - runs-on: ${{ matrix.os }} - env: - CRAFT_FONTS_DIR: ${{ github.workspace }}/.test-inputs/craft-fonts - steps: - - name: Harden runner - uses: step-security/harden-runner@v2.22.4 - with: - egress-policy: audit - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - name: Checkout test fonts - uses: actions/checkout@v7.0.1 - with: - repository: storytold/craft-fonts - ref: b076417b5880bd4154bd22f00695b1d28c6bea83 - path: .test-inputs/craft-fonts - persist-credentials: false - - uses: step-security/dtolnay-rust-toolchain@v1.0.0 - with: - toolchain: stable - - uses: step-security/rust-cache@v2.9.2 - with: - key: craft-keymap-${{ matrix.os }} - save-if: ${{ github.ref == 'refs/heads/main' }} - - run: cargo test -p craft-keymap --features ui --locked wasm: timeout-minutes: 10 name: Keymap WASM @@ -108,8 +70,4 @@ jobs: with: toolchain: stable targets: wasm32-unknown-unknown - - uses: step-security/rust-cache@v2.9.2 - with: - key: craft-keymap-wasm - save-if: ${{ github.ref == 'refs/heads/main' }} - run: cargo check -p craft-keymap --features ui --locked --target wasm32-unknown-unknown diff --git a/.github/workflows/media.yml b/.github/workflows/media.yml index 4814e35ca..1b5afaebd 100644 --- a/.github/workflows/media.yml +++ b/.github/workflows/media.yml @@ -11,17 +11,22 @@ on: permissions: contents: read +# A fraction of the fuzz-style cases, as in the Workspace gate; nightly.yml runs them in full. +env: + CRAFT_HOSTILE_DIVISOR: "8" + PROPTEST_CASES: "64" + concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it - # always gets to save the cache. - group: shared-media-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + # One group per pull request and per branch: a newer push cancels the run it supersedes, on main + # too, since the newest run checks everything before it. + group: shared-media-${{ github.ref }} + cancel-in-progress: true jobs: lint: timeout-minutes: 10 - name: Media lint, feature sets, WASM + # fmt, and clippy with default and with all features, run in the Workspace gate on every OS. + name: Media feature sets, WASM runs-on: ubuntu-latest steps: - name: Harden runner @@ -34,20 +39,12 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: stable - components: rustfmt, clippy + components: clippy targets: wasm32-unknown-unknown - - uses: step-security/rust-cache@v2.9.2 - with: - key: craft-media-lint - save-if: ${{ github.ref == 'refs/heads/main' }} - - run: cargo fmt -p craft-media --check - run: cargo clippy -p craft-media --all-targets --locked --no-default-features -- -D warnings - run: cargo clippy -p craft-media --all-targets --locked --no-default-features --features bitstream -- -D warnings - - run: cargo clippy -p craft-media --all-targets --locked -- -D warnings - run: cargo clippy -p craft-media --all-targets --locked --no-default-features --features matroska -- -D warnings - run: cargo clippy -p craft-media --all-targets --locked --no-default-features --features ogg -- -D warnings - - run: cargo clippy -p craft-media --all-targets --locked --all-features -- -D warnings - - run: cargo clippy -p craft-media-testkit --all-targets --locked -- -D warnings - run: cargo clippy -p craft-media --all-targets --locked --no-default-features --features serde -- -D warnings - run: cargo check -p craft-media --all-features --locked --target wasm32-unknown-unknown test: @@ -65,10 +62,6 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: stable - - uses: step-security/rust-cache@v2.9.2 - with: - key: craft-media-test - save-if: ${{ github.ref == 'refs/heads/main' }} - name: Build tests while installing ffmpeg run: | # The man-db index rebuild after dpkg is slow on hosted runners. diff --git a/.github/workflows/menu.yml b/.github/workflows/menu.yml index 8d1058731..b12c8bcc6 100644 --- a/.github/workflows/menu.yml +++ b/.github/workflows/menu.yml @@ -18,7 +18,8 @@ concurrency: jobs: linux: timeout-minutes: 15 - name: Menu (Linux) + # fmt, native clippy and the tests run in the Workspace gate on every OS. + name: Menu macOS and WASM lint runs-on: ubuntu-latest steps: - uses: actions/checkout@v7.0.1 @@ -26,14 +27,8 @@ jobs: persist-credentials: false - uses: dtolnay/rust-toolchain@stable with: - components: rustfmt, clippy + components: clippy targets: aarch64-apple-darwin, wasm32-unknown-unknown - - uses: Swatinem/rust-cache@v2.9.3 - with: - key: craft-menu-linux - - run: cargo fmt -p craft-menu --check - - run: cargo clippy -p craft-menu --all-targets --locked -- -D warnings # The macOS backend type-checks from Linux; it needs no linker or SDK. - run: cargo clippy -p craft-menu --lib --locked --target aarch64-apple-darwin -- -D warnings - run: cargo clippy -p craft-menu --lib --locked --target wasm32-unknown-unknown -- -D warnings - - run: cargo test -p craft-menu --locked diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 000000000..ed9d36127 --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,34 @@ +name: Nightly full tests + +# The tests the Workspace gate shortens, in full, once a day: every hostile-input hammer and sweep +# without CRAFT_HOSTILE_DIVISOR, and every proptest at its own case count (no PROPTEST_CASES). The +# rest of the workspace's tests run with them, on Linux, with the gate's setup. +on: + schedule: + - cron: "37 2 * * *" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: nightly-${{ github.ref }} + cancel-in-progress: false + +jobs: + full: + name: Full tests + uses: storytold/craft-common-actions/.github/workflows/gate.yml@main + with: + os: '["ubuntu-latest"]' + timeout-minutes: 120 + tools: cargo-nextest@0.9.150 + packages: mesa-vulkan-drivers libvulkan1 + craft-fonts-ref: b076417b5880bd4154bd22f00695b1d28c6bea83 + linux-env: WGPU_BACKEND=vulkan + env: CRAFT_PRINT_REQUIRE_QUEUE=0 + setup-command: bash scripts/ci-test-inputs.sh + cache-paths: .test-inputs/photocraft/corpus + cache-key: photocraft-corpus + cache-key-files: scripts/ci-test-inputs.sh + command: cargo nextest run --workspace --all-features --locked --profile ci diff --git a/.github/workflows/pcraft-format.yml b/.github/workflows/pcraft-format.yml index 9c66b5a10..0e4feabc9 100644 --- a/.github/workflows/pcraft-format.yml +++ b/.github/workflows/pcraft-format.yml @@ -12,20 +12,18 @@ permissions: contents: read concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued. - group: pcraft-format-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + # One group per pull request and per branch: a newer push cancels the run it supersedes, on main + # too, since the newest run checks everything before it. + group: pcraft-format-${{ github.ref }} + cancel-in-progress: true +# fmt, clippy and the tests run in the Workspace gate on every OS, and its msrv job checks the +# workspace on 1.95; this checks the web build on 1.95. jobs: pcraft-format: + name: PhotoCraft format WASM timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - # macOS runners are kept for releases: macOS runs only when started by hand. - os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }} - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest steps: - name: Harden runner uses: step-security/harden-runner@v2.22.4 @@ -37,9 +35,5 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: "1.95" - components: rustfmt, clippy targets: wasm32-unknown-unknown - - run: cargo fmt --check - - run: cargo clippy -p craft-pcraft-format --all-targets --locked -- -D warnings - - run: cargo test -p craft-pcraft-format --locked -- --nocapture - run: cargo check -p craft-pcraft-format --locked --target wasm32-unknown-unknown diff --git a/.github/workflows/pdf.yml b/.github/workflows/pdf.yml index 09ddfabca..7b7397757 100644 --- a/.github/workflows/pdf.yml +++ b/.github/workflows/pdf.yml @@ -11,22 +11,23 @@ on: permissions: contents: read +# A fraction of the proptest cases, as in the Workspace gate; nightly.yml runs them in full. +env: + PROPTEST_CASES: "64" + concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it - # always gets to save the cache. - group: shared-pdf-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + # One group per pull request and per branch: a newer push cancels the run it supersedes, on main + # too, since the newest run checks everything before it. + group: shared-pdf-${{ github.ref }} + cancel-in-progress: true jobs: native: - name: PDF (${{ matrix.os }}) - strategy: - fail-fast: false - matrix: - # macOS runners are kept for releases: macOS runs only when started by hand. - os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }} - runs-on: ${{ matrix.os }} + # fmt, clippy and the tests run in the Workspace gate on every OS, with the licensed fonts and + # the plate tests; this runs the tests behind craft_pdf_unicode_fixtures and the read-only + # contract, which is its own workspace. + name: PDF Unicode fixtures and read-only contract + runs-on: ubuntu-latest env: RUSTFLAGS: --cfg craft_pdf_unicode_fixtures steps: @@ -38,11 +39,6 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: stable - components: rustfmt, clippy - - uses: step-security/rust-cache@v2.9.2 - with: - key: craft-pdf - save-if: ${{ github.ref == 'refs/heads/main' }} - uses: actions/setup-python@v7.1.0 with: python-version: "3.12" @@ -52,9 +48,7 @@ jobs: python scripts/test-fetch-pdf-test-fonts.py python scripts/fetch-pdf-test-fonts.py "$RUNNER_TEMP/craft-pdf-test-fonts" echo "CRAFT_PDF_TEST_FONT_DIR=$RUNNER_TEMP/craft-pdf-test-fonts" >> "$GITHUB_ENV" - - run: cargo fmt --check - - run: cargo clippy -p craft-pdf --all-targets --all-features --locked -- -D warnings - - run: cargo test -p craft-pdf --all-features --locked + - run: cargo test -p craft-pdf --all-features --locked --test unicode_writing --test upstream_compatibility --test upstream_reading - run: cargo test --manifest-path crates/pdf/read-only-contract/Cargo.toml --locked backends: name: PDF vendored contracts @@ -68,10 +62,6 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: stable - - uses: step-security/rust-cache@v2.9.2 - with: - key: craft-pdf-backends - save-if: ${{ github.ref == 'refs/heads/main' }} - run: bash scripts/test-pdf-backends.sh wasm: name: PDF WASM runtime @@ -89,10 +79,6 @@ jobs: with: toolchain: stable targets: wasm32-unknown-unknown - - uses: step-security/rust-cache@v2.9.2 - with: - key: craft-pdf-wasm - save-if: ${{ github.ref == 'refs/heads/main' }} - uses: actions/setup-python@v7.1.0 with: python-version: "3.12" @@ -117,31 +103,3 @@ jobs: fallback: cargo-install - run: cargo test -p craft-pdf --all-features --locked --target wasm32-unknown-unknown - run: cargo test --manifest-path crates/pdf/read-only-contract/Cargo.toml --locked --target wasm32-unknown-unknown - plate-test: - name: Plate tests and craft-prepress - runs-on: ubuntu-latest - steps: - - name: Harden runner - uses: step-security/harden-runner@v2.22.4 - with: - egress-policy: audit - - uses: actions/checkout@v7.0.1 - - uses: step-security/dtolnay-rust-toolchain@v1.0.0 - with: - toolchain: stable - components: clippy - - uses: step-security/rust-cache@v2.9.2 - with: - key: craft-plate-test - save-if: ${{ github.ref == 'refs/heads/main' }} - - uses: actions/setup-python@v7.1.0 - with: - python-version: "3.12" - - name: Provision licensed PDF test fonts - shell: bash - run: | - python scripts/test-fetch-pdf-test-fonts.py - python scripts/fetch-pdf-test-fonts.py "$RUNNER_TEMP/craft-pdf-test-fonts" - echo "CRAFT_PDF_TEST_FONT_DIR=$RUNNER_TEMP/craft-pdf-test-fonts" >> "$GITHUB_ENV" - - run: cargo clippy -p craft-plate-test -p craft-prepress --all-targets --all-features --locked -- -D warnings - - run: cargo test -p craft-plate-test -p craft-prepress --all-features --locked diff --git a/.github/workflows/print.yml b/.github/workflows/print.yml deleted file mode 100644 index 8e2108e09..000000000 --- a/.github/workflows/print.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: Print spooler -on: - workflow_dispatch: - push: - branches: [main] - paths: ['crates/print/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/print.yml'] - pull_request: - paths: ['crates/print/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/print.yml'] -permissions: - contents: read -concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued. - group: print-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} -jobs: - print: - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - # macOS runners are kept for releases: macOS runs only when started by hand. - os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }} - runs-on: ${{ matrix.os }} - steps: - - name: Harden runner - uses: step-security/harden-runner@v2.22.4 - with: - egress-policy: audit - - uses: actions/checkout@v7.0.1 - - uses: step-security/dtolnay-rust-toolchain@v1.0.0 - with: - toolchain: stable - components: rustfmt, clippy - - run: cargo fmt --check - - run: cargo clippy -p craft-print --all-targets -- -D warnings - - name: Prepare the Windows test driver - if: runner.os == 'Windows' - shell: powershell - run: | - Start-Service Spooler - Add-PrinterDriver -Name 'Generic / Text Only' - - run: cargo test -p craft-print --locked -- --nocapture - timeout-minutes: 10 - env: - CRAFT_PRINT_REQUIRE_QUEUE: ${{ runner.os == 'Windows' && '1' || '0' }} diff --git a/.github/workflows/proof.yml b/.github/workflows/proof.yml index 10365a230..ea5edb8a5 100644 --- a/.github/workflows/proof.yml +++ b/.github/workflows/proof.yml @@ -12,16 +12,16 @@ permissions: contents: read concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it - # always gets to save the cache. - group: craft-proof-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + # One group per pull request and per branch: a newer push cancels the run it supersedes, on main + # too, since the newest run checks everything before it. + group: craft-proof-${{ github.ref }} + cancel-in-progress: true jobs: check: timeout-minutes: 20 - name: Proofing lint, tests with and without Language data, WASM + # fmt, and clippy and the tests without Language data, run in the Workspace gate on every OS. + name: Proofing with Language data, WASM runs-on: ubuntu-latest steps: - name: Harden runner @@ -43,18 +43,10 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: stable - components: rustfmt, clippy + components: clippy targets: wasm32-unknown-unknown - - uses: step-security/rust-cache@v2.9.2 - with: - key: craft-proof - save-if: ${{ github.ref == 'refs/heads/main' }} - - run: cargo fmt -p craft-proof --check - - name: Without Language data (a fork, docs.rs, offline) - run: | - cargo clippy -p craft-proof --all-targets --locked -- -D warnings - cargo test -p craft-proof --locked - cargo check -p craft-proof --locked --target wasm32-unknown-unknown + - name: The web build without Language data (a fork, docs.rs, offline) + run: cargo check -p craft-proof --locked --target wasm32-unknown-unknown - name: A release without Language data fails run: | if CRAFT_I18N_REQUIRED=1 cargo check -p craft-proof --locked 2>build.log; then diff --git a/.github/workflows/ui.yml b/.github/workflows/ui.yml deleted file mode 100644 index f15ef6ddc..000000000 --- a/.github/workflows/ui.yml +++ /dev/null @@ -1,74 +0,0 @@ -name: Shared UI - -on: - push: - branches: [main] - paths: ["crates/ui/**", "crates/i18n/**", "Cargo.toml", "Cargo.lock", ".github/workflows/ui.yml"] - pull_request: - paths: ["crates/ui/**", "crates/i18n/**", "Cargo.toml", "Cargo.lock", ".github/workflows/ui.yml"] - workflow_dispatch: - -permissions: - contents: read - -concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it - # always gets to save the cache. - group: shared-ui-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -jobs: - native: - timeout-minutes: 30 - name: UI (${{ matrix.os }}) - strategy: - fail-fast: false - matrix: - # macOS runners are kept for releases: macOS runs only when started by hand. - os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }} - runs-on: ${{ matrix.os }} - steps: - - name: Harden runner - uses: step-security/harden-runner@v2.22.4 - with: - egress-policy: audit - - uses: actions/checkout@v7.0.1 - - name: Install software Vulkan renderer - if: runner.os == 'Linux' - run: | - sudo apt-get update - sudo apt-get install -y mesa-vulkan-drivers libvulkan1 - echo 'WGPU_BACKEND=vulkan' >> "$GITHUB_ENV" - - uses: step-security/dtolnay-rust-toolchain@v1.0.0 - with: - toolchain: stable - components: rustfmt, clippy - - uses: step-security/rust-cache@v2.9.2 - with: - key: craft-ui - save-if: ${{ github.ref == 'refs/heads/main' }} - - run: cargo fmt --check - - run: cargo clippy -p craft-ui --all-targets --locked -- -D warnings - - run: cargo clippy -p craft-ui --all-targets --all-features --locked -- -D warnings - - run: cargo test -p craft-ui --locked - - run: cargo test -p craft-ui --all-features --locked - wasm: - timeout-minutes: 15 - name: UI WASM - runs-on: ubuntu-latest - steps: - - name: Harden runner - uses: step-security/harden-runner@v2.22.4 - with: - egress-policy: audit - - uses: actions/checkout@v7.0.1 - - uses: step-security/dtolnay-rust-toolchain@v1.0.0 - with: - toolchain: stable - targets: wasm32-unknown-unknown - - uses: step-security/rust-cache@v2.9.2 - with: - key: craft-ui-wasm - save-if: ${{ github.ref == 'refs/heads/main' }} - - run: cargo check -p craft-ui --all-features --locked --target wasm32-unknown-unknown diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml index 9a7139026..96ab4d6ad 100644 --- a/.github/workflows/update.yml +++ b/.github/workflows/update.yml @@ -12,20 +12,18 @@ permissions: contents: read concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued. - group: update-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + # One group per pull request and per branch: a newer push cancels the run it supersedes, on main + # too, since the newest run checks everything before it. + group: update-${{ github.ref }} + cancel-in-progress: true +# fmt, clippy and the tests with default and with all features run in the Workspace gate on every +# OS; this runs the egui feature alone. jobs: update: + name: Update with egui alone timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - # macOS runners are kept for releases: macOS runs only when started by hand. - os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }} - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest steps: - name: Harden runner uses: step-security/harden-runner@v2.22.4 @@ -39,10 +37,6 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: stable - components: rustfmt, clippy - - run: cargo fmt --check - # Default (no egui) and the egui feature both build and lint cleanly. - - run: cargo clippy -p craft-update --all-targets --locked -- -D warnings + components: clippy - run: cargo clippy -p craft-update --all-targets --locked --features egui -- -D warnings - - run: cargo test -p craft-update --locked -- --nocapture - run: cargo test -p craft-update --locked --features egui -- --nocapture diff --git a/.github/workflows/vectorcraft-format.yml b/.github/workflows/vectorcraft-format.yml index 74144f30f..cd66615a5 100644 --- a/.github/workflows/vectorcraft-format.yml +++ b/.github/workflows/vectorcraft-format.yml @@ -12,20 +12,18 @@ permissions: contents: read concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued. - group: vectorcraft-format-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + # One group per pull request and per branch: a newer push cancels the run it supersedes, on main + # too, since the newest run checks everything before it. + group: vectorcraft-format-${{ github.ref }} + cancel-in-progress: true +# fmt, clippy and the tests run in the Workspace gate on every OS, and its msrv job checks the +# workspace on 1.95; this checks the web build on 1.95. jobs: vectorcraft-format: + name: VectorCraft format WASM timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - # macOS runners are kept for releases: macOS runs only when started by hand. - os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }} - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest steps: - name: Harden runner uses: step-security/harden-runner@v2.22.4 @@ -37,9 +35,5 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: "1.95" - components: rustfmt, clippy targets: wasm32-unknown-unknown - - run: cargo fmt --check - - run: cargo clippy -p craft-vectorcraft-format --all-targets --locked -- -D warnings - - run: cargo test -p craft-vectorcraft-format --locked -- --nocapture - run: cargo check -p craft-vectorcraft-format --locked --target wasm32-unknown-unknown diff --git a/.github/workflows/video.yml b/.github/workflows/video.yml index fa667bdfc..20afe0d70 100644 --- a/.github/workflows/video.yml +++ b/.github/workflows/video.yml @@ -11,6 +11,11 @@ on: permissions: contents: read +# A fraction of the fuzz-style cases, as in the Workspace gate; nightly.yml runs them in full. +env: + CRAFT_HOSTILE_DIVISOR: "8" + PROPTEST_CASES: "64" + concurrency: group: shared-video-${{ github.ref }} cancel-in-progress: true @@ -18,7 +23,8 @@ concurrency: jobs: lint: timeout-minutes: 20 - name: Video lint, feature sets, docs, WASM + # fmt and clippy with all features run in the Workspace gate on every OS. + name: Video feature sets, docs, WASM runs-on: ubuntu-latest steps: - name: Harden runner @@ -31,12 +37,8 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: "1.95" - components: rustfmt, clippy + components: clippy targets: wasm32-unknown-unknown - - uses: step-security/rust-cache@v2.9.2 - with: - key: craft-video-lint - - run: cargo fmt -p craft-video --check # Each feature on its own, so code that needs another feature fails here. - name: Clippy, each feature alone run: | @@ -45,19 +47,15 @@ jobs: cargo clippy -p craft-video --all-targets --locked --no-default-features ${f:+--features "$f"} -- -D warnings echo "::endgroup::" done - - run: cargo clippy -p craft-video --all-targets --locked --all-features -- -D warnings - run: cargo doc -p craft-video --all-features --no-deps --locked env: RUSTDOCFLAGS: "-D warnings" - run: cargo check -p craft-video --all-features --locked --target wasm32-unknown-unknown test: timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, windows-latest] - runs-on: ${{ matrix.os }} - name: Video tests (${{ matrix.os }}) + # The Workspace gate runs these tests on every OS without ffmpeg; this runs them with it. + runs-on: ubuntu-latest + name: Video tests with ffmpeg steps: - name: Harden runner uses: step-security/harden-runner@v2.22.4 @@ -69,11 +67,7 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: "1.95" - - uses: step-security/rust-cache@v2.9.2 - with: - key: craft-video-${{ matrix.os }} - - name: Install ffmpeg (Linux) - if: runner.os == 'Linux' + - name: Install ffmpeg run: | sudo rm -f /var/lib/man-db/auto-update sudo apt-get update -qq && sudo apt-get install -y -qq --no-install-recommends ffmpeg diff --git a/.github/workflows/web-api.yml b/.github/workflows/web-api.yml index 4bf1a30fe..f0f8e22da 100644 --- a/.github/workflows/web-api.yml +++ b/.github/workflows/web-api.yml @@ -12,10 +12,10 @@ permissions: contents: read concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued. - group: web-api-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + # One group per pull request and per branch: a newer push cancels the run it supersedes, on main + # too, since the newest run checks everything before it. + group: web-api-${{ github.ref }} + cancel-in-progress: true jobs: web-api: @@ -35,7 +35,7 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: stable - components: rustfmt, clippy + components: clippy targets: wasm32-unknown-unknown - name: Read the wasm-bindgen version from Cargo.lock id: wasm-bindgen @@ -50,8 +50,6 @@ jobs: with: tool: wasm-bindgen@${{ steps.wasm-bindgen.outputs.version }} fallback: cargo-install - - run: cargo fmt --check - - run: cargo clippy -p craft-web-api --all-targets --locked -- -D warnings + # fmt, native clippy and the native tests run in the Workspace gate. - run: cargo clippy -p craft-web-api --all-targets --locked --target wasm32-unknown-unknown -- -D warnings - - run: cargo test -p craft-web-api --locked - run: cargo test -p craft-web-api --locked --target wasm32-unknown-unknown diff --git a/.github/workflows/web-host.yml b/.github/workflows/web-host.yml index 76e6701e5..dc0254850 100644 --- a/.github/workflows/web-host.yml +++ b/.github/workflows/web-host.yml @@ -12,16 +12,16 @@ permissions: contents: read concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it - # always gets to save the cache. - group: shared-web-host-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + # One group per pull request and per branch: a newer push cancels the run it supersedes, on main + # too, since the newest run checks everything before it. + group: shared-web-host-${{ github.ref }} + cancel-in-progress: true jobs: check: timeout-minutes: 20 - name: Web host lint, tests, WASM tests + # fmt, native clippy and the native tests run in the Workspace gate. + name: Web host WASM lint and tests runs-on: ubuntu-latest env: CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUNNER: wasm-bindgen-test-runner @@ -36,16 +36,9 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: stable - components: rustfmt, clippy + components: clippy targets: wasm32-unknown-unknown - - uses: step-security/rust-cache@v2.9.2 - with: - key: craft-web-host - save-if: ${{ github.ref == 'refs/heads/main' }} - - run: cargo fmt -p craft-web-host --check - - run: cargo clippy -p craft-web-host --all-targets --locked -- -D warnings - run: cargo clippy -p craft-web-host --all-targets --locked --target wasm32-unknown-unknown -- -D warnings - - run: cargo test -p craft-web-host --locked - name: Read the wasm-bindgen version from Cargo.lock id: wasm-bindgen shell: bash diff --git a/.github/workflows/window.yml b/.github/workflows/window.yml index fdd28e203..c13b1ac3f 100644 --- a/.github/workflows/window.yml +++ b/.github/workflows/window.yml @@ -12,20 +12,18 @@ permissions: contents: read concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued. - group: window-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + # One group per pull request and per branch: a newer push cancels the run it supersedes, on main + # too, since the newest run checks everything before it. + group: window-${{ github.ref }} + cancel-in-progress: true +# fmt, clippy and the tests with default and with all features (egui and restore together) run in +# the Workspace gate on every OS; this runs each feature alone. jobs: window: + name: Window with each feature alone timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - # macOS runners are kept for releases: macOS runs only when started by hand. - os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }} - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest steps: - name: Harden runner uses: step-security/harden-runner@v2.22.4 @@ -39,12 +37,8 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: stable - components: rustfmt, clippy - - run: cargo fmt --check - # Default (no egui) and the egui feature both build and lint cleanly. - - run: cargo clippy -p craft-window --all-targets --locked -- -D warnings + components: clippy - run: cargo clippy -p craft-window --all-targets --locked --features egui -- -D warnings - - run: cargo test -p craft-window --locked -- --nocapture - run: cargo test -p craft-window --locked --features egui -- --nocapture - run: cargo clippy -p craft-window --all-targets --locked --features restore -- -D warnings - - run: cargo test -p craft-window --locked --features restore,egui -- --nocapture + - run: cargo test -p craft-window --locked --features restore -- --nocapture diff --git a/.github/workflows/workspace.yml b/.github/workflows/workspace.yml index 4d717fed8..af7e04769 100644 --- a/.github/workflows/workspace.yml +++ b/.github/workflows/workspace.yml @@ -6,17 +6,21 @@ on: tags: ["v*"] pull_request: workflow_dispatch: + # A full run on main every 6 hours that is never cancelled, so the caches are saved while pushes + # to main cancel each other. + schedule: + - cron: "17 */6 * * *" permissions: contents: read concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (push to main, a release tag, a manual run) gets a group of its own: it is neither - # cancelled nor replaced while queued, so a main run always finishes and saves the dependency - # cache, and a release run is never cancelled. - group: workspace-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + # A pull request, and main, each have one group, so a newer push cancels the run it supersedes; + # the newest push's run still tests everything before it. Every other run (a release tag, a + # manual run, the scheduled run) gets a group of its own and is neither cancelled nor replaced + # while queued: a release run always finishes, and the scheduled run saves the caches. + group: workspace-${{ (github.event_name == 'pull_request' || github.event_name == 'push' && github.ref == 'refs/heads/main') && github.ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || github.event_name == 'push' && github.ref == 'refs/heads/main' }} jobs: native: @@ -24,47 +28,58 @@ jobs: uses: storytold/craft-common-actions/.github/workflows/gate.yml@main with: os: '["ubuntu-latest","windows-latest","macos-latest"]' + # On a pull request, macOS and Windows run only when it can change platform behaviour: a + # crate with platform cfg code or one that depends on it, the build setup, or a file outside + # the crates (craft-common-actions' select-platforms). The full-ci label runs them anyway + # from the next push or manual run; pushes to main always run every leg. + select-os: '["windows-latest","macos-latest"]' timeout-minutes: 120 - # The Windows leg runs its tests with nextest (.config/nextest.toml). + # Every leg runs its tests with nextest, and its doctests with cargo test (.config/nextest.toml). tools: cargo-nextest@0.9.150 packages: mesa-vulkan-drivers libvulkan1 craft-fonts-ref: b076417b5880bd4154bd22f00695b1d28c6bea83 linux-env: WGPU_BACKEND=vulkan - windows-env: CRAFT_PRINT_REQUIRE_QUEUE=1 - env: CRAFT_PRINT_REQUIRE_QUEUE=0 - setup-command: | - set -euo pipefail - corpus_fetcher=.test-inputs/photocraft - mkdir -p "$corpus_fetcher" - git -C "$corpus_fetcher" init - git -C "$corpus_fetcher" remote add origin https://github.com/storytold/PhotoCraft.git - git -C "$corpus_fetcher" fetch --quiet --depth=1 origin 74330157cb601573ae4523a95bb9132822fc3918 - git -C "$corpus_fetcher" checkout --quiet --detach FETCH_HEAD - (cd "$corpus_fetcher" && cargo xtask corpus --psd --psd-tools --atn --exr --heif --pngsuite) - mkdir -p corpus - cp -a "$corpus_fetcher/corpus/." corpus/ - python scripts/test-fetch-pdf-test-fonts.py - python scripts/fetch-pdf-test-fonts.py "$RUNNER_TEMP/craft-pdf-test-fonts" - printf 'CRAFT_PSD_CORPUS=%s\n' "$GITHUB_WORKSPACE/$corpus_fetcher/corpus" >> "$GITHUB_ENV" - printf 'CRAFT_PDF_TEST_FONT_DIR=%s\n' "$RUNNER_TEMP/craft-pdf-test-fonts" >> "$GITHUB_ENV" + # No debuginfo on Windows: writing PDBs for every test binary is a large part of its link + # time. A panic still names its file and line; only backtrace frames lose theirs, and the + # Linux and macOS legs keep line tables. + windows-env: | + CRAFT_PRINT_REQUIRE_QUEUE=1 + CARGO_PROFILE_DEV_DEBUG=0 + # The fuzz-style tests run a fraction of their cases here: every hostile-input hammer and + # sweep 1/8 (craft-testkit's CRAFT_HOSTILE_DIVISOR), every proptest without its own case + # count 64 cases instead of 256. nightly.yml runs them all in full. + env: | + CRAFT_PRINT_REQUIRE_QUEUE=0 + CRAFT_HOSTILE_DIVISOR=8 + PROPTEST_CASES=64 + setup-command: bash scripts/ci-test-inputs.sh + # The PhotoCraft corpora, keyed on the script that pins them: a hit skips the PhotoCraft + # checkout and its xtask build (1:51 on Windows). + cache-paths: .test-inputs/photocraft/corpus + cache-key: photocraft-corpus + cache-key-files: scripts/ci-test-inputs.sh + # A `lint ()` leg beside each `gate ()` leg, so clippy is off the tests' critical path. + lint-command: | + if [ "$RUNNER_OS" = Linux ]; then + python3 scripts/test-workspace-concurrency.py + python3 scripts/test-workspace-tests.py + fi + cargo fmt --check + cargo clippy --workspace --all-targets --locked -- -D warnings + cargo clippy --workspace --all-targets --all-features --locked -- -D warnings command: | - python3 scripts/test-workspace-concurrency.py - cargo fmt --check - cargo clippy --workspace --all-targets --locked -- -D warnings - cargo clippy --workspace --all-targets --all-features --locked -- -D warnings - cargo test --workspace --locked - cargo test --workspace --all-features --locked - windows-command: | - powershell.exe -NoProfile -Command '$ErrorActionPreference = "Stop"; Start-Service Spooler; Add-PrinterDriver -Name "Generic / Text Only"' - cargo fmt --check - cargo clippy --workspace --all-targets --locked -- -D warnings - cargo clippy --workspace --all-targets --all-features --locked -- -D warnings - # The same tests as `cargo test`, run in parallel across test binaries instead of one - # binary at a time; nextest skips doctests, so `cargo test --doc` runs them. - cargo nextest run --workspace --locked --profile ci - cargo test --workspace --doc --locked + # The tests run once, with every feature, in parallel across test binaries; nextest skips + # doctests, so `cargo test --doc` runs them. The default-feature compile is the lint leg's + # first clippy run; the last command runs the crates whose tests check behaviour with a + # feature off, which an all-features build cannot reach. cargo nextest run --workspace --all-features --locked --profile ci cargo test --workspace --doc --all-features --locked + cargo nextest run -p craft-image-codecs -p craft-text --locked --profile ci + windows-command: | + powershell.exe -NoProfile -Command '$ErrorActionPreference = "Stop"; Start-Service Spooler; Add-PrinterDriver -Name "Generic / Text Only"' + cargo nextest run --workspace --all-features --locked --profile ci + cargo test --workspace --doc --all-features --locked + cargo nextest run -p craft-image-codecs -p craft-text --locked --profile ci msrv: name: Rust 1.95 diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index f8208725c..2b467131e 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -119,6 +119,7 @@ The XML reader in `crates/plist/src/xml.rs` and its base64 compatibility helper `crates/palette` extracts the GPL text parser from AniCraft `crates/doc/src/swatch.rs` at `d875a7feb250e6ab06aa28a37fb78a0699e88330` (MIT OR Apache-2.0), introduced by Tanish Manku in `a64130a9da9bdde545463b852d95d6a6774b1dfd` (AniCraft #270). The import preserves Tanish as primary author. Neutral RGB/name adaptation separates application display-name sanitization and the `Untitled` policy; validation tests and integration are subsequent Dylan M. Taylor changes. +The legacy ZIP Zstandard method-20 mapping and bounded-read regression in `crates/zip/src/lenient` are ported from RRFZ's Craft Anything commit `29badae7c9280fdf4c24795927bf3a91f51694c6` (MIT OR Apache-2.0). RRFZ remains the primary Git author. Firmware scanner detection remains app-owned and is not part of this port. ## TAR/CPIO compatibility updates The old GNU sparse-extension traversal, GNU CRC-symlink compatibility and PAX-size precedence fixes and their regression tests in `crates/tar` are ported from RRFZ's Craft Anything PRs #1973 (`8ea2ec66a1bc80f0420f3695baac877a0cb9b9a5`), #1984 (`b226a475f7dce9710e3af7a069386fe7a9720df7`) and #2007 (`d29465c20b0d570ea4973be512a0cb4e212bb60f`). Source is MIT OR Apache-2.0; RRFZ remains the primary Git author of each port. The PAX update adapts to the shared parser's existing `PaxOverrides` structure without changing the source policy. @@ -130,14 +131,26 @@ The USDZ alignment requirement and compatibility fixture for `Writer::add_stored The raw gradient wire ordering in `craft-lottie::Gradient::flat_values` follows AniCraft's Lottie exporter by Tanish Manku (`63b39355`, `b9258b7b`), MIT OR Apache-2.0. Sanitization, animation fitting and numeric output policy remain caller-owned. + ## Screen Video key frames `crates/video/src/screen_video.rs` extracts AniCraft `crates/xfl/src/video.rs` codec-3 decoding and its synthetic partial-block fixture, introduced by Tanish Manku in `1f3e603b7ff53a3da364e41ec4da8d43dc83aab0` with pixel-loop cleanup `2b7571363b4b647266df758f95b6e397021cfd4d` (MIT OR Apache-2.0). Import commits preserve Tanish as primary author. Caller-owned bounds and neutral RGBA output separate codec work from the consuming app's FLV first-frame/report policy. Boundary tests and integration are subsequent Dylan M. Taylor changes. +## Source Map v3 parser and lookup + +`crates/sourcemap` extracts the bounded parser, source inventory and generated-position lookup from Craft Anything `crates/formats/src/sourcemap.rs` at `e5c9c94106747ed44b9a3fb0decce867c2285f93` (MIT OR Apache-2.0). Core-history ports preserve Brandon Thomas, BFlat, Afloat, RRFZ, Nicolas Bohn and Alex Joaquim Pereira as primary authors, with exact source revisions in each commit. Six regression files preserve their source authors; nine core unit tests retain the original tests. App annotation scanning, data URLs, base64 and file I/O are intentionally not imported. Workspace wiring is a subsequent change; existing limits and errors are unchanged. ## JSONC span-preserving editing `crates/jsonc` is extracted from Craft Anything `crates/formats/src/jsonc.rs` at `e5c9c94106747ed44b9a3fb0decce867c2285f93` (MIT OR Apache-2.0). The original implementation and hardening by BFlat, newline preservation by NetterSilver, and subsequent RRFZ/Afloat correctness fixes retain separate primary-author port commits with immutable source revisions. Shared API wiring and optional input-byte admission are subsequent changes. Compatibility parsing and all existing formatting/editing tests are retained; application file I/O and policy are not imported. +## Java class inspection + +`crates/java-class` extracts Craft Anything `java_class.rs` and `java_ops.rs` at `5979f9b13f16dcf1df3c7ae7f8e7735f452866f2` (MIT OR Apache-2.0), preserving separate primary-author source-history ports by Brandon Thomas, BFlat, Afloat and RRFZ. The checked byte-reader, modified UTF-8 unit decoder, descriptor display helper and synthetic class fixtures come from the same source snapshot. Seven regression files retain their source authors. Neutral error wiring and shared crate integration are subsequent changes. No JAR archive, application I/O or bytecode execution is imported. + +## WebAssembly inspection + +`crates/wasm` extracts Craft Anything `crates/formats/src/wasm.rs` at `5979f9b13f16dcf1df3c7ae7f8e7735f452866f2`, with its checked byte-reader subset and original synthetic WASM/testkit fixtures (MIT OR Apache-2.0). Eight core-history ports preserve Brandon Thomas, Afloat and RRFZ as primary authors; component, repeated-custom-section and referenced-string regression imports retain source authors and commit references. Fixtures come from `crates/testkit/src/formats.rs` and deterministic mutation helpers from `crates/testkit/src/lib.rs`; no third-party binaries. Shared error naming and workspace wiring are subsequent changes. Existing inspection limits and report behavior are unchanged; this crate validates/inspects/renders, never executes modules. + ## Packet capture and protocol inspection `crates/pcap` extracts Craft Anything `crates/formats/src/pcap.rs`, its checked byte reader primitives and synthetic packet fixtures/regressions at `5979f9b13f16dcf1df3c7ae7f8e7735f452866f2` (MIT OR Apache-2.0). Brandon Thomas introduced the implementation, byte reader and fixtures in `f64ae3bf3c8863a982af1304e06de20653803968`. Subsequent fixes by phatloi0010, GGWA, BFlat, Afloat, NetterSilver, Max Mittelstädt and RRFZ retain separate primary-author source-history commits with immutable source revisions. Shared error/API wiring and chrono-based timestamp formatting with exhaustive day-boundary parity tests are subsequent Dylan M. Taylor changes. No packet reassembly, decryption, live capture or app analysis policy is imported. @@ -148,3 +161,13 @@ Sanitization, animation fitting and numeric output policy remain caller-owned. ## Dalvik DEX inspection `crates/dex` extracts Craft Anything `crates/formats/src/dex.rs` and `dex_ops.rs` at `5979f9b13f16dcf1df3c7ae7f8e7735f452866f2` (MIT OR Apache-2.0). Five source-history ports preserve Brandon Thomas, BFlat and Afloat as primary authors. Five regression imports retain BFlat/Afloat authors and exact source revisions. The checked byte-reader/MUTF-8 subset and synthetic DEX fixture come from that app's `bytes.rs` and `testkit/src/android.rs`; deterministic noise/mutations come from its testkit. No application file I/O, APK logic or execution is imported. Shared neutral errors and workspace wiring are subsequent changes; existing per-file strings, member/handler and bytecode payload budgets are retained. + +## EVM bytecode inspection + +`crates/evm` extracts Craft Anything `crates/formats/src/evm.rs` at `a31e995858f4312e0f1d59a007112da191ef2d26` (MIT OR Apache-2.0). Eight separate source-history ports preserve Brandon Thomas, Afloat, RRFZ and NetterSilver as primary authors with immutable source references. Synthetic EVM fixtures and deterministic mutation helpers come from the same app source; regression files retain their source provenance. Neutral errors and workspace integration are subsequent changes. No code execution or transaction behavior is imported. + +## XFDF field reader + +`crates/xfdf` extracts PDFCraft field reading from #5937 refresh `cb564fd827f572818e9ddb0ea5989a432b1f39fd` (MIT OR Apache-2.0). App-side annotation and form policy remains local; source history is retained in the original repository. + +Retained XFDF source contributors (move-aware blame): Brandon Thomas, Marco Arnulfo, pkibiego and storybored379; module split by aiongg. Subsequent budget/integration work by Dylan M. Taylor. diff --git a/Cargo.lock b/Cargo.lock index eee3e7ce5..62f2866e4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1920,6 +1920,16 @@ dependencies = [ "zip 9.1.3", ] +[[package]] +name = "craft-evm" +version = "0.1.0" +dependencies = [ + "craft-testkit", + "serde", + "serde_json", + "thiserror 2.0.21", +] + [[package]] name = "craft-ews" version = "0.1.0" @@ -2300,6 +2310,15 @@ dependencies = [ "widestring", ] +[[package]] +name = "craft-java-class" +version = "0.1.0" +dependencies = [ + "craft-testkit", + "serde", + "thiserror 2.0.21", +] + [[package]] name = "craft-jmap" version = "0.1.0" @@ -3111,6 +3130,15 @@ dependencies = [ name = "craft-sniff" version = "0.1.0" +[[package]] +name = "craft-sourcemap" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", + "thiserror 2.0.21", +] + [[package]] name = "craft-ssip" version = "0.1.0" @@ -3157,11 +3185,14 @@ dependencies = [ "craft-xml", "data-url", "flate2", + "gif", + "image-webp", "kurbo 0.13.1", "quick-xml 0.41.0", "resvg 0.48.1", "roxmltree 0.21.2", "thiserror 2.0.21", + "toml", "usvg 0.48.1", "vectorcraft-geom", "vectorcraft-pathops", @@ -3179,6 +3210,7 @@ dependencies = [ name = "craft-tar" version = "0.1.0" dependencies = [ + "craft-testkit", "serde", ] @@ -3365,6 +3397,17 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "craft-wasm" +version = "0.1.0" +dependencies = [ + "craft-testkit", + "serde", + "thiserror 2.0.21", + "wasmparser 0.261.0", + "wasmprinter", +] + [[package]] name = "craft-web-api" version = "0.1.0" @@ -3439,6 +3482,14 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "craft-xfdf" +version = "0.1.0" +dependencies = [ + "craft-xml", + "thiserror 2.0.21", +] + [[package]] name = "craft-xfl" version = "0.1.0" @@ -12048,7 +12099,7 @@ dependencies = [ "wasmi_collections", "wasmi_core", "wasmi_ir", - "wasmparser", + "wasmparser 0.228.0", ] [[package]] @@ -12088,6 +12139,28 @@ dependencies = [ "indexmap", ] +[[package]] +name = "wasmparser" +version = "0.261.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f20f20e44f7e8aeb6744823ea9d869ede51e51be4fdaedede2852282e54d2d8" +dependencies = [ + "bitflags 2.13.2", + "indexmap", + "semver", +] + +[[package]] +name = "wasmprinter" +version = "0.261.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86a21c9e3325655ba02de3e645bc8357a8b3651a2e3ada25b6dbc37560deb9b7" +dependencies = [ + "anyhow", + "termcolor", + "wasmparser 0.261.0", +] + [[package]] name = "wayland-backend" version = "0.3.17" diff --git a/Cargo.toml b/Cargo.toml index 44960fcbd..ddb423c09 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,8 +11,12 @@ rust-version = "1.95" repository = "https://github.com/storytold/craft-libs" [workspace.dependencies] +craft-evm = { path = "crates/evm" } +craft-java-class = { path = "crates/java-class" } +craft-sourcemap = { path = "crates/sourcemap" } craft-pcap = { path = "crates/pcap" } craft-har = { path = "crates/har" } +craft-xfdf = { path = "crates/xfdf" } craft-jsonc = { path = "crates/jsonc" } craft-palette = { path = "crates/palette" } craft-score = { path = "crates/score" } @@ -28,6 +32,7 @@ acadrust = "=0.6.6" log = "0.4" # internal: add each crate here as `craft- = { path = "crates/" }` craft-plist = { path = "crates/plist" } +craft-wasm = { path = "crates/wasm" } craft-platform = { path = "crates/platform" } craft-aep = { path = "crates/aep" } craft-affinity = { path = "crates/affinity" } diff --git a/crates/cms/README.md b/crates/cms/README.md index 71f3146d5..cbee391cd 100644 --- a/crates/cms/README.md +++ b/crates/cms/README.md @@ -81,6 +81,12 @@ Buffers are checked once per call, before any pixel is written. the exact and float paths compare each pixel with white; all other colours are left as the profiles give them. With a profile whose white falls between its nodes (`coated-cmyk`: 0.4 % C), the last step to white is that much larger than its neighbours. +* Black preservation (opt-in, `BlackPreserving`): for CMYK → CMYK, a colour with exactly zero + C, M and Y comes out as exactly `(0, 0, 0, K')`, where `K'` is the destination black whose + K-only L* equals the source's (both K ramps sampled at 1024 points when the transform is + built, with its intent and BPC). The test is exact zero, so an input with any chromatic ink + takes the ordinary path. Absolute colorimetric, non-CMYK profiles and K ramps that reverse at + one of the 1024 samples are refused (a reversal between samples is not detected). ## Built-in profiles (all CC0-1.0, generated by this crate) diff --git a/crates/cms/profiles/craft-coated-fogra39.icc b/crates/cms/profiles/craft-coated-fogra39.icc index 13590238e..1ef3cad1f 100644 Binary files a/crates/cms/profiles/craft-coated-fogra39.icc and b/crates/cms/profiles/craft-coated-fogra39.icc differ diff --git a/crates/cms/src/black.rs b/crates/cms/src/black.rs new file mode 100644 index 000000000..f80defeef --- /dev/null +++ b/crates/cms/src/black.rs @@ -0,0 +1,205 @@ +//! K-only black preservation for CMYK → CMYK transforms. +//! +//! A CMYK → CMYK transform goes through the three-component PCS, so black alone (C = M = Y = 0) +//! comes out as a four-ink mix. ISO 32000-2 §8.6.5.7 notes that this loses fidelity in the black +//! component. ICC defines no black preservation; this is a named policy the caller opts into. +//! +//! [`BlackPreserving`] converts like its [`Transform`], except that a colour with exactly zero +//! cyan, magenta and yellow becomes exactly `(0, 0, 0, K')`. `K'` is the destination black whose +//! K-only L* equals the source's, both measured to Lab with the transform's own intent and BPC +//! setting: both ramps are sampled once, at construction, and the destination ramp is inverted. +//! A source black darker than the destination's maximum black maps to K = 1. The test is exact zero, so the transform is discontinuous at C = M = Y = 0: an input +//! with any chromatic ink takes the ordinary path. + +use crate::transform::{Transform, TransformOptions}; +use crate::{Builtin, CmsError, ColorSpace, Intent, Profile, invalid_arg}; + +/// Samples along each K-only ramp. Linear interpolation between samples of a smooth ramp is far +/// below the 16-bit quantum. +const K_SAMPLES: usize = 1024; +/// Slack for the monotonicity checks (L* units, K units). +const MONOTONIC_EPS: f64 = 1e-6; + +/// A CMYK → CMYK transform that keeps black-only colours black-only. +#[derive(Debug)] +pub struct BlackPreserving { + transform: Transform, + /// Destination K for source K at `i / (K_SAMPLES - 1)`; `K_SAMPLES` values in `[0, 1]`. + curve: Vec, + /// Source K from which the source black is at least as dark as the destination's darkest: + /// mapped to exactly 1. Infinite when the destination's black is the darker. + saturate: f32, + /// The mapping at the 256 8-bit source codes, already quantised. + curve8: [u8; 256], +} + +impl BlackPreserving { + /// Transform from `src` to `dst` with black preservation. + /// + /// Errors with [`CmsError::InvalidArgument`] when either profile is not CMYK or the intent is + /// absolute colorimetric (which reproduces the source's paper and black, so there is no + /// black to map), and with [`CmsError::Unsupported`] when a profile's K-only ramp is not + /// monotonic at its 1024 samples (black that gets lighter as K rises has no equal-lightness + /// inverse; a reversal between samples is not detected). Other errors + /// are those of [`Transform::with_options`]. + pub fn new(src: &Profile, dst: &Profile, opts: TransformOptions) -> Result { + if src.color_space != ColorSpace::Cmyk || dst.color_space != ColorSpace::Cmyk { + return Err(invalid_arg("black preservation needs CMYK source and destination profiles")); + } + if opts.intent == Intent::AbsoluteColorimetric { + return Err(invalid_arg("black preservation does not apply to absolute colorimetric")); + } + let transform = Transform::with_options(src, dst, opts)?; + let src_l = k_ramp_lightness(src, opts)?; + let dst_l = k_ramp_lightness(dst, opts)?; + let (curve, saturate) = k_curve(&src_l, &dst_l)?; + let mut curve8 = [0u8; 256]; + for (code, slot) in curve8.iter_mut().enumerate() { + *slot = quantise(map(&curve, saturate, code as f32 / 255.0), 255.0) as u8; + } + Ok(BlackPreserving { transform, curve, saturate, curve8 }) + } + + /// The ordinary transform, which every colour with chromatic ink goes through. Converting with + /// it directly does not preserve black. + pub fn transform(&self) -> &Transform { + &self.transform + } + + /// Destination K for a black-only source colour of `k` (clamped to `[0, 1]`). + pub fn map_k(&self, k: f32) -> f32 { + map(&self.curve, self.saturate, k) + } + + /// Exact evaluation of one colour; slices as in [`Transform::eval`]. + pub fn eval(&self, input: &[f32], out: &mut [f32]) -> Result<(), CmsError> { + self.transform.eval(input, out)?; + self.patch_one(input, out); + Ok(()) + } + + /// Lookup-table evaluation of one colour; slices as in [`Transform::eval`]. + pub fn eval_fast(&self, input: &[f32], out: &mut [f32]) -> Result<(), CmsError> { + self.transform.eval_fast(input, out)?; + self.patch_one(input, out); + Ok(()) + } + + /// Converts float pixels; layout and errors as in [`Transform::convert_f32`]. + pub fn convert_f32(&self, src: &[f32], src_stride: usize, dst: &mut [f32], dst_stride: usize, copy_extra: bool) -> Result<(), CmsError> { + self.transform.convert_f32(src, src_stride, dst, dst_stride, copy_extra)?; + for (s, d) in src.chunks_exact(src_stride).zip(dst.chunks_exact_mut(dst_stride)) { + self.patch_one(s, d); + } + Ok(()) + } + + /// Converts 8-bit pixels; layout and errors as in [`Transform::convert_u8`]. + pub fn convert_u8(&self, src: &[u8], src_stride: usize, dst: &mut [u8], dst_stride: usize, copy_extra: bool) -> Result<(), CmsError> { + self.transform.convert_u8(src, src_stride, dst, dst_stride, copy_extra)?; + for (s, d) in src.chunks_exact(src_stride).zip(dst.chunks_exact_mut(dst_stride)) { + if let ([0, 0, 0, k, ..], [c, m, y, kd, ..]) = (s, d) { + (*c, *m, *y) = (0, 0, 0); + *kd = self.curve8[usize::from(*k)]; + } + } + Ok(()) + } + + /// Converts 16-bit pixels; layout and errors as in [`Transform::convert_u16`]. + pub fn convert_u16(&self, src: &[u16], src_stride: usize, dst: &mut [u16], dst_stride: usize, copy_extra: bool) -> Result<(), CmsError> { + self.transform.convert_u16(src, src_stride, dst, dst_stride, copy_extra)?; + for (s, d) in src.chunks_exact(src_stride).zip(dst.chunks_exact_mut(dst_stride)) { + if let ([0, 0, 0, k, ..], [c, m, y, kd, ..]) = (s, d) { + (*c, *m, *y) = (0, 0, 0); + *kd = quantise(self.map_k(f32::from(*k) / 65535.0), 65535.0) as u16; + } + } + Ok(()) + } + + fn patch_one(&self, input: &[f32], out: &mut [f32]) { + if let ([0.0, 0.0, 0.0, k, ..], [c, m, y, kd, ..]) = (input, out) { + (*c, *m, *y) = (0.0, 0.0, 0.0); + *kd = self.map_k(*k); + } + } +} + +/// The sampled mapping from the source to the destination K-only ramp, and the source K at +/// which the destination's darkest black is reached (the `saturate` field). +fn k_curve(src_l: &[f64], dst_l: &[f64]) -> Result<(Vec, f32), CmsError> { + let curve: Vec = src_l.iter().map(|&l| invert(dst_l, l) as f32).collect(); + if !curve.windows(2).all(|w| f64::from(w[0]) - f64::from(w[1]) <= MONOTONIC_EPS) { + return Err(CmsError::Unsupported("black preservation: the K mapping is not monotonic".into())); + } + let saturate = match (src_l.last(), dst_l.last()) { + (Some(&s), Some(&d)) if s < d => invert(src_l, d) as f32, + _ => f32::INFINITY, + }; + Ok((curve, saturate)) +} + +/// L* of `p`'s K-only ramp at `K_SAMPLES` points, light to dark; refused when it is not +/// non-increasing. +fn k_ramp_lightness(p: &Profile, opts: TransformOptions) -> Result, CmsError> { + let to_lab = Transform::with_options(p, Builtin::LabD50.profile(), TransformOptions { precise_float: true, ..opts })?; + let mut ramp = Vec::with_capacity(K_SAMPLES); + let mut lab = [0.0f32; 3]; + for i in 0..K_SAMPLES { + let k = i as f32 / (K_SAMPLES - 1) as f32; + to_lab.eval(&[0.0, 0.0, 0.0, k], &mut lab)?; + ramp.push(f64::from(lab[0]) * 100.0); + } + if !ramp.iter().all(|l| l.is_finite()) || !ramp.windows(2).all(|w| w[1] - w[0] <= MONOTONIC_EPS) { + return Err(CmsError::Unsupported("black preservation: a K-only ramp is not monotonic".into())); + } + Ok(ramp) +} + +/// The K in `[0, 1]` at which the non-increasing `ramp` reaches `target`, clamped to its ends. +fn invert(ramp: &[f64], target: f64) -> f64 { + let (Some(&first), Some(&last)) = (ramp.first(), ramp.last()) else { return 0.0 }; + if ramp.len() < 2 || target >= first { + return 0.0; + } + if target <= last { + return 1.0; + } + // ramp[lo] > target >= ramp[hi] + let (mut lo, mut hi) = (0usize, ramp.len() - 1); + while hi - lo > 1 { + let mid = lo + (hi - lo) / 2; + if ramp.get(mid).is_some_and(|&v| v > target) { + lo = mid; + } else { + hi = mid; + } + } + let (a, b) = (ramp.get(lo).copied().unwrap_or(first), ramp.get(hi).copied().unwrap_or(last)); + let t = if a - b > MONOTONIC_EPS { (a - target) / (a - b) } else { 0.0 }; + (lo as f64 + t) / (ramp.len() - 1) as f64 +} + +/// Linear interpolation in `curve` (samples over `[0, 1]`) at `k`, clamped; 1 from `saturate`. +fn map(curve: &[f32], saturate: f32, k: f32) -> f32 { + let Some(&last) = curve.last() else { return 0.0 }; + let k = if k.is_nan() { 0.0 } else { k.clamp(0.0, 1.0) }; + if k >= saturate { + return 1.0; + } + let x = k * (curve.len() - 1) as f32; + let i = x.floor() as usize; + match (curve.get(i), curve.get(i + 1)) { + (Some(&a), Some(&b)) => a + (b - a) * (x - i as f32), + _ => last, + } +} + +fn quantise(v: f32, max: f32) -> f32 { + (v.clamp(0.0, 1.0) * max + 0.5).floor() +} + +#[cfg(test)] +#[path = "black_tests.rs"] +mod tests; diff --git a/crates/cms/src/black_tests.rs b/crates/cms/src/black_tests.rs new file mode 100644 index 000000000..8752cdaa2 --- /dev/null +++ b/crates/cms/src/black_tests.rs @@ -0,0 +1,149 @@ +use super::*; + +fn coated() -> &'static Profile { + Builtin::CoatedCmyk.profile() +} +fn fogra() -> &'static Profile { + Builtin::CoatedFogra39.profile() +} +fn opts() -> TransformOptions { + TransformOptions::default() +} + +/// L* of a CMYK colour under `p`, measured as the transform with `o` measures it. +fn lightness(p: &Profile, o: TransformOptions, cmyk: [f32; 4]) -> f32 { + let t = Transform::with_options(p, Builtin::LabD50.profile(), o).unwrap(); + let mut lab = [0.0f32; 3]; + t.eval(&cmyk, &mut lab).unwrap(); + lab[0] * 100.0 +} + +#[test] +fn the_plain_transform_turns_black_into_four_inks() { + // What the policy is for: without it, black alone gains chromatic ink across presses. + let t = Transform::with_options(coated(), fogra(), opts()).unwrap(); + let mut out = [0.0f32; 4]; + t.eval(&[0.0, 0.0, 0.0, 0.5], &mut out).unwrap(); + assert!(out[..3].iter().any(|v| *v > 0.01), "{out:?}"); +} + +#[test] +fn black_only_stays_black_only_on_every_path() { + for (src, dst) in [(coated(), fogra()), (fogra(), coated()), (fogra(), fogra())] { + let bp = BlackPreserving::new(src, dst, opts()).unwrap(); + for k in [0.0f32, 0.1, 0.5, 0.9, 1.0] { + for fast in [false, true] { + let mut out = [1.0f32; 4]; + if fast { bp.eval_fast(&[0.0, 0.0, 0.0, k], &mut out) } else { bp.eval(&[0.0, 0.0, 0.0, k], &mut out) }.unwrap(); + assert_eq!(out[..3], [0.0; 3], "k {k} fast {fast}"); + assert_eq!(out[3], bp.map_k(k)); + } + } + let src8: Vec = (0..=255u8).flat_map(|k| [0, 0, 0, k, 7]).collect(); + let mut dst8 = vec![0u8; src8.len()]; + bp.convert_u8(&src8, 5, &mut dst8, 5, true).unwrap(); + for (k, px) in dst8.as_chunks::<5>().0.iter().enumerate() { + assert_eq!(px[..3], [0; 3], "u8 k {k}"); + assert!((f32::from(px[3]) - bp.map_k(k as f32 / 255.0) * 255.0).abs() <= 0.5, "u8 k {k}: {}", px[3]); + assert_eq!(px[4], 7, "extra channel copied"); + } + let src16: Vec = (0..=64u16).flat_map(|k| [0, 0, 0, k * 1023]).collect(); + let mut dst16 = vec![0u16; src16.len()]; + bp.convert_u16(&src16, 4, &mut dst16, 4, false).unwrap(); + for (px, k) in dst16.as_chunks::<4>().0.iter().zip(src16.as_chunks::<4>().0) { + assert_eq!(px[..3], [0; 3], "u16 {k:?}"); + assert!((f32::from(px[3]) - bp.map_k(f32::from(k[3]) / 65535.0) * 65535.0).abs() <= 0.5, "u16 {k:?}: {}", px[3]); + } + let srcf: Vec = (0..=10).flat_map(|k| [0.0, 0.0, 0.0, k as f32 / 10.0]).collect(); + let mut dstf = vec![0.0f32; srcf.len()]; + bp.convert_f32(&srcf, 4, &mut dstf, 4, false).unwrap(); + for (px, k) in dstf.as_chunks::<4>().0.iter().zip(srcf.as_chunks::<4>().0) { + assert_eq!(*px, [0.0, 0.0, 0.0, bp.map_k(k[3])], "f32 {k:?}"); + } + } +} + +#[test] +fn the_destination_black_matches_the_source_lightness() { + // Measured worst error over 201 K steps, both directions, both intents, BPC on and off: + // 0.007 L*. + for intent in [Intent::RelativeColorimetric, Intent::Perceptual] { + for bpc in [true, false] { + let o = TransformOptions { intent, bpc, ..opts() }; + for (src, dst) in [(coated(), fogra()), (fogra(), coated())] { + let bp = BlackPreserving::new(src, dst, o).unwrap(); + let dst_black = lightness(dst, o, [0.0, 0.0, 0.0, 1.0]); + for k in [0.1f32, 0.3, 0.5, 0.7, 0.9, 1.0] { + let want = lightness(src, o, [0.0, 0.0, 0.0, k]); + if want <= dst_black { + assert_eq!(bp.map_k(k), 1.0, "darker than the destination's black clamps to K = 1"); + continue; + } + let got = lightness(dst, o, [0.0, 0.0, 0.0, bp.map_k(k)]); + assert!((got - want).abs() < 0.02, "{intent:?} bpc {bpc} k {k}: L* {got} for {want}"); + } + assert_eq!(bp.map_k(0.0), 0.0, "paper stays paper"); + assert!(bp.map_k(1.0) > 0.95, "full black stays near full black: {}", bp.map_k(1.0)); + assert!(bp.map_k(0.25) < bp.map_k(0.75)); + } + } + } + let same = BlackPreserving::new(fogra(), fogra(), opts()).unwrap(); + for k in [0.0f32, 0.2, 0.6, 1.0] { + assert!((same.map_k(k) - k).abs() < 1e-3, "{k} -> {}", same.map_k(k)); + } +} + +#[test] +fn colours_with_any_chromatic_ink_take_the_ordinary_path() { + let bp = BlackPreserving::new(coated(), fogra(), opts()).unwrap(); + for input in [[1e-6f32, 0.0, 0.0, 0.5], [0.0, 0.2, 0.0, 0.5], [0.3, 0.3, 0.3, 0.0], [0.0, 0.0, 1.0, 1.0]] { + let (mut a, mut b) = ([0.0f32; 4], [0.0f32; 4]); + bp.eval(&input, &mut a).unwrap(); + bp.transform().eval(&input, &mut b).unwrap(); + assert_eq!(a, b, "{input:?}"); + } + let src = [1u8, 0, 0, 128, 0, 1, 0, 128]; + let (mut a, mut b) = ([0u8; 8], [0u8; 8]); + bp.convert_u8(&src, 4, &mut a, 4, false).unwrap(); + bp.transform().convert_u8(&src, 4, &mut b, 4, false).unwrap(); + assert_eq!(a, b); +} + +#[test] +fn refusals_and_hostile_values() { + let srgb = Builtin::Srgb.profile(); + assert!(matches!(BlackPreserving::new(srgb, fogra(), opts()), Err(CmsError::InvalidArgument(_)))); + assert!(matches!(BlackPreserving::new(fogra(), srgb, opts()), Err(CmsError::InvalidArgument(_)))); + let abs = TransformOptions { intent: Intent::AbsoluteColorimetric, ..opts() }; + assert!(matches!(BlackPreserving::new(fogra(), coated(), abs), Err(CmsError::InvalidArgument(_)))); + let bp = BlackPreserving::new(coated(), fogra(), opts()).unwrap(); + for k in [f32::NAN, f32::INFINITY, f32::NEG_INFINITY, -1.0, 2.0] { + assert!((0.0..=1.0).contains(&bp.map_k(k)), "{k}"); + } + let mut out = [0.0f32; 4]; + assert!(bp.eval(&[0.0; 3], &mut out).is_err(), "short input"); + assert!(bp.convert_u8(&[0, 0, 0, 9, 0], 4, &mut [0u8; 4], 4, false).is_err(), "partial pixel"); + assert_eq!(invert(&[], 50.0), 0.0); + assert_eq!(invert(&[100.0], 50.0), 0.0); + assert_eq!(invert(&[100.0, 0.0], 150.0), 0.0); + assert_eq!(invert(&[100.0, 0.0], -5.0), 1.0); + assert!((invert(&[100.0, 50.0, 0.0], 25.0) - 0.75).abs() < 1e-12); + assert_eq!(map(&[], f32::INFINITY, 0.5), 0.0); +} + +#[test] +fn black_beyond_the_destination_maximum_maps_to_full_black_between_samples() { + // Source L* = 100 (1 - K), destination L* = 100 - 50 K: from K = 0.5 the source is darker + // than any destination black, and 0.5 falls between the 1024 samples. + let ramp = |f: fn(f64) -> f64| (0..K_SAMPLES).map(|i| f(i as f64 / (K_SAMPLES - 1) as f64)).collect::>(); + let (src, dst) = (ramp(|k| 100.0 * (1.0 - k)), ramp(|k| 100.0 - 50.0 * k)); + let (curve, saturate) = k_curve(&src, &dst).unwrap(); + assert!((saturate - 0.5).abs() < 1e-6, "{saturate}"); + for k in [0.5001f32, 0.6, 1.0] { + assert_eq!(map(&curve, saturate, k), 1.0, "{k}"); + } + assert!((map(&curve, saturate, 0.25) - 0.5).abs() < 1e-4); + let (_, never) = k_curve(&dst, &src).unwrap(); + assert_eq!(never, f32::INFINITY, "a lighter source never saturates"); +} diff --git a/crates/cms/src/lib.rs b/crates/cms/src/lib.rs index b082f67a5..38f926dcf 100644 --- a/crates/cms/src/lib.rs +++ b/crates/cms/src/lib.rs @@ -19,6 +19,7 @@ //! CIE 15, and Adobe's published black point compensation paper. #![forbid(unsafe_code)] +pub mod black; pub mod builtin; mod builtin_v2; pub mod clut; @@ -39,6 +40,7 @@ pub mod transform; mod write; pub mod write_v2; +pub use black::BlackPreserving; pub use builtin::Builtin; pub use clut::Clut; pub use curve::Curve; diff --git a/crates/cms/src/synth.rs b/crates/cms/src/synth.rs index b1535f78c..d192127ec 100644 --- a/crates/cms/src/synth.rs +++ b/crates/cms/src/synth.rs @@ -19,6 +19,9 @@ //! L* is scaled onto the device black, chroma is softly compressed against a gamut boundary //! descriptor sampled from the forward model — then solved as above. //! +//! That is [`GamutMapping::Legacy`]. [`GamutMapping::Oklab`] maps out-of-gamut colours in Oklab +//! instead and gives saturation its own table; the Coated FOGRA39 built-in uses it. +//! //! The model is plausible for coated offset, not a measured characterisation: use a real //! profile from your print provider for production print. //! @@ -49,6 +52,11 @@ pub struct CmykParams { pub k_max: f64, /// Exponent of the black generation curve above `gcr_start`. pub gcr_gamma: f64, + /// Black generation as a curve of `[gray component, black]` points with increasing gray + /// components in `[0, 1]`, interpolated linearly and held flat beyond the ends. Empty by + /// default: black is then `k_max` times the ramp of `gcr_start` and `gcr_gamma` (the curve + /// of the shipped Photocraft profile). + pub gcr_curve: Vec<[f64; 2]>, /// Tone value increase at 50 % for C, M, Y, K. pub tvi: [f64; 4], /// Grid points of the AToB (4D) and BToA (3D) tables. @@ -60,6 +68,32 @@ pub struct CmykParams { /// encoding and separates to a trace of ink. Off by default, which keeps the bytes of the /// shipped Photocraft profile. pub white_on_node: bool, + /// How the BToA tables bring colours into the gamut. [`GamutMapping::Legacy`] by default, + /// which keeps the bytes of the shipped Photocraft profile. + pub mapping: GamutMapping, +} + +/// How the BToA tables map colours the printer cannot reach. +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum GamutMapping { + /// The shipped Photocraft profile: out-of-gamut colours land on the closest reachable + /// colour (ΔE76); perceptual scales L* onto the device black and compresses Lab chroma; + /// saturation shares the perceptual table. + Legacy, + /// Gamut mapping in Oklab, whose hue lines stay perceptually constant (a CIELAB hue line + /// from a saturated blue toward grey drifts toward purple; an Oklab one does not): + /// + /// * **Colorimetric**: in-gamut colours separate exactly. An out-of-gamut colour takes the + /// closest colour the printer reaches in Oklab, lightness and hue weighted above chroma, + /// and more so the further outside it is; black is generated from the clipped colour, so + /// inks the clip drives to 100 % stay there. + /// * **Perceptual** and **saturation**: the ICC v4 perceptual reference black is mapped + /// onto the device black linearly in XYZ (as black point compensation does) for neutrals, + /// fading out with Oklab chroma, so greys separate as with colorimetric + BPC and colourful + /// colours keep their lightness. Perceptual then compresses Oklab chroma above `knee` × + /// the gamut boundary into the boundary; saturation scales Oklab chroma by `saturation` + /// and clips as colorimetric does. + Oklab { knee: f64, saturation: f64 }, } impl Default for CmykParams { @@ -71,10 +105,12 @@ impl Default for CmykParams { gcr_start: 0.2, k_max: 0.95, gcr_gamma: 1.1, + gcr_curve: Vec::new(), tvi: [0.14, 0.14, 0.14, 0.17], grid_a2b: 11, grid_b2a: 21, white_on_node: false, + mapping: GamutMapping::Legacy, } } } @@ -183,6 +219,11 @@ impl Forward for TableForward { /// Bounded least-squares solve for C, M, Y given K, minimising ΔE76 to `target`, subject to /// `C + M + Y ≤ limit` (projected Levenberg–Marquardt with a numerical Jacobian). fn solve_cmy(model: &F, target: [f64; 3], k: f64, limit: f64, seeds: &[[f64; 3]]) -> ([f64; 3], f64) { + solve_cmy_by(model, k, limit, seeds, |l| [l[0] - target[0], l[1] - target[1], l[2] - target[2]]) +} + +/// [`solve_cmy`] minimising the norm of `residual(lab)` instead of ΔE76. +fn solve_cmy_by(model: &F, k: f64, limit: f64, seeds: &[[f64; 3]], residual: impl Fn([f64; 3]) -> [f64; 3]) -> ([f64; 3], f64) { let project = |mut x: [f64; 3]| { for _ in 0..4 { for v in &mut x { @@ -202,10 +243,7 @@ fn solve_cmy(model: &F, target: [f64; 3], k: f64, limit: f6 } x }; - let err = |x: [f64; 3]| { - let l = model.lab([x[0], x[1], x[2], k]); - [l[0] - target[0], l[1] - target[1], l[2] - target[2]] - }; + let err = |x: [f64; 3]| residual(model.lab([x[0], x[1], x[2], k])); let norm2 = |e: [f64; 3]| e[0] * e[0] + e[1] * e[1] + e[2] * e[2]; let mut best = ([0.0; 3], f64::INFINITY); for seed in seeds { @@ -275,13 +313,32 @@ fn solve_cmy(model: &F, target: [f64; 3], k: f64, limit: f6 const SEEDS: [[f64; 3]; 5] = [[0.3, 0.3, 0.3], [0.05, 0.05, 0.05], [0.8, 0.8, 0.8], [0.9, 0.2, 0.2], [0.2, 0.9, 0.9]]; +/// Black for a gray component `g` (the smallest of C, M, Y without black). +fn gcr(p: &CmykParams, g: f64) -> f64 { + let Some(first) = p.gcr_curve.first() else { + // The Photocraft profile's curve. + return if g <= p.gcr_start { 0.0 } else { p.k_max * ((g - p.gcr_start) / (1.0 - p.gcr_start)).powf(p.gcr_gamma) }; + }; + let mut k = first[1]; + for w in p.gcr_curve.windows(2) { + let ([g0, k0], [g1, k1]) = (w[0], w[1]); + if g >= g1 { + k = k1; + } else if g > g0 { + k = k0 + (k1 - k0) * (g - g0) / (g1 - g0); + break; + } + } + k.clamp(0.0, 1.0) +} + /// Colorimetric separation of a relative Lab target: GCR black generation then a CMY solve. pub fn separate(model: &F, p: &CmykParams, target: [f64; 3], warm: Option<[f64; 4]>) -> ([f64; 4], f64) { let mut seeds: Vec<[f64; 3]> = warm.map(|w| vec![[w[0], w[1], w[2]]]).unwrap_or_default(); seeds.extend_from_slice(&SEEDS); let (cmy0, _) = solve_cmy(model, target, 0.0, 3.0, &seeds); let g = cmy0[0].min(cmy0[1]).min(cmy0[2]); - let k = if g <= p.gcr_start { 0.0 } else { p.k_max * ((g - p.gcr_start) / (1.0 - p.gcr_start)).powf(p.gcr_gamma) }; + let k = gcr(p, g); let mut seeds2 = vec![[(cmy0[0] - k).max(0.0), (cmy0[1] - k).max(0.0), (cmy0[2] - k).max(0.0)], cmy0]; seeds2.extend_from_slice(&seeds); let (cmy, de) = solve_cmy(model, target, k, (p.tac - k).max(0.0), &seeds2); @@ -383,6 +440,187 @@ impl Gbd { } } +/// ΔE76 under which a separation counts as reaching its target. +const REACH: f64 = 1.5; +/// Oklab chroma beyond any printable colour: the top of the boundary search. +const OKLAB_CHROMA_MAX: f64 = 0.5; +/// Oklab chroma above which the perceptual and saturation tables no longer lift black: they lift +/// neutrals fully, and less the more colourful a colour is. +const LIFT_CHROMA: f64 = 0.1; +/// Weight of lightness and hue against chroma when a colour just outside the gamut is clipped. +const CLIP_WEIGHT: f64 = 1.5; +/// Growth of that weight with the squared share of chroma the clip loses. +const CLIP_WEIGHT_GROWTH: f64 = 200.0; + +// Oklab (Björn Ottosson, 2020): XYZ (D65) → cone response → cube root → Lab-like axes. +const OKLAB_M1: math::Mat3 = + [[0.8189330101, 0.3618667424, -0.1288597137], [0.0329845436, 0.9293118715, 0.0361456387], [0.0482003018, 0.2643662691, 0.6338517070]]; +const OKLAB_M2: math::Mat3 = + [[0.2104542553, 0.7936177850, -0.0040720468], [1.9779984951, -2.4285922050, 0.4505937099], [0.0259040371, 0.7827717662, -0.8086757660]]; +const OKLAB_M1_INV: math::Mat3 = + [[1.2270138511, -0.5577999807, 0.2812561490], [-0.0405801784, 1.1122568696, -0.0716766787], [-0.0763812845, -0.4214819784, 1.5861632204]]; +const OKLAB_M2_INV: math::Mat3 = [[1.0, 0.3963377774, 0.2158037573], [1.0, -0.1055613458, -0.0638541728], [1.0, -0.0894841775, -1.2914855480]]; + +/// The [`GamutMapping::Oklab`] separations of one forward model. +struct OklabMapper<'a, F: Forward + ?Sized> { + model: &'a F, + p: &'a CmykParams, + knee: f64, + saturation: f64, + /// XYZ scale that maps the v4 perceptual reference black onto the device black (the white + /// stays fixed): `v' = a·v + w·(1 − a)`. + black_scale: f64, + to_d65: math::Mat3, + to_d50: math::Mat3, +} + +impl<'a, F: Forward + ?Sized> OklabMapper<'a, F> { + fn new(model: &'a F, p: &'a CmykParams, knee: f64, saturation: f64) -> Self { + let d65 = math::xy_to_xyz(math::D65_XY); + // The darkest neutral the printer reaches. + let (black, _) = separate(model, p, [0.0, 0.0, 0.0], None); + let device = math::lab_to_xyz([model.lab(black)[0], 0.0, 0.0], math::D50)[1]; + let reference = crate::transform::PERCEPTUAL_BLACK[1]; + OklabMapper { + model, + p, + knee, + saturation, + black_scale: (1.0 - device) / (1.0 - reference), + to_d65: math::bradford(math::D50, d65), + to_d50: math::bradford(d65, math::D50), + } + } + + fn oklab(&self, lab: [f64; 3]) -> [f64; 3] { + let xyz = math::apply(&self.to_d65, math::lab_to_xyz(lab, math::D50)); + math::apply(&OKLAB_M2, math::apply(&OKLAB_M1, xyz).map(f64::cbrt)) + } + + fn lab(&self, ok: [f64; 3]) -> [f64; 3] { + let lms = math::apply(&OKLAB_M2_INV, ok).map(|v| v * v * v); + math::xyz_to_lab(math::apply(&self.to_d50, math::apply(&OKLAB_M1_INV, lms)), math::D50) + } + + /// The most chroma (scale of `ok`'s chroma in `[0, top]`) the printer reaches at `ok`'s + /// lightness and hue, and its separation; `None` when even the neutral is out of reach. + fn boundary(&self, ok: [f64; 3], top: f64, warm: Option<[f64; 4]>) -> Option<(f64, [f64; 4])> { + let (grey, d) = separate(self.model, self.p, self.lab([ok[0], 0.0, 0.0]), warm); + if d > REACH { + return None; + } + let (mut lo, mut hi, mut best) = (0.0, top, grey); + for _ in 0..14 { + let mid = 0.5 * (lo + hi); + let (c, d) = separate(self.model, self.p, self.lab([ok[0], ok[1] * mid, ok[2] * mid]), Some(best)); + if d <= REACH { + lo = mid; + best = c; + } else { + hi = mid; + } + } + Some((lo, best)) + } + + /// Colorimetric: exact in gamut; outside, the closest colour the printer reaches in Oklab, + /// lightness and hue weighted above chroma, and more so the further outside the colour is. + fn clip(&self, lab: [f64; 3], warm: Option<[f64; 4]>) -> [f64; 4] { + let (cmyk, de) = separate(self.model, self.p, lab, warm); + if de <= REACH { + return cmyk; + } + let ok = self.oklab(lab); + let c0 = ok[1].hypot(ok[2]); + // Unit vectors along the colour's chroma and hue directions. + let (u, v) = if c0 > 1e-9 { ([ok[1] / c0, ok[2] / c0], [-ok[2] / c0, ok[1] / c0]) } else { ([1.0, 0.0], [0.0, 1.0]) }; + // Lightness and hue errors weighted by `w`, chroma error by 1; scaled by 100 so the + // solver's step tolerances suit it as they suit ΔE76. + let residual = |w: f64| { + move |l: [f64; 3]| { + let o = self.oklab(l); + let (da, db) = (o[1] - ok[1], o[2] - ok[2]); + [100.0 * w * (o[0] - ok[0]), 100.0 * (da * u[0] + db * u[1]), 100.0 * w * (da * v[0] + db * v[1])] + } + }; + let solve = |w: f64, k: f64, from: [f64; 3]| { + let mut seeds = vec![from]; + seeds.extend_from_slice(&SEEDS); + solve_cmy_by(self.model, k, (self.p.tac - k).max(0.0), &seeds, residual(w)).0 + }; + // A first clip without black measures how far outside the colour is. A colour just + // outside may give up a little lightness and hue to keep its chroma (a red reaches the + // full magenta and yellow edge); one far outside keeps its lightness and hue and loses + // chroma (a saturated blue does not slide along the gamut surface). + let first = solve(CLIP_WEIGHT, 0.0, [cmyk[0], cmyk[1], cmyk[2]]); + let o = self.oklab(self.model.lab([first[0], first[1], first[2], 0.0])); + let loss = if c0 > 1e-9 { (1.0 - o[1].hypot(o[2]) / c0).clamp(0.0, 1.0) } else { 0.0 }; + let w = CLIP_WEIGHT * (1.0 + CLIP_WEIGHT_GROWTH * loss * loss); + let free = solve(w, 0.0, first); + let norm = |c: [f64; 4]| residual(w)(self.model.lab(c)).iter().map(|v| v * v).sum::(); + // Candidates, the closest kept: the separation itself (a colour it nearly reaches stays + // continuous with its in-gamut neighbours), the clip with the separation's black, and + // the clip with black generated from the black-free clip's gray component (inks the + // clip drives to 100 % stay there and black replaces the rest). + let mut best = (norm(cmyk), cmyk); + for k in [cmyk[3], gcr(self.p, free[0].min(free[1]).min(free[2]))] { + for from in [[cmyk[0], cmyk[1], cmyk[2]], free] { + let cmy = solve(w, k, from); + let c = [cmy[0], cmy[1], cmy[2], k]; + let n = norm(c); + if n < best.0 { + best = (n, c); + } + } + } + // Near paper white the gamut narrows to a point: the separation's closest colour keeps + // light tints' chroma where holding lightness would drop them toward bare paper, and + // the table nodes there carry the in-gamut tints between them. The inks blend from the + // clip at L* 85 to the separation at L* 95. + let clip_share = ((95.0 - lab[0]) / 10.0).clamp(0.0, 1.0); + std::array::from_fn(|i| clip_share * best.1[i] + (1.0 - clip_share) * cmyk[i]) + } + + /// Maps the v4 perceptual reference black onto the device black, linearly in XYZ, for + /// neutrals; colourful colours keep their lightness. + fn lift_black(&self, lab: [f64; 3]) -> [f64; 3] { + let ok = self.oklab(lab); + let neutral = (1.0 - ok[1].hypot(ok[2]) / LIFT_CHROMA).clamp(0.0, 1.0); + let a = 1.0 - neutral * (1.0 - self.black_scale); + let xyz = math::lab_to_xyz(lab, math::D50); + math::xyz_to_lab(std::array::from_fn(|i| a * xyz[i] + math::D50[i] * (1.0 - a)), math::D50) + } + + fn perceptual(&self, lab: [f64; 3], warm: Option<[f64; 4]>) -> [f64; 4] { + let lab = self.lift_black(lab); + let ok = self.oklab(lab); + let c = ok[1].hypot(ok[2]); + if c < 1e-9 { + return self.clip(lab, warm); + } + let Some((s, edge)) = self.boundary(ok, OKLAB_CHROMA_MAX / c, warm) else { + return self.clip(lab, warm); + }; + // Chroma above the knee is compressed smoothly into the boundary chroma `cb`. + let cb = s * c; + let knee = self.knee * cb; + if c <= knee { + return self.clip(lab, warm); + } + let c2 = knee + (cb - knee) * ((c - knee) / (cb - knee).max(1e-9)).tanh(); + let f = c2 / c; + if f >= s { + return edge; + } + self.clip(self.lab([ok[0], ok[1] * f, ok[2] * f]), Some(edge)) + } + + fn saturation(&self, lab: [f64; 3], warm: Option<[f64; 4]>) -> [f64; 4] { + let ok = self.oklab(self.lift_black(lab)); + self.clip(self.lab([ok[0], ok[1] * self.saturation, ok[2] * self.saturation]), warm) + } +} + fn lab_node_legacy(i: usize, n: usize) -> f64 { i as f64 / (n - 1) as f64 } @@ -419,6 +657,18 @@ fn white_node_curve(l_axis: bool, x: f64) -> f64 { } fn b2a_table(model: &F, p: &CmykParams, gbd: Option<&Gbd>) -> Result { + b2a_table_with(p, |lab, warm| { + let target = match gbd { + Some(g) => g.compress(lab), + None => lab, + }; + separate(model, p, target, warm).0 + }) +} + +/// Samples a BToA table: `node` separates each relative Lab node, warm-started from the +/// previous node along b. +fn b2a_table_with(p: &CmykParams, node: impl Fn([f64; 3], Option<[f64; 4]>) -> [f64; 4] + Sync) -> Result { let n = p.grid_b2a.max(2); let nodes = n * n * n; let step = (n - 1) as f64; @@ -434,11 +684,7 @@ fn b2a_table(model: &F, p: &CmykParams, gbd: Option<&Gbd>) } else { decode_v2([lab_node_legacy(li, n), lab_node_legacy(ai, n), lab_node_legacy(bi, n)]) }; - let target = match gbd { - Some(g) => g.compress(lab), - None => lab, - }; - let (cmyk, _) = if p.white_on_node && lab == [100.0, 0.0, 0.0] { ([0.0; 4], 0.0) } else { separate(model, p, target, warm) }; + let cmyk = if p.white_on_node && lab == [100.0, 0.0, 0.0] { [0.0; 4] } else { node(lab, warm) }; warm = Some(cmyk); out.extend(cmyk.map(|v| v as f32)); } @@ -460,6 +706,15 @@ fn check_grids(p: &CmykParams) -> Result<(), CmsError> { if p.white_on_node && p.grid_b2a.is_multiple_of(2) { return Err(invalid_arg(format!("white on a node needs an odd BToA grid, not {}", p.grid_b2a))); } + let valid = |v: &[f64; 2]| v.iter().all(|x| (0.0..=1.0).contains(x)); + if !p.gcr_curve.iter().all(valid) || p.gcr_curve.windows(2).any(|w| w[1][0] <= w[0][0]) { + return Err(invalid_arg(format!("black generation curve {:?} needs points in [0, 1] with increasing gray components", p.gcr_curve))); + } + if let GamutMapping::Oklab { knee, saturation } = p.mapping + && !((0.0..1.0).contains(&knee) && saturation.is_finite() && saturation > 0.0) + { + return Err(invalid_arg(format!("Oklab gamut mapping needs a knee in [0, 1) and a positive saturation, not {knee} and {saturation}"))); + } Ok(()) } @@ -518,11 +773,23 @@ pub fn cmyk_profile_from_model(model: &F, paper: [f64; 3], } fn lut_profile(model: &F, a2b: Clut, paper: [f64; 3], p: &CmykParams) -> Result { - let gbd = Gbd::new(model, p); - let colorimetric = b2a_table(model, p, None)?; - let perceptual = b2a_table(model, p, Some(&gbd))?; + let (perceptual, colorimetric, saturation) = match p.mapping { + GamutMapping::Legacy => { + let gbd = Gbd::new(model, p); + let colorimetric = b2a_table(model, p, None)?; + let perceptual = b2a_table(model, p, Some(&gbd))?; + (perceptual.clone(), colorimetric, perceptual) + } + GamutMapping::Oklab { knee, saturation } => { + let m = OklabMapper::new(model, p, knee, saturation); + let colorimetric = b2a_table_with(p, |lab, warm| m.clip(lab, warm))?; + let perceptual = b2a_table_with(p, |lab, warm| m.perceptual(lab, warm))?; + let saturation = b2a_table_with(p, |lab, warm| m.saturation(lab, warm))?; + (perceptual, colorimetric, saturation) + } + }; let a2b = lut16(a2b, false); - let (b0, b1) = (lut16(perceptual, p.white_on_node), lut16(colorimetric, p.white_on_node)); + let (b0, b1, b2) = (lut16(perceptual, p.white_on_node), lut16(colorimetric, p.white_on_node), lut16(saturation, p.white_on_node)); Ok(Profile { version: (4, 0x30), class: ProfileClass::Output, @@ -538,7 +805,7 @@ fn lut_profile(model: &F, a2b: Clut, paper: [f64; 3], p: &C trc: None, gray_trc: None, a2b: [Some(a2b.clone()), Some(a2b.clone()), Some(a2b)], - b2a: [Some(b0.clone()), Some(b1), Some(b0)], + b2a: [Some(b0), Some(b1), Some(b2)], bytes: None, hash: Default::default(), } @@ -576,6 +843,46 @@ mod tests { } } + #[test] + fn oklab_mapping_refuses_bad_parameters() { + for (knee, saturation) in [(1.0, 1.0), (-0.1, 1.0), (f64::NAN, 1.0), (0.5, 0.0), (0.5, f64::INFINITY), (0.5, f64::NAN)] { + let p = CmykParams { grid_a2b: 2, grid_b2a: 2, mapping: GamutMapping::Oklab { knee, saturation }, ..CmykParams::default() }; + assert!(cmyk_profile(&p).is_err(), "knee {knee}, saturation {saturation}"); + } + } + + #[test] + fn black_generation_curve() { + let p = CmykParams { gcr_curve: vec![[0.0, 0.0], [0.5, 0.2], [1.0, 0.9]], ..CmykParams::default() }; + for (g, k) in [(-1.0, 0.0), (0.0, 0.0), (0.25, 0.1), (0.5, 0.2), (0.75, 0.55), (1.0, 0.9), (2.0, 0.9)] { + assert!((gcr(&p, g) - k).abs() < 1e-12, "g {g}: {} != {k}", gcr(&p, g)); + } + for curve in [vec![[0.5, 0.2], [0.5, 0.3]], vec![[0.5, 0.2], [0.2, 0.3]], vec![[0.0, 1.5]], vec![[f64::NAN, 0.0]]] { + let p = CmykParams { grid_a2b: 2, grid_b2a: 2, gcr_curve: curve.clone(), ..CmykParams::default() }; + assert!(cmyk_profile(&p).is_err(), "{curve:?}"); + } + } + + #[test] + fn oklab_round_trips() { + let m = CmykModel::coated(CmykParams::default().tvi); + let p = CmykParams::default(); + let mapper = OklabMapper::new(&m, &p, 0.65, 1.05); + for lab in [[50.0, 0.0, 0.0], [44.0, 63.7, 47.0], [31.9, 32.9, -76.8], [90.0, -5.0, 80.0]] { + let back = mapper.lab(mapper.oklab(lab)); + // The published matrices carry 10 digits, so the inverses are exact to about 1e-5 ΔE. + assert!(math::delta_e76(lab, back) < 1e-3, "{lab:?} -> {back:?}"); + } + } + + #[test] + fn oklab_profile_has_three_tables() { + let p = CmykParams { grid_a2b: 5, grid_b2a: 5, mapping: GamutMapping::Oklab { knee: 0.65, saturation: 1.05 }, ..CmykParams::default() }; + let profile = cmyk_profile(&p).unwrap(); + let [Some(b0), Some(b1), Some(b2)] = &profile.b2a else { panic!("missing BToA tables") }; + assert!(b0 != b1 && b0 != b2, "perceptual shares a table"); + } + #[test] fn separation_hits_in_gamut_targets() { let m = CmykModel::coated(CmykParams::default().tvi); diff --git a/crates/cms/src/transform.rs b/crates/cms/src/transform.rs index c4f6e11cc..38a3223d9 100644 --- a/crates/cms/src/transform.rs +++ b/crates/cms/src/transform.rs @@ -46,7 +46,7 @@ const OUT_TABLE: usize = GpuMatrixShaper::OUTPUT_ENTRIES; const _: () = assert!(OUT_TABLE.is_power_of_two()); /// ICC v4 perceptual reference medium black (ICC.1:2010 §6.3.4.4), XYZ. -const PERCEPTUAL_BLACK: [f64; 3] = [0.00336, 0.0034731, 0.00287]; +pub(crate) const PERCEPTUAL_BLACK: [f64; 3] = [0.00336, 0.0034731, 0.00287]; /// Accelerated representation of a transform. #[derive(Debug)] diff --git a/crates/cms/tests/fogra39.rs b/crates/cms/tests/fogra39.rs index f958ffc51..d08e3bae6 100644 --- a/crates/cms/tests/fogra39.rs +++ b/crates/cms/tests/fogra39.rs @@ -12,7 +12,7 @@ //! ([`synth::cmyk_profile_from_model`]). use craft_cms::math::{self, delta_e2000}; -use craft_cms::synth::{self, CmykParams, Forward}; +use craft_cms::synth::{self, CmykParams, Forward, GamutMapping}; use craft_cms::{Builtin, Intent, Profile, Transform}; const DATA: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/data/FOGRA39.txt"); @@ -346,8 +346,10 @@ fn regen_coated_fogra39() { description: Builtin::CoatedFogra39.description().into(), copyright: craft_cms::builtin::COATED_FOGRA39_COPYRIGHT.into(), tac: 3.3, + gcr_curve: vec![[0.0, 0.0], [0.2, 0.05], [0.5, 0.205], [0.67, 0.5], [1.0, 0.95]], grid_b2a: 33, white_on_node: true, + mapping: GamutMapping::Oklab { knee: 0.65, saturation: 1.05 }, ..CmykParams::default() }; let profile = synth::cmyk_profile_from_model(&f, paper(&p), ¶ms).unwrap(); @@ -564,3 +566,60 @@ fn stand_in_separations_respect_the_ink_limit() { } } } + +/// sRGB patches and their separations by the reference application (Convert to Profile to +/// Coated FOGRA39, dither on), CMYK % per intent: relative + BPC, relative, perceptual, +/// saturation, absolute. Measured for storytold/photocraft#14267. +const REFERENCE_SEPARATIONS: [([u8; 3], [[f32; 4]; 5]); 3] = [ + ( + [200, 30, 30], + [[14.5, 97.7, 95.3, 4.3], [13.3, 100.0, 100.0, 5.5], [18.0, 96.9, 92.9, 8.6], [14.9, 100.0, 100.0, 5.9], [8.6, 100.0, 100.0, 2.4]], + ), + ([30, 60, 200], [[91.8, 75.3, 0.0, 0.0], [94.5, 76.5, 0.0, 0.0], [94.5, 77.7, 0.0, 0.0], [93.7, 76.9, 0.0, 0.0], [92.6, 74.9, 0.0, 0.0]]), + ( + [128, 128, 128], + [[49.0, 38.4, 38.4, 20.0], [49.8, 39.2, 38.8, 21.2], [49.0, 38.4, 38.4, 20.0], [49.0, 38.4, 38.4, 20.0], [46.7, 36.9, 38.4, 18.4]], + ), +]; + +const SEPARATION_INTENTS: [(Intent, bool); 5] = [ + (Intent::RelativeColorimetric, true), + (Intent::RelativeColorimetric, false), + (Intent::Perceptual, true), + (Intent::Saturation, true), + (Intent::AbsoluteColorimetric, false), +]; + +fn separation(intent: Intent, bpc: bool, rgb: [u8; 3]) -> [f32; 4] { + let t = Transform::new(Builtin::Srgb.profile(), fogra39(), intent, bpc).unwrap(); + let mut o = [0.0f32; 4]; + t.eval(&rgb.map(|v| v as f32 / 255.0), &mut o).unwrap(); + o.map(|v| v * 100.0) +} + +/// sRGB → Coated FOGRA39 matches the reference application's inks on a red, a saturated blue +/// and a mid grey for every intent. Most land within 2 % per ink; red at the magenta–yellow +/// edge lands between table nodes that are and are not clipped, which costs up to 2.2 % with +/// the 33³ BToA tables. +#[test] +fn separates_like_the_reference_application() { + let mut worst = 0.0f32; + for (rgb, want) in REFERENCE_SEPARATIONS { + for ((intent, bpc), want) in SEPARATION_INTENTS.into_iter().zip(want) { + let got = separation(intent, bpc, rgb); + let diff = (0..4).map(|i| (got[i] - want[i]).abs()).fold(0.0f32, f32::max); + eprintln!("{rgb:?} {intent:?} bpc {bpc}: {got:.1?} reference {want:?} (max {diff:.1})"); + worst = worst.max(diff); + } + } + assert!(worst <= 2.5, "worst ink difference {worst} %"); +} + +/// Saturation has its own table: a red separates with more magenta and yellow than +/// perceptual, whose chroma compression takes some back. +#[test] +fn saturation_and_perceptual_separate_differently() { + let red = [200, 30, 30]; + let (perceptual, saturation) = (separation(Intent::Perceptual, true, red), separation(Intent::Saturation, true, red)); + assert!(saturation[1] > perceptual[1] + 1.0 && saturation[2] > perceptual[2] + 1.0, "perceptual {perceptual:?}, saturation {saturation:?}"); +} diff --git a/crates/evm/Cargo.toml b/crates/evm/Cargo.toml new file mode 100644 index 000000000..018f5dff0 --- /dev/null +++ b/crates/evm/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "craft-evm" +description = "Read-only EVM bytecode dispatcher and compiler metadata inspection" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true +[lints] +workspace = true +[dependencies] +serde = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } + +[dev-dependencies] +craft-testkit = { workspace = true } diff --git a/crates/evm/README.md b/crates/evm/README.md new file mode 100644 index 000000000..6214c45b4 --- /dev/null +++ b/crates/evm/README.md @@ -0,0 +1,7 @@ +# craft-evm + +Read-only EVM bytecode inspection: strict raw/hex admission, opcode listing, dispatcher selector and jump-pattern facts, and Solidity/Vyper CBOR metadata. No VM execution, RPC, wallet, signing or transaction support. + +Preserves Craft Anything's parser behavior and limits: 16 MiB decoded bytecode, 4,096 reported functions and 400 listing entries, plus existing bounded CBOR recursion/collection parsing. These are not a total allocator-byte guarantee. Dispatcher facts are heuristics, not a full control-flow or semantic proof. EOF containers remain unsupported. + +Application file I/O, evidence policy and selector naming remain outside this crate. diff --git a/crates/evm/src/lib.rs b/crates/evm/src/lib.rs new file mode 100644 index 000000000..29d254b40 --- /dev/null +++ b/crates/evm/src/lib.rs @@ -0,0 +1,895 @@ +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +#![forbid(unsafe_code)] + +//! EVM bytecode: the public function selectors a contract's dispatcher compares against, where +//! each one jumps, whether it refuses payments, fallback/receive presence, and the compiler +//! metadata (CBOR) appended to the code. Nothing is executed; this is a linear sweep that +//! respects PUSH data plus pattern matching over the instruction stream. + +use EvmError as FormatError; + +/// Neutral inspection failure; no code is executed. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum EvmError { + #[error("truncated input: {0}")] + Truncated(String), + #[error("malformed input: {0}")] + Malformed(String), + #[error("unsupported: {0}")] + Unsupported(String), + #[error("input too large: {0}")] + TooLarge(String), +} + +#[cfg(test)] +mod test_support; +use serde::Serialize; +use serde_json::{Map, Value}; +use std::collections::BTreeSet; + +/// Largest bytecode we analyse (16 MiB; deployed contracts are at most 24 KiB). +pub const MAX_CODE: usize = 16 << 20; +const MAX_FUNCTIONS: usize = 4096; +const LISTING_CAP: usize = 400; + +/// How input bytes are encoded. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Encoding { + /// Hex when the bytes are all hex digits (with an optional 0x), raw otherwise. + Auto, + Raw, + Hex, +} + +/// Turn input bytes into bytecode. Hex is strict: optional `0x`, even length, only hex digits; +/// surrounding whitespace is ignored. +pub fn decode_input(input: &[u8], enc: Encoding) -> Result<(Vec, &'static str), FormatError> { + let trimmed = input.trim_ascii(); + let body = trimmed.strip_prefix(b"0x").or_else(|| trimmed.strip_prefix(b"0X")).unwrap_or(trimmed); + let looks_hex = !body.is_empty() && body.iter().all(u8::is_ascii_hexdigit); + let hex = match enc { + Encoding::Hex => true, + Encoding::Raw => false, + Encoding::Auto => looks_hex, + }; + if !hex { + if input.len() > MAX_CODE { + return Err(FormatError::TooLarge(format!("{} bytes; the limit is {MAX_CODE}", input.len()))); + } + return Ok((input.to_vec(), "raw")); + } + if body.is_empty() { + return Err(FormatError::Malformed("no hex digits".into())); + } + if let Some(p) = body.iter().position(|b| !b.is_ascii_hexdigit()) { + return Err(FormatError::Malformed(format!("not a hex digit at position {p}"))); + } + if !body.len().is_multiple_of(2) { + return Err(FormatError::Malformed(format!("odd number of hex digits ({})", body.len()))); + } + if body.len() / 2 > MAX_CODE { + return Err(FormatError::TooLarge(format!("{} bytes; the limit is {MAX_CODE}", body.len() / 2))); + } + let nib = |c: u8| -> u8 { + match c { + b'0'..=b'9' => c - b'0', + b'a'..=b'f' => c - b'a' + 10, + _ => c.wrapping_sub(b'A').wrapping_add(10) & 0x0f, + } + }; + Ok((body.chunks(2).map(|p| (nib(p.first().copied().unwrap_or(b'0')) << 4) | nib(p.get(1).copied().unwrap_or(b'0'))).collect(), "hex")) +} + +#[derive(Debug, Clone, Copy)] +struct Ins { + off: usize, + op: u8, + /// Immediate (PUSH data) start and length. + imm: (usize, usize), +} + +fn push_len(op: u8) -> usize { + if (0x60..=0x7f).contains(&op) { usize::from(op - 0x5f) } else { 0 } +} + +/// The mnemonic of an opcode (`UNKNOWN_0x…` for unassigned ones). +pub fn mnemonic(op: u8) -> String { + let s = match op { + 0x00 => "STOP", + 0x01 => "ADD", + 0x02 => "MUL", + 0x03 => "SUB", + 0x04 => "DIV", + 0x05 => "SDIV", + 0x06 => "MOD", + 0x07 => "SMOD", + 0x08 => "ADDMOD", + 0x09 => "MULMOD", + 0x0a => "EXP", + 0x0b => "SIGNEXTEND", + 0x10 => "LT", + 0x11 => "GT", + 0x12 => "SLT", + 0x13 => "SGT", + 0x14 => "EQ", + 0x15 => "ISZERO", + 0x16 => "AND", + 0x17 => "OR", + 0x18 => "XOR", + 0x19 => "NOT", + 0x1a => "BYTE", + 0x1b => "SHL", + 0x1c => "SHR", + 0x1d => "SAR", + 0x20 => "KECCAK256", + 0x30 => "ADDRESS", + 0x31 => "BALANCE", + 0x32 => "ORIGIN", + 0x33 => "CALLER", + 0x34 => "CALLVALUE", + 0x35 => "CALLDATALOAD", + 0x36 => "CALLDATASIZE", + 0x37 => "CALLDATACOPY", + 0x38 => "CODESIZE", + 0x39 => "CODECOPY", + 0x3a => "GASPRICE", + 0x3b => "EXTCODESIZE", + 0x3c => "EXTCODECOPY", + 0x3d => "RETURNDATASIZE", + 0x3e => "RETURNDATACOPY", + 0x3f => "EXTCODEHASH", + 0x40 => "BLOCKHASH", + 0x41 => "COINBASE", + 0x42 => "TIMESTAMP", + 0x43 => "NUMBER", + 0x44 => "PREVRANDAO", + 0x45 => "GASLIMIT", + 0x46 => "CHAINID", + 0x47 => "SELFBALANCE", + 0x48 => "BASEFEE", + 0x49 => "BLOBHASH", + 0x4a => "BLOBBASEFEE", + 0x50 => "POP", + 0x51 => "MLOAD", + 0x52 => "MSTORE", + 0x53 => "MSTORE8", + 0x54 => "SLOAD", + 0x55 => "SSTORE", + 0x56 => "JUMP", + 0x57 => "JUMPI", + 0x58 => "PC", + 0x59 => "MSIZE", + 0x5a => "GAS", + 0x5b => "JUMPDEST", + 0x5c => "TLOAD", + 0x5d => "TSTORE", + 0x5e => "MCOPY", + 0x5f => "PUSH0", + 0x60..=0x7f => return format!("PUSH{}", op - 0x5f), + 0x80..=0x8f => return format!("DUP{}", op - 0x7f), + 0x90..=0x9f => return format!("SWAP{}", op - 0x8f), + 0xa0..=0xa4 => return format!("LOG{}", op - 0xa0), + 0xf0 => "CREATE", + 0xf1 => "CALL", + 0xf2 => "CALLCODE", + 0xf3 => "RETURN", + 0xf4 => "DELEGATECALL", + 0xf5 => "CREATE2", + 0xfa => "STATICCALL", + 0xfd => "REVERT", + 0xfe => "INVALID", + 0xff => "SELFDESTRUCT", + _ => return format!("UNKNOWN_{op:#04x}"), + }; + s.to_string() +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct EvmFunction { + /// `0x` + 8 hex digits. + pub selector: String, + /// Where the dispatcher sends calls with this selector. + pub bytecode_offset: usize, + /// Offset of the comparison in the dispatcher. + pub compare_offset: usize, + pub dispatch: &'static str, + /// `eq-jumpi` (jump on match) or `eq-iszero-fallthrough` (body follows the check). + pub pattern: &'static str, + /// `nonpayable`, `payable` or `unknown` (view/pure cannot be told from the dispatcher). + pub mutability: &'static str, + /// Why the mutability was chosen. + pub mutability_reason: String, + /// False when the destination is not a JUMPDEST (the match is then doubtful). + pub jumpdest_valid: bool, +} + +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct EvmMetadata { + pub offset: usize, + pub length: usize, + pub entries: Value, + pub compiler: Option, + pub ipfs: Option, + pub swarm: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct EvmInterface { + pub code_size: usize, + /// Bytes analysed after removing the metadata tail. + pub analysed_size: usize, + pub metadata: Option, + pub instructions: usize, + pub jumpdests: usize, + /// How the selector is pulled from calldata: `shr-224`, `div-2^224`, `and-mask`, or none. + pub selector_extraction: Option<&'static str>, + pub dispatcher_offset: Option, + /// `solc-like`, `vyper-like`, `mixed` or `none`. + pub style: &'static str, + pub functions: Vec, + /// Comparisons that split the selector range (binary-search dispatchers). + pub binary_search_splits: usize, + /// A CALLVALUE check that runs before any selector comparison (all functions non-payable). + pub callvalue_check_before_dispatch: bool, + pub short_calldata_check: bool, + /// `present`, `absent` or `unknown`. + pub fallback: &'static str, + /// `present` or `not-detected`. + pub receive: &'static str, + /// The dispatcher region as text (first instructions only). + pub dispatcher_listing: Vec, + pub warnings: Vec, +} + +fn sweep(code: &[u8]) -> Vec { + let mut out = Vec::with_capacity(code.len().min(1 << 20) / 2); + let mut i = 0usize; + while i < code.len() { + let op = code.get(i).copied().unwrap_or(0); + let n = push_len(op); + out.push(Ins { off: i, op, imm: (i + 1, n) }); + i = i.saturating_add(1).saturating_add(n); + } + out +} + +fn imm<'a>(code: &'a [u8], ins: &Ins) -> &'a [u8] { + // Truncated PUSH data at the end of code reads as zero-padded; we only use what exists. + let (s, n) = ins.imm; + code.get(s..s.saturating_add(n).min(code.len())).unwrap_or(&[]) +} + +fn imm_u64(code: &[u8], ins: &Ins) -> Option { + let b = imm(code, ins); + if ins.imm.1 > 8 || b.len() != ins.imm.1 { + return None; + } + Some(b.iter().fold(0u64, |a, &x| (a << 8) | u64::from(x))) +} + +fn is_push(op: u8) -> bool { + (0x60..=0x7f).contains(&op) +} + +/// Strip and decode a trailing CBOR metadata block. +fn split_metadata(code: &[u8]) -> (usize, Option) { + let n = code.len(); + if n < 4 { + return (n, None); + } + let len = usize::from(u16::from_be_bytes([code.get(n - 2).copied().unwrap_or(0), code.get(n - 1).copied().unwrap_or(0)])); + if len == 0 || len.saturating_add(2) > n { + return (n, None); + } + let start = n - 2 - len; + let Some(cbor) = code.get(start..n - 2) else { return (n, None) }; + if !matches!(cbor.first(), Some(0xa1..=0xb8 | 0xbf)) { + return (n, None); + } + let mut pos = 0usize; + let Some(v) = cbor_item(cbor, &mut pos, 0) else { return (n, None) }; + if pos != cbor.len() { + return (n, None); + } + let Value::Object(map) = &v else { return (n, None) }; + let compiler = map + .get("solc") + .and_then(|s| s.as_str().map(String::from)) + .map(|v| format!("solc {v}")) + .or_else(|| map.get("vyper").and_then(Value::as_str).map(|v| format!("vyper {v}"))); + let ipfs = map.get("ipfs").and_then(Value::as_str).map(String::from); + let swarm = map.get("bzzr1").or_else(|| map.get("bzzr0")).and_then(Value::as_str).map(String::from); + (start, Some(EvmMetadata { offset: start, length: len + 2, entries: v.clone(), compiler, ipfs, swarm })) +} + +fn cbor_arg(b: &[u8], pos: &mut usize, info: u8) -> Option { + let take = |pos: &mut usize, k: usize| -> Option { + let s = b.get(*pos..pos.checked_add(k)?)?; + *pos += k; + Some(s.iter().fold(0u64, |a, &x| (a << 8) | u64::from(x))) + }; + match info { + 0..=23 => Some(u64::from(info)), + 24 => take(pos, 1), + 25 => take(pos, 2), + 26 => take(pos, 4), + 27 => take(pos, 8), + _ => None, + } +} + +/// Decode one bounded metadata item. An incomplete or unsupported value invalidates the metadata trailer. +fn cbor_item(b: &[u8], pos: &mut usize, depth: u32) -> Option { + if depth > 4 { + return None; + } + let ib = *b.get(*pos)?; + *pos += 1; + let major = ib >> 5; + let info = ib & 0x1f; + match major { + 0 => cbor_arg(b, pos, info).map(Value::from), + 2 | 3 => { + let n = usize::try_from(cbor_arg(b, pos, info)?).ok()?; + let s = b.get(*pos..pos.checked_add(n)?)?; + *pos += n; + Some(if major == 3 { Value::from(std::str::from_utf8(s).ok()?) } else { Value::from(bytes_value(s)) }) + } + 4 => { + let n = usize::try_from(cbor_arg(b, pos, info)?).ok()?; + if n > 64 { + return None; + } + let mut items = Vec::new(); + for _ in 0..n { + items.push(cbor_item(b, pos, depth + 1)?); + } + Some(Value::Array(items)) + } + 5 => { + let n = usize::try_from(cbor_arg(b, pos, info)?).ok()?; + if n > 32 { + return None; + } + let mut m = Map::new(); + for _ in 0..n { + // The metadata view uses JSON object keys. Coercing other CBOR key + // types to strings can merge distinct keys and hide bytecode. + if b.get(*pos).map(|byte| byte >> 5) != Some(3) { + return None; + } + let Value::String(k) = cbor_item(b, pos, depth + 1)? else { return None }; + if m.contains_key(&k) { + return None; + } + let v = cbor_item(b, pos, depth + 1)?; + let v = match (k.as_str(), &v) { + ("solc" | "vyper", Value::Array(items)) if items.len() == 3 && items.iter().all(Value::is_u64) => { + Value::from(items.iter().map(|v| v.to_string()).collect::>().join(".")) + } + ("solc", Value::String(s)) if s.starts_with("0x") && s.len() == 8 => { + let d: Vec = (0..3).filter_map(|i| hex_byte(s.get(2 + 2 * i..4 + 2 * i)?)).collect(); + if let [a, bb, c] = d.as_slice() { Value::from(format!("{a}.{bb}.{c}")) } else { v } + } + ("ipfs", Value::String(s)) if s.starts_with("0x1220") => Value::from(base58(&hex_decode(s.get(2..).unwrap_or("")))), + _ => v, + }; + m.insert(k, v); + } + Some(Value::Object(m)) + } + 7 => match info { + 20 => Some(Value::Bool(false)), + 21 => Some(Value::Bool(true)), + 22 => Some(Value::Null), + _ => None, + }, + _ => None, + } +} + +fn bytes_value(s: &[u8]) -> String { + let mut out = String::with_capacity(2 + s.len() * 2); + out.push_str("0x"); + for b in s { + out.push_str(&format!("{b:02x}")); + } + out +} + +/// One byte from two hex digits; `from_str_radix` alone would also accept a leading `+`. +fn hex_byte(pair: &str) -> Option { + if pair.bytes().all(|c| c.is_ascii_hexdigit()) { u8::from_str_radix(pair, 16).ok() } else { None } +} + +fn hex_decode(s: &str) -> Vec { + (0..s.len() / 2).filter_map(|i| hex_byte(s.get(2 * i..2 * i + 2)?)).collect() +} + +fn base58(b: &[u8]) -> String { + const A: &[u8; 58] = b"123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"; + let zeros = b.iter().take_while(|&&x| x == 0).count(); + let mut digits: Vec = Vec::new(); + for &byte in b.iter().take(128) { + let mut carry = u32::from(byte); + for d in digits.iter_mut() { + carry += u32::from(*d) << 8; + *d = (carry % 58) as u8; + carry /= 58; + } + while carry > 0 { + digits.push((carry % 58) as u8); + carry /= 58; + } + } + let mut s: String = std::iter::repeat_n('1', zeros).collect(); + s.extend(digits.iter().rev().map(|&d| char::from(*A.get(usize::from(d)).unwrap_or(&b'1')))); + s +} + +fn selector_extraction(body: &[u8], ins: &[Ins]) -> Option<(&'static str, usize)> { + let at = |k: usize| ins.get(k); + let op_at = |k: usize| at(k).map(|i| i.op); + // Selector extraction: CALLDATALOAD followed by SHR 224, DIV 2^224 or a 4-byte mask. + let mut extraction: Option<(&'static str, usize)> = None; + for (k, i) in ins.iter().enumerate() { + if i.op != 0x35 { + continue; + } + for d in 1..=6 { + let Some(n) = at(k + d) else { break }; + let kind = match n.op { + 0x60 if imm(body, n) == [0xe0] && op_at(k + d + 1) == Some(0x1c) => Some("shr-224"), + 0x7c if d == 1 + && imm(body, n).first() == Some(&1) + && imm(body, n).iter().skip(1).all(|&b| b == 0) + && op_at(k + d + 1) == Some(0x90) + && op_at(k + d + 2) == Some(0x04) => + { + Some("div-2^224") + } + 0x63 if imm(body, n) == [0xff, 0xff, 0xff, 0xff] && (op_at(k + d + 1) == Some(0x16) || op_at(k + d + 2) == Some(0x16)) => { + Some("and-mask") + } + _ => None, + }; + if let Some(kind) = kind { + extraction = Some((kind, k)); + break; + } + } + if extraction.is_some() { + break; + } + } + extraction +} + +fn short_calldata_path(body: &[u8], ins: &[Ins], disp_start: usize) -> (bool, &'static str) { + let at = |k: usize| ins.get(k); + let op_at = |k: usize| at(k).map(|i| i.op); + // Short calldata path and receive(). + let mut short_calldata = false; + let mut receive = "not-detected"; + for (k, i) in ins.iter().enumerate().take(disp_start.saturating_add(64)) { + if i.op != 0x36 { + continue; + } + if op_at(k + 1) == Some(0x15) && at(k + 2).is_some_and(|p| is_push(p.op)) && op_at(k + 3) == Some(0x57) { + receive = "present"; + } + // LT pops the size first in PUSH 4, CALLDATASIZE, LT. + if k.checked_sub(1).and_then(at).is_some_and(|p| is_push(p.op) && imm_u64(body, p) == Some(4)) + && op_at(k + 1) == Some(0x10) + && at(k + 2).is_some_and(|p| is_push(p.op)) + && op_at(k + 3) == Some(0x57) + { + short_calldata = true; + } + } + (short_calldata, receive) +} + +/// Analyse bytecode (already decoded from hex if needed). +pub fn analyze(code: &[u8]) -> Result { + if code.is_empty() { + return Err(FormatError::Truncated("no bytecode".into())); + } + if code.len() > MAX_CODE { + return Err(FormatError::TooLarge(format!("{} bytes; the limit is {MAX_CODE}", code.len()))); + } + if code.starts_with(&[0xef, 0x00]) { + return Err(FormatError::Unsupported("EVM Object Format (EOF, 0xEF00) containers are not analysed".into())); + } + let mut warnings = Vec::new(); + let (end, metadata) = split_metadata(code); + let body = code.get(..end).unwrap_or(code); + let ins = sweep(body); + let jumpdests = ins.iter().filter(|i| i.op == 0x5b).count(); + let at = |k: usize| ins.get(k); + let op_at = |k: usize| at(k).map(|i| i.op); + let index_of = |off: usize| ins.binary_search_by_key(&off, |i| i.off).ok(); + + let extraction = selector_extraction(body, &ins); + let disp_start = extraction.map(|(_, k)| k).unwrap_or(0); + if extraction.is_none() { + warnings.push("no selector extraction (CALLDATALOAD then SHR/DIV/AND) was seen; matches below are less certain".into()); + } + + // Only a guard on the straight-line entry path can apply to every selector. + let callvalue_before = ins + .iter() + .take(disp_start) + .enumerate() + .take_while(|(_, i)| !matches!(i.op, 0x00 | 0x56 | 0x57 | 0xf3 | 0xfd | 0xfe | 0xff)) + .any(|(k, i)| i.op == 0x34 && rejects_value(body, &ins, k, &index_of)); + + let mut functions: Vec = Vec::new(); + let mut selectors = BTreeSet::new(); + let mut splits = 0usize; + let mut last_jumpi: Option = None; + let mut solc_like = 0usize; + let mut vyper_like = 0usize; + for k in disp_start..ins.len() { + let Some(i) = at(k) else { break }; + let n = push_len(i.op); + if !(1..=4).contains(&n) { + continue; + } + let Some(sel) = imm_u64(body, i) else { continue }; + let found = |dk: usize| -> Option { + let p = at(dk)?; + (is_push(p.op) && p.imm.1 <= 3 && op_at(dk + 1) == Some(0x57)).then_some(dk) + }; + // PUSH sel, [DUP2], EQ, PUSH dest, JUMPI + let eq_k = if op_at(k + 1) == Some(0x14) { + Some(k + 1) + } else if matches!(op_at(k + 1), Some(0x80..=0x8f)) && op_at(k + 2) == Some(0x14) { + Some(k + 2) + } else { + None + }; + let mut hit: Option<(usize, usize, &'static str)> = None; + if let Some(e) = eq_k { + if let Some(pk) = found(e + 1) + && let Some(dest) = at(pk).and_then(|p| imm_u64(body, p)) + { + hit = Some((usize::try_from(dest).unwrap_or(usize::MAX), pk + 1, "eq-jumpi")); + } else if n == 4 && op_at(e + 1) == Some(0x15) && found(e + 2).is_some() { + // Vyper-style: if selector != x jump to the next check; the body follows. + if let Some(next) = at(e + 4) { + hit = Some((next.off, e + 3, "eq-iszero-fallthrough")); + } + } + } else if n == 4 { + // Vyper also loads the selector from memory: PUSH sel, PUSH1 0, MLOAD, EQ, ISZERO … + let load = at(k + 1).is_some_and(|p| (p.op == 0x5f || is_push(p.op)) && imm_u64(body, p) == Some(0)); + if load + && op_at(k + 2) == Some(0x51) + && op_at(k + 3) == Some(0x14) + && op_at(k + 4) == Some(0x15) + && found(k + 5).is_some() + && let Some(next) = at(k + 7) + { + hit = Some((next.off, k + 6, "eq-iszero-fallthrough")); + } + let direct = matches!(op_at(k + 1), Some(0x10 | 0x11)) && found(k + 2).is_some(); + let via_dup = matches!(op_at(k + 1), Some(0x80..=0x8f)) && matches!(op_at(k + 2), Some(0x10 | 0x11)) && found(k + 3).is_some(); + if hit.is_none() && (direct || via_dup) { + splits += 1; + } + } + let Some((dest, jumpi_k, pattern)) = hit else { continue }; + if n < 4 && extraction.is_none() { + continue; + } + // Duplicate identities still advance the end of the recognized dispatcher. + // Otherwise the fallback scan starts inside a later comparison, not after it. + last_jumpi = Some(jumpi_k); + if !selectors.insert(sel) { + continue; + } + let selector = format!("0x{sel:08x}"); + if pattern == "eq-jumpi" { + solc_like += 1; + } else { + vyper_like += 1; + } + let di = index_of(dest); + let jumpdest_valid = pattern != "eq-jumpi" || di.and_then(at).is_some_and(|x| x.op == 0x5b); + let (mutability, reason) = if !jumpdest_valid { + ("unknown", "the destination is not a JUMPDEST".to_owned()) + } else if callvalue_before { + ("nonpayable", "a CALLVALUE check runs before the dispatcher, so every function refuses payments".to_string()) + } else { + match di { + Some(d) => payability(body, &ins, d, &index_of), + None => ("unknown", "the destination is not an instruction boundary".to_string()), + } + }; + functions.push(EvmFunction { + selector, + bytecode_offset: dest, + compare_offset: i.off, + dispatch: "abi", + pattern, + mutability, + mutability_reason: reason, + jumpdest_valid, + }); + if functions.len() >= MAX_FUNCTIONS { + warnings.push(format!("stopped at {MAX_FUNCTIONS} selectors")); + break; + } + } + if functions.iter().any(|f| !f.jumpdest_valid) { + warnings.push("some destinations are not JUMPDESTs; those matches may be data, not dispatch".into()); + } + + let (short_calldata, receive) = short_calldata_path(body, &ins, disp_start); + let mut fallback = "unknown"; + // At the selector cap the remaining dispatcher has not been scanned. + // Its next comparison is not evidence that a fallback body exists. + if let Some(j) = last_jumpi + && functions.len() < MAX_FUNCTIONS + && solc_like > 0 + { + fallback = if reverts_soon(body, &ins, j + 1, &index_of) { "absent" } else { "present" }; + } + let style = match (solc_like > 0, vyper_like > 0) { + (true, true) => "mixed", + (true, false) => "solc-like", + (false, true) => "vyper-like", + _ => "none", + }; + let listing_end = last_jumpi.map(|j| j + 8).unwrap_or(LISTING_CAP).min(LISTING_CAP); + let dispatcher_listing = ins + .iter() + .take(listing_end) + .map(|i| { + let d = imm(body, i); + if d.is_empty() && push_len(i.op) == 0 { + format!("{:#06x}: {}", i.off, mnemonic(i.op)) + } else { + format!("{:#06x}: {} {}", i.off, mnemonic(i.op), bytes_value(d)) + } + }) + .collect(); + if functions.is_empty() { + warnings.push( + "no selector dispatcher was recognised (the code may be creation code without runtime, a proxy, a library, or use a jump-table dispatcher)".into(), + ); + } + Ok(EvmInterface { + code_size: code.len(), + analysed_size: body.len(), + metadata, + instructions: ins.len(), + jumpdests, + selector_extraction: extraction.map(|(k, _)| k), + dispatcher_offset: extraction.and_then(|(_, k)| at(k)).map(|i| i.off), + style, + functions, + binary_search_splits: splits, + callvalue_check_before_dispatch: callvalue_before, + short_calldata_check: short_calldata, + fallback, + receive, + dispatcher_listing, + warnings, + }) +} + +/// Resolve the destination of a JUMP to an instruction index, when the PUSH right before it is +/// readable and lands on an instruction boundary. +fn jump_destination(code: &[u8], ins: &[Ins], k: usize, index_of: &dyn Fn(usize) -> Option) -> Option { + let push = k.checked_sub(1).and_then(|p| ins.get(p)).filter(|p| is_push(p.op))?; + index_of(usize::try_from(imm_u64(code, push)?).ok()?) +} + +/// Follow a function's entry for a few instructions (and up to two unconditional jumps) looking +/// for a CALLVALUE check. +fn payability(code: &[u8], ins: &[Ins], start: usize, index_of: &dyn Fn(usize) -> Option) -> (&'static str, String) { + let mut k = start; + let mut hops = 0; + for _ in 0..48 { + let Some(i) = ins.get(k) else { break }; + match i.op { + 0x34 if rejects_value(code, ins, k, index_of) => return ("nonpayable", format!("rejects nonzero CALLVALUE at {:#x}", i.off)), + 0x56 => match jump_destination(code, ins, k, index_of) { + Some(t) if hops < 2 && ins.get(t).is_some_and(|target| target.op == 0x5b) => { + hops += 1; + k = t; + continue; + } + _ => return ("unknown", "jumps somewhere we cannot follow before any CALLVALUE check".into()), + }, + 0x57 => return ("unknown", "branches before any CALLVALUE check".into()), + 0xfd | 0xfe => return ("unknown", "halts with an error before any CALLVALUE check".into()), + 0x00 | 0xf3 | 0xff | 0x35 | 0x37 | 0x54 | 0x55 | 0xf1 | 0xf4 | 0xfa | 0xa0..=0xa4 => { + return ("payable", format!("reaches {} at {:#x} without a CALLVALUE check", mnemonic(i.op), i.off)); + } + _ => {} + } + k += 1; + } + ("unknown", "no CALLVALUE check in the first instructions".into()) +} + +/// CALLVALUE only reads the payment. Recognize a direct test that sends the nonzero path +/// to a rejection, allowing the common DUP1 that preserves the original value for cleanup. +fn rejects_value(code: &[u8], ins: &[Ins], k: usize, index_of: &dyn Fn(usize) -> Option) -> bool { + let mut next = k + 1; + if ins.get(next).is_some_and(|i| i.op == 0x80) { + next += 1; + } + let jumps_when_zero = ins.get(next).is_some_and(|i| i.op == 0x15); + next += usize::from(jumps_when_zero); + let Some(push) = ins.get(next).filter(|i| is_push(i.op)) else { return false }; + if ins.get(next + 1).is_none_or(|i| i.op != 0x57) { + return false; + } + let Some(target) = imm_u64(code, push).and_then(|v| usize::try_from(v).ok()).and_then(index_of) else { return false }; + if ins.get(target).is_none_or(|i| i.op != 0x5b) { + return false; + } + reverts_soon(code, ins, if jumps_when_zero { next + 2 } else { target }, index_of) +} + +fn reverts_soon(code: &[u8], ins: &[Ins], mut k: usize, index_of: &dyn Fn(usize) -> Option) -> bool { + let mut hops = 0; + for _ in 0..12 { + let Some(i) = ins.get(k) else { return false }; + match i.op { + 0xfd | 0xfe => return true, + 0x56 => match jump_destination(code, ins, k, index_of) { + Some(t) if hops < 2 => { + hops += 1; + k = t; + continue; + } + _ => return false, + }, + 0x5b | 0x50 | 0x80..=0x9f | 0x5f | 0x60..=0x7f => {} + _ => return false, + } + k += 1; + } + false +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_support::formats as fx; + + #[test] + fn hex_signs_are_not_hex_digits() { + let solc = |v: &str| { + let mut b = vec![0xa1, 0x64]; + b.extend(b"solc"); + b.push(0x60 + v.len() as u8); + b.extend(v.as_bytes()); + cbor_item(&b, &mut 0, 0).unwrap()["solc"].clone() + }; + assert_eq!(solc("0x+1+2+3"), "0x+1+2+3"); + assert_eq!(solc("0x000008"), "0.0.8"); + assert!(hex_decode("+f").is_empty()); + assert_eq!(hex_decode("0f"), vec![15]); + } + + /// A plausible trailer length alone must not remove instructions from the analyzed bytecode. + #[test] + fn incomplete_cbor_must_not_shorten_the_analyzed_contract() { + // A nested array declares 65 integers but supplies just 64. The top-level map + // and its length suffix otherwise look complete. + let mut short = vec![0xa1, 0x61, b'x', 0x98, 65]; + short.extend([0; 64]); + let mut short_map = vec![0xa1, 0x61, b'x', 0xb8, 33]; + for key in 0..32 { + short_map.extend([0x18, key, 0]); + } + for cbor in [short, short_map, vec![0xa1, 0x61, b'x', 0x61, 0xff]] { + let mut code = vec![0x00]; + code.extend(&cbor); + code.extend((cbor.len() as u16).to_be_bytes()); + assert!(split_metadata(&code).1.is_none()); + assert_eq!(analyze(&code).unwrap().analysed_size, code.len()); + } + } + + #[test] + fn dispatcher_selectors_and_metadata() { + let (code, dests) = fx::evm_runtime(); + let r = analyze(&code).unwrap(); + assert_eq!(r.selector_extraction, Some("shr-224")); + assert_eq!(r.style, "solc-like"); + let sels: Vec<_> = r.functions.iter().map(|f| f.selector.clone()).collect(); + assert_eq!(sels, fx::EVM_SELECTORS.iter().map(|s| format!("0x{s:08x}")).collect::>()); + for (f, d) in r.functions.iter().zip(dests) { + assert_eq!(f.bytecode_offset, d); + assert!(f.jumpdest_valid); + } + assert_eq!(r.functions[0].mutability, "nonpayable"); + assert_eq!(r.functions[1].mutability, "nonpayable"); + assert_eq!(r.functions[2].mutability, "payable"); + assert!(r.short_calldata_check); + assert_eq!(r.fallback, "absent"); + let m = r.metadata.unwrap(); + assert_eq!(m.compiler.as_deref(), Some("solc 0.8.19")); + assert!(m.ipfs.unwrap().starts_with("Qm")); + assert_eq!(r.analysed_size + m.length, code.len()); + } + + #[test] + fn hex_input_rules() { + let (code, _) = fx::evm_runtime(); + let hex = fx::to_hex(&code); + assert_eq!(decode_input(format!(" {hex}\n").as_bytes(), Encoding::Auto).unwrap(), (code.clone(), "hex")); + assert_eq!(decode_input(&code, Encoding::Auto).unwrap().1, "raw"); + assert!(decode_input(b"0x123", Encoding::Hex).is_err()); + assert!(decode_input(b"0xzz", Encoding::Hex).is_err()); + assert!(decode_input(b"", Encoding::Hex).is_err()); + assert!(matches!(analyze(&[0xef, 0x00, 0x01]), Err(FormatError::Unsupported(_)))); + assert_eq!(base58(&[0, 0, 1]), "112"); + } + + #[test] + fn vyper_style_fallthrough() { + // PUSH1 0 CALLDATALOAD PUSH1 0xe0 SHR PUSH1 0 MSTORE PUSH4 sel PUSH1 0 MLOAD EQ ISZERO PUSH2 next JUMPI + let mut c = vec![ + 0x60, 0x00, 0x35, 0x60, 0xe0, 0x1c, 0x60, 0x00, 0x52, 0x63, 0x12, 0x34, 0x56, 0x78, 0x60, 0x00, 0x51, 0x14, 0x15, 0x61, 0x00, 0x20, 0x57, + ]; + let body = c.len(); + c.extend_from_slice(&[0x34, 0x15, 0x00]); + let r = analyze(&c).unwrap(); + assert_eq!(r.style, "vyper-like"); + assert_eq!(r.functions[0].selector, "0x12345678"); + assert_eq!(r.functions[0].bytecode_offset, body); + assert_eq!(r.functions[0].mutability, "payable", "ISZERO followed by STOP never rejects payment"); + } + + #[test] + fn nonpayable_needs_a_rejecting_nonzero_branch() { + fn dispatch(prefix: &[u8]) -> EvmInterface { + let mut code = prefix.to_vec(); + code.extend([0x5f, 0x35, 0x60, 0xe0, 0x1c, 0x63, 1, 2, 3, 4, 0x14, 0x60]); + code.extend([(code.len() + 3) as u8, 0x57, 0x00, 0x5b, 0x00]); + analyze(&code).unwrap() + } + let read_only = dispatch(&[0x34, 0x50]); // CALLVALUE POP + assert!(!read_only.callvalue_check_before_dispatch); + assert_eq!(read_only.functions[0].mutability, "payable"); + let guard = dispatch(&[0x34, 0x80, 0x15, 0x60, 9, 0x57, 0x5f, 0x80, 0xfd, 0x5b, 0x50]); + assert!(guard.callvalue_check_before_dispatch); + assert_eq!(guard.functions[0].mutability, "nonpayable"); + // The other common form jumps to REVERT for a nonzero value and otherwise skips it. + let direct = dispatch(&[0x34, 0x60, 7, 0x57, 0x60, 11, 0x56, 0x5b, 0x5f, 0x80, 0xfd, 0x5b]); + assert!(direct.callvalue_check_before_dispatch); + // Reversing that conditional rejects zero-value calls, not payments. + let reversed = dispatch(&[0x34, 0x15, 0x60, 8, 0x57, 0x60, 12, 0x56, 0x5b, 0x5f, 0x80, 0xfd, 0x5b]); + assert!(!reversed.callvalue_check_before_dispatch); + // An unconditional jump bypasses the otherwise valid guard before the dispatcher. + let bypassed = dispatch(&[0x60, 12, 0x56, 0x34, 0x80, 0x15, 0x60, 12, 0x57, 0x5f, 0x80, 0xfd, 0x5b]); + assert!(!bypassed.callvalue_check_before_dispatch); + } + + #[test] + fn hostile_inputs_do_not_panic() { + let (code, _) = fx::evm_runtime(); + for m in crate::test_support::mutations(&code) { + let _ = analyze(&m); + } + for n in 0..code.len() { + let _ = analyze(&code[..n]); + } + for seed in 0..200 { + let n = crate::test_support::noise(seed, 300); + let _ = analyze(&n); + let _ = decode_input(&n, Encoding::Auto); + } + // A metadata length that claims everything, and nested CBOR. + let _ = analyze(&[0xa1, 0xa1, 0xa1, 0xa1, 0xa1, 0xa1, 0xa1, 0x00, 0x07]); + let _ = analyze(&[0x7f; 40]); + } +} diff --git a/crates/evm/src/test_support.rs b/crates/evm/src/test_support.rs new file mode 100644 index 000000000..0dc9861ef --- /dev/null +++ b/crates/evm/src/test_support.rs @@ -0,0 +1,63 @@ +pub mod formats { + pub const EVM_SELECTORS: [u32; 3] = [0xa905_9cbb, 0x70a0_8231, 0xd0e3_0db0]; + + /// Runtime bytecode with a compiler-style dispatcher, a fallback that reverts, two non-payable + /// functions (CALLVALUE check), one payable function, and a CBOR metadata tail (ipfs + solc 0.8.19). + /// Returns (bytecode, jump destinations of the three functions). + pub fn evm_runtime() -> (Vec, [usize; 3]) { + let mut c: Vec = vec![0x60, 0x80, 0x60, 0x40, 0x52]; + // if calldatasize < 4 goto fallback + c.extend_from_slice(&[0x60, 0x04, 0x36, 0x10, 0x61, 0, 0]); + let fb_patch = c.len() - 2; + c.push(0x57); + c.extend_from_slice(&[0x60, 0x00, 0x35, 0x60, 0xe0, 0x1c]); + let mut patches = Vec::new(); + for sel in EVM_SELECTORS { + c.push(0x80); + c.push(0x63); + c.extend_from_slice(&sel.to_be_bytes()); + c.extend_from_slice(&[0x14, 0x61, 0, 0]); + patches.push(c.len() - 2); + c.push(0x57); + } + let fallback = c.len(); + c.extend_from_slice(&[0x5b, 0x60, 0x00, 0x80, 0xfd]); + let mut dests = [0usize; 3]; + for (i, dest) in dests.iter_mut().enumerate() { + *dest = c.len(); + if i < 2 { + // JUMPDEST CALLVALUE DUP1 ISZERO PUSH2 ok JUMPI PUSH1 0 DUP1 REVERT JUMPDEST POP STOP + c.extend_from_slice(&[0x5b, 0x34, 0x80, 0x15, 0x61, 0, 0]); + let ok_patch = c.len() - 2; + c.extend_from_slice(&[0x57, 0x60, 0x00, 0x80, 0xfd]); + let ok = c.len(); + c.extend_from_slice(&[0x5b, 0x50, 0x00]); + c[ok_patch..ok_patch + 2].copy_from_slice(&(ok as u16).to_be_bytes()); + } else { + c.extend_from_slice(&[0x5b, 0x60, 0x01, 0x60, 0x00, 0x55, 0x00]); + } + } + c[fb_patch..fb_patch + 2].copy_from_slice(&(fallback as u16).to_be_bytes()); + for (p, d) in patches.iter().zip(dests) { + c[*p..*p + 2].copy_from_slice(&(d as u16).to_be_bytes()); + } + c.push(0xfe); + let mut meta = vec![0xa2, 0x64, b'i', b'p', b'f', b's', 0x58, 0x22, 0x12, 0x20]; + meta.extend((0..32u8).map(|i| i.wrapping_mul(7))); + meta.extend_from_slice(&[0x64, b's', b'o', b'l', b'c', 0x43, 0x00, 0x08, 0x13]); + let len = meta.len() as u16; + c.extend_from_slice(&meta); + c.extend_from_slice(&len.to_be_bytes()); + (c, dests) + } + + /// Bytecode as `0x`-prefixed lowercase hex. + pub fn to_hex(b: &[u8]) -> String { + let mut s = String::from("0x"); + for x in b { + s.push_str(&format!("{x:02x}")); + } + s + } +} +pub use craft_testkit::sample_damage::{mutations, noise}; diff --git a/crates/evm/tests/evm_metadata_arrays.rs b/crates/evm/tests/evm_metadata_arrays.rs new file mode 100644 index 000000000..ceeda5c07 --- /dev/null +++ b/crates/evm/tests/evm_metadata_arrays.rs @@ -0,0 +1,39 @@ +use craft_evm as evm; +use serde_json::json; + +/// Unrelated three-integer arrays stay arrays while compiler versions stay readable. +#[test] +fn cbor_arrays_are_not_all_compiler_versions() { + // {"sizes": [1, 2, 3], "vyper": [0, 3, 10]} + for (cbor, compiler) in [ + (b"\xa2\x65sizes\x83\x01\x02\x03\x65vyper\x83\x00\x03\x0a".as_slice(), "vyper 0.3.10"), + (b"\xa2\x65sizes\x83\x01\x02\x03\x64solc\x83\x00\x08\x13".as_slice(), "solc 0.8.19"), + ] { + let mut code = vec![0x00]; + code.extend(cbor); + code.extend((cbor.len() as u16).to_be_bytes()); + let result = evm::analyze(&code).unwrap(); + let meta = result.metadata.unwrap(); + assert_eq!(meta.entries["sizes"], json!([1, 2, 3])); + assert_eq!(meta.compiler.as_deref(), Some(compiler)); + assert_eq!(result.analysed_size, 1); + } +} + +/// A non-version Vyper array stays typed, and a truncated array stays bytecode. +#[test] +fn invalid_version_shapes_are_not_invented_or_silently_consumed() { + let cbor = b"\xa1\x65vyper\x82\x00\x03"; + let mut code = vec![0x00]; + code.extend(cbor); + code.extend((cbor.len() as u16).to_be_bytes()); + let meta = evm::analyze(&code).unwrap().metadata.unwrap(); + assert_eq!(meta.compiler, None); + assert_eq!(meta.entries["vyper"], json!([0, 3])); + + let mut truncated = code; + truncated[8] = 0x83; // Declare three items but keep only two. + let result = evm::analyze(&truncated).unwrap(); + assert!(result.metadata.is_none()); + assert_eq!(result.analysed_size, truncated.len()); +} diff --git a/crates/evm/tests/evm_metadata_map_keys.rs b/crates/evm/tests/evm_metadata_map_keys.rs new file mode 100644 index 000000000..17ee4822c --- /dev/null +++ b/crates/evm/tests/evm_metadata_map_keys.rs @@ -0,0 +1,93 @@ +use craft_evm::analyze; + +fn contract_with_trailer(cbor: &[u8]) -> Vec { + let mut code = vec![0x5b, 0x00]; + code.extend_from_slice(cbor); + code.extend_from_slice(&u16::try_from(cbor.len()).unwrap().to_be_bytes()); + code +} + +fn assert_unrecognized(cbor: &[u8]) { + let code = contract_with_trailer(cbor); + let result = analyze(&code).unwrap(); + assert!(result.metadata.is_none(), "ambiguous map must remain bytecode: {cbor:02x?}"); + assert_eq!(result.analysed_size, code.len()); +} + +#[test] +fn duplicate_compiler_keys_do_not_replace_the_reported_version() { + assert_unrecognized(b"\xa2\x64solc\x43\x00\x08\x13\x64solc\x43\x00\x08\x1a"); +} + +#[test] +fn identical_duplicate_keys_are_also_ambiguous() { + assert_unrecognized(b"\xa2\x64solc\x43\x00\x08\x13\x64solc\x43\x00\x08\x13"); +} + +#[test] +fn numeric_keys_cannot_collide_with_text_keys() { + assert_unrecognized(b"\xa2\x01\x00\x61\x31\x01"); +} + +#[test] +fn metadata_map_keys_must_be_encoded_as_text() { + for key in [vec![0x01], vec![0x41, b'x'], vec![0x83, 0, 1, 2], vec![0xf5], vec![0xf6], vec![0xa0]] { + let mut cbor = vec![0xa1]; + cbor.extend(key); + cbor.push(0); + assert_unrecognized(&cbor); + } +} + +#[test] +fn nested_maps_do_not_silently_overwrite_keys() { + assert_unrecognized(b"\xa1\x65extra\xa2\x61x\x00\x61x\x01"); + assert_unrecognized(b"\xa1\x65extra\xa1\x01\x00"); +} + +#[test] +fn ordinary_compiler_metadata_and_unique_nested_text_keys_still_decode() { + let code = contract_with_trailer(b"\xa2\x64solc\x43\x00\x08\x13\x65extra\xa2\x61x\x00\x61y\x01"); + let result = analyze(&code).unwrap(); + assert_eq!(result.analysed_size, 2); + let metadata = result.metadata.unwrap(); + assert_eq!(metadata.compiler.as_deref(), Some("solc 0.8.19")); + assert_eq!(metadata.entries["extra"]["x"], 0); + assert_eq!(metadata.entries["extra"]["y"], 1); + assert_eq!(metadata.length, code.len() - 2); +} + +#[test] +fn equal_keys_in_separate_maps_are_independent() { + let code = contract_with_trailer(b"\xa2\x61a\xa1\x61x\x00\x61b\xa1\x61x\x01"); + let result = analyze(&code).unwrap(); + assert_eq!(result.analysed_size, 2); + let entries = result.metadata.unwrap().entries; + assert_eq!(entries["a"]["x"], 0); + assert_eq!(entries["b"]["x"], 1); +} + +#[test] +fn extended_length_metadata_maps_respect_the_pair_limit() { + let map = |count: u8| { + let mut cbor = vec![0xb8, count]; + for index in 0..count { + let key = format!("k{index:02}"); + cbor.push(0x63); + cbor.extend_from_slice(key.as_bytes()); + cbor.push(0); + } + cbor + }; + for count in [24, 32] { + let code = contract_with_trailer(&map(count)); + let report = analyze(&code).unwrap(); + assert_eq!(report.analysed_size, 2); + assert_eq!(report.metadata.unwrap().entries.as_object().unwrap().len(), usize::from(count)); + } + assert_unrecognized(&map(33)); + assert_unrecognized(&[0xb8]); + let mut truncated = map(24); + truncated.pop(); + assert_unrecognized(&truncated); +} diff --git a/crates/evm/tests/evm_selector_dedup.rs b/crates/evm/tests/evm_selector_dedup.rs new file mode 100644 index 000000000..99aac44ba --- /dev/null +++ b/crates/evm/tests/evm_selector_dedup.rs @@ -0,0 +1,115 @@ +use craft_evm as evm; + +fn prefix() -> Vec { + vec![0x5b, 0x00, 0x60, 0x00, 0x35, 0x60, 0xe0, 0x1c] +} + +fn comparison(code: &mut Vec, selector: u32, width: usize, destination: u8) { + code.push(0x5f + u8::try_from(width).unwrap()); + code.extend_from_slice(&selector.to_be_bytes()[4 - width..]); + code.extend([0x14, 0x60, destination, 0x57]); +} + +#[test] +fn repeated_comparisons_keep_the_first_destination_and_encounter_order() { + let mut code = prefix(); + comparison(&mut code, 0xfeed0002, 4, 0); + comparison(&mut code, 0xfeed0001, 4, 0); + let expected = evm::analyze(&code).unwrap().functions; + for _ in 0..2000 { + comparison(&mut code, 0xfeed0001, 4, 1); + comparison(&mut code, 0xfeed0002, 4, 1); + } + assert_eq!(evm::analyze(&code).unwrap().functions, expected); +} + +#[test] +fn different_push_widths_share_the_same_numeric_selector() { + let mut code = prefix(); + for width in 1..=4 { + comparison(&mut code, 17, width, if width == 1 { 0 } else { 1 }); + } + let result = evm::analyze(&code).unwrap(); + assert_eq!(result.functions.len(), 1); + assert_eq!(result.functions[0].selector, "0x00000011"); + assert_eq!(result.functions[0].bytecode_offset, 0); +} + +#[test] +fn rejected_candidates_do_not_spend_selector_identity() { + let mut code = vec![0x5b, 0x00]; + // Short comparisons without selector extraction are not considered functions. + comparison(&mut code, 17, 1, 1); + // A selector-shaped immediate without a comparison is not a function either. + code.extend([0x63, 0, 0, 0, 17, 0x50]); + comparison(&mut code, 17, 4, 0); + let result = evm::analyze(&code).unwrap(); + assert_eq!(result.functions.len(), 1); + assert_eq!(result.functions[0].bytecode_offset, 0); + assert!(result.functions[0].jumpdest_valid); +} + +#[test] +fn selector_cap_counts_unique_matches() { + let mut code = prefix(); + for selector in 0..4095 { + comparison(&mut code, selector, 4, 0); + } + for _ in 0..10000 { + comparison(&mut code, 4094, 4, 1); + } + let below = evm::analyze(&code).unwrap(); + assert_eq!(below.functions.len(), 4095); + assert!(!below.warnings.iter().any(|warning| warning.contains("stopped at"))); + comparison(&mut code, 4095, 4, 0); + comparison(&mut code, 4096, 4, 0); + let capped = evm::analyze(&code).unwrap(); + assert_eq!(capped.functions.len(), 4096); + assert_eq!(capped.functions.last().unwrap().selector, "0x00000fff"); + assert!(capped.warnings.iter().any(|warning| warning == "stopped at 4096 selectors")); +} + +#[test] +fn repeated_comparisons_keep_fallback_scan_after_the_last_dispatch() { + for (ending, expected) in [(&[0x5f, 0x5f, 0xfd][..], "absent"), (&[0x00][..], "present")] { + let mut code = prefix(); + comparison(&mut code, 0xfeed0001, 4, 0); + comparison(&mut code, 0xfeed0001, 4, 1); + code.extend_from_slice(ending); + let report = evm::analyze(&code).unwrap(); + assert_eq!(report.functions.len(), 1); + assert_eq!(report.functions[0].bytecode_offset, 0); + assert_eq!(report.fallback, expected); + } +} + +#[test] +fn selector_cap_leaves_fallback_unknown() { + for ending in [&[0x5f, 0x5f, 0xfd][..], &[0x00][..]] { + let mut code = prefix(); + for selector in 0..4097 { + comparison(&mut code, selector, 4, 0); + } + code.extend_from_slice(ending); + let report = evm::analyze(&code).unwrap(); + assert_eq!(report.functions.len(), 4096); + assert_eq!(report.fallback, "unknown"); + } +} + +#[test] +fn payability_only_follows_jumps_to_jumpdest() { + for (target, expected) in [(&[0x5b, 0x00][..], "payable"), (&[0x00][..], "unknown")] { + let mut code = prefix(); + let entry = u8::try_from(code.len() + 10).unwrap(); + comparison(&mut code, 0xfeed0001, 4, entry); + code.push(0x00); + assert_eq!(code.len(), usize::from(entry)); + code.extend([0x5b, 0x60, entry + 4, 0x56]); + code.extend_from_slice(target); + let report = evm::analyze(&code).unwrap(); + assert_eq!(report.functions.len(), 1); + assert!(report.functions[0].jumpdest_valid); + assert_eq!(report.functions[0].mutability, expected); + } +} diff --git a/crates/image-codecs/src/bmp_probe.rs b/crates/image-codecs/src/bmp_probe.rs index 5b47721bc..89f6a3b97 100644 --- a/crates/image-codecs/src/bmp_probe.rs +++ b/crates/image-codecs/src/bmp_probe.rs @@ -1,5 +1,7 @@ //! Header-only BMP dimensions and pixel-data coverage, without allocating a canvas. +use crate::error::CodecError; + /// Recognize a BMP file header with a supported DIB header kind. /// /// This is a format-routing check, not full validation: dimensions and pixel data may still @@ -13,6 +15,115 @@ pub fn is_supported_header(b: &[u8]) -> bool { false } +/// Wrap a DIB (a BMP without its 14-byte `BITMAPFILEHEADER`, as the Windows clipboard holds one +/// for `CF_DIB` and `CF_DIBV5`) as a BMP file the decoders here read. The pixel offset is computed +/// from the header with saturating arithmetic, so a hostile header cannot overflow; whether the +/// pixel data exists is left to the decoder. Apply [`dib_layout`] first for clipboard DIBs. +/// +/// # Errors +/// +/// Returns [`CodecError::LimitExceeded`] when the output (`dib.len() + 14` bytes) cannot be allocated. +pub fn dib_to_bmp(dib: &[u8]) -> Result, CodecError> { + let u16_at = |at: usize| dib.get(at..at + 2).and_then(|b| b.try_into().ok()).map(u16::from_le_bytes); + let u32_at = |at: usize| dib.get(at..at + 4).and_then(|b| b.try_into().ok()).map(u32::from_le_bytes); + let bi_size = u32_at(0).unwrap_or(40); + let offset = if bi_size == 12 { + let bpp = u16_at(10).unwrap_or(24); + let palette = if bpp <= 8 { (1u32 << bpp) * 3 } else { 0 }; + 14 + bi_size + palette + } else { + let bpp = u16_at(14).unwrap_or(24); + let compression = u32_at(16).unwrap_or(0); + let clr_used = u32_at(32).unwrap_or(0); + let palette_entries = if clr_used != 0 { + clr_used + } else if bpp <= 8 { + 1u32 << bpp + } else { + 0 + }; + // The masks of a 52- or 56-byte (V2/V3) header are inside the header; only the other + // header kinds carry them as 12 further bytes before the pixels. + let masks = if compression == 3 && bi_size >= 40 && !matches!(bi_size, 52 | 56) { 12 } else { 0 }; + 14u32.saturating_add(bi_size).saturating_add(masks).saturating_add(palette_entries.saturating_mul(4)) + }; + let total = dib.len().saturating_add(14); + let file_size = u32::try_from(total).unwrap_or(u32::MAX); + let mut bmp = Vec::new(); + bmp.try_reserve_exact(total).map_err(|_| CodecError::LimitExceeded(format!("a {total}-byte BMP")))?; + bmp.extend_from_slice(b"BM"); + bmp.extend_from_slice(&file_size.to_le_bytes()); + bmp.extend_from_slice(&[0u8; 4]); + bmp.extend_from_slice(&offset.to_le_bytes()); + bmp.extend_from_slice(dib); + Ok(bmp) +} + +/// Put a clipboard DIB with a V4 or V5 header (108 bytes or more) in the layout the decoder reads. +/// A 32-bit one whose alpha mask is set but whose compression says `BI_RGB` is marked +/// `BI_BITFIELDS` (with the usual colour masks when it gives none), so its alpha is kept as +/// Windows programs mean it; arboard makes the same change. Bit fields then need the colour masks +/// after the header too, where Windows puts a copy and the decoder skips them; a DIB without that +/// copy gets it. Any other DIB comes back unchanged, as does one whose 12 extra bytes cannot be +/// allocated. +pub fn dib_layout(mut dib: Vec) -> Vec { + let u32_at = |d: &[u8], at: usize| d.get(at..at + 4).and_then(|b| b.try_into().ok()).map(u32::from_le_bytes); + let Some(size) = u32_at(&dib, 0).filter(|&s| s >= 108).and_then(|s| usize::try_from(s).ok()) else { return dib }; + let put = |d: &mut Vec, at: usize, v: u32| { + if let Some(b) = d.get_mut(at..at + 4) { + b.copy_from_slice(&v.to_le_bytes()); + } + }; + // Decide everything first and mutate only after the 12-byte reservation succeeded, so a failed + // allocation returns the DIB unchanged. + let promote = dib.get(14..16) == Some(&[32, 0]) && u32_at(&dib, 16) == Some(0) && u32_at(&dib, 52) == Some(0xFF00_0000); + let default_masks = promote && (40..52).step_by(4).all(|at| u32_at(&dib, at) == Some(0)); + let mut masks = dib.get(40..52).and_then(|b| <[u8; 12]>::try_from(b).ok()); + if default_masks { + let mut m = [0u8; 12]; + m[0..4].copy_from_slice(&0x00FF_0000u32.to_le_bytes()); + m[4..8].copy_from_slice(&0x0000_FF00u32.to_le_bytes()); + m[8..12].copy_from_slice(&0x0000_00FFu32.to_le_bytes()); + masks = Some(m); + } + let bitfields = promote || u32_at(&dib, 16) == Some(3); + // Bytes equal to the masks right after the header are a copy only when more bytes follow than + // the pixels need; otherwise they are the pixels (e.g. a 3x1 V5 DIB with pixels equal to the masks). + let has_copy = |m: &[u8; 12]| { + dib.get(size..size.saturating_add(12)) == Some(m.as_slice()) && pixel_bytes(&dib).is_none_or(|need| dib.len().saturating_sub(size) > need) + }; + let insert = match masks { + Some(m) if bitfields && size <= dib.len() && !has_copy(&m) => Some(m), + _ => None, + }; + if insert.is_some() && dib.try_reserve(12).is_err() { + return dib; + } + if promote { + put(&mut dib, 16, 3); + if default_masks { + put(&mut dib, 40, 0x00FF_0000); + put(&mut dib, 44, 0x0000_FF00); + put(&mut dib, 48, 0x0000_00FF); + } + } + if let Some(m) = insert { + dib.splice(size..size, m); + } + dib +} + +/// Bytes of pixel rows (4-byte aligned stride) the info-header dimensions of `dib` call for, or +/// `None` when width, height or bit count cannot be read. +fn pixel_bytes(dib: &[u8]) -> Option { + let i32_at = |at: usize| dib.get(at..at + 4).and_then(|b| b.try_into().ok()).map(i32::from_le_bytes); + let w = u64::try_from(i32_at(4)?).ok()?; + let h = u64::from(i32_at(8)?.unsigned_abs()); + let bpp = u64::from(u16::from_le_bytes(dib.get(14..16)?.try_into().ok()?)); + let stride = w.checked_mul(bpp)?.div_ceil(32).checked_mul(4)?; + usize::try_from(stride.checked_mul(h)?).ok() +} + /// Coverage of the declared pixel array. Row byte counts include row padding; RLE coverage /// counts the pixels drawn by commands and may be smaller than the canvas. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -174,4 +285,335 @@ mod tests { b[0] = b'X'; assert!(!is_supported_header(&b)); } + + // The WordCraft originals, kept verbatim as the oracle. + fn oracle_dib_to_bmp(dib: &[u8]) -> Vec { + let u16_at = |at: usize| dib.get(at..at + 2).and_then(|b| b.try_into().ok()).map(u16::from_le_bytes); + let u32_at = |at: usize| dib.get(at..at + 4).and_then(|b| b.try_into().ok()).map(u32::from_le_bytes); + let bi_size = u32_at(0).unwrap_or(40); + let offset = if bi_size == 12 { + let bpp = u16_at(10).unwrap_or(24); + let palette = if bpp <= 8 { (1u32 << bpp) * 3 } else { 0 }; + 14 + bi_size + palette + } else { + let bpp = u16_at(14).unwrap_or(24); + let compression = u32_at(16).unwrap_or(0); + let clr_used = u32_at(32).unwrap_or(0); + let palette_entries = if clr_used != 0 { + clr_used + } else if bpp <= 8 { + 1u32 << bpp + } else { + 0 + }; + let masks = if compression == 3 && bi_size >= 40 { 12 } else { 0 }; + 14u32.saturating_add(bi_size).saturating_add(masks).saturating_add(palette_entries.saturating_mul(4)) + }; + let file_size = u32::try_from(14 + dib.len()).unwrap_or(u32::MAX); + let mut bmp = Vec::with_capacity(14 + dib.len()); + bmp.extend_from_slice(b"BM"); + bmp.extend_from_slice(&file_size.to_le_bytes()); + bmp.extend_from_slice(&[0u8; 4]); + bmp.extend_from_slice(&offset.to_le_bytes()); + bmp.extend_from_slice(dib); + bmp + } + + fn oracle_dib_layout(mut dib: Vec) -> Vec { + let u32_at = |d: &[u8], at: usize| d.get(at..at + 4).and_then(|b| b.try_into().ok()).map(u32::from_le_bytes); + let Some(size) = u32_at(&dib, 0).filter(|&s| s >= 108).and_then(|s| usize::try_from(s).ok()) else { return dib }; + let put = |d: &mut Vec, at: usize, v: u32| { + if let Some(b) = d.get_mut(at..at + 4) { + b.copy_from_slice(&v.to_le_bytes()); + } + }; + if dib.get(14..16) == Some(&[32, 0]) && u32_at(&dib, 16) == Some(0) && u32_at(&dib, 52) == Some(0xFF00_0000) { + put(&mut dib, 16, 3); + if (40..52).step_by(4).all(|at| u32_at(&dib, at) == Some(0)) { + put(&mut dib, 40, 0x00FF_0000); + put(&mut dib, 44, 0x0000_FF00); + put(&mut dib, 48, 0x0000_00FF); + } + } + if u32_at(&dib, 16) == Some(3) + && size <= dib.len() + && let Some(masks) = dib.get(40..52).map(<[u8]>::to_vec) + && dib.get(size..size + 12) != Some(masks.as_slice()) + { + dib.splice(size..size, masks); + } + dib + } + + fn core(bpp: u16) -> Vec { + let mut d = Vec::new(); + d.extend_from_slice(&12u32.to_le_bytes()); + d.extend_from_slice(&2u16.to_le_bytes()); + d.extend_from_slice(&2u16.to_le_bytes()); + d.extend_from_slice(&1u16.to_le_bytes()); + d.extend_from_slice(&bpp.to_le_bytes()); + d.resize(12 + 64, 7); + d + } + + #[allow(clippy::too_many_arguments)] + fn info(size: u32, bpp: u16, comp: u32, clr_used: u32, masks: [u32; 4], tail: usize) -> Vec { + let mut d = vec![0u8; size.clamp(40, 124) as usize]; + d[0..4].copy_from_slice(&size.to_le_bytes()); + d[4..8].copy_from_slice(&2i32.to_le_bytes()); + d[8..12].copy_from_slice(&2i32.to_le_bytes()); + d[12..14].copy_from_slice(&1u16.to_le_bytes()); + d[14..16].copy_from_slice(&bpp.to_le_bytes()); + d[16..20].copy_from_slice(&comp.to_le_bytes()); + d[32..36].copy_from_slice(&clr_used.to_le_bytes()); + for (i, m) in masks.iter().enumerate() { + if let Some(b) = d.get_mut(40 + 4 * i..44 + 4 * i) { + b.copy_from_slice(&m.to_le_bytes()); + } + } + d.extend((0..tail).map(|i| (i * 31 + 5) as u8)); + d + } + + fn corpus() -> Vec> { + let mut v = Vec::new(); + for bpp in [1, 4, 8, 24, 0, 99] { + v.push(core(bpp)); + } + for bpp in [1u16, 4, 8, 16, 24, 32, 0, 255] { + for clr in [0u32, 1, 5, 256, u32::MAX] { + v.push(info(40, bpp, 0, clr, [0; 4], 40)); + } + } + for (size, comp, masks) in [ + (40, 3, [0xFF0000, 0xFF00, 0xFF, 0]), + (40, 3, [0; 4]), + (52, 3, [0xFF0000, 0xFF00, 0xFF, 0]), + (108, 3, [0xFF0000, 0xFF00, 0xFF, 0xFF00_0000]), + (108, 0, [0xFF0000, 0xFF00, 0xFF, 0xFF00_0000]), + (108, 0, [0, 0, 0, 0xFF00_0000]), + (108, 0, [0; 4]), + (124, 3, [0xFF0000, 0xFF00, 0xFF, 0xFF00_0000]), + (124, 0, [0, 0, 0, 0xFF00_0000]), + (124, 0, [1, 0, 0, 0xFF00_0000]), + (124, 0, [0xFF0000, 0xFF00, 0xFF, 0]), + (124, 1, [0, 0, 0, 0xFF00_0000]), + (124, 3, [1, 2, 3, 4]), + (0, 0, [0; 4]), + (12, 0, [0; 4]), + (u32::MAX, 3, [0; 4]), + (u32::MAX, 0, [0; 4]), + (200, 3, [9; 4]), + ] { + for bpp in [8u16, 16, 32] { + for clr in [0u32, 2, u32::MAX] { + v.push(info(size, bpp, comp, clr, masks, 48)); + } + } + } + // 52/56-byte bitfields headers with and without a pixel tail, and 3x1 V5 DIBs. + for size in [52, 56] { + v.push(info(size, 32, 3, 0, [0xFF0000, 0xFF00, 0xFF, 0xFF00_0000], 4)); + v.push(info(size, 32, 3, 0, [0xFF0000, 0xFF00, 0xFF, 0xFF00_0000], 0)); + } + v.push(v5_3x1(false)); + v.push(v5_3x1(true)); + v + } + + /// 3x1 32-bit V5 BI_BITFIELDS whose three pixels equal the colour masks; with `copy` the DIB + /// also holds the real 12-byte mask copy before the pixels. + fn v5_3x1(copy: bool) -> Vec { + let masks = [0x00FF_0000u32, 0x0000_FF00, 0x0000_00FF]; + let mut d = info(124, 32, 3, 0, [masks[0], masks[1], masks[2], 0xFF00_0000], 0); + d[4..8].copy_from_slice(&3i32.to_le_bytes()); + d[8..12].copy_from_slice(&1i32.to_le_bytes()); + for _ in 0..usize::from(copy) + 1 { + for m in masks { + d.extend_from_slice(&m.to_le_bytes()); + } + } + d + } + + fn dib_u32(d: &[u8], at: usize) -> Option { + d.get(at..at + 4).and_then(|b| b.try_into().ok()).map(u32::from_le_bytes) + } + + /// Fixed case 1: a 52- or 56-byte BI_BITFIELDS header holds its masks itself, so `dib_to_bmp` + /// no longer adds the 12 appended-mask bytes the WordCraft original added to the pixel offset. + fn bmp_offset_fixed(dib: &[u8]) -> bool { + matches!(dib_u32(dib, 0).unwrap_or(40), 52 | 56) && dib_u32(dib, 16) == Some(3) + } + + /// Fixed case 2: the original read 12 pixel bytes equal to the masks (exact-length payload) as a + /// mask copy and inserted nothing. Returns the corrected output when `cut` is such a case. + fn exact_length_fixed(cut: &[u8]) -> Option> { + let o = oracle_dib_layout(cut.to_vec()); + let size = usize::try_from(dib_u32(&o, 0).filter(|&s| s >= 108)?).ok()?; + let need = pixel_bytes(&o)?; + let same = o.len() == cut.len() + && dib_u32(&o, 16) == Some(3) + && size <= o.len() + && o.get(size..size + 12).is_some() + && o.get(size..size + 12) == o.get(40..52) + && o.len() - size <= need; + if !same { + return None; + } + let mut want = o.clone(); + want.splice(size..size, o[40..52].to_vec()); + Some(want) + } + + #[test] + fn dib_helpers_match_the_wordcraft_originals_byte_for_byte() { + let (mut total, mut offset_cases, mut exact_cases) = (0usize, 0usize, 0usize); + for dib in corpus() { + for n in 0..=dib.len() { + let cut = &dib[..n]; + total += 1; + let fixed_layout = exact_length_fixed(cut); + match &fixed_layout { + Some(want) => { + exact_cases += 1; + assert_eq!(&dib_layout(cut.to_vec()), want, "exact-length layout len {n}"); + } + None => assert_eq!(dib_layout(cut.to_vec()), oracle_dib_layout(cut.to_vec()), "layout len {n}"), + } + if bmp_offset_fixed(cut) { + offset_cases += 1; + let (new, old) = (dib_to_bmp(cut).unwrap(), oracle_dib_to_bmp(cut)); + assert_eq!(new[..10], old[..10]); + assert_eq!(new[14..], old[14..]); + let (a, b) = (dib_u32(&new, 10).unwrap(), dib_u32(&old, 10).unwrap()); + assert_eq!(a.saturating_add(12), b, "offset len {n}"); + } else { + assert_eq!(dib_to_bmp(cut).unwrap(), oracle_dib_to_bmp(cut), "bmp len {n}"); + } + let both = dib_layout(cut.to_vec()); + let old_both = oracle_dib_layout(cut.to_vec()); + if fixed_layout.is_none() && !bmp_offset_fixed(&both) { + assert_eq!(dib_to_bmp(&both).unwrap(), oracle_dib_to_bmp(&old_both)); + } + } + } + eprintln!("corpus cases {total}: 52/56-byte bitfields offset {offset_cases}, exact-length payload {exact_cases}"); + assert!(offset_cases > 0 && exact_cases > 0); + } + + #[test] + fn dib_helpers_survive_pseudo_random_bytes() { + let mut x = 0x9E37_79B9_7F4A_7C15u64; + for len in 0..300usize { + let d: Vec = (0..len) + .map(|_| { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + x as u8 + }) + .collect(); + let laid = dib_layout(d.clone()); + if exact_length_fixed(&d).is_none() && !bmp_offset_fixed(&laid) { + assert_eq!(dib_to_bmp(&laid).unwrap(), oracle_dib_to_bmp(&oracle_dib_layout(d))); + } + } + } + + #[test] + fn huge_palette_count_saturates_the_offset() { + let b = dib_to_bmp(&info(40, 8, 0, u32::MAX, [0; 4], 8)).unwrap(); + assert_eq!(u32::from_le_bytes([b[10], b[11], b[12], b[13]]), u32::MAX); + let b = dib_to_bmp(&info(u32::MAX, 8, 3, u32::MAX, [0; 4], 8)).unwrap(); + assert_eq!(u32::from_le_bytes([b[10], b[11], b[12], b[13]]), u32::MAX); + } + + #[test] + fn wrapped_dibs_decode_and_keep_alpha() { + // 2x2 32-bit V5, BI_RGB with an alpha mask, bottom-up BGRA pixels. + let mut d = info(124, 32, 0, 0, [0, 0, 0, 0xFF00_0000], 0); + let px = [[1u8, 2, 3, 255], [4, 5, 6, 128], [7, 8, 9, 64], [10, 11, 12, 0]]; + for p in px { + d.extend_from_slice(&p); + } + let bmp = dib_to_bmp(&dib_layout(d)).unwrap(); + let img = crate::decode(&bmp).unwrap(); + let alphas: Vec = img.to_rgba8().chunks(4).map(|p| u16::from(p[3])).collect(); + assert_eq!(alphas, [64, 0, 255, 128]); + // 24-bit plain DIB and a 12-byte core 8-bit one open too. + let mut d = info(40, 24, 0, 0, [0; 4], 0); + d.extend_from_slice(&[0, 0, 255, 0, 255, 0, 0, 0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]); + assert!(crate::decode(&dib_to_bmp(&d).unwrap()).is_ok()); + let mut c = core(8); + c.truncate(12); + c.extend((0..256 * 3).map(|i| i as u8)); + c.extend_from_slice(&[0, 1, 0, 0, 2, 3, 0, 0]); + assert!(crate::decode(&dib_to_bmp(&c).unwrap()).is_ok()); + } + + /// 1x1 32-bit BI_BITFIELDS DIB, one pixel B=10 G=20 R=30 A=128, with the given header size. + fn one_pixel(size: u32) -> Vec { + let mut d = info(size, 32, 3, 0, [0xFF0000, 0xFF00, 0xFF, 0xFF00_0000], 0); + d[4..8].copy_from_slice(&1i32.to_le_bytes()); + d[8..12].copy_from_slice(&1i32.to_le_bytes()); + if size == 40 { + for m in [0xFF0000u32, 0xFF00, 0xFF] { + d.extend_from_slice(&m.to_le_bytes()); + } + } + d.extend_from_slice(&[10, 20, 30, 128]); + d + } + + #[test] + fn bitfields_pixel_offset_for_each_header_size_decodes() { + for size in [40u32, 52, 56] { + let bmp = dib_to_bmp(&one_pixel(size)).unwrap(); + let off = 14 + size + if size == 40 { 12 } else { 0 }; + assert_eq!(dib_u32(&bmp, 10), Some(off), "offset for {size}"); + let img = crate::decode(&bmp).unwrap_or_else(|e| panic!("{size}-byte header: {e:?}")); + let rgba = img.to_rgba8(); + assert_eq!(&rgba[..3], &[30, 20, 10], "size {size}"); + if size == 56 { + assert_eq!(rgba[3], 128); + } + } + // V5: the header's masks, a copy inserted by `dib_layout`, then the pixel. + let bmp = dib_to_bmp(&dib_layout(one_pixel(124))).unwrap(); + assert_eq!(&crate::decode(&bmp).unwrap().to_rgba8()[..4], &[30, 20, 10, 128]); + } + + #[test] + fn exact_length_pixels_equal_to_the_masks_are_not_a_mask_copy() { + let d = v5_3x1(false); + let laid = dib_layout(d.clone()); + assert_eq!(laid.len(), d.len() + 12); + assert_eq!(laid[124..136], d[40..52]); + assert_eq!(oracle_dib_layout(d).len(), laid.len() - 12, "the original skipped the insertion"); + let img = crate::decode(&dib_to_bmp(&laid).unwrap()).unwrap().to_rgba8(); + assert_eq!(img.len(), 12); + // pixel 0 is 0x00FF0000 (blue 0, green 0, red 255 under the masks). + assert_eq!(&img[..3], &[255, 0, 0]); + // A real copy followed by the pixels is left alone. + let real = v5_3x1(true); + assert_eq!(dib_layout(real.clone()), real); + assert!(crate::decode(&dib_to_bmp(&real).unwrap()).is_ok()); + } + + #[test] + fn layout_decides_before_it_mutates() { + // BI_RGB V5 with an alpha mask and no colour masks: promoted and given a copy together. + let mut d = info(124, 32, 0, 0, [0, 0, 0, 0xFF00_0000], 16); + d[4..8].copy_from_slice(&2i32.to_le_bytes()); + let laid = dib_layout(d.clone()); + assert_eq!(laid.len(), d.len() + 12); + assert_eq!(dib_u32(&laid, 16), Some(3)); + // Unreadable dimensions keep the byte-equality rule. + let mut odd = info(108, 32, 3, 0, [1, 2, 3, 0], 0); + odd[4..8].copy_from_slice(&(-1i32).to_le_bytes()); + let copy = odd[40..52].to_vec(); + odd.extend_from_slice(©); + assert_eq!(dib_layout(odd.clone()), odd); + } } diff --git a/crates/image-codecs/src/codecs/jpeg.rs b/crates/image-codecs/src/codecs/jpeg.rs index 415a5d717..989956182 100644 --- a/crates/image-codecs/src/codecs/jpeg.rs +++ b/crates/image-codecs/src/codecs/jpeg.rs @@ -46,6 +46,45 @@ pub(crate) struct JpegMeta { pub held_coefficients: u64, } +/// Validate a JPEG by decoding the 8-bit RGBA output used by renderers. +/// +/// This deliberately uses the existing zune backend's default decode policy. +/// Unlike general JPEG import it does not fall back to additional JPEG modes, +/// apply orientation, or retain pixels. The caller supplies the admitted pixel count. +pub fn validate_jpeg_rgba8(bytes: &[u8], max_pixels: u64) -> Result<(), CodecError> { + validate_jpeg_rgba8_with_panic_policy(bytes, max_pixels, cfg!(panic = "abort")) +} + +fn validate_jpeg_rgba8_with_panic_policy(bytes: &[u8], max_pixels: u64, panic_aborts: bool) -> Result<(), CodecError> { + // The decoder can assert while upsampling these frames. On abort builds the + // unwind guard below cannot turn that assertion into an image omission. + if panic_aborts && first_component_subsampled(bytes) { + return Err(CodecError::malformed(F, "JPEG first component is subsampled below the frame maximum")); + } + let decode = || { + let options = DecoderOptions::default().jpeg_set_out_colorspace(ColorSpace::RGBA); + let mut decoder = zune_jpeg::JpegDecoder::new_with_options(std::io::Cursor::new(bytes), options); + decoder.decode_headers().map_err(|error| CodecError::malformed(F, format_args!("{error:?}")))?; + let info = decoder.info().ok_or_else(|| CodecError::malformed(F, "missing frame dimensions"))?; + let pixels = u64::from(info.width) * u64::from(info.height); + if pixels == 0 { + return Err(CodecError::malformed(F, "empty frame")); + } + if pixels > max_pixels { + return Err(CodecError::LimitExceeded("JPEG exceeds the admitted pixel count".into())); + } + let length = usize::try_from(pixels * 4).map_err(|_| CodecError::LimitExceeded("JPEG RGBA output is too large".into()))?; + if decoder.output_colorspace() != Some(ColorSpace::RGBA) || decoder.output_buffer_size() != Some(length) { + return Err(CodecError::malformed(F, "unexpected RGBA output")); + } + let mut output = Vec::new(); + output.try_reserve_exact(length).map_err(|_| CodecError::LimitExceeded("could not allocate JPEG RGBA output".into()))?; + output.resize(length, 0); + decoder.decode_into(&mut output).map_err(|error| CodecError::malformed(F, format_args!("{error:?}"))) + }; + std::panic::catch_unwind(decode).map_err(|_| CodecError::malformed(F, "JPEG decoder failed on this frame"))? +} + /// The first frame of `b` as [`crate::JpegInfo`], from its markers alone (see [`crate::jpeg_info`]). pub(crate) fn info(b: &[u8]) -> Option { let m = scan_metadata(b); @@ -86,6 +125,83 @@ fn held_coefficients(marker: u8, seg: &[u8], scan_components: Option) -> u64 factors.iter().map(|&(fh, fv)| mcus * fh * fv * 64 * 2).sum() } +/// Bytes of DCT coefficients to budget for before decoding hostile `b`: the larger of +/// [`crate::JpegInfo::held_coefficients`]'s rule (first frame; SOF2, or SOF0/SOF1 whose first +/// scan lacks components; 8-bit only; sampling factors as written) and a coarser one that +/// keeps a progressive JPEG's coefficients for the whole picture: +/// - every SOF2, SOF6, SOF10 and SOF14 header before the first scan counts, at any precision, and +/// the largest wins; each charges 2 bytes a coefficient: per component, `2 * 64 * h * v` per +/// MCU, with `h` and `v` clamped to 1..=4 and the MCUs covering the frame at the largest `h`, `v`; +/// - a header that can't be followed (a segment that is cut, a length under 2, a frame header +/// without its components, no `0xFF` where a marker belongs) charges the costliest frame, +/// four components sampled 4 × 4, at the first frame's size (as `scan_metadata` reads it), and 0 without one. +/// +/// Never less than either rule. Frames `craft-jpeg` decodes (arithmetic, lossless, 12-bit) +/// enforce `max_alloc` on their own buffers, and SOF6/SOF14 (hierarchical) are not decoded at all, +/// so charging them is only a conservative budget. Arithmetic saturates; the scan is linear. +pub(crate) fn worst_case_coefficients(b: &[u8]) -> u64 { + let m = scan_metadata(b); + let mut at = 2usize; + let mut most = 0u64; + let first: Option<(u64, u64)> = m.frame.map(|(w, h, _)| (u64::from(w), u64::from(h))); + let mut broken = false; + while !broken { + if b.get(at) != Some(&0xFF) { + broken = true; + break; + } + while b.get(at) == Some(&0xFF) { + at += 1; + } + let Some(&code) = b.get(at) else { + broken = true; + break; + }; + at += 1; + match code { + 0xD8 | 0x01 | 0xD0..=0xD7 => continue, + 0xD9 | 0xDA => break, + _ => {} + } + let len = match (b.get(at), b.get(at + 1)) { + (Some(&hi), Some(&lo)) => usize::from(u16::from_be_bytes([hi, lo])), + _ => 0, + }; + let Some(seg) = (len >= 2).then(|| b.get(at + 2..at + len)).flatten() else { + broken = true; + break; + }; + if matches!(code, 0xC2 | 0xC6 | 0xCA | 0xCE) { + match progressive_frame_coefficients(seg) { + Some(held) => most = most.max(held), + None => { + broken = true; + break; + } + } + } + at += len; + } + if broken && let Some((w, h)) = first { + let mcus = w.div_ceil(32).saturating_mul(h.div_ceil(32)); + most = most.max(mcus.saturating_mul(2 * 64 * 16 * 4)); + } + most.max(m.held_coefficients) +} + +/// [`worst_case_coefficients`]'s charge for one frame header `sof` (after its length). +fn progressive_frame_coefficients(sof: &[u8]) -> Option { + let height = u64::from(u16::from_be_bytes([*sof.get(1)?, *sof.get(2)?])); + let width = u64::from(u16::from_be_bytes([*sof.get(3)?, *sof.get(4)?])); + let n = usize::from(*sof.get(5)?); + let comps = sof.get(6..6usize.checked_add(n.checked_mul(3)?)?)?; + let factors: Vec<(u64, u64)> = + comps.as_chunks::<3>().0.iter().map(|&[_, f, _]| (u64::from(f >> 4).clamp(1, 4), u64::from(f & 15).clamp(1, 4))).collect(); + let (hmax, vmax) = factors.iter().fold((1, 1), |(h, v), &(a, b)| (h.max(a), v.max(b))); + let mcus = width.div_ceil(8 * hmax).saturating_mul(height.div_ceil(8 * vmax)); + Some(factors.iter().fold(0u64, |sum, &(h, v)| sum.saturating_add(mcus.saturating_mul(2 * 64 * h * v)))) +} + /// Walk the marker segments up to SOS and collect metadata. pub(crate) fn scan_metadata(b: &[u8]) -> JpegMeta { let mut m = JpegMeta::default(); @@ -340,10 +456,17 @@ fn first_component_subsampled(b: &[u8]) -> bool { return false; } let mut i = 2; - while let (Some(&0xFF), Some(&m)) = (b.get(i), b.get(i + 1)) { + while let Some(&prefix) = b.get(i) { + // The decoder searches forward between segments. Stray bytes and + // stuffed FF 00 pairs must not hide a frame from the abort-build guard. + if prefix != 0xFF { + i += 1; + continue; + } + let Some(&m) = b.get(i + 1) else { return false }; match m { 0xFF => i += 1, - 0x01 | 0xD0..=0xD8 => i += 2, + 0x00 | 0x01 | 0xD0..=0xD8 => i += 2, 0xD9 | 0xDA => return false, _ => { let Some(len) = b.get(i + 2..i + 4).map(|l| usize::from(u16::from_be_bytes([l[0], l[1]]))) else { return false }; @@ -825,6 +948,170 @@ mod tests { assert_eq!(held(&frame(0xC2, 12, &[0x11, 0x11, 0x11], None)), Some(0)); } + /// The consumer's own estimate (WordCraft's `jpeg_coefficients`), ported as the oracle that + /// `worst_case_coefficients` must never undercut. + fn oracle(bytes: &[u8]) -> u64 { + fn worst(bytes: &[u8]) -> u64 { + let Some((w, h)) = info(bytes).map(|i| (i.width, i.height)) else { return 0 }; + let mcus = u64::from(w).div_ceil(32) * u64::from(h).div_ceil(32); + mcus.saturating_mul(2 * 64 * 16 * 4) + } + fn frame(sof: &[u8]) -> Option { + let height = u64::from(u16::from_be_bytes([*sof.get(1)?, *sof.get(2)?])); + let width = u64::from(u16::from_be_bytes([*sof.get(3)?, *sof.get(4)?])); + let n = usize::from(*sof.get(5)?); + let s: Vec<(u64, u64)> = sof + .get(6..6 + 3 * n)? + .as_chunks::<3>() + .0 + .iter() + .map(|&[_, s, _]| (u64::from(s >> 4).clamp(1, 4), u64::from(s & 15).clamp(1, 4))) + .collect(); + let (hm, vm) = s.iter().fold((1, 1), |(h, v), &(a, b)| (h.max(a), v.max(b))); + let mcus = width.div_ceil(8 * hm).checked_mul(height.div_ceil(8 * vm))?; + s.iter().try_fold(0u64, |sum, &(h, v)| sum.checked_add(mcus.checked_mul(2 * 64 * h * v)?)) + } + let mut at = 2; + let mut most = 0u64; + loop { + if bytes.get(at) != Some(&0xFF) { + return worst(bytes).max(most); + } + while bytes.get(at) == Some(&0xFF) { + at += 1; + } + let Some(&code) = bytes.get(at) else { return worst(bytes).max(most) }; + at += 1; + match code { + 0xD8 | 0x01 | 0xD0..=0xD7 => continue, + 0xD9 | 0xDA => return most, + _ => {} + } + let len = match (bytes.get(at), bytes.get(at + 1)) { + (Some(&hi), Some(&lo)) => usize::from(u16::from_be_bytes([hi, lo])), + _ => 0, + }; + let Some(seg) = bytes.get(at + 2..at + len).filter(|_| len >= 2) else { return worst(bytes).max(most) }; + if matches!(code, 0xC2 | 0xC6 | 0xCA | 0xCE) { + let Some(h) = frame(seg) else { return worst(bytes).max(most) }; + most = most.max(h); + } + at += len; + } + } + + /// A JPEG header: each frame is (marker, precision, width, height, sampling bytes), then an + /// optional scan of `scan` components, then EOI. + fn header(frames: &[(u8, u8, u16, u16, &[u8])], scan: Option) -> Vec { + let mut b = vec![0xFF, 0xD8]; + for &(marker, precision, w, h, f) in frames { + b.extend_from_slice(&[0xFF, marker]); + b.extend_from_slice(&(8 + 3 * f.len() as u16).to_be_bytes()); + b.push(precision); + b.extend_from_slice(&h.to_be_bytes()); + b.extend_from_slice(&w.to_be_bytes()); + b.push(f.len() as u8); + for (i, &s) in f.iter().enumerate() { + b.extend_from_slice(&[i as u8 + 1, s, 0]); + } + } + if let Some(n) = scan { + b.extend_from_slice(&[0xFF, 0xDA, 0, 6 + 2 * n, n]); + (1..=n).for_each(|c| b.extend_from_slice(&[c, 0])); + b.extend_from_slice(&[0, 63, 0]); + } + b.extend_from_slice(&[0xFF, 0xD9]); + b + } + + /// Never less than the oracle or `held_coefficients`; equal to the larger on valid headers. + #[test] + fn worst_case_coefficients_covers_both_rule_sets() { + const C3: &[u8] = &[0x22, 0x11, 0x11]; + const G: &[u8] = &[0x11]; + let cases: Vec> = vec![ + header(&[(0xC0, 8, 640, 480, C3)], Some(3)), + header(&[(0xC0, 8, 640, 480, G)], Some(1)), + header(&[(0xC0, 8, 640, 480, C3)], Some(1)), + header(&[(0xC1, 8, 640, 480, C3)], None), + header(&[(0xC2, 8, 640, 480, C3)], Some(1)), + header(&[(0xC2, 12, 640, 480, C3)], Some(1)), + header(&[(0xC6, 8, 640, 480, C3)], None), + header(&[(0xCA, 8, 640, 480, C3)], None), + header(&[(0xCE, 8, 640, 480, C3)], None), + header(&[(0xC0, 8, 64, 64, G), (0xC2, 8, 6000, 4000, C3), (0xC2, 8, 100, 100, C3)], Some(1)), + header(&[(0xC2, 8, 640, 480, &[0x00, 0x00, 0x00])], None), + header(&[(0xC2, 8, 640, 480, &[0xFF, 0xFF, 0xFF, 0xFF])], None), + header(&[(0xC2, 8, 65535, 65535, &[0x44; 4])], None), + header(&[(0xC0, 8, 65535, 65535, &[0xFF; 4])], Some(1)), + header(&[], None), + vec![0xFF, 0xD8], + vec![], + b"not a jpeg at all".to_vec(), + ]; + for b in &cases { + let held = scan_metadata(b).held_coefficients; + let got = worst_case_coefficients(b); + assert!(got >= oracle(b) && got >= held, "{b:02x?}"); + assert!(got >= info(b).map_or(0, |i| i.held_coefficients)); + } + let at = |i: usize| worst_case_coefficients(&cases[i]); + // Equal to the oracle where the shared rule charges nothing extra. + for i in [0, 1, 6, 7, 8, 9, 10, 12, 14] { + assert_eq!(at(i), oracle(&cases[i]), "case {i}"); + } + // Where it does, the shared figure wins (the oracle misses a short first scan). + assert_eq!(oracle(&cases[2]), 0); + assert_eq!(at(2), scan_metadata(&cases[2]).held_coefficients); + // Unclamped factors (15 × 15) charge the shared rule more than the clamped oracle. + assert_eq!(at(11), scan_metadata(&cases[11]).held_coefficients); + assert!(at(11) > oracle(&cases[11])); + assert!(at(2) > 0 && at(3) == 0 && at(0) == 0); + // 12-bit and arithmetic progressive frames are charged though craft-jpeg limits itself. + assert_eq!(at(5), at(4)); + // Sampling factors clamp: 0 acts as 1, 15 as 4. + assert_eq!(at(10), 80 * 60 * 3 * 128); + assert_eq!(at(12), 2048 * 2048 * 4 * 2 * 64 * 16); + // The largest of several frames wins. + assert_eq!(at(9), oracle(&header(&[(0xC2, 8, 6000, 4000, C3)], None))); + } + + /// A cut header charges the costliest frame at its size; every prefix and random damage stays + /// bounded, never panics and never undercuts the oracle. + #[test] + fn worst_case_coefficients_hostile_input() { + let full = header(&[(0xC2, 8, 6000, 4000, &[0x22, 0x11, 0x11])], Some(1)); + let cut = &full[..full.len() - 8]; + // Cut inside the component list: a frame the scan can't read, at its own size. + let sof_cut = &full[..14]; + assert_eq!(worst_case_coefficients(sof_cut), oracle(sof_cut)); + assert_eq!(worst_case_coefficients(cut), oracle(cut)); + let mut bases = vec![full.clone(), header(&[(0xC0, 8, 6000, 4000, &[0x11; 3])], Some(1))]; + bases.push(header(&[(0xC2, 8, 65535, 65535, &[0xFF; 4])], None)); + let mut seed = 0x9E37_79B9_7F4A_7C15u64; + let mut next = move || { + seed = seed.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1_442_695_040_888_963_407); + (seed >> 33) as usize + }; + for base in &bases { + for n in 0..=base.len() { + let b = &base[..n]; + let got = worst_case_coefficients(b); + assert!(got >= oracle(b) && got < 1 << 50); + } + for _ in 0..2000 { + let mut b = base.clone(); + for _ in 0..=next() % 4 { + let i = next() % b.len(); + b[i] = next() as u8; + } + b.truncate(next() % (b.len() + 1)); + let got = worst_case_coefficients(&b); + assert!(got >= oracle(&b) && got >= scan_metadata(&b).held_coefficients && got < 1 << 50); + } + } + } + /// An APP1 Extended XMP fragment (GUID, total size, offset, data). fn extended_xmp(offset: u32) -> Vec { let mut seg = EXTENDED_XMP_HEADER.to_vec(); @@ -1073,25 +1360,61 @@ mod tests { (v * 65535.0 / 4095.0).round() as u16 } + fn progressive_frame_header(comps: &[(u8, u8)]) -> Vec { + let mut s = vec![0xFF, 0xD8, 0xFF, 0xE0, 0, 4, 0, 0, 0xFF, 0xC2, 0, (8 + 3 * comps.len()) as u8, 8, 0, 8, 0, 8, comps.len() as u8]; + for (i, &(h, v)) in comps.iter().enumerate() { + s.extend([i as u8 + 1, (h << 4) | v, 0]); + } + s + } + #[test] fn first_component_subsampling() { - let sof = |comps: &[(u8, u8)]| { - let mut s = vec![0xFF, 0xD8, 0xFF, 0xE0, 0, 4, 0, 0, 0xFF, 0xC2, 0, (8 + 3 * comps.len()) as u8, 8, 0, 8, 0, 8, comps.len() as u8]; - for (i, &(h, v)) in comps.iter().enumerate() { - s.extend([i as u8 + 1, (h << 4) | v, 0]); - } - s - }; - assert!(first_component_subsampled(&sof(&[(1, 1), (1, 1), (1, 1), (2, 2)]))); - assert!(first_component_subsampled(&sof(&[(1, 2), (1, 2), (2, 1)]))); - assert!(!first_component_subsampled(&sof(&[(2, 2), (1, 1), (1, 1)]))); - assert!(!first_component_subsampled(&sof(&[(1, 1); 4]))); - assert!(!first_component_subsampled(&sof(&[]))); + assert!(first_component_subsampled(&progressive_frame_header(&[(1, 1), (1, 1), (1, 1), (2, 2)]))); + assert!(first_component_subsampled(&progressive_frame_header(&[(1, 2), (1, 2), (2, 1)]))); + assert!(!first_component_subsampled(&progressive_frame_header(&[(2, 2), (1, 1), (1, 1)]))); + assert!(!first_component_subsampled(&progressive_frame_header(&[(1, 1); 4]))); + assert!(!first_component_subsampled(&progressive_frame_header(&[]))); assert!(!first_component_subsampled(&[0xFF, 0xD8, 0xFF, 0xDA, 0, 2])); assert!(!first_component_subsampled(&[0xFF, 0xD8, 0xFF, 0xC0, 0, 20, 8])); assert!(!first_component_subsampled(b"not a jpeg")); } + #[test] + fn renderer_validation_refuses_unsafe_sampling_under_abort_policy() { + // The frame header alone exercises refusal before any pixel decode. + let header = progressive_frame_header(&[(1, 1), (1, 1), (1, 1), (2, 2)]); + let refuses_sampling = |result| { + matches!( + result, + Err(CodecError::Malformed { format: Format::Jpeg, message }) + if message == "JPEG first component is subsampled below the frame maximum" + ) + }; + assert!(refuses_sampling(validate_jpeg_rgba8_with_panic_policy(&header, 64, true))); + assert!(!refuses_sampling(validate_jpeg_rgba8_with_panic_policy(&header, 64, false))); + + let mut ordinary = Vec::new(); + jpeg_encoder::Encoder::new(&mut ordinary, 90).encode(&[128], 1, 1, jpeg_encoder::ColorType::Luma).unwrap(); + validate_jpeg_rgba8_with_panic_policy(&ordinary, 1, true).unwrap(); + } + + #[test] + fn jpeg_sampling_guard_finds_frame_after_marker_padding() { + let header = progressive_frame_header(&[(1, 1), (1, 1), (1, 1), (2, 2)]); + // Exercise the shared header guard, without a pixel payload. + for split in [2, 8] { + let (before, after) = header.split_at(split); + for padding in [&[0][..], &[0x7F, 0][..], &[0xFF, 0][..], &[0xFF][..]] { + let padded = [before, padding, after].concat(); + assert!(first_component_subsampled(&padded), "offset {split}, padding {padding:?}"); + let got = validate_jpeg_rgba8_with_panic_policy(&padded, 64, true); + assert!(matches!(got, Err(CodecError::Malformed { message, .. }) if message.contains("frame maximum"))); + } + } + assert!(!first_component_subsampled(&[0xFF, 0xD8, 0, 0xFF])); + } + #[test] fn routing() { assert!(!uses_craft_jpeg(&[0xFF, 0xD8, 0xFF, 0xC0, 0, 11, 8, 0, 8, 0, 8, 1, 1, 0x11, 0])); diff --git a/crates/image-codecs/src/lib.rs b/crates/image-codecs/src/lib.rs index dc2ac2933..032289c7c 100644 --- a/crates/image-codecs/src/lib.rs +++ b/crates/image-codecs/src/lib.rs @@ -45,6 +45,7 @@ pub use crate::cicp::ContentLightLevel; pub use crate::codecs::avif::{HDR_REFERENCE_WHITE_NITS, HdrTransfer, encode_avif_hdr, encode_avif_with_speed}; pub use crate::codecs::exr_cryptomatte::{cryptomatte_id, cryptomatte_key, cryptomatte_preview_color}; pub use crate::codecs::gif_frames::{AnimationFrame, MAX_GIF_FRAMES}; +pub use crate::codecs::jpeg::validate_jpeg_rgba8; pub use crate::codecs::png::{ PngDecodeRows, PngExtras, PngHeader, PngPalette, PngRowSink, PngRows, SharedBytes, encode_indexed as encode_png_indexed, encode_pq as encode_png_pq, palette as png_palette, stereo as png_stereo, @@ -307,6 +308,15 @@ pub fn jpeg_info(bytes: &[u8]) -> Option { jpeg::info(bytes) } +/// Bytes of DCT coefficients to budget for a JPEG before decoding it: never less than +/// [`JpegInfo::held_coefficients`] nor than the whole-picture estimate of every progressive +/// frame header (SOF2, SOF6, SOF10, SOF14 at any precision, sampling clamped to 1..=4), and the +/// costliest frame at the first size read when the header can't be followed. 0 without a +/// readable frame. See the function body's rules for the precise charges. +pub fn jpeg_worst_case_coefficients(bytes: &[u8]) -> u64 { + jpeg::worst_case_coefficients(bytes) +} + /// PCX's dimensions are available from its fixed-size header without decoding the RLE rows. pub fn pcx_dimensions(bytes: &[u8]) -> Option<(u32, u32)> { codecs::pcx::dimensions(bytes) diff --git a/crates/image-codecs/src/png_probe.rs b/crates/image-codecs/src/png_probe.rs index 73f1199eb..e7af74334 100644 --- a/crates/image-codecs/src/png_probe.rs +++ b/crates/image-codecs/src/png_probe.rs @@ -136,6 +136,93 @@ pub fn color_chunks(b: &[u8]) -> Option { Some(result) } +/// Most chunks [`pixel_model`] walks before the image data; each chunk is at least 12 bytes. +const MAX_PROBE_CHUNKS: usize = 1 << 16; + +/// Pixel model of a PNG, from `IHDR` and the chunk headers before the first `IDAT`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct PixelModel { + pub width: u32, + pub height: u32, + /// `IHDR` bit depth (1, 2, 4, 8 or 16). + pub bit_depth: u8, + /// `IHDR` colour type (0 grey, 2 RGB, 3 palette, 4 grey+alpha, 6 RGBA). + pub color_type: u8, + /// A `tRNS` chunk precedes the image data; it adds alpha to grey, RGB and palette images. + pub has_trns: bool, +} + +impl PixelModel { + /// Channels of the decoded buffer (palette expands to RGB; `tRNS` adds an alpha channel). + pub fn channels(&self) -> u8 { + let base = match self.color_type { + 0 => 1, + 2 | 3 => 3, + 4 => 2, + _ => 4, + }; + base + u8::from(self.has_trns && matches!(self.color_type, 0 | 2 | 3)) + } + + /// Bytes per pixel of the buffer a decoder that expands sub-byte samples to 8 bits gives: + /// `channels * (2 if bit_depth == 16 else 1)`. + pub fn bytes_per_pixel(&self) -> u8 { + self.channels() * if self.bit_depth == 16 { 2 } else { 1 } + } + + /// Whether the image has transparency (alpha colour type or `tRNS`). + pub fn has_alpha(&self) -> bool { + matches!(self.color_type, 4 | 6) || self.has_trns + } +} + +/// Dimensions, bit depth, colour type and `tRNS` presence without inflating anything (an `iCCP` +/// profile is skipped, not decompressed). Walks at most 65,536 chunk headers and stops at `IDAT`; +/// `IEND` or the end of the input before `IDAT` is an error. Does not validate CRCs. +/// A caller gets its `(width, height, bytes_per_pixel, alpha)` as +/// `(m.width, m.height, u64::from(m.bytes_per_pixel()), m.has_alpha())`. +pub fn pixel_model(b: &[u8]) -> Result { + if !b.starts_with(b"\x89PNG\r\n\x1a\n") { + return Err(err("bad signature")); + } + let be32 = |p: usize| b.get(p..p.checked_add(4)?).and_then(|v| <[u8; 4]>::try_from(v).ok()).map(u32::from_be_bytes); + if be32(8) != Some(13) || b.get(12..16) != Some(b"IHDR") { + return Err(err("missing IHDR")); + } + let ihdr = b.get(16..29).ok_or_else(|| err("truncated IHDR"))?; + let (width, height) = (be32(16).unwrap_or(0), be32(20).unwrap_or(0)); + let (bit_depth, color_type) = (ihdr[8], ihdr[9]); + if width == 0 || height == 0 || width > i32::MAX as u32 || height > i32::MAX as u32 { + return Err(err("dimensions out of range")); + } + let depth_ok = match color_type { + 0 => matches!(bit_depth, 1 | 2 | 4 | 8 | 16), + 3 => matches!(bit_depth, 1 | 2 | 4 | 8), + 2 | 4 | 6 => matches!(bit_depth, 8 | 16), + _ => false, + }; + if !depth_ok { + return Err(err("invalid IHDR")); + } + let mut has_trns = false; + let mut p = 8usize; + for _ in 0..MAX_PROBE_CHUNKS { + let len = be32(p).ok_or_else(|| err("truncated"))?; + let kind = p.checked_add(4).and_then(|k| b.get(k..k.checked_add(4)?)).ok_or_else(|| err("truncated"))?; + match kind { + b"IDAT" => return Ok(PixelModel { width, height, bit_depth, color_type, has_trns }), + b"IEND" => return Err(err("no image data")), + b"tRNS" => has_trns = true, + _ => {} + } + if len > i32::MAX as u32 { + return Err(err("chunk length out of range")); + } + p = p.checked_add(12).and_then(|q| q.checked_add(len as usize)).ok_or_else(|| err("truncated"))?; + } + Err(CodecError::LimitExceeded("PNG chunk count".into())) +} + #[cfg(test)] mod tests { use super::*; @@ -187,4 +274,140 @@ mod tests { assert!(!color.srgb); assert!(color_chunks(b"not png").is_none()); } + + /// WordCraft's `png_chunks` (crates/geom/src/picture.rs) as the differential oracle. + fn oracle(bytes: &[u8]) -> Option<(u32, u32, u64, bool)> { + let be = |at: usize| bytes.get(at..at.checked_add(4)?).and_then(|b| <[u8; 4]>::try_from(b).ok()).map(u32::from_be_bytes); + if bytes.get(12..16) != Some(b"IHDR".as_slice()) { + return None; + } + let (width, height) = (be(16)?, be(20)?); + let (depth, color) = (*bytes.get(24)?, *bytes.get(25)?); + let mut trns = false; + let mut at = 8usize; + loop { + let len = usize::try_from(be(at)?).ok()?; + match bytes.get(at.checked_add(4)?..at.checked_add(8)?)? { + b"IDAT" => break, + b"IEND" => return None, + b"tRNS" => trns = true, + _ => {} + } + at = at.checked_add(12)?.checked_add(len)?; + } + let (channels, alpha) = match color { + 0 => (1 + u64::from(trns), trns), + 2 | 3 => (3 + u64::from(trns), trns), + 4 => (2, true), + 6 => (4, true), + _ => return None, + }; + Some((width, height, channels * if depth == 16 { 2 } else { 1 }, alpha)) + } + + fn tuple(b: &[u8]) -> Option<(u32, u32, u64, bool)> { + pixel_model(b).ok().map(|m| (m.width, m.height, u64::from(m.bytes_per_pixel()), m.has_alpha())) + } + + fn png_with(depth: u8, color: u8, before_idat: &[(&[u8; 4], Vec)]) -> Vec { + let mut bytes = b"\x89PNG\r\n\x1a\n".to_vec(); + let mut ihdr = Vec::new(); + ihdr.extend_from_slice(&5u32.to_be_bytes()); + ihdr.extend_from_slice(&7u32.to_be_bytes()); + ihdr.extend_from_slice(&[depth, color, 0, 0, 0]); + chunk(&mut bytes, b"IHDR", &ihdr); + for (k, d) in before_idat { + chunk(&mut bytes, k, d); + } + chunk(&mut bytes, b"IDAT", &[1, 2, 3]); + chunk(&mut bytes, b"IEND", &[]); + bytes + } + + #[test] + fn pixel_model_matches_oracle_for_every_color_type_depth_and_trns() { + let valid: [(u8, &[u8]); 5] = [(0, &[1, 2, 4, 8, 16]), (2, &[8, 16]), (3, &[1, 2, 4, 8]), (4, &[8, 16]), (6, &[8, 16])]; + for (color, depths) in valid { + for &depth in depths { + for trns in [false, true] { + let extra: Vec<(&[u8; 4], Vec)> = if trns { vec![(b"tRNS", vec![0, 0])] } else { vec![] }; + let bytes = png_with(depth, color, &extra); + assert!(tuple(&bytes).is_some(), "color {color} depth {depth}"); + assert_eq!(tuple(&bytes), oracle(&bytes), "color {color} depth {depth} trns {trns}"); + } + } + } + } + + #[test] + fn pixel_model_known_values() { + let m = pixel_model(&png_with(16, 2, &[(b"tRNS", vec![0; 6])])).unwrap(); + assert_eq!((m.width, m.height, m.channels(), m.bytes_per_pixel(), m.has_alpha()), (5, 7, 4, 8, true)); + let m = pixel_model(&png_with(4, 3, &[])).unwrap(); + assert_eq!((m.bytes_per_pixel(), m.has_alpha()), (3, false)); + } + + #[test] + fn pixel_model_skips_a_huge_iccp_without_inflating() { + let junk = vec![0xAB; 20 << 20]; + let bytes = png_with(8, 2, &[(b"iCCP", junk), (b"tRNS", vec![0; 6])]); + assert_eq!(tuple(&bytes), oracle(&bytes)); + assert_eq!(tuple(&bytes), Some((5, 7, 4, true))); + } + + #[test] + fn pixel_model_malformed_inputs_agree_with_oracle_and_never_panic() { + let full = png_with(8, 6, &[(b"tRNS", vec![0]), (b"gAMA", vec![0; 4])]); + for n in 0..=full.len() { + let cut = &full[..n]; + let got = tuple(cut); + // an Err where the oracle answers can only be a stricter header check + if got.is_some() { + assert_eq!(got, oracle(cut), "prefix {n}"); + } + } + assert!(pixel_model(&full[..33]).is_err()); + // missing IHDR + let mut bad = full.clone(); + bad[12..16].copy_from_slice(b"IHDX"); + assert!(pixel_model(&bad).is_err() && oracle(&bad).is_none()); + // chunk length overflow: a length past the input, then one past i32::MAX + for len in [u32::MAX, i32::MAX as u32, 0x7fff_fff0] { + let mut o = full.clone(); + o[33..37].copy_from_slice(&len.to_be_bytes()); + assert!(pixel_model(&o).is_err(), "len {len:#x}"); + assert_eq!(tuple(&o), oracle(&o)); + } + // IEND before IDAT, unknown colour type, zero size, bad signature + let mut iend = png_with(8, 2, &[]); + let at = iend.windows(4).position(|w| w == b"IDAT").unwrap(); + iend[at..at + 4].copy_from_slice(b"IEND"); + assert!(pixel_model(&iend).is_err() && oracle(&iend).is_none()); + assert!(pixel_model(&png_with(8, 5, &[])).is_err()); + let mut zero = png_with(8, 2, &[]); + zero[16..20].copy_from_slice(&0u32.to_be_bytes()); + assert!(pixel_model(&zero).is_err()); + assert!(pixel_model(b"not a png").is_err()); + // deterministic pseudo-random corruption + let mut s = 0x9e37_79b9u32; + for _ in 0..2000 { + let mut f = full.clone(); + for _ in 0..3 { + s = s.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + let i = (s >> 8) as usize % f.len(); + f[i] = (s >> 24) as u8; + } + let _ = pixel_model(&f); + } + } + + #[test] + fn pixel_model_many_tiny_chunks_and_the_chunk_cap() { + let many: Vec<(&[u8; 4], Vec)> = (0..10_000).map(|_| (b"tEXt", Vec::new())).chain([(b"tRNS", vec![0])]).collect(); + let bytes = png_with(8, 0, &many); + assert_eq!(tuple(&bytes), oracle(&bytes)); + assert_eq!(tuple(&bytes), Some((5, 7, 2, true))); + let over: Vec<(&[u8; 4], Vec)> = (0..MAX_PROBE_CHUNKS + 1).map(|_| (b"tEXt", Vec::new())).collect(); + assert!(matches!(pixel_model(&png_with(8, 0, &over)), Err(CodecError::LimitExceeded(_)))); + } } diff --git a/crates/image-codecs/src/webp_probe.rs b/crates/image-codecs/src/webp_probe.rs index 4a31def00..1c22411dc 100644 --- a/crates/image-codecs/src/webp_probe.rs +++ b/crates/image-codecs/src/webp_probe.rs @@ -32,6 +32,25 @@ pub fn header(bytes: &[u8]) -> Result<(u32, u32, u16), CodecError> { Ok((w, h, orientation)) } +/// Whether a WebP carries transparency, from the first chunk only (nothing is decoded): +/// `VP8X` flag bit 0x10, `VP8L` `alpha_is_used`, plain `VP8` never. +pub fn has_alpha(bytes: &[u8]) -> Result { + let err = |s: &str| CodecError::malformed(Format::WebP, s); + if bytes.get(..4) != Some(b"RIFF") || bytes.get(8..12) != Some(b"WEBP") { + return Err(err("bad signature")); + } + match bytes.get(12..16) { + Some(b"VP8X") => Ok(bytes.get(20).ok_or_else(|| err("truncated VP8X"))? & 0x10 != 0), + // 0x2f signature byte, then 14+14 size bits, then the alpha_is_used bit (bit 28) + Some(b"VP8L") => match bytes.get(20..25) { + Some([0x2f, a, b, c, d]) => Ok(u32::from_le_bytes([*a, *b, *c, *d]) >> 28 & 1 != 0), + _ => Err(err("bad VP8L header")), + }, + Some(b"VP8 ") => Ok(false), + _ => Err(err("no image data")), + } +} + #[cfg(test)] mod tests { use super::*; @@ -71,4 +90,39 @@ mod tests { bytes.push(0); assert!(header(&bytes).is_err()); } + + #[test] + fn has_alpha_reads_vp8x_vp8l_and_plain_vp8() { + for (layout, want) in [(crate::ChannelLayout::Rgb, false), (crate::ChannelLayout::Rgba, true)] { + let n = layout.channels(); + let image = crate::Image::from_u8(2, 3, layout, vec![42; 6 * n]).unwrap(); + let bytes = crate::encode(&image, Format::WebP, &crate::EncodeOptions::default()).unwrap(); + assert_eq!(has_alpha(&bytes).unwrap(), want); + } + let riff = |chunk: &[u8; 4], body: &[u8]| { + let mut v = b"RIFF\0\0\0\0WEBP".to_vec(); + v.extend_from_slice(chunk); + v.extend_from_slice(&(body.len() as u32).to_le_bytes()); + v.extend_from_slice(body); + v + }; + let mut x = [0u8; 10]; + assert!(!has_alpha(&riff(b"VP8X", &x)).unwrap()); + x[0] = 0x10; + assert!(has_alpha(&riff(b"VP8X", &x)).unwrap()); + assert!(!has_alpha(&riff(b"VP8 ", &[0; 10])).unwrap()); + assert!(has_alpha(&riff(b"VP8L", &[0x2f, 0, 0, 0, 0x10])).unwrap()); + assert!(!has_alpha(&riff(b"VP8L", &[0x2f, 0, 0, 0, 0])).unwrap()); + } + + #[test] + fn has_alpha_rejects_malformed_prefixes() { + let full = [b"RIFF\0\0\0\0WEBPVP8X\x0a\0\0\0".as_slice(), &[0x10; 10]].concat(); + for n in 0..20 { + assert!(has_alpha(&full[..n]).is_err(), "prefix {n}"); + } + assert!(has_alpha(&full[..21]).is_ok()); + assert!(has_alpha(b"RIFF\0\0\0\0WEBPVP8L\x05\0\0\0\x2e").is_err()); + assert!(has_alpha(b"junk").is_err()); + } } diff --git a/crates/image-codecs/tests/it/malformed.rs b/crates/image-codecs/tests/it/malformed.rs index ec06109e0..b247bdfc6 100644 --- a/crates/image-codecs/tests/it/malformed.rs +++ b/crates/image-codecs/tests/it/malformed.rs @@ -76,9 +76,11 @@ fn truncated_files_mostly_error() { #[test] fn bit_flips_never_panic() { + // CRAFT_HOSTILE_DIVISOR=N (pull requests) runs 1/N of the flips. + let rounds = 150usize.div_ceil(craft_media_testkit::hostile::divisor()); let mut rng = common::Rng::new(42); for (f, bytes) in samples() { - for _ in 0..150 { + for _ in 0..rounds { let mut b = bytes.clone(); let flips = 1 + (rng.next_u64() % 4) as usize; for _ in 0..flips { @@ -92,8 +94,11 @@ fn bit_flips_never_panic() { #[test] fn byte_overwrites_in_header_never_panic() { - for (f, bytes) in samples() { - for i in 0..bytes.len().min(64) { + // CRAFT_HOSTILE_DIVISOR=N (pull requests) overwrites every N-th byte, from a different offset + // for each sample, so the samples together still reach every header position. + let step = craft_media_testkit::hostile::divisor(); + for (k, (f, bytes)) in samples().into_iter().enumerate() { + for i in (0..bytes.len().min(64)).filter(|i| (i + k) % step == 0) { for v in [0x00, 0x7F, 0xFF] { let mut b = bytes.clone(); b[i] = v; diff --git a/crates/image-codecs/tests/it/tiff_pages.rs b/crates/image-codecs/tests/it/tiff_pages.rs index 7501be936..7f6708f88 100644 --- a/crates/image-codecs/tests/it/tiff_pages.rs +++ b/crates/image-codecs/tests/it/tiff_pages.rs @@ -820,15 +820,18 @@ fn poke(bytes: &[u8]) { #[test] fn zoo_decodes_and_never_panics_when_cut_or_corrupted() { - for f in zoo() { + // CRAFT_HOSTILE_DIVISOR=N (pull requests) runs every N-th cut and flip, from a different offset + // for each file, and 1/N of the random rounds. + let step = craft_media_testkit::hostile::divisor(); + for (k, f) in zoo().into_iter().enumerate() { let info = tiff_info(&f).unwrap(); for i in 0..info.pages.len() { decode_tiff_page(&f, i, &DecodeOptions::default()).unwrap(); } - for cut in 0..f.len() { + for cut in (0..f.len()).filter(|cut| (cut + k) % step == 0) { poke(&f[..cut]); } - for i in 0..f.len() { + for i in (0..f.len()).filter(|i| (i + k) % step == 0) { for flip in [0xffu8, 0x80, 0x01] { let mut g = f.clone(); g[i] ^= flip; @@ -836,7 +839,7 @@ fn zoo_decodes_and_never_panics_when_cut_or_corrupted() { } } let mut r = Rng::new(f.len() as u64); - for _ in 0..300 { + for _ in 0..300usize.div_ceil(step) { let mut g = f.clone(); for _ in 0..4 { let i = (r.next_u64() as usize) % g.len(); diff --git a/crates/java-class/Cargo.toml b/crates/java-class/Cargo.toml new file mode 100644 index 000000000..1a5144fc3 --- /dev/null +++ b/crates/java-class/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name="craft-java-class" +description="Java class-file inspection and JVM bytecode decoding with retained bounds and partial reports" +version.workspace=true +edition.workspace=true +license.workspace=true +rust-version.workspace=true +repository.workspace=true +[lints] +workspace=true +[dependencies] +serde={workspace=true} +thiserror={workspace=true} +[dev-dependencies] +craft-testkit = { workspace = true } diff --git a/crates/java-class/src/bytes.rs b/crates/java-class/src/bytes.rs new file mode 100644 index 000000000..bc26aea19 --- /dev/null +++ b/crates/java-class/src/bytes.rs @@ -0,0 +1,51 @@ +//! Checked little/big-endian reads over untrusted bytes. Every read returns `None` when out of range. + +#[derive(Debug, Clone, Copy)] +pub struct Reader<'a> { + data: &'a [u8], +} + +impl<'a> Reader<'a> { + pub fn new(data: &'a [u8]) -> Self { + Reader { data } + } + + pub fn slice(&self, off: usize, len: usize) -> Option<&'a [u8]> { + let end = off.checked_add(len)?; + self.data.get(off..end) + } + + pub fn u8(&self, off: usize) -> Option { + self.data.get(off).copied() + } + + fn arr(&self, off: usize) -> Option<[u8; N]> { + self.slice(off, N)?.try_into().ok() + } + + pub fn u16_be(&self, off: usize) -> Option { + self.arr(off).map(u16::from_be_bytes) + } + + pub fn u32_be(&self, off: usize) -> Option { + self.arr(off).map(u32::from_be_bytes) + } + + pub fn u64_be(&self, off: usize) -> Option { + self.arr(off).map(u64::from_be_bytes) + } +} + +/// Decode one modified UTF-8 code unit. Invalid bytes consume only themselves, +/// so a damaged sequence cannot hide valid text or borrow missing bytes. +pub(crate) fn mutf8_unit(bytes: &[u8]) -> (u16, usize) { + match bytes { + [a @ 0x00..=0x7f, ..] => (u16::from(*a), 1), + [0xc0, 0x80, ..] => (0, 2), + [a @ 0xc2..=0xdf, b @ 0x80..=0xbf, ..] => ((u16::from(*a & 0x1f) << 6) | u16::from(*b & 0x3f), 2), + [a @ 0xe0..=0xef, b @ 0x80..=0xbf, c @ 0x80..=0xbf, ..] if *a != 0xe0 || *b >= 0xa0 => { + ((u16::from(*a & 0x0f) << 12) | (u16::from(*b & 0x3f) << 6) | u16::from(*c & 0x3f), 3) + } + _ => (0xfffd, 1), + } +} diff --git a/crates/java-class/src/descriptor.rs b/crates/java-class/src/descriptor.rs new file mode 100644 index 000000000..3f4b32bd7 --- /dev/null +++ b/crates/java-class/src/descriptor.rs @@ -0,0 +1,19 @@ +// Descriptor display helper retained from the source DEX reader. +pub fn java_name(desc: &str) -> String { + let dims = desc.chars().take_while(|&c| c == '[').count(); + let base = desc.get(dims..).unwrap_or(""); + let name = match base { + "V" => "void".to_string(), + "Z" => "boolean".to_string(), + "B" => "byte".to_string(), + "S" => "short".to_string(), + "C" => "char".to_string(), + "I" => "int".to_string(), + "J" => "long".to_string(), + "F" => "float".to_string(), + "D" => "double".to_string(), + b if b.starts_with('L') && b.ends_with(';') => b.get(1..b.len() - 1).unwrap_or("").replace('/', "."), + b => b.to_string(), + }; + format!("{name}{}", "[]".repeat(dims.min(255))) +} diff --git a/crates/java-class/src/java_class.rs b/crates/java-class/src/java_class.rs new file mode 100644 index 000000000..4b751ec3e --- /dev/null +++ b/crates/java-class/src/java_class.rs @@ -0,0 +1,1082 @@ +//! Java `.class` files (JVMS): the magic and version, the constant pool (every tag including +//! `MethodHandle`/`MethodType`/`Dynamic`/`InvokeDynamic`/`Module`/`Package`), access flags, the +//! this/super class and interfaces, fields and methods with their attributes (`Code` with its +//! exception table and line numbers, `ConstantValue`, `Exceptions`, `Signature`), and the +//! class-level `SourceFile`, `InnerClasses` and annotation attributes (type names, best-effort). +//! +//! Counts and lengths are checked against the file size, strings are decoded from modified UTF-8, +//! and every read is bounds-checked, so hostile input yields warnings, not panics. + +use crate::bytes::Reader; +use crate::java_ops::{self, Insn}; +use crate::{FormatError, descriptor as dex}; +use serde::Serialize; + +/// Most constant-pool entries, fields, methods, attributes or interfaces we read from one file. +const MAX_ITEMS: usize = 1 << 20; +/// Most attributes per element. +const MAX_ATTRS: usize = 64 * 1024; +/// Deepest annotation/element-value nesting we walk. +const MAX_ANNO_DEPTH: usize = 32; + +/// One constant-pool entry. Index 0 and the slot after a `Long`/`Double` are `Unusable`. +#[derive(Debug, Clone, PartialEq)] +pub enum Const { + Unusable, + Utf8(String), + Integer(i32), + Float(f32), + Long(i64), + Double(f64), + /// `name_index` → a Utf8 internal name. + Class(u16), + /// `string_index` → a Utf8. + String(u16), + /// `class_index`, `name_and_type_index`. + FieldRef(u16, u16), + MethodRef(u16, u16), + InterfaceMethodRef(u16, u16), + /// `name_index`, `descriptor_index`. + NameAndType(u16, u16), + /// `reference_kind`, `reference_index`. + MethodHandle(u8, u16), + /// `descriptor_index`. + MethodType(u16), + /// `bootstrap_method_attr_index`, `name_and_type_index`. + Dynamic(u16, u16), + InvokeDynamic(u16, u16), + Module(u16), + Package(u16), +} + +/// The class-file version. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +pub struct Version { + pub major: u16, + pub minor: u16, +} + +impl Version { + /// A friendly Java platform label (`Java 8`, `Java 17`, `Java 1.3`). + pub fn java_label(self) -> String { + match self.major { + 45..=48 => format!("Java 1.{}", self.major - 44), + m if m >= 49 => format!("Java {}", m - 44), + m => format!("pre-standard (major {m})"), + } + } +} + +/// An exception-table row of a `Code` attribute. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ExceptionEntry { + pub start_pc: u16, + pub end_pc: u16, + pub handler_pc: u16, + /// Dotted caught type; `None` is catch-all (a `finally` or `catch (Throwable)` with index 0). + pub catch_type: Option, +} + +/// A parsed `Code` attribute. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Code { + pub max_stack: u16, + pub max_locals: u16, + pub bytecode: Vec, + pub exceptions: Vec, + /// `(start_pc, line)` from the `LineNumberTable`. + pub line_numbers: Vec<(u16, u16)>, +} + +/// A field or method. +#[derive(Debug, Clone, Default, PartialEq)] +pub struct Member { + pub access: u16, + pub name: String, + pub descriptor: String, + /// The generic `Signature` attribute, if present. + pub signature: Option, + /// A field's `ConstantValue`, rendered to text. + pub constant_value: Option, + /// A method's declared `throws` types (dotted). + pub exceptions: Vec, + /// A method's code. + pub code: Option, + pub deprecated: bool, + /// The `Synthetic` attribute (separate from the access flag). + pub synthetic_attr: bool, +} + +/// An `InnerClasses` row. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct InnerClass { + pub inner: Option, + pub outer: Option, + /// Simple name; `None` for an anonymous class. + pub name: Option, + pub access: u16, +} + +/// A parsed class file. +#[derive(Debug, Clone)] +pub struct ClassFile { + pub version: Version, + pub access: u16, + /// Internal name of this class (`com/example/Main`). + pub this_class: String, + /// Internal name of the superclass; `None` only for `java/lang/Object` and modules. + pub super_class: Option, + /// Internal names of the implemented interfaces. + pub interfaces: Vec, + pub fields: Vec, + pub methods: Vec, + pub source_file: Option, + pub signature: Option, + pub inner_classes: Vec, + /// Class-level annotation types (dotted), with `(runtime)`/`(class)` visibility. + pub annotations: Vec, + /// Bootstrap method handle and static arguments, indexed by dynamic pool constants. + pub bootstrap_methods: Vec<(u16, Vec)>, + pub pool: Vec, + pub warnings: Vec, +} + +// --------------------------------------------------------------------------------------------- +// Names, descriptors and access flags + +/// `java/lang/String` → `java.lang.String`; `[Ljava/lang/String;` → `java.lang.String[]`. +pub fn internal_to_dotted(name: &str) -> String { + if name.starts_with('[') { dex::java_name(name) } else { name.replace('/', ".") } +} + +/// A field descriptor (`Ljava/lang/String;`, `[I`, `I`) as a Java type. +pub fn field_type(desc: &str) -> String { + dex::java_name(desc) +} + +/// Read parameter descriptors up to the closing parenthesis and return the remaining return descriptor. +fn split_types(s: &str) -> Option<(Vec, &str)> { + let b = s.as_bytes(); + let mut out = Vec::new(); + let mut i = 0usize; + while i < b.len() && out.len() < MAX_ITEMS { + if b.get(i) == Some(&b')') { + return Some((out, s.get(i + 1..)?)); + } + let start = i; + while b.get(i) == Some(&b'[') { + i += 1; + } + match b.get(i) { + Some(b'L') => match s.get(i..).and_then(|r| r.find(';')) { + Some(semi) => i = i + semi + 1, + None => i = b.len(), + }, + Some(_) => i += 1, + None => break, + } + match s.get(start..i) { + Some(t) if !t.is_empty() => out.push(t.to_string()), + _ => break, + } + } + None +} + +/// Parameter types and return type of a method descriptor (`(Ljava/lang/String;I)V`). +pub fn method_params(desc: &str) -> (Vec, String) { + let Some(rest) = desc.strip_prefix('(') else { return (Vec::new(), "?".into()) }; + let Some((params, ret)) = split_types(rest) else { return (Vec::new(), "?".into()) }; + (params.iter().map(|t| field_type(t)).collect(), field_type(ret)) +} + +/// What a set of access flags belongs to. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AccessKind { + Class, + Field, + Method, + InnerClass, +} + +/// Human names of the set access flags. +pub fn access_flags(flags: u16, kind: AccessKind) -> Vec<&'static str> { + let mut v = Vec::new(); + let common: &[(u16, &str)] = &[(0x0001, "public"), (0x0002, "private"), (0x0004, "protected"), (0x0008, "static"), (0x0010, "final")]; + for (bit, name) in common { + if flags & bit != 0 { + v.push(*name); + } + } + match kind { + AccessKind::Method => { + for (bit, name) in [ + (0x0020u16, "synchronized"), + (0x0040, "bridge"), + (0x0080, "varargs"), + (0x0100, "native"), + (0x0400, "abstract"), + (0x0800, "strictfp"), + (0x1000, "synthetic"), + ] { + if flags & bit != 0 { + v.push(name); + } + } + } + AccessKind::Field => { + for (bit, name) in [(0x0040u16, "volatile"), (0x0080, "transient"), (0x1000, "synthetic"), (0x4000, "enum")] { + if flags & bit != 0 { + v.push(name); + } + } + } + AccessKind::Class | AccessKind::InnerClass => { + for (bit, name) in [ + (0x0020u16, "super"), + (0x0200, "interface"), + (0x0400, "abstract"), + (0x1000, "synthetic"), + (0x2000, "annotation"), + (0x4000, "enum"), + (0x8000, "module"), + ] { + if flags & bit != 0 { + // "super" is an internal marker on classes; hide it unless it is the only flag of note. + if name == "super" && kind == AccessKind::Class { + continue; + } + v.push(name); + } + } + } + } + v +} + +/// A resolved reference to a field or method. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct Ref { + /// Internal owner name (`java/lang/System`). + pub owner: String, + pub name: String, + pub descriptor: String, +} + +impl Ref { + /// `java.lang.System.out:java.io.PrintStream` style display. A field descriptor is shown as a + /// Java type; a method descriptor is left in its raw `(args)ret` form. + pub fn display(&self) -> String { + let ty = if self.descriptor.starts_with('(') { self.descriptor.clone() } else { field_type(&self.descriptor) }; + format!("{}.{}:{}", internal_to_dotted(&self.owner), self.name, ty) + } +} + +// --------------------------------------------------------------------------------------------- +// Modified UTF-8 + +/// Decode a modified-UTF-8 byte run (class-file `Utf8` content) to a String. +fn decode_mutf8(bytes: &[u8]) -> String { + let mut units: Vec = Vec::new(); + let mut i = 0usize; + while i < bytes.len() { + let (unit, consumed) = crate::bytes::mutf8_unit(bytes.get(i..).unwrap_or_default()); + units.push(unit); + i += consumed; + } + String::from_utf16_lossy(&units) +} + +// --------------------------------------------------------------------------------------------- +// Cursor + +struct Cursor<'a> { + r: Reader<'a>, + pos: usize, +} + +impl<'a> Cursor<'a> { + fn new(data: &'a [u8]) -> Self { + Cursor { r: Reader::new(data), pos: 0 } + } + fn u8(&mut self) -> Option { + let v = self.r.u8(self.pos)?; + self.pos += 1; + Some(v) + } + fn u16(&mut self) -> Option { + let v = self.r.u16_be(self.pos)?; + self.pos += 2; + Some(v) + } + fn u32(&mut self) -> Option { + let v = self.r.u32_be(self.pos)?; + self.pos += 4; + Some(v) + } + fn u64(&mut self) -> Option { + let v = self.r.u64_be(self.pos)?; + self.pos += 8; + Some(v) + } + fn take(&mut self, n: usize) -> Option<&'a [u8]> { + let s = self.r.slice(self.pos, n)?; + self.pos += n; + Some(s) + } +} + +// --------------------------------------------------------------------------------------------- +// Parsing + +/// True when the bytes begin a Java class file (0xCAFEBABE and a plausible major version). +/// A big-endian Mach-O fat binary shares the magic but has a small arch count where the version +/// is, so it fails the major-version check. +pub fn is_class(data: &[u8]) -> bool { + if !data.starts_with(&[0xca, 0xfe, 0xba, 0xbe]) { + return false; + } + let r = Reader::new(data); + matches!(r.u16_be(6), Some(major) if (45..=255).contains(&major)) +} + +fn utf8_of(pool: &[Const], idx: u16) -> &str { + match pool.get(idx as usize) { + Some(Const::Utf8(s)) => s.as_str(), + _ => "", + } +} + +/// Parse a class file. +pub fn parse(data: &[u8]) -> Result { + if !data.starts_with(&[0xca, 0xfe, 0xba, 0xbe]) { + return Err(FormatError::Malformed("not a Java class file (no 0xCAFEBABE magic)".into())); + } + let mut c = Cursor::new(data); + let mut warnings = Vec::new(); + let _magic = c.u32(); + let minor = c.u16().ok_or_else(|| FormatError::Truncated("version".into()))?; + let major = c.u16().ok_or_else(|| FormatError::Truncated("version".into()))?; + if major < 45 { + return Err(FormatError::Unsupported(format!("class major version {major} is below 45 (not a standard class file)"))); + } + let version = Version { major, minor }; + let cp_count = c.u16().ok_or_else(|| FormatError::Truncated("constant pool count".into()))?; + if cp_count == 0 { + return Err(FormatError::Malformed("constant pool count is zero".into())); + } + let pool = parse_pool(&mut c, cp_count, &mut warnings)?; + let access = c.u16().ok_or_else(|| FormatError::Truncated("access flags".into()))?; + let this_idx = c.u16().ok_or_else(|| FormatError::Truncated("this_class".into()))?; + let super_idx = c.u16().ok_or_else(|| FormatError::Truncated("super_class".into()))?; + let this_class = class_name_at(&pool, this_idx); + let super_class = if super_idx == 0 { None } else { Some(class_name_at(&pool, super_idx)) }; + // Interfaces + let n_iface = c.u16().ok_or_else(|| FormatError::Truncated("interfaces count".into()))? as usize; + let mut interfaces = Vec::new(); + for _ in 0..n_iface.min(MAX_ITEMS) { + let Some(i) = c.u16() else { + warnings.push("interface list ended early".into()); + break; + }; + interfaces.push(class_name_at(&pool, i)); + } + // Fields + let fields = parse_members(&mut c, &pool, &mut warnings, "field")?; + // Methods + let methods = parse_members(&mut c, &pool, &mut warnings, "method")?; + // Class attributes + let (attrs, complete) = read_attrs(&mut c, &pool); + if !complete { + warnings.push("class attribute table ended early".into()); + } + let mut source_file = None; + let mut signature = None; + let mut inner_classes = Vec::new(); + let mut annotations = Vec::new(); + let mut bootstrap_methods = Vec::new(); + for a in &attrs { + match a.name { + "SourceFile" => { + if let Some(i) = Reader::new(a.info).u16_be(0) { + source_file = Some(utf8_of(&pool, i).to_string()); + } + } + "Signature" => { + if let Some(i) = Reader::new(a.info).u16_be(0) { + signature = Some(utf8_of(&pool, i).to_string()); + } + } + "BootstrapMethods" => match bootstrap_table(a.info) { + Some(table) => bootstrap_methods = table, + None => warnings.push("invalid BootstrapMethods table".into()), + }, + "InnerClasses" => inner_classes = parse_inner_classes(a.info, &pool), + "RuntimeVisibleAnnotations" => annotations.extend(parse_annotation_types(a.info, &pool, "runtime")), + "RuntimeInvisibleAnnotations" => annotations.extend(parse_annotation_types(a.info, &pool, "class")), + _ => {} + } + } + Ok(ClassFile { + version, + access, + this_class, + super_class, + interfaces, + fields, + methods, + source_file, + signature, + inner_classes, + annotations, + bootstrap_methods, + pool, + warnings, + }) +} + +fn parse_pool(c: &mut Cursor<'_>, cp_count: u16, warnings: &mut Vec) -> Result, FormatError> { + let mut pool = vec![Const::Unusable]; // index 0 + let mut i = 1u16; + while i < cp_count { + let tag = c.u8().ok_or_else(|| FormatError::Truncated("constant pool entry".into()))?; + let entry = match tag { + 1 => { + let len = c.u16().ok_or_else(|| FormatError::Truncated("Utf8 length".into()))? as usize; + let bytes = c.take(len).ok_or_else(|| FormatError::Truncated("Utf8 bytes".into()))?; + Const::Utf8(decode_mutf8(bytes)) + } + 3 => Const::Integer(c.u32().ok_or_else(|| trunc("Integer"))? as i32), + 4 => Const::Float(f32::from_bits(c.u32().ok_or_else(|| trunc("Float"))?)), + 5 => Const::Long(c.u64().ok_or_else(|| trunc("Long"))? as i64), + 6 => Const::Double(f64::from_bits(c.u64().ok_or_else(|| trunc("Double"))?)), + 7 => Const::Class(c.u16().ok_or_else(|| trunc("Class"))?), + 8 => Const::String(c.u16().ok_or_else(|| trunc("String"))?), + 9 => Const::FieldRef(c.u16().ok_or_else(|| trunc("Fieldref"))?, c.u16().ok_or_else(|| trunc("Fieldref"))?), + 10 => Const::MethodRef(c.u16().ok_or_else(|| trunc("Methodref"))?, c.u16().ok_or_else(|| trunc("Methodref"))?), + 11 => Const::InterfaceMethodRef(c.u16().ok_or_else(|| trunc("InterfaceMethodref"))?, c.u16().ok_or_else(|| trunc("InterfaceMethodref"))?), + 12 => Const::NameAndType(c.u16().ok_or_else(|| trunc("NameAndType"))?, c.u16().ok_or_else(|| trunc("NameAndType"))?), + 15 => Const::MethodHandle(c.u8().ok_or_else(|| trunc("MethodHandle"))?, c.u16().ok_or_else(|| trunc("MethodHandle"))?), + 16 => Const::MethodType(c.u16().ok_or_else(|| trunc("MethodType"))?), + 17 => Const::Dynamic(c.u16().ok_or_else(|| trunc("Dynamic"))?, c.u16().ok_or_else(|| trunc("Dynamic"))?), + 18 => Const::InvokeDynamic(c.u16().ok_or_else(|| trunc("InvokeDynamic"))?, c.u16().ok_or_else(|| trunc("InvokeDynamic"))?), + 19 => Const::Module(c.u16().ok_or_else(|| trunc("Module"))?), + 20 => Const::Package(c.u16().ok_or_else(|| trunc("Package"))?), + other => { + return Err(FormatError::Malformed(format!("unknown constant pool tag {other} at index {i}"))); + } + }; + let wide = matches!(entry, Const::Long(_) | Const::Double(_)); + let next = i + .checked_add(if wide { 2 } else { 1 }) + .filter(|next| *next <= cp_count) + .ok_or_else(|| FormatError::Malformed(format!("constant pool entry {i} has no reserved second slot")))?; + pool.push(entry); + if wide { + pool.push(Const::Unusable); + } + i = next; + } + if pool.len() > MAX_ITEMS { + warnings.push("constant pool is unusually large".into()); + } + Ok(pool) +} + +fn trunc(what: &str) -> FormatError { + FormatError::Truncated(format!("constant pool {what}")) +} + +struct RawAttr<'a, 'p> { + // Many attributes can reference one long constant-pool name. The pool + // already owns it for the entire parse; a per-attribute copy amplifies it. + name: &'p str, + info: &'a [u8], +} + +/// Return the complete attributes in the readable prefix and whether the whole table was read. +fn read_attrs<'a, 'p>(c: &mut Cursor<'a>, pool: &'p [Const]) -> (Vec>, bool) { + let mut out = Vec::new(); + let Some(n) = c.u16().map(usize::from) else { return (out, false) }; + for _ in 0..n.min(MAX_ATTRS) { + let (Some(ni), Some(len)) = (c.u16(), c.u32()) else { return (out, false) }; + let Some(info) = c.take(len as usize) else { return (out, false) }; + out.push(RawAttr { name: utf8_of(pool, ni), info }); + } + (out, n <= MAX_ATTRS) +} + +fn bootstrap_table(data: &[u8]) -> Option)>> { + let mut cursor = Cursor::new(data); + let mut table = Vec::new(); + for _ in 0..cursor.u16()? { + let handle = cursor.u16()?; + let count = usize::from(cursor.u16()?); + let args = cursor.take(count * 2)?.as_chunks::<2>().0.iter().map(|b| u16::from_be_bytes(*b)).collect(); + table.push((handle, args)); + } + (cursor.pos == data.len()).then_some(table) +} + +fn parse_members(c: &mut Cursor<'_>, pool: &[Const], warnings: &mut Vec, what: &str) -> Result, FormatError> { + let n = c.u16().ok_or_else(|| FormatError::Truncated(format!("{what} count")))? as usize; + let mut out = Vec::new(); + for _ in 0..n.min(MAX_ITEMS) { + let (Some(access), Some(name_idx), Some(desc_idx)) = (c.u16(), c.u16(), c.u16()) else { + warnings.push(format!("{what} table ended early")); + break; + }; + let mut m = + Member { access, name: utf8_of(pool, name_idx).to_string(), descriptor: utf8_of(pool, desc_idx).to_string(), ..Default::default() }; + let (attrs, _) = read_attrs(c, pool); + for a in attrs { + match a.name { + "Code" => m.code = parse_code(a.info, pool), + "ConstantValue" => { + if let Some(i) = Reader::new(a.info).u16_be(0) { + m.constant_value = Some(const_display(pool, i)); + } + } + "Exceptions" => m.exceptions = parse_exceptions_attr(a.info, pool), + "Signature" => { + if let Some(i) = Reader::new(a.info).u16_be(0) { + m.signature = Some(utf8_of(pool, i).to_string()); + } + } + "Deprecated" => m.deprecated = true, + "Synthetic" => m.synthetic_attr = true, + _ => {} + } + } + out.push(m); + } + Ok(out) +} + +fn parse_code(info: &[u8], pool: &[Const]) -> Option { + let mut c = Cursor::new(info); + let max_stack = c.u16()?; + let max_locals = c.u16()?; + let code_len = c.u32()? as usize; + let bytecode = c.take(code_len)?.to_vec(); + let n_exc = c.u16()? as usize; + let mut exceptions = Vec::new(); + for _ in 0..n_exc.min(MAX_ITEMS) { + let (Some(start_pc), Some(end_pc), Some(handler_pc), Some(catch)) = (c.u16(), c.u16(), c.u16(), c.u16()) else { break }; + exceptions.push(ExceptionEntry { start_pc, end_pc, handler_pc, catch_type: (catch != 0).then(|| class_name_at(pool, catch)) }); + } + let mut line_numbers = Vec::new(); + let (attrs, _) = read_attrs(&mut c, pool); + for a in attrs { + if a.name == "LineNumberTable" { + let mut lc = Cursor::new(a.info); + let n = lc.u16().unwrap_or(0) as usize; + for _ in 0..n.min(MAX_ITEMS) { + let (Some(pc), Some(line)) = (lc.u16(), lc.u16()) else { break }; + line_numbers.push((pc, line)); + } + } + } + Some(Code { max_stack, max_locals, bytecode, exceptions, line_numbers }) +} + +fn parse_exceptions_attr(info: &[u8], pool: &[Const]) -> Vec { + let mut c = Cursor::new(info); + let n = c.u16().unwrap_or(0) as usize; + let mut out = Vec::new(); + for _ in 0..n.min(MAX_ITEMS) { + let Some(i) = c.u16() else { break }; + out.push(class_name_at(pool, i)); + } + out +} + +fn parse_inner_classes(info: &[u8], pool: &[Const]) -> Vec { + let mut c = Cursor::new(info); + let n = c.u16().unwrap_or(0) as usize; + let mut out = Vec::new(); + for _ in 0..n.min(MAX_ITEMS) { + let (Some(inner), Some(outer), Some(name), Some(access)) = (c.u16(), c.u16(), c.u16(), c.u16()) else { break }; + out.push(InnerClass { + inner: (inner != 0).then(|| class_name_at(pool, inner)), + outer: (outer != 0).then(|| class_name_at(pool, outer)), + name: (name != 0).then(|| utf8_of(pool, name).to_string()), + access, + }); + } + out +} + +/// Extract annotation type names (dotted) from a `Runtime*Annotations` attribute, best-effort. +fn parse_annotation_types(info: &[u8], pool: &[Const], visibility: &str) -> Vec { + let mut c = Cursor::new(info); + let n = c.u16().unwrap_or(0) as usize; + let mut out = Vec::new(); + for _ in 0..n.min(MAX_ITEMS) { + match read_annotation(&mut c, pool, 0) { + Some(t) => out.push(format!("{t} ({visibility})")), + None => break, + } + } + out +} + +/// Read one `annotation` structure, returning its (dotted) type name; advances past its values. +fn read_annotation(c: &mut Cursor<'_>, pool: &[Const], depth: usize) -> Option { + if depth > MAX_ANNO_DEPTH { + return None; + } + let type_idx = c.u16()?; + let n_pairs = c.u16()? as usize; + let type_name = field_type(utf8_of(pool, type_idx)); + for _ in 0..n_pairs.min(MAX_ITEMS) { + let _name_idx = c.u16()?; + skip_element_value(c, pool, depth + 1)?; + } + Some(type_name) +} + +fn skip_element_value(c: &mut Cursor<'_>, pool: &[Const], depth: usize) -> Option<()> { + if depth > MAX_ANNO_DEPTH { + return None; + } + let tag = c.u8()?; + match tag { + b'B' | b'C' | b'D' | b'F' | b'I' | b'J' | b'S' | b'Z' | b's' | b'c' => { + c.u16()?; + } + b'e' => { + c.u16()?; + c.u16()?; + } + b'@' => { + read_annotation(c, pool, depth + 1)?; + } + b'[' => { + let n = c.u16()? as usize; + for _ in 0..n.min(MAX_ITEMS) { + skip_element_value(c, pool, depth + 1)?; + } + } + _ => return None, + } + Some(()) +} + +/// Internal class name at a `Class` constant index (`java/lang/String` or `[I`). +fn class_name_at(pool: &[Const], idx: u16) -> String { + match pool.get(idx as usize) { + Some(Const::Class(name_idx)) => utf8_of(pool, *name_idx).to_string(), + _ => format!(""), + } +} + +/// Render a loadable constant (for `ldc`/`ConstantValue`) to text. +fn const_display(pool: &[Const], idx: u16) -> String { + match pool.get(idx as usize) { + Some(Const::Integer(v)) => v.to_string(), + Some(Const::Float(v)) => format!("{v}f"), + Some(Const::Long(v)) => format!("{v}L"), + Some(Const::Double(v)) => v.to_string(), + Some(Const::String(s)) => format!("{:?}", utf8_of(pool, *s)), + Some(Const::Class(n)) => format!("{}.class", internal_to_dotted(utf8_of(pool, *n))), + Some(Const::MethodHandle(k, _)) => format!("MethodHandle(kind {k})"), + Some(Const::MethodType(d)) => format!("MethodType({})", utf8_of(pool, *d)), + _ => format!("#{idx}"), + } +} + +// --------------------------------------------------------------------------------------------- +// Resolution and formatting on a parsed class + +impl ClassFile { + /// The Utf8 at `idx`, or an empty string. + pub fn utf8(&self, idx: u16) -> &str { + utf8_of(&self.pool, idx) + } + + /// The dotted name of this class. + pub fn name_dotted(&self) -> String { + internal_to_dotted(&self.this_class) + } + + /// Resolve a `Fieldref`/`Methodref`/`InterfaceMethodref` index to owner, name and descriptor. + pub fn resolve_ref(&self, idx: u16) -> Option { + let (class_idx, nat_idx) = match self.pool.get(idx as usize)? { + Const::FieldRef(c, n) | Const::MethodRef(c, n) | Const::InterfaceMethodRef(c, n) => (*c, *n), + _ => return None, + }; + let owner = class_name_at(&self.pool, class_idx); + let (name, descriptor) = self.name_and_type(nat_idx); + Some(Ref { owner, name, descriptor }) + } + + /// Resolve a `NameAndType` index to (name, descriptor). + pub fn name_and_type(&self, idx: u16) -> (String, String) { + match self.pool.get(idx as usize) { + Some(Const::NameAndType(n, d)) => (self.utf8(*n).to_string(), self.utf8(*d).to_string()), + _ => (format!(""), String::new()), + } + } + + /// The (name, descriptor) of an `InvokeDynamic` (or `Dynamic`) constant's `NameAndType`. + pub fn name_and_type_of_indy(&self, idx: u16) -> Option<(String, String)> { + match self.pool.get(idx as usize)? { + Const::InvokeDynamic(_, nat) | Const::Dynamic(_, nat) => Some(self.name_and_type(*nat)), + _ => None, + } + } + + /// Dotted class name at a `Class` constant index. + pub fn class_dotted(&self, idx: u16) -> String { + internal_to_dotted(&class_name_at(&self.pool, idx)) + } + + /// A Java-style method signature (`public static void log(java.lang.String)`). + pub fn method_signature(&self, m: &Member) -> String { + let (params, ret) = method_params(&m.descriptor); + let flags = access_flags(m.access, AccessKind::Method).join(" "); + let params = params.join(", "); + if m.name == "" || m.name == "" { + format!("{}{}{}({params})", flags, if flags.is_empty() { "" } else { " " }, m.name).trim().to_string() + } else { + format!("{}{}{} {}({params})", flags, if flags.is_empty() { "" } else { " " }, ret, m.name).trim().to_string() + } + } + + /// Render one decoded instruction with its constant-pool reference resolved. + pub fn format_insn(&self, i: &Insn) -> String { + let mut operand = String::new(); + if let Some(cp) = i.cp_index { + operand = match i.opcode { + 0x12..=0x14 => const_display(&self.pool, cp), + 0xb2..=0xb5 => self.resolve_ref(cp).map(|r| r.display()).unwrap_or_else(|| format!("#{cp}")), + 0xb6..=0xb9 => self + .resolve_ref(cp) + .map(|r| format!("{}.{}{}", internal_to_dotted(&r.owner), r.name, r.descriptor)) + .unwrap_or_else(|| format!("#{cp}")), + 0xba => { + // invokedynamic: show the bootstrap name:descriptor + match self.pool.get(cp as usize) { + Some(Const::InvokeDynamic(_, nat)) => { + let (n, d) = self.name_and_type(*nat); + format!("{n}{d}") + } + _ => format!("#{cp}"), + } + } + 0xbb | 0xbd | 0xc0 | 0xc1 | 0xc5 => self.class_dotted(cp), + _ => format!("#{cp}"), + }; + } else if i.opcode == 0xbc { + // newarray + operand = java_ops::array_type(i.int.unwrap_or(0)).to_string(); + } else if i.opcode == 0x84 { + // iinc local, const + operand = format!("{}, {}", i.local.unwrap_or(0), i.int.unwrap_or(0)); + } else if let Some(l) = i.local { + operand = l.to_string(); + } else if let Some(v) = i.int { + operand = v.to_string(); + } else if let Some(b) = i.branch { + operand = format!("{b}"); + } + if i.opcode == 0xc5 { + // multianewarray class, dims + operand = format!("{}, {}", operand, i.int.unwrap_or(0)); + } + let mut s = if operand.is_empty() { i.name.to_string() } else { format!("{} {}", i.name, operand) }; + if let Some(sw) = &i.switch { + let cases: Vec = sw.pairs.iter().take(32).map(|(k, t)| format!("{k} -> {t}")).collect(); + s.push_str(&format!(" # default -> {}; {}", sw.default, cases.join(", "))); + } + s + } +} + +// --------------------------------------------------------------------------------------------- +// References + +/// One place a method's bytecode refers to a class, method or field. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct Reference { + /// The calling method, `owner.name:descriptor`. + pub caller: String, + /// Byte offset in the caller's code. + pub pc: u32, + pub instruction: &'static str, + /// Direct member/type use or a method handle, method type or bootstrap linkage. + pub kind: &'static str, + pub target: String, +} + +fn class_component(name: &str) -> Option<&str> { + if name.starts_with('[') { name.trim_start_matches('[').strip_prefix('L')?.strip_suffix(';') } else { Some(name) } +} + +fn descriptor_mentions(mut descriptor: &str, owner: &str) -> bool { + while let Some((_, rest)) = descriptor.split_once('L') { + let Some((class, tail)) = rest.split_once(';') else { return false }; + if class == owner { + return true; + } + descriptor = tail; + } + false +} + +/// Resolve a loadable constant to its named dependency, retaining the linkage role. +fn constant_dependency(cf: &ClassFile, idx: u16, owner: &str, method: Option<&str>, bootstrap: bool) -> Option<(&'static str, String)> { + match cf.pool.get(usize::from(idx))? { + Const::Class(utf) if method.is_none() => { + let name = cf.utf8(*utf); + (class_component(name) == Some(owner)).then(|| ("type", internal_to_dotted(name))) + } + Const::MethodType(utf) if method.is_none() => { + let desc = cf.utf8(*utf); + descriptor_mentions(desc, owner).then(|| ("method-type", desc.to_owned())) + } + Const::MethodHandle(kind, target) => { + let member = cf.pool.get(usize::from(*target))?; + let valid = matches!( + (kind, member), + (1..=4, Const::FieldRef(..)) + | (5 | 8, Const::MethodRef(..)) + | (6 | 7, Const::MethodRef(..) | Const::InterfaceMethodRef(..)) + | (9, Const::InterfaceMethodRef(..)) + ); + let target = cf.resolve_ref(*target)?; + (valid && (!bootstrap || matches!(*kind, 6 | 8)) && target.owner == owner && method.is_none_or(|m| *kind >= 5 && target.name == m)) + .then(|| (if bootstrap { "bootstrap-method" } else { "method-handle" }, target.display())) + } + _ => None, + } +} + +/// Every instruction in `cf` that refers to the internal class `owner` (optionally only calls to +/// its methods named `method`). The bool is true when decoding or the reference limit stops a scan early. +pub fn references(cf: &ClassFile, owner: &str, method: Option<&str>, max: usize) -> (Vec, bool) { + let mut out = Vec::new(); + let mut incomplete = false; + // A single bootstrap record may be referenced by every instruction in a class. + let mut linkage_budget = 200_000usize; + for m in &cf.methods { + let Some(code) = &m.code else { continue }; + let caller = Ref { owner: cf.this_class.clone(), name: m.name.clone(), descriptor: m.descriptor.clone() }.display(); + let (insns, complete) = java_ops::decode(&code.bytecode, 200_000); + incomplete |= !complete; + for i in &insns { + let Some(cp) = i.cp_index else { continue }; + if matches!(i.opcode, 0x12..=0x14 | 0xba) { + let link = match (cf.pool.get(usize::from(cp)), i.opcode) { + (Some(Const::Dynamic(n, _)), 0x12..=0x14) | (Some(Const::InvokeDynamic(n, _)), 0xba) => Some(*n), + (Some(Const::Class(_) | Const::MethodType(_) | Const::MethodHandle(..)), 0x12 | 0x13) => None, + _ => continue, + }; + let record = link.and_then(|idx| cf.bootstrap_methods.get(usize::from(idx))); + let (handle, args): (u16, &[u16]) = record.map_or((cp, &[][..]), |(h, a)| (*h, a)); + let mut seen = std::collections::BTreeSet::new(); + for (index, &operand) in std::iter::once(&handle).chain(args).enumerate() { + let Some(left) = linkage_budget.checked_sub(1) else { return (out, true) }; + linkage_budget = left; + let is_bootstrap = index == 0 && record.is_some(); + if seen.insert((operand, is_bootstrap)) + && let Some((kind, target)) = constant_dependency(cf, operand, owner, method, is_bootstrap) + { + if out.len() >= max { + return (out, true); + } + out.push(Reference { caller: caller.clone(), pc: i.pc, instruction: i.name, kind, target }); + } + } + continue; + } + let hit = match i.opcode { + 0xb6..=0xb9 => { + cf.resolve_ref(cp).and_then(|r| (r.owner == owner && method.is_none_or(|n| r.name == n)).then(|| ("method", r.display()))) + } + 0xb2..=0xb5 if method.is_none() => cf.resolve_ref(cp).and_then(|r| (r.owner == owner).then(|| ("field", r.display()))), + 0xbb | 0xbd | 0xc0 | 0xc1 | 0xc5 if method.is_none() => { + let name = class_name_at(&cf.pool, cp); + (class_component(&name) == Some(owner)).then_some(("type", internal_to_dotted(&name))) + } + _ => None, + }; + if let Some((kind, target)) = hit { + if out.len() >= max { + return (out, true); + } + out.push(Reference { caller: caller.clone(), pc: i.pc, instruction: i.name, kind, target }); + } + } + } + (out, incomplete) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_support::java as fx; + + /// Two valid static methods: a one-dimensional allocation and a two-dimensional one. + fn array_reference_class(component: &str, array: &str) -> ClassFile { + let utf8 = |s: &str| { + let mut bytes = vec![1]; + bytes.extend_from_slice(&(s.len() as u16).to_be_bytes()); + bytes.extend_from_slice(s.as_bytes()); + bytes + }; + let pool = [ + utf8("sample/ArrayUses"), + vec![7, 0, 1], + utf8("java/lang/Object"), + vec![7, 0, 3], + utf8("Code"), + utf8("row"), + utf8(&format!("()[L{component};")), + utf8("grid"), + utf8(&format!("(){array}")), + utf8(component), + vec![7, 0, 10], + utf8(array), + vec![7, 0, 12], + ]; + let mut bytes = vec![0xca, 0xfe, 0xba, 0xbe, 0, 0, 0, 52, 0, 14]; + bytes.extend(pool.into_iter().flatten()); + // public/super, this, super, no interfaces or fields, two methods. + bytes.extend_from_slice(&[0, 0x21, 0, 2, 0, 4, 0, 0, 0, 0, 0, 2]); + for (name, descriptor, code) in [(6u16, 7u16, vec![4, 0xbd, 0, 11, 0xb0]), (8, 9, vec![4, 4, 0xc5, 0, 13, 2, 0xb0])] { + for word in [9u16, name, descriptor, 1, 5] { + bytes.extend_from_slice(&word.to_be_bytes()); + } + bytes.extend_from_slice(&(12u32 + code.len() as u32).to_be_bytes()); + bytes.extend_from_slice(&[0, 2, 0, 0]); // max_stack, max_locals + bytes.extend_from_slice(&(code.len() as u32).to_be_bytes()); + bytes.extend_from_slice(&code); + bytes.extend_from_slice(&[0, 0, 0, 0]); // exception table and nested attributes + } + bytes.extend_from_slice(&[0, 0]); // class attributes + let cf = parse(&bytes).unwrap(); + assert!(cf.warnings.is_empty(), "{:?}", cf.warnings); + cf + } + + #[test] + fn parentheses_in_reference_types_are_part_of_the_class_name() { + let (params, result) = method_params("(Lpkg/A)B;[Lpkg/C(D;J)Lpkg/R)S;"); + assert_eq!(params, ["pkg.A)B", "pkg.C(D[]", "long"]); + assert_eq!(result, "pkg.R)S"); + } + + #[test] + fn object_arrays_reference_the_component_class() { + let cf = array_reference_class("java/lang/String", "[[Ljava/lang/String;"); + let (refs, more) = references(&cf, "java/lang/String", None, 10); + assert!(!more); + assert_eq!(refs.len(), 2, "{refs:?}"); + assert_eq!((refs[0].instruction, refs[0].pc), ("anewarray", 1)); + assert_eq!((refs[1].instruction, refs[1].pc), ("multianewarray", 2)); + assert_eq!(refs[1].target, "java.lang.String[][]"); + let (limited, more) = references(&cf, "java/lang/String", None, 1); + assert_eq!(limited, refs[..1]); + assert!(more); + assert_eq!(references(&cf, "java/lang/String", Some("grid"), 10), (Vec::new(), false)); + assert_eq!(references(&cf, "java/lang/Integer", None, 10), (Vec::new(), false)); + } + + #[test] + fn array_components_keep_class_names_beginning_with_l() { + for component in ["Leaf", "LLeaf"] { + let cf = array_reference_class(component, &format!("[[L{component};")); + let (refs, more) = references(&cf, component, None, 10); + assert_eq!(refs.len(), 2, "{component}: {refs:?}"); + assert!(!more); + } + } + + #[test] + fn primitive_arrays_do_not_reference_a_class_named_after_the_descriptor() { + let cf = array_reference_class("I", "[[I"); + let (refs, more) = references(&cf, "I", None, 10); + assert!(!more); + assert_eq!(refs.len(), 1, "{refs:?}"); + assert_eq!(refs[0].instruction, "anewarray"); + } + + #[test] + fn parses_fixture() { + let data = fx::hello_class(); + let cf = parse(&data).unwrap(); + assert!(cf.warnings.is_empty(), "{:?}", cf.warnings); + assert_eq!(cf.this_class, "com/example/Hello"); + assert_eq!(cf.super_class.as_deref(), Some("java/lang/Object")); + assert_eq!(cf.interfaces, vec!["java/lang/Runnable"]); + assert_eq!(cf.source_file.as_deref(), Some("Hello.java")); + assert!(cf.version.major >= 45); + // field + let f = cf.fields.iter().find(|f| f.name == "count").unwrap(); + assert_eq!(field_type(&f.descriptor), "int"); + // run() loads a string and calls println + let run = cf.methods.iter().find(|m| m.name == "run").unwrap(); + assert_eq!(cf.method_signature(run), "public void run()"); + let code = run.code.as_ref().unwrap(); + let (ins, complete) = java_ops::decode(&code.bytecode, 1000); + assert!(complete, "{ins:?}"); + let text: Vec = ins.iter().map(|i| cf.format_insn(i)).collect(); + assert!(text.iter().any(|t| t == "getstatic java.lang.System.out:java.io.PrintStream"), "{text:?}"); + assert!(text.iter().any(|t| t.starts_with("ldc \"Hello")), "{text:?}"); + assert!(text.iter().any(|t| t == "invokevirtual java.io.PrintStream.println(Ljava/lang/String;)V"), "{text:?}"); + // references to System from this class + let (refs, more) = references(&cf, "java/lang/System", None, 100); + assert!(!more); + assert!(refs.iter().any(|r| r.kind == "field" && r.target.contains("out")), "{refs:?}"); + } + + #[test] + fn helpers() { + assert_eq!(internal_to_dotted("java/lang/String"), "java.lang.String"); + assert_eq!(internal_to_dotted("[Ljava/lang/String;"), "java.lang.String[]"); + assert_eq!(method_params("(Ljava/lang/String;I)V"), (vec!["java.lang.String".to_string(), "int".to_string()], "void".to_string())); + assert_eq!(method_params("()I"), (Vec::new(), "int".to_string())); + assert_eq!(Version { major: 52, minor: 0 }.java_label(), "Java 8"); + assert_eq!(Version { major: 65, minor: 0 }.java_label(), "Java 21"); + assert_eq!(decode_mutf8(&[0xc0, 0x80, 0x41]), "\0A"); + } + + #[test] + fn hostile() { + let data = fx::hello_class(); + let walk = |cf: &ClassFile| { + let _ = cf.name_dotted(); + for m in cf.methods.iter().chain(&cf.fields) { + let _ = cf.method_signature(m); + if let Some(code) = &m.code { + let (ins, _) = java_ops::decode(&code.bytecode, 10_000); + for i in &ins { + let _ = cf.format_insn(i); + } + } + } + let _ = references(cf, "java/lang/System", None, 100); + }; + for m in crate::test_support::mutations(&data) { + if let Ok(cf) = parse(&m) { + walk(&cf); + } + } + for len in 0..data.len() { + if let Ok(cf) = parse(&data[..len]) { + walk(&cf); + } + } + for seed in 0..64 { + let mut n = crate::test_support::noise(seed, 2048); + n[..4].copy_from_slice(&[0xca, 0xfe, 0xba, 0xbe]); + n[6] = 0; + n[7] = 52; + if let Ok(cf) = parse(&n) { + walk(&cf); + } + } + } +} diff --git a/crates/java-class/src/java_ops.rs b/crates/java-class/src/java_ops.rs new file mode 100644 index 000000000..46e5cdf30 --- /dev/null +++ b/crates/java-class/src/java_ops.rs @@ -0,0 +1,711 @@ +//! JVM bytecode: the opcode table (every defined opcode plus the `wide`, `tableswitch` and +//! `lookupswitch` variable-length forms) and a decoder for each instruction. Pure decoding; +//! constant-pool index operands are resolved by the caller (see [`crate::java_class`]). + +use serde::Serialize; + +/// The shape of an opcode's operands. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +pub enum Operands { + /// No operands. + None, + /// One signed byte immediate (`bipush`). + Byte, + /// One signed short immediate (`sipush`). + Short, + /// A one-byte local-variable index. + Local, + /// `iinc`: a one-byte local index and a signed byte increment. + LocalConst, + /// A two-byte signed branch offset. + Branch2, + /// A four-byte signed branch offset (`goto_w`, `jsr_w`). + Branch4, + /// A one-byte constant-pool index (`ldc`). + Cp1, + /// A two-byte constant-pool index. + Cp2, + /// `newarray`: a one-byte primitive array type. + NewArray, + /// `invokeinterface`: a two-byte index, a count byte and a zero byte. + InvokeInterface, + /// `invokedynamic`: a two-byte index and two zero bytes. + InvokeDynamic, + /// `multianewarray`: a two-byte class index and a dimension byte. + MultiANewArray, + /// `tableswitch` (variable length, 4-byte aligned). + TableSwitch, + /// `lookupswitch` (variable length, 4-byte aligned). + LookupSwitch, + /// `wide`: a prefix that widens the following load/store/ret/iinc. + Wide, + /// A reserved or unknown opcode. + Reserved, +} + +/// One opcode slot. +#[derive(Debug, Clone, Copy)] +pub struct Op { + pub name: &'static str, + pub operands: Operands, +} + +const fn op(name: &'static str, operands: Operands) -> Op { + Op { name, operands } +} + +/// The opcode table entry for `b`. +pub fn opcode(b: u8) -> Op { + use Operands as O; + match b { + 0x00 => op("nop", O::None), + 0x01 => op("aconst_null", O::None), + 0x02 => op("iconst_m1", O::None), + 0x03 => op("iconst_0", O::None), + 0x04 => op("iconst_1", O::None), + 0x05 => op("iconst_2", O::None), + 0x06 => op("iconst_3", O::None), + 0x07 => op("iconst_4", O::None), + 0x08 => op("iconst_5", O::None), + 0x09 => op("lconst_0", O::None), + 0x0a => op("lconst_1", O::None), + 0x0b => op("fconst_0", O::None), + 0x0c => op("fconst_1", O::None), + 0x0d => op("fconst_2", O::None), + 0x0e => op("dconst_0", O::None), + 0x0f => op("dconst_1", O::None), + 0x10 => op("bipush", O::Byte), + 0x11 => op("sipush", O::Short), + 0x12 => op("ldc", O::Cp1), + 0x13 => op("ldc_w", O::Cp2), + 0x14 => op("ldc2_w", O::Cp2), + 0x15 => op("iload", O::Local), + 0x16 => op("lload", O::Local), + 0x17 => op("fload", O::Local), + 0x18 => op("dload", O::Local), + 0x19 => op("aload", O::Local), + 0x1a => op("iload_0", O::None), + 0x1b => op("iload_1", O::None), + 0x1c => op("iload_2", O::None), + 0x1d => op("iload_3", O::None), + 0x1e => op("lload_0", O::None), + 0x1f => op("lload_1", O::None), + 0x20 => op("lload_2", O::None), + 0x21 => op("lload_3", O::None), + 0x22 => op("fload_0", O::None), + 0x23 => op("fload_1", O::None), + 0x24 => op("fload_2", O::None), + 0x25 => op("fload_3", O::None), + 0x26 => op("dload_0", O::None), + 0x27 => op("dload_1", O::None), + 0x28 => op("dload_2", O::None), + 0x29 => op("dload_3", O::None), + 0x2a => op("aload_0", O::None), + 0x2b => op("aload_1", O::None), + 0x2c => op("aload_2", O::None), + 0x2d => op("aload_3", O::None), + 0x2e => op("iaload", O::None), + 0x2f => op("laload", O::None), + 0x30 => op("faload", O::None), + 0x31 => op("daload", O::None), + 0x32 => op("aaload", O::None), + 0x33 => op("baload", O::None), + 0x34 => op("caload", O::None), + 0x35 => op("saload", O::None), + 0x36 => op("istore", O::Local), + 0x37 => op("lstore", O::Local), + 0x38 => op("fstore", O::Local), + 0x39 => op("dstore", O::Local), + 0x3a => op("astore", O::Local), + 0x3b => op("istore_0", O::None), + 0x3c => op("istore_1", O::None), + 0x3d => op("istore_2", O::None), + 0x3e => op("istore_3", O::None), + 0x3f => op("lstore_0", O::None), + 0x40 => op("lstore_1", O::None), + 0x41 => op("lstore_2", O::None), + 0x42 => op("lstore_3", O::None), + 0x43 => op("fstore_0", O::None), + 0x44 => op("fstore_1", O::None), + 0x45 => op("fstore_2", O::None), + 0x46 => op("fstore_3", O::None), + 0x47 => op("dstore_0", O::None), + 0x48 => op("dstore_1", O::None), + 0x49 => op("dstore_2", O::None), + 0x4a => op("dstore_3", O::None), + 0x4b => op("astore_0", O::None), + 0x4c => op("astore_1", O::None), + 0x4d => op("astore_2", O::None), + 0x4e => op("astore_3", O::None), + 0x4f => op("iastore", O::None), + 0x50 => op("lastore", O::None), + 0x51 => op("fastore", O::None), + 0x52 => op("dastore", O::None), + 0x53 => op("aastore", O::None), + 0x54 => op("bastore", O::None), + 0x55 => op("castore", O::None), + 0x56 => op("sastore", O::None), + 0x57 => op("pop", O::None), + 0x58 => op("pop2", O::None), + 0x59 => op("dup", O::None), + 0x5a => op("dup_x1", O::None), + 0x5b => op("dup_x2", O::None), + 0x5c => op("dup2", O::None), + 0x5d => op("dup2_x1", O::None), + 0x5e => op("dup2_x2", O::None), + 0x5f => op("swap", O::None), + 0x60 => op("iadd", O::None), + 0x61 => op("ladd", O::None), + 0x62 => op("fadd", O::None), + 0x63 => op("dadd", O::None), + 0x64 => op("isub", O::None), + 0x65 => op("lsub", O::None), + 0x66 => op("fsub", O::None), + 0x67 => op("dsub", O::None), + 0x68 => op("imul", O::None), + 0x69 => op("lmul", O::None), + 0x6a => op("fmul", O::None), + 0x6b => op("dmul", O::None), + 0x6c => op("idiv", O::None), + 0x6d => op("ldiv", O::None), + 0x6e => op("fdiv", O::None), + 0x6f => op("ddiv", O::None), + 0x70 => op("irem", O::None), + 0x71 => op("lrem", O::None), + 0x72 => op("frem", O::None), + 0x73 => op("drem", O::None), + 0x74 => op("ineg", O::None), + 0x75 => op("lneg", O::None), + 0x76 => op("fneg", O::None), + 0x77 => op("dneg", O::None), + 0x78 => op("ishl", O::None), + 0x79 => op("lshl", O::None), + 0x7a => op("ishr", O::None), + 0x7b => op("lshr", O::None), + 0x7c => op("iushr", O::None), + 0x7d => op("lushr", O::None), + 0x7e => op("iand", O::None), + 0x7f => op("land", O::None), + 0x80 => op("ior", O::None), + 0x81 => op("lor", O::None), + 0x82 => op("ixor", O::None), + 0x83 => op("lxor", O::None), + 0x84 => op("iinc", O::LocalConst), + 0x85 => op("i2l", O::None), + 0x86 => op("i2f", O::None), + 0x87 => op("i2d", O::None), + 0x88 => op("l2i", O::None), + 0x89 => op("l2f", O::None), + 0x8a => op("l2d", O::None), + 0x8b => op("f2i", O::None), + 0x8c => op("f2l", O::None), + 0x8d => op("f2d", O::None), + 0x8e => op("d2i", O::None), + 0x8f => op("d2l", O::None), + 0x90 => op("d2f", O::None), + 0x91 => op("i2b", O::None), + 0x92 => op("i2c", O::None), + 0x93 => op("i2s", O::None), + 0x94 => op("lcmp", O::None), + 0x95 => op("fcmpl", O::None), + 0x96 => op("fcmpg", O::None), + 0x97 => op("dcmpl", O::None), + 0x98 => op("dcmpg", O::None), + 0x99 => op("ifeq", O::Branch2), + 0x9a => op("ifne", O::Branch2), + 0x9b => op("iflt", O::Branch2), + 0x9c => op("ifge", O::Branch2), + 0x9d => op("ifgt", O::Branch2), + 0x9e => op("ifle", O::Branch2), + 0x9f => op("if_icmpeq", O::Branch2), + 0xa0 => op("if_icmpne", O::Branch2), + 0xa1 => op("if_icmplt", O::Branch2), + 0xa2 => op("if_icmpge", O::Branch2), + 0xa3 => op("if_icmpgt", O::Branch2), + 0xa4 => op("if_icmple", O::Branch2), + 0xa5 => op("if_acmpeq", O::Branch2), + 0xa6 => op("if_acmpne", O::Branch2), + 0xa7 => op("goto", O::Branch2), + 0xa8 => op("jsr", O::Branch2), + 0xa9 => op("ret", O::Local), + 0xaa => op("tableswitch", O::TableSwitch), + 0xab => op("lookupswitch", O::LookupSwitch), + 0xac => op("ireturn", O::None), + 0xad => op("lreturn", O::None), + 0xae => op("freturn", O::None), + 0xaf => op("dreturn", O::None), + 0xb0 => op("areturn", O::None), + 0xb1 => op("return", O::None), + 0xb2 => op("getstatic", O::Cp2), + 0xb3 => op("putstatic", O::Cp2), + 0xb4 => op("getfield", O::Cp2), + 0xb5 => op("putfield", O::Cp2), + 0xb6 => op("invokevirtual", O::Cp2), + 0xb7 => op("invokespecial", O::Cp2), + 0xb8 => op("invokestatic", O::Cp2), + 0xb9 => op("invokeinterface", O::InvokeInterface), + 0xba => op("invokedynamic", O::InvokeDynamic), + 0xbb => op("new", O::Cp2), + 0xbc => op("newarray", O::NewArray), + 0xbd => op("anewarray", O::Cp2), + 0xbe => op("arraylength", O::None), + 0xbf => op("athrow", O::None), + 0xc0 => op("checkcast", O::Cp2), + 0xc1 => op("instanceof", O::Cp2), + 0xc2 => op("monitorenter", O::None), + 0xc3 => op("monitorexit", O::None), + 0xc4 => op("wide", O::Wide), + 0xc5 => op("multianewarray", O::MultiANewArray), + 0xc6 => op("ifnull", O::Branch2), + 0xc7 => op("ifnonnull", O::Branch2), + 0xc8 => op("goto_w", O::Branch4), + 0xc9 => op("jsr_w", O::Branch4), + 0xca => op("breakpoint", O::None), + 0xfe => op("impdep1", O::None), + 0xff => op("impdep2", O::None), + _ => op("unknown", O::Reserved), + } +} + +/// The primitive array type for a `newarray` atype operand. +pub fn array_type(atype: i32) -> &'static str { + match atype { + 4 => "boolean", + 5 => "char", + 6 => "float", + 7 => "double", + 8 => "byte", + 9 => "short", + 10 => "int", + 11 => "long", + _ => "?", + } +} + +/// A decoded `tableswitch`/`lookupswitch`. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct Switch { + /// Absolute byte offset of the default branch. + pub default: i64, + /// `(key, absolute target)` pairs in listing order. + pub pairs: Vec<(i32, i64)>, +} + +/// One decoded instruction. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct Insn { + /// Byte offset from the start of the method's code. + pub pc: u32, + pub opcode: u8, + pub name: &'static str, + /// Total length in bytes, including operands. + pub len: u32, + /// Local-variable index operand (load/store/ret/iinc). + #[serde(skip_serializing_if = "Option::is_none")] + pub local: Option, + /// Integer immediate (`bipush`/`sipush`/`iinc` increment/`newarray` type/`multianewarray` dims). + #[serde(skip_serializing_if = "Option::is_none")] + pub int: Option, + /// Constant-pool index operand (resolved by the caller). + #[serde(skip_serializing_if = "Option::is_none")] + pub cp_index: Option, + /// Absolute byte offset of a branch target. + #[serde(skip_serializing_if = "Option::is_none")] + pub branch: Option, + /// A switch table when the opcode is `tableswitch`/`lookupswitch`. + #[serde(skip_serializing_if = "Option::is_none")] + pub switch: Option, + /// True when this instruction was prefixed by `wide`. + #[serde(skip_serializing_if = "std::ops::Not::not")] + pub wide: bool, +} + +/// Most entries decoded from one switch table. +const MAX_SWITCH: usize = 1 << 20; + +fn s8(code: &[u8], at: usize) -> Option { + code.get(at).map(|&b| i32::from(b as i8)) +} +fn u16b(code: &[u8], at: usize) -> Option { + Some(u16::from(*code.get(at)?) << 8 | u16::from(*code.get(at + 1)?)) +} +fn s16(code: &[u8], at: usize) -> Option { + u16b(code, at).map(|v| i32::from(v as i16)) +} +fn s32(code: &[u8], at: usize) -> Option { + Some(i32::from(*code.get(at)?) << 24 | i32::from(*code.get(at + 1)?) << 16 | i32::from(*code.get(at + 2)?) << 8 | i32::from(*code.get(at + 3)?)) +} + +fn base(pc: usize, opcode: u8, name: &'static str) -> Insn { + Insn { pc: pc as u32, opcode, name, len: 1, local: None, int: None, cp_index: None, branch: None, switch: None, wide: false } +} + +/// Decode one instruction at `pc`, or `None` for invalid or incomplete operands. +pub fn decode_one(code: &[u8], pc: usize) -> Option { + let b = *code.get(pc)?; + let o = opcode(b); + let mut insn = base(pc, b, o.name); + let rel = |d: i32| pc as i64 + i64::from(d); + match o.operands { + Operands::None | Operands::Reserved => {} + Operands::Byte => { + insn.int = Some(s8(code, pc + 1)?); + insn.len = 2; + } + Operands::Short => { + insn.int = Some(s16(code, pc + 1)?); + insn.len = 3; + } + Operands::Local => { + insn.local = Some(u16::from(*code.get(pc + 1)?)); + insn.len = 2; + } + Operands::LocalConst => { + insn.local = Some(u16::from(*code.get(pc + 1)?)); + insn.int = Some(s8(code, pc + 2)?); + insn.len = 3; + } + Operands::Branch2 => { + insn.branch = Some(rel(s16(code, pc + 1)?)); + insn.len = 3; + } + Operands::Branch4 => { + insn.branch = Some(rel(s32(code, pc + 1)?)); + insn.len = 5; + } + Operands::Cp1 => { + insn.cp_index = Some(u16::from(*code.get(pc + 1)?)); + insn.len = 2; + } + Operands::Cp2 => { + insn.cp_index = Some(u16b(code, pc + 1)?); + insn.len = 3; + } + Operands::NewArray => { + let atype = *code.get(pc + 1)?; + if !(4..=11).contains(&atype) { + return None; + } + insn.int = Some(i32::from(atype)); + insn.len = 2; + } + Operands::InvokeInterface => { + insn.cp_index = Some(u16b(code, pc + 1)?); + if *code.get(pc + 3)? == 0 || *code.get(pc + 4)? != 0 { + return None; + } + insn.len = 5; + } + Operands::InvokeDynamic => { + insn.cp_index = Some(u16b(code, pc + 1)?); + if *code.get(pc + 3)? != 0 || *code.get(pc + 4)? != 0 { + return None; + } + insn.len = 5; + } + Operands::MultiANewArray => { + insn.cp_index = Some(u16b(code, pc + 1)?); + let dimensions = *code.get(pc + 3)?; + if dimensions == 0 { + return None; + } + insn.int = Some(i32::from(dimensions)); + insn.len = 4; + } + Operands::Wide => return decode_wide(code, pc), + Operands::TableSwitch => return decode_tableswitch(code, pc), + Operands::LookupSwitch => return decode_lookupswitch(code, pc), + } + if pc.checked_add(insn.len as usize)? > code.len() { + return None; + } + Some(insn) +} + +fn decode_wide(code: &[u8], pc: usize) -> Option { + let real = *code.get(pc + 1)?; + if !matches!(real, 0x15..=0x19 | 0x36..=0x3a | 0x84 | 0xa9) { + return None; + } + let o = opcode(real); + let mut insn = base(pc, real, o.name); + insn.wide = true; + if real == 0x84 { + // wide iinc: 2-byte local, 2-byte signed const + insn.local = Some(u16b(code, pc + 2)?); + insn.int = Some(s16(code, pc + 4)?); + insn.len = 6; + } else { + // wide load/store/ret: 2-byte local + insn.local = Some(u16b(code, pc + 2)?); + insn.len = 4; + } + if pc.checked_add(insn.len as usize)? > code.len() { + return None; + } + Some(insn) +} + +/// Padding so the operand word after a switch opcode is 4-byte aligned from the code start. +fn switch_pad(pc: usize) -> usize { + (4 - ((pc + 1) % 4)) % 4 +} + +fn decode_tableswitch(code: &[u8], pc: usize) -> Option { + let mut at = pc + 1 + switch_pad(pc); + let default = rel_i64(pc, s32(code, at)?); + let low = s32(code, at + 4)?; + let high = s32(code, at + 8)?; + at += 12; + if high < low { + return None; + } + let count = usize::try_from(i64::from(high) - i64::from(low) + 1).ok()?; + if count > MAX_SWITCH { + return None; + } + let end = at.checked_add(count.checked_mul(4)?)?; + code.get(at..end)?; + let mut pairs = Vec::with_capacity(count); + for i in 0..count { + let off = s32(code, at + i * 4)?; + pairs.push((low.wrapping_add(i as i32), rel_i64(pc, off))); + } + let mut insn = base(pc, 0xaa, "tableswitch"); + insn.switch = Some(Switch { default, pairs }); + insn.len = (end - pc) as u32; + Some(insn) +} + +fn decode_lookupswitch(code: &[u8], pc: usize) -> Option { + let mut at = pc + 1 + switch_pad(pc); + let default = rel_i64(pc, s32(code, at)?); + let npairs = s32(code, at + 4)?; + at += 8; + if npairs < 0 { + return None; + } + let count = npairs as usize; + if count > MAX_SWITCH { + return None; + } + let end = at.checked_add(count.checked_mul(8)?)?; + code.get(at..end)?; + let mut pairs = Vec::with_capacity(count); + let mut previous = None; + for i in 0..count { + let key = s32(code, at + i * 8)?; + if previous.is_some_and(|previous| key <= previous) { + return None; + } + previous = Some(key); + let off = s32(code, at + i * 8 + 4)?; + pairs.push((key, rel_i64(pc, off))); + } + let mut insn = base(pc, 0xab, "lookupswitch"); + insn.switch = Some(Switch { default, pairs }); + insn.len = (end - pc) as u32; + Some(insn) +} + +fn rel_i64(pc: usize, off: i32) -> i64 { + pc as i64 + i64::from(off) +} + +/// Decode a whole method body, stopping at `max` instructions. `complete` is false when a +/// instruction has invalid/incomplete operands or the cap was hit. +pub fn decode(code: &[u8], max: usize) -> (Vec, bool) { + let mut out = Vec::new(); + let mut pc = 0usize; + let mut complete = true; + while pc < code.len() { + if out.len() >= max { + complete = false; + break; + } + match decode_one(code, pc) { + Some(i) => { + pc += (i.len as usize).max(1); + out.push(i); + } + None => { + complete = false; + break; + } + } + } + (out, complete) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn lookup(pc: usize, keys: &[i32]) -> Vec { + let mut code = vec![0; pc]; + code.push(0xab); + while !code.len().is_multiple_of(4) { + code.push(0); + } + code.extend_from_slice(&0i32.to_be_bytes()); + code.extend_from_slice(&(keys.len() as i32).to_be_bytes()); + for key in keys { + code.extend_from_slice(&key.to_be_bytes()); + code.extend_from_slice(&0i32.to_be_bytes()); + } + code + } + + #[test] + fn lookup_keys_must_increase_strictly() { + for keys in [&[1, 1][..], &[1, 0], &[0, -1], &[i32::MIN, i32::MAX, i32::MIN]] { + for pc in 0..4 { + let mut code = lookup(pc, keys); + code.push(0xb1); + assert!(decode_one(&code, pc).is_none(), "pc {pc}, keys {keys:?}"); + let (decoded, complete) = decode(&code, 100); + assert!(!complete); + assert_eq!(decoded.len(), pc, "preserve preceding nops, stop before the bad switch"); + } + } + } + + #[test] + fn lookup_keys_use_signed_order_at_every_alignment() { + for keys in [&[][..], &[i32::MIN, -1, 0, i32::MAX]] { + for pc in 0..4 { + let code = lookup(pc, keys); + let insn = decode_one(&code, pc).unwrap(); + assert_eq!(insn.len as usize, code.len() - pc); + let switch = insn.switch.unwrap(); + assert_eq!(switch.default, pc as i64); + assert_eq!(switch.pairs, keys.iter().map(|&key| (key, pc as i64)).collect::>()); + assert!(decode(&code, 100).1); + } + } + } + + #[test] + fn switch_payloads_must_be_complete() { + for pc in 0..4 { + let lookup = lookup(pc, &[-1, 1]); + let mut dense = vec![0; pc]; + dense.push(0xaa); + while !dense.len().is_multiple_of(4) { + dense.push(0); + } + for word in [0i32, i32::MAX - 1, i32::MAX, 0, 0] { + dense.extend_from_slice(&word.to_be_bytes()); + } + for code in [lookup, dense] { + assert!(decode_one(&code, pc).is_some()); + for end in pc + 1..code.len() { + assert!(decode_one(&code[..end], pc).is_none(), "pc {pc}, end {end}"); + } + } + } + } + + #[test] + fn switch_counts_cannot_reserve_an_absent_table() { + let mut lookup = lookup(0, &[]); + lookup[8..12].copy_from_slice(&(MAX_SWITCH as i32).to_be_bytes()); + assert!(decode_one(&lookup, 0).is_none()); + for (low, high) in [(0i32, MAX_SWITCH as i32 - 1), (i32::MIN, i32::MAX)] { + let mut dense = vec![0xaa, 0, 0, 0]; + for word in [0, low, high] { + dense.extend_from_slice(&word.to_be_bytes()); + } + assert!(decode_one(&dense, 0).is_none()); + } + } + + #[test] + fn table_covers_defined_opcodes() { + let mut named = 0; + for b in 0..=255u8 { + if opcode(b).name != "unknown" { + named += 1; + } + } + // 0x00..=0xca is 203 opcodes, plus impdep1/impdep2. + assert_eq!(named, 203 + 2); + assert_eq!(opcode(0xb6).name, "invokevirtual"); + assert_eq!(opcode(0x10).name, "bipush"); + assert_eq!(opcode(0xc4).name, "wide"); + } + + #[test] + fn decodes_core_formats() { + // iconst_1; bipush 7; istore_1; goto +3 ; (pad to reach len) return + let code = [0x04, 0x10, 0x07, 0x3c, 0xa7, 0x00, 0x03, 0xb1]; + let (ins, complete) = decode(&code, 100); + assert!(complete, "{ins:?}"); + assert_eq!(ins[0].name, "iconst_1"); + assert_eq!(ins[1].name, "bipush"); + assert_eq!(ins[1].int, Some(7)); + assert_eq!(ins[2].name, "istore_1"); + assert_eq!(ins[3].name, "goto"); + assert_eq!(ins[3].branch, Some(7)); // pc 4 + 3 + assert_eq!(ins[4].name, "return"); + } + + #[test] + fn decodes_invoke_and_wide() { + // invokevirtual #5 ; invokeinterface #6 count 2 ; wide iload 300 + let code = [0xb6, 0x00, 0x05, 0xb9, 0x00, 0x06, 0x02, 0x00, 0xc4, 0x15, 0x01, 0x2c]; + let (ins, complete) = decode(&code, 100); + assert!(complete); + assert_eq!(ins[0].name, "invokevirtual"); + assert_eq!(ins[0].cp_index, Some(5)); + assert_eq!(ins[0].len, 3); + assert_eq!(ins[1].name, "invokeinterface"); + assert_eq!(ins[1].cp_index, Some(6)); + assert_eq!(ins[1].len, 5); + assert_eq!(ins[2].name, "iload"); + assert!(ins[2].wide); + assert_eq!(ins[2].local, Some(300)); + assert_eq!(ins[2].len, 4); + } + + #[test] + fn decodes_switches() { + // tableswitch at pc 0: pad 3, default +40, low 1, high 2, offsets +10, +20 + let mut code = vec![0xaa, 0, 0, 0]; + code.extend_from_slice(&40i32.to_be_bytes()); + code.extend_from_slice(&1i32.to_be_bytes()); + code.extend_from_slice(&2i32.to_be_bytes()); + code.extend_from_slice(&10i32.to_be_bytes()); + code.extend_from_slice(&20i32.to_be_bytes()); + let i = decode_one(&code, 0).unwrap(); + assert_eq!(i.name, "tableswitch"); + let sw = i.switch.unwrap(); + assert_eq!(sw.default, 40); + assert_eq!(sw.pairs, vec![(1, 10), (2, 20)]); + assert_eq!(i.len as usize, code.len()); + + // lookupswitch at pc 0: pad 3, default +4, npairs 1, (key 99 -> +8) + let mut lk = vec![0xab, 0, 0, 0]; + lk.extend_from_slice(&4i32.to_be_bytes()); + lk.extend_from_slice(&1i32.to_be_bytes()); + lk.extend_from_slice(&99i32.to_be_bytes()); + lk.extend_from_slice(&8i32.to_be_bytes()); + let l = decode_one(&lk, 0).unwrap(); + let sw = l.switch.unwrap(); + assert_eq!(sw.default, 4); + assert_eq!(sw.pairs, vec![(99, 8)]); + } + + #[test] + fn hostile() { + for seed in 0..256 { + let n = crate::test_support::noise(seed, 512); + let _ = decode(&n, 10_000); + } + // truncated operands never panic + let _ = decode(&[0xb6, 0x00], 10); + let _ = decode(&[0xaa, 0, 0, 0, 0, 0, 0, 0], 10); + let _ = decode(&[0xab, 0, 0, 0, 0, 0, 0, 0, 0x7f, 0xff, 0xff, 0xff], 10); + let _ = decode(&[0xc4], 10); + } +} diff --git a/crates/java-class/src/lib.rs b/crates/java-class/src/lib.rs new file mode 100644 index 000000000..4d135e16b --- /dev/null +++ b/crates/java-class/src/lib.rs @@ -0,0 +1,50 @@ +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +#![forbid(unsafe_code)] +//! Java class-file inspection and JVM bytecode decoding. Never executes bytecode. +//! Retains source item, attribute and annotation-depth limits and partial reports. +mod bytes; +mod descriptor; +pub mod java_class; +pub mod java_ops; +pub use java_class::*; +#[cfg(test)] +mod test_support; +/// File admission errors; damaged optional structures retain partial-report warnings. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum JavaClassError { + #[error("the file ends early ({0})")] + Truncated(String), + #[error("the file is malformed ({0})")] + Malformed(String), + #[error("not supported: {0}")] + Unsupported(String), +} +use JavaClassError as FormatError; + +#[cfg(test)] +#[path = "regressions/java_array_operands.rs"] +mod java_array_operands; + +#[cfg(test)] +#[path = "regressions/java_attribute_names.rs"] +mod java_attribute_names; + +#[cfg(test)] +#[path = "regressions/java_constant_pool_slots.rs"] +mod java_constant_pool_slots; + +#[cfg(test)] +#[path = "regressions/java_invoke_operands.rs"] +mod java_invoke_operands; + +#[cfg(test)] +#[path = "regressions/java_linkage_references.rs"] +mod java_linkage_references; + +#[cfg(test)] +#[path = "regressions/java_truncated_attributes.rs"] +mod java_truncated_attributes; + +#[cfg(test)] +#[path = "regressions/java_wide_opcodes.rs"] +mod java_wide_opcodes; diff --git a/crates/java-class/src/regressions/java_array_operands.rs b/crates/java-class/src/regressions/java_array_operands.rs new file mode 100644 index 000000000..b5867db52 --- /dev/null +++ b/crates/java-class/src/regressions/java_array_operands.rs @@ -0,0 +1,48 @@ +use crate::java_ops::{decode, decode_one}; + +#[test] +fn invalid_primitive_array_types_stop_before_the_allocation() { + for atype in (0..=u8::MAX).filter(|value| !(4..=11).contains(value)) { + assert!(decode_one(&[0xbc, atype], 0).is_none(), "atype {atype}"); + let (instructions, complete) = decode(&[0, 0xbc, atype, 0xb1], 100); + assert!(!complete, "atype {atype}"); + assert_eq!(instructions.len(), 1); + assert_eq!(instructions[0].name, "nop"); + } +} + +#[test] +fn zero_dimension_multidimensional_allocations_are_not_complete_bytecode() { + assert!(decode_one(&[0xc5, 0, 1, 0], 0).is_none()); + let (instructions, complete) = decode(&[0, 0xc5, 0, 1, 0, 0xb1], 100); + assert!(!complete); + assert_eq!(instructions.len(), 1); + assert_eq!(instructions[0].name, "nop"); +} + +#[test] +fn every_defined_primitive_type_and_nonzero_dimension_remains_decodable() { + for atype in 4..=11 { + let (instructions, complete) = decode(&[0xbc, atype, 0xb1], 100); + assert!(complete); + assert_eq!(instructions.len(), 2); + assert_eq!(instructions[0].int, Some(i32::from(atype))); + assert_eq!(instructions[0].len, 2); + } + for dimensions in 1..=u8::MAX { + let instruction = decode_one(&[0, 0xc5, 0x12, 0x34, dimensions], 1).unwrap(); + assert_eq!(instruction.pc, 1); + assert_eq!(instruction.cp_index, Some(0x1234)); + assert_eq!(instruction.int, Some(i32::from(dimensions))); + assert_eq!(instruction.len, 4); + } +} + +#[test] +fn incomplete_array_operands_remain_rejected() { + for instruction in [&[0xbc, 10][..], &[0xc5, 0, 1, 2][..]] { + for end in 1..instruction.len() { + assert!(decode_one(&instruction[..end], 0).is_none()); + } + } +} diff --git a/crates/java-class/src/regressions/java_attribute_names.rs b/crates/java-class/src/regressions/java_attribute_names.rs new file mode 100644 index 000000000..c506ed0c9 --- /dev/null +++ b/crates/java-class/src/regressions/java_attribute_names.rs @@ -0,0 +1,132 @@ +use crate::java_class::{self, ClassFile, Const}; + +#[derive(Clone, Copy, Debug)] +enum Scope { + Class, + Field, + Method, + Code, +} + +fn u16b(out: &mut Vec, n: u16) { + out.extend_from_slice(&n.to_be_bytes()); +} + +fn u32b(out: &mut Vec, n: u32) { + out.extend_from_slice(&n.to_be_bytes()); +} + +fn utf8(out: &mut Vec, text: &str) { + out.push(1); + u16b(out, text.len() as u16); + out.extend_from_slice(text.as_bytes()); +} + +fn attr(name: u16, info: &[u8]) -> Vec { + let mut out = Vec::new(); + u16b(&mut out, name); + u32b(&mut out, info.len() as u32); + out.extend_from_slice(info); + out +} + +fn attributes(out: &mut Vec, repeated: u16, known: &[Vec]) { + u16b(out, repeated + known.len() as u16); + for _ in 0..repeated { + out.extend_from_slice(&attr(5, &[])); + } + for bytes in known { + out.extend_from_slice(bytes); + } +} + +// A valid shared CONSTANT_Utf8 is named by every unknown attribute. Known +// attributes follow it, so avoiding name copies must not discard their facts. +fn fixture(scope: Scope, repeated: u16) -> Vec { + let mut out = vec![0xca, 0xfe, 0xba, 0xbe, 0, 0, 0, 52]; + u16b(&mut out, 17); + utf8(&mut out, "AttributeFixture"); // 1 + out.push(7); + u16b(&mut out, 1); // 2: this class + utf8(&mut out, "java/lang/Object"); // 3 + out.push(7); + u16b(&mut out, 3); // 4: superclass + utf8(&mut out, &format!("org.example.{}", "A".repeat(usize::from(u16::MAX) - 12))); // 5 + for text in ["SourceFile", "AttributeFixture.java", "field", "I"] { + utf8(&mut out, text); // 6..9 + } + out.push(3); + u32b(&mut out, 42); // 10 + for text in ["ConstantValue", "run", "()V", "Code", "LineNumberTable", "Deprecated"] { + utf8(&mut out, text); // 11..16 + } + for word in [0x0021, 2, 4, 0, 1, 0x0019, 8, 9] { + u16b(&mut out, word); + } + attributes(&mut out, if matches!(scope, Scope::Field) { repeated } else { 0 }, &[attr(11, &10u16.to_be_bytes()), attr(16, &[])]); + for word in [1, 0x0009, 12, 13] { + u16b(&mut out, word); + } + let mut code = vec![0, 0, 0, 0, 0, 0, 0, 1, 0xb1, 0, 0]; + attributes(&mut code, if matches!(scope, Scope::Code) { repeated } else { 0 }, &[attr(15, &[0, 1, 0, 0, 0, 7])]); + attributes(&mut out, if matches!(scope, Scope::Method) { repeated } else { 0 }, &[attr(14, &code)]); + attributes(&mut out, if matches!(scope, Scope::Class) { repeated } else { 0 }, &[attr(6, &7u16.to_be_bytes())]); + out +} + +fn check_facts(parsed: &ClassFile) { + assert!(parsed.warnings.is_empty(), "{:?}", parsed.warnings); + assert_eq!(parsed.this_class, "AttributeFixture"); + assert_eq!(parsed.super_class.as_deref(), Some("java/lang/Object")); + assert_eq!(parsed.source_file.as_deref(), Some("AttributeFixture.java")); + assert_eq!(parsed.fields.len(), 1); + assert_eq!(parsed.fields[0].name, "field"); + assert_eq!(parsed.fields[0].constant_value.as_deref(), Some("42")); + assert!(parsed.fields[0].deprecated); + assert_eq!(parsed.methods.len(), 1); + assert_eq!(parsed.methods[0].name, "run"); + let code = parsed.methods[0].code.as_ref().unwrap(); + assert_eq!(code.bytecode, [0xb1]); + assert_eq!(code.line_numbers, [(0, 7)]); + assert!(matches!(&parsed.pool[5], Const::Utf8(name) if name.len() == usize::from(u16::MAX))); +} + +#[test] +fn known_attributes_survive_repeated_unknown_names_at_every_scope() { + for scope in [Scope::Class, Scope::Field, Scope::Method, Scope::Code] { + check_facts(&java_class::parse(&fixture(scope, 64)).unwrap()); + } +} + +fn memory_case(scope: Scope) { + let bytes = fixture(scope, 4096); + assert!(bytes.len() < 100_000); + check_facts(&java_class::parse(&bytes).unwrap()); +} + +// Run each ignored case separately with a 128 MiB process address-space limit. +// The original parser copies 256 MiB of one attribute name; the fixture itself +// is under 100 KiB. These checks deliberately make no timing assertion. +#[test] +#[ignore = "run separately under a process memory limit"] +fn repeated_class_attribute_names_fit_in_bounded_memory() { + memory_case(Scope::Class); +} + +#[test] +#[ignore = "run separately under a process memory limit"] +fn repeated_field_attribute_names_fit_in_bounded_memory() { + memory_case(Scope::Field); +} + +#[test] +#[ignore = "run separately under a process memory limit"] +fn repeated_method_attribute_names_fit_in_bounded_memory() { + memory_case(Scope::Method); +} + +#[test] +#[ignore = "run separately under a process memory limit"] +fn repeated_code_attribute_names_fit_in_bounded_memory() { + memory_case(Scope::Code); +} diff --git a/crates/java-class/src/regressions/java_constant_pool_slots.rs b/crates/java-class/src/regressions/java_constant_pool_slots.rs new file mode 100644 index 000000000..15ad88567 --- /dev/null +++ b/crates/java-class/src/regressions/java_constant_pool_slots.rs @@ -0,0 +1,53 @@ +use crate::{FormatError, java_class}; + +fn class_with_pool(count: u16, pool: &[u8]) -> Vec { + let mut bytes = vec![0xca, 0xfe, 0xba, 0xbe, 0, 0, 0, 52]; + bytes.extend_from_slice(&count.to_be_bytes()); + bytes.extend_from_slice(pool); + bytes.extend_from_slice(&[0; 14]); + bytes +} + +#[test] +fn a_wide_constant_needs_its_reserved_slot() { + for tag in [5, 6] { + let mut pool = vec![tag]; + pool.extend_from_slice(&[0; 8]); + let bytes = class_with_pool(2, &pool); + assert!(matches!(java_class::parse(&bytes), Err(FormatError::Malformed(message)) if message.contains("second slot"))); + } +} + +#[test] +fn the_largest_pool_cannot_overflow_on_a_trailing_wide_constant() { + for tag in [5, 6] { + let mut pool = [3, 0, 0, 0, 0].repeat(usize::from(u16::MAX) - 2); + pool.push(tag); + pool.extend_from_slice(&[0; 8]); + let bytes = class_with_pool(u16::MAX, &pool); + assert!(matches!(java_class::parse(&bytes), Err(FormatError::Malformed(message)) if message.contains("second slot"))); + } +} + +#[test] +fn a_wide_constant_with_two_remaining_slots_is_retained() { + for tag in [5, 6] { + for count in [3, u16::MAX] { + let mut pool = [3, 0, 0, 0, 0].repeat(usize::from(count) - 3); + pool.push(tag); + pool.extend_from_slice(&[0; 8]); + let parsed = java_class::parse(&class_with_pool(count, &pool)).unwrap(); + assert_eq!(parsed.pool.len(), usize::from(count)); + assert!(matches!(parsed.pool.last(), Some(java_class::Const::Unusable))); + assert!(matches!(parsed.pool[usize::from(count) - 2], java_class::Const::Long(_) | java_class::Const::Double(_))); + } + } +} + +#[test] +fn ordinary_final_constants_and_existing_classes_still_parse() { + let parsed = java_class::parse(&class_with_pool(2, &[3, 0, 0, 0, 7])).unwrap(); + assert_eq!(parsed.pool, vec![java_class::Const::Unusable, java_class::Const::Integer(7)]); + let parsed = java_class::parse(&crate::test_support::java::hello_class()).unwrap(); + assert!(!parsed.methods.is_empty()); +} diff --git a/crates/java-class/src/regressions/java_invoke_operands.rs b/crates/java-class/src/regressions/java_invoke_operands.rs new file mode 100644 index 000000000..a99d8803b --- /dev/null +++ b/crates/java-class/src/regressions/java_invoke_operands.rs @@ -0,0 +1,76 @@ +use crate::java_class; +use crate::java_ops::{decode, decode_one}; + +#[test] +fn invokeinterface_requires_a_count_and_zero_reserved_byte() { + for code in [[0xb9, 0, 1, 0, 0], [0xb9, 0, 1, 1, 1], [0xb9, 0, 1, 255, 255]] { + assert!(decode_one(&code, 0).is_none(), "{code:?}"); + assert!(!decode(&code, 16).1); + } +} + +#[test] +fn invokedynamic_requires_both_reserved_bytes_to_be_zero() { + for code in [[0xba, 0, 1, 1, 0], [0xba, 0, 1, 0, 1], [0xba, 0, 1, 255, 255]] { + assert!(decode_one(&code, 0).is_none(), "{code:?}"); + assert!(!decode(&code, 16).1); + } +} + +#[test] +fn valid_and_truncated_invocations_keep_their_boundaries() { + for count in [1, 2, 255] { + let instruction = decode_one(&[0xb9, 0x12, 0x34, count, 0], 0).unwrap(); + assert_eq!(instruction.cp_index, Some(0x1234)); + assert_eq!(instruction.len, 5); + } + let instruction = decode_one(&[0xba, 0x12, 0x34, 0, 0], 0).unwrap(); + assert_eq!(instruction.cp_index, Some(0x1234)); + for code in [[0xb9, 0, 1, 1, 0], [0xba, 0, 1, 0, 0]] { + for len in 1..5 { + assert!(decode_one(&code[..len], 0).is_none()); + } + } + let (instructions, complete) = decode(&[0xb9, 0, 1, 1, 0, 0xba, 0, 2, 0, 0, 0xb1], 16); + assert!(complete); + assert_eq!(instructions.len(), 3); + assert_eq!(instructions[2].pc, 10); +} + +fn reference_class() -> java_class::ClassFile { + let mut class = java_class::parse(&crate::test_support::java::hello_class()).unwrap(); + class.methods.retain(|method| method.name == "run"); + class +} + +#[test] +fn malformed_invocations_mark_reference_scans_incomplete() { + let valid = reference_class(); + let (expected, incomplete) = java_class::references(&valid, "java/io/PrintStream", None, 10); + assert!(!incomplete); + assert_eq!(expected.len(), 1); + for invalid in [[0xb9, 0, 1, 0, 0], [0xb9, 0, 1, 1, 1], [0xba, 0, 1, 1, 0], [0xba, 0, 1, 0, 1]] { + let mut class = valid.clone(); + let code = &mut class.methods[0].code.as_mut().unwrap().bytecode; + let later_reference = code.clone(); + *code = invalid.to_vec(); + code.extend_from_slice(&later_reference); + let (found, incomplete) = java_class::references(&class, "java/io/PrintStream", None, 10); + assert!(found.is_empty()); + assert!(incomplete, "invalid invocation must not hide an unfinished reference scan: {invalid:?}"); + } +} + +#[test] +fn incomplete_scans_keep_earlier_references_and_scan_other_methods() { + let mut class = reference_class(); + let mut later_method = class.methods[0].clone(); + later_method.name = "later".into(); + class.methods[0].code.as_mut().unwrap().bytecode.extend_from_slice(&[0xba, 0, 1, 1, 0]); + class.methods.push(later_method); + let (found, incomplete) = java_class::references(&class, "java/io/PrintStream", None, 10); + assert!(incomplete); + assert_eq!(found.len(), 2); + assert!(found[0].caller.contains(".run:(")); + assert!(found[1].caller.contains(".later:(")); +} diff --git a/crates/java-class/src/regressions/java_linkage_references.rs b/crates/java-class/src/regressions/java_linkage_references.rs new file mode 100644 index 000000000..7c05abe94 --- /dev/null +++ b/crates/java-class/src/regressions/java_linkage_references.rs @@ -0,0 +1,97 @@ +use crate::java_class::{parse, references}; + +/// A method with a chosen load or call, plus a dynamic call site and a dynamic constant with distinct bootstrap signatures. +fn class(code: &[u8]) -> Vec { + fn utf(s: &str) -> Vec { + let mut b = vec![1]; + b.extend((s.len() as u16).to_be_bytes()); + b.extend(s.bytes()); + b + } + let pool = [ + utf("Uses"), + vec![7, 0, 1], + utf("java/lang/Object"), + vec![7, 0, 3], + utf("run"), + utf("()V"), + utf("Code"), + utf("BootstrapMethods"), + utf("Target"), + vec![7, 0, 9], + utf("apply"), + vec![12, 0, 11, 0, 6], + vec![10, 0, 10, 0, 12], + vec![15, 6, 0, 13], + utf("Factory"), + vec![7, 0, 15], + utf("boot"), + vec![12, 0, 17, 0, 28], + vec![10, 0, 16, 0, 18], + vec![15, 6, 0, 19], + utf("([LTarget;)V"), + vec![16, 0, 21], + vec![18, 0, 0, 0, 12], + utf("value"), + utf("I"), + vec![12, 0, 24, 0, 25], + vec![17, 0, 1, 0, 26], + utf( + "(Ljava/lang/invoke/MethodHandles$Lookup;Ljava/lang/String;Ljava/lang/invoke/MethodType;Ljava/lang/invoke/MethodHandle;)Ljava/lang/invoke/CallSite;", + ), + utf("(Ljava/lang/invoke/MethodHandles$Lookup;Ljava/lang/String;Ljava/lang/Class;)I"), + vec![12, 0, 17, 0, 29], + vec![10, 0, 16, 0, 30], + vec![15, 6, 0, 31], + ]; + let mut bytes = vec![0xca, 0xfe, 0xba, 0xbe, 0, 0, 0, 55]; + bytes.extend((pool.len() as u16 + 1).to_be_bytes()); + for e in pool { + bytes.extend(e); + } + bytes.extend([0, 0x21, 0, 2, 0, 4, 0, 0, 0, 0, 0, 1, 0, 9, 0, 5, 0, 6, 0, 1, 0, 7]); + bytes.extend(((12 + code.len()) as u32).to_be_bytes()); + bytes.extend([0, 3, 0, 0]); + bytes.extend((code.len() as u32).to_be_bytes()); + bytes.extend(code); + bytes.extend([0, 0, 0, 0]); + let boots = [0, 2, 0, 20, 0, 1, 0, 14, 0, 32, 0, 0]; + bytes.extend([0, 1, 0, 8]); + bytes.extend((boots.len() as u32).to_be_bytes()); + bytes.extend(boots); + bytes +} +fn kinds(code: &[u8], owner: &str, method: Option<&str>) -> Vec<&'static str> { + let cf = parse(&class(code)).unwrap(); + let (hits, partial) = references(&cf, owner, method, 10); + assert!(!partial); + hits.into_iter().map(|r| r.kind).collect() +} +#[test] +fn class_literal_has_a_use_site() { + assert_eq!(kinds(&[0x12, 10, 0x57, 0xb1], "Target", None), ["type"]); +} +#[test] +fn literal_method_handle_finds_its_member() { + assert_eq!(kinds(&[0x12, 14, 0x57, 0xb1], "Target", Some("apply")), ["method-handle"]); +} +#[test] +fn method_type_mentions_object_array_components() { + assert_eq!(kinds(&[0x12, 22, 0x57, 0xb1], "Target", None), ["method-type"]); +} +#[test] +fn dynamic_call_exposes_factory_dependency() { + assert_eq!(kinds(&[0xba, 0, 23, 0, 0, 0xb1], "Factory", Some("boot")), ["bootstrap-method"]); +} +#[test] +fn dynamic_call_exposes_static_method_handle_argument() { + assert_eq!(kinds(&[0xba, 0, 23, 0, 0, 0xb1], "Target", Some("apply")), ["method-handle"]); +} +#[test] +fn dynamic_constant_exposes_bootstrap_dependency() { + assert_eq!(kinds(&[0x12, 27, 0x57, 0xb1], "Factory", Some("boot")), ["bootstrap-method"]); +} +#[test] +fn unused_bootstrap_does_not_introduce_a_reference() { + assert!(kinds(&[0xb1], "Factory", None).is_empty()); +} diff --git a/crates/java-class/src/regressions/java_truncated_attributes.rs b/crates/java-class/src/regressions/java_truncated_attributes.rs new file mode 100644 index 000000000..700640f33 --- /dev/null +++ b/crates/java-class/src/regressions/java_truncated_attributes.rs @@ -0,0 +1,46 @@ +use crate::java_class::parse; +use crate::test_support::java::hello_class; + +/// The fixture ends in a class attribute count and a single two-byte SourceFile attribute. +const ATTRIBUTE_TABLE_BYTES: usize = 10; + +/// Every cut of the attribute count, header or body leaves a partial class report. +#[test] +fn incomplete_class_attribute_table_reports_a_warning() { + let data = hello_class(); + let start = data.len() - ATTRIBUTE_TABLE_BYTES; + for end in start..data.len() { + let class = parse(&data[..end]).unwrap(); + assert_eq!(class.methods.len(), 4); + assert!(class.source_file.is_none()); + assert!(class.warnings.iter().any(|warning| warning.contains("class attribute table")), "cut at {}", end - start); + } +} + +/// A complete attribute stays available when a subsequent attribute is cut short. +#[test] +fn completed_class_attributes_survive_a_truncated_record() { + let mut data = hello_class(); + let start = data.len() - ATTRIBUTE_TABLE_BYTES; + data[start..start + 2].copy_from_slice(&2u16.to_be_bytes()); + data.extend_from_slice(&[0, 1, 0, 0, 0, 2, 0]); + let class = parse(&data).unwrap(); + assert_eq!(class.source_file.as_deref(), Some("Hello.java")); + assert!(class.warnings.iter().any(|warning| warning.contains("class attribute table"))); +} + +/// Complete, empty and unknown class attributes do not trigger a truncation warning. +#[test] +fn complete_class_attribute_tables_are_accepted() { + let mut data = hello_class(); + let start = data.len() - ATTRIBUTE_TABLE_BYTES; + let original = parse(&data).unwrap(); + assert_eq!(original.source_file.as_deref(), Some("Hello.java")); + assert!(original.warnings.is_empty()); + data[start..start + 2].copy_from_slice(&2u16.to_be_bytes()); + data.extend_from_slice(&[0, 1, 0, 0, 0, 1, 0]); + assert!(parse(&data).unwrap().warnings.is_empty()); + data.truncate(start + 2); + data[start..].fill(0); + assert!(parse(&data).unwrap().warnings.is_empty()); +} diff --git a/crates/java-class/src/regressions/java_wide_opcodes.rs b/crates/java-class/src/regressions/java_wide_opcodes.rs new file mode 100644 index 000000000..85c03ffaa --- /dev/null +++ b/crates/java-class/src/regressions/java_wide_opcodes.rs @@ -0,0 +1,33 @@ +use crate::java_ops::{decode, decode_one}; + +#[test] +fn only_defined_wide_opcodes_are_accepted() { + for op in 0..=255u8 { + let valid = matches!(op, 0x15..=0x19 | 0x36..=0x3a | 0x84 | 0xa9); + let code = [0xc4, op, 0x01, 0x2c, 0xff, 0xfe]; + assert_eq!(decode_one(&code, 0).is_some(), valid, "opcode {op:#x}"); + } +} + +#[test] +fn invalid_wide_does_not_invent_a_method_body() { + let (instructions, complete) = decode(&[0x00, 0xc4, 0xb1, 0, 1, 0xb1], 32); + assert!(!complete); + assert_eq!(instructions.len(), 1); + assert_eq!(instructions[0].name, "nop"); +} + +#[test] +fn wide_locals_and_signed_increments_are_preserved() { + for op in [0x15, 0x16, 0x17, 0x18, 0x19, 0x36, 0x37, 0x38, 0x39, 0x3a, 0xa9] { + let instruction = decode_one(&[0xc4, op, 0xff, 0xff], 0).unwrap(); + assert_eq!(instruction.local, Some(u16::MAX)); + assert_eq!(instruction.len, 4); + assert!(instruction.wide); + } + let instruction = decode_one(&[0xc4, 0x84, 0x01, 0x2c, 0xff, 0xfe], 0).unwrap(); + assert_eq!(instruction.local, Some(300)); + assert_eq!(instruction.int, Some(-2)); + assert_eq!(instruction.len, 6); + assert!(decode_one(&[0xc4, 0x84, 0x00, 1], 0).is_none()); +} diff --git a/crates/java-class/src/test_support.rs b/crates/java-class/src/test_support.rs new file mode 100644 index 000000000..e2e13ddbd --- /dev/null +++ b/crates/java-class/src/test_support.rs @@ -0,0 +1,276 @@ +pub use craft_testkit::sample_damage::{mutations, noise}; + +pub mod java { + use std::collections::HashMap; + + /// A constant-pool builder that interns entries and tracks the double slot after `Long`/`Double`. + #[derive(Default)] + struct Pool { + items: Vec>, + /// Next free index (1-based; accounts for long/double taking two slots). + count: u16, + cache: HashMap, + } + + impl Pool { + fn new() -> Self { + Pool { items: Vec::new(), count: 1, cache: HashMap::new() } + } + fn add(&mut self, bytes: Vec, slots: u16) -> u16 { + let idx = self.count; + self.items.push(bytes); + self.count += slots; + idx + } + fn utf8(&mut self, s: &str) -> u16 { + let key = format!("u:{s}"); + if let Some(&i) = self.cache.get(&key) { + return i; + } + let mut b = vec![1u8]; + b.extend_from_slice(&(s.len() as u16).to_be_bytes()); + b.extend_from_slice(s.as_bytes()); + let i = self.add(b, 1); + self.cache.insert(key, i); + i + } + fn class(&mut self, name: &str) -> u16 { + let n = self.utf8(name); + let key = format!("c:{name}"); + if let Some(&i) = self.cache.get(&key) { + return i; + } + let mut b = vec![7u8]; + b.extend_from_slice(&n.to_be_bytes()); + let i = self.add(b, 1); + self.cache.insert(key, i); + i + } + fn string(&mut self, s: &str) -> u16 { + let u = self.utf8(s); + let mut b = vec![8u8]; + b.extend_from_slice(&u.to_be_bytes()); + self.add(b, 1) + } + fn nat(&mut self, name: &str, desc: &str) -> u16 { + let n = self.utf8(name); + let d = self.utf8(desc); + let mut b = vec![12u8]; + b.extend_from_slice(&n.to_be_bytes()); + b.extend_from_slice(&d.to_be_bytes()); + self.add(b, 1) + } + fn reff(&mut self, tag: u8, owner: &str, name: &str, desc: &str) -> u16 { + let c = self.class(owner); + let nt = self.nat(name, desc); + let mut b = vec![tag]; + b.extend_from_slice(&c.to_be_bytes()); + b.extend_from_slice(&nt.to_be_bytes()); + self.add(b, 1) + } + fn long(&mut self, v: i64) -> u16 { + let mut b = vec![5u8]; + b.extend_from_slice(&v.to_be_bytes()); + self.add(b, 2) + } + /// The serialized constant pool and the count field. + fn finish(&self) -> (Vec, u16) { + (self.items.concat(), self.count) + } + } + + fn u16b(v: &mut Vec, x: u16) { + v.extend_from_slice(&x.to_be_bytes()); + } + fn u32b(v: &mut Vec, x: u32) { + v.extend_from_slice(&x.to_be_bytes()); + } + + /// Build an attribute: name_index, length, body. + fn attr(name_idx: u16, body: Vec) -> Vec { + let mut v = Vec::new(); + u16b(&mut v, name_idx); + u32b(&mut v, body.len() as u32); + v.extend_from_slice(&body); + v + } + + /// A method's `Code` attribute body (max_stack, max_locals, code, exception table, no nested attrs). + fn code_body(max_stack: u16, max_locals: u16, code: &[u8], exc: &[(u16, u16, u16, u16)]) -> Vec { + let mut v = Vec::new(); + u16b(&mut v, max_stack); + u16b(&mut v, max_locals); + u32b(&mut v, code.len() as u32); + v.extend_from_slice(code); + u16b(&mut v, exc.len() as u16); + for (s, e, h, t) in exc { + u16b(&mut v, *s); + u16b(&mut v, *e); + u16b(&mut v, *h); + u16b(&mut v, *t); + } + u16b(&mut v, 0); // nested attributes count + v + } + + /// A method_info: access, name, descriptor, and one already-built Code attribute. + fn method(p: &mut Pool, access: u16, name: &str, desc: &str, code_attribute: Vec) -> Vec { + let name_idx = p.utf8(name); + let desc_idx = p.utf8(desc); + let mut v = Vec::new(); + u16b(&mut v, access); + u16b(&mut v, name_idx); + u16b(&mut v, desc_idx); + u16b(&mut v, 1); // one attribute + v.extend_from_slice(&code_attribute); + v + } + + /// Assemble a whole class from its parts. + #[allow(clippy::too_many_arguments)] + fn assemble( + pool: &Pool, + access: u16, + this_c: u16, + super_c: u16, + interfaces: &[u16], + fields: &[Vec], + methods: &[Vec], + class_attrs: &[Vec], + ) -> Vec { + let (cp, count) = pool.finish(); + let mut v = Vec::new(); + v.extend_from_slice(&[0xca, 0xfe, 0xba, 0xbe]); + u16b(&mut v, 0); // minor + u16b(&mut v, 52); // major (Java 8) + u16b(&mut v, count); + v.extend_from_slice(&cp); + u16b(&mut v, access); + u16b(&mut v, this_c); + u16b(&mut v, super_c); + u16b(&mut v, interfaces.len() as u16); + for i in interfaces { + u16b(&mut v, *i); + } + u16b(&mut v, fields.len() as u16); + for f in fields { + v.extend_from_slice(f); + } + u16b(&mut v, methods.len() as u16); + for m in methods { + v.extend_from_slice(m); + } + u16b(&mut v, class_attrs.len() as u16); + for a in class_attrs { + v.extend_from_slice(a); + } + v + } + + pub const HELLO: &str = "Hello, Craft!"; + + /// `public class com.example.Hello implements Runnable` with a `count` field, a `BIG` long + /// constant, ``, `run()` (System.out.println("Hello, Craft!")), `pick(int)` (a tableswitch) + /// and `guarded()` (a try/catch over the println call). + pub fn hello_class() -> Vec { + let mut p = Pool::new(); + let this_c = p.class("com/example/Hello"); + let obj_c = p.class("java/lang/Object"); + let runnable_c = p.class("java/lang/Runnable"); + let code_attr = p.utf8("Code"); + + // fields: int count; static final long BIG; + let count_name = p.utf8("count"); + let i_desc = p.utf8("I"); + let big_name = p.utf8("BIG"); + let j_desc = p.utf8("J"); + let constval_attr = p.utf8("ConstantValue"); + let big_val = p.long(0x1122_3344_5566_7788); + + // refs used by code + let obj_init = p.reff(10, "java/lang/Object", "", "()V"); + let sysout = p.reff(9, "java/lang/System", "out", "Ljava/io/PrintStream;"); + let hello = p.string(HELLO); + let println = p.reff(10, "java/io/PrintStream", "println", "(Ljava/lang/String;)V"); + let rt_exc = p.class("java/lang/RuntimeException"); + + // (): aload_0; invokespecial Object.; return + let mut init = vec![0x2a, 0xb7]; + init.extend_from_slice(&obj_init.to_be_bytes()); + init.push(0xb1); + let init_m = method(&mut p, 0x0001, "", "()V", attr(code_attr, code_body(1, 1, &init, &[]))); + + // run(): getstatic System.out; ldc "Hello"; invokevirtual println; return + let mut run = vec![0xb2]; + run.extend_from_slice(&sysout.to_be_bytes()); + run.push(0x12); + run.push(hello as u8); // ldc (1-byte index; pool is small) + run.push(0xb6); + run.extend_from_slice(&println.to_be_bytes()); + run.push(0xb1); + let run_m = method(&mut p, 0x0001, "run", "()V", attr(code_attr, code_body(2, 1, &run, &[]))); + + // pick(int): iload_0; tableswitch{0->L0,1->L1,default->Ldef}; L0: bipush 10; ireturn; L1: bipush 20; ireturn; Ldef: iconst_m1; ireturn + // offsets: iload_1@0; tableswitch@1 (pad 2) ends @24; L0@24; L1@27; Ldef@30 + let mut pick = vec![0x1b, 0xaa, 0x00, 0x00]; + u32b(&mut pick, 29); // default -> 1+29 = 30 + u32b(&mut pick, 0); // low + u32b(&mut pick, 1); // high + u32b(&mut pick, 23); // case 0 -> 1+23 = 24 + u32b(&mut pick, 26); // case 1 -> 1+26 = 27 + debug_assert_eq!(pick.len(), 24); + pick.extend_from_slice(&[0x10, 10, 0xac]); // @24: bipush 10; ireturn + pick.extend_from_slice(&[0x10, 20, 0xac]); // @27: bipush 20; ireturn + pick.extend_from_slice(&[0x02, 0xac]); // @30: iconst_m1; ireturn + let pick_m = method(&mut p, 0x0001, "pick", "(I)I", attr(code_attr, code_body(1, 1, &pick, &[]))); + + // guarded(): try { System.out.println("Hello"); } catch (RuntimeException e) {} (then return) + // 0: getstatic(3) ; 3: ldc(2) ; 5: invokevirtual(3) ; 8: goto +5 ->13 ; 11: astore_1 ; 12: return ; 13: return + let mut g = vec![0xb2]; + g.extend_from_slice(&sysout.to_be_bytes()); + g.push(0x12); + g.push(hello as u8); + g.push(0xb6); + g.extend_from_slice(&println.to_be_bytes()); + g.extend_from_slice(&[0xa7, 0x00, 0x05]); // @8 goto -> 13 + g.push(0x4c); // @11 astore_1 + g.push(0xb1); // @12 return + g.push(0xb1); // @13 return + let guarded_m = method(&mut p, 0x0001, "guarded", "()V", attr(code_attr, code_body(2, 2, &g, &[(0, 8, 11, rt_exc)]))); + + // field infos + let field = |p: &mut Pool, access: u16, name_idx: u16, desc_idx: u16, attrs: &[Vec]| -> Vec { + let _ = p; + let mut v = Vec::new(); + u16b(&mut v, access); + u16b(&mut v, name_idx); + u16b(&mut v, desc_idx); + u16b(&mut v, attrs.len() as u16); + for a in attrs { + v.extend_from_slice(a); + } + v + }; + let count_f = field(&mut p, 0x0002, count_name, i_desc, &[]); // private int count + let mut cv = Vec::new(); + u16b(&mut cv, big_val); + let big_f = field(&mut p, 0x0019, big_name, j_desc, &[attr(constval_attr, cv)]); // public static final long BIG + + let source_attr = p.utf8("SourceFile"); + let source_val = p.utf8("Hello.java"); + let mut sf = Vec::new(); + u16b(&mut sf, source_val); + let source = attr(source_attr, sf); + + assemble( + &p, + 0x0021, // public super + this_c, + obj_c, + &[runnable_c], + &[count_f, big_f], + &[init_m, run_m, pick_m, guarded_m], + &[source], + ) + } +} diff --git a/crates/jpeg/src/encode.rs b/crates/jpeg/src/encode.rs index 548f4c9f9..fb5fe183c 100644 --- a/crates/jpeg/src/encode.rs +++ b/crates/jpeg/src/encode.rs @@ -419,6 +419,34 @@ pub fn encode_baseline( sub: ChromaSubsampling, app_segments: &[(u8, Vec)], ) -> Result, EncodeError> { + validate_encode(data, width, height, channels, app_segments)?; + Ok(encode(data, width, height, channels, quality, sub, app_segments)) +} + +/// Encode the same optimized baseline JPEG as [`encode_baseline`], trading a second DCT pass +/// for working memory independent of image height. This path is sequential even when the +/// `parallel` feature is enabled. It retains only one MCU row of conversion planes and writes +/// directly into the returned encoded buffer: no image-wide tokens or duplicate coded buffer. +/// +/// Scratch consists of at most `24 * width + 8` floats (odd-width 4:2:0), plus fixed-size coding tables. +/// The caller still owns the input pixels and the returned JPEG; this is not a cap on output +/// size or a replacement for the caller's image/admission budget. Geometry, quality, metadata, +/// Huffman tables and restart boundaries match [`encode_baseline`] byte for byte. +#[allow(clippy::too_many_arguments)] +pub fn encode_baseline_low_memory( + data: &[u8], + width: usize, + height: usize, + channels: usize, + quality: u8, + sub: ChromaSubsampling, + app_segments: &[(u8, Vec)], +) -> Result, EncodeError> { + validate_encode(data, width, height, channels, app_segments)?; + Ok(encode_impl(data, width, height, channels, quality, sub, app_segments, true, true)) +} + +fn validate_encode(data: &[u8], width: usize, height: usize, channels: usize, app_segments: &[(u8, Vec)]) -> Result<(), EncodeError> { let fits = width.checked_mul(height).and_then(|n| n.checked_mul(channels)).is_some_and(|n| n <= data.len()); if !fits || width == 0 || height == 0 || width > 65_535 || height > 65_535 || !matches!(channels, 1 | 3 | 4) { return Err(EncodeError::InvalidInput(format!("{width}x{height}, {channels} channels, {} bytes", data.len()))); @@ -426,7 +454,7 @@ pub fn encode_baseline( if app_segments.iter().any(|(marker, payload)| !(0xE0..=0xEF).contains(marker) || payload.len() > 65_533) { return Err(EncodeError::InvalidInput("APP markers must be 0xE0..=0xEF with payloads no larger than 65533 bytes".into())); } - Ok(encode(data, width, height, channels, quality, sub, app_segments)) + Ok(()) } /// [`encode`], with optimized Huffman tables or (`optimize` false) the Annex K example tables. @@ -440,6 +468,21 @@ fn encode_with( sub: ChromaSubsampling, app_segments: &[(u8, Vec)], optimize: bool, +) -> Vec { + encode_impl(data, width, height, channels, quality, sub, app_segments, optimize, false) +} + +#[allow(clippy::too_many_arguments)] +fn encode_impl( + data: &[u8], + width: usize, + height: usize, + channels: usize, + quality: u8, + sub: ChromaSubsampling, + app_segments: &[(u8, Vec)], + optimize: bool, + low_memory: bool, ) -> Vec { // The public entry takes caller-sized dimensions: refuse (an empty output) what would size an allocation from // them or index past `data`, rather than panic or allocate for them. @@ -509,6 +552,8 @@ fn encode_with( planes.push(Plane { w: cw, h: crows.len(), data: cb }); planes.push(Plane { w: cw, h: crows.len(), data: cr }); } + #[cfg(test)] + low_memory_tests::record_planes(planes.iter().map(|p| p.data.len()).sum()); planes }; @@ -519,45 +564,58 @@ fn encode_with( let (mx, my) = (width.div_ceil(mcu_w), height.div_ceil(mcu_h)); // restart interval: whole MCU rows per band, ≤ 65535 MCUs, enough bands to keep all cores busy let rows_per_band = (my / 64).clamp(1, (65_535 / mx.max(1)).max(1)); - let bands: Vec = (0..my.div_ceil(rows_per_band)).collect(); + let num_bands = my.div_ceil(rows_per_band); + let bands: Vec = if low_memory { Vec::new() } else { (0..num_bands).collect() }; // first pass: each band's quantized blocks as symbols, with their counts per table (DC // prediction restarts per band, as the restart markers say) - let tokenize_band = |band: &usize| -> (Vec, [Freq; 4]) { + let visit_band = |band: &usize, emit: &mut dyn FnMut(&[i16; 64], &mut i32, bool)| { let (mut blk, mut q) = ([0f32; 64], [0i16; 64]); - let (mut tokens, mut freq) = (Vec::new(), [[0u64; 256]; 4]); let mut pred = [0i32; 3]; let (m0, m1) = (band * rows_per_band, ((band + 1) * rows_per_band).min(my)); - // the band's planes, its MCU row `m0` at their top (blocks past the image's bottom repeat its last row, - // which is the band's last row then) - let planes = convert(m0 * mcu_h..(m1 * mcu_h).min(height), m0 * 8..(m1 * 8).min(ch)); for mcu_y in m0..m1 { - let mcu_y = mcu_y - m0; + // Convert only the current MCU row. DC predictors still span the original band, + // so its symbols/restart decisions are identical to whole-band conversion. + let planes = convert(mcu_y * mcu_h..((mcu_y + 1) * mcu_h).min(height), mcu_y * 8..((mcu_y + 1) * 8).min(ch)); for mcu_x in 0..mx { for vy in 0..vs { for hx in 0..hs { - planes[0].block(mcu_x * hs + hx, mcu_y * vs + vy, &mut blk); + planes[0].block(mcu_x * hs + hx, vy, &mut blk); fdct_quant(&blk, &rl, &mut q); - tokenize_block(&q, &mut pred[0], false, &mut tokens, &mut freq); + emit(&q, &mut pred[0], false); } } if !gray { for (plane, pred) in planes.iter().zip(pred.iter_mut()).take(3).skip(1) { - plane.block(mcu_x, mcu_y, &mut blk); + plane.block(mcu_x, 0, &mut blk); fdct_quant(&blk, &rc, &mut q); - tokenize_block(&q, pred, true, &mut tokens, &mut freq); + emit(&q, pred, true); } } } } + }; + let tokenize_band = |band: &usize| -> (Vec, [Freq; 4]) { + let (mut tokens, mut freq) = (Vec::new(), [[0u64; 256]; 4]); + visit_band(band, &mut |q, pred, chroma| tokenize_block(q, pred, chroma, &mut tokens, &mut freq)); + #[cfg(test)] + low_memory_tests::record_tokens(tokens.len()); (tokens, freq) }; #[cfg(feature = "parallel")] - let tokenized: Vec<(Vec, [Freq; 4])> = bands.par_iter().map(tokenize_band).collect(); + let tokenized: Vec<(Vec, [Freq; 4])> = if low_memory { Vec::new() } else { bands.par_iter().map(tokenize_band).collect() }; #[cfg(not(feature = "parallel"))] - let tokenized: Vec<(Vec, [Freq; 4])> = bands.iter().map(tokenize_band).collect(); + let tokenized: Vec<(Vec, [Freq; 4])> = if low_memory { Vec::new() } else { bands.iter().map(tokenize_band).collect() }; // the tables: DC luma, AC luma, DC chroma, AC chroma let tables: [([u8; 16], Vec); 4] = if optimize { let mut total = [[0u64; 256]; 4]; + if low_memory { + for band in 0..num_bands { + visit_band(&band, &mut |q, pred, chroma| { + let base = if chroma { 2 } else { 0 }; + block_symbols(q, pred, |ac, symbol, _, _| total[base + ac as usize][symbol & 0xFF] += 1); + }); + } + } for (_, f) in &tokenized { for (t, f) in total.iter_mut().zip(f) { for (t, f) in t.iter_mut().zip(f) { @@ -577,9 +635,9 @@ fn encode_with( let huff = tables.each_ref().map(|(bits, vals)| Huff::new(bits, vals)); // second pass: code the symbols #[cfg(feature = "parallel")] - let coded: Vec> = tokenized.into_par_iter().map(|(t, _)| write_tokens(&t, &huff)).collect(); + let coded: Vec> = if low_memory { Vec::new() } else { tokenized.into_par_iter().map(|(t, _)| write_tokens(&t, &huff)).collect() }; #[cfg(not(feature = "parallel"))] - let coded: Vec> = tokenized.into_iter().map(|(t, _)| write_tokens(&t, &huff)).collect(); + let coded: Vec> = if low_memory { Vec::new() } else { tokenized.into_iter().map(|(t, _)| write_tokens(&t, &huff)).collect() }; // headers let mut out = Vec::with_capacity(coded.iter().map(Vec::len).sum::() + 4096); @@ -617,7 +675,7 @@ fn encode_with( dht(&mut out, 0x11, acb, acv); } let interval = (rows_per_band * mx) as u16; - if coded.len() > 1 { + if num_bands > 1 { segment(&mut out, 0xDD, &interval.to_be_bytes()); } if gray { @@ -625,11 +683,32 @@ fn encode_with( } else { segment(&mut out, 0xDA, &[3, 1, 0x00, 2, 0x11, 3, 0x11, 0, 63, 0]); } - let last = coded.len().saturating_sub(1); - for (i, c) in coded.iter().enumerate() { - out.extend_from_slice(c); - if i < last { - out.extend_from_slice(&[0xFF, 0xD0 + (i % 8) as u8]); + if low_memory { + let last = num_bands.saturating_sub(1); + for i in 0..num_bands { + // Continue directly in the final buffer, resetting entropy bits at the same band + // boundaries as the retained-token path. Only the output grows with image height. + let mut bits = Bits { out, acc: 0, n: 0 }; + visit_band(&i, &mut |q, pred, chroma| { + let base = if chroma { 2 } else { 0 }; + block_symbols(q, pred, |ac, symbol, extra, n| { + let (code, len) = huff[base + ac as usize].code[symbol & 0xFF]; + bits.put(code as u32, len as u32); + bits.put(extra & 0xFFFF, n & 0xF); + }); + }); + out = bits.finish(); + if i < last { + out.extend_from_slice(&[0xFF, 0xD0 + (i % 8) as u8]); + } + } + } else { + let last = coded.len().saturating_sub(1); + for (i, c) in coded.iter().enumerate() { + out.extend_from_slice(c); + if i < last { + out.extend_from_slice(&[0xFF, 0xD0 + (i % 8) as u8]); + } } } out.extend_from_slice(&[0xFF, 0xD9]); @@ -877,3 +956,7 @@ mod tests { assert!(psnr(&src, &px) > 35.0); } } + +#[cfg(test)] +#[path = "encode_low_memory_tests.rs"] +mod low_memory_tests; diff --git a/crates/jpeg/src/encode_low_memory_tests.rs b/crates/jpeg/src/encode_low_memory_tests.rs new file mode 100644 index 000000000..de45f8dfe --- /dev/null +++ b/crates/jpeg/src/encode_low_memory_tests.rs @@ -0,0 +1,155 @@ +use super::*; +use std::cell::Cell; + +thread_local! { + static STATS: Cell<(usize, usize)> = const { Cell::new((0, 0)) }; +} + +pub(super) fn record_planes(samples: usize) { + STATS.with(|s| { + let (peak, tokens) = s.get(); + s.set((peak.max(samples), tokens)); + }); +} +pub(super) fn record_tokens(count: usize) { + STATS.with(|s| { + let (peak, tokens) = s.get(); + s.set((peak, tokens + count)); + }); +} + +fn noise(len: usize) -> Vec { + let mut seed = 0x89ab_cdefu32; + (0..len) + .map(|_| { + seed ^= seed << 13; + seed ^= seed >> 17; + seed ^= seed << 5; + seed as u8 + }) + .collect() +} + +fn compare(src: &[u8], w: usize, h: usize, ch: usize, q: u8, sub: ChromaSubsampling, app: &[(u8, Vec)]) { + let expected = encode_baseline(src, w, h, ch, q, sub, app).unwrap(); + STATS.with(|s| s.set((0, 0))); + let actual = encode_baseline_low_memory(src, w, h, ch, q, sub, app).unwrap(); + assert_eq!(actual, expected, "{w}x{h} channels={ch} quality={q} subsampling={sub:?}"); + let (peak, tokens) = STATS.with(Cell::get); + assert!(peak <= 24 * w + 8, "MCU-row scratch exceeded: {peak}"); + assert_eq!(tokens, 0, "low-memory mode retained entropy tokens"); +} + +#[test] +fn all_sampling_and_channels_match_retained_tokens_byte_for_byte() { + for (w, h) in [(1, 1), (7, 9), (17, 31), (33, 65), (17, 1025)] { + for ch in [1, 3, 4] { + let src = noise(w * h * ch); + for sub in [ChromaSubsampling::S444, ChromaSubsampling::S422, ChromaSubsampling::S420] { + for q in [0, 1, 25, 50, 90, 100, 255] { + compare(&src, w, h, ch, q, sub, &[]); + } + } + } + } +} + +#[test] +fn high_entropy_hd_and_wide_frames_keep_only_one_mcu_row() { + for (w, h) in [(1920, 1080), (65_535, 1)] { + let src = noise(w * h * 3); + compare(&src, w, h, 3, 100, ChromaSubsampling::S444, &[]); + } +} + +#[test] +fn metadata_empty_tables_and_extra_source_bytes_keep_exact_output() { + let app = [(0xe0, b"custom density".to_vec()), (0xe2, vec![73; 65_533])]; + for ch in [1, 3, 4] { + let src = vec![127; 19 * 23 * ch + 5]; + compare(&src, 19, 23, ch, 75, ChromaSubsampling::S420, &app); + } +} + +#[test] +fn checked_admission_matches_original_without_panics() { + for (w, h, ch) in [(0, 1, 3), (1, 0, 3), (1, 1, 0), (1, 1, 2), (usize::MAX, 2, 4), (65_536, 1, 1), (1, 65_536, 1), (4, 4, 3)] { + let src = vec![0; if w == 65_536 || h == 65_536 { 65_536 } else { 8 }]; + let old = encode_baseline(&src, w, h, ch, 90, ChromaSubsampling::S444, &[]); + let new = encode_baseline_low_memory(&src, w, h, ch, 90, ChromaSubsampling::S444, &[]); + assert!(new.is_err()); + assert_eq!(new, old); + } + for app in [vec![(0xdf, vec![])], vec![(0xf0, vec![])], vec![(0xe1, vec![0; 65_534])]] { + assert_eq!( + encode_baseline_low_memory(&[0; 3], 1, 1, 3, 90, ChromaSubsampling::S444, &app), + encode_baseline(&[0; 3], 1, 1, 3, 90, ChromaSubsampling::S444, &app) + ); + } +} + +// Generated with the immutable pre-change encoder at craft-libs +// 937ac93c5cf3cd4b6a81e195b078fe161eb081b9, not the refactored token-retaining path. +// Odd chroma edges and 1025/2049-row inputs exercise multi-row restart bands. +#[test] +fn prechange_encoded_bytes_remain_pinned_across_band_boundaries() { + let cases = [ + (1, 1, 1, 1, 444, 159, 0xe3bd59145381ba57), + (1, 1, 1, 75, 422, 159, 0xcb2b103914bbdbd3), + (1, 1, 1, 100, 420, 160, 0xd9f640613efb6abf), + (1, 1, 3, 1, 444, 284, 0x61d2772b8fd9df17), + (1, 1, 3, 75, 422, 286, 0xa0e03ab7efa23296), + (1, 1, 3, 100, 420, 288, 0x5660667f1abd0a63), + (1, 1, 4, 1, 444, 284, 0x61d2772b8fd9df17), + (1, 1, 4, 75, 422, 286, 0xa0e03ab7efa23296), + (1, 1, 4, 100, 420, 288, 0x5660667f1abd0a63), + (17, 31, 1, 1, 444, 240, 0xf8d277e9499253d4), + (17, 31, 1, 75, 422, 515, 0xa6946d4f144bf243), + (17, 31, 1, 100, 420, 793, 0xbb3db259708d8951), + (17, 31, 3, 1, 444, 363, 0xff2b5b41ffb3f81b), + (17, 31, 3, 75, 422, 877, 0x0374ffb527378dfe), + (17, 31, 3, 100, 420, 1220, 0x2feeedfa90885737), + (17, 31, 4, 1, 444, 366, 0xde7f8d1330e436eb), + (17, 31, 4, 75, 422, 879, 0xb1559fbfc3ee0cf6), + (17, 31, 4, 100, 420, 1214, 0xaf5dcf12d28c8e88), + (33, 65, 1, 1, 444, 388, 0xbee80491e42c8fd2), + (33, 65, 1, 75, 422, 1470, 0x284f4eb78b785ca5), + (33, 65, 1, 100, 420, 2576, 0xa81cc94015f326dc), + (33, 65, 3, 1, 444, 526, 0xf75b4036acc30874), + (33, 65, 3, 75, 422, 2322, 0x6fbc56bd9a1eec70), + (33, 65, 3, 100, 420, 3796, 0xcad25051562ac816), + (33, 65, 4, 1, 444, 527, 0x4bfedc11153de4c1), + (33, 65, 4, 75, 422, 2309, 0x47bf11bf32bac6a4), + (33, 65, 4, 100, 420, 3775, 0x64c2303fde273cd1), + (17, 1025, 1, 1, 444, 2017, 0x242f4a95d8579612), + (17, 1025, 1, 75, 422, 10728, 0xfe49408f379bdffb), + (17, 1025, 1, 100, 420, 19686, 0x1355b995e38d6467), + (17, 1025, 3, 1, 444, 2137, 0x4e319440cb6946c2), + (17, 1025, 3, 75, 422, 17682, 0xf456c63b731dde8f), + (17, 1025, 3, 100, 420, 29015, 0x0631001154b128cf), + (17, 1025, 4, 1, 444, 2155, 0xa2de9c1d77e62aa6), + (17, 1025, 4, 75, 422, 17648, 0x7d85fbdf56061f0c), + (17, 1025, 4, 100, 420, 29010, 0x4848eeb93daf1df2), + (19, 2049, 1, 1, 444, 3531, 0x9b93cf219ff85796), + (19, 2049, 1, 75, 422, 26283, 0xb160523444397608), + (19, 2049, 1, 100, 420, 51883, 0xc5c26836c0a976c6), + (19, 2049, 3, 1, 444, 3303, 0x26eaac2272e1fdb8), + (19, 2049, 3, 75, 422, 42745, 0x38b9843fb54db698), + (19, 2049, 3, 100, 420, 79868, 0xd6a7917af19682f7), + (19, 2049, 4, 1, 444, 3343, 0x6bf577903528692f), + (19, 2049, 4, 75, 422, 42706, 0x0e62366274593a93), + (19, 2049, 4, 100, 420, 79804, 0x867d05cf9d944599), + ]; + for (w, h, ch, q, sampling, len, hash) in cases { + let sub = match sampling { + 444 => ChromaSubsampling::S444, + 422 => ChromaSubsampling::S422, + _ => ChromaSubsampling::S420, + }; + let src = noise(w * h * ch); + for actual in [encode_baseline(&src, w, h, ch, q, sub, &[]).unwrap(), encode_baseline_low_memory(&src, w, h, ch, q, sub, &[]).unwrap()] { + let actual_hash = actual.iter().fold(0xcbf29ce484222325u64, |h, b| (h ^ u64::from(*b)).wrapping_mul(0x100000001b3)); + assert_eq!((actual.len(), actual_hash), (len, hash), "{w}x{h} channels={ch} quality={q} sampling={sampling}"); + } + } +} diff --git a/crates/jpeg/src/lib.rs b/crates/jpeg/src/lib.rs index 23434fdd0..ed2ee1988 100644 --- a/crates/jpeg/src/lib.rs +++ b/crates/jpeg/src/lib.rs @@ -1705,7 +1705,7 @@ mod tests; /// Baseline JPEG encoder (8-bit, Huffman-coded sequential DCT). mod encode; -pub use encode::{ChromaSubsampling, EncodeError, encode_baseline}; +pub use encode::{ChromaSubsampling, EncodeError, encode_baseline, encode_baseline_low_memory}; mod app14; pub use app14::adobe_transform; diff --git a/crates/mcp-server/src/lib.rs b/crates/mcp-server/src/lib.rs index 17e6827c4..66568fc05 100644 --- a/crates/mcp-server/src/lib.rs +++ b/crates/mcp-server/src/lib.rs @@ -15,7 +15,7 @@ mod server; pub use content::{Contents, base64, image_result, text_result}; pub use schema::{Effect, check_arguments, object, string, tool}; -pub use server::{Incoming, Info, PROTOCOL_VERSION, Reference, SUPPORTED_VERSIONS, Server, Tools, read_message, response}; +pub use server::{Incoming, Info, Options, PROTOCOL_VERSION, Reference, SUPPORTED_VERSIONS, Server, Tools, read_message, response}; #[cfg(test)] mod tests; diff --git a/crates/mcp-server/src/server.rs b/crates/mcp-server/src/server.rs index b7ed05027..e0d064da5 100644 --- a/crates/mcp-server/src/server.rs +++ b/crates/mcp-server/src/server.rs @@ -4,7 +4,7 @@ use std::io::{BufRead, Write}; use serde_json::{Value, json}; -use crate::content::Contents; +use crate::content::{Contents, text_result}; use crate::schema::{check_arguments, check_prompt_arguments}; /// The newest protocol revision we speak; offered when the client asks for one we don't know. @@ -88,9 +88,25 @@ pub struct Info { pub max_line: usize, } +/// Opt-in behaviour for apps with their own conventions; `Options::default()` is what [`Server::new`] +/// does. Set with [`Server::with_options`]. +#[derive(Clone, Debug, Default)] +pub struct Options { + /// Sent as `serverInfo.websiteUrl` at `initialize` (MCP 2025-06-18) when set. + pub website_url: Option, + /// Report a `tools/call` whose arguments fail the check against the tool's schema, or whose + /// tool is unknown, as a tool result with `isError: true` and the message as text, not as a + /// JSON-RPC -32602 error. For apps whose clients read every tool failure from the result. + pub tool_errors_as_results: bool, + /// Refuse a request whose `id` is neither a string nor a number (JSON-RPC 2.0) with -32600 and + /// a null id, instead of echoing the id back. + pub strict_request_ids: bool, +} + /// An MCP server over the app's [`Tools`]. pub struct Server { info: Info, + options: Options, tools: T, initialized: bool, } @@ -157,7 +173,13 @@ pub fn response(id: Value, result: Value) -> Value { impl Server { pub fn new(info: Info, tools: T) -> Self { - Self { info, tools, initialized: false } + Self { info, options: Options::default(), tools, initialized: false } + } + /// This server with `options` ([`Options`]) in place of the defaults. + #[must_use] + pub fn with_options(mut self, options: Options) -> Self { + self.options = options; + self } pub fn tools(&mut self) -> &mut T { &mut self.tools @@ -215,7 +237,11 @@ impl Server { if o.contains_key("result") || o.contains_key("error") { return None; } - return Some(error(id.unwrap_or(Value::Null), INVALID_REQUEST, "missing `method`")); + let echoed = match id { + Some(id) if !self.options.strict_request_ids || id.is_string() || id.is_number() => id, + _ => Value::Null, + }; + return Some(error(echoed, INVALID_REQUEST, "missing `method`")); }; let params = o.get("params").cloned().unwrap_or(Value::Null); let Some(id) = id else { @@ -224,6 +250,9 @@ impl Server { } return None; }; + if self.options.strict_request_ids && !(id.is_string() || id.is_number()) { + return Some(error(Value::Null, INVALID_REQUEST, "`id` must be a string or number")); + } Some(match self.request(method, ¶ms) { Ok(r) => response(id, r), Err((code, m)) => error(id, code, m), @@ -236,10 +265,14 @@ impl Server { let asked = params.get("protocolVersion").and_then(Value::as_str).unwrap_or(PROTOCOL_VERSION); let version = if SUPPORTED_VERSIONS.contains(&asked) { asked } else { PROTOCOL_VERSION }; let Info { name, title, version: server_version, .. } = self.info; + let mut server_info = json!({"name": name, "title": title, "version": server_version}); + if let Some(url) = &self.options.website_url { + server_info["websiteUrl"] = json!(url); + } Ok(json!({ "protocolVersion": version, "capabilities": self.capabilities(), - "serverInfo": {"name": name, "title": title, "version": server_version}, + "serverInfo": server_info, "instructions": self.tools.instructions(), })) } @@ -251,7 +284,9 @@ impl Server { None | Some(Value::Null) => json!({}), Some(a) => a.clone(), }; - check_arguments(&self.tools.list(), name, &args).map_err(|e| (INVALID_PARAMS, e))?; + if let Err(e) = check_arguments(&self.tools.list(), name, &args) { + return if self.options.tool_errors_as_results { Ok(text_result(e, true)) } else { Err((INVALID_PARAMS, e)) }; + } Ok(self.tools.call(name, &args)) } "resources/list" => Ok(json!({"resources": self.tools.resources()})), diff --git a/crates/mcp-server/src/tests.rs b/crates/mcp-server/src/tests.rs index b687ee2e8..3f8f8d315 100644 --- a/crates/mcp-server/src/tests.rs +++ b/crates/mcp-server/src/tests.rs @@ -1,7 +1,8 @@ use serde_json::{Value, json}; use crate::{ - Contents, Effect, Info, PROTOCOL_VERSION, Reference, SUPPORTED_VERSIONS, Server, Tools, check_arguments, object, string, text_result, tool, + Contents, Effect, Info, Options, PROTOCOL_VERSION, Reference, SUPPORTED_VERSIONS, Server, Tools, check_arguments, object, string, text_result, + tool, }; /// One app's tools: `echo` and `level`, nothing behind them (no engine, no design). `log` records @@ -366,3 +367,131 @@ fn every_call_runs_right_after_its_listing() { reply(&mut s, &json!([call("level", json!({})), call("echo", json!({"text": "b"}))])); assert_eq!(s.tools().log, ["list", "list", "call echo", "list", "list", "call level", "list", "call echo"]); } + +fn with(options: Options) -> Server { + Server::new(info(), Echo::default()).with_options(options) +} + +fn call_msg(id: u64, name: &str, args: Value) -> Value { + json!({"jsonrpc": "2.0", "id": id, "method": "tools/call", "params": {"name": name, "arguments": args}}) +} + +/// `websiteUrl` is sent only when set. +#[test] +fn website_url_is_sent_only_when_set() { + let init = json!({"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {}}); + let off = reply(&mut with(Options::default()), &init).unwrap(); + assert!(off["result"]["serverInfo"].get("websiteUrl").is_none(), "{off}"); + let on = reply(&mut with(Options { website_url: Some("https://example.org/echo".into()), ..Options::default() }), &init).unwrap(); + assert_eq!(on["result"]["serverInfo"]["websiteUrl"], "https://example.org/echo"); + assert_eq!(on["result"]["serverInfo"]["name"], "echo"); +} + +/// With `tool_errors_as_results`, argument failures and unknown tools are `isError` results that +/// carry the message and never run the tool; other errors stay JSON-RPC errors. +#[test] +fn tool_errors_as_results_replaces_invalid_params() { + let mut s = with(Options { tool_errors_as_results: true, ..Options::default() }); + let bad = [ + call_msg(1, "echo", json!({})), + call_msg(2, "echo", json!({"text": 3})), + call_msg(3, "echo", json!({"text": "a", "loud": true})), + call_msg(4, "shout", json!({})), + call_msg(5, "echo", json!([])), + call_msg(6, "echo", json!("text")), + call_msg(7, "echo", json!(7)), + ]; + for msg in &bad { + let r = reply(&mut s, msg).unwrap(); + assert!(r.get("error").is_none(), "{r}"); + assert_eq!(r["result"]["isError"], true, "{r}"); + assert!(r["result"]["content"][0]["text"].as_str().is_some_and(|t| !t.is_empty()), "{r}"); + } + let r = reply(&mut s, &bad[0]).unwrap(); + assert_eq!(r["result"]["content"][0]["text"], "echo: missing required argument `text`"); + assert!(s.tools().calls.is_empty()); + let ok = reply(&mut s, &call_msg(8, "echo", json!({"text": "hi"}))).unwrap(); + assert_eq!(ok["result"], json!({"content": [{"type": "text", "text": "hi"}]})); + let nomethod = reply(&mut s, &json!({"jsonrpc": "2.0", "id": 9, "method": "nope"})).unwrap(); + assert_eq!(nomethod["error"]["code"], -32601); + let noname = reply(&mut s, &json!({"jsonrpc": "2.0", "id": 10, "method": "tools/call", "params": {}})).unwrap(); + assert_eq!(noname["error"]["code"], -32602); + // Unchanged with the option off. + let mut off = with(Options::default()); + for msg in &bad { + assert_eq!(reply(&mut off, msg).unwrap()["error"]["code"], -32602); + } +} + +/// With `strict_request_ids`, an id that is not a string or number is -32600 with a null id; the +/// tool never runs. Off, such ids are echoed as before. Notifications have no id and are unaffected. +#[test] +fn strict_request_ids_refuse_other_id_types() { + let odd = [json!(null), json!(true), json!({"a": 1}), json!([1])]; + let mut on = with(Options { strict_request_ids: true, ..Options::default() }); + let mut off = with(Options::default()); + for id in &odd { + let msg = json!({"jsonrpc": "2.0", "id": id, "method": "tools/call", "params": {"name": "echo", "arguments": {"text": "x"}}}); + let r = reply(&mut on, &msg).unwrap(); + assert_eq!(r["error"]["code"], -32600, "{id}: {r}"); + assert_eq!(r["id"], Value::Null); + let r = reply(&mut off, &json!({"jsonrpc": "2.0", "id": id, "method": "ping"})).unwrap(); + assert_eq!(r["id"], *id); + assert_eq!(r["result"], json!({})); + } + assert!(on.tools().calls.is_empty()); + for id in [json!("a"), json!(1), json!(1.5), json!(-3)] { + let r = reply(&mut on, &json!({"jsonrpc": "2.0", "id": id, "method": "ping"})).unwrap(); + assert_eq!(r["result"], json!({}), "{id}"); + } + assert!(reply(&mut on, ¬ification()).is_none()); +} + +/// The missing-`method` error follows the same id rule: strict mode echoes only string and number +/// ids and answers with a null id for the rest; off, ids are echoed as before. +#[test] +fn strict_request_ids_apply_to_the_missing_method_error() { + let mut on = with(Options { strict_request_ids: true, ..Options::default() }); + let mut off = with(Options::default()); + for id in [json!(true), json!({"a": 1}), json!([1])] { + let msg = json!({"jsonrpc": "2.0", "id": id}); + let r = reply(&mut on, &msg).unwrap(); + assert_eq!((r["error"]["code"].clone(), r["id"].clone()), (json!(-32600), Value::Null), "{id}"); + assert_eq!(reply(&mut off, &msg).unwrap()["id"], id); + } + for id in [json!("a"), json!(7)] { + assert_eq!(reply(&mut on, &json!({"jsonrpc": "2.0", "id": id})).unwrap()["id"], id); + } + assert_eq!(reply(&mut on, &json!({"jsonrpc": "2.0"})).unwrap()["id"], Value::Null); +} + +/// Hostile input with every option on: oversized and non-UTF-8 lines, odd ids and wrongly typed +/// arguments are answered and serving goes on, and nothing panics. +#[test] +fn every_option_on_survives_hostile_input() { + let mut input = vec![b'x'; (1 << 16) + 10]; + input.push(b'\n'); + input.extend_from_slice(b"\xff\xfe\n"); + for id in ["null", "true", "{}", "[]", "\"s\"", "1e999"] { + input.extend_from_slice( + format!( + "{{\"jsonrpc\":\"2.0\",\"id\":{id},\"method\":\"tools/call\",\"params\":{{\"name\":\"echo\",\"arguments\":{{\"text\":{id}}}}}}}\n" + ) + .as_bytes(), + ); + } + for args in ["null", "1", "\"a\"", "[]", "{\"text\":[1]}", "{\"text\":{}}"] { + input.extend_from_slice( + format!("{{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/call\",\"params\":{{\"name\":\"level\",\"arguments\":{args}}}}}\n").as_bytes(), + ); + } + input.extend_from_slice(json!({"jsonrpc": "2.0", "id": 9, "method": "ping"}).to_string().as_bytes()); + let mut output = Vec::new(); + let options = Options { website_url: Some("https://example.org".into()), tool_errors_as_results: true, strict_request_ids: true }; + with(options).serve(std::io::Cursor::new(input), &mut output).unwrap(); + let replies: Vec = String::from_utf8(output).unwrap().lines().map(|l| serde_json::from_str(l).unwrap()).collect(); + assert_eq!(replies.len(), 2 + 6 + 6 + 1, "{replies:?}"); + assert_eq!(replies[0]["error"]["code"], -32600); + assert_eq!(replies[1]["error"]["code"], -32700); + assert_eq!(replies.last().map(|r| &r["result"]), Some(&json!({}))); +} diff --git a/crates/media-testkit/src/hostile.rs b/crates/media-testkit/src/hostile.rs index f40021986..ddfc55f79 100644 --- a/crates/media-testkit/src/hostile.rs +++ b/crates/media-testkit/src/hostile.rs @@ -3,9 +3,9 @@ //! thread with a time limit for each case. [`Rng`], [`mutations`] and the other names here are //! `craft_testkit`'s. +pub use craft_testkit::hostile::{DIVISOR_VARIABLE, MAX_CASES, Mutation, Mutations, Rng, mutations}; #[cfg(not(target_arch = "wasm32"))] -pub use craft_testkit::hostile::{HammerConfig, HammerError}; -pub use craft_testkit::hostile::{MAX_CASES, Mutation, Mutations, Rng, mutations}; +pub use craft_testkit::hostile::{HammerConfig, HammerError, divisor}; /// The variable that multiplies [`hammer`]'s iterations. #[cfg(not(target_arch = "wasm32"))] @@ -17,8 +17,10 @@ const MULTIPLIER_VARIABLE: &str = "CRAFT_MEDIA_HOSTILE_MULT"; /// Panics on the first case that panics or does not finish, naming `name`, the case, its length and /// the panic's message. /// -/// `CRAFT_MEDIA_HOSTILE_MULT=10` runs ten times the iterations (soak runs). A value that is not a -/// whole number above zero panics, naming the variable, before `run` is called. +/// `CRAFT_MEDIA_HOSTILE_MULT=10` runs ten times the iterations (soak runs), and +/// `CRAFT_HOSTILE_DIVISOR=8` every eighth truncation and an eighth of the iterations (pull +/// requests). A value that is not a whole number above zero panics, naming the variable, before +/// `run` is called. #[cfg(not(target_arch = "wasm32"))] #[track_caller] pub fn hammer(name: &str, seed: &[u8], iters: usize, run: impl Fn(&[u8]) + Send + Sync + 'static) { diff --git a/crates/media-testkit/src/lib.rs b/crates/media-testkit/src/lib.rs index 3fa99c54f..db4c844f9 100644 --- a/crates/media-testkit/src/lib.rs +++ b/crates/media-testkit/src/lib.rs @@ -13,7 +13,8 @@ //! //! [`hostile::hammer`] runs a parser over every truncation and mutations of a valid input and fails //! on a panic or a case that does not finish in time; `CRAFT_MEDIA_HOSTILE_MULT=10` runs ten times -//! the iterations. It and [`with_deadline`] call `craft_testkit`. +//! the iterations and `CRAFT_HOSTILE_DIVISOR=8` an eighth of them. It and [`with_deadline`] call +//! `craft_testkit`. //! [`vmd`] writes Sierra VMD fixtures, which ffmpeg can read but not write. //! //! Failing loudly is what a test helper is for: a wrong `CRAFT_MEDIA_*` setting or an unwritable diff --git a/crates/media-testkit/tests/hostile.rs b/crates/media-testkit/tests/hostile.rs index 8e34cd5af..5281f4a41 100644 --- a/crates/media-testkit/tests/hostile.rs +++ b/crates/media-testkit/tests/hostile.rs @@ -5,7 +5,7 @@ use std::process::Command; use std::sync::{Arc, Mutex}; use std::time::Duration; -use craft_media_testkit::hostile::hammer; +use craft_media_testkit::hostile::{divisor, hammer}; /// Set only in the child processes of `run_child`: the value names what `child_hammer` does. const CHILD_VARIABLE: &str = "CRAFT_MEDIA_TESTKIT_HOSTILE_CHILD"; @@ -60,10 +60,11 @@ fn a_bad_multiplier_names_the_variable() { #[test] fn a_panic_keeps_its_message() { - let payload = std::panic::catch_unwind(|| hammer("demo", &[9, 8, 7, 6], 0, |d| assert!(d.len() != 2, "boom at {}", d.len()))).unwrap_err(); + // Truncation to 0 runs under any CRAFT_HOSTILE_DIVISOR. + let payload = std::panic::catch_unwind(|| hammer("demo", &[9, 8, 7, 6], 0, |d| assert!(!d.is_empty(), "boom at {}", d.len()))).unwrap_err(); let message = payload.downcast_ref::().cloned().unwrap_or_default(); - assert!(message.starts_with("demo: truncation to 2 (len 2): the parser panicked: "), "{message}"); - assert!(message.contains("boom at 2"), "{message}"); + assert!(message.starts_with("demo: truncation to 0 (len 0): the parser panicked: "), "{message}"); + assert!(message.contains("boom at 0"), "{message}"); } #[test] @@ -74,8 +75,8 @@ fn every_truncation_of_a_large_seed_runs() { hammer("large", &seed, 0, move |d| { record.lock().unwrap().insert(d.len()); }); - // The seed and its 8,192 truncations. - assert_eq!(lengths.lock().unwrap().len(), 8_193); + // The seed and its 8,192 truncations, every N-th of them under CRAFT_HOSTILE_DIVISOR=N. + assert_eq!(lengths.lock().unwrap().len(), 1 + 8_192usize.div_ceil(divisor())); } #[test] diff --git a/crates/media/src/mxf/tests/hostile.rs b/crates/media/src/mxf/tests/hostile.rs index 015413881..6ff89ed15 100644 --- a/crates/media/src/mxf/tests/hostile.rs +++ b/crates/media/src/mxf/tests/hostile.rs @@ -51,27 +51,53 @@ fn sync_before_on_an_empty_track() { } } -#[test] -fn hostile_input_never_panics() { +// Keep each deterministic hammer independent so test runners can schedule the seeds separately. +// The labels, mutation counts, every-truncation coverage and per-case watchdog stay unchanged. +fn hostile_seeds() -> [Vec; 3] { let (to, flags) = ipbb(); - let seeds = [build(7, |i| vec![i as u8; 100 + i], &to, &flags), build(5, prores_like, &to, &flags), build(4, prores_like, &[], &flags)]; - for (i, s) in seeds.iter().enumerate() { - craft_media_testkit::hostile::hammer(&format!("mxf seed {i}"), s, 6_000, exercise); - } - // The metadata sets and index segments, parsed directly. - let seed = &seeds[0]; + [build(7, |i| vec![i as u8; 100 + i], &to, &flags), build(5, prores_like, &to, &flags), build(4, prores_like, &[], &flags)] +} + +#[test] +fn hostile_input_seed_0_never_panics() { + craft_media_testkit::hostile::hammer("mxf seed 0", &hostile_seeds()[0], 6_000, exercise); +} + +#[test] +fn hostile_input_seed_1_never_panics() { + craft_media_testkit::hostile::hammer("mxf seed 1", &hostile_seeds()[1], 6_000, exercise); +} + +#[test] +fn hostile_input_seed_2_never_panics() { + craft_media_testkit::hostile::hammer("mxf seed 2", &hostile_seeds()[2], 6_000, exercise); +} + +#[test] +fn hostile_index_segment_never_panics() { + let seed = &hostile_seeds()[0]; let index_key = [0x06, 0x0E, 0x2B, 0x34, 0x02, 0x53, 0x01, 0x01, 0x0D, 0x01, 0x02, 0x01, 0x01, 0x10, 0x01, 0x00]; if let Some(v) = value_of(seed, &index_key) { craft_media_testkit::hostile::hammer("mxf index segment", v, 30_000, |d| { let _ = IndexSegment::parse(d); }); } +} + +#[test] +fn hostile_primer_never_panics() { + let seed = &hostile_seeds()[0]; let primer_key = [0x06, 0x0E, 0x2B, 0x34, 0x02, 0x05, 0x01, 0x01, 0x0D, 0x01, 0x02, 0x01, 0x01, 0x05, 0x01, 0x00]; if let Some(v) = value_of(seed, &primer_key) { craft_media_testkit::hostile::hammer("mxf primer", v, 5_000, |d| { let _ = Primer::parse(d); }); } +} + +#[test] +fn hostile_set_never_panics() { + let seed = &hostile_seeds()[0]; let set = seed.windows(16).position(|w| w == set_key(0x18)).and_then(|at| value_of(&seed[at..], &set_key(0x18))); if let Some(v) = set { craft_media_testkit::hostile::hammer("mxf set", v, 20_000, |d| { diff --git a/crates/prepress/src/output.rs b/crates/prepress/src/output.rs index 9bc9a0296..428597d33 100644 --- a/crates/prepress/src/output.rs +++ b/crates/prepress/src/output.rs @@ -191,10 +191,12 @@ mod tests { let p = Plates { inks, unseparated: Vec::new(), width: 2, height: 1, dpi: 72.0, conversion: Conversion::DeviceFormulas, planes }; let output = Output::new(fogra39(), fogra39(), Intent::RelativeColorimetric, false).unwrap(); let cmyk = p.composite_to(&[], &output); - // Magenta's round trip through the profile adds a little yellow. - assert!(near(&cmyk[0..4], &[255, 255, 8, 0], 8), "spot over cyan: {:?}", &cmyk[0..4]); - assert!(near(&cmyk[4..8], &[0, 255, 8, 0], 8), "spot alone: {:?}", &cmyk[4..8]); - assert!(near(&p.composite_to(&[true, true, true, true, false], &output)[0..4], &[255, 0, 0, 0], 6), "spot hidden"); + // The press profile's round trip: its BToA tables generate their own black, so a + // full-tone ink comes back a little short with a trace of black, as a printer would + // separate it. The tolerance is the profile's own round trip, not an exchange total. + assert!(near(&cmyk[0..4], &[250, 247, 17, 2], 8), "spot over cyan: {:?}", &cmyk[0..4]); + assert!(near(&cmyk[4..8], &[8, 247, 11, 1], 8), "spot alone: {:?}", &cmyk[4..8]); + assert!(near(&p.composite_to(&[true, true, true, true, false], &output)[0..4], &[250, 7, 6, 1], 8), "spot hidden"); } #[test] diff --git a/crates/print/README.md b/crates/print/README.md index eb7af885e..56ba5ed86 100644 --- a/crates/print/README.md +++ b/crates/print/README.md @@ -199,10 +199,10 @@ job) and `an_installed_printer_prints_a_file` (`CRAFT_PRINT_TEST_PRINTER`, `CRAFT_PRINT_TEST_FILE`, optional `CRAFT_PRINT_TEST_SCALING=shrink` and `CRAFT_PRINT_TEST_DUPLEX=long`). Physical printer fidelity remains a hardware check. -The cross-platform GitHub Actions workflow is defined in `.github/workflows/print.yml`. -It runs formatting, Clippy and this crate's own tests on Windows, Linux and macOS when print -code changes, and supports manual runs. The Windows job executes the spooler regression above; -physical printer fidelity still requires hardware verification. +The Workspace gate (`.github/workflows/workspace.yml`) runs formatting, Clippy and this crate's +tests on Windows, Linux and macOS. Its Windows leg installs the test print queue and runs the +spooler regression above with `CRAFT_PRINT_REQUIRE_QUEUE=1`; physical printer fidelity still +requires hardware verification. Windows discovery reports driver duplex support in `Printer::two_sided` (a two-sided option in the driver's PrintCapabilities); CUPS leaves it unknown. Explicit simplex/duplex choices that the driver diff --git a/crates/psd/src/guides.rs b/crates/psd/src/guides.rs index 5f100c480..2b654ec8f 100644 --- a/crates/psd/src/guides.rs +++ b/crates/psd/src/guides.rs @@ -5,6 +5,48 @@ //! Unknown direction values retain PhotoCraft's vertical interpretation. Grid cycles, reserved //! bytes and trailing data are not interpreted. Resource 1097 remains opaque except its count. +/// One legacy Grid and Guides record (resource 1032). +/// Position is signed 1/32 pixel; direction 1 is horizontal, all others vertical. +/// Adapted from PhotoCraft's reader by Brandon Thomas and writer by aiongg. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct GuideV1 { + /// Signed position in units of 1/32 pixel. + pub location: i32, + /// Raw direction byte, retained without normalization. + pub direction: u8, +} + +impl GuideV1 { + /// Encode exactly one five-byte record without allocation. + pub fn to_bytes(self) -> [u8; 5] { + let p = self.location.to_be_bytes(); + [p[0], p[1], p[2], p[3], self.direction] + } +} + +/// Encode a legacy resource header. Grid cycles and count are caller supplied. +pub fn header_v1(horizontal_cycle: u32, vertical_cycle: u32, count: u32) -> [u8; 16] { + let mut out = [0; 16]; + for (chunk, value) in out.as_chunks_mut::<4>().0.iter_mut().zip([1u32, horizontal_cycle, vertical_cycle, count]) { + chunk.copy_from_slice(&value.to_be_bytes()); + } + out +} + +/// Read complete legacy records, preserving the valid prefix on truncation. +/// Matches PhotoCraft recovery: version/grid fields are ignored. Declared counts +/// cannot drive allocation or scanning beyond complete records in the input. +pub fn parse_v1_prefix(data: &[u8]) -> impl Iterator + '_ { + let count = be_u32(data, 12).unwrap_or(0); + data.get(16..) + .unwrap_or_default() + .as_chunks::<5>() + .0 + .iter() + .zip(0..count) + .map(|(g, _)| GuideV1 { location: i32::from_be_bytes([g[0], g[1], g[2], g[3]]), direction: g[4] }) +} + /// Grid and Guides version 2 resource ID. pub const GUIDES_V2: u16 = 1092; /// Associated guide identifiers resource ID. @@ -101,3 +143,31 @@ mod tests { assert_eq!(ids_count(&[0, 0, 0]), None); } } + +#[cfg(test)] +mod v1_tests { + use super::*; + + #[test] + fn wire_roundtrip_and_signed_unknown_direction() { + let entries = [GuideV1 { location: -33, direction: 1 }, GuideV1 { location: i32::MAX, direction: 255 }]; + let mut data = header_v1(576, 576, 2).to_vec(); + assert_eq!(data, [0, 0, 0, 1, 0, 0, 2, 64, 0, 0, 2, 64, 0, 0, 0, 2]); + for e in entries { + data.extend(e.to_bytes()); + } + assert_eq!(&data[16..21], &[255, 255, 255, 223, 1]); + assert_eq!(parse_v1_prefix(&data).collect::>(), entries); + for end in 0..=data.len() { + assert_eq!(parse_v1_prefix(&data[..end]).count(), end.saturating_sub(16) / 5); + } + data[..4].copy_from_slice(&99u32.to_be_bytes()); + data[12..16].copy_from_slice(&u32::MAX.to_be_bytes()); + data.extend([9, 9, 9, 9]); + assert_eq!(parse_v1_prefix(&data).collect::>(), entries); + data[12..16].copy_from_slice(&1u32.to_be_bytes()); + assert_eq!(parse_v1_prefix(&data).collect::>(), &entries[..1]); + data[12..16].fill(0); + assert_eq!(parse_v1_prefix(&data).count(), 0); + } +} diff --git a/crates/sourcemap/Cargo.toml b/crates/sourcemap/Cargo.toml new file mode 100644 index 000000000..b36df0107 --- /dev/null +++ b/crates/sourcemap/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "craft-sourcemap" +description = "Bounded Source Map v3 parser, source inventory and generated-position lookup" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true +[lints] +workspace = true +[dependencies] +serde = { workspace = true } +serde_json = { workspace = true } + +thiserror = { workspace = true } diff --git a/crates/sourcemap/src/lib.rs b/crates/sourcemap/src/lib.rs new file mode 100644 index 000000000..a3c23ffa2 --- /dev/null +++ b/crates/sourcemap/src/lib.rs @@ -0,0 +1,816 @@ +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +#![forbid(unsafe_code)] + +//! Source maps (revision 3), including indexed maps with sections. +//! +//! Lookups follow the "original positions for a generated position" rule: on the generated line, +//! take the greatest segment column not after the requested column and return every segment that +//! starts there. Lines are 0-based here (callers add one for display); columns are 0-based UTF-16 +//! code units as the format defines. + +use serde::de::{self, DeserializeSeed, MapAccess, SeqAccess, Visitor}; +use std::cell::Cell; +use std::fmt; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Limits { + /// Biggest map JSON accepted. + pub max_bytes: usize, + /// Most decoded segments across the whole map. + pub max_segments: usize, + /// Most generated lines across the whole map, including empty lines and inline sections. + pub max_lines: usize, + /// Deepest section nesting (the format allows one level; we tolerate a few). + pub max_depth: usize, + /// Most `sources`, `sourcesContent`, `names` and ignore-list entries and index-map sections + /// across the whole map, counted while the JSON is read, so a map of tiny entries is refused + /// before it is kept. An entry a repeated key replaces gives its slot back. + pub max_entries: usize, +} + +impl Default for Limits { + fn default() -> Self { + Limits { max_bytes: 64 * 1024 * 1024, max_segments: 4_000_000, max_lines: 1_000_000, max_depth: 8, max_entries: 4_000_000 } + } +} + +/// Limits on retained source names after applying each map's root. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct InventoryLimits { + pub max_sources: usize, + /// Total UTF-8 bytes in the expanded source paths, across all sections. + pub max_bytes: usize, +} + +impl Default for InventoryLimits { + fn default() -> Self { + Self { max_sources: 1_000_000, max_bytes: 64 * 1024 * 1024 } + } +} + +/// An ordered prefix of the map's source names, with their complete identities. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct SourceInventory { + pub sources: Vec, + /// More names existed than the limits or available allocations could retain. + pub truncated: bool, + /// Index-map sections that pointed at an external map nobody read, so the sources they + /// would add are unknown (not absent). + pub external_sections: usize, +} + +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum SourceMapError { + #[error("source map is {0} bytes; the limit is {1}")] + TooLarge(usize, usize), + #[error("source map is not valid JSON: {0}")] + BadJson(String), + #[error("not a version 3 source map")] + BadVersion, + #[error("source map mappings are malformed at segment {0}")] + BadMappings(usize), + #[error("source map index has invalid sections")] + BadSections, + #[error("source map has more than {0} segments")] + TooManySegments(usize), + #[error("source map has more than {0} generated lines")] + TooManyLines(usize), + #[error("could not reserve memory for generated source-map lines")] + LineAllocationFailed, + #[error("source map sections nest deeper than {0}")] + TooDeep(usize), + #[error("source map has more than {0} source, content, name, ignore-list and section entries")] + TooManyEntries(usize), +} + +/// One decoded mapping segment. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Segment { + pub generated_column: u32, + /// (source index, original line, original column, name index) + pub original: Option<(u32, u32, u32, Option)>, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct SourceMap { + pub file: Option, + pub source_root: Option, + pub sources: Vec>, + pub sources_content: Vec>, + pub names: Vec, + pub ignore_list: Vec, + /// Segments per generated line, sorted by column. + pub lines: Vec>, + pub segments: usize, + /// Indexed maps: (line offset, column offset, map). + pub sections: Vec<(u32, u32, SourceMap)>, + /// Index-map sections that point at an external map (`url`), which isn't fetched here: the + /// sources that map would add are unknown, not absent. + pub external_sections: usize, +} + +/// A position in an original source. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)] +pub struct OriginalPosition { + pub source_index: u32, + pub source: Option, + /// 0-based. + pub line: u32, + /// 0-based. + pub column: u32, + pub name: Option, + /// 0-based generated column of the matched segment. + pub generated_column: u32, +} + +impl SourceMap { + /// The source path with `sourceRoot` prefixed before URL interpretation. + pub fn source_path(&self, index: u32) -> Option { + let s = self.sources.get(index as usize)?.as_ref()?; + // Generators commonly emit an empty root to mean no root. Keep those source identities + // unchanged even though the literal ECMA-426 prefix algorithm would prepend a slash. + Some(match self.source_root.as_deref().filter(|root| !root.is_empty()) { + Some(root) => { + let join = if root.ends_with('/') { "" } else { "/" }; + format!("{root}{join}{s}") + } + None => s.clone(), + }) + } + + pub fn source_content(&self, index: u32) -> Option<&str> { + self.sources_content.get(index as usize)?.as_deref() + } + + /// Every original position for a generated (line, column), both 0-based. + pub fn original_positions(&self, line: u32, column: u32) -> Vec { + if !self.sections.is_empty() { + // Last section that starts at or before the position. + let end = self.sections.partition_point(|(l, c, _)| (*l, *c) <= (line, column)); + let Some((l, c, map)) = end.checked_sub(1).and_then(|i| self.sections.get(i)) else { return Vec::new() }; + let rel_line = line.saturating_sub(*l); + let rel_col = if rel_line == 0 { column.saturating_sub(*c) } else { column }; + return map + .original_positions(rel_line, rel_col) + .into_iter() + .filter_map(|mut position| { + if rel_line == 0 { + position.generated_column = position.generated_column.checked_add(*c)?; + } + Some(position) + }) + .collect(); + } + let Some(segs) = self.lines.get(line as usize) else { return Vec::new() }; + // Greatest generated column <= column. + let idx = segs.partition_point(|s| s.generated_column <= column); + let Some(best) = idx.checked_sub(1).and_then(|i| segs.get(i)) else { return Vec::new() }; + let col = best.generated_column; + let first = segs.partition_point(|s| s.generated_column < col); + segs.get(first..idx) + .unwrap_or_default() + .iter() + .filter_map(|s| { + let (src, l, c, n) = s.original?; + Some(OriginalPosition { + source_index: src, + source: self.source_path(src), + line: l, + column: c, + name: n.and_then(|n| self.names.get(n as usize).cloned()), + generated_column: col, + }) + }) + .collect() + } + + /// Sources flattened across sections, within the default inventory limits. + /// Use `source_inventory` to choose limits and observe truncation. + pub fn all_sources(&self) -> Vec { + self.source_inventory(InventoryLimits::default()).sources + } + + /// Retain a bounded prefix, checking expanded lengths before constructing paths. + pub fn source_inventory(&self, limits: InventoryLimits) -> SourceInventory { + let mut sources = Vec::new(); + let mut bytes_left = limits.max_bytes; + let truncated = !self.collect_sources(limits.max_sources, &mut bytes_left, &mut sources); + SourceInventory { sources, truncated, external_sections: self.external_sections_total() } + } + + /// This map's external sections plus those of every map it holds. + fn external_sections_total(&self) -> usize { + self.sections.iter().fold(self.external_sections, |n, (_, _, map)| n.saturating_add(map.external_sections_total())) + } + + /// Append this map and its sections using one shared entry and byte budget. + fn collect_sources(&self, max_sources: usize, bytes_left: &mut usize, out: &mut Vec) -> bool { + let root = self.source_root.as_deref().unwrap_or_default(); + let join = if root.is_empty() || root.ends_with('/') { "" } else { "/" }; + for source in self.sources.iter().flatten() { + let Some(bytes) = root.len().checked_add(join.len()).and_then(|n| n.checked_add(source.len())) else { return false }; + if out.len() >= max_sources || bytes > *bytes_left { + return false; + } + let mut path = String::new(); + if out.try_reserve(1).is_err() || path.try_reserve_exact(bytes).is_err() { + return false; + } + path.push_str(root); + path.push_str(join); + path.push_str(source); + out.push(path); + *bytes_left = bytes_left.saturating_sub(bytes); + } + for (_, _, map) in &self.sections { + if !map.collect_sources(max_sources, bytes_left, out) { + return false; + } + } + true + } +} + +/// Parse a source map. +pub fn parse(json: &[u8], limits: Limits) -> Result { + read(json, limits, true) +} + +/// Read a map's sources, their content, names and ignore list without decoding its mappings or +/// checking its section offsets, so a map [`parse`] refuses for those +/// ([`SourceMapError::BadMappings`] or [`SourceMapError::BadSections`]) can still list its sources. +/// The map has no lines or segments. A map whose mappings, as far as they decode, name a source +/// past the end of its `sources` list is still refused, since the list is known to be short, and so are a section +/// that has neither a map nor a URL, or both, and a `sections` value that isn't an array. +pub fn parse_sources(json: &[u8], limits: Limits) -> Result { + read(json, limits, false) +} + +fn read(json: &[u8], limits: Limits, mappings: bool) -> Result { + if json.len() > limits.max_bytes { + return Err(SourceMapError::TooLarge(json.len(), limits.max_bytes)); + } + // Some servers prefix maps with `)]}'` to defeat XSSI; the format allows skipping that line. + let body = if json.starts_with(b")]}") { json.iter().position(|b| *b == b'\n').and_then(|p| json.get(p + 1..)).unwrap_or(&[]) } else { json }; + let entries = Entries { left: Cell::new(limits.max_entries), exhausted: Cell::new(false) }; + let mut de = serde_json::Deserializer::from_slice(body); + let raw = RawMapSeed(&entries).deserialize(&mut de).and_then(|raw| de.end().map(|()| raw)); + let raw = match raw { + Ok(raw) => raw, + Err(_) if entries.exhausted.get() => return Err(SourceMapError::TooManyEntries(limits.max_entries)), + Err(e) => return Err(SourceMapError::BadJson(e.to_string())), + }; + let mut budget = limits.max_segments; + let mut line_budget = limits.max_lines; + from_raw(raw, 0, limits, &mut budget, &mut line_budget, mappings) +} + +/// `mappings`: decode the mappings and check section offsets, as [`parse`] does. +fn from_raw( + raw: Option, + depth: usize, + limits: Limits, + budget: &mut usize, + line_budget: &mut usize, + mappings: bool, +) -> Result { + if depth > limits.max_depth { + return Err(SourceMapError::TooDeep(limits.max_depth)); + } + let Some(raw) = raw.filter(|raw| raw.version == Some(3)) else { + return Err(SourceMapError::BadVersion); + }; + if let Some(sections) = raw.sections { + // The index has its own file name. Only its section maps describe original sources. + let mut map = SourceMap { file: raw.file, ..Default::default() }; + let sections = sections.ok_or(SourceMapError::BadSections)?; + for section in sections { + let offset = |n: Option| n.and_then(|n| u32::try_from(n).ok()).ok_or(SourceMapError::BadSections); + let (line, col) = section.offset.unwrap_or_default(); + let (line, col) = if mappings { (offset(line)?, offset(col)?) } else { (offset(line).unwrap_or(0), offset(col).unwrap_or(0)) }; + if mappings && map.sections.last().is_some_and(|(l, c, _)| (*l, *c) > (line, col)) { + return Err(SourceMapError::BadSections); + } + let (sub, external) = match (section.map, section.url) { + (Some(m), None) => (from_raw(m.map(|m| *m), depth + 1, limits, budget, line_budget, mappings)?, false), + // External maps are never fetched here. Retain their start so a lookup + // cannot leak through to the previous inline map, and count it so the + // sources it would add are reported unknown rather than absent. + (None, Some(true)) => (SourceMap::default(), true), + _ => return Err(SourceMapError::BadSections), + }; + if external { + map.external_sections = map.external_sections.saturating_add(1); + } + map.segments = map.segments.saturating_add(sub.segments); + map.sections.push((line, col, sub)); + } + return Ok(map); + } + let mut map = SourceMap { + file: raw.file, + source_root: raw.source_root, + sources: raw.sources, + sources_content: raw.sources_content, + names: raw.names.into_iter().map(Option::unwrap_or_default).collect(), + ignore_list: raw + .ignore_list + .or(raw.google_ignore_list) + .unwrap_or_default() + .into_iter() + .filter_map(|n| n.and_then(|n| u32::try_from(n).ok())) + .collect(), + ..Default::default() + }; + let m = raw.mappings.as_deref().unwrap_or_default(); + if mappings { + decode_mappings(m, &mut map, budget, line_budget, limits.max_lines)?; + } else if let Some(count) = past_sources(m, map.sources.len()) { + return Err(SourceMapError::BadMappings(count)); + } + Ok(map) +} + +/// The entry budget shared by every list in a map and its sections while the JSON is read. +struct Entries { + left: Cell, + exhausted: Cell, +} + +impl Entries { + fn take(&self) -> Result<(), E> { + let Some(left) = self.left.get().checked_sub(1) else { + self.exhausted.set(true); + return Err(E::custom("too many source-map entries")); + }; + self.left.set(left); + Ok(()) + } + + /// Return the slots of entries that won't be kept after all. Callers drop those entries + /// first, so a replaced list and its replacement are never held together. + fn give_back(&self, n: usize) { + self.left.set(self.left.get().saturating_add(n)); + } +} + +/// The fields a map is built from, read without keeping anything else. Any JSON value is +/// accepted where the format expects another kind; it reads as absent, as before. +#[derive(Default)] +struct RawMap { + version: Option, + file: Option, + source_root: Option, + sources: Vec>, + sources_content: Vec>, + names: Vec>, + /// Present even when it isn't an array, so `x_google_ignoreList` stays unused then. + ignore_list: Option>>, + google_ignore_list: Option>>, + mappings: Option, + /// `Some(None)`: present but not an array. + sections: Option>>, +} + +impl RawMap { + /// The entry slots this map and its sections hold. + fn held(&self) -> usize { + let lists = |l: &Option>>| l.as_ref().map_or(0, Vec::len); + self.sources.len() + + self.sources_content.len() + + self.names.len() + + lists(&self.ignore_list) + + lists(&self.google_ignore_list) + + held_by(&self.sections) + } +} + +fn held_by(sections: &Option>>) -> usize { + sections.iter().flatten().flatten().map(|s| 1 + s.map.iter().flatten().map(|m| m.held()).sum::()).sum() +} + +#[derive(Default)] +struct RawSection { + offset: Option<(Option, Option)>, + /// `Some(None)`: present but not an object. Boxed, so a section that isn't a map stays small. + map: Option>>, + /// Whether the URL is a string. + url: Option, +} + +/// Visitor methods that read any other kind of value, without keeping it, as the default. +macro_rules! skip { + ($de:lifetime; bool) => { fn visit_bool(self, _: bool) -> Result { Ok(Default::default()) } }; + ($de:lifetime; i64) => { fn visit_i64(self, _: i64) -> Result { Ok(Default::default()) } }; + ($de:lifetime; u64) => { fn visit_u64(self, _: u64) -> Result { Ok(Default::default()) } }; + ($de:lifetime; f64) => { fn visit_f64(self, _: f64) -> Result { Ok(Default::default()) } }; + ($de:lifetime; str) => { fn visit_str(self, _: &str) -> Result { Ok(Default::default()) } }; + ($de:lifetime; unit) => { fn visit_unit(self) -> Result { Ok(Default::default()) } }; + ($de:lifetime; seq) => { + fn visit_seq>(self, mut seq: A) -> Result { + while seq.next_element_seed(Skip)?.is_some() {} + Ok(Default::default()) + } + }; + ($de:lifetime; map) => { + fn visit_map>(self, mut map: A) -> Result { + while map.next_key_seed(Skip)?.is_some() { + map.next_value_seed(Skip)?; + } + Ok(Default::default()) + } + }; + ($de:lifetime; $($kind:ident),+) => { $(skip!($de; $kind);)+ }; +} + +/// Read a value the map doesn't keep, with the checks a full read makes (string escapes and +/// UTF-8, number range, nesting depth), so skipping it can't let a broken map through. +#[derive(Clone, Copy)] +struct Skip; + +impl<'de> DeserializeSeed<'de> for Skip { + type Value = (); + fn deserialize>(self, d: D) -> Result<(), D::Error> { + d.deserialize_any(self) + } +} + +impl<'de> Visitor<'de> for Skip { + type Value = (); + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("any JSON value") + } + skip!('de; bool, i64, u64, f64, str, unit, seq, map); +} + +/// A string, or `None` for any other value. +#[derive(Clone, Copy)] +struct MaybeStr; + +impl<'de> Visitor<'de> for MaybeStr { + type Value = Option; + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("any JSON value") + } + fn visit_str(self, s: &str) -> Result { + Ok(Some(s.to_owned())) + } + fn visit_string(self, s: String) -> Result { + Ok(Some(s)) + } + skip!('de; bool, i64, u64, f64, unit, seq, map); +} + +/// A non-negative integer, or `None` for any other value. +#[derive(Clone, Copy)] +struct MaybeU64; + +impl<'de> Visitor<'de> for MaybeU64 { + type Value = Option; + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("any JSON value") + } + fn visit_u64(self, n: u64) -> Result { + Ok(Some(n)) + } + skip!('de; bool, i64, f64, str, unit, seq, map); +} + +/// One value read through `V`, after taking its slot from the entry budget. +struct Entry<'a, V>(&'a Entries, V); + +impl<'de, V: Visitor<'de>> DeserializeSeed<'de> for Entry<'_, V> { + type Value = V::Value; + fn deserialize>(self, d: D) -> Result { + self.0.take()?; + d.deserialize_any(self.1) + } +} + +/// A list whose entries count against the budget; any other value reads as an empty list. +struct EntryList<'a, V>(&'a Entries, V); + +impl<'de, V: Visitor<'de> + Copy> Visitor<'de> for EntryList<'_, V> { + type Value = Vec; + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("any JSON value") + } + fn visit_seq>(self, mut seq: A) -> Result { + let mut out = Vec::new(); + while let Some(value) = seq.next_element_seed(Entry(self.0, self.1))? { + out.push(value); + } + Ok(out) + } + skip!('de; bool, i64, u64, f64, str, unit, map); +} + +impl<'de, V: Visitor<'de> + Copy> DeserializeSeed<'de> for EntryList<'_, V> { + type Value = Vec; + fn deserialize>(self, d: D) -> Result { + d.deserialize_any(self) + } +} + +/// A map object, or `None` for any other value. +#[derive(Clone, Copy)] +struct RawMapSeed<'a>(&'a Entries); + +impl<'de> Visitor<'de> for RawMapSeed<'_> { + type Value = Option; + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("any JSON value") + } + fn visit_map>(self, mut fields: A) -> Result { + let mut raw = RawMap::default(); + // A repeated key replaces the earlier value, as it did when the whole JSON was read. + while let Some(key) = fields.next_key::()? { + match key { + Field::Version => raw.version = fields.next_value_seed(Any(MaybeU64))?, + Field::File => raw.file = fields.next_value_seed(Any(MaybeStr))?, + Field::SourceRoot => raw.source_root = fields.next_value_seed(Any(MaybeStr))?, + Field::Sources => { + self.0.give_back(std::mem::take(&mut raw.sources).len()); + raw.sources = fields.next_value_seed(EntryList(self.0, MaybeStr))?; + } + Field::SourcesContent => { + self.0.give_back(std::mem::take(&mut raw.sources_content).len()); + raw.sources_content = fields.next_value_seed(EntryList(self.0, MaybeStr))?; + } + Field::Names => { + self.0.give_back(std::mem::take(&mut raw.names).len()); + raw.names = fields.next_value_seed(EntryList(self.0, MaybeStr))?; + } + // `ignoreList` wins over `x_google_ignoreList` wherever either appears. + Field::IgnoreList => { + let unused = raw.ignore_list.take().into_iter().chain(raw.google_ignore_list.take()); + self.0.give_back(unused.map(|l| l.len()).sum()); + raw.ignore_list = Some(fields.next_value_seed(EntryList(self.0, MaybeU64))?); + } + Field::GoogleIgnoreList if raw.ignore_list.is_some() => fields.next_value_seed(Skip)?, + Field::GoogleIgnoreList => { + self.0.give_back(raw.google_ignore_list.take().map_or(0, |l| l.len())); + raw.google_ignore_list = Some(fields.next_value_seed(EntryList(self.0, MaybeU64))?); + } + Field::Mappings => raw.mappings = fields.next_value_seed(Any(MaybeStr))?, + Field::Sections => { + self.0.give_back(held_by(&raw.sections.take())); + raw.sections = Some(fields.next_value_seed(Any(Sections(self.0)))?); + } + Field::Other => fields.next_value_seed(Skip)?, + } + } + Ok(Some(raw)) + } + skip!('de; bool, i64, u64, f64, str, unit, seq); +} + +impl<'de> DeserializeSeed<'de> for RawMapSeed<'_> { + type Value = Option; + fn deserialize>(self, d: D) -> Result { + d.deserialize_any(self) + } +} + +/// Read any JSON value through `V`. +struct Any(V); + +impl<'de, V: Visitor<'de>> DeserializeSeed<'de> for Any { + type Value = V::Value; + fn deserialize>(self, d: D) -> Result { + d.deserialize_any(self.0) + } +} + +#[derive(serde::Deserialize)] +#[serde(field_identifier, rename_all = "camelCase")] +enum Field { + Version, + File, + SourceRoot, + Sources, + SourcesContent, + Names, + IgnoreList, + #[serde(rename = "x_google_ignoreList")] + GoogleIgnoreList, + Mappings, + Sections, + #[serde(other)] + Other, +} + +/// The sections array, or `None` for any other value. +struct Sections<'a>(&'a Entries); + +impl<'de> Visitor<'de> for Sections<'_> { + type Value = Option>; + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("any JSON value") + } + fn visit_seq>(self, mut seq: A) -> Result { + let mut out = Vec::new(); + while let Some(section) = seq.next_element_seed(Entry(self.0, Section(self.0)))? { + out.push(section); + } + Ok(Some(out)) + } + skip!('de; bool, i64, u64, f64, str, unit, map); +} + +/// One section; any value that isn't an object reads as a section without an offset. +struct Section<'a>(&'a Entries); + +impl<'de> Visitor<'de> for Section<'_> { + type Value = RawSection; + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("any JSON value") + } + fn visit_map>(self, mut fields: A) -> Result { + let mut section = RawSection::default(); + while let Some(key) = fields.next_key::()? { + match key { + SectionField::Offset => section.offset = Some(fields.next_value_seed(Any(Offset))?), + SectionField::Map => { + self.0.give_back(section.map.take().flatten().map_or(0, |m| m.held())); + section.map = Some(fields.next_value_seed(RawMapSeed(self.0))?.map(Box::new)); + } + SectionField::Url => section.url = Some(fields.next_value_seed(Any(MaybeStr))?.is_some()), + SectionField::Other => fields.next_value_seed(Skip)?, + } + } + Ok(section) + } + skip!('de; bool, i64, u64, f64, str, unit, seq); +} + +#[derive(serde::Deserialize)] +#[serde(field_identifier, rename_all = "lowercase")] +enum SectionField { + Offset, + Map, + Url, + #[serde(other)] + Other, +} + +/// A section offset's line and column; any value that isn't an object has neither. +struct Offset; + +impl<'de> Visitor<'de> for Offset { + type Value = (Option, Option); + fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str("any JSON value") + } + fn visit_map>(self, mut fields: A) -> Result { + let (mut line, mut column) = (None, None); + while let Some(key) = fields.next_key::()? { + match key { + OffsetField::Line => line = fields.next_value_seed(Any(MaybeU64))?, + OffsetField::Column => column = fields.next_value_seed(Any(MaybeU64))?, + OffsetField::Other => fields.next_value_seed(Skip)?, + } + } + Ok((line, column)) + } + skip!('de; bool, i64, u64, f64, str, unit, seq); +} + +#[derive(serde::Deserialize)] +#[serde(field_identifier, rename_all = "lowercase")] +enum OffsetField { + Line, + Column, + #[serde(other)] + Other, +} + +fn b64(c: u8) -> Option { + Some(match c { + b'A'..=b'Z' => i64::from(c - b'A'), + b'a'..=b'z' => i64::from(c - b'a') + 26, + b'0'..=b'9' => i64::from(c - b'0') + 52, + b'+' => 62, + b'/' => 63, + _ => return None, + }) +} + +/// Decode one VLQ field from `bytes` at `*pos`. +fn vlq(bytes: &[u8], pos: &mut usize) -> Option { + let mut result = 0u32; + let mut shift = 0u32; + loop { + let d = b64(*bytes.get(*pos)?)? as u32; + *pos += 1; + let bits = d & 31; + if bits != 0 { + if bits > u32::MAX.checked_shr(shift)? { + return None; + } + result |= bits << shift; + } + if d & 32 == 0 { + break; + } + // Redundant zero groups do not change the value. + shift = shift.saturating_add(5); + } + let magnitude = i64::from(result >> 1); + Some(match (result & 1, magnitude) { + (1, 0) => i64::from(i32::MIN), + (1, n) => -n, + _ => magnitude, + }) +} + +/// Decode mapping deltas, rejecting fields that cannot name a declared source or symbol. +fn decode_mappings(m: &str, map: &mut SourceMap, budget: &mut usize, line_budget: &mut usize, line_limit: usize) -> Result<(), SourceMapError> { + let bytes = m.as_bytes(); + let (mut src, mut ol, mut oc, mut name) = (0i64, 0i64, 0i64, 0i64); + let mut pos = 0usize; + let mut line: Vec = Vec::new(); + let mut col = 0i64; + let mut count = 0usize; + let to_u32 = |v: i64, count: usize| u32::try_from(v).map_err(|_| SourceMapError::BadMappings(count)); + while pos <= bytes.len() { + match bytes.get(pos) { + None | Some(b';') => { + if *line_budget == 0 { + return Err(SourceMapError::TooManyLines(line_limit)); + } + *line_budget -= 1; + line.sort_by_key(|s| s.generated_column); + map.lines.try_reserve(1).map_err(|_| SourceMapError::LineAllocationFailed)?; + map.lines.push(std::mem::take(&mut line)); + col = 0; + pos += 1; + continue; + } + Some(b',') => { + pos += 1; + continue; + } + _ => {} + } + if *budget == 0 { + return Err(SourceMapError::TooManySegments(count)); + } + *budget -= 1; + count += 1; + col += vlq(bytes, &mut pos).ok_or(SourceMapError::BadMappings(count))?; + let mut seg = Segment { generated_column: to_u32(col, count)?, original: None }; + if !matches!(bytes.get(pos), None | Some(b',') | Some(b';')) { + src += vlq(bytes, &mut pos).ok_or(SourceMapError::BadMappings(count))?; + ol += vlq(bytes, &mut pos).ok_or(SourceMapError::BadMappings(count))?; + oc += vlq(bytes, &mut pos).ok_or(SourceMapError::BadMappings(count))?; + let mut n = None; + if !matches!(bytes.get(pos), None | Some(b',') | Some(b';')) { + name += vlq(bytes, &mut pos).ok_or(SourceMapError::BadMappings(count))?; + n = Some(to_u32(name, count)?); + } + let source = to_u32(src, count)?; + if source as usize >= map.sources.len() || n.is_some_and(|index| index as usize >= map.names.len()) { + return Err(SourceMapError::BadMappings(count)); + } + seg.original = Some((source, to_u32(ol, count)?, to_u32(oc, count)?, n)); + } + if !matches!(bytes.get(pos), None | Some(b',') | Some(b';')) { + return Err(SourceMapError::BadMappings(count)); + } + line.push(seg); + } + map.segments = count; + Ok(()) +} + +/// The segment, counted as [`decode_mappings`] does, that names a source past the end of a list +/// of `sources`. Mappings that can't be decoded end the search: past them nothing is known. +fn past_sources(m: &str, sources: usize) -> Option { + let bytes = m.as_bytes(); + let (mut pos, mut src, mut count) = (0usize, 0i64, 0usize); + let at_end = |pos: usize| matches!(bytes.get(pos), None | Some(b',') | Some(b';')); + while pos < bytes.len() { + if matches!(bytes[pos], b',' | b';') { + pos += 1; + continue; + } + count += 1; + vlq(bytes, &mut pos)?; + if at_end(pos) { + continue; + } + src += vlq(bytes, &mut pos)?; + // A negative index is wrong too, but the deltas after it still decode. + if usize::try_from(src).is_ok_and(|src| src >= sources) { + return Some(count); + } + while !at_end(pos) { + vlq(bytes, &mut pos)?; + } + } + None +} + +#[cfg(test)] +mod tests; diff --git a/crates/sourcemap/src/tests.rs b/crates/sourcemap/src/tests.rs new file mode 100644 index 000000000..414f3b64c --- /dev/null +++ b/crates/sourcemap/src/tests.rs @@ -0,0 +1,122 @@ +use super::*; +#[test] +fn mapping_indices_must_refer_to_declared_entries() { + for mappings in ["ACAA", "AAAAC"] { + let json = serde_json::json!({"version":3,"sources":["source.js"],"names":["name"],"mappings":mappings}); + assert!(matches!(parse(json.to_string().as_bytes(), Limits::default()), Err(SourceMapError::BadMappings(1)))); + } + let valid = br#"{"version":3,"sources":[null],"names":["name"],"mappings":"AAAAA"}"#; + assert!(parse(valid, Limits::default()).is_ok()); +} + +#[test] +fn lookup_keeps_coincident_mappings_and_stops_at_an_unmapped_segment() { + let map = parse(br#"{"version":3,"sources":["a","b"],"mappings":"EAAA,ACAA,E"}"#, Limits::default()).unwrap(); + assert!(map.original_positions(0, 1).is_empty()); + let matches = map.original_positions(0, 3); + assert_eq!(matches.iter().map(|p| p.source_index).collect::>(), [0, 1]); + assert!(map.original_positions(0, 4).is_empty()); +} + +#[test] +fn relative_sources_keep_the_url_root_delimiter() { + for (root, expected) in [ + ("webpack:///", "webpack:///src/main.js"), + ("file:///", "file:///src/main.js"), + ("https://host/scripts/", "https://host/scripts/src/main.js"), + ("src", "src/src/main.js"), + ] { + let json = serde_json::json!({"version":3,"sourceRoot":root,"sources":["src/main.js"],"mappings":"AAAA"}); + let map = parse(json.to_string().as_bytes(), Limits::default()).unwrap(); + assert_eq!(map.all_sources(), [expected]); + assert_eq!(map.original_positions(0, 0)[0].source.as_deref(), Some(expected)); + } +} + +#[test] +fn decodes_and_looks_up() { + // Generated "a;b" from two lines of src.js. + let m = br#"{"version":3,"sources":["src.js"],"names":["foo"],"mappings":"AAAAA,EAAE;AACA","sourcesContent":["x\ny"]}"#; + let map = parse(m, Limits::default()).unwrap(); + assert_eq!(map.lines.len(), 2); + let p = map.original_positions(0, 1); + assert_eq!(p[0].source.as_deref(), Some("src.js")); + assert_eq!(p[0].name.as_deref(), Some("foo")); + assert_eq!((p[0].line, p[0].column), (0, 0)); + let q = map.original_positions(0, 2); + assert_eq!(q[0].column, 2); + let r = map.original_positions(1, 0); + assert_eq!(r[0].line, 1); + assert!(map.original_positions(5, 0).is_empty()); + assert_eq!(map.source_content(0), Some("x\ny")); +} + +#[test] +fn indexed_maps() { + let m = br#"{"version":3,"sections":[{"offset":{"line":0,"column":0},"map":{"version":3,"sources":["a.js"],"names":[],"mappings":"AAAA"}},{"offset":{"line":1,"column":0},"map":{"version":3,"sources":["b.js"],"names":[],"mappings":"AAAA"}}]}"#; + let map = parse(m, Limits::default()).unwrap(); + assert_eq!(map.original_positions(1, 3)[0].source.as_deref(), Some("b.js")); + assert_eq!(map.all_sources(), vec!["a.js", "b.js"]); +} + +#[test] +fn validates_index_map_shape_and_offsets() { + let inline = serde_json::json!({"version":3,"mappings":"A"}); + for sections in [ + serde_json::json!({}), + serde_json::json!([{"map":inline}]), + serde_json::json!([{"offset":{"line":-1,"column":0},"map":inline}]), + serde_json::json!([{"offset":{"line":0,"column":4294967296_u64},"map":inline}]), + serde_json::json!([{"offset":{"line":0,"column":0}}]), + serde_json::json!([{"offset":{"line":0,"column":0},"map":inline,"url":"later.map"}]), + serde_json::json!([{"offset":{"line":0,"column":2},"map":inline},{"offset":{"line":0,"column":1},"map":inline}]), + ] { + let json = serde_json::json!({"version":3,"sections":sections}).to_string(); + assert_eq!(parse(json.as_bytes(), Limits::default()), Err(SourceMapError::BadSections)); + } +} + +#[test] +fn an_external_section_ends_the_previous_inline_mapping() { + let inline = serde_json::json!({"version":3,"sources":["first.js"],"mappings":"AAAA"}); + let sections = serde_json::json!([ + {"offset":{"line":0,"column":0},"map":inline}, + {"offset":{"line":0,"column":4},"url":"external.map"}, + {"offset":{"line":1,"column":0},"map":inline} + ]); + let json = serde_json::json!({"version":3,"sections":sections}).to_string(); + let map = parse(json.as_bytes(), Limits::default()).unwrap(); + assert_eq!(map.original_positions(0, 3)[0].source.as_deref(), Some("first.js")); + assert!(map.original_positions(0, 4).is_empty()); + assert_eq!(map.original_positions(1, 0)[0].generated_column, 0); +} + +#[test] +fn vlq_signed_extremes_and_overflow() { + for (field, value) in [ + ("B", Some(i64::from(i32::MIN))), + ("+/////D", Some(i64::from(i32::MAX))), + ("//////D", Some(-i64::from(i32::MAX))), + ("ggggggE", None), + ("ggggggggggA", Some(0)), + ] { + assert_eq!(vlq(field.as_bytes(), &mut 0), value, "{field}"); + } + let map = parse(br#"{"version":3,"mappings":"+/////D,C,B"}"#, Limits::default()).unwrap(); + assert_eq!(map.lines[0].iter().map(|s| s.generated_column).collect::>(), [0, 2147483647, 2147483648]); + assert!(parse(br#"{"version":3,"mappings":"B"}"#, Limits::default()).is_err()); +} + +#[test] +fn hostile_mappings_do_not_panic() { + for m in ["", ";;;", "A", "gggggggggggggggggggggggggg", "AAAA,", "////", "AAAAAAAAAAAAAAA", "D", "AD,AA", "+", "!!"] { + let j = format!(r#"{{"version":3,"sources":[],"names":[],"mappings":"{m}"}}"#); + if let Ok(map) = parse(j.as_bytes(), Limits::default()) { + let _ = map.original_positions(0, 0); + let _ = map.original_positions(u32::MAX, u32::MAX); + } + } + let tight = Limits { max_segments: 1, ..Limits::default() }; + assert!(parse(br#"{"version":3,"sources":[],"names":[],"mappings":"A,A"}"#, tight).is_err()); + assert!(parse(b"[]", Limits::default()).is_err()); +} diff --git a/crates/sourcemap/tests/entry_budgets.rs b/crates/sourcemap/tests/entry_budgets.rs new file mode 100644 index 000000000..65ae533ba --- /dev/null +++ b/crates/sourcemap/tests/entry_budgets.rs @@ -0,0 +1,145 @@ +use craft_sourcemap::{Limits, SourceMap, SourceMapError, parse}; +use serde_json::json; + +fn with(max_entries: usize) -> Limits { + Limits { max_entries, ..Limits::default() } +} + +fn decode(value: serde_json::Value, max_entries: usize) -> Result { + parse(&serde_json::to_vec(&value).unwrap(), with(max_entries)) +} + +#[test] +fn entries_up_to_the_limit_are_kept() { + let value = json!({"version":3,"sources":["a.js","b.js"],"sourcesContent":["a();",null],"names":["x"],"mappings":"AAAAA,CCAA"}); + let map = decode(value.clone(), 5).unwrap(); + assert_eq!(map.sources, vec![Some("a.js".to_string()), Some("b.js".to_string())]); + assert_eq!(map.sources_content, vec![Some("a();".to_string()), None]); + assert_eq!(map.names, vec!["x".to_string()]); + assert_eq!(decode(value, 4), Err(SourceMapError::TooManyEntries(4))); +} + +#[test] +fn every_kind_of_entry_counts_once() { + for value in [ + json!({"version":3,"sources":["a","b","c"],"mappings":""}), + json!({"version":3,"sourcesContent":["a","b","c"],"mappings":""}), + json!({"version":3,"names":["a","b","c"],"mappings":""}), + json!({"version":3,"ignoreList":[0,1,2],"mappings":""}), + json!({"version":3,"x_google_ignoreList":[0,1,2],"mappings":""}), + // Entries that are not strings still take a slot each. + json!({"version":3,"sources":[null, 1, {"a":[1,2,3]}],"mappings":""}), + ] { + assert_eq!(decode(value.clone(), 2), Err(SourceMapError::TooManyEntries(2)), "{value}"); + assert!(decode(value.clone(), 3).is_ok(), "{value}"); + } +} + +#[test] +fn entries_a_repeated_key_replaces_give_their_slots_back() { + let json = br#"{"version":3,"mappings":"","sources":["a","b"],"sources":["c"],"names":["x"],"names":[]}"#; + let map = parse(json, with(2)).unwrap(); + assert_eq!(map.sources, vec![Some("c".to_string())]); + assert!(map.names.is_empty()); + // `ignoreList` is kept over `x_google_ignoreList`, before or after it. + let first = br#"{"version":3,"mappings":"","x_google_ignoreList":[0,0],"ignoreList":[0]}"#; + assert_eq!(parse(first, with(2)).unwrap().ignore_list, vec![0]); + let after = br#"{"version":3,"mappings":"","ignoreList":[0],"x_google_ignoreList":[0,0,0]}"#; + assert_eq!(parse(after, with(1)).unwrap().ignore_list, vec![0]); + // A replaced sections array gives back its sections and what their maps held: without that, + // the second array would need slots four and five. + let sections = br#"{"version":3,"sections":[{"offset":{"line":0,"column":0},"map":{"version":3,"sources":["a"],"mappings":""}},{"offset":{"line":1,"column":0},"url":"b.map"}],"sections":[{"offset":{"line":0,"column":0},"url":"c.map"}]}"#; + assert_eq!(parse(sections, with(3)).unwrap().sections.len(), 1); + // So does a section's replaced map: one section, then one source instead of two. + let map = br#"{"version":3,"sections":[{"offset":{"line":0,"column":0},"map":{"version":3,"sources":["a","b"],"mappings":""},"map":{"version":3,"sources":["c"],"mappings":""}}]}"#; + assert_eq!(parse(map, with(3)).unwrap().sections[0].2.sources, vec![Some("c".to_string())]); +} + +#[test] +fn sections_and_their_maps_share_the_budget() { + let section = |line: u32, name: &str| json!({"offset":{"line":line,"column":0},"map":{"version":3,"sources":[name],"mappings":"AAAA"}}); + let value = json!({"version":3,"sections":[section(0, "a.js"), section(1, "b.js")]}); + // Two sections and one source in each. + assert_eq!(decode(value.clone(), 4).unwrap().sections.len(), 2); + assert_eq!(decode(value, 3), Err(SourceMapError::TooManyEntries(3))); + // Sections without a map count too, so a long list of them is refused early. + let external = json!({"offset":{"line":0,"column":0},"url":"a.map"}); + let value = json!({"version":3,"sections":[external.clone(), external.clone(), external]}); + assert_eq!(decode(value.clone(), 3).unwrap().sections.len(), 3); + assert_eq!(decode(value, 2), Err(SourceMapError::TooManyEntries(2))); + let not_sections = br#"{"version":3,"sections":[0,0,0]}"#; + assert_eq!(parse(not_sections, with(2)), Err(SourceMapError::TooManyEntries(2))); + assert_eq!(parse(not_sections, with(3)), Err(SourceMapError::BadSections)); +} + +#[test] +fn reading_stops_at_the_limit() { + // Everything after the entry that goes over the limit is never read: the broken tail would + // otherwise be reported as bad JSON. + let json = br#"{"version":3,"sources":["a","b","c","d" this is not JSON"#; + assert_eq!(parse(json, with(2)), Err(SourceMapError::TooManyEntries(2))); + assert!(matches!(parse(json, Limits::default()), Err(SourceMapError::BadJson(_)))); +} + +#[test] +fn unknown_fields_are_skipped_without_counting() { + let value = json!({"version":3,"x_vendor":[1,2,3,4,5,6,7,8],"sources":["a.js"],"mappings":"AAAA"}); + assert_eq!(decode(value, 1).unwrap().sources.len(), 1); +} + +#[test] +fn values_the_map_does_not_keep_are_still_checked() { + let deep = format!("{{\"version\":3,\"mappings\":\"\",\"x\":{}{}}}", "[".repeat(200), "]".repeat(200)); + for json in [ + br#"{"version":3,"mappings":"","x":"\ud800"}"#.as_slice(), + br#"{"version":3,"mappings":"","sources":{"a":"\udc00"}}"#, + br#"{"version":3,"mappings":"","x":1e400}"#, + br#"{"version":3,"mappings":"","sources":[{"a":1e400}]}"#, + b"{\"version\":3,\"mappings\":\"\",\"x\":\"\xff\"}", + br#"{"version":3,"mappings":"","x":[1,]}"#, + deep.as_bytes(), + ] { + let got = parse(json, Limits::default()); + let full = serde_json::from_slice::(json).unwrap_err().to_string(); + assert_eq!(got, Err(SourceMapError::BadJson(full)), "{}", String::from_utf8_lossy(json)); + } +} + +#[test] +fn a_hostile_map_of_empty_names_is_refused_by_default() { + // About 21 million empty strings fit in the 64 MiB a map may have. + let limit = Limits::default().max_entries; + let names = vec!["\"\""; limit + 1].join(","); + let json = format!("{{\"version\":3,\"mappings\":\"\",\"names\":[{names}]}}").into_bytes(); + assert_eq!(parse(&json, Limits::default()), Err(SourceMapError::TooManyEntries(limit))); + // The same map is well formed: with room for every name it reads. + assert_eq!(parse(&json, with(limit + 1)).unwrap().names.len(), limit + 1); +} + +#[test] +fn sources_are_read_from_a_map_with_broken_mappings_or_offsets() { + use craft_sourcemap::parse_sources; + let bad = br#"{"version":3,"sourceRoot":"src","sources":["a.js"],"sourcesContent":["a();"],"names":["x"],"mappings":"!!"}"#; + assert_eq!(parse(bad, Limits::default()), Err(SourceMapError::BadMappings(1))); + let map = parse_sources(bad, Limits::default()).unwrap(); + assert_eq!((map.source_path(0).as_deref(), map.sources_content.len(), map.names.len(), map.lines.len()), (Some("src/a.js"), 1, 1, 0)); + let unordered = br#"{"version":3,"sections":[{"offset":{"line":2,"column":0},"map":{"version":3,"sources":["a.js"],"mappings":"AAAA"}},{"offset":{"line":-1,"column":0},"map":{"version":3,"sources":["b.js"],"mappings":""}},{"offset":{"line":0,"column":0},"map":{"version":3,"sources":["c.js"],"mappings":"!"}}]}"#; + assert_eq!(parse(unordered, Limits::default()), Err(SourceMapError::BadSections)); + let map = parse_sources(unordered, Limits::default()).unwrap(); + assert_eq!(map.source_inventory(Default::default()).sources, vec!["a.js".to_string(), "b.js".to_string(), "c.js".to_string()]); + // Past mappings that can't be decoded, the indices aren't known. + assert_eq!(parse_sources(br#"{"version":3,"sources":["a"],"mappings":"AAAA,!,ACAA"}"#, Limits::default()).unwrap().sources.len(), 1); + // What a section holds must still be known, and the version and limits still apply. + for (json, limit, err) in [ + (br#"{"version":3,"sections":[{"offset":{"line":0,"column":0}}]}"#.as_slice(), 9, SourceMapError::BadSections), + (br#"{"version":3,"sections":{}}"#, 9, SourceMapError::BadSections), + (br#"{"version":3,"sections":[{"offset":{"line":0,"column":0},"map":{"version":2,"sources":["a"]}}]}"#, 9, SourceMapError::BadVersion), + (br#"{"version":3,"sources":["a","b"],"mappings":"!"}"#, 1, SourceMapError::TooManyEntries(1)), + // Mappings that name a source past the end of the list show it is short. + (br#"{"version":3,"mappings":"AAAA"}"#, 9, SourceMapError::BadMappings(1)), + (br#"{"version":3,"sources":["a"],"mappings":"AAAA;ACAA,!"}"#, 9, SourceMapError::BadMappings(2)), + (br#"{"version":3,"sources":["a"],"mappings":"AAAA,ADAA,AEAA"}"#, 9, SourceMapError::BadMappings(3)), + ] { + assert_eq!(parse_sources(json, with(limit)), Err(err), "{}", String::from_utf8_lossy(json)); + } +} diff --git a/crates/sourcemap/tests/external_sections.rs b/crates/sourcemap/tests/external_sections.rs new file mode 100644 index 000000000..643cd52b1 --- /dev/null +++ b/crates/sourcemap/tests/external_sections.rs @@ -0,0 +1,84 @@ +//! An index-map section that points at an external map (`url`) adds no sources here, but its +//! sources are unknown rather than absent: the inventory says how many such sections it saw. + +use craft_sourcemap::{InventoryLimits, Limits, SourceMap, parse, parse_sources}; +use serde_json::{Value, json}; + +fn decode(value: Value) -> SourceMap { + parse(&serde_json::to_vec(&value).unwrap(), Limits::default()).unwrap() +} + +/// An external section adds no sources but is counted, so the inventory can say part of the map +/// was never read. +#[test] +fn an_external_section_is_counted_and_adds_no_sources() { + let map = decode(json!({ + "version": 3, + "sections": [ + {"offset": {"line": 0, "column": 0}, "map": {"version": 3, "sources": ["a.js"], "mappings": ""}}, + {"offset": {"line": 10, "column": 0}, "url": "vendor.js.map"} + ] + })); + assert_eq!(map.all_sources(), ["a.js"]); + assert_eq!(map.external_sections, 1); + let inventory = map.source_inventory(InventoryLimits::default()); + assert_eq!(inventory.sources, ["a.js"]); + assert!(!inventory.truncated); + assert_eq!(inventory.external_sections, 1); +} + +/// The count reaches an external section inside a nested index map. +#[test] +fn external_sections_are_counted_through_nested_index_maps() { + let map = decode(json!({ + "version": 3, + "sections": [ + {"offset": {"line": 0, "column": 0}, "map": {"version": 3, "sections": [ + {"offset": {"line": 0, "column": 0}, "map": {"version": 3, "sources": ["a.js"], "mappings": ""}}, + {"offset": {"line": 4, "column": 0}, "url": "inner.js.map"} + ]}}, + {"offset": {"line": 20, "column": 0}, "url": "outer.js.map"} + ] + })); + assert_eq!(map.all_sources(), ["a.js"]); + assert_eq!(map.source_inventory(InventoryLimits::default()).external_sections, 2); +} + +/// A map with only inline sections reports nothing unread. +#[test] +fn an_inline_only_map_counts_no_external_sections() { + let map = decode(json!({ + "version": 3, + "sections": [{"offset": {"line": 0, "column": 0}, "map": {"version": 3, "sources": ["a.js"], "mappings": ""}}] + })); + assert_eq!(map.source_inventory(InventoryLimits::default()).external_sections, 0); +} + +/// A map read for its sources alone (its mappings are refused) still says an external section +/// left part of the list unread. +#[test] +fn a_sources_only_read_still_counts_external_sections() { + let bytes = serde_json::to_vec(&json!({ + "version": 3, + "sections": [ + {"offset": {"line": 0, "column": 0}, "map": {"version": 3, "sources": ["a.js"]}}, + {"offset": {"line": 8, "column": 0}, "url": "vendor.js.map"} + ] + })) + .unwrap(); + let map = parse_sources(&bytes, Limits::default()).unwrap(); + assert_eq!(map.source_inventory(InventoryLimits::default()).external_sections, 1); +} + +/// The count spends no inventory budget: a map with many external sections still retains its +/// inline sources in full. +#[test] +fn many_external_sections_do_not_spend_the_source_budget() { + let mut sections: Vec = (0..1_000).map(|i| json!({"offset": {"line": i, "column": 0}, "url": format!("{i}.map")})).collect(); + sections.push(json!({"offset": {"line": 1_000, "column": 0}, "map": {"version": 3, "sources": ["a.js", "b.js"], "mappings": ""}})); + let map = decode(json!({"version": 3, "sections": sections})); + let inventory = map.source_inventory(InventoryLimits { max_sources: 2, max_bytes: 16 }); + assert_eq!(inventory.sources, ["a.js", "b.js"]); + assert!(!inventory.truncated); + assert_eq!(inventory.external_sections, 1_000); +} diff --git a/crates/sourcemap/tests/inventory_budgets.rs b/crates/sourcemap/tests/inventory_budgets.rs new file mode 100644 index 000000000..7491108e4 --- /dev/null +++ b/crates/sourcemap/tests/inventory_budgets.rs @@ -0,0 +1,104 @@ +use craft_sourcemap::{InventoryLimits, Limits, SourceMap, parse}; +use serde_json::{Value, json}; + +fn decode(value: Value) -> SourceMap { + parse(&serde_json::to_vec(&value).unwrap(), Limits::default()).unwrap() +} + +#[test] +fn all_sources_bounds_expanded_path_bytes_before_collecting() { + let root = "r".repeat(2 * 1024 * 1024); + let bytes = serde_json::to_vec(&json!({ + "version": 3, "sourceRoot": root, "sources": vec!["/x.js"; 40], "mappings": "" + })) + .unwrap(); + let map = parse(&bytes, Limits::default()).unwrap(); + let sources = map.all_sources(); + assert!(sources.iter().map(String::len).sum::() <= 64 * 1024 * 1024); + assert_eq!(sources.len(), 31); + assert!(sources.iter().all(|source| source.ends_with("//x.js"))); +} + +#[test] +fn repeated_large_roots_stop_before_relative_and_absolute_looking_paths_expand() { + for source in ["x.js", "/x.js", "https://example.test/x.js"] { + let bytes = serde_json::to_vec(&json!({ + "version": 3, "sourceRoot": "r".repeat(1024 * 1024), + "sources": vec![source; 10_000], "mappings": "" + })) + .unwrap(); + assert!(bytes.len() < 2 * 1024 * 1024); + let map = parse(&bytes, Limits::default()).unwrap(); + let inventory = map.source_inventory(InventoryLimits { max_sources: 5000, max_bytes: 2 * 1024 * 1024 }); + assert_eq!(inventory.sources.len(), 1); + assert_eq!(inventory.sources[0].len(), 1024 * 1024 + 1 + source.len()); + assert!(inventory.sources[0].ends_with(source)); + assert!(inventory.truncated); + } +} + +#[test] +fn byte_limits_preserve_complete_utf8_identities_at_the_exact_boundary() { + let map = decode(json!({"version": 3, "sourceRoot": "é/", "sources": ["東京.js", "東京.js", null], "mappings": ""})); + let expected = "é/東京.js"; + let limits = InventoryLimits { max_sources: 2, max_bytes: 2 * expected.len() }; + let exact = map.source_inventory(limits); + assert_eq!(exact.sources, [expected, expected]); + assert!(!exact.truncated); + let short = map.source_inventory(InventoryLimits { max_bytes: limits.max_bytes - 1, ..limits }); + assert_eq!(short.sources, [expected]); + assert!(short.truncated); + let too_small = map.source_inventory(InventoryLimits { max_bytes: expected.len() - 1, ..limits }); + assert!(too_small.sources.is_empty()); + assert!(too_small.truncated); +} + +#[test] +fn entry_limits_count_empty_paths_but_not_null_sources() { + let map = decode(json!({"version": 3, "sourceRoot": "", "sources": [null, "", "", null], "mappings": ""})); + let exact = map.source_inventory(InventoryLimits { max_sources: 2, max_bytes: 0 }); + assert_eq!(exact.sources, ["", ""]); + assert!(!exact.truncated); + let one = map.source_inventory(InventoryLimits { max_sources: 1, max_bytes: 0 }); + assert_eq!(one.sources, [""]); + assert!(one.truncated); + let zero = map.source_inventory(InventoryLimits { max_sources: 0, max_bytes: 0 }); + assert!(zero.sources.is_empty()); + assert!(zero.truncated); + let absent = decode(json!({"version": 3, "sources": [null], "mappings": ""})); + assert!(!absent.source_inventory(InventoryLimits { max_sources: 0, max_bytes: 0 }).truncated); +} + +#[test] +fn indexed_sections_share_entry_and_byte_limits_in_source_order() { + let map = decode(json!({"version": 3, "sourceRoot": "outer", "sections": [ + {"offset": {"line": 0, "column": 0}, "map": { + "version": 3, "sourceRoot": "src", "sources": [null, "/first.js"], "mappings": "" + }}, + {"offset": {"line": 1, "column": 0}, "url": "external.map"}, + {"offset": {"line": 2, "column": 0}, "map": {"version": 3, "sections": [ + {"offset": {"line": 0, "column": 0}, "map": { + "version": 3, "sourceRoot": "file:///", "sources": ["second.js", "third.js"], "mappings": "" + }} + ]}} + ]})); + let expected = ["src//first.js", "file:///second.js", "file:///third.js"]; + let complete = map.source_inventory(InventoryLimits { max_sources: 3, max_bytes: expected.iter().map(|s| s.len()).sum() }); + assert_eq!(complete.sources, expected); + assert!(!complete.truncated); + let bytes = map.source_inventory(InventoryLimits { max_sources: 3, max_bytes: expected[0].len() + expected[1].len() }); + assert_eq!(bytes.sources, expected[..2]); + assert!(bytes.truncated); + let entries = map.source_inventory(InventoryLimits { max_sources: 1, max_bytes: 100 }); + assert_eq!(entries.sources, expected[..1]); + assert!(entries.truncated); +} + +#[test] +fn an_oversized_path_stops_the_prefix_without_shortening_or_skipping_identities() { + let map = decode(json!({"version": 3, "sources": ["a.js", "b".repeat(40), "c.js"], "mappings": ""})); + let inventory = map.source_inventory(InventoryLimits { max_sources: 3, max_bytes: 8 }); + assert_eq!(inventory.sources, ["a.js"]); + assert!(inventory.truncated); + assert_eq!(map.source_path(1).unwrap(), "b".repeat(40)); +} diff --git a/crates/sourcemap/tests/line_budgets.rs b/crates/sourcemap/tests/line_budgets.rs new file mode 100644 index 000000000..f987cd765 --- /dev/null +++ b/crates/sourcemap/tests/line_budgets.rs @@ -0,0 +1,72 @@ +use craft_sourcemap as sourcemap; +use craft_sourcemap::{Limits, SourceMapError}; +use serde_json::{Value, json}; + +fn decode(value: Value, limits: Limits) -> Result { + sourcemap::parse(&serde_json::to_vec(&value).unwrap(), limits) +} + +fn map(mappings: &str) -> Value { + json!({"version":3,"sources":["input.js"],"names":[],"mappings":mappings}) +} + +#[test] +fn empty_lines_spend_the_default_budget_even_with_no_segments() { + let limits = Limits { max_segments: 0, ..Limits::default() }; + assert_eq!(decode(map(&";".repeat(1_000_000)), limits), Err(SourceMapError::TooManyLines(1_000_000))); +} + +#[test] +fn exact_and_zero_line_limits_include_the_final_empty_line() { + let limits = Limits { max_lines: 3, max_segments: 0, ..Limits::default() }; + let parsed = decode(map(";;"), limits).unwrap(); + assert_eq!(parsed.lines.len(), 3); + assert_eq!(parsed.segments, 0); + assert_eq!(decode(map(";;;"), limits), Err(SourceMapError::TooManyLines(3))); + assert_eq!(decode(map(""), Limits { max_lines: 0, ..limits }), Err(SourceMapError::TooManyLines(0))); + assert_eq!(decode(map(""), Limits { max_lines: 1, ..limits }).unwrap().lines.len(), 1); +} + +#[test] +fn nested_sections_share_one_generated_line_budget() { + let value = json!({"version":3,"sections":[ + {"offset":{"line":0,"column":0},"map":map(";")}, + {"offset":{"line":2,"column":0},"map":{"version":3,"sections":[ + {"offset":{"line":0,"column":0},"map":map(";")} + ]}} + ]}); + let limits = Limits { max_lines: 3, max_segments: 0, ..Limits::default() }; + assert_eq!(decode(value.clone(), limits), Err(SourceMapError::TooManyLines(3))); + let parsed = decode(value, Limits { max_lines: 4, ..limits }).unwrap(); + assert_eq!(parsed.sections[0].2.lines.len(), 2); + assert_eq!(parsed.sections[1].2.sections[0].2.lines.len(), 2); +} + +#[test] +fn external_sections_do_not_consume_undecoded_lines() { + let value = json!({"version":3,"sections":[ + {"offset":{"line":0,"column":0},"url":"other.map"}, + {"offset":{"line":2,"column":0},"map":map("")} + ]}); + let parsed = decode(value, Limits { max_lines: 1, max_segments: 0, ..Limits::default() }).unwrap(); + assert!(parsed.sections[0].2.lines.is_empty()); + assert_eq!(parsed.sections[1].2.lines.len(), 1); +} + +#[test] +fn positions_after_empty_lines_keep_their_coordinates() { + let parsed = decode(map("AAAA;;AACA"), Limits { max_lines: 3, max_segments: 2, ..Limits::default() }).unwrap(); + assert_eq!(parsed.lines.len(), 3); + assert!(parsed.original_positions(1, 0).is_empty()); + let position = &parsed.original_positions(2, 0)[0]; + assert_eq!(position.source.as_deref(), Some("input.js")); + assert_eq!((position.line, position.column), (1, 0)); +} + +#[test] +fn line_and_segment_limits_are_independent() { + let limits = Limits { max_lines: 10, max_segments: 1, ..Limits::default() }; + assert!(matches!(decode(map("A,A"), limits), Err(SourceMapError::TooManySegments(_)))); + assert_eq!(decode(map(";"), Limits { max_lines: 1, max_segments: 0, ..limits }), Err(SourceMapError::TooManyLines(1))); + assert_eq!(decode(map(";"), Limits { max_lines: 2, max_segments: 0, ..limits }).unwrap().segments, 0); +} diff --git a/crates/sourcemap/tests/root_prefix.rs b/crates/sourcemap/tests/root_prefix.rs new file mode 100644 index 000000000..fd8a8a814 --- /dev/null +++ b/crates/sourcemap/tests/root_prefix.rs @@ -0,0 +1,80 @@ +use craft_sourcemap::{Limits, SourceMap, parse}; +use serde_json::{Value, json}; + +fn decode(value: Value) -> SourceMap { + parse(&serde_json::to_vec(&value).unwrap(), Limits::default()).unwrap() +} + +#[test] +fn root_prefix_precedes_absolute_looking_sources() { + for (root, source, expected) in [ + ("webpack:///", "/src/main.js", "webpack:////src/main.js"), + ("https://example.test/build", "/src/main.js", "https://example.test/build//src/main.js"), + ("src", "https://example.test/main.js", "src/https://example.test/main.js"), + ("src/", "//example.test/main.js", "src///example.test/main.js"), + ("sources", "node:internal/main", "sources/node:internal/main"), + ] { + let map = decode(json!({"version": 3, "sourceRoot": root, "sources": [source], "mappings": "AAAA"})); + assert_eq!(map.original_positions(0, 0)[0].source.as_deref(), Some(expected), "root {root:?}, source {source:?}"); + assert_eq!(map.all_sources(), [expected]); + } +} + +#[test] +fn empty_root_keeps_the_same_identity_as_an_absent_or_null_root() { + for source in ["src/main.js", "/src/main.js", "https://example.test/main.js"] { + let mut value = json!({"version": 3, "sources": [source], "mappings": "AAAA"}); + assert_eq!(decode(value.clone()).all_sources(), [source]); + value["sourceRoot"] = Value::Null; + assert_eq!(decode(value.clone()).all_sources(), [source]); + value["sourceRoot"] = json!(""); + let map = decode(value); + assert_eq!(map.all_sources(), [source]); + assert_eq!(map.original_positions(0, 0)[0].source.as_deref(), Some(source)); + } +} + +#[test] +fn each_source_keeps_its_index_and_embedded_content() { + let map = decode(json!({ + "version": 3, "sourceRoot": "https://example.test/originals/", + "sources": ["/first.js", null, "https://other.test/third.js"], + "sourcesContent": ["first();", null, "third();"], "mappings": "AAAA,CEAA" + })); + let first = map.original_positions(0, 0); + let third = map.original_positions(0, 1); + assert_eq!(first[0].source_index, 0); + assert_eq!(third[0].source_index, 2); + assert_eq!(first[0].source.as_deref(), Some("https://example.test/originals//first.js")); + assert_eq!(third[0].source.as_deref(), Some("https://example.test/originals/https://other.test/third.js")); + assert_eq!(map.source_content(0), Some("first();")); + assert_eq!(map.source_content(2), Some("third();")); + assert_eq!(map.source_path(1), None); + assert_eq!(map.source_path(3), None); +} + +#[test] +fn indexed_maps_use_each_embedded_maps_own_root() { + let map = decode(json!({ + "version": 3, "sourceRoot": "outer", "sections": [ + {"offset": {"line": 0, "column": 3}, "map": { + "version": 3, "sourceRoot": "file:///", "sources": ["/first.js"], "mappings": "AAAA" + }}, + {"offset": {"line": 1, "column": 0}, "map": { + "version": 3, "sources": ["second.js"], "mappings": "AAAA" + }} + ] + })); + assert_eq!(map.original_positions(0, 3)[0].source.as_deref(), Some("file:////first.js")); + assert_eq!(map.original_positions(1, 0)[0].source.as_deref(), Some("second.js")); + assert_eq!(map.all_sources(), ["file:////first.js", "second.js"]); +} + +#[test] +fn relative_sources_still_preserve_url_root_delimiters() { + for (root, expected) in [("webpack:///", "webpack:///main.js"), ("file:///", "file:///main.js"), ("src", "src/main.js"), ("src/", "src/main.js")] + { + let map = decode(json!({"version": 3, "sourceRoot": root, "sources": ["main.js"], "mappings": "AAAA"})); + assert_eq!(map.all_sources(), [expected]); + } +} diff --git a/crates/sourcemap/tests/section_columns.rs b/crates/sourcemap/tests/section_columns.rs new file mode 100644 index 000000000..d3dff81cd --- /dev/null +++ b/crates/sourcemap/tests/section_columns.rs @@ -0,0 +1,24 @@ +use craft_sourcemap::{Limits, parse}; +use serde_json::json; + +#[test] +fn indexed_lookup_reports_columns_in_the_generated_file() { + let json = json!({"version":3,"sections":[{"offset":{"line":3,"column":11},"map":{"version":3,"sources":["one.js"],"mappings":"IAAA;IACA"}}]}); + let map = parse(json.to_string().as_bytes(), Limits::default()).unwrap(); + assert!(map.original_positions(3, 14).is_empty()); + let first = map.original_positions(3, 17); + assert_eq!(first[0].generated_column, 15); + assert_eq!((first[0].line, first[0].column), (0, 0)); + assert_eq!(map.original_positions(4, 4)[0].generated_column, 4); +} + +#[test] +fn nested_section_columns_are_added_only_on_each_first_line() { + for (inner_line, query_line, query_column) in [(0, 3, 18), (1, 4, 7)] { + let json = json!({"version":3,"sections":[{"offset":{"line":3,"column":11},"map":{"version":3,"sections":[{"offset":{"line":inner_line,"column":3},"map":{"version":3,"sources":["nested.js"],"mappings":"IAAA"}}]}}]}); + let map = parse(json.to_string().as_bytes(), Limits::default()).unwrap(); + let positions = map.original_positions(query_line, query_column); + assert_eq!(positions[0].generated_column, query_column); + assert_eq!(positions[0].source.as_deref(), Some("nested.js")); + } +} diff --git a/crates/svg/Cargo.toml b/crates/svg/Cargo.toml index b35c4de41..eb5e01c91 100644 --- a/crates/svg/Cargo.toml +++ b/crates/svg/Cargo.toml @@ -31,6 +31,11 @@ usvg = { version = "=0.48.1", default-features = false, optional = true } resvg = { version = "=0.48.1", default-features = false, features = ["text", "raster-images"], optional = true } data-url = { version = "0.3.2", optional = true } +# Renderer-specific validation uses the same resolved backends as resvg0.48.1. +# JPEG validation uses the existing backend in craft-image-codecs. +gif = { version = "=0.14.2", optional = true } +image-webp = { version = "=0.2.4", optional = true } + [features] default = ["paths"] @@ -45,5 +50,9 @@ tree-text = ["tree", "usvg/text"] # The system's fonts in `usvg::fontdb`. tree-system-fonts = ["tree-text", "usvg/system-fonts"] # Bounded rendering of a parsed tree through the matching resvg/tiny-skia versions. -tree-render = ["tree", "dep:resvg"] +tree-render = ["tree", "dep:resvg", "dep:gif", "dep:image-webp"] tree-render-system-fonts = ["tree-render", "resvg/system-fonts"] + +[dev-dependencies] +# Already resolved in the workspace; test-only Cargo.lock edge inspection. +toml = "=1.1.8" diff --git a/crates/svg/README.md b/crates/svg/README.md index 508081367..a82d88028 100644 --- a/crates/svg/README.md +++ b/crates/svg/README.md @@ -16,3 +16,59 @@ The shared adapter adds an aggregate 64 MiB stylesheet-text limit and idempotent ### Transform compatibility `transform_compat::parse_transform` preserves EffectCraft transform-list recovery and exact composition. It retains six numeric arguments per operation; scanning is not time/byte capped and matrix overflow behavior is unchanged. The strict geometry reader is unchanged. Source: Brandon Thomas, EffectCraft94e1c86d89a866771f80ad1610afda9aa12e85d8; bounded safe argument matching by Altynzhan Karybai,4035d181329e847d7a57bf3d059502ea2c91b560. + + +### Opt-in validated embedded images + +With `tree-render`, `tree::ValidatedImageResolver` owns one drawing's immutable +`Limits`, original child-source work, data-URL normalization and renderer-specific +raster validation. Existing `ImageResolver::new`/`resolver` and legacy parse entry +points retain their behavior. Opt in explicitly: + +```rust +fn example() -> Result<(), craft_svg::tree::Error> { + use craft_svg::tree::{self, Limits, ValidatedImageResolver, usvg}; + let images = ValidatedImageResolver::new(Limits::default()); + let checked = tree::preflight( + "", + &images.limits(), + )?; + let drawing = images.parse_preflight(checked, usvg::Options::default())?; + let usage = images.usage(); + // Importers disclose usage's invalid_* and *_limit_images; strict consumers refuse them. + let _ = (drawing, usage); + Ok(()) +} +``` + +The caller supplies fonts and a sufficient stack/worker for its depth policy. +`parse_preflight` accepts only an opaque preflight made with exactly this context's +limits. It consumes options and installs its own data-only resolver, including for +all children; unrelated callbacks cannot fetch a linked file or bypass admission. +Expanded root ancestor frames (checked height minus the callback leaf) and each +child's complete expanded height share one depth allowance. The callback leaf is +not counted twice; source nesting alone cannot hide expanded reference chains. +An overlapping root conversion on the same context is refused. + +Unchanged XML converts from its original checked DOM. A modified data URL requires +one checked replacement DOM. Invalid URLs become bounded resolver-visible image +failures. The normal path counts only resolved images. If a repair cannot fit, +the checked original can still convert when all its MIME headers meet the work +limit. In that case malformed image declarations are counted conservatively. +Original UTF-8 child bytes are charged before XML conversion or shorter repairs, +with per-document and aggregate bounds. A malformed plain encoding spends its +input bytes and leaves the remaining work for siblings. Failed inflation can +exhaust its attempted allowance, and the separate bounded retry classification +never admits another child. Encoded raster callbacks spend at most `limits.bytes + limits.embedded_bytes` cumulatively, +without retaining their sources. Pixel work and decoder allocation are admitted +before decoding; attempted failures keep their charge. An oversized unattempted +image does not prevent a later smaller image from fitting. + +`resolve_text` accepts an already-decoded child SVG through the same admission +context, rather than an unchecked DOM or a precharged-budget override. `usage()` +reports omissions and work after conversion. The context is for one drawing, not a +global rendering cache. + +The validator uses the existing JPEG backend in craft-image-codecs. The lockfile +test checks that it and the other validation backends match the selected resvg +dependency edges. resvg decodes the image again when it paints. diff --git a/crates/svg/src/lib.rs b/crates/svg/src/lib.rs index ccb2b6837..79a706052 100644 --- a/crates/svg/src/lib.rs +++ b/crates/svg/src/lib.rs @@ -8,7 +8,7 @@ //! - `paths` (feature `paths`, on by default): the geometry of a document as Bézier paths in //! millimetres, for cutters, plotters and CAD import. //! - [`color`]: CSS colours. -//! - [`write`]: a writer for SVG documents, their path data and numbers. +//! - [`mod@write`]: a writer for SVG documents, their path data and numbers. //! - `tree` (feature `tree`): the full parser. usvg at the one version the Craft apps share, //! re-exported, behind an entry point that runs the checks above before it parses. //! diff --git a/crates/svg/src/tree.rs b/crates/svg/src/tree.rs index ab04a3002..3a549e773 100644 --- a/crates/svg/src/tree.rs +++ b/crates/svg/src/tree.rs @@ -22,10 +22,10 @@ //! 16 MB or so, and a 1 MB stack takes a depth of about 34. use std::borrow::Cow; +use std::collections::HashMap; use std::collections::hash_map::Entry; -use std::collections::{HashMap, HashSet}; use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; -use std::sync::{Arc, Mutex, PoisonError}; +use std::sync::{Arc, Mutex, PoisonError, Weak}; pub use usvg; @@ -34,7 +34,13 @@ use crate::{guard, svgz}; pub mod document; pub mod draw; +#[cfg(feature = "tree-render")] +mod image_href; pub mod primitive; +#[cfg(feature = "tree-render")] +mod validated; +#[cfg(feature = "tree-render")] +pub use validated::ValidatedImageResolver; /// What one import may use. The defaults suit a desktop app that parses on a thread with a /// large stack (see the module text). @@ -64,9 +70,9 @@ pub struct Limits { pub embedded_bytes: usize, /// SVG documents embedded inside one another, for [`image_resolver`]. pub embedded_depth: usize, - /// Decoded RGBA pixels retained across the raster images embedded in one drawing. Dimensions - /// are probed before usvg hands the bytes to a decoder; repeated references to the same data - /// URL share one charge. + /// Cumulative raster decode pixels for a drawing. usvg sends each data-URL + /// resolution separately, so repeated uses normally spend a separate charge. + /// The legacy resolver can share a charge for the same live input allocation. pub embedded_raster_pixels: u64, } @@ -161,6 +167,8 @@ pub fn root_unitless_size(svg: &str, limits: &Limits) -> Result { + #[cfg(feature = "tree-render")] + limits: Limits, xml: roxmltree::Document<'s>, depth: usize, /// [`height`]: the nesting usvg converts, references followed. @@ -219,6 +227,8 @@ const UNSUPPORTED: &[&str] = &[ /// Checks already-decoded SVG text once, retaining the parsed XML for both depth policy and tree conversion. pub fn preflight<'s>(svg: &'s str, limits: &Limits) -> Result, Error> { + #[cfg(test)] + PREFLIGHT_PASSES.with(|count| count.set(count.get().saturating_add(1))); let preflight_limits = crate::preflight::Limits { bytes: limits.bytes, depth: limits.depth, @@ -239,7 +249,13 @@ pub fn preflight<'s>(svg: &'s str, limits: &Limits) -> Result, Err let Some(height) = height(&xml, limits.tree_depth)? else { return Err(Error::TooDeep(limits.tree_depth)); }; - Ok(Preflight { xml, depth, height }) + Ok(Preflight { + #[cfg(feature = "tree-render")] + limits: *limits, + xml, + depth, + height, + }) } /// Dimensions of an SVG viewport in SVG user units. @@ -421,7 +437,7 @@ struct Used { svg_count: AtomicUsize, svg_depth: AtomicUsize, raster_pixels: AtomicU64, - raster_data: Mutex>, + raster_data: Mutex>>>, invalid_raster: AtomicUsize, raster_limit: AtomicUsize, invalid_svg: AtomicUsize, @@ -432,9 +448,12 @@ struct Used { /// What resolving one drawing's embedded images used or refused. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub struct ImageUsage { - /// Unique decoded raster pixels reserved before decode. + /// Decoded raster pixel work reserved before decode. The legacy resolver + /// shares a charge across repeated data; the validated context charges each + /// attempted decode, including failures. pub raster_pixels: u64, - /// Unpacked SVG bytes consumed by embedded SVG data URLs. + /// Decoded SVG text work charged to embedded data URLs, including the + /// remaining allowance consumed by a failed inflation attempt. pub embedded_svg_bytes: usize, /// Embedded SVG images parsed, including nested images. pub embedded_svg_count: usize, @@ -474,7 +493,7 @@ impl ImageResolver { svg_count: AtomicUsize::new(0), svg_depth: AtomicUsize::new(0), raster_pixels: AtomicU64::new(0), - raster_data: Mutex::new(HashSet::new()), + raster_data: Mutex::new(HashMap::new()), invalid_raster: AtomicUsize::new(0), raster_limit: AtomicUsize::new(0), invalid_svg: AtomicUsize::new(0), @@ -542,28 +561,51 @@ fn account_invalid_data_urls(xml: &roxmltree::Document<'_>, used: &Used) { } } -fn raster_format(data: &[u8]) -> Option { - match craft_image_codecs::detect(data)? { - format @ (craft_image_codecs::Format::Png - | craft_image_codecs::Format::Jpeg - | craft_image_codecs::Format::Gif - | craft_image_codecs::Format::WebP) => Some(format), +#[derive(Clone, Copy)] +enum SvgRaster { + Png, + Jpeg, + Gif, + WebP, +} + +impl SvgRaster { + fn codec(self) -> craft_image_codecs::Format { + match self { + Self::Png => craft_image_codecs::Format::Png, + Self::Jpeg => craft_image_codecs::Format::Jpeg, + Self::Gif => craft_image_codecs::Format::Gif, + Self::WebP => craft_image_codecs::Format::WebP, + } + } + + fn image_kind(self, data: Arc>) -> usvg::ImageKind { + match self { + Self::Png => usvg::ImageKind::PNG(data), + Self::Jpeg => usvg::ImageKind::JPEG(data), + Self::Gif => usvg::ImageKind::GIF(data), + Self::WebP => usvg::ImageKind::WEBP(data), + } + } +} + +/// Match supported raster signatures to the MIME types both image resolvers accept. +fn declared_raster(mime: &str, data: &[u8]) -> Option { + use craft_image_codecs::Format; + match (mime, craft_image_codecs::detect(data)?) { + ("image/png" | "text/plain", Format::Png) => Some(SvgRaster::Png), + ("image/jpg" | "image/jpeg" | "text/plain", Format::Jpeg) => Some(SvgRaster::Jpeg), + ("image/gif" | "text/plain", Format::Gif) => Some(SvgRaster::Gif), + ("image/webp" | "text/plain", Format::WebP) => Some(SvgRaster::WebP), _ => None, } } -fn raster_image(used: &Used, limits: &Limits, data: Arc>, format: craft_image_codecs::Format) -> Option { +fn raster_image(used: &Used, limits: &Limits, data: Arc>, format: SvgRaster) -> Option { let Ok((width, height)) = craft_image_codecs::probe_allocation_size(&data) else { used.invalid_raster.fetch_add(1, Ordering::Relaxed); return None; }; - if !matches!( - format, - craft_image_codecs::Format::Png | craft_image_codecs::Format::Jpeg | craft_image_codecs::Format::Gif | craft_image_codecs::Format::WebP - ) { - used.invalid_raster.fetch_add(1, Ordering::Relaxed); - return None; - } let pixels = u64::from(width).checked_mul(u64::from(height))?; if pixels == 0 || pixels > limits.embedded_raster_pixels { used.raster_limit.fetch_add(1, Ordering::Relaxed); @@ -571,12 +613,11 @@ fn raster_image(used: &Used, limits: &Limits, data: Arc>, format: craft_ } let pointer = Arc::as_ptr(&data) as usize; let mut seen = used.raster_data.lock().unwrap_or_else(PoisonError::into_inner); - if seen.insert(pointer) { + if let Entry::Vacant(entry) = seen.entry(pointer) { let mut current = used.raster_pixels.load(Ordering::Relaxed); loop { let next = current.checked_add(pixels)?; if next > limits.embedded_raster_pixels { - seen.remove(&pointer); used.raster_limit.fetch_add(1, Ordering::Relaxed); return None; } @@ -585,13 +626,16 @@ fn raster_image(used: &Used, limits: &Limits, data: Arc>, format: craft_ Err(actual) => current = actual, } } + // The weak reference pins the allocation identity, not its encoded bytes. + // At most one entry is added for each nonrefundable pixel charge. + entry.insert(Arc::downgrade(&data)); } drop(seen); // A valid header is not proof of valid pixels. resvg can silently omit a damaged // image later; decode now under explicit limits so omission accounting is honest. let decode_limits = craft_image_codecs::Limits::new(width, height, pixels, limits.embedded_raster_pixels.saturating_mul(8)); let options = craft_image_codecs::DecodeOptions { limits: decode_limits, keep_orientation: true }; - if let Err(error) = craft_image_codecs::decode_as_with(format, &data, &options) { + if let Err(error) = craft_image_codecs::decode_as_with(format.codec(), &data, &options) { // No tree retains this allocation on failure. Forget its address before it // can be reused, but keep the pixel charge for the decode work already spent. used.raster_data.lock().unwrap_or_else(PoisonError::into_inner).remove(&pointer); @@ -605,15 +649,37 @@ fn raster_image(used: &Used, limits: &Limits, data: Arc>, format: craft_ } return None; } - Some(match format { - craft_image_codecs::Format::Png => usvg::ImageKind::PNG(data), - craft_image_codecs::Format::Jpeg => usvg::ImageKind::JPEG(data), - craft_image_codecs::Format::Gif => usvg::ImageKind::GIF(data), - craft_image_codecs::Format::WebP => usvg::ImageKind::WEBP(data), - _ => return None, + Some(format.image_kind(data)) +} + +/// Decode one child within the original byte allowance, including encoding expansion. +fn embedded_text<'a>(data: &'a [u8], room: usize, limits: &Limits) -> Result, svgz::Error> { + let text = svgz::text(data, room.min(limits.bytes))?; + if text.len() > room { + return Err(svgz::Error::TooLarge(room)); + } + Ok(text) +} + +enum ChildRefusal { + Depth, + Bytes, + Invalid, +} + +fn preflight_child<'a>(text: &'a str, limits: &Limits) -> Result, ChildRefusal> { + preflight(text, limits).map_err(|error| match error { + Error::TooDeep(_) => ChildRefusal::Depth, + Error::Text(svgz::Error::TooLarge(_)) => ChildRefusal::Bytes, + _ => ChildRefusal::Invalid, }) } +/// Reserve the complete checked height within the parent's remaining XML room. +fn child_total_depth(parent: usize, checked: &Preflight<'_>, limits: &Limits) -> Option { + parent.checked_add(checked.height).filter(|depth| *depth <= limits.tree_depth) +} + /// An embedded SVG document, if the budgets allow it. fn embedded(used: &Used, limits: &Limits, data: &[u8], options: &usvg::Options<'_>) -> Option { if used.depth.load(Ordering::Relaxed) >= limits.embedded_depth { @@ -622,7 +688,7 @@ fn embedded(used: &Used, limits: &Limits, data: &[u8], options: &usvg::Options<' return None; } let left = limits.embedded_bytes.saturating_sub(used.bytes.load(Ordering::Relaxed)); - let text = match svgz::text(data, left.min(limits.bytes)) { + let text = match embedded_text(data, left, limits) { Ok(text) => text, // What a refused document unpacked before it was refused was work too: it uses up what // is left, so the next image can't repeat it. @@ -635,25 +701,17 @@ fn embedded(used: &Used, limits: &Limits, data: &[u8], options: &usvg::Options<' return None; } }; - // The raw/inflated bytes can fit while UTF-16/Latin-1 decoding expands - // beyond the aggregate allowance. A refusal consumes the remaining work. - if text.len() > left { - used.bytes.fetch_add(left, Ordering::Relaxed); - used.svg_limit.fetch_add(1, Ordering::Relaxed); - used.svg_resource_limit.fetch_add(1, Ordering::Relaxed); - return None; - } used.bytes.fetch_add(text.len(), Ordering::Relaxed); - let checked = match preflight(&text, limits) { + let checked = match preflight_child(&text, limits) { Ok(checked) => checked, // Decoding UTF-16 or Latin-1 can exceed the text cap after the // raw/inflated bytes fitted. This is a resource refusal, not invalid XML. - Err(Error::Text(svgz::Error::TooLarge(_))) => { + Err(ChildRefusal::Bytes) => { used.svg_limit.fetch_add(1, Ordering::Relaxed); used.svg_resource_limit.fetch_add(1, Ordering::Relaxed); return None; } - Err(Error::TooDeep(_)) => { + Err(ChildRefusal::Depth) => { used.svg_limit.fetch_add(1, Ordering::Relaxed); return None; } @@ -663,9 +721,7 @@ fn embedded(used: &Used, limits: &Limits, data: &[u8], options: &usvg::Options<' } }; // The nesting usvg converts, references followed, not the source depth alone. - let xml_depth = checked.height; - let total_depth = used.xml_depth.load(Ordering::Relaxed).checked_add(xml_depth); - let Some(total_depth) = total_depth.filter(|depth| *depth <= limits.tree_depth) else { + let Some(total_depth) = child_total_depth(used.xml_depth.load(Ordering::Relaxed), &checked, limits) else { used.svg_limit.fetch_add(1, Ordering::Relaxed); return None; }; @@ -700,16 +756,7 @@ pub fn image_resolver(limits: Limits) -> usvg::ImageHrefResolver<'static> { fn image_resolver_for(limits: Limits, used: Arc) -> usvg::ImageHrefResolver<'static> { usvg::ImageHrefResolver { resolve_data: Box::new(move |mime, data, options| { - let detected = raster_format(&data); - let format = match mime { - "image/jpg" | "image/jpeg" => (detected == Some(craft_image_codecs::Format::Jpeg)).then_some(craft_image_codecs::Format::Jpeg), - "image/png" => (detected == Some(craft_image_codecs::Format::Png)).then_some(craft_image_codecs::Format::Png), - "image/gif" => (detected == Some(craft_image_codecs::Format::Gif)).then_some(craft_image_codecs::Format::Gif), - "image/webp" => (detected == Some(craft_image_codecs::Format::WebP)).then_some(craft_image_codecs::Format::WebP), - "text/plain" => detected, - _ => None, - }; - if let Some(format) = format { + if let Some(format) = declared_raster(mime, &data) { return raster_image(&used, &limits, data, format); } // usvg can decode only the raster variants above. Count every other declared @@ -730,3 +777,8 @@ fn image_resolver_for(limits: Limits, used: Arc) -> usvg::ImageHrefResolve #[cfg(test)] mod tests; + +#[cfg(test)] +thread_local! { + static PREFLIGHT_PASSES: std::cell::Cell = const { std::cell::Cell::new(0) }; +} diff --git a/crates/svg/src/tree/draw.rs b/crates/svg/src/tree/draw.rs index f83653284..642a3888b 100644 --- a/crates/svg/src/tree/draw.rs +++ b/crates/svg/src/tree/draw.rs @@ -29,6 +29,13 @@ pub use resvg; #[cfg(feature = "tree-render")] pub use resvg::tiny_skia; +#[cfg(feature = "tree-render")] +mod raster; +#[cfg(feature = "tree-render")] +pub use raster::{RasterValidationFailure, validate_raster}; +#[cfg(feature = "tree-render")] +pub(super) use raster::{decode_raster, probe_raster_pixels}; + /// Filter primitives per filter. A result of every primitive is kept; real filters have 1 to 12 /// (Inkscape's drop shadow has 5), and 64 leaves headroom for generated ones. const MAX_PRIMITIVES: usize = 64; @@ -546,3 +553,6 @@ fn deeper(cx: Cx) -> Result<(), Refusal> { } Ok(()) } + +#[cfg(all(test, feature = "tree-render"))] +mod decoder_edges; diff --git a/crates/svg/src/tree/draw/decoder_edges.rs b/crates/svg/src/tree/draw/decoder_edges.rs new file mode 100644 index 000000000..f83130cec --- /dev/null +++ b/crates/svg/src/tree/draw/decoder_edges.rs @@ -0,0 +1,194 @@ +//! Test-only resolved-edge checks. No Cargo subprocess or global version count. +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::indexing_slicing)] + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct PackageId { + name: String, + version: String, + source: Option, +} +struct Package { + id: PackageId, + dependencies: Vec, +} +pub(crate) struct LockGraph { + packages: Vec, +} +// Cargo dependency edges may omit the resolved Git fragment even though the +// selected package ID always records it. Other source identities stay exact. +fn source_matches(edge: &str, package: Option<&str>) -> bool { + let Some(package) = package else { return false }; + if edge.starts_with("git+") && !edge.contains('#') { package.split_once('#').map_or(package, |(base, _)| base) == edge } else { package == edge } +} +impl LockGraph { + pub(crate) fn parse(text: &str) -> Self { + let lock: toml::Value = toml::from_str(text).unwrap(); + let packages = lock + .get("package") + .unwrap() + .as_array() + .unwrap() + .iter() + .map(|package| { + let string = |key| package.get(key).unwrap().as_str().unwrap().to_owned(); + Package { + id: PackageId { + name: string("name"), + version: string("version"), + source: package.get("source").map(|v| v.as_str().unwrap().to_owned()), + }, + dependencies: package + .get("dependencies") + .map(|d| d.as_array().unwrap().iter().map(|v| v.as_str().unwrap().to_owned()).collect()) + .unwrap_or_default(), + } + }) + .collect(); + Self { packages } + } + pub(crate) fn workspace_package(&self, name: &str) -> PackageId { + let found: Vec<_> = self.packages.iter().filter(|p| p.id.name == name && p.id.source.is_none()).collect(); + assert_eq!(found.len(), 1, "one exact workspace package required: {name}"); + found[0].id.clone() + } + pub(crate) fn dependency(&self, from: &PackageId, name: &str) -> PackageId { + let package = self.packages.iter().find(|p| &p.id == from).unwrap(); + let edges: Vec<_> = package.dependencies.iter().filter(|edge| edge.split_whitespace().next() == Some(name)).collect(); + assert_eq!(edges.len(), 1, "one {name} dependency edge required from {from:?}"); + let fields: Vec<_> = edges[0].split_whitespace().collect(); + let version = fields.get(1).copied(); + let source = fields.get(2).map(|s| s.trim_start_matches('(').trim_end_matches(')')); + let found: Vec<_> = self + .packages + .iter() + .filter(|p| { + p.id.name == name && version.is_none_or(|v| p.id.version == v) && source.is_none_or(|s| source_matches(s, p.id.source.as_deref())) + }) + .collect(); + assert_eq!(found.len(), 1, "ambiguous or absent resolved edge {} from {from:?}", edges[0]); + found[0].id.clone() + } +} + +pub(crate) fn assert_renderer_edges(graph: &LockGraph, svg: &PackageId) { + let renderer = graph.dependency(svg, "resvg"); + assert_eq!(graph.dependency(svg, "usvg"), graph.dependency(&renderer, "usvg"), "shared and renderer trees must be the same resolved package"); + for codec in ["gif", "image-webp"] { + assert_eq!(graph.dependency(svg, codec), graph.dependency(&renderer, codec), "validator and selected renderer differ for {codec}"); + } + let codecs = graph.dependency(svg, "craft-image-codecs"); + assert_eq!( + graph.dependency(&codecs, "zune-jpeg"), + graph.dependency(&renderer, "zune-jpeg"), + "JPEG validator and renderer must use the same backend" + ); +} + +#[test] +fn validated_renderer_dependency_edges_are_the_same_resolved_packages() { + let file = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../Cargo.lock"); + let graph = LockGraph::parse(&std::fs::read_to_string(file).unwrap()); + assert_renderer_edges(&graph, &graph.workspace_package("craft-svg")); +} + +#[test] +fn dependency_lookup_follows_the_selected_edge_and_rejects_ambiguity() { + let graph = LockGraph::parse( + r#" +[[package]] +name = "consumer" +version = "1" +dependencies = ["codec 1 (registry+one)"] +[[package]] +name = "codec" +version = "1" +source = "registry+one" +[[package]] +name = "codec" +version = "1" +source = "registry+two" +[[package]] +name = "other" +version = "1" +dependencies = ["codec"] +"#, + ); + let selected = graph.dependency(&graph.workspace_package("consumer"), "codec"); + assert_eq!(selected.source.as_deref(), Some("registry+one")); + assert!(std::panic::catch_unwind(|| graph.dependency(&graph.workspace_package("other"), "codec")).is_err()); +} + +#[test] +fn selected_renderer_graph_rejects_a_split_validator_backend() { + use std::fmt::Write; + let make = |split: bool| { + let mut text = String::from( + "[[package]]\nname='craft-svg'\nversion='1'\ndependencies=['resvg', 'usvg', 'gif', 'image-webp', 'craft-image-codecs']\n\ + [[package]]\nname='craft-image-codecs'\nversion='1'\ndependencies=['", + ); + text.push_str(if split { "zune-jpeg 2" } else { "zune-jpeg 1" }); + text.push_str( + "']\n[[package]]\nname='resvg'\nversion='1'\nsource='registry+one'\ndependencies=['usvg', 'gif', 'image-webp', 'zune-jpeg 1']\n", + ); + for name in ["usvg", "gif", "image-webp", "zune-jpeg"] { + write!(&mut text, "[[package]]\nname='{name}'\nversion='1'\nsource='registry+one'\n").unwrap(); + } + // An unrelated second codec version is valid; only the selected edges count. + text.push_str("[[package]]\nname='zune-jpeg'\nversion='2'\nsource='registry+one'\n"); + LockGraph::parse(&text) + }; + let matching = make(false); + assert_renderer_edges(&matching, &matching.workspace_package("craft-svg")); + let split = make(true); + assert!(std::panic::catch_unwind(|| assert_renderer_edges(&split, &split.workspace_package("craft-svg"))).is_err()); +} + +#[test] +fn git_edges_resolve_full_package_ids_and_reject_canonical_ambiguity() { + let fixture = |second: bool| { + let mut text = String::from( + r#" +[[package]] +name = "consumer" +version = "1" +dependencies = ["codec 1 (git+https://example.invalid/codecs?rev=chosen)"] +[[package]] +name = "explicit" +version = "1" +dependencies = ["codec 1 (git+https://example.invalid/codecs?rev=chosen#aaaa)"] +[[package]] +name = "wrong" +version = "1" +dependencies = ["codec 1 (git+https://example.invalid/codecs?rev=chosen#cccc)"] +[[package]] +name = "codec" +version = "1" +source = "git+https://example.invalid/codecs?rev=chosen#aaaa" +"#, + ); + if second { + text.push_str( + r#" +[[package]] +name = "codec" +version = "1" +source = "git+https://example.invalid/codecs?rev=chosen#bbbb" +"#, + ); + } + LockGraph::parse(&text) + }; + let unique = fixture(false); + let selected = unique.dependency(&unique.workspace_package("consumer"), "codec"); + assert_eq!(selected.source.as_deref(), Some("git+https://example.invalid/codecs?rev=chosen#aaaa")); + assert_eq!(selected, unique.dependency(&unique.workspace_package("explicit"), "codec")); + assert!(std::panic::catch_unwind(|| unique.dependency(&unique.workspace_package("wrong"), "codec")).is_err()); + let ambiguous = fixture(true); + assert!(std::panic::catch_unwind(|| ambiguous.dependency(&ambiguous.workspace_package("consumer"), "codec")).is_err()); + assert_eq!(selected, ambiguous.dependency(&ambiguous.workspace_package("explicit"), "codec")); + assert!(!source_matches("registry+one", Some("registry+one#aaaa")), "registry sources remain exact"); + assert!( + !source_matches("git+https://example.invalid/codecs?rev=other", selected.source.as_deref()), + "requested Git revision remains part of the identity" + ); +} diff --git a/crates/svg/src/tree/draw/raster.rs b/crates/svg/src/tree/draw/raster.rs new file mode 100644 index 000000000..48bff6ae9 --- /dev/null +++ b/crates/svg/src/tree/draw/raster.rs @@ -0,0 +1,132 @@ +//! Validate optional raster images with the same backends used by the SVG renderer. +//! +//! Header probing alone cannot detect damaged pixel data. Admit dimensions and +//! reserve work before decoding: even a failed decode keeps its charge. + +use std::io::Cursor; +use std::num::NonZeroU64; + +use crate::tree::{SvgRaster, draw::tiny_skia, usvg}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum RasterValidationFailure { + /// The renderer cannot decode the encoded image. + Invalid, + /// Header dimensions, cumulative work or decoder allocation exceed admission. + Limit, +} + +/// Validate the renderer's first frame within per-image and remaining work limits. +/// Failed attempted decodes keep their pixel charge. No decoded pixels are retained. SVG +/// children use the same wrapped resolver and need no second recursive walk. +pub fn validate_raster(kind: &usvg::ImageKind, max_pixels: u64, pixels_left: &mut u64) -> Result<(), RasterValidationFailure> { + let (format, data) = match kind { + usvg::ImageKind::PNG(data) => (SvgRaster::Png, data), + usvg::ImageKind::JPEG(data) => (SvgRaster::Jpeg, data), + usvg::ImageKind::GIF(data) => (SvgRaster::Gif, data), + usvg::ImageKind::WEBP(data) => (SvgRaster::WebP, data), + usvg::ImageKind::SVG(_) => return Ok(()), + }; + let pixels = probe_raster_pixels(data, max_pixels)?; + *pixels_left = pixels_left.checked_sub(pixels).ok_or(RasterValidationFailure::Limit)?; + decode_raster(format, data, pixels) +} + +pub(in crate::tree) fn probe_raster_pixels(data: &[u8], max_pixels: u64) -> Result { + // The allocation probe accounts for oversized GIF/WebP frames as well as the canvas. + let (width, height) = craft_image_codecs::probe_allocation_size(data).map_err(|_| RasterValidationFailure::Invalid)?; + let pixels = area(width, height)?; + if pixels > max_pixels { + return Err(RasterValidationFailure::Limit); + } + Ok(pixels) +} + +pub(in crate::tree) fn decode_raster(format: SvgRaster, data: &[u8], pixels: u64) -> Result<(), RasterValidationFailure> { + match format { + SvgRaster::Png => { + let decoded = tiny_skia::Pixmap::decode_png(data).map_err(|_| RasterValidationFailure::Invalid)?; + rgba_bytes(decoded.width(), decoded.height(), pixels)?; + Ok(()) + } + SvgRaster::Jpeg => craft_image_codecs::validate_jpeg_rgba8(data, pixels).map_err(|error| match error { + craft_image_codecs::CodecError::LimitExceeded(_) => RasterValidationFailure::Limit, + _ => RasterValidationFailure::Invalid, + }), + SvgRaster::Gif => gif(data, pixels), + SvgRaster::WebP => webp(data, pixels), + } +} + +fn area(width: u32, height: u32) -> Result { + if width == 0 || height == 0 { + return Err(RasterValidationFailure::Invalid); + } + u64::from(width).checked_mul(u64::from(height)).ok_or(RasterValidationFailure::Limit) +} + +/// Decoder headers must not request more output than the allocation probe paid for. +fn rgba_bytes(width: u32, height: u32, admitted: u64) -> Result { + let pixels = area(width, height)?; + if pixels > admitted { + return Err(RasterValidationFailure::Limit); + } + usize::try_from(pixels.checked_mul(4).ok_or(RasterValidationFailure::Limit)?).map_err(|_| RasterValidationFailure::Limit) +} + +fn buffer(length: usize) -> Result, RasterValidationFailure> { + let mut output = Vec::new(); + output.try_reserve_exact(length).map_err(|_| RasterValidationFailure::Limit)?; + output.resize(length, 0); + Ok(output) +} + +fn gif(data: &[u8], admitted: u64) -> Result<(), RasterValidationFailure> { + let bytes = admitted.checked_mul(4).ok_or(RasterValidationFailure::Limit)?; + // Do not loosen the renderer's default 50 MB GIF frame limit. Smaller + // imports also keep their own admitted allocation ceiling during decode. + let limit = NonZeroU64::new(bytes.min(50_000_000)).ok_or(RasterValidationFailure::Invalid)?; + let mut options = gif::DecodeOptions::new(); + options.set_color_output(gif::ColorOutput::RGBA); + options.set_memory_limit(gif::MemoryLimit::Bytes(limit)); + let mut decoder = options.read_info(data).map_err(gif_error)?; + let frame = decoder.read_next_frame().map_err(gif_error)?.ok_or(RasterValidationFailure::Invalid)?; + let expected = rgba_bytes(u32::from(frame.width), u32::from(frame.height), admitted)?; + if frame.buffer.len() != expected { + return Err(RasterValidationFailure::Invalid); + } + Ok(()) +} + +fn gif_error(error: gif::DecodingError) -> RasterValidationFailure { + match error { + gif::DecodingError::MemoryLimit | gif::DecodingError::OutOfMemory => RasterValidationFailure::Limit, + _ => RasterValidationFailure::Invalid, + } +} + +fn webp(data: &[u8], admitted: u64) -> Result<(), RasterValidationFailure> { + let mut decoder = image_webp::WebPDecoder::new(Cursor::new(data)).map_err(webp_error)?; + let (width, height) = decoder.dimensions(); + let rgba = rgba_bytes(width, height, admitted)?; + let channels = if decoder.has_alpha() { 4 } else { 3 }; + let expected = rgba / 4 * channels; + if decoder.output_buffer_size() != Some(expected) { + return Err(RasterValidationFailure::Invalid); + } + // The decoder's metadata memory limit does not cap read_image scratch. + // Admit the frame and exact output buffer before calling read_image. + let mut output = buffer(expected)?; + decoder.read_image(&mut output).map_err(webp_error) +} + +fn webp_error(error: image_webp::DecodingError) -> RasterValidationFailure { + match error { + image_webp::DecodingError::MemoryLimitExceeded | image_webp::DecodingError::ImageTooLarge => RasterValidationFailure::Limit, + _ => RasterValidationFailure::Invalid, + } +} + +#[cfg(test)] +#[path = "raster_tests.rs"] +mod raster_tests; diff --git a/crates/svg/src/tree/draw/raster_tests.rs b/crates/svg/src/tree/draw/raster_tests.rs new file mode 100644 index 000000000..be84e4738 --- /dev/null +++ b/crates/svg/src/tree/draw/raster_tests.rs @@ -0,0 +1,186 @@ +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::indexing_slicing)] +use super::*; +use craft_image_codecs::{ChannelLayout, Format, Image}; +use std::sync::Arc; + +fn kind(format: Format, bytes: Vec) -> usvg::ImageKind { + let bytes = Arc::new(bytes); + match format { + Format::Png => usvg::ImageKind::PNG(bytes), + Format::Jpeg => usvg::ImageKind::JPEG(bytes), + Format::Gif => usvg::ImageKind::GIF(bytes), + Format::WebP => usvg::ImageKind::WEBP(bytes), + _ => panic!("test raster format"), + } +} + +fn encoded(format: Format) -> Vec { + let image = Image::from_u8(2, 2, ChannelLayout::Rgb, [17, 83, 149].repeat(4)).unwrap(); + craft_image_codecs::encode(&image, format, &Default::default()).unwrap() +} + +fn segment(out: &mut Vec, marker: u8, payload: &[u8]) { + out.extend_from_slice(&[0xff, marker]); + out.extend_from_slice(&u16::try_from(payload.len() + 2).unwrap().to_be_bytes()); + out.extend_from_slice(payload); +} + +// ISO/IEC10918-1 framing, one gray 8x8 block. The single one-bit Huffman codes +// encode DC category 0 and AC EOB. No third-party asset or camera data is used. +fn gray_jpeg(bits: u8, entropy: &[u8]) -> Vec { + let mut data = vec![0xff, 0xd8]; + let mut quantization = vec![0]; + quantization.extend_from_slice(&[1; 64]); + segment(&mut data, 0xdb, &quantization); + segment(&mut data, if bits == 8 { 0xc0 } else { 0xc1 }, &[bits, 0, 8, 0, 8, 1, 1, 0x11, 0]); + let mut tables = Vec::new(); + for class in [0x00, 0x10] { + tables.extend_from_slice(&[class, 1]); + tables.extend_from_slice(&[0; 15]); + tables.push(0); + } + segment(&mut data, 0xc4, &tables); + segment(&mut data, 0xda, &[1, 1, 0, 0, 63, 0]); + data.extend_from_slice(entropy); + data.extend_from_slice(&[0xff, 0xd9]); + data +} + +fn corrupt_jpeg() -> Vec { + let mut data = gray_jpeg(8, &[0x3f]); + let scan = data.windows(2).position(|marker| marker == b"\xff\xda").unwrap(); + // Keep the complete scan header and valid entropy, but reference table 1 + // while DHT defines only table 0. The renderer accepts these headers, then + // fails pixel decoding when checking the selected Huffman tables. Random + // all-one entropy is tolerated by its default policy and is not this guard. + assert_eq!(data[scan + 6], 0); + data[scan + 6] = 0x11; + data +} + +fn crc32(bytes: &[u8]) -> u32 { + let mut value = !0u32; + for byte in bytes { + value ^= u32::from(*byte); + for _ in 0..8 { + value = (value >> 1) ^ (0xedb8_8320 & 0u32.wrapping_sub(value & 1)); + } + } + !value +} + +fn corrupt_png() -> Vec { + let mut data = encoded(Format::Png); + let mut at = 8; + loop { + let length = u32::from_be_bytes(data[at..at + 4].try_into().unwrap()) as usize; + if &data[at + 4..at + 8] == b"IDAT" { + // Keep the complete container and an honest CRC, but destroy the + // zlib header in the pixel stream rather than the dimension header. + assert!(length >= 2); + data[at + 8..at + 10].fill(0); + let crc = crc32(&data[at + 4..at + 8 + length]); + data[at + 8 + length..at + 12 + length].copy_from_slice(&crc.to_be_bytes()); + return data; + } + at += length + 12; + } +} + +fn corrupt_gif() -> Vec { + // Complete 2x2 GIF89a framing and palette, followed by a code7 when only + // codes 0..5 are initially defined by an LZW minimum code size of 2. + let mut data = b"GIF89a".to_vec(); + data.extend_from_slice(&[2, 0, 2, 0, 0x80, 0, 0]); + data.extend_from_slice(&[0, 0, 0, 0, 255, 0]); + data.extend_from_slice(&[0x2c, 0, 0, 0, 0, 2, 0, 2, 0, 0]); + data.extend_from_slice(&[2, 1, 7, 0, 0x3b]); + data +} + +fn corrupt_webp() -> Vec { + // Complete RIFF/VP8L dimensions, with no transform/Huffman/pixel bits. + // Chunk length and odd-byte padding remain internally consistent. + let mut data = b"RIFF".to_vec(); + data.extend_from_slice(&18u32.to_le_bytes()); + data.extend_from_slice(b"WEBPVP8L"); + data.extend_from_slice(&5u32.to_le_bytes()); + data.extend_from_slice(&[0x2f, 1, 0x40, 0, 0, 0]); + data +} + +#[test] +fn renderer_validator_accepts_complete_png_jpeg_gif_and_webp_at_exact_pixel_allowance() { + for format in [Format::Png, Format::Jpeg, Format::Gif, Format::WebP] { + let data = encoded(format); + assert_eq!(craft_image_codecs::probe_allocation_size(&data).unwrap(), (2, 2)); + let mut remaining = 4; + assert_eq!(validate_raster(&kind(format, data), 4, &mut remaining), Ok(()), "{format:?}"); + assert_eq!(remaining, 0, "{format:?}: exact area must be charged"); + } +} + +#[test] +fn renderer_validator_rejects_corrupt_pixels_after_valid_dimensions_without_refunding_work() { + let cases = [ + (Format::Png, corrupt_png(), (2, 2)), + // Header-only admission cannot detect this missing scan-table dependency. + (Format::Jpeg, corrupt_jpeg(), (8, 8)), + (Format::Gif, corrupt_gif(), (2, 2)), + (Format::WebP, corrupt_webp(), (2, 2)), + ]; + for (format, data, dimensions) in cases { + assert_eq!(craft_image_codecs::probe_allocation_size(&data).unwrap(), dimensions, "{format:?}"); + match format { + Format::Gif => { + gif::DecodeOptions::new().read_info(data.as_slice()).expect("intact GIF headers before corrupt LZW pixels"); + } + Format::WebP => { + image_webp::WebPDecoder::new(Cursor::new(&data)).expect("intact WebP headers before missing pixel bits"); + } + _ => {} + } + let pixels = u64::from(dimensions.0) * u64::from(dimensions.1); + let image = kind(format, data); + let mut remaining = pixels * 2; + for left in [pixels, 0] { + assert_eq!(validate_raster(&image, pixels, &mut remaining), Err(RasterValidationFailure::Invalid), "{format:?}"); + assert_eq!(remaining, left, "{format:?}: failed decode work is nonrefundable"); + } + assert_eq!(validate_raster(&image, pixels, &mut remaining), Err(RasterValidationFailure::Limit), "{format:?}"); + assert_eq!(remaining, 0); + } +} + +#[test] +fn twelve_bit_jpeg_is_valid_for_the_general_codec_but_not_the_svg_renderer() { + for bits in [8, 12] { + // DC=0, EOB=0, then six one-pad bits: a constant mid-gray block. + let data = gray_jpeg(bits, &[0x3f]); + let decoded = craft_image_codecs::decode(&data).unwrap(); + assert_eq!(decoded.dimensions(), (8, 8)); + let mut remaining = 64; + let expected = if bits == 8 { Ok(()) } else { Err(RasterValidationFailure::Invalid) }; + assert_eq!(validate_raster(&kind(Format::Jpeg, data), 64, &mut remaining), expected, "{bits}-bit"); + assert_eq!(remaining, 0, "capability refusal still spends attempted decode work"); + } +} + +#[test] +fn renderer_validator_refuses_unadmitted_headers_and_pixels_without_charging_unattempted_decode() { + for format in [Format::Png, Format::Jpeg, Format::Gif, Format::WebP] { + let image = kind(format, encoded(format)); + let mut remaining = 4; + assert_eq!(validate_raster(&image, 3, &mut remaining), Err(RasterValidationFailure::Limit), "{format:?}: per-image ceiling"); + assert_eq!(remaining, 4); + remaining = 3; + assert_eq!(validate_raster(&image, 4, &mut remaining), Err(RasterValidationFailure::Limit), "{format:?}: cumulative allowance"); + assert_eq!(remaining, 3); + assert_eq!( + validate_raster(&kind(format, b"invalid raster header".to_vec()), 4, &mut remaining), + Err(RasterValidationFailure::Invalid), + "{format:?}" + ); + assert_eq!(remaining, 3, "invalid dimensions cannot reserve an invented area"); + } +} diff --git a/crates/svg/src/tree/image_href.rs b/crates/svg/src/tree/image_href.rs new file mode 100644 index 000000000..620289a80 --- /dev/null +++ b/crates/svg/src/tree/image_href.rs @@ -0,0 +1,458 @@ +//! Bounded data-URL normalization over the shared checked XML document. +//! Only attribute values change; actual usvg callbacks decide image usage. +use super::Error; +#[cfg(test)] +use super::Limits; +use std::borrow::Cow; +use std::ops::Range; + +#[cfg(test)] +const SVG_NS: &str = "http://www.w3.org/2000/svg"; +const XLINK_NS: &str = "http://www.w3.org/1999/xlink"; +const INVALID_IMAGE: &str = "data:image/png,%00"; +const IGNORED_DATA: &str = "data:application/octet-stream,%00"; + +struct Patch { + range: Range, + replacement: Cow<'static, str>, +} + +fn invalid(message: &str) -> Error { + Error::Parse(message.into()) +} + +#[cfg(test)] +fn repair<'a>(text: &'a str, limits: &Limits) -> Result, Error> { + normalize(&super::preflight(text, limits)?) +} + +pub(super) fn normalize<'a>(checked: &super::Preflight<'a>) -> Result, Error> { + let text = checked.xml.input_text(); + let limits = &checked.limits; + let xml = &checked.xml; + if !text.contains("href") { + return Ok(Cow::Borrowed(text)); + } + // Expanded values fit source + entity work. Twelve linear passes cover URL + // scans/streaming decode, full attribute projection and worst-case XML escaping; + // parameter compaction never allocates one record per original parameter. + let mut work = text.len().saturating_add(limits.entity_bytes).saturating_mul(12); + let max_patches = (limits.bytes / std::mem::size_of::()).min(usize::try_from(limits.xml_nodes).unwrap_or(usize::MAX)); + let mut patches: Vec = Vec::new(); + for node in xml.descendants().filter(|node| node.is_element()) { + if !matches!(node.tag_name().name(), "image" | "feImage") { + continue; + } + // Shared preflight inspects local image names in every namespace and + // checks XLink before plain href. Normalize both representations, even + // when one is inert or unselected. Only actual resolver callbacks count + // omissions, so this does not turn hidden/unused resources into warnings. + for href in node.attributes().filter(|a| a.name() == "href" && matches!(a.namespace(), None | Some(XLINK_NS))) { + let Some(replacement) = data_url_replacement(href.value(), &mut work)? else { continue }; + let range = verified_value_range(text, node.range(), href.range(), href.name(), &mut work)?; + if patches.len() == patches.capacity() { + if patches.len() >= max_patches { + return Err(invalid("SVG data-URL repair metadata exceeds its limit")); + } + let capacity = patches.capacity().max(8).saturating_mul(2).min(max_patches); + patches + .try_reserve_exact(capacity.saturating_sub(patches.len())) + .map_err(|_| invalid("could not allocate bounded SVG data-URL repairs"))?; + } + patches.push(Patch { range, replacement }); + } + } + if patches.is_empty() { + return Ok(Cow::Borrowed(text)); + } + patches.sort_unstable_by_key(|patch| (patch.range.start, patch.range.end)); + patches.dedup_by_key(|patch| (patch.range.start, patch.range.end)); + let mut end = 0; + let mut length = text.len(); + for patch in &patches { + let range = &patch.range; + if range.start < end { + return Err(invalid("SVG data-URL source ranges overlap")); + } + end = range.end; + length = length + .checked_sub(range.len()) + .and_then(|n| n.checked_add(patch.replacement.len())) + .ok_or_else(|| invalid("SVG data-URL repair size overflows"))?; + } + if length > limits.bytes { + return Err(invalid("repaired SVG exceeds the text byte limit")); + } + let mut result = String::new(); + result.try_reserve_exact(length).map_err(|_| invalid("could not allocate bounded repaired SVG text"))?; + let mut cursor = 0; + for patch in patches { + result.push_str(text.get(cursor..patch.range.start).ok_or_else(|| invalid("invalid SVG data-URL source range"))?); + result.push_str(&patch.replacement); + cursor = patch.range.end; + } + result.push_str(text.get(cursor..).ok_or_else(|| invalid("invalid SVG data-URL source range"))?); + Ok(Cow::Owned(result)) +} + +/// Fall back to the original, checked DOM when a repair cannot be projected. +/// Only MIME headers already safe for usvg's parser can take this route. Count +/// malformed declarations conservatively because usvg may drop them before +/// invoking a callback. +pub(super) fn unrepaired_image_omissions(checked: &super::Preflight<'_>) -> Option { + let mut work = checked.xml.input_text().len().saturating_add(checked.limits.entity_bytes).saturating_mul(6); + let mut omitted = 0usize; + for node in checked.xml.descendants().filter(|node| node.is_element() && matches!(node.tag_name().name(), "image" | "feImage")) { + for href in node.attributes().filter(|a| a.name() == "href" && matches!(a.namespace(), None | Some(XLINK_NS))) { + let value = href.value(); + work = work.checked_sub(value.len())?; + if !mime_header_admitted(value) { + return None; + } + } + if let Some(value) = node.attribute((XLINK_NS, "href")).or_else(|| node.attribute("href")) + && malformed_data_url_replacement(value, &mut work).ok()? == Some(INVALID_IMAGE) + { + omitted = omitted.saturating_add(1); + } + } + Some(omitted) +} + +// The pinned MIME parser searches all earlier parameters for each new name. +// Cap even quoted semicolons before invoking it: source-byte work is already +// charged, so at most this constant times the bounded XML work reaches MIME +// parsing. Bodies can remain large without acquiring MIME parameter records. +const MAX_MIME_SEPARATORS: usize = 64; + +fn mime_header_admitted(value: &str) -> bool { + let trimmed = value.trim_start_matches(|ch| ch <= ' '); + let mut bytes = trimmed.bytes(); + for expected in b"data:" { + let Some(actual) = bytes.by_ref().find(|&byte| !matches!(byte, b'\t' | b'\n' | b'\r')) else { return true }; + if !actual.eq_ignore_ascii_case(expected) { + return true; // Not a data URL: preserve external/fragment handling. + } + } + let mut separators = 0; + for byte in bytes { + if matches!(byte, b',' | b'#') { + break; + } + if byte == b';' { + separators += 1; + if separators > MAX_MIME_SEPARATORS { + return false; + } + } + } + true +} + +/// A long parameter list must not reach the quadratic MIME parser, but its +/// essence, final base64 marker and body still determine the original image. +/// Compact only that header, then use the same parser/streaming decoder. Escape +/// the decoded attribute value once when projecting it back into XML. +fn data_url_replacement(value: &str, work: &mut usize) -> Result>, Error> { + if mime_header_admitted(value) { + return malformed_data_url_replacement(value, work).map(|replacement| replacement.map(Cow::Borrowed)); + } + *work = work.checked_sub(value.len()).ok_or_else(|| invalid("SVG data-URL validation work exceeds its limit"))?; + let normalized = value.trim_matches(|ch| ch <= ' '); + let delimiter = normalized.find([',', '#']); + let Some(comma) = delimiter.filter(|&at| normalized.as_bytes().get(at) == Some(&b',')) else { + return missing_comma_replacement(value, work).map(|replacement| Some(Cow::Borrowed(replacement))); + }; + let header = normalized.get(..comma).ok_or_else(|| invalid("invalid SVG data-URL header framing"))?; + let essence = header.split_once(';').map_or(header, |(essence, _)| essence); + // Retaining the final parameter lets the pinned parser decide whether its + // suffix is a base64 marker, including its case/ASCII-whitespace rules. + let suffix = header.rfind(';').and_then(|at| header.get(at..)).unwrap_or(""); + let body = normalized.get(comma..).ok_or_else(|| invalid("invalid SVG data-URL body framing"))?; + let length = essence + .len() + .checked_add(suffix.len()) + .and_then(|n| n.checked_add(body.len())) + .ok_or_else(|| invalid("SVG data-URL repair size overflows"))?; + let mut compact = String::new(); + compact.try_reserve_exact(length).map_err(|_| invalid("could not allocate bounded SVG data-URL header"))?; + compact.push_str(essence); + compact.push_str(suffix); + compact.push_str(body); + if let Some(replacement) = malformed_data_url_replacement(&compact, work)? { + return Ok(Some(Cow::Borrowed(replacement))); + } + let escaped_len = compact + .chars() + .try_fold(0usize, |n, ch| { + n.checked_add(match ch { + '&' | '\t' | '\n' | '\r' => 5, + '\'' | '"' => 6, + '<' | '>' => 4, + _ => ch.len_utf8(), + }) + }) + .ok_or_else(|| invalid("SVG data-URL repair size overflows"))?; + *work = work.checked_sub(escaped_len).ok_or_else(|| invalid("SVG data-URL validation work exceeds its limit"))?; + let mut escaped = String::new(); + escaped.try_reserve_exact(escaped_len).map_err(|_| invalid("could not allocate bounded SVG data-URL repair"))?; + for ch in compact.chars() { + match ch { + '&' => escaped.push_str("&"), + '\'' => escaped.push_str("'"), + '"' => escaped.push_str("""), + '<' => escaped.push_str("<"), + '>' => escaped.push_str(">"), + '\t' => escaped.push_str(" "), + '\n' => escaped.push_str(" "), + '\r' => escaped.push_str(" "), + _ => escaped.push(ch), + } + } + Ok(Some(Cow::Owned(escaped))) +} + +fn malformed_data_url_replacement(value: &str, work: &mut usize) -> Result, Error> { + malformed_data_url_replacement_with(value, work, data_url::DataUrl::process) +} + +fn malformed_data_url_replacement_with<'a>( + value: &'a str, + work: &mut usize, + parse: impl FnOnce(&'a str) -> Result, data_url::DataUrlError>, +) -> Result, Error> { + *work = work.checked_sub(value.len()).ok_or_else(|| invalid("SVG data-URL validation work exceeds its limit"))?; + if !mime_header_admitted(value) { + // Strip parameters before recovering MIME disposition; no original + // parameter list reaches either our parser or shared whole-DOM preflight. + return missing_comma_replacement(value, work).map(Some); + } + let url = match parse(value) { + Ok(url) => url, + Err(data_url::DataUrlError::NoComma) => return missing_comma_replacement(value, work).map(Some), + Err(data_url::DataUrlError::NotADataUrl) => return Ok(None), + }; + let mut exhausted = false; + let decoded = url.decode(|bytes| { + if let Some(left) = work.checked_sub(bytes.len()) { + *work = left; + Ok(()) + } else { + exhausted = true; + Err(()) + } + }); + if exhausted { + return Err(invalid("SVG data-URL validation work exceeds its limit")); + } + Ok(decoded.is_err().then(|| replacement_for_mime(url.mime_type()))) +} + +/// Preflight must see a valid URL, while the resolver must keep its original +/// disposition. Non-image MIME types are ignored even when their body is bad. +fn replacement_for_mime(mime: &data_url::mime::Mime) -> &'static str { + if mime.type_ == "image" || mime.matches("text", "plain") { INVALID_IMAGE } else { IGNORED_DATA } +} + +fn missing_comma_replacement(value: &str, work: &mut usize) -> Result<&'static str, Error> { + // NoComma does not expose its MIME type. Ask the same parser about a header + // with an empty body rather than guessing MIME defaults, case or parameters. + // Match its URL C0 trimming; fragments and MIME parameters cannot affect + // the image/text-plain essence. Do not allocate parameter bookkeeping for a + // hostile missing-comma header just to recover that disposition. + let normalized = value.trim_matches(|ch| ch <= ' '); + let header = normalized.split_once('#').map_or(normalized, |(header, _)| header); + let essence = header.split_once(';').map_or(header, |(essence, _)| essence); + *work = work.checked_sub(essence.len()).ok_or_else(|| invalid("SVG data-URL validation work exceeds its limit"))?; + let length = essence.len().checked_add(1).ok_or_else(|| invalid("SVG data-URL repair size overflows"))?; + let mut candidate = String::new(); + candidate.try_reserve_exact(length).map_err(|_| invalid("could not allocate bounded SVG data-URL header"))?; + candidate.push_str(essence); + candidate.push(','); + // This candidate has no MIME parameters, including for oversized original + // lists. Its byte scan/allocation is charged above before the linear parse. + if !mime_header_admitted(&candidate) { + return Err(invalid("SVG data-URL MIME header exceeds its limit")); + } + let url = data_url::DataUrl::process(&candidate).map_err(|_| invalid("invalid SVG data-URL header framing"))?; + Ok(replacement_for_mime(url.mime_type())) +} + +/// Use the complete source range rather than roxmltree's compact QName/equals +/// offsets (u16/u8). Scan only this validated attribute, never a DTD declaration +/// or another node. Charge the extra source scan before examining it. +fn verified_value_range(text: &str, owner: Range, attribute: Range, local_name: &str, work: &mut usize) -> Result, Error> { + let error = || invalid("SVG data-URL attribute source positions cannot be represented safely"); + if attribute.start < owner.start || attribute.end > owner.end { + return Err(error()); + } + *work = work.checked_sub(attribute.len()).ok_or_else(|| invalid("SVG data-URL validation work exceeds its limit"))?; + let source = text.get(attribute.clone()).ok_or_else(error)?; + let name_end = source.find([' ', '\t', '\r', '\n', '=']).ok_or_else(error)?; + let name = source.get(..name_end).ok_or_else(error)?; + if name.rsplit(':').next() != Some(local_name) { + return Err(error()); + } + let after_name = source.get(name_end..).ok_or_else(error)?.trim_start_matches([' ', '\t', '\r', '\n']); + let after_equals = after_name.strip_prefix('=').ok_or_else(error)?.trim_start_matches([' ', '\t', '\r', '\n']); + let quote = after_equals.chars().next().ok_or_else(error)?; + if !matches!(quote, '\'' | '"') || !after_equals.ends_with(quote) || after_equals.len() < 2 { + return Err(error()); + } + let start = attribute.end.checked_sub(after_equals.len()).and_then(|n| n.checked_add(1)).ok_or_else(error)?; + let end = attribute.end.checked_sub(1).ok_or_else(error)?; + if text.get(start..end).is_none_or(|value| value.contains(quote)) { + return Err(error()); + } + Ok(start..end) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn limits() -> Limits { + Limits { bytes: 1 << 20, depth: 32, tree_depth: 32, entity_bytes: 65_536, ..Default::default() } + } + + #[test] + fn original_dom_fallback_does_not_bypass_mime_admission() { + let ordinary = ""; + assert_eq!(unrepaired_image_omissions(&super::super::preflight(ordinary, &limits()).unwrap()), Some(1)); + + let header = "a=b;".repeat(MAX_MIME_SEPARATORS + 1); + let excessive = format!(""); + assert!(unrepaired_image_omissions(&super::super::preflight(&excessive, &limits()).unwrap()).is_none()); + } + + #[test] + fn malformed_image_urls_are_repaired_in_both_svg_namespaces_and_filter_resources() { + for tag in ["image", "feImage"] { + for text in [ + format!("<{tag} href='data:image/png;base64,A'/>"), + format!("<{tag} href='data:image/png;base64,A'/>"), + format!(""), + ] { + let repaired = repair(&text, &limits()).unwrap(); + assert!(matches!(repaired, Cow::Owned(_))); + assert_eq!(repaired.matches(INVALID_IMAGE).count(), 1); + } + } + } + + #[test] + fn unselected_href_and_foreign_local_image_names_are_normalized_for_shared_preflight() { + for attrs in ["href='data:image/png,%00' x:href='data:image/png;base64,A'", "x:href='data:image/png;base64,A' href='data:image/png,%00'"] { + let text = format!(""); + let repaired = repair(&text, &limits()).unwrap(); + assert!(matches!(repaired, Cow::Owned(_))); + assert_eq!(repaired.matches(INVALID_IMAGE).count(), 2); + assert!(!repaired.contains("base64,A")); + } + let text = format!( + "" + ); + let repaired = repair(&text, &limits()).unwrap(); + assert_eq!(repaired.matches(INVALID_IMAGE).count(), 3); + assert!(repaired.contains("f:href='data:image/png;base64,A'"), "foreign href attributes remain inert and unchanged"); + } + + #[test] + fn both_href_forms_on_hidden_unused_and_foreign_resources_do_not_preflight_as_omissions() { + for tag in ["image", "feImage"] { + let text = format!( + "<{tag} id='unused' href='data:image/png;base64,A' x:href='data:image/png;base64,A'/><{tag} display='none' x:href='data:image/png;base64,A' href='data:image/png;base64,A'/>" + ); + let original = text.clone(); + let repaired = repair(&text, &limits()).unwrap(); + assert_eq!(text, original, "repair never mutates the original source"); + assert_eq!(repaired.matches(INVALID_IMAGE).count(), 6); + let resolver = crate::tree::ImageResolver::new(limits()); + resolver.check_document(&repaired).unwrap(); + let usage = resolver.usage(); + assert_eq!( + (usage.invalid_raster_images, usage.invalid_svg_images), + (0, 0), + "shared whole-document scanning cannot invent omissions before resolution" + ); + assert!(matches!(repair(&repaired, &limits()).unwrap(), Cow::Borrowed(_)), "normalization is idempotent"); + } + } + + #[test] + fn entity_expanded_urls_patch_the_owning_attributes_without_rewriting_the_doctype() { + let text = format!( + "]>" + ); + let repaired = repair(&text, &limits()).unwrap(); + assert!(repaired.contains("")); + assert_eq!(repaired.matches(INVALID_IMAGE).count(), 2); + assert!(!repaired.contains("href='&bad;'")); + assert!(matches!(repair(&repaired, &limits()).unwrap(), Cow::Borrowed(_))); + } + + #[test] + fn valid_data_and_external_urls_are_not_rewritten() { + for value in ["data:image/png;base64,AA==", "data:image/svg+xml,%3Csvg%2F%3E", "file.png", "#local"] { + let text = format!(""); + assert!(matches!(repair(&text, &limits()).unwrap(), Cow::Borrowed(_))); + } + let text = ""; + assert!(repair(text, &limits()).unwrap().contains(INVALID_IMAGE)); + } + + #[test] + fn malformed_non_image_urls_keep_ignored_replacements_even_without_a_comma() { + for value in [ + "data:application/octet-stream;base64,A", + "data:application/octet-stream;base64", + "data:APPLICATION/OCTET-STREAM;BASE64,A", + "data:text/css;base64,A", + "data:audio/ogg;base64,A", + "data:application/octet-stream;base64#fragment,not-the-delimiter", + ] { + let text = format!(""); + let repaired = repair(&text, &limits()).unwrap(); + assert!(repaired.contains(IGNORED_DATA), "{value}: {repaired}"); + assert!(!repaired.contains(INVALID_IMAGE)); + assert!(matches!(repair(&repaired, &limits()).unwrap(), Cow::Borrowed(_))); + let resolver = crate::tree::ImageResolver::new(limits()); + resolver.check_document(&repaired).unwrap(); + assert_eq!(resolver.usage().invalid_raster_images, 0); + } + } + + #[test] + fn image_and_default_plain_mime_failures_remain_disclosed() { + for value in [ + "data:IMAGE/PNG;base64,A", + "data:image/png;base64", + "data:text/plain;base64,A", + "data:;base64,A", + "data:;base64", + "data:not-a-mime;base64", + ] { + let text = format!(""); + assert!(repair(&text, &limits()).unwrap().contains(INVALID_IMAGE), "{value}"); + } + let value = "data:application/octet-stream;base64#fragment"; + let mut insufficient = value.len(); + assert!(malformed_data_url_replacement(value, &mut insufficient).is_err(), "the extra MIME-header parse has its own work charge"); + let mut sufficient = value.len() * 2; + assert_eq!(malformed_data_url_replacement(value, &mut sufficient).unwrap(), Some(IGNORED_DATA)); + } + + #[test] + fn long_equals_spacing_is_repaired_and_output_growth_stays_bounded() { + let text = format!("", " ".repeat(300)); + let repaired = repair(&text, &limits()).unwrap(); + assert!(repaired.contains(INVALID_IMAGE)); + assert!(repaired.contains(&format!("href{}=", " ".repeat(300)))); + let text = ""; + assert!(matches!(repair(text, &Limits { bytes: text.len(), ..limits() }), Err(Error::Parse(message)) if message.contains("byte limit"))); + } +} + +#[cfg(test)] +#[path = "mime_header_tests.rs"] +mod mime_header_tests; diff --git a/crates/svg/src/tree/mime_header_tests.rs b/crates/svg/src/tree/mime_header_tests.rs new file mode 100644 index 000000000..c22ef7d4a --- /dev/null +++ b/crates/svg/src/tree/mime_header_tests.rs @@ -0,0 +1,158 @@ +use super::*; +use std::fmt::Write; + +fn parameters(count: usize) -> String { + let mut text = String::new(); + for index in 0..count { + write!(&mut text, ";p{index}=x").unwrap(); + } + text +} + +#[test] +fn a_hundred_thousand_unique_mime_parameters_are_refused_before_the_original_parser() { + let params = parameters(100_000); + for (essence, expected) in [ + ("image/png", INVALID_IMAGE), + ("text/plain", INVALID_IMAGE), + ("", INVALID_IMAGE), + ("not-a-mime", INVALID_IMAGE), + ("application/octet-stream", IGNORED_DATA), + ("audio/ogg", IGNORED_DATA), + ] { + for delimiter in [",AA==", "#fragment,AA==", ""] { + let value = format!("data:{essence}{params};base64{delimiter}"); + let mut work = value.len() * 2; + assert!(!mime_header_admitted(&value)); + let result = + malformed_data_url_replacement_with(&value, &mut work, |_| panic!("the original parameter flood must not reach DataUrl::process")) + .unwrap(); + assert_eq!(result, Some(expected), "{essence} {delimiter}"); + } + } +} + +#[test] +fn mime_parameter_admission_preserves_the_boundary_and_ignores_body_semicolons() { + let accepted = format!("data:image/png{};base64,AA==", parameters(MAX_MIME_SEPARATORS - 1)); + assert!(mime_header_admitted(&accepted)); + let called = std::cell::Cell::new(false); + let mut work = accepted.len() * 2; + assert_eq!( + malformed_data_url_replacement_with(&accepted, &mut work, |value| { + called.set(true); + data_url::DataUrl::process(value) + }) + .unwrap(), + None + ); + assert!(called.get(), "an admitted original URL still reaches the real parser"); + let refused = format!("data:image/png{};base64,AA==", parameters(MAX_MIME_SEPARATORS)); + assert!(!mime_header_admitted(&refused)); + let mut work = refused.len() * 2; + assert_eq!(malformed_data_url_replacement(&refused, &mut work).unwrap(), Some(INVALID_IMAGE)); + + for value in [format!("data:text/plain,{}", ";".repeat(100_000)), format!("external.png?{}", parameters(100_000))] { + assert!(mime_header_admitted(&value), "payloads/external URLs are not MIME parameters"); + let mut work = value.len() * 2; + assert_eq!(malformed_data_url_replacement(&value, &mut work).unwrap(), None); + } + let value = format!(" \td\ta\nt\ra:image/png{};base64,A", parameters(MAX_MIME_SEPARATORS)); + assert!(!mime_header_admitted(&value), "scheme normalization matches the pinned parser"); + let mut work = value.len() * 2; + assert_eq!(malformed_data_url_replacement(&value, &mut work).unwrap(), Some(INVALID_IMAGE)); +} + +#[test] +fn compacted_parameter_lists_preserve_both_hrefs_and_foreign_resources_without_preflight_warnings() { + let value = format!("data:image/png{};base64,AA==", parameters(MAX_MIME_SEPARATORS)); + let text = format!( + "" + ); + let limits = Limits { bytes: 1 << 20, ..Default::default() }; + let repaired = repair(&text, &limits).unwrap(); + assert_eq!(repaired.matches("data:image/png;base64,AA==").count(), 6); + assert!(!repaired.contains(";p0=")); + let resolver = crate::tree::ImageResolver::new(limits); + resolver.check_document(&repaired).unwrap(); + let usage = resolver.usage(); + assert_eq!((usage.invalid_raster_images, usage.invalid_svg_images), (0, 0)); + assert!(matches!(repair(&repaired, &Limits { bytes: 1 << 20, ..Default::default() }).unwrap(), Cow::Borrowed(_))); +} + +#[test] +fn duplicate_and_quoted_parameters_share_the_same_conservative_boundary() { + for (accepted, refused) in [ + ( + format!("data:image/png{},%00", ";p=x".repeat(MAX_MIME_SEPARATORS)), + format!("data:image/png{},%00", ";p=x".repeat(MAX_MIME_SEPARATORS + 1)), + ), + ( + format!("data:image/png;p=\"{}\",%00", ";".repeat(MAX_MIME_SEPARATORS - 1)), + format!("data:image/png;p=\"{}\",%00", ";".repeat(MAX_MIME_SEPARATORS)), + ), + ] { + assert!(mime_header_admitted(&accepted)); + let mut work = accepted.len() * 2; + assert_eq!(malformed_data_url_replacement(&accepted, &mut work).unwrap(), None); + assert!(!mime_header_admitted(&refused)); + let mut work = refused.len() * 2; + assert_eq!(malformed_data_url_replacement(&refused, &mut work).unwrap(), Some(INVALID_IMAGE)); + } + for value in ["external.png", "#local", "datx:image/png;param=x,body"] { + let mut work = value.len() * 2; + assert_eq!(malformed_data_url_replacement(value, &mut work).unwrap(), None); + } + let value = "data:image/png;base64"; + let mut work = value.len() * 2; + assert_eq!(malformed_data_url_replacement(value, &mut work).unwrap(), Some(INVALID_IMAGE)); +} + +#[test] +fn parameter_flood_scans_and_essence_classification_spend_one_shared_work_ledger() { + let value = format!("data:application/octet-stream{};base64,AA==", parameters(MAX_MIME_SEPARATORS)); + let essence_len = "data:application/octet-stream".len(); + let charge = value.len() + essence_len; + let mut work = charge * 2; + for remaining in [charge, 0] { + assert_eq!(malformed_data_url_replacement_with(&value, &mut work, |_| panic!("not admitted")).unwrap(), Some(IGNORED_DATA)); + assert_eq!(work, remaining, "the original header scan and parameter-free fallback each retain their charge"); + } + assert!(malformed_data_url_replacement_with(&value, &mut work, |_| panic!("no work remains")).is_err()); + assert_eq!(work, 0); + let mut not_enough_for_essence = value.len(); + assert!(malformed_data_url_replacement_with(&value, &mut not_enough_for_essence, |_| panic!("not admitted")).is_err()); + assert_eq!(not_enough_for_essence, 0); +} + +#[test] +fn many_parameter_urls_preserve_bodies_and_base64_disposition() { + let params = parameters(100_000); + for (suffix, body, expected) in [ + (";base64", "AA==", vec![0]), + ("; BASE64 ", "AA==", vec![0]), + (";p=last", "%00%26%27%22", vec![0, b'&', b'\'', b'"']), + (";p=last", "&'" z", b"&\'\"z".to_vec()), + ] { + let text = format!(""); + let repaired = repair(&text, &Limits { bytes: 2 << 20, ..Default::default() }).unwrap(); + let checked = super::super::preflight(&repaired, &Limits { bytes: 2 << 20, ..Default::default() }).unwrap(); + let value = checked.xml.descendants().find(|node| node.has_tag_name("image")).unwrap().attribute("href").unwrap(); + assert!(!value.contains(";p0=")); + assert_eq!(data_url::DataUrl::process(value).unwrap().decode_to_vec().unwrap().0, expected); + assert!(matches!(repair(&repaired, &Limits { bytes: 2 << 20, ..Default::default() }).unwrap(), Cow::Borrowed(_))); + } + let malformed = format!(""); + assert!(repair(&malformed, &Limits { bytes: 2 << 20, ..Default::default() }).unwrap().contains(INVALID_IMAGE)); +} + +#[test] +fn header_compaction_has_a_shared_nonrefundable_work_charge() { + let value = format!("data:image/png{};base64,AA==", parameters(100)); + let mut work = value.len(); + assert!(data_url_replacement(&value, &mut work).is_err()); + assert_eq!(work, 0, "the original parameter scan stays spent on refusal"); + let mut work = value.len() * 12; + assert!(data_url_replacement(&value, &mut work).unwrap().is_some()); + assert!(work < value.len() * 12 - value.len(), "compact parse, streaming decode and XML projection are charged too"); +} diff --git a/crates/svg/src/tree/tests.rs b/crates/svg/src/tree/tests.rs index abdf0f772..7663ab301 100644 --- a/crates/svg/src/tree/tests.rs +++ b/crates/svg/src/tree/tests.rs @@ -547,7 +547,7 @@ mod invalid_image_accounting { } #[test] - fn failed_decode_forgets_address_but_preserves_spent_budget() { + fn failed_decode_releases_input_but_preserves_spent_budget() { let valid = png(); let idat = valid.windows(4).position(|b| b == b"IDAT").unwrap(); let invalid = Arc::new(valid[..idat + 4].to_vec()); @@ -555,11 +555,12 @@ mod invalid_image_accounting { let limits = Limits { embedded_raster_pixels: 3, ..shallow() }; let resolver = ImageResolver::new(limits); let used = &resolver.used; - let format = craft_image_codecs::Format::Png; + let format = SvgRaster::Png; assert!(raster_image(used, &limits, invalid, format).is_none()); - assert!(!used.raster_data.lock().unwrap().contains(&pointer)); + assert!(!used.raster_data.lock().unwrap().contains_key(&pointer)); assert_eq!(resolver.usage().raster_pixels, 1); let first_data = Arc::new(valid.clone()); + let first_pointer = Arc::as_ptr(&first_data) as usize; let first = raster_image(used, &limits, first_data.clone(), format).unwrap(); assert_eq!(resolver.usage().raster_pixels, 2); let repeated = raster_image(used, &limits, first_data, format).unwrap(); @@ -569,6 +570,9 @@ mod invalid_image_accounting { assert!(raster_image(used, &limits, Arc::new(valid), format).is_none()); assert_eq!(resolver.usage().raster_limit_images, 1); drop((first, repeated, second)); + let seen = used.raster_data.lock().unwrap(); + let identity = seen.get(&first_pointer).expect("the cache holds the allocation identity"); + assert!(identity.upgrade().is_none(), "the cache must not retain encoded bytes"); } #[test] diff --git a/crates/svg/src/tree/validated.rs b/crates/svg/src/tree/validated.rs new file mode 100644 index 000000000..c6c591f5d --- /dev/null +++ b/crates/svg/src/tree/validated.rs @@ -0,0 +1,401 @@ +//! Opt-in complete image validation and checked-DOM normalization. +//! Legacy ImageResolver entry points retain their existing behavior. +use super::draw::{RasterValidationFailure, decode_raster, probe_raster_pixels}; +use super::{ + ChildRefusal, Error, ImageUsage, Limits, Preflight, child_total_depth, declared_raster, embedded_text, image_href, preflight, preflight_child, + usvg, +}; +use crate::svgz; +use std::sync::{Arc, Mutex, PoisonError}; + +struct State { + document_bytes: usize, + pixel_limit: u64, + pixels_left: u64, + svg_byte_limit: usize, + svg_bytes_left: usize, + svg_failure_exhausted: bool, + classification_bytes_left: usize, + classification_content_left: usize, + classification_text_bytes_left: usize, + encoded_left: usize, + invalid_raster: usize, + invalid_svg: usize, + raster_limits: usize, + svg_limits: usize, + svg_resource_limits: usize, + root_active: bool, + xml_depth: usize, + depth: usize, + svg_count: usize, + svg_depth: usize, +} + +impl State { + fn new(limits: Limits) -> Self { + Self { + document_bytes: limits.bytes, + pixel_limit: limits.embedded_raster_pixels, + pixels_left: limits.embedded_raster_pixels, + svg_byte_limit: limits.embedded_bytes, + svg_bytes_left: limits.embedded_bytes, + svg_failure_exhausted: false, + classification_bytes_left: limits.embedded_bytes, + classification_content_left: limits.embedded_bytes, + classification_text_bytes_left: limits.embedded_bytes, + encoded_left: limits.bytes.saturating_add(limits.embedded_bytes), + invalid_raster: 0, + invalid_svg: 0, + raster_limits: 0, + svg_limits: 0, + svg_resource_limits: 0, + root_active: false, + xml_depth: 0, + depth: 0, + svg_count: 0, + svg_depth: 0, + } + } + + fn usage(&self) -> ImageUsage { + ImageUsage { + raster_pixels: self.pixel_limit.saturating_sub(self.pixels_left), + embedded_svg_bytes: self.svg_byte_limit.saturating_sub(self.svg_bytes_left), + embedded_svg_count: self.svg_count, + embedded_svg_depth: self.svg_depth, + invalid_raster_images: self.invalid_raster, + raster_limit_images: self.raster_limits, + invalid_svg_images: self.invalid_svg, + svg_limit_images: self.svg_limits, + svg_resource_limit_images: self.svg_resource_limits, + } + } + + fn classify_after_svg_failure(&mut self, data: &[u8]) { + if self.svg_resource_limits != 0 { + self.svg_limits = self.svg_limits.saturating_add(1); + self.svg_resource_limits = self.svg_resource_limits.saturating_add(1); + return; + } + let compressed = data.starts_with(&[0x1f, 0x8b]); + let content_left = self.classification_content_left; + let limit = if compressed { content_left.min(self.classification_bytes_left) } else { content_left }.min(self.document_bytes); + if compressed && limit == 0 { + self.svg_limits = self.svg_limits.saturating_add(1); + self.svg_resource_limits = self.svg_resource_limits.saturating_add(1); + return; + } + let resource_refusal = match crate::svgz::inflate(data, limit) { + Ok(bytes) => { + if compressed { + self.classification_bytes_left = self.classification_bytes_left.saturating_sub(bytes.len()); + } + let Some(text_left) = self.classification_text_bytes_left.checked_sub(bytes.len()) else { + self.svg_limits = self.svg_limits.saturating_add(1); + self.svg_resource_limits = self.svg_resource_limits.saturating_add(1); + return; + }; + self.classification_text_bytes_left = text_left; + match crate::svgz::decode(&bytes) { + Ok(text) => text.len() > content_left || text.len() > self.document_bytes, + Err(_) => { + self.invalid_svg = self.invalid_svg.saturating_add(1); + return; + } + } + } + Err(crate::svgz::Error::TooLarge(_)) => { + if compressed { + self.classification_bytes_left = self.classification_bytes_left.saturating_sub(limit); + } + true + } + Err(error) => { + // Nested data was fully inflated before refusal. Other failures + // may have consumed the entire attempted inflation allowance. + if compressed { + let charged = match error { + crate::svgz::Error::Nested(bytes) => bytes, + _ => limit, + }; + self.classification_bytes_left = self.classification_bytes_left.saturating_sub(charged); + } + self.invalid_svg = self.invalid_svg.saturating_add(1); + return; + } + }; + self.svg_limits = self.svg_limits.saturating_add(1); + if resource_refusal { + self.svg_resource_limits = self.svg_resource_limits.saturating_add(1); + } + } +} + +/// One opt-in drawing's image work, normalization and exact rendering validation. +/// +/// Unlike legacy ImageResolver, every attempted raster callback spends encoded +/// and decoded work, even on failure. Rendering still decodes pixels afterwards: +/// resvg does not expose a decoded-pixel injection or decode-result hook. +/// All root and child conversions install this same resolver themselves. +#[derive(Clone)] +pub struct ValidatedImageResolver { + limits: Limits, + state: Arc>, +} + +impl ValidatedImageResolver { + /// Create a separate budget for one drawing. Encoded raster work is bounded + /// by root text bytes plus aggregate child SVG bytes, without retaining data. + pub fn new(limits: Limits) -> Self { + Self { limits, state: Arc::new(Mutex::new(State::new(limits))) } + } + + /// The immutable admission policy captured by this instance. Remaining work + /// never changes these limits; use them for the matching opaque preflight. + pub fn limits(&self) -> Limits { + self.limits + } + + /// Snapshot admitted work and optional-image omissions. + pub fn usage(&self) -> ImageUsage { + self.state.lock().unwrap_or_else(PoisonError::into_inner).usage() + } + + /// Convert exactly the DOM checked under these limits, with this instance's + /// resolver. A caller-supplied image callback cannot bypass image admission. + /// The caller still supplies the stack/worker and font policy. + pub fn parse_preflight(&self, checked: Preflight<'_>, mut options: usvg::Options<'_>) -> Result { + if checked.limits != self.limits { + return Err(Error::Parse("checked SVG limits differ from the validated image resolver".into())); + } + // A shallow source can expand through use/paint references. Reserve its + // expanded ancestor frames, excluding the callback leaf itself. The child + // contributes its complete checked height, as in legacy child admission. + let _scope = self.root_scope(checked.height.saturating_sub(1))?; + options.image_href_resolver = self.resolver(); + self.convert_checked(checked, &options) + } + + /// Resolve an already-decoded child SVG, charging original text before XML + /// parsing or repair. This is not an unchecked/precharged admission hook. + /// As with parse_preflight, unrelated image callbacks are replaced. + pub fn resolve_text(&self, text: &str, mut options: usvg::Options<'_>) -> Result, Error> { + let _scope = self.root_scope(0)?; + options.image_href_resolver = self.resolver(); + Ok(self.embedded_text(text, &options)) + } + + fn root_scope(&self, depth: usize) -> Result { + let mut state = self.state.lock().unwrap_or_else(PoisonError::into_inner); + if state.root_active { + return Err(Error::Parse("the validated SVG resolver is already converting a drawing".into())); + } + let previous = state.xml_depth; + state.root_active = true; + state.xml_depth = depth; + Ok(Scope { state: Arc::clone(&self.state), previous, child: false }) + } + + fn resolver(&self) -> usvg::ImageHrefResolver<'static> { + let this = self.clone(); + usvg::ImageHrefResolver { + resolve_string: Box::new(|_, _| None), + resolve_data: Box::new(move |mime, data, options| { + if let Some(format) = declared_raster(mime, &data) { + return this.raster_with(&data, |pixels| decode_raster(format, &data, pixels)).then(|| format.image_kind(data)); + } + if !matches!(mime, "image/svg+xml" | "text/plain") { + if mime.starts_with("image/") { + let mut state = this.state.lock().unwrap_or_else(PoisonError::into_inner); + state.invalid_raster = state.invalid_raster.saturating_add(1); + } + return None; + } + this.embedded_bytes(&data, options) + }), + } + } + + fn raster_with(&self, data: &[u8], decode: impl FnOnce(u64) -> Result<(), RasterValidationFailure>) -> bool { + let result = self.reserve_raster(data).and_then(decode); + if let Err(error) = result { + let mut state = self.state.lock().unwrap_or_else(PoisonError::into_inner); + match error { + RasterValidationFailure::Invalid => state.invalid_raster = state.invalid_raster.saturating_add(1), + RasterValidationFailure::Limit => state.raster_limits = state.raster_limits.saturating_add(1), + } + return false; + } + true + } + + fn reserve_raster(&self, data: &[u8]) -> Result { + { + let mut state = self.state.lock().unwrap_or_else(PoisonError::into_inner); + // usvg uses a fresh Arc for each data-URL callback. Each actual validation + // spends encoded work, including when the header or decoder refuses it. + state.encoded_left = state.encoded_left.checked_sub(data.len()).ok_or(RasterValidationFailure::Limit)?; + } + let pixels = probe_raster_pixels(data, self.limits.embedded_raster_pixels)?; + let mut state = self.state.lock().unwrap_or_else(PoisonError::into_inner); + state.pixels_left = state.pixels_left.checked_sub(pixels).ok_or(RasterValidationFailure::Limit)?; + // The caller decodes after releasing the mutex. The work stays charged. + Ok(pixels) + } + + fn can_start_child(&self) -> bool { + let mut state = self.state.lock().unwrap_or_else(PoisonError::into_inner); + if state.depth >= self.limits.embedded_depth { + state.svg_limits = state.svg_limits.saturating_add(1); + state.svg_resource_limits = state.svg_resource_limits.saturating_add(1); + return false; + } + true + } + + fn embedded_bytes(&self, data: &[u8], options: &usvg::Options<'_>) -> Option { + if !self.can_start_child() { + return None; + } + let room = { + let mut state = self.state.lock().unwrap_or_else(PoisonError::into_inner); + if state.svg_failure_exhausted { + state.classify_after_svg_failure(data); + return None; + } + state.svg_bytes_left + }; + let text = match embedded_text(data, room, &self.limits) { + Ok(text) => text, + Err(error) => { + let mut state = self.state.lock().unwrap_or_else(PoisonError::into_inner); + if matches!(error, svgz::Error::Encoding) && !crate::sniff::is_svgz(data) { + // Plain text validation can spend at most the input's bytes. + // An invalid encoding does not exhaust a later sibling's room. + state.svg_bytes_left = state.svg_bytes_left.saturating_sub(data.len()); + state.invalid_svg = state.invalid_svg.saturating_add(1); + return None; + } + state.svg_bytes_left = 0; + if matches!(error, svgz::Error::TooLarge(_)) { + state.svg_limits = state.svg_limits.saturating_add(1); + state.svg_resource_limits = state.svg_resource_limits.saturating_add(1); + } else { + // An unreadable compressed stream may have used the entire + // allowed inflation work. Later callbacks can only classify. + state.invalid_svg = state.invalid_svg.saturating_add(1); + state.svg_failure_exhausted = true; + state.classification_content_left = room; + } + return None; + } + }; + self.precharge_and_parse(&text, options) + } + + fn embedded_text(&self, text: &str, options: &usvg::Options<'_>) -> Option { + if !self.can_start_child() { + return None; + } + self.precharge_and_parse(text, options) + } + + fn precharge_and_parse(&self, text: &str, options: &usvg::Options<'_>) -> Option { + { + let mut state = self.state.lock().unwrap_or_else(PoisonError::into_inner); + let Some(left) = state.svg_bytes_left.checked_sub(text.len()) else { + state.svg_bytes_left = 0; + state.svg_limits = state.svg_limits.saturating_add(1); + state.svg_resource_limits = state.svg_resource_limits.saturating_add(1); + return None; + }; + // Original UTF-8 bytes/work are never refunded when repair shortens + // an attribute or conversion subsequently fails. + state.svg_bytes_left = left; + } + let checked = match preflight_child(text, &self.limits) { + Ok(checked) => checked, + Err(error) => { + let mut state = self.state.lock().unwrap_or_else(PoisonError::into_inner); + match error { + ChildRefusal::Depth => state.svg_limits = state.svg_limits.saturating_add(1), + ChildRefusal::Bytes => { + state.svg_limits = state.svg_limits.saturating_add(1); + state.svg_resource_limits = state.svg_resource_limits.saturating_add(1); + } + ChildRefusal::Invalid => state.invalid_svg = state.invalid_svg.saturating_add(1), + } + return None; + } + }; + let scope = { + let mut state = self.state.lock().unwrap_or_else(PoisonError::into_inner); + let Some(total) = child_total_depth(state.xml_depth, &checked, &self.limits) else { + state.svg_limits = state.svg_limits.saturating_add(1); + return None; + }; + let previous = state.xml_depth; + // Admission includes the child's complete height, but recursive + // conversion reserves ancestors only, not its image callback leaf. + state.xml_depth = total.saturating_sub(1); + state.depth = state.depth.saturating_add(1); + state.svg_depth = state.svg_depth.max(state.depth); + Scope { state: Arc::clone(&self.state), previous, child: true } + }; + let result = self.convert_checked(checked, options); + drop(scope); + let mut state = self.state.lock().unwrap_or_else(PoisonError::into_inner); + match result { + Ok(tree) => { + state.svg_count = state.svg_count.saturating_add(1); + Some(usvg::ImageKind::SVG(tree)) + } + Err(_) => { + state.invalid_svg = state.invalid_svg.saturating_add(1); + None + } + } + } + fn convert_checked(&self, checked: Preflight<'_>, options: &usvg::Options<'_>) -> Result { + match image_href::normalize(&checked) { + Ok(std::borrow::Cow::Borrowed(_)) => checked.parse(options), + Ok(std::borrow::Cow::Owned(text)) => { + let limits = checked.limits; + drop(checked); + preflight(&text, &limits)?.parse(options) + } + Err(repair_error) => { + // Keep the original checked DOM if its URLs are safe to pass to + // usvg. Malformed optional URLs are dropped by its parser. + let omitted = image_href::unrepaired_image_omissions(&checked).ok_or(repair_error)?; + let tree = checked.parse(options)?; + let mut state = self.state.lock().unwrap_or_else(PoisonError::into_inner); + state.invalid_raster = state.invalid_raster.saturating_add(omitted); + Ok(tree) + } + } + } +} + +struct Scope { + state: Arc>, + previous: usize, + child: bool, +} +impl Drop for Scope { + fn drop(&mut self) { + let mut state = self.state.lock().unwrap_or_else(PoisonError::into_inner); + state.xml_depth = self.previous; + if self.child { + state.depth = state.depth.saturating_sub(1); + } else { + state.root_active = false; + } + } +} + +#[cfg(test)] +mod tests; + +#[cfg(test)] +mod xml_tests; diff --git a/crates/svg/src/tree/validated/tests.rs b/crates/svg/src/tree/validated/tests.rs new file mode 100644 index 000000000..7eaa47dab --- /dev/null +++ b/crates/svg/src/tree/validated/tests.rs @@ -0,0 +1,352 @@ +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::indexing_slicing)] +use super::*; +use std::io::Write; +const PIXELS: u64 = 24_000_000; +const SVG_BYTES: usize = 16 << 20; + +fn test_limits(pixels: u64) -> Limits { + Limits { + bytes: 1 << 30, + depth: 513, + tree_depth: 513, + embedded_bytes: SVG_BYTES, + embedded_depth: 32, + embedded_raster_pixels: pixels, + ..Default::default() + } +} +fn wrapped_options(pixels: u64) -> (usvg::Options<'static>, Arc>, Arc) { + let resolver = Arc::new(ValidatedImageResolver::new(test_limits(pixels))); + let options = usvg::Options { image_href_resolver: resolver.resolver(), ..Default::default() }; + (options, Arc::clone(&resolver.state), resolver) +} +fn on_test_stack(run: impl FnOnce() -> Result + Send) -> Result { + std::thread::scope(|scope| std::thread::Builder::new().stack_size(16 << 20).spawn_scoped(scope, run).unwrap().join().unwrap()) +} +fn png() -> Vec { + let image = craft_image_codecs::Image::from_u8(2, 2, craft_image_codecs::ChannelLayout::Rgb, [17, 83, 149].repeat(4)).unwrap(); + craft_image_codecs::encode(&image, craft_image_codecs::Format::Png, &Default::default()).unwrap() +} +fn data_url(mime: &str, bytes: &[u8]) -> String { + use std::fmt::Write; + let mut result = format!("data:{mime},"); + for byte in bytes { + write!(&mut result, "%{byte:02X}").unwrap(); + } + result +} + +#[test] +fn original_nested_xml_bytes_are_not_refunded_when_malformed_hrefs_shrink() { + on_test_stack(|| { + let (options, state, _resolver) = wrapped_options(PIXELS); + let prefix = ""; + let original_size = 3 << 20; + let source = Arc::new([prefix.as_bytes(), &vec![b'A'; original_size - prefix.len() - suffix.len()], suffix.as_bytes()].concat()); + assert_eq!(source.len(), original_size); + for accepted in 1..=5 { + let kind = (options.image_href_resolver.resolve_data)("image/svg+xml", Arc::clone(&source), &options); + assert!(matches!(kind, Some(usvg::ImageKind::SVG(_))), "source {accepted} fits the original-text allowance"); + drop(kind); + let state = state.lock().unwrap_or_else(PoisonError::into_inner); + assert_eq!(state.svg_bytes_left, SVG_BYTES - accepted * original_size); + assert!(!state.svg_failure_exhausted); + assert_eq!(state.classification_bytes_left, SVG_BYTES); + let usage = state.usage(); + assert_eq!(usage.embedded_svg_bytes, accepted * original_size, "the short replacement cannot refund original XML work"); + } + assert!((options.image_href_resolver.resolve_data)("image/svg+xml", Arc::clone(&source), &options).is_none()); + let state = state.lock().unwrap_or_else(PoisonError::into_inner); + assert_eq!(state.svg_bytes_left, 0); + assert_eq!((state.svg_limits, state.svg_resource_limits), (1, 1)); + assert!(!state.svg_failure_exhausted, "ordinary byte exhaustion cannot enter compatibility classification"); + assert_eq!(state.classification_bytes_left, SVG_BYTES); + assert_eq!(state.usage().svg_resource_limit_images, 1); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn invalid_utf8_gzip_consumes_remaining_svg_work_before_the_next_child() { + on_test_stack(|| { + let (options, state, _resolver) = wrapped_options(PIXELS); + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + encoder.write_all(&[0xff; 4096]).unwrap(); + let invalid = Arc::new(encoder.finish().unwrap()); + assert!((options.image_href_resolver.resolve_data)("image/svg+xml", invalid, &options).is_none()); + { + let state = state.lock().unwrap_or_else(PoisonError::into_inner); + assert_eq!(state.svg_bytes_left, 0, "a failed decode cannot retry the full work allowance"); + assert_eq!(state.invalid_svg, 1); + assert!(state.svg_failure_exhausted); + assert_eq!(state.svg_resource_limits, 0); + assert_eq!(state.classification_bytes_left, SVG_BYTES); + } + let valid = Arc::new(b"".to_vec()); + let valid_size = valid.len(); + assert!((options.image_href_resolver.resolve_data)("image/svg+xml", valid, &options).is_none()); + let state = state.lock().unwrap_or_else(PoisonError::into_inner); + assert_eq!((state.invalid_svg, state.svg_limits, state.svg_resource_limits), (1, 1, 0)); + assert_eq!(state.classification_bytes_left, SVG_BYTES, "plain input does not spend retry inflation work"); + let usage = state.usage(); + assert_eq!(usage.embedded_svg_bytes, SVG_BYTES); + assert_eq!(usage.svg_resource_limit_images, 0, "a later small child is omitted, not a resource error"); + let charged = 2 * SVG_BYTES - state.svg_bytes_left - state.classification_bytes_left; + assert_eq!(charged, SVG_BYTES); + assert!(valid_size < SVG_BYTES); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn post_failure_classification_is_cumulative_and_never_admits_later_children() { + on_test_stack(|| { + let (options, state, _resolver) = wrapped_options(PIXELS); + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + encoder.write_all(&[0xff; 256]).unwrap(); + assert!((options.image_href_resolver.resolve_data)("image/svg+xml", Arc::new(encoder.finish().unwrap()), &options).is_none()); + let text = b""; + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + encoder.write_all(text).unwrap(); + let valid = Arc::new(encoder.finish().unwrap()); + // Exercise the cumulative inflated-byte boundary with a small injected + // second pool; public tests cover the full 16 MiB allowance. + let classification_limit = text.len() * 2 - 1; + state.lock().unwrap_or_else(PoisonError::into_inner).classification_bytes_left = classification_limit; + assert!((options.image_href_resolver.resolve_data)("image/svg+xml", Arc::clone(&valid), &options).is_none()); + { + let state = state.lock().unwrap_or_else(PoisonError::into_inner); + assert_eq!(state.classification_bytes_left, text.len() - 1); + assert_eq!((state.svg_limits, state.svg_resource_limits), (1, 0)); + } + assert!((options.image_href_resolver.resolve_data)("image/svg+xml", Arc::clone(&valid), &options).is_none()); + let state = state.lock().unwrap_or_else(PoisonError::into_inner); + assert_eq!(state.classification_bytes_left, 0); + assert_eq!((state.svg_limits, state.svg_resource_limits), (2, 1)); + assert_eq!(state.svg_bytes_left, 0, "classification never refunds original failed work"); + assert!(state.svg_failure_exhausted); + let charged = 2 * SVG_BYTES - state.svg_bytes_left - state.classification_bytes_left; + assert_eq!(charged, 2 * SVG_BYTES, "original content plus aggregate retry inflation allowances remain bounded"); + assert_eq!(state.usage().embedded_svg_count, 0, "classified children must never reach the shared parser"); + Ok(()) + }) + .unwrap(); +} + +#[test] +fn retry_inflation_room_and_saved_decoded_content_room_remain_independent() { + on_test_stack(|| { + let mut latin1 = b""); + let decoded_len = crate::svgz::decode(&latin1).unwrap().len(); + assert!(latin1.len() < 256 && decoded_len > 256 && decoded_len < 512); + let gzip = |bytes: &[u8]| { + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + encoder.write_all(bytes).unwrap(); + Arc::new(encoder.finish().unwrap()) + }; + for (content_room, expected_resource) in [(512usize, false), (256, true)] { + let (options, state, _resolver) = wrapped_options(PIXELS); + // Model preceding genuine content consumption before an unreadable + // stream burns the rest. Classification must save this exact room. + state.lock().unwrap_or_else(PoisonError::into_inner).svg_bytes_left = content_room; + assert!((options.image_href_resolver.resolve_data)("image/svg+xml", gzip(&[0xff; 4]), &options).is_none()); + { + let mut state = state.lock().unwrap_or_else(PoisonError::into_inner); + assert_eq!(state.classification_content_left, content_room); + assert_eq!(state.svg_bytes_left, 0); + // Enough compressed retry work for the raw Latin-1 bytes, but + // less than their decoded UTF-8 length, just like main's case. + state.classification_bytes_left = latin1.len() + 1; + } + assert!((options.image_href_resolver.resolve_data)("image/svg+xml", gzip(&latin1), &options).is_none()); + let state = state.lock().unwrap_or_else(PoisonError::into_inner); + assert_eq!(state.classification_bytes_left, 1, "retry work charges inflated bytes, not UTF-8 bytes"); + assert_eq!(state.svg_resource_limits, usize::from(expected_resource)); + assert_eq!(state.svg_bytes_left, 0, "classification never restores original content allowance"); + assert_eq!(state.usage().embedded_svg_count, 0, "no classified child is parsed or drawn"); + } + Ok(()) + }) + .unwrap(); +} + +#[test] +fn failed_raster_callbacks_release_sources_and_keep_every_decode_charge() { + let mut broken = png(); + let idat = broken.windows(4).position(|bytes| bytes == b"IDAT").unwrap(); + broken[idat + 4] ^= 1; + assert_eq!(craft_image_codecs::probe_allocation_size(&broken).unwrap(), (2, 2)); + let (options, state, resolver) = wrapped_options(8); + let encoded_before = state.lock().unwrap().encoded_left; + for attempt in 1..=2 { + // Match usvg's actual callback boundary: each href gets a new Arc. + let encoded = Arc::new(broken.clone()); + let weak = Arc::downgrade(&encoded); + assert!((options.image_href_resolver.resolve_data)("image/png", Arc::clone(&encoded), &options).is_none()); + let usage = resolver.usage(); + assert_eq!(usage.raster_pixels, attempt * 4); + assert_eq!(state.lock().unwrap().encoded_left, encoded_before - attempt as usize * encoded.len()); + assert_eq!(usage.invalid_raster_images, attempt as usize); + drop(encoded); + assert!(weak.upgrade().is_none(), "validation must not retain failed callback buffers until parse completion"); + } + assert!((options.image_href_resolver.resolve_data)("image/png", Arc::new(png()), &options).is_none()); + let usage = resolver.usage(); + assert_eq!((usage.invalid_raster_images, usage.raster_limit_images, usage.raster_pixels), (2, 1, 8)); +} + +#[test] +fn valid_raster_callbacks_spend_exact_pixel_and_encoded_allowances_per_call() { + let encoded = Arc::new(png()); + let (options, state, resolver) = wrapped_options(8); + let call = || (options.image_href_resolver.resolve_data)("image/png", Arc::clone(&encoded), &options); + state.lock().unwrap().encoded_left = encoded.len() * 2; + assert!(call().is_some()); + assert_eq!(resolver.usage().raster_pixels, 4); + assert_eq!(state.lock().unwrap().encoded_left, encoded.len()); + assert!(call().is_some(), "even a directly reused Arc is a new actual validation call"); + assert_eq!(state.lock().unwrap().encoded_left, 0); + assert!(call().is_none()); + let usage = resolver.usage(); + assert_eq!((usage.raster_pixels, usage.invalid_raster_images, usage.raster_limit_images), (8, 0, 1)); +} + +#[test] +fn raster_decode_happens_after_the_work_is_charged_and_mutex_released() { + let (_, state, resolver) = wrapped_options(4); + let image = png(); + assert!(resolver.raster_with(&image, |admitted| { + assert_eq!(admitted, 4); + assert!(state.try_lock().is_ok(), "a usage snapshot must not wait for the raster decode"); + assert_eq!(resolver.usage().raster_pixels, 4); + Ok(()) + })); +} + +#[test] +fn raster_callbacks_own_per_call_pixel_charges_and_keep_mime_aliases_separate() { + let png = Arc::new(png()); + let (options, state, resolver) = wrapped_options(4); + assert!((options.image_href_resolver.resolve_data)("image/png", Arc::clone(&png), &options).is_some()); + assert!((options.image_href_resolver.resolve_data)("image/png", Arc::clone(&png), &options).is_none()); + assert_eq!((resolver.usage().raster_pixels, resolver.usage().invalid_raster_images, resolver.usage().raster_limit_images), (4, 0, 1)); + assert!((options.image_href_resolver.resolve_data)("image/jpeg", png, &options).is_none()); + let usage = state.lock().unwrap().usage(); + assert_eq!((usage.raster_pixels, usage.invalid_raster_images), (4, 1)); +} + +#[test] +fn encoded_raster_refusal_is_per_image_and_does_not_consume_unattempted_work() { + let bytes = png(); + let mut oversized = bytes.clone(); + oversized.extend([0; 32]); + let (options, state, resolver) = wrapped_options(4); + state.lock().unwrap().encoded_left = bytes.len(); + assert!((options.image_href_resolver.resolve_data)("image/png", Arc::new(oversized), &options).is_none()); + assert_eq!(state.lock().unwrap().encoded_left, bytes.len()); + assert_eq!(resolver.usage().raster_pixels, 0); + assert!((options.image_href_resolver.resolve_data)("image/png", Arc::new(bytes), &options).is_some()); + assert_eq!(state.lock().unwrap().encoded_left, 0); + let usage = resolver.usage(); + assert_eq!((usage.raster_pixels, usage.raster_limit_images, usage.invalid_raster_images), (4, 1, 0)); +} + +#[test] +fn encoded_raster_exhaustion_keeps_ordinary_drawing_and_reports_only_omitted_images() { + let bytes = png(); + let href = data_url("image/png", &bytes); + let text = format!( + r##""## + ); + let resolver = ValidatedImageResolver::new(test_limits(8)); + resolver.state.lock().unwrap().encoded_left = bytes.len(); + let tree = resolver.parse_preflight(preflight(&text, &resolver.limits()).unwrap(), usvg::Options::default()).unwrap(); + let usage = resolver.usage(); + assert_eq!((usage.raster_pixels, usage.raster_limit_images, usage.invalid_raster_images), (4, 1, 0)); + let output = super::super::draw::render(&tree, usvg::Transform::identity(), 4, 2, 0, 1 << 20).unwrap(); + let background = output.pixel(3, 1).unwrap().demultiply(); + assert_eq!((background.red(), background.green(), background.blue(), background.alpha()), (0, 255, 0, 255)); + let pixel = output.pixel(0, 0).unwrap().demultiply(); + assert_eq!((pixel.red(), pixel.green(), pixel.blue(), pixel.alpha()), (17, 83, 149, 255)); + assert_eq!(resolver.state.lock().unwrap().encoded_left, 0); +} + +// Preserve upstream's text_rechecked boundary using the integrated real resolver. +// Plain and gzip retries spend source bytes before decoding, independently of +// inflation work and the drawing's genuine decoded-content capacity. +#[test] +fn plain_legacy_encoding_rechecks_have_a_separate_bounded_work_allowance() { + on_test_stack(|| { + let xml = b""; + for compressed in [false, true] { + let (options, state, _resolver) = wrapped_options(PIXELS); + let encode = |bytes: &[u8]| { + if compressed { + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + encoder.write_all(bytes).unwrap(); + encoder.finish().unwrap() + } else { + bytes.to_vec() + } + }; + // A failed compressed decode exhausts the primary pool. The + // callbacks under test may use plain or compressed encodings. + let invalid = Arc::new(crate::svgz::compress(&[0xff])); + assert!((options.image_href_resolver.resolve_data)("image/svg+xml", invalid, &options).is_none()); + state.lock().unwrap_or_else(PoisonError::into_inner).classification_text_bytes_left = xml.len(); + assert!((options.image_href_resolver.resolve_data)("image/svg+xml", Arc::new(encode(xml)), &options).is_none()); + { + let state = state.lock().unwrap_or_else(PoisonError::into_inner); + assert_eq!(state.classification_text_bytes_left, 0); + assert_eq!(state.svg_resource_limits, 0, "exactly fitting text recheck is permitted"); + assert_eq!(state.classification_bytes_left, SVG_BYTES - usize::from(compressed) * xml.len()); + } + assert!((options.image_href_resolver.resolve_data)("image/svg+xml", Arc::new(encode(xml)), &options).is_none()); + let state = state.lock().unwrap_or_else(PoisonError::into_inner); + assert_eq!(state.classification_text_bytes_left, 0); + assert_eq!(state.svg_resource_limits, 1, "the next source cannot repeat text-conversion work"); + assert_eq!(state.invalid_svg, 1, "the original encoding error stays invalid"); + assert_eq!(state.svg_limits, 2); + assert_eq!(state.usage().embedded_svg_count, 0, "rechecked children are never admitted"); + } + Ok(()) + }) + .unwrap(); +} + +#[test] +fn embedded_gzip_obeys_document_bytes_before_aggregate_inflation_room() { + on_test_stack(|| { + let mut text = b"".len(), b' '); + text.extend_from_slice(b"-->"); + assert_eq!(text.len(), 256); + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + encoder.write_all(&text).unwrap(); + let encoded = Arc::new(encoder.finish().unwrap()); + for bytes in [text.len() - 1, text.len()] { + let limits = Limits { bytes, embedded_bytes: 1024, ..test_limits(4) }; + assert!(limits.embedded_bytes > text.len()); + let resolver = ValidatedImageResolver::new(limits); + let options = usvg::Options { image_href_resolver: resolver.resolver(), ..Default::default() }; + let image = (options.image_href_resolver.resolve_data)("image/svg+xml", Arc::clone(&encoded), &options); + let usage = resolver.usage(); + if bytes == text.len() { + assert!(matches!(image, Some(usvg::ImageKind::SVG(_))), "exact document boundary remains admissible"); + assert_eq!((usage.embedded_svg_count, usage.embedded_svg_bytes), (1, text.len())); + assert_eq!((usage.svg_resource_limit_images, usage.invalid_svg_images), (0, 0)); + } else { + assert!(image.is_none(), "aggregate room must not lift one document's inflate limit"); + assert_eq!((usage.embedded_svg_count, usage.svg_resource_limit_images, usage.invalid_svg_images), (0, 1, 0)); + assert_eq!(usage.embedded_svg_bytes, 1024, "refused work retains its aggregate charge"); + } + } + Ok(()) + }) + .unwrap(); +} diff --git a/crates/svg/src/tree/validated/xml_tests.rs b/crates/svg/src/tree/validated/xml_tests.rs new file mode 100644 index 000000000..e156c42e6 --- /dev/null +++ b/crates/svg/src/tree/validated/xml_tests.rs @@ -0,0 +1,250 @@ +//! Public validated-context contracts; private byte/raster accounting lives in tests.rs. +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::indexing_slicing)] +use super::super::{Limits, PREFLIGHT_PASSES, preflight, usvg}; +use super::ValidatedImageResolver; +use std::io::Write; +use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, +}; + +const NS: &str = "http://www.w3.org/2000/svg"; + +fn drawing(contents: &str) -> String { + format!("{contents}") +} + +fn url(bytes: &[u8]) -> String { + use std::fmt::Write; + let mut value = String::from("data:image/svg+xml,"); + for byte in bytes { + write!(&mut value, "%{byte:02X}").unwrap(); + } + value +} + +fn gzip(bytes: &[u8]) -> Vec { + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + encoder.write_all(bytes).unwrap(); + encoder.finish().unwrap() +} + +fn image(href: &str) -> String { + format!("") +} + +fn parse_count(text: &str, limits: Limits) -> (super::super::ImageUsage, usize) { + PREFLIGHT_PASSES.with(|count| count.set(0)); + let context = ValidatedImageResolver::new(limits); + let checked = preflight(text, &limits).unwrap(); + let tree = context.parse_preflight(checked, usvg::Options::default()).unwrap(); + assert!(!tree.root().children().is_empty(), "the visible drawing survives validation"); + (context.usage(), PREFLIGHT_PASSES.with(|count| count.get())) +} + +#[test] +fn malformed_plain_child_leaves_work_for_valid_siblings() { + let child = drawing(""); + for invalid in [&[0xff][..], &[0xff, 0xfe, 0xd8][..]] { + let contents = format!("{}{}", image(&url(invalid)), image(&url(child.as_bytes()))); + let root = drawing(&contents); + let limits = Limits { embedded_bytes: invalid.len() + child.len(), ..Default::default() }; + let context = ValidatedImageResolver::new(limits); + let tree = context.parse_preflight(preflight(&root, &limits).unwrap(), usvg::Options::default()).unwrap(); + let usage = context.usage(); + assert_eq!((usage.invalid_svg_images, usage.embedded_svg_count, usage.svg_limit_images), (1, 1, 0)); + assert_eq!(usage.embedded_svg_bytes, limits.embedded_bytes); + let mut pixmap = super::super::draw::tiny_skia::Pixmap::new(4, 4).unwrap(); + super::super::draw::resvg::render(&tree, Default::default(), &mut pixmap.as_mut()); + assert_eq!(pixmap.pixel(2, 2).unwrap().red(), 255); + } +} + +#[test] +fn a_malformed_image_at_the_svg_byte_cap_does_not_drop_the_drawing() { + let root = drawing(""); + let limits = Limits { bytes: root.len(), ..Default::default() }; + let context = ValidatedImageResolver::new(limits); + let tree = context.parse_preflight(preflight(&root, &limits).unwrap(), usvg::Options::default()).unwrap(); + assert_eq!(context.usage().invalid_raster_images, 1); + let mut pixmap = super::super::draw::tiny_skia::Pixmap::new(4, 4).unwrap(); + super::super::draw::resvg::render(&tree, Default::default(), &mut pixmap.as_mut()); + assert_eq!(pixmap.pixel(2, 2).unwrap().red(), 255); +} + +#[test] +fn validated_root_reuses_checked_xml_and_rechecks_only_changed_attributes() { + let ordinary = drawing(""); + let repaired = drawing(""); + let (usage, count) = parse_count(&ordinary, Limits::default()); + assert_eq!(count, 1); + assert_eq!(usage.invalid_raster_images, 0); + let (usage, count) = parse_count(&repaired, Limits::default()); + assert_eq!(count, 2); + assert_eq!(usage.invalid_raster_images, 1); +} + +#[test] +fn validated_plain_and_gzip_children_reuse_their_original_preflight() { + for repaired in [false, true] { + let mut contents = String::from(""); + if repaired { + contents.push_str(""); + } + let child = drawing(&contents); + for compressed in [false, true] { + let encoded = if compressed { gzip(child.as_bytes()) } else { child.as_bytes().to_vec() }; + let root = drawing(&image(&url(&encoded))); + let (usage, count) = parse_count(&root, Limits::default()); + assert_eq!(count, if repaired { 3 } else { 2 }, "compressed={compressed}, repaired={repaired}"); + assert_eq!(usage.embedded_svg_count, 1); + assert_eq!(usage.embedded_svg_bytes, child.len()); + assert_eq!(usage.invalid_raster_images, usize::from(repaired)); + assert_eq!(usage.invalid_svg_images, 0); + } + } +} + +#[test] +fn validated_original_xml_guards_precede_normalization_and_conversion() { + let normal = drawing(""); + let entity = format!("]>{}", drawing("")); + let markup = format!("'>]>{}", drawing("&a;")); + let referenced = drawing(""); + for (text, limits) in [ + (&normal, Limits { bytes: normal.len() - 1, ..Default::default() }), + (&normal, Limits { attributes: 0, ..Default::default() }), + (&normal, Limits { xml_nodes: 1, ..Default::default() }), + (&entity, Limits { entity_bytes: 1, ..Default::default() }), + (&markup, Limits { refuse_entity_markup: true, ..Default::default() }), + (&referenced, Limits { tree_depth: 2, ..Default::default() }), + ] { + assert!(preflight(text, &limits).is_err(), "original source must be refused before normalization: {text}"); + } + let limits = Limits { entity_bytes: 1024, ..Default::default() }; + let context = ValidatedImageResolver::new(limits); + let checked = preflight(&entity, &limits).unwrap(); + context.parse_preflight(checked, usvg::Options::default()).unwrap(); + assert_eq!(context.usage().invalid_raster_images, 1, "admitted entity data reaches omission accounting"); +} + +#[test] +fn validated_context_requires_exact_preflight_limits() { + let text = drawing(""); + let limits = Limits { bytes: 4096, tree_depth: 16, embedded_bytes: 2048, ..Default::default() }; + let context = ValidatedImageResolver::new(limits); + assert_eq!(context.limits(), limits); + for changed in [ + Limits { bytes: 4095, ..limits }, + Limits { bytes: 4097, ..limits }, + Limits { tree_depth: 15, ..limits }, + Limits { embedded_bytes: 2049, ..limits }, + ] { + let checked = preflight(&text, &changed).unwrap(); + assert!(context.parse_preflight(checked, usvg::Options::default()).is_err()); + assert_eq!(context.usage(), Default::default(), "policy mismatch must not start conversion"); + } + context.parse_preflight(preflight(&text, &limits).unwrap(), usvg::Options::default()).unwrap(); +} + +fn hostile_options(calls: &Arc) -> usvg::Options<'static> { + let string_calls = Arc::clone(calls); + let data_calls = Arc::clone(calls); + usvg::Options { + image_href_resolver: usvg::ImageHrefResolver { + resolve_string: Box::new(move |_, _| { + string_calls.fetch_add(1, Ordering::SeqCst); + panic!("ambient string resolver must not run") + }), + resolve_data: Box::new(move |_, _, _| { + data_calls.fetch_add(1, Ordering::SeqCst); + panic!("caller data resolver must not bypass validation") + }), + }, + ..Default::default() + } +} + +#[test] +fn validated_root_and_nested_context_override_untrusted_image_callbacks() { + let calls = Arc::new(AtomicUsize::new(0)); + let child = drawing( + "", + ); + let root = drawing(&format!("{}", image(&url(child.as_bytes())))); + let limits = Limits::default(); + let context = ValidatedImageResolver::new(limits); + context.parse_preflight(preflight(&root, &limits).unwrap(), hostile_options(&calls)).unwrap(); + assert_eq!(calls.load(Ordering::SeqCst), 0); + assert_eq!(context.usage().embedded_svg_count, 1); + assert_eq!(context.usage().invalid_raster_images, 1); + let direct = ValidatedImageResolver::new(limits); + assert!(matches!(direct.resolve_text(&child, hostile_options(&calls)).unwrap(), Some(usvg::ImageKind::SVG(_)))); + assert_eq!(calls.load(Ordering::SeqCst), 0); + assert_eq!(direct.usage().invalid_raster_images, 1); +} + +#[test] +fn expanded_root_references_and_child_share_the_converted_depth_limit() { + let child = drawing(""); // height 3 + let href = url(child.as_bytes()); + let root = drawing(&format!("{}", image(&href))); // height 6 + assert!(preflight(&root, &Limits { tree_depth: 5, ..Default::default() }).is_err()); + assert!(preflight(&root, &Limits { tree_depth: 6, ..Default::default() }).is_ok()); + assert!(preflight(&child, &Limits { tree_depth: 3, ..Default::default() }).is_ok()); + // Expanded root height6 includes the callback leaf: reserve5 ancestors, + // then admit the child's complete height3. Source depth alone would wrongly + // accept the cap7 case, while reserving the leaf twice would reject cap8. + for (depth, admitted) in [(7, false), (8, true)] { + let limits = Limits { tree_depth: depth, ..Default::default() }; + let checked = preflight(&root, &limits).unwrap(); + assert!(checked.depth() < 6, "source nesting is shallower than expanded references"); + let context = ValidatedImageResolver::new(limits); + context.parse_preflight(checked, usvg::Options::default()).unwrap(); + let usage = context.usage(); + assert_eq!(usage.embedded_svg_count, usize::from(admitted)); + assert_eq!(usage.svg_limit_images, usize::from(!admitted)); + assert_eq!(usage.invalid_svg_images, 0); + } +} + +#[test] +fn plain_root_callback_leaf_does_not_spend_the_child_boundary_twice() { + let child = drawing(""); // height3 + let root = drawing(&image(&url(child.as_bytes()))); // height2, one ancestor + for (depth, admitted) in [(3, false), (4, true)] { + let limits = Limits { tree_depth: depth, ..Default::default() }; + assert!(preflight(&root, &limits).is_ok()); + assert!(preflight(&child, &limits).is_ok()); + let context = ValidatedImageResolver::new(limits); + let tree = context.parse_preflight(preflight(&root, &limits).unwrap(), usvg::Options::default()).unwrap(); + assert_eq!(context.usage().embedded_svg_count, usize::from(admitted)); + assert_eq!(context.usage().svg_limit_images, usize::from(!admitted)); + assert_eq!(context.usage().invalid_svg_images, 0); + let pixels = super::super::draw::render(&tree, usvg::Transform::identity(), 4, 4, 0, 1 << 20).unwrap(); + let sample = pixels.pixel(1, 1).unwrap(); + assert_eq!((sample.red(), sample.green(), sample.blue()), (0, 0, 0)); + assert_eq!(sample.alpha(), if admitted { 255 } else { 0 }, "exact-boundary child must actually draw"); + } +} + +#[test] +fn nested_callback_leaves_do_not_spend_the_descendant_boundary_twice() { + let leaf = drawing(""); // height2 + let middle = drawing(&image(&url(leaf.as_bytes()))); // height2 + let root = drawing(&image(&url(middle.as_bytes()))); // height2 + // Root and middle each contribute one ancestor, then the leaf's full + // height2: the visible rectangle fits exactly at depth4, but not depth3. + for (depth, admitted) in [(3, false), (4, true)] { + let limits = Limits { tree_depth: depth, ..Default::default() }; + let context = ValidatedImageResolver::new(limits); + let tree = context.parse_preflight(preflight(&root, &limits).unwrap(), usvg::Options::default()).unwrap(); + let usage = context.usage(); + assert_eq!(usage.embedded_svg_count, if admitted { 2 } else { 1 }); + assert_eq!(usage.svg_limit_images, usize::from(!admitted)); + assert_eq!(usage.invalid_svg_images, 0); + let pixels = super::super::draw::render(&tree, usvg::Transform::identity(), 4, 4, 0, 1 << 20).unwrap(); + let sample = pixels.pixel(1, 1).unwrap(); + assert_eq!(sample.alpha(), if admitted { 255 } else { 0 }, "exact-boundary grandchild must actually draw"); + } +} diff --git a/crates/tar/Cargo.toml b/crates/tar/Cargo.toml index e0628a35a..e76bb9e4c 100644 --- a/crates/tar/Cargo.toml +++ b/crates/tar/Cargo.toml @@ -16,3 +16,6 @@ serde = { workspace = true, optional = true } [features] serde = ["dep:serde"] + +[dev-dependencies] +craft-testkit = { workspace = true } diff --git a/crates/tar/src/testkit.rs b/crates/tar/src/testkit.rs index 828644a4b..f435b55e5 100644 --- a/crates/tar/src/testkit.rs +++ b/crates/tar/src/testkit.rs @@ -92,44 +92,4 @@ pub fn cpio_newc(files: &[(&str, &[u8], u32)]) -> Vec { out } -/// Deterministic pseudo-random bytes (xorshift) for fuzz-like loops. -pub fn noise(seed: u64, len: usize) -> Vec { - let mut x = seed.wrapping_mul(0x9e37_79b9_7f4a_7c15) | 1; - (0..len) - .map(|_| { - x ^= x << 13; - x ^= x >> 7; - x ^= x << 17; - x as u8 - }) - .collect() -} - -/// Every way to damage a sample used by robustness tests: truncations at many lengths and -/// single-byte corruptions spread across the file. -pub fn mutations(sample: &[u8]) -> Vec> { - let mut out = Vec::new(); - let n = sample.len(); - let step = (n / 64).max(1); - let mut len = 0; - while len < n { - out.push(sample[..len].to_vec()); - len += step; - } - for (i, seed) in (0..n).step_by(step).zip(1u64..) { - let mut m = sample.to_vec(); - let noise = noise(seed, 8); - for (j, v) in noise.iter().enumerate() { - if let Some(b) = m.get_mut(i + j) { - *b = *v; - } - } - out.push(m); - let mut m = sample.to_vec(); - if let Some(b) = m.get_mut(i) { - *b = 0xff; - } - out.push(m); - } - out -} +pub use craft_testkit::sample_damage::{mutations, noise}; diff --git a/crates/testkit/README.md b/crates/testkit/README.md index 8b8e9b208..1f58a7af9 100644 --- a/crates/testkit/README.md +++ b/crates/testkit/README.md @@ -192,6 +192,16 @@ A product of the iterations and the multiplier that does not fit in a `usize`, a more than `MAX_CASES` (10,000,000) cases in all, and a limit variable that `deadline` would reject also fail the test before `parse` is called. +### Quick runs + +`CRAFT_HOSTILE_DIVISOR=N` runs a fraction of every sweep: the unmodified seed, every N-th truncation +(lengths 0, N, 2N, ...) and the rounds divided by N, rounded up. It takes the same values as the +multiplier. The Workspace gate sets it on every run and the nightly run leaves it unset, so every +case still runs once a day. A sweep written without `hammer` reads it with `hostile::divisor()` and +runs the cases whose index plus an offset is a multiple of it, with a different offset for each input +so that the inputs together still reach every index. A failing case is found again with the same +value. + Changes from `craft-media-testkit`'s `hammer`: - A case that does not finish fails the test after its limit with the case's label. Before, a case diff --git a/crates/testkit/src/hostile.rs b/crates/testkit/src/hostile.rs index 4480d6aaf..2b34819e6 100644 --- a/crates/testkit/src/hostile.rs +++ b/crates/testkit/src/hostile.rs @@ -11,7 +11,10 @@ //! mutations, one case after another on a worker thread, with a time limit for each case. It fails //! the test, naming the case and the input length, when the parser panics or does not finish in //! time. The number of rounds is a count the caller gives, multiplied by a variable the caller -//! names. [`hammer`] and [`HammerConfig`] are not built for `wasm32`. +//! names. [`DIVISOR_VARIABLE`] (`CRAFT_HOSTILE_DIVISOR`) runs a fraction of every sweep for quick +//! runs such as pull requests: every N-th truncation and 1/N of the rounds; [`divisor`] reads it +//! for sweeps written without [`hammer`]. [`hammer`], [`HammerConfig`] and [`divisor`] are not +//! built for `wasm32`. use std::fmt::Write as _; @@ -40,6 +43,12 @@ const WORKER_NAME: &str = "craft-testkit hostile"; /// [`HammerConfig::with_multiplier_variable`] names another. pub const DEFAULT_MULTIPLIER_VARIABLE: &str = "CRAFT_HOSTILE_MULT"; +/// The variable that divides every hostile-input sweep: with `N`, [`hammer`] runs the unmodified +/// seed, every N-th truncation (lengths 0, N, 2N, ...) and 1/N of its rounds, rounded up. Unset, or +/// 1, runs everything. The Workspace gate sets it and the nightly full run leaves it unset; a +/// failing case is found again with the same value. +pub const DIVISOR_VARIABLE: &str = "CRAFT_HOSTILE_DIVISOR"; + /// The default limit for one case of [`hammer`]. The limit's variable, `CRAFT_TEST_TIMEOUT_SECS` /// unless [`HammerConfig::with_timeout`] names another, replaces it. #[cfg(not(target_arch = "wasm32"))] @@ -339,6 +348,36 @@ fn parse_multiplier(value: &str) -> Result { } } +/// The value of [`DIVISOR_VARIABLE`], or 1 when it is not set, for a sweep written without +/// [`hammer`]: run the cases whose index plus an offset is a multiple of it, and give each input of +/// the sweep a different offset so that together they still reach every index. Panics, naming the +/// variable, when the value is not a whole number above zero. +/// +/// ``` +/// let step = craft_testkit::hostile::divisor(); +/// let positions: Vec = (0..64).filter(|i| (i + 3) % step == 0).collect(); +/// assert!(!positions.is_empty()); +/// ``` +// Failing the calling test is what this function is for. +#[cfg(not(target_arch = "wasm32"))] +#[allow(clippy::panic)] +#[track_caller] +pub fn divisor() -> usize { + match divisor_with(read_variable) { + Ok(divisor) => divisor, + Err(error) => panic!("{error}"), + } +} + +/// [`divisor`] with the variable lookup passed in. +#[cfg(not(target_arch = "wasm32"))] +fn divisor_with(lookup: impl FnOnce(&str) -> Option) -> Result { + let Some(raw) = lookup(DIVISOR_VARIABLE) else { return Ok(1) }; + let variable = DIVISOR_VARIABLE.to_owned(); + let value = raw.into_string().map_err(|_| HammerError::NotUnicode { variable: variable.clone() })?; + parse_multiplier(&value).map_err(|reason| HammerError::BadValue { variable, value, reason }) +} + /// Reads the variable `name` from the process environment. #[cfg(not(target_arch = "wasm32"))] fn read_variable(name: &str) -> Option { @@ -388,15 +427,16 @@ where { let label = config.label; let rounds = config.rounds_with(&lookup).map_err(|error| format!("{label}: {error}"))?; - let rounds = if seed.is_empty() { 0 } else { rounds }; - let cases = 1 + seed.len() as u128 + rounds as u128; + let divisor = divisor_with(&lookup).map_err(|error| format!("{label}: {error}"))?; + let rounds = if seed.is_empty() { 0 } else { rounds.div_ceil(divisor) }; + let cases = 1 + seed.len().div_ceil(divisor) as u128 + rounds as u128; if cases > MAX_CASES as u128 { return Err(format!("{label}: {}", HammerError::TooManyCases { cases })); } let start = Instant::now(); let limit = resolve(&config.timeout, &lookup, start).map(Limit::from).map_err(|error| format!("{label}: {error}"))?; let clock: Clock = Arc::new(move || start.elapsed()); - run_cases(label.to_owned(), limit, Arc::from(seed), rounds, parse, clock) + run_cases(label.to_owned(), limit, Arc::from(seed), rounds, divisor, parse, clock) } /// A clock that returns the time since a [`hammer`] call started. @@ -467,7 +507,7 @@ impl Drop for Report { /// Starts the worker that runs every case, then waits for it, failing when a case runs past the /// limit. #[cfg(not(target_arch = "wasm32"))] -fn run_cases(label: String, limit: Limit, seed: Arc<[u8]>, rounds: usize, parse: F, clock: Clock) -> Result<(), String> +fn run_cases(label: String, limit: Limit, seed: Arc<[u8]>, rounds: usize, divisor: usize, parse: F, clock: Clock) -> Result<(), String> where F: Fn(&[u8]) + Send + Sync + 'static, { @@ -480,7 +520,7 @@ where .name(WORKER_NAME.to_owned()) .spawn(move || { let mut report = Report { watch: worker_watch, outcome: Err(format!("{worker_label}: the worker thread stopped without a result")) }; - report.outcome = run_all(&report.watch, &worker_clock, &worker_label, &worker_limit, &seed, rounds, &parse); + report.outcome = run_all(&report.watch, &worker_clock, &worker_label, &worker_limit, &seed, rounds, divisor, &parse); }) .map_err(|error| format!("{label}: could not start the worker thread: {error}"))?; let mut state = watch.lock(); @@ -498,11 +538,21 @@ where } } -/// Runs the unmodified seed, its truncations and its mutations in order, stopping at the first -/// failure. +/// Runs the unmodified seed, every `divisor`-th truncation and its mutations in order, stopping at +/// the first failure. #[cfg(not(target_arch = "wasm32"))] -fn run_all(watch: &Watch, clock: &Clock, label: &str, limit: &Limit, seed: &[u8], rounds: usize, parse: &F) -> Result<(), String> { - for n in std::iter::once(seed.len()).chain(0..seed.len()) { +#[allow(clippy::too_many_arguments)] +fn run_all( + watch: &Watch, + clock: &Clock, + label: &str, + limit: &Limit, + seed: &[u8], + rounds: usize, + divisor: usize, + parse: &F, +) -> Result<(), String> { + for n in std::iter::once(seed.len()).chain((0..seed.len()).step_by(divisor)) { let case = if n == seed.len() { "the unmodified seed".to_owned() } else { format!("truncation to {n}") }; run_one(watch, clock, label, limit, case, seed.get(..n).unwrap_or_default(), parse)?; } @@ -675,7 +725,7 @@ mod tests { Duration::ZERO }); let limit = Limit { limit: Duration::from_secs(5), text: "5s".to_owned() }; - assert_eq!(run_cases("late start".to_owned(), limit, Arc::from(&[1u8][..]), 0, |_: &[u8]| {}, clock), Ok(())); + assert_eq!(run_cases("late start".to_owned(), limit, Arc::from(&[1u8][..]), 0, 1, |_: &[u8]| {}, clock), Ok(())); } #[test] @@ -742,6 +792,29 @@ mod tests { assert_eq!(config.rounds_with(unset), Ok(7)); } + #[test] + fn divisor_runs_every_nth_truncation_and_a_fraction_of_the_rounds() { + let lookup = |name: &str| (name == DIVISOR_VARIABLE).then(|| OsString::from("4")); + let seen = Arc::new(Mutex::new(Vec::new())); + let record = Arc::clone(&seen); + hammer_with(&config("divided", 10), &[0; 10], move |d| record.lock().unwrap().push(d.len()), lookup).unwrap(); + let lengths = seen.lock().unwrap().clone(); + // The seed, truncations to 0, 4 and 8, then 10 / 4 = 3 rounds rounded up. + assert_eq!(lengths.len(), 7, "{lengths:?}"); + assert_eq!(lengths[..4], [10, 0, 4, 8]); + assert_eq!(divisor_with(unset), Ok(1)); + } + + #[test] + fn invalid_divisor_fails_before_any_parse() { + let called = Arc::new(Mutex::new(false)); + let flag = Arc::clone(&called); + let lookup = |name: &str| (name == DIVISOR_VARIABLE).then(|| OsString::from("0")); + let message = hammer_with(&config("bad divisor", 3), &[1, 2], move |_| *flag.lock().unwrap() = true, lookup).unwrap_err(); + assert_eq!(message, r#"bad divisor: CRAFT_HOSTILE_DIVISOR="0" is zero; set it to a whole number above zero"#); + assert!(!*called.lock().unwrap()); + } + #[test] fn invalid_multiplier_names_variable() { for (value, reason) in [ diff --git a/crates/testkit/src/lib.rs b/crates/testkit/src/lib.rs index 020938371..910f25518 100644 --- a/crates/testkit/src/lib.rs +++ b/crates/testkit/src/lib.rs @@ -39,6 +39,7 @@ pub mod deadline; pub mod guards; pub mod hostile; +pub mod sample_damage; #[cfg(not(target_arch = "wasm32"))] pub use deadline::{DeadlineLimit, wait_until, with_deadline}; diff --git a/crates/testkit/src/sample_damage.rs b/crates/testkit/src/sample_damage.rs new file mode 100644 index 000000000..f0bdec729 --- /dev/null +++ b/crates/testkit/src/sample_damage.rs @@ -0,0 +1,74 @@ +//! Stable sampled truncations and corruptions for existing parser regression corpora. +//! +//! Unlike `hostile::mutations`, this preserves the historical TAR/EVM fixture sequence. + +/// Deterministic pseudo-random bytes (xorshift) for fuzz-like loops. +pub fn noise(seed: u64, len: usize) -> Vec { + let mut x = seed.wrapping_mul(0x9e37_79b9_7f4a_7c15) | 1; + (0..len) + .map(|_| { + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + x as u8 + }) + .collect() +} + +/// Every way to damage a sample used by robustness tests: truncations at many lengths and +/// single-byte corruptions spread across the file. +pub fn mutations(sample: &[u8]) -> Vec> { + let mut out = Vec::new(); + let n = sample.len(); + let step = (n / 64).max(1); + let mut len = 0; + while len < n { + out.push(sample[..len].to_vec()); + len += step; + } + for (i, seed) in (0..n).step_by(step).zip(1u64..) { + let mut m = sample.to_vec(); + let noise = noise(seed, 8); + for (j, v) in noise.iter().enumerate() { + if let Some(b) = m.get_mut(i + j) { + *b = *v; + } + } + out.push(m); + let mut m = sample.to_vec(); + if let Some(b) = m.get_mut(i) { + *b = 0xff; + } + out.push(m); + } + out +} + +#[cfg(test)] +mod tests { + use super::{mutations, noise}; + + #[test] + fn fixture_sequence_stays_stable() { + assert_eq!(noise(1, 8), [173, 118, 54, 116, 236, 121, 207, 234]); + assert!(noise(1, 0).is_empty()); + assert!(mutations(&[]).is_empty()); + assert_eq!(mutations(&[42]), vec![vec![], vec![173], vec![255]]); + } + + #[test] + fn sampling_stride_preserves_truncation_and_corruption_order() { + let sample = vec![42; 128]; + let cases = mutations(&sample); + assert_eq!(cases.len(), 192); + for (index, prefix) in cases.iter().take(64).enumerate() { + assert_eq!(prefix, &sample[..index * 2]); + } + for (index, pair) in cases[64..].as_chunks::<2>().0.iter().enumerate() { + assert_eq!(pair[0].len(), sample.len()); + let mut expected = sample.clone(); + expected[index * 2] = 255; + assert_eq!(pair[1], expected); + } + } +} diff --git a/crates/vectordrawable/src/write.rs b/crates/vectordrawable/src/write.rs index ec14fca97..5a883df77 100644 --- a/crates/vectordrawable/src/write.rs +++ b/crates/vectordrawable/src/write.rs @@ -13,6 +13,10 @@ use crate::model::{Cap, Color, Fill, GradientKind, Group, Item, ItemKind, Join, /// never recurses without bound. Android Studio's own drawables nest a handful of levels. pub const MAX_DEPTH: usize = 256; +/// Hard ceiling for [`document_with_depth_limit`]. A caller may need extra XML groups for +/// transforms and clips around each bounded application layer, but cannot disable the guard. +pub const MAX_CONFIGURED_DEPTH: usize = 1024; + const ANDROID: &str = "http://schemas.android.com/apk/res/android"; const AAPT: &str = "http://schemas.android.com/aapt"; @@ -110,18 +114,37 @@ impl Out { } /// Whether a path that is written (see [`MAX_DEPTH`]) has a gradient fill. -fn uses_gradient(items: &[Item], depth: usize) -> bool { - items.iter().any(|it| match &it.kind { - ItemKind::Path(p) => matches!(p.fill, Some(Fill::Gradient(_))), - ItemKind::Group(g) => depth < MAX_DEPTH && uses_gradient(&g.children, depth + 1), - }) +fn uses_gradient(items: &[Item], max_depth: usize) -> bool { + let mut pending = vec![(1, items.iter())]; + while let Some((depth, items)) = pending.last_mut() { + let depth = *depth; + match items.next().map(|it| &it.kind) { + Some(ItemKind::Path(p)) if matches!(p.fill, Some(Fill::Gradient(_))) => return true, + Some(ItemKind::Group(g)) if depth < max_depth => pending.push((depth + 1, g.children.iter())), + Some(_) => {} + None => { + pending.pop(); + } + } + } + false } /// The whole file: a `width`×`height` dp drawable with a viewport of the same size. pub fn document(width: f64, height: f64, items: &[Item]) -> String { + document_with_depth_limit(width, height, items, MAX_DEPTH) +} + +/// Write with a caller-selected XML group depth cap, clamped to `1..=MAX_CONFIGURED_DEPTH`. +/// +/// A group at the cap keeps its attributes and clip but not its children. The default +/// [`document`] cap is unchanged. Use this when one bounded source layer projects to several +/// XML wrapper groups; the caller must include those wrappers in its depth calculation. +pub fn document_with_depth_limit(width: f64, height: f64, items: &[Item], max_depth: usize) -> String { + let max_depth = max_depth.clamp(1, MAX_CONFIGURED_DEPTH); let mut o = Out { s: String::from("\n") }; let mut attrs = vec![("xmlns:android", ANDROID.to_string())]; - if uses_gradient(items, 1) { + if uses_gradient(items, max_depth) { attrs.push(("xmlns:aapt", AAPT.to_string())); } attrs.extend([ @@ -135,21 +158,35 @@ pub fn document(width: f64, height: f64, items: &[Item]) -> String { // Android refuses a without a ("no path defined"): one empty, unfilled path. o.open(1, "path", &[("android:pathData", "M0,0".to_string())], true); } - for it in items { - item(&mut o, 1, it); - } + write_items(&mut o, items, max_depth); o.close(0, "vector"); o.s } -fn item(o: &mut Out, depth: usize, it: &Item) { - match &it.kind { - ItemKind::Path(p) => path(o, depth, p), - ItemKind::Group(g) => group(o, depth, g), +fn write_items(o: &mut Out, items: &[Item], max_depth: usize) { + // One iterator per open group, not one pending entry per sibling. The configured depth + // bounds this stack, and wide documents do not require a second list of their items. + let mut pending = vec![(1, items.iter())]; + while let Some((depth, items)) = pending.last_mut() { + let depth = *depth; + match items.next().map(|it| &it.kind) { + Some(ItemKind::Path(p)) => path(o, depth, p), + Some(ItemKind::Group(g)) => { + if group(o, depth, g, max_depth) { + pending.push((depth + 1, g.children.iter())); + } + } + None => { + pending.pop(); + if depth > 1 { + o.close(depth - 1, "group"); + } + } + } } } -fn group(o: &mut Out, depth: usize, g: &Group) { +fn group(o: &mut Out, depth: usize, g: &Group, max_depth: usize) -> bool { let mut attrs = Vec::new(); if let Some(n) = &g.name { attrs.push(("android:name", esc(n))); @@ -172,19 +209,21 @@ fn group(o: &mut Out, depth: usize, g: &Group) { attrs.push(("android:translateY", num(x.translate.1))); } } - let children: &[Item] = if depth < MAX_DEPTH { &g.children } else { &[] }; + let children: &[Item] = if depth < max_depth { &g.children } else { &[] }; if children.is_empty() && g.clip.is_none() { o.open(depth, "group", &attrs, true); - return; + return false; } o.open(depth, "group", &attrs, false); if let Some(c) = &g.clip { o.open(depth + 1, "clip-path", &[("android:pathData", path_data(c))], true); } - for c in children { - item(o, depth + 1, c); + if children.is_empty() { + o.close(depth, "group"); + false + } else { + true } - o.close(depth, "group"); } fn stroke_attrs(attrs: &mut Vec<(&str, String)>, s: &Stroke) { @@ -445,3 +484,61 @@ mod tests { assert!(xml.contains(" Item { + let mut item = Item { + bbox: Rect::ZERO, + kind: ItemKind::Path(Path { + name: Some("deep".into()), + data: BezPath::new(), + stroke: None, + even_odd: false, + fill: Some(Fill::Gradient(Gradient { kind: GradientKind::Sweep { center: (0.0, 0.0) }, stops: vec![] })), + }), + }; + for _ in 0..groups { + item = Item { bbox: Rect::ZERO, kind: ItemKind::Group(Group { name: None, transform: None, clip: None, children: vec![item] }) }; + } + item + } + + #[test] + fn expanded_source_layers_keep_their_leaf_and_namespace() { + let items = [nested(514)]; + assert!(!document(1.0, 1.0, &items).contains("aapt")); + let xml = document_with_depth_limit(1.0, 1.0, &items, 516); + assert_eq!(xml.matches(" { + data: &'a [u8], +} +impl<'a> Reader<'a> { + pub fn new(data: &'a [u8]) -> Self { + Self { data } + } + pub fn u8(&self, off: usize) -> Option { + self.data.get(off).copied() + } + pub fn slice(&self, off: usize, len: usize) -> Option<&'a [u8]> { + self.data.get(off..off.checked_add(len)?) + } + /// Unsigned LEB128 at `off`: (value, bytes consumed). At most 10 bytes. + pub fn uleb128(&self, off: usize) -> Option<(u64, usize)> { + let mut result = 0u64; + for i in 0..10usize { + let b = self.u8(off.checked_add(i)?)?; + let payload = u64::from(b & 0x7f); + let shift = 7 * i as u32; + if shift > 0 && payload.checked_shl(shift)? >> shift != payload { + return None; + } + result |= payload << shift; + if b & 0x80 == 0 { + return Some((result, i + 1)); + } + } + None + } +} diff --git a/crates/wasm/src/lib.rs b/crates/wasm/src/lib.rs new file mode 100644 index 000000000..0c552fd21 --- /dev/null +++ b/crates/wasm/src/lib.rs @@ -0,0 +1,973 @@ +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +#![forbid(unsafe_code)] + +//! WebAssembly modules: validation, sections, imports, exports, functions with their signatures +//! and names, memories, tables, globals, tags, custom sections (names, producers, target features, +//! source map URL) and a text (WAT) rendering. Built on the pure-Rust `wasmparser` and +//! `wasmprinter` crates; never executes anything. + +use crate::bytes::Reader; +use WasmError as FormatError; +use serde::Serialize; +use std::collections::{BTreeMap, btree_map::Entry}; +use wasmparser::{CompositeInnerType, Encoding, ExternalKind, FuncType, KnownCustom, Name, Parser, Payload, TypeRef, Validator, WasmFeatures}; + +/// Largest module we read (256 MiB). +pub const MAX_INPUT: usize = 256 << 20; +/// Most entries kept per list. +pub const MAX_ITEMS: usize = 100_000; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct WasmSection { + pub id: u8, + pub name: String, + pub offset: u64, + pub size: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct WasmImport { + pub module: String, + pub name: String, + /// func, table, memory, global or tag. + pub kind: &'static str, + pub type_index: Option, + /// Function signature or a short description of the imported item. + pub signature: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct WasmExport { + pub name: String, + pub kind: &'static str, + pub index: u32, + /// Name of the exported function from the name section, when known. + pub target_name: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct WasmFunction { + /// Index in the function index space (imports first). + pub index: u32, + pub type_index: u32, + pub signature: String, + pub body_offset: u64, + pub body_size: u64, + pub locals: u64, + pub name: Option, + pub exported_as: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct WasmMemory { + pub index: u32, + pub initial: u64, + pub maximum: Option, + pub shared: bool, + pub memory64: bool, + pub imported: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct WasmTable { + pub index: u32, + pub element_type: String, + pub initial: u64, + pub maximum: Option, + pub imported: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct WasmGlobal { + pub index: u32, + pub value_type: String, + pub mutable: bool, + pub imported: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct WasmTag { + /// Index in the tag index space (imports first). + pub index: u32, + pub type_index: u32, + pub signature: String, + pub imported: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct WasmCustom { + pub name: String, + pub offset: u64, + pub size: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ProducerValue { + pub name: String, + pub version: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct ProducerField { + pub field: String, + pub values: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct WasmValidation { + pub valid: bool, + pub error: Option, + pub offset: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct WasmReport { + /// `module` or `component`. + pub encoding: &'static str, + pub version: u16, + pub size: usize, + pub validation: WasmValidation, + /// Set when the structure could not be read to the end. + pub parse_error: Option, + pub sections: Vec, + pub types: usize, + pub imports: Vec, + pub imported_functions: u32, + pub exports: Vec, + /// Functions defined in the module (imports are listed under `imports`). + pub functions: Vec, + pub memories: Vec, + pub tables: Vec, + pub globals: Vec, + pub tags: Vec, + pub start_function: Option, + pub data_segments: u32, + pub element_segments: u32, + pub data_count: Option, + pub custom_sections: Vec, + pub module_name: Option, + pub producers: Vec, + pub target_features: Vec, + pub source_mapping_url: Option, + pub external_debug_info: Option, + pub truncated: bool, + pub warnings: Vec, +} + +/// True when the bytes start with the WebAssembly magic. +pub fn is_wasm(data: &[u8]) -> bool { + data.starts_with(b"\0asm") +} + +fn sig(f: &FuncType) -> String { + let p: Vec = f.params().iter().map(|v| v.to_string()).collect(); + let r: Vec = f.results().iter().map(|v| v.to_string()).collect(); + format!("({}) -> ({})", p.join(", "), r.join(", ")) +} + +struct ReferencedStrings { + remaining: usize, + exhausted: bool, +} + +impl ReferencedStrings { + fn claim(&mut self, bytes: usize) -> bool { + if self.exhausted || bytes > self.remaining { + self.exhausted = true; + return false; + } + self.remaining -= bytes; + true + } + + fn copy(&mut self, value: &str) -> Option { + self.claim(value.len()).then(|| value.to_owned()) + } +} + +fn type_sig(types: &[Option], t: u32, cache: &mut BTreeMap, strings: &mut ReferencedStrings) -> Option { + if strings.exhausted { + return None; + } + let signature = match cache.entry(t) { + Entry::Occupied(entry) => entry.into_mut(), + Entry::Vacant(entry) => { + let value = match types.get(t as usize) { + Some(Some(f)) => sig(f), + Some(None) => format!("type {t} (not a function type)"), + None => format!("type {t} (missing)"), + }; + if !strings.claim(value.len()) { + return None; + } + entry.insert(value) + } + }; + strings.copy(signature) +} + +fn section_name(id: u8) -> &'static str { + match id { + 0 => "custom", + 1 => "type", + 2 => "import", + 3 => "function", + 4 => "table", + 5 => "memory", + 6 => "global", + 7 => "export", + 8 => "start", + 9 => "element", + 10 => "code", + 11 => "data", + 12 => "datacount", + 13 => "tag", + _ => "unknown", + } +} + +/// Read the outer component layout without interpreting the contents of embedded binaries. +fn component_sections(reader: &mut wasmparser::BinaryReader<'_>, rep: &mut WasmReport) -> Result<(), wasmparser::BinaryReaderError> { + while !reader.eof() { + if rep.sections.len() == MAX_ITEMS { + rep.truncated = true; + break; + } + let id = reader.read_u8()?; + let body = reader.read_reader()?; + let range = body.range(); + let name = match id { + 0 => format!("custom:{}", wasmparser::CustomSectionReader::new(body)?.name()), + 1 => "core module".into(), + 2 => "core instance".into(), + 3 => "core type".into(), + 4 => "component".into(), + 5 => "instance".into(), + 6 => "alias".into(), + 7 => "type".into(), + 8 => "canonical".into(), + 9 => "start".into(), + 10 => "import".into(), + 11 => "export".into(), + _ => "unknown".into(), + }; + rep.sections.push(WasmSection { id, name, offset: range.start, size: range.end.saturating_sub(range.start) }); + } + Ok(()) +} + +fn kind_name(k: ExternalKind) -> &'static str { + match k { + ExternalKind::Func | ExternalKind::FuncExact => "func", + ExternalKind::Table => "table", + ExternalKind::Memory => "memory", + ExternalKind::Global => "global", + ExternalKind::Tag => "tag", + } +} + +/// Validate with every feature enabled; the result never fails the caller. +pub fn validate(data: &[u8]) -> WasmValidation { + match Validator::new_with_features(WasmFeatures::all()).validate_all(data) { + Ok(_) => WasmValidation { valid: true, error: None, offset: None }, + Err(e) => WasmValidation { valid: false, error: Some(e.message().to_string()), offset: Some(e.offset()) }, + } +} + +/// A length-prefixed UTF-8 string at the start of a custom section payload. +fn leading_string(data: &[u8]) -> Option { + // inspect bounds the enclosing module by MAX_INPUT, including any inline source map. + wasmparser::BinaryReader::new(data, 0).read_unlimited_string().ok().map(str::to_owned) +} + +fn target_features(data: &[u8]) -> Vec { + let r = Reader::new(data); + let mut out = Vec::new(); + let Some((n, mut p)) = r.uleb128(0) else { return out }; + for _ in 0..n.min(1024) { + let Some(prefix) = r.u8(p) else { break }; + let Some((len, used)) = r.uleb128(p + 1) else { break }; + let Some(s) = usize::try_from(len).ok().and_then(|l| r.slice(p + 1 + used, l)) else { break }; + out.push(format!("{}{}", char::from(prefix), String::from_utf8_lossy(s))); + p = p + 1 + used + s.len(); + } + out +} + +fn cap(len: usize, truncated: &mut bool) -> bool { + if len >= MAX_ITEMS { + *truncated = true; + false + } else { + true + } +} +fn read_types( + r: wasmparser::TypeSectionReader<'_>, + types: &mut Vec>, + rep: &mut WasmReport, +) -> Result<(), wasmparser::BinaryReaderError> { + for rg in r { + for st in rg?.into_types() { + if types.len() >= MAX_ITEMS { + rep.truncated = true; + break; + } + types.push(match st.composite_type.inner { + CompositeInnerType::Func(f) => Some(f), + _ => None, + }); + } + } + rep.types = types.len(); + + Ok(()) +} +fn read_custom(c: wasmparser::CustomSectionReader<'_>, rep: &mut WasmReport, fn_names: &mut BTreeMap, producer_values: &mut usize) { + if cap(rep.custom_sections.len(), &mut rep.truncated) { + let range = c.range(); + rep.custom_sections.push(WasmCustom { name: c.name().to_string(), offset: range.start, size: range.end.saturating_sub(range.start) }); + } + match c.name() { + "sourceMappingURL" => rep.source_mapping_url = leading_string(c.data()), + "external_debug_info" => rep.external_debug_info = leading_string(c.data()), + "target_features" => rep.target_features = target_features(c.data()), + _ => {} + } + match c.as_known() { + KnownCustom::Name(names) => { + // A damaged name section is reported but does not stop the rest. + for (i, sub) in names.into_iter().take(65).enumerate() { + if i == 64 { + rep.truncated = true; + break; + } + let Ok(sub) = sub else { + rep.warnings.push("the name section is damaged".into()); + break; + }; + match sub { + Name::Module { name, .. } => rep.module_name = Some(name.to_string()), + Name::Function(map) => { + for (i, n) in map.into_iter().take(MAX_ITEMS + 1).enumerate() { + if i == MAX_ITEMS { + rep.truncated = true; + break; + } + match n { + Ok(n) => { + if let Some(name) = fn_names.get_mut(&n.index) { + // Repeated sections may replace an already retained name at the limit. + *name = n.name.to_string(); + } else if cap(fn_names.len(), &mut rep.truncated) { + fn_names.insert(n.index, n.name.to_string()); + } + } + Err(_) => { + rep.warnings.push("a function name entry is damaged".into()); + break; + } + } + } + } + _ => {} + } + } + } + KnownCustom::Producers(p) => { + for (i, f) in p.into_iter().take(65).enumerate() { + if i == 64 || !cap(rep.producers.len(), &mut rep.truncated) { + rep.truncated = true; + break; + } + let Ok(f) = f else { + rep.warnings.push("the producers section is damaged".into()); + break; + }; + let mut values = Vec::new(); + for (i, value) in f.values.into_iter().take(65).enumerate() { + if i == 64 || !cap(*producer_values, &mut rep.truncated) { + rep.truncated = true; + break; + } + if let Ok(value) = value { + values.push(ProducerValue { name: value.name.to_string(), version: value.version.to_string() }); + *producer_values += 1; + } + } + rep.producers.push(ProducerField { field: f.name.to_string(), values }); + } + } + _ => {} + } +} + +fn read_code( + body: wasmparser::FunctionBody<'_>, + types: &[Option], + func_types: &[u32], + code_index: &mut u32, + rep: &mut WasmReport, + type_signatures: &mut BTreeMap, + strings: &mut ReferencedStrings, +) -> Result<(), wasmparser::BinaryReaderError> { + let ti = func_types.get(*code_index as usize).copied(); + let mut locals = 0u64; + let lr = body.get_locals_reader()?; + for l in lr { + let (n, _) = l?; + locals = locals.saturating_add(u64::from(n)); + } + let range = body.range(); + if cap(rep.functions.len(), &mut rep.truncated) { + let signature = match ti { + Some(t) => type_sig(types, t, type_signatures, strings), + None => strings.copy("no matching function section entry"), + }; + if let Some(signature) = signature { + rep.functions.push(WasmFunction { + index: rep.imported_functions.saturating_add(*code_index), + type_index: ti.unwrap_or(u32::MAX), + signature, + body_offset: range.start, + body_size: range.end.saturating_sub(range.start), + locals, + name: None, + exported_as: Vec::new(), + }); + } + } + *code_index = code_index.saturating_add(1); + + Ok(()) +} +fn read_functions( + r: wasmparser::FunctionSectionReader<'_>, + func_types: &mut Vec, + declared_functions: &mut u32, + rep: &mut WasmReport, +) -> Result<(), wasmparser::BinaryReaderError> { + *declared_functions = r.count(); + for t in r { + if func_types.len() >= MAX_ITEMS { + rep.truncated = true; + break; + } + func_types.push(t?); + } + + Ok(()) +} + +/// Read a module. Fails only when the bytes are not WebAssembly at all or too large; structural +/// damage is reported in `parse_error` and `validation`. +pub fn inspect(data: &[u8]) -> Result { + if data.len() > MAX_INPUT { + return Err(FormatError::TooLarge(format!("{} bytes; the limit is {MAX_INPUT}", data.len()))); + } + if !is_wasm(data) { + return Err(FormatError::Malformed("no WebAssembly magic (\\0asm)".into())); + } + let mut rep = WasmReport { + encoding: "module", + version: 0, + size: data.len(), + validation: validate(data), + parse_error: None, + sections: Vec::new(), + types: 0, + imports: Vec::new(), + imported_functions: 0, + exports: Vec::new(), + functions: Vec::new(), + memories: Vec::new(), + tables: Vec::new(), + globals: Vec::new(), + tags: Vec::new(), + start_function: None, + data_segments: 0, + element_segments: 0, + data_count: None, + custom_sections: Vec::new(), + module_name: None, + producers: Vec::new(), + target_features: Vec::new(), + source_mapping_url: None, + external_debug_info: None, + truncated: false, + warnings: Vec::new(), + }; + let mut types: Vec> = Vec::new(); + let mut func_types: Vec = Vec::new(); + let mut fn_names: BTreeMap = BTreeMap::new(); + let mut producer_values = 0usize; + let mut imported_tables = 0u32; + let mut imported_memories = 0u32; + let mut imported_globals = 0u32; + let mut imported_tags = 0u32; + let mut declared_functions = 0; + let mut code_index = 0u32; + // Type and name references can expand a small module into much larger display strings. + // Charge both the signature cache and every retained copy before allocating that copy. + let reference_budget = (64 << 20).max(data.len().saturating_mul(3)); + let mut strings = ReferencedStrings { remaining: reference_budget, exhausted: false }; + let mut type_signatures = BTreeMap::new(); + + let result: Result<(), wasmparser::BinaryReaderError> = (|| { + for payload in Parser::new(0).parse_all(data) { + let payload = payload?; + if let Some((id, range)) = payload.as_section() + && cap(rep.sections.len(), &mut rep.truncated) + { + let name = match &payload { + Payload::CustomSection(c) => format!("custom:{}", c.name()), + _ => section_name(id).to_string(), + }; + rep.sections.push(WasmSection { id, name, offset: range.start, size: range.end.saturating_sub(range.start) }); + } + match payload { + Payload::Version { num, encoding, range } => { + rep.version = num; + if encoding == Encoding::Component { + rep.encoding = "component"; + rep.warnings.push("this is a component, not a core module; only validation and the top-level layout are reported".into()); + if let Some(body) = usize::try_from(range.end).ok().and_then(|end| data.get(end..)) { + component_sections(&mut wasmparser::BinaryReader::new(body, range.end), &mut rep)?; + } + break; + } + } + Payload::TypeSection(r) => { + read_types(r, &mut types, &mut rep)?; + } + Payload::ImportSection(r) => { + for imp in r.into_imports() { + let imp = imp?; + let (kind, type_index, signature) = match imp.ty { + TypeRef::Func(t) | TypeRef::FuncExact(t) => { + rep.imported_functions = rep.imported_functions.saturating_add(1); + let signature = + if rep.imports.len() < MAX_ITEMS { type_sig(&types, t, &mut type_signatures, &mut strings) } else { None }; + ("func", Some(t), signature) + } + TypeRef::Table(t) => { + if cap(rep.tables.len(), &mut rep.truncated) { + rep.tables.push(WasmTable { + index: imported_tables, + element_type: format!("{:?}", t.element_type), + initial: t.initial, + maximum: t.maximum, + imported: true, + }); + } + imported_tables = imported_tables.saturating_add(1); + ("table", None, Some(format!("table {} min {}", format!("{:?}", t.element_type).to_lowercase(), t.initial))) + } + TypeRef::Memory(m) => { + if cap(rep.memories.len(), &mut rep.truncated) { + rep.memories.push(WasmMemory { + index: imported_memories, + initial: m.initial, + maximum: m.maximum, + shared: m.shared, + memory64: m.memory64, + imported: true, + }); + } + imported_memories = imported_memories.saturating_add(1); + ("memory", None, Some(format!("memory min {} pages", m.initial))) + } + TypeRef::Global(g) => { + if cap(rep.globals.len(), &mut rep.truncated) { + rep.globals.push(WasmGlobal { + index: imported_globals, + value_type: g.content_type.to_string(), + mutable: g.mutable, + imported: true, + }); + } + imported_globals = imported_globals.saturating_add(1); + ("global", None, Some(format!("{}{}", if g.mutable { "mut " } else { "" }, g.content_type))) + } + TypeRef::Tag(t) => { + let keep_tag = cap(rep.tags.len(), &mut rep.truncated); + let signature = if rep.imports.len() < MAX_ITEMS { + type_sig(&types, t.func_type_idx, &mut type_signatures, &mut strings) + } else { + None + }; + if keep_tag { + let tag_signature = match &signature { + Some(value) => strings.copy(value), + None => type_sig(&types, t.func_type_idx, &mut type_signatures, &mut strings), + }; + if let Some(signature) = tag_signature { + rep.tags.push(WasmTag { index: imported_tags, type_index: t.func_type_idx, signature, imported: true }); + } + } + // The import list and tag list are capped independently. + imported_tags = imported_tags.saturating_add(1); + ("tag", Some(t.func_type_idx), signature) + } + }; + if cap(rep.imports.len(), &mut rep.truncated) + && let Some(signature) = signature + { + rep.imports.push(WasmImport { module: imp.module.to_string(), name: imp.name.to_string(), kind, type_index, signature }); + } + } + } + Payload::FunctionSection(r) => { + read_functions(r, &mut func_types, &mut declared_functions, &mut rep)?; + } + Payload::TableSection(r) => { + for (i, t) in r.into_iter().enumerate() { + let t = t?; + if !cap(rep.tables.len(), &mut rep.truncated) { + break; + } + rep.tables.push(WasmTable { + index: imported_tables.saturating_add(u32::try_from(i).unwrap_or(u32::MAX)), + element_type: format!("{:?}", t.ty.element_type), + initial: t.ty.initial, + maximum: t.ty.maximum, + imported: false, + }); + } + } + Payload::MemorySection(r) => { + for (i, m) in r.into_iter().enumerate() { + let m = m?; + if !cap(rep.memories.len(), &mut rep.truncated) { + break; + } + rep.memories.push(WasmMemory { + index: imported_memories.saturating_add(u32::try_from(i).unwrap_or(u32::MAX)), + initial: m.initial, + maximum: m.maximum, + shared: m.shared, + memory64: m.memory64, + imported: false, + }); + } + } + Payload::TagSection(r) => { + for (i, t) in r.into_iter().enumerate() { + let t = t?; + if !cap(rep.tags.len(), &mut rep.truncated) { + break; + } + if let Some(signature) = type_sig(&types, t.func_type_idx, &mut type_signatures, &mut strings) { + rep.tags.push(WasmTag { + index: imported_tags.saturating_add(u32::try_from(i).unwrap_or(u32::MAX)), + type_index: t.func_type_idx, + signature, + imported: false, + }); + } + } + } + Payload::GlobalSection(r) => { + for (i, g) in r.into_iter().enumerate() { + let g = g?; + if !cap(rep.globals.len(), &mut rep.truncated) { + break; + } + rep.globals.push(WasmGlobal { + index: imported_globals.saturating_add(u32::try_from(i).unwrap_or(u32::MAX)), + value_type: g.ty.content_type.to_string(), + mutable: g.ty.mutable, + imported: false, + }); + } + } + Payload::ExportSection(r) => { + for e in r { + let e = e?; + if !cap(rep.exports.len(), &mut rep.truncated) { + break; + } + rep.exports.push(WasmExport { name: e.name.to_string(), kind: kind_name(e.kind), index: e.index, target_name: None }); + } + } + Payload::StartSection { func, .. } => rep.start_function = Some(func), + Payload::ElementSection(r) => rep.element_segments = r.count(), + Payload::DataCountSection { count, .. } => rep.data_count = Some(count), + Payload::DataSection(r) => rep.data_segments = r.count(), + Payload::CodeSectionEntry(body) => { + read_code(body, &types, &func_types, &mut code_index, &mut rep, &mut type_signatures, &mut strings)?; + } + Payload::CustomSection(c) => { + read_custom(c, &mut rep, &mut fn_names, &mut producer_values); + } + Payload::End(_) => {} + _ => {} + } + } + Ok(()) + })(); + if let Err(e) = result { + rep.parse_error = Some(format!("{} (at byte {})", e.message(), e.offset())); + } + // Attach names and exports to functions. + let mut exported: BTreeMap> = BTreeMap::new(); + for e in rep.exports.iter_mut() { + if e.kind == "func" { + e.target_name = fn_names.get(&e.index).and_then(|name| strings.copy(name)); + exported.entry(e.index).or_default().push(e.name.clone()); + } + } + for f in rep.functions.iter_mut() { + f.name = fn_names.get(&f.index).and_then(|name| strings.copy(name)); + f.exported_as = exported.remove(&f.index).unwrap_or_default(); + } + if declared_functions != code_index && rep.parse_error.is_none() { + rep.warnings.push(format!("{declared_functions} function declarations but {code_index} bodies")); + } + if strings.exhausted { + rep.truncated = true; + rep.warnings + .push(format!("referenced type signatures and function names exceed the {reference_budget}-byte budget; some metadata is omitted")); + } + Ok(rep) +} + +struct CappedText { + out: String, + cap: usize, + truncated: bool, +} + +impl wasmprinter::Print for CappedText { + fn write_str(&mut self, s: &str) -> std::io::Result<()> { + if self.out.len().saturating_add(s.len()) > self.cap { + let room = self.cap.saturating_sub(self.out.len()); + let mut end = room.min(s.len()); + while end > 0 && !s.is_char_boundary(end) { + end -= 1; + } + self.out.push_str(s.get(..end).unwrap_or("")); + self.truncated = true; + return Err(std::io::Error::other("output cap reached")); + } + self.out.push_str(s); + Ok(()) + } +} + +/// The text format of a module, at most `cap` bytes: (text, truncated). Only call this on +/// modules that passed [`validate`]; damaged modules are not rendered. +pub fn to_wat(data: &[u8], cap: usize) -> Result<(String, bool), String> { + let mut p = CappedText { out: String::new(), cap, truncated: false }; + let mut cfg = wasmprinter::Config::new(); + cfg.name_unnamed(false); + match cfg.print(data, &mut p) { + Ok(()) => Ok((p.out, false)), + Err(_) if p.truncated => Ok((p.out, true)), + Err(e) => Err(e.to_string()), + } +} + +/// What a JavaScript glue file says about the module. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct GlueScan { + /// String literals ending in `.wasm`. + pub wasm_references: Vec, + /// Imports (`module.name`) whose module and field names both appear in the glue. + pub matched_imports: Vec, + pub mentions_instantiate: bool, +} + +/// Scan glue JavaScript for `.wasm` literals and the import names the module expects. +pub fn scan_glue(text: &str, imports: &[WasmImport]) -> GlueScan { + let mut refs = Vec::new(); + let b = text.as_bytes(); + let mut i = 0usize; + while let Some(p) = text.get(i..).and_then(|t| t.find(".wasm")) { + let at = i + p; + let end = at + 5; + if let Some(&q) = b.get(end) + && matches!(q, b'"' | b'\'' | b'`') + { + let start = b.get(..at).and_then(|h| h.iter().rposition(|&c| c == q)).map(|s| s + 1); + if let Some(s) = start + && at - s <= 256 + && let Some(lit) = text.get(s..end) + && !lit.contains(['\n', ' ']) + && !refs.iter().any(|r: &String| r == lit) + && refs.len() < 256 + { + refs.push(lit.to_string()); + } + } + i = end; + } + let matched = imports + .iter() + .filter(|im| text.contains(im.module.as_str()) && text.contains(im.name.as_str())) + .take(MAX_ITEMS) + .map(|im| format!("{}.{}", im.module, im.name)) + .collect(); + GlueScan { wasm_references: refs, matched_imports: matched, mentions_instantiate: text.contains("WebAssembly.instantiate") } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_support::formats as fx; + + /// The output inventory limit applies to retained functions, not the local groups within one retained function. + #[test] + fn local_counts_include_groups_after_ten_thousand() { + // One valid () -> () function containing 10,001 runs of one i32 local. + let mut bytes = b"\0asm\x01\0\0\0\x01\x04\x01\x60\0\0\x03\x02\x01\0".to_vec(); + // section size = 1 + 3 + 20005, body size = 2 + 20002 + 1. + bytes.extend([10, 0xa9, 0x9c, 1, 1, 0xa5, 0x9c, 1, 0x91, 0x4e]); + for _ in 0..10_001 { + bytes.extend([1, 0x7f]); + } + bytes.push(0x0b); + let report = inspect(&bytes).unwrap(); + assert!(report.validation.valid, "{:?}", report.validation); + assert_eq!(report.functions[0].locals, 10_001); + } + + #[test] + fn debug_links_need_valid_wasm_names() { + assert_eq!(leading_string(b"\x04test").as_deref(), Some("test")); + assert!(leading_string(&[1, 0xff]).is_none()); + assert!(leading_string(b"\x81\x80\x80\x80\x80\0x").is_none()); + assert!(leading_string(b"\x04abc").is_none()); + } + + #[test] + fn inspect_preserves_long_source_mapping_urls() { + let url = format!("data:application/json;base64,{}", "A".repeat(100_000)); + let report = inspect(&fx::wasm_module_with_source_mapping_url(&url)).unwrap(); + assert_eq!(report.source_mapping_url.as_deref(), Some(url.as_str())); + } + + #[test] + fn large_function_tables_do_not_report_missing_declarations() { + fn leb(mut value: usize) -> Vec { + let mut out = Vec::new(); + while value >= 128 { + out.push((value as u8 & 0x7f) | 0x80); + value >>= 7; + } + out.push(value as u8); + out + } + let count = MAX_ITEMS + 1; + let mut bytes = b"\0asm\x01\0\0\0\x01\x04\x01\x60\0\0".to_vec(); + let mut declarations = leb(count); + declarations.resize(declarations.len() + count, 0); + let mut bodies = leb(count); + for _ in 0..count { + bodies.extend([2, 0, 0x0b]); + } + for (id, section) in [(3, declarations), (10, bodies)] { + bytes.push(id); + bytes.extend(leb(section.len())); + bytes.extend(section); + } + let report = inspect(&bytes).unwrap(); + assert!(report.validation.valid && report.truncated); + assert_eq!(report.functions.len(), MAX_ITEMS); + assert!(!report.warnings.iter().any(|s| s.contains("function declarations")), "{:?}", report.warnings); + } + + #[test] + fn reads_the_sample_module() { + let m = fx::wasm_module(); + let r = inspect(&m).unwrap(); + assert!(r.validation.valid, "{:?}", r.validation); + assert!(r.parse_error.is_none()); + assert_eq!(r.types, 3); + assert_eq!(r.imports.len(), 1); + assert_eq!(r.imports[0].signature, "(i32) -> ()"); + assert_eq!(r.imported_functions, 1); + assert_eq!(r.functions.len(), 2); + assert_eq!(r.functions[0].name.as_deref(), Some("add")); + assert_eq!(r.functions[0].signature, "(i32, i32) -> (i32)"); + assert_eq!(r.functions[0].exported_as, vec!["add"]); + assert_eq!(r.exports.len(), 3); + assert_eq!(r.memories.len(), 1); + assert_eq!(r.globals.len(), 1); + assert!(r.globals[0].mutable); + assert_eq!(r.data_segments, 1); + assert_eq!(r.module_name.as_deref(), Some("demo")); + assert_eq!(r.producers.len(), 2); + assert_eq!(r.producers[1].values[0].name, "rustc"); + assert_eq!(r.source_mapping_url.as_deref(), Some("demo.wasm.map")); + assert_eq!(r.target_features, vec!["+mutable-globals"]); + let (wat, trunc) = to_wat(&m, 1 << 20).unwrap(); + assert!(!trunc); + assert!(wat.contains("(module $demo"), "{wat}"); + assert!(wat.contains("i32.add")); + let (short, trunc) = to_wat(&m, 40).unwrap(); + assert!(trunc && short.len() <= 40); + let g = scan_glue(fx::WASM_GLUE, &r.imports); + assert_eq!(g.wasm_references, vec!["./demo.wasm"]); + assert_eq!(g.matched_imports, vec!["env.log"]); + assert!(g.mentions_instantiate); + } + + #[test] + fn damaged_modules_are_reported_not_fatal() { + let m = fx::wasm_module(); + let mut bad = m.clone(); + let len = bad.len(); + bad.truncate(len - 10); + let r = inspect(&bad).unwrap(); + assert!(!r.validation.valid); + assert!(r.parse_error.is_some()); + assert!(inspect(b"not wasm").is_err()); + } + + #[test] + fn hostile_inputs_do_not_panic() { + let m = fx::wasm_module(); + for x in crate::test_support::mutations(&m) { + if let Ok(r) = inspect(&x) + && r.validation.valid + { + let _ = to_wat(&x, 4096); + } + } + for n in 0..m.len() { + let _ = inspect(&m[..n]); + } + for seed in 0..100 { + let mut n = crate::test_support::noise(seed, 256); + n[..8].copy_from_slice(b"\0asm\x01\0\0\0"); + let _ = inspect(&n); + let _ = scan_glue(&String::from_utf8_lossy(&n), &[]); + } + } +} + +mod bytes; +#[cfg(test)] +mod test_support; + +/// Admission errors; structural damage stays in the inspection report. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum WasmError { + #[error("the file is malformed ({0})")] + Malformed(String), + #[error("too large: {0}")] + TooLarge(String), +} + +#[cfg(test)] +#[path = "regressions/component_layout.rs"] +mod component_layout; + +#[cfg(test)] +#[path = "regressions/custom_inventory_budget.rs"] +mod custom_inventory_budget; + +#[cfg(test)] +#[path = "regressions/reference_bytes.rs"] +mod reference_bytes; diff --git a/crates/wasm/src/regressions/component_layout.rs b/crates/wasm/src/regressions/component_layout.rs new file mode 100644 index 000000000..6f2f9532f --- /dev/null +++ b/crates/wasm/src/regressions/component_layout.rs @@ -0,0 +1,42 @@ +use crate::{MAX_ITEMS, inspect}; + +const COMPONENT: &[u8] = b"\0asm\x0d\0\x01\0"; + +/// Only outer sections appear, including sections after embedded binaries. +#[test] +fn lists_component_sections_without_descending_into_embedded_modules() { + let mut bytes = COMPONENT.to_vec(); + bytes.extend(b"\x00\x04\x03one"); + bytes.extend(b"\x01\x0b\0asm\x01\0\0\0\x01\x01\0"); + bytes.extend(b"\x04\x0e\0asm\x0d\0\x01\0\x00\x04\x03two"); + bytes.extend(b"\x00\x06\x05three"); + let report = inspect(&bytes).unwrap(); + assert!(report.validation.valid, "{:?}", report.validation); + let layout: Vec<_> = report.sections.iter().map(|s| (s.id, s.name.as_str(), s.offset, s.size)).collect(); + assert_eq!(layout, [(0, "custom:one", 10, 4), (1, "core module", 16, 11), (4, "component", 29, 14), (0, "custom:three", 45, 6)]); + assert!(report.functions.is_empty()); + assert!(report.parse_error.is_none()); +} + +/// A short section reports damage without losing the preceding layout. +#[test] +fn truncated_component_keeps_its_complete_prefix() { + let mut bytes = COMPONENT.to_vec(); + bytes.extend(b"\x00\x04\x03one\x01\x08\0asm"); + let report = inspect(&bytes).unwrap(); + assert_eq!(report.sections.len(), 1); + assert!(report.parse_error.is_some()); +} + +/// Component layouts use the same cap as core module layouts. +#[test] +fn component_section_limit_is_disclosed() { + let mut bytes = COMPONENT.to_vec(); + for _ in 0..=MAX_ITEMS { + bytes.extend(b"\x00\x02\x01x"); + } + let report = inspect(&bytes).unwrap(); + assert!(report.validation.valid); + assert_eq!(report.sections.len(), MAX_ITEMS); + assert!(report.truncated); +} diff --git a/crates/wasm/src/regressions/custom_inventory_budget.rs b/crates/wasm/src/regressions/custom_inventory_budget.rs new file mode 100644 index 000000000..d9c92d87e --- /dev/null +++ b/crates/wasm/src/regressions/custom_inventory_budget.rs @@ -0,0 +1,115 @@ +use crate::test_support::formats::wasm_module; +use crate::{MAX_ITEMS, inspect}; + +use crate::test_support::{leb, string}; + +fn custom(module: &mut Vec, name: &str, data: &[u8]) { + let mut body = Vec::new(); + string(&mut body, name); + body.extend(data); + module.push(0); + module.extend(leb(body.len())); + module.extend(body); +} + +fn names(module: &mut Vec, entries: &[(usize, &str)]) { + let mut map = leb(entries.len()); + for (index, name) in entries { + map.extend(leb(*index)); + string(&mut map, name); + } + let mut body = vec![1]; + body.extend(leb(map.len())); + body.extend(map); + custom(module, "name", &body); +} + +fn producers(fields: usize, values: usize) -> Vec { + let mut body = leb(fields); + for _ in 0..fields { + string(&mut body, "language"); + body.extend(leb(values)); + for _ in 0..values { + string(&mut body, "fixture"); + string(&mut body, "1"); + } + } + body +} + +#[test] +fn repeated_name_sections_share_a_limit_but_can_update_retained_functions() { + let mut bytes = b"\0asm\x01\0\0\0\x01\x04\x01\x60\0\0\x03\x02\x01\0\x0a\x04\x01\x02\0\x0b".to_vec(); + let entries: Vec<_> = (0..MAX_ITEMS).map(|index| (index, "kept")).collect(); + names(&mut bytes, &entries); + let complete = inspect(&bytes).unwrap(); + assert!(complete.validation.valid); + assert!(!complete.truncated); + assert_eq!(complete.functions[0].name.as_deref(), Some("kept")); + names(&mut bytes, &[(0, "updated"), (MAX_ITEMS, "omitted")]); + let report = inspect(&bytes).unwrap(); + assert!(report.validation.valid); + assert!(report.parse_error.is_none()); + assert!(report.truncated); + assert_eq!(report.functions[0].name.as_deref(), Some("updated")); +} + +#[test] +fn producer_values_are_bounded_across_repeated_sections() { + let mut bytes = b"\0asm\x01\0\0\0".to_vec(); + let section = producers(1, 64); + for _ in 0..MAX_ITEMS / 64 + 1 { + custom(&mut bytes, "producers", §ion); + } + let report = inspect(&bytes).unwrap(); + assert!(report.validation.valid); + assert!(report.parse_error.is_none()); + assert!(report.truncated); + assert_eq!(report.producers.iter().map(|field| field.values.len()).sum::(), MAX_ITEMS); + assert_eq!(report.producers[0].values[0].name, "fixture"); +} + +#[test] +fn empty_producer_fields_still_spend_the_global_field_limit() { + let mut bytes = b"\0asm\x01\0\0\0".to_vec(); + let section = producers(64, 0); + for _ in 0..MAX_ITEMS / 64 + 1 { + custom(&mut bytes, "producers", §ion); + } + let report = inspect(&bytes).unwrap(); + assert!(report.validation.valid); + assert_eq!(report.producers.len(), MAX_ITEMS); + assert!(report.truncated); +} + +#[test] +fn existing_per_section_limits_are_reported_as_truncation() { + for (fields, values) in [(65, 0), (1, 65)] { + let mut bytes = b"\0asm\x01\0\0\0".to_vec(); + custom(&mut bytes, "producers", &producers(fields, values)); + let report = inspect(&bytes).unwrap(); + assert!(report.truncated, "{fields} fields with {values} values"); + } + let mut bytes = b"\0asm\x01\0\0\0".to_vec(); + let mut subsections = vec![0, 2, 1, b'm']; + // Name subsection IDs must increase, including unknown extension IDs. + for id in 20..84 { + subsections.extend([id, 0]); + } + custom(&mut bytes, "name", &subsections); + let report = inspect(&bytes).unwrap(); + assert!(report.truncated); + assert_eq!(report.module_name.as_deref(), Some("m")); +} + +#[test] +fn ordinary_names_and_producers_remain_complete() { + let report = inspect(&wasm_module()).unwrap(); + assert!(report.validation.valid); + assert!(!report.truncated); + assert_eq!(report.functions[0].name.as_deref(), Some("add")); + assert_eq!(report.module_name.as_deref(), Some("demo")); + assert_eq!(report.producers.len(), 2); + assert_eq!(report.producers[1].values[0].name, "rustc"); + assert_eq!(report.exports.iter().find(|export| export.name == "add").unwrap().target_name.as_deref(), Some("add")); +} diff --git a/crates/wasm/src/regressions/reference_bytes.rs b/crates/wasm/src/regressions/reference_bytes.rs new file mode 100644 index 000000000..f5ff9a4ba --- /dev/null +++ b/crates/wasm/src/regressions/reference_bytes.rs @@ -0,0 +1,140 @@ +use crate::{WasmReport, inspect}; + +use crate::test_support::{leb, string}; + +fn section(module: &mut Vec, id: u8, body: &[u8]) { + module.push(id); + module.extend(leb(body.len())); + module.extend(body); +} + +fn shared_type(functions: usize, imports: usize, imported_tags: usize, defined_tags: usize, params: usize) -> Vec { + let mut module = b"\0asm\x01\0\0\0".to_vec(); + let mut ty = vec![1, 0x60]; + ty.extend(leb(params)); + ty.extend(std::iter::repeat_n(0x7f, params)); + ty.push(0); + section(&mut module, 1, &ty); + if imports + imported_tags > 0 { + let mut body = leb(imports + imported_tags); + for i in 0..imports + imported_tags { + string(&mut body, "env"); + string(&mut body, "item"); + if i < imports { + body.extend([0, 0]); + } else { + body.extend([4, 0, 0]); + } + } + section(&mut module, 2, &body); + } + if functions > 0 { + let mut body = leb(functions); + body.extend(std::iter::repeat_n(0, functions)); + section(&mut module, 3, &body); + } + if defined_tags > 0 { + let mut body = leb(defined_tags); + for _ in 0..defined_tags { + body.extend([0, 0]); + } + section(&mut module, 13, &body); + } + if functions > 0 { + let mut body = leb(functions); + for _ in 0..functions { + body.extend([2, 0, 0x0b]); + } + section(&mut module, 10, &body); + } + module +} + +fn derived_bytes(report: &WasmReport) -> usize { + report.imports.iter().map(|item| item.signature.len()).sum::() + + report.tags.iter().map(|item| item.signature.len()).sum::() + + report.functions.iter().map(|item| item.signature.len() + item.name.as_ref().map_or(0, String::len)).sum::() + + report.exports.iter().map(|item| item.target_name.as_ref().map_or(0, String::len)).sum::() +} + +fn bounded(module: &[u8]) -> WasmReport { + assert!(module.len() < 1 << 20); + let report = inspect(module).unwrap(); + assert!(report.parse_error.is_none()); + assert!(derived_bytes(&report) <= 64 << 20, "{} input bytes retain {} derived string bytes", module.len(), derived_bytes(&report)); + assert!(report.truncated); + assert!(report.warnings.iter().any(|warning| warning.contains("byte budget")), "{:?}", report.warnings); + report +} + +#[test] +fn repeated_function_signatures_have_a_byte_budget() { + let report = bounded(&shared_type(16_384, 0, 0, 0, 1000)); + assert!(report.validation.valid, "{:?}", report.validation); + assert!(!report.functions.is_empty()); + assert!(report.functions.len() < 16_384); + assert!( + report.functions.iter().enumerate().all(|(index, function)| function.index as usize == index && function.signature.starts_with("(i32, i32")) + ); + assert!(!report.warnings.iter().any(|warning| warning.contains("function declarations"))); +} + +#[test] +fn repeated_import_signatures_stay_bounded_even_when_validation_fails() { + let report = bounded(&shared_type(0, 16_384, 0, 0, 1000)); + assert!(!report.validation.valid); + assert!(report.validation.error.as_deref().unwrap().contains("effective type size")); + assert_eq!(report.imported_functions, 16_384); + assert!(!report.imports.is_empty()); + assert!(report.imports.len() < 16_384); +} + +#[test] +fn imported_and_defined_tags_share_the_same_budget_as_functions() { + for (functions, imports, imported_tags, defined_tags) in [(0, 0, 8192, 0), (0, 0, 0, 16_384), (4096, 4096, 4096, 4096)] { + let report = bounded(&shared_type(functions, imports, imported_tags, defined_tags, 1000)); + assert_eq!(report.imported_functions as usize, imports); + assert!(report.tags.len() < imported_tags + defined_tags); + assert!(report.tags.iter().all(|tag| tag.signature.starts_with("(i32, i32"))); + assert!(report.tags.iter().filter(|tag| !tag.imported).all(|tag| tag.index as usize >= imported_tags)); + } +} + +#[test] +fn exported_function_names_cannot_multiply_a_small_name_section() { + let mut module = shared_type(0, 1, 0, 0, 0); + let name = "λ".repeat(32_768); + let mut exports = leb(1280); + for index in 0..1280 { + string(&mut exports, &format!("export_{index}")); + exports.extend([0, 0]); + } + section(&mut module, 7, &exports); + let mut map = vec![1, 0]; + string(&mut map, &name); + let mut names = vec![4, b'n', b'a', b'm', b'e', 1]; + names.extend(leb(map.len())); + names.extend(map); + section(&mut module, 0, &names); + let report = bounded(&module); + assert!(report.validation.valid, "{:?}", report.validation); + assert_eq!(report.exports.len(), 1280); + assert!(report.exports.iter().any(|export| export.target_name.is_none())); + assert!(report.exports.iter().filter_map(|export| export.target_name.as_deref()).all(|value| value == name)); +} + +#[test] +fn ordinary_shared_signatures_and_names_remain_complete() { + let report = inspect(&shared_type(2, 2, 2, 2, 3)).unwrap(); + assert!(report.validation.valid, "{:?}", report.validation); + assert!(!report.truncated, "{:?}", report.warnings); + assert_eq!(report.imported_functions, 2); + assert_eq!(report.functions.len(), 2); + assert_eq!(report.tags.len(), 4); + assert!(report.imports.iter().all(|item| item.signature == "(i32, i32, i32) -> ()")); + assert!(report.tags.iter().all(|tag| tag.signature == "(i32, i32, i32) -> ()")); + assert_eq!(report.functions.iter().map(|function| function.index).collect::>(), [2, 3]); + assert_eq!(report.tags.iter().map(|tag| tag.index).collect::>(), [0, 1, 2, 3]); + assert!(report.tags[..2].iter().all(|tag| tag.imported)); + assert!(report.tags[2..].iter().all(|tag| !tag.imported)); +} diff --git a/crates/wasm/src/test_support.rs b/crates/wasm/src/test_support.rs new file mode 100644 index 000000000..800f368fd --- /dev/null +++ b/crates/wasm/src/test_support.rs @@ -0,0 +1,123 @@ +// Synthetic fixtures and mutations from Craft Anything testkit, same source revision. +pub use craft_testkit::sample_damage::{mutations, noise}; + +pub mod formats { + fn leb(mut v: u32, out: &mut Vec) { + loop { + let b = (v & 0x7f) as u8; + v >>= 7; + if v == 0 { + out.push(b); + return; + } + out.push(b | 0x80); + } + } + + fn name(s: &str, out: &mut Vec) { + leb(s.len() as u32, out); + out.extend_from_slice(s.as_bytes()); + } + + fn section(id: u8, body: &[u8], out: &mut Vec) { + out.push(id); + leb(body.len() as u32, out); + out.extend_from_slice(body); + } + + fn custom(sname: &str, body: &[u8], out: &mut Vec) { + let mut c = Vec::new(); + name(sname, &mut c); + c.extend_from_slice(body); + section(0, &c, out); + } + + /// A small valid module: imports `env.log(i32)`, defines `add(i32,i32)->i32` and `_start`, + /// one memory, one mutable global, one data segment, exports `add`, `memory` and `_start`, and + /// carries `name`, `producers` and `sourceMappingURL` custom sections. + pub fn wasm_module() -> Vec { + wasm_module_with_source_mapping_url("demo.wasm.map") + } + + /// The sample module with a caller-provided source map URL. + pub fn wasm_module_with_source_mapping_url(url: &str) -> Vec { + let mut m = b"\0asm\x01\0\0\0".to_vec(); + section(1, &[3, 0x60, 2, 0x7f, 0x7f, 1, 0x7f, 0x60, 0, 0, 0x60, 1, 0x7f, 0], &mut m); + let mut imp = vec![1]; + name("env", &mut imp); + name("log", &mut imp); + imp.extend_from_slice(&[0, 2]); + section(2, &imp, &mut m); + section(3, &[2, 0, 1], &mut m); + section(5, &[1, 0, 1], &mut m); + section(6, &[1, 0x7f, 1, 0x41, 42, 0x0b], &mut m); + let mut exp = vec![3]; + name("add", &mut exp); + exp.extend_from_slice(&[0, 1]); + name("memory", &mut exp); + exp.extend_from_slice(&[2, 0]); + name("_start", &mut exp); + exp.extend_from_slice(&[0, 2]); + section(7, &exp, &mut m); + let add = [0u8, 0x20, 0, 0x20, 1, 0x6a, 0x0b]; + let start = [0u8, 0x41, 7, 0x10, 0, 0x0b]; + let mut code = vec![2, add.len() as u8]; + code.extend_from_slice(&add); + code.push(start.len() as u8); + code.extend_from_slice(&start); + section(10, &code, &mut m); + section(11, &[1, 0, 0x41, 0, 0x0b, 2, b'h', b'i'], &mut m); + // name section + let mut n = Vec::new(); + let mut module_name = Vec::new(); + name("demo", &mut module_name); + n.push(0); + leb(module_name.len() as u32, &mut n); + n.extend_from_slice(&module_name); + let mut fnames = vec![3]; + for (i, f) in ["log", "add", "_start"].iter().enumerate() { + leb(i as u32, &mut fnames); + name(f, &mut fnames); + } + n.push(1); + leb(fnames.len() as u32, &mut n); + n.extend_from_slice(&fnames); + custom("name", &n, &mut m); + let mut p = vec![2]; + name("language", &mut p); + p.push(1); + name("Rust", &mut p); + name("", &mut p); + name("processed-by", &mut p); + p.push(1); + name("rustc", &mut p); + name("1.80.0", &mut p); + custom("producers", &p, &mut m); + let mut s = Vec::new(); + name(url, &mut s); + custom("sourceMappingURL", &s, &mut m); + let mut tf = vec![1, b'+']; + name("mutable-globals", &mut tf); + custom("target_features", &tf, &mut m); + m + } + + /// JavaScript glue that loads the module. + pub const WASM_GLUE: &str = + "const imports = { env: { log: (x) => console.log(x) } };\nfetch(\"./demo.wasm\").then(r => WebAssembly.instantiateStreaming(r, imports));\n"; +} + +pub fn leb(mut value: usize) -> Vec { + let mut out = Vec::new(); + while value >= 128 { + out.push((value as u8 & 0x7f) | 0x80); + value >>= 7; + } + out.push(value as u8); + out +} + +pub fn string(out: &mut Vec, value: &str) { + out.extend(leb(value.len())); + out.extend(value.as_bytes()); +} diff --git a/crates/xfdf/Cargo.toml b/crates/xfdf/Cargo.toml new file mode 100644 index 000000000..9b1304d89 --- /dev/null +++ b/crates/xfdf/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "craft-xfdf" +description = "Bounded XFDF field-value reader without PDF or form policy" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true +[lints] +workspace = true +[dependencies] +craft-xml = { workspace = true } +thiserror = { workspace = true } diff --git a/crates/xfdf/README.md b/crates/xfdf/README.md new file mode 100644 index 000000000..ce8c2802e --- /dev/null +++ b/crates/xfdf/README.md @@ -0,0 +1,11 @@ +# craft-xfdf + +Scoped XFDF field-value reader extracted from PDFCraft #5937 refresh cb564fd82. Does not read FDF/PDF, parse annotations or apply field values. PDF application, permissions, calculations and password filtering stay caller-owned. + +Preserves local-name matching, duplicate field order, dotted names, empty leaf clearing and ignored rich-text-only fields, direct split text concatenation and source UTF-8 replacement policy. XML depth1024, field depth64,2million nodes and64MiB charged expanded names/text. Names and values are charged before copying; temporary parents and emitted name copies both count. XML input/tree memory remains governed by craft-xml defaults, not this output budget alone. + +Current strict craft-xml also rejects duplicate raw/expanded attributes and more than1024 active inherited namespace declarations; older app XML accepted those. These deliberate tighter admissions must be disclosed by consumers. No complete XFDF or corpus parity claim. + +Source history remains in PDFCraft. Shared move and preallocation budget reconciliation by Dylan M. Taylor, MIT OR Apache-2.0. No assets are imported. + +Retained source contributors (move-aware blame): Brandon Thomas, Marco Arnulfo, pkibiego and storybored379; module split by aiongg. Subsequent budget/integration work by Dylan M. Taylor. diff --git a/crates/xfdf/src/lib.rs b/crates/xfdf/src/lib.rs new file mode 100644 index 000000000..002376106 --- /dev/null +++ b/crates/xfdf/src/lib.rs @@ -0,0 +1,177 @@ +//! Bounded XFDF field values; PDF annotations and form application remain caller-owned. +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +#![forbid(unsafe_code)] +const MAX_NODES: u32 = 2_000_000; +const MAX_DEPTH: usize = 64; +const MAX_XML_DEPTH: usize = 1024; +const MAX_XFDF_TEXT: usize = 64 << 20; +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +pub enum Error { + #[error("this file isn't XFDF, FDF, XML or text form data")] + UnknownFormat, + #[error("the file can't be read: {0}")] + Malformed(String), +} +/// Read field names and values in file order. This does not apply values to a PDF. +/// UTF-8 replacement and leading BOM/space handling retain the source app policy. +pub fn read_fields(bytes: &[u8]) -> Result)>, Error> { + let text = String::from_utf8_lossy(bytes); + let text = text.trim_start_matches('\u{feff}').trim_start(); + check_form_xml_depth(text)?; + let root = parse_form_xml(text).map_err(|_| Error::UnknownFormat)?; + if root.local() != "xfdf" { + return Err(Error::UnknownFormat); + } + let mut values = Vec::new(); + if let Some(fields) = root.child(craft_xml::Local("fields")) { + walk_xfdf(fields, "", &mut values, 0, &mut 0)?; + } + Ok(values) +} +fn xml_nesting_depth(text: &str) -> usize { + fn find(hay: &[u8], from: usize, needle: &[u8]) -> Option { + (from..hay.len()).find(|&p| hay[p..].starts_with(needle)) + } + let b = text.as_bytes(); + let (mut i, mut depth, mut max) = (0usize, 0usize, 0usize); + while i < b.len() { + if b[i] != b'<' { + i += 1; + } else if b[i..].starts_with(b"").map_or(b.len(), |p| p + 3); + } else if b[i..].starts_with(b"").map_or(b.len(), |p| p + 3); + } else if b[i..].starts_with(b"").map_or(b.len(), |p| p + 2); + } else if b[i..].starts_with(b"").map_or(b.len(), |p| p + 1); + } else if b[i..].starts_with(b"").map_or(b.len(), |p| p + 1); + } else { + depth += 1; + max = max.max(depth); + // Skip the rest of the start tag, over any quoted attribute value, and remember its last + // significant byte: a tag that ends in `/` closes itself, so it is one level deep while it + // is read rather than a level left open. + i += 1; + let mut quote = 0u8; + let mut last = 0u8; + while i < b.len() { + let c = b[i]; + if quote != 0 { + if c == quote { + quote = 0; + } + } else if c == b'"' || c == b'\'' { + quote = c; + } else if c == b'>' { + i += 1; + break; + } else if !c.is_ascii_whitespace() { + last = c; + } + i += 1; + } + if last == b'/' { + depth -= 1; + } + } + } + max +} + +fn check_form_xml_depth(text: &str) -> Result<(), Error> { + let depth = xml_nesting_depth(text); + if depth > MAX_XML_DEPTH { + return Err(Error::Malformed(format!("the file nests XML elements {depth} deep, past the {MAX_XML_DEPTH} the reader accepts"))); + } + Ok(()) +} + +fn parse_form_xml(text: &str) -> Result { + let mut opts = craft_xml::Options { strict: true, doctype: craft_xml::Doctype::Reject, ..Default::default() }; + opts.limits.max_nodes = MAX_NODES as usize; + opts.limits.max_depth = MAX_XML_DEPTH.max(1024); + opts.parse(text.as_bytes()) +} + +fn el_text(el: &craft_xml::El) -> String { + el.children + .iter() + .filter_map(|n| match n { + craft_xml::Node::Text(t) => Some(t.as_str()), + _ => None, + }) + .collect() +} + +fn walk_xfdf(node: &craft_xml::El, prefix: &str, out: &mut Vec<(String, Vec)>, depth: usize, bytes: &mut usize) -> Result<(), Error> { + if depth > MAX_DEPTH { + return Err(Error::Malformed(format!("XFDF fields nest more than {MAX_DEPTH} deep"))); + } + for f in node.children(craft_xml::Local("field")) { + let Some(name) = f.attr("name") else { continue }; + // Charge temporary parent names too, before allocation/recursion can amplify them. + let full_len = if prefix.is_empty() { name.len() } else { prefix.len().saturating_add(1).saturating_add(name.len()) }; + charge_xfdf_text(bytes, full_len)?; + let full = if prefix.is_empty() { name.to_string() } else { format!("{prefix}.{name}") }; + let mut vals = Vec::new(); + for value in f.children(craft_xml::Local("value")) { + let len = value + .children + .iter() + .filter_map(|n| match n { + craft_xml::Node::Text(t) => Some(t.len()), + _ => None, + }) + .fold(0, usize::saturating_add); + charge_xfdf_text(bytes, len)?; + vals.push(el_text(value)); + } + // A leaf with no `` is an empty field (the export writes one that way), so it + // clears the field like FDF's `/V ()`; a field with kids only names their parent. A + // leaf holding only `` is not read yet, so it leaves the field alone. + let names_kids_or_rich = f.elements().any(|c| matches!(c.local(), "field" | "value-richtext")); + if !vals.is_empty() || !names_kids_or_rich { + charge_xfdf_text(bytes, full.len())?; // The emitted name is a second owned copy. + out.push((full.clone(), vals)); + } + walk_xfdf(f, &full, out, depth + 1, bytes)?; + } + Ok(()) +} + +fn charge_xfdf_text(bytes: &mut usize, amount: usize) -> Result<(), Error> { + *bytes = bytes.saturating_add(amount); + if *bytes > MAX_XFDF_TEXT { + return Err(Error::Malformed(format!("the file expands to more than {} MiB of field names and values", MAX_XFDF_TEXT >> 20))); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn field_order_empty_rich_and_split_values() { + let xml = b"abcskiplast"; + assert_eq!( + read_fields(xml).unwrap(), + vec![("parent.child".into(), vec!["ab".into(), "c".into()]), ("empty".into(), vec![]), ("parent.child".into(), vec!["last".into()])] + ); + } + #[test] + fn bounds_and_strict_xml_hardening() { + assert_eq!(read_fields(b""), Err(Error::UnknownFormat)); + assert!(read_fields(b"").is_err()); + assert!(read_fields(b"").is_err()); + let text = format!("{}{}{}", "".repeat(65), "x", "".repeat(65)); + assert!(read_fields(text.as_bytes()).is_err()); + let fields = parse_form_xml("").unwrap(); + let mut out = Vec::new(); + assert!(walk_xfdf(&fields, "", &mut out, 0, &mut (MAX_XFDF_TEXT - 5)).is_err()); + assert!(out.is_empty()); + walk_xfdf(&fields, "", &mut out, 0, &mut (MAX_XFDF_TEXT - 6)).unwrap(); + } +} diff --git a/crates/zip/Cargo.toml b/crates/zip/Cargo.toml index 89cb2d1a7..cfdb333b4 100644 --- a/crates/zip/Cargo.toml +++ b/crates/zip/Cargo.toml @@ -35,7 +35,7 @@ lenient = ["dep:miniz_oxide"] lenient-bzip2 = ["lenient", "dep:bzip2-rs"] # Methods 14 (ZIP LZMA) and 95 (XZ). lenient-lzma = ["lenient", "dep:lzma-rs"] -# Method 93. +# Zstandard methods 20 (legacy) and 93. lenient-zstd = ["lenient", "dep:ruzstd"] serde = ["dep:serde"] diff --git a/crates/zip/src/lenient.rs b/crates/zip/src/lenient.rs index c9bed3764..20da277cd 100644 --- a/crates/zip/src/lenient.rs +++ b/crates/zip/src/lenient.rs @@ -91,7 +91,7 @@ impl ZipEntry { 9 => "deflate64", 12 => "bzip2", 14 => "lzma", - 93 => "zstd", + 20 | 93 => "zstd", 95 => "xz", 99 => "aes", _ => "other", @@ -463,7 +463,7 @@ pub fn read_entry_unverified(data: &[u8], e: &ZipEntry, max_bytes: u64) -> Resul })?, 12 | 95 => decode_bzip2_or_xz(raw, e, max_bytes)?, 14 => decode_lzma(raw, e, max_bytes)?, - 93 => decode_zstd(raw, e, max_bytes)?, + 20 | 93 => decode_zstd(raw, e, max_bytes)?, m => return Err(ZipError::Unsupported(format!("compression method {m} ({}) for {}", e.method_name(), e.name))), }; if out.len() as u64 != e.uncompressed_size { diff --git a/crates/zip/src/lenient/tests.rs b/crates/zip/src/lenient/tests.rs index c2ca97394..ded7c0ecd 100644 --- a/crates/zip/src/lenient/tests.rs +++ b/crates/zip/src/lenient/tests.rs @@ -391,3 +391,40 @@ fn a_method_left_out_of_the_build_is_unsupported() { data[8..10].copy_from_slice(&93u16.to_le_bytes()); assert!(matches!(read_entry_unverified(&data, &entry, 100), Err(ZipError::Unsupported(m)) if m.contains("not built in"))); } + +// RRFZ, Craft Anything 29badae7: deprecated ZIP method 20 is Zstandard. +#[cfg(feature = "lenient-zstd")] +mod legacy_zstandard { + use crate::lenient as zip; + use crate::lenient::testkit::zip_stored; + + const BODY: &[u8] = b"zstandard payload"; + + /// Build a ZIP with one single-segment Zstandard frame containing a raw final block. + fn sample(method: u16) -> Vec { + let size = BODY.len() as u8; + let mut frame = vec![0x28, 0xb5, 0x2f, 0xfd, 0x20, size, (size << 3) | 1, 0, 0]; + frame.extend_from_slice(BODY); + let mut data = zip_stored(&[("a.txt", &frame)]); + let cd = 30 + "a.txt".len() + frame.len(); + data[8..10].copy_from_slice(&method.to_le_bytes()); + data[14..18].copy_from_slice(&zip::crc32(BODY).to_le_bytes()); + data[22..26].copy_from_slice(&u32::from(size).to_le_bytes()); + data[cd + 10..cd + 12].copy_from_slice(&method.to_le_bytes()); + data[cd + 16..cd + 20].copy_from_slice(&zip::crc32(BODY).to_le_bytes()); + data[cd + 24..cd + 28].copy_from_slice(&u32::from(size).to_le_bytes()); + data + } + + /// The deprecated and current method IDs both decode through the bounded ZIP reader. + #[test] + fn reads_zstandard_members_under_both_method_ids() { + for method in [93, 20] { + let data = sample(method); + let entry = zip::parse(&data, 1).unwrap().entries.remove(0); + assert_eq!(entry.method_name(), "zstd", "method {method}"); + assert_eq!(zip::read_entry(&data, &entry, 100).unwrap(), BODY); + assert!(matches!(zip::read_entry(&data, &entry, 2), Err(zip::ZipError::TooLarge(2)))); + } + } +} diff --git a/scripts/ci-test-inputs.sh b/scripts/ci-test-inputs.sh new file mode 100755 index 000000000..32af3ea77 --- /dev/null +++ b/scripts/ci-test-inputs.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# The test inputs the Workspace gate and the nightly full run fetch before their tests: the +# PhotoCraft test corpora (PSD, PSD tools, ATN, EXR, HEIF, PngSuite) at a pinned PhotoCraft commit, +# and the licensed PDF test fonts. Exports CRAFT_PSD_CORPUS and CRAFT_PDF_TEST_FONT_DIR through +# $GITHUB_ENV. Run from the repository root. +# +# A corpus already in .test-inputs/photocraft/corpus (restored from the CI cache, which is keyed on +# this file) is used as it is, without the PhotoCraft checkout and its xtask build. +set -euo pipefail + +# The PhotoCraft commit whose `cargo xtask corpus` pins the corpora. Changing it changes this file, +# which starts a new corpus cache entry. +photocraft=74330157cb601573ae4523a95bb9132822fc3918 +fetcher=.test-inputs/photocraft + +if [ -z "$(ls -A "$fetcher/corpus" 2>/dev/null)" ]; then + rm -rf "$fetcher" + mkdir -p "$fetcher" + git -C "$fetcher" init --quiet + git -C "$fetcher" remote add origin https://github.com/storytold/PhotoCraft.git + git -C "$fetcher" fetch --quiet --depth=1 origin "$photocraft" + git -C "$fetcher" checkout --quiet --detach FETCH_HEAD + (cd "$fetcher" && cargo xtask corpus --psd --psd-tools --atn --exr --heif --pngsuite) +fi +mkdir -p corpus +cp -a "$fetcher/corpus/." corpus/ + +python scripts/test-fetch-pdf-test-fonts.py +python scripts/fetch-pdf-test-fonts.py "$RUNNER_TEMP/craft-pdf-test-fonts" + +printf 'CRAFT_PSD_CORPUS=%s\n' "$GITHUB_WORKSPACE/$fetcher/corpus" >> "$GITHUB_ENV" +printf 'CRAFT_PDF_TEST_FONT_DIR=%s\n' "$RUNNER_TEMP/craft-pdf-test-fonts" >> "$GITHUB_ENV" diff --git a/scripts/test-workspace-concurrency.py b/scripts/test-workspace-concurrency.py index 96bfae304..db20031e9 100644 --- a/scripts/test-workspace-concurrency.py +++ b/scripts/test-workspace-concurrency.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 -"""Check the Workspace workflow's pull-request-only cancellation policy. +"""Check the Workspace workflow's cancellation policy: pull requests and pushes to main cancel the +run they supersede; release tags, manual runs and scheduled runs are never cancelled. The policy is the top-level `concurrency` block of .github/workflows/workspace.yml. This script reads those fields, requires the documented expression shapes, and checks which group each event @@ -21,16 +22,28 @@ except ImportError: yaml = None DEFAULT_WORKFLOW = Path(__file__).resolve().parent.parent / ".github" / "workflows" / "workspace.yml" -PULL_REQUEST, PUSH, DISPATCH = "pull_request", "push", "workflow_dispatch" +PULL_REQUEST, PUSH, DISPATCH, SCHEDULE = "pull_request", "push", "workflow_dispatch", "schedule" -# The two shapes this policy may use. GitHub reads `a && b || c` as "b, or c when a is false"; these -# patterns are the whole of that expression language this script understands, so a reworded policy -# has to be reworded here too, on purpose. github.ref, never github.head_ref or github.sha. +# The runs that share a group per ref and cancel the run they supersede. GitHub reads `a || b && c` +# as `a || (b && c)`, and `x && github.ref || github.run_id` as "github.ref, or the run id when x is +# false". This exact condition is the whole of the expression language this script understands, so +# a reworded policy has to be reworded here too, on purpose. github.ref, never github.head_ref or +# github.sha. +SUPERSEDED = "github.event_name == 'pull_request' || github.event_name == 'push' && github.ref == 'refs/heads/main'" GROUP_SHAPE = re.compile( - r"(?P[A-Za-z0-9._-]+)-\$\{\{\s*github\.event_name\s*==\s*'(?P[a-z_]+)'\s*" + r"(?P[A-Za-z0-9._-]+)-\$\{\{\s*\(\s*(?P[^()]+?)\s*\)\s*" r"&&\s*github\.ref\s*\|\|\s*github\.run_id\s*\}\}" ) -CANCEL_SHAPE = re.compile(r"\$\{\{\s*github\.event_name\s*==\s*'(?P[a-z_]+)'\s*\}\}") +CANCEL_SHAPE = re.compile(r"\$\{\{\s*(?P[^{}]+?)\s*\}\}") + + +def normal(expression): + return re.sub(r"\s+", " ", expression).strip() + + +def superseded(event_name, ref): + """SUPERSEDED, evaluated.""" + return event_name == PULL_REQUEST or (event_name == PUSH and ref == "refs/heads/main") def read_concurrency(text): @@ -72,22 +85,25 @@ def other_run(event_name, ref, run_id): MAIN, TAG = ("refs/heads/main", "refs/tags/v1.2.3") SAME_GROUP = [ ("a newer commit on the same pull request", pull_request(1316, 43599251877), pull_request(1316, 43600899878)), - ("a rerun of one main run", other_run(PUSH, MAIN, 7001), other_run(PUSH, MAIN, 7001)), + ("two pushes to main", other_run(PUSH, MAIN, 3), other_run(PUSH, MAIN, 4)), ] DIFFERENT_GROUPS = [ ("two pull requests", pull_request(1316, 1), pull_request(1317, 1)), ("the same branch name in two forks", pull_request(1316, 2), pull_request(1400, 2)), - ("two pushes to main", other_run(PUSH, MAIN, 3), other_run(PUSH, MAIN, 4)), ("two release tag runs", other_run(PUSH, TAG, 5), other_run(PUSH, TAG, 6)), ("two manual runs", other_run(DISPATCH, MAIN, 7), other_run(DISPATCH, MAIN, 8)), + ("two scheduled runs", other_run(SCHEDULE, MAIN, 15), other_run(SCHEDULE, MAIN, 16)), + ("a main push and a scheduled run", other_run(PUSH, MAIN, 17), other_run(SCHEDULE, MAIN, 17)), + ("a main push and a manual run of main", other_run(PUSH, MAIN, 18), other_run(DISPATCH, MAIN, 18)), ("a main push and a pull request", other_run(PUSH, MAIN, 9), pull_request(1316, 9)), ("a release tag and a pull request", other_run(PUSH, TAG, 10), pull_request(1316, 10)), ] CANCELLATION = [ ("a push supersedes the previous run of its pull request", pull_request(1316, 11), True), - ("a push to main is never cancelled", other_run(PUSH, MAIN, 12), False), + ("a push to main supersedes the previous main run", other_run(PUSH, MAIN, 12), True), ("a release tag run is never cancelled", other_run(PUSH, TAG, 13), False), ("a manual run is never cancelled", other_run(DISPATCH, MAIN, 14), False), + ("the scheduled run that saves the caches is never cancelled", other_run(SCHEDULE, MAIN, 19), False), ] @@ -103,17 +119,17 @@ class ConcurrencyTests(unittest.TestCase): if not cls.group_shape or not cls.cancel_shape: raise AssertionError(f"the concurrency fields are not the documented shapes: {cls.fields}") - def cancels(self, event_name): - return event_name == self.cancel_shape["event"] + def cancels(self, event_name, ref): + return superseded(event_name, ref) def group(self, event_name, ref, run_id): - return self.group_shape["prefix"] + "-" + (ref if event_name == self.group_shape["event"] else str(run_id)) + return self.group_shape["prefix"] + "-" + (ref if superseded(event_name, ref) else str(run_id)) def test_policy_is_the_two_documented_fields(self): self.assertEqual(sorted(self.fields), ["cancel-in-progress", "group"]) self.assertEqual(self.group_shape["prefix"], "workspace") - self.assertEqual(self.group_shape["event"], PULL_REQUEST) - self.assertEqual(self.cancel_shape["event"], PULL_REQUEST) + self.assertEqual(normal(self.group_shape["condition"]), SUPERSEDED) + self.assertEqual(normal(self.cancel_shape["condition"]), SUPERSEDED) def test_a_scalar_concurrency_value_is_not_read_as_a_mapping(self): cases = { @@ -133,10 +149,10 @@ class ConcurrencyTests(unittest.TestCase): with self.subTest(description): self.assertNotEqual(self.group(*first), self.group(*second), description) - def test_only_a_pull_request_cancels(self): + def test_only_pull_requests_and_main_pushes_cancel(self): for description, context, expected in CANCELLATION: with self.subTest(description): - self.assertEqual(self.cancels(context[0]), expected, description) + self.assertEqual(self.cancels(context[0], context[1]), expected, description) def test_pyyaml_reads_the_same_fields(self): if yaml is None: diff --git a/scripts/test-workspace-tests.py b/scripts/test-workspace-tests.py new file mode 100644 index 000000000..fada6dc9b --- /dev/null +++ b/scripts/test-workspace-tests.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +"""Exercise the native workflow's command selection without compiling Rust.""" +import os +import re +import subprocess +import tomllib +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +WORKFLOW = ROOT / ".github/workflows/workspace.yml" + + +def block(text, name): + match = re.search(r"^ " + re.escape(name) + r": \|\n((?: .*\n|\n)+)", text, re.M) + if match is None: + raise AssertionError(f"missing command block: {name}") + return "\n".join(line[8:] for line in match[1].splitlines()) + + +def commands(os_name, fail="", name="command"): + # Functions avoid executing any caller-supplied cargo command or changing the workspace. + # The real bash -euo pipefail semantics still apply, including failures inside a branch. + stub = """ +cargo() { + printf '%s\\n' "$*" + if [ -n "$FAIL" ] && [ "$*" = "$FAIL" ]; then return 17; fi +} +python3() { :; } +""" + return subprocess.run( + ["bash", "-euo", "pipefail", "-c", stub + block(WORKFLOW.read_text(), name)], + env={**os.environ, "RUNNER_OS": os_name, "FAIL": fail}, + text=True, capture_output=True, check=False, + ) + + +COMMON = [ + "fmt --check", + "clippy --workspace --all-targets --locked -- -D warnings", + "clippy --workspace --all-targets --all-features --locked -- -D warnings", +] +TESTS = [ + "nextest run --workspace --all-features --locked --profile ci", + "test --workspace --doc --all-features --locked", + "nextest run -p craft-image-codecs -p craft-text --locked --profile ci", +] + + +class WorkspaceTests(unittest.TestCase): + def test_every_leg_lints_with_both_feature_sets(self): + for os_name in ("Linux", "macOS", "Windows"): + with self.subTest(os=os_name): + result = commands(os_name, name="lint-command") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.splitlines(), COMMON) + + def test_linux_and_macos_run_one_feature_set_with_doctests(self): + for os_name in ("Linux", "macOS"): + with self.subTest(os=os_name): + result = commands(os_name) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.splitlines(), TESTS) + + def test_each_stage_failure_remains_fatal(self): + for name, stages in (("lint-command", COMMON), ("command", TESTS)): + for index, command in enumerate(stages): + with self.subTest(block=name, command=command): + result = commands("Linux", command, name) + self.assertEqual(result.returncode, 17, result.stderr) + self.assertEqual(result.stdout.splitlines(), stages[:index + 1]) + + def test_windows_runs_the_same_tests(self): + source = block(WORKFLOW.read_text(), "windows-command") + cargo = [line.strip() for line in source.splitlines() if line.strip().startswith("cargo ")] + self.assertEqual(cargo, ["cargo " + command for command in TESTS]) + + def test_ci_profile_keeps_its_failure_policy_without_retries_or_filtering(self): + config = tomllib.loads((ROOT / ".config/nextest.toml").read_text()) + self.assertIs(config["profile"]["ci"]["fail-fast"], False) + for profile in config["profile"].values(): + self.assertNotIn("default-filter", profile) + self.assertNotIn("retries", profile) + self.assertEqual(config["profile"]["ci"]["slow-timeout"], {"period": "60s"}) + + def test_unix_resource_groups_are_serial_and_leave_windows_alone(self): + config = tomllib.loads((ROOT / ".config/nextest.toml").read_text()) + overrides = config["profile"]["ci"]["overrides"] + self.assertEqual(overrides, [ + {"platform": "cfg(unix)", + "filter": "package(craft-ui) | package(craft-keymap) | package(craft-update)", + "test-group": "gpu"}, + {"platform": "cfg(unix)", "filter": "package(craft-video) & kind(test)", + "test-group": "video-integration"}, + ]) + for override in overrides: + self.assertEqual(config["test-groups"][override["test-group"]]["max-threads"], 1) + + def test_workflow_stays_shared_and_keeps_platform_gates(self): + source = WORKFLOW.read_text() + self.assertIn("uses: storytold/craft-common-actions/.github/workflows/gate.yml@main", source) + self.assertIn('os: \'["ubuntu-latest","windows-latest","macos-latest"]\'', source) + self.assertIn(" msrv:", source) + self.assertIn(" wasm:", source) + self.assertIn("python3 scripts/test-workspace-tests.py", block(source, "lint-command")) + + +if __name__ == "__main__": + unittest.main() +