Merge pull request #2150 from FredApps/fix/dex-main-integration

DEX: refresh main without losing safety repair
This commit is contained in:
FredApps authored and GitHub committed 2026-11-18 10:13:03 +07:00
commit e5942cd706
129 files changed
+12638 -975

No files matched your search

+20 -3
View File
@@ -1,7 +1,6 @@
# cargo-nextest (https://nexte.st) runs every test in its own process, in parallel across test
# binaries. The Windows leg of the Workspace gate uses it (.github/workflows/workspace.yml); every
# other leg, and local runs, keep `cargo test`. Nextest does not run doctests, so the Windows leg
# also runs `cargo test --doc`.
# binaries. Every leg of the Workspace gate uses it (.github/workflows/workspace.yml); local runs may
# keep `cargo test`. Nextest does not run doctests, so the gate also runs `cargo test --doc`.
#
# cargo nextest run --workspace --profile ci
# cargo test --workspace --doc
@@ -22,6 +21,9 @@ video-fixtures = { max-threads = 1 }
# craft-print: `PRINT_TICKET_PROVIDER` serializes the Windows PrintTicket provider, and the tests
# share the one test print queue the gate installs.
print = { max-threads = 1 }
# craft-video integration tests on Linux and macOS: the binaries generate and read the same fixture
# directories (see the `ci` overrides below).
video-integration = { max-threads = 1 }
[[profile.default.overrides]]
filter = 'package(craft-ui) & (binary(visual_regression) | test(/^combo::tests::|selected_tab_erases_a_baseline_centered_on_its_bottom_edge/))'
@@ -42,3 +44,18 @@ test-group = 'print'
fail-fast = false
slow-timeout = { period = "60s" }
failure-output = "immediate-final"
# Linux and macOS (the resource audit in craft-libs#2058): every GPU consumer shares the one `gpu`
# slot, and the craft-video integration binaries share their fixture directories. Conservative on
# purpose: the in-process mutexes these tests rely on do not reach across nextest's processes.
# Windows keeps the default groups above, which its leg has passed with since it moved to nextest.
[[profile.ci.overrides]]
platform = 'cfg(unix)'
filter = 'package(craft-ui) | package(craft-keymap) | package(craft-update)'
test-group = 'gpu'
[[profile.ci.overrides]]
platform = 'cfg(unix)'
filter = 'package(craft-video) & kind(test)'
test-group = 'video-integration'
+1 -28
View File
@@ -18,7 +18,6 @@ workflows:
- 'actions/checkout@v7.0.1'
- 'step-security/dtolnay-rust-toolchain@v1.0.0'
- 'step-security/harden-runner@v2.22.4'
- 'step-security/rust-cache@v2.9.2'
'.github/workflows/cms.yml':
- 'actions/checkout@v4.4.0'
- 'dtolnay/rust-toolchain@stable'
@@ -27,7 +26,6 @@ workflows:
- 'actions/checkout@v7.0.1'
- 'step-security/dtolnay-rust-toolchain@v1.0.0'
- 'step-security/harden-runner@v2.22.4'
- 'step-security/rust-cache@v2.9.2'
'.github/workflows/dialogs.yml':
- 'actions/checkout@v7.0.1'
- 'step-security/dtolnay-rust-toolchain@v1.0.0'
@@ -36,10 +34,6 @@ workflows:
- 'actions/checkout@v7.0.1'
- 'step-security/dtolnay-rust-toolchain@v1.0.0'
- 'step-security/harden-runner@v2.22.4'
'.github/workflows/docpath.yml':
- 'actions/checkout@v7.0.1'
- 'step-security/dtolnay-rust-toolchain@v1.0.0'
- 'step-security/harden-runner@v2.22.4'
'.github/workflows/ecproj-format.yml':
- 'actions/checkout@v7.0.1'
- 'step-security/dtolnay-rust-toolchain@v1.0.0'
@@ -52,22 +46,15 @@ workflows:
- 'actions/checkout@v7.0.1'
- 'dtolnay/rust-toolchain@stable'
- 'step-security/harden-runner@v2.22.4'
- 'swatinem/rust-cache@v2.9.3'
'.github/workflows/house-rules.yml': []
'.github/workflows/i18n.yml':
- 'actions/checkout@v7.0.1'
- 'step-security/dtolnay-rust-toolchain@v1.0.0'
- 'step-security/harden-runner@v2.22.4'
'.github/workflows/jpeg.yml':
- 'actions/checkout@v7.0.1'
- 'step-security/dtolnay-rust-toolchain@v1.0.0'
- 'step-security/harden-runner@v2.22.4'
- 'step-security/rust-cache@v2.9.2'
'.github/workflows/keymap.yml':
- 'actions/checkout@v7.0.1'
- 'step-security/dtolnay-rust-toolchain@v1.0.0'
- 'step-security/harden-runner@v2.22.4'
- 'step-security/rust-cache@v2.9.2'
'.github/workflows/lockfile.yml':
- 'actions/checkout@v7.0.1'
- 'step-security/harden-runner@v2.22.4'
@@ -75,11 +62,10 @@ workflows:
- 'actions/checkout@v7.0.1'
- 'step-security/dtolnay-rust-toolchain@v1.0.0'
- 'step-security/harden-runner@v2.22.4'
- 'step-security/rust-cache@v2.9.2'
'.github/workflows/menu.yml':
- 'actions/checkout@v7.0.1'
- 'dtolnay/rust-toolchain@stable'
- 'swatinem/rust-cache@v2.9.3'
'.github/workflows/nightly.yml': []
'.github/workflows/pcraft-format.yml':
- 'actions/checkout@v7.0.1'
- 'step-security/dtolnay-rust-toolchain@v1.0.0'
@@ -89,23 +75,12 @@ workflows:
- 'actions/setup-python@v7.1.0'
- 'step-security/dtolnay-rust-toolchain@v1.0.0'
- 'step-security/harden-runner@v2.22.4'
- 'step-security/rust-cache@v2.9.2'
- 'step-security/taiki-e-install-action@v2.87.32'
'.github/workflows/print.yml':
- 'actions/checkout@v7.0.1'
- 'step-security/dtolnay-rust-toolchain@v1.0.0'
- 'step-security/harden-runner@v2.22.4'
'.github/workflows/proof.yml':
- 'actions/checkout@v7.0.1'
- 'step-security/dtolnay-rust-toolchain@v1.0.0'
- 'step-security/harden-runner@v2.22.4'
- 'step-security/rust-cache@v2.9.2'
'.github/workflows/sonarcloud.yml': []
'.github/workflows/ui.yml':
- 'actions/checkout@v7.0.1'
- 'step-security/dtolnay-rust-toolchain@v1.0.0'
- 'step-security/harden-runner@v2.22.4'
- 'step-security/rust-cache@v2.9.2'
'.github/workflows/update.yml':
- 'actions/checkout@v7.0.1'
- 'step-security/dtolnay-rust-toolchain@v1.0.0'
@@ -118,7 +93,6 @@ workflows:
- 'actions/checkout@v7.0.1'
- 'step-security/dtolnay-rust-toolchain@v1.0.0'
- 'step-security/harden-runner@v2.22.4'
- 'step-security/rust-cache@v2.9.2'
'.github/workflows/web-api.yml':
- 'actions/checkout@v7.0.1'
- 'step-security/dtolnay-rust-toolchain@v1.0.0'
@@ -128,7 +102,6 @@ workflows:
- 'actions/checkout@v7.0.1'
- 'step-security/dtolnay-rust-toolchain@v1.0.0'
- 'step-security/harden-runner@v2.22.4'
- 'step-security/rust-cache@v2.9.2'
- 'step-security/taiki-e-install-action@v2.87.32'
'.github/workflows/window.yml':
- 'actions/checkout@v7.0.1'
+7 -14
View File
@@ -12,20 +12,17 @@ permissions:
contents: read
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued.
group: ai-core-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# One group per pull request and per branch: a newer push cancels the run it supersedes, on main
# too, since the newest run checks everything before it.
group: ai-core-${{ github.ref }}
cancel-in-progress: true
# fmt, clippy and the tests run in the Workspace gate on every OS; this checks what it does not.
jobs:
ai-core:
name: AI model WASM
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
# macOS runners are kept for releases: macOS runs only when started by hand.
os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }}
runs-on: ${{ matrix.os }}
runs-on: ubuntu-latest
steps:
- name: Harden runner
uses: step-security/harden-runner@v2.22.4
@@ -35,10 +32,6 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
components: rustfmt, clippy
targets: wasm32-unknown-unknown
- run: cargo fmt --check
- run: cargo clippy -p craft-ai-core --all-targets --locked -- -D warnings
- run: cargo test -p craft-ai-core --locked
# The model promises to compile for the web.
- run: cargo check -p craft-ai-core --target wasm32-unknown-unknown --locked
+7 -16
View File
@@ -12,20 +12,17 @@ permissions:
contents: read
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued.
group: atomic-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# One group per pull request and per branch: a newer push cancels the run it supersedes, on main
# too, since the newest run checks everything before it.
group: atomic-${{ github.ref }}
cancel-in-progress: true
# fmt, clippy and the tests run in the Workspace gate on every OS; this runs what it does not.
jobs:
atomic:
name: Atomic inherited Linux ACLs
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
# macOS runners are kept for releases: macOS runs only when started by hand.
os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }}
runs-on: ${{ matrix.os }}
runs-on: ubuntu-latest
steps:
- name: Harden runner
uses: step-security/harden-runner@v2.22.4
@@ -35,15 +32,9 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
components: rustfmt, clippy
- name: Install ACL tools
if: runner.os == 'Linux'
run: sudo apt-get update && sudo apt-get install -y acl
- run: cargo fmt --check
- run: cargo clippy -p craft-atomic -p craft-settings --all-targets --locked -- -D warnings
- run: cargo test -p craft-atomic -p craft-settings --locked -- --nocapture
- name: Check inherited Linux ACLs
if: runner.os == 'Linux'
run: cargo test -p craft-atomic --locked -- --ignored --nocapture
wasm:
+12 -40
View File
@@ -11,17 +11,22 @@ on:
permissions:
contents: read
# A fraction of the fuzz-style cases, as in the Workspace gate; nightly.yml runs them in full.
env:
CRAFT_HOSTILE_DIVISOR: "8"
PROPTEST_CASES: "64"
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it
# always gets to save the cache.
group: shared-audio-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# One group per pull request and per branch: a newer push cancels the run it supersedes, on main
# too, since the newest run checks everything before it.
group: shared-audio-${{ github.ref }}
cancel-in-progress: true
jobs:
lint:
timeout-minutes: 15
name: Audio lint, feature sets, docs, WASM
# fmt, clippy with all features and the minimum Rust version run in the Workspace gate.
name: Audio feature sets, docs, WASM
runs-on: ubuntu-latest
steps:
- name: Harden runner
@@ -34,13 +39,8 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
components: rustfmt, clippy
components: clippy
targets: wasm32-unknown-unknown
- uses: step-security/rust-cache@v2.9.2
with:
key: craft-audio-lint
save-if: ${{ github.ref == 'refs/heads/main' }}
- run: cargo fmt -p craft-audio --check
# Each feature on its own, so code that needs another feature fails here.
- name: Clippy, each feature alone
run: |
@@ -49,7 +49,6 @@ jobs:
cargo clippy -p craft-audio --all-targets --locked --no-default-features ${f:+--features "$f"} -- -D warnings
echo "::endgroup::"
done
- run: cargo clippy -p craft-audio --all-targets --locked --all-features -- -D warnings
# missing_docs is a warning in the crate; -D warnings makes it (and broken links) fail here.
- run: cargo doc -p craft-audio --all-features --no-deps --locked
env:
@@ -63,25 +62,6 @@ jobs:
RUSTDOCFLAGS: "-D warnings"
- run: cargo check -p craft-audio --all-features --locked --target wasm32-unknown-unknown
- run: cargo check -p craft-audio --no-default-features --features lossless-encode --locked --target wasm32-unknown-unknown
msrv:
timeout-minutes: 10
name: Audio on the minimum Rust version
runs-on: ubuntu-latest
steps:
- name: Harden runner
uses: step-security/harden-runner@v2.22.4
with:
egress-policy: audit
- uses: actions/checkout@v7.0.1
with:
persist-credentials: false
# The workspace's rust-version.
- run: rustup toolchain install 1.95.0 --profile minimal
- uses: step-security/rust-cache@v2.9.2
with:
key: craft-audio-msrv
save-if: ${{ github.ref == 'refs/heads/main' }}
- run: cargo +1.95.0 check -p craft-audio -p craft-media --all-features --locked
features:
timeout-minutes: 10
name: Audio tests without symphonia and lossless-encode only
@@ -97,10 +77,6 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
- uses: step-security/rust-cache@v2.9.2
with:
key: craft-audio-features
save-if: ${{ github.ref == 'refs/heads/main' }}
- run: cargo test -p craft-audio --no-default-features --features lossless-encode --locked
- run: cargo test -p craft-audio --no-default-features --features pcm-encode --locked --lib
- run: cargo test -p craft-audio --no-default-features --features flac-encode --locked --lib
@@ -122,10 +98,6 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
- uses: step-security/rust-cache@v2.9.2
with:
key: craft-audio-test
save-if: ${{ github.ref == 'refs/heads/main' }}
- uses: actions/cache@v6.1.0
with:
path: ${{ runner.temp }}/media-fixtures/opus/vectors
+7 -3
View File
@@ -11,6 +11,13 @@ on:
permissions:
contents: read
concurrency:
# One group per pull request and per branch: a newer push cancels the run it supersedes, on main
# too, since the newest run checks everything before it.
group: cms-${{ github.ref }}
cancel-in-progress: true
# clippy and the debug tests run in the Workspace gate; this runs the release build of the tests.
jobs:
cms:
timeout-minutes: 20
@@ -18,9 +25,6 @@ jobs:
steps:
- uses: actions/checkout@v4.4.0
- uses: dtolnay/rust-toolchain@stable
with:
components: clippy
- run: cargo clippy -p craft-cms --all-targets --all-features --locked -- -D warnings
# Release: the FOGRA39 fit and ink-limit tests take about 35 s in debug and 4 s here. Tests that need
# installed system profiles print "skipped:" when none are present.
- run: cargo test -p craft-cms --all-features --release --locked -- --nocapture
+5 -41
View File
@@ -12,41 +12,13 @@ permissions:
contents: read
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it
# always gets to save the cache.
group: shared-crw-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# One group per pull request and per branch: a newer push cancels the run it supersedes, on main
# too, since the newest run checks everything before it.
group: shared-crw-${{ github.ref }}
cancel-in-progress: true
# fmt, clippy and the tests run in the Workspace gate on every OS; this checks what it does not.
jobs:
native:
timeout-minutes: 20
name: CRW lint and tests (${{ matrix.os }})
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, windows-latest]
runs-on: ${{ matrix.os }}
steps:
- name: Harden runner
uses: step-security/harden-runner@v2.22.4
with:
egress-policy: audit
- uses: actions/checkout@v7.0.1
with:
persist-credentials: false
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
components: rustfmt, clippy
- uses: step-security/rust-cache@v2.9.2
with:
key: craft-crw-${{ matrix.os }}
save-if: ${{ github.ref == 'refs/heads/main' }}
- run: cargo fmt -p craft-crw --check
- run: cargo clippy -p craft-crw --all-targets --locked -- -D warnings
- run: cargo test -p craft-crw --locked
wasm:
timeout-minutes: 10
name: CRW WASM
@@ -63,10 +35,6 @@ jobs:
with:
toolchain: stable
targets: wasm32-unknown-unknown
- uses: step-security/rust-cache@v2.9.2
with:
key: craft-crw-wasm
save-if: ${{ github.ref == 'refs/heads/main' }}
- run: cargo check -p craft-crw --locked --target wasm32-unknown-unknown
samples:
@@ -84,10 +52,6 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
- uses: step-security/rust-cache@v2.9.2
with:
key: craft-crw-samples
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Cache the CC0 sample files
uses: actions/cache@v6.1.0
with:
+5 -6
View File
@@ -12,10 +12,10 @@ permissions:
contents: read
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued.
group: dialogs-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# One group per pull request and per branch: a newer push cancels the run it supersedes, on main
# too, since the newest run checks everything before it.
group: dialogs-${{ github.ref }}
cancel-in-progress: true
jobs:
dialogs:
@@ -37,6 +37,5 @@ jobs:
components: clippy
# The macOS code is linted on Linux: it compiles for the target without the SDK.
targets: aarch64-apple-darwin
- run: cargo clippy -p craft-dialogs --all-targets --locked -- -D warnings
# Native clippy and the tests run in the Workspace gate.
- run: cargo clippy -p craft-dialogs --target aarch64-apple-darwin --locked -- -D warnings
- run: cargo test -p craft-dialogs --locked
+7 -16
View File
@@ -12,20 +12,17 @@ permissions:
contents: read
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued.
group: doc-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# One group per pull request and per branch: a newer push cancels the run it supersedes, on main
# too, since the newest run checks everything before it.
group: doc-${{ github.ref }}
cancel-in-progress: true
# fmt, clippy and the tests run in the Workspace gate on every OS; this checks what it does not.
jobs:
doc:
name: Craft documents WASM
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
# macOS runners are kept for releases: macOS runs only when started by hand.
os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }}
runs-on: ${{ matrix.os }}
runs-on: ubuntu-latest
steps:
- name: Harden runner
uses: step-security/harden-runner@v2.22.4
@@ -37,11 +34,5 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
components: rustfmt, clippy
targets: wasm32-unknown-unknown
- run: cargo fmt --check
- run: cargo clippy -p craft-doc --all-targets --locked -- -D warnings
- run: cargo clippy -p craft-doc --all-targets --all-features --locked -- -D warnings
- run: cargo test -p craft-doc --locked -- --nocapture
- run: cargo test -p craft-doc --all-features --locked -- --nocapture
- run: cargo check -p craft-doc --all-features --locked --target wasm32-unknown-unknown
-41
View File
@@ -1,41 +0,0 @@
name: Document paths
on:
workflow_dispatch:
push:
branches: [main]
paths: ['crates/docpath/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/docpath.yml']
pull_request:
paths: ['crates/docpath/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/docpath.yml']
permissions:
contents: read
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued.
group: docpath-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
docpath:
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
# macOS runners are kept for releases: macOS runs only when started by hand.
os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }}
runs-on: ${{ matrix.os }}
steps:
- name: Harden runner
uses: step-security/harden-runner@v2.22.4
with:
egress-policy: audit
- uses: actions/checkout@v7.0.1
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
components: rustfmt, clippy
- run: cargo fmt --check
- run: cargo clippy -p craft-docpath --all-targets --locked -- -D warnings
- run: cargo test -p craft-docpath --locked -- --nocapture
+7 -14
View File
@@ -12,20 +12,17 @@ permissions:
contents: read
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued.
group: ecproj-format-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# One group per pull request and per branch: a newer push cancels the run it supersedes, on main
# too, since the newest run checks everything before it.
group: ecproj-format-${{ github.ref }}
cancel-in-progress: true
# fmt, clippy and the tests run in the Workspace gate on every OS; this checks what it does not.
jobs:
ecproj-format:
name: EffectCraft project format WASM
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
# macOS runners are kept for releases: macOS runs only when started by hand.
os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }}
runs-on: ${{ matrix.os }}
runs-on: ubuntu-latest
steps:
- name: Harden runner
uses: step-security/harden-runner@v2.22.4
@@ -37,9 +34,5 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
components: rustfmt, clippy
targets: wasm32-unknown-unknown
- run: cargo fmt --check
- run: cargo clippy -p craft-ecproj-format --all-targets --locked -- -D warnings
- run: cargo test -p craft-ecproj-format --locked -- --nocapture
- run: cargo check -p craft-ecproj-format --locked --target wasm32-unknown-unknown
+7 -17
View File
@@ -12,20 +12,17 @@ permissions:
contents: read
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued.
group: font-store-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# One group per pull request and per branch: a newer push cancels the run it supersedes, on main
# too, since the newest run checks everything before it.
group: font-store-${{ github.ref }}
cancel-in-progress: true
# fmt, clippy and the tests run in the Workspace gate on every OS; this checks what it does not.
jobs:
font-store:
name: Font store WASM
timeout-minutes: 20
strategy:
fail-fast: false
matrix:
# macOS runners are kept for releases: macOS runs only when started by hand.
os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }}
runs-on: ${{ matrix.os }}
runs-on: ubuntu-latest
steps:
- name: Harden runner
uses: step-security/harden-runner@v2.22.4
@@ -35,13 +32,6 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
components: rustfmt, clippy
targets: wasm32-unknown-unknown
- run: cargo fmt --check
- run: cargo clippy -p craft-font-store --all-targets --locked -- -D warnings
- run: cargo test -p craft-font-store --locked
- run: cargo clippy -p craft-font-catalog --all-targets --locked -- -D warnings
- run: cargo test -p craft-font-catalog --locked
- name: Check the web build
if: runner.os == 'Linux'
run: cargo check -p craft-font-store --target wasm32-unknown-unknown --locked
+10 -22
View File
@@ -42,11 +42,10 @@ permissions:
contents: read
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it
# always gets to save the cache.
group: shared-formats-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# One group per pull request and per branch: a newer push cancels the run it supersedes, on main
# too, since the newest run checks everything before it.
group: shared-formats-${{ github.ref }}
cancel-in-progress: true
jobs:
check:
@@ -68,7 +67,9 @@ jobs:
- { crate: craft-tiff, features: "" }
- { crate: craft-heif, features: "testdata" }
- { crate: craft-cms, features: "" }
name: ${{ matrix.crate }} lint, tests, WASM
# fmt, clippy with default and with all features, and the tests run in the Workspace gate on
# every OS; this lints each feature alone and checks the web build.
name: ${{ matrix.crate }} feature sets, WASM
runs-on: ubuntu-latest
steps:
- name: Harden runner
@@ -80,32 +81,19 @@ jobs:
persist-credentials: false
- uses: dtolnay/rust-toolchain@stable
with:
components: rustfmt, clippy
components: clippy
targets: wasm32-unknown-unknown
- uses: Swatinem/rust-cache@v2.9.3
with:
key: ${{ matrix.crate }}
save-if: ${{ github.ref == 'refs/heads/main' }}
- run: cargo fmt -p "$CRATE" --check
env:
CRATE: ${{ matrix.crate }}
- name: Clippy, default features and each feature alone
- name: Clippy, each feature alone
if: matrix.features != ''
env:
CRATE: ${{ matrix.crate }}
FEATURES: ${{ matrix.features }}
run: |
cargo clippy -p "$CRATE" --all-targets --locked -- -D warnings
for f in $FEATURES; do
echo "::group::feature: $f"
cargo clippy -p "$CRATE" --all-targets --locked --no-default-features --features "$f" -- -D warnings
echo "::endgroup::"
done
- run: cargo clippy -p "$CRATE" --all-targets --locked --all-features -- -D warnings
env:
CRATE: ${{ matrix.crate }}
- run: cargo test -p "$CRATE" --all-features --locked
env:
CRATE: ${{ matrix.crate }}
- run: cargo check -p "$CRATE" --all-features --locked --target wasm32-unknown-unknown
env:
CRATE: ${{ matrix.crate }}
-41
View File
@@ -1,41 +0,0 @@
name: Translations
on:
workflow_dispatch:
push:
branches: [main]
paths: ['crates/i18n/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/i18n.yml']
pull_request:
paths: ['crates/i18n/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/i18n.yml']
permissions:
contents: read
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued.
group: i18n-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
i18n:
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
# macOS runners are kept for releases: macOS runs only when started by hand.
os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }}
runs-on: ${{ matrix.os }}
steps:
- name: Harden runner
uses: step-security/harden-runner@v2.22.4
with:
egress-policy: audit
- uses: actions/checkout@v7.0.1
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
components: rustfmt, clippy
- run: cargo fmt --check
- run: cargo clippy -p craft-i18n --all-targets --features collate --locked -- -D warnings
- run: cargo test -p craft-i18n --features collate --locked -- --nocapture
+8 -15
View File
@@ -12,16 +12,17 @@ permissions:
contents: read
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it
# always gets to save the cache.
group: shared-jpeg-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# One group per pull request and per branch: a newer push cancels the run it supersedes, on main
# too, since the newest run checks everything before it.
group: shared-jpeg-${{ github.ref }}
cancel-in-progress: true
jobs:
check:
timeout-minutes: 10
name: JPEG lint, tests, WASM
# Uncached: the wasm check builds its dependencies on every run (about 10 minutes with setup).
timeout-minutes: 20
# fmt, clippy and the tests run in the Workspace gate on every OS.
name: JPEG WASM
runs-on: ubuntu-latest
steps:
- name: Harden runner
@@ -34,13 +35,5 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
components: rustfmt, clippy
targets: wasm32-unknown-unknown
- uses: step-security/rust-cache@v2.9.2
with:
key: craft-jpeg
save-if: ${{ github.ref == 'refs/heads/main' }}
- run: cargo fmt -p craft-jpeg --check
- run: cargo clippy -p craft-jpeg --all-targets --locked -- -D warnings
- run: cargo test -p craft-jpeg --locked
- run: cargo check -p craft-jpeg --locked --target wasm32-unknown-unknown
+8 -50
View File
@@ -12,16 +12,17 @@ permissions:
contents: read
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it
# always gets to save the cache.
group: keymap-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# One group per pull request and per branch: a newer push cancels the run it supersedes, on main
# too, since the newest run checks everything before it.
group: keymap-${{ github.ref }}
cancel-in-progress: true
jobs:
linux:
timeout-minutes: 15
name: Keymap (Linux)
# fmt, and clippy and the tests with default and with all features, run in the Workspace gate
# on every OS; this runs the feature subsets.
name: Keymap feature sets
runs-on: ubuntu-latest
env:
CRAFT_FONTS_DIR: ${{ github.workspace }}/.test-inputs/craft-fonts
@@ -43,12 +44,7 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
components: rustfmt, clippy
- uses: step-security/rust-cache@v2.9.2
with:
key: craft-keymap-linux
save-if: ${{ github.ref == 'refs/heads/main' }}
- run: cargo fmt -p craft-keymap --check
components: clippy
- run: cargo clippy -p craft-keymap --all-targets --no-default-features --locked -- -D warnings
# `snapshot` covers `ui` and `egui` and compiles the dialog screenshots, which need a GPU
# to run, so the test steps leave them out.
@@ -58,40 +54,6 @@ jobs:
run: cargo test -p craft-keymap --no-default-features --lib --tests --locked
- name: Test egui dispatch and the shortcuts dialog
run: cargo test -p craft-keymap --features ui --locked
native:
timeout-minutes: 15
name: Keymap (${{ matrix.os }})
strategy:
fail-fast: false
matrix:
# macOS runners are kept for releases: macOS runs only when started by hand.
os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["macos-latest","windows-latest"]') || fromJSON('["windows-latest"]') }}
runs-on: ${{ matrix.os }}
env:
CRAFT_FONTS_DIR: ${{ github.workspace }}/.test-inputs/craft-fonts
steps:
- name: Harden runner
uses: step-security/harden-runner@v2.22.4
with:
egress-policy: audit
- uses: actions/checkout@v7.0.1
with:
persist-credentials: false
- name: Checkout test fonts
uses: actions/checkout@v7.0.1
with:
repository: storytold/craft-fonts
ref: b076417b5880bd4154bd22f00695b1d28c6bea83
path: .test-inputs/craft-fonts
persist-credentials: false
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
- uses: step-security/rust-cache@v2.9.2
with:
key: craft-keymap-${{ matrix.os }}
save-if: ${{ github.ref == 'refs/heads/main' }}
- run: cargo test -p craft-keymap --features ui --locked
wasm:
timeout-minutes: 10
name: Keymap WASM
@@ -108,8 +70,4 @@ jobs:
with:
toolchain: stable
targets: wasm32-unknown-unknown
- uses: step-security/rust-cache@v2.9.2
with:
key: craft-keymap-wasm
save-if: ${{ github.ref == 'refs/heads/main' }}
- run: cargo check -p craft-keymap --features ui --locked --target wasm32-unknown-unknown
+12 -19
View File
@@ -11,17 +11,22 @@ on:
permissions:
contents: read
# A fraction of the fuzz-style cases, as in the Workspace gate; nightly.yml runs them in full.
env:
CRAFT_HOSTILE_DIVISOR: "8"
PROPTEST_CASES: "64"
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it
# always gets to save the cache.
group: shared-media-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# One group per pull request and per branch: a newer push cancels the run it supersedes, on main
# too, since the newest run checks everything before it.
group: shared-media-${{ github.ref }}
cancel-in-progress: true
jobs:
lint:
timeout-minutes: 10
name: Media lint, feature sets, WASM
# fmt, and clippy with default and with all features, run in the Workspace gate on every OS.
name: Media feature sets, WASM
runs-on: ubuntu-latest
steps:
- name: Harden runner
@@ -34,20 +39,12 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
components: rustfmt, clippy
components: clippy
targets: wasm32-unknown-unknown
- uses: step-security/rust-cache@v2.9.2
with:
key: craft-media-lint
save-if: ${{ github.ref == 'refs/heads/main' }}
- run: cargo fmt -p craft-media --check
- run: cargo clippy -p craft-media --all-targets --locked --no-default-features -- -D warnings
- run: cargo clippy -p craft-media --all-targets --locked --no-default-features --features bitstream -- -D warnings
- run: cargo clippy -p craft-media --all-targets --locked -- -D warnings
- run: cargo clippy -p craft-media --all-targets --locked --no-default-features --features matroska -- -D warnings
- run: cargo clippy -p craft-media --all-targets --locked --no-default-features --features ogg -- -D warnings
- run: cargo clippy -p craft-media --all-targets --locked --all-features -- -D warnings
- run: cargo clippy -p craft-media-testkit --all-targets --locked -- -D warnings
- run: cargo clippy -p craft-media --all-targets --locked --no-default-features --features serde -- -D warnings
- run: cargo check -p craft-media --all-features --locked --target wasm32-unknown-unknown
test:
@@ -65,10 +62,6 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
- uses: step-security/rust-cache@v2.9.2
with:
key: craft-media-test
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Build tests while installing ffmpeg
run: |
# The man-db index rebuild after dpkg is slow on hosted runners.
+3 -8
View File
@@ -18,7 +18,8 @@ concurrency:
jobs:
linux:
timeout-minutes: 15
name: Menu (Linux)
# fmt, native clippy and the tests run in the Workspace gate on every OS.
name: Menu macOS and WASM lint
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7.0.1
@@ -26,14 +27,8 @@ jobs:
persist-credentials: false
- uses: dtolnay/rust-toolchain@stable
with:
components: rustfmt, clippy
components: clippy
targets: aarch64-apple-darwin, wasm32-unknown-unknown
- uses: Swatinem/rust-cache@v2.9.3
with:
key: craft-menu-linux
- run: cargo fmt -p craft-menu --check
- run: cargo clippy -p craft-menu --all-targets --locked -- -D warnings
# The macOS backend type-checks from Linux; it needs no linker or SDK.
- run: cargo clippy -p craft-menu --lib --locked --target aarch64-apple-darwin -- -D warnings
- run: cargo clippy -p craft-menu --lib --locked --target wasm32-unknown-unknown -- -D warnings
- run: cargo test -p craft-menu --locked
+34
View File
@@ -0,0 +1,34 @@
name: Nightly full tests
# The tests the Workspace gate shortens, in full, once a day: every hostile-input hammer and sweep
# without CRAFT_HOSTILE_DIVISOR, and every proptest at its own case count (no PROPTEST_CASES). The
# rest of the workspace's tests run with them, on Linux, with the gate's setup.
on:
schedule:
- cron: "37 2 * * *"
workflow_dispatch:
permissions:
contents: read
concurrency:
group: nightly-${{ github.ref }}
cancel-in-progress: false
jobs:
full:
name: Full tests
uses: storytold/craft-common-actions/.github/workflows/gate.yml@main
with:
os: '["ubuntu-latest"]'
timeout-minutes: 120
tools: cargo-nextest@0.9.150
packages: mesa-vulkan-drivers libvulkan1
craft-fonts-ref: b076417b5880bd4154bd22f00695b1d28c6bea83
linux-env: WGPU_BACKEND=vulkan
env: CRAFT_PRINT_REQUIRE_QUEUE=0
setup-command: bash scripts/ci-test-inputs.sh
cache-paths: .test-inputs/photocraft/corpus
cache-key: photocraft-corpus
cache-key-files: scripts/ci-test-inputs.sh
command: cargo nextest run --workspace --all-features --locked --profile ci
+8 -14
View File
@@ -12,20 +12,18 @@ permissions:
contents: read
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued.
group: pcraft-format-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# One group per pull request and per branch: a newer push cancels the run it supersedes, on main
# too, since the newest run checks everything before it.
group: pcraft-format-${{ github.ref }}
cancel-in-progress: true
# fmt, clippy and the tests run in the Workspace gate on every OS, and its msrv job checks the
# workspace on 1.95; this checks the web build on 1.95.
jobs:
pcraft-format:
name: PhotoCraft format WASM
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
# macOS runners are kept for releases: macOS runs only when started by hand.
os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }}
runs-on: ${{ matrix.os }}
runs-on: ubuntu-latest
steps:
- name: Harden runner
uses: step-security/harden-runner@v2.22.4
@@ -37,9 +35,5 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: "1.95"
components: rustfmt, clippy
targets: wasm32-unknown-unknown
- run: cargo fmt --check
- run: cargo clippy -p craft-pcraft-format --all-targets --locked -- -D warnings
- run: cargo test -p craft-pcraft-format --locked -- --nocapture
- run: cargo check -p craft-pcraft-format --locked --target wasm32-unknown-unknown
+14 -56
View File
@@ -11,22 +11,23 @@ on:
permissions:
contents: read
# A fraction of the proptest cases, as in the Workspace gate; nightly.yml runs them in full.
env:
PROPTEST_CASES: "64"
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it
# always gets to save the cache.
group: shared-pdf-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# One group per pull request and per branch: a newer push cancels the run it supersedes, on main
# too, since the newest run checks everything before it.
group: shared-pdf-${{ github.ref }}
cancel-in-progress: true
jobs:
native:
name: PDF (${{ matrix.os }})
strategy:
fail-fast: false
matrix:
# macOS runners are kept for releases: macOS runs only when started by hand.
os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }}
runs-on: ${{ matrix.os }}
# fmt, clippy and the tests run in the Workspace gate on every OS, with the licensed fonts and
# the plate tests; this runs the tests behind craft_pdf_unicode_fixtures and the read-only
# contract, which is its own workspace.
name: PDF Unicode fixtures and read-only contract
runs-on: ubuntu-latest
env:
RUSTFLAGS: --cfg craft_pdf_unicode_fixtures
steps:
@@ -38,11 +39,6 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
components: rustfmt, clippy
- uses: step-security/rust-cache@v2.9.2
with:
key: craft-pdf
save-if: ${{ github.ref == 'refs/heads/main' }}
- uses: actions/setup-python@v7.1.0
with:
python-version: "3.12"
@@ -52,9 +48,7 @@ jobs:
python scripts/test-fetch-pdf-test-fonts.py
python scripts/fetch-pdf-test-fonts.py "$RUNNER_TEMP/craft-pdf-test-fonts"
echo "CRAFT_PDF_TEST_FONT_DIR=$RUNNER_TEMP/craft-pdf-test-fonts" >> "$GITHUB_ENV"
- run: cargo fmt --check
- run: cargo clippy -p craft-pdf --all-targets --all-features --locked -- -D warnings
- run: cargo test -p craft-pdf --all-features --locked
- run: cargo test -p craft-pdf --all-features --locked --test unicode_writing --test upstream_compatibility --test upstream_reading
- run: cargo test --manifest-path crates/pdf/read-only-contract/Cargo.toml --locked
backends:
name: PDF vendored contracts
@@ -68,10 +62,6 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
- uses: step-security/rust-cache@v2.9.2
with:
key: craft-pdf-backends
save-if: ${{ github.ref == 'refs/heads/main' }}
- run: bash scripts/test-pdf-backends.sh
wasm:
name: PDF WASM runtime
@@ -89,10 +79,6 @@ jobs:
with:
toolchain: stable
targets: wasm32-unknown-unknown
- uses: step-security/rust-cache@v2.9.2
with:
key: craft-pdf-wasm
save-if: ${{ github.ref == 'refs/heads/main' }}
- uses: actions/setup-python@v7.1.0
with:
python-version: "3.12"
@@ -117,31 +103,3 @@ jobs:
fallback: cargo-install
- run: cargo test -p craft-pdf --all-features --locked --target wasm32-unknown-unknown
- run: cargo test --manifest-path crates/pdf/read-only-contract/Cargo.toml --locked --target wasm32-unknown-unknown
plate-test:
name: Plate tests and craft-prepress
runs-on: ubuntu-latest
steps:
- name: Harden runner
uses: step-security/harden-runner@v2.22.4
with:
egress-policy: audit
- uses: actions/checkout@v7.0.1
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
components: clippy
- uses: step-security/rust-cache@v2.9.2
with:
key: craft-plate-test
save-if: ${{ github.ref == 'refs/heads/main' }}
- uses: actions/setup-python@v7.1.0
with:
python-version: "3.12"
- name: Provision licensed PDF test fonts
shell: bash
run: |
python scripts/test-fetch-pdf-test-fonts.py
python scripts/fetch-pdf-test-fonts.py "$RUNNER_TEMP/craft-pdf-test-fonts"
echo "CRAFT_PDF_TEST_FONT_DIR=$RUNNER_TEMP/craft-pdf-test-fonts" >> "$GITHUB_ENV"
- run: cargo clippy -p craft-plate-test -p craft-prepress --all-targets --all-features --locked -- -D warnings
- run: cargo test -p craft-plate-test -p craft-prepress --all-features --locked
-46
View File
@@ -1,46 +0,0 @@
name: Print spooler
on:
workflow_dispatch:
push:
branches: [main]
paths: ['crates/print/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/print.yml']
pull_request:
paths: ['crates/print/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/print.yml']
permissions:
contents: read
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued.
group: print-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
print:
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
# macOS runners are kept for releases: macOS runs only when started by hand.
os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }}
runs-on: ${{ matrix.os }}
steps:
- name: Harden runner
uses: step-security/harden-runner@v2.22.4
with:
egress-policy: audit
- uses: actions/checkout@v7.0.1
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
components: rustfmt, clippy
- run: cargo fmt --check
- run: cargo clippy -p craft-print --all-targets -- -D warnings
- name: Prepare the Windows test driver
if: runner.os == 'Windows'
shell: powershell
run: |
Start-Service Spooler
Add-PrinterDriver -Name 'Generic / Text Only'
- run: cargo test -p craft-print --locked -- --nocapture
timeout-minutes: 10
env:
CRAFT_PRINT_REQUIRE_QUEUE: ${{ runner.os == 'Windows' && '1' || '0' }}
+9 -17
View File
@@ -12,16 +12,16 @@ permissions:
contents: read
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it
# always gets to save the cache.
group: craft-proof-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# One group per pull request and per branch: a newer push cancels the run it supersedes, on main
# too, since the newest run checks everything before it.
group: craft-proof-${{ github.ref }}
cancel-in-progress: true
jobs:
check:
timeout-minutes: 20
name: Proofing lint, tests with and without Language data, WASM
# fmt, and clippy and the tests without Language data, run in the Workspace gate on every OS.
name: Proofing with Language data, WASM
runs-on: ubuntu-latest
steps:
- name: Harden runner
@@ -43,18 +43,10 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
components: rustfmt, clippy
components: clippy
targets: wasm32-unknown-unknown
- uses: step-security/rust-cache@v2.9.2
with:
key: craft-proof
save-if: ${{ github.ref == 'refs/heads/main' }}
- run: cargo fmt -p craft-proof --check
- name: Without Language data (a fork, docs.rs, offline)
run: |
cargo clippy -p craft-proof --all-targets --locked -- -D warnings
cargo test -p craft-proof --locked
cargo check -p craft-proof --locked --target wasm32-unknown-unknown
- name: The web build without Language data (a fork, docs.rs, offline)
run: cargo check -p craft-proof --locked --target wasm32-unknown-unknown
- name: A release without Language data fails
run: |
if CRAFT_I18N_REQUIRED=1 cargo check -p craft-proof --locked 2>build.log; then
-74
View File
@@ -1,74 +0,0 @@
name: Shared UI
on:
push:
branches: [main]
paths: ["crates/ui/**", "crates/i18n/**", "Cargo.toml", "Cargo.lock", ".github/workflows/ui.yml"]
pull_request:
paths: ["crates/ui/**", "crates/i18n/**", "Cargo.toml", "Cargo.lock", ".github/workflows/ui.yml"]
workflow_dispatch:
permissions:
contents: read
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it
# always gets to save the cache.
group: shared-ui-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
native:
timeout-minutes: 30
name: UI (${{ matrix.os }})
strategy:
fail-fast: false
matrix:
# macOS runners are kept for releases: macOS runs only when started by hand.
os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }}
runs-on: ${{ matrix.os }}
steps:
- name: Harden runner
uses: step-security/harden-runner@v2.22.4
with:
egress-policy: audit
- uses: actions/checkout@v7.0.1
- name: Install software Vulkan renderer
if: runner.os == 'Linux'
run: |
sudo apt-get update
sudo apt-get install -y mesa-vulkan-drivers libvulkan1
echo 'WGPU_BACKEND=vulkan' >> "$GITHUB_ENV"
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
components: rustfmt, clippy
- uses: step-security/rust-cache@v2.9.2
with:
key: craft-ui
save-if: ${{ github.ref == 'refs/heads/main' }}
- run: cargo fmt --check
- run: cargo clippy -p craft-ui --all-targets --locked -- -D warnings
- run: cargo clippy -p craft-ui --all-targets --all-features --locked -- -D warnings
- run: cargo test -p craft-ui --locked
- run: cargo test -p craft-ui --all-features --locked
wasm:
timeout-minutes: 15
name: UI WASM
runs-on: ubuntu-latest
steps:
- name: Harden runner
uses: step-security/harden-runner@v2.22.4
with:
egress-policy: audit
- uses: actions/checkout@v7.0.1
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
targets: wasm32-unknown-unknown
- uses: step-security/rust-cache@v2.9.2
with:
key: craft-ui-wasm
save-if: ${{ github.ref == 'refs/heads/main' }}
- run: cargo check -p craft-ui --all-features --locked --target wasm32-unknown-unknown
+9 -15
View File
@@ -12,20 +12,18 @@ permissions:
contents: read
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued.
group: update-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# One group per pull request and per branch: a newer push cancels the run it supersedes, on main
# too, since the newest run checks everything before it.
group: update-${{ github.ref }}
cancel-in-progress: true
# fmt, clippy and the tests with default and with all features run in the Workspace gate on every
# OS; this runs the egui feature alone.
jobs:
update:
name: Update with egui alone
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
# macOS runners are kept for releases: macOS runs only when started by hand.
os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }}
runs-on: ${{ matrix.os }}
runs-on: ubuntu-latest
steps:
- name: Harden runner
uses: step-security/harden-runner@v2.22.4
@@ -39,10 +37,6 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
components: rustfmt, clippy
- run: cargo fmt --check
# Default (no egui) and the egui feature both build and lint cleanly.
- run: cargo clippy -p craft-update --all-targets --locked -- -D warnings
components: clippy
- run: cargo clippy -p craft-update --all-targets --locked --features egui -- -D warnings
- run: cargo test -p craft-update --locked -- --nocapture
- run: cargo test -p craft-update --locked --features egui -- --nocapture
+8 -14
View File
@@ -12,20 +12,18 @@ permissions:
contents: read
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued.
group: vectorcraft-format-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# One group per pull request and per branch: a newer push cancels the run it supersedes, on main
# too, since the newest run checks everything before it.
group: vectorcraft-format-${{ github.ref }}
cancel-in-progress: true
# fmt, clippy and the tests run in the Workspace gate on every OS, and its msrv job checks the
# workspace on 1.95; this checks the web build on 1.95.
jobs:
vectorcraft-format:
name: VectorCraft format WASM
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
# macOS runners are kept for releases: macOS runs only when started by hand.
os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }}
runs-on: ${{ matrix.os }}
runs-on: ubuntu-latest
steps:
- name: Harden runner
uses: step-security/harden-runner@v2.22.4
@@ -37,9 +35,5 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: "1.95"
components: rustfmt, clippy
targets: wasm32-unknown-unknown
- run: cargo fmt --check
- run: cargo clippy -p craft-vectorcraft-format --all-targets --locked -- -D warnings
- run: cargo test -p craft-vectorcraft-format --locked -- --nocapture
- run: cargo check -p craft-vectorcraft-format --locked --target wasm32-unknown-unknown
+12 -18
View File
@@ -11,6 +11,11 @@ on:
permissions:
contents: read
# A fraction of the fuzz-style cases, as in the Workspace gate; nightly.yml runs them in full.
env:
CRAFT_HOSTILE_DIVISOR: "8"
PROPTEST_CASES: "64"
concurrency:
group: shared-video-${{ github.ref }}
cancel-in-progress: true
@@ -18,7 +23,8 @@ concurrency:
jobs:
lint:
timeout-minutes: 20
name: Video lint, feature sets, docs, WASM
# fmt and clippy with all features run in the Workspace gate on every OS.
name: Video feature sets, docs, WASM
runs-on: ubuntu-latest
steps:
- name: Harden runner
@@ -31,12 +37,8 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: "1.95"
components: rustfmt, clippy
components: clippy
targets: wasm32-unknown-unknown
- uses: step-security/rust-cache@v2.9.2
with:
key: craft-video-lint
- run: cargo fmt -p craft-video --check
# Each feature on its own, so code that needs another feature fails here.
- name: Clippy, each feature alone
run: |
@@ -45,19 +47,15 @@ jobs:
cargo clippy -p craft-video --all-targets --locked --no-default-features ${f:+--features "$f"} -- -D warnings
echo "::endgroup::"
done
- run: cargo clippy -p craft-video --all-targets --locked --all-features -- -D warnings
- run: cargo doc -p craft-video --all-features --no-deps --locked
env:
RUSTDOCFLAGS: "-D warnings"
- run: cargo check -p craft-video --all-features --locked --target wasm32-unknown-unknown
test:
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, windows-latest]
runs-on: ${{ matrix.os }}
name: Video tests (${{ matrix.os }})
# The Workspace gate runs these tests on every OS without ffmpeg; this runs them with it.
runs-on: ubuntu-latest
name: Video tests with ffmpeg
steps:
- name: Harden runner
uses: step-security/harden-runner@v2.22.4
@@ -69,11 +67,7 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: "1.95"
- uses: step-security/rust-cache@v2.9.2
with:
key: craft-video-${{ matrix.os }}
- name: Install ffmpeg (Linux)
if: runner.os == 'Linux'
- name: Install ffmpeg
run: |
sudo rm -f /var/lib/man-db/auto-update
sudo apt-get update -qq && sudo apt-get install -y -qq --no-install-recommends ffmpeg
+6 -8
View File
@@ -12,10 +12,10 @@ permissions:
contents: read
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued.
group: web-api-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# One group per pull request and per branch: a newer push cancels the run it supersedes, on main
# too, since the newest run checks everything before it.
group: web-api-${{ github.ref }}
cancel-in-progress: true
jobs:
web-api:
@@ -35,7 +35,7 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
components: rustfmt, clippy
components: clippy
targets: wasm32-unknown-unknown
- name: Read the wasm-bindgen version from Cargo.lock
id: wasm-bindgen
@@ -50,8 +50,6 @@ jobs:
with:
tool: wasm-bindgen@${{ steps.wasm-bindgen.outputs.version }}
fallback: cargo-install
- run: cargo fmt --check
- run: cargo clippy -p craft-web-api --all-targets --locked -- -D warnings
# fmt, native clippy and the native tests run in the Workspace gate.
- run: cargo clippy -p craft-web-api --all-targets --locked --target wasm32-unknown-unknown -- -D warnings
- run: cargo test -p craft-web-api --locked
- run: cargo test -p craft-web-api --locked --target wasm32-unknown-unknown
+7 -14
View File
@@ -12,16 +12,16 @@ permissions:
contents: read
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it
# always gets to save the cache.
group: shared-web-host-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# One group per pull request and per branch: a newer push cancels the run it supersedes, on main
# too, since the newest run checks everything before it.
group: shared-web-host-${{ github.ref }}
cancel-in-progress: true
jobs:
check:
timeout-minutes: 20
name: Web host lint, tests, WASM tests
# fmt, native clippy and the native tests run in the Workspace gate.
name: Web host WASM lint and tests
runs-on: ubuntu-latest
env:
CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUNNER: wasm-bindgen-test-runner
@@ -36,16 +36,9 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
components: rustfmt, clippy
components: clippy
targets: wasm32-unknown-unknown
- uses: step-security/rust-cache@v2.9.2
with:
key: craft-web-host
save-if: ${{ github.ref == 'refs/heads/main' }}
- run: cargo fmt -p craft-web-host --check
- run: cargo clippy -p craft-web-host --all-targets --locked -- -D warnings
- run: cargo clippy -p craft-web-host --all-targets --locked --target wasm32-unknown-unknown -- -D warnings
- run: cargo test -p craft-web-host --locked
- name: Read the wasm-bindgen version from Cargo.lock
id: wasm-bindgen
shell: bash
+10 -16
View File
@@ -12,20 +12,18 @@ permissions:
contents: read
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued.
group: window-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# One group per pull request and per branch: a newer push cancels the run it supersedes, on main
# too, since the newest run checks everything before it.
group: window-${{ github.ref }}
cancel-in-progress: true
# fmt, clippy and the tests with default and with all features (egui and restore together) run in
# the Workspace gate on every OS; this runs each feature alone.
jobs:
window:
name: Window with each feature alone
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
# macOS runners are kept for releases: macOS runs only when started by hand.
os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }}
runs-on: ${{ matrix.os }}
runs-on: ubuntu-latest
steps:
- name: Harden runner
uses: step-security/harden-runner@v2.22.4
@@ -39,12 +37,8 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
components: rustfmt, clippy
- run: cargo fmt --check
# Default (no egui) and the egui feature both build and lint cleanly.
- run: cargo clippy -p craft-window --all-targets --locked -- -D warnings
components: clippy
- run: cargo clippy -p craft-window --all-targets --locked --features egui -- -D warnings
- run: cargo test -p craft-window --locked -- --nocapture
- run: cargo test -p craft-window --locked --features egui -- --nocapture
- run: cargo clippy -p craft-window --all-targets --locked --features restore -- -D warnings
- run: cargo test -p craft-window --locked --features restore,egui -- --nocapture
- run: cargo test -p craft-window --locked --features restore -- --nocapture
+54 -39
View File
@@ -6,17 +6,21 @@ on:
tags: ["v*"]
pull_request:
workflow_dispatch:
# A full run on main every 6 hours that is never cancelled, so the caches are saved while pushes
# to main cancel each other.
schedule:
- cron: "17 */6 * * *"
permissions:
contents: read
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (push to main, a release tag, a manual run) gets a group of its own: it is neither
# cancelled nor replaced while queued, so a main run always finishes and saves the dependency
# cache, and a release run is never cancelled.
group: workspace-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# A pull request, and main, each have one group, so a newer push cancels the run it supersedes;
# the newest push's run still tests everything before it. Every other run (a release tag, a
# manual run, the scheduled run) gets a group of its own and is neither cancelled nor replaced
# while queued: a release run always finishes, and the scheduled run saves the caches.
group: workspace-${{ (github.event_name == 'pull_request' || github.event_name == 'push' && github.ref == 'refs/heads/main') && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || github.event_name == 'push' && github.ref == 'refs/heads/main' }}
jobs:
native:
@@ -24,47 +28,58 @@ jobs:
uses: storytold/craft-common-actions/.github/workflows/gate.yml@main
with:
os: '["ubuntu-latest","windows-latest","macos-latest"]'
# On a pull request, macOS and Windows run only when it can change platform behaviour: a
# crate with platform cfg code or one that depends on it, the build setup, or a file outside
# the crates (craft-common-actions' select-platforms). The full-ci label runs them anyway
# from the next push or manual run; pushes to main always run every leg.
select-os: '["windows-latest","macos-latest"]'
timeout-minutes: 120
# The Windows leg runs its tests with nextest (.config/nextest.toml).
# Every leg runs its tests with nextest, and its doctests with cargo test (.config/nextest.toml).
tools: cargo-nextest@0.9.150
packages: mesa-vulkan-drivers libvulkan1
craft-fonts-ref: b076417b5880bd4154bd22f00695b1d28c6bea83
linux-env: WGPU_BACKEND=vulkan
windows-env: CRAFT_PRINT_REQUIRE_QUEUE=1
env: CRAFT_PRINT_REQUIRE_QUEUE=0
setup-command: |
set -euo pipefail
corpus_fetcher=.test-inputs/photocraft
mkdir -p "$corpus_fetcher"
git -C "$corpus_fetcher" init
git -C "$corpus_fetcher" remote add origin https://github.com/storytold/PhotoCraft.git
git -C "$corpus_fetcher" fetch --quiet --depth=1 origin 74330157cb601573ae4523a95bb9132822fc3918
git -C "$corpus_fetcher" checkout --quiet --detach FETCH_HEAD
(cd "$corpus_fetcher" && cargo xtask corpus --psd --psd-tools --atn --exr --heif --pngsuite)
mkdir -p corpus
cp -a "$corpus_fetcher/corpus/." corpus/
python scripts/test-fetch-pdf-test-fonts.py
python scripts/fetch-pdf-test-fonts.py "$RUNNER_TEMP/craft-pdf-test-fonts"
printf 'CRAFT_PSD_CORPUS=%s\n' "$GITHUB_WORKSPACE/$corpus_fetcher/corpus" >> "$GITHUB_ENV"
printf 'CRAFT_PDF_TEST_FONT_DIR=%s\n' "$RUNNER_TEMP/craft-pdf-test-fonts" >> "$GITHUB_ENV"
# No debuginfo on Windows: writing PDBs for every test binary is a large part of its link
# time. A panic still names its file and line; only backtrace frames lose theirs, and the
# Linux and macOS legs keep line tables.
windows-env: |
CRAFT_PRINT_REQUIRE_QUEUE=1
CARGO_PROFILE_DEV_DEBUG=0
# The fuzz-style tests run a fraction of their cases here: every hostile-input hammer and
# sweep 1/8 (craft-testkit's CRAFT_HOSTILE_DIVISOR), every proptest without its own case
# count 64 cases instead of 256. nightly.yml runs them all in full.
env: |
CRAFT_PRINT_REQUIRE_QUEUE=0
CRAFT_HOSTILE_DIVISOR=8
PROPTEST_CASES=64
setup-command: bash scripts/ci-test-inputs.sh
# The PhotoCraft corpora, keyed on the script that pins them: a hit skips the PhotoCraft
# checkout and its xtask build (1:51 on Windows).
cache-paths: .test-inputs/photocraft/corpus
cache-key: photocraft-corpus
cache-key-files: scripts/ci-test-inputs.sh
# A `lint (<os>)` leg beside each `gate (<os>)` leg, so clippy is off the tests' critical path.
lint-command: |
if [ "$RUNNER_OS" = Linux ]; then
python3 scripts/test-workspace-concurrency.py
python3 scripts/test-workspace-tests.py
fi
cargo fmt --check
cargo clippy --workspace --all-targets --locked -- -D warnings
cargo clippy --workspace --all-targets --all-features --locked -- -D warnings
command: |
python3 scripts/test-workspace-concurrency.py
cargo fmt --check
cargo clippy --workspace --all-targets --locked -- -D warnings
cargo clippy --workspace --all-targets --all-features --locked -- -D warnings
cargo test --workspace --locked
cargo test --workspace --all-features --locked
windows-command: |
powershell.exe -NoProfile -Command '$ErrorActionPreference = "Stop"; Start-Service Spooler; Add-PrinterDriver -Name "Generic / Text Only"'
cargo fmt --check
cargo clippy --workspace --all-targets --locked -- -D warnings
cargo clippy --workspace --all-targets --all-features --locked -- -D warnings
# The same tests as `cargo test`, run in parallel across test binaries instead of one
# binary at a time; nextest skips doctests, so `cargo test --doc` runs them.
cargo nextest run --workspace --locked --profile ci
cargo test --workspace --doc --locked
# The tests run once, with every feature, in parallel across test binaries; nextest skips
# doctests, so `cargo test --doc` runs them. The default-feature compile is the lint leg's
# first clippy run; the last command runs the crates whose tests check behaviour with a
# feature off, which an all-features build cannot reach.
cargo nextest run --workspace --all-features --locked --profile ci
cargo test --workspace --doc --all-features --locked
cargo nextest run -p craft-image-codecs -p craft-text --locked --profile ci
windows-command: |
powershell.exe -NoProfile -Command '$ErrorActionPreference = "Stop"; Start-Service Spooler; Add-PrinterDriver -Name "Generic / Text Only"'
cargo nextest run --workspace --all-features --locked --profile ci
cargo test --workspace --doc --all-features --locked
cargo nextest run -p craft-image-codecs -p craft-text --locked --profile ci
msrv:
name: Rust 1.95
+23
View File
@@ -119,6 +119,7 @@ The XML reader in `crates/plist/src/xml.rs` and its base64 compatibility helper
`crates/palette` extracts the GPL text parser from AniCraft `crates/doc/src/swatch.rs` at `d875a7feb250e6ab06aa28a37fb78a0699e88330` (MIT OR Apache-2.0), introduced by Tanish Manku in `a64130a9da9bdde545463b852d95d6a6774b1dfd` (AniCraft #270). The import preserves Tanish as primary author. Neutral RGB/name adaptation separates application display-name sanitization and the `Untitled` policy; validation tests and integration are subsequent Dylan M. Taylor changes.
The legacy ZIP Zstandard method-20 mapping and bounded-read regression in `crates/zip/src/lenient` are ported from RRFZ's Craft Anything commit `29badae7c9280fdf4c24795927bf3a91f51694c6` (MIT OR Apache-2.0). RRFZ remains the primary Git author. Firmware scanner detection remains app-owned and is not part of this port.
## TAR/CPIO compatibility updates
The old GNU sparse-extension traversal, GNU CRC-symlink compatibility and PAX-size precedence fixes and their regression tests in `crates/tar` are ported from RRFZ's Craft Anything PRs #1973 (`8ea2ec66a1bc80f0420f3695baac877a0cb9b9a5`), #1984 (`b226a475f7dce9710e3af7a069386fe7a9720df7`) and #2007 (`d29465c20b0d570ea4973be512a0cb4e212bb60f`). Source is MIT OR Apache-2.0; RRFZ remains the primary Git author of each port. The PAX update adapts to the shared parser's existing `PaxOverrides` structure without changing the source policy.
@@ -130,14 +131,26 @@ The USDZ alignment requirement and compatibility fixture for `Writer::add_stored
The raw gradient wire ordering in `craft-lottie::Gradient::flat_values` follows
AniCraft's Lottie exporter by Tanish Manku (`63b39355`, `b9258b7b`), MIT OR Apache-2.0.
Sanitization, animation fitting and numeric output policy remain caller-owned.
## Screen Video key frames
`crates/video/src/screen_video.rs` extracts AniCraft `crates/xfl/src/video.rs` codec-3 decoding and its synthetic partial-block fixture, introduced by Tanish Manku in `1f3e603b7ff53a3da364e41ec4da8d43dc83aab0` with pixel-loop cleanup `2b7571363b4b647266df758f95b6e397021cfd4d` (MIT OR Apache-2.0). Import commits preserve Tanish as primary author. Caller-owned bounds and neutral RGBA output separate codec work from the consuming app's FLV first-frame/report policy. Boundary tests and integration are subsequent Dylan M. Taylor changes.
## Source Map v3 parser and lookup
`crates/sourcemap` extracts the bounded parser, source inventory and generated-position lookup from Craft Anything `crates/formats/src/sourcemap.rs` at `e5c9c94106747ed44b9a3fb0decce867c2285f93` (MIT OR Apache-2.0). Core-history ports preserve Brandon Thomas, BFlat, Afloat, RRFZ, Nicolas Bohn and Alex Joaquim Pereira as primary authors, with exact source revisions in each commit. Six regression files preserve their source authors; nine core unit tests retain the original tests. App annotation scanning, data URLs, base64 and file I/O are intentionally not imported. Workspace wiring is a subsequent change; existing limits and errors are unchanged.
## JSONC span-preserving editing
`crates/jsonc` is extracted from Craft Anything `crates/formats/src/jsonc.rs` at `e5c9c94106747ed44b9a3fb0decce867c2285f93` (MIT OR Apache-2.0). The original implementation and hardening by BFlat, newline preservation by NetterSilver, and subsequent RRFZ/Afloat correctness fixes retain separate primary-author port commits with immutable source revisions. Shared API wiring and optional input-byte admission are subsequent changes. Compatibility parsing and all existing formatting/editing tests are retained; application file I/O and policy are not imported.
## Java class inspection
`crates/java-class` extracts Craft Anything `java_class.rs` and `java_ops.rs` at `5979f9b13f16dcf1df3c7ae7f8e7735f452866f2` (MIT OR Apache-2.0), preserving separate primary-author source-history ports by Brandon Thomas, BFlat, Afloat and RRFZ. The checked byte-reader, modified UTF-8 unit decoder, descriptor display helper and synthetic class fixtures come from the same source snapshot. Seven regression files retain their source authors. Neutral error wiring and shared crate integration are subsequent changes. No JAR archive, application I/O or bytecode execution is imported.
## WebAssembly inspection
`crates/wasm` extracts Craft Anything `crates/formats/src/wasm.rs` at `5979f9b13f16dcf1df3c7ae7f8e7735f452866f2`, with its checked byte-reader subset and original synthetic WASM/testkit fixtures (MIT OR Apache-2.0). Eight core-history ports preserve Brandon Thomas, Afloat and RRFZ as primary authors; component, repeated-custom-section and referenced-string regression imports retain source authors and commit references. Fixtures come from `crates/testkit/src/formats.rs` and deterministic mutation helpers from `crates/testkit/src/lib.rs`; no third-party binaries. Shared error naming and workspace wiring are subsequent changes. Existing inspection limits and report behavior are unchanged; this crate validates/inspects/renders, never executes modules.
## Packet capture and protocol inspection
`crates/pcap` extracts Craft Anything `crates/formats/src/pcap.rs`, its checked byte reader primitives and synthetic packet fixtures/regressions at `5979f9b13f16dcf1df3c7ae7f8e7735f452866f2` (MIT OR Apache-2.0). Brandon Thomas introduced the implementation, byte reader and fixtures in `f64ae3bf3c8863a982af1304e06de20653803968`. Subsequent fixes by phatloi0010, GGWA, BFlat, Afloat, NetterSilver, Max Mittelstädt and RRFZ retain separate primary-author source-history commits with immutable source revisions. Shared error/API wiring and chrono-based timestamp formatting with exhaustive day-boundary parity tests are subsequent Dylan M. Taylor changes. No packet reassembly, decryption, live capture or app analysis policy is imported.
@@ -148,3 +161,13 @@ Sanitization, animation fitting and numeric output policy remain caller-owned.
## Dalvik DEX inspection
`crates/dex` extracts Craft Anything `crates/formats/src/dex.rs` and `dex_ops.rs` at `5979f9b13f16dcf1df3c7ae7f8e7735f452866f2` (MIT OR Apache-2.0). Five source-history ports preserve Brandon Thomas, BFlat and Afloat as primary authors. Five regression imports retain BFlat/Afloat authors and exact source revisions. The checked byte-reader/MUTF-8 subset and synthetic DEX fixture come from that app's `bytes.rs` and `testkit/src/android.rs`; deterministic noise/mutations come from its testkit. No application file I/O, APK logic or execution is imported. Shared neutral errors and workspace wiring are subsequent changes; existing per-file strings, member/handler and bytecode payload budgets are retained.
## EVM bytecode inspection
`crates/evm` extracts Craft Anything `crates/formats/src/evm.rs` at `a31e995858f4312e0f1d59a007112da191ef2d26` (MIT OR Apache-2.0). Eight separate source-history ports preserve Brandon Thomas, Afloat, RRFZ and NetterSilver as primary authors with immutable source references. Synthetic EVM fixtures and deterministic mutation helpers come from the same app source; regression files retain their source provenance. Neutral errors and workspace integration are subsequent changes. No code execution or transaction behavior is imported.
## XFDF field reader
`crates/xfdf` extracts PDFCraft field reading from #5937 refresh `cb564fd827f572818e9ddb0ea5989a432b1f39fd` (MIT OR Apache-2.0). App-side annotation and form policy remains local; source history is retained in the original repository.
Retained XFDF source contributors (move-aware blame): Brandon Thomas, Marco Arnulfo, pkibiego and storybored379; module split by aiongg. Subsequent budget/integration work by Dylan M. Taylor.
Generated
+74 -1
View File
@@ -1920,6 +1920,16 @@ dependencies = [
"zip 9.1.3",
]
[[package]]
name = "craft-evm"
version = "0.1.0"
dependencies = [
"craft-testkit",
"serde",
"serde_json",
"thiserror 2.0.21",
]
[[package]]
name = "craft-ews"
version = "0.1.0"
@@ -2300,6 +2310,15 @@ dependencies = [
"widestring",
]
[[package]]
name = "craft-java-class"
version = "0.1.0"
dependencies = [
"craft-testkit",
"serde",
"thiserror 2.0.21",
]
[[package]]
name = "craft-jmap"
version = "0.1.0"
@@ -3111,6 +3130,15 @@ dependencies = [
name = "craft-sniff"
version = "0.1.0"
[[package]]
name = "craft-sourcemap"
version = "0.1.0"
dependencies = [
"serde",
"serde_json",
"thiserror 2.0.21",
]
[[package]]
name = "craft-ssip"
version = "0.1.0"
@@ -3157,11 +3185,14 @@ dependencies = [
"craft-xml",
"data-url",
"flate2",
"gif",
"image-webp",
"kurbo 0.13.1",
"quick-xml 0.41.0",
"resvg 0.48.1",
"roxmltree 0.21.2",
"thiserror 2.0.21",
"toml",
"usvg 0.48.1",
"vectorcraft-geom",
"vectorcraft-pathops",
@@ -3179,6 +3210,7 @@ dependencies = [
name = "craft-tar"
version = "0.1.0"
dependencies = [
"craft-testkit",
"serde",
]
@@ -3365,6 +3397,17 @@ dependencies = [
"thiserror 2.0.21",
]
[[package]]
name = "craft-wasm"
version = "0.1.0"
dependencies = [
"craft-testkit",
"serde",
"thiserror 2.0.21",
"wasmparser 0.261.0",
"wasmprinter",
]
[[package]]
name = "craft-web-api"
version = "0.1.0"
@@ -3439,6 +3482,14 @@ dependencies = [
"thiserror 2.0.21",
]
[[package]]
name = "craft-xfdf"
version = "0.1.0"
dependencies = [
"craft-xml",
"thiserror 2.0.21",
]
[[package]]
name = "craft-xfl"
version = "0.1.0"
@@ -12048,7 +12099,7 @@ dependencies = [
"wasmi_collections",
"wasmi_core",
"wasmi_ir",
"wasmparser",
"wasmparser 0.228.0",
]
[[package]]
@@ -12088,6 +12139,28 @@ dependencies = [
"indexmap",
]
[[package]]
name = "wasmparser"
version = "0.261.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4f20f20e44f7e8aeb6744823ea9d869ede51e51be4fdaedede2852282e54d2d8"
dependencies = [
"bitflags 2.13.2",
"indexmap",
"semver",
]
[[package]]
name = "wasmprinter"
version = "0.261.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86a21c9e3325655ba02de3e645bc8357a8b3651a2e3ada25b6dbc37560deb9b7"
dependencies = [
"anyhow",
"termcolor",
"wasmparser 0.261.0",
]
[[package]]
name = "wayland-backend"
version = "0.3.17"
+5
View File
@@ -11,8 +11,12 @@ rust-version = "1.95"
repository = "https://github.com/storytold/craft-libs"
[workspace.dependencies]
craft-evm = { path = "crates/evm" }
craft-java-class = { path = "crates/java-class" }
craft-sourcemap = { path = "crates/sourcemap" }
craft-pcap = { path = "crates/pcap" }
craft-har = { path = "crates/har" }
craft-xfdf = { path = "crates/xfdf" }
craft-jsonc = { path = "crates/jsonc" }
craft-palette = { path = "crates/palette" }
craft-score = { path = "crates/score" }
@@ -28,6 +32,7 @@ acadrust = "=0.6.6"
log = "0.4"
# internal: add each crate here as `craft-<name> = { path = "crates/<name>" }`
craft-plist = { path = "crates/plist" }
craft-wasm = { path = "crates/wasm" }
craft-platform = { path = "crates/platform" }
craft-aep = { path = "crates/aep" }
craft-affinity = { path = "crates/affinity" }
+6
View File
@@ -81,6 +81,12 @@ Buffers are checked once per call, before any pixel is written.
the exact and float paths compare each pixel with white; all other colours are left as the
profiles give them. With a profile whose white falls between its nodes (`coated-cmyk`: 0.4 %
C), the last step to white is that much larger than its neighbours.
* Black preservation (opt-in, `BlackPreserving`): for CMYK → CMYK, a colour with exactly zero
C, M and Y comes out as exactly `(0, 0, 0, K')`, where `K'` is the destination black whose
K-only L* equals the source's (both K ramps sampled at 1024 points when the transform is
built, with its intent and BPC). The test is exact zero, so an input with any chromatic ink
takes the ordinary path. Absolute colorimetric, non-CMYK profiles and K ramps that reverse at
one of the 1024 samples are refused (a reversal between samples is not detected).
## Built-in profiles (all CC0-1.0, generated by this crate)
Binary file not shown.
+205
View File
@@ -0,0 +1,205 @@
//! K-only black preservation for CMYK → CMYK transforms.
//!
//! A CMYK → CMYK transform goes through the three-component PCS, so black alone (C = M = Y = 0)
//! comes out as a four-ink mix. ISO 32000-2 §8.6.5.7 notes that this loses fidelity in the black
//! component. ICC defines no black preservation; this is a named policy the caller opts into.
//!
//! [`BlackPreserving`] converts like its [`Transform`], except that a colour with exactly zero
//! cyan, magenta and yellow becomes exactly `(0, 0, 0, K')`. `K'` is the destination black whose
//! K-only L* equals the source's, both measured to Lab with the transform's own intent and BPC
//! setting: both ramps are sampled once, at construction, and the destination ramp is inverted.
//! A source black darker than the destination's maximum black maps to K = 1. The test is exact zero, so the transform is discontinuous at C = M = Y = 0: an input
//! with any chromatic ink takes the ordinary path.
use crate::transform::{Transform, TransformOptions};
use crate::{Builtin, CmsError, ColorSpace, Intent, Profile, invalid_arg};
/// Samples along each K-only ramp. Linear interpolation between samples of a smooth ramp is far
/// below the 16-bit quantum.
const K_SAMPLES: usize = 1024;
/// Slack for the monotonicity checks (L* units, K units).
const MONOTONIC_EPS: f64 = 1e-6;
/// A CMYK → CMYK transform that keeps black-only colours black-only.
#[derive(Debug)]
pub struct BlackPreserving {
transform: Transform,
/// Destination K for source K at `i / (K_SAMPLES - 1)`; `K_SAMPLES` values in `[0, 1]`.
curve: Vec<f32>,
/// Source K from which the source black is at least as dark as the destination's darkest:
/// mapped to exactly 1. Infinite when the destination's black is the darker.
saturate: f32,
/// The mapping at the 256 8-bit source codes, already quantised.
curve8: [u8; 256],
}
impl BlackPreserving {
/// Transform from `src` to `dst` with black preservation.
///
/// Errors with [`CmsError::InvalidArgument`] when either profile is not CMYK or the intent is
/// absolute colorimetric (which reproduces the source's paper and black, so there is no
/// black to map), and with [`CmsError::Unsupported`] when a profile's K-only ramp is not
/// monotonic at its 1024 samples (black that gets lighter as K rises has no equal-lightness
/// inverse; a reversal between samples is not detected). Other errors
/// are those of [`Transform::with_options`].
pub fn new(src: &Profile, dst: &Profile, opts: TransformOptions) -> Result<BlackPreserving, CmsError> {
if src.color_space != ColorSpace::Cmyk || dst.color_space != ColorSpace::Cmyk {
return Err(invalid_arg("black preservation needs CMYK source and destination profiles"));
}
if opts.intent == Intent::AbsoluteColorimetric {
return Err(invalid_arg("black preservation does not apply to absolute colorimetric"));
}
let transform = Transform::with_options(src, dst, opts)?;
let src_l = k_ramp_lightness(src, opts)?;
let dst_l = k_ramp_lightness(dst, opts)?;
let (curve, saturate) = k_curve(&src_l, &dst_l)?;
let mut curve8 = [0u8; 256];
for (code, slot) in curve8.iter_mut().enumerate() {
*slot = quantise(map(&curve, saturate, code as f32 / 255.0), 255.0) as u8;
}
Ok(BlackPreserving { transform, curve, saturate, curve8 })
}
/// The ordinary transform, which every colour with chromatic ink goes through. Converting with
/// it directly does not preserve black.
pub fn transform(&self) -> &Transform {
&self.transform
}
/// Destination K for a black-only source colour of `k` (clamped to `[0, 1]`).
pub fn map_k(&self, k: f32) -> f32 {
map(&self.curve, self.saturate, k)
}
/// Exact evaluation of one colour; slices as in [`Transform::eval`].
pub fn eval(&self, input: &[f32], out: &mut [f32]) -> Result<(), CmsError> {
self.transform.eval(input, out)?;
self.patch_one(input, out);
Ok(())
}
/// Lookup-table evaluation of one colour; slices as in [`Transform::eval`].
pub fn eval_fast(&self, input: &[f32], out: &mut [f32]) -> Result<(), CmsError> {
self.transform.eval_fast(input, out)?;
self.patch_one(input, out);
Ok(())
}
/// Converts float pixels; layout and errors as in [`Transform::convert_f32`].
pub fn convert_f32(&self, src: &[f32], src_stride: usize, dst: &mut [f32], dst_stride: usize, copy_extra: bool) -> Result<(), CmsError> {
self.transform.convert_f32(src, src_stride, dst, dst_stride, copy_extra)?;
for (s, d) in src.chunks_exact(src_stride).zip(dst.chunks_exact_mut(dst_stride)) {
self.patch_one(s, d);
}
Ok(())
}
/// Converts 8-bit pixels; layout and errors as in [`Transform::convert_u8`].
pub fn convert_u8(&self, src: &[u8], src_stride: usize, dst: &mut [u8], dst_stride: usize, copy_extra: bool) -> Result<(), CmsError> {
self.transform.convert_u8(src, src_stride, dst, dst_stride, copy_extra)?;
for (s, d) in src.chunks_exact(src_stride).zip(dst.chunks_exact_mut(dst_stride)) {
if let ([0, 0, 0, k, ..], [c, m, y, kd, ..]) = (s, d) {
(*c, *m, *y) = (0, 0, 0);
*kd = self.curve8[usize::from(*k)];
}
}
Ok(())
}
/// Converts 16-bit pixels; layout and errors as in [`Transform::convert_u16`].
pub fn convert_u16(&self, src: &[u16], src_stride: usize, dst: &mut [u16], dst_stride: usize, copy_extra: bool) -> Result<(), CmsError> {
self.transform.convert_u16(src, src_stride, dst, dst_stride, copy_extra)?;
for (s, d) in src.chunks_exact(src_stride).zip(dst.chunks_exact_mut(dst_stride)) {
if let ([0, 0, 0, k, ..], [c, m, y, kd, ..]) = (s, d) {
(*c, *m, *y) = (0, 0, 0);
*kd = quantise(self.map_k(f32::from(*k) / 65535.0), 65535.0) as u16;
}
}
Ok(())
}
fn patch_one(&self, input: &[f32], out: &mut [f32]) {
if let ([0.0, 0.0, 0.0, k, ..], [c, m, y, kd, ..]) = (input, out) {
(*c, *m, *y) = (0.0, 0.0, 0.0);
*kd = self.map_k(*k);
}
}
}
/// The sampled mapping from the source to the destination K-only ramp, and the source K at
/// which the destination's darkest black is reached (the `saturate` field).
fn k_curve(src_l: &[f64], dst_l: &[f64]) -> Result<(Vec<f32>, f32), CmsError> {
let curve: Vec<f32> = src_l.iter().map(|&l| invert(dst_l, l) as f32).collect();
if !curve.windows(2).all(|w| f64::from(w[0]) - f64::from(w[1]) <= MONOTONIC_EPS) {
return Err(CmsError::Unsupported("black preservation: the K mapping is not monotonic".into()));
}
let saturate = match (src_l.last(), dst_l.last()) {
(Some(&s), Some(&d)) if s < d => invert(src_l, d) as f32,
_ => f32::INFINITY,
};
Ok((curve, saturate))
}
/// L* of `p`'s K-only ramp at `K_SAMPLES` points, light to dark; refused when it is not
/// non-increasing.
fn k_ramp_lightness(p: &Profile, opts: TransformOptions) -> Result<Vec<f64>, CmsError> {
let to_lab = Transform::with_options(p, Builtin::LabD50.profile(), TransformOptions { precise_float: true, ..opts })?;
let mut ramp = Vec::with_capacity(K_SAMPLES);
let mut lab = [0.0f32; 3];
for i in 0..K_SAMPLES {
let k = i as f32 / (K_SAMPLES - 1) as f32;
to_lab.eval(&[0.0, 0.0, 0.0, k], &mut lab)?;
ramp.push(f64::from(lab[0]) * 100.0);
}
if !ramp.iter().all(|l| l.is_finite()) || !ramp.windows(2).all(|w| w[1] - w[0] <= MONOTONIC_EPS) {
return Err(CmsError::Unsupported("black preservation: a K-only ramp is not monotonic".into()));
}
Ok(ramp)
}
/// The K in `[0, 1]` at which the non-increasing `ramp` reaches `target`, clamped to its ends.
fn invert(ramp: &[f64], target: f64) -> f64 {
let (Some(&first), Some(&last)) = (ramp.first(), ramp.last()) else { return 0.0 };
if ramp.len() < 2 || target >= first {
return 0.0;
}
if target <= last {
return 1.0;
}
// ramp[lo] > target >= ramp[hi]
let (mut lo, mut hi) = (0usize, ramp.len() - 1);
while hi - lo > 1 {
let mid = lo + (hi - lo) / 2;
if ramp.get(mid).is_some_and(|&v| v > target) {
lo = mid;
} else {
hi = mid;
}
}
let (a, b) = (ramp.get(lo).copied().unwrap_or(first), ramp.get(hi).copied().unwrap_or(last));
let t = if a - b > MONOTONIC_EPS { (a - target) / (a - b) } else { 0.0 };
(lo as f64 + t) / (ramp.len() - 1) as f64
}
/// Linear interpolation in `curve` (samples over `[0, 1]`) at `k`, clamped; 1 from `saturate`.
fn map(curve: &[f32], saturate: f32, k: f32) -> f32 {
let Some(&last) = curve.last() else { return 0.0 };
let k = if k.is_nan() { 0.0 } else { k.clamp(0.0, 1.0) };
if k >= saturate {
return 1.0;
}
let x = k * (curve.len() - 1) as f32;
let i = x.floor() as usize;
match (curve.get(i), curve.get(i + 1)) {
(Some(&a), Some(&b)) => a + (b - a) * (x - i as f32),
_ => last,
}
}
fn quantise(v: f32, max: f32) -> f32 {
(v.clamp(0.0, 1.0) * max + 0.5).floor()
}
#[cfg(test)]
#[path = "black_tests.rs"]
mod tests;
+149
View File
@@ -0,0 +1,149 @@
use super::*;
fn coated() -> &'static Profile {
Builtin::CoatedCmyk.profile()
}
fn fogra() -> &'static Profile {
Builtin::CoatedFogra39.profile()
}
fn opts() -> TransformOptions {
TransformOptions::default()
}
/// L* of a CMYK colour under `p`, measured as the transform with `o` measures it.
fn lightness(p: &Profile, o: TransformOptions, cmyk: [f32; 4]) -> f32 {
let t = Transform::with_options(p, Builtin::LabD50.profile(), o).unwrap();
let mut lab = [0.0f32; 3];
t.eval(&cmyk, &mut lab).unwrap();
lab[0] * 100.0
}
#[test]
fn the_plain_transform_turns_black_into_four_inks() {
// What the policy is for: without it, black alone gains chromatic ink across presses.
let t = Transform::with_options(coated(), fogra(), opts()).unwrap();
let mut out = [0.0f32; 4];
t.eval(&[0.0, 0.0, 0.0, 0.5], &mut out).unwrap();
assert!(out[..3].iter().any(|v| *v > 0.01), "{out:?}");
}
#[test]
fn black_only_stays_black_only_on_every_path() {
for (src, dst) in [(coated(), fogra()), (fogra(), coated()), (fogra(), fogra())] {
let bp = BlackPreserving::new(src, dst, opts()).unwrap();
for k in [0.0f32, 0.1, 0.5, 0.9, 1.0] {
for fast in [false, true] {
let mut out = [1.0f32; 4];
if fast { bp.eval_fast(&[0.0, 0.0, 0.0, k], &mut out) } else { bp.eval(&[0.0, 0.0, 0.0, k], &mut out) }.unwrap();
assert_eq!(out[..3], [0.0; 3], "k {k} fast {fast}");
assert_eq!(out[3], bp.map_k(k));
}
}
let src8: Vec<u8> = (0..=255u8).flat_map(|k| [0, 0, 0, k, 7]).collect();
let mut dst8 = vec![0u8; src8.len()];
bp.convert_u8(&src8, 5, &mut dst8, 5, true).unwrap();
for (k, px) in dst8.as_chunks::<5>().0.iter().enumerate() {
assert_eq!(px[..3], [0; 3], "u8 k {k}");
assert!((f32::from(px[3]) - bp.map_k(k as f32 / 255.0) * 255.0).abs() <= 0.5, "u8 k {k}: {}", px[3]);
assert_eq!(px[4], 7, "extra channel copied");
}
let src16: Vec<u16> = (0..=64u16).flat_map(|k| [0, 0, 0, k * 1023]).collect();
let mut dst16 = vec![0u16; src16.len()];
bp.convert_u16(&src16, 4, &mut dst16, 4, false).unwrap();
for (px, k) in dst16.as_chunks::<4>().0.iter().zip(src16.as_chunks::<4>().0) {
assert_eq!(px[..3], [0; 3], "u16 {k:?}");
assert!((f32::from(px[3]) - bp.map_k(f32::from(k[3]) / 65535.0) * 65535.0).abs() <= 0.5, "u16 {k:?}: {}", px[3]);
}
let srcf: Vec<f32> = (0..=10).flat_map(|k| [0.0, 0.0, 0.0, k as f32 / 10.0]).collect();
let mut dstf = vec![0.0f32; srcf.len()];
bp.convert_f32(&srcf, 4, &mut dstf, 4, false).unwrap();
for (px, k) in dstf.as_chunks::<4>().0.iter().zip(srcf.as_chunks::<4>().0) {
assert_eq!(*px, [0.0, 0.0, 0.0, bp.map_k(k[3])], "f32 {k:?}");
}
}
}
#[test]
fn the_destination_black_matches_the_source_lightness() {
// Measured worst error over 201 K steps, both directions, both intents, BPC on and off:
// 0.007 L*.
for intent in [Intent::RelativeColorimetric, Intent::Perceptual] {
for bpc in [true, false] {
let o = TransformOptions { intent, bpc, ..opts() };
for (src, dst) in [(coated(), fogra()), (fogra(), coated())] {
let bp = BlackPreserving::new(src, dst, o).unwrap();
let dst_black = lightness(dst, o, [0.0, 0.0, 0.0, 1.0]);
for k in [0.1f32, 0.3, 0.5, 0.7, 0.9, 1.0] {
let want = lightness(src, o, [0.0, 0.0, 0.0, k]);
if want <= dst_black {
assert_eq!(bp.map_k(k), 1.0, "darker than the destination's black clamps to K = 1");
continue;
}
let got = lightness(dst, o, [0.0, 0.0, 0.0, bp.map_k(k)]);
assert!((got - want).abs() < 0.02, "{intent:?} bpc {bpc} k {k}: L* {got} for {want}");
}
assert_eq!(bp.map_k(0.0), 0.0, "paper stays paper");
assert!(bp.map_k(1.0) > 0.95, "full black stays near full black: {}", bp.map_k(1.0));
assert!(bp.map_k(0.25) < bp.map_k(0.75));
}
}
}
let same = BlackPreserving::new(fogra(), fogra(), opts()).unwrap();
for k in [0.0f32, 0.2, 0.6, 1.0] {
assert!((same.map_k(k) - k).abs() < 1e-3, "{k} -> {}", same.map_k(k));
}
}
#[test]
fn colours_with_any_chromatic_ink_take_the_ordinary_path() {
let bp = BlackPreserving::new(coated(), fogra(), opts()).unwrap();
for input in [[1e-6f32, 0.0, 0.0, 0.5], [0.0, 0.2, 0.0, 0.5], [0.3, 0.3, 0.3, 0.0], [0.0, 0.0, 1.0, 1.0]] {
let (mut a, mut b) = ([0.0f32; 4], [0.0f32; 4]);
bp.eval(&input, &mut a).unwrap();
bp.transform().eval(&input, &mut b).unwrap();
assert_eq!(a, b, "{input:?}");
}
let src = [1u8, 0, 0, 128, 0, 1, 0, 128];
let (mut a, mut b) = ([0u8; 8], [0u8; 8]);
bp.convert_u8(&src, 4, &mut a, 4, false).unwrap();
bp.transform().convert_u8(&src, 4, &mut b, 4, false).unwrap();
assert_eq!(a, b);
}
#[test]
fn refusals_and_hostile_values() {
let srgb = Builtin::Srgb.profile();
assert!(matches!(BlackPreserving::new(srgb, fogra(), opts()), Err(CmsError::InvalidArgument(_))));
assert!(matches!(BlackPreserving::new(fogra(), srgb, opts()), Err(CmsError::InvalidArgument(_))));
let abs = TransformOptions { intent: Intent::AbsoluteColorimetric, ..opts() };
assert!(matches!(BlackPreserving::new(fogra(), coated(), abs), Err(CmsError::InvalidArgument(_))));
let bp = BlackPreserving::new(coated(), fogra(), opts()).unwrap();
for k in [f32::NAN, f32::INFINITY, f32::NEG_INFINITY, -1.0, 2.0] {
assert!((0.0..=1.0).contains(&bp.map_k(k)), "{k}");
}
let mut out = [0.0f32; 4];
assert!(bp.eval(&[0.0; 3], &mut out).is_err(), "short input");
assert!(bp.convert_u8(&[0, 0, 0, 9, 0], 4, &mut [0u8; 4], 4, false).is_err(), "partial pixel");
assert_eq!(invert(&[], 50.0), 0.0);
assert_eq!(invert(&[100.0], 50.0), 0.0);
assert_eq!(invert(&[100.0, 0.0], 150.0), 0.0);
assert_eq!(invert(&[100.0, 0.0], -5.0), 1.0);
assert!((invert(&[100.0, 50.0, 0.0], 25.0) - 0.75).abs() < 1e-12);
assert_eq!(map(&[], f32::INFINITY, 0.5), 0.0);
}
#[test]
fn black_beyond_the_destination_maximum_maps_to_full_black_between_samples() {
// Source L* = 100 (1 - K), destination L* = 100 - 50 K: from K = 0.5 the source is darker
// than any destination black, and 0.5 falls between the 1024 samples.
let ramp = |f: fn(f64) -> f64| (0..K_SAMPLES).map(|i| f(i as f64 / (K_SAMPLES - 1) as f64)).collect::<Vec<_>>();
let (src, dst) = (ramp(|k| 100.0 * (1.0 - k)), ramp(|k| 100.0 - 50.0 * k));
let (curve, saturate) = k_curve(&src, &dst).unwrap();
assert!((saturate - 0.5).abs() < 1e-6, "{saturate}");
for k in [0.5001f32, 0.6, 1.0] {
assert_eq!(map(&curve, saturate, k), 1.0, "{k}");
}
assert!((map(&curve, saturate, 0.25) - 0.5).abs() < 1e-4);
let (_, never) = k_curve(&dst, &src).unwrap();
assert_eq!(never, f32::INFINITY, "a lighter source never saturates");
}
+2
View File
@@ -19,6 +19,7 @@
//! CIE 15, and Adobe's published black point compensation paper.
#![forbid(unsafe_code)]
pub mod black;
pub mod builtin;
mod builtin_v2;
pub mod clut;
@@ -39,6 +40,7 @@ pub mod transform;
mod write;
pub mod write_v2;
pub use black::BlackPreserving;
pub use builtin::Builtin;
pub use clut::Clut;
pub use curve::Curve;
+322 -15
View File
@@ -19,6 +19,9 @@
//! L* is scaled onto the device black, chroma is softly compressed against a gamut boundary
//! descriptor sampled from the forward model — then solved as above.
//!
//! That is [`GamutMapping::Legacy`]. [`GamutMapping::Oklab`] maps out-of-gamut colours in Oklab
//! instead and gives saturation its own table; the Coated FOGRA39 built-in uses it.
//!
//! The model is plausible for coated offset, not a measured characterisation: use a real
//! profile from your print provider for production print.
//!
@@ -49,6 +52,11 @@ pub struct CmykParams {
pub k_max: f64,
/// Exponent of the black generation curve above `gcr_start`.
pub gcr_gamma: f64,
/// Black generation as a curve of `[gray component, black]` points with increasing gray
/// components in `[0, 1]`, interpolated linearly and held flat beyond the ends. Empty by
/// default: black is then `k_max` times the ramp of `gcr_start` and `gcr_gamma` (the curve
/// of the shipped Photocraft profile).
pub gcr_curve: Vec<[f64; 2]>,
/// Tone value increase at 50 % for C, M, Y, K.
pub tvi: [f64; 4],
/// Grid points of the AToB (4D) and BToA (3D) tables.
@@ -60,6 +68,32 @@ pub struct CmykParams {
/// encoding and separates to a trace of ink. Off by default, which keeps the bytes of the
/// shipped Photocraft profile.
pub white_on_node: bool,
/// How the BToA tables bring colours into the gamut. [`GamutMapping::Legacy`] by default,
/// which keeps the bytes of the shipped Photocraft profile.
pub mapping: GamutMapping,
}
/// How the BToA tables map colours the printer cannot reach.
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum GamutMapping {
/// The shipped Photocraft profile: out-of-gamut colours land on the closest reachable
/// colour (ΔE76); perceptual scales L* onto the device black and compresses Lab chroma;
/// saturation shares the perceptual table.
Legacy,
/// Gamut mapping in Oklab, whose hue lines stay perceptually constant (a CIELAB hue line
/// from a saturated blue toward grey drifts toward purple; an Oklab one does not):
///
/// * **Colorimetric**: in-gamut colours separate exactly. An out-of-gamut colour takes the
/// closest colour the printer reaches in Oklab, lightness and hue weighted above chroma,
/// and more so the further outside it is; black is generated from the clipped colour, so
/// inks the clip drives to 100 % stay there.
/// * **Perceptual** and **saturation**: the ICC v4 perceptual reference black is mapped
/// onto the device black linearly in XYZ (as black point compensation does) for neutrals,
/// fading out with Oklab chroma, so greys separate as with colorimetric + BPC and colourful
/// colours keep their lightness. Perceptual then compresses Oklab chroma above `knee` ×
/// the gamut boundary into the boundary; saturation scales Oklab chroma by `saturation`
/// and clips as colorimetric does.
Oklab { knee: f64, saturation: f64 },
}
impl Default for CmykParams {
@@ -71,10 +105,12 @@ impl Default for CmykParams {
gcr_start: 0.2,
k_max: 0.95,
gcr_gamma: 1.1,
gcr_curve: Vec::new(),
tvi: [0.14, 0.14, 0.14, 0.17],
grid_a2b: 11,
grid_b2a: 21,
white_on_node: false,
mapping: GamutMapping::Legacy,
}
}
}
@@ -183,6 +219,11 @@ impl Forward for TableForward {
/// Bounded least-squares solve for C, M, Y given K, minimising ΔE76 to `target`, subject to
/// `C + M + Y ≤ limit` (projected Levenberg–Marquardt with a numerical Jacobian).
fn solve_cmy<F: Forward + ?Sized>(model: &F, target: [f64; 3], k: f64, limit: f64, seeds: &[[f64; 3]]) -> ([f64; 3], f64) {
solve_cmy_by(model, k, limit, seeds, |l| [l[0] - target[0], l[1] - target[1], l[2] - target[2]])
}
/// [`solve_cmy`] minimising the norm of `residual(lab)` instead of ΔE76.
fn solve_cmy_by<F: Forward + ?Sized>(model: &F, k: f64, limit: f64, seeds: &[[f64; 3]], residual: impl Fn([f64; 3]) -> [f64; 3]) -> ([f64; 3], f64) {
let project = |mut x: [f64; 3]| {
for _ in 0..4 {
for v in &mut x {
@@ -202,10 +243,7 @@ fn solve_cmy<F: Forward + ?Sized>(model: &F, target: [f64; 3], k: f64, limit: f6
}
x
};
let err = |x: [f64; 3]| {
let l = model.lab([x[0], x[1], x[2], k]);
[l[0] - target[0], l[1] - target[1], l[2] - target[2]]
};
let err = |x: [f64; 3]| residual(model.lab([x[0], x[1], x[2], k]));
let norm2 = |e: [f64; 3]| e[0] * e[0] + e[1] * e[1] + e[2] * e[2];
let mut best = ([0.0; 3], f64::INFINITY);
for seed in seeds {
@@ -275,13 +313,32 @@ fn solve_cmy<F: Forward + ?Sized>(model: &F, target: [f64; 3], k: f64, limit: f6
const SEEDS: [[f64; 3]; 5] = [[0.3, 0.3, 0.3], [0.05, 0.05, 0.05], [0.8, 0.8, 0.8], [0.9, 0.2, 0.2], [0.2, 0.9, 0.9]];
/// Black for a gray component `g` (the smallest of C, M, Y without black).
fn gcr(p: &CmykParams, g: f64) -> f64 {
let Some(first) = p.gcr_curve.first() else {
// The Photocraft profile's curve.
return if g <= p.gcr_start { 0.0 } else { p.k_max * ((g - p.gcr_start) / (1.0 - p.gcr_start)).powf(p.gcr_gamma) };
};
let mut k = first[1];
for w in p.gcr_curve.windows(2) {
let ([g0, k0], [g1, k1]) = (w[0], w[1]);
if g >= g1 {
k = k1;
} else if g > g0 {
k = k0 + (k1 - k0) * (g - g0) / (g1 - g0);
break;
}
}
k.clamp(0.0, 1.0)
}
/// Colorimetric separation of a relative Lab target: GCR black generation then a CMY solve.
pub fn separate<F: Forward + ?Sized>(model: &F, p: &CmykParams, target: [f64; 3], warm: Option<[f64; 4]>) -> ([f64; 4], f64) {
let mut seeds: Vec<[f64; 3]> = warm.map(|w| vec![[w[0], w[1], w[2]]]).unwrap_or_default();
seeds.extend_from_slice(&SEEDS);
let (cmy0, _) = solve_cmy(model, target, 0.0, 3.0, &seeds);
let g = cmy0[0].min(cmy0[1]).min(cmy0[2]);
let k = if g <= p.gcr_start { 0.0 } else { p.k_max * ((g - p.gcr_start) / (1.0 - p.gcr_start)).powf(p.gcr_gamma) };
let k = gcr(p, g);
let mut seeds2 = vec![[(cmy0[0] - k).max(0.0), (cmy0[1] - k).max(0.0), (cmy0[2] - k).max(0.0)], cmy0];
seeds2.extend_from_slice(&seeds);
let (cmy, de) = solve_cmy(model, target, k, (p.tac - k).max(0.0), &seeds2);
@@ -383,6 +440,187 @@ impl Gbd {
}
}
/// ΔE76 under which a separation counts as reaching its target.
const REACH: f64 = 1.5;
/// Oklab chroma beyond any printable colour: the top of the boundary search.
const OKLAB_CHROMA_MAX: f64 = 0.5;
/// Oklab chroma above which the perceptual and saturation tables no longer lift black: they lift
/// neutrals fully, and less the more colourful a colour is.
const LIFT_CHROMA: f64 = 0.1;
/// Weight of lightness and hue against chroma when a colour just outside the gamut is clipped.
const CLIP_WEIGHT: f64 = 1.5;
/// Growth of that weight with the squared share of chroma the clip loses.
const CLIP_WEIGHT_GROWTH: f64 = 200.0;
// Oklab (Björn Ottosson, 2020): XYZ (D65) → cone response → cube root → Lab-like axes.
const OKLAB_M1: math::Mat3 =
[[0.8189330101, 0.3618667424, -0.1288597137], [0.0329845436, 0.9293118715, 0.0361456387], [0.0482003018, 0.2643662691, 0.6338517070]];
const OKLAB_M2: math::Mat3 =
[[0.2104542553, 0.7936177850, -0.0040720468], [1.9779984951, -2.4285922050, 0.4505937099], [0.0259040371, 0.7827717662, -0.8086757660]];
const OKLAB_M1_INV: math::Mat3 =
[[1.2270138511, -0.5577999807, 0.2812561490], [-0.0405801784, 1.1122568696, -0.0716766787], [-0.0763812845, -0.4214819784, 1.5861632204]];
const OKLAB_M2_INV: math::Mat3 = [[1.0, 0.3963377774, 0.2158037573], [1.0, -0.1055613458, -0.0638541728], [1.0, -0.0894841775, -1.2914855480]];
/// The [`GamutMapping::Oklab`] separations of one forward model.
struct OklabMapper<'a, F: Forward + ?Sized> {
model: &'a F,
p: &'a CmykParams,
knee: f64,
saturation: f64,
/// XYZ scale that maps the v4 perceptual reference black onto the device black (the white
/// stays fixed): `v' = a·v + w·(1 − a)`.
black_scale: f64,
to_d65: math::Mat3,
to_d50: math::Mat3,
}
impl<'a, F: Forward + ?Sized> OklabMapper<'a, F> {
fn new(model: &'a F, p: &'a CmykParams, knee: f64, saturation: f64) -> Self {
let d65 = math::xy_to_xyz(math::D65_XY);
// The darkest neutral the printer reaches.
let (black, _) = separate(model, p, [0.0, 0.0, 0.0], None);
let device = math::lab_to_xyz([model.lab(black)[0], 0.0, 0.0], math::D50)[1];
let reference = crate::transform::PERCEPTUAL_BLACK[1];
OklabMapper {
model,
p,
knee,
saturation,
black_scale: (1.0 - device) / (1.0 - reference),
to_d65: math::bradford(math::D50, d65),
to_d50: math::bradford(d65, math::D50),
}
}
fn oklab(&self, lab: [f64; 3]) -> [f64; 3] {
let xyz = math::apply(&self.to_d65, math::lab_to_xyz(lab, math::D50));
math::apply(&OKLAB_M2, math::apply(&OKLAB_M1, xyz).map(f64::cbrt))
}
fn lab(&self, ok: [f64; 3]) -> [f64; 3] {
let lms = math::apply(&OKLAB_M2_INV, ok).map(|v| v * v * v);
math::xyz_to_lab(math::apply(&self.to_d50, math::apply(&OKLAB_M1_INV, lms)), math::D50)
}
/// The most chroma (scale of `ok`'s chroma in `[0, top]`) the printer reaches at `ok`'s
/// lightness and hue, and its separation; `None` when even the neutral is out of reach.
fn boundary(&self, ok: [f64; 3], top: f64, warm: Option<[f64; 4]>) -> Option<(f64, [f64; 4])> {
let (grey, d) = separate(self.model, self.p, self.lab([ok[0], 0.0, 0.0]), warm);
if d > REACH {
return None;
}
let (mut lo, mut hi, mut best) = (0.0, top, grey);
for _ in 0..14 {
let mid = 0.5 * (lo + hi);
let (c, d) = separate(self.model, self.p, self.lab([ok[0], ok[1] * mid, ok[2] * mid]), Some(best));
if d <= REACH {
lo = mid;
best = c;
} else {
hi = mid;
}
}
Some((lo, best))
}
/// Colorimetric: exact in gamut; outside, the closest colour the printer reaches in Oklab,
/// lightness and hue weighted above chroma, and more so the further outside the colour is.
fn clip(&self, lab: [f64; 3], warm: Option<[f64; 4]>) -> [f64; 4] {
let (cmyk, de) = separate(self.model, self.p, lab, warm);
if de <= REACH {
return cmyk;
}
let ok = self.oklab(lab);
let c0 = ok[1].hypot(ok[2]);
// Unit vectors along the colour's chroma and hue directions.
let (u, v) = if c0 > 1e-9 { ([ok[1] / c0, ok[2] / c0], [-ok[2] / c0, ok[1] / c0]) } else { ([1.0, 0.0], [0.0, 1.0]) };
// Lightness and hue errors weighted by `w`, chroma error by 1; scaled by 100 so the
// solver's step tolerances suit it as they suit ΔE76.
let residual = |w: f64| {
move |l: [f64; 3]| {
let o = self.oklab(l);
let (da, db) = (o[1] - ok[1], o[2] - ok[2]);
[100.0 * w * (o[0] - ok[0]), 100.0 * (da * u[0] + db * u[1]), 100.0 * w * (da * v[0] + db * v[1])]
}
};
let solve = |w: f64, k: f64, from: [f64; 3]| {
let mut seeds = vec![from];
seeds.extend_from_slice(&SEEDS);
solve_cmy_by(self.model, k, (self.p.tac - k).max(0.0), &seeds, residual(w)).0
};
// A first clip without black measures how far outside the colour is. A colour just
// outside may give up a little lightness and hue to keep its chroma (a red reaches the
// full magenta and yellow edge); one far outside keeps its lightness and hue and loses
// chroma (a saturated blue does not slide along the gamut surface).
let first = solve(CLIP_WEIGHT, 0.0, [cmyk[0], cmyk[1], cmyk[2]]);
let o = self.oklab(self.model.lab([first[0], first[1], first[2], 0.0]));
let loss = if c0 > 1e-9 { (1.0 - o[1].hypot(o[2]) / c0).clamp(0.0, 1.0) } else { 0.0 };
let w = CLIP_WEIGHT * (1.0 + CLIP_WEIGHT_GROWTH * loss * loss);
let free = solve(w, 0.0, first);
let norm = |c: [f64; 4]| residual(w)(self.model.lab(c)).iter().map(|v| v * v).sum::<f64>();
// Candidates, the closest kept: the separation itself (a colour it nearly reaches stays
// continuous with its in-gamut neighbours), the clip with the separation's black, and
// the clip with black generated from the black-free clip's gray component (inks the
// clip drives to 100 % stay there and black replaces the rest).
let mut best = (norm(cmyk), cmyk);
for k in [cmyk[3], gcr(self.p, free[0].min(free[1]).min(free[2]))] {
for from in [[cmyk[0], cmyk[1], cmyk[2]], free] {
let cmy = solve(w, k, from);
let c = [cmy[0], cmy[1], cmy[2], k];
let n = norm(c);
if n < best.0 {
best = (n, c);
}
}
}
// Near paper white the gamut narrows to a point: the separation's closest colour keeps
// light tints' chroma where holding lightness would drop them toward bare paper, and
// the table nodes there carry the in-gamut tints between them. The inks blend from the
// clip at L* 85 to the separation at L* 95.
let clip_share = ((95.0 - lab[0]) / 10.0).clamp(0.0, 1.0);
std::array::from_fn(|i| clip_share * best.1[i] + (1.0 - clip_share) * cmyk[i])
}
/// Maps the v4 perceptual reference black onto the device black, linearly in XYZ, for
/// neutrals; colourful colours keep their lightness.
fn lift_black(&self, lab: [f64; 3]) -> [f64; 3] {
let ok = self.oklab(lab);
let neutral = (1.0 - ok[1].hypot(ok[2]) / LIFT_CHROMA).clamp(0.0, 1.0);
let a = 1.0 - neutral * (1.0 - self.black_scale);
let xyz = math::lab_to_xyz(lab, math::D50);
math::xyz_to_lab(std::array::from_fn(|i| a * xyz[i] + math::D50[i] * (1.0 - a)), math::D50)
}
fn perceptual(&self, lab: [f64; 3], warm: Option<[f64; 4]>) -> [f64; 4] {
let lab = self.lift_black(lab);
let ok = self.oklab(lab);
let c = ok[1].hypot(ok[2]);
if c < 1e-9 {
return self.clip(lab, warm);
}
let Some((s, edge)) = self.boundary(ok, OKLAB_CHROMA_MAX / c, warm) else {
return self.clip(lab, warm);
};
// Chroma above the knee is compressed smoothly into the boundary chroma `cb`.
let cb = s * c;
let knee = self.knee * cb;
if c <= knee {
return self.clip(lab, warm);
}
let c2 = knee + (cb - knee) * ((c - knee) / (cb - knee).max(1e-9)).tanh();
let f = c2 / c;
if f >= s {
return edge;
}
self.clip(self.lab([ok[0], ok[1] * f, ok[2] * f]), Some(edge))
}
fn saturation(&self, lab: [f64; 3], warm: Option<[f64; 4]>) -> [f64; 4] {
let ok = self.oklab(self.lift_black(lab));
self.clip(self.lab([ok[0], ok[1] * self.saturation, ok[2] * self.saturation]), warm)
}
}
fn lab_node_legacy(i: usize, n: usize) -> f64 {
i as f64 / (n - 1) as f64
}
@@ -419,6 +657,18 @@ fn white_node_curve(l_axis: bool, x: f64) -> f64 {
}
fn b2a_table<F: Forward + ?Sized>(model: &F, p: &CmykParams, gbd: Option<&Gbd>) -> Result<Clut, CmsError> {
b2a_table_with(p, |lab, warm| {
let target = match gbd {
Some(g) => g.compress(lab),
None => lab,
};
separate(model, p, target, warm).0
})
}
/// Samples a BToA table: `node` separates each relative Lab node, warm-started from the
/// previous node along b.
fn b2a_table_with(p: &CmykParams, node: impl Fn([f64; 3], Option<[f64; 4]>) -> [f64; 4] + Sync) -> Result<Clut, CmsError> {
let n = p.grid_b2a.max(2);
let nodes = n * n * n;
let step = (n - 1) as f64;
@@ -434,11 +684,7 @@ fn b2a_table<F: Forward + ?Sized>(model: &F, p: &CmykParams, gbd: Option<&Gbd>)
} else {
decode_v2([lab_node_legacy(li, n), lab_node_legacy(ai, n), lab_node_legacy(bi, n)])
};
let target = match gbd {
Some(g) => g.compress(lab),
None => lab,
};
let (cmyk, _) = if p.white_on_node && lab == [100.0, 0.0, 0.0] { ([0.0; 4], 0.0) } else { separate(model, p, target, warm) };
let cmyk = if p.white_on_node && lab == [100.0, 0.0, 0.0] { [0.0; 4] } else { node(lab, warm) };
warm = Some(cmyk);
out.extend(cmyk.map(|v| v as f32));
}
@@ -460,6 +706,15 @@ fn check_grids(p: &CmykParams) -> Result<(), CmsError> {
if p.white_on_node && p.grid_b2a.is_multiple_of(2) {
return Err(invalid_arg(format!("white on a node needs an odd BToA grid, not {}", p.grid_b2a)));
}
let valid = |v: &[f64; 2]| v.iter().all(|x| (0.0..=1.0).contains(x));
if !p.gcr_curve.iter().all(valid) || p.gcr_curve.windows(2).any(|w| w[1][0] <= w[0][0]) {
return Err(invalid_arg(format!("black generation curve {:?} needs points in [0, 1] with increasing gray components", p.gcr_curve)));
}
if let GamutMapping::Oklab { knee, saturation } = p.mapping
&& !((0.0..1.0).contains(&knee) && saturation.is_finite() && saturation > 0.0)
{
return Err(invalid_arg(format!("Oklab gamut mapping needs a knee in [0, 1) and a positive saturation, not {knee} and {saturation}")));
}
Ok(())
}
@@ -518,11 +773,23 @@ pub fn cmyk_profile_from_model<F: Forward + ?Sized>(model: &F, paper: [f64; 3],
}
fn lut_profile<F: Forward + ?Sized>(model: &F, a2b: Clut, paper: [f64; 3], p: &CmykParams) -> Result<Profile, CmsError> {
let gbd = Gbd::new(model, p);
let colorimetric = b2a_table(model, p, None)?;
let perceptual = b2a_table(model, p, Some(&gbd))?;
let (perceptual, colorimetric, saturation) = match p.mapping {
GamutMapping::Legacy => {
let gbd = Gbd::new(model, p);
let colorimetric = b2a_table(model, p, None)?;
let perceptual = b2a_table(model, p, Some(&gbd))?;
(perceptual.clone(), colorimetric, perceptual)
}
GamutMapping::Oklab { knee, saturation } => {
let m = OklabMapper::new(model, p, knee, saturation);
let colorimetric = b2a_table_with(p, |lab, warm| m.clip(lab, warm))?;
let perceptual = b2a_table_with(p, |lab, warm| m.perceptual(lab, warm))?;
let saturation = b2a_table_with(p, |lab, warm| m.saturation(lab, warm))?;
(perceptual, colorimetric, saturation)
}
};
let a2b = lut16(a2b, false);
let (b0, b1) = (lut16(perceptual, p.white_on_node), lut16(colorimetric, p.white_on_node));
let (b0, b1, b2) = (lut16(perceptual, p.white_on_node), lut16(colorimetric, p.white_on_node), lut16(saturation, p.white_on_node));
Ok(Profile {
version: (4, 0x30),
class: ProfileClass::Output,
@@ -538,7 +805,7 @@ fn lut_profile<F: Forward + ?Sized>(model: &F, a2b: Clut, paper: [f64; 3], p: &C
trc: None,
gray_trc: None,
a2b: [Some(a2b.clone()), Some(a2b.clone()), Some(a2b)],
b2a: [Some(b0.clone()), Some(b1), Some(b0)],
b2a: [Some(b0), Some(b1), Some(b2)],
bytes: None,
hash: Default::default(),
}
@@ -576,6 +843,46 @@ mod tests {
}
}
#[test]
fn oklab_mapping_refuses_bad_parameters() {
for (knee, saturation) in [(1.0, 1.0), (-0.1, 1.0), (f64::NAN, 1.0), (0.5, 0.0), (0.5, f64::INFINITY), (0.5, f64::NAN)] {
let p = CmykParams { grid_a2b: 2, grid_b2a: 2, mapping: GamutMapping::Oklab { knee, saturation }, ..CmykParams::default() };
assert!(cmyk_profile(&p).is_err(), "knee {knee}, saturation {saturation}");
}
}
#[test]
fn black_generation_curve() {
let p = CmykParams { gcr_curve: vec![[0.0, 0.0], [0.5, 0.2], [1.0, 0.9]], ..CmykParams::default() };
for (g, k) in [(-1.0, 0.0), (0.0, 0.0), (0.25, 0.1), (0.5, 0.2), (0.75, 0.55), (1.0, 0.9), (2.0, 0.9)] {
assert!((gcr(&p, g) - k).abs() < 1e-12, "g {g}: {} != {k}", gcr(&p, g));
}
for curve in [vec![[0.5, 0.2], [0.5, 0.3]], vec![[0.5, 0.2], [0.2, 0.3]], vec![[0.0, 1.5]], vec![[f64::NAN, 0.0]]] {
let p = CmykParams { grid_a2b: 2, grid_b2a: 2, gcr_curve: curve.clone(), ..CmykParams::default() };
assert!(cmyk_profile(&p).is_err(), "{curve:?}");
}
}
#[test]
fn oklab_round_trips() {
let m = CmykModel::coated(CmykParams::default().tvi);
let p = CmykParams::default();
let mapper = OklabMapper::new(&m, &p, 0.65, 1.05);
for lab in [[50.0, 0.0, 0.0], [44.0, 63.7, 47.0], [31.9, 32.9, -76.8], [90.0, -5.0, 80.0]] {
let back = mapper.lab(mapper.oklab(lab));
// The published matrices carry 10 digits, so the inverses are exact to about 1e-5 ΔE.
assert!(math::delta_e76(lab, back) < 1e-3, "{lab:?} -> {back:?}");
}
}
#[test]
fn oklab_profile_has_three_tables() {
let p = CmykParams { grid_a2b: 5, grid_b2a: 5, mapping: GamutMapping::Oklab { knee: 0.65, saturation: 1.05 }, ..CmykParams::default() };
let profile = cmyk_profile(&p).unwrap();
let [Some(b0), Some(b1), Some(b2)] = &profile.b2a else { panic!("missing BToA tables") };
assert!(b0 != b1 && b0 != b2, "perceptual shares a table");
}
#[test]
fn separation_hits_in_gamut_targets() {
let m = CmykModel::coated(CmykParams::default().tvi);
+1 -1
View File
@@ -46,7 +46,7 @@ const OUT_TABLE: usize = GpuMatrixShaper::OUTPUT_ENTRIES;
const _: () = assert!(OUT_TABLE.is_power_of_two());
/// ICC v4 perceptual reference medium black (ICC.1:2010 §6.3.4.4), XYZ.
const PERCEPTUAL_BLACK: [f64; 3] = [0.00336, 0.0034731, 0.00287];
pub(crate) const PERCEPTUAL_BLACK: [f64; 3] = [0.00336, 0.0034731, 0.00287];
/// Accelerated representation of a transform.
#[derive(Debug)]
+60 -1
View File
@@ -12,7 +12,7 @@
//! ([`synth::cmyk_profile_from_model`]).
use craft_cms::math::{self, delta_e2000};
use craft_cms::synth::{self, CmykParams, Forward};
use craft_cms::synth::{self, CmykParams, Forward, GamutMapping};
use craft_cms::{Builtin, Intent, Profile, Transform};
const DATA: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/data/FOGRA39.txt");
@@ -346,8 +346,10 @@ fn regen_coated_fogra39() {
description: Builtin::CoatedFogra39.description().into(),
copyright: craft_cms::builtin::COATED_FOGRA39_COPYRIGHT.into(),
tac: 3.3,
gcr_curve: vec![[0.0, 0.0], [0.2, 0.05], [0.5, 0.205], [0.67, 0.5], [1.0, 0.95]],
grid_b2a: 33,
white_on_node: true,
mapping: GamutMapping::Oklab { knee: 0.65, saturation: 1.05 },
..CmykParams::default()
};
let profile = synth::cmyk_profile_from_model(&f, paper(&p), &params).unwrap();
@@ -564,3 +566,60 @@ fn stand_in_separations_respect_the_ink_limit() {
}
}
}
/// sRGB patches and their separations by the reference application (Convert to Profile to
/// Coated FOGRA39, dither on), CMYK % per intent: relative + BPC, relative, perceptual,
/// saturation, absolute. Measured for storytold/photocraft#14267.
const REFERENCE_SEPARATIONS: [([u8; 3], [[f32; 4]; 5]); 3] = [
(
[200, 30, 30],
[[14.5, 97.7, 95.3, 4.3], [13.3, 100.0, 100.0, 5.5], [18.0, 96.9, 92.9, 8.6], [14.9, 100.0, 100.0, 5.9], [8.6, 100.0, 100.0, 2.4]],
),
([30, 60, 200], [[91.8, 75.3, 0.0, 0.0], [94.5, 76.5, 0.0, 0.0], [94.5, 77.7, 0.0, 0.0], [93.7, 76.9, 0.0, 0.0], [92.6, 74.9, 0.0, 0.0]]),
(
[128, 128, 128],
[[49.0, 38.4, 38.4, 20.0], [49.8, 39.2, 38.8, 21.2], [49.0, 38.4, 38.4, 20.0], [49.0, 38.4, 38.4, 20.0], [46.7, 36.9, 38.4, 18.4]],
),
];
const SEPARATION_INTENTS: [(Intent, bool); 5] = [
(Intent::RelativeColorimetric, true),
(Intent::RelativeColorimetric, false),
(Intent::Perceptual, true),
(Intent::Saturation, true),
(Intent::AbsoluteColorimetric, false),
];
fn separation(intent: Intent, bpc: bool, rgb: [u8; 3]) -> [f32; 4] {
let t = Transform::new(Builtin::Srgb.profile(), fogra39(), intent, bpc).unwrap();
let mut o = [0.0f32; 4];
t.eval(&rgb.map(|v| v as f32 / 255.0), &mut o).unwrap();
o.map(|v| v * 100.0)
}
/// sRGB → Coated FOGRA39 matches the reference application's inks on a red, a saturated blue
/// and a mid grey for every intent. Most land within 2 % per ink; red at the magenta–yellow
/// edge lands between table nodes that are and are not clipped, which costs up to 2.2 % with
/// the 33³ BToA tables.
#[test]
fn separates_like_the_reference_application() {
let mut worst = 0.0f32;
for (rgb, want) in REFERENCE_SEPARATIONS {
for ((intent, bpc), want) in SEPARATION_INTENTS.into_iter().zip(want) {
let got = separation(intent, bpc, rgb);
let diff = (0..4).map(|i| (got[i] - want[i]).abs()).fold(0.0f32, f32::max);
eprintln!("{rgb:?} {intent:?} bpc {bpc}: {got:.1?} reference {want:?} (max {diff:.1})");
worst = worst.max(diff);
}
}
assert!(worst <= 2.5, "worst ink difference {worst} %");
}
/// Saturation has its own table: a red separates with more magenta and yellow than
/// perceptual, whose chroma compression takes some back.
#[test]
fn saturation_and_perceptual_separate_differently() {
let red = [200, 30, 30];
let (perceptual, saturation) = (separation(Intent::Perceptual, true, red), separation(Intent::Saturation, true, red));
assert!(saturation[1] > perceptual[1] + 1.0 && saturation[2] > perceptual[2] + 1.0, "perceptual {perceptual:?}, saturation {saturation:?}");
}
+17
View File
@@ -0,0 +1,17 @@
[package]
name = "craft-evm"
description = "Read-only EVM bytecode dispatcher and compiler metadata inspection"
version.workspace = true
edition.workspace = true
license.workspace = true
rust-version.workspace = true
repository.workspace = true
[lints]
workspace = true
[dependencies]
serde = { workspace = true }
serde_json = { workspace = true }
thiserror = { workspace = true }
[dev-dependencies]
craft-testkit = { workspace = true }
+7
View File
@@ -0,0 +1,7 @@
# craft-evm
Read-only EVM bytecode inspection: strict raw/hex admission, opcode listing, dispatcher selector and jump-pattern facts, and Solidity/Vyper CBOR metadata. No VM execution, RPC, wallet, signing or transaction support.
Preserves Craft Anything's parser behavior and limits: 16 MiB decoded bytecode, 4,096 reported functions and 400 listing entries, plus existing bounded CBOR recursion/collection parsing. These are not a total allocator-byte guarantee. Dispatcher facts are heuristics, not a full control-flow or semantic proof. EOF containers remain unsupported.
Application file I/O, evidence policy and selector naming remain outside this crate.
+895
View File
@@ -0,0 +1,895 @@
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
#![forbid(unsafe_code)]
//! EVM bytecode: the public function selectors a contract's dispatcher compares against, where
//! each one jumps, whether it refuses payments, fallback/receive presence, and the compiler
//! metadata (CBOR) appended to the code. Nothing is executed; this is a linear sweep that
//! respects PUSH data plus pattern matching over the instruction stream.
use EvmError as FormatError;
/// Neutral inspection failure; no code is executed.
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
pub enum EvmError {
#[error("truncated input: {0}")]
Truncated(String),
#[error("malformed input: {0}")]
Malformed(String),
#[error("unsupported: {0}")]
Unsupported(String),
#[error("input too large: {0}")]
TooLarge(String),
}
#[cfg(test)]
mod test_support;
use serde::Serialize;
use serde_json::{Map, Value};
use std::collections::BTreeSet;
/// Largest bytecode we analyse (16 MiB; deployed contracts are at most 24 KiB).
pub const MAX_CODE: usize = 16 << 20;
const MAX_FUNCTIONS: usize = 4096;
const LISTING_CAP: usize = 400;
/// How input bytes are encoded.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Encoding {
/// Hex when the bytes are all hex digits (with an optional 0x), raw otherwise.
Auto,
Raw,
Hex,
}
/// Turn input bytes into bytecode. Hex is strict: optional `0x`, even length, only hex digits;
/// surrounding whitespace is ignored.
pub fn decode_input(input: &[u8], enc: Encoding) -> Result<(Vec<u8>, &'static str), FormatError> {
let trimmed = input.trim_ascii();
let body = trimmed.strip_prefix(b"0x").or_else(|| trimmed.strip_prefix(b"0X")).unwrap_or(trimmed);
let looks_hex = !body.is_empty() && body.iter().all(u8::is_ascii_hexdigit);
let hex = match enc {
Encoding::Hex => true,
Encoding::Raw => false,
Encoding::Auto => looks_hex,
};
if !hex {
if input.len() > MAX_CODE {
return Err(FormatError::TooLarge(format!("{} bytes; the limit is {MAX_CODE}", input.len())));
}
return Ok((input.to_vec(), "raw"));
}
if body.is_empty() {
return Err(FormatError::Malformed("no hex digits".into()));
}
if let Some(p) = body.iter().position(|b| !b.is_ascii_hexdigit()) {
return Err(FormatError::Malformed(format!("not a hex digit at position {p}")));
}
if !body.len().is_multiple_of(2) {
return Err(FormatError::Malformed(format!("odd number of hex digits ({})", body.len())));
}
if body.len() / 2 > MAX_CODE {
return Err(FormatError::TooLarge(format!("{} bytes; the limit is {MAX_CODE}", body.len() / 2)));
}
let nib = |c: u8| -> u8 {
match c {
b'0'..=b'9' => c - b'0',
b'a'..=b'f' => c - b'a' + 10,
_ => c.wrapping_sub(b'A').wrapping_add(10) & 0x0f,
}
};
Ok((body.chunks(2).map(|p| (nib(p.first().copied().unwrap_or(b'0')) << 4) | nib(p.get(1).copied().unwrap_or(b'0'))).collect(), "hex"))
}
#[derive(Debug, Clone, Copy)]
struct Ins {
off: usize,
op: u8,
/// Immediate (PUSH data) start and length.
imm: (usize, usize),
}
fn push_len(op: u8) -> usize {
if (0x60..=0x7f).contains(&op) { usize::from(op - 0x5f) } else { 0 }
}
/// The mnemonic of an opcode (`UNKNOWN_0x…` for unassigned ones).
pub fn mnemonic(op: u8) -> String {
let s = match op {
0x00 => "STOP",
0x01 => "ADD",
0x02 => "MUL",
0x03 => "SUB",
0x04 => "DIV",
0x05 => "SDIV",
0x06 => "MOD",
0x07 => "SMOD",
0x08 => "ADDMOD",
0x09 => "MULMOD",
0x0a => "EXP",
0x0b => "SIGNEXTEND",
0x10 => "LT",
0x11 => "GT",
0x12 => "SLT",
0x13 => "SGT",
0x14 => "EQ",
0x15 => "ISZERO",
0x16 => "AND",
0x17 => "OR",
0x18 => "XOR",
0x19 => "NOT",
0x1a => "BYTE",
0x1b => "SHL",
0x1c => "SHR",
0x1d => "SAR",
0x20 => "KECCAK256",
0x30 => "ADDRESS",
0x31 => "BALANCE",
0x32 => "ORIGIN",
0x33 => "CALLER",
0x34 => "CALLVALUE",
0x35 => "CALLDATALOAD",
0x36 => "CALLDATASIZE",
0x37 => "CALLDATACOPY",
0x38 => "CODESIZE",
0x39 => "CODECOPY",
0x3a => "GASPRICE",
0x3b => "EXTCODESIZE",
0x3c => "EXTCODECOPY",
0x3d => "RETURNDATASIZE",
0x3e => "RETURNDATACOPY",
0x3f => "EXTCODEHASH",
0x40 => "BLOCKHASH",
0x41 => "COINBASE",
0x42 => "TIMESTAMP",
0x43 => "NUMBER",
0x44 => "PREVRANDAO",
0x45 => "GASLIMIT",
0x46 => "CHAINID",
0x47 => "SELFBALANCE",
0x48 => "BASEFEE",
0x49 => "BLOBHASH",
0x4a => "BLOBBASEFEE",
0x50 => "POP",
0x51 => "MLOAD",
0x52 => "MSTORE",
0x53 => "MSTORE8",
0x54 => "SLOAD",
0x55 => "SSTORE",
0x56 => "JUMP",
0x57 => "JUMPI",
0x58 => "PC",
0x59 => "MSIZE",
0x5a => "GAS",
0x5b => "JUMPDEST",
0x5c => "TLOAD",
0x5d => "TSTORE",
0x5e => "MCOPY",
0x5f => "PUSH0",
0x60..=0x7f => return format!("PUSH{}", op - 0x5f),
0x80..=0x8f => return format!("DUP{}", op - 0x7f),
0x90..=0x9f => return format!("SWAP{}", op - 0x8f),
0xa0..=0xa4 => return format!("LOG{}", op - 0xa0),
0xf0 => "CREATE",
0xf1 => "CALL",
0xf2 => "CALLCODE",
0xf3 => "RETURN",
0xf4 => "DELEGATECALL",
0xf5 => "CREATE2",
0xfa => "STATICCALL",
0xfd => "REVERT",
0xfe => "INVALID",
0xff => "SELFDESTRUCT",
_ => return format!("UNKNOWN_{op:#04x}"),
};
s.to_string()
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct EvmFunction {
/// `0x` + 8 hex digits.
pub selector: String,
/// Where the dispatcher sends calls with this selector.
pub bytecode_offset: usize,
/// Offset of the comparison in the dispatcher.
pub compare_offset: usize,
pub dispatch: &'static str,
/// `eq-jumpi` (jump on match) or `eq-iszero-fallthrough` (body follows the check).
pub pattern: &'static str,
/// `nonpayable`, `payable` or `unknown` (view/pure cannot be told from the dispatcher).
pub mutability: &'static str,
/// Why the mutability was chosen.
pub mutability_reason: String,
/// False when the destination is not a JUMPDEST (the match is then doubtful).
pub jumpdest_valid: bool,
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct EvmMetadata {
pub offset: usize,
pub length: usize,
pub entries: Value,
pub compiler: Option<String>,
pub ipfs: Option<String>,
pub swarm: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct EvmInterface {
pub code_size: usize,
/// Bytes analysed after removing the metadata tail.
pub analysed_size: usize,
pub metadata: Option<EvmMetadata>,
pub instructions: usize,
pub jumpdests: usize,
/// How the selector is pulled from calldata: `shr-224`, `div-2^224`, `and-mask`, or none.
pub selector_extraction: Option<&'static str>,
pub dispatcher_offset: Option<usize>,
/// `solc-like`, `vyper-like`, `mixed` or `none`.
pub style: &'static str,
pub functions: Vec<EvmFunction>,
/// Comparisons that split the selector range (binary-search dispatchers).
pub binary_search_splits: usize,
/// A CALLVALUE check that runs before any selector comparison (all functions non-payable).
pub callvalue_check_before_dispatch: bool,
pub short_calldata_check: bool,
/// `present`, `absent` or `unknown`.
pub fallback: &'static str,
/// `present` or `not-detected`.
pub receive: &'static str,
/// The dispatcher region as text (first instructions only).
pub dispatcher_listing: Vec<String>,
pub warnings: Vec<String>,
}
fn sweep(code: &[u8]) -> Vec<Ins> {
let mut out = Vec::with_capacity(code.len().min(1 << 20) / 2);
let mut i = 0usize;
while i < code.len() {
let op = code.get(i).copied().unwrap_or(0);
let n = push_len(op);
out.push(Ins { off: i, op, imm: (i + 1, n) });
i = i.saturating_add(1).saturating_add(n);
}
out
}
fn imm<'a>(code: &'a [u8], ins: &Ins) -> &'a [u8] {
// Truncated PUSH data at the end of code reads as zero-padded; we only use what exists.
let (s, n) = ins.imm;
code.get(s..s.saturating_add(n).min(code.len())).unwrap_or(&[])
}
fn imm_u64(code: &[u8], ins: &Ins) -> Option<u64> {
let b = imm(code, ins);
if ins.imm.1 > 8 || b.len() != ins.imm.1 {
return None;
}
Some(b.iter().fold(0u64, |a, &x| (a << 8) | u64::from(x)))
}
fn is_push(op: u8) -> bool {
(0x60..=0x7f).contains(&op)
}
/// Strip and decode a trailing CBOR metadata block.
fn split_metadata(code: &[u8]) -> (usize, Option<EvmMetadata>) {
let n = code.len();
if n < 4 {
return (n, None);
}
let len = usize::from(u16::from_be_bytes([code.get(n - 2).copied().unwrap_or(0), code.get(n - 1).copied().unwrap_or(0)]));
if len == 0 || len.saturating_add(2) > n {
return (n, None);
}
let start = n - 2 - len;
let Some(cbor) = code.get(start..n - 2) else { return (n, None) };
if !matches!(cbor.first(), Some(0xa1..=0xb8 | 0xbf)) {
return (n, None);
}
let mut pos = 0usize;
let Some(v) = cbor_item(cbor, &mut pos, 0) else { return (n, None) };
if pos != cbor.len() {
return (n, None);
}
let Value::Object(map) = &v else { return (n, None) };
let compiler = map
.get("solc")
.and_then(|s| s.as_str().map(String::from))
.map(|v| format!("solc {v}"))
.or_else(|| map.get("vyper").and_then(Value::as_str).map(|v| format!("vyper {v}")));
let ipfs = map.get("ipfs").and_then(Value::as_str).map(String::from);
let swarm = map.get("bzzr1").or_else(|| map.get("bzzr0")).and_then(Value::as_str).map(String::from);
(start, Some(EvmMetadata { offset: start, length: len + 2, entries: v.clone(), compiler, ipfs, swarm }))
}
fn cbor_arg(b: &[u8], pos: &mut usize, info: u8) -> Option<u64> {
let take = |pos: &mut usize, k: usize| -> Option<u64> {
let s = b.get(*pos..pos.checked_add(k)?)?;
*pos += k;
Some(s.iter().fold(0u64, |a, &x| (a << 8) | u64::from(x)))
};
match info {
0..=23 => Some(u64::from(info)),
24 => take(pos, 1),
25 => take(pos, 2),
26 => take(pos, 4),
27 => take(pos, 8),
_ => None,
}
}
/// Decode one bounded metadata item. An incomplete or unsupported value invalidates the metadata trailer.
fn cbor_item(b: &[u8], pos: &mut usize, depth: u32) -> Option<Value> {
if depth > 4 {
return None;
}
let ib = *b.get(*pos)?;
*pos += 1;
let major = ib >> 5;
let info = ib & 0x1f;
match major {
0 => cbor_arg(b, pos, info).map(Value::from),
2 | 3 => {
let n = usize::try_from(cbor_arg(b, pos, info)?).ok()?;
let s = b.get(*pos..pos.checked_add(n)?)?;
*pos += n;
Some(if major == 3 { Value::from(std::str::from_utf8(s).ok()?) } else { Value::from(bytes_value(s)) })
}
4 => {
let n = usize::try_from(cbor_arg(b, pos, info)?).ok()?;
if n > 64 {
return None;
}
let mut items = Vec::new();
for _ in 0..n {
items.push(cbor_item(b, pos, depth + 1)?);
}
Some(Value::Array(items))
}
5 => {
let n = usize::try_from(cbor_arg(b, pos, info)?).ok()?;
if n > 32 {
return None;
}
let mut m = Map::new();
for _ in 0..n {
// The metadata view uses JSON object keys. Coercing other CBOR key
// types to strings can merge distinct keys and hide bytecode.
if b.get(*pos).map(|byte| byte >> 5) != Some(3) {
return None;
}
let Value::String(k) = cbor_item(b, pos, depth + 1)? else { return None };
if m.contains_key(&k) {
return None;
}
let v = cbor_item(b, pos, depth + 1)?;
let v = match (k.as_str(), &v) {
("solc" | "vyper", Value::Array(items)) if items.len() == 3 && items.iter().all(Value::is_u64) => {
Value::from(items.iter().map(|v| v.to_string()).collect::<Vec<_>>().join("."))
}
("solc", Value::String(s)) if s.starts_with("0x") && s.len() == 8 => {
let d: Vec<u8> = (0..3).filter_map(|i| hex_byte(s.get(2 + 2 * i..4 + 2 * i)?)).collect();
if let [a, bb, c] = d.as_slice() { Value::from(format!("{a}.{bb}.{c}")) } else { v }
}
("ipfs", Value::String(s)) if s.starts_with("0x1220") => Value::from(base58(&hex_decode(s.get(2..).unwrap_or("")))),
_ => v,
};
m.insert(k, v);
}
Some(Value::Object(m))
}
7 => match info {
20 => Some(Value::Bool(false)),
21 => Some(Value::Bool(true)),
22 => Some(Value::Null),
_ => None,
},
_ => None,
}
}
fn bytes_value(s: &[u8]) -> String {
let mut out = String::with_capacity(2 + s.len() * 2);
out.push_str("0x");
for b in s {
out.push_str(&format!("{b:02x}"));
}
out
}
/// One byte from two hex digits; `from_str_radix` alone would also accept a leading `+`.
fn hex_byte(pair: &str) -> Option<u8> {
if pair.bytes().all(|c| c.is_ascii_hexdigit()) { u8::from_str_radix(pair, 16).ok() } else { None }
}
fn hex_decode(s: &str) -> Vec<u8> {
(0..s.len() / 2).filter_map(|i| hex_byte(s.get(2 * i..2 * i + 2)?)).collect()
}
fn base58(b: &[u8]) -> String {
const A: &[u8; 58] = b"123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz";
let zeros = b.iter().take_while(|&&x| x == 0).count();
let mut digits: Vec<u8> = Vec::new();
for &byte in b.iter().take(128) {
let mut carry = u32::from(byte);
for d in digits.iter_mut() {
carry += u32::from(*d) << 8;
*d = (carry % 58) as u8;
carry /= 58;
}
while carry > 0 {
digits.push((carry % 58) as u8);
carry /= 58;
}
}
let mut s: String = std::iter::repeat_n('1', zeros).collect();
s.extend(digits.iter().rev().map(|&d| char::from(*A.get(usize::from(d)).unwrap_or(&b'1'))));
s
}
fn selector_extraction(body: &[u8], ins: &[Ins]) -> Option<(&'static str, usize)> {
let at = |k: usize| ins.get(k);
let op_at = |k: usize| at(k).map(|i| i.op);
// Selector extraction: CALLDATALOAD followed by SHR 224, DIV 2^224 or a 4-byte mask.
let mut extraction: Option<(&'static str, usize)> = None;
for (k, i) in ins.iter().enumerate() {
if i.op != 0x35 {
continue;
}
for d in 1..=6 {
let Some(n) = at(k + d) else { break };
let kind = match n.op {
0x60 if imm(body, n) == [0xe0] && op_at(k + d + 1) == Some(0x1c) => Some("shr-224"),
0x7c if d == 1
&& imm(body, n).first() == Some(&1)
&& imm(body, n).iter().skip(1).all(|&b| b == 0)
&& op_at(k + d + 1) == Some(0x90)
&& op_at(k + d + 2) == Some(0x04) =>
{
Some("div-2^224")
}
0x63 if imm(body, n) == [0xff, 0xff, 0xff, 0xff] && (op_at(k + d + 1) == Some(0x16) || op_at(k + d + 2) == Some(0x16)) => {
Some("and-mask")
}
_ => None,
};
if let Some(kind) = kind {
extraction = Some((kind, k));
break;
}
}
if extraction.is_some() {
break;
}
}
extraction
}
fn short_calldata_path(body: &[u8], ins: &[Ins], disp_start: usize) -> (bool, &'static str) {
let at = |k: usize| ins.get(k);
let op_at = |k: usize| at(k).map(|i| i.op);
// Short calldata path and receive().
let mut short_calldata = false;
let mut receive = "not-detected";
for (k, i) in ins.iter().enumerate().take(disp_start.saturating_add(64)) {
if i.op != 0x36 {
continue;
}
if op_at(k + 1) == Some(0x15) && at(k + 2).is_some_and(|p| is_push(p.op)) && op_at(k + 3) == Some(0x57) {
receive = "present";
}
// LT pops the size first in PUSH 4, CALLDATASIZE, LT.
if k.checked_sub(1).and_then(at).is_some_and(|p| is_push(p.op) && imm_u64(body, p) == Some(4))
&& op_at(k + 1) == Some(0x10)
&& at(k + 2).is_some_and(|p| is_push(p.op))
&& op_at(k + 3) == Some(0x57)
{
short_calldata = true;
}
}
(short_calldata, receive)
}
/// Analyse bytecode (already decoded from hex if needed).
pub fn analyze(code: &[u8]) -> Result<EvmInterface, FormatError> {
if code.is_empty() {
return Err(FormatError::Truncated("no bytecode".into()));
}
if code.len() > MAX_CODE {
return Err(FormatError::TooLarge(format!("{} bytes; the limit is {MAX_CODE}", code.len())));
}
if code.starts_with(&[0xef, 0x00]) {
return Err(FormatError::Unsupported("EVM Object Format (EOF, 0xEF00) containers are not analysed".into()));
}
let mut warnings = Vec::new();
let (end, metadata) = split_metadata(code);
let body = code.get(..end).unwrap_or(code);
let ins = sweep(body);
let jumpdests = ins.iter().filter(|i| i.op == 0x5b).count();
let at = |k: usize| ins.get(k);
let op_at = |k: usize| at(k).map(|i| i.op);
let index_of = |off: usize| ins.binary_search_by_key(&off, |i| i.off).ok();
let extraction = selector_extraction(body, &ins);
let disp_start = extraction.map(|(_, k)| k).unwrap_or(0);
if extraction.is_none() {
warnings.push("no selector extraction (CALLDATALOAD then SHR/DIV/AND) was seen; matches below are less certain".into());
}
// Only a guard on the straight-line entry path can apply to every selector.
let callvalue_before = ins
.iter()
.take(disp_start)
.enumerate()
.take_while(|(_, i)| !matches!(i.op, 0x00 | 0x56 | 0x57 | 0xf3 | 0xfd | 0xfe | 0xff))
.any(|(k, i)| i.op == 0x34 && rejects_value(body, &ins, k, &index_of));
let mut functions: Vec<EvmFunction> = Vec::new();
let mut selectors = BTreeSet::new();
let mut splits = 0usize;
let mut last_jumpi: Option<usize> = None;
let mut solc_like = 0usize;
let mut vyper_like = 0usize;
for k in disp_start..ins.len() {
let Some(i) = at(k) else { break };
let n = push_len(i.op);
if !(1..=4).contains(&n) {
continue;
}
let Some(sel) = imm_u64(body, i) else { continue };
let found = |dk: usize| -> Option<usize> {
let p = at(dk)?;
(is_push(p.op) && p.imm.1 <= 3 && op_at(dk + 1) == Some(0x57)).then_some(dk)
};
// PUSH sel, [DUP2], EQ, PUSH dest, JUMPI
let eq_k = if op_at(k + 1) == Some(0x14) {
Some(k + 1)
} else if matches!(op_at(k + 1), Some(0x80..=0x8f)) && op_at(k + 2) == Some(0x14) {
Some(k + 2)
} else {
None
};
let mut hit: Option<(usize, usize, &'static str)> = None;
if let Some(e) = eq_k {
if let Some(pk) = found(e + 1)
&& let Some(dest) = at(pk).and_then(|p| imm_u64(body, p))
{
hit = Some((usize::try_from(dest).unwrap_or(usize::MAX), pk + 1, "eq-jumpi"));
} else if n == 4 && op_at(e + 1) == Some(0x15) && found(e + 2).is_some() {
// Vyper-style: if selector != x jump to the next check; the body follows.
if let Some(next) = at(e + 4) {
hit = Some((next.off, e + 3, "eq-iszero-fallthrough"));
}
}
} else if n == 4 {
// Vyper also loads the selector from memory: PUSH sel, PUSH1 0, MLOAD, EQ, ISZERO …
let load = at(k + 1).is_some_and(|p| (p.op == 0x5f || is_push(p.op)) && imm_u64(body, p) == Some(0));
if load
&& op_at(k + 2) == Some(0x51)
&& op_at(k + 3) == Some(0x14)
&& op_at(k + 4) == Some(0x15)
&& found(k + 5).is_some()
&& let Some(next) = at(k + 7)
{
hit = Some((next.off, k + 6, "eq-iszero-fallthrough"));
}
let direct = matches!(op_at(k + 1), Some(0x10 | 0x11)) && found(k + 2).is_some();
let via_dup = matches!(op_at(k + 1), Some(0x80..=0x8f)) && matches!(op_at(k + 2), Some(0x10 | 0x11)) && found(k + 3).is_some();
if hit.is_none() && (direct || via_dup) {
splits += 1;
}
}
let Some((dest, jumpi_k, pattern)) = hit else { continue };
if n < 4 && extraction.is_none() {
continue;
}
// Duplicate identities still advance the end of the recognized dispatcher.
// Otherwise the fallback scan starts inside a later comparison, not after it.
last_jumpi = Some(jumpi_k);
if !selectors.insert(sel) {
continue;
}
let selector = format!("0x{sel:08x}");
if pattern == "eq-jumpi" {
solc_like += 1;
} else {
vyper_like += 1;
}
let di = index_of(dest);
let jumpdest_valid = pattern != "eq-jumpi" || di.and_then(at).is_some_and(|x| x.op == 0x5b);
let (mutability, reason) = if !jumpdest_valid {
("unknown", "the destination is not a JUMPDEST".to_owned())
} else if callvalue_before {
("nonpayable", "a CALLVALUE check runs before the dispatcher, so every function refuses payments".to_string())
} else {
match di {
Some(d) => payability(body, &ins, d, &index_of),
None => ("unknown", "the destination is not an instruction boundary".to_string()),
}
};
functions.push(EvmFunction {
selector,
bytecode_offset: dest,
compare_offset: i.off,
dispatch: "abi",
pattern,
mutability,
mutability_reason: reason,
jumpdest_valid,
});
if functions.len() >= MAX_FUNCTIONS {
warnings.push(format!("stopped at {MAX_FUNCTIONS} selectors"));
break;
}
}
if functions.iter().any(|f| !f.jumpdest_valid) {
warnings.push("some destinations are not JUMPDESTs; those matches may be data, not dispatch".into());
}
let (short_calldata, receive) = short_calldata_path(body, &ins, disp_start);
let mut fallback = "unknown";
// At the selector cap the remaining dispatcher has not been scanned.
// Its next comparison is not evidence that a fallback body exists.
if let Some(j) = last_jumpi
&& functions.len() < MAX_FUNCTIONS
&& solc_like > 0
{
fallback = if reverts_soon(body, &ins, j + 1, &index_of) { "absent" } else { "present" };
}
let style = match (solc_like > 0, vyper_like > 0) {
(true, true) => "mixed",
(true, false) => "solc-like",
(false, true) => "vyper-like",
_ => "none",
};
let listing_end = last_jumpi.map(|j| j + 8).unwrap_or(LISTING_CAP).min(LISTING_CAP);
let dispatcher_listing = ins
.iter()
.take(listing_end)
.map(|i| {
let d = imm(body, i);
if d.is_empty() && push_len(i.op) == 0 {
format!("{:#06x}: {}", i.off, mnemonic(i.op))
} else {
format!("{:#06x}: {} {}", i.off, mnemonic(i.op), bytes_value(d))
}
})
.collect();
if functions.is_empty() {
warnings.push(
"no selector dispatcher was recognised (the code may be creation code without runtime, a proxy, a library, or use a jump-table dispatcher)".into(),
);
}
Ok(EvmInterface {
code_size: code.len(),
analysed_size: body.len(),
metadata,
instructions: ins.len(),
jumpdests,
selector_extraction: extraction.map(|(k, _)| k),
dispatcher_offset: extraction.and_then(|(_, k)| at(k)).map(|i| i.off),
style,
functions,
binary_search_splits: splits,
callvalue_check_before_dispatch: callvalue_before,
short_calldata_check: short_calldata,
fallback,
receive,
dispatcher_listing,
warnings,
})
}
/// Resolve the destination of a JUMP to an instruction index, when the PUSH right before it is
/// readable and lands on an instruction boundary.
fn jump_destination(code: &[u8], ins: &[Ins], k: usize, index_of: &dyn Fn(usize) -> Option<usize>) -> Option<usize> {
let push = k.checked_sub(1).and_then(|p| ins.get(p)).filter(|p| is_push(p.op))?;
index_of(usize::try_from(imm_u64(code, push)?).ok()?)
}
/// Follow a function's entry for a few instructions (and up to two unconditional jumps) looking
/// for a CALLVALUE check.
fn payability(code: &[u8], ins: &[Ins], start: usize, index_of: &dyn Fn(usize) -> Option<usize>) -> (&'static str, String) {
let mut k = start;
let mut hops = 0;
for _ in 0..48 {
let Some(i) = ins.get(k) else { break };
match i.op {
0x34 if rejects_value(code, ins, k, index_of) => return ("nonpayable", format!("rejects nonzero CALLVALUE at {:#x}", i.off)),
0x56 => match jump_destination(code, ins, k, index_of) {
Some(t) if hops < 2 && ins.get(t).is_some_and(|target| target.op == 0x5b) => {
hops += 1;
k = t;
continue;
}
_ => return ("unknown", "jumps somewhere we cannot follow before any CALLVALUE check".into()),
},
0x57 => return ("unknown", "branches before any CALLVALUE check".into()),
0xfd | 0xfe => return ("unknown", "halts with an error before any CALLVALUE check".into()),
0x00 | 0xf3 | 0xff | 0x35 | 0x37 | 0x54 | 0x55 | 0xf1 | 0xf4 | 0xfa | 0xa0..=0xa4 => {
return ("payable", format!("reaches {} at {:#x} without a CALLVALUE check", mnemonic(i.op), i.off));
}
_ => {}
}
k += 1;
}
("unknown", "no CALLVALUE check in the first instructions".into())
}
/// CALLVALUE only reads the payment. Recognize a direct test that sends the nonzero path
/// to a rejection, allowing the common DUP1 that preserves the original value for cleanup.
fn rejects_value(code: &[u8], ins: &[Ins], k: usize, index_of: &dyn Fn(usize) -> Option<usize>) -> bool {
let mut next = k + 1;
if ins.get(next).is_some_and(|i| i.op == 0x80) {
next += 1;
}
let jumps_when_zero = ins.get(next).is_some_and(|i| i.op == 0x15);
next += usize::from(jumps_when_zero);
let Some(push) = ins.get(next).filter(|i| is_push(i.op)) else { return false };
if ins.get(next + 1).is_none_or(|i| i.op != 0x57) {
return false;
}
let Some(target) = imm_u64(code, push).and_then(|v| usize::try_from(v).ok()).and_then(index_of) else { return false };
if ins.get(target).is_none_or(|i| i.op != 0x5b) {
return false;
}
reverts_soon(code, ins, if jumps_when_zero { next + 2 } else { target }, index_of)
}
fn reverts_soon(code: &[u8], ins: &[Ins], mut k: usize, index_of: &dyn Fn(usize) -> Option<usize>) -> bool {
let mut hops = 0;
for _ in 0..12 {
let Some(i) = ins.get(k) else { return false };
match i.op {
0xfd | 0xfe => return true,
0x56 => match jump_destination(code, ins, k, index_of) {
Some(t) if hops < 2 => {
hops += 1;
k = t;
continue;
}
_ => return false,
},
0x5b | 0x50 | 0x80..=0x9f | 0x5f | 0x60..=0x7f => {}
_ => return false,
}
k += 1;
}
false
}
#[cfg(test)]
mod tests {
use super::*;
use crate::test_support::formats as fx;
#[test]
fn hex_signs_are_not_hex_digits() {
let solc = |v: &str| {
let mut b = vec![0xa1, 0x64];
b.extend(b"solc");
b.push(0x60 + v.len() as u8);
b.extend(v.as_bytes());
cbor_item(&b, &mut 0, 0).unwrap()["solc"].clone()
};
assert_eq!(solc("0x+1+2+3"), "0x+1+2+3");
assert_eq!(solc("0x000008"), "0.0.8");
assert!(hex_decode("+f").is_empty());
assert_eq!(hex_decode("0f"), vec![15]);
}
/// A plausible trailer length alone must not remove instructions from the analyzed bytecode.
#[test]
fn incomplete_cbor_must_not_shorten_the_analyzed_contract() {
// A nested array declares 65 integers but supplies just 64. The top-level map
// and its length suffix otherwise look complete.
let mut short = vec![0xa1, 0x61, b'x', 0x98, 65];
short.extend([0; 64]);
let mut short_map = vec![0xa1, 0x61, b'x', 0xb8, 33];
for key in 0..32 {
short_map.extend([0x18, key, 0]);
}
for cbor in [short, short_map, vec![0xa1, 0x61, b'x', 0x61, 0xff]] {
let mut code = vec![0x00];
code.extend(&cbor);
code.extend((cbor.len() as u16).to_be_bytes());
assert!(split_metadata(&code).1.is_none());
assert_eq!(analyze(&code).unwrap().analysed_size, code.len());
}
}
#[test]
fn dispatcher_selectors_and_metadata() {
let (code, dests) = fx::evm_runtime();
let r = analyze(&code).unwrap();
assert_eq!(r.selector_extraction, Some("shr-224"));
assert_eq!(r.style, "solc-like");
let sels: Vec<_> = r.functions.iter().map(|f| f.selector.clone()).collect();
assert_eq!(sels, fx::EVM_SELECTORS.iter().map(|s| format!("0x{s:08x}")).collect::<Vec<_>>());
for (f, d) in r.functions.iter().zip(dests) {
assert_eq!(f.bytecode_offset, d);
assert!(f.jumpdest_valid);
}
assert_eq!(r.functions[0].mutability, "nonpayable");
assert_eq!(r.functions[1].mutability, "nonpayable");
assert_eq!(r.functions[2].mutability, "payable");
assert!(r.short_calldata_check);
assert_eq!(r.fallback, "absent");
let m = r.metadata.unwrap();
assert_eq!(m.compiler.as_deref(), Some("solc 0.8.19"));
assert!(m.ipfs.unwrap().starts_with("Qm"));
assert_eq!(r.analysed_size + m.length, code.len());
}
#[test]
fn hex_input_rules() {
let (code, _) = fx::evm_runtime();
let hex = fx::to_hex(&code);
assert_eq!(decode_input(format!(" {hex}\n").as_bytes(), Encoding::Auto).unwrap(), (code.clone(), "hex"));
assert_eq!(decode_input(&code, Encoding::Auto).unwrap().1, "raw");
assert!(decode_input(b"0x123", Encoding::Hex).is_err());
assert!(decode_input(b"0xzz", Encoding::Hex).is_err());
assert!(decode_input(b"", Encoding::Hex).is_err());
assert!(matches!(analyze(&[0xef, 0x00, 0x01]), Err(FormatError::Unsupported(_))));
assert_eq!(base58(&[0, 0, 1]), "112");
}
#[test]
fn vyper_style_fallthrough() {
// PUSH1 0 CALLDATALOAD PUSH1 0xe0 SHR PUSH1 0 MSTORE PUSH4 sel PUSH1 0 MLOAD EQ ISZERO PUSH2 next JUMPI <body: CALLVALUE ...>
let mut c = vec![
0x60, 0x00, 0x35, 0x60, 0xe0, 0x1c, 0x60, 0x00, 0x52, 0x63, 0x12, 0x34, 0x56, 0x78, 0x60, 0x00, 0x51, 0x14, 0x15, 0x61, 0x00, 0x20, 0x57,
];
let body = c.len();
c.extend_from_slice(&[0x34, 0x15, 0x00]);
let r = analyze(&c).unwrap();
assert_eq!(r.style, "vyper-like");
assert_eq!(r.functions[0].selector, "0x12345678");
assert_eq!(r.functions[0].bytecode_offset, body);
assert_eq!(r.functions[0].mutability, "payable", "ISZERO followed by STOP never rejects payment");
}
#[test]
fn nonpayable_needs_a_rejecting_nonzero_branch() {
fn dispatch(prefix: &[u8]) -> EvmInterface {
let mut code = prefix.to_vec();
code.extend([0x5f, 0x35, 0x60, 0xe0, 0x1c, 0x63, 1, 2, 3, 4, 0x14, 0x60]);
code.extend([(code.len() + 3) as u8, 0x57, 0x00, 0x5b, 0x00]);
analyze(&code).unwrap()
}
let read_only = dispatch(&[0x34, 0x50]); // CALLVALUE POP
assert!(!read_only.callvalue_check_before_dispatch);
assert_eq!(read_only.functions[0].mutability, "payable");
let guard = dispatch(&[0x34, 0x80, 0x15, 0x60, 9, 0x57, 0x5f, 0x80, 0xfd, 0x5b, 0x50]);
assert!(guard.callvalue_check_before_dispatch);
assert_eq!(guard.functions[0].mutability, "nonpayable");
// The other common form jumps to REVERT for a nonzero value and otherwise skips it.
let direct = dispatch(&[0x34, 0x60, 7, 0x57, 0x60, 11, 0x56, 0x5b, 0x5f, 0x80, 0xfd, 0x5b]);
assert!(direct.callvalue_check_before_dispatch);
// Reversing that conditional rejects zero-value calls, not payments.
let reversed = dispatch(&[0x34, 0x15, 0x60, 8, 0x57, 0x60, 12, 0x56, 0x5b, 0x5f, 0x80, 0xfd, 0x5b]);
assert!(!reversed.callvalue_check_before_dispatch);
// An unconditional jump bypasses the otherwise valid guard before the dispatcher.
let bypassed = dispatch(&[0x60, 12, 0x56, 0x34, 0x80, 0x15, 0x60, 12, 0x57, 0x5f, 0x80, 0xfd, 0x5b]);
assert!(!bypassed.callvalue_check_before_dispatch);
}
#[test]
fn hostile_inputs_do_not_panic() {
let (code, _) = fx::evm_runtime();
for m in crate::test_support::mutations(&code) {
let _ = analyze(&m);
}
for n in 0..code.len() {
let _ = analyze(&code[..n]);
}
for seed in 0..200 {
let n = crate::test_support::noise(seed, 300);
let _ = analyze(&n);
let _ = decode_input(&n, Encoding::Auto);
}
// A metadata length that claims everything, and nested CBOR.
let _ = analyze(&[0xa1, 0xa1, 0xa1, 0xa1, 0xa1, 0xa1, 0xa1, 0x00, 0x07]);
let _ = analyze(&[0x7f; 40]);
}
}
+63
View File
@@ -0,0 +1,63 @@
pub mod formats {
pub const EVM_SELECTORS: [u32; 3] = [0xa905_9cbb, 0x70a0_8231, 0xd0e3_0db0];
/// Runtime bytecode with a compiler-style dispatcher, a fallback that reverts, two non-payable
/// functions (CALLVALUE check), one payable function, and a CBOR metadata tail (ipfs + solc 0.8.19).
/// Returns (bytecode, jump destinations of the three functions).
pub fn evm_runtime() -> (Vec<u8>, [usize; 3]) {
let mut c: Vec<u8> = vec![0x60, 0x80, 0x60, 0x40, 0x52];
// if calldatasize < 4 goto fallback
c.extend_from_slice(&[0x60, 0x04, 0x36, 0x10, 0x61, 0, 0]);
let fb_patch = c.len() - 2;
c.push(0x57);
c.extend_from_slice(&[0x60, 0x00, 0x35, 0x60, 0xe0, 0x1c]);
let mut patches = Vec::new();
for sel in EVM_SELECTORS {
c.push(0x80);
c.push(0x63);
c.extend_from_slice(&sel.to_be_bytes());
c.extend_from_slice(&[0x14, 0x61, 0, 0]);
patches.push(c.len() - 2);
c.push(0x57);
}
let fallback = c.len();
c.extend_from_slice(&[0x5b, 0x60, 0x00, 0x80, 0xfd]);
let mut dests = [0usize; 3];
for (i, dest) in dests.iter_mut().enumerate() {
*dest = c.len();
if i < 2 {
// JUMPDEST CALLVALUE DUP1 ISZERO PUSH2 ok JUMPI PUSH1 0 DUP1 REVERT JUMPDEST POP STOP
c.extend_from_slice(&[0x5b, 0x34, 0x80, 0x15, 0x61, 0, 0]);
let ok_patch = c.len() - 2;
c.extend_from_slice(&[0x57, 0x60, 0x00, 0x80, 0xfd]);
let ok = c.len();
c.extend_from_slice(&[0x5b, 0x50, 0x00]);
c[ok_patch..ok_patch + 2].copy_from_slice(&(ok as u16).to_be_bytes());
} else {
c.extend_from_slice(&[0x5b, 0x60, 0x01, 0x60, 0x00, 0x55, 0x00]);
}
}
c[fb_patch..fb_patch + 2].copy_from_slice(&(fallback as u16).to_be_bytes());
for (p, d) in patches.iter().zip(dests) {
c[*p..*p + 2].copy_from_slice(&(d as u16).to_be_bytes());
}
c.push(0xfe);
let mut meta = vec![0xa2, 0x64, b'i', b'p', b'f', b's', 0x58, 0x22, 0x12, 0x20];
meta.extend((0..32u8).map(|i| i.wrapping_mul(7)));
meta.extend_from_slice(&[0x64, b's', b'o', b'l', b'c', 0x43, 0x00, 0x08, 0x13]);
let len = meta.len() as u16;
c.extend_from_slice(&meta);
c.extend_from_slice(&len.to_be_bytes());
(c, dests)
}
/// Bytecode as `0x`-prefixed lowercase hex.
pub fn to_hex(b: &[u8]) -> String {
let mut s = String::from("0x");
for x in b {
s.push_str(&format!("{x:02x}"));
}
s
}
}
pub use craft_testkit::sample_damage::{mutations, noise};
+39
View File
@@ -0,0 +1,39 @@
use craft_evm as evm;
use serde_json::json;
/// Unrelated three-integer arrays stay arrays while compiler versions stay readable.
#[test]
fn cbor_arrays_are_not_all_compiler_versions() {
// {"sizes": [1, 2, 3], "vyper": [0, 3, 10]}
for (cbor, compiler) in [
(b"\xa2\x65sizes\x83\x01\x02\x03\x65vyper\x83\x00\x03\x0a".as_slice(), "vyper 0.3.10"),
(b"\xa2\x65sizes\x83\x01\x02\x03\x64solc\x83\x00\x08\x13".as_slice(), "solc 0.8.19"),
] {
let mut code = vec![0x00];
code.extend(cbor);
code.extend((cbor.len() as u16).to_be_bytes());
let result = evm::analyze(&code).unwrap();
let meta = result.metadata.unwrap();
assert_eq!(meta.entries["sizes"], json!([1, 2, 3]));
assert_eq!(meta.compiler.as_deref(), Some(compiler));
assert_eq!(result.analysed_size, 1);
}
}
/// A non-version Vyper array stays typed, and a truncated array stays bytecode.
#[test]
fn invalid_version_shapes_are_not_invented_or_silently_consumed() {
let cbor = b"\xa1\x65vyper\x82\x00\x03";
let mut code = vec![0x00];
code.extend(cbor);
code.extend((cbor.len() as u16).to_be_bytes());
let meta = evm::analyze(&code).unwrap().metadata.unwrap();
assert_eq!(meta.compiler, None);
assert_eq!(meta.entries["vyper"], json!([0, 3]));
let mut truncated = code;
truncated[8] = 0x83; // Declare three items but keep only two.
let result = evm::analyze(&truncated).unwrap();
assert!(result.metadata.is_none());
assert_eq!(result.analysed_size, truncated.len());
}
+93
View File
@@ -0,0 +1,93 @@
use craft_evm::analyze;
fn contract_with_trailer(cbor: &[u8]) -> Vec<u8> {
let mut code = vec![0x5b, 0x00];
code.extend_from_slice(cbor);
code.extend_from_slice(&u16::try_from(cbor.len()).unwrap().to_be_bytes());
code
}
fn assert_unrecognized(cbor: &[u8]) {
let code = contract_with_trailer(cbor);
let result = analyze(&code).unwrap();
assert!(result.metadata.is_none(), "ambiguous map must remain bytecode: {cbor:02x?}");
assert_eq!(result.analysed_size, code.len());
}
#[test]
fn duplicate_compiler_keys_do_not_replace_the_reported_version() {
assert_unrecognized(b"\xa2\x64solc\x43\x00\x08\x13\x64solc\x43\x00\x08\x1a");
}
#[test]
fn identical_duplicate_keys_are_also_ambiguous() {
assert_unrecognized(b"\xa2\x64solc\x43\x00\x08\x13\x64solc\x43\x00\x08\x13");
}
#[test]
fn numeric_keys_cannot_collide_with_text_keys() {
assert_unrecognized(b"\xa2\x01\x00\x61\x31\x01");
}
#[test]
fn metadata_map_keys_must_be_encoded_as_text() {
for key in [vec![0x01], vec![0x41, b'x'], vec![0x83, 0, 1, 2], vec![0xf5], vec![0xf6], vec![0xa0]] {
let mut cbor = vec![0xa1];
cbor.extend(key);
cbor.push(0);
assert_unrecognized(&cbor);
}
}
#[test]
fn nested_maps_do_not_silently_overwrite_keys() {
assert_unrecognized(b"\xa1\x65extra\xa2\x61x\x00\x61x\x01");
assert_unrecognized(b"\xa1\x65extra\xa1\x01\x00");
}
#[test]
fn ordinary_compiler_metadata_and_unique_nested_text_keys_still_decode() {
let code = contract_with_trailer(b"\xa2\x64solc\x43\x00\x08\x13\x65extra\xa2\x61x\x00\x61y\x01");
let result = analyze(&code).unwrap();
assert_eq!(result.analysed_size, 2);
let metadata = result.metadata.unwrap();
assert_eq!(metadata.compiler.as_deref(), Some("solc 0.8.19"));
assert_eq!(metadata.entries["extra"]["x"], 0);
assert_eq!(metadata.entries["extra"]["y"], 1);
assert_eq!(metadata.length, code.len() - 2);
}
#[test]
fn equal_keys_in_separate_maps_are_independent() {
let code = contract_with_trailer(b"\xa2\x61a\xa1\x61x\x00\x61b\xa1\x61x\x01");
let result = analyze(&code).unwrap();
assert_eq!(result.analysed_size, 2);
let entries = result.metadata.unwrap().entries;
assert_eq!(entries["a"]["x"], 0);
assert_eq!(entries["b"]["x"], 1);
}
#[test]
fn extended_length_metadata_maps_respect_the_pair_limit() {
let map = |count: u8| {
let mut cbor = vec![0xb8, count];
for index in 0..count {
let key = format!("k{index:02}");
cbor.push(0x63);
cbor.extend_from_slice(key.as_bytes());
cbor.push(0);
}
cbor
};
for count in [24, 32] {
let code = contract_with_trailer(&map(count));
let report = analyze(&code).unwrap();
assert_eq!(report.analysed_size, 2);
assert_eq!(report.metadata.unwrap().entries.as_object().unwrap().len(), usize::from(count));
}
assert_unrecognized(&map(33));
assert_unrecognized(&[0xb8]);
let mut truncated = map(24);
truncated.pop();
assert_unrecognized(&truncated);
}
+115
View File
@@ -0,0 +1,115 @@
use craft_evm as evm;
fn prefix() -> Vec<u8> {
vec![0x5b, 0x00, 0x60, 0x00, 0x35, 0x60, 0xe0, 0x1c]
}
fn comparison(code: &mut Vec<u8>, selector: u32, width: usize, destination: u8) {
code.push(0x5f + u8::try_from(width).unwrap());
code.extend_from_slice(&selector.to_be_bytes()[4 - width..]);
code.extend([0x14, 0x60, destination, 0x57]);
}
#[test]
fn repeated_comparisons_keep_the_first_destination_and_encounter_order() {
let mut code = prefix();
comparison(&mut code, 0xfeed0002, 4, 0);
comparison(&mut code, 0xfeed0001, 4, 0);
let expected = evm::analyze(&code).unwrap().functions;
for _ in 0..2000 {
comparison(&mut code, 0xfeed0001, 4, 1);
comparison(&mut code, 0xfeed0002, 4, 1);
}
assert_eq!(evm::analyze(&code).unwrap().functions, expected);
}
#[test]
fn different_push_widths_share_the_same_numeric_selector() {
let mut code = prefix();
for width in 1..=4 {
comparison(&mut code, 17, width, if width == 1 { 0 } else { 1 });
}
let result = evm::analyze(&code).unwrap();
assert_eq!(result.functions.len(), 1);
assert_eq!(result.functions[0].selector, "0x00000011");
assert_eq!(result.functions[0].bytecode_offset, 0);
}
#[test]
fn rejected_candidates_do_not_spend_selector_identity() {
let mut code = vec![0x5b, 0x00];
// Short comparisons without selector extraction are not considered functions.
comparison(&mut code, 17, 1, 1);
// A selector-shaped immediate without a comparison is not a function either.
code.extend([0x63, 0, 0, 0, 17, 0x50]);
comparison(&mut code, 17, 4, 0);
let result = evm::analyze(&code).unwrap();
assert_eq!(result.functions.len(), 1);
assert_eq!(result.functions[0].bytecode_offset, 0);
assert!(result.functions[0].jumpdest_valid);
}
#[test]
fn selector_cap_counts_unique_matches() {
let mut code = prefix();
for selector in 0..4095 {
comparison(&mut code, selector, 4, 0);
}
for _ in 0..10000 {
comparison(&mut code, 4094, 4, 1);
}
let below = evm::analyze(&code).unwrap();
assert_eq!(below.functions.len(), 4095);
assert!(!below.warnings.iter().any(|warning| warning.contains("stopped at")));
comparison(&mut code, 4095, 4, 0);
comparison(&mut code, 4096, 4, 0);
let capped = evm::analyze(&code).unwrap();
assert_eq!(capped.functions.len(), 4096);
assert_eq!(capped.functions.last().unwrap().selector, "0x00000fff");
assert!(capped.warnings.iter().any(|warning| warning == "stopped at 4096 selectors"));
}
#[test]
fn repeated_comparisons_keep_fallback_scan_after_the_last_dispatch() {
for (ending, expected) in [(&[0x5f, 0x5f, 0xfd][..], "absent"), (&[0x00][..], "present")] {
let mut code = prefix();
comparison(&mut code, 0xfeed0001, 4, 0);
comparison(&mut code, 0xfeed0001, 4, 1);
code.extend_from_slice(ending);
let report = evm::analyze(&code).unwrap();
assert_eq!(report.functions.len(), 1);
assert_eq!(report.functions[0].bytecode_offset, 0);
assert_eq!(report.fallback, expected);
}
}
#[test]
fn selector_cap_leaves_fallback_unknown() {
for ending in [&[0x5f, 0x5f, 0xfd][..], &[0x00][..]] {
let mut code = prefix();
for selector in 0..4097 {
comparison(&mut code, selector, 4, 0);
}
code.extend_from_slice(ending);
let report = evm::analyze(&code).unwrap();
assert_eq!(report.functions.len(), 4096);
assert_eq!(report.fallback, "unknown");
}
}
#[test]
fn payability_only_follows_jumps_to_jumpdest() {
for (target, expected) in [(&[0x5b, 0x00][..], "payable"), (&[0x00][..], "unknown")] {
let mut code = prefix();
let entry = u8::try_from(code.len() + 10).unwrap();
comparison(&mut code, 0xfeed0001, 4, entry);
code.push(0x00);
assert_eq!(code.len(), usize::from(entry));
code.extend([0x5b, 0x60, entry + 4, 0x56]);
code.extend_from_slice(target);
let report = evm::analyze(&code).unwrap();
assert_eq!(report.functions.len(), 1);
assert!(report.functions[0].jumpdest_valid);
assert_eq!(report.functions[0].mutability, expected);
}
}
+442
View File
@@ -1,5 +1,7 @@
//! Header-only BMP dimensions and pixel-data coverage, without allocating a canvas.
use crate::error::CodecError;
/// Recognize a BMP file header with a supported DIB header kind.
///
/// This is a format-routing check, not full validation: dimensions and pixel data may still
@@ -13,6 +15,115 @@ pub fn is_supported_header(b: &[u8]) -> bool {
false
}
/// Wrap a DIB (a BMP without its 14-byte `BITMAPFILEHEADER`, as the Windows clipboard holds one
/// for `CF_DIB` and `CF_DIBV5`) as a BMP file the decoders here read. The pixel offset is computed
/// from the header with saturating arithmetic, so a hostile header cannot overflow; whether the
/// pixel data exists is left to the decoder. Apply [`dib_layout`] first for clipboard DIBs.
///
/// # Errors
///
/// Returns [`CodecError::LimitExceeded`] when the output (`dib.len() + 14` bytes) cannot be allocated.
pub fn dib_to_bmp(dib: &[u8]) -> Result<Vec<u8>, CodecError> {
let u16_at = |at: usize| dib.get(at..at + 2).and_then(|b| b.try_into().ok()).map(u16::from_le_bytes);
let u32_at = |at: usize| dib.get(at..at + 4).and_then(|b| b.try_into().ok()).map(u32::from_le_bytes);
let bi_size = u32_at(0).unwrap_or(40);
let offset = if bi_size == 12 {
let bpp = u16_at(10).unwrap_or(24);
let palette = if bpp <= 8 { (1u32 << bpp) * 3 } else { 0 };
14 + bi_size + palette
} else {
let bpp = u16_at(14).unwrap_or(24);
let compression = u32_at(16).unwrap_or(0);
let clr_used = u32_at(32).unwrap_or(0);
let palette_entries = if clr_used != 0 {
clr_used
} else if bpp <= 8 {
1u32 << bpp
} else {
0
};
// The masks of a 52- or 56-byte (V2/V3) header are inside the header; only the other
// header kinds carry them as 12 further bytes before the pixels.
let masks = if compression == 3 && bi_size >= 40 && !matches!(bi_size, 52 | 56) { 12 } else { 0 };
14u32.saturating_add(bi_size).saturating_add(masks).saturating_add(palette_entries.saturating_mul(4))
};
let total = dib.len().saturating_add(14);
let file_size = u32::try_from(total).unwrap_or(u32::MAX);
let mut bmp = Vec::new();
bmp.try_reserve_exact(total).map_err(|_| CodecError::LimitExceeded(format!("a {total}-byte BMP")))?;
bmp.extend_from_slice(b"BM");
bmp.extend_from_slice(&file_size.to_le_bytes());
bmp.extend_from_slice(&[0u8; 4]);
bmp.extend_from_slice(&offset.to_le_bytes());
bmp.extend_from_slice(dib);
Ok(bmp)
}
/// Put a clipboard DIB with a V4 or V5 header (108 bytes or more) in the layout the decoder reads.
/// A 32-bit one whose alpha mask is set but whose compression says `BI_RGB` is marked
/// `BI_BITFIELDS` (with the usual colour masks when it gives none), so its alpha is kept as
/// Windows programs mean it; arboard makes the same change. Bit fields then need the colour masks
/// after the header too, where Windows puts a copy and the decoder skips them; a DIB without that
/// copy gets it. Any other DIB comes back unchanged, as does one whose 12 extra bytes cannot be
/// allocated.
pub fn dib_layout(mut dib: Vec<u8>) -> Vec<u8> {
let u32_at = |d: &[u8], at: usize| d.get(at..at + 4).and_then(|b| b.try_into().ok()).map(u32::from_le_bytes);
let Some(size) = u32_at(&dib, 0).filter(|&s| s >= 108).and_then(|s| usize::try_from(s).ok()) else { return dib };
let put = |d: &mut Vec<u8>, at: usize, v: u32| {
if let Some(b) = d.get_mut(at..at + 4) {
b.copy_from_slice(&v.to_le_bytes());
}
};
// Decide everything first and mutate only after the 12-byte reservation succeeded, so a failed
// allocation returns the DIB unchanged.
let promote = dib.get(14..16) == Some(&[32, 0]) && u32_at(&dib, 16) == Some(0) && u32_at(&dib, 52) == Some(0xFF00_0000);
let default_masks = promote && (40..52).step_by(4).all(|at| u32_at(&dib, at) == Some(0));
let mut masks = dib.get(40..52).and_then(|b| <[u8; 12]>::try_from(b).ok());
if default_masks {
let mut m = [0u8; 12];
m[0..4].copy_from_slice(&0x00FF_0000u32.to_le_bytes());
m[4..8].copy_from_slice(&0x0000_FF00u32.to_le_bytes());
m[8..12].copy_from_slice(&0x0000_00FFu32.to_le_bytes());
masks = Some(m);
}
let bitfields = promote || u32_at(&dib, 16) == Some(3);
// Bytes equal to the masks right after the header are a copy only when more bytes follow than
// the pixels need; otherwise they are the pixels (e.g. a 3x1 V5 DIB with pixels equal to the masks).
let has_copy = |m: &[u8; 12]| {
dib.get(size..size.saturating_add(12)) == Some(m.as_slice()) && pixel_bytes(&dib).is_none_or(|need| dib.len().saturating_sub(size) > need)
};
let insert = match masks {
Some(m) if bitfields && size <= dib.len() && !has_copy(&m) => Some(m),
_ => None,
};
if insert.is_some() && dib.try_reserve(12).is_err() {
return dib;
}
if promote {
put(&mut dib, 16, 3);
if default_masks {
put(&mut dib, 40, 0x00FF_0000);
put(&mut dib, 44, 0x0000_FF00);
put(&mut dib, 48, 0x0000_00FF);
}
}
if let Some(m) = insert {
dib.splice(size..size, m);
}
dib
}
/// Bytes of pixel rows (4-byte aligned stride) the info-header dimensions of `dib` call for, or
/// `None` when width, height or bit count cannot be read.
fn pixel_bytes(dib: &[u8]) -> Option<usize> {
let i32_at = |at: usize| dib.get(at..at + 4).and_then(|b| b.try_into().ok()).map(i32::from_le_bytes);
let w = u64::try_from(i32_at(4)?).ok()?;
let h = u64::from(i32_at(8)?.unsigned_abs());
let bpp = u64::from(u16::from_le_bytes(dib.get(14..16)?.try_into().ok()?));
let stride = w.checked_mul(bpp)?.div_ceil(32).checked_mul(4)?;
usize::try_from(stride.checked_mul(h)?).ok()
}
/// Coverage of the declared pixel array. Row byte counts include row padding; RLE coverage
/// counts the pixels drawn by commands and may be smaller than the canvas.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
@@ -174,4 +285,335 @@ mod tests {
b[0] = b'X';
assert!(!is_supported_header(&b));
}
// The WordCraft originals, kept verbatim as the oracle.
fn oracle_dib_to_bmp(dib: &[u8]) -> Vec<u8> {
let u16_at = |at: usize| dib.get(at..at + 2).and_then(|b| b.try_into().ok()).map(u16::from_le_bytes);
let u32_at = |at: usize| dib.get(at..at + 4).and_then(|b| b.try_into().ok()).map(u32::from_le_bytes);
let bi_size = u32_at(0).unwrap_or(40);
let offset = if bi_size == 12 {
let bpp = u16_at(10).unwrap_or(24);
let palette = if bpp <= 8 { (1u32 << bpp) * 3 } else { 0 };
14 + bi_size + palette
} else {
let bpp = u16_at(14).unwrap_or(24);
let compression = u32_at(16).unwrap_or(0);
let clr_used = u32_at(32).unwrap_or(0);
let palette_entries = if clr_used != 0 {
clr_used
} else if bpp <= 8 {
1u32 << bpp
} else {
0
};
let masks = if compression == 3 && bi_size >= 40 { 12 } else { 0 };
14u32.saturating_add(bi_size).saturating_add(masks).saturating_add(palette_entries.saturating_mul(4))
};
let file_size = u32::try_from(14 + dib.len()).unwrap_or(u32::MAX);
let mut bmp = Vec::with_capacity(14 + dib.len());
bmp.extend_from_slice(b"BM");
bmp.extend_from_slice(&file_size.to_le_bytes());
bmp.extend_from_slice(&[0u8; 4]);
bmp.extend_from_slice(&offset.to_le_bytes());
bmp.extend_from_slice(dib);
bmp
}
fn oracle_dib_layout(mut dib: Vec<u8>) -> Vec<u8> {
let u32_at = |d: &[u8], at: usize| d.get(at..at + 4).and_then(|b| b.try_into().ok()).map(u32::from_le_bytes);
let Some(size) = u32_at(&dib, 0).filter(|&s| s >= 108).and_then(|s| usize::try_from(s).ok()) else { return dib };
let put = |d: &mut Vec<u8>, at: usize, v: u32| {
if let Some(b) = d.get_mut(at..at + 4) {
b.copy_from_slice(&v.to_le_bytes());
}
};
if dib.get(14..16) == Some(&[32, 0]) && u32_at(&dib, 16) == Some(0) && u32_at(&dib, 52) == Some(0xFF00_0000) {
put(&mut dib, 16, 3);
if (40..52).step_by(4).all(|at| u32_at(&dib, at) == Some(0)) {
put(&mut dib, 40, 0x00FF_0000);
put(&mut dib, 44, 0x0000_FF00);
put(&mut dib, 48, 0x0000_00FF);
}
}
if u32_at(&dib, 16) == Some(3)
&& size <= dib.len()
&& let Some(masks) = dib.get(40..52).map(<[u8]>::to_vec)
&& dib.get(size..size + 12) != Some(masks.as_slice())
{
dib.splice(size..size, masks);
}
dib
}
fn core(bpp: u16) -> Vec<u8> {
let mut d = Vec::new();
d.extend_from_slice(&12u32.to_le_bytes());
d.extend_from_slice(&2u16.to_le_bytes());
d.extend_from_slice(&2u16.to_le_bytes());
d.extend_from_slice(&1u16.to_le_bytes());
d.extend_from_slice(&bpp.to_le_bytes());
d.resize(12 + 64, 7);
d
}
#[allow(clippy::too_many_arguments)]
fn info(size: u32, bpp: u16, comp: u32, clr_used: u32, masks: [u32; 4], tail: usize) -> Vec<u8> {
let mut d = vec![0u8; size.clamp(40, 124) as usize];
d[0..4].copy_from_slice(&size.to_le_bytes());
d[4..8].copy_from_slice(&2i32.to_le_bytes());
d[8..12].copy_from_slice(&2i32.to_le_bytes());
d[12..14].copy_from_slice(&1u16.to_le_bytes());
d[14..16].copy_from_slice(&bpp.to_le_bytes());
d[16..20].copy_from_slice(&comp.to_le_bytes());
d[32..36].copy_from_slice(&clr_used.to_le_bytes());
for (i, m) in masks.iter().enumerate() {
if let Some(b) = d.get_mut(40 + 4 * i..44 + 4 * i) {
b.copy_from_slice(&m.to_le_bytes());
}
}
d.extend((0..tail).map(|i| (i * 31 + 5) as u8));
d
}
fn corpus() -> Vec<Vec<u8>> {
let mut v = Vec::new();
for bpp in [1, 4, 8, 24, 0, 99] {
v.push(core(bpp));
}
for bpp in [1u16, 4, 8, 16, 24, 32, 0, 255] {
for clr in [0u32, 1, 5, 256, u32::MAX] {
v.push(info(40, bpp, 0, clr, [0; 4], 40));
}
}
for (size, comp, masks) in [
(40, 3, [0xFF0000, 0xFF00, 0xFF, 0]),
(40, 3, [0; 4]),
(52, 3, [0xFF0000, 0xFF00, 0xFF, 0]),
(108, 3, [0xFF0000, 0xFF00, 0xFF, 0xFF00_0000]),
(108, 0, [0xFF0000, 0xFF00, 0xFF, 0xFF00_0000]),
(108, 0, [0, 0, 0, 0xFF00_0000]),
(108, 0, [0; 4]),
(124, 3, [0xFF0000, 0xFF00, 0xFF, 0xFF00_0000]),
(124, 0, [0, 0, 0, 0xFF00_0000]),
(124, 0, [1, 0, 0, 0xFF00_0000]),
(124, 0, [0xFF0000, 0xFF00, 0xFF, 0]),
(124, 1, [0, 0, 0, 0xFF00_0000]),
(124, 3, [1, 2, 3, 4]),
(0, 0, [0; 4]),
(12, 0, [0; 4]),
(u32::MAX, 3, [0; 4]),
(u32::MAX, 0, [0; 4]),
(200, 3, [9; 4]),
] {
for bpp in [8u16, 16, 32] {
for clr in [0u32, 2, u32::MAX] {
v.push(info(size, bpp, comp, clr, masks, 48));
}
}
}
// 52/56-byte bitfields headers with and without a pixel tail, and 3x1 V5 DIBs.
for size in [52, 56] {
v.push(info(size, 32, 3, 0, [0xFF0000, 0xFF00, 0xFF, 0xFF00_0000], 4));
v.push(info(size, 32, 3, 0, [0xFF0000, 0xFF00, 0xFF, 0xFF00_0000], 0));
}
v.push(v5_3x1(false));
v.push(v5_3x1(true));
v
}
/// 3x1 32-bit V5 BI_BITFIELDS whose three pixels equal the colour masks; with `copy` the DIB
/// also holds the real 12-byte mask copy before the pixels.
fn v5_3x1(copy: bool) -> Vec<u8> {
let masks = [0x00FF_0000u32, 0x0000_FF00, 0x0000_00FF];
let mut d = info(124, 32, 3, 0, [masks[0], masks[1], masks[2], 0xFF00_0000], 0);
d[4..8].copy_from_slice(&3i32.to_le_bytes());
d[8..12].copy_from_slice(&1i32.to_le_bytes());
for _ in 0..usize::from(copy) + 1 {
for m in masks {
d.extend_from_slice(&m.to_le_bytes());
}
}
d
}
fn dib_u32(d: &[u8], at: usize) -> Option<u32> {
d.get(at..at + 4).and_then(|b| b.try_into().ok()).map(u32::from_le_bytes)
}
/// Fixed case 1: a 52- or 56-byte BI_BITFIELDS header holds its masks itself, so `dib_to_bmp`
/// no longer adds the 12 appended-mask bytes the WordCraft original added to the pixel offset.
fn bmp_offset_fixed(dib: &[u8]) -> bool {
matches!(dib_u32(dib, 0).unwrap_or(40), 52 | 56) && dib_u32(dib, 16) == Some(3)
}
/// Fixed case 2: the original read 12 pixel bytes equal to the masks (exact-length payload) as a
/// mask copy and inserted nothing. Returns the corrected output when `cut` is such a case.
fn exact_length_fixed(cut: &[u8]) -> Option<Vec<u8>> {
let o = oracle_dib_layout(cut.to_vec());
let size = usize::try_from(dib_u32(&o, 0).filter(|&s| s >= 108)?).ok()?;
let need = pixel_bytes(&o)?;
let same = o.len() == cut.len()
&& dib_u32(&o, 16) == Some(3)
&& size <= o.len()
&& o.get(size..size + 12).is_some()
&& o.get(size..size + 12) == o.get(40..52)
&& o.len() - size <= need;
if !same {
return None;
}
let mut want = o.clone();
want.splice(size..size, o[40..52].to_vec());
Some(want)
}
#[test]
fn dib_helpers_match_the_wordcraft_originals_byte_for_byte() {
let (mut total, mut offset_cases, mut exact_cases) = (0usize, 0usize, 0usize);
for dib in corpus() {
for n in 0..=dib.len() {
let cut = &dib[..n];
total += 1;
let fixed_layout = exact_length_fixed(cut);
match &fixed_layout {
Some(want) => {
exact_cases += 1;
assert_eq!(&dib_layout(cut.to_vec()), want, "exact-length layout len {n}");
}
None => assert_eq!(dib_layout(cut.to_vec()), oracle_dib_layout(cut.to_vec()), "layout len {n}"),
}
if bmp_offset_fixed(cut) {
offset_cases += 1;
let (new, old) = (dib_to_bmp(cut).unwrap(), oracle_dib_to_bmp(cut));
assert_eq!(new[..10], old[..10]);
assert_eq!(new[14..], old[14..]);
let (a, b) = (dib_u32(&new, 10).unwrap(), dib_u32(&old, 10).unwrap());
assert_eq!(a.saturating_add(12), b, "offset len {n}");
} else {
assert_eq!(dib_to_bmp(cut).unwrap(), oracle_dib_to_bmp(cut), "bmp len {n}");
}
let both = dib_layout(cut.to_vec());
let old_both = oracle_dib_layout(cut.to_vec());
if fixed_layout.is_none() && !bmp_offset_fixed(&both) {
assert_eq!(dib_to_bmp(&both).unwrap(), oracle_dib_to_bmp(&old_both));
}
}
}
eprintln!("corpus cases {total}: 52/56-byte bitfields offset {offset_cases}, exact-length payload {exact_cases}");
assert!(offset_cases > 0 && exact_cases > 0);
}
#[test]
fn dib_helpers_survive_pseudo_random_bytes() {
let mut x = 0x9E37_79B9_7F4A_7C15u64;
for len in 0..300usize {
let d: Vec<u8> = (0..len)
.map(|_| {
x ^= x << 13;
x ^= x >> 7;
x ^= x << 17;
x as u8
})
.collect();
let laid = dib_layout(d.clone());
if exact_length_fixed(&d).is_none() && !bmp_offset_fixed(&laid) {
assert_eq!(dib_to_bmp(&laid).unwrap(), oracle_dib_to_bmp(&oracle_dib_layout(d)));
}
}
}
#[test]
fn huge_palette_count_saturates_the_offset() {
let b = dib_to_bmp(&info(40, 8, 0, u32::MAX, [0; 4], 8)).unwrap();
assert_eq!(u32::from_le_bytes([b[10], b[11], b[12], b[13]]), u32::MAX);
let b = dib_to_bmp(&info(u32::MAX, 8, 3, u32::MAX, [0; 4], 8)).unwrap();
assert_eq!(u32::from_le_bytes([b[10], b[11], b[12], b[13]]), u32::MAX);
}
#[test]
fn wrapped_dibs_decode_and_keep_alpha() {
// 2x2 32-bit V5, BI_RGB with an alpha mask, bottom-up BGRA pixels.
let mut d = info(124, 32, 0, 0, [0, 0, 0, 0xFF00_0000], 0);
let px = [[1u8, 2, 3, 255], [4, 5, 6, 128], [7, 8, 9, 64], [10, 11, 12, 0]];
for p in px {
d.extend_from_slice(&p);
}
let bmp = dib_to_bmp(&dib_layout(d)).unwrap();
let img = crate::decode(&bmp).unwrap();
let alphas: Vec<u16> = img.to_rgba8().chunks(4).map(|p| u16::from(p[3])).collect();
assert_eq!(alphas, [64, 0, 255, 128]);
// 24-bit plain DIB and a 12-byte core 8-bit one open too.
let mut d = info(40, 24, 0, 0, [0; 4], 0);
d.extend_from_slice(&[0, 0, 255, 0, 255, 0, 0, 0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]);
assert!(crate::decode(&dib_to_bmp(&d).unwrap()).is_ok());
let mut c = core(8);
c.truncate(12);
c.extend((0..256 * 3).map(|i| i as u8));
c.extend_from_slice(&[0, 1, 0, 0, 2, 3, 0, 0]);
assert!(crate::decode(&dib_to_bmp(&c).unwrap()).is_ok());
}
/// 1x1 32-bit BI_BITFIELDS DIB, one pixel B=10 G=20 R=30 A=128, with the given header size.
fn one_pixel(size: u32) -> Vec<u8> {
let mut d = info(size, 32, 3, 0, [0xFF0000, 0xFF00, 0xFF, 0xFF00_0000], 0);
d[4..8].copy_from_slice(&1i32.to_le_bytes());
d[8..12].copy_from_slice(&1i32.to_le_bytes());
if size == 40 {
for m in [0xFF0000u32, 0xFF00, 0xFF] {
d.extend_from_slice(&m.to_le_bytes());
}
}
d.extend_from_slice(&[10, 20, 30, 128]);
d
}
#[test]
fn bitfields_pixel_offset_for_each_header_size_decodes() {
for size in [40u32, 52, 56] {
let bmp = dib_to_bmp(&one_pixel(size)).unwrap();
let off = 14 + size + if size == 40 { 12 } else { 0 };
assert_eq!(dib_u32(&bmp, 10), Some(off), "offset for {size}");
let img = crate::decode(&bmp).unwrap_or_else(|e| panic!("{size}-byte header: {e:?}"));
let rgba = img.to_rgba8();
assert_eq!(&rgba[..3], &[30, 20, 10], "size {size}");
if size == 56 {
assert_eq!(rgba[3], 128);
}
}
// V5: the header's masks, a copy inserted by `dib_layout`, then the pixel.
let bmp = dib_to_bmp(&dib_layout(one_pixel(124))).unwrap();
assert_eq!(&crate::decode(&bmp).unwrap().to_rgba8()[..4], &[30, 20, 10, 128]);
}
#[test]
fn exact_length_pixels_equal_to_the_masks_are_not_a_mask_copy() {
let d = v5_3x1(false);
let laid = dib_layout(d.clone());
assert_eq!(laid.len(), d.len() + 12);
assert_eq!(laid[124..136], d[40..52]);
assert_eq!(oracle_dib_layout(d).len(), laid.len() - 12, "the original skipped the insertion");
let img = crate::decode(&dib_to_bmp(&laid).unwrap()).unwrap().to_rgba8();
assert_eq!(img.len(), 12);
// pixel 0 is 0x00FF0000 (blue 0, green 0, red 255 under the masks).
assert_eq!(&img[..3], &[255, 0, 0]);
// A real copy followed by the pixels is left alone.
let real = v5_3x1(true);
assert_eq!(dib_layout(real.clone()), real);
assert!(crate::decode(&dib_to_bmp(&real).unwrap()).is_ok());
}
#[test]
fn layout_decides_before_it_mutates() {
// BI_RGB V5 with an alpha mask and no colour masks: promoted and given a copy together.
let mut d = info(124, 32, 0, 0, [0, 0, 0, 0xFF00_0000], 16);
d[4..8].copy_from_slice(&2i32.to_le_bytes());
let laid = dib_layout(d.clone());
assert_eq!(laid.len(), d.len() + 12);
assert_eq!(dib_u32(&laid, 16), Some(3));
// Unreadable dimensions keep the byte-equality rule.
let mut odd = info(108, 32, 3, 0, [1, 2, 3, 0], 0);
odd[4..8].copy_from_slice(&(-1i32).to_le_bytes());
let copy = odd[40..52].to_vec();
odd.extend_from_slice(&copy);
assert_eq!(dib_layout(odd.clone()), odd);
}
}
+337 -14
View File
@@ -46,6 +46,45 @@ pub(crate) struct JpegMeta {
pub held_coefficients: u64,
}
/// Validate a JPEG by decoding the 8-bit RGBA output used by renderers.
///
/// This deliberately uses the existing zune backend's default decode policy.
/// Unlike general JPEG import it does not fall back to additional JPEG modes,
/// apply orientation, or retain pixels. The caller supplies the admitted pixel count.
pub fn validate_jpeg_rgba8(bytes: &[u8], max_pixels: u64) -> Result<(), CodecError> {
validate_jpeg_rgba8_with_panic_policy(bytes, max_pixels, cfg!(panic = "abort"))
}
fn validate_jpeg_rgba8_with_panic_policy(bytes: &[u8], max_pixels: u64, panic_aborts: bool) -> Result<(), CodecError> {
// The decoder can assert while upsampling these frames. On abort builds the
// unwind guard below cannot turn that assertion into an image omission.
if panic_aborts && first_component_subsampled(bytes) {
return Err(CodecError::malformed(F, "JPEG first component is subsampled below the frame maximum"));
}
let decode = || {
let options = DecoderOptions::default().jpeg_set_out_colorspace(ColorSpace::RGBA);
let mut decoder = zune_jpeg::JpegDecoder::new_with_options(std::io::Cursor::new(bytes), options);
decoder.decode_headers().map_err(|error| CodecError::malformed(F, format_args!("{error:?}")))?;
let info = decoder.info().ok_or_else(|| CodecError::malformed(F, "missing frame dimensions"))?;
let pixels = u64::from(info.width) * u64::from(info.height);
if pixels == 0 {
return Err(CodecError::malformed(F, "empty frame"));
}
if pixels > max_pixels {
return Err(CodecError::LimitExceeded("JPEG exceeds the admitted pixel count".into()));
}
let length = usize::try_from(pixels * 4).map_err(|_| CodecError::LimitExceeded("JPEG RGBA output is too large".into()))?;
if decoder.output_colorspace() != Some(ColorSpace::RGBA) || decoder.output_buffer_size() != Some(length) {
return Err(CodecError::malformed(F, "unexpected RGBA output"));
}
let mut output = Vec::new();
output.try_reserve_exact(length).map_err(|_| CodecError::LimitExceeded("could not allocate JPEG RGBA output".into()))?;
output.resize(length, 0);
decoder.decode_into(&mut output).map_err(|error| CodecError::malformed(F, format_args!("{error:?}")))
};
std::panic::catch_unwind(decode).map_err(|_| CodecError::malformed(F, "JPEG decoder failed on this frame"))?
}
/// The first frame of `b` as [`crate::JpegInfo`], from its markers alone (see [`crate::jpeg_info`]).
pub(crate) fn info(b: &[u8]) -> Option<crate::JpegInfo> {
let m = scan_metadata(b);
@@ -86,6 +125,83 @@ fn held_coefficients(marker: u8, seg: &[u8], scan_components: Option<u8>) -> u64
factors.iter().map(|&(fh, fv)| mcus * fh * fv * 64 * 2).sum()
}
/// Bytes of DCT coefficients to budget for before decoding hostile `b`: the larger of
/// [`crate::JpegInfo::held_coefficients`]'s rule (first frame; SOF2, or SOF0/SOF1 whose first
/// scan lacks components; 8-bit only; sampling factors as written) and a coarser one that
/// keeps a progressive JPEG's coefficients for the whole picture:
/// - every SOF2, SOF6, SOF10 and SOF14 header before the first scan counts, at any precision, and
/// the largest wins; each charges 2 bytes a coefficient: per component, `2 * 64 * h * v` per
/// MCU, with `h` and `v` clamped to 1..=4 and the MCUs covering the frame at the largest `h`, `v`;
/// - a header that can't be followed (a segment that is cut, a length under 2, a frame header
/// without its components, no `0xFF` where a marker belongs) charges the costliest frame,
/// four components sampled 4 × 4, at the first frame's size (as `scan_metadata` reads it), and 0 without one.
///
/// Never less than either rule. Frames `craft-jpeg` decodes (arithmetic, lossless, 12-bit)
/// enforce `max_alloc` on their own buffers, and SOF6/SOF14 (hierarchical) are not decoded at all,
/// so charging them is only a conservative budget. Arithmetic saturates; the scan is linear.
pub(crate) fn worst_case_coefficients(b: &[u8]) -> u64 {
let m = scan_metadata(b);
let mut at = 2usize;
let mut most = 0u64;
let first: Option<(u64, u64)> = m.frame.map(|(w, h, _)| (u64::from(w), u64::from(h)));
let mut broken = false;
while !broken {
if b.get(at) != Some(&0xFF) {
broken = true;
break;
}
while b.get(at) == Some(&0xFF) {
at += 1;
}
let Some(&code) = b.get(at) else {
broken = true;
break;
};
at += 1;
match code {
0xD8 | 0x01 | 0xD0..=0xD7 => continue,
0xD9 | 0xDA => break,
_ => {}
}
let len = match (b.get(at), b.get(at + 1)) {
(Some(&hi), Some(&lo)) => usize::from(u16::from_be_bytes([hi, lo])),
_ => 0,
};
let Some(seg) = (len >= 2).then(|| b.get(at + 2..at + len)).flatten() else {
broken = true;
break;
};
if matches!(code, 0xC2 | 0xC6 | 0xCA | 0xCE) {
match progressive_frame_coefficients(seg) {
Some(held) => most = most.max(held),
None => {
broken = true;
break;
}
}
}
at += len;
}
if broken && let Some((w, h)) = first {
let mcus = w.div_ceil(32).saturating_mul(h.div_ceil(32));
most = most.max(mcus.saturating_mul(2 * 64 * 16 * 4));
}
most.max(m.held_coefficients)
}
/// [`worst_case_coefficients`]'s charge for one frame header `sof` (after its length).
fn progressive_frame_coefficients(sof: &[u8]) -> Option<u64> {
let height = u64::from(u16::from_be_bytes([*sof.get(1)?, *sof.get(2)?]));
let width = u64::from(u16::from_be_bytes([*sof.get(3)?, *sof.get(4)?]));
let n = usize::from(*sof.get(5)?);
let comps = sof.get(6..6usize.checked_add(n.checked_mul(3)?)?)?;
let factors: Vec<(u64, u64)> =
comps.as_chunks::<3>().0.iter().map(|&[_, f, _]| (u64::from(f >> 4).clamp(1, 4), u64::from(f & 15).clamp(1, 4))).collect();
let (hmax, vmax) = factors.iter().fold((1, 1), |(h, v), &(a, b)| (h.max(a), v.max(b)));
let mcus = width.div_ceil(8 * hmax).saturating_mul(height.div_ceil(8 * vmax));
Some(factors.iter().fold(0u64, |sum, &(h, v)| sum.saturating_add(mcus.saturating_mul(2 * 64 * h * v))))
}
/// Walk the marker segments up to SOS and collect metadata.
pub(crate) fn scan_metadata(b: &[u8]) -> JpegMeta {
let mut m = JpegMeta::default();
@@ -340,10 +456,17 @@ fn first_component_subsampled(b: &[u8]) -> bool {
return false;
}
let mut i = 2;
while let (Some(&0xFF), Some(&m)) = (b.get(i), b.get(i + 1)) {
while let Some(&prefix) = b.get(i) {
// The decoder searches forward between segments. Stray bytes and
// stuffed FF 00 pairs must not hide a frame from the abort-build guard.
if prefix != 0xFF {
i += 1;
continue;
}
let Some(&m) = b.get(i + 1) else { return false };
match m {
0xFF => i += 1,
0x01 | 0xD0..=0xD8 => i += 2,
0x00 | 0x01 | 0xD0..=0xD8 => i += 2,
0xD9 | 0xDA => return false,
_ => {
let Some(len) = b.get(i + 2..i + 4).map(|l| usize::from(u16::from_be_bytes([l[0], l[1]]))) else { return false };
@@ -825,6 +948,170 @@ mod tests {
assert_eq!(held(&frame(0xC2, 12, &[0x11, 0x11, 0x11], None)), Some(0));
}
/// The consumer's own estimate (WordCraft's `jpeg_coefficients`), ported as the oracle that
/// `worst_case_coefficients` must never undercut.
fn oracle(bytes: &[u8]) -> u64 {
fn worst(bytes: &[u8]) -> u64 {
let Some((w, h)) = info(bytes).map(|i| (i.width, i.height)) else { return 0 };
let mcus = u64::from(w).div_ceil(32) * u64::from(h).div_ceil(32);
mcus.saturating_mul(2 * 64 * 16 * 4)
}
fn frame(sof: &[u8]) -> Option<u64> {
let height = u64::from(u16::from_be_bytes([*sof.get(1)?, *sof.get(2)?]));
let width = u64::from(u16::from_be_bytes([*sof.get(3)?, *sof.get(4)?]));
let n = usize::from(*sof.get(5)?);
let s: Vec<(u64, u64)> = sof
.get(6..6 + 3 * n)?
.as_chunks::<3>()
.0
.iter()
.map(|&[_, s, _]| (u64::from(s >> 4).clamp(1, 4), u64::from(s & 15).clamp(1, 4)))
.collect();
let (hm, vm) = s.iter().fold((1, 1), |(h, v), &(a, b)| (h.max(a), v.max(b)));
let mcus = width.div_ceil(8 * hm).checked_mul(height.div_ceil(8 * vm))?;
s.iter().try_fold(0u64, |sum, &(h, v)| sum.checked_add(mcus.checked_mul(2 * 64 * h * v)?))
}
let mut at = 2;
let mut most = 0u64;
loop {
if bytes.get(at) != Some(&0xFF) {
return worst(bytes).max(most);
}
while bytes.get(at) == Some(&0xFF) {
at += 1;
}
let Some(&code) = bytes.get(at) else { return worst(bytes).max(most) };
at += 1;
match code {
0xD8 | 0x01 | 0xD0..=0xD7 => continue,
0xD9 | 0xDA => return most,
_ => {}
}
let len = match (bytes.get(at), bytes.get(at + 1)) {
(Some(&hi), Some(&lo)) => usize::from(u16::from_be_bytes([hi, lo])),
_ => 0,
};
let Some(seg) = bytes.get(at + 2..at + len).filter(|_| len >= 2) else { return worst(bytes).max(most) };
if matches!(code, 0xC2 | 0xC6 | 0xCA | 0xCE) {
let Some(h) = frame(seg) else { return worst(bytes).max(most) };
most = most.max(h);
}
at += len;
}
}
/// A JPEG header: each frame is (marker, precision, width, height, sampling bytes), then an
/// optional scan of `scan` components, then EOI.
fn header(frames: &[(u8, u8, u16, u16, &[u8])], scan: Option<u8>) -> Vec<u8> {
let mut b = vec![0xFF, 0xD8];
for &(marker, precision, w, h, f) in frames {
b.extend_from_slice(&[0xFF, marker]);
b.extend_from_slice(&(8 + 3 * f.len() as u16).to_be_bytes());
b.push(precision);
b.extend_from_slice(&h.to_be_bytes());
b.extend_from_slice(&w.to_be_bytes());
b.push(f.len() as u8);
for (i, &s) in f.iter().enumerate() {
b.extend_from_slice(&[i as u8 + 1, s, 0]);
}
}
if let Some(n) = scan {
b.extend_from_slice(&[0xFF, 0xDA, 0, 6 + 2 * n, n]);
(1..=n).for_each(|c| b.extend_from_slice(&[c, 0]));
b.extend_from_slice(&[0, 63, 0]);
}
b.extend_from_slice(&[0xFF, 0xD9]);
b
}
/// Never less than the oracle or `held_coefficients`; equal to the larger on valid headers.
#[test]
fn worst_case_coefficients_covers_both_rule_sets() {
const C3: &[u8] = &[0x22, 0x11, 0x11];
const G: &[u8] = &[0x11];
let cases: Vec<Vec<u8>> = vec![
header(&[(0xC0, 8, 640, 480, C3)], Some(3)),
header(&[(0xC0, 8, 640, 480, G)], Some(1)),
header(&[(0xC0, 8, 640, 480, C3)], Some(1)),
header(&[(0xC1, 8, 640, 480, C3)], None),
header(&[(0xC2, 8, 640, 480, C3)], Some(1)),
header(&[(0xC2, 12, 640, 480, C3)], Some(1)),
header(&[(0xC6, 8, 640, 480, C3)], None),
header(&[(0xCA, 8, 640, 480, C3)], None),
header(&[(0xCE, 8, 640, 480, C3)], None),
header(&[(0xC0, 8, 64, 64, G), (0xC2, 8, 6000, 4000, C3), (0xC2, 8, 100, 100, C3)], Some(1)),
header(&[(0xC2, 8, 640, 480, &[0x00, 0x00, 0x00])], None),
header(&[(0xC2, 8, 640, 480, &[0xFF, 0xFF, 0xFF, 0xFF])], None),
header(&[(0xC2, 8, 65535, 65535, &[0x44; 4])], None),
header(&[(0xC0, 8, 65535, 65535, &[0xFF; 4])], Some(1)),
header(&[], None),
vec![0xFF, 0xD8],
vec![],
b"not a jpeg at all".to_vec(),
];
for b in &cases {
let held = scan_metadata(b).held_coefficients;
let got = worst_case_coefficients(b);
assert!(got >= oracle(b) && got >= held, "{b:02x?}");
assert!(got >= info(b).map_or(0, |i| i.held_coefficients));
}
let at = |i: usize| worst_case_coefficients(&cases[i]);
// Equal to the oracle where the shared rule charges nothing extra.
for i in [0, 1, 6, 7, 8, 9, 10, 12, 14] {
assert_eq!(at(i), oracle(&cases[i]), "case {i}");
}
// Where it does, the shared figure wins (the oracle misses a short first scan).
assert_eq!(oracle(&cases[2]), 0);
assert_eq!(at(2), scan_metadata(&cases[2]).held_coefficients);
// Unclamped factors (15 × 15) charge the shared rule more than the clamped oracle.
assert_eq!(at(11), scan_metadata(&cases[11]).held_coefficients);
assert!(at(11) > oracle(&cases[11]));
assert!(at(2) > 0 && at(3) == 0 && at(0) == 0);
// 12-bit and arithmetic progressive frames are charged though craft-jpeg limits itself.
assert_eq!(at(5), at(4));
// Sampling factors clamp: 0 acts as 1, 15 as 4.
assert_eq!(at(10), 80 * 60 * 3 * 128);
assert_eq!(at(12), 2048 * 2048 * 4 * 2 * 64 * 16);
// The largest of several frames wins.
assert_eq!(at(9), oracle(&header(&[(0xC2, 8, 6000, 4000, C3)], None)));
}
/// A cut header charges the costliest frame at its size; every prefix and random damage stays
/// bounded, never panics and never undercuts the oracle.
#[test]
fn worst_case_coefficients_hostile_input() {
let full = header(&[(0xC2, 8, 6000, 4000, &[0x22, 0x11, 0x11])], Some(1));
let cut = &full[..full.len() - 8];
// Cut inside the component list: a frame the scan can't read, at its own size.
let sof_cut = &full[..14];
assert_eq!(worst_case_coefficients(sof_cut), oracle(sof_cut));
assert_eq!(worst_case_coefficients(cut), oracle(cut));
let mut bases = vec![full.clone(), header(&[(0xC0, 8, 6000, 4000, &[0x11; 3])], Some(1))];
bases.push(header(&[(0xC2, 8, 65535, 65535, &[0xFF; 4])], None));
let mut seed = 0x9E37_79B9_7F4A_7C15u64;
let mut next = move || {
seed = seed.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1_442_695_040_888_963_407);
(seed >> 33) as usize
};
for base in &bases {
for n in 0..=base.len() {
let b = &base[..n];
let got = worst_case_coefficients(b);
assert!(got >= oracle(b) && got < 1 << 50);
}
for _ in 0..2000 {
let mut b = base.clone();
for _ in 0..=next() % 4 {
let i = next() % b.len();
b[i] = next() as u8;
}
b.truncate(next() % (b.len() + 1));
let got = worst_case_coefficients(&b);
assert!(got >= oracle(&b) && got >= scan_metadata(&b).held_coefficients && got < 1 << 50);
}
}
}
/// An APP1 Extended XMP fragment (GUID, total size, offset, data).
fn extended_xmp(offset: u32) -> Vec<u8> {
let mut seg = EXTENDED_XMP_HEADER.to_vec();
@@ -1073,25 +1360,61 @@ mod tests {
(v * 65535.0 / 4095.0).round() as u16
}
fn progressive_frame_header(comps: &[(u8, u8)]) -> Vec<u8> {
let mut s = vec![0xFF, 0xD8, 0xFF, 0xE0, 0, 4, 0, 0, 0xFF, 0xC2, 0, (8 + 3 * comps.len()) as u8, 8, 0, 8, 0, 8, comps.len() as u8];
for (i, &(h, v)) in comps.iter().enumerate() {
s.extend([i as u8 + 1, (h << 4) | v, 0]);
}
s
}
#[test]
fn first_component_subsampling() {
let sof = |comps: &[(u8, u8)]| {
let mut s = vec![0xFF, 0xD8, 0xFF, 0xE0, 0, 4, 0, 0, 0xFF, 0xC2, 0, (8 + 3 * comps.len()) as u8, 8, 0, 8, 0, 8, comps.len() as u8];
for (i, &(h, v)) in comps.iter().enumerate() {
s.extend([i as u8 + 1, (h << 4) | v, 0]);
}
s
};
assert!(first_component_subsampled(&sof(&[(1, 1), (1, 1), (1, 1), (2, 2)])));
assert!(first_component_subsampled(&sof(&[(1, 2), (1, 2), (2, 1)])));
assert!(!first_component_subsampled(&sof(&[(2, 2), (1, 1), (1, 1)])));
assert!(!first_component_subsampled(&sof(&[(1, 1); 4])));
assert!(!first_component_subsampled(&sof(&[])));
assert!(first_component_subsampled(&progressive_frame_header(&[(1, 1), (1, 1), (1, 1), (2, 2)])));
assert!(first_component_subsampled(&progressive_frame_header(&[(1, 2), (1, 2), (2, 1)])));
assert!(!first_component_subsampled(&progressive_frame_header(&[(2, 2), (1, 1), (1, 1)])));
assert!(!first_component_subsampled(&progressive_frame_header(&[(1, 1); 4])));
assert!(!first_component_subsampled(&progressive_frame_header(&[])));
assert!(!first_component_subsampled(&[0xFF, 0xD8, 0xFF, 0xDA, 0, 2]));
assert!(!first_component_subsampled(&[0xFF, 0xD8, 0xFF, 0xC0, 0, 20, 8]));
assert!(!first_component_subsampled(b"not a jpeg"));
}
#[test]
fn renderer_validation_refuses_unsafe_sampling_under_abort_policy() {
// The frame header alone exercises refusal before any pixel decode.
let header = progressive_frame_header(&[(1, 1), (1, 1), (1, 1), (2, 2)]);
let refuses_sampling = |result| {
matches!(
result,
Err(CodecError::Malformed { format: Format::Jpeg, message })
if message == "JPEG first component is subsampled below the frame maximum"
)
};
assert!(refuses_sampling(validate_jpeg_rgba8_with_panic_policy(&header, 64, true)));
assert!(!refuses_sampling(validate_jpeg_rgba8_with_panic_policy(&header, 64, false)));
let mut ordinary = Vec::new();
jpeg_encoder::Encoder::new(&mut ordinary, 90).encode(&[128], 1, 1, jpeg_encoder::ColorType::Luma).unwrap();
validate_jpeg_rgba8_with_panic_policy(&ordinary, 1, true).unwrap();
}
#[test]
fn jpeg_sampling_guard_finds_frame_after_marker_padding() {
let header = progressive_frame_header(&[(1, 1), (1, 1), (1, 1), (2, 2)]);
// Exercise the shared header guard, without a pixel payload.
for split in [2, 8] {
let (before, after) = header.split_at(split);
for padding in [&[0][..], &[0x7F, 0][..], &[0xFF, 0][..], &[0xFF][..]] {
let padded = [before, padding, after].concat();
assert!(first_component_subsampled(&padded), "offset {split}, padding {padding:?}");
let got = validate_jpeg_rgba8_with_panic_policy(&padded, 64, true);
assert!(matches!(got, Err(CodecError::Malformed { message, .. }) if message.contains("frame maximum")));
}
}
assert!(!first_component_subsampled(&[0xFF, 0xD8, 0, 0xFF]));
}
#[test]
fn routing() {
assert!(!uses_craft_jpeg(&[0xFF, 0xD8, 0xFF, 0xC0, 0, 11, 8, 0, 8, 0, 8, 1, 1, 0x11, 0]));
+10
View File
@@ -45,6 +45,7 @@ pub use crate::cicp::ContentLightLevel;
pub use crate::codecs::avif::{HDR_REFERENCE_WHITE_NITS, HdrTransfer, encode_avif_hdr, encode_avif_with_speed};
pub use crate::codecs::exr_cryptomatte::{cryptomatte_id, cryptomatte_key, cryptomatte_preview_color};
pub use crate::codecs::gif_frames::{AnimationFrame, MAX_GIF_FRAMES};
pub use crate::codecs::jpeg::validate_jpeg_rgba8;
pub use crate::codecs::png::{
PngDecodeRows, PngExtras, PngHeader, PngPalette, PngRowSink, PngRows, SharedBytes, encode_indexed as encode_png_indexed,
encode_pq as encode_png_pq, palette as png_palette, stereo as png_stereo,
@@ -307,6 +308,15 @@ pub fn jpeg_info(bytes: &[u8]) -> Option<JpegInfo> {
jpeg::info(bytes)
}
/// Bytes of DCT coefficients to budget for a JPEG before decoding it: never less than
/// [`JpegInfo::held_coefficients`] nor than the whole-picture estimate of every progressive
/// frame header (SOF2, SOF6, SOF10, SOF14 at any precision, sampling clamped to 1..=4), and the
/// costliest frame at the first size read when the header can't be followed. 0 without a
/// readable frame. See the function body's rules for the precise charges.
pub fn jpeg_worst_case_coefficients(bytes: &[u8]) -> u64 {
jpeg::worst_case_coefficients(bytes)
}
/// PCX's dimensions are available from its fixed-size header without decoding the RLE rows.
pub fn pcx_dimensions(bytes: &[u8]) -> Option<(u32, u32)> {
codecs::pcx::dimensions(bytes)
+223
View File
@@ -136,6 +136,93 @@ pub fn color_chunks(b: &[u8]) -> Option<ColorChunks> {
Some(result)
}
/// Most chunks [`pixel_model`] walks before the image data; each chunk is at least 12 bytes.
const MAX_PROBE_CHUNKS: usize = 1 << 16;
/// Pixel model of a PNG, from `IHDR` and the chunk headers before the first `IDAT`.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct PixelModel {
pub width: u32,
pub height: u32,
/// `IHDR` bit depth (1, 2, 4, 8 or 16).
pub bit_depth: u8,
/// `IHDR` colour type (0 grey, 2 RGB, 3 palette, 4 grey+alpha, 6 RGBA).
pub color_type: u8,
/// A `tRNS` chunk precedes the image data; it adds alpha to grey, RGB and palette images.
pub has_trns: bool,
}
impl PixelModel {
/// Channels of the decoded buffer (palette expands to RGB; `tRNS` adds an alpha channel).
pub fn channels(&self) -> u8 {
let base = match self.color_type {
0 => 1,
2 | 3 => 3,
4 => 2,
_ => 4,
};
base + u8::from(self.has_trns && matches!(self.color_type, 0 | 2 | 3))
}
/// Bytes per pixel of the buffer a decoder that expands sub-byte samples to 8 bits gives:
/// `channels * (2 if bit_depth == 16 else 1)`.
pub fn bytes_per_pixel(&self) -> u8 {
self.channels() * if self.bit_depth == 16 { 2 } else { 1 }
}
/// Whether the image has transparency (alpha colour type or `tRNS`).
pub fn has_alpha(&self) -> bool {
matches!(self.color_type, 4 | 6) || self.has_trns
}
}
/// Dimensions, bit depth, colour type and `tRNS` presence without inflating anything (an `iCCP`
/// profile is skipped, not decompressed). Walks at most 65,536 chunk headers and stops at `IDAT`;
/// `IEND` or the end of the input before `IDAT` is an error. Does not validate CRCs.
/// A caller gets its `(width, height, bytes_per_pixel, alpha)` as
/// `(m.width, m.height, u64::from(m.bytes_per_pixel()), m.has_alpha())`.
pub fn pixel_model(b: &[u8]) -> Result<PixelModel, CodecError> {
if !b.starts_with(b"\x89PNG\r\n\x1a\n") {
return Err(err("bad signature"));
}
let be32 = |p: usize| b.get(p..p.checked_add(4)?).and_then(|v| <[u8; 4]>::try_from(v).ok()).map(u32::from_be_bytes);
if be32(8) != Some(13) || b.get(12..16) != Some(b"IHDR") {
return Err(err("missing IHDR"));
}
let ihdr = b.get(16..29).ok_or_else(|| err("truncated IHDR"))?;
let (width, height) = (be32(16).unwrap_or(0), be32(20).unwrap_or(0));
let (bit_depth, color_type) = (ihdr[8], ihdr[9]);
if width == 0 || height == 0 || width > i32::MAX as u32 || height > i32::MAX as u32 {
return Err(err("dimensions out of range"));
}
let depth_ok = match color_type {
0 => matches!(bit_depth, 1 | 2 | 4 | 8 | 16),
3 => matches!(bit_depth, 1 | 2 | 4 | 8),
2 | 4 | 6 => matches!(bit_depth, 8 | 16),
_ => false,
};
if !depth_ok {
return Err(err("invalid IHDR"));
}
let mut has_trns = false;
let mut p = 8usize;
for _ in 0..MAX_PROBE_CHUNKS {
let len = be32(p).ok_or_else(|| err("truncated"))?;
let kind = p.checked_add(4).and_then(|k| b.get(k..k.checked_add(4)?)).ok_or_else(|| err("truncated"))?;
match kind {
b"IDAT" => return Ok(PixelModel { width, height, bit_depth, color_type, has_trns }),
b"IEND" => return Err(err("no image data")),
b"tRNS" => has_trns = true,
_ => {}
}
if len > i32::MAX as u32 {
return Err(err("chunk length out of range"));
}
p = p.checked_add(12).and_then(|q| q.checked_add(len as usize)).ok_or_else(|| err("truncated"))?;
}
Err(CodecError::LimitExceeded("PNG chunk count".into()))
}
#[cfg(test)]
mod tests {
use super::*;
@@ -187,4 +274,140 @@ mod tests {
assert!(!color.srgb);
assert!(color_chunks(b"not png").is_none());
}
/// WordCraft's `png_chunks` (crates/geom/src/picture.rs) as the differential oracle.
fn oracle(bytes: &[u8]) -> Option<(u32, u32, u64, bool)> {
let be = |at: usize| bytes.get(at..at.checked_add(4)?).and_then(|b| <[u8; 4]>::try_from(b).ok()).map(u32::from_be_bytes);
if bytes.get(12..16) != Some(b"IHDR".as_slice()) {
return None;
}
let (width, height) = (be(16)?, be(20)?);
let (depth, color) = (*bytes.get(24)?, *bytes.get(25)?);
let mut trns = false;
let mut at = 8usize;
loop {
let len = usize::try_from(be(at)?).ok()?;
match bytes.get(at.checked_add(4)?..at.checked_add(8)?)? {
b"IDAT" => break,
b"IEND" => return None,
b"tRNS" => trns = true,
_ => {}
}
at = at.checked_add(12)?.checked_add(len)?;
}
let (channels, alpha) = match color {
0 => (1 + u64::from(trns), trns),
2 | 3 => (3 + u64::from(trns), trns),
4 => (2, true),
6 => (4, true),
_ => return None,
};
Some((width, height, channels * if depth == 16 { 2 } else { 1 }, alpha))
}
fn tuple(b: &[u8]) -> Option<(u32, u32, u64, bool)> {
pixel_model(b).ok().map(|m| (m.width, m.height, u64::from(m.bytes_per_pixel()), m.has_alpha()))
}
fn png_with(depth: u8, color: u8, before_idat: &[(&[u8; 4], Vec<u8>)]) -> Vec<u8> {
let mut bytes = b"\x89PNG\r\n\x1a\n".to_vec();
let mut ihdr = Vec::new();
ihdr.extend_from_slice(&5u32.to_be_bytes());
ihdr.extend_from_slice(&7u32.to_be_bytes());
ihdr.extend_from_slice(&[depth, color, 0, 0, 0]);
chunk(&mut bytes, b"IHDR", &ihdr);
for (k, d) in before_idat {
chunk(&mut bytes, k, d);
}
chunk(&mut bytes, b"IDAT", &[1, 2, 3]);
chunk(&mut bytes, b"IEND", &[]);
bytes
}
#[test]
fn pixel_model_matches_oracle_for_every_color_type_depth_and_trns() {
let valid: [(u8, &[u8]); 5] = [(0, &[1, 2, 4, 8, 16]), (2, &[8, 16]), (3, &[1, 2, 4, 8]), (4, &[8, 16]), (6, &[8, 16])];
for (color, depths) in valid {
for &depth in depths {
for trns in [false, true] {
let extra: Vec<(&[u8; 4], Vec<u8>)> = if trns { vec![(b"tRNS", vec![0, 0])] } else { vec![] };
let bytes = png_with(depth, color, &extra);
assert!(tuple(&bytes).is_some(), "color {color} depth {depth}");
assert_eq!(tuple(&bytes), oracle(&bytes), "color {color} depth {depth} trns {trns}");
}
}
}
}
#[test]
fn pixel_model_known_values() {
let m = pixel_model(&png_with(16, 2, &[(b"tRNS", vec![0; 6])])).unwrap();
assert_eq!((m.width, m.height, m.channels(), m.bytes_per_pixel(), m.has_alpha()), (5, 7, 4, 8, true));
let m = pixel_model(&png_with(4, 3, &[])).unwrap();
assert_eq!((m.bytes_per_pixel(), m.has_alpha()), (3, false));
}
#[test]
fn pixel_model_skips_a_huge_iccp_without_inflating() {
let junk = vec![0xAB; 20 << 20];
let bytes = png_with(8, 2, &[(b"iCCP", junk), (b"tRNS", vec![0; 6])]);
assert_eq!(tuple(&bytes), oracle(&bytes));
assert_eq!(tuple(&bytes), Some((5, 7, 4, true)));
}
#[test]
fn pixel_model_malformed_inputs_agree_with_oracle_and_never_panic() {
let full = png_with(8, 6, &[(b"tRNS", vec![0]), (b"gAMA", vec![0; 4])]);
for n in 0..=full.len() {
let cut = &full[..n];
let got = tuple(cut);
// an Err where the oracle answers can only be a stricter header check
if got.is_some() {
assert_eq!(got, oracle(cut), "prefix {n}");
}
}
assert!(pixel_model(&full[..33]).is_err());
// missing IHDR
let mut bad = full.clone();
bad[12..16].copy_from_slice(b"IHDX");
assert!(pixel_model(&bad).is_err() && oracle(&bad).is_none());
// chunk length overflow: a length past the input, then one past i32::MAX
for len in [u32::MAX, i32::MAX as u32, 0x7fff_fff0] {
let mut o = full.clone();
o[33..37].copy_from_slice(&len.to_be_bytes());
assert!(pixel_model(&o).is_err(), "len {len:#x}");
assert_eq!(tuple(&o), oracle(&o));
}
// IEND before IDAT, unknown colour type, zero size, bad signature
let mut iend = png_with(8, 2, &[]);
let at = iend.windows(4).position(|w| w == b"IDAT").unwrap();
iend[at..at + 4].copy_from_slice(b"IEND");
assert!(pixel_model(&iend).is_err() && oracle(&iend).is_none());
assert!(pixel_model(&png_with(8, 5, &[])).is_err());
let mut zero = png_with(8, 2, &[]);
zero[16..20].copy_from_slice(&0u32.to_be_bytes());
assert!(pixel_model(&zero).is_err());
assert!(pixel_model(b"not a png").is_err());
// deterministic pseudo-random corruption
let mut s = 0x9e37_79b9u32;
for _ in 0..2000 {
let mut f = full.clone();
for _ in 0..3 {
s = s.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
let i = (s >> 8) as usize % f.len();
f[i] = (s >> 24) as u8;
}
let _ = pixel_model(&f);
}
}
#[test]
fn pixel_model_many_tiny_chunks_and_the_chunk_cap() {
let many: Vec<(&[u8; 4], Vec<u8>)> = (0..10_000).map(|_| (b"tEXt", Vec::new())).chain([(b"tRNS", vec![0])]).collect();
let bytes = png_with(8, 0, &many);
assert_eq!(tuple(&bytes), oracle(&bytes));
assert_eq!(tuple(&bytes), Some((5, 7, 2, true)));
let over: Vec<(&[u8; 4], Vec<u8>)> = (0..MAX_PROBE_CHUNKS + 1).map(|_| (b"tEXt", Vec::new())).collect();
assert!(matches!(pixel_model(&png_with(8, 0, &over)), Err(CodecError::LimitExceeded(_))));
}
}
+54
View File
@@ -32,6 +32,25 @@ pub fn header(bytes: &[u8]) -> Result<(u32, u32, u16), CodecError> {
Ok((w, h, orientation))
}
/// Whether a WebP carries transparency, from the first chunk only (nothing is decoded):
/// `VP8X` flag bit 0x10, `VP8L` `alpha_is_used`, plain `VP8` never.
pub fn has_alpha(bytes: &[u8]) -> Result<bool, CodecError> {
let err = |s: &str| CodecError::malformed(Format::WebP, s);
if bytes.get(..4) != Some(b"RIFF") || bytes.get(8..12) != Some(b"WEBP") {
return Err(err("bad signature"));
}
match bytes.get(12..16) {
Some(b"VP8X") => Ok(bytes.get(20).ok_or_else(|| err("truncated VP8X"))? & 0x10 != 0),
// 0x2f signature byte, then 14+14 size bits, then the alpha_is_used bit (bit 28)
Some(b"VP8L") => match bytes.get(20..25) {
Some([0x2f, a, b, c, d]) => Ok(u32::from_le_bytes([*a, *b, *c, *d]) >> 28 & 1 != 0),
_ => Err(err("bad VP8L header")),
},
Some(b"VP8 ") => Ok(false),
_ => Err(err("no image data")),
}
}
#[cfg(test)]
mod tests {
use super::*;
@@ -71,4 +90,39 @@ mod tests {
bytes.push(0);
assert!(header(&bytes).is_err());
}
#[test]
fn has_alpha_reads_vp8x_vp8l_and_plain_vp8() {
for (layout, want) in [(crate::ChannelLayout::Rgb, false), (crate::ChannelLayout::Rgba, true)] {
let n = layout.channels();
let image = crate::Image::from_u8(2, 3, layout, vec![42; 6 * n]).unwrap();
let bytes = crate::encode(&image, Format::WebP, &crate::EncodeOptions::default()).unwrap();
assert_eq!(has_alpha(&bytes).unwrap(), want);
}
let riff = |chunk: &[u8; 4], body: &[u8]| {
let mut v = b"RIFF\0\0\0\0WEBP".to_vec();
v.extend_from_slice(chunk);
v.extend_from_slice(&(body.len() as u32).to_le_bytes());
v.extend_from_slice(body);
v
};
let mut x = [0u8; 10];
assert!(!has_alpha(&riff(b"VP8X", &x)).unwrap());
x[0] = 0x10;
assert!(has_alpha(&riff(b"VP8X", &x)).unwrap());
assert!(!has_alpha(&riff(b"VP8 ", &[0; 10])).unwrap());
assert!(has_alpha(&riff(b"VP8L", &[0x2f, 0, 0, 0, 0x10])).unwrap());
assert!(!has_alpha(&riff(b"VP8L", &[0x2f, 0, 0, 0, 0])).unwrap());
}
#[test]
fn has_alpha_rejects_malformed_prefixes() {
let full = [b"RIFF\0\0\0\0WEBPVP8X\x0a\0\0\0".as_slice(), &[0x10; 10]].concat();
for n in 0..20 {
assert!(has_alpha(&full[..n]).is_err(), "prefix {n}");
}
assert!(has_alpha(&full[..21]).is_ok());
assert!(has_alpha(b"RIFF\0\0\0\0WEBPVP8L\x05\0\0\0\x2e").is_err());
assert!(has_alpha(b"junk").is_err());
}
}
+8 -3
View File
@@ -76,9 +76,11 @@ fn truncated_files_mostly_error() {
#[test]
fn bit_flips_never_panic() {
// CRAFT_HOSTILE_DIVISOR=N (pull requests) runs 1/N of the flips.
let rounds = 150usize.div_ceil(craft_media_testkit::hostile::divisor());
let mut rng = common::Rng::new(42);
for (f, bytes) in samples() {
for _ in 0..150 {
for _ in 0..rounds {
let mut b = bytes.clone();
let flips = 1 + (rng.next_u64() % 4) as usize;
for _ in 0..flips {
@@ -92,8 +94,11 @@ fn bit_flips_never_panic() {
#[test]
fn byte_overwrites_in_header_never_panic() {
for (f, bytes) in samples() {
for i in 0..bytes.len().min(64) {
// CRAFT_HOSTILE_DIVISOR=N (pull requests) overwrites every N-th byte, from a different offset
// for each sample, so the samples together still reach every header position.
let step = craft_media_testkit::hostile::divisor();
for (k, (f, bytes)) in samples().into_iter().enumerate() {
for i in (0..bytes.len().min(64)).filter(|i| (i + k) % step == 0) {
for v in [0x00, 0x7F, 0xFF] {
let mut b = bytes.clone();
b[i] = v;
+7 -4
View File
@@ -820,15 +820,18 @@ fn poke(bytes: &[u8]) {
#[test]
fn zoo_decodes_and_never_panics_when_cut_or_corrupted() {
for f in zoo() {
// CRAFT_HOSTILE_DIVISOR=N (pull requests) runs every N-th cut and flip, from a different offset
// for each file, and 1/N of the random rounds.
let step = craft_media_testkit::hostile::divisor();
for (k, f) in zoo().into_iter().enumerate() {
let info = tiff_info(&f).unwrap();
for i in 0..info.pages.len() {
decode_tiff_page(&f, i, &DecodeOptions::default()).unwrap();
}
for cut in 0..f.len() {
for cut in (0..f.len()).filter(|cut| (cut + k) % step == 0) {
poke(&f[..cut]);
}
for i in 0..f.len() {
for i in (0..f.len()).filter(|i| (i + k) % step == 0) {
for flip in [0xffu8, 0x80, 0x01] {
let mut g = f.clone();
g[i] ^= flip;
@@ -836,7 +839,7 @@ fn zoo_decodes_and_never_panics_when_cut_or_corrupted() {
}
}
let mut r = Rng::new(f.len() as u64);
for _ in 0..300 {
for _ in 0..300usize.div_ceil(step) {
let mut g = f.clone();
for _ in 0..4 {
let i = (r.next_u64() as usize) % g.len();
+15
View File
@@ -0,0 +1,15 @@
[package]
name="craft-java-class"
description="Java class-file inspection and JVM bytecode decoding with retained bounds and partial reports"
version.workspace=true
edition.workspace=true
license.workspace=true
rust-version.workspace=true
repository.workspace=true
[lints]
workspace=true
[dependencies]
serde={workspace=true}
thiserror={workspace=true}
[dev-dependencies]
craft-testkit = { workspace = true }
+51
View File
@@ -0,0 +1,51 @@
//! Checked little/big-endian reads over untrusted bytes. Every read returns `None` when out of range.
#[derive(Debug, Clone, Copy)]
pub struct Reader<'a> {
data: &'a [u8],
}
impl<'a> Reader<'a> {
pub fn new(data: &'a [u8]) -> Self {
Reader { data }
}
pub fn slice(&self, off: usize, len: usize) -> Option<&'a [u8]> {
let end = off.checked_add(len)?;
self.data.get(off..end)
}
pub fn u8(&self, off: usize) -> Option<u8> {
self.data.get(off).copied()
}
fn arr<const N: usize>(&self, off: usize) -> Option<[u8; N]> {
self.slice(off, N)?.try_into().ok()
}
pub fn u16_be(&self, off: usize) -> Option<u16> {
self.arr(off).map(u16::from_be_bytes)
}
pub fn u32_be(&self, off: usize) -> Option<u32> {
self.arr(off).map(u32::from_be_bytes)
}
pub fn u64_be(&self, off: usize) -> Option<u64> {
self.arr(off).map(u64::from_be_bytes)
}
}
/// Decode one modified UTF-8 code unit. Invalid bytes consume only themselves,
/// so a damaged sequence cannot hide valid text or borrow missing bytes.
pub(crate) fn mutf8_unit(bytes: &[u8]) -> (u16, usize) {
match bytes {
[a @ 0x00..=0x7f, ..] => (u16::from(*a), 1),
[0xc0, 0x80, ..] => (0, 2),
[a @ 0xc2..=0xdf, b @ 0x80..=0xbf, ..] => ((u16::from(*a & 0x1f) << 6) | u16::from(*b & 0x3f), 2),
[a @ 0xe0..=0xef, b @ 0x80..=0xbf, c @ 0x80..=0xbf, ..] if *a != 0xe0 || *b >= 0xa0 => {
((u16::from(*a & 0x0f) << 12) | (u16::from(*b & 0x3f) << 6) | u16::from(*c & 0x3f), 3)
}
_ => (0xfffd, 1),
}
}
+19
View File
@@ -0,0 +1,19 @@
// Descriptor display helper retained from the source DEX reader.
pub fn java_name(desc: &str) -> String {
let dims = desc.chars().take_while(|&c| c == '[').count();
let base = desc.get(dims..).unwrap_or("");
let name = match base {
"V" => "void".to_string(),
"Z" => "boolean".to_string(),
"B" => "byte".to_string(),
"S" => "short".to_string(),
"C" => "char".to_string(),
"I" => "int".to_string(),
"J" => "long".to_string(),
"F" => "float".to_string(),
"D" => "double".to_string(),
b if b.starts_with('L') && b.ends_with(';') => b.get(1..b.len() - 1).unwrap_or("").replace('/', "."),
b => b.to_string(),
};
format!("{name}{}", "[]".repeat(dims.min(255)))
}
File diff suppressed because it is too large. Load diff
+711
View File
@@ -0,0 +1,711 @@
//! JVM bytecode: the opcode table (every defined opcode plus the `wide`, `tableswitch` and
//! `lookupswitch` variable-length forms) and a decoder for each instruction. Pure decoding;
//! constant-pool index operands are resolved by the caller (see [`crate::java_class`]).
use serde::Serialize;
/// The shape of an opcode's operands.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
pub enum Operands {
/// No operands.
None,
/// One signed byte immediate (`bipush`).
Byte,
/// One signed short immediate (`sipush`).
Short,
/// A one-byte local-variable index.
Local,
/// `iinc`: a one-byte local index and a signed byte increment.
LocalConst,
/// A two-byte signed branch offset.
Branch2,
/// A four-byte signed branch offset (`goto_w`, `jsr_w`).
Branch4,
/// A one-byte constant-pool index (`ldc`).
Cp1,
/// A two-byte constant-pool index.
Cp2,
/// `newarray`: a one-byte primitive array type.
NewArray,
/// `invokeinterface`: a two-byte index, a count byte and a zero byte.
InvokeInterface,
/// `invokedynamic`: a two-byte index and two zero bytes.
InvokeDynamic,
/// `multianewarray`: a two-byte class index and a dimension byte.
MultiANewArray,
/// `tableswitch` (variable length, 4-byte aligned).
TableSwitch,
/// `lookupswitch` (variable length, 4-byte aligned).
LookupSwitch,
/// `wide`: a prefix that widens the following load/store/ret/iinc.
Wide,
/// A reserved or unknown opcode.
Reserved,
}
/// One opcode slot.
#[derive(Debug, Clone, Copy)]
pub struct Op {
pub name: &'static str,
pub operands: Operands,
}
const fn op(name: &'static str, operands: Operands) -> Op {
Op { name, operands }
}
/// The opcode table entry for `b`.
pub fn opcode(b: u8) -> Op {
use Operands as O;
match b {
0x00 => op("nop", O::None),
0x01 => op("aconst_null", O::None),
0x02 => op("iconst_m1", O::None),
0x03 => op("iconst_0", O::None),
0x04 => op("iconst_1", O::None),
0x05 => op("iconst_2", O::None),
0x06 => op("iconst_3", O::None),
0x07 => op("iconst_4", O::None),
0x08 => op("iconst_5", O::None),
0x09 => op("lconst_0", O::None),
0x0a => op("lconst_1", O::None),
0x0b => op("fconst_0", O::None),
0x0c => op("fconst_1", O::None),
0x0d => op("fconst_2", O::None),
0x0e => op("dconst_0", O::None),
0x0f => op("dconst_1", O::None),
0x10 => op("bipush", O::Byte),
0x11 => op("sipush", O::Short),
0x12 => op("ldc", O::Cp1),
0x13 => op("ldc_w", O::Cp2),
0x14 => op("ldc2_w", O::Cp2),
0x15 => op("iload", O::Local),
0x16 => op("lload", O::Local),
0x17 => op("fload", O::Local),
0x18 => op("dload", O::Local),
0x19 => op("aload", O::Local),
0x1a => op("iload_0", O::None),
0x1b => op("iload_1", O::None),
0x1c => op("iload_2", O::None),
0x1d => op("iload_3", O::None),
0x1e => op("lload_0", O::None),
0x1f => op("lload_1", O::None),
0x20 => op("lload_2", O::None),
0x21 => op("lload_3", O::None),
0x22 => op("fload_0", O::None),
0x23 => op("fload_1", O::None),
0x24 => op("fload_2", O::None),
0x25 => op("fload_3", O::None),
0x26 => op("dload_0", O::None),
0x27 => op("dload_1", O::None),
0x28 => op("dload_2", O::None),
0x29 => op("dload_3", O::None),
0x2a => op("aload_0", O::None),
0x2b => op("aload_1", O::None),
0x2c => op("aload_2", O::None),
0x2d => op("aload_3", O::None),
0x2e => op("iaload", O::None),
0x2f => op("laload", O::None),
0x30 => op("faload", O::None),
0x31 => op("daload", O::None),
0x32 => op("aaload", O::None),
0x33 => op("baload", O::None),
0x34 => op("caload", O::None),
0x35 => op("saload", O::None),
0x36 => op("istore", O::Local),
0x37 => op("lstore", O::Local),
0x38 => op("fstore", O::Local),
0x39 => op("dstore", O::Local),
0x3a => op("astore", O::Local),
0x3b => op("istore_0", O::None),
0x3c => op("istore_1", O::None),
0x3d => op("istore_2", O::None),
0x3e => op("istore_3", O::None),
0x3f => op("lstore_0", O::None),
0x40 => op("lstore_1", O::None),
0x41 => op("lstore_2", O::None),
0x42 => op("lstore_3", O::None),
0x43 => op("fstore_0", O::None),
0x44 => op("fstore_1", O::None),
0x45 => op("fstore_2", O::None),
0x46 => op("fstore_3", O::None),
0x47 => op("dstore_0", O::None),
0x48 => op("dstore_1", O::None),
0x49 => op("dstore_2", O::None),
0x4a => op("dstore_3", O::None),
0x4b => op("astore_0", O::None),
0x4c => op("astore_1", O::None),
0x4d => op("astore_2", O::None),
0x4e => op("astore_3", O::None),
0x4f => op("iastore", O::None),
0x50 => op("lastore", O::None),
0x51 => op("fastore", O::None),
0x52 => op("dastore", O::None),
0x53 => op("aastore", O::None),
0x54 => op("bastore", O::None),
0x55 => op("castore", O::None),
0x56 => op("sastore", O::None),
0x57 => op("pop", O::None),
0x58 => op("pop2", O::None),
0x59 => op("dup", O::None),
0x5a => op("dup_x1", O::None),
0x5b => op("dup_x2", O::None),
0x5c => op("dup2", O::None),
0x5d => op("dup2_x1", O::None),
0x5e => op("dup2_x2", O::None),
0x5f => op("swap", O::None),
0x60 => op("iadd", O::None),
0x61 => op("ladd", O::None),
0x62 => op("fadd", O::None),
0x63 => op("dadd", O::None),
0x64 => op("isub", O::None),
0x65 => op("lsub", O::None),
0x66 => op("fsub", O::None),
0x67 => op("dsub", O::None),
0x68 => op("imul", O::None),
0x69 => op("lmul", O::None),
0x6a => op("fmul", O::None),
0x6b => op("dmul", O::None),
0x6c => op("idiv", O::None),
0x6d => op("ldiv", O::None),
0x6e => op("fdiv", O::None),
0x6f => op("ddiv", O::None),
0x70 => op("irem", O::None),
0x71 => op("lrem", O::None),
0x72 => op("frem", O::None),
0x73 => op("drem", O::None),
0x74 => op("ineg", O::None),
0x75 => op("lneg", O::None),
0x76 => op("fneg", O::None),
0x77 => op("dneg", O::None),
0x78 => op("ishl", O::None),
0x79 => op("lshl", O::None),
0x7a => op("ishr", O::None),
0x7b => op("lshr", O::None),
0x7c => op("iushr", O::None),
0x7d => op("lushr", O::None),
0x7e => op("iand", O::None),
0x7f => op("land", O::None),
0x80 => op("ior", O::None),
0x81 => op("lor", O::None),
0x82 => op("ixor", O::None),
0x83 => op("lxor", O::None),
0x84 => op("iinc", O::LocalConst),
0x85 => op("i2l", O::None),
0x86 => op("i2f", O::None),
0x87 => op("i2d", O::None),
0x88 => op("l2i", O::None),
0x89 => op("l2f", O::None),
0x8a => op("l2d", O::None),
0x8b => op("f2i", O::None),
0x8c => op("f2l", O::None),
0x8d => op("f2d", O::None),
0x8e => op("d2i", O::None),
0x8f => op("d2l", O::None),
0x90 => op("d2f", O::None),
0x91 => op("i2b", O::None),
0x92 => op("i2c", O::None),
0x93 => op("i2s", O::None),
0x94 => op("lcmp", O::None),
0x95 => op("fcmpl", O::None),
0x96 => op("fcmpg", O::None),
0x97 => op("dcmpl", O::None),
0x98 => op("dcmpg", O::None),
0x99 => op("ifeq", O::Branch2),
0x9a => op("ifne", O::Branch2),
0x9b => op("iflt", O::Branch2),
0x9c => op("ifge", O::Branch2),
0x9d => op("ifgt", O::Branch2),
0x9e => op("ifle", O::Branch2),
0x9f => op("if_icmpeq", O::Branch2),
0xa0 => op("if_icmpne", O::Branch2),
0xa1 => op("if_icmplt", O::Branch2),
0xa2 => op("if_icmpge", O::Branch2),
0xa3 => op("if_icmpgt", O::Branch2),
0xa4 => op("if_icmple", O::Branch2),
0xa5 => op("if_acmpeq", O::Branch2),
0xa6 => op("if_acmpne", O::Branch2),
0xa7 => op("goto", O::Branch2),
0xa8 => op("jsr", O::Branch2),
0xa9 => op("ret", O::Local),
0xaa => op("tableswitch", O::TableSwitch),
0xab => op("lookupswitch", O::LookupSwitch),
0xac => op("ireturn", O::None),
0xad => op("lreturn", O::None),
0xae => op("freturn", O::None),
0xaf => op("dreturn", O::None),
0xb0 => op("areturn", O::None),
0xb1 => op("return", O::None),
0xb2 => op("getstatic", O::Cp2),
0xb3 => op("putstatic", O::Cp2),
0xb4 => op("getfield", O::Cp2),
0xb5 => op("putfield", O::Cp2),
0xb6 => op("invokevirtual", O::Cp2),
0xb7 => op("invokespecial", O::Cp2),
0xb8 => op("invokestatic", O::Cp2),
0xb9 => op("invokeinterface", O::InvokeInterface),
0xba => op("invokedynamic", O::InvokeDynamic),
0xbb => op("new", O::Cp2),
0xbc => op("newarray", O::NewArray),
0xbd => op("anewarray", O::Cp2),
0xbe => op("arraylength", O::None),
0xbf => op("athrow", O::None),
0xc0 => op("checkcast", O::Cp2),
0xc1 => op("instanceof", O::Cp2),
0xc2 => op("monitorenter", O::None),
0xc3 => op("monitorexit", O::None),
0xc4 => op("wide", O::Wide),
0xc5 => op("multianewarray", O::MultiANewArray),
0xc6 => op("ifnull", O::Branch2),
0xc7 => op("ifnonnull", O::Branch2),
0xc8 => op("goto_w", O::Branch4),
0xc9 => op("jsr_w", O::Branch4),
0xca => op("breakpoint", O::None),
0xfe => op("impdep1", O::None),
0xff => op("impdep2", O::None),
_ => op("unknown", O::Reserved),
}
}
/// The primitive array type for a `newarray` atype operand.
pub fn array_type(atype: i32) -> &'static str {
match atype {
4 => "boolean",
5 => "char",
6 => "float",
7 => "double",
8 => "byte",
9 => "short",
10 => "int",
11 => "long",
_ => "?",
}
}
/// A decoded `tableswitch`/`lookupswitch`.
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct Switch {
/// Absolute byte offset of the default branch.
pub default: i64,
/// `(key, absolute target)` pairs in listing order.
pub pairs: Vec<(i32, i64)>,
}
/// One decoded instruction.
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct Insn {
/// Byte offset from the start of the method's code.
pub pc: u32,
pub opcode: u8,
pub name: &'static str,
/// Total length in bytes, including operands.
pub len: u32,
/// Local-variable index operand (load/store/ret/iinc).
#[serde(skip_serializing_if = "Option::is_none")]
pub local: Option<u16>,
/// Integer immediate (`bipush`/`sipush`/`iinc` increment/`newarray` type/`multianewarray` dims).
#[serde(skip_serializing_if = "Option::is_none")]
pub int: Option<i32>,
/// Constant-pool index operand (resolved by the caller).
#[serde(skip_serializing_if = "Option::is_none")]
pub cp_index: Option<u16>,
/// Absolute byte offset of a branch target.
#[serde(skip_serializing_if = "Option::is_none")]
pub branch: Option<i64>,
/// A switch table when the opcode is `tableswitch`/`lookupswitch`.
#[serde(skip_serializing_if = "Option::is_none")]
pub switch: Option<Switch>,
/// True when this instruction was prefixed by `wide`.
#[serde(skip_serializing_if = "std::ops::Not::not")]
pub wide: bool,
}
/// Most entries decoded from one switch table.
const MAX_SWITCH: usize = 1 << 20;
fn s8(code: &[u8], at: usize) -> Option<i32> {
code.get(at).map(|&b| i32::from(b as i8))
}
fn u16b(code: &[u8], at: usize) -> Option<u16> {
Some(u16::from(*code.get(at)?) << 8 | u16::from(*code.get(at + 1)?))
}
fn s16(code: &[u8], at: usize) -> Option<i32> {
u16b(code, at).map(|v| i32::from(v as i16))
}
fn s32(code: &[u8], at: usize) -> Option<i32> {
Some(i32::from(*code.get(at)?) << 24 | i32::from(*code.get(at + 1)?) << 16 | i32::from(*code.get(at + 2)?) << 8 | i32::from(*code.get(at + 3)?))
}
fn base(pc: usize, opcode: u8, name: &'static str) -> Insn {
Insn { pc: pc as u32, opcode, name, len: 1, local: None, int: None, cp_index: None, branch: None, switch: None, wide: false }
}
/// Decode one instruction at `pc`, or `None` for invalid or incomplete operands.
pub fn decode_one(code: &[u8], pc: usize) -> Option<Insn> {
let b = *code.get(pc)?;
let o = opcode(b);
let mut insn = base(pc, b, o.name);
let rel = |d: i32| pc as i64 + i64::from(d);
match o.operands {
Operands::None | Operands::Reserved => {}
Operands::Byte => {
insn.int = Some(s8(code, pc + 1)?);
insn.len = 2;
}
Operands::Short => {
insn.int = Some(s16(code, pc + 1)?);
insn.len = 3;
}
Operands::Local => {
insn.local = Some(u16::from(*code.get(pc + 1)?));
insn.len = 2;
}
Operands::LocalConst => {
insn.local = Some(u16::from(*code.get(pc + 1)?));
insn.int = Some(s8(code, pc + 2)?);
insn.len = 3;
}
Operands::Branch2 => {
insn.branch = Some(rel(s16(code, pc + 1)?));
insn.len = 3;
}
Operands::Branch4 => {
insn.branch = Some(rel(s32(code, pc + 1)?));
insn.len = 5;
}
Operands::Cp1 => {
insn.cp_index = Some(u16::from(*code.get(pc + 1)?));
insn.len = 2;
}
Operands::Cp2 => {
insn.cp_index = Some(u16b(code, pc + 1)?);
insn.len = 3;
}
Operands::NewArray => {
let atype = *code.get(pc + 1)?;
if !(4..=11).contains(&atype) {
return None;
}
insn.int = Some(i32::from(atype));
insn.len = 2;
}
Operands::InvokeInterface => {
insn.cp_index = Some(u16b(code, pc + 1)?);
if *code.get(pc + 3)? == 0 || *code.get(pc + 4)? != 0 {
return None;
}
insn.len = 5;
}
Operands::InvokeDynamic => {
insn.cp_index = Some(u16b(code, pc + 1)?);
if *code.get(pc + 3)? != 0 || *code.get(pc + 4)? != 0 {
return None;
}
insn.len = 5;
}
Operands::MultiANewArray => {
insn.cp_index = Some(u16b(code, pc + 1)?);
let dimensions = *code.get(pc + 3)?;
if dimensions == 0 {
return None;
}
insn.int = Some(i32::from(dimensions));
insn.len = 4;
}
Operands::Wide => return decode_wide(code, pc),
Operands::TableSwitch => return decode_tableswitch(code, pc),
Operands::LookupSwitch => return decode_lookupswitch(code, pc),
}
if pc.checked_add(insn.len as usize)? > code.len() {
return None;
}
Some(insn)
}
fn decode_wide(code: &[u8], pc: usize) -> Option<Insn> {
let real = *code.get(pc + 1)?;
if !matches!(real, 0x15..=0x19 | 0x36..=0x3a | 0x84 | 0xa9) {
return None;
}
let o = opcode(real);
let mut insn = base(pc, real, o.name);
insn.wide = true;
if real == 0x84 {
// wide iinc: 2-byte local, 2-byte signed const
insn.local = Some(u16b(code, pc + 2)?);
insn.int = Some(s16(code, pc + 4)?);
insn.len = 6;
} else {
// wide load/store/ret: 2-byte local
insn.local = Some(u16b(code, pc + 2)?);
insn.len = 4;
}
if pc.checked_add(insn.len as usize)? > code.len() {
return None;
}
Some(insn)
}
/// Padding so the operand word after a switch opcode is 4-byte aligned from the code start.
fn switch_pad(pc: usize) -> usize {
(4 - ((pc + 1) % 4)) % 4
}
fn decode_tableswitch(code: &[u8], pc: usize) -> Option<Insn> {
let mut at = pc + 1 + switch_pad(pc);
let default = rel_i64(pc, s32(code, at)?);
let low = s32(code, at + 4)?;
let high = s32(code, at + 8)?;
at += 12;
if high < low {
return None;
}
let count = usize::try_from(i64::from(high) - i64::from(low) + 1).ok()?;
if count > MAX_SWITCH {
return None;
}
let end = at.checked_add(count.checked_mul(4)?)?;
code.get(at..end)?;
let mut pairs = Vec::with_capacity(count);
for i in 0..count {
let off = s32(code, at + i * 4)?;
pairs.push((low.wrapping_add(i as i32), rel_i64(pc, off)));
}
let mut insn = base(pc, 0xaa, "tableswitch");
insn.switch = Some(Switch { default, pairs });
insn.len = (end - pc) as u32;
Some(insn)
}
fn decode_lookupswitch(code: &[u8], pc: usize) -> Option<Insn> {
let mut at = pc + 1 + switch_pad(pc);
let default = rel_i64(pc, s32(code, at)?);
let npairs = s32(code, at + 4)?;
at += 8;
if npairs < 0 {
return None;
}
let count = npairs as usize;
if count > MAX_SWITCH {
return None;
}
let end = at.checked_add(count.checked_mul(8)?)?;
code.get(at..end)?;
let mut pairs = Vec::with_capacity(count);
let mut previous = None;
for i in 0..count {
let key = s32(code, at + i * 8)?;
if previous.is_some_and(|previous| key <= previous) {
return None;
}
previous = Some(key);
let off = s32(code, at + i * 8 + 4)?;
pairs.push((key, rel_i64(pc, off)));
}
let mut insn = base(pc, 0xab, "lookupswitch");
insn.switch = Some(Switch { default, pairs });
insn.len = (end - pc) as u32;
Some(insn)
}
fn rel_i64(pc: usize, off: i32) -> i64 {
pc as i64 + i64::from(off)
}
/// Decode a whole method body, stopping at `max` instructions. `complete` is false when a
/// instruction has invalid/incomplete operands or the cap was hit.
pub fn decode(code: &[u8], max: usize) -> (Vec<Insn>, bool) {
let mut out = Vec::new();
let mut pc = 0usize;
let mut complete = true;
while pc < code.len() {
if out.len() >= max {
complete = false;
break;
}
match decode_one(code, pc) {
Some(i) => {
pc += (i.len as usize).max(1);
out.push(i);
}
None => {
complete = false;
break;
}
}
}
(out, complete)
}
#[cfg(test)]
mod tests {
use super::*;
fn lookup(pc: usize, keys: &[i32]) -> Vec<u8> {
let mut code = vec![0; pc];
code.push(0xab);
while !code.len().is_multiple_of(4) {
code.push(0);
}
code.extend_from_slice(&0i32.to_be_bytes());
code.extend_from_slice(&(keys.len() as i32).to_be_bytes());
for key in keys {
code.extend_from_slice(&key.to_be_bytes());
code.extend_from_slice(&0i32.to_be_bytes());
}
code
}
#[test]
fn lookup_keys_must_increase_strictly() {
for keys in [&[1, 1][..], &[1, 0], &[0, -1], &[i32::MIN, i32::MAX, i32::MIN]] {
for pc in 0..4 {
let mut code = lookup(pc, keys);
code.push(0xb1);
assert!(decode_one(&code, pc).is_none(), "pc {pc}, keys {keys:?}");
let (decoded, complete) = decode(&code, 100);
assert!(!complete);
assert_eq!(decoded.len(), pc, "preserve preceding nops, stop before the bad switch");
}
}
}
#[test]
fn lookup_keys_use_signed_order_at_every_alignment() {
for keys in [&[][..], &[i32::MIN, -1, 0, i32::MAX]] {
for pc in 0..4 {
let code = lookup(pc, keys);
let insn = decode_one(&code, pc).unwrap();
assert_eq!(insn.len as usize, code.len() - pc);
let switch = insn.switch.unwrap();
assert_eq!(switch.default, pc as i64);
assert_eq!(switch.pairs, keys.iter().map(|&key| (key, pc as i64)).collect::<Vec<_>>());
assert!(decode(&code, 100).1);
}
}
}
#[test]
fn switch_payloads_must_be_complete() {
for pc in 0..4 {
let lookup = lookup(pc, &[-1, 1]);
let mut dense = vec![0; pc];
dense.push(0xaa);
while !dense.len().is_multiple_of(4) {
dense.push(0);
}
for word in [0i32, i32::MAX - 1, i32::MAX, 0, 0] {
dense.extend_from_slice(&word.to_be_bytes());
}
for code in [lookup, dense] {
assert!(decode_one(&code, pc).is_some());
for end in pc + 1..code.len() {
assert!(decode_one(&code[..end], pc).is_none(), "pc {pc}, end {end}");
}
}
}
}
#[test]
fn switch_counts_cannot_reserve_an_absent_table() {
let mut lookup = lookup(0, &[]);
lookup[8..12].copy_from_slice(&(MAX_SWITCH as i32).to_be_bytes());
assert!(decode_one(&lookup, 0).is_none());
for (low, high) in [(0i32, MAX_SWITCH as i32 - 1), (i32::MIN, i32::MAX)] {
let mut dense = vec![0xaa, 0, 0, 0];
for word in [0, low, high] {
dense.extend_from_slice(&word.to_be_bytes());
}
assert!(decode_one(&dense, 0).is_none());
}
}
#[test]
fn table_covers_defined_opcodes() {
let mut named = 0;
for b in 0..=255u8 {
if opcode(b).name != "unknown" {
named += 1;
}
}
// 0x00..=0xca is 203 opcodes, plus impdep1/impdep2.
assert_eq!(named, 203 + 2);
assert_eq!(opcode(0xb6).name, "invokevirtual");
assert_eq!(opcode(0x10).name, "bipush");
assert_eq!(opcode(0xc4).name, "wide");
}
#[test]
fn decodes_core_formats() {
// iconst_1; bipush 7; istore_1; goto +3 ; (pad to reach len) return
let code = [0x04, 0x10, 0x07, 0x3c, 0xa7, 0x00, 0x03, 0xb1];
let (ins, complete) = decode(&code, 100);
assert!(complete, "{ins:?}");
assert_eq!(ins[0].name, "iconst_1");
assert_eq!(ins[1].name, "bipush");
assert_eq!(ins[1].int, Some(7));
assert_eq!(ins[2].name, "istore_1");
assert_eq!(ins[3].name, "goto");
assert_eq!(ins[3].branch, Some(7)); // pc 4 + 3
assert_eq!(ins[4].name, "return");
}
#[test]
fn decodes_invoke_and_wide() {
// invokevirtual #5 ; invokeinterface #6 count 2 ; wide iload 300
let code = [0xb6, 0x00, 0x05, 0xb9, 0x00, 0x06, 0x02, 0x00, 0xc4, 0x15, 0x01, 0x2c];
let (ins, complete) = decode(&code, 100);
assert!(complete);
assert_eq!(ins[0].name, "invokevirtual");
assert_eq!(ins[0].cp_index, Some(5));
assert_eq!(ins[0].len, 3);
assert_eq!(ins[1].name, "invokeinterface");
assert_eq!(ins[1].cp_index, Some(6));
assert_eq!(ins[1].len, 5);
assert_eq!(ins[2].name, "iload");
assert!(ins[2].wide);
assert_eq!(ins[2].local, Some(300));
assert_eq!(ins[2].len, 4);
}
#[test]
fn decodes_switches() {
// tableswitch at pc 0: pad 3, default +40, low 1, high 2, offsets +10, +20
let mut code = vec![0xaa, 0, 0, 0];
code.extend_from_slice(&40i32.to_be_bytes());
code.extend_from_slice(&1i32.to_be_bytes());
code.extend_from_slice(&2i32.to_be_bytes());
code.extend_from_slice(&10i32.to_be_bytes());
code.extend_from_slice(&20i32.to_be_bytes());
let i = decode_one(&code, 0).unwrap();
assert_eq!(i.name, "tableswitch");
let sw = i.switch.unwrap();
assert_eq!(sw.default, 40);
assert_eq!(sw.pairs, vec![(1, 10), (2, 20)]);
assert_eq!(i.len as usize, code.len());
// lookupswitch at pc 0: pad 3, default +4, npairs 1, (key 99 -> +8)
let mut lk = vec![0xab, 0, 0, 0];
lk.extend_from_slice(&4i32.to_be_bytes());
lk.extend_from_slice(&1i32.to_be_bytes());
lk.extend_from_slice(&99i32.to_be_bytes());
lk.extend_from_slice(&8i32.to_be_bytes());
let l = decode_one(&lk, 0).unwrap();
let sw = l.switch.unwrap();
assert_eq!(sw.default, 4);
assert_eq!(sw.pairs, vec![(99, 8)]);
}
#[test]
fn hostile() {
for seed in 0..256 {
let n = crate::test_support::noise(seed, 512);
let _ = decode(&n, 10_000);
}
// truncated operands never panic
let _ = decode(&[0xb6, 0x00], 10);
let _ = decode(&[0xaa, 0, 0, 0, 0, 0, 0, 0], 10);
let _ = decode(&[0xab, 0, 0, 0, 0, 0, 0, 0, 0x7f, 0xff, 0xff, 0xff], 10);
let _ = decode(&[0xc4], 10);
}
}
+50
View File
@@ -0,0 +1,50 @@
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
#![forbid(unsafe_code)]
//! Java class-file inspection and JVM bytecode decoding. Never executes bytecode.
//! Retains source item, attribute and annotation-depth limits and partial reports.
mod bytes;
mod descriptor;
pub mod java_class;
pub mod java_ops;
pub use java_class::*;
#[cfg(test)]
mod test_support;
/// File admission errors; damaged optional structures retain partial-report warnings.
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
pub enum JavaClassError {
#[error("the file ends early ({0})")]
Truncated(String),
#[error("the file is malformed ({0})")]
Malformed(String),
#[error("not supported: {0}")]
Unsupported(String),
}
use JavaClassError as FormatError;
#[cfg(test)]
#[path = "regressions/java_array_operands.rs"]
mod java_array_operands;
#[cfg(test)]
#[path = "regressions/java_attribute_names.rs"]
mod java_attribute_names;
#[cfg(test)]
#[path = "regressions/java_constant_pool_slots.rs"]
mod java_constant_pool_slots;
#[cfg(test)]
#[path = "regressions/java_invoke_operands.rs"]
mod java_invoke_operands;
#[cfg(test)]
#[path = "regressions/java_linkage_references.rs"]
mod java_linkage_references;
#[cfg(test)]
#[path = "regressions/java_truncated_attributes.rs"]
mod java_truncated_attributes;
#[cfg(test)]
#[path = "regressions/java_wide_opcodes.rs"]
mod java_wide_opcodes;
@@ -0,0 +1,48 @@
use crate::java_ops::{decode, decode_one};
#[test]
fn invalid_primitive_array_types_stop_before_the_allocation() {
for atype in (0..=u8::MAX).filter(|value| !(4..=11).contains(value)) {
assert!(decode_one(&[0xbc, atype], 0).is_none(), "atype {atype}");
let (instructions, complete) = decode(&[0, 0xbc, atype, 0xb1], 100);
assert!(!complete, "atype {atype}");
assert_eq!(instructions.len(), 1);
assert_eq!(instructions[0].name, "nop");
}
}
#[test]
fn zero_dimension_multidimensional_allocations_are_not_complete_bytecode() {
assert!(decode_one(&[0xc5, 0, 1, 0], 0).is_none());
let (instructions, complete) = decode(&[0, 0xc5, 0, 1, 0, 0xb1], 100);
assert!(!complete);
assert_eq!(instructions.len(), 1);
assert_eq!(instructions[0].name, "nop");
}
#[test]
fn every_defined_primitive_type_and_nonzero_dimension_remains_decodable() {
for atype in 4..=11 {
let (instructions, complete) = decode(&[0xbc, atype, 0xb1], 100);
assert!(complete);
assert_eq!(instructions.len(), 2);
assert_eq!(instructions[0].int, Some(i32::from(atype)));
assert_eq!(instructions[0].len, 2);
}
for dimensions in 1..=u8::MAX {
let instruction = decode_one(&[0, 0xc5, 0x12, 0x34, dimensions], 1).unwrap();
assert_eq!(instruction.pc, 1);
assert_eq!(instruction.cp_index, Some(0x1234));
assert_eq!(instruction.int, Some(i32::from(dimensions)));
assert_eq!(instruction.len, 4);
}
}
#[test]
fn incomplete_array_operands_remain_rejected() {
for instruction in [&[0xbc, 10][..], &[0xc5, 0, 1, 2][..]] {
for end in 1..instruction.len() {
assert!(decode_one(&instruction[..end], 0).is_none());
}
}
}
@@ -0,0 +1,132 @@
use crate::java_class::{self, ClassFile, Const};
#[derive(Clone, Copy, Debug)]
enum Scope {
Class,
Field,
Method,
Code,
}
fn u16b(out: &mut Vec<u8>, n: u16) {
out.extend_from_slice(&n.to_be_bytes());
}
fn u32b(out: &mut Vec<u8>, n: u32) {
out.extend_from_slice(&n.to_be_bytes());
}
fn utf8(out: &mut Vec<u8>, text: &str) {
out.push(1);
u16b(out, text.len() as u16);
out.extend_from_slice(text.as_bytes());
}
fn attr(name: u16, info: &[u8]) -> Vec<u8> {
let mut out = Vec::new();
u16b(&mut out, name);
u32b(&mut out, info.len() as u32);
out.extend_from_slice(info);
out
}
fn attributes(out: &mut Vec<u8>, repeated: u16, known: &[Vec<u8>]) {
u16b(out, repeated + known.len() as u16);
for _ in 0..repeated {
out.extend_from_slice(&attr(5, &[]));
}
for bytes in known {
out.extend_from_slice(bytes);
}
}
// A valid shared CONSTANT_Utf8 is named by every unknown attribute. Known
// attributes follow it, so avoiding name copies must not discard their facts.
fn fixture(scope: Scope, repeated: u16) -> Vec<u8> {
let mut out = vec![0xca, 0xfe, 0xba, 0xbe, 0, 0, 0, 52];
u16b(&mut out, 17);
utf8(&mut out, "AttributeFixture"); // 1
out.push(7);
u16b(&mut out, 1); // 2: this class
utf8(&mut out, "java/lang/Object"); // 3
out.push(7);
u16b(&mut out, 3); // 4: superclass
utf8(&mut out, &format!("org.example.{}", "A".repeat(usize::from(u16::MAX) - 12))); // 5
for text in ["SourceFile", "AttributeFixture.java", "field", "I"] {
utf8(&mut out, text); // 6..9
}
out.push(3);
u32b(&mut out, 42); // 10
for text in ["ConstantValue", "run", "()V", "Code", "LineNumberTable", "Deprecated"] {
utf8(&mut out, text); // 11..16
}
for word in [0x0021, 2, 4, 0, 1, 0x0019, 8, 9] {
u16b(&mut out, word);
}
attributes(&mut out, if matches!(scope, Scope::Field) { repeated } else { 0 }, &[attr(11, &10u16.to_be_bytes()), attr(16, &[])]);
for word in [1, 0x0009, 12, 13] {
u16b(&mut out, word);
}
let mut code = vec![0, 0, 0, 0, 0, 0, 0, 1, 0xb1, 0, 0];
attributes(&mut code, if matches!(scope, Scope::Code) { repeated } else { 0 }, &[attr(15, &[0, 1, 0, 0, 0, 7])]);
attributes(&mut out, if matches!(scope, Scope::Method) { repeated } else { 0 }, &[attr(14, &code)]);
attributes(&mut out, if matches!(scope, Scope::Class) { repeated } else { 0 }, &[attr(6, &7u16.to_be_bytes())]);
out
}
fn check_facts(parsed: &ClassFile) {
assert!(parsed.warnings.is_empty(), "{:?}", parsed.warnings);
assert_eq!(parsed.this_class, "AttributeFixture");
assert_eq!(parsed.super_class.as_deref(), Some("java/lang/Object"));
assert_eq!(parsed.source_file.as_deref(), Some("AttributeFixture.java"));
assert_eq!(parsed.fields.len(), 1);
assert_eq!(parsed.fields[0].name, "field");
assert_eq!(parsed.fields[0].constant_value.as_deref(), Some("42"));
assert!(parsed.fields[0].deprecated);
assert_eq!(parsed.methods.len(), 1);
assert_eq!(parsed.methods[0].name, "run");
let code = parsed.methods[0].code.as_ref().unwrap();
assert_eq!(code.bytecode, [0xb1]);
assert_eq!(code.line_numbers, [(0, 7)]);
assert!(matches!(&parsed.pool[5], Const::Utf8(name) if name.len() == usize::from(u16::MAX)));
}
#[test]
fn known_attributes_survive_repeated_unknown_names_at_every_scope() {
for scope in [Scope::Class, Scope::Field, Scope::Method, Scope::Code] {
check_facts(&java_class::parse(&fixture(scope, 64)).unwrap());
}
}
fn memory_case(scope: Scope) {
let bytes = fixture(scope, 4096);
assert!(bytes.len() < 100_000);
check_facts(&java_class::parse(&bytes).unwrap());
}
// Run each ignored case separately with a 128 MiB process address-space limit.
// The original parser copies 256 MiB of one attribute name; the fixture itself
// is under 100 KiB. These checks deliberately make no timing assertion.
#[test]
#[ignore = "run separately under a process memory limit"]
fn repeated_class_attribute_names_fit_in_bounded_memory() {
memory_case(Scope::Class);
}
#[test]
#[ignore = "run separately under a process memory limit"]
fn repeated_field_attribute_names_fit_in_bounded_memory() {
memory_case(Scope::Field);
}
#[test]
#[ignore = "run separately under a process memory limit"]
fn repeated_method_attribute_names_fit_in_bounded_memory() {
memory_case(Scope::Method);
}
#[test]
#[ignore = "run separately under a process memory limit"]
fn repeated_code_attribute_names_fit_in_bounded_memory() {
memory_case(Scope::Code);
}
@@ -0,0 +1,53 @@
use crate::{FormatError, java_class};
fn class_with_pool(count: u16, pool: &[u8]) -> Vec<u8> {
let mut bytes = vec![0xca, 0xfe, 0xba, 0xbe, 0, 0, 0, 52];
bytes.extend_from_slice(&count.to_be_bytes());
bytes.extend_from_slice(pool);
bytes.extend_from_slice(&[0; 14]);
bytes
}
#[test]
fn a_wide_constant_needs_its_reserved_slot() {
for tag in [5, 6] {
let mut pool = vec![tag];
pool.extend_from_slice(&[0; 8]);
let bytes = class_with_pool(2, &pool);
assert!(matches!(java_class::parse(&bytes), Err(FormatError::Malformed(message)) if message.contains("second slot")));
}
}
#[test]
fn the_largest_pool_cannot_overflow_on_a_trailing_wide_constant() {
for tag in [5, 6] {
let mut pool = [3, 0, 0, 0, 0].repeat(usize::from(u16::MAX) - 2);
pool.push(tag);
pool.extend_from_slice(&[0; 8]);
let bytes = class_with_pool(u16::MAX, &pool);
assert!(matches!(java_class::parse(&bytes), Err(FormatError::Malformed(message)) if message.contains("second slot")));
}
}
#[test]
fn a_wide_constant_with_two_remaining_slots_is_retained() {
for tag in [5, 6] {
for count in [3, u16::MAX] {
let mut pool = [3, 0, 0, 0, 0].repeat(usize::from(count) - 3);
pool.push(tag);
pool.extend_from_slice(&[0; 8]);
let parsed = java_class::parse(&class_with_pool(count, &pool)).unwrap();
assert_eq!(parsed.pool.len(), usize::from(count));
assert!(matches!(parsed.pool.last(), Some(java_class::Const::Unusable)));
assert!(matches!(parsed.pool[usize::from(count) - 2], java_class::Const::Long(_) | java_class::Const::Double(_)));
}
}
}
#[test]
fn ordinary_final_constants_and_existing_classes_still_parse() {
let parsed = java_class::parse(&class_with_pool(2, &[3, 0, 0, 0, 7])).unwrap();
assert_eq!(parsed.pool, vec![java_class::Const::Unusable, java_class::Const::Integer(7)]);
let parsed = java_class::parse(&crate::test_support::java::hello_class()).unwrap();
assert!(!parsed.methods.is_empty());
}
@@ -0,0 +1,76 @@
use crate::java_class;
use crate::java_ops::{decode, decode_one};
#[test]
fn invokeinterface_requires_a_count_and_zero_reserved_byte() {
for code in [[0xb9, 0, 1, 0, 0], [0xb9, 0, 1, 1, 1], [0xb9, 0, 1, 255, 255]] {
assert!(decode_one(&code, 0).is_none(), "{code:?}");
assert!(!decode(&code, 16).1);
}
}
#[test]
fn invokedynamic_requires_both_reserved_bytes_to_be_zero() {
for code in [[0xba, 0, 1, 1, 0], [0xba, 0, 1, 0, 1], [0xba, 0, 1, 255, 255]] {
assert!(decode_one(&code, 0).is_none(), "{code:?}");
assert!(!decode(&code, 16).1);
}
}
#[test]
fn valid_and_truncated_invocations_keep_their_boundaries() {
for count in [1, 2, 255] {
let instruction = decode_one(&[0xb9, 0x12, 0x34, count, 0], 0).unwrap();
assert_eq!(instruction.cp_index, Some(0x1234));
assert_eq!(instruction.len, 5);
}
let instruction = decode_one(&[0xba, 0x12, 0x34, 0, 0], 0).unwrap();
assert_eq!(instruction.cp_index, Some(0x1234));
for code in [[0xb9, 0, 1, 1, 0], [0xba, 0, 1, 0, 0]] {
for len in 1..5 {
assert!(decode_one(&code[..len], 0).is_none());
}
}
let (instructions, complete) = decode(&[0xb9, 0, 1, 1, 0, 0xba, 0, 2, 0, 0, 0xb1], 16);
assert!(complete);
assert_eq!(instructions.len(), 3);
assert_eq!(instructions[2].pc, 10);
}
fn reference_class() -> java_class::ClassFile {
let mut class = java_class::parse(&crate::test_support::java::hello_class()).unwrap();
class.methods.retain(|method| method.name == "run");
class
}
#[test]
fn malformed_invocations_mark_reference_scans_incomplete() {
let valid = reference_class();
let (expected, incomplete) = java_class::references(&valid, "java/io/PrintStream", None, 10);
assert!(!incomplete);
assert_eq!(expected.len(), 1);
for invalid in [[0xb9, 0, 1, 0, 0], [0xb9, 0, 1, 1, 1], [0xba, 0, 1, 1, 0], [0xba, 0, 1, 0, 1]] {
let mut class = valid.clone();
let code = &mut class.methods[0].code.as_mut().unwrap().bytecode;
let later_reference = code.clone();
*code = invalid.to_vec();
code.extend_from_slice(&later_reference);
let (found, incomplete) = java_class::references(&class, "java/io/PrintStream", None, 10);
assert!(found.is_empty());
assert!(incomplete, "invalid invocation must not hide an unfinished reference scan: {invalid:?}");
}
}
#[test]
fn incomplete_scans_keep_earlier_references_and_scan_other_methods() {
let mut class = reference_class();
let mut later_method = class.methods[0].clone();
later_method.name = "later".into();
class.methods[0].code.as_mut().unwrap().bytecode.extend_from_slice(&[0xba, 0, 1, 1, 0]);
class.methods.push(later_method);
let (found, incomplete) = java_class::references(&class, "java/io/PrintStream", None, 10);
assert!(incomplete);
assert_eq!(found.len(), 2);
assert!(found[0].caller.contains(".run:("));
assert!(found[1].caller.contains(".later:("));
}
@@ -0,0 +1,97 @@
use crate::java_class::{parse, references};
/// A method with a chosen load or call, plus a dynamic call site and a dynamic constant with distinct bootstrap signatures.
fn class(code: &[u8]) -> Vec<u8> {
fn utf(s: &str) -> Vec<u8> {
let mut b = vec![1];
b.extend((s.len() as u16).to_be_bytes());
b.extend(s.bytes());
b
}
let pool = [
utf("Uses"),
vec![7, 0, 1],
utf("java/lang/Object"),
vec![7, 0, 3],
utf("run"),
utf("()V"),
utf("Code"),
utf("BootstrapMethods"),
utf("Target"),
vec![7, 0, 9],
utf("apply"),
vec![12, 0, 11, 0, 6],
vec![10, 0, 10, 0, 12],
vec![15, 6, 0, 13],
utf("Factory"),
vec![7, 0, 15],
utf("boot"),
vec![12, 0, 17, 0, 28],
vec![10, 0, 16, 0, 18],
vec![15, 6, 0, 19],
utf("([LTarget;)V"),
vec![16, 0, 21],
vec![18, 0, 0, 0, 12],
utf("value"),
utf("I"),
vec![12, 0, 24, 0, 25],
vec![17, 0, 1, 0, 26],
utf(
"(Ljava/lang/invoke/MethodHandles$Lookup;Ljava/lang/String;Ljava/lang/invoke/MethodType;Ljava/lang/invoke/MethodHandle;)Ljava/lang/invoke/CallSite;",
),
utf("(Ljava/lang/invoke/MethodHandles$Lookup;Ljava/lang/String;Ljava/lang/Class;)I"),
vec![12, 0, 17, 0, 29],
vec![10, 0, 16, 0, 30],
vec![15, 6, 0, 31],
];
let mut bytes = vec![0xca, 0xfe, 0xba, 0xbe, 0, 0, 0, 55];
bytes.extend((pool.len() as u16 + 1).to_be_bytes());
for e in pool {
bytes.extend(e);
}
bytes.extend([0, 0x21, 0, 2, 0, 4, 0, 0, 0, 0, 0, 1, 0, 9, 0, 5, 0, 6, 0, 1, 0, 7]);
bytes.extend(((12 + code.len()) as u32).to_be_bytes());
bytes.extend([0, 3, 0, 0]);
bytes.extend((code.len() as u32).to_be_bytes());
bytes.extend(code);
bytes.extend([0, 0, 0, 0]);
let boots = [0, 2, 0, 20, 0, 1, 0, 14, 0, 32, 0, 0];
bytes.extend([0, 1, 0, 8]);
bytes.extend((boots.len() as u32).to_be_bytes());
bytes.extend(boots);
bytes
}
fn kinds(code: &[u8], owner: &str, method: Option<&str>) -> Vec<&'static str> {
let cf = parse(&class(code)).unwrap();
let (hits, partial) = references(&cf, owner, method, 10);
assert!(!partial);
hits.into_iter().map(|r| r.kind).collect()
}
#[test]
fn class_literal_has_a_use_site() {
assert_eq!(kinds(&[0x12, 10, 0x57, 0xb1], "Target", None), ["type"]);
}
#[test]
fn literal_method_handle_finds_its_member() {
assert_eq!(kinds(&[0x12, 14, 0x57, 0xb1], "Target", Some("apply")), ["method-handle"]);
}
#[test]
fn method_type_mentions_object_array_components() {
assert_eq!(kinds(&[0x12, 22, 0x57, 0xb1], "Target", None), ["method-type"]);
}
#[test]
fn dynamic_call_exposes_factory_dependency() {
assert_eq!(kinds(&[0xba, 0, 23, 0, 0, 0xb1], "Factory", Some("boot")), ["bootstrap-method"]);
}
#[test]
fn dynamic_call_exposes_static_method_handle_argument() {
assert_eq!(kinds(&[0xba, 0, 23, 0, 0, 0xb1], "Target", Some("apply")), ["method-handle"]);
}
#[test]
fn dynamic_constant_exposes_bootstrap_dependency() {
assert_eq!(kinds(&[0x12, 27, 0x57, 0xb1], "Factory", Some("boot")), ["bootstrap-method"]);
}
#[test]
fn unused_bootstrap_does_not_introduce_a_reference() {
assert!(kinds(&[0xb1], "Factory", None).is_empty());
}
@@ -0,0 +1,46 @@
use crate::java_class::parse;
use crate::test_support::java::hello_class;
/// The fixture ends in a class attribute count and a single two-byte SourceFile attribute.
const ATTRIBUTE_TABLE_BYTES: usize = 10;
/// Every cut of the attribute count, header or body leaves a partial class report.
#[test]
fn incomplete_class_attribute_table_reports_a_warning() {
let data = hello_class();
let start = data.len() - ATTRIBUTE_TABLE_BYTES;
for end in start..data.len() {
let class = parse(&data[..end]).unwrap();
assert_eq!(class.methods.len(), 4);
assert!(class.source_file.is_none());
assert!(class.warnings.iter().any(|warning| warning.contains("class attribute table")), "cut at {}", end - start);
}
}
/// A complete attribute stays available when a subsequent attribute is cut short.
#[test]
fn completed_class_attributes_survive_a_truncated_record() {
let mut data = hello_class();
let start = data.len() - ATTRIBUTE_TABLE_BYTES;
data[start..start + 2].copy_from_slice(&2u16.to_be_bytes());
data.extend_from_slice(&[0, 1, 0, 0, 0, 2, 0]);
let class = parse(&data).unwrap();
assert_eq!(class.source_file.as_deref(), Some("Hello.java"));
assert!(class.warnings.iter().any(|warning| warning.contains("class attribute table")));
}
/// Complete, empty and unknown class attributes do not trigger a truncation warning.
#[test]
fn complete_class_attribute_tables_are_accepted() {
let mut data = hello_class();
let start = data.len() - ATTRIBUTE_TABLE_BYTES;
let original = parse(&data).unwrap();
assert_eq!(original.source_file.as_deref(), Some("Hello.java"));
assert!(original.warnings.is_empty());
data[start..start + 2].copy_from_slice(&2u16.to_be_bytes());
data.extend_from_slice(&[0, 1, 0, 0, 0, 1, 0]);
assert!(parse(&data).unwrap().warnings.is_empty());
data.truncate(start + 2);
data[start..].fill(0);
assert!(parse(&data).unwrap().warnings.is_empty());
}
@@ -0,0 +1,33 @@
use crate::java_ops::{decode, decode_one};
#[test]
fn only_defined_wide_opcodes_are_accepted() {
for op in 0..=255u8 {
let valid = matches!(op, 0x15..=0x19 | 0x36..=0x3a | 0x84 | 0xa9);
let code = [0xc4, op, 0x01, 0x2c, 0xff, 0xfe];
assert_eq!(decode_one(&code, 0).is_some(), valid, "opcode {op:#x}");
}
}
#[test]
fn invalid_wide_does_not_invent_a_method_body() {
let (instructions, complete) = decode(&[0x00, 0xc4, 0xb1, 0, 1, 0xb1], 32);
assert!(!complete);
assert_eq!(instructions.len(), 1);
assert_eq!(instructions[0].name, "nop");
}
#[test]
fn wide_locals_and_signed_increments_are_preserved() {
for op in [0x15, 0x16, 0x17, 0x18, 0x19, 0x36, 0x37, 0x38, 0x39, 0x3a, 0xa9] {
let instruction = decode_one(&[0xc4, op, 0xff, 0xff], 0).unwrap();
assert_eq!(instruction.local, Some(u16::MAX));
assert_eq!(instruction.len, 4);
assert!(instruction.wide);
}
let instruction = decode_one(&[0xc4, 0x84, 0x01, 0x2c, 0xff, 0xfe], 0).unwrap();
assert_eq!(instruction.local, Some(300));
assert_eq!(instruction.int, Some(-2));
assert_eq!(instruction.len, 6);
assert!(decode_one(&[0xc4, 0x84, 0x00, 1], 0).is_none());
}
+276
View File
@@ -0,0 +1,276 @@
pub use craft_testkit::sample_damage::{mutations, noise};
pub mod java {
use std::collections::HashMap;
/// A constant-pool builder that interns entries and tracks the double slot after `Long`/`Double`.
#[derive(Default)]
struct Pool {
items: Vec<Vec<u8>>,
/// Next free index (1-based; accounts for long/double taking two slots).
count: u16,
cache: HashMap<String, u16>,
}
impl Pool {
fn new() -> Self {
Pool { items: Vec::new(), count: 1, cache: HashMap::new() }
}
fn add(&mut self, bytes: Vec<u8>, slots: u16) -> u16 {
let idx = self.count;
self.items.push(bytes);
self.count += slots;
idx
}
fn utf8(&mut self, s: &str) -> u16 {
let key = format!("u:{s}");
if let Some(&i) = self.cache.get(&key) {
return i;
}
let mut b = vec![1u8];
b.extend_from_slice(&(s.len() as u16).to_be_bytes());
b.extend_from_slice(s.as_bytes());
let i = self.add(b, 1);
self.cache.insert(key, i);
i
}
fn class(&mut self, name: &str) -> u16 {
let n = self.utf8(name);
let key = format!("c:{name}");
if let Some(&i) = self.cache.get(&key) {
return i;
}
let mut b = vec![7u8];
b.extend_from_slice(&n.to_be_bytes());
let i = self.add(b, 1);
self.cache.insert(key, i);
i
}
fn string(&mut self, s: &str) -> u16 {
let u = self.utf8(s);
let mut b = vec![8u8];
b.extend_from_slice(&u.to_be_bytes());
self.add(b, 1)
}
fn nat(&mut self, name: &str, desc: &str) -> u16 {
let n = self.utf8(name);
let d = self.utf8(desc);
let mut b = vec![12u8];
b.extend_from_slice(&n.to_be_bytes());
b.extend_from_slice(&d.to_be_bytes());
self.add(b, 1)
}
fn reff(&mut self, tag: u8, owner: &str, name: &str, desc: &str) -> u16 {
let c = self.class(owner);
let nt = self.nat(name, desc);
let mut b = vec![tag];
b.extend_from_slice(&c.to_be_bytes());
b.extend_from_slice(&nt.to_be_bytes());
self.add(b, 1)
}
fn long(&mut self, v: i64) -> u16 {
let mut b = vec![5u8];
b.extend_from_slice(&v.to_be_bytes());
self.add(b, 2)
}
/// The serialized constant pool and the count field.
fn finish(&self) -> (Vec<u8>, u16) {
(self.items.concat(), self.count)
}
}
fn u16b(v: &mut Vec<u8>, x: u16) {
v.extend_from_slice(&x.to_be_bytes());
}
fn u32b(v: &mut Vec<u8>, x: u32) {
v.extend_from_slice(&x.to_be_bytes());
}
/// Build an attribute: name_index, length, body.
fn attr(name_idx: u16, body: Vec<u8>) -> Vec<u8> {
let mut v = Vec::new();
u16b(&mut v, name_idx);
u32b(&mut v, body.len() as u32);
v.extend_from_slice(&body);
v
}
/// A method's `Code` attribute body (max_stack, max_locals, code, exception table, no nested attrs).
fn code_body(max_stack: u16, max_locals: u16, code: &[u8], exc: &[(u16, u16, u16, u16)]) -> Vec<u8> {
let mut v = Vec::new();
u16b(&mut v, max_stack);
u16b(&mut v, max_locals);
u32b(&mut v, code.len() as u32);
v.extend_from_slice(code);
u16b(&mut v, exc.len() as u16);
for (s, e, h, t) in exc {
u16b(&mut v, *s);
u16b(&mut v, *e);
u16b(&mut v, *h);
u16b(&mut v, *t);
}
u16b(&mut v, 0); // nested attributes count
v
}
/// A method_info: access, name, descriptor, and one already-built Code attribute.
fn method(p: &mut Pool, access: u16, name: &str, desc: &str, code_attribute: Vec<u8>) -> Vec<u8> {
let name_idx = p.utf8(name);
let desc_idx = p.utf8(desc);
let mut v = Vec::new();
u16b(&mut v, access);
u16b(&mut v, name_idx);
u16b(&mut v, desc_idx);
u16b(&mut v, 1); // one attribute
v.extend_from_slice(&code_attribute);
v
}
/// Assemble a whole class from its parts.
#[allow(clippy::too_many_arguments)]
fn assemble(
pool: &Pool,
access: u16,
this_c: u16,
super_c: u16,
interfaces: &[u16],
fields: &[Vec<u8>],
methods: &[Vec<u8>],
class_attrs: &[Vec<u8>],
) -> Vec<u8> {
let (cp, count) = pool.finish();
let mut v = Vec::new();
v.extend_from_slice(&[0xca, 0xfe, 0xba, 0xbe]);
u16b(&mut v, 0); // minor
u16b(&mut v, 52); // major (Java 8)
u16b(&mut v, count);
v.extend_from_slice(&cp);
u16b(&mut v, access);
u16b(&mut v, this_c);
u16b(&mut v, super_c);
u16b(&mut v, interfaces.len() as u16);
for i in interfaces {
u16b(&mut v, *i);
}
u16b(&mut v, fields.len() as u16);
for f in fields {
v.extend_from_slice(f);
}
u16b(&mut v, methods.len() as u16);
for m in methods {
v.extend_from_slice(m);
}
u16b(&mut v, class_attrs.len() as u16);
for a in class_attrs {
v.extend_from_slice(a);
}
v
}
pub const HELLO: &str = "Hello, Craft!";
/// `public class com.example.Hello implements Runnable` with a `count` field, a `BIG` long
/// constant, `<init>`, `run()` (System.out.println("Hello, Craft!")), `pick(int)` (a tableswitch)
/// and `guarded()` (a try/catch over the println call).
pub fn hello_class() -> Vec<u8> {
let mut p = Pool::new();
let this_c = p.class("com/example/Hello");
let obj_c = p.class("java/lang/Object");
let runnable_c = p.class("java/lang/Runnable");
let code_attr = p.utf8("Code");
// fields: int count; static final long BIG;
let count_name = p.utf8("count");
let i_desc = p.utf8("I");
let big_name = p.utf8("BIG");
let j_desc = p.utf8("J");
let constval_attr = p.utf8("ConstantValue");
let big_val = p.long(0x1122_3344_5566_7788);
// refs used by code
let obj_init = p.reff(10, "java/lang/Object", "<init>", "()V");
let sysout = p.reff(9, "java/lang/System", "out", "Ljava/io/PrintStream;");
let hello = p.string(HELLO);
let println = p.reff(10, "java/io/PrintStream", "println", "(Ljava/lang/String;)V");
let rt_exc = p.class("java/lang/RuntimeException");
// <init>(): aload_0; invokespecial Object.<init>; return
let mut init = vec![0x2a, 0xb7];
init.extend_from_slice(&obj_init.to_be_bytes());
init.push(0xb1);
let init_m = method(&mut p, 0x0001, "<init>", "()V", attr(code_attr, code_body(1, 1, &init, &[])));
// run(): getstatic System.out; ldc "Hello"; invokevirtual println; return
let mut run = vec![0xb2];
run.extend_from_slice(&sysout.to_be_bytes());
run.push(0x12);
run.push(hello as u8); // ldc (1-byte index; pool is small)
run.push(0xb6);
run.extend_from_slice(&println.to_be_bytes());
run.push(0xb1);
let run_m = method(&mut p, 0x0001, "run", "()V", attr(code_attr, code_body(2, 1, &run, &[])));
// pick(int): iload_0; tableswitch{0->L0,1->L1,default->Ldef}; L0: bipush 10; ireturn; L1: bipush 20; ireturn; Ldef: iconst_m1; ireturn
// offsets: iload_1@0; tableswitch@1 (pad 2) ends @24; L0@24; L1@27; Ldef@30
let mut pick = vec![0x1b, 0xaa, 0x00, 0x00];
u32b(&mut pick, 29); // default -> 1+29 = 30
u32b(&mut pick, 0); // low
u32b(&mut pick, 1); // high
u32b(&mut pick, 23); // case 0 -> 1+23 = 24
u32b(&mut pick, 26); // case 1 -> 1+26 = 27
debug_assert_eq!(pick.len(), 24);
pick.extend_from_slice(&[0x10, 10, 0xac]); // @24: bipush 10; ireturn
pick.extend_from_slice(&[0x10, 20, 0xac]); // @27: bipush 20; ireturn
pick.extend_from_slice(&[0x02, 0xac]); // @30: iconst_m1; ireturn
let pick_m = method(&mut p, 0x0001, "pick", "(I)I", attr(code_attr, code_body(1, 1, &pick, &[])));
// guarded(): try { System.out.println("Hello"); } catch (RuntimeException e) {} (then return)
// 0: getstatic(3) ; 3: ldc(2) ; 5: invokevirtual(3) ; 8: goto +5 ->13 ; 11: astore_1 ; 12: return ; 13: return
let mut g = vec![0xb2];
g.extend_from_slice(&sysout.to_be_bytes());
g.push(0x12);
g.push(hello as u8);
g.push(0xb6);
g.extend_from_slice(&println.to_be_bytes());
g.extend_from_slice(&[0xa7, 0x00, 0x05]); // @8 goto -> 13
g.push(0x4c); // @11 astore_1
g.push(0xb1); // @12 return
g.push(0xb1); // @13 return
let guarded_m = method(&mut p, 0x0001, "guarded", "()V", attr(code_attr, code_body(2, 2, &g, &[(0, 8, 11, rt_exc)])));
// field infos
let field = |p: &mut Pool, access: u16, name_idx: u16, desc_idx: u16, attrs: &[Vec<u8>]| -> Vec<u8> {
let _ = p;
let mut v = Vec::new();
u16b(&mut v, access);
u16b(&mut v, name_idx);
u16b(&mut v, desc_idx);
u16b(&mut v, attrs.len() as u16);
for a in attrs {
v.extend_from_slice(a);
}
v
};
let count_f = field(&mut p, 0x0002, count_name, i_desc, &[]); // private int count
let mut cv = Vec::new();
u16b(&mut cv, big_val);
let big_f = field(&mut p, 0x0019, big_name, j_desc, &[attr(constval_attr, cv)]); // public static final long BIG
let source_attr = p.utf8("SourceFile");
let source_val = p.utf8("Hello.java");
let mut sf = Vec::new();
u16b(&mut sf, source_val);
let source = attr(source_attr, sf);
assemble(
&p,
0x0021, // public super
this_c,
obj_c,
&[runnable_c],
&[count_f, big_f],
&[init_m, run_m, pick_m, guarded_m],
&[source],
)
}
}
+105 -22
View File
@@ -419,6 +419,34 @@ pub fn encode_baseline(
sub: ChromaSubsampling,
app_segments: &[(u8, Vec<u8>)],
) -> Result<Vec<u8>, EncodeError> {
validate_encode(data, width, height, channels, app_segments)?;
Ok(encode(data, width, height, channels, quality, sub, app_segments))
}
/// Encode the same optimized baseline JPEG as [`encode_baseline`], trading a second DCT pass
/// for working memory independent of image height. This path is sequential even when the
/// `parallel` feature is enabled. It retains only one MCU row of conversion planes and writes
/// directly into the returned encoded buffer: no image-wide tokens or duplicate coded buffer.
///
/// Scratch consists of at most `24 * width + 8` floats (odd-width 4:2:0), plus fixed-size coding tables.
/// The caller still owns the input pixels and the returned JPEG; this is not a cap on output
/// size or a replacement for the caller's image/admission budget. Geometry, quality, metadata,
/// Huffman tables and restart boundaries match [`encode_baseline`] byte for byte.
#[allow(clippy::too_many_arguments)]
pub fn encode_baseline_low_memory(
data: &[u8],
width: usize,
height: usize,
channels: usize,
quality: u8,
sub: ChromaSubsampling,
app_segments: &[(u8, Vec<u8>)],
) -> Result<Vec<u8>, EncodeError> {
validate_encode(data, width, height, channels, app_segments)?;
Ok(encode_impl(data, width, height, channels, quality, sub, app_segments, true, true))
}
fn validate_encode(data: &[u8], width: usize, height: usize, channels: usize, app_segments: &[(u8, Vec<u8>)]) -> Result<(), EncodeError> {
let fits = width.checked_mul(height).and_then(|n| n.checked_mul(channels)).is_some_and(|n| n <= data.len());
if !fits || width == 0 || height == 0 || width > 65_535 || height > 65_535 || !matches!(channels, 1 | 3 | 4) {
return Err(EncodeError::InvalidInput(format!("{width}x{height}, {channels} channels, {} bytes", data.len())));
@@ -426,7 +454,7 @@ pub fn encode_baseline(
if app_segments.iter().any(|(marker, payload)| !(0xE0..=0xEF).contains(marker) || payload.len() > 65_533) {
return Err(EncodeError::InvalidInput("APP markers must be 0xE0..=0xEF with payloads no larger than 65533 bytes".into()));
}
Ok(encode(data, width, height, channels, quality, sub, app_segments))
Ok(())
}
/// [`encode`], with optimized Huffman tables or (`optimize` false) the Annex K example tables.
@@ -440,6 +468,21 @@ fn encode_with(
sub: ChromaSubsampling,
app_segments: &[(u8, Vec<u8>)],
optimize: bool,
) -> Vec<u8> {
encode_impl(data, width, height, channels, quality, sub, app_segments, optimize, false)
}
#[allow(clippy::too_many_arguments)]
fn encode_impl(
data: &[u8],
width: usize,
height: usize,
channels: usize,
quality: u8,
sub: ChromaSubsampling,
app_segments: &[(u8, Vec<u8>)],
optimize: bool,
low_memory: bool,
) -> Vec<u8> {
// The public entry takes caller-sized dimensions: refuse (an empty output) what would size an allocation from
// them or index past `data`, rather than panic or allocate for them.
@@ -509,6 +552,8 @@ fn encode_with(
planes.push(Plane { w: cw, h: crows.len(), data: cb });
planes.push(Plane { w: cw, h: crows.len(), data: cr });
}
#[cfg(test)]
low_memory_tests::record_planes(planes.iter().map(|p| p.data.len()).sum());
planes
};
@@ -519,45 +564,58 @@ fn encode_with(
let (mx, my) = (width.div_ceil(mcu_w), height.div_ceil(mcu_h));
// restart interval: whole MCU rows per band, ≤ 65535 MCUs, enough bands to keep all cores busy
let rows_per_band = (my / 64).clamp(1, (65_535 / mx.max(1)).max(1));
let bands: Vec<usize> = (0..my.div_ceil(rows_per_band)).collect();
let num_bands = my.div_ceil(rows_per_band);
let bands: Vec<usize> = if low_memory { Vec::new() } else { (0..num_bands).collect() };
// first pass: each band's quantized blocks as symbols, with their counts per table (DC
// prediction restarts per band, as the restart markers say)
let tokenize_band = |band: &usize| -> (Vec<Token>, [Freq; 4]) {
let visit_band = |band: &usize, emit: &mut dyn FnMut(&[i16; 64], &mut i32, bool)| {
let (mut blk, mut q) = ([0f32; 64], [0i16; 64]);
let (mut tokens, mut freq) = (Vec::new(), [[0u64; 256]; 4]);
let mut pred = [0i32; 3];
let (m0, m1) = (band * rows_per_band, ((band + 1) * rows_per_band).min(my));
// the band's planes, its MCU row `m0` at their top (blocks past the image's bottom repeat its last row,
// which is the band's last row then)
let planes = convert(m0 * mcu_h..(m1 * mcu_h).min(height), m0 * 8..(m1 * 8).min(ch));
for mcu_y in m0..m1 {
let mcu_y = mcu_y - m0;
// Convert only the current MCU row. DC predictors still span the original band,
// so its symbols/restart decisions are identical to whole-band conversion.
let planes = convert(mcu_y * mcu_h..((mcu_y + 1) * mcu_h).min(height), mcu_y * 8..((mcu_y + 1) * 8).min(ch));
for mcu_x in 0..mx {
for vy in 0..vs {
for hx in 0..hs {
planes[0].block(mcu_x * hs + hx, mcu_y * vs + vy, &mut blk);
planes[0].block(mcu_x * hs + hx, vy, &mut blk);
fdct_quant(&blk, &rl, &mut q);
tokenize_block(&q, &mut pred[0], false, &mut tokens, &mut freq);
emit(&q, &mut pred[0], false);
}
}
if !gray {
for (plane, pred) in planes.iter().zip(pred.iter_mut()).take(3).skip(1) {
plane.block(mcu_x, mcu_y, &mut blk);
plane.block(mcu_x, 0, &mut blk);
fdct_quant(&blk, &rc, &mut q);
tokenize_block(&q, pred, true, &mut tokens, &mut freq);
emit(&q, pred, true);
}
}
}
}
};
let tokenize_band = |band: &usize| -> (Vec<Token>, [Freq; 4]) {
let (mut tokens, mut freq) = (Vec::new(), [[0u64; 256]; 4]);
visit_band(band, &mut |q, pred, chroma| tokenize_block(q, pred, chroma, &mut tokens, &mut freq));
#[cfg(test)]
low_memory_tests::record_tokens(tokens.len());
(tokens, freq)
};
#[cfg(feature = "parallel")]
let tokenized: Vec<(Vec<Token>, [Freq; 4])> = bands.par_iter().map(tokenize_band).collect();
let tokenized: Vec<(Vec<Token>, [Freq; 4])> = if low_memory { Vec::new() } else { bands.par_iter().map(tokenize_band).collect() };
#[cfg(not(feature = "parallel"))]
let tokenized: Vec<(Vec<Token>, [Freq; 4])> = bands.iter().map(tokenize_band).collect();
let tokenized: Vec<(Vec<Token>, [Freq; 4])> = if low_memory { Vec::new() } else { bands.iter().map(tokenize_band).collect() };
// the tables: DC luma, AC luma, DC chroma, AC chroma
let tables: [([u8; 16], Vec<u8>); 4] = if optimize {
let mut total = [[0u64; 256]; 4];
if low_memory {
for band in 0..num_bands {
visit_band(&band, &mut |q, pred, chroma| {
let base = if chroma { 2 } else { 0 };
block_symbols(q, pred, |ac, symbol, _, _| total[base + ac as usize][symbol & 0xFF] += 1);
});
}
}
for (_, f) in &tokenized {
for (t, f) in total.iter_mut().zip(f) {
for (t, f) in t.iter_mut().zip(f) {
@@ -577,9 +635,9 @@ fn encode_with(
let huff = tables.each_ref().map(|(bits, vals)| Huff::new(bits, vals));
// second pass: code the symbols
#[cfg(feature = "parallel")]
let coded: Vec<Vec<u8>> = tokenized.into_par_iter().map(|(t, _)| write_tokens(&t, &huff)).collect();
let coded: Vec<Vec<u8>> = if low_memory { Vec::new() } else { tokenized.into_par_iter().map(|(t, _)| write_tokens(&t, &huff)).collect() };
#[cfg(not(feature = "parallel"))]
let coded: Vec<Vec<u8>> = tokenized.into_iter().map(|(t, _)| write_tokens(&t, &huff)).collect();
let coded: Vec<Vec<u8>> = if low_memory { Vec::new() } else { tokenized.into_iter().map(|(t, _)| write_tokens(&t, &huff)).collect() };
// headers
let mut out = Vec::with_capacity(coded.iter().map(Vec::len).sum::<usize>() + 4096);
@@ -617,7 +675,7 @@ fn encode_with(
dht(&mut out, 0x11, acb, acv);
}
let interval = (rows_per_band * mx) as u16;
if coded.len() > 1 {
if num_bands > 1 {
segment(&mut out, 0xDD, &interval.to_be_bytes());
}
if gray {
@@ -625,11 +683,32 @@ fn encode_with(
} else {
segment(&mut out, 0xDA, &[3, 1, 0x00, 2, 0x11, 3, 0x11, 0, 63, 0]);
}
let last = coded.len().saturating_sub(1);
for (i, c) in coded.iter().enumerate() {
out.extend_from_slice(c);
if i < last {
out.extend_from_slice(&[0xFF, 0xD0 + (i % 8) as u8]);
if low_memory {
let last = num_bands.saturating_sub(1);
for i in 0..num_bands {
// Continue directly in the final buffer, resetting entropy bits at the same band
// boundaries as the retained-token path. Only the output grows with image height.
let mut bits = Bits { out, acc: 0, n: 0 };
visit_band(&i, &mut |q, pred, chroma| {
let base = if chroma { 2 } else { 0 };
block_symbols(q, pred, |ac, symbol, extra, n| {
let (code, len) = huff[base + ac as usize].code[symbol & 0xFF];
bits.put(code as u32, len as u32);
bits.put(extra & 0xFFFF, n & 0xF);
});
});
out = bits.finish();
if i < last {
out.extend_from_slice(&[0xFF, 0xD0 + (i % 8) as u8]);
}
}
} else {
let last = coded.len().saturating_sub(1);
for (i, c) in coded.iter().enumerate() {
out.extend_from_slice(c);
if i < last {
out.extend_from_slice(&[0xFF, 0xD0 + (i % 8) as u8]);
}
}
}
out.extend_from_slice(&[0xFF, 0xD9]);
@@ -877,3 +956,7 @@ mod tests {
assert!(psnr(&src, &px) > 35.0);
}
}
#[cfg(test)]
#[path = "encode_low_memory_tests.rs"]
mod low_memory_tests;
+155
View File
@@ -0,0 +1,155 @@
use super::*;
use std::cell::Cell;
thread_local! {
static STATS: Cell<(usize, usize)> = const { Cell::new((0, 0)) };
}
pub(super) fn record_planes(samples: usize) {
STATS.with(|s| {
let (peak, tokens) = s.get();
s.set((peak.max(samples), tokens));
});
}
pub(super) fn record_tokens(count: usize) {
STATS.with(|s| {
let (peak, tokens) = s.get();
s.set((peak, tokens + count));
});
}
fn noise(len: usize) -> Vec<u8> {
let mut seed = 0x89ab_cdefu32;
(0..len)
.map(|_| {
seed ^= seed << 13;
seed ^= seed >> 17;
seed ^= seed << 5;
seed as u8
})
.collect()
}
fn compare(src: &[u8], w: usize, h: usize, ch: usize, q: u8, sub: ChromaSubsampling, app: &[(u8, Vec<u8>)]) {
let expected = encode_baseline(src, w, h, ch, q, sub, app).unwrap();
STATS.with(|s| s.set((0, 0)));
let actual = encode_baseline_low_memory(src, w, h, ch, q, sub, app).unwrap();
assert_eq!(actual, expected, "{w}x{h} channels={ch} quality={q} subsampling={sub:?}");
let (peak, tokens) = STATS.with(Cell::get);
assert!(peak <= 24 * w + 8, "MCU-row scratch exceeded: {peak}");
assert_eq!(tokens, 0, "low-memory mode retained entropy tokens");
}
#[test]
fn all_sampling_and_channels_match_retained_tokens_byte_for_byte() {
for (w, h) in [(1, 1), (7, 9), (17, 31), (33, 65), (17, 1025)] {
for ch in [1, 3, 4] {
let src = noise(w * h * ch);
for sub in [ChromaSubsampling::S444, ChromaSubsampling::S422, ChromaSubsampling::S420] {
for q in [0, 1, 25, 50, 90, 100, 255] {
compare(&src, w, h, ch, q, sub, &[]);
}
}
}
}
}
#[test]
fn high_entropy_hd_and_wide_frames_keep_only_one_mcu_row() {
for (w, h) in [(1920, 1080), (65_535, 1)] {
let src = noise(w * h * 3);
compare(&src, w, h, 3, 100, ChromaSubsampling::S444, &[]);
}
}
#[test]
fn metadata_empty_tables_and_extra_source_bytes_keep_exact_output() {
let app = [(0xe0, b"custom density".to_vec()), (0xe2, vec![73; 65_533])];
for ch in [1, 3, 4] {
let src = vec![127; 19 * 23 * ch + 5];
compare(&src, 19, 23, ch, 75, ChromaSubsampling::S420, &app);
}
}
#[test]
fn checked_admission_matches_original_without_panics() {
for (w, h, ch) in [(0, 1, 3), (1, 0, 3), (1, 1, 0), (1, 1, 2), (usize::MAX, 2, 4), (65_536, 1, 1), (1, 65_536, 1), (4, 4, 3)] {
let src = vec![0; if w == 65_536 || h == 65_536 { 65_536 } else { 8 }];
let old = encode_baseline(&src, w, h, ch, 90, ChromaSubsampling::S444, &[]);
let new = encode_baseline_low_memory(&src, w, h, ch, 90, ChromaSubsampling::S444, &[]);
assert!(new.is_err());
assert_eq!(new, old);
}
for app in [vec![(0xdf, vec![])], vec![(0xf0, vec![])], vec![(0xe1, vec![0; 65_534])]] {
assert_eq!(
encode_baseline_low_memory(&[0; 3], 1, 1, 3, 90, ChromaSubsampling::S444, &app),
encode_baseline(&[0; 3], 1, 1, 3, 90, ChromaSubsampling::S444, &app)
);
}
}
// Generated with the immutable pre-change encoder at craft-libs
// 937ac93c5cf3cd4b6a81e195b078fe161eb081b9, not the refactored token-retaining path.
// Odd chroma edges and 1025/2049-row inputs exercise multi-row restart bands.
#[test]
fn prechange_encoded_bytes_remain_pinned_across_band_boundaries() {
let cases = [
(1, 1, 1, 1, 444, 159, 0xe3bd59145381ba57),
(1, 1, 1, 75, 422, 159, 0xcb2b103914bbdbd3),
(1, 1, 1, 100, 420, 160, 0xd9f640613efb6abf),
(1, 1, 3, 1, 444, 284, 0x61d2772b8fd9df17),
(1, 1, 3, 75, 422, 286, 0xa0e03ab7efa23296),
(1, 1, 3, 100, 420, 288, 0x5660667f1abd0a63),
(1, 1, 4, 1, 444, 284, 0x61d2772b8fd9df17),
(1, 1, 4, 75, 422, 286, 0xa0e03ab7efa23296),
(1, 1, 4, 100, 420, 288, 0x5660667f1abd0a63),
(17, 31, 1, 1, 444, 240, 0xf8d277e9499253d4),
(17, 31, 1, 75, 422, 515, 0xa6946d4f144bf243),
(17, 31, 1, 100, 420, 793, 0xbb3db259708d8951),
(17, 31, 3, 1, 444, 363, 0xff2b5b41ffb3f81b),
(17, 31, 3, 75, 422, 877, 0x0374ffb527378dfe),
(17, 31, 3, 100, 420, 1220, 0x2feeedfa90885737),
(17, 31, 4, 1, 444, 366, 0xde7f8d1330e436eb),
(17, 31, 4, 75, 422, 879, 0xb1559fbfc3ee0cf6),
(17, 31, 4, 100, 420, 1214, 0xaf5dcf12d28c8e88),
(33, 65, 1, 1, 444, 388, 0xbee80491e42c8fd2),
(33, 65, 1, 75, 422, 1470, 0x284f4eb78b785ca5),
(33, 65, 1, 100, 420, 2576, 0xa81cc94015f326dc),
(33, 65, 3, 1, 444, 526, 0xf75b4036acc30874),
(33, 65, 3, 75, 422, 2322, 0x6fbc56bd9a1eec70),
(33, 65, 3, 100, 420, 3796, 0xcad25051562ac816),
(33, 65, 4, 1, 444, 527, 0x4bfedc11153de4c1),
(33, 65, 4, 75, 422, 2309, 0x47bf11bf32bac6a4),
(33, 65, 4, 100, 420, 3775, 0x64c2303fde273cd1),
(17, 1025, 1, 1, 444, 2017, 0x242f4a95d8579612),
(17, 1025, 1, 75, 422, 10728, 0xfe49408f379bdffb),
(17, 1025, 1, 100, 420, 19686, 0x1355b995e38d6467),
(17, 1025, 3, 1, 444, 2137, 0x4e319440cb6946c2),
(17, 1025, 3, 75, 422, 17682, 0xf456c63b731dde8f),
(17, 1025, 3, 100, 420, 29015, 0x0631001154b128cf),
(17, 1025, 4, 1, 444, 2155, 0xa2de9c1d77e62aa6),
(17, 1025, 4, 75, 422, 17648, 0x7d85fbdf56061f0c),
(17, 1025, 4, 100, 420, 29010, 0x4848eeb93daf1df2),
(19, 2049, 1, 1, 444, 3531, 0x9b93cf219ff85796),
(19, 2049, 1, 75, 422, 26283, 0xb160523444397608),
(19, 2049, 1, 100, 420, 51883, 0xc5c26836c0a976c6),
(19, 2049, 3, 1, 444, 3303, 0x26eaac2272e1fdb8),
(19, 2049, 3, 75, 422, 42745, 0x38b9843fb54db698),
(19, 2049, 3, 100, 420, 79868, 0xd6a7917af19682f7),
(19, 2049, 4, 1, 444, 3343, 0x6bf577903528692f),
(19, 2049, 4, 75, 422, 42706, 0x0e62366274593a93),
(19, 2049, 4, 100, 420, 79804, 0x867d05cf9d944599),
];
for (w, h, ch, q, sampling, len, hash) in cases {
let sub = match sampling {
444 => ChromaSubsampling::S444,
422 => ChromaSubsampling::S422,
_ => ChromaSubsampling::S420,
};
let src = noise(w * h * ch);
for actual in [encode_baseline(&src, w, h, ch, q, sub, &[]).unwrap(), encode_baseline_low_memory(&src, w, h, ch, q, sub, &[]).unwrap()] {
let actual_hash = actual.iter().fold(0xcbf29ce484222325u64, |h, b| (h ^ u64::from(*b)).wrapping_mul(0x100000001b3));
assert_eq!((actual.len(), actual_hash), (len, hash), "{w}x{h} channels={ch} quality={q} sampling={sampling}");
}
}
}
+1 -1
View File
@@ -1705,7 +1705,7 @@ mod tests;
/// Baseline JPEG encoder (8-bit, Huffman-coded sequential DCT).
mod encode;
pub use encode::{ChromaSubsampling, EncodeError, encode_baseline};
pub use encode::{ChromaSubsampling, EncodeError, encode_baseline, encode_baseline_low_memory};
mod app14;
pub use app14::adobe_transform;
+1 -1
View File
@@ -15,7 +15,7 @@ mod server;
pub use content::{Contents, base64, image_result, text_result};
pub use schema::{Effect, check_arguments, object, string, tool};
pub use server::{Incoming, Info, PROTOCOL_VERSION, Reference, SUPPORTED_VERSIONS, Server, Tools, read_message, response};
pub use server::{Incoming, Info, Options, PROTOCOL_VERSION, Reference, SUPPORTED_VERSIONS, Server, Tools, read_message, response};
#[cfg(test)]
mod tests;
+40 -5
View File
@@ -4,7 +4,7 @@ use std::io::{BufRead, Write};
use serde_json::{Value, json};
use crate::content::Contents;
use crate::content::{Contents, text_result};
use crate::schema::{check_arguments, check_prompt_arguments};
/// The newest protocol revision we speak; offered when the client asks for one we don't know.
@@ -88,9 +88,25 @@ pub struct Info {
pub max_line: usize,
}
/// Opt-in behaviour for apps with their own conventions; `Options::default()` is what [`Server::new`]
/// does. Set with [`Server::with_options`].
#[derive(Clone, Debug, Default)]
pub struct Options {
/// Sent as `serverInfo.websiteUrl` at `initialize` (MCP 2025-06-18) when set.
pub website_url: Option<String>,
/// Report a `tools/call` whose arguments fail the check against the tool's schema, or whose
/// tool is unknown, as a tool result with `isError: true` and the message as text, not as a
/// JSON-RPC -32602 error. For apps whose clients read every tool failure from the result.
pub tool_errors_as_results: bool,
/// Refuse a request whose `id` is neither a string nor a number (JSON-RPC 2.0) with -32600 and
/// a null id, instead of echoing the id back.
pub strict_request_ids: bool,
}
/// An MCP server over the app's [`Tools`].
pub struct Server<T: Tools> {
info: Info,
options: Options,
tools: T,
initialized: bool,
}
@@ -157,7 +173,13 @@ pub fn response(id: Value, result: Value) -> Value {
impl<T: Tools> Server<T> {
pub fn new(info: Info, tools: T) -> Self {
Self { info, tools, initialized: false }
Self { info, options: Options::default(), tools, initialized: false }
}
/// This server with `options` ([`Options`]) in place of the defaults.
#[must_use]
pub fn with_options(mut self, options: Options) -> Self {
self.options = options;
self
}
pub fn tools(&mut self) -> &mut T {
&mut self.tools
@@ -215,7 +237,11 @@ impl<T: Tools> Server<T> {
if o.contains_key("result") || o.contains_key("error") {
return None;
}
return Some(error(id.unwrap_or(Value::Null), INVALID_REQUEST, "missing `method`"));
let echoed = match id {
Some(id) if !self.options.strict_request_ids || id.is_string() || id.is_number() => id,
_ => Value::Null,
};
return Some(error(echoed, INVALID_REQUEST, "missing `method`"));
};
let params = o.get("params").cloned().unwrap_or(Value::Null);
let Some(id) = id else {
@@ -224,6 +250,9 @@ impl<T: Tools> Server<T> {
}
return None;
};
if self.options.strict_request_ids && !(id.is_string() || id.is_number()) {
return Some(error(Value::Null, INVALID_REQUEST, "`id` must be a string or number"));
}
Some(match self.request(method, &params) {
Ok(r) => response(id, r),
Err((code, m)) => error(id, code, m),
@@ -236,10 +265,14 @@ impl<T: Tools> Server<T> {
let asked = params.get("protocolVersion").and_then(Value::as_str).unwrap_or(PROTOCOL_VERSION);
let version = if SUPPORTED_VERSIONS.contains(&asked) { asked } else { PROTOCOL_VERSION };
let Info { name, title, version: server_version, .. } = self.info;
let mut server_info = json!({"name": name, "title": title, "version": server_version});
if let Some(url) = &self.options.website_url {
server_info["websiteUrl"] = json!(url);
}
Ok(json!({
"protocolVersion": version,
"capabilities": self.capabilities(),
"serverInfo": {"name": name, "title": title, "version": server_version},
"serverInfo": server_info,
"instructions": self.tools.instructions(),
}))
}
@@ -251,7 +284,9 @@ impl<T: Tools> Server<T> {
None | Some(Value::Null) => json!({}),
Some(a) => a.clone(),
};
check_arguments(&self.tools.list(), name, &args).map_err(|e| (INVALID_PARAMS, e))?;
if let Err(e) = check_arguments(&self.tools.list(), name, &args) {
return if self.options.tool_errors_as_results { Ok(text_result(e, true)) } else { Err((INVALID_PARAMS, e)) };
}
Ok(self.tools.call(name, &args))
}
"resources/list" => Ok(json!({"resources": self.tools.resources()})),
+130 -1
View File
@@ -1,7 +1,8 @@
use serde_json::{Value, json};
use crate::{
Contents, Effect, Info, PROTOCOL_VERSION, Reference, SUPPORTED_VERSIONS, Server, Tools, check_arguments, object, string, text_result, tool,
Contents, Effect, Info, Options, PROTOCOL_VERSION, Reference, SUPPORTED_VERSIONS, Server, Tools, check_arguments, object, string, text_result,
tool,
};
/// One app's tools: `echo` and `level`, nothing behind them (no engine, no design). `log` records
@@ -366,3 +367,131 @@ fn every_call_runs_right_after_its_listing() {
reply(&mut s, &json!([call("level", json!({})), call("echo", json!({"text": "b"}))]));
assert_eq!(s.tools().log, ["list", "list", "call echo", "list", "list", "call level", "list", "call echo"]);
}
fn with(options: Options) -> Server<Echo> {
Server::new(info(), Echo::default()).with_options(options)
}
fn call_msg(id: u64, name: &str, args: Value) -> Value {
json!({"jsonrpc": "2.0", "id": id, "method": "tools/call", "params": {"name": name, "arguments": args}})
}
/// `websiteUrl` is sent only when set.
#[test]
fn website_url_is_sent_only_when_set() {
let init = json!({"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {}});
let off = reply(&mut with(Options::default()), &init).unwrap();
assert!(off["result"]["serverInfo"].get("websiteUrl").is_none(), "{off}");
let on = reply(&mut with(Options { website_url: Some("https://example.org/echo".into()), ..Options::default() }), &init).unwrap();
assert_eq!(on["result"]["serverInfo"]["websiteUrl"], "https://example.org/echo");
assert_eq!(on["result"]["serverInfo"]["name"], "echo");
}
/// With `tool_errors_as_results`, argument failures and unknown tools are `isError` results that
/// carry the message and never run the tool; other errors stay JSON-RPC errors.
#[test]
fn tool_errors_as_results_replaces_invalid_params() {
let mut s = with(Options { tool_errors_as_results: true, ..Options::default() });
let bad = [
call_msg(1, "echo", json!({})),
call_msg(2, "echo", json!({"text": 3})),
call_msg(3, "echo", json!({"text": "a", "loud": true})),
call_msg(4, "shout", json!({})),
call_msg(5, "echo", json!([])),
call_msg(6, "echo", json!("text")),
call_msg(7, "echo", json!(7)),
];
for msg in &bad {
let r = reply(&mut s, msg).unwrap();
assert!(r.get("error").is_none(), "{r}");
assert_eq!(r["result"]["isError"], true, "{r}");
assert!(r["result"]["content"][0]["text"].as_str().is_some_and(|t| !t.is_empty()), "{r}");
}
let r = reply(&mut s, &bad[0]).unwrap();
assert_eq!(r["result"]["content"][0]["text"], "echo: missing required argument `text`");
assert!(s.tools().calls.is_empty());
let ok = reply(&mut s, &call_msg(8, "echo", json!({"text": "hi"}))).unwrap();
assert_eq!(ok["result"], json!({"content": [{"type": "text", "text": "hi"}]}));
let nomethod = reply(&mut s, &json!({"jsonrpc": "2.0", "id": 9, "method": "nope"})).unwrap();
assert_eq!(nomethod["error"]["code"], -32601);
let noname = reply(&mut s, &json!({"jsonrpc": "2.0", "id": 10, "method": "tools/call", "params": {}})).unwrap();
assert_eq!(noname["error"]["code"], -32602);
// Unchanged with the option off.
let mut off = with(Options::default());
for msg in &bad {
assert_eq!(reply(&mut off, msg).unwrap()["error"]["code"], -32602);
}
}
/// With `strict_request_ids`, an id that is not a string or number is -32600 with a null id; the
/// tool never runs. Off, such ids are echoed as before. Notifications have no id and are unaffected.
#[test]
fn strict_request_ids_refuse_other_id_types() {
let odd = [json!(null), json!(true), json!({"a": 1}), json!([1])];
let mut on = with(Options { strict_request_ids: true, ..Options::default() });
let mut off = with(Options::default());
for id in &odd {
let msg = json!({"jsonrpc": "2.0", "id": id, "method": "tools/call", "params": {"name": "echo", "arguments": {"text": "x"}}});
let r = reply(&mut on, &msg).unwrap();
assert_eq!(r["error"]["code"], -32600, "{id}: {r}");
assert_eq!(r["id"], Value::Null);
let r = reply(&mut off, &json!({"jsonrpc": "2.0", "id": id, "method": "ping"})).unwrap();
assert_eq!(r["id"], *id);
assert_eq!(r["result"], json!({}));
}
assert!(on.tools().calls.is_empty());
for id in [json!("a"), json!(1), json!(1.5), json!(-3)] {
let r = reply(&mut on, &json!({"jsonrpc": "2.0", "id": id, "method": "ping"})).unwrap();
assert_eq!(r["result"], json!({}), "{id}");
}
assert!(reply(&mut on, &notification()).is_none());
}
/// The missing-`method` error follows the same id rule: strict mode echoes only string and number
/// ids and answers with a null id for the rest; off, ids are echoed as before.
#[test]
fn strict_request_ids_apply_to_the_missing_method_error() {
let mut on = with(Options { strict_request_ids: true, ..Options::default() });
let mut off = with(Options::default());
for id in [json!(true), json!({"a": 1}), json!([1])] {
let msg = json!({"jsonrpc": "2.0", "id": id});
let r = reply(&mut on, &msg).unwrap();
assert_eq!((r["error"]["code"].clone(), r["id"].clone()), (json!(-32600), Value::Null), "{id}");
assert_eq!(reply(&mut off, &msg).unwrap()["id"], id);
}
for id in [json!("a"), json!(7)] {
assert_eq!(reply(&mut on, &json!({"jsonrpc": "2.0", "id": id})).unwrap()["id"], id);
}
assert_eq!(reply(&mut on, &json!({"jsonrpc": "2.0"})).unwrap()["id"], Value::Null);
}
/// Hostile input with every option on: oversized and non-UTF-8 lines, odd ids and wrongly typed
/// arguments are answered and serving goes on, and nothing panics.
#[test]
fn every_option_on_survives_hostile_input() {
let mut input = vec![b'x'; (1 << 16) + 10];
input.push(b'\n');
input.extend_from_slice(b"\xff\xfe\n");
for id in ["null", "true", "{}", "[]", "\"s\"", "1e999"] {
input.extend_from_slice(
format!(
"{{\"jsonrpc\":\"2.0\",\"id\":{id},\"method\":\"tools/call\",\"params\":{{\"name\":\"echo\",\"arguments\":{{\"text\":{id}}}}}}}\n"
)
.as_bytes(),
);
}
for args in ["null", "1", "\"a\"", "[]", "{\"text\":[1]}", "{\"text\":{}}"] {
input.extend_from_slice(
format!("{{\"jsonrpc\":\"2.0\",\"id\":1,\"method\":\"tools/call\",\"params\":{{\"name\":\"level\",\"arguments\":{args}}}}}\n").as_bytes(),
);
}
input.extend_from_slice(json!({"jsonrpc": "2.0", "id": 9, "method": "ping"}).to_string().as_bytes());
let mut output = Vec::new();
let options = Options { website_url: Some("https://example.org".into()), tool_errors_as_results: true, strict_request_ids: true };
with(options).serve(std::io::Cursor::new(input), &mut output).unwrap();
let replies: Vec<Value> = String::from_utf8(output).unwrap().lines().map(|l| serde_json::from_str(l).unwrap()).collect();
assert_eq!(replies.len(), 2 + 6 + 6 + 1, "{replies:?}");
assert_eq!(replies[0]["error"]["code"], -32600);
assert_eq!(replies[1]["error"]["code"], -32700);
assert_eq!(replies.last().map(|r| &r["result"]), Some(&json!({})));
}
+6 -4
View File
@@ -3,9 +3,9 @@
//! thread with a time limit for each case. [`Rng`], [`mutations`] and the other names here are
//! `craft_testkit`'s.
pub use craft_testkit::hostile::{DIVISOR_VARIABLE, MAX_CASES, Mutation, Mutations, Rng, mutations};
#[cfg(not(target_arch = "wasm32"))]
pub use craft_testkit::hostile::{HammerConfig, HammerError};
pub use craft_testkit::hostile::{MAX_CASES, Mutation, Mutations, Rng, mutations};
pub use craft_testkit::hostile::{HammerConfig, HammerError, divisor};
/// The variable that multiplies [`hammer`]'s iterations.
#[cfg(not(target_arch = "wasm32"))]
@@ -17,8 +17,10 @@ const MULTIPLIER_VARIABLE: &str = "CRAFT_MEDIA_HOSTILE_MULT";
/// Panics on the first case that panics or does not finish, naming `name`, the case, its length and
/// the panic's message.
///
/// `CRAFT_MEDIA_HOSTILE_MULT=10` runs ten times the iterations (soak runs). A value that is not a
/// whole number above zero panics, naming the variable, before `run` is called.
/// `CRAFT_MEDIA_HOSTILE_MULT=10` runs ten times the iterations (soak runs), and
/// `CRAFT_HOSTILE_DIVISOR=8` every eighth truncation and an eighth of the iterations (pull
/// requests). A value that is not a whole number above zero panics, naming the variable, before
/// `run` is called.
#[cfg(not(target_arch = "wasm32"))]
#[track_caller]
pub fn hammer(name: &str, seed: &[u8], iters: usize, run: impl Fn(&[u8]) + Send + Sync + 'static) {
+2 -1
View File
@@ -13,7 +13,8 @@
//!
//! [`hostile::hammer`] runs a parser over every truncation and mutations of a valid input and fails
//! on a panic or a case that does not finish in time; `CRAFT_MEDIA_HOSTILE_MULT=10` runs ten times
//! the iterations. It and [`with_deadline`] call `craft_testkit`.
//! the iterations and `CRAFT_HOSTILE_DIVISOR=8` an eighth of them. It and [`with_deadline`] call
//! `craft_testkit`.
//! [`vmd`] writes Sierra VMD fixtures, which ffmpeg can read but not write.
//!
//! Failing loudly is what a test helper is for: a wrong `CRAFT_MEDIA_*` setting or an unwritable
+7 -6
View File
@@ -5,7 +5,7 @@ use std::process::Command;
use std::sync::{Arc, Mutex};
use std::time::Duration;
use craft_media_testkit::hostile::hammer;
use craft_media_testkit::hostile::{divisor, hammer};
/// Set only in the child processes of `run_child`: the value names what `child_hammer` does.
const CHILD_VARIABLE: &str = "CRAFT_MEDIA_TESTKIT_HOSTILE_CHILD";
@@ -60,10 +60,11 @@ fn a_bad_multiplier_names_the_variable() {
#[test]
fn a_panic_keeps_its_message() {
let payload = std::panic::catch_unwind(|| hammer("demo", &[9, 8, 7, 6], 0, |d| assert!(d.len() != 2, "boom at {}", d.len()))).unwrap_err();
// Truncation to 0 runs under any CRAFT_HOSTILE_DIVISOR.
let payload = std::panic::catch_unwind(|| hammer("demo", &[9, 8, 7, 6], 0, |d| assert!(!d.is_empty(), "boom at {}", d.len()))).unwrap_err();
let message = payload.downcast_ref::<String>().cloned().unwrap_or_default();
assert!(message.starts_with("demo: truncation to 2 (len 2): the parser panicked: "), "{message}");
assert!(message.contains("boom at 2"), "{message}");
assert!(message.starts_with("demo: truncation to 0 (len 0): the parser panicked: "), "{message}");
assert!(message.contains("boom at 0"), "{message}");
}
#[test]
@@ -74,8 +75,8 @@ fn every_truncation_of_a_large_seed_runs() {
hammer("large", &seed, 0, move |d| {
record.lock().unwrap().insert(d.len());
});
// The seed and its 8,192 truncations.
assert_eq!(lengths.lock().unwrap().len(), 8_193);
// The seed and its 8,192 truncations, every N-th of them under CRAFT_HOSTILE_DIVISOR=N.
assert_eq!(lengths.lock().unwrap().len(), 1 + 8_192usize.div_ceil(divisor()));
}
#[test]
+34 -8
View File
@@ -51,27 +51,53 @@ fn sync_before_on_an_empty_track() {
}
}
#[test]
fn hostile_input_never_panics() {
// Keep each deterministic hammer independent so test runners can schedule the seeds separately.
// The labels, mutation counts, every-truncation coverage and per-case watchdog stay unchanged.
fn hostile_seeds() -> [Vec<u8>; 3] {
let (to, flags) = ipbb();
let seeds = [build(7, |i| vec![i as u8; 100 + i], &to, &flags), build(5, prores_like, &to, &flags), build(4, prores_like, &[], &flags)];
for (i, s) in seeds.iter().enumerate() {
craft_media_testkit::hostile::hammer(&format!("mxf seed {i}"), s, 6_000, exercise);
}
// The metadata sets and index segments, parsed directly.
let seed = &seeds[0];
[build(7, |i| vec![i as u8; 100 + i], &to, &flags), build(5, prores_like, &to, &flags), build(4, prores_like, &[], &flags)]
}
#[test]
fn hostile_input_seed_0_never_panics() {
craft_media_testkit::hostile::hammer("mxf seed 0", &hostile_seeds()[0], 6_000, exercise);
}
#[test]
fn hostile_input_seed_1_never_panics() {
craft_media_testkit::hostile::hammer("mxf seed 1", &hostile_seeds()[1], 6_000, exercise);
}
#[test]
fn hostile_input_seed_2_never_panics() {
craft_media_testkit::hostile::hammer("mxf seed 2", &hostile_seeds()[2], 6_000, exercise);
}
#[test]
fn hostile_index_segment_never_panics() {
let seed = &hostile_seeds()[0];
let index_key = [0x06, 0x0E, 0x2B, 0x34, 0x02, 0x53, 0x01, 0x01, 0x0D, 0x01, 0x02, 0x01, 0x01, 0x10, 0x01, 0x00];
if let Some(v) = value_of(seed, &index_key) {
craft_media_testkit::hostile::hammer("mxf index segment", v, 30_000, |d| {
let _ = IndexSegment::parse(d);
});
}
}
#[test]
fn hostile_primer_never_panics() {
let seed = &hostile_seeds()[0];
let primer_key = [0x06, 0x0E, 0x2B, 0x34, 0x02, 0x05, 0x01, 0x01, 0x0D, 0x01, 0x02, 0x01, 0x01, 0x05, 0x01, 0x00];
if let Some(v) = value_of(seed, &primer_key) {
craft_media_testkit::hostile::hammer("mxf primer", v, 5_000, |d| {
let _ = Primer::parse(d);
});
}
}
#[test]
fn hostile_set_never_panics() {
let seed = &hostile_seeds()[0];
let set = seed.windows(16).position(|w| w == set_key(0x18)).and_then(|at| value_of(&seed[at..], &set_key(0x18)));
if let Some(v) = set {
craft_media_testkit::hostile::hammer("mxf set", v, 20_000, |d| {
+6 -4
View File
@@ -191,10 +191,12 @@ mod tests {
let p = Plates { inks, unseparated: Vec::new(), width: 2, height: 1, dpi: 72.0, conversion: Conversion::DeviceFormulas, planes };
let output = Output::new(fogra39(), fogra39(), Intent::RelativeColorimetric, false).unwrap();
let cmyk = p.composite_to(&[], &output);
// Magenta's round trip through the profile adds a little yellow.
assert!(near(&cmyk[0..4], &[255, 255, 8, 0], 8), "spot over cyan: {:?}", &cmyk[0..4]);
assert!(near(&cmyk[4..8], &[0, 255, 8, 0], 8), "spot alone: {:?}", &cmyk[4..8]);
assert!(near(&p.composite_to(&[true, true, true, true, false], &output)[0..4], &[255, 0, 0, 0], 6), "spot hidden");
// The press profile's round trip: its BToA tables generate their own black, so a
// full-tone ink comes back a little short with a trace of black, as a printer would
// separate it. The tolerance is the profile's own round trip, not an exchange total.
assert!(near(&cmyk[0..4], &[250, 247, 17, 2], 8), "spot over cyan: {:?}", &cmyk[0..4]);
assert!(near(&cmyk[4..8], &[8, 247, 11, 1], 8), "spot alone: {:?}", &cmyk[4..8]);
assert!(near(&p.composite_to(&[true, true, true, true, false], &output)[0..4], &[250, 7, 6, 1], 8), "spot hidden");
}
#[test]
+4 -4
View File
@@ -199,10 +199,10 @@ job) and `an_installed_printer_prints_a_file`
(`CRAFT_PRINT_TEST_PRINTER`, `CRAFT_PRINT_TEST_FILE`, optional `CRAFT_PRINT_TEST_SCALING=shrink`
and `CRAFT_PRINT_TEST_DUPLEX=long`). Physical printer fidelity remains a hardware check.
The cross-platform GitHub Actions workflow is defined in `.github/workflows/print.yml`.
It runs formatting, Clippy and this crate's own tests on Windows, Linux and macOS when print
code changes, and supports manual runs. The Windows job executes the spooler regression above;
physical printer fidelity still requires hardware verification.
The Workspace gate (`.github/workflows/workspace.yml`) runs formatting, Clippy and this crate's
tests on Windows, Linux and macOS. Its Windows leg installs the test print queue and runs the
spooler regression above with `CRAFT_PRINT_REQUIRE_QUEUE=1`; physical printer fidelity still
requires hardware verification.
Windows discovery reports driver duplex support in `Printer::two_sided` (a two-sided option in
the driver's PrintCapabilities); CUPS leaves it unknown. Explicit simplex/duplex choices that the driver
+70
View File
@@ -5,6 +5,48 @@
//! Unknown direction values retain PhotoCraft's vertical interpretation. Grid cycles, reserved
//! bytes and trailing data are not interpreted. Resource 1097 remains opaque except its count.
/// One legacy Grid and Guides record (resource 1032).
/// Position is signed 1/32 pixel; direction 1 is horizontal, all others vertical.
/// Adapted from PhotoCraft's reader by Brandon Thomas and writer by aiongg.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct GuideV1 {
/// Signed position in units of 1/32 pixel.
pub location: i32,
/// Raw direction byte, retained without normalization.
pub direction: u8,
}
impl GuideV1 {
/// Encode exactly one five-byte record without allocation.
pub fn to_bytes(self) -> [u8; 5] {
let p = self.location.to_be_bytes();
[p[0], p[1], p[2], p[3], self.direction]
}
}
/// Encode a legacy resource header. Grid cycles and count are caller supplied.
pub fn header_v1(horizontal_cycle: u32, vertical_cycle: u32, count: u32) -> [u8; 16] {
let mut out = [0; 16];
for (chunk, value) in out.as_chunks_mut::<4>().0.iter_mut().zip([1u32, horizontal_cycle, vertical_cycle, count]) {
chunk.copy_from_slice(&value.to_be_bytes());
}
out
}
/// Read complete legacy records, preserving the valid prefix on truncation.
/// Matches PhotoCraft recovery: version/grid fields are ignored. Declared counts
/// cannot drive allocation or scanning beyond complete records in the input.
pub fn parse_v1_prefix(data: &[u8]) -> impl Iterator<Item = GuideV1> + '_ {
let count = be_u32(data, 12).unwrap_or(0);
data.get(16..)
.unwrap_or_default()
.as_chunks::<5>()
.0
.iter()
.zip(0..count)
.map(|(g, _)| GuideV1 { location: i32::from_be_bytes([g[0], g[1], g[2], g[3]]), direction: g[4] })
}
/// Grid and Guides version 2 resource ID.
pub const GUIDES_V2: u16 = 1092;
/// Associated guide identifiers resource ID.
@@ -101,3 +143,31 @@ mod tests {
assert_eq!(ids_count(&[0, 0, 0]), None);
}
}
#[cfg(test)]
mod v1_tests {
use super::*;
#[test]
fn wire_roundtrip_and_signed_unknown_direction() {
let entries = [GuideV1 { location: -33, direction: 1 }, GuideV1 { location: i32::MAX, direction: 255 }];
let mut data = header_v1(576, 576, 2).to_vec();
assert_eq!(data, [0, 0, 0, 1, 0, 0, 2, 64, 0, 0, 2, 64, 0, 0, 0, 2]);
for e in entries {
data.extend(e.to_bytes());
}
assert_eq!(&data[16..21], &[255, 255, 255, 223, 1]);
assert_eq!(parse_v1_prefix(&data).collect::<Vec<_>>(), entries);
for end in 0..=data.len() {
assert_eq!(parse_v1_prefix(&data[..end]).count(), end.saturating_sub(16) / 5);
}
data[..4].copy_from_slice(&99u32.to_be_bytes());
data[12..16].copy_from_slice(&u32::MAX.to_be_bytes());
data.extend([9, 9, 9, 9]);
assert_eq!(parse_v1_prefix(&data).collect::<Vec<_>>(), entries);
data[12..16].copy_from_slice(&1u32.to_be_bytes());
assert_eq!(parse_v1_prefix(&data).collect::<Vec<_>>(), &entries[..1]);
data[12..16].fill(0);
assert_eq!(parse_v1_prefix(&data).count(), 0);
}
}
+15
View File
@@ -0,0 +1,15 @@
[package]
name = "craft-sourcemap"
description = "Bounded Source Map v3 parser, source inventory and generated-position lookup"
version.workspace = true
edition.workspace = true
license.workspace = true
rust-version.workspace = true
repository.workspace = true
[lints]
workspace = true
[dependencies]
serde = { workspace = true }
serde_json = { workspace = true }
thiserror = { workspace = true }
+816
View File
@@ -0,0 +1,816 @@
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
#![forbid(unsafe_code)]
//! Source maps (revision 3), including indexed maps with sections.
//!
//! Lookups follow the "original positions for a generated position" rule: on the generated line,
//! take the greatest segment column not after the requested column and return every segment that
//! starts there. Lines are 0-based here (callers add one for display); columns are 0-based UTF-16
//! code units as the format defines.
use serde::de::{self, DeserializeSeed, MapAccess, SeqAccess, Visitor};
use std::cell::Cell;
use std::fmt;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Limits {
/// Biggest map JSON accepted.
pub max_bytes: usize,
/// Most decoded segments across the whole map.
pub max_segments: usize,
/// Most generated lines across the whole map, including empty lines and inline sections.
pub max_lines: usize,
/// Deepest section nesting (the format allows one level; we tolerate a few).
pub max_depth: usize,
/// Most `sources`, `sourcesContent`, `names` and ignore-list entries and index-map sections
/// across the whole map, counted while the JSON is read, so a map of tiny entries is refused
/// before it is kept. An entry a repeated key replaces gives its slot back.
pub max_entries: usize,
}
impl Default for Limits {
fn default() -> Self {
Limits { max_bytes: 64 * 1024 * 1024, max_segments: 4_000_000, max_lines: 1_000_000, max_depth: 8, max_entries: 4_000_000 }
}
}
/// Limits on retained source names after applying each map's root.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct InventoryLimits {
pub max_sources: usize,
/// Total UTF-8 bytes in the expanded source paths, across all sections.
pub max_bytes: usize,
}
impl Default for InventoryLimits {
fn default() -> Self {
Self { max_sources: 1_000_000, max_bytes: 64 * 1024 * 1024 }
}
}
/// An ordered prefix of the map's source names, with their complete identities.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct SourceInventory {
pub sources: Vec<String>,
/// More names existed than the limits or available allocations could retain.
pub truncated: bool,
/// Index-map sections that pointed at an external map nobody read, so the sources they
/// would add are unknown (not absent).
pub external_sections: usize,
}
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
pub enum SourceMapError {
#[error("source map is {0} bytes; the limit is {1}")]
TooLarge(usize, usize),
#[error("source map is not valid JSON: {0}")]
BadJson(String),
#[error("not a version 3 source map")]
BadVersion,
#[error("source map mappings are malformed at segment {0}")]
BadMappings(usize),
#[error("source map index has invalid sections")]
BadSections,
#[error("source map has more than {0} segments")]
TooManySegments(usize),
#[error("source map has more than {0} generated lines")]
TooManyLines(usize),
#[error("could not reserve memory for generated source-map lines")]
LineAllocationFailed,
#[error("source map sections nest deeper than {0}")]
TooDeep(usize),
#[error("source map has more than {0} source, content, name, ignore-list and section entries")]
TooManyEntries(usize),
}
/// One decoded mapping segment.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Segment {
pub generated_column: u32,
/// (source index, original line, original column, name index)
pub original: Option<(u32, u32, u32, Option<u32>)>,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct SourceMap {
pub file: Option<String>,
pub source_root: Option<String>,
pub sources: Vec<Option<String>>,
pub sources_content: Vec<Option<String>>,
pub names: Vec<String>,
pub ignore_list: Vec<u32>,
/// Segments per generated line, sorted by column.
pub lines: Vec<Vec<Segment>>,
pub segments: usize,
/// Indexed maps: (line offset, column offset, map).
pub sections: Vec<(u32, u32, SourceMap)>,
/// Index-map sections that point at an external map (`url`), which isn't fetched here: the
/// sources that map would add are unknown, not absent.
pub external_sections: usize,
}
/// A position in an original source.
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize)]
pub struct OriginalPosition {
pub source_index: u32,
pub source: Option<String>,
/// 0-based.
pub line: u32,
/// 0-based.
pub column: u32,
pub name: Option<String>,
/// 0-based generated column of the matched segment.
pub generated_column: u32,
}
impl SourceMap {
/// The source path with `sourceRoot` prefixed before URL interpretation.
pub fn source_path(&self, index: u32) -> Option<String> {
let s = self.sources.get(index as usize)?.as_ref()?;
// Generators commonly emit an empty root to mean no root. Keep those source identities
// unchanged even though the literal ECMA-426 prefix algorithm would prepend a slash.
Some(match self.source_root.as_deref().filter(|root| !root.is_empty()) {
Some(root) => {
let join = if root.ends_with('/') { "" } else { "/" };
format!("{root}{join}{s}")
}
None => s.clone(),
})
}
pub fn source_content(&self, index: u32) -> Option<&str> {
self.sources_content.get(index as usize)?.as_deref()
}
/// Every original position for a generated (line, column), both 0-based.
pub fn original_positions(&self, line: u32, column: u32) -> Vec<OriginalPosition> {
if !self.sections.is_empty() {
// Last section that starts at or before the position.
let end = self.sections.partition_point(|(l, c, _)| (*l, *c) <= (line, column));
let Some((l, c, map)) = end.checked_sub(1).and_then(|i| self.sections.get(i)) else { return Vec::new() };
let rel_line = line.saturating_sub(*l);
let rel_col = if rel_line == 0 { column.saturating_sub(*c) } else { column };
return map
.original_positions(rel_line, rel_col)
.into_iter()
.filter_map(|mut position| {
if rel_line == 0 {
position.generated_column = position.generated_column.checked_add(*c)?;
}
Some(position)
})
.collect();
}
let Some(segs) = self.lines.get(line as usize) else { return Vec::new() };
// Greatest generated column <= column.
let idx = segs.partition_point(|s| s.generated_column <= column);
let Some(best) = idx.checked_sub(1).and_then(|i| segs.get(i)) else { return Vec::new() };
let col = best.generated_column;
let first = segs.partition_point(|s| s.generated_column < col);
segs.get(first..idx)
.unwrap_or_default()
.iter()
.filter_map(|s| {
let (src, l, c, n) = s.original?;
Some(OriginalPosition {
source_index: src,
source: self.source_path(src),
line: l,
column: c,
name: n.and_then(|n| self.names.get(n as usize).cloned()),
generated_column: col,
})
})
.collect()
}
/// Sources flattened across sections, within the default inventory limits.
/// Use `source_inventory` to choose limits and observe truncation.
pub fn all_sources(&self) -> Vec<String> {
self.source_inventory(InventoryLimits::default()).sources
}
/// Retain a bounded prefix, checking expanded lengths before constructing paths.
pub fn source_inventory(&self, limits: InventoryLimits) -> SourceInventory {
let mut sources = Vec::new();
let mut bytes_left = limits.max_bytes;
let truncated = !self.collect_sources(limits.max_sources, &mut bytes_left, &mut sources);
SourceInventory { sources, truncated, external_sections: self.external_sections_total() }
}
/// This map's external sections plus those of every map it holds.
fn external_sections_total(&self) -> usize {
self.sections.iter().fold(self.external_sections, |n, (_, _, map)| n.saturating_add(map.external_sections_total()))
}
/// Append this map and its sections using one shared entry and byte budget.
fn collect_sources(&self, max_sources: usize, bytes_left: &mut usize, out: &mut Vec<String>) -> bool {
let root = self.source_root.as_deref().unwrap_or_default();
let join = if root.is_empty() || root.ends_with('/') { "" } else { "/" };
for source in self.sources.iter().flatten() {
let Some(bytes) = root.len().checked_add(join.len()).and_then(|n| n.checked_add(source.len())) else { return false };
if out.len() >= max_sources || bytes > *bytes_left {
return false;
}
let mut path = String::new();
if out.try_reserve(1).is_err() || path.try_reserve_exact(bytes).is_err() {
return false;
}
path.push_str(root);
path.push_str(join);
path.push_str(source);
out.push(path);
*bytes_left = bytes_left.saturating_sub(bytes);
}
for (_, _, map) in &self.sections {
if !map.collect_sources(max_sources, bytes_left, out) {
return false;
}
}
true
}
}
/// Parse a source map.
pub fn parse(json: &[u8], limits: Limits) -> Result<SourceMap, SourceMapError> {
read(json, limits, true)
}
/// Read a map's sources, their content, names and ignore list without decoding its mappings or
/// checking its section offsets, so a map [`parse`] refuses for those
/// ([`SourceMapError::BadMappings`] or [`SourceMapError::BadSections`]) can still list its sources.
/// The map has no lines or segments. A map whose mappings, as far as they decode, name a source
/// past the end of its `sources` list is still refused, since the list is known to be short, and so are a section
/// that has neither a map nor a URL, or both, and a `sections` value that isn't an array.
pub fn parse_sources(json: &[u8], limits: Limits) -> Result<SourceMap, SourceMapError> {
read(json, limits, false)
}
fn read(json: &[u8], limits: Limits, mappings: bool) -> Result<SourceMap, SourceMapError> {
if json.len() > limits.max_bytes {
return Err(SourceMapError::TooLarge(json.len(), limits.max_bytes));
}
// Some servers prefix maps with `)]}'` to defeat XSSI; the format allows skipping that line.
let body = if json.starts_with(b")]}") { json.iter().position(|b| *b == b'\n').and_then(|p| json.get(p + 1..)).unwrap_or(&[]) } else { json };
let entries = Entries { left: Cell::new(limits.max_entries), exhausted: Cell::new(false) };
let mut de = serde_json::Deserializer::from_slice(body);
let raw = RawMapSeed(&entries).deserialize(&mut de).and_then(|raw| de.end().map(|()| raw));
let raw = match raw {
Ok(raw) => raw,
Err(_) if entries.exhausted.get() => return Err(SourceMapError::TooManyEntries(limits.max_entries)),
Err(e) => return Err(SourceMapError::BadJson(e.to_string())),
};
let mut budget = limits.max_segments;
let mut line_budget = limits.max_lines;
from_raw(raw, 0, limits, &mut budget, &mut line_budget, mappings)
}
/// `mappings`: decode the mappings and check section offsets, as [`parse`] does.
fn from_raw(
raw: Option<RawMap>,
depth: usize,
limits: Limits,
budget: &mut usize,
line_budget: &mut usize,
mappings: bool,
) -> Result<SourceMap, SourceMapError> {
if depth > limits.max_depth {
return Err(SourceMapError::TooDeep(limits.max_depth));
}
let Some(raw) = raw.filter(|raw| raw.version == Some(3)) else {
return Err(SourceMapError::BadVersion);
};
if let Some(sections) = raw.sections {
// The index has its own file name. Only its section maps describe original sources.
let mut map = SourceMap { file: raw.file, ..Default::default() };
let sections = sections.ok_or(SourceMapError::BadSections)?;
for section in sections {
let offset = |n: Option<u64>| n.and_then(|n| u32::try_from(n).ok()).ok_or(SourceMapError::BadSections);
let (line, col) = section.offset.unwrap_or_default();
let (line, col) = if mappings { (offset(line)?, offset(col)?) } else { (offset(line).unwrap_or(0), offset(col).unwrap_or(0)) };
if mappings && map.sections.last().is_some_and(|(l, c, _)| (*l, *c) > (line, col)) {
return Err(SourceMapError::BadSections);
}
let (sub, external) = match (section.map, section.url) {
(Some(m), None) => (from_raw(m.map(|m| *m), depth + 1, limits, budget, line_budget, mappings)?, false),
// External maps are never fetched here. Retain their start so a lookup
// cannot leak through to the previous inline map, and count it so the
// sources it would add are reported unknown rather than absent.
(None, Some(true)) => (SourceMap::default(), true),
_ => return Err(SourceMapError::BadSections),
};
if external {
map.external_sections = map.external_sections.saturating_add(1);
}
map.segments = map.segments.saturating_add(sub.segments);
map.sections.push((line, col, sub));
}
return Ok(map);
}
let mut map = SourceMap {
file: raw.file,
source_root: raw.source_root,
sources: raw.sources,
sources_content: raw.sources_content,
names: raw.names.into_iter().map(Option::unwrap_or_default).collect(),
ignore_list: raw
.ignore_list
.or(raw.google_ignore_list)
.unwrap_or_default()
.into_iter()
.filter_map(|n| n.and_then(|n| u32::try_from(n).ok()))
.collect(),
..Default::default()
};
let m = raw.mappings.as_deref().unwrap_or_default();
if mappings {
decode_mappings(m, &mut map, budget, line_budget, limits.max_lines)?;
} else if let Some(count) = past_sources(m, map.sources.len()) {
return Err(SourceMapError::BadMappings(count));
}
Ok(map)
}
/// The entry budget shared by every list in a map and its sections while the JSON is read.
struct Entries {
left: Cell<usize>,
exhausted: Cell<bool>,
}
impl Entries {
fn take<E: de::Error>(&self) -> Result<(), E> {
let Some(left) = self.left.get().checked_sub(1) else {
self.exhausted.set(true);
return Err(E::custom("too many source-map entries"));
};
self.left.set(left);
Ok(())
}
/// Return the slots of entries that won't be kept after all. Callers drop those entries
/// first, so a replaced list and its replacement are never held together.
fn give_back(&self, n: usize) {
self.left.set(self.left.get().saturating_add(n));
}
}
/// The fields a map is built from, read without keeping anything else. Any JSON value is
/// accepted where the format expects another kind; it reads as absent, as before.
#[derive(Default)]
struct RawMap {
version: Option<u64>,
file: Option<String>,
source_root: Option<String>,
sources: Vec<Option<String>>,
sources_content: Vec<Option<String>>,
names: Vec<Option<String>>,
/// Present even when it isn't an array, so `x_google_ignoreList` stays unused then.
ignore_list: Option<Vec<Option<u64>>>,
google_ignore_list: Option<Vec<Option<u64>>>,
mappings: Option<String>,
/// `Some(None)`: present but not an array.
sections: Option<Option<Vec<RawSection>>>,
}
impl RawMap {
/// The entry slots this map and its sections hold.
fn held(&self) -> usize {
let lists = |l: &Option<Vec<Option<u64>>>| l.as_ref().map_or(0, Vec::len);
self.sources.len()
+ self.sources_content.len()
+ self.names.len()
+ lists(&self.ignore_list)
+ lists(&self.google_ignore_list)
+ held_by(&self.sections)
}
}
fn held_by(sections: &Option<Option<Vec<RawSection>>>) -> usize {
sections.iter().flatten().flatten().map(|s| 1 + s.map.iter().flatten().map(|m| m.held()).sum::<usize>()).sum()
}
#[derive(Default)]
struct RawSection {
offset: Option<(Option<u64>, Option<u64>)>,
/// `Some(None)`: present but not an object. Boxed, so a section that isn't a map stays small.
map: Option<Option<Box<RawMap>>>,
/// Whether the URL is a string.
url: Option<bool>,
}
/// Visitor methods that read any other kind of value, without keeping it, as the default.
macro_rules! skip {
($de:lifetime; bool) => { fn visit_bool<E: de::Error>(self, _: bool) -> Result<Self::Value, E> { Ok(Default::default()) } };
($de:lifetime; i64) => { fn visit_i64<E: de::Error>(self, _: i64) -> Result<Self::Value, E> { Ok(Default::default()) } };
($de:lifetime; u64) => { fn visit_u64<E: de::Error>(self, _: u64) -> Result<Self::Value, E> { Ok(Default::default()) } };
($de:lifetime; f64) => { fn visit_f64<E: de::Error>(self, _: f64) -> Result<Self::Value, E> { Ok(Default::default()) } };
($de:lifetime; str) => { fn visit_str<E: de::Error>(self, _: &str) -> Result<Self::Value, E> { Ok(Default::default()) } };
($de:lifetime; unit) => { fn visit_unit<E: de::Error>(self) -> Result<Self::Value, E> { Ok(Default::default()) } };
($de:lifetime; seq) => {
fn visit_seq<A: SeqAccess<$de>>(self, mut seq: A) -> Result<Self::Value, A::Error> {
while seq.next_element_seed(Skip)?.is_some() {}
Ok(Default::default())
}
};
($de:lifetime; map) => {
fn visit_map<A: MapAccess<$de>>(self, mut map: A) -> Result<Self::Value, A::Error> {
while map.next_key_seed(Skip)?.is_some() {
map.next_value_seed(Skip)?;
}
Ok(Default::default())
}
};
($de:lifetime; $($kind:ident),+) => { $(skip!($de; $kind);)+ };
}
/// Read a value the map doesn't keep, with the checks a full read makes (string escapes and
/// UTF-8, number range, nesting depth), so skipping it can't let a broken map through.
#[derive(Clone, Copy)]
struct Skip;
impl<'de> DeserializeSeed<'de> for Skip {
type Value = ();
fn deserialize<D: de::Deserializer<'de>>(self, d: D) -> Result<(), D::Error> {
d.deserialize_any(self)
}
}
impl<'de> Visitor<'de> for Skip {
type Value = ();
fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.write_str("any JSON value")
}
skip!('de; bool, i64, u64, f64, str, unit, seq, map);
}
/// A string, or `None` for any other value.
#[derive(Clone, Copy)]
struct MaybeStr;
impl<'de> Visitor<'de> for MaybeStr {
type Value = Option<String>;
fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.write_str("any JSON value")
}
fn visit_str<E: de::Error>(self, s: &str) -> Result<Self::Value, E> {
Ok(Some(s.to_owned()))
}
fn visit_string<E: de::Error>(self, s: String) -> Result<Self::Value, E> {
Ok(Some(s))
}
skip!('de; bool, i64, u64, f64, unit, seq, map);
}
/// A non-negative integer, or `None` for any other value.
#[derive(Clone, Copy)]
struct MaybeU64;
impl<'de> Visitor<'de> for MaybeU64 {
type Value = Option<u64>;
fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.write_str("any JSON value")
}
fn visit_u64<E: de::Error>(self, n: u64) -> Result<Self::Value, E> {
Ok(Some(n))
}
skip!('de; bool, i64, f64, str, unit, seq, map);
}
/// One value read through `V`, after taking its slot from the entry budget.
struct Entry<'a, V>(&'a Entries, V);
impl<'de, V: Visitor<'de>> DeserializeSeed<'de> for Entry<'_, V> {
type Value = V::Value;
fn deserialize<D: de::Deserializer<'de>>(self, d: D) -> Result<Self::Value, D::Error> {
self.0.take()?;
d.deserialize_any(self.1)
}
}
/// A list whose entries count against the budget; any other value reads as an empty list.
struct EntryList<'a, V>(&'a Entries, V);
impl<'de, V: Visitor<'de> + Copy> Visitor<'de> for EntryList<'_, V> {
type Value = Vec<V::Value>;
fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.write_str("any JSON value")
}
fn visit_seq<A: SeqAccess<'de>>(self, mut seq: A) -> Result<Self::Value, A::Error> {
let mut out = Vec::new();
while let Some(value) = seq.next_element_seed(Entry(self.0, self.1))? {
out.push(value);
}
Ok(out)
}
skip!('de; bool, i64, u64, f64, str, unit, map);
}
impl<'de, V: Visitor<'de> + Copy> DeserializeSeed<'de> for EntryList<'_, V> {
type Value = Vec<V::Value>;
fn deserialize<D: de::Deserializer<'de>>(self, d: D) -> Result<Self::Value, D::Error> {
d.deserialize_any(self)
}
}
/// A map object, or `None` for any other value.
#[derive(Clone, Copy)]
struct RawMapSeed<'a>(&'a Entries);
impl<'de> Visitor<'de> for RawMapSeed<'_> {
type Value = Option<RawMap>;
fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.write_str("any JSON value")
}
fn visit_map<A: MapAccess<'de>>(self, mut fields: A) -> Result<Self::Value, A::Error> {
let mut raw = RawMap::default();
// A repeated key replaces the earlier value, as it did when the whole JSON was read.
while let Some(key) = fields.next_key::<Field>()? {
match key {
Field::Version => raw.version = fields.next_value_seed(Any(MaybeU64))?,
Field::File => raw.file = fields.next_value_seed(Any(MaybeStr))?,
Field::SourceRoot => raw.source_root = fields.next_value_seed(Any(MaybeStr))?,
Field::Sources => {
self.0.give_back(std::mem::take(&mut raw.sources).len());
raw.sources = fields.next_value_seed(EntryList(self.0, MaybeStr))?;
}
Field::SourcesContent => {
self.0.give_back(std::mem::take(&mut raw.sources_content).len());
raw.sources_content = fields.next_value_seed(EntryList(self.0, MaybeStr))?;
}
Field::Names => {
self.0.give_back(std::mem::take(&mut raw.names).len());
raw.names = fields.next_value_seed(EntryList(self.0, MaybeStr))?;
}
// `ignoreList` wins over `x_google_ignoreList` wherever either appears.
Field::IgnoreList => {
let unused = raw.ignore_list.take().into_iter().chain(raw.google_ignore_list.take());
self.0.give_back(unused.map(|l| l.len()).sum());
raw.ignore_list = Some(fields.next_value_seed(EntryList(self.0, MaybeU64))?);
}
Field::GoogleIgnoreList if raw.ignore_list.is_some() => fields.next_value_seed(Skip)?,
Field::GoogleIgnoreList => {
self.0.give_back(raw.google_ignore_list.take().map_or(0, |l| l.len()));
raw.google_ignore_list = Some(fields.next_value_seed(EntryList(self.0, MaybeU64))?);
}
Field::Mappings => raw.mappings = fields.next_value_seed(Any(MaybeStr))?,
Field::Sections => {
self.0.give_back(held_by(&raw.sections.take()));
raw.sections = Some(fields.next_value_seed(Any(Sections(self.0)))?);
}
Field::Other => fields.next_value_seed(Skip)?,
}
}
Ok(Some(raw))
}
skip!('de; bool, i64, u64, f64, str, unit, seq);
}
impl<'de> DeserializeSeed<'de> for RawMapSeed<'_> {
type Value = Option<RawMap>;
fn deserialize<D: de::Deserializer<'de>>(self, d: D) -> Result<Self::Value, D::Error> {
d.deserialize_any(self)
}
}
/// Read any JSON value through `V`.
struct Any<V>(V);
impl<'de, V: Visitor<'de>> DeserializeSeed<'de> for Any<V> {
type Value = V::Value;
fn deserialize<D: de::Deserializer<'de>>(self, d: D) -> Result<Self::Value, D::Error> {
d.deserialize_any(self.0)
}
}
#[derive(serde::Deserialize)]
#[serde(field_identifier, rename_all = "camelCase")]
enum Field {
Version,
File,
SourceRoot,
Sources,
SourcesContent,
Names,
IgnoreList,
#[serde(rename = "x_google_ignoreList")]
GoogleIgnoreList,
Mappings,
Sections,
#[serde(other)]
Other,
}
/// The sections array, or `None` for any other value.
struct Sections<'a>(&'a Entries);
impl<'de> Visitor<'de> for Sections<'_> {
type Value = Option<Vec<RawSection>>;
fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.write_str("any JSON value")
}
fn visit_seq<A: SeqAccess<'de>>(self, mut seq: A) -> Result<Self::Value, A::Error> {
let mut out = Vec::new();
while let Some(section) = seq.next_element_seed(Entry(self.0, Section(self.0)))? {
out.push(section);
}
Ok(Some(out))
}
skip!('de; bool, i64, u64, f64, str, unit, map);
}
/// One section; any value that isn't an object reads as a section without an offset.
struct Section<'a>(&'a Entries);
impl<'de> Visitor<'de> for Section<'_> {
type Value = RawSection;
fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.write_str("any JSON value")
}
fn visit_map<A: MapAccess<'de>>(self, mut fields: A) -> Result<Self::Value, A::Error> {
let mut section = RawSection::default();
while let Some(key) = fields.next_key::<SectionField>()? {
match key {
SectionField::Offset => section.offset = Some(fields.next_value_seed(Any(Offset))?),
SectionField::Map => {
self.0.give_back(section.map.take().flatten().map_or(0, |m| m.held()));
section.map = Some(fields.next_value_seed(RawMapSeed(self.0))?.map(Box::new));
}
SectionField::Url => section.url = Some(fields.next_value_seed(Any(MaybeStr))?.is_some()),
SectionField::Other => fields.next_value_seed(Skip)?,
}
}
Ok(section)
}
skip!('de; bool, i64, u64, f64, str, unit, seq);
}
#[derive(serde::Deserialize)]
#[serde(field_identifier, rename_all = "lowercase")]
enum SectionField {
Offset,
Map,
Url,
#[serde(other)]
Other,
}
/// A section offset's line and column; any value that isn't an object has neither.
struct Offset;
impl<'de> Visitor<'de> for Offset {
type Value = (Option<u64>, Option<u64>);
fn expecting(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.write_str("any JSON value")
}
fn visit_map<A: MapAccess<'de>>(self, mut fields: A) -> Result<Self::Value, A::Error> {
let (mut line, mut column) = (None, None);
while let Some(key) = fields.next_key::<OffsetField>()? {
match key {
OffsetField::Line => line = fields.next_value_seed(Any(MaybeU64))?,
OffsetField::Column => column = fields.next_value_seed(Any(MaybeU64))?,
OffsetField::Other => fields.next_value_seed(Skip)?,
}
}
Ok((line, column))
}
skip!('de; bool, i64, u64, f64, str, unit, seq);
}
#[derive(serde::Deserialize)]
#[serde(field_identifier, rename_all = "lowercase")]
enum OffsetField {
Line,
Column,
#[serde(other)]
Other,
}
fn b64(c: u8) -> Option<i64> {
Some(match c {
b'A'..=b'Z' => i64::from(c - b'A'),
b'a'..=b'z' => i64::from(c - b'a') + 26,
b'0'..=b'9' => i64::from(c - b'0') + 52,
b'+' => 62,
b'/' => 63,
_ => return None,
})
}
/// Decode one VLQ field from `bytes` at `*pos`.
fn vlq(bytes: &[u8], pos: &mut usize) -> Option<i64> {
let mut result = 0u32;
let mut shift = 0u32;
loop {
let d = b64(*bytes.get(*pos)?)? as u32;
*pos += 1;
let bits = d & 31;
if bits != 0 {
if bits > u32::MAX.checked_shr(shift)? {
return None;
}
result |= bits << shift;
}
if d & 32 == 0 {
break;
}
// Redundant zero groups do not change the value.
shift = shift.saturating_add(5);
}
let magnitude = i64::from(result >> 1);
Some(match (result & 1, magnitude) {
(1, 0) => i64::from(i32::MIN),
(1, n) => -n,
_ => magnitude,
})
}
/// Decode mapping deltas, rejecting fields that cannot name a declared source or symbol.
fn decode_mappings(m: &str, map: &mut SourceMap, budget: &mut usize, line_budget: &mut usize, line_limit: usize) -> Result<(), SourceMapError> {
let bytes = m.as_bytes();
let (mut src, mut ol, mut oc, mut name) = (0i64, 0i64, 0i64, 0i64);
let mut pos = 0usize;
let mut line: Vec<Segment> = Vec::new();
let mut col = 0i64;
let mut count = 0usize;
let to_u32 = |v: i64, count: usize| u32::try_from(v).map_err(|_| SourceMapError::BadMappings(count));
while pos <= bytes.len() {
match bytes.get(pos) {
None | Some(b';') => {
if *line_budget == 0 {
return Err(SourceMapError::TooManyLines(line_limit));
}
*line_budget -= 1;
line.sort_by_key(|s| s.generated_column);
map.lines.try_reserve(1).map_err(|_| SourceMapError::LineAllocationFailed)?;
map.lines.push(std::mem::take(&mut line));
col = 0;
pos += 1;
continue;
}
Some(b',') => {
pos += 1;
continue;
}
_ => {}
}
if *budget == 0 {
return Err(SourceMapError::TooManySegments(count));
}
*budget -= 1;
count += 1;
col += vlq(bytes, &mut pos).ok_or(SourceMapError::BadMappings(count))?;
let mut seg = Segment { generated_column: to_u32(col, count)?, original: None };
if !matches!(bytes.get(pos), None | Some(b',') | Some(b';')) {
src += vlq(bytes, &mut pos).ok_or(SourceMapError::BadMappings(count))?;
ol += vlq(bytes, &mut pos).ok_or(SourceMapError::BadMappings(count))?;
oc += vlq(bytes, &mut pos).ok_or(SourceMapError::BadMappings(count))?;
let mut n = None;
if !matches!(bytes.get(pos), None | Some(b',') | Some(b';')) {
name += vlq(bytes, &mut pos).ok_or(SourceMapError::BadMappings(count))?;
n = Some(to_u32(name, count)?);
}
let source = to_u32(src, count)?;
if source as usize >= map.sources.len() || n.is_some_and(|index| index as usize >= map.names.len()) {
return Err(SourceMapError::BadMappings(count));
}
seg.original = Some((source, to_u32(ol, count)?, to_u32(oc, count)?, n));
}
if !matches!(bytes.get(pos), None | Some(b',') | Some(b';')) {
return Err(SourceMapError::BadMappings(count));
}
line.push(seg);
}
map.segments = count;
Ok(())
}
/// The segment, counted as [`decode_mappings`] does, that names a source past the end of a list
/// of `sources`. Mappings that can't be decoded end the search: past them nothing is known.
fn past_sources(m: &str, sources: usize) -> Option<usize> {
let bytes = m.as_bytes();
let (mut pos, mut src, mut count) = (0usize, 0i64, 0usize);
let at_end = |pos: usize| matches!(bytes.get(pos), None | Some(b',') | Some(b';'));
while pos < bytes.len() {
if matches!(bytes[pos], b',' | b';') {
pos += 1;
continue;
}
count += 1;
vlq(bytes, &mut pos)?;
if at_end(pos) {
continue;
}
src += vlq(bytes, &mut pos)?;
// A negative index is wrong too, but the deltas after it still decode.
if usize::try_from(src).is_ok_and(|src| src >= sources) {
return Some(count);
}
while !at_end(pos) {
vlq(bytes, &mut pos)?;
}
}
None
}
#[cfg(test)]
mod tests;
+122
View File
@@ -0,0 +1,122 @@
use super::*;
#[test]
fn mapping_indices_must_refer_to_declared_entries() {
for mappings in ["ACAA", "AAAAC"] {
let json = serde_json::json!({"version":3,"sources":["source.js"],"names":["name"],"mappings":mappings});
assert!(matches!(parse(json.to_string().as_bytes(), Limits::default()), Err(SourceMapError::BadMappings(1))));
}
let valid = br#"{"version":3,"sources":[null],"names":["name"],"mappings":"AAAAA"}"#;
assert!(parse(valid, Limits::default()).is_ok());
}
#[test]
fn lookup_keeps_coincident_mappings_and_stops_at_an_unmapped_segment() {
let map = parse(br#"{"version":3,"sources":["a","b"],"mappings":"EAAA,ACAA,E"}"#, Limits::default()).unwrap();
assert!(map.original_positions(0, 1).is_empty());
let matches = map.original_positions(0, 3);
assert_eq!(matches.iter().map(|p| p.source_index).collect::<Vec<_>>(), [0, 1]);
assert!(map.original_positions(0, 4).is_empty());
}
#[test]
fn relative_sources_keep_the_url_root_delimiter() {
for (root, expected) in [
("webpack:///", "webpack:///src/main.js"),
("file:///", "file:///src/main.js"),
("https://host/scripts/", "https://host/scripts/src/main.js"),
("src", "src/src/main.js"),
] {
let json = serde_json::json!({"version":3,"sourceRoot":root,"sources":["src/main.js"],"mappings":"AAAA"});
let map = parse(json.to_string().as_bytes(), Limits::default()).unwrap();
assert_eq!(map.all_sources(), [expected]);
assert_eq!(map.original_positions(0, 0)[0].source.as_deref(), Some(expected));
}
}
#[test]
fn decodes_and_looks_up() {
// Generated "a;b" from two lines of src.js.
let m = br#"{"version":3,"sources":["src.js"],"names":["foo"],"mappings":"AAAAA,EAAE;AACA","sourcesContent":["x\ny"]}"#;
let map = parse(m, Limits::default()).unwrap();
assert_eq!(map.lines.len(), 2);
let p = map.original_positions(0, 1);
assert_eq!(p[0].source.as_deref(), Some("src.js"));
assert_eq!(p[0].name.as_deref(), Some("foo"));
assert_eq!((p[0].line, p[0].column), (0, 0));
let q = map.original_positions(0, 2);
assert_eq!(q[0].column, 2);
let r = map.original_positions(1, 0);
assert_eq!(r[0].line, 1);
assert!(map.original_positions(5, 0).is_empty());
assert_eq!(map.source_content(0), Some("x\ny"));
}
#[test]
fn indexed_maps() {
let m = br#"{"version":3,"sections":[{"offset":{"line":0,"column":0},"map":{"version":3,"sources":["a.js"],"names":[],"mappings":"AAAA"}},{"offset":{"line":1,"column":0},"map":{"version":3,"sources":["b.js"],"names":[],"mappings":"AAAA"}}]}"#;
let map = parse(m, Limits::default()).unwrap();
assert_eq!(map.original_positions(1, 3)[0].source.as_deref(), Some("b.js"));
assert_eq!(map.all_sources(), vec!["a.js", "b.js"]);
}
#[test]
fn validates_index_map_shape_and_offsets() {
let inline = serde_json::json!({"version":3,"mappings":"A"});
for sections in [
serde_json::json!({}),
serde_json::json!([{"map":inline}]),
serde_json::json!([{"offset":{"line":-1,"column":0},"map":inline}]),
serde_json::json!([{"offset":{"line":0,"column":4294967296_u64},"map":inline}]),
serde_json::json!([{"offset":{"line":0,"column":0}}]),
serde_json::json!([{"offset":{"line":0,"column":0},"map":inline,"url":"later.map"}]),
serde_json::json!([{"offset":{"line":0,"column":2},"map":inline},{"offset":{"line":0,"column":1},"map":inline}]),
] {
let json = serde_json::json!({"version":3,"sections":sections}).to_string();
assert_eq!(parse(json.as_bytes(), Limits::default()), Err(SourceMapError::BadSections));
}
}
#[test]
fn an_external_section_ends_the_previous_inline_mapping() {
let inline = serde_json::json!({"version":3,"sources":["first.js"],"mappings":"AAAA"});
let sections = serde_json::json!([
{"offset":{"line":0,"column":0},"map":inline},
{"offset":{"line":0,"column":4},"url":"external.map"},
{"offset":{"line":1,"column":0},"map":inline}
]);
let json = serde_json::json!({"version":3,"sections":sections}).to_string();
let map = parse(json.as_bytes(), Limits::default()).unwrap();
assert_eq!(map.original_positions(0, 3)[0].source.as_deref(), Some("first.js"));
assert!(map.original_positions(0, 4).is_empty());
assert_eq!(map.original_positions(1, 0)[0].generated_column, 0);
}
#[test]
fn vlq_signed_extremes_and_overflow() {
for (field, value) in [
("B", Some(i64::from(i32::MIN))),
("+/////D", Some(i64::from(i32::MAX))),
("//////D", Some(-i64::from(i32::MAX))),
("ggggggE", None),
("ggggggggggA", Some(0)),
] {
assert_eq!(vlq(field.as_bytes(), &mut 0), value, "{field}");
}
let map = parse(br#"{"version":3,"mappings":"+/////D,C,B"}"#, Limits::default()).unwrap();
assert_eq!(map.lines[0].iter().map(|s| s.generated_column).collect::<Vec<_>>(), [0, 2147483647, 2147483648]);
assert!(parse(br#"{"version":3,"mappings":"B"}"#, Limits::default()).is_err());
}
#[test]
fn hostile_mappings_do_not_panic() {
for m in ["", ";;;", "A", "gggggggggggggggggggggggggg", "AAAA,", "////", "AAAAAAAAAAAAAAA", "D", "AD,AA", "+", "!!"] {
let j = format!(r#"{{"version":3,"sources":[],"names":[],"mappings":"{m}"}}"#);
if let Ok(map) = parse(j.as_bytes(), Limits::default()) {
let _ = map.original_positions(0, 0);
let _ = map.original_positions(u32::MAX, u32::MAX);
}
}
let tight = Limits { max_segments: 1, ..Limits::default() };
assert!(parse(br#"{"version":3,"sources":[],"names":[],"mappings":"A,A"}"#, tight).is_err());
assert!(parse(b"[]", Limits::default()).is_err());
}
+145
View File
@@ -0,0 +1,145 @@
use craft_sourcemap::{Limits, SourceMap, SourceMapError, parse};
use serde_json::json;
fn with(max_entries: usize) -> Limits {
Limits { max_entries, ..Limits::default() }
}
fn decode(value: serde_json::Value, max_entries: usize) -> Result<SourceMap, SourceMapError> {
parse(&serde_json::to_vec(&value).unwrap(), with(max_entries))
}
#[test]
fn entries_up_to_the_limit_are_kept() {
let value = json!({"version":3,"sources":["a.js","b.js"],"sourcesContent":["a();",null],"names":["x"],"mappings":"AAAAA,CCAA"});
let map = decode(value.clone(), 5).unwrap();
assert_eq!(map.sources, vec![Some("a.js".to_string()), Some("b.js".to_string())]);
assert_eq!(map.sources_content, vec![Some("a();".to_string()), None]);
assert_eq!(map.names, vec!["x".to_string()]);
assert_eq!(decode(value, 4), Err(SourceMapError::TooManyEntries(4)));
}
#[test]
fn every_kind_of_entry_counts_once() {
for value in [
json!({"version":3,"sources":["a","b","c"],"mappings":""}),
json!({"version":3,"sourcesContent":["a","b","c"],"mappings":""}),
json!({"version":3,"names":["a","b","c"],"mappings":""}),
json!({"version":3,"ignoreList":[0,1,2],"mappings":""}),
json!({"version":3,"x_google_ignoreList":[0,1,2],"mappings":""}),
// Entries that are not strings still take a slot each.
json!({"version":3,"sources":[null, 1, {"a":[1,2,3]}],"mappings":""}),
] {
assert_eq!(decode(value.clone(), 2), Err(SourceMapError::TooManyEntries(2)), "{value}");
assert!(decode(value.clone(), 3).is_ok(), "{value}");
}
}
#[test]
fn entries_a_repeated_key_replaces_give_their_slots_back() {
let json = br#"{"version":3,"mappings":"","sources":["a","b"],"sources":["c"],"names":["x"],"names":[]}"#;
let map = parse(json, with(2)).unwrap();
assert_eq!(map.sources, vec![Some("c".to_string())]);
assert!(map.names.is_empty());
// `ignoreList` is kept over `x_google_ignoreList`, before or after it.
let first = br#"{"version":3,"mappings":"","x_google_ignoreList":[0,0],"ignoreList":[0]}"#;
assert_eq!(parse(first, with(2)).unwrap().ignore_list, vec![0]);
let after = br#"{"version":3,"mappings":"","ignoreList":[0],"x_google_ignoreList":[0,0,0]}"#;
assert_eq!(parse(after, with(1)).unwrap().ignore_list, vec![0]);
// A replaced sections array gives back its sections and what their maps held: without that,
// the second array would need slots four and five.
let sections = br#"{"version":3,"sections":[{"offset":{"line":0,"column":0},"map":{"version":3,"sources":["a"],"mappings":""}},{"offset":{"line":1,"column":0},"url":"b.map"}],"sections":[{"offset":{"line":0,"column":0},"url":"c.map"}]}"#;
assert_eq!(parse(sections, with(3)).unwrap().sections.len(), 1);
// So does a section's replaced map: one section, then one source instead of two.
let map = br#"{"version":3,"sections":[{"offset":{"line":0,"column":0},"map":{"version":3,"sources":["a","b"],"mappings":""},"map":{"version":3,"sources":["c"],"mappings":""}}]}"#;
assert_eq!(parse(map, with(3)).unwrap().sections[0].2.sources, vec![Some("c".to_string())]);
}
#[test]
fn sections_and_their_maps_share_the_budget() {
let section = |line: u32, name: &str| json!({"offset":{"line":line,"column":0},"map":{"version":3,"sources":[name],"mappings":"AAAA"}});
let value = json!({"version":3,"sections":[section(0, "a.js"), section(1, "b.js")]});
// Two sections and one source in each.
assert_eq!(decode(value.clone(), 4).unwrap().sections.len(), 2);
assert_eq!(decode(value, 3), Err(SourceMapError::TooManyEntries(3)));
// Sections without a map count too, so a long list of them is refused early.
let external = json!({"offset":{"line":0,"column":0},"url":"a.map"});
let value = json!({"version":3,"sections":[external.clone(), external.clone(), external]});
assert_eq!(decode(value.clone(), 3).unwrap().sections.len(), 3);
assert_eq!(decode(value, 2), Err(SourceMapError::TooManyEntries(2)));
let not_sections = br#"{"version":3,"sections":[0,0,0]}"#;
assert_eq!(parse(not_sections, with(2)), Err(SourceMapError::TooManyEntries(2)));
assert_eq!(parse(not_sections, with(3)), Err(SourceMapError::BadSections));
}
#[test]
fn reading_stops_at_the_limit() {
// Everything after the entry that goes over the limit is never read: the broken tail would
// otherwise be reported as bad JSON.
let json = br#"{"version":3,"sources":["a","b","c","d" this is not JSON"#;
assert_eq!(parse(json, with(2)), Err(SourceMapError::TooManyEntries(2)));
assert!(matches!(parse(json, Limits::default()), Err(SourceMapError::BadJson(_))));
}
#[test]
fn unknown_fields_are_skipped_without_counting() {
let value = json!({"version":3,"x_vendor":[1,2,3,4,5,6,7,8],"sources":["a.js"],"mappings":"AAAA"});
assert_eq!(decode(value, 1).unwrap().sources.len(), 1);
}
#[test]
fn values_the_map_does_not_keep_are_still_checked() {
let deep = format!("{{\"version\":3,\"mappings\":\"\",\"x\":{}{}}}", "[".repeat(200), "]".repeat(200));
for json in [
br#"{"version":3,"mappings":"","x":"\ud800"}"#.as_slice(),
br#"{"version":3,"mappings":"","sources":{"a":"\udc00"}}"#,
br#"{"version":3,"mappings":"","x":1e400}"#,
br#"{"version":3,"mappings":"","sources":[{"a":1e400}]}"#,
b"{\"version\":3,\"mappings\":\"\",\"x\":\"\xff\"}",
br#"{"version":3,"mappings":"","x":[1,]}"#,
deep.as_bytes(),
] {
let got = parse(json, Limits::default());
let full = serde_json::from_slice::<serde_json::Value>(json).unwrap_err().to_string();
assert_eq!(got, Err(SourceMapError::BadJson(full)), "{}", String::from_utf8_lossy(json));
}
}
#[test]
fn a_hostile_map_of_empty_names_is_refused_by_default() {
// About 21 million empty strings fit in the 64 MiB a map may have.
let limit = Limits::default().max_entries;
let names = vec!["\"\""; limit + 1].join(",");
let json = format!("{{\"version\":3,\"mappings\":\"\",\"names\":[{names}]}}").into_bytes();
assert_eq!(parse(&json, Limits::default()), Err(SourceMapError::TooManyEntries(limit)));
// The same map is well formed: with room for every name it reads.
assert_eq!(parse(&json, with(limit + 1)).unwrap().names.len(), limit + 1);
}
#[test]
fn sources_are_read_from_a_map_with_broken_mappings_or_offsets() {
use craft_sourcemap::parse_sources;
let bad = br#"{"version":3,"sourceRoot":"src","sources":["a.js"],"sourcesContent":["a();"],"names":["x"],"mappings":"!!"}"#;
assert_eq!(parse(bad, Limits::default()), Err(SourceMapError::BadMappings(1)));
let map = parse_sources(bad, Limits::default()).unwrap();
assert_eq!((map.source_path(0).as_deref(), map.sources_content.len(), map.names.len(), map.lines.len()), (Some("src/a.js"), 1, 1, 0));
let unordered = br#"{"version":3,"sections":[{"offset":{"line":2,"column":0},"map":{"version":3,"sources":["a.js"],"mappings":"AAAA"}},{"offset":{"line":-1,"column":0},"map":{"version":3,"sources":["b.js"],"mappings":""}},{"offset":{"line":0,"column":0},"map":{"version":3,"sources":["c.js"],"mappings":"!"}}]}"#;
assert_eq!(parse(unordered, Limits::default()), Err(SourceMapError::BadSections));
let map = parse_sources(unordered, Limits::default()).unwrap();
assert_eq!(map.source_inventory(Default::default()).sources, vec!["a.js".to_string(), "b.js".to_string(), "c.js".to_string()]);
// Past mappings that can't be decoded, the indices aren't known.
assert_eq!(parse_sources(br#"{"version":3,"sources":["a"],"mappings":"AAAA,!,ACAA"}"#, Limits::default()).unwrap().sources.len(), 1);
// What a section holds must still be known, and the version and limits still apply.
for (json, limit, err) in [
(br#"{"version":3,"sections":[{"offset":{"line":0,"column":0}}]}"#.as_slice(), 9, SourceMapError::BadSections),
(br#"{"version":3,"sections":{}}"#, 9, SourceMapError::BadSections),
(br#"{"version":3,"sections":[{"offset":{"line":0,"column":0},"map":{"version":2,"sources":["a"]}}]}"#, 9, SourceMapError::BadVersion),
(br#"{"version":3,"sources":["a","b"],"mappings":"!"}"#, 1, SourceMapError::TooManyEntries(1)),
// Mappings that name a source past the end of the list show it is short.
(br#"{"version":3,"mappings":"AAAA"}"#, 9, SourceMapError::BadMappings(1)),
(br#"{"version":3,"sources":["a"],"mappings":"AAAA;ACAA,!"}"#, 9, SourceMapError::BadMappings(2)),
(br#"{"version":3,"sources":["a"],"mappings":"AAAA,ADAA,AEAA"}"#, 9, SourceMapError::BadMappings(3)),
] {
assert_eq!(parse_sources(json, with(limit)), Err(err), "{}", String::from_utf8_lossy(json));
}
}
@@ -0,0 +1,84 @@
//! An index-map section that points at an external map (`url`) adds no sources here, but its
//! sources are unknown rather than absent: the inventory says how many such sections it saw.
use craft_sourcemap::{InventoryLimits, Limits, SourceMap, parse, parse_sources};
use serde_json::{Value, json};
fn decode(value: Value) -> SourceMap {
parse(&serde_json::to_vec(&value).unwrap(), Limits::default()).unwrap()
}
/// An external section adds no sources but is counted, so the inventory can say part of the map
/// was never read.
#[test]
fn an_external_section_is_counted_and_adds_no_sources() {
let map = decode(json!({
"version": 3,
"sections": [
{"offset": {"line": 0, "column": 0}, "map": {"version": 3, "sources": ["a.js"], "mappings": ""}},
{"offset": {"line": 10, "column": 0}, "url": "vendor.js.map"}
]
}));
assert_eq!(map.all_sources(), ["a.js"]);
assert_eq!(map.external_sections, 1);
let inventory = map.source_inventory(InventoryLimits::default());
assert_eq!(inventory.sources, ["a.js"]);
assert!(!inventory.truncated);
assert_eq!(inventory.external_sections, 1);
}
/// The count reaches an external section inside a nested index map.
#[test]
fn external_sections_are_counted_through_nested_index_maps() {
let map = decode(json!({
"version": 3,
"sections": [
{"offset": {"line": 0, "column": 0}, "map": {"version": 3, "sections": [
{"offset": {"line": 0, "column": 0}, "map": {"version": 3, "sources": ["a.js"], "mappings": ""}},
{"offset": {"line": 4, "column": 0}, "url": "inner.js.map"}
]}},
{"offset": {"line": 20, "column": 0}, "url": "outer.js.map"}
]
}));
assert_eq!(map.all_sources(), ["a.js"]);
assert_eq!(map.source_inventory(InventoryLimits::default()).external_sections, 2);
}
/// A map with only inline sections reports nothing unread.
#[test]
fn an_inline_only_map_counts_no_external_sections() {
let map = decode(json!({
"version": 3,
"sections": [{"offset": {"line": 0, "column": 0}, "map": {"version": 3, "sources": ["a.js"], "mappings": ""}}]
}));
assert_eq!(map.source_inventory(InventoryLimits::default()).external_sections, 0);
}
/// A map read for its sources alone (its mappings are refused) still says an external section
/// left part of the list unread.
#[test]
fn a_sources_only_read_still_counts_external_sections() {
let bytes = serde_json::to_vec(&json!({
"version": 3,
"sections": [
{"offset": {"line": 0, "column": 0}, "map": {"version": 3, "sources": ["a.js"]}},
{"offset": {"line": 8, "column": 0}, "url": "vendor.js.map"}
]
}))
.unwrap();
let map = parse_sources(&bytes, Limits::default()).unwrap();
assert_eq!(map.source_inventory(InventoryLimits::default()).external_sections, 1);
}
/// The count spends no inventory budget: a map with many external sections still retains its
/// inline sources in full.
#[test]
fn many_external_sections_do_not_spend_the_source_budget() {
let mut sections: Vec<Value> = (0..1_000).map(|i| json!({"offset": {"line": i, "column": 0}, "url": format!("{i}.map")})).collect();
sections.push(json!({"offset": {"line": 1_000, "column": 0}, "map": {"version": 3, "sources": ["a.js", "b.js"], "mappings": ""}}));
let map = decode(json!({"version": 3, "sections": sections}));
let inventory = map.source_inventory(InventoryLimits { max_sources: 2, max_bytes: 16 });
assert_eq!(inventory.sources, ["a.js", "b.js"]);
assert!(!inventory.truncated);
assert_eq!(inventory.external_sections, 1_000);
}
+104
View File
@@ -0,0 +1,104 @@
use craft_sourcemap::{InventoryLimits, Limits, SourceMap, parse};
use serde_json::{Value, json};
fn decode(value: Value) -> SourceMap {
parse(&serde_json::to_vec(&value).unwrap(), Limits::default()).unwrap()
}
#[test]
fn all_sources_bounds_expanded_path_bytes_before_collecting() {
let root = "r".repeat(2 * 1024 * 1024);
let bytes = serde_json::to_vec(&json!({
"version": 3, "sourceRoot": root, "sources": vec!["/x.js"; 40], "mappings": ""
}))
.unwrap();
let map = parse(&bytes, Limits::default()).unwrap();
let sources = map.all_sources();
assert!(sources.iter().map(String::len).sum::<usize>() <= 64 * 1024 * 1024);
assert_eq!(sources.len(), 31);
assert!(sources.iter().all(|source| source.ends_with("//x.js")));
}
#[test]
fn repeated_large_roots_stop_before_relative_and_absolute_looking_paths_expand() {
for source in ["x.js", "/x.js", "https://example.test/x.js"] {
let bytes = serde_json::to_vec(&json!({
"version": 3, "sourceRoot": "r".repeat(1024 * 1024),
"sources": vec![source; 10_000], "mappings": ""
}))
.unwrap();
assert!(bytes.len() < 2 * 1024 * 1024);
let map = parse(&bytes, Limits::default()).unwrap();
let inventory = map.source_inventory(InventoryLimits { max_sources: 5000, max_bytes: 2 * 1024 * 1024 });
assert_eq!(inventory.sources.len(), 1);
assert_eq!(inventory.sources[0].len(), 1024 * 1024 + 1 + source.len());
assert!(inventory.sources[0].ends_with(source));
assert!(inventory.truncated);
}
}
#[test]
fn byte_limits_preserve_complete_utf8_identities_at_the_exact_boundary() {
let map = decode(json!({"version": 3, "sourceRoot": "é/", "sources": ["東京.js", "東京.js", null], "mappings": ""}));
let expected = "é/東京.js";
let limits = InventoryLimits { max_sources: 2, max_bytes: 2 * expected.len() };
let exact = map.source_inventory(limits);
assert_eq!(exact.sources, [expected, expected]);
assert!(!exact.truncated);
let short = map.source_inventory(InventoryLimits { max_bytes: limits.max_bytes - 1, ..limits });
assert_eq!(short.sources, [expected]);
assert!(short.truncated);
let too_small = map.source_inventory(InventoryLimits { max_bytes: expected.len() - 1, ..limits });
assert!(too_small.sources.is_empty());
assert!(too_small.truncated);
}
#[test]
fn entry_limits_count_empty_paths_but_not_null_sources() {
let map = decode(json!({"version": 3, "sourceRoot": "", "sources": [null, "", "", null], "mappings": ""}));
let exact = map.source_inventory(InventoryLimits { max_sources: 2, max_bytes: 0 });
assert_eq!(exact.sources, ["", ""]);
assert!(!exact.truncated);
let one = map.source_inventory(InventoryLimits { max_sources: 1, max_bytes: 0 });
assert_eq!(one.sources, [""]);
assert!(one.truncated);
let zero = map.source_inventory(InventoryLimits { max_sources: 0, max_bytes: 0 });
assert!(zero.sources.is_empty());
assert!(zero.truncated);
let absent = decode(json!({"version": 3, "sources": [null], "mappings": ""}));
assert!(!absent.source_inventory(InventoryLimits { max_sources: 0, max_bytes: 0 }).truncated);
}
#[test]
fn indexed_sections_share_entry_and_byte_limits_in_source_order() {
let map = decode(json!({"version": 3, "sourceRoot": "outer", "sections": [
{"offset": {"line": 0, "column": 0}, "map": {
"version": 3, "sourceRoot": "src", "sources": [null, "/first.js"], "mappings": ""
}},
{"offset": {"line": 1, "column": 0}, "url": "external.map"},
{"offset": {"line": 2, "column": 0}, "map": {"version": 3, "sections": [
{"offset": {"line": 0, "column": 0}, "map": {
"version": 3, "sourceRoot": "file:///", "sources": ["second.js", "third.js"], "mappings": ""
}}
]}}
]}));
let expected = ["src//first.js", "file:///second.js", "file:///third.js"];
let complete = map.source_inventory(InventoryLimits { max_sources: 3, max_bytes: expected.iter().map(|s| s.len()).sum() });
assert_eq!(complete.sources, expected);
assert!(!complete.truncated);
let bytes = map.source_inventory(InventoryLimits { max_sources: 3, max_bytes: expected[0].len() + expected[1].len() });
assert_eq!(bytes.sources, expected[..2]);
assert!(bytes.truncated);
let entries = map.source_inventory(InventoryLimits { max_sources: 1, max_bytes: 100 });
assert_eq!(entries.sources, expected[..1]);
assert!(entries.truncated);
}
#[test]
fn an_oversized_path_stops_the_prefix_without_shortening_or_skipping_identities() {
let map = decode(json!({"version": 3, "sources": ["a.js", "b".repeat(40), "c.js"], "mappings": ""}));
let inventory = map.source_inventory(InventoryLimits { max_sources: 3, max_bytes: 8 });
assert_eq!(inventory.sources, ["a.js"]);
assert!(inventory.truncated);
assert_eq!(map.source_path(1).unwrap(), "b".repeat(40));
}
+72
View File
@@ -0,0 +1,72 @@
use craft_sourcemap as sourcemap;
use craft_sourcemap::{Limits, SourceMapError};
use serde_json::{Value, json};
fn decode(value: Value, limits: Limits) -> Result<sourcemap::SourceMap, SourceMapError> {
sourcemap::parse(&serde_json::to_vec(&value).unwrap(), limits)
}
fn map(mappings: &str) -> Value {
json!({"version":3,"sources":["input.js"],"names":[],"mappings":mappings})
}
#[test]
fn empty_lines_spend_the_default_budget_even_with_no_segments() {
let limits = Limits { max_segments: 0, ..Limits::default() };
assert_eq!(decode(map(&";".repeat(1_000_000)), limits), Err(SourceMapError::TooManyLines(1_000_000)));
}
#[test]
fn exact_and_zero_line_limits_include_the_final_empty_line() {
let limits = Limits { max_lines: 3, max_segments: 0, ..Limits::default() };
let parsed = decode(map(";;"), limits).unwrap();
assert_eq!(parsed.lines.len(), 3);
assert_eq!(parsed.segments, 0);
assert_eq!(decode(map(";;;"), limits), Err(SourceMapError::TooManyLines(3)));
assert_eq!(decode(map(""), Limits { max_lines: 0, ..limits }), Err(SourceMapError::TooManyLines(0)));
assert_eq!(decode(map(""), Limits { max_lines: 1, ..limits }).unwrap().lines.len(), 1);
}
#[test]
fn nested_sections_share_one_generated_line_budget() {
let value = json!({"version":3,"sections":[
{"offset":{"line":0,"column":0},"map":map(";")},
{"offset":{"line":2,"column":0},"map":{"version":3,"sections":[
{"offset":{"line":0,"column":0},"map":map(";")}
]}}
]});
let limits = Limits { max_lines: 3, max_segments: 0, ..Limits::default() };
assert_eq!(decode(value.clone(), limits), Err(SourceMapError::TooManyLines(3)));
let parsed = decode(value, Limits { max_lines: 4, ..limits }).unwrap();
assert_eq!(parsed.sections[0].2.lines.len(), 2);
assert_eq!(parsed.sections[1].2.sections[0].2.lines.len(), 2);
}
#[test]
fn external_sections_do_not_consume_undecoded_lines() {
let value = json!({"version":3,"sections":[
{"offset":{"line":0,"column":0},"url":"other.map"},
{"offset":{"line":2,"column":0},"map":map("")}
]});
let parsed = decode(value, Limits { max_lines: 1, max_segments: 0, ..Limits::default() }).unwrap();
assert!(parsed.sections[0].2.lines.is_empty());
assert_eq!(parsed.sections[1].2.lines.len(), 1);
}
#[test]
fn positions_after_empty_lines_keep_their_coordinates() {
let parsed = decode(map("AAAA;;AACA"), Limits { max_lines: 3, max_segments: 2, ..Limits::default() }).unwrap();
assert_eq!(parsed.lines.len(), 3);
assert!(parsed.original_positions(1, 0).is_empty());
let position = &parsed.original_positions(2, 0)[0];
assert_eq!(position.source.as_deref(), Some("input.js"));
assert_eq!((position.line, position.column), (1, 0));
}
#[test]
fn line_and_segment_limits_are_independent() {
let limits = Limits { max_lines: 10, max_segments: 1, ..Limits::default() };
assert!(matches!(decode(map("A,A"), limits), Err(SourceMapError::TooManySegments(_))));
assert_eq!(decode(map(";"), Limits { max_lines: 1, max_segments: 0, ..limits }), Err(SourceMapError::TooManyLines(1)));
assert_eq!(decode(map(";"), Limits { max_lines: 2, max_segments: 0, ..limits }).unwrap().segments, 0);
}
+80
View File
@@ -0,0 +1,80 @@
use craft_sourcemap::{Limits, SourceMap, parse};
use serde_json::{Value, json};
fn decode(value: Value) -> SourceMap {
parse(&serde_json::to_vec(&value).unwrap(), Limits::default()).unwrap()
}
#[test]
fn root_prefix_precedes_absolute_looking_sources() {
for (root, source, expected) in [
("webpack:///", "/src/main.js", "webpack:////src/main.js"),
("https://example.test/build", "/src/main.js", "https://example.test/build//src/main.js"),
("src", "https://example.test/main.js", "src/https://example.test/main.js"),
("src/", "//example.test/main.js", "src///example.test/main.js"),
("sources", "node:internal/main", "sources/node:internal/main"),
] {
let map = decode(json!({"version": 3, "sourceRoot": root, "sources": [source], "mappings": "AAAA"}));
assert_eq!(map.original_positions(0, 0)[0].source.as_deref(), Some(expected), "root {root:?}, source {source:?}");
assert_eq!(map.all_sources(), [expected]);
}
}
#[test]
fn empty_root_keeps_the_same_identity_as_an_absent_or_null_root() {
for source in ["src/main.js", "/src/main.js", "https://example.test/main.js"] {
let mut value = json!({"version": 3, "sources": [source], "mappings": "AAAA"});
assert_eq!(decode(value.clone()).all_sources(), [source]);
value["sourceRoot"] = Value::Null;
assert_eq!(decode(value.clone()).all_sources(), [source]);
value["sourceRoot"] = json!("");
let map = decode(value);
assert_eq!(map.all_sources(), [source]);
assert_eq!(map.original_positions(0, 0)[0].source.as_deref(), Some(source));
}
}
#[test]
fn each_source_keeps_its_index_and_embedded_content() {
let map = decode(json!({
"version": 3, "sourceRoot": "https://example.test/originals/",
"sources": ["/first.js", null, "https://other.test/third.js"],
"sourcesContent": ["first();", null, "third();"], "mappings": "AAAA,CEAA"
}));
let first = map.original_positions(0, 0);
let third = map.original_positions(0, 1);
assert_eq!(first[0].source_index, 0);
assert_eq!(third[0].source_index, 2);
assert_eq!(first[0].source.as_deref(), Some("https://example.test/originals//first.js"));
assert_eq!(third[0].source.as_deref(), Some("https://example.test/originals/https://other.test/third.js"));
assert_eq!(map.source_content(0), Some("first();"));
assert_eq!(map.source_content(2), Some("third();"));
assert_eq!(map.source_path(1), None);
assert_eq!(map.source_path(3), None);
}
#[test]
fn indexed_maps_use_each_embedded_maps_own_root() {
let map = decode(json!({
"version": 3, "sourceRoot": "outer", "sections": [
{"offset": {"line": 0, "column": 3}, "map": {
"version": 3, "sourceRoot": "file:///", "sources": ["/first.js"], "mappings": "AAAA"
}},
{"offset": {"line": 1, "column": 0}, "map": {
"version": 3, "sources": ["second.js"], "mappings": "AAAA"
}}
]
}));
assert_eq!(map.original_positions(0, 3)[0].source.as_deref(), Some("file:////first.js"));
assert_eq!(map.original_positions(1, 0)[0].source.as_deref(), Some("second.js"));
assert_eq!(map.all_sources(), ["file:////first.js", "second.js"]);
}
#[test]
fn relative_sources_still_preserve_url_root_delimiters() {
for (root, expected) in [("webpack:///", "webpack:///main.js"), ("file:///", "file:///main.js"), ("src", "src/main.js"), ("src/", "src/main.js")]
{
let map = decode(json!({"version": 3, "sourceRoot": root, "sources": ["main.js"], "mappings": "AAAA"}));
assert_eq!(map.all_sources(), [expected]);
}
}
+24
View File
@@ -0,0 +1,24 @@
use craft_sourcemap::{Limits, parse};
use serde_json::json;
#[test]
fn indexed_lookup_reports_columns_in_the_generated_file() {
let json = json!({"version":3,"sections":[{"offset":{"line":3,"column":11},"map":{"version":3,"sources":["one.js"],"mappings":"IAAA;IACA"}}]});
let map = parse(json.to_string().as_bytes(), Limits::default()).unwrap();
assert!(map.original_positions(3, 14).is_empty());
let first = map.original_positions(3, 17);
assert_eq!(first[0].generated_column, 15);
assert_eq!((first[0].line, first[0].column), (0, 0));
assert_eq!(map.original_positions(4, 4)[0].generated_column, 4);
}
#[test]
fn nested_section_columns_are_added_only_on_each_first_line() {
for (inner_line, query_line, query_column) in [(0, 3, 18), (1, 4, 7)] {
let json = json!({"version":3,"sections":[{"offset":{"line":3,"column":11},"map":{"version":3,"sections":[{"offset":{"line":inner_line,"column":3},"map":{"version":3,"sources":["nested.js"],"mappings":"IAAA"}}]}}]});
let map = parse(json.to_string().as_bytes(), Limits::default()).unwrap();
let positions = map.original_positions(query_line, query_column);
assert_eq!(positions[0].generated_column, query_column);
assert_eq!(positions[0].source.as_deref(), Some("nested.js"));
}
}
+10 -1
View File
@@ -31,6 +31,11 @@ usvg = { version = "=0.48.1", default-features = false, optional = true }
resvg = { version = "=0.48.1", default-features = false, features = ["text", "raster-images"], optional = true }
data-url = { version = "0.3.2", optional = true }
# Renderer-specific validation uses the same resolved backends as resvg0.48.1.
# JPEG validation uses the existing backend in craft-image-codecs.
gif = { version = "=0.14.2", optional = true }
image-webp = { version = "=0.2.4", optional = true }
[features]
default = ["paths"]
@@ -45,5 +50,9 @@ tree-text = ["tree", "usvg/text"]
# The system's fonts in `usvg::fontdb`.
tree-system-fonts = ["tree-text", "usvg/system-fonts"]
# Bounded rendering of a parsed tree through the matching resvg/tiny-skia versions.
tree-render = ["tree", "dep:resvg"]
tree-render = ["tree", "dep:resvg", "dep:gif", "dep:image-webp"]
tree-render-system-fonts = ["tree-render", "resvg/system-fonts"]
[dev-dependencies]
# Already resolved in the workspace; test-only Cargo.lock edge inspection.
toml = "=1.1.8"
+56
View File
@@ -16,3 +16,59 @@ The shared adapter adds an aggregate 64 MiB stylesheet-text limit and idempotent
### Transform compatibility
`transform_compat::parse_transform` preserves EffectCraft transform-list recovery and exact composition. It retains six numeric arguments per operation; scanning is not time/byte capped and matrix overflow behavior is unchanged. The strict geometry reader is unchanged. Source: Brandon Thomas, EffectCraft94e1c86d89a866771f80ad1610afda9aa12e85d8; bounded safe argument matching by Altynzhan Karybai,4035d181329e847d7a57bf3d059502ea2c91b560.
### Opt-in validated embedded images
With `tree-render`, `tree::ValidatedImageResolver` owns one drawing's immutable
`Limits`, original child-source work, data-URL normalization and renderer-specific
raster validation. Existing `ImageResolver::new`/`resolver` and legacy parse entry
points retain their behavior. Opt in explicitly:
```rust
fn example() -> Result<(), craft_svg::tree::Error> {
use craft_svg::tree::{self, Limits, ValidatedImageResolver, usvg};
let images = ValidatedImageResolver::new(Limits::default());
let checked = tree::preflight(
"<svg xmlns='http://www.w3.org/2000/svg' width='1' height='1'/>",
&images.limits(),
)?;
let drawing = images.parse_preflight(checked, usvg::Options::default())?;
let usage = images.usage();
// Importers disclose usage's invalid_* and *_limit_images; strict consumers refuse them.
let _ = (drawing, usage);
Ok(())
}
```
The caller supplies fonts and a sufficient stack/worker for its depth policy.
`parse_preflight` accepts only an opaque preflight made with exactly this context's
limits. It consumes options and installs its own data-only resolver, including for
all children; unrelated callbacks cannot fetch a linked file or bypass admission.
Expanded root ancestor frames (checked height minus the callback leaf) and each
child's complete expanded height share one depth allowance. The callback leaf is
not counted twice; source nesting alone cannot hide expanded reference chains.
An overlapping root conversion on the same context is refused.
Unchanged XML converts from its original checked DOM. A modified data URL requires
one checked replacement DOM. Invalid URLs become bounded resolver-visible image
failures. The normal path counts only resolved images. If a repair cannot fit,
the checked original can still convert when all its MIME headers meet the work
limit. In that case malformed image declarations are counted conservatively.
Original UTF-8 child bytes are charged before XML conversion or shorter repairs,
with per-document and aggregate bounds. A malformed plain encoding spends its
input bytes and leaves the remaining work for siblings. Failed inflation can
exhaust its attempted allowance, and the separate bounded retry classification
never admits another child. Encoded raster callbacks spend at most `limits.bytes + limits.embedded_bytes` cumulatively,
without retaining their sources. Pixel work and decoder allocation are admitted
before decoding; attempted failures keep their charge. An oversized unattempted
image does not prevent a later smaller image from fitting.
`resolve_text` accepts an already-decoded child SVG through the same admission
context, rather than an unchecked DOM or a precharged-budget override. `usage()`
reports omissions and work after conversion. The context is for one drawing, not a
global rendering cache.
The validator uses the existing JPEG backend in craft-image-codecs. The lockfile
test checks that it and the other validation backends match the selected resvg
dependency edges. resvg decodes the image again when it paints.
+1 -1
View File
@@ -8,7 +8,7 @@
//! - `paths` (feature `paths`, on by default): the geometry of a document as Bézier paths in
//! millimetres, for cutters, plotters and CAD import.
//! - [`color`]: CSS colours.
//! - [`write`]: a writer for SVG documents, their path data and numbers.
//! - [`mod@write`]: a writer for SVG documents, their path data and numbers.
//! - `tree` (feature `tree`): the full parser. usvg at the one version the Craft apps share,
//! re-exported, behind an entry point that runs the checks above before it parses.
//!
+110 -58
View File
@@ -22,10 +22,10 @@
//! 16 MB or so, and a 1 MB stack takes a depth of about 34.
use std::borrow::Cow;
use std::collections::HashMap;
use std::collections::hash_map::Entry;
use std::collections::{HashMap, HashSet};
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
use std::sync::{Arc, Mutex, PoisonError};
use std::sync::{Arc, Mutex, PoisonError, Weak};
pub use usvg;
@@ -34,7 +34,13 @@ use crate::{guard, svgz};
pub mod document;
pub mod draw;
#[cfg(feature = "tree-render")]
mod image_href;
pub mod primitive;
#[cfg(feature = "tree-render")]
mod validated;
#[cfg(feature = "tree-render")]
pub use validated::ValidatedImageResolver;
/// What one import may use. The defaults suit a desktop app that parses on a thread with a
/// large stack (see the module text).
@@ -64,9 +70,9 @@ pub struct Limits {
pub embedded_bytes: usize,
/// SVG documents embedded inside one another, for [`image_resolver`].
pub embedded_depth: usize,
/// Decoded RGBA pixels retained across the raster images embedded in one drawing. Dimensions
/// are probed before usvg hands the bytes to a decoder; repeated references to the same data
/// URL share one charge.
/// Cumulative raster decode pixels for a drawing. usvg sends each data-URL
/// resolution separately, so repeated uses normally spend a separate charge.
/// The legacy resolver can share a charge for the same live input allocation.
pub embedded_raster_pixels: u64,
}
@@ -161,6 +167,8 @@ pub fn root_unitless_size(svg: &str, limits: &Limits) -> Result<Option<(f64, f64
/// SVG text after shared byte, entity, attribute, XML and expansion-depth checks.
/// Keep it opaque so consumers use the same checked tree and don't reparse XML locally.
pub struct Preflight<'s> {
#[cfg(feature = "tree-render")]
limits: Limits,
xml: roxmltree::Document<'s>,
depth: usize,
/// [`height`]: the nesting usvg converts, references followed.
@@ -219,6 +227,8 @@ const UNSUPPORTED: &[&str] = &[
/// Checks already-decoded SVG text once, retaining the parsed XML for both depth policy and tree conversion.
pub fn preflight<'s>(svg: &'s str, limits: &Limits) -> Result<Preflight<'s>, Error> {
#[cfg(test)]
PREFLIGHT_PASSES.with(|count| count.set(count.get().saturating_add(1)));
let preflight_limits = crate::preflight::Limits {
bytes: limits.bytes,
depth: limits.depth,
@@ -239,7 +249,13 @@ pub fn preflight<'s>(svg: &'s str, limits: &Limits) -> Result<Preflight<'s>, Err
let Some(height) = height(&xml, limits.tree_depth)? else {
return Err(Error::TooDeep(limits.tree_depth));
};
Ok(Preflight { xml, depth, height })
Ok(Preflight {
#[cfg(feature = "tree-render")]
limits: *limits,
xml,
depth,
height,
})
}
/// Dimensions of an SVG viewport in SVG user units.
@@ -421,7 +437,7 @@ struct Used {
svg_count: AtomicUsize,
svg_depth: AtomicUsize,
raster_pixels: AtomicU64,
raster_data: Mutex<HashSet<usize>>,
raster_data: Mutex<HashMap<usize, Weak<Vec<u8>>>>,
invalid_raster: AtomicUsize,
raster_limit: AtomicUsize,
invalid_svg: AtomicUsize,
@@ -432,9 +448,12 @@ struct Used {
/// What resolving one drawing's embedded images used or refused.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub struct ImageUsage {
/// Unique decoded raster pixels reserved before decode.
/// Decoded raster pixel work reserved before decode. The legacy resolver
/// shares a charge across repeated data; the validated context charges each
/// attempted decode, including failures.
pub raster_pixels: u64,
/// Unpacked SVG bytes consumed by embedded SVG data URLs.
/// Decoded SVG text work charged to embedded data URLs, including the
/// remaining allowance consumed by a failed inflation attempt.
pub embedded_svg_bytes: usize,
/// Embedded SVG images parsed, including nested images.
pub embedded_svg_count: usize,
@@ -474,7 +493,7 @@ impl ImageResolver {
svg_count: AtomicUsize::new(0),
svg_depth: AtomicUsize::new(0),
raster_pixels: AtomicU64::new(0),
raster_data: Mutex::new(HashSet::new()),
raster_data: Mutex::new(HashMap::new()),
invalid_raster: AtomicUsize::new(0),
raster_limit: AtomicUsize::new(0),
invalid_svg: AtomicUsize::new(0),
@@ -542,28 +561,51 @@ fn account_invalid_data_urls(xml: &roxmltree::Document<'_>, used: &Used) {
}
}
fn raster_format(data: &[u8]) -> Option<craft_image_codecs::Format> {
match craft_image_codecs::detect(data)? {
format @ (craft_image_codecs::Format::Png
| craft_image_codecs::Format::Jpeg
| craft_image_codecs::Format::Gif
| craft_image_codecs::Format::WebP) => Some(format),
#[derive(Clone, Copy)]
enum SvgRaster {
Png,
Jpeg,
Gif,
WebP,
}
impl SvgRaster {
fn codec(self) -> craft_image_codecs::Format {
match self {
Self::Png => craft_image_codecs::Format::Png,
Self::Jpeg => craft_image_codecs::Format::Jpeg,
Self::Gif => craft_image_codecs::Format::Gif,
Self::WebP => craft_image_codecs::Format::WebP,
}
}
fn image_kind(self, data: Arc<Vec<u8>>) -> usvg::ImageKind {
match self {
Self::Png => usvg::ImageKind::PNG(data),
Self::Jpeg => usvg::ImageKind::JPEG(data),
Self::Gif => usvg::ImageKind::GIF(data),
Self::WebP => usvg::ImageKind::WEBP(data),
}
}
}
/// Match supported raster signatures to the MIME types both image resolvers accept.
fn declared_raster(mime: &str, data: &[u8]) -> Option<SvgRaster> {
use craft_image_codecs::Format;
match (mime, craft_image_codecs::detect(data)?) {
("image/png" | "text/plain", Format::Png) => Some(SvgRaster::Png),
("image/jpg" | "image/jpeg" | "text/plain", Format::Jpeg) => Some(SvgRaster::Jpeg),
("image/gif" | "text/plain", Format::Gif) => Some(SvgRaster::Gif),
("image/webp" | "text/plain", Format::WebP) => Some(SvgRaster::WebP),
_ => None,
}
}
fn raster_image(used: &Used, limits: &Limits, data: Arc<Vec<u8>>, format: craft_image_codecs::Format) -> Option<usvg::ImageKind> {
fn raster_image(used: &Used, limits: &Limits, data: Arc<Vec<u8>>, format: SvgRaster) -> Option<usvg::ImageKind> {
let Ok((width, height)) = craft_image_codecs::probe_allocation_size(&data) else {
used.invalid_raster.fetch_add(1, Ordering::Relaxed);
return None;
};
if !matches!(
format,
craft_image_codecs::Format::Png | craft_image_codecs::Format::Jpeg | craft_image_codecs::Format::Gif | craft_image_codecs::Format::WebP
) {
used.invalid_raster.fetch_add(1, Ordering::Relaxed);
return None;
}
let pixels = u64::from(width).checked_mul(u64::from(height))?;
if pixels == 0 || pixels > limits.embedded_raster_pixels {
used.raster_limit.fetch_add(1, Ordering::Relaxed);
@@ -571,12 +613,11 @@ fn raster_image(used: &Used, limits: &Limits, data: Arc<Vec<u8>>, format: craft_
}
let pointer = Arc::as_ptr(&data) as usize;
let mut seen = used.raster_data.lock().unwrap_or_else(PoisonError::into_inner);
if seen.insert(pointer) {
if let Entry::Vacant(entry) = seen.entry(pointer) {
let mut current = used.raster_pixels.load(Ordering::Relaxed);
loop {
let next = current.checked_add(pixels)?;
if next > limits.embedded_raster_pixels {
seen.remove(&pointer);
used.raster_limit.fetch_add(1, Ordering::Relaxed);
return None;
}
@@ -585,13 +626,16 @@ fn raster_image(used: &Used, limits: &Limits, data: Arc<Vec<u8>>, format: craft_
Err(actual) => current = actual,
}
}
// The weak reference pins the allocation identity, not its encoded bytes.
// At most one entry is added for each nonrefundable pixel charge.
entry.insert(Arc::downgrade(&data));
}
drop(seen);
// A valid header is not proof of valid pixels. resvg can silently omit a damaged
// image later; decode now under explicit limits so omission accounting is honest.
let decode_limits = craft_image_codecs::Limits::new(width, height, pixels, limits.embedded_raster_pixels.saturating_mul(8));
let options = craft_image_codecs::DecodeOptions { limits: decode_limits, keep_orientation: true };
if let Err(error) = craft_image_codecs::decode_as_with(format, &data, &options) {
if let Err(error) = craft_image_codecs::decode_as_with(format.codec(), &data, &options) {
// No tree retains this allocation on failure. Forget its address before it
// can be reused, but keep the pixel charge for the decode work already spent.
used.raster_data.lock().unwrap_or_else(PoisonError::into_inner).remove(&pointer);
@@ -605,15 +649,37 @@ fn raster_image(used: &Used, limits: &Limits, data: Arc<Vec<u8>>, format: craft_
}
return None;
}
Some(match format {
craft_image_codecs::Format::Png => usvg::ImageKind::PNG(data),
craft_image_codecs::Format::Jpeg => usvg::ImageKind::JPEG(data),
craft_image_codecs::Format::Gif => usvg::ImageKind::GIF(data),
craft_image_codecs::Format::WebP => usvg::ImageKind::WEBP(data),
_ => return None,
Some(format.image_kind(data))
}
/// Decode one child within the original byte allowance, including encoding expansion.
fn embedded_text<'a>(data: &'a [u8], room: usize, limits: &Limits) -> Result<Cow<'a, str>, svgz::Error> {
let text = svgz::text(data, room.min(limits.bytes))?;
if text.len() > room {
return Err(svgz::Error::TooLarge(room));
}
Ok(text)
}
enum ChildRefusal {
Depth,
Bytes,
Invalid,
}
fn preflight_child<'a>(text: &'a str, limits: &Limits) -> Result<Preflight<'a>, ChildRefusal> {
preflight(text, limits).map_err(|error| match error {
Error::TooDeep(_) => ChildRefusal::Depth,
Error::Text(svgz::Error::TooLarge(_)) => ChildRefusal::Bytes,
_ => ChildRefusal::Invalid,
})
}
/// Reserve the complete checked height within the parent's remaining XML room.
fn child_total_depth(parent: usize, checked: &Preflight<'_>, limits: &Limits) -> Option<usize> {
parent.checked_add(checked.height).filter(|depth| *depth <= limits.tree_depth)
}
/// An embedded SVG document, if the budgets allow it.
fn embedded(used: &Used, limits: &Limits, data: &[u8], options: &usvg::Options<'_>) -> Option<usvg::ImageKind> {
if used.depth.load(Ordering::Relaxed) >= limits.embedded_depth {
@@ -622,7 +688,7 @@ fn embedded(used: &Used, limits: &Limits, data: &[u8], options: &usvg::Options<'
return None;
}
let left = limits.embedded_bytes.saturating_sub(used.bytes.load(Ordering::Relaxed));
let text = match svgz::text(data, left.min(limits.bytes)) {
let text = match embedded_text(data, left, limits) {
Ok(text) => text,
// What a refused document unpacked before it was refused was work too: it uses up what
// is left, so the next image can't repeat it.
@@ -635,25 +701,17 @@ fn embedded(used: &Used, limits: &Limits, data: &[u8], options: &usvg::Options<'
return None;
}
};
// The raw/inflated bytes can fit while UTF-16/Latin-1 decoding expands
// beyond the aggregate allowance. A refusal consumes the remaining work.
if text.len() > left {
used.bytes.fetch_add(left, Ordering::Relaxed);
used.svg_limit.fetch_add(1, Ordering::Relaxed);
used.svg_resource_limit.fetch_add(1, Ordering::Relaxed);
return None;
}
used.bytes.fetch_add(text.len(), Ordering::Relaxed);
let checked = match preflight(&text, limits) {
let checked = match preflight_child(&text, limits) {
Ok(checked) => checked,
// Decoding UTF-16 or Latin-1 can exceed the text cap after the
// raw/inflated bytes fitted. This is a resource refusal, not invalid XML.
Err(Error::Text(svgz::Error::TooLarge(_))) => {
Err(ChildRefusal::Bytes) => {
used.svg_limit.fetch_add(1, Ordering::Relaxed);
used.svg_resource_limit.fetch_add(1, Ordering::Relaxed);
return None;
}
Err(Error::TooDeep(_)) => {
Err(ChildRefusal::Depth) => {
used.svg_limit.fetch_add(1, Ordering::Relaxed);
return None;
}
@@ -663,9 +721,7 @@ fn embedded(used: &Used, limits: &Limits, data: &[u8], options: &usvg::Options<'
}
};
// The nesting usvg converts, references followed, not the source depth alone.
let xml_depth = checked.height;
let total_depth = used.xml_depth.load(Ordering::Relaxed).checked_add(xml_depth);
let Some(total_depth) = total_depth.filter(|depth| *depth <= limits.tree_depth) else {
let Some(total_depth) = child_total_depth(used.xml_depth.load(Ordering::Relaxed), &checked, limits) else {
used.svg_limit.fetch_add(1, Ordering::Relaxed);
return None;
};
@@ -700,16 +756,7 @@ pub fn image_resolver(limits: Limits) -> usvg::ImageHrefResolver<'static> {
fn image_resolver_for(limits: Limits, used: Arc<Used>) -> usvg::ImageHrefResolver<'static> {
usvg::ImageHrefResolver {
resolve_data: Box::new(move |mime, data, options| {
let detected = raster_format(&data);
let format = match mime {
"image/jpg" | "image/jpeg" => (detected == Some(craft_image_codecs::Format::Jpeg)).then_some(craft_image_codecs::Format::Jpeg),
"image/png" => (detected == Some(craft_image_codecs::Format::Png)).then_some(craft_image_codecs::Format::Png),
"image/gif" => (detected == Some(craft_image_codecs::Format::Gif)).then_some(craft_image_codecs::Format::Gif),
"image/webp" => (detected == Some(craft_image_codecs::Format::WebP)).then_some(craft_image_codecs::Format::WebP),
"text/plain" => detected,
_ => None,
};
if let Some(format) = format {
if let Some(format) = declared_raster(mime, &data) {
return raster_image(&used, &limits, data, format);
}
// usvg can decode only the raster variants above. Count every other declared
@@ -730,3 +777,8 @@ fn image_resolver_for(limits: Limits, used: Arc<Used>) -> usvg::ImageHrefResolve
#[cfg(test)]
mod tests;
#[cfg(test)]
thread_local! {
static PREFLIGHT_PASSES: std::cell::Cell<usize> = const { std::cell::Cell::new(0) };
}
+10
View File
@@ -29,6 +29,13 @@ pub use resvg;
#[cfg(feature = "tree-render")]
pub use resvg::tiny_skia;
#[cfg(feature = "tree-render")]
mod raster;
#[cfg(feature = "tree-render")]
pub use raster::{RasterValidationFailure, validate_raster};
#[cfg(feature = "tree-render")]
pub(super) use raster::{decode_raster, probe_raster_pixels};
/// Filter primitives per filter. A result of every primitive is kept; real filters have 1 to 12
/// (Inkscape's drop shadow has 5), and 64 leaves headroom for generated ones.
const MAX_PRIMITIVES: usize = 64;
@@ -546,3 +553,6 @@ fn deeper(cx: Cx) -> Result<(), Refusal> {
}
Ok(())
}
#[cfg(all(test, feature = "tree-render"))]
mod decoder_edges;
+194
View File
@@ -0,0 +1,194 @@
//! Test-only resolved-edge checks. No Cargo subprocess or global version count.
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::indexing_slicing)]
#[derive(Clone, Debug, PartialEq, Eq)]
pub(crate) struct PackageId {
name: String,
version: String,
source: Option<String>,
}
struct Package {
id: PackageId,
dependencies: Vec<String>,
}
pub(crate) struct LockGraph {
packages: Vec<Package>,
}
// Cargo dependency edges may omit the resolved Git fragment even though the
// selected package ID always records it. Other source identities stay exact.
fn source_matches(edge: &str, package: Option<&str>) -> bool {
let Some(package) = package else { return false };
if edge.starts_with("git+") && !edge.contains('#') { package.split_once('#').map_or(package, |(base, _)| base) == edge } else { package == edge }
}
impl LockGraph {
pub(crate) fn parse(text: &str) -> Self {
let lock: toml::Value = toml::from_str(text).unwrap();
let packages = lock
.get("package")
.unwrap()
.as_array()
.unwrap()
.iter()
.map(|package| {
let string = |key| package.get(key).unwrap().as_str().unwrap().to_owned();
Package {
id: PackageId {
name: string("name"),
version: string("version"),
source: package.get("source").map(|v| v.as_str().unwrap().to_owned()),
},
dependencies: package
.get("dependencies")
.map(|d| d.as_array().unwrap().iter().map(|v| v.as_str().unwrap().to_owned()).collect())
.unwrap_or_default(),
}
})
.collect();
Self { packages }
}
pub(crate) fn workspace_package(&self, name: &str) -> PackageId {
let found: Vec<_> = self.packages.iter().filter(|p| p.id.name == name && p.id.source.is_none()).collect();
assert_eq!(found.len(), 1, "one exact workspace package required: {name}");
found[0].id.clone()
}
pub(crate) fn dependency(&self, from: &PackageId, name: &str) -> PackageId {
let package = self.packages.iter().find(|p| &p.id == from).unwrap();
let edges: Vec<_> = package.dependencies.iter().filter(|edge| edge.split_whitespace().next() == Some(name)).collect();
assert_eq!(edges.len(), 1, "one {name} dependency edge required from {from:?}");
let fields: Vec<_> = edges[0].split_whitespace().collect();
let version = fields.get(1).copied();
let source = fields.get(2).map(|s| s.trim_start_matches('(').trim_end_matches(')'));
let found: Vec<_> = self
.packages
.iter()
.filter(|p| {
p.id.name == name && version.is_none_or(|v| p.id.version == v) && source.is_none_or(|s| source_matches(s, p.id.source.as_deref()))
})
.collect();
assert_eq!(found.len(), 1, "ambiguous or absent resolved edge {} from {from:?}", edges[0]);
found[0].id.clone()
}
}
pub(crate) fn assert_renderer_edges(graph: &LockGraph, svg: &PackageId) {
let renderer = graph.dependency(svg, "resvg");
assert_eq!(graph.dependency(svg, "usvg"), graph.dependency(&renderer, "usvg"), "shared and renderer trees must be the same resolved package");
for codec in ["gif", "image-webp"] {
assert_eq!(graph.dependency(svg, codec), graph.dependency(&renderer, codec), "validator and selected renderer differ for {codec}");
}
let codecs = graph.dependency(svg, "craft-image-codecs");
assert_eq!(
graph.dependency(&codecs, "zune-jpeg"),
graph.dependency(&renderer, "zune-jpeg"),
"JPEG validator and renderer must use the same backend"
);
}
#[test]
fn validated_renderer_dependency_edges_are_the_same_resolved_packages() {
let file = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../Cargo.lock");
let graph = LockGraph::parse(&std::fs::read_to_string(file).unwrap());
assert_renderer_edges(&graph, &graph.workspace_package("craft-svg"));
}
#[test]
fn dependency_lookup_follows_the_selected_edge_and_rejects_ambiguity() {
let graph = LockGraph::parse(
r#"
[[package]]
name = "consumer"
version = "1"
dependencies = ["codec 1 (registry+one)"]
[[package]]
name = "codec"
version = "1"
source = "registry+one"
[[package]]
name = "codec"
version = "1"
source = "registry+two"
[[package]]
name = "other"
version = "1"
dependencies = ["codec"]
"#,
);
let selected = graph.dependency(&graph.workspace_package("consumer"), "codec");
assert_eq!(selected.source.as_deref(), Some("registry+one"));
assert!(std::panic::catch_unwind(|| graph.dependency(&graph.workspace_package("other"), "codec")).is_err());
}
#[test]
fn selected_renderer_graph_rejects_a_split_validator_backend() {
use std::fmt::Write;
let make = |split: bool| {
let mut text = String::from(
"[[package]]\nname='craft-svg'\nversion='1'\ndependencies=['resvg', 'usvg', 'gif', 'image-webp', 'craft-image-codecs']\n\
[[package]]\nname='craft-image-codecs'\nversion='1'\ndependencies=['",
);
text.push_str(if split { "zune-jpeg 2" } else { "zune-jpeg 1" });
text.push_str(
"']\n[[package]]\nname='resvg'\nversion='1'\nsource='registry+one'\ndependencies=['usvg', 'gif', 'image-webp', 'zune-jpeg 1']\n",
);
for name in ["usvg", "gif", "image-webp", "zune-jpeg"] {
write!(&mut text, "[[package]]\nname='{name}'\nversion='1'\nsource='registry+one'\n").unwrap();
}
// An unrelated second codec version is valid; only the selected edges count.
text.push_str("[[package]]\nname='zune-jpeg'\nversion='2'\nsource='registry+one'\n");
LockGraph::parse(&text)
};
let matching = make(false);
assert_renderer_edges(&matching, &matching.workspace_package("craft-svg"));
let split = make(true);
assert!(std::panic::catch_unwind(|| assert_renderer_edges(&split, &split.workspace_package("craft-svg"))).is_err());
}
#[test]
fn git_edges_resolve_full_package_ids_and_reject_canonical_ambiguity() {
let fixture = |second: bool| {
let mut text = String::from(
r#"
[[package]]
name = "consumer"
version = "1"
dependencies = ["codec 1 (git+https://example.invalid/codecs?rev=chosen)"]
[[package]]
name = "explicit"
version = "1"
dependencies = ["codec 1 (git+https://example.invalid/codecs?rev=chosen#aaaa)"]
[[package]]
name = "wrong"
version = "1"
dependencies = ["codec 1 (git+https://example.invalid/codecs?rev=chosen#cccc)"]
[[package]]
name = "codec"
version = "1"
source = "git+https://example.invalid/codecs?rev=chosen#aaaa"
"#,
);
if second {
text.push_str(
r#"
[[package]]
name = "codec"
version = "1"
source = "git+https://example.invalid/codecs?rev=chosen#bbbb"
"#,
);
}
LockGraph::parse(&text)
};
let unique = fixture(false);
let selected = unique.dependency(&unique.workspace_package("consumer"), "codec");
assert_eq!(selected.source.as_deref(), Some("git+https://example.invalid/codecs?rev=chosen#aaaa"));
assert_eq!(selected, unique.dependency(&unique.workspace_package("explicit"), "codec"));
assert!(std::panic::catch_unwind(|| unique.dependency(&unique.workspace_package("wrong"), "codec")).is_err());
let ambiguous = fixture(true);
assert!(std::panic::catch_unwind(|| ambiguous.dependency(&ambiguous.workspace_package("consumer"), "codec")).is_err());
assert_eq!(selected, ambiguous.dependency(&ambiguous.workspace_package("explicit"), "codec"));
assert!(!source_matches("registry+one", Some("registry+one#aaaa")), "registry sources remain exact");
assert!(
!source_matches("git+https://example.invalid/codecs?rev=other", selected.source.as_deref()),
"requested Git revision remains part of the identity"
);
}
+132
View File
@@ -0,0 +1,132 @@
//! Validate optional raster images with the same backends used by the SVG renderer.
//!
//! Header probing alone cannot detect damaged pixel data. Admit dimensions and
//! reserve work before decoding: even a failed decode keeps its charge.
use std::io::Cursor;
use std::num::NonZeroU64;
use crate::tree::{SvgRaster, draw::tiny_skia, usvg};
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum RasterValidationFailure {
/// The renderer cannot decode the encoded image.
Invalid,
/// Header dimensions, cumulative work or decoder allocation exceed admission.
Limit,
}
/// Validate the renderer's first frame within per-image and remaining work limits.
/// Failed attempted decodes keep their pixel charge. No decoded pixels are retained. SVG
/// children use the same wrapped resolver and need no second recursive walk.
pub fn validate_raster(kind: &usvg::ImageKind, max_pixels: u64, pixels_left: &mut u64) -> Result<(), RasterValidationFailure> {
let (format, data) = match kind {
usvg::ImageKind::PNG(data) => (SvgRaster::Png, data),
usvg::ImageKind::JPEG(data) => (SvgRaster::Jpeg, data),
usvg::ImageKind::GIF(data) => (SvgRaster::Gif, data),
usvg::ImageKind::WEBP(data) => (SvgRaster::WebP, data),
usvg::ImageKind::SVG(_) => return Ok(()),
};
let pixels = probe_raster_pixels(data, max_pixels)?;
*pixels_left = pixels_left.checked_sub(pixels).ok_or(RasterValidationFailure::Limit)?;
decode_raster(format, data, pixels)
}
pub(in crate::tree) fn probe_raster_pixels(data: &[u8], max_pixels: u64) -> Result<u64, RasterValidationFailure> {
// The allocation probe accounts for oversized GIF/WebP frames as well as the canvas.
let (width, height) = craft_image_codecs::probe_allocation_size(data).map_err(|_| RasterValidationFailure::Invalid)?;
let pixels = area(width, height)?;
if pixels > max_pixels {
return Err(RasterValidationFailure::Limit);
}
Ok(pixels)
}
pub(in crate::tree) fn decode_raster(format: SvgRaster, data: &[u8], pixels: u64) -> Result<(), RasterValidationFailure> {
match format {
SvgRaster::Png => {
let decoded = tiny_skia::Pixmap::decode_png(data).map_err(|_| RasterValidationFailure::Invalid)?;
rgba_bytes(decoded.width(), decoded.height(), pixels)?;
Ok(())
}
SvgRaster::Jpeg => craft_image_codecs::validate_jpeg_rgba8(data, pixels).map_err(|error| match error {
craft_image_codecs::CodecError::LimitExceeded(_) => RasterValidationFailure::Limit,
_ => RasterValidationFailure::Invalid,
}),
SvgRaster::Gif => gif(data, pixels),
SvgRaster::WebP => webp(data, pixels),
}
}
fn area(width: u32, height: u32) -> Result<u64, RasterValidationFailure> {
if width == 0 || height == 0 {
return Err(RasterValidationFailure::Invalid);
}
u64::from(width).checked_mul(u64::from(height)).ok_or(RasterValidationFailure::Limit)
}
/// Decoder headers must not request more output than the allocation probe paid for.
fn rgba_bytes(width: u32, height: u32, admitted: u64) -> Result<usize, RasterValidationFailure> {
let pixels = area(width, height)?;
if pixels > admitted {
return Err(RasterValidationFailure::Limit);
}
usize::try_from(pixels.checked_mul(4).ok_or(RasterValidationFailure::Limit)?).map_err(|_| RasterValidationFailure::Limit)
}
fn buffer(length: usize) -> Result<Vec<u8>, RasterValidationFailure> {
let mut output = Vec::new();
output.try_reserve_exact(length).map_err(|_| RasterValidationFailure::Limit)?;
output.resize(length, 0);
Ok(output)
}
fn gif(data: &[u8], admitted: u64) -> Result<(), RasterValidationFailure> {
let bytes = admitted.checked_mul(4).ok_or(RasterValidationFailure::Limit)?;
// Do not loosen the renderer's default 50 MB GIF frame limit. Smaller
// imports also keep their own admitted allocation ceiling during decode.
let limit = NonZeroU64::new(bytes.min(50_000_000)).ok_or(RasterValidationFailure::Invalid)?;
let mut options = gif::DecodeOptions::new();
options.set_color_output(gif::ColorOutput::RGBA);
options.set_memory_limit(gif::MemoryLimit::Bytes(limit));
let mut decoder = options.read_info(data).map_err(gif_error)?;
let frame = decoder.read_next_frame().map_err(gif_error)?.ok_or(RasterValidationFailure::Invalid)?;
let expected = rgba_bytes(u32::from(frame.width), u32::from(frame.height), admitted)?;
if frame.buffer.len() != expected {
return Err(RasterValidationFailure::Invalid);
}
Ok(())
}
fn gif_error(error: gif::DecodingError) -> RasterValidationFailure {
match error {
gif::DecodingError::MemoryLimit | gif::DecodingError::OutOfMemory => RasterValidationFailure::Limit,
_ => RasterValidationFailure::Invalid,
}
}
fn webp(data: &[u8], admitted: u64) -> Result<(), RasterValidationFailure> {
let mut decoder = image_webp::WebPDecoder::new(Cursor::new(data)).map_err(webp_error)?;
let (width, height) = decoder.dimensions();
let rgba = rgba_bytes(width, height, admitted)?;
let channels = if decoder.has_alpha() { 4 } else { 3 };
let expected = rgba / 4 * channels;
if decoder.output_buffer_size() != Some(expected) {
return Err(RasterValidationFailure::Invalid);
}
// The decoder's metadata memory limit does not cap read_image scratch.
// Admit the frame and exact output buffer before calling read_image.
let mut output = buffer(expected)?;
decoder.read_image(&mut output).map_err(webp_error)
}
fn webp_error(error: image_webp::DecodingError) -> RasterValidationFailure {
match error {
image_webp::DecodingError::MemoryLimitExceeded | image_webp::DecodingError::ImageTooLarge => RasterValidationFailure::Limit,
_ => RasterValidationFailure::Invalid,
}
}
#[cfg(test)]
#[path = "raster_tests.rs"]
mod raster_tests;
+186
View File
@@ -0,0 +1,186 @@
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::indexing_slicing)]
use super::*;
use craft_image_codecs::{ChannelLayout, Format, Image};
use std::sync::Arc;
fn kind(format: Format, bytes: Vec<u8>) -> usvg::ImageKind {
let bytes = Arc::new(bytes);
match format {
Format::Png => usvg::ImageKind::PNG(bytes),
Format::Jpeg => usvg::ImageKind::JPEG(bytes),
Format::Gif => usvg::ImageKind::GIF(bytes),
Format::WebP => usvg::ImageKind::WEBP(bytes),
_ => panic!("test raster format"),
}
}
fn encoded(format: Format) -> Vec<u8> {
let image = Image::from_u8(2, 2, ChannelLayout::Rgb, [17, 83, 149].repeat(4)).unwrap();
craft_image_codecs::encode(&image, format, &Default::default()).unwrap()
}
fn segment(out: &mut Vec<u8>, marker: u8, payload: &[u8]) {
out.extend_from_slice(&[0xff, marker]);
out.extend_from_slice(&u16::try_from(payload.len() + 2).unwrap().to_be_bytes());
out.extend_from_slice(payload);
}
// ISO/IEC10918-1 framing, one gray 8x8 block. The single one-bit Huffman codes
// encode DC category 0 and AC EOB. No third-party asset or camera data is used.
fn gray_jpeg(bits: u8, entropy: &[u8]) -> Vec<u8> {
let mut data = vec![0xff, 0xd8];
let mut quantization = vec![0];
quantization.extend_from_slice(&[1; 64]);
segment(&mut data, 0xdb, &quantization);
segment(&mut data, if bits == 8 { 0xc0 } else { 0xc1 }, &[bits, 0, 8, 0, 8, 1, 1, 0x11, 0]);
let mut tables = Vec::new();
for class in [0x00, 0x10] {
tables.extend_from_slice(&[class, 1]);
tables.extend_from_slice(&[0; 15]);
tables.push(0);
}
segment(&mut data, 0xc4, &tables);
segment(&mut data, 0xda, &[1, 1, 0, 0, 63, 0]);
data.extend_from_slice(entropy);
data.extend_from_slice(&[0xff, 0xd9]);
data
}
fn corrupt_jpeg() -> Vec<u8> {
let mut data = gray_jpeg(8, &[0x3f]);
let scan = data.windows(2).position(|marker| marker == b"\xff\xda").unwrap();
// Keep the complete scan header and valid entropy, but reference table 1
// while DHT defines only table 0. The renderer accepts these headers, then
// fails pixel decoding when checking the selected Huffman tables. Random
// all-one entropy is tolerated by its default policy and is not this guard.
assert_eq!(data[scan + 6], 0);
data[scan + 6] = 0x11;
data
}
fn crc32(bytes: &[u8]) -> u32 {
let mut value = !0u32;
for byte in bytes {
value ^= u32::from(*byte);
for _ in 0..8 {
value = (value >> 1) ^ (0xedb8_8320 & 0u32.wrapping_sub(value & 1));
}
}
!value
}
fn corrupt_png() -> Vec<u8> {
let mut data = encoded(Format::Png);
let mut at = 8;
loop {
let length = u32::from_be_bytes(data[at..at + 4].try_into().unwrap()) as usize;
if &data[at + 4..at + 8] == b"IDAT" {
// Keep the complete container and an honest CRC, but destroy the
// zlib header in the pixel stream rather than the dimension header.
assert!(length >= 2);
data[at + 8..at + 10].fill(0);
let crc = crc32(&data[at + 4..at + 8 + length]);
data[at + 8 + length..at + 12 + length].copy_from_slice(&crc.to_be_bytes());
return data;
}
at += length + 12;
}
}
fn corrupt_gif() -> Vec<u8> {
// Complete 2x2 GIF89a framing and palette, followed by a code7 when only
// codes 0..5 are initially defined by an LZW minimum code size of 2.
let mut data = b"GIF89a".to_vec();
data.extend_from_slice(&[2, 0, 2, 0, 0x80, 0, 0]);
data.extend_from_slice(&[0, 0, 0, 0, 255, 0]);
data.extend_from_slice(&[0x2c, 0, 0, 0, 0, 2, 0, 2, 0, 0]);
data.extend_from_slice(&[2, 1, 7, 0, 0x3b]);
data
}
fn corrupt_webp() -> Vec<u8> {
// Complete RIFF/VP8L dimensions, with no transform/Huffman/pixel bits.
// Chunk length and odd-byte padding remain internally consistent.
let mut data = b"RIFF".to_vec();
data.extend_from_slice(&18u32.to_le_bytes());
data.extend_from_slice(b"WEBPVP8L");
data.extend_from_slice(&5u32.to_le_bytes());
data.extend_from_slice(&[0x2f, 1, 0x40, 0, 0, 0]);
data
}
#[test]
fn renderer_validator_accepts_complete_png_jpeg_gif_and_webp_at_exact_pixel_allowance() {
for format in [Format::Png, Format::Jpeg, Format::Gif, Format::WebP] {
let data = encoded(format);
assert_eq!(craft_image_codecs::probe_allocation_size(&data).unwrap(), (2, 2));
let mut remaining = 4;
assert_eq!(validate_raster(&kind(format, data), 4, &mut remaining), Ok(()), "{format:?}");
assert_eq!(remaining, 0, "{format:?}: exact area must be charged");
}
}
#[test]
fn renderer_validator_rejects_corrupt_pixels_after_valid_dimensions_without_refunding_work() {
let cases = [
(Format::Png, corrupt_png(), (2, 2)),
// Header-only admission cannot detect this missing scan-table dependency.
(Format::Jpeg, corrupt_jpeg(), (8, 8)),
(Format::Gif, corrupt_gif(), (2, 2)),
(Format::WebP, corrupt_webp(), (2, 2)),
];
for (format, data, dimensions) in cases {
assert_eq!(craft_image_codecs::probe_allocation_size(&data).unwrap(), dimensions, "{format:?}");
match format {
Format::Gif => {
gif::DecodeOptions::new().read_info(data.as_slice()).expect("intact GIF headers before corrupt LZW pixels");
}
Format::WebP => {
image_webp::WebPDecoder::new(Cursor::new(&data)).expect("intact WebP headers before missing pixel bits");
}
_ => {}
}
let pixels = u64::from(dimensions.0) * u64::from(dimensions.1);
let image = kind(format, data);
let mut remaining = pixels * 2;
for left in [pixels, 0] {
assert_eq!(validate_raster(&image, pixels, &mut remaining), Err(RasterValidationFailure::Invalid), "{format:?}");
assert_eq!(remaining, left, "{format:?}: failed decode work is nonrefundable");
}
assert_eq!(validate_raster(&image, pixels, &mut remaining), Err(RasterValidationFailure::Limit), "{format:?}");
assert_eq!(remaining, 0);
}
}
#[test]
fn twelve_bit_jpeg_is_valid_for_the_general_codec_but_not_the_svg_renderer() {
for bits in [8, 12] {
// DC=0, EOB=0, then six one-pad bits: a constant mid-gray block.
let data = gray_jpeg(bits, &[0x3f]);
let decoded = craft_image_codecs::decode(&data).unwrap();
assert_eq!(decoded.dimensions(), (8, 8));
let mut remaining = 64;
let expected = if bits == 8 { Ok(()) } else { Err(RasterValidationFailure::Invalid) };
assert_eq!(validate_raster(&kind(Format::Jpeg, data), 64, &mut remaining), expected, "{bits}-bit");
assert_eq!(remaining, 0, "capability refusal still spends attempted decode work");
}
}
#[test]
fn renderer_validator_refuses_unadmitted_headers_and_pixels_without_charging_unattempted_decode() {
for format in [Format::Png, Format::Jpeg, Format::Gif, Format::WebP] {
let image = kind(format, encoded(format));
let mut remaining = 4;
assert_eq!(validate_raster(&image, 3, &mut remaining), Err(RasterValidationFailure::Limit), "{format:?}: per-image ceiling");
assert_eq!(remaining, 4);
remaining = 3;
assert_eq!(validate_raster(&image, 4, &mut remaining), Err(RasterValidationFailure::Limit), "{format:?}: cumulative allowance");
assert_eq!(remaining, 3);
assert_eq!(
validate_raster(&kind(format, b"invalid raster header".to_vec()), 4, &mut remaining),
Err(RasterValidationFailure::Invalid),
"{format:?}"
);
assert_eq!(remaining, 3, "invalid dimensions cannot reserve an invented area");
}
}
Loaded 100 of 129 files, more files were not shown because too many files have changed in this diff. Show more