Merge pull request #2151 from FredApps/fix/xfa-main-integration

XFA: refresh main without losing safety repair
This commit is contained in:
FredApps authored and GitHub committed 2026-11-18 10:37:51 +07:00
commit f34fa55847
202 files changed
+30363 -994

No files matched your search

+20 -3
View File
@@ -1,7 +1,6 @@
# cargo-nextest (https://nexte.st) runs every test in its own process, in parallel across test
# binaries. The Windows leg of the Workspace gate uses it (.github/workflows/workspace.yml); every
# other leg, and local runs, keep `cargo test`. Nextest does not run doctests, so the Windows leg
# also runs `cargo test --doc`.
# binaries. Every leg of the Workspace gate uses it (.github/workflows/workspace.yml); local runs may
# keep `cargo test`. Nextest does not run doctests, so the gate also runs `cargo test --doc`.
#
# cargo nextest run --workspace --profile ci
# cargo test --workspace --doc
@@ -22,6 +21,9 @@ video-fixtures = { max-threads = 1 }
# craft-print: `PRINT_TICKET_PROVIDER` serializes the Windows PrintTicket provider, and the tests
# share the one test print queue the gate installs.
print = { max-threads = 1 }
# craft-video integration tests on Linux and macOS: the binaries generate and read the same fixture
# directories (see the `ci` overrides below).
video-integration = { max-threads = 1 }
[[profile.default.overrides]]
filter = 'package(craft-ui) & (binary(visual_regression) | test(/^combo::tests::|selected_tab_erases_a_baseline_centered_on_its_bottom_edge/))'
@@ -42,3 +44,18 @@ test-group = 'print'
fail-fast = false
slow-timeout = { period = "60s" }
failure-output = "immediate-final"
# Linux and macOS (the resource audit in craft-libs#2058): every GPU consumer shares the one `gpu`
# slot, and the craft-video integration binaries share their fixture directories. Conservative on
# purpose: the in-process mutexes these tests rely on do not reach across nextest's processes.
# Windows keeps the default groups above, which its leg has passed with since it moved to nextest.
[[profile.ci.overrides]]
platform = 'cfg(unix)'
filter = 'package(craft-ui) | package(craft-keymap) | package(craft-update)'
test-group = 'gpu'
[[profile.ci.overrides]]
platform = 'cfg(unix)'
filter = 'package(craft-video) & kind(test)'
test-group = 'video-integration'
+1 -28
View File
@@ -18,7 +18,6 @@ workflows:
- 'actions/checkout@v7.0.1'
- 'step-security/dtolnay-rust-toolchain@v1.0.0'
- 'step-security/harden-runner@v2.22.4'
- 'step-security/rust-cache@v2.9.2'
'.github/workflows/cms.yml':
- 'actions/checkout@v4.4.0'
- 'dtolnay/rust-toolchain@stable'
@@ -27,7 +26,6 @@ workflows:
- 'actions/checkout@v7.0.1'
- 'step-security/dtolnay-rust-toolchain@v1.0.0'
- 'step-security/harden-runner@v2.22.4'
- 'step-security/rust-cache@v2.9.2'
'.github/workflows/dialogs.yml':
- 'actions/checkout@v7.0.1'
- 'step-security/dtolnay-rust-toolchain@v1.0.0'
@@ -36,10 +34,6 @@ workflows:
- 'actions/checkout@v7.0.1'
- 'step-security/dtolnay-rust-toolchain@v1.0.0'
- 'step-security/harden-runner@v2.22.4'
'.github/workflows/docpath.yml':
- 'actions/checkout@v7.0.1'
- 'step-security/dtolnay-rust-toolchain@v1.0.0'
- 'step-security/harden-runner@v2.22.4'
'.github/workflows/ecproj-format.yml':
- 'actions/checkout@v7.0.1'
- 'step-security/dtolnay-rust-toolchain@v1.0.0'
@@ -52,22 +46,15 @@ workflows:
- 'actions/checkout@v7.0.1'
- 'dtolnay/rust-toolchain@stable'
- 'step-security/harden-runner@v2.22.4'
- 'swatinem/rust-cache@v2.9.3'
'.github/workflows/house-rules.yml': []
'.github/workflows/i18n.yml':
- 'actions/checkout@v7.0.1'
- 'step-security/dtolnay-rust-toolchain@v1.0.0'
- 'step-security/harden-runner@v2.22.4'
'.github/workflows/jpeg.yml':
- 'actions/checkout@v7.0.1'
- 'step-security/dtolnay-rust-toolchain@v1.0.0'
- 'step-security/harden-runner@v2.22.4'
- 'step-security/rust-cache@v2.9.2'
'.github/workflows/keymap.yml':
- 'actions/checkout@v7.0.1'
- 'step-security/dtolnay-rust-toolchain@v1.0.0'
- 'step-security/harden-runner@v2.22.4'
- 'step-security/rust-cache@v2.9.2'
'.github/workflows/lockfile.yml':
- 'actions/checkout@v7.0.1'
- 'step-security/harden-runner@v2.22.4'
@@ -75,11 +62,10 @@ workflows:
- 'actions/checkout@v7.0.1'
- 'step-security/dtolnay-rust-toolchain@v1.0.0'
- 'step-security/harden-runner@v2.22.4'
- 'step-security/rust-cache@v2.9.2'
'.github/workflows/menu.yml':
- 'actions/checkout@v7.0.1'
- 'dtolnay/rust-toolchain@stable'
- 'swatinem/rust-cache@v2.9.3'
'.github/workflows/nightly.yml': []
'.github/workflows/pcraft-format.yml':
- 'actions/checkout@v7.0.1'
- 'step-security/dtolnay-rust-toolchain@v1.0.0'
@@ -89,23 +75,12 @@ workflows:
- 'actions/setup-python@v7.1.0'
- 'step-security/dtolnay-rust-toolchain@v1.0.0'
- 'step-security/harden-runner@v2.22.4'
- 'step-security/rust-cache@v2.9.2'
- 'step-security/taiki-e-install-action@v2.87.32'
'.github/workflows/print.yml':
- 'actions/checkout@v7.0.1'
- 'step-security/dtolnay-rust-toolchain@v1.0.0'
- 'step-security/harden-runner@v2.22.4'
'.github/workflows/proof.yml':
- 'actions/checkout@v7.0.1'
- 'step-security/dtolnay-rust-toolchain@v1.0.0'
- 'step-security/harden-runner@v2.22.4'
- 'step-security/rust-cache@v2.9.2'
'.github/workflows/sonarcloud.yml': []
'.github/workflows/ui.yml':
- 'actions/checkout@v7.0.1'
- 'step-security/dtolnay-rust-toolchain@v1.0.0'
- 'step-security/harden-runner@v2.22.4'
- 'step-security/rust-cache@v2.9.2'
'.github/workflows/update.yml':
- 'actions/checkout@v7.0.1'
- 'step-security/dtolnay-rust-toolchain@v1.0.0'
@@ -118,7 +93,6 @@ workflows:
- 'actions/checkout@v7.0.1'
- 'step-security/dtolnay-rust-toolchain@v1.0.0'
- 'step-security/harden-runner@v2.22.4'
- 'step-security/rust-cache@v2.9.2'
'.github/workflows/web-api.yml':
- 'actions/checkout@v7.0.1'
- 'step-security/dtolnay-rust-toolchain@v1.0.0'
@@ -128,7 +102,6 @@ workflows:
- 'actions/checkout@v7.0.1'
- 'step-security/dtolnay-rust-toolchain@v1.0.0'
- 'step-security/harden-runner@v2.22.4'
- 'step-security/rust-cache@v2.9.2'
- 'step-security/taiki-e-install-action@v2.87.32'
'.github/workflows/window.yml':
- 'actions/checkout@v7.0.1'
+7 -14
View File
@@ -12,20 +12,17 @@ permissions:
contents: read
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued.
group: ai-core-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# One group per pull request and per branch: a newer push cancels the run it supersedes, on main
# too, since the newest run checks everything before it.
group: ai-core-${{ github.ref }}
cancel-in-progress: true
# fmt, clippy and the tests run in the Workspace gate on every OS; this checks what it does not.
jobs:
ai-core:
name: AI model WASM
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
# macOS runners are kept for releases: macOS runs only when started by hand.
os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }}
runs-on: ${{ matrix.os }}
runs-on: ubuntu-latest
steps:
- name: Harden runner
uses: step-security/harden-runner@v2.22.4
@@ -35,10 +32,6 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
components: rustfmt, clippy
targets: wasm32-unknown-unknown
- run: cargo fmt --check
- run: cargo clippy -p craft-ai-core --all-targets --locked -- -D warnings
- run: cargo test -p craft-ai-core --locked
# The model promises to compile for the web.
- run: cargo check -p craft-ai-core --target wasm32-unknown-unknown --locked
+7 -16
View File
@@ -12,20 +12,17 @@ permissions:
contents: read
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued.
group: atomic-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# One group per pull request and per branch: a newer push cancels the run it supersedes, on main
# too, since the newest run checks everything before it.
group: atomic-${{ github.ref }}
cancel-in-progress: true
# fmt, clippy and the tests run in the Workspace gate on every OS; this runs what it does not.
jobs:
atomic:
name: Atomic inherited Linux ACLs
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
# macOS runners are kept for releases: macOS runs only when started by hand.
os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }}
runs-on: ${{ matrix.os }}
runs-on: ubuntu-latest
steps:
- name: Harden runner
uses: step-security/harden-runner@v2.22.4
@@ -35,15 +32,9 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
components: rustfmt, clippy
- name: Install ACL tools
if: runner.os == 'Linux'
run: sudo apt-get update && sudo apt-get install -y acl
- run: cargo fmt --check
- run: cargo clippy -p craft-atomic -p craft-settings --all-targets --locked -- -D warnings
- run: cargo test -p craft-atomic -p craft-settings --locked -- --nocapture
- name: Check inherited Linux ACLs
if: runner.os == 'Linux'
run: cargo test -p craft-atomic --locked -- --ignored --nocapture
wasm:
+12 -40
View File
@@ -11,17 +11,22 @@ on:
permissions:
contents: read
# A fraction of the fuzz-style cases, as in the Workspace gate; nightly.yml runs them in full.
env:
CRAFT_HOSTILE_DIVISOR: "8"
PROPTEST_CASES: "64"
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it
# always gets to save the cache.
group: shared-audio-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# One group per pull request and per branch: a newer push cancels the run it supersedes, on main
# too, since the newest run checks everything before it.
group: shared-audio-${{ github.ref }}
cancel-in-progress: true
jobs:
lint:
timeout-minutes: 15
name: Audio lint, feature sets, docs, WASM
# fmt, clippy with all features and the minimum Rust version run in the Workspace gate.
name: Audio feature sets, docs, WASM
runs-on: ubuntu-latest
steps:
- name: Harden runner
@@ -34,13 +39,8 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
components: rustfmt, clippy
components: clippy
targets: wasm32-unknown-unknown
- uses: step-security/rust-cache@v2.9.2
with:
key: craft-audio-lint
save-if: ${{ github.ref == 'refs/heads/main' }}
- run: cargo fmt -p craft-audio --check
# Each feature on its own, so code that needs another feature fails here.
- name: Clippy, each feature alone
run: |
@@ -49,7 +49,6 @@ jobs:
cargo clippy -p craft-audio --all-targets --locked --no-default-features ${f:+--features "$f"} -- -D warnings
echo "::endgroup::"
done
- run: cargo clippy -p craft-audio --all-targets --locked --all-features -- -D warnings
# missing_docs is a warning in the crate; -D warnings makes it (and broken links) fail here.
- run: cargo doc -p craft-audio --all-features --no-deps --locked
env:
@@ -63,25 +62,6 @@ jobs:
RUSTDOCFLAGS: "-D warnings"
- run: cargo check -p craft-audio --all-features --locked --target wasm32-unknown-unknown
- run: cargo check -p craft-audio --no-default-features --features lossless-encode --locked --target wasm32-unknown-unknown
msrv:
timeout-minutes: 10
name: Audio on the minimum Rust version
runs-on: ubuntu-latest
steps:
- name: Harden runner
uses: step-security/harden-runner@v2.22.4
with:
egress-policy: audit
- uses: actions/checkout@v7.0.1
with:
persist-credentials: false
# The workspace's rust-version.
- run: rustup toolchain install 1.95.0 --profile minimal
- uses: step-security/rust-cache@v2.9.2
with:
key: craft-audio-msrv
save-if: ${{ github.ref == 'refs/heads/main' }}
- run: cargo +1.95.0 check -p craft-audio -p craft-media --all-features --locked
features:
timeout-minutes: 10
name: Audio tests without symphonia and lossless-encode only
@@ -97,10 +77,6 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
- uses: step-security/rust-cache@v2.9.2
with:
key: craft-audio-features
save-if: ${{ github.ref == 'refs/heads/main' }}
- run: cargo test -p craft-audio --no-default-features --features lossless-encode --locked
- run: cargo test -p craft-audio --no-default-features --features pcm-encode --locked --lib
- run: cargo test -p craft-audio --no-default-features --features flac-encode --locked --lib
@@ -122,10 +98,6 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
- uses: step-security/rust-cache@v2.9.2
with:
key: craft-audio-test
save-if: ${{ github.ref == 'refs/heads/main' }}
- uses: actions/cache@v6.1.0
with:
path: ${{ runner.temp }}/media-fixtures/opus/vectors
+7 -3
View File
@@ -11,6 +11,13 @@ on:
permissions:
contents: read
concurrency:
# One group per pull request and per branch: a newer push cancels the run it supersedes, on main
# too, since the newest run checks everything before it.
group: cms-${{ github.ref }}
cancel-in-progress: true
# clippy and the debug tests run in the Workspace gate; this runs the release build of the tests.
jobs:
cms:
timeout-minutes: 20
@@ -18,9 +25,6 @@ jobs:
steps:
- uses: actions/checkout@v4.4.0
- uses: dtolnay/rust-toolchain@stable
with:
components: clippy
- run: cargo clippy -p craft-cms --all-targets --all-features --locked -- -D warnings
# Release: the FOGRA39 fit and ink-limit tests take about 35 s in debug and 4 s here. Tests that need
# installed system profiles print "skipped:" when none are present.
- run: cargo test -p craft-cms --all-features --release --locked -- --nocapture
+5 -41
View File
@@ -12,41 +12,13 @@ permissions:
contents: read
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it
# always gets to save the cache.
group: shared-crw-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# One group per pull request and per branch: a newer push cancels the run it supersedes, on main
# too, since the newest run checks everything before it.
group: shared-crw-${{ github.ref }}
cancel-in-progress: true
# fmt, clippy and the tests run in the Workspace gate on every OS; this checks what it does not.
jobs:
native:
timeout-minutes: 20
name: CRW lint and tests (${{ matrix.os }})
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, windows-latest]
runs-on: ${{ matrix.os }}
steps:
- name: Harden runner
uses: step-security/harden-runner@v2.22.4
with:
egress-policy: audit
- uses: actions/checkout@v7.0.1
with:
persist-credentials: false
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
components: rustfmt, clippy
- uses: step-security/rust-cache@v2.9.2
with:
key: craft-crw-${{ matrix.os }}
save-if: ${{ github.ref == 'refs/heads/main' }}
- run: cargo fmt -p craft-crw --check
- run: cargo clippy -p craft-crw --all-targets --locked -- -D warnings
- run: cargo test -p craft-crw --locked
wasm:
timeout-minutes: 10
name: CRW WASM
@@ -63,10 +35,6 @@ jobs:
with:
toolchain: stable
targets: wasm32-unknown-unknown
- uses: step-security/rust-cache@v2.9.2
with:
key: craft-crw-wasm
save-if: ${{ github.ref == 'refs/heads/main' }}
- run: cargo check -p craft-crw --locked --target wasm32-unknown-unknown
samples:
@@ -84,10 +52,6 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
- uses: step-security/rust-cache@v2.9.2
with:
key: craft-crw-samples
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Cache the CC0 sample files
uses: actions/cache@v6.1.0
with:
+5 -6
View File
@@ -12,10 +12,10 @@ permissions:
contents: read
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued.
group: dialogs-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# One group per pull request and per branch: a newer push cancels the run it supersedes, on main
# too, since the newest run checks everything before it.
group: dialogs-${{ github.ref }}
cancel-in-progress: true
jobs:
dialogs:
@@ -37,6 +37,5 @@ jobs:
components: clippy
# The macOS code is linted on Linux: it compiles for the target without the SDK.
targets: aarch64-apple-darwin
- run: cargo clippy -p craft-dialogs --all-targets --locked -- -D warnings
# Native clippy and the tests run in the Workspace gate.
- run: cargo clippy -p craft-dialogs --target aarch64-apple-darwin --locked -- -D warnings
- run: cargo test -p craft-dialogs --locked
+7 -16
View File
@@ -12,20 +12,17 @@ permissions:
contents: read
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued.
group: doc-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# One group per pull request and per branch: a newer push cancels the run it supersedes, on main
# too, since the newest run checks everything before it.
group: doc-${{ github.ref }}
cancel-in-progress: true
# fmt, clippy and the tests run in the Workspace gate on every OS; this checks what it does not.
jobs:
doc:
name: Craft documents WASM
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
# macOS runners are kept for releases: macOS runs only when started by hand.
os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }}
runs-on: ${{ matrix.os }}
runs-on: ubuntu-latest
steps:
- name: Harden runner
uses: step-security/harden-runner@v2.22.4
@@ -37,11 +34,5 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
components: rustfmt, clippy
targets: wasm32-unknown-unknown
- run: cargo fmt --check
- run: cargo clippy -p craft-doc --all-targets --locked -- -D warnings
- run: cargo clippy -p craft-doc --all-targets --all-features --locked -- -D warnings
- run: cargo test -p craft-doc --locked -- --nocapture
- run: cargo test -p craft-doc --all-features --locked -- --nocapture
- run: cargo check -p craft-doc --all-features --locked --target wasm32-unknown-unknown
-41
View File
@@ -1,41 +0,0 @@
name: Document paths
on:
workflow_dispatch:
push:
branches: [main]
paths: ['crates/docpath/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/docpath.yml']
pull_request:
paths: ['crates/docpath/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/docpath.yml']
permissions:
contents: read
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued.
group: docpath-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
docpath:
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
# macOS runners are kept for releases: macOS runs only when started by hand.
os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }}
runs-on: ${{ matrix.os }}
steps:
- name: Harden runner
uses: step-security/harden-runner@v2.22.4
with:
egress-policy: audit
- uses: actions/checkout@v7.0.1
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
components: rustfmt, clippy
- run: cargo fmt --check
- run: cargo clippy -p craft-docpath --all-targets --locked -- -D warnings
- run: cargo test -p craft-docpath --locked -- --nocapture
+7 -14
View File
@@ -12,20 +12,17 @@ permissions:
contents: read
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued.
group: ecproj-format-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# One group per pull request and per branch: a newer push cancels the run it supersedes, on main
# too, since the newest run checks everything before it.
group: ecproj-format-${{ github.ref }}
cancel-in-progress: true
# fmt, clippy and the tests run in the Workspace gate on every OS; this checks what it does not.
jobs:
ecproj-format:
name: EffectCraft project format WASM
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
# macOS runners are kept for releases: macOS runs only when started by hand.
os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }}
runs-on: ${{ matrix.os }}
runs-on: ubuntu-latest
steps:
- name: Harden runner
uses: step-security/harden-runner@v2.22.4
@@ -37,9 +34,5 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
components: rustfmt, clippy
targets: wasm32-unknown-unknown
- run: cargo fmt --check
- run: cargo clippy -p craft-ecproj-format --all-targets --locked -- -D warnings
- run: cargo test -p craft-ecproj-format --locked -- --nocapture
- run: cargo check -p craft-ecproj-format --locked --target wasm32-unknown-unknown
+7 -17
View File
@@ -12,20 +12,17 @@ permissions:
contents: read
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued.
group: font-store-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# One group per pull request and per branch: a newer push cancels the run it supersedes, on main
# too, since the newest run checks everything before it.
group: font-store-${{ github.ref }}
cancel-in-progress: true
# fmt, clippy and the tests run in the Workspace gate on every OS; this checks what it does not.
jobs:
font-store:
name: Font store WASM
timeout-minutes: 20
strategy:
fail-fast: false
matrix:
# macOS runners are kept for releases: macOS runs only when started by hand.
os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }}
runs-on: ${{ matrix.os }}
runs-on: ubuntu-latest
steps:
- name: Harden runner
uses: step-security/harden-runner@v2.22.4
@@ -35,13 +32,6 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
components: rustfmt, clippy
targets: wasm32-unknown-unknown
- run: cargo fmt --check
- run: cargo clippy -p craft-font-store --all-targets --locked -- -D warnings
- run: cargo test -p craft-font-store --locked
- run: cargo clippy -p craft-font-catalog --all-targets --locked -- -D warnings
- run: cargo test -p craft-font-catalog --locked
- name: Check the web build
if: runner.os == 'Linux'
run: cargo check -p craft-font-store --target wasm32-unknown-unknown --locked
+10 -22
View File
@@ -42,11 +42,10 @@ permissions:
contents: read
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it
# always gets to save the cache.
group: shared-formats-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# One group per pull request and per branch: a newer push cancels the run it supersedes, on main
# too, since the newest run checks everything before it.
group: shared-formats-${{ github.ref }}
cancel-in-progress: true
jobs:
check:
@@ -68,7 +67,9 @@ jobs:
- { crate: craft-tiff, features: "" }
- { crate: craft-heif, features: "testdata" }
- { crate: craft-cms, features: "" }
name: ${{ matrix.crate }} lint, tests, WASM
# fmt, clippy with default and with all features, and the tests run in the Workspace gate on
# every OS; this lints each feature alone and checks the web build.
name: ${{ matrix.crate }} feature sets, WASM
runs-on: ubuntu-latest
steps:
- name: Harden runner
@@ -80,32 +81,19 @@ jobs:
persist-credentials: false
- uses: dtolnay/rust-toolchain@stable
with:
components: rustfmt, clippy
components: clippy
targets: wasm32-unknown-unknown
- uses: Swatinem/rust-cache@v2.9.3
with:
key: ${{ matrix.crate }}
save-if: ${{ github.ref == 'refs/heads/main' }}
- run: cargo fmt -p "$CRATE" --check
env:
CRATE: ${{ matrix.crate }}
- name: Clippy, default features and each feature alone
- name: Clippy, each feature alone
if: matrix.features != ''
env:
CRATE: ${{ matrix.crate }}
FEATURES: ${{ matrix.features }}
run: |
cargo clippy -p "$CRATE" --all-targets --locked -- -D warnings
for f in $FEATURES; do
echo "::group::feature: $f"
cargo clippy -p "$CRATE" --all-targets --locked --no-default-features --features "$f" -- -D warnings
echo "::endgroup::"
done
- run: cargo clippy -p "$CRATE" --all-targets --locked --all-features -- -D warnings
env:
CRATE: ${{ matrix.crate }}
- run: cargo test -p "$CRATE" --all-features --locked
env:
CRATE: ${{ matrix.crate }}
- run: cargo check -p "$CRATE" --all-features --locked --target wasm32-unknown-unknown
env:
CRATE: ${{ matrix.crate }}
-41
View File
@@ -1,41 +0,0 @@
name: Translations
on:
workflow_dispatch:
push:
branches: [main]
paths: ['crates/i18n/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/i18n.yml']
pull_request:
paths: ['crates/i18n/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/i18n.yml']
permissions:
contents: read
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued.
group: i18n-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
i18n:
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
# macOS runners are kept for releases: macOS runs only when started by hand.
os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }}
runs-on: ${{ matrix.os }}
steps:
- name: Harden runner
uses: step-security/harden-runner@v2.22.4
with:
egress-policy: audit
- uses: actions/checkout@v7.0.1
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
components: rustfmt, clippy
- run: cargo fmt --check
- run: cargo clippy -p craft-i18n --all-targets --features collate --locked -- -D warnings
- run: cargo test -p craft-i18n --features collate --locked -- --nocapture
+8 -15
View File
@@ -12,16 +12,17 @@ permissions:
contents: read
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it
# always gets to save the cache.
group: shared-jpeg-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# One group per pull request and per branch: a newer push cancels the run it supersedes, on main
# too, since the newest run checks everything before it.
group: shared-jpeg-${{ github.ref }}
cancel-in-progress: true
jobs:
check:
timeout-minutes: 10
name: JPEG lint, tests, WASM
# Uncached: the wasm check builds its dependencies on every run (about 10 minutes with setup).
timeout-minutes: 20
# fmt, clippy and the tests run in the Workspace gate on every OS.
name: JPEG WASM
runs-on: ubuntu-latest
steps:
- name: Harden runner
@@ -34,13 +35,5 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
components: rustfmt, clippy
targets: wasm32-unknown-unknown
- uses: step-security/rust-cache@v2.9.2
with:
key: craft-jpeg
save-if: ${{ github.ref == 'refs/heads/main' }}
- run: cargo fmt -p craft-jpeg --check
- run: cargo clippy -p craft-jpeg --all-targets --locked -- -D warnings
- run: cargo test -p craft-jpeg --locked
- run: cargo check -p craft-jpeg --locked --target wasm32-unknown-unknown
+8 -50
View File
@@ -12,16 +12,17 @@ permissions:
contents: read
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it
# always gets to save the cache.
group: keymap-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# One group per pull request and per branch: a newer push cancels the run it supersedes, on main
# too, since the newest run checks everything before it.
group: keymap-${{ github.ref }}
cancel-in-progress: true
jobs:
linux:
timeout-minutes: 15
name: Keymap (Linux)
# fmt, and clippy and the tests with default and with all features, run in the Workspace gate
# on every OS; this runs the feature subsets.
name: Keymap feature sets
runs-on: ubuntu-latest
env:
CRAFT_FONTS_DIR: ${{ github.workspace }}/.test-inputs/craft-fonts
@@ -43,12 +44,7 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
components: rustfmt, clippy
- uses: step-security/rust-cache@v2.9.2
with:
key: craft-keymap-linux
save-if: ${{ github.ref == 'refs/heads/main' }}
- run: cargo fmt -p craft-keymap --check
components: clippy
- run: cargo clippy -p craft-keymap --all-targets --no-default-features --locked -- -D warnings
# `snapshot` covers `ui` and `egui` and compiles the dialog screenshots, which need a GPU
# to run, so the test steps leave them out.
@@ -58,40 +54,6 @@ jobs:
run: cargo test -p craft-keymap --no-default-features --lib --tests --locked
- name: Test egui dispatch and the shortcuts dialog
run: cargo test -p craft-keymap --features ui --locked
native:
timeout-minutes: 15
name: Keymap (${{ matrix.os }})
strategy:
fail-fast: false
matrix:
# macOS runners are kept for releases: macOS runs only when started by hand.
os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["macos-latest","windows-latest"]') || fromJSON('["windows-latest"]') }}
runs-on: ${{ matrix.os }}
env:
CRAFT_FONTS_DIR: ${{ github.workspace }}/.test-inputs/craft-fonts
steps:
- name: Harden runner
uses: step-security/harden-runner@v2.22.4
with:
egress-policy: audit
- uses: actions/checkout@v7.0.1
with:
persist-credentials: false
- name: Checkout test fonts
uses: actions/checkout@v7.0.1
with:
repository: storytold/craft-fonts
ref: b076417b5880bd4154bd22f00695b1d28c6bea83
path: .test-inputs/craft-fonts
persist-credentials: false
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
- uses: step-security/rust-cache@v2.9.2
with:
key: craft-keymap-${{ matrix.os }}
save-if: ${{ github.ref == 'refs/heads/main' }}
- run: cargo test -p craft-keymap --features ui --locked
wasm:
timeout-minutes: 10
name: Keymap WASM
@@ -108,8 +70,4 @@ jobs:
with:
toolchain: stable
targets: wasm32-unknown-unknown
- uses: step-security/rust-cache@v2.9.2
with:
key: craft-keymap-wasm
save-if: ${{ github.ref == 'refs/heads/main' }}
- run: cargo check -p craft-keymap --features ui --locked --target wasm32-unknown-unknown
+12 -19
View File
@@ -11,17 +11,22 @@ on:
permissions:
contents: read
# A fraction of the fuzz-style cases, as in the Workspace gate; nightly.yml runs them in full.
env:
CRAFT_HOSTILE_DIVISOR: "8"
PROPTEST_CASES: "64"
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it
# always gets to save the cache.
group: shared-media-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# One group per pull request and per branch: a newer push cancels the run it supersedes, on main
# too, since the newest run checks everything before it.
group: shared-media-${{ github.ref }}
cancel-in-progress: true
jobs:
lint:
timeout-minutes: 10
name: Media lint, feature sets, WASM
# fmt, and clippy with default and with all features, run in the Workspace gate on every OS.
name: Media feature sets, WASM
runs-on: ubuntu-latest
steps:
- name: Harden runner
@@ -34,20 +39,12 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
components: rustfmt, clippy
components: clippy
targets: wasm32-unknown-unknown
- uses: step-security/rust-cache@v2.9.2
with:
key: craft-media-lint
save-if: ${{ github.ref == 'refs/heads/main' }}
- run: cargo fmt -p craft-media --check
- run: cargo clippy -p craft-media --all-targets --locked --no-default-features -- -D warnings
- run: cargo clippy -p craft-media --all-targets --locked --no-default-features --features bitstream -- -D warnings
- run: cargo clippy -p craft-media --all-targets --locked -- -D warnings
- run: cargo clippy -p craft-media --all-targets --locked --no-default-features --features matroska -- -D warnings
- run: cargo clippy -p craft-media --all-targets --locked --no-default-features --features ogg -- -D warnings
- run: cargo clippy -p craft-media --all-targets --locked --all-features -- -D warnings
- run: cargo clippy -p craft-media-testkit --all-targets --locked -- -D warnings
- run: cargo clippy -p craft-media --all-targets --locked --no-default-features --features serde -- -D warnings
- run: cargo check -p craft-media --all-features --locked --target wasm32-unknown-unknown
test:
@@ -65,10 +62,6 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
- uses: step-security/rust-cache@v2.9.2
with:
key: craft-media-test
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Build tests while installing ffmpeg
run: |
# The man-db index rebuild after dpkg is slow on hosted runners.
+3 -8
View File
@@ -18,7 +18,8 @@ concurrency:
jobs:
linux:
timeout-minutes: 15
name: Menu (Linux)
# fmt, native clippy and the tests run in the Workspace gate on every OS.
name: Menu macOS and WASM lint
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7.0.1
@@ -26,14 +27,8 @@ jobs:
persist-credentials: false
- uses: dtolnay/rust-toolchain@stable
with:
components: rustfmt, clippy
components: clippy
targets: aarch64-apple-darwin, wasm32-unknown-unknown
- uses: Swatinem/rust-cache@v2.9.3
with:
key: craft-menu-linux
- run: cargo fmt -p craft-menu --check
- run: cargo clippy -p craft-menu --all-targets --locked -- -D warnings
# The macOS backend type-checks from Linux; it needs no linker or SDK.
- run: cargo clippy -p craft-menu --lib --locked --target aarch64-apple-darwin -- -D warnings
- run: cargo clippy -p craft-menu --lib --locked --target wasm32-unknown-unknown -- -D warnings
- run: cargo test -p craft-menu --locked
+34
View File
@@ -0,0 +1,34 @@
name: Nightly full tests
# The tests the Workspace gate shortens, in full, once a day: every hostile-input hammer and sweep
# without CRAFT_HOSTILE_DIVISOR, and every proptest at its own case count (no PROPTEST_CASES). The
# rest of the workspace's tests run with them, on Linux, with the gate's setup.
on:
schedule:
- cron: "37 2 * * *"
workflow_dispatch:
permissions:
contents: read
concurrency:
group: nightly-${{ github.ref }}
cancel-in-progress: false
jobs:
full:
name: Full tests
uses: storytold/craft-common-actions/.github/workflows/gate.yml@main
with:
os: '["ubuntu-latest"]'
timeout-minutes: 120
tools: cargo-nextest@0.9.150
packages: mesa-vulkan-drivers libvulkan1
craft-fonts-ref: b076417b5880bd4154bd22f00695b1d28c6bea83
linux-env: WGPU_BACKEND=vulkan
env: CRAFT_PRINT_REQUIRE_QUEUE=0
setup-command: bash scripts/ci-test-inputs.sh
cache-paths: .test-inputs/photocraft/corpus
cache-key: photocraft-corpus
cache-key-files: scripts/ci-test-inputs.sh
command: cargo nextest run --workspace --all-features --locked --profile ci
+8 -14
View File
@@ -12,20 +12,18 @@ permissions:
contents: read
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued.
group: pcraft-format-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# One group per pull request and per branch: a newer push cancels the run it supersedes, on main
# too, since the newest run checks everything before it.
group: pcraft-format-${{ github.ref }}
cancel-in-progress: true
# fmt, clippy and the tests run in the Workspace gate on every OS, and its msrv job checks the
# workspace on 1.95; this checks the web build on 1.95.
jobs:
pcraft-format:
name: PhotoCraft format WASM
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
# macOS runners are kept for releases: macOS runs only when started by hand.
os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }}
runs-on: ${{ matrix.os }}
runs-on: ubuntu-latest
steps:
- name: Harden runner
uses: step-security/harden-runner@v2.22.4
@@ -37,9 +35,5 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: "1.95"
components: rustfmt, clippy
targets: wasm32-unknown-unknown
- run: cargo fmt --check
- run: cargo clippy -p craft-pcraft-format --all-targets --locked -- -D warnings
- run: cargo test -p craft-pcraft-format --locked -- --nocapture
- run: cargo check -p craft-pcraft-format --locked --target wasm32-unknown-unknown
+14 -56
View File
@@ -11,22 +11,23 @@ on:
permissions:
contents: read
# A fraction of the proptest cases, as in the Workspace gate; nightly.yml runs them in full.
env:
PROPTEST_CASES: "64"
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it
# always gets to save the cache.
group: shared-pdf-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# One group per pull request and per branch: a newer push cancels the run it supersedes, on main
# too, since the newest run checks everything before it.
group: shared-pdf-${{ github.ref }}
cancel-in-progress: true
jobs:
native:
name: PDF (${{ matrix.os }})
strategy:
fail-fast: false
matrix:
# macOS runners are kept for releases: macOS runs only when started by hand.
os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }}
runs-on: ${{ matrix.os }}
# fmt, clippy and the tests run in the Workspace gate on every OS, with the licensed fonts and
# the plate tests; this runs the tests behind craft_pdf_unicode_fixtures and the read-only
# contract, which is its own workspace.
name: PDF Unicode fixtures and read-only contract
runs-on: ubuntu-latest
env:
RUSTFLAGS: --cfg craft_pdf_unicode_fixtures
steps:
@@ -38,11 +39,6 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
components: rustfmt, clippy
- uses: step-security/rust-cache@v2.9.2
with:
key: craft-pdf
save-if: ${{ github.ref == 'refs/heads/main' }}
- uses: actions/setup-python@v7.1.0
with:
python-version: "3.12"
@@ -52,9 +48,7 @@ jobs:
python scripts/test-fetch-pdf-test-fonts.py
python scripts/fetch-pdf-test-fonts.py "$RUNNER_TEMP/craft-pdf-test-fonts"
echo "CRAFT_PDF_TEST_FONT_DIR=$RUNNER_TEMP/craft-pdf-test-fonts" >> "$GITHUB_ENV"
- run: cargo fmt --check
- run: cargo clippy -p craft-pdf --all-targets --all-features --locked -- -D warnings
- run: cargo test -p craft-pdf --all-features --locked
- run: cargo test -p craft-pdf --all-features --locked --test unicode_writing --test upstream_compatibility --test upstream_reading
- run: cargo test --manifest-path crates/pdf/read-only-contract/Cargo.toml --locked
backends:
name: PDF vendored contracts
@@ -68,10 +62,6 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
- uses: step-security/rust-cache@v2.9.2
with:
key: craft-pdf-backends
save-if: ${{ github.ref == 'refs/heads/main' }}
- run: bash scripts/test-pdf-backends.sh
wasm:
name: PDF WASM runtime
@@ -89,10 +79,6 @@ jobs:
with:
toolchain: stable
targets: wasm32-unknown-unknown
- uses: step-security/rust-cache@v2.9.2
with:
key: craft-pdf-wasm
save-if: ${{ github.ref == 'refs/heads/main' }}
- uses: actions/setup-python@v7.1.0
with:
python-version: "3.12"
@@ -117,31 +103,3 @@ jobs:
fallback: cargo-install
- run: cargo test -p craft-pdf --all-features --locked --target wasm32-unknown-unknown
- run: cargo test --manifest-path crates/pdf/read-only-contract/Cargo.toml --locked --target wasm32-unknown-unknown
plate-test:
name: Plate tests and craft-prepress
runs-on: ubuntu-latest
steps:
- name: Harden runner
uses: step-security/harden-runner@v2.22.4
with:
egress-policy: audit
- uses: actions/checkout@v7.0.1
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
components: clippy
- uses: step-security/rust-cache@v2.9.2
with:
key: craft-plate-test
save-if: ${{ github.ref == 'refs/heads/main' }}
- uses: actions/setup-python@v7.1.0
with:
python-version: "3.12"
- name: Provision licensed PDF test fonts
shell: bash
run: |
python scripts/test-fetch-pdf-test-fonts.py
python scripts/fetch-pdf-test-fonts.py "$RUNNER_TEMP/craft-pdf-test-fonts"
echo "CRAFT_PDF_TEST_FONT_DIR=$RUNNER_TEMP/craft-pdf-test-fonts" >> "$GITHUB_ENV"
- run: cargo clippy -p craft-plate-test -p craft-prepress --all-targets --all-features --locked -- -D warnings
- run: cargo test -p craft-plate-test -p craft-prepress --all-features --locked
-46
View File
@@ -1,46 +0,0 @@
name: Print spooler
on:
workflow_dispatch:
push:
branches: [main]
paths: ['crates/print/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/print.yml']
pull_request:
paths: ['crates/print/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/print.yml']
permissions:
contents: read
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued.
group: print-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
print:
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
# macOS runners are kept for releases: macOS runs only when started by hand.
os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }}
runs-on: ${{ matrix.os }}
steps:
- name: Harden runner
uses: step-security/harden-runner@v2.22.4
with:
egress-policy: audit
- uses: actions/checkout@v7.0.1
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
components: rustfmt, clippy
- run: cargo fmt --check
- run: cargo clippy -p craft-print --all-targets -- -D warnings
- name: Prepare the Windows test driver
if: runner.os == 'Windows'
shell: powershell
run: |
Start-Service Spooler
Add-PrinterDriver -Name 'Generic / Text Only'
- run: cargo test -p craft-print --locked -- --nocapture
timeout-minutes: 10
env:
CRAFT_PRINT_REQUIRE_QUEUE: ${{ runner.os == 'Windows' && '1' || '0' }}
+9 -17
View File
@@ -12,16 +12,16 @@ permissions:
contents: read
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it
# always gets to save the cache.
group: craft-proof-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# One group per pull request and per branch: a newer push cancels the run it supersedes, on main
# too, since the newest run checks everything before it.
group: craft-proof-${{ github.ref }}
cancel-in-progress: true
jobs:
check:
timeout-minutes: 20
name: Proofing lint, tests with and without Language data, WASM
# fmt, and clippy and the tests without Language data, run in the Workspace gate on every OS.
name: Proofing with Language data, WASM
runs-on: ubuntu-latest
steps:
- name: Harden runner
@@ -43,18 +43,10 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
components: rustfmt, clippy
components: clippy
targets: wasm32-unknown-unknown
- uses: step-security/rust-cache@v2.9.2
with:
key: craft-proof
save-if: ${{ github.ref == 'refs/heads/main' }}
- run: cargo fmt -p craft-proof --check
- name: Without Language data (a fork, docs.rs, offline)
run: |
cargo clippy -p craft-proof --all-targets --locked -- -D warnings
cargo test -p craft-proof --locked
cargo check -p craft-proof --locked --target wasm32-unknown-unknown
- name: The web build without Language data (a fork, docs.rs, offline)
run: cargo check -p craft-proof --locked --target wasm32-unknown-unknown
- name: A release without Language data fails
run: |
if CRAFT_I18N_REQUIRED=1 cargo check -p craft-proof --locked 2>build.log; then
-74
View File
@@ -1,74 +0,0 @@
name: Shared UI
on:
push:
branches: [main]
paths: ["crates/ui/**", "crates/i18n/**", "Cargo.toml", "Cargo.lock", ".github/workflows/ui.yml"]
pull_request:
paths: ["crates/ui/**", "crates/i18n/**", "Cargo.toml", "Cargo.lock", ".github/workflows/ui.yml"]
workflow_dispatch:
permissions:
contents: read
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it
# always gets to save the cache.
group: shared-ui-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
native:
timeout-minutes: 30
name: UI (${{ matrix.os }})
strategy:
fail-fast: false
matrix:
# macOS runners are kept for releases: macOS runs only when started by hand.
os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }}
runs-on: ${{ matrix.os }}
steps:
- name: Harden runner
uses: step-security/harden-runner@v2.22.4
with:
egress-policy: audit
- uses: actions/checkout@v7.0.1
- name: Install software Vulkan renderer
if: runner.os == 'Linux'
run: |
sudo apt-get update
sudo apt-get install -y mesa-vulkan-drivers libvulkan1
echo 'WGPU_BACKEND=vulkan' >> "$GITHUB_ENV"
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
components: rustfmt, clippy
- uses: step-security/rust-cache@v2.9.2
with:
key: craft-ui
save-if: ${{ github.ref == 'refs/heads/main' }}
- run: cargo fmt --check
- run: cargo clippy -p craft-ui --all-targets --locked -- -D warnings
- run: cargo clippy -p craft-ui --all-targets --all-features --locked -- -D warnings
- run: cargo test -p craft-ui --locked
- run: cargo test -p craft-ui --all-features --locked
wasm:
timeout-minutes: 15
name: UI WASM
runs-on: ubuntu-latest
steps:
- name: Harden runner
uses: step-security/harden-runner@v2.22.4
with:
egress-policy: audit
- uses: actions/checkout@v7.0.1
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
targets: wasm32-unknown-unknown
- uses: step-security/rust-cache@v2.9.2
with:
key: craft-ui-wasm
save-if: ${{ github.ref == 'refs/heads/main' }}
- run: cargo check -p craft-ui --all-features --locked --target wasm32-unknown-unknown
+9 -15
View File
@@ -12,20 +12,18 @@ permissions:
contents: read
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued.
group: update-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# One group per pull request and per branch: a newer push cancels the run it supersedes, on main
# too, since the newest run checks everything before it.
group: update-${{ github.ref }}
cancel-in-progress: true
# fmt, clippy and the tests with default and with all features run in the Workspace gate on every
# OS; this runs the egui feature alone.
jobs:
update:
name: Update with egui alone
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
# macOS runners are kept for releases: macOS runs only when started by hand.
os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }}
runs-on: ${{ matrix.os }}
runs-on: ubuntu-latest
steps:
- name: Harden runner
uses: step-security/harden-runner@v2.22.4
@@ -39,10 +37,6 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
components: rustfmt, clippy
- run: cargo fmt --check
# Default (no egui) and the egui feature both build and lint cleanly.
- run: cargo clippy -p craft-update --all-targets --locked -- -D warnings
components: clippy
- run: cargo clippy -p craft-update --all-targets --locked --features egui -- -D warnings
- run: cargo test -p craft-update --locked -- --nocapture
- run: cargo test -p craft-update --locked --features egui -- --nocapture
+8 -14
View File
@@ -12,20 +12,18 @@ permissions:
contents: read
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued.
group: vectorcraft-format-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# One group per pull request and per branch: a newer push cancels the run it supersedes, on main
# too, since the newest run checks everything before it.
group: vectorcraft-format-${{ github.ref }}
cancel-in-progress: true
# fmt, clippy and the tests run in the Workspace gate on every OS, and its msrv job checks the
# workspace on 1.95; this checks the web build on 1.95.
jobs:
vectorcraft-format:
name: VectorCraft format WASM
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
# macOS runners are kept for releases: macOS runs only when started by hand.
os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }}
runs-on: ${{ matrix.os }}
runs-on: ubuntu-latest
steps:
- name: Harden runner
uses: step-security/harden-runner@v2.22.4
@@ -37,9 +35,5 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: "1.95"
components: rustfmt, clippy
targets: wasm32-unknown-unknown
- run: cargo fmt --check
- run: cargo clippy -p craft-vectorcraft-format --all-targets --locked -- -D warnings
- run: cargo test -p craft-vectorcraft-format --locked -- --nocapture
- run: cargo check -p craft-vectorcraft-format --locked --target wasm32-unknown-unknown
+12 -18
View File
@@ -11,6 +11,11 @@ on:
permissions:
contents: read
# A fraction of the fuzz-style cases, as in the Workspace gate; nightly.yml runs them in full.
env:
CRAFT_HOSTILE_DIVISOR: "8"
PROPTEST_CASES: "64"
concurrency:
group: shared-video-${{ github.ref }}
cancel-in-progress: true
@@ -18,7 +23,8 @@ concurrency:
jobs:
lint:
timeout-minutes: 20
name: Video lint, feature sets, docs, WASM
# fmt and clippy with all features run in the Workspace gate on every OS.
name: Video feature sets, docs, WASM
runs-on: ubuntu-latest
steps:
- name: Harden runner
@@ -31,12 +37,8 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: "1.95"
components: rustfmt, clippy
components: clippy
targets: wasm32-unknown-unknown
- uses: step-security/rust-cache@v2.9.2
with:
key: craft-video-lint
- run: cargo fmt -p craft-video --check
# Each feature on its own, so code that needs another feature fails here.
- name: Clippy, each feature alone
run: |
@@ -45,19 +47,15 @@ jobs:
cargo clippy -p craft-video --all-targets --locked --no-default-features ${f:+--features "$f"} -- -D warnings
echo "::endgroup::"
done
- run: cargo clippy -p craft-video --all-targets --locked --all-features -- -D warnings
- run: cargo doc -p craft-video --all-features --no-deps --locked
env:
RUSTDOCFLAGS: "-D warnings"
- run: cargo check -p craft-video --all-features --locked --target wasm32-unknown-unknown
test:
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, windows-latest]
runs-on: ${{ matrix.os }}
name: Video tests (${{ matrix.os }})
# The Workspace gate runs these tests on every OS without ffmpeg; this runs them with it.
runs-on: ubuntu-latest
name: Video tests with ffmpeg
steps:
- name: Harden runner
uses: step-security/harden-runner@v2.22.4
@@ -69,11 +67,7 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: "1.95"
- uses: step-security/rust-cache@v2.9.2
with:
key: craft-video-${{ matrix.os }}
- name: Install ffmpeg (Linux)
if: runner.os == 'Linux'
- name: Install ffmpeg
run: |
sudo rm -f /var/lib/man-db/auto-update
sudo apt-get update -qq && sudo apt-get install -y -qq --no-install-recommends ffmpeg
+6 -8
View File
@@ -12,10 +12,10 @@ permissions:
contents: read
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued.
group: web-api-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# One group per pull request and per branch: a newer push cancels the run it supersedes, on main
# too, since the newest run checks everything before it.
group: web-api-${{ github.ref }}
cancel-in-progress: true
jobs:
web-api:
@@ -35,7 +35,7 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
components: rustfmt, clippy
components: clippy
targets: wasm32-unknown-unknown
- name: Read the wasm-bindgen version from Cargo.lock
id: wasm-bindgen
@@ -50,8 +50,6 @@ jobs:
with:
tool: wasm-bindgen@${{ steps.wasm-bindgen.outputs.version }}
fallback: cargo-install
- run: cargo fmt --check
- run: cargo clippy -p craft-web-api --all-targets --locked -- -D warnings
# fmt, native clippy and the native tests run in the Workspace gate.
- run: cargo clippy -p craft-web-api --all-targets --locked --target wasm32-unknown-unknown -- -D warnings
- run: cargo test -p craft-web-api --locked
- run: cargo test -p craft-web-api --locked --target wasm32-unknown-unknown
+7 -14
View File
@@ -12,16 +12,16 @@ permissions:
contents: read
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it
# always gets to save the cache.
group: shared-web-host-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# One group per pull request and per branch: a newer push cancels the run it supersedes, on main
# too, since the newest run checks everything before it.
group: shared-web-host-${{ github.ref }}
cancel-in-progress: true
jobs:
check:
timeout-minutes: 20
name: Web host lint, tests, WASM tests
# fmt, native clippy and the native tests run in the Workspace gate.
name: Web host WASM lint and tests
runs-on: ubuntu-latest
env:
CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUNNER: wasm-bindgen-test-runner
@@ -36,16 +36,9 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
components: rustfmt, clippy
components: clippy
targets: wasm32-unknown-unknown
- uses: step-security/rust-cache@v2.9.2
with:
key: craft-web-host
save-if: ${{ github.ref == 'refs/heads/main' }}
- run: cargo fmt -p craft-web-host --check
- run: cargo clippy -p craft-web-host --all-targets --locked -- -D warnings
- run: cargo clippy -p craft-web-host --all-targets --locked --target wasm32-unknown-unknown -- -D warnings
- run: cargo test -p craft-web-host --locked
- name: Read the wasm-bindgen version from Cargo.lock
id: wasm-bindgen
shell: bash
+10 -16
View File
@@ -12,20 +12,18 @@ permissions:
contents: read
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued.
group: window-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# One group per pull request and per branch: a newer push cancels the run it supersedes, on main
# too, since the newest run checks everything before it.
group: window-${{ github.ref }}
cancel-in-progress: true
# fmt, clippy and the tests with default and with all features (egui and restore together) run in
# the Workspace gate on every OS; this runs each feature alone.
jobs:
window:
name: Window with each feature alone
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
# macOS runners are kept for releases: macOS runs only when started by hand.
os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }}
runs-on: ${{ matrix.os }}
runs-on: ubuntu-latest
steps:
- name: Harden runner
uses: step-security/harden-runner@v2.22.4
@@ -39,12 +37,8 @@ jobs:
- uses: step-security/dtolnay-rust-toolchain@v1.0.0
with:
toolchain: stable
components: rustfmt, clippy
- run: cargo fmt --check
# Default (no egui) and the egui feature both build and lint cleanly.
- run: cargo clippy -p craft-window --all-targets --locked -- -D warnings
components: clippy
- run: cargo clippy -p craft-window --all-targets --locked --features egui -- -D warnings
- run: cargo test -p craft-window --locked -- --nocapture
- run: cargo test -p craft-window --locked --features egui -- --nocapture
- run: cargo clippy -p craft-window --all-targets --locked --features restore -- -D warnings
- run: cargo test -p craft-window --locked --features restore,egui -- --nocapture
- run: cargo test -p craft-window --locked --features restore -- --nocapture
+54 -39
View File
@@ -6,17 +6,21 @@ on:
tags: ["v*"]
pull_request:
workflow_dispatch:
# A full run on main every 6 hours that is never cancelled, so the caches are saved while pushes
# to main cancel each other.
schedule:
- cron: "17 */6 * * *"
permissions:
contents: read
concurrency:
# Pull requests share one group per branch, so a newer push cancels the older run. Every other
# run (push to main, a release tag, a manual run) gets a group of its own: it is neither
# cancelled nor replaced while queued, so a main run always finishes and saves the dependency
# cache, and a release run is never cancelled.
group: workspace-${{ github.event_name == 'pull_request' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
# A pull request, and main, each have one group, so a newer push cancels the run it supersedes;
# the newest push's run still tests everything before it. Every other run (a release tag, a
# manual run, the scheduled run) gets a group of its own and is neither cancelled nor replaced
# while queued: a release run always finishes, and the scheduled run saves the caches.
group: workspace-${{ (github.event_name == 'pull_request' || github.event_name == 'push' && github.ref == 'refs/heads/main') && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || github.event_name == 'push' && github.ref == 'refs/heads/main' }}
jobs:
native:
@@ -24,47 +28,58 @@ jobs:
uses: storytold/craft-common-actions/.github/workflows/gate.yml@main
with:
os: '["ubuntu-latest","windows-latest","macos-latest"]'
# On a pull request, macOS and Windows run only when it can change platform behaviour: a
# crate with platform cfg code or one that depends on it, the build setup, or a file outside
# the crates (craft-common-actions' select-platforms). The full-ci label runs them anyway
# from the next push or manual run; pushes to main always run every leg.
select-os: '["windows-latest","macos-latest"]'
timeout-minutes: 120
# The Windows leg runs its tests with nextest (.config/nextest.toml).
# Every leg runs its tests with nextest, and its doctests with cargo test (.config/nextest.toml).
tools: cargo-nextest@0.9.150
packages: mesa-vulkan-drivers libvulkan1
craft-fonts-ref: b076417b5880bd4154bd22f00695b1d28c6bea83
linux-env: WGPU_BACKEND=vulkan
windows-env: CRAFT_PRINT_REQUIRE_QUEUE=1
env: CRAFT_PRINT_REQUIRE_QUEUE=0
setup-command: |
set -euo pipefail
corpus_fetcher=.test-inputs/photocraft
mkdir -p "$corpus_fetcher"
git -C "$corpus_fetcher" init
git -C "$corpus_fetcher" remote add origin https://github.com/storytold/PhotoCraft.git
git -C "$corpus_fetcher" fetch --quiet --depth=1 origin 74330157cb601573ae4523a95bb9132822fc3918
git -C "$corpus_fetcher" checkout --quiet --detach FETCH_HEAD
(cd "$corpus_fetcher" && cargo xtask corpus --psd --psd-tools --atn --exr --heif --pngsuite)
mkdir -p corpus
cp -a "$corpus_fetcher/corpus/." corpus/
python scripts/test-fetch-pdf-test-fonts.py
python scripts/fetch-pdf-test-fonts.py "$RUNNER_TEMP/craft-pdf-test-fonts"
printf 'CRAFT_PSD_CORPUS=%s\n' "$GITHUB_WORKSPACE/$corpus_fetcher/corpus" >> "$GITHUB_ENV"
printf 'CRAFT_PDF_TEST_FONT_DIR=%s\n' "$RUNNER_TEMP/craft-pdf-test-fonts" >> "$GITHUB_ENV"
# No debuginfo on Windows: writing PDBs for every test binary is a large part of its link
# time. A panic still names its file and line; only backtrace frames lose theirs, and the
# Linux and macOS legs keep line tables.
windows-env: |
CRAFT_PRINT_REQUIRE_QUEUE=1
CARGO_PROFILE_DEV_DEBUG=0
# The fuzz-style tests run a fraction of their cases here: every hostile-input hammer and
# sweep 1/8 (craft-testkit's CRAFT_HOSTILE_DIVISOR), every proptest without its own case
# count 64 cases instead of 256. nightly.yml runs them all in full.
env: |
CRAFT_PRINT_REQUIRE_QUEUE=0
CRAFT_HOSTILE_DIVISOR=8
PROPTEST_CASES=64
setup-command: bash scripts/ci-test-inputs.sh
# The PhotoCraft corpora, keyed on the script that pins them: a hit skips the PhotoCraft
# checkout and its xtask build (1:51 on Windows).
cache-paths: .test-inputs/photocraft/corpus
cache-key: photocraft-corpus
cache-key-files: scripts/ci-test-inputs.sh
# A `lint (<os>)` leg beside each `gate (<os>)` leg, so clippy is off the tests' critical path.
lint-command: |
if [ "$RUNNER_OS" = Linux ]; then
python3 scripts/test-workspace-concurrency.py
python3 scripts/test-workspace-tests.py
fi
cargo fmt --check
cargo clippy --workspace --all-targets --locked -- -D warnings
cargo clippy --workspace --all-targets --all-features --locked -- -D warnings
command: |
python3 scripts/test-workspace-concurrency.py
cargo fmt --check
cargo clippy --workspace --all-targets --locked -- -D warnings
cargo clippy --workspace --all-targets --all-features --locked -- -D warnings
cargo test --workspace --locked
cargo test --workspace --all-features --locked
windows-command: |
powershell.exe -NoProfile -Command '$ErrorActionPreference = "Stop"; Start-Service Spooler; Add-PrinterDriver -Name "Generic / Text Only"'
cargo fmt --check
cargo clippy --workspace --all-targets --locked -- -D warnings
cargo clippy --workspace --all-targets --all-features --locked -- -D warnings
# The same tests as `cargo test`, run in parallel across test binaries instead of one
# binary at a time; nextest skips doctests, so `cargo test --doc` runs them.
cargo nextest run --workspace --locked --profile ci
cargo test --workspace --doc --locked
# The tests run once, with every feature, in parallel across test binaries; nextest skips
# doctests, so `cargo test --doc` runs them. The default-feature compile is the lint leg's
# first clippy run; the last command runs the crates whose tests check behaviour with a
# feature off, which an all-features build cannot reach.
cargo nextest run --workspace --all-features --locked --profile ci
cargo test --workspace --doc --all-features --locked
cargo nextest run -p craft-image-codecs -p craft-text --locked --profile ci
windows-command: |
powershell.exe -NoProfile -Command '$ErrorActionPreference = "Stop"; Start-Service Spooler; Add-PrinterDriver -Name "Generic / Text Only"'
cargo nextest run --workspace --all-features --locked --profile ci
cargo test --workspace --doc --all-features --locked
cargo nextest run -p craft-image-codecs -p craft-text --locked --profile ci
msrv:
name: Rust 1.95
+28 -1
View File
@@ -119,6 +119,7 @@ The XML reader in `crates/plist/src/xml.rs` and its base64 compatibility helper
`crates/palette` extracts the GPL text parser from AniCraft `crates/doc/src/swatch.rs` at `d875a7feb250e6ab06aa28a37fb78a0699e88330` (MIT OR Apache-2.0), introduced by Tanish Manku in `a64130a9da9bdde545463b852d95d6a6774b1dfd` (AniCraft #270). The import preserves Tanish as primary author. Neutral RGB/name adaptation separates application display-name sanitization and the `Untitled` policy; validation tests and integration are subsequent Dylan M. Taylor changes.
The legacy ZIP Zstandard method-20 mapping and bounded-read regression in `crates/zip/src/lenient` are ported from RRFZ's Craft Anything commit `29badae7c9280fdf4c24795927bf3a91f51694c6` (MIT OR Apache-2.0). RRFZ remains the primary Git author. Firmware scanner detection remains app-owned and is not part of this port.
## TAR/CPIO compatibility updates
The old GNU sparse-extension traversal, GNU CRC-symlink compatibility and PAX-size precedence fixes and their regression tests in `crates/tar` are ported from RRFZ's Craft Anything PRs #1973 (`8ea2ec66a1bc80f0420f3695baac877a0cb9b9a5`), #1984 (`b226a475f7dce9710e3af7a069386fe7a9720df7`) and #2007 (`d29465c20b0d570ea4973be512a0cb4e212bb60f`). Source is MIT OR Apache-2.0; RRFZ remains the primary Git author of each port. The PAX update adapts to the shared parser's existing `PaxOverrides` structure without changing the source policy.
@@ -130,17 +131,43 @@ The USDZ alignment requirement and compatibility fixture for `Writer::add_stored
The raw gradient wire ordering in `craft-lottie::Gradient::flat_values` follows
AniCraft's Lottie exporter by Tanish Manku (`63b39355`, `b9258b7b`), MIT OR Apache-2.0.
Sanitization, animation fitting and numeric output policy remain caller-owned.
## Screen Video key frames
`crates/video/src/screen_video.rs` extracts AniCraft `crates/xfl/src/video.rs` codec-3 decoding and its synthetic partial-block fixture, introduced by Tanish Manku in `1f3e603b7ff53a3da364e41ec4da8d43dc83aab0` with pixel-loop cleanup `2b7571363b4b647266df758f95b6e397021cfd4d` (MIT OR Apache-2.0). Import commits preserve Tanish as primary author. Caller-owned bounds and neutral RGBA output separate codec work from the consuming app's FLV first-frame/report policy. Boundary tests and integration are subsequent Dylan M. Taylor changes.
## Source Map v3 parser and lookup
`crates/sourcemap` extracts the bounded parser, source inventory and generated-position lookup from Craft Anything `crates/formats/src/sourcemap.rs` at `e5c9c94106747ed44b9a3fb0decce867c2285f93` (MIT OR Apache-2.0). Core-history ports preserve Brandon Thomas, BFlat, Afloat, RRFZ, Nicolas Bohn and Alex Joaquim Pereira as primary authors, with exact source revisions in each commit. Six regression files preserve their source authors; nine core unit tests retain the original tests. App annotation scanning, data URLs, base64 and file I/O are intentionally not imported. Workspace wiring is a subsequent change; existing limits and errors are unchanged.
## JSONC span-preserving editing
`crates/jsonc` is extracted from Craft Anything `crates/formats/src/jsonc.rs` at `e5c9c94106747ed44b9a3fb0decce867c2285f93` (MIT OR Apache-2.0). The original implementation and hardening by BFlat, newline preservation by NetterSilver, and subsequent RRFZ/Afloat correctness fixes retain separate primary-author port commits with immutable source revisions. Shared API wiring and optional input-byte admission are subsequent changes. Compatibility parsing and all existing formatting/editing tests are retained; application file I/O and policy are not imported.
## XFA XML DOM
`crates/xfa/src/xml.rs` is extracted from PDFCraft at `271e479c53723b1b294c866d5cb2f27604601b29`, originally by Dylan M. Taylor (`eead4541`, `7d44e654`), MIT OR Apache-2.0. Only visibility and module boundaries change; PDF/layout/form policy remains in the app.
`crates/xfa/src/xml.rs` is extracted from PDFCraft at `271e479c53723b1b294c866d5cb2f27604601b29`, originally by Dylan M. Taylor (`eead4541`, `7d44e654`), MIT OR Apache-2.0. The initial extraction changes visibility and module boundaries; subsequent namespace-retention budget hardening intentionally tightens resource admission. PDF/layout/form policy remains in the app.
## Java class inspection
`crates/java-class` extracts Craft Anything `java_class.rs` and `java_ops.rs` at `5979f9b13f16dcf1df3c7ae7f8e7735f452866f2` (MIT OR Apache-2.0), preserving separate primary-author source-history ports by Brandon Thomas, BFlat, Afloat and RRFZ. The checked byte-reader, modified UTF-8 unit decoder, descriptor display helper and synthetic class fixtures come from the same source snapshot. Seven regression files retain their source authors. Neutral error wiring and shared crate integration are subsequent changes. No JAR archive, application I/O or bytecode execution is imported.
## WebAssembly inspection
`crates/wasm` extracts Craft Anything `crates/formats/src/wasm.rs` at `5979f9b13f16dcf1df3c7ae7f8e7735f452866f2`, with its checked byte-reader subset and original synthetic WASM/testkit fixtures (MIT OR Apache-2.0). Eight core-history ports preserve Brandon Thomas, Afloat and RRFZ as primary authors; component, repeated-custom-section and referenced-string regression imports retain source authors and commit references. Fixtures come from `crates/testkit/src/formats.rs` and deterministic mutation helpers from `crates/testkit/src/lib.rs`; no third-party binaries. Shared error naming and workspace wiring are subsequent changes. Existing inspection limits and report behavior are unchanged; this crate validates/inspects/renders, never executes modules.
## Packet capture and protocol inspection
`crates/pcap` extracts Craft Anything `crates/formats/src/pcap.rs`, its checked byte reader primitives and synthetic packet fixtures/regressions at `5979f9b13f16dcf1df3c7ae7f8e7735f452866f2` (MIT OR Apache-2.0). Brandon Thomas introduced the implementation, byte reader and fixtures in `f64ae3bf3c8863a982af1304e06de20653803968`. Subsequent fixes by phatloi0010, GGWA, BFlat, Afloat, NetterSilver, Max Mittelstädt and RRFZ retain separate primary-author source-history commits with immutable source revisions. Shared error/API wiring and chrono-based timestamp formatting with exhaustive day-boundary parity tests are subsequent Dylan M. Taylor changes. No packet reassembly, decryption, live capture or app analysis policy is imported.
## HTTP Archive reader
`crates/har` preserves Craft Anything `crates/formats/src/har.rs` from `5979f9b13f16dcf1df3c7ae7f8e7735f452866f2` (MIT OR Apache-2.0). Brandon Thomas introduced the typed reader in `f64ae3bf3c8863a982af1304e06de20653803968`; BFlat added UTF-8 BOM compatibility in `cb0bb9d8fa4536ba44ef2c86f5ed759b879cc2d3`. Both source snapshots retain primary author attribution. Shared wiring, explicit byte admission and its tests are subsequent Dylan M. Taylor changes. Redaction and analysis remain app-owned.
## EVM bytecode inspection
`crates/evm` extracts Craft Anything `crates/formats/src/evm.rs` at `a31e995858f4312e0f1d59a007112da191ef2d26` (MIT OR Apache-2.0). Eight separate source-history ports preserve Brandon Thomas, Afloat, RRFZ and NetterSilver as primary authors with immutable source references. Synthetic EVM fixtures and deterministic mutation helpers come from the same app source; regression files retain their source provenance. Neutral errors and workspace integration are subsequent changes. No code execution or transaction behavior is imported.
## XFDF field reader
`crates/xfdf` extracts PDFCraft field reading from #5937 refresh `cb564fd827f572818e9ddb0ea5989a432b1f39fd` (MIT OR Apache-2.0). App-side annotation and form policy remains local; source history is retained in the original repository.
Retained XFDF source contributors (move-aware blame): Brandon Thomas, Marco Arnulfo, pkibiego and storybored379; module split by aiongg. Subsequent budget/integration work by Dylan M. Taylor.
Generated
+123 -8
View File
@@ -1074,6 +1074,18 @@ dependencies = [
"winx",
]
[[package]]
name = "cap-std"
version = "4.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1ec78e242cfa2cfe276807ac2ecc00315a6c97786977414bcd1c3963b6c91b8"
dependencies = [
"cap-primitives",
"io-extras",
"io-lifetimes 3.0.1",
"rustix 1.1.5",
]
[[package]]
name = "cast"
version = "0.3.0"
@@ -1616,6 +1628,25 @@ dependencies = [
"craft-media-testkit",
]
[[package]]
name = "craft-cloud"
version = "0.1.0"
dependencies = [
"serde",
"thiserror 2.0.21",
]
[[package]]
name = "craft-cloud-local"
version = "0.1.0"
dependencies = [
"cap-std",
"craft-cloud",
"craft-ui",
"egui",
"tempfile",
]
[[package]]
name = "craft-cms"
version = "0.1.0"
@@ -1881,6 +1912,16 @@ dependencies = [
"zip 9.1.3",
]
[[package]]
name = "craft-evm"
version = "0.1.0"
dependencies = [
"craft-testkit",
"serde",
"serde_json",
"thiserror 2.0.21",
]
[[package]]
name = "craft-ews"
version = "0.1.0"
@@ -2122,13 +2163,6 @@ dependencies = [
"flate2",
]
[[package]]
name = "craft-hdr"
version = "0.1.0"
dependencies = [
"thiserror 2.0.21",
]
[[package]]
name = "craft-har"
version = "0.1.0"
@@ -2137,6 +2171,13 @@ dependencies = [
"thiserror 2.0.21",
]
[[package]]
name = "craft-hdr"
version = "0.1.0"
dependencies = [
"thiserror 2.0.21",
]
[[package]]
name = "craft-heif"
version = "0.1.0"
@@ -2261,6 +2302,15 @@ dependencies = [
"widestring",
]
[[package]]
name = "craft-java-class"
version = "0.1.0"
dependencies = [
"craft-testkit",
"serde",
"thiserror 2.0.21",
]
[[package]]
name = "craft-jmap"
version = "0.1.0"
@@ -2611,6 +2661,16 @@ dependencies = [
name = "craft-palette"
version = "0.1.0"
[[package]]
name = "craft-pcap"
version = "0.1.0"
dependencies = [
"chrono",
"serde",
"serde_json",
"thiserror 2.0.21",
]
[[package]]
name = "craft-pcraft-bundle"
version = "0.1.0"
@@ -3062,6 +3122,15 @@ dependencies = [
name = "craft-sniff"
version = "0.1.0"
[[package]]
name = "craft-sourcemap"
version = "0.1.0"
dependencies = [
"serde",
"serde_json",
"thiserror 2.0.21",
]
[[package]]
name = "craft-ssip"
version = "0.1.0"
@@ -3108,11 +3177,14 @@ dependencies = [
"craft-xml",
"data-url",
"flate2",
"gif",
"image-webp",
"kurbo 0.13.1",
"quick-xml 0.41.0",
"resvg 0.48.1",
"roxmltree 0.21.2",
"thiserror 2.0.21",
"toml",
"usvg 0.48.1",
"vectorcraft-geom",
"vectorcraft-pathops",
@@ -3130,6 +3202,7 @@ dependencies = [
name = "craft-tar"
version = "0.1.0"
dependencies = [
"craft-testkit",
"serde",
]
@@ -3214,6 +3287,7 @@ dependencies = [
name = "craft-ui"
version = "0.1.0"
dependencies = [
"craft-cloud",
"craft-fonts-build",
"craft-i18n",
"eframe",
@@ -3315,6 +3389,17 @@ dependencies = [
"thiserror 2.0.21",
]
[[package]]
name = "craft-wasm"
version = "0.1.0"
dependencies = [
"craft-testkit",
"serde",
"thiserror 2.0.21",
"wasmparser 0.261.0",
"wasmprinter",
]
[[package]]
name = "craft-web-api"
version = "0.1.0"
@@ -3396,6 +3481,14 @@ dependencies = [
"craft-xml",
]
[[package]]
name = "craft-xfdf"
version = "0.1.0"
dependencies = [
"craft-xml",
"thiserror 2.0.21",
]
[[package]]
name = "craft-xfl"
version = "0.1.0"
@@ -12005,7 +12098,7 @@ dependencies = [
"wasmi_collections",
"wasmi_core",
"wasmi_ir",
"wasmparser",
"wasmparser 0.228.0",
]
[[package]]
@@ -12045,6 +12138,28 @@ dependencies = [
"indexmap",
]
[[package]]
name = "wasmparser"
version = "0.261.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4f20f20e44f7e8aeb6744823ea9d869ede51e51be4fdaedede2852282e54d2d8"
dependencies = [
"bitflags 2.13.2",
"indexmap",
"semver",
]
[[package]]
name = "wasmprinter"
version = "0.261.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86a21c9e3325655ba02de3e645bc8357a8b3651a2e3ada25b6dbc37560deb9b7"
dependencies = [
"anyhow",
"termcolor",
"wasmparser 0.261.0",
]
[[package]]
name = "wayland-backend"
version = "0.3.17"
+8
View File
@@ -12,7 +12,12 @@ repository = "https://github.com/storytold/craft-libs"
[workspace.dependencies]
craft-xfa = { path = "crates/xfa" }
craft-evm = { path = "crates/evm" }
craft-java-class = { path = "crates/java-class" }
craft-sourcemap = { path = "crates/sourcemap" }
craft-pcap = { path = "crates/pcap" }
craft-har = { path = "crates/har" }
craft-xfdf = { path = "crates/xfdf" }
craft-jsonc = { path = "crates/jsonc" }
craft-palette = { path = "crates/palette" }
craft-score = { path = "crates/score" }
@@ -28,6 +33,7 @@ acadrust = "=0.6.6"
log = "0.4"
# internal: add each crate here as `craft-<name> = { path = "crates/<name>" }`
craft-plist = { path = "crates/plist" }
craft-wasm = { path = "crates/wasm" }
craft-platform = { path = "crates/platform" }
craft-aep = { path = "crates/aep" }
craft-affinity = { path = "crates/affinity" }
@@ -59,6 +65,8 @@ craft-captions = { path = "crates/captions" }
craft-cfb = { path = "crates/cfb" }
craft-codepage = { path = "crates/codepage" }
craft-cms = { path = "crates/cms" }
craft-cloud = { path = "crates/cloud" }
craft-cloud-local = { path = "crates/cloud-local" }
craft-color = { path = "crates/color" }
craft-frame = { path = "crates/frame" }
craft-crs = { path = "crates/crs" }
+20
View File
@@ -36,6 +36,8 @@ flac-encode = ["craft-media/bitstream"]
# The pure-Rust Vorbis encoder (`vorbis::encoder`) and Ogg Vorbis files through `encode` and
# `encode_ogg_vorbis`.
vorbis-encode = ["craft-media/ogg"]
# The pure-Rust Vorbis decoder (`vorbis::decoder`), packets and whole Ogg Vorbis files.
vorbis-decode = ["craft-media/ogg"]
# Codec modules on their own, for packet-level use.
aac-decode = ["craft-media/bitstream"]
# The AAC encoder (`aac::Encoder`), and AAC in M4A through `encode` and `FileEncoder`.
@@ -49,6 +51,8 @@ qoa-decode = []
qoa-encode = []
# ITU-T G.722 wideband sub-band ADPCM speech codec (64 kbit/s).
g722 = []
# ITU-T G.726 ADPCM speech codec (16/24/32/40 kbit/s).
g726 = []
opus-encode = []
# GSM 06.10 full-rate speech: decoder and encoder in the `gsm` module, each on its own.
gsm = ["gsm-decode", "gsm-encode"]
@@ -121,6 +125,18 @@ required-features = ["adpcm"]
name = "flac_decode"
required-features = ["flac-decode", "flac-encode"]
[[test]]
name = "vorbis_decode_oracle"
required-features = ["vorbis-decode", "vorbis-encode", "decode"]
[[test]]
name = "vorbis_decode_hostile"
required-features = ["vorbis-decode", "vorbis-encode"]
[[test]]
name = "vorbis_decode_synthetic"
required-features = ["vorbis-decode"]
[[test]]
name = "gsm_oracle"
required-features = ["gsm"]
@@ -133,6 +149,10 @@ required-features = ["qoa"]
name = "g722_oracle"
required-features = ["g722"]
[[test]]
name = "g726_oracle"
required-features = ["g726"]
[[test]]
name = "sbc_oracle"
required-features = ["sbc"]
+1
View File
@@ -39,6 +39,7 @@ No default features. The shared types (`AudioBuffer`, `AudioInfo`, `ChannelLayou
| `alac-encode` | `alac::Encoder`; `encode`, `FileEncoder` for ALAC in M4A | craft-media `isobmff` |
| `opus-decode`, `opus-encode` | `opus::Decoder`/`StreamDecoder`, `opus::Encoder` | none |
| `vorbis-encode` | `vorbis::encoder::Encoder` (pure Rust); `encode_ogg_vorbis`, `encode_ogg` and `encode` for Ogg Vorbis files (whole buffer; not `FileEncoder`) | craft-media `ogg` |
| `vorbis-decode` | `vorbis::decoder::Decoder` (packets), `vorbis::decoder::decode_ogg` (whole Ogg Vorbis files); native Vorbis I: floors 0 and 1, residues 0 to 2, channel coupling, both block sizes. `decode` still reads Vorbis files through symphonia | craft-media `ogg` |
| `ac3-decode` | `ac3::Decoder`, `ac3::ProgrammeDecoder` | craft-media `bitstream` |
| `ac3-encode`, `eac3-encode` | `ac3::encoder::Encoder`; `encode`, `FileEncoder` for AC-3 and E-AC-3 streams | craft-media `bitstream` |
| `mpa-decode` | `mpa::Decoder` | craft-media `bitstream` |
+996
View File
@@ -0,0 +1,996 @@
//! ITU-T Recommendation G.726 ADPCM speech codec (16, 24, 32, 40 kbit/s).
//!
//! Clean-room implementation based on ITU-T Recommendation G.726 (12/1990)
//! "40, 32, 24, 16 kbit/s Adaptive Differential Pulse Code Modulation (ADPCM)".
//!
//! Provides encoding from 8 kHz 16-bit linear PCM to G.726 bitstreams, and decoding
//! from G.726 bitstreams to 8 kHz 16-bit linear PCM.
//!
//! Supports both standard big-endian bit packing (network order / RFC 3551) and
//! little-endian bit packing (RIFF WAV / AVID format tag 0x0045 / 0x0014).
const MAX_CHANNELS: usize = 8;
const MAX_INPUT_BYTES: usize = 64 * 1024 * 1024;
const MAX_SAMPLES: usize = 64 * 1024 * 1024;
/// Bit rate for G.726 ADPCM.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum BitRate {
/// 16 kbit/s (2 bits per sample).
Rate16k,
/// 24 kbit/s (3 bits per sample).
Rate24k,
/// 32 kbit/s (4 bits per sample).
Rate32k,
/// 40 kbit/s (5 bits per sample).
Rate40k,
}
impl BitRate {
/// Number of bits per code (sample).
#[inline]
#[must_use]
pub const fn code_size(self) -> usize {
match self {
Self::Rate16k => 2,
Self::Rate24k => 3,
Self::Rate32k => 4,
Self::Rate40k => 5,
}
}
/// Bit rate in bits per second.
#[inline]
#[must_use]
pub const fn bit_rate_bps(self) -> u32 {
match self {
Self::Rate16k => 16_000,
Self::Rate24k => 24_000,
Self::Rate32k => 32_000,
Self::Rate40k => 40_000,
}
}
}
/// Bit packing order within bytes.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum BitPacking {
/// Big-endian packing: MSB to LSB (standard RFC 3551 / network order).
#[default]
BigEndian,
/// Little-endian packing: LSB to MSB (WAV / RIFF order).
LittleEndian,
}
/// Errors returned by G.726 operations.
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
pub enum Error {
/// Channel count must be between 1 and 8.
#[error("unsupported channel count: {0} (must be 1..=8)")]
InvalidChannels(usize),
/// Input exceeds safe maximum memory bound.
#[error("input exceeds safe maximum processing limit")]
InputTooLarge,
/// Output buffer allocation failed.
#[error("memory allocation failed")]
AllocationFailed,
}
// ---------------------------------------------------------------------------
// G.726 Tables
// ---------------------------------------------------------------------------
// Table 11..14 (I -> DQS, DQLN)
static OUT_16: [(u8, u16); 4] = [(0, 116), (0, 365), (1, 365), (1, 116)];
static OUT_24: [(u8, u16); 8] = [(0, 2048), (0, 135), (0, 273), (0, 373), (1, 373), (1, 273), (1, 135), (1, 2048)];
static OUT_32: [(u8, u16); 16] = [
(0, 2048),
(0, 4),
(0, 135),
(0, 213),
(0, 273),
(0, 323),
(0, 373),
(0, 425),
(1, 425),
(1, 373),
(1, 323),
(1, 273),
(1, 213),
(1, 135),
(1, 4),
(1, 2048),
];
static OUT_40: [(u8, u16); 32] = [
(0, 2048),
(0, 4030),
(0, 28),
(0, 104),
(0, 169),
(0, 224),
(0, 274),
(0, 318),
(0, 358),
(0, 395),
(0, 429),
(0, 459),
(0, 488),
(0, 514),
(0, 539),
(0, 566),
(1, 566),
(1, 539),
(1, 514),
(1, 488),
(1, 459),
(1, 429),
(1, 395),
(1, 358),
(1, 318),
(1, 274),
(1, 224),
(1, 169),
(1, 104),
(1, 28),
(1, 4030),
(1, 2048),
];
// FUNCTW: scale factor multipliers WI
static WI_16: [u16; 2] = [4074, 439];
static WI_24: [u16; 4] = [4092, 30, 137, 582];
static WI_32: [u16; 8] = [4084, 18, 41, 64, 112, 198, 355, 1122];
static WI_40: [u16; 16] = [14, 14, 24, 39, 40, 41, 58, 100, 141, 179, 219, 280, 358, 440, 529, 696];
#[inline]
fn get_wi(rate: BitRate, code: u8) -> u16 {
match rate {
BitRate::Rate16k => {
let im = if (code >> 1) == 0 { (code & 1) as usize } else { ((3 - code) & 1) as usize };
WI_16[im]
}
BitRate::Rate24k => {
let im = if (code >> 2) == 0 { (code & 3) as usize } else { ((7 - code) & 3) as usize };
WI_24[im]
}
BitRate::Rate32k => {
let im = if (code >> 3) == 0 { (code & 7) as usize } else { ((15 - code) & 7) as usize };
WI_32[im]
}
BitRate::Rate40k => {
let im = if (code >> 4) == 0 { (code & 15) as usize } else { ((31 - code) & 15) as usize };
WI_40[im]
}
}
}
#[inline]
fn get_fi(rate: BitRate, code: u8) -> u8 {
match rate {
BitRate::Rate16k => {
let im = if (code >> 1) == 0 { code & 1 } else { (3 - code) & 1 };
if im == 1 { 7 } else { 0 }
}
BitRate::Rate24k => {
let im = if (code >> 2) == 0 { code & 3 } else { (7 - code) & 3 };
match im {
0 => 0,
1 => 1,
2 => 2,
_ => 7,
}
}
BitRate::Rate32k => {
let im = if (code >> 3) == 0 { code & 7 } else { (15 - code) & 7 };
if im <= 2 {
0
} else if im <= 5 {
1
} else if im == 6 {
3
} else {
7
}
}
BitRate::Rate40k => {
let im = if (code >> 4) == 0 { code & 15 } else { (31 - code) & 15 };
if im <= 4 {
0
} else if im <= 9 {
1
} else {
match im {
10 => 2,
11 => 3,
12 => 4,
13 => 5,
_ => 6,
}
}
}
}
}
// Quantizer decision intervals (lo, hi, positive_code, negative_code)
static THRESH_16: [(i32, i32, u8, u8); 1] = [(261, 2047, 1, 2)];
static THRESH_24: [(i32, i32, u8, u8); 3] = [(331, 2047, 3, 4), (218, 330, 2, 5), (8, 217, 1, 6)];
static THRESH_32: [(i32, i32, u8, u8); 8] = [
(400, 2047, 7, 8),
(349, 399, 6, 9),
(300, 348, 5, 10),
(246, 299, 4, 11),
(178, 245, 3, 12),
(80, 177, 2, 13),
(0, 79, 1, 14),
(3972, 4095, 1, 14),
];
static THRESH_40: [(i32, i32, u8, u8); 16] = [
(553, 2047, 15, 16),
(528, 552, 14, 17),
(502, 527, 13, 18),
(475, 501, 12, 19),
(445, 474, 11, 20),
(413, 444, 10, 21),
(378, 412, 9, 22),
(339, 377, 8, 23),
(298, 338, 7, 24),
(250, 297, 6, 25),
(198, 249, 5, 26),
(139, 197, 4, 27),
(68, 138, 3, 28),
(0, 67, 2, 29),
(4080, 4095, 2, 29),
(3974, 4079, 1, 30),
];
#[inline]
fn quantize_16(dln: i32, ds: u8) -> u8 {
for &(lo, hi, pos, neg) in &THRESH_16 {
if (lo..=hi).contains(&dln) {
return if ds == 0 { pos } else { neg };
}
}
if ds == 0 { 0 } else { 3 }
}
#[inline]
fn quantize_24(dln: i32, ds: u8) -> u8 {
for &(lo, hi, pos, neg) in &THRESH_24 {
if (lo..=hi).contains(&dln) {
return if ds == 0 { pos } else { neg };
}
}
7
}
#[inline]
fn quantize_32(dln: i32, ds: u8) -> u8 {
for &(lo, hi, pos, neg) in &THRESH_32 {
if (lo..=hi).contains(&dln) {
return if ds == 0 { pos } else { neg };
}
}
15
}
#[inline]
fn quantize_40(dln: i32, ds: u8) -> u8 {
for &(lo, hi, pos, neg) in &THRESH_40 {
if (lo..=hi).contains(&dln) {
return if ds == 0 { pos } else { neg };
}
}
31
}
// ---------------------------------------------------------------------------
// G.726 Core State
// ---------------------------------------------------------------------------
#[derive(Debug, Clone)]
struct Core {
rate: BitRate,
is_40k: bool,
a: [u16; 2],
b: [u16; 6],
dms: i16,
dml: i16,
ap: i16,
yu: i16,
yl: i32,
sr: [u16; 2],
dq: [u16; 6],
pk: [u8; 2],
td: i16,
}
impl Core {
fn new(rate: BitRate) -> Self {
let is_40k = matches!(rate, BitRate::Rate40k);
let mut core =
Self { rate, is_40k, a: [0; 2], b: [0; 6], dms: 0, dml: 0, ap: 0, yu: 544, yl: 34816, sr: [32, 32], dq: [32; 6], pk: [0; 2], td: 0 };
core.reset();
core
}
fn reset(&mut self) {
self.a = [0; 2];
self.b = [0; 6];
self.dms = 0;
self.dml = 0;
self.ap = 0;
self.yu = 544;
self.yl = 34816;
self.sr = [32, 32];
self.dq = [32; 6];
self.pk = [0; 2];
self.td = 0;
}
#[inline]
fn fmult(an: u16, srn: u16) -> u16 {
let an_s = (an >> 15) & 1;
let an_mag = if an_s == 0 { (an >> 2) & 0x1FFF } else { (16384_u32.wrapping_sub((an >> 2) as u32) & 0x1FFF) as u16 };
let an_exp = if an_mag > 0 { (16 - an_mag.leading_zeros()).min(13) as u16 } else { 0 };
let an_mant = if an_mag == 0 { 1 << 5 } else { (((an_mag as u32) << 6) >> an_exp) as u16 & 0x3F };
let srn_s = (srn >> 10) & 1;
let srn_exp = (srn >> 6) & 15;
let srn_mant = srn & 63;
let wan_s = srn_s ^ an_s;
let wan_exp = srn_exp + an_exp;
let wan_mant = (((srn_mant as u32 * an_mant as u32) + 48) >> 4) as u16;
let wan_mag =
if wan_exp <= 26 { ((wan_mant as u32) << 7) >> (26 - wan_exp) } else { (((wan_mant as u32) << 7) << (wan_exp - 26)) & 0x7FFF } as u16;
if wan_s == 0 { wan_mag } else { (65536_u32.wrapping_sub(wan_mag as u32) & 0xFFFF) as u16 }
}
#[inline]
fn quantize(&self, dln: i32, ds: u8) -> u8 {
match self.rate {
BitRate::Rate16k => quantize_16(dln, ds),
BitRate::Rate24k => quantize_24(dln, ds),
BitRate::Rate32k => quantize_32(dln, ds),
BitRate::Rate40k => quantize_40(dln, ds),
}
}
#[inline]
fn table_lookup(&self, code: u8) -> (u8, u16) {
match self.rate {
BitRate::Rate16k => OUT_16[(code & 3) as usize],
BitRate::Rate24k => OUT_24[(code & 7) as usize],
BitRate::Rate32k => OUT_32[(code & 15) as usize],
BitRate::Rate40k => OUT_40[(code & 31) as usize],
}
}
#[inline]
fn compute_accum(&self) -> (i16, i16) {
let wa0 = Self::fmult(self.a[0], self.sr[0]);
let wa1 = Self::fmult(self.a[1], self.sr[1]);
let mut sezi: u32 = 0;
for i in 0..6 {
sezi = (sezi + Self::fmult(self.b[i], self.dq[i]) as u32) & 0xFFFF;
}
let sei = (sezi + wa1 as u32 + wa0 as u32) & 0xFFFF;
let sez = (sezi as i16) >> 1;
let se = (sei as i16) >> 1;
(sez, se)
}
#[inline]
fn compute_mix(&self) -> i32 {
let al = if self.ap >= 256 { 64 } else { self.ap >> 2 } as i32;
let dif = (self.yu as i32 + 16384 - (self.yl >> 6)) & 16383;
let difs = (dif >> 13) & 1;
let difm = if difs == 0 { dif } else { (16384 - dif) & 8191 };
let prodm = (difm * al) >> 6;
let prod = if difs == 0 { prodm } else { (16384 - prodm) & 8191 };
((self.yl >> 6) + prod) & 8191
}
#[inline]
fn quantize_sample(&self, sample: i16, se: i16, y: i32) -> u8 {
let sl = (sample >> 2) as i32;
let sl_ext = if sl >= 0 { sl } else { 65536 + sl };
let se_ext = if se >= 0 { se as i32 } else { 65536 + se as i32 };
let d = ((sl_ext + 65536 - se_ext) & 0xFFFF) as u16;
let ds = ((d >> 15) & 1) as u8;
let dqm = if ds == 0 { d } else { (65536_u32.wrapping_sub(d as u32) & 0x7FFF) as u16 };
let exp = if dqm > 1 { (15 - dqm.leading_zeros()).min(14) } else { 0 };
let mant = (((dqm as u32) << 7) >> exp) & 127;
let dl = ((exp << 7) + mant) as u16;
let dln = (dl as i32 + 4096 - (y >> 2)) & 4095;
self.quantize(dln, ds)
}
#[inline]
fn step(&mut self, sample: i16, is_encoder: bool, code_in: u8) -> (u8, i16) {
let (sez, se) = self.compute_accum();
let y = self.compute_mix();
let code = if is_encoder { self.quantize_sample(sample, se, y) } else { code_in };
// 4. RECONST & ADDA & ANTILOG
let (dqs, dqln) = self.table_lookup(code);
let dql = (dqln as i32 + (y >> 2)) & 4095;
let dex = (dql >> 7) & 15;
let dmn = dql & 127;
let dqt = 128 + dmn;
let dqmag = if (dql >> 11) == 0 { ((dqt << 7) >> (14 - dex)) as u16 } else { 0 };
let dq = if self.is_40k { ((dqs as u16) << 15) | (dqmag & 0x7FFF) } else { ((dqs as u16) << 14) | (dqmag & 0x3FFF) };
// 5. ADDB & ADDC
let dqi = if dqs == 0 {
dq as u32
} else if self.is_40k {
65536_u32.wrapping_sub((dq & 0x7FFF) as u32) & 0xFFFF
} else {
65536_u32.wrapping_sub((dq & 0x3FFF) as u32) & 0xFFFF
};
let sei_ext = if se >= 0 { se as u32 } else { (65536 + se as i32) as u32 };
let sr = ((dqi + sei_ext) & 0xFFFF) as u16;
let sezi_ext = if sez >= 0 { sez as u32 } else { (65536 + sez as i32) as u32 };
let dqsez = ((dqi + sezi_ext) & 0xFFFF) as u16;
let pk0 = ((dqsez >> 15) & 1) as u8;
let sigpk = if dqsez == 0 { 1 } else { 0 };
// 6. FLOATA & FLOATB
let dq_mag = if self.is_40k { dq & 0x7FFF } else { dq & 0x3FFF };
let (dq_exp, dq_mant) = if dq_mag == 0 {
(0, 32)
} else {
let max_exp = if self.is_40k { 15 } else { 14 };
let exp = (16 - dq_mag.leading_zeros()).min(max_exp);
let mant = (((dq_mag as u32) << 6) >> exp) as u16;
(exp as u16, mant)
};
let dq0 = ((dqs as u16) << 10) | (dq_exp << 6) | (dq_mant & 63);
let sr_s = ((sr >> 15) & 1) as u8;
let sr_mag = if sr_s == 0 { sr } else { (65536_u32.wrapping_sub(sr as u32) & 0x7FFF) as u16 };
let (sr_exp, sr_mant) = if sr_mag == 0 {
(0, 32)
} else {
let exp = (16 - sr_mag.leading_zeros()).min(15);
let mant = (((sr_mag as u32) << 6) >> exp) as u16;
(exp as u16, mant)
};
let sr0 = ((sr_s as u16) << 10) | (sr_exp << 6) | (sr_mant & 63);
self.update_adaptation_and_predictors(code, y, pk0, sigpk, dqs, dq_mag);
self.sr = [sr0, self.sr[0]];
self.dq = [dq0, self.dq[0], self.dq[1], self.dq[2], self.dq[3], self.dq[4]];
self.pk = [pk0, self.pk[0]];
let sr_signed = if sr < 32768 { sr as i32 } else { sr as i32 - 65536 };
let out_pcm = (sr_signed << 2).clamp(-32768, 32767) as i16;
(code, out_pcm)
}
#[inline]
fn adapt_scale(&mut self, code: u8, y: i32) {
let wi = get_wi(self.rate, code);
let dif = (((wi as i32) << 5) + 131072 - y) & 131071;
let difs = (dif >> 16) & 1;
let difsx = if difs == 0 { dif >> 5 } else { (dif >> 5) + 4096 };
let yut = (y + difsx) & 8191;
let yup = yut.clamp(544, 5120);
let dif = (yup + ((1048576 - self.yl) >> 6)) & 16383;
let difs = (dif >> 13) & 1;
let difsx = if difs == 0 { dif } else { dif + 507904 };
self.yu = yup as i16;
self.yl = (self.yl + difsx) & 524287;
}
#[inline]
fn adapt_speed(&mut self, code: u8, y: i32, dq_mag: u16) -> (i16, i16, u8) {
let fi = get_fi(self.rate, code);
let dif = (((fi as i32) << 9) + 8192 - self.dms as i32) & 8191;
let difs = (dif >> 12) & 1;
let difsx = if difs == 0 { dif >> 5 } else { (dif >> 5) + 3840 };
let dmsp = ((difsx + self.dms as i32) & 4095) as i16;
let dif = (((fi as i32) << 11) + 32768 - self.dml as i32) & 32767;
let difs = (dif >> 14) & 1;
let difsx = if difs == 0 { dif >> 7 } else { (dif >> 7) + 16128 };
let dmlp = ((difsx + self.dml as i32) & 16383) as i16;
let dif = (((dmsp as i32) << 2) + 32768 - dmlp as i32) & 32767;
let difs = (dif >> 14) & 1;
let difm = if difs == 0 { dif } else { (32768 - dif) & 16383 };
let dthr = (dmlp as i32) >> 3;
let tdp = if (32768..53760).contains(&self.a[1]) { 1 } else { 0 };
let ax = if y >= 1536 && tdp == 0 && difm < dthr { 0 } else { 1 };
let dif = ((ax << 9) + 2048 - self.ap as i32) & 2047;
let difs = (dif >> 10) & 1;
let difsx = if difs == 0 { dif >> 4 } else { (dif >> 4) + 1920 };
let app = ((difsx + self.ap as i32) & 1023) as i16;
let ylint = (self.yl >> 15) as u32;
let ylfrac = ((self.yl >> 10) & 31) as u32;
let thr1 = (32 + ylfrac) << ylint;
let thr2 = if self.is_40k && ylint > 9 {
31 << 10
} else if !self.is_40k && ylint > 8 {
31 << 9
} else {
thr1
};
let dqthr = (thr2 + (thr2 >> 1)) >> 1;
let tr = if (dq_mag as u32) > dqthr && self.td == 1 { 1 } else { 0 };
let apr = if tr == 1 { 256 } else { app };
self.dms = dmsp;
self.dml = dmlp;
self.ap = apr;
self.td = if tr == 1 { 0 } else { tdp };
(apr, tdp, tr)
}
#[inline]
fn update_predictors(&mut self, pk0: u8, sigpk: u8, dqs: u8, dq_mag: u16, tr: u8) {
let pks = pk0 ^ self.pk[0];
let uga1 = if sigpk == 1 {
0
} else if pks == 0 {
192
} else {
65344
};
let a1s = (self.a[0] >> 15) & 1;
let ula1 = if a1s == 0 {
(65536_u32.wrapping_sub((self.a[0] >> 8) as u32) & 0xFFFF) as u16
} else {
(65536_u32.wrapping_sub(((self.a[0] >> 8) + 65280) as u32) & 0xFFFF) as u16
};
let ua1 = (uga1 as u32 + ula1 as u32) & 0xFFFF;
let a1t = ((self.a[0] as u32 + ua1) & 0xFFFF) as u16;
let pks1 = pk0 ^ self.pk[0];
let pks2 = pk0 ^ self.pk[1];
let uga2a = if pks2 == 0 { 16384_u32 } else { 114688_u32 };
let a1_u = self.a[0];
let fa1 = if a1s == 0 {
if a1_u <= 8191 { (a1_u as u32) << 2 } else { 8191 << 2 }
} else if a1_u >= 57345 {
((a1_u as u32) << 2) & 131071
} else {
24577 << 2
};
let fa = if pks1 == 1 { fa1 } else { (131072_u32.wrapping_sub(fa1)) & 131071 };
let uga2b = (uga2a + fa) & 131071;
let uga2s = (uga2b >> 16) & 1;
let uga2 = if sigpk == 1 {
0
} else if uga2s == 0 {
uga2b >> 7
} else {
(uga2b >> 7) + 64512
} as u16;
let a2s = (self.a[1] >> 15) & 1;
let ula2 = if a2s == 0 {
(65536_u32.wrapping_sub((self.a[1] >> 7) as u32) & 0xFFFF) as u16
} else {
(65536_u32.wrapping_sub(((self.a[1] >> 7) + 65024) as u32) & 0xFFFF) as u16
};
let ua2 = (uga2 as u32 + ula2 as u32) & 0xFFFF;
let a2t = ((self.a[1] as u32 + ua2) & 0xFFFF) as u16;
let a2p = if (32768..=53248).contains(&a2t) {
53248
} else if (12288..=32767).contains(&a2t) {
12288
} else {
a2t
};
let ome = 15360_u32;
let a1ul = ((ome + 65536 - a2p as u32) & 0xFFFF) as u16;
let a1ll = ((a2p as u32 + 65536 - ome) & 0xFFFF) as u16;
let a1p = if a1t >= 32768 && a1t <= a1ll {
a1ll
} else if a1t >= a1ul && a1t <= 32767 {
a1ul
} else {
a1t
};
let a1r = if tr == 1 { 0 } else { a1p };
let a2r = if tr == 1 { 0 } else { a2p };
let mut new_b = [0u16; 6];
for (i, slot) in new_b.iter_mut().enumerate() {
let un = (dqs ^ ((self.dq[i] >> 10) as u8)) & 1;
let ugbn = if dq_mag == 0 {
0
} else if un == 0 {
128
} else {
65408
};
let bns = (self.b[i] >> 15) & 1;
let ulbn = if self.is_40k {
if bns == 0 {
(65536_u32.wrapping_sub((self.b[i] >> 9) as u32) & 0xFFFF) as u16
} else {
(65536_u32.wrapping_sub(((self.b[i] >> 9) + 65408) as u32) & 0xFFFF) as u16
}
} else if bns == 0 {
(65536_u32.wrapping_sub((self.b[i] >> 8) as u32) & 0xFFFF) as u16
} else {
(65536_u32.wrapping_sub(((self.b[i] >> 8) + 65280) as u32) & 0xFFFF) as u16
};
let ubn = (ugbn as u32 + ulbn as u32) & 0xFFFF;
let bnp = ((self.b[i] as u32 + ubn) & 0xFFFF) as u16;
*slot = if tr == 1 { 0 } else { bnp };
}
self.a[0] = a1r;
self.a[1] = a2r;
self.b = new_b;
}
#[inline]
fn update_adaptation_and_predictors(&mut self, code: u8, y: i32, pk0: u8, sigpk: u8, dqs: u8, dq_mag: u16) {
self.adapt_scale(code, y);
let (_, _, tr) = self.adapt_speed(code, y, dq_mag);
self.update_predictors(pk0, sigpk, dqs, dq_mag, tr);
}
}
// ---------------------------------------------------------------------------
// Decoder & Encoder structs
// ---------------------------------------------------------------------------
/// G.726 ADPCM decoder.
#[derive(Debug, Clone)]
pub struct Decoder {
core: Core,
packing: BitPacking,
bit_buf: u32,
num_bits: usize,
}
impl Decoder {
/// Create a new G.726 decoder at the specified bit rate and bit packing.
#[must_use]
pub fn new(rate: BitRate, packing: BitPacking) -> Self {
Self { core: Core::new(rate), packing, bit_buf: 0, num_bits: 0 }
}
/// Reset decoder internal predictor state.
pub fn reset(&mut self) {
self.core.reset();
self.bit_buf = 0;
self.num_bits = 0;
}
/// Decode a single quantized ADPCM code into an 8 kHz 16-bit linear PCM sample.
/// Bits above the configured code width are ignored, as in packed-stream decoding.
#[inline]
pub fn decode_sample(&mut self, code: u8) -> i16 {
// Adaptation must see the same bounded codeword as the reconstruction table;
// otherwise unused high bits can underflow its unsigned inverse-code arithmetic.
let code = code & ((1u8 << self.core.rate.code_size()) - 1);
self.core.step(0, false, code).1
}
/// Decode G.726 bitstream bytes into 8 kHz 16-bit linear PCM samples, appending to `dst`.
pub fn decode(&mut self, src: &[u8], dst: &mut Vec<i16>) -> Result<(), Error> {
if src.len() > MAX_INPUT_BYTES {
return Err(Error::InputTooLarge);
}
let cs = self.core.rate.code_size();
let est_samples = (src.len() * 8) / cs;
dst.try_reserve(est_samples).map_err(|_| Error::AllocationFailed)?;
match self.packing {
BitPacking::BigEndian => {
for &b in src {
self.bit_buf = ((self.bit_buf & ((1 << self.num_bits) - 1)) << 8) | (b as u32);
self.num_bits += 8;
while self.num_bits >= cs {
let code = ((self.bit_buf >> (self.num_bits - cs)) & ((1 << cs) - 1)) as u8;
self.num_bits -= cs;
let s = self.decode_sample(code);
dst.push(s);
}
self.bit_buf &= (1 << self.num_bits) - 1;
}
}
BitPacking::LittleEndian => {
for &b in src {
self.bit_buf |= (b as u32) << self.num_bits;
self.num_bits += 8;
while self.num_bits >= cs {
let code = (self.bit_buf & ((1 << cs) - 1)) as u8;
self.bit_buf >>= cs;
self.num_bits -= cs;
let s = self.decode_sample(code);
dst.push(s);
}
}
}
}
Ok(())
}
}
/// G.726 ADPCM encoder.
#[derive(Debug, Clone)]
pub struct Encoder {
core: Core,
packing: BitPacking,
bit_buf: u32,
num_bits: usize,
}
impl Encoder {
/// Create a new G.726 encoder at the specified bit rate and bit packing.
#[must_use]
pub fn new(rate: BitRate, packing: BitPacking) -> Self {
Self { core: Core::new(rate), packing, bit_buf: 0, num_bits: 0 }
}
/// Reset encoder internal predictor state.
pub fn reset(&mut self) {
self.core.reset();
self.bit_buf = 0;
self.num_bits = 0;
}
/// Encode a single 8 kHz 16-bit linear PCM sample into an ADPCM code word.
#[inline]
pub fn encode_sample(&mut self, sample: i16) -> u8 {
self.core.step(sample, true, 0).0
}
/// Encode 8 kHz 16-bit linear PCM samples into G.726 bitstream bytes, appending to `dst`.
pub fn encode(&mut self, src: &[i16], dst: &mut Vec<u8>) -> Result<(), Error> {
if src.len() > MAX_SAMPLES {
return Err(Error::InputTooLarge);
}
let cs = self.core.rate.code_size();
let est_bytes = (src.len() * cs).div_ceil(8);
dst.try_reserve(est_bytes).map_err(|_| Error::AllocationFailed)?;
match self.packing {
BitPacking::BigEndian => {
for &s in src {
let code = self.encode_sample(s);
self.bit_buf = ((self.bit_buf & ((1 << self.num_bits) - 1)) << cs) | ((code as u32) & ((1 << cs) - 1));
self.num_bits += cs;
while self.num_bits >= 8 {
dst.push(((self.bit_buf >> (self.num_bits - 8)) & 0xFF) as u8);
self.num_bits -= 8;
}
self.bit_buf &= (1 << self.num_bits) - 1;
}
}
BitPacking::LittleEndian => {
for &s in src {
let code = self.encode_sample(s);
self.bit_buf |= (code as u32) << self.num_bits;
self.num_bits += cs;
while self.num_bits >= 8 {
dst.push((self.bit_buf & 0xFF) as u8);
self.bit_buf >>= 8;
self.num_bits -= 8;
}
}
}
}
Ok(())
}
/// Flush any remaining buffered bits into `dst`, padding with zeroes.
pub fn flush(&mut self, dst: &mut Vec<u8>) -> Result<(), Error> {
if self.num_bits > 0 {
dst.try_reserve(1).map_err(|_| Error::AllocationFailed)?;
match self.packing {
BitPacking::BigEndian => {
let b = ((self.bit_buf << (8 - self.num_bits)) & 0xFF) as u8;
dst.push(b);
}
BitPacking::LittleEndian => {
let b = (self.bit_buf & 0xFF) as u8;
dst.push(b);
}
}
self.bit_buf = 0;
self.num_bits = 0;
}
Ok(())
}
}
// ---------------------------------------------------------------------------
// Public Top-Level Functions
// ---------------------------------------------------------------------------
/// Decode a mono G.726 bitstream into 8 kHz 16-bit linear PCM samples.
pub fn decode(bytes: &[u8], rate: BitRate, packing: BitPacking) -> Result<Vec<i16>, Error> {
let mut decoder = Decoder::new(rate, packing);
let mut out = Vec::new();
decoder.decode(bytes, &mut out)?;
Ok(out)
}
/// Encode mono 8 kHz 16-bit linear PCM samples into a G.726 bitstream.
pub fn encode(samples: &[i16], rate: BitRate, packing: BitPacking) -> Result<Vec<u8>, Error> {
let mut encoder = Encoder::new(rate, packing);
let mut out = Vec::new();
encoder.encode(samples, &mut out)?;
encoder.flush(&mut out)?;
Ok(out)
}
/// Decode an interleaved multi-channel G.726 stream into interleaved 8 kHz 16-bit PCM samples.
pub fn decode_interleaved(bytes: &[u8], channels: usize, rate: BitRate, packing: BitPacking) -> Result<Vec<i16>, Error> {
if channels == 0 || channels > MAX_CHANNELS {
return Err(Error::InvalidChannels(channels));
}
if channels == 1 {
return decode(bytes, rate, packing);
}
if bytes.len() > MAX_INPUT_BYTES {
return Err(Error::InputTooLarge);
}
let mut decoders: Vec<Decoder> = Vec::with_capacity(channels);
for _ in 0..channels {
decoders.push(Decoder::new(rate, packing));
}
let cs = rate.code_size();
let est_samples = (bytes.len() * 8) / cs;
let mut out = Vec::new();
out.try_reserve(est_samples).map_err(|_| Error::AllocationFailed)?;
let mut ch_outs: Vec<Vec<i16>> = Vec::with_capacity(channels);
for _ in 0..channels {
ch_outs.push(Vec::new());
}
for (i, &b) in bytes.iter().enumerate() {
let ch = i % channels;
decoders[ch].decode(&[b], &mut ch_outs[ch])?;
}
let min_len = ch_outs.iter().map(Vec::len).min().unwrap_or(0);
for i in 0..min_len {
for ch_out in ch_outs.iter().take(channels) {
out.push(ch_out[i]);
}
}
Ok(out)
}
/// Encode interleaved 8 kHz 16-bit PCM samples into an interleaved multi-channel G.726 stream.
pub fn encode_interleaved(samples: &[i16], channels: usize, rate: BitRate, packing: BitPacking) -> Result<Vec<u8>, Error> {
if channels == 0 || channels > MAX_CHANNELS {
return Err(Error::InvalidChannels(channels));
}
if channels == 1 {
return encode(samples, rate, packing);
}
if samples.len() > MAX_SAMPLES {
return Err(Error::InputTooLarge);
}
let mut encoders: Vec<Encoder> = Vec::with_capacity(channels);
for _ in 0..channels {
encoders.push(Encoder::new(rate, packing));
}
let cs = rate.code_size();
let est_bytes = (samples.len() * cs).div_ceil(8);
let mut out = Vec::new();
out.try_reserve(est_bytes).map_err(|_| Error::AllocationFailed)?;
let mut ch_samples: Vec<Vec<i16>> = Vec::with_capacity(channels);
for _ in 0..channels {
ch_samples.push(Vec::new());
}
for (i, &s) in samples.iter().enumerate() {
let ch = i % channels;
ch_samples[ch].push(s);
}
let mut ch_bytes: Vec<Vec<u8>> = Vec::with_capacity(channels);
for ch in 0..channels {
let mut b = Vec::new();
encoders[ch].encode(&ch_samples[ch], &mut b)?;
encoders[ch].flush(&mut b)?;
ch_bytes.push(b);
}
let min_bytes = ch_bytes.iter().map(Vec::len).min().unwrap_or(0);
for i in 0..min_bytes {
for ch_b in ch_bytes.iter().take(channels) {
out.push(ch_b[i]);
}
}
Ok(out)
}
// ---------------------------------------------------------------------------
// Unit tests
// ---------------------------------------------------------------------------
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn sample_api_ignores_unused_codeword_bits() {
for rate in [BitRate::Rate16k, BitRate::Rate24k, BitRate::Rate32k, BitRate::Rate40k] {
let mask = (1u8 << rate.code_size()) - 1;
let mut actual = Decoder::new(rate, BitPacking::BigEndian);
let mut expected = Decoder::new(rate, BitPacking::BigEndian);
for code in 0..=u8::MAX {
assert_eq!(actual.decode_sample(code), expected.decode_sample(code & mask));
}
// High bits must not leave a different predictor state for later valid codes.
for code in 0..=mask {
assert_eq!(actual.decode_sample(code), expected.decode_sample(code));
}
}
}
#[test]
fn reject_invalid_channels() {
assert!(decode_interleaved(&[], 0, BitRate::Rate32k, BitPacking::BigEndian).is_err());
assert!(decode_interleaved(&[], 9, BitRate::Rate32k, BitPacking::BigEndian).is_err());
assert!(encode_interleaved(&[], 0, BitRate::Rate32k, BitPacking::BigEndian).is_err());
assert!(encode_interleaved(&[], 9, BitRate::Rate32k, BitPacking::BigEndian).is_err());
}
#[test]
fn roundtrip_silence_all_rates() {
let pcm = vec![0i16; 64];
for rate in [BitRate::Rate16k, BitRate::Rate24k, BitRate::Rate32k, BitRate::Rate40k] {
for packing in [BitPacking::BigEndian, BitPacking::LittleEndian] {
let encoded = encode(&pcm, rate, packing).expect("encode silence");
let decoded = decode(&encoded, rate, packing).expect("decode silence");
assert_eq!(decoded.len(), pcm.len());
for &s in &decoded {
assert!(s.abs() <= 16, "silence sample too large: {s}");
}
}
}
}
#[test]
fn hostile_input_does_not_panic() {
let mut decoder = Decoder::new(BitRate::Rate32k, BitPacking::BigEndian);
let mut out = Vec::new();
let hostile = vec![0xFF; 1024];
let _ = decoder.decode(&hostile, &mut out);
let mut decoder = Decoder::new(BitRate::Rate16k, BitPacking::LittleEndian);
let mut out = Vec::new();
let _ = decoder.decode(&[0x00, 0xAA, 0x55, 0x12, 0x34], &mut out);
}
}
+4 -1
View File
@@ -13,6 +13,7 @@
//! demuxers and [`TrackDecoder`]; everything else (FLAC, Vorbis, CAF, ADPCM, A-law, µ-law) through
//! `symphonia`, which is to be replaced by native decoders. Feature `decode-native` is the
//! same without symphonia.
//! - [`vorbis::decoder`] (feature `vorbis-decode`): the pure-Rust Vorbis I decoder, packets and Ogg files.
//! - [`encode_ogg_vorbis`] and [`vorbis::encoder`] (feature `vorbis-encode`): Ogg Vorbis files
//! and Vorbis packets from a pure-Rust encoder, 1 to 8 channels, quality -1 to 10, with tags.
//! - [`TrackDecoder`] (any decoder feature): one track's packets from any demuxer to planar
@@ -48,6 +49,8 @@ pub mod alac;
mod buffer;
#[cfg(feature = "g722")]
pub mod g722;
#[cfg(feature = "g726")]
pub mod g726;
#[cfg(any(feature = "gsm-decode", feature = "gsm-encode"))]
pub mod gsm;
mod layout;
@@ -71,7 +74,7 @@ pub mod sbc;
mod track;
#[cfg(feature = "vmd-decode")]
pub mod vmd;
#[cfg(feature = "vorbis-encode")]
#[cfg(any(feature = "vorbis-decode", feature = "vorbis-encode"))]
pub mod vorbis;
#[cfg(any(feature = "decode-native", feature = "pcm-encode"))]
mod wav;
+96
View File
@@ -0,0 +1,96 @@
//! The Vorbis bit reader (Vorbis I §2): bits are taken least significant first from each byte.
/// Reading past the end of the packet ("end-of-packet", §2.1.8).
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(super) struct Eop;
pub(super) struct BitReader<'a> {
data: &'a [u8],
/// The position of the next bit.
pos: usize,
}
impl<'a> BitReader<'a> {
pub(super) fn new(data: &'a [u8]) -> Self {
Self { data, pos: 0 }
}
/// The bits not yet read.
pub(super) fn remaining(&self) -> usize {
(self.data.len() * 8).saturating_sub(self.pos)
}
/// Read `n` (at most 32) bits as an unsigned integer. A read that does not fit takes nothing.
pub(super) fn read(&mut self, n: u32) -> Result<u32, Eop> {
if n == 0 {
return Ok(0);
}
if (n as usize) > self.remaining() {
self.pos = self.data.len() * 8;
return Err(Eop);
}
let mut value = 0u64;
let mut got = 0u32;
while got < n {
let byte = u64::from(self.data[self.pos / 8]);
let shift = (self.pos % 8) as u32;
let take = (8 - shift).min(n - got);
value |= ((byte >> shift) & ((1 << take) - 1)) << got;
got += take;
self.pos += take as usize;
}
Ok(value as u32)
}
pub(super) fn bit(&mut self) -> Result<bool, Eop> {
Ok(self.read(1)? == 1)
}
}
/// `ilog(x)` (§9.2.1): the position of the highest set bit, 0 for values below 1.
pub(super) fn ilog(x: i64) -> u32 {
if x <= 0 { 0 } else { 64 - (x as u64).leading_zeros() }
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn reads_least_significant_bit_first() {
// Fields come out of each byte from its least significant bit.
let mut r = BitReader::new(&[0xFC, 0x08]);
assert_eq!(r.read(4), Ok(0xC));
assert_eq!(r.read(4), Ok(0xF));
assert_eq!(r.read(1), Ok(0));
assert_eq!(r.read(3), Ok(4));
assert_eq!(r.read(4), Ok(0));
assert_eq!(r.read(1), Err(Eop));
}
#[test]
fn wide_reads_span_bytes_and_zero_bits_are_free() {
let mut r = BitReader::new(&[0x01, 0x02, 0x03, 0x04, 0x05]);
assert_eq!(r.read(0), Ok(0));
assert_eq!(r.read(32), Ok(0x0403_0201));
assert_eq!(r.read(8), Ok(5));
assert_eq!(r.read(0), Ok(0));
assert_eq!(r.read(1), Err(Eop));
assert_eq!(r.remaining(), 0);
}
#[test]
fn a_failed_read_consumes_the_rest() {
let mut r = BitReader::new(&[0xFF]);
assert_eq!(r.read(3), Ok(7));
assert_eq!(r.read(6), Err(Eop));
assert_eq!(r.read(1), Err(Eop));
}
#[test]
fn ilog_matches_the_specification_examples() {
for (x, want) in [(0, 0), (1, 1), (2, 2), (3, 2), (4, 3), (7, 3), (-5, 0), (255, 8), (256, 9)] {
assert_eq!(ilog(x), want, "ilog({x})");
}
}
}
+409
View File
@@ -0,0 +1,409 @@
//! Codebooks (Vorbis I §3): a Huffman code over entry numbers and an optional table of vectors.
use super::Error;
use super::bits::{BitReader, Eop, ilog};
const NONE: u32 = u32::MAX;
const LEAF: u32 = 1 << 31;
/// Entries a codebook may declare.
const MAX_ENTRIES: u32 = 1 << 20;
/// Nodes of one codeword tree.
const MAX_NODES: usize = 1 << 22;
#[derive(Clone, Copy)]
struct Node {
child: [u32; 2],
full: bool,
}
pub(super) struct Codebook {
pub(super) dimensions: usize,
pub(super) entries: usize,
tree: Vec<Node>,
/// The only used entry of a single-entry book.
single: Option<u32>,
/// `entries * dimensions` values when the book has a lookup table.
vectors: Vec<f32>,
}
/// `float32_unpack` (§9.2.2).
fn float32_unpack(x: u32) -> f64 {
let mantissa = f64::from(x & 0x001F_FFFF);
let exponent = ((x & 0x7FE0_0000) >> 21) as i32;
let m = if x & 0x8000_0000 != 0 { -mantissa } else { mantissa };
m * 2f64.powi(exponent - 788)
}
/// `lookup1_values` (§9.2.3): the greatest `v` with `v^dimensions <= entries`.
fn lookup1_values(entries: u32, dimensions: u32) -> u32 {
let mut v = 1u32;
loop {
let next = u128::from(v) + 1;
let mut p = 1u128;
for _ in 0..dimensions {
p *= next;
if p > u128::from(entries) {
return v;
}
}
v += 1;
}
}
fn insert(tree: &mut Vec<Node>, at: usize, depth: u32, len: u32, entry: u32) -> bool {
for b in 0..2 {
let child = tree[at].child[b];
if depth + 1 == len {
if child == NONE {
tree[at].child[b] = LEAF | entry;
refresh(tree, at);
return true;
}
continue;
}
if child == NONE {
if tree.len() >= MAX_NODES {
return false;
}
tree.push(Node { child: [NONE; 2], full: false });
let new = tree.len() - 1;
tree[at].child[b] = new as u32;
if insert(tree, new, depth + 1, len, entry) {
refresh(tree, at);
return true;
}
return false;
}
if child & LEAF != 0 || tree[child as usize].full {
continue;
}
if insert(tree, child as usize, depth + 1, len, entry) {
refresh(tree, at);
return true;
}
}
false
}
fn refresh(tree: &mut [Node], at: usize) {
let done = |c: u32, t: &[Node]| c != NONE && (c & LEAF != 0 || t[c as usize].full);
let full = done(tree[at].child[0], tree) && done(tree[at].child[1], tree);
tree[at].full = full;
}
impl Codebook {
/// Read one codebook from a setup header. `budget` is the bytes the setup's codebooks may
/// still take in all.
pub(super) fn read(r: &mut BitReader<'_>, budget: &mut usize) -> Result<Self, Error> {
let bad = |m: &'static str| Error::Setup(m);
let eop = |_: Eop| Error::Setup("setup header ends inside a codebook");
if r.read(24).map_err(eop)? != 0x0056_4342 {
return Err(bad("codebook sync pattern missing"));
}
let dimensions = r.read(16).map_err(eop)?;
let entries = r.read(24).map_err(eop)?;
if dimensions == 0 || entries == 0 || entries > MAX_ENTRIES {
return Err(bad("codebook with no dimensions, no entries or too many"));
}
let mut lengths: Vec<u8> = Vec::with_capacity(entries as usize);
if r.bit().map_err(eop)? {
// Ordered: the number of codewords of each length.
let mut current = 0u32;
let mut len = r.read(5).map_err(eop)? + 1;
while current < entries {
let number = r.read(ilog(i64::from(entries - current))).map_err(eop)?;
if number > entries - current || len > 32 {
return Err(bad("ordered codebook lengths overrun the entries"));
}
lengths.extend(std::iter::repeat_n(len as u8, number as usize));
current += number;
len += 1;
}
} else {
let sparse = r.bit().map_err(eop)?;
for _ in 0..entries {
if sparse && !r.bit().map_err(eop)? {
lengths.push(0);
} else {
lengths.push((r.read(5).map_err(eop)? + 1) as u8);
}
}
}
// The codeword tree.
let used = lengths.iter().filter(|&&l| l != 0).count();
let mut tree = vec![Node { child: [NONE; 2], full: false }];
let mut single = None;
if used == 1 {
let entry = lengths.iter().position(|&l| l != 0).unwrap_or(0);
if lengths[entry] != 1 {
return Err(bad("a codebook with one used entry must give it length 1"));
}
single = Some(entry as u32);
} else if used > 1 {
for (entry, &len) in lengths.iter().enumerate() {
if len != 0 && !insert(&mut tree, 0, 0, u32::from(len), entry as u32) {
return Err(bad("overspecified codebook"));
}
}
if !tree[0].full {
return Err(bad("underspecified codebook"));
}
}
let tree_bytes = tree.len() * std::mem::size_of::<Node>();
if tree_bytes > *budget {
return Err(Error::Limit("the codebooks are too large"));
}
*budget -= tree_bytes;
// The lookup table.
let lookup_type = r.read(4).map_err(eop)?;
let mut vectors = Vec::new();
if lookup_type > 2 {
return Err(bad("reserved codebook lookup type"));
}
if lookup_type != 0 {
let minimum = float32_unpack(r.read(32).map_err(eop)?);
let delta = float32_unpack(r.read(32).map_err(eop)?);
let value_bits = r.read(4).map_err(eop)? + 1;
let sequence = r.bit().map_err(eop)?;
// Check before multiplying or reading multiplicands: the product can overflow
// usize on 32-bit targets, and a type-2 table otherwise allocates before admission.
let total = (entries as usize)
.checked_mul(dimensions as usize)
.filter(|&n| n <= *budget / std::mem::size_of::<f32>())
.ok_or(Error::Limit("the codebooks' vector tables are too large"))?;
let values = if lookup_type == 1 { lookup1_values(entries, dimensions) as usize } else { total };
if !minimum.is_finite() || !delta.is_finite() || values > r.remaining() / value_bits as usize {
return Err(bad("codebook lookup table is damaged or truncated"));
}
let multiplicands: Vec<u32> = (0..values).map(|_| r.read(value_bits).unwrap_or(0)).collect();
*budget -= total * std::mem::size_of::<f32>();
vectors = vec![0f32; total];
for e in 0..entries as usize {
let out = &mut vectors[e * dimensions as usize..(e + 1) * dimensions as usize];
let mut last = 0f64;
let mut divisor = 1usize;
for (i, slot) in out.iter_mut().enumerate() {
let m = if lookup_type == 1 {
let at = (e / divisor) % values;
divisor = divisor.saturating_mul(values);
multiplicands[at]
} else {
multiplicands[e * dimensions as usize + i]
};
let v = f64::from(m) * delta + minimum + last;
if sequence {
last = v;
}
*slot = v as f32;
}
}
}
Ok(Self { dimensions: dimensions as usize, entries: entries as usize, tree, single, vectors })
}
/// Whether the book has a lookup table.
pub(super) fn has_vectors(&self) -> bool {
!self.vectors.is_empty()
}
/// Read a codeword; the entry number (scalar context).
pub(super) fn scalar(&self, r: &mut BitReader<'_>) -> Result<u32, Eop> {
if let Some(e) = self.single {
r.read(1)?;
return Ok(e);
}
let mut at = 0usize;
loop {
let c = self.tree[at].child[usize::from(r.bit()?)];
if c == NONE {
return Err(Eop);
}
if c & LEAF != 0 {
return Ok(c & !LEAF);
}
at = c as usize;
}
}
/// Read a codeword and return its vector (VQ context); `None` for a book without a table.
pub(super) fn vector(&self, r: &mut BitReader<'_>) -> Result<Option<&[f32]>, Eop> {
let e = self.scalar(r)? as usize;
Ok(self.vectors.get(e * self.dimensions..(e + 1) * self.dimensions))
}
}
#[cfg(test)]
mod tests {
use super::*;
/// Pack fields least significant bit first.
struct W(Vec<u8>, usize);
impl W {
fn put(&mut self, v: u64, n: u32) {
for i in 0..n {
if self.1.is_multiple_of(8) {
self.0.push(0);
}
let last = self.0.len() - 1;
self.0[last] |= (((v >> i) & 1) as u8) << (self.1 % 8);
self.1 += 1;
}
}
}
fn book_bytes(lengths: &[u32], lookup: Option<(u32, u32, &[u32])>) -> Vec<u8> {
let mut w = W(Vec::new(), 0);
w.put(0x56_4342, 24);
w.put(1, 16);
w.put(lengths.len() as u64, 24);
w.put(0, 1); // not ordered
w.put(0, 1); // not sparse
for &l in lengths {
w.put(u64::from(l - 1), 5);
}
match lookup {
None => w.put(0, 4),
Some((ty, bits, mult)) => {
w.put(u64::from(ty), 4);
w.put(0, 32); // minimum: 0.0
w.put(0, 32);
w.put(u64::from(bits - 1), 4);
w.put(0, 1);
for &m in mult {
w.put(u64::from(m), bits);
}
}
}
w.0
}
#[test]
fn the_specification_example_assigns_its_codewords() {
// §3.2.1: lengths 2 4 4 4 4 2 3 3 -> 00, 0100, 0101, 0110, 0111, 10, 110, 111.
let bytes = book_bytes(&[2, 4, 4, 4, 4, 2, 3, 3], None);
let mut budget = 1 << 26;
let book = Codebook::read(&mut BitReader::new(&bytes), &mut budget).expect("book");
let code = |bits: &[u8]| {
let mut w = W(Vec::new(), 0);
for &b in bits {
w.put(u64::from(b), 1);
}
let mut r = BitReader::new(&w.0);
book.scalar(&mut r).expect("entry")
};
assert_eq!(code(&[0, 0]), 0);
assert_eq!(code(&[0, 1, 0, 0]), 1);
assert_eq!(code(&[0, 1, 1, 1]), 4);
assert_eq!(code(&[1, 0]), 5);
assert_eq!(code(&[1, 1, 0]), 6);
assert_eq!(code(&[1, 1, 1]), 7);
}
#[test]
fn over_and_underspecified_books_are_errors() {
let mut budget = 1 << 26;
let over = book_bytes(&[1, 1, 1], None);
assert!(Codebook::read(&mut BitReader::new(&over), &mut budget).is_err());
let under = book_bytes(&[2, 2, 2], None);
assert!(Codebook::read(&mut BitReader::new(&under), &mut budget).is_err());
}
#[test]
fn a_single_entry_book_sinks_one_bit_of_either_value() {
let mut budget = 1 << 26;
let bytes = book_bytes(&[1], None);
let book = Codebook::read(&mut BitReader::new(&bytes), &mut budget).expect("book");
for byte in [0u8, 1] {
let data = [byte];
let mut r = BitReader::new(&data);
assert_eq!(book.scalar(&mut r), Ok(0));
assert_eq!(r.remaining(), 7);
}
let bad = book_bytes(&[2], None);
assert!(Codebook::read(&mut BitReader::new(&bad), &mut budget).is_err());
}
#[test]
fn lookup_type_one_builds_a_lattice() {
// 4 entries, 2 dimensions, 2 values per dimension: minimum 0 and delta 1 in the packed
// float format are written below by hand (mantissa 1, exponent 788 -> 1.0).
let mut w = W(Vec::new(), 0);
w.put(0x56_4342, 24);
w.put(2, 16);
w.put(4, 24);
w.put(0, 1);
w.put(0, 1);
for _ in 0..4 {
w.put(1, 5); // length 2 each
}
w.put(1, 4); // lookup type 1
w.put(0, 32); // minimum 0
w.put((788u64 << 21) | 1, 32); // delta 1.0
w.put(0, 4); // 1 bit per value
w.put(0, 1);
w.put(0b10, 2); // multiplicands 0, 1
let mut budget = 1 << 26;
let book = Codebook::read(&mut BitReader::new(&w.0), &mut budget).expect("book");
assert_eq!(lookup1_values(4, 2), 2);
// Entry e is (m[e % 2], m[(e / 2) % 2]).
assert_eq!(book.vectors, vec![0.0, 0.0, 1.0, 0.0, 0.0, 1.0, 1.0, 1.0]);
}
#[test]
fn vector_budget_is_checked_before_reading_lookup_values() {
// A valid single-entry tree, but two values exceed the remaining budget.
// No multiplicands follow: admission must reject it before reading them.
for lookup in [1, 2] {
let mut w = W(Vec::new(), 0);
w.put(0x56_4342, 24);
w.put(2, 16);
w.put(1, 24);
w.put(0, 1);
w.put(0, 1);
w.put(0, 5);
w.put(lookup, 4);
w.put(0, 32);
w.put(0, 32);
w.put(15, 4);
w.put(0, 1);
let mut budget = std::mem::size_of::<Node>() + 4;
assert!(matches!(Codebook::read(&mut BitReader::new(&w.0), &mut budget), Err(Error::Limit(_))));
}
}
#[test]
fn vector_size_that_overflows_32_bits_is_refused() {
for lookup in [1, 2] {
let mut w = W(Vec::new(), 0);
w.put(0x56_4342, 24);
w.put(65535, 16);
w.put(65538, 24); // product is above u32::MAX
w.put(0, 1); // unordered
w.put(1, 1); // sparse: only one entry is used
w.put(1, 1);
w.put(0, 5); // length 1
for _ in 1..65538 {
w.put(0, 1);
}
w.put(lookup, 4);
w.put(0, 32);
w.put(0, 32);
w.put(0, 4);
w.put(0, 1);
w.put(0, 1); // lookup type 1's single multiplicand
let mut budget = 256 << 20;
assert!(matches!(Codebook::read(&mut BitReader::new(&w.0), &mut budget), Err(Error::Limit(_))));
}
}
#[test]
fn helper_functions() {
assert_eq!(float32_unpack((788u32 << 21) | 5), 5.0);
assert_eq!(float32_unpack(0x8000_0000 | (789u32 << 21) | 3), -6.0);
for (entries, dims, want) in [(8, 3, 2), (9, 2, 3), (1, 5, 1), (10, 1, 10), (1000, 3, 10), (4_000_000, 2, 2000)] {
assert_eq!(lookup1_values(entries, dims), want, "{entries} entries, {dims} dims");
}
}
}
+385
View File
@@ -0,0 +1,385 @@
//! Floors (Vorbis I §6 and §7): the spectral envelope each channel's residue is multiplied by.
use super::Error;
use super::bits::{BitReader, Eop, ilog};
use super::codebook::Codebook;
use super::tables::INVERSE_DB;
/// Why a floor could not be decoded from a packet.
pub(super) enum Fail {
/// The packet ended (a nominal event: the channel is silent).
Eop,
/// The packet names a codebook that does not exist: it cannot be decoded (§6.2.2).
Invalid,
}
impl From<Eop> for Fail {
fn from(_: Eop) -> Self {
Fail::Eop
}
}
/// What a decoded floor gives to synthesis.
pub(super) enum FloorData {
One { y: Vec<i32>, step2: Vec<bool> },
Zero { amplitude: u32, coefficients: Vec<f32> },
}
pub(super) struct Floor0 {
order: usize,
rate: u32,
bark_map_size: u32,
amplitude_bits: u32,
amplitude_offset: u32,
books: Vec<usize>,
}
pub(super) struct Floor1 {
partition_class: Vec<usize>,
class_dimensions: Vec<usize>,
class_subclasses: Vec<u32>,
class_masterbook: Vec<usize>,
subclass_books: Vec<Vec<i32>>,
multiplier: i32,
range: i32,
xlist: Vec<i32>,
/// `(low, high)` neighbour positions of each X from index 2 on.
neighbours: Vec<(usize, usize)>,
/// Indices of `xlist` in ascending X order.
sorted: Vec<usize>,
}
pub(super) enum Floor {
Zero(Floor0),
One(Floor1),
}
fn eop(_: Eop) -> Error {
Error::Setup("setup header ends inside a floor")
}
impl Floor {
pub(super) fn read(floor_type: u32, r: &mut BitReader<'_>, books: &[Codebook]) -> Result<Self, Error> {
match floor_type {
0 => {
let order = r.read(8).map_err(eop)? as usize;
let rate = r.read(16).map_err(eop)?;
let bark_map_size = r.read(16).map_err(eop)?;
let amplitude_bits = r.read(6).map_err(eop)?;
let amplitude_offset = r.read(8).map_err(eop)?;
let count = r.read(4).map_err(eop)? as usize + 1;
let mut list = Vec::with_capacity(count);
for _ in 0..count {
let b = r.read(8).map_err(eop)? as usize;
match books.get(b) {
Some(book) if book.has_vectors() => list.push(b),
_ => return Err(Error::Setup("floor 0 names a missing codebook or one without vectors")),
}
}
if order == 0 || rate == 0 || bark_map_size == 0 || amplitude_bits > 32 {
return Err(Error::Setup("floor 0 with a zero order, rate or map size, or more than 32 amplitude bits"));
}
Ok(Self::Zero(Floor0 { order, rate, bark_map_size, amplitude_bits, amplitude_offset, books: list }))
}
1 => {
let partitions = r.read(5).map_err(eop)? as usize;
let mut partition_class = Vec::with_capacity(partitions);
for _ in 0..partitions {
partition_class.push(r.read(4).map_err(eop)? as usize);
}
let classes = partition_class.iter().max().map_or(0, |m| m + 1);
let (mut class_dimensions, mut class_subclasses, mut class_masterbook, mut subclass_books) = (vec![], vec![], vec![], vec![]);
for _ in 0..classes {
class_dimensions.push(r.read(3).map_err(eop)? as usize + 1);
let subs = r.read(2).map_err(eop)?;
class_subclasses.push(subs);
let master = if subs != 0 { r.read(8).map_err(eop)? as usize } else { 0 };
if subs != 0 && master >= books.len() {
return Err(Error::Setup("floor 1 master book out of range"));
}
class_masterbook.push(master);
let mut list = Vec::new();
for _ in 0..(1u32 << subs) {
let b = r.read(8).map_err(eop)? as i32 - 1;
if b >= books.len() as i32 {
return Err(Error::Setup("floor 1 subclass book out of range"));
}
list.push(b);
}
subclass_books.push(list);
}
let multiplier = r.read(2).map_err(eop)? as i32 + 1;
let rangebits = r.read(4).map_err(eop)?;
let mut xlist = vec![0, 1i32 << rangebits];
for &c in &partition_class {
for _ in 0..class_dimensions[c] {
xlist.push(r.read(rangebits).map_err(eop)? as i32);
}
}
if xlist.len() > 65 {
return Err(Error::Setup("floor 1 has more than 65 points"));
}
let mut seen = xlist.clone();
seen.sort_unstable();
if seen.windows(2).any(|w| w[0] == w[1]) {
return Err(Error::Setup("floor 1 repeats an X position"));
}
// Low and high neighbours (§9.2.4, §9.2.5) of each point, among those before it.
let mut neighbours = Vec::new();
for i in 2..xlist.len() {
let (mut lo, mut hi) = (0usize, 1usize);
for j in 0..i {
if xlist[j] < xlist[i] && xlist[j] > xlist[lo] {
lo = j;
}
if xlist[j] > xlist[i] && xlist[j] < xlist[hi] {
hi = j;
}
}
neighbours.push((lo, hi));
}
let mut sorted: Vec<usize> = (0..xlist.len()).collect();
sorted.sort_by_key(|&i| xlist[i]);
let range = [256, 128, 86, 64][multiplier as usize - 1];
Ok(Self::One(Floor1 {
partition_class,
class_dimensions,
class_subclasses,
class_masterbook,
subclass_books,
multiplier,
range,
xlist,
neighbours,
sorted,
}))
}
_ => Err(Error::Setup("unknown floor type")),
}
}
/// Decode one channel's floor from a packet: `None` when the channel is unused this block.
pub(super) fn decode(&self, r: &mut BitReader<'_>, books: &[Codebook]) -> Result<Option<FloorData>, Fail> {
match self {
Self::One(f) => Ok(f.decode(r, books)?),
Self::Zero(f) => f.decode(r, books),
}
}
/// The floor curve over `n` bins (`n` is half the block size).
pub(super) fn curve(&self, data: &FloorData, n: usize) -> Vec<f32> {
match (self, data) {
(Self::One(f), FloorData::One { y, step2 }) => f.curve(y, step2, n),
(Self::Zero(f), FloorData::Zero { amplitude, coefficients }) => f.curve(*amplitude, coefficients, n),
_ => vec![0.0; n],
}
}
}
/// `render_point` (§9.2.6).
fn render_point(x0: i32, y0: i32, x1: i32, y1: i32, x: i32) -> i32 {
let dy = i64::from(y1 - y0);
let adx = i64::from(x1 - x0);
let err = dy.abs() * i64::from(x - x0);
let off = (err / adx) as i32;
if dy < 0 { y0 - off } else { y0 + off }
}
/// `render_line` (§9.2.7), writing the bins that fall inside `v`.
fn render_line(x0: i32, y0: i32, x1: i32, y1: i32, v: &mut [i32]) {
let dy = y1 - y0;
let adx = x1 - x0;
let base = dy / adx;
let ady = dy.abs() - base.abs() * adx;
let sy = if dy < 0 { base - 1 } else { base + 1 };
let (mut y, mut err) = (y0, 0);
if let Some(slot) = usize::try_from(x0).ok().and_then(|x| v.get_mut(x)) {
*slot = y;
}
for x in x0 + 1..x1 {
err += ady;
if err >= adx {
err -= adx;
y += sy;
} else {
y += base;
}
match usize::try_from(x).ok().and_then(|x| v.get_mut(x)) {
Some(slot) => *slot = y,
None => break,
}
}
}
impl Floor1 {
fn decode(&self, r: &mut BitReader<'_>, books: &[Codebook]) -> Result<Option<FloorData>, Eop> {
if !r.bit()? {
return Ok(None);
}
let bits = ilog(i64::from(self.range) - 1);
let mut y = vec![0i32; self.xlist.len()];
y[0] = r.read(bits)? as i32;
y[1] = r.read(bits)? as i32;
let mut offset = 2usize;
for &class in &self.partition_class {
let cdim = self.class_dimensions[class];
let cbits = self.class_subclasses[class];
let csub = (1u32 << cbits) - 1;
let mut cval = 0u32;
if cbits > 0 {
cval = books[self.class_masterbook[class]].scalar(r)?;
}
for j in 0..cdim {
let book = self.subclass_books[class][(cval & csub) as usize];
cval >>= cbits;
y[offset + j] = if book >= 0 { books[book as usize].scalar(r)? as i32 } else { 0 };
}
offset += cdim;
}
// Step 1: amplitude synthesis (§7.2.4).
let mut step2 = vec![false; self.xlist.len()];
step2[0] = true;
step2[1] = true;
let mut fin = y.clone();
for i in 2..self.xlist.len() {
let (lo, hi) = self.neighbours[i - 2];
let predicted = render_point(self.xlist[lo], fin[lo], self.xlist[hi], fin[hi], self.xlist[i]);
let val = y[i];
let highroom = self.range - predicted;
let lowroom = predicted;
let room = if highroom < lowroom { highroom * 2 } else { lowroom * 2 };
if val != 0 {
step2[lo] = true;
step2[hi] = true;
step2[i] = true;
fin[i] = if val >= room {
if highroom > lowroom { val - lowroom + predicted } else { predicted - val + highroom - 1 }
} else if val % 2 == 1 {
predicted - (val + 1) / 2
} else {
predicted + val / 2
};
} else {
fin[i] = predicted;
}
// Valid setups never leave the range; a damaged one must not carry out-of-range values on.
fin[i] = fin[i].clamp(0, self.range - 1);
}
Ok(Some(FloorData::One { y: fin, step2 }))
}
fn curve(&self, fin: &[i32], step2: &[bool], n: usize) -> Vec<f32> {
let mut floor = vec![0i32; n];
let (mut hx, mut hy) = (0i32, 0i32);
let (mut lx, mut ly) = (0i32, fin[self.sorted[0]] * self.multiplier);
for &i in &self.sorted[1..] {
if step2[i] {
hy = fin[i] * self.multiplier;
hx = self.xlist[i];
render_line(lx, ly, hx, hy, &mut floor);
lx = hx;
ly = hy;
}
}
if (hx as usize) < n {
render_line(hx, hy, n as i32, hy, &mut floor);
}
floor.iter().map(|&v| INVERSE_DB[v.clamp(0, 255) as usize]).collect()
}
}
impl Floor0 {
fn decode(&self, r: &mut BitReader<'_>, books: &[Codebook]) -> Result<Option<FloorData>, Fail> {
let amplitude = r.read(self.amplitude_bits)?;
if amplitude == 0 {
return Ok(None);
}
let book = r.read(ilog(self.books.len() as i64))? as usize;
let Some(&book) = self.books.get(book) else { return Err(Fail::Invalid) };
let book = &books[book];
let mut coefficients: Vec<f32> = Vec::with_capacity(self.order + book.dimensions);
let mut last = 0f32;
while coefficients.len() < self.order {
let Some(v) = book.vector(r)? else { return Err(Fail::Invalid) };
coefficients.extend(v.iter().map(|&x| x + last));
last = coefficients[coefficients.len() - 1];
}
Ok(Some(FloorData::Zero { amplitude, coefficients }))
}
fn curve(&self, amplitude: u32, coefficients: &[f32], n: usize) -> Vec<f32> {
let bark = |x: f64| 13.1 * (0.00074 * x).atan() + 2.24 * (0.000_000_018_5 * x * x).atan() + 0.0001 * x;
let scale = f64::from(self.bark_map_size) / bark(0.5 * f64::from(self.rate));
let map: Vec<i64> = (0..n)
.map(|i| {
let foobar = (bark(f64::from(self.rate) * i as f64 / (2.0 * n as f64)) * scale).floor() as i64;
foobar.min(i64::from(self.bark_map_size) - 1)
})
.collect();
let cos_c: Vec<f64> = coefficients.iter().take(self.order).map(|&c| f64::from(c).cos()).collect();
let amp = f64::from(amplitude);
let offset = f64::from(self.amplitude_offset);
let top = (2f64).powi(self.amplitude_bits as i32) - 1.0;
let mut out = vec![0f32; n];
let mut i = 0usize;
while i < n {
let omega = std::f64::consts::PI * map[i] as f64 / f64::from(self.bark_map_size);
let cw = omega.cos();
let (p, q);
if self.order % 2 == 1 {
let mut pp = 1.0 - cw * cw;
for j in 0..(self.order - 1) / 2 {
let d = cos_c[2 * j + 1] - cw;
pp *= 4.0 * d * d;
}
let mut qq = 0.25;
for j in 0..=(self.order - 1) / 2 {
let d = cos_c[2 * j] - cw;
qq *= 4.0 * d * d;
}
p = pp;
q = qq;
} else {
let mut pp = (1.0 - cw) / 2.0;
let mut qq = (1.0 + cw) / 2.0;
for j in 0..self.order / 2 {
let d1 = cos_c[2 * j + 1] - cw;
pp *= 4.0 * d1 * d1;
let d0 = cos_c[2 * j] - cw;
qq *= 4.0 * d0 * d0;
}
p = pp;
q = qq;
}
let linear = (0.115_129_25 * (amp * offset / (top * (p + q).sqrt()) - offset)).exp() as f32;
let key = map[i];
while i < n && map[i] == key {
out[i] = linear;
i += 1;
}
}
out
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn render_line_matches_a_bresenham_walk() {
let mut v = vec![0i32; 16];
render_line(0, 0, 8, 4, &mut v);
assert_eq!(&v[..8], &[0, 0, 1, 1, 2, 2, 3, 3]);
let mut v = vec![0i32; 16];
render_line(2, 10, 6, 2, &mut v);
assert_eq!(&v[2..6], &[10, 8, 6, 4]);
// Lines running past the vector stop at its end.
let mut v = vec![0i32; 4];
render_line(0, 0, 100, 100, &mut v);
assert_eq!(v, vec![0, 1, 2, 3]);
assert_eq!(render_point(0, 0, 10, 10, 5), 5);
assert_eq!(render_point(0, 10, 10, 0, 5), 5);
}
}
+164
View File
@@ -0,0 +1,164 @@
//! The inverse MDCT of a block (Vorbis I §4.3.7) and the window slopes.
//!
//! `y[j] = sum_k X[k] cos(2*pi/n * (j + 1/2 + n/4) * (k + 1/2))` for `n/2` coefficients and `n`
//! samples, computed as a DCT-IV of the coefficients through an `n/8`-point complex FFT, then
//! unfolded.
use std::f64::consts::PI;
#[derive(Clone, Copy, Default)]
struct Cpx {
re: f64,
im: f64,
}
impl Cpx {
fn mul(self, o: Cpx) -> Cpx {
Cpx { re: self.re * o.re - self.im * o.im, im: self.re * o.im + self.im * o.re }
}
}
fn cis(t: f64) -> Cpx {
Cpx { re: t.cos(), im: t.sin() }
}
pub(super) struct Imdct {
n: usize,
twiddle: Vec<Cpx>,
post: Vec<Cpx>,
fft_tw: Vec<Cpx>,
rev: Vec<usize>,
}
impl Imdct {
/// For blocks of `n` samples, a power of two of at least 16.
pub(super) fn new(n: usize) -> Self {
let m = n / 2;
let h = m / 2;
let twiddle = (0..h).map(|j| cis(-PI * (4.0 * j as f64 + 1.0) / (4.0 * m as f64))).collect();
let post = (0..h).map(|k| cis(-PI * k as f64 / m as f64)).collect();
let fft_tw = (0..h / 2).map(|j| cis(-2.0 * PI * j as f64 / h as f64)).collect();
let bits = h.trailing_zeros();
let rev = (0..h).map(|i| if bits == 0 { 0 } else { i.reverse_bits() >> (usize::BITS - bits) }).collect();
Self { n, twiddle, post, fft_tw, rev }
}
/// `x`: `n/2` coefficients; `out`: `n` samples.
pub(super) fn inverse(&self, x: &[f32], out: &mut [f32]) {
let m = self.n / 2;
let q = self.n / 4;
let h = m / 2;
let mut z: Vec<Cpx> = (0..h).map(|j| Cpx { re: f64::from(x[2 * j]), im: f64::from(x[m - 1 - 2 * j]) }.mul(self.twiddle[j])).collect();
self.fft(&mut z);
let mut v = vec![0f64; m];
for (k, c) in z.iter().enumerate() {
let d = c.mul(self.post[k]);
v[2 * k] = d.re;
v[m - 1 - 2 * k] = -d.im;
}
// Unfold the DCT-IV output to the block.
for i in 0..q {
out[3 * q - 1 - i] = (-v[i]) as f32;
out[3 * q + i] = (-v[i]) as f32;
}
for i in q..2 * q {
out[i - q] = v[i] as f32;
out[3 * q - 1 - i] = (-v[i]) as f32;
}
}
fn fft(&self, a: &mut [Cpx]) {
let h = a.len();
for (i, &j) in self.rev.iter().enumerate() {
if i < j {
a.swap(i, j);
}
}
let mut len = 2;
while len <= h {
let step = h / len;
for start in (0..h).step_by(len) {
for k in 0..len / 2 {
let w = self.fft_tw[k * step];
let u = a[start + k];
let t = a[start + k + len / 2].mul(w);
a[start + k] = Cpx { re: u.re + t.re, im: u.im + t.im };
a[start + k + len / 2] = Cpx { re: u.re - t.re, im: u.im - t.im };
}
}
len *= 2;
}
}
}
/// The rising half of the Vorbis window over `len` samples (§4.3.1):
/// `sin(pi/2 * sin^2((i + 0.5) / len * pi/2))`.
pub(super) fn slope(len: usize) -> Vec<f32> {
(0..len)
.map(|i| {
let s = (PI / 2.0 * (i as f64 + 0.5) / len as f64).sin();
(PI / 2.0 * s * s).sin() as f32
})
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn matches_the_direct_formula() {
for n in [16usize, 64, 256, 2048] {
let x: Vec<f32> = (0..n / 2).map(|i| ((i * 7919 % 211) as f32 - 105.0) / 50.0).collect();
let mut fast = vec![0f32; n];
Imdct::new(n).inverse(&x, &mut fast);
for j in [0usize, 1, 2, n / 4, n / 2, n - 1] {
let direct: f64 =
(0..n / 2).map(|k| f64::from(x[k]) * (2.0 * PI / n as f64 * (j as f64 + 0.5 + n as f64 / 4.0) * (k as f64 + 0.5)).cos()).sum();
assert!((direct - f64::from(fast[j])).abs() < 1e-3 * (1.0 + direct.abs()), "n={n} j={j}: {direct} vs {}", fast[j]);
}
}
}
#[test]
fn windowed_overlap_add_reconstructs_the_signal() {
// Forward MDCT by the definition with the 2/M scale an encoder uses, then this inverse.
let n = 64usize;
let m = n / 2;
let signal: Vec<f64> = (0..n * 6).map(|i| ((i * 37 % 101) as f64 - 50.0) / 50.0).collect();
let w: Vec<f32> = slope(m).into_iter().chain(slope(m).into_iter().rev()).collect();
let imdct = Imdct::new(n);
let mut out = vec![0f64; signal.len()];
let mut start = 0;
while start + n <= signal.len() {
let coeffs: Vec<f32> = (0..m)
.map(|k| {
let s: f64 = (0..n)
.map(|j| {
signal[start + j] * f64::from(w[j]) * (2.0 * PI / n as f64 * (j as f64 + 0.5 + n as f64 / 4.0) * (k as f64 + 0.5)).cos()
})
.sum();
(s * 2.0 / m as f64) as f32
})
.collect();
let mut y = vec![0f32; n];
imdct.inverse(&coeffs, &mut y);
for j in 0..n {
out[start + j] += f64::from(y[j]) * f64::from(w[j]);
}
start += m;
}
for i in m..signal.len() - m {
assert!((out[i] - signal[i]).abs() < 1e-4, "sample {i}: {} vs {}", out[i], signal[i]);
}
}
#[test]
fn the_slope_is_power_complementary() {
let s = slope(1024);
for i in 0..1024 {
let sum = s[i] * s[i] + s[1023 - i] * s[1023 - i];
assert!((sum - 1.0).abs() < 1e-5, "{i}: {sum}");
}
}
}
+552
View File
@@ -0,0 +1,552 @@
//! A clean-room Vorbis I decoder (feature `vorbis-decode`), written from the Vorbis I
//! specification (Xiph.Org, 2020).
//!
//! [`Decoder`] works on packets: build it from the three header packets (identification, comment
//! and setup), then give it audio packets in order. Each call returns the PCM the packet
//! completes, planar `f32` in the nominal range -1.0 to 1.0 (not clipped), channels in the order of
//! the stream. The first packet only primes the overlap and returns nothing. [`decode_ogg`] reads a
//! whole Ogg Vorbis file.
//!
//! Codebooks, floors 0 and 1, residues 0, 1 and 2, channel coupling and both block sizes with their
//! hybrid windows are implemented. Damaged input is an error, never a panic: setup sizes are
//! bounded before anything is allocated, and the codeword trees are checked for completeness.
mod bits;
mod codebook;
mod floor;
mod mdct;
mod residue;
mod tables;
use bits::{BitReader, Eop, ilog};
use codebook::Codebook;
use floor::{Fail, Floor, FloorData};
use mdct::{Imdct, slope};
use residue::Residue;
/// Errors of the Vorbis decoder.
#[non_exhaustive]
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
pub enum Error {
/// The identification or comment header is not valid Vorbis.
#[error("invalid Vorbis header: {0}")]
Header(&'static str),
/// The setup header is damaged or uses something Vorbis I does not allow.
#[error("invalid Vorbis setup: {0}")]
Setup(&'static str),
/// The setup or decoded PCM asks for more memory than the decoder allows.
#[error("Vorbis decode above the limits: {0}")]
Limit(&'static str),
/// The packet is not an audio packet (a header, or a packet with its first bit set).
#[error("not an audio packet")]
NotAudio,
/// The packet is damaged: it ends before its mode and window flags are read, or names a mode
/// that does not exist.
#[error("damaged audio packet: {0}")]
Packet(&'static str),
/// An Ogg file with no Vorbis stream, or one that cannot be read.
#[error("Ogg Vorbis file: {0}")]
Ogg(String),
}
/// `Result` of this module.
pub type Result<T> = std::result::Result<T, Error>;
/// The identification header (§4.2.2).
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Identification {
/// The channel count, at least 1.
pub channels: u8,
/// Samples per second per channel.
pub sample_rate: u32,
/// The maximum bit rate in bits per second, 0 or negative when not set.
pub bitrate_maximum: i32,
/// The nominal bit rate in bits per second, 0 or negative when not set.
pub bitrate_nominal: i32,
/// The minimum bit rate in bits per second, 0 or negative when not set.
pub bitrate_minimum: i32,
/// The short block size in samples.
pub blocksize_0: u32,
/// The long block size in samples.
pub blocksize_1: u32,
}
/// The comment header (§5): a vendor string and `NAME=value` comments.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct Comments {
/// The vendor string of the encoder.
pub vendor: String,
/// The user comments, each `NAME=value`.
pub comments: Vec<String>,
}
impl Comments {
/// The first comment named `name` (case-insensitive), its value.
#[must_use]
pub fn get(&self, name: &str) -> Option<&str> {
self.comments.iter().find_map(|c| {
let (k, v) = c.split_once('=')?;
k.eq_ignore_ascii_case(name).then_some(v)
})
}
}
fn header_start(packet: &[u8], kind: u8) -> Result<BitReader<'_>> {
if packet.len() < 7 || packet[0] != kind || &packet[1..7] != b"vorbis" {
return Err(Error::Header("wrong packet type or missing \"vorbis\" tag"));
}
let mut r = BitReader::new(packet);
let _ = r.read(8);
for _ in 0..6 {
let _ = r.read(8);
}
Ok(r)
}
fn read_identification(packet: &[u8]) -> Result<Identification> {
let mut r = header_start(packet, 1)?;
let eop = |_: Eop| Error::Header("identification header is truncated");
if r.read(32).map_err(eop)? != 0 {
return Err(Error::Header("unsupported Vorbis version"));
}
let channels = r.read(8).map_err(eop)? as u8;
let sample_rate = r.read(32).map_err(eop)?;
let bitrate_maximum = r.read(32).map_err(eop)? as i32;
let bitrate_nominal = r.read(32).map_err(eop)? as i32;
let bitrate_minimum = r.read(32).map_err(eop)? as i32;
let e0 = r.read(4).map_err(eop)?;
let e1 = r.read(4).map_err(eop)?;
if !r.bit().map_err(eop)? {
return Err(Error::Header("identification header framing bit is not set"));
}
if channels == 0 || sample_rate == 0 {
return Err(Error::Header("no channels or a zero sample rate"));
}
if !(6..=13).contains(&e0) || !(6..=13).contains(&e1) || e0 > e1 {
return Err(Error::Header("block sizes must be powers of two from 64 to 8192, the short one not above the long one"));
}
Ok(Identification { channels, sample_rate, bitrate_maximum, bitrate_nominal, bitrate_minimum, blocksize_0: 1 << e0, blocksize_1: 1 << e1 })
}
/// Comments are best effort: damage here is not fatal (§4.2.1).
fn read_comments(packet: &[u8]) -> Result<Comments> {
header_start(packet, 3)?;
let mut at = 7usize;
let mut take = |n: usize| -> Option<&[u8]> {
let s = packet.get(at..at.checked_add(n)?)?;
at += n;
Some(s)
};
let word = |b: &[u8]| u32::from_le_bytes([b[0], b[1], b[2], b[3]]) as usize;
let mut out = Comments::default();
let Some(n) = take(4).map(word) else { return Ok(out) };
let Some(vendor) = take(n) else { return Ok(out) };
out.vendor = String::from_utf8_lossy(vendor).into_owned();
let Some(count) = take(4).map(word) else { return Ok(out) };
for _ in 0..count {
let Some(n) = take(4).map(word) else { break };
let Some(c) = take(n) else { break };
out.comments.push(String::from_utf8_lossy(c).into_owned());
}
Ok(out)
}
struct Mapping {
coupling: Vec<(usize, usize)>,
mux: Vec<usize>,
/// `(floor, residue)` of each submap.
submaps: Vec<(usize, usize)>,
}
struct Mode {
long: bool,
mapping: usize,
}
struct Setup {
books: Vec<Codebook>,
floors: Vec<Floor>,
residues: Vec<Residue>,
mappings: Vec<Mapping>,
modes: Vec<Mode>,
}
/// Bytes the codebooks of a setup may take in all (codeword trees and vector tables).
const CODEBOOK_BUDGET: usize = 256 << 20;
fn read_setup(packet: &[u8], channels: usize) -> Result<Setup> {
let mut r = header_start(packet, 5)?;
let eop = |_: Eop| Error::Setup("setup header is truncated");
let count = r.read(8).map_err(eop)? as usize + 1;
let mut budget = CODEBOOK_BUDGET;
let mut books = Vec::with_capacity(count);
for _ in 0..count {
books.push(Codebook::read(&mut r, &mut budget)?);
}
// Time domain transforms: placeholders.
for _ in 0..r.read(6).map_err(eop)? + 1 {
if r.read(16).map_err(eop)? != 0 {
return Err(Error::Setup("a time domain transform is not zero"));
}
}
let count = r.read(6).map_err(eop)? as usize + 1;
let mut floors = Vec::with_capacity(count);
for _ in 0..count {
let kind = r.read(16).map_err(eop)?;
floors.push(Floor::read(kind, &mut r, &books)?);
}
let count = r.read(6).map_err(eop)? as usize + 1;
let mut residues = Vec::with_capacity(count);
for _ in 0..count {
let kind = r.read(16).map_err(eop)?;
if kind > 2 {
return Err(Error::Setup("unknown residue type"));
}
residues.push(Residue::read(kind, &mut r, &books)?);
}
let count = r.read(6).map_err(eop)? as usize + 1;
let mut mappings = Vec::with_capacity(count);
for _ in 0..count {
if r.read(16).map_err(eop)? != 0 {
return Err(Error::Setup("unknown mapping type"));
}
let submaps = if r.bit().map_err(eop)? { r.read(4).map_err(eop)? as usize + 1 } else { 1 };
let mut coupling = Vec::new();
if r.bit().map_err(eop)? {
let steps = r.read(8).map_err(eop)? as usize + 1;
let bits = ilog(channels as i64 - 1);
for _ in 0..steps {
let magnitude = r.read(bits).map_err(eop)? as usize;
let angle = r.read(bits).map_err(eop)? as usize;
if magnitude == angle || magnitude >= channels || angle >= channels {
return Err(Error::Setup("channel coupling names an invalid channel pair"));
}
coupling.push((magnitude, angle));
}
}
if r.read(2).map_err(eop)? != 0 {
return Err(Error::Setup("reserved mapping bits are set"));
}
let mut mux = vec![0usize; channels];
if submaps > 1 {
for m in &mut mux {
*m = r.read(4).map_err(eop)? as usize;
if *m >= submaps {
return Err(Error::Setup("channel multiplexed to a submap that does not exist"));
}
}
}
let mut list = Vec::with_capacity(submaps);
for _ in 0..submaps {
r.read(8).map_err(eop)?;
let floor = r.read(8).map_err(eop)? as usize;
let residue = r.read(8).map_err(eop)? as usize;
if floor >= floors.len() || residue >= residues.len() {
return Err(Error::Setup("submap names a floor or residue that does not exist"));
}
list.push((floor, residue));
}
mappings.push(Mapping { coupling, mux, submaps: list });
}
let count = r.read(6).map_err(eop)? as usize + 1;
let mut modes = Vec::with_capacity(count);
for _ in 0..count {
let long = r.bit().map_err(eop)?;
let window = r.read(16).map_err(eop)?;
let transform = r.read(16).map_err(eop)?;
let mapping = r.read(8).map_err(eop)? as usize;
if window != 0 || transform != 0 || mapping >= mappings.len() {
return Err(Error::Setup("a mode has a nonzero window or transform type or an unknown mapping"));
}
modes.push(Mode { long, mapping });
}
if !r.bit().map_err(eop)? {
return Err(Error::Setup("setup header framing bit is not set"));
}
Ok(Setup { books, floors, residues, mappings, modes })
}
/// The previous block, windowed, kept for overlap-add.
struct Previous {
n: usize,
data: Vec<Vec<f32>>,
}
/// A Vorbis I packet decoder.
pub struct Decoder {
id: Identification,
comments: Comments,
setup: Setup,
imdct: [Imdct; 2],
slopes: [Vec<f32>; 2],
previous: Option<Previous>,
}
impl Decoder {
/// A decoder from the three header packets, in order. A damaged comment header is tolerated
/// (its comments are then empty); the other two must be valid.
pub fn new(identification: &[u8], comment: &[u8], setup: &[u8]) -> Result<Self> {
let id = read_identification(identification)?;
let comments = read_comments(comment)?;
let setup = read_setup(setup, usize::from(id.channels))?;
let (b0, b1) = (id.blocksize_0 as usize, id.blocksize_1 as usize);
Ok(Self { id, comments, setup, imdct: [Imdct::new(b0), Imdct::new(b1)], slopes: [slope(b0 / 2), slope(b1 / 2)], previous: None })
}
/// The identification header.
#[must_use]
pub fn identification(&self) -> &Identification {
&self.id
}
/// The comment header.
#[must_use]
pub fn comments(&self) -> &Comments {
&self.comments
}
/// Forget the previous block, as after a seek: the next packet only primes the overlap again.
pub fn reset(&mut self) {
self.previous = None;
}
/// The window of a block (§4.3.1).
fn window(&self, n: usize, long: bool, prev_long: bool, next_long: bool) -> Vec<f32> {
let half = n / 2;
let b0 = self.id.blocksize_0 as usize;
let mut w = vec![0f32; n];
// Left half.
if long && !prev_long {
let (start, end) = (n / 4 - b0 / 4, n / 4 + b0 / 4);
w[start..end].copy_from_slice(&self.slopes[0]);
w[end..half].fill(1.0);
} else {
w[..half].copy_from_slice(&self.slopes[usize::from(long)]);
}
// Right half.
if long && !next_long {
let start = 3 * n / 4 - b0 / 4;
w[half..start].fill(1.0);
for (i, s) in self.slopes[0].iter().rev().enumerate() {
w[start + i] = *s;
}
} else {
for (i, s) in self.slopes[usize::from(long)].iter().rev().enumerate() {
w[half + i] = *s;
}
}
w
}
/// Decode one audio packet. The result holds one `Vec` per channel, all of the same length:
/// the samples this packet completes (none for the first packet after a start or reset).
pub fn decode_packet(&mut self, packet: &[u8]) -> Result<Vec<Vec<f32>>> {
let channels = usize::from(self.id.channels);
let mut r = BitReader::new(packet);
match r.bit() {
Ok(false) => {}
Ok(true) => return Err(Error::NotAudio),
Err(_) => return Err(Error::Packet("empty packet")),
}
let mode_bits = ilog(self.setup.modes.len() as i64 - 1);
let mode_number = r.read(mode_bits).map_err(|_| Error::Packet("packet ends inside the mode number"))? as usize;
let Some(mode) = self.setup.modes.get(mode_number) else { return Err(Error::Packet("mode number out of range")) };
let long = mode.long;
let n = if long { self.id.blocksize_1 } else { self.id.blocksize_0 } as usize;
let (mut prev_long, mut next_long) = (true, true);
if long {
prev_long = r.bit().map_err(|_| Error::Packet("packet ends inside the window flags"))?;
next_long = r.bit().map_err(|_| Error::Packet("packet ends inside the window flags"))?;
}
let window = self.window(n, long, prev_long, next_long);
let half = n / 2;
let mapping = &self.setup.mappings[mode.mapping];
let spectrum = self.spectrum(&mut r, mapping, half, channels)?;
// Inverse transform and window.
let which = usize::from(long);
let mut blocks: Vec<Vec<f32>> = Vec::with_capacity(channels);
for s in &spectrum {
let mut y = vec![0f32; n];
self.imdct[which].inverse(s, &mut y);
for (v, w) in y.iter_mut().zip(&window) {
*v *= *w;
}
blocks.push(y);
}
// Overlap-add with the previous block (§4.3.8).
let out = match &self.previous {
None => vec![Vec::new(); channels],
Some(prev) => {
let count = prev.n / 4 + n / 4;
blocks
.iter()
.zip(&prev.data)
.map(|(cur, old)| {
(0..count)
.map(|k| {
let a = old.get(prev.n / 2 + k).copied().unwrap_or(0.0);
let b = (n / 4 + k).checked_sub(prev.n / 4).and_then(|i| cur.get(i)).copied().unwrap_or(0.0);
a + b
})
.collect()
})
.collect()
}
};
self.previous = Some(Previous { n, data: blocks });
Ok(out)
}
/// Floors, residues, coupling and the dot product: the spectrum of each channel (§4.3.2 to 4.3.6).
fn spectrum(&self, r: &mut BitReader<'_>, mapping: &Mapping, half: usize, channels: usize) -> Result<Vec<Vec<f32>>> {
let zero = || vec![vec![0f32; half]; channels];
let setup = &self.setup;
// Floors, in channel order.
let mut floors: Vec<Option<FloorData>> = Vec::with_capacity(channels);
for ch in 0..channels {
let (floor, _) = mapping.submaps[mapping.mux[ch]];
match setup.floors[floor].decode(r, &setup.books) {
Ok(f) => floors.push(f),
Err(Fail::Eop) => return Ok(zero()),
Err(Fail::Invalid) => return Err(Error::Packet("a floor names a codebook that does not exist")),
}
}
let mut no_residue: Vec<bool> = floors.iter().map(Option::is_none).collect();
for &(m, a) in &mapping.coupling {
if !no_residue[m] || !no_residue[a] {
no_residue[m] = false;
no_residue[a] = false;
}
}
// Residues, in submap order.
let mut residue = zero();
for (i, &(_, res)) in mapping.submaps.iter().enumerate() {
let members: Vec<usize> = (0..channels).filter(|&c| mapping.mux[c] == i).collect();
if members.is_empty() {
continue;
}
let skip: Vec<bool> = members.iter().map(|&c| no_residue[c]).collect();
let mut vectors = vec![vec![0f32; half]; members.len()];
setup.residues[res].decode(r, &setup.books, half, &skip, &mut vectors);
for (v, &c) in vectors.into_iter().zip(&members) {
residue[c] = v;
}
}
// Inverse channel coupling (§4.3.5), last step first.
for &(m, a) in mapping.coupling.iter().rev() {
let mut mag = std::mem::take(&mut residue[m]);
let mut ang = std::mem::take(&mut residue[a]);
for (mv, av) in mag.iter_mut().zip(ang.iter_mut()) {
let (big, small) = (*mv, *av);
let (new_m, new_a);
if big > 0.0 {
if small > 0.0 {
new_m = big;
new_a = big - small;
} else {
new_a = big;
new_m = big + small;
}
} else if small > 0.0 {
new_m = big;
new_a = big + small;
} else {
new_a = big;
new_m = big - small;
}
*mv = new_m;
*av = new_a;
}
residue[m] = mag;
residue[a] = ang;
}
// The floor times the residue.
for ch in 0..channels {
match &floors[ch] {
Some(data) => {
let (floor, _) = mapping.submaps[mapping.mux[ch]];
let curve = setup.floors[floor].curve(data, half);
for (v, c) in residue[ch].iter_mut().zip(curve) {
*v *= c;
}
}
None => residue[ch].fill(0.0),
}
}
Ok(residue)
}
}
/// A decoded Ogg Vorbis file.
#[derive(Debug, Clone, PartialEq)]
pub struct Decoded {
/// Samples per second per channel.
pub sample_rate: u32,
/// One `Vec` of samples per channel.
pub channels: Vec<Vec<f32>>,
/// The stream's comment header.
pub comments: Comments,
/// Audio packets that could not be decoded and were skipped.
pub damaged_packets: usize,
}
/// Decode the first Vorbis stream of an Ogg file. The result is trimmed to the stream's final
/// granule position, so an encoder's padding at the end is dropped. Damaged packets are skipped
/// and counted, not fatal. Collected PCM is limited to 256 MiB; use the packet
/// decoder for longer streams without retaining the entire output.
pub fn decode_ogg(data: &[u8]) -> Result<Decoded> {
let ogg = craft_media::ogg::open(data).map_err(|e| Error::Ogg(e.to_string()))?;
let index = ogg.stream_of(craft_media::ogg::Codec::Vorbis).ok_or_else(|| Error::Ogg("no Vorbis stream".into()))?;
let stream = &ogg.streams[index];
let [id, comment, setup, ..] = stream.headers.as_slice() else { return Err(Error::Ogg("the stream has fewer than three header packets".into())) };
let mut decoder = Decoder::new(id, comment, setup)?;
let channel_count = usize::from(decoder.id.channels);
let mut channels = vec![Vec::new(); channel_count];
let mut damaged = 0;
for i in 0..stream.packets.len() {
let packet = ogg.read_packet(data, index, i).map_err(|e| Error::Ogg(e.to_string()))?;
match decoder.decode_packet(&packet) {
Ok(pcm) => {
append_pcm(&mut channels, pcm, 256 << 20)?;
}
Err(_) => damaged += 1,
}
}
if let Some(total) = stream.last_granule.and_then(|g| usize::try_from(g).ok()) {
for c in &mut channels {
c.truncate(total);
}
}
Ok(Decoded { sample_rate: decoder.id.sample_rate, channels, comments: decoder.comments.clone(), damaged_packets: damaged })
}
// Admit every channel together before extending any of them. Limit retained PCM,
// not the compressed input, setup tables or the decoder's bounded packet scratch.
fn append_pcm(channels: &mut [Vec<f32>], pcm: Vec<Vec<f32>>, budget: usize) -> Result<()> {
let samples = channels.iter().chain(&pcm).try_fold(0usize, |n, c| n.checked_add(c.len()));
if samples.is_none_or(|n| n > budget / std::mem::size_of::<f32>()) {
return Err(Error::Limit("decoded PCM exceeds the whole-file budget"));
}
for (all, new) in channels.iter_mut().zip(pcm) {
all.try_reserve_exact(new.len()).map_err(|_| Error::Limit("cannot allocate decoded PCM"))?;
all.extend(new);
}
Ok(())
}
#[cfg(test)]
mod budget_tests {
use super::*;
#[test]
fn pcm_budget_counts_all_channels_before_extending() {
let mut channels = vec![vec![1.0], vec![2.0]];
append_pcm(&mut channels, vec![vec![3.0], vec![4.0]], 16).expect("exact budget");
let before = channels.clone();
assert!(matches!(append_pcm(&mut channels, vec![vec![5.0], vec![6.0]], 23), Err(Error::Limit(_))));
assert_eq!(channels, before);
append_pcm(&mut channels, vec![Vec::new(), Vec::new()], 16).expect("empty packet");
}
}
+196
View File
@@ -0,0 +1,196 @@
//! Residues (Vorbis I §8): the fine detail of the spectrum, coded as vectors in passes.
use super::Error;
use super::bits::{BitReader, Eop};
use super::codebook::Codebook;
pub(super) struct Residue {
kind: u32,
begin: usize,
end: usize,
partition_size: usize,
classifications: usize,
classbook: usize,
/// `books[class][pass]`.
books: Vec<[Option<usize>; 8]>,
}
fn eop(_: Eop) -> Error {
Error::Setup("setup header ends inside a residue")
}
impl Residue {
pub(super) fn read(kind: u32, r: &mut BitReader<'_>, books: &[Codebook]) -> Result<Self, Error> {
let begin = r.read(24).map_err(eop)? as usize;
let end = r.read(24).map_err(eop)? as usize;
let partition_size = r.read(24).map_err(eop)? as usize + 1;
let classifications = r.read(6).map_err(eop)? as usize + 1;
let classbook = r.read(8).map_err(eop)? as usize;
let Some(cb) = books.get(classbook) else { return Err(Error::Setup("residue class book out of range")) };
// classifications ^ dimensions must not exceed the book's entries.
let mut reach = 1u128;
for _ in 0..cb.dimensions {
reach = reach.saturating_mul(classifications as u128);
if reach > cb.entries as u128 {
return Err(Error::Setup("residue class book has too few entries for its classifications"));
}
}
let mut cascade = Vec::with_capacity(classifications);
for _ in 0..classifications {
let low = r.read(3).map_err(eop)?;
let high = if r.bit().map_err(eop)? { r.read(5).map_err(eop)? } else { 0 };
cascade.push(high * 8 + low);
}
let mut list = Vec::with_capacity(classifications);
for &c in &cascade {
let mut row = [None; 8];
for (j, slot) in row.iter_mut().enumerate() {
if c & (1 << j) != 0 {
let b = r.read(8).map_err(eop)? as usize;
match books.get(b) {
Some(book) if book.has_vectors() => *slot = Some(b),
_ => return Err(Error::Setup("residue names a missing codebook or one without vectors")),
}
}
}
list.push(row);
}
Ok(Self { kind, begin, end, partition_size, classifications, classbook, books: list })
}
/// Decode a bundle of vectors, each `n` long (half the block size), into `out`; a vector marked
/// in `skip` is left zero. End of packet ends decoding where it is (a nominal event).
pub(super) fn decode(&self, r: &mut BitReader<'_>, books: &[Codebook], n: usize, skip: &[bool], out: &mut [Vec<f32>]) {
let ch = out.len();
if skip.iter().all(|&s| s) {
return;
}
if self.kind == 2 {
let mut flat = vec![vec![0f32; n * ch]];
let _ = self.decode_vectors(r, books, n * ch, &[false], &mut flat);
for i in 0..n {
for (j, v) in out.iter_mut().enumerate() {
v[i] = flat[0][i * ch + j];
}
}
} else {
let _ = self.decode_vectors(r, books, n, skip, out);
}
}
fn decode_vectors(&self, r: &mut BitReader<'_>, books: &[Codebook], actual: usize, skip: &[bool], out: &mut [Vec<f32>]) -> Result<(), Eop> {
let ch = out.len();
let begin = self.begin.min(actual);
let end = self.end.min(actual);
if end <= begin {
return Ok(());
}
let classbook = &books[self.classbook];
let cpw = classbook.dimensions;
let partitions = (end - begin) / self.partition_size;
// Classbook dimensions may greatly exceed the actual partition count.
// Retain only digits that this block can use, not padding per channel.
let mut classes = vec![vec![0usize; partitions]; ch];
for pass in 0..8 {
let mut count = 0usize;
while count < partitions {
if pass == 0 {
for j in 0..ch {
if skip[j] {
continue;
}
let mut temp = classbook.scalar(r)? as usize;
for i in (0..cpw).rev() {
if i + count < partitions {
classes[j][i + count] = temp % self.classifications;
}
temp /= self.classifications;
}
}
}
let mut i = 0;
while i < cpw && count < partitions {
for j in 0..ch {
if skip[j] {
continue;
}
let Some(book) = self.books[classes[j][count]][pass] else { continue };
let book = &books[book];
let offset = begin + count * self.partition_size;
self.partition(r, book, &mut out[j][offset..offset + self.partition_size])?;
}
count += 1;
i += 1;
}
}
}
Ok(())
}
/// One partition into `v` (the slice of the vector it covers).
fn partition(&self, r: &mut BitReader<'_>, book: &Codebook, v: &mut [f32]) -> Result<(), Eop> {
let n = v.len();
let dims = book.dimensions;
let step = n / dims;
if self.kind == 0 {
for i in 0..step {
let Some(entry) = book.vector(r)? else { return Ok(()) };
for (j, &e) in entry.iter().enumerate() {
v[i + j * step] += e;
}
}
} else {
let mut i = 0;
for _ in 0..step {
let Some(entry) = book.vector(r)? else { return Ok(()) };
for &e in entry {
v[i] += e;
i += 1;
}
}
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn huge_class_dimensions_keep_only_actual_partitions() {
let mut data = Vec::<u8>::new();
let mut bits = 0usize;
let mut put = |value: u32, n: usize| {
for i in 0..n {
if bits.is_multiple_of(8) {
data.push(0);
}
let last = data.len() - 1;
data[last] |= (((value >> i) & 1) as u8) << (bits % 8);
bits += 1;
}
};
put(0x56_4342, 24);
put(65535, 16);
put(1, 24);
put(0, 1); // unordered, not sparse, one length-1 entry
put(0, 1);
put(0, 5);
put(0, 4); // scalar-only classbook
let book = Codebook::read(&mut BitReader::new(&data), &mut (1 << 20)).expect("classbook");
let books = [book];
for kind in [0, 1] {
let residue = Residue { kind, begin: 0, end: 32, partition_size: 1, classifications: 1, classbook: 0, books: vec![[None; 8]] };
let mut out = vec![vec![0.0; 32]; 255];
let packet = [0u8; 32];
let mut r = BitReader::new(&packet);
residue.decode(&mut r, &books, 32, &[false; 255], &mut out);
assert_eq!(r.remaining(), 1); // one classword per channel, not 32
assert!(out.iter().flatten().all(|&x| x == 0.0));
let mut r = BitReader::new(&packet);
residue.decode(&mut r, &books, 32, &[true; 255], &mut out);
assert_eq!(r.remaining(), 256); // skipped channels consume nothing
}
}
}
+263
View File
@@ -0,0 +1,263 @@
//! `floor1_inverse_dB_table` (Vorbis I §10.1): the 256 linear amplitudes of the floor 1 integer values.
// The values are printed to the digits the specification gives them.
#![allow(clippy::excessive_precision)]
pub(super) const INVERSE_DB: [f32; 256] = [
1.0649863e-07,
1.1341951e-07,
1.2079015e-07,
1.2863978e-07,
1.3699951e-07,
1.4590251e-07,
1.5538408e-07,
1.6548181e-07,
1.7623575e-07,
1.8768855e-07,
1.9988561e-07,
2.1287530e-07,
2.2670913e-07,
2.4144197e-07,
2.5713223e-07,
2.7384213e-07,
2.9163793e-07,
3.1059021e-07,
3.3077411e-07,
3.5226968e-07,
3.7516214e-07,
3.9954229e-07,
4.2550680e-07,
4.5315863e-07,
4.8260743e-07,
5.1396998e-07,
5.4737065e-07,
5.8294187e-07,
6.2082472e-07,
6.6116941e-07,
7.0413592e-07,
7.4989464e-07,
7.9862701e-07,
8.5052630e-07,
9.0579828e-07,
9.6466216e-07,
1.0273513e-06,
1.0941144e-06,
1.1652161e-06,
1.2409384e-06,
1.3215816e-06,
1.4074654e-06,
1.4989305e-06,
1.5963394e-06,
1.7000785e-06,
1.8105592e-06,
1.9282195e-06,
2.0535261e-06,
2.1869758e-06,
2.3290978e-06,
2.4804557e-06,
2.6416497e-06,
2.8133190e-06,
2.9961443e-06,
3.1908506e-06,
3.3982101e-06,
3.6190449e-06,
3.8542308e-06,
4.1047004e-06,
4.3714470e-06,
4.6555282e-06,
4.9580707e-06,
5.2802740e-06,
5.6234160e-06,
5.9888572e-06,
6.3780469e-06,
6.7925283e-06,
7.2339451e-06,
7.7040476e-06,
8.2047000e-06,
8.7378876e-06,
9.3057248e-06,
9.9104632e-06,
1.0554501e-05,
1.1240392e-05,
1.1970856e-05,
1.2748789e-05,
1.3577278e-05,
1.4459606e-05,
1.5399272e-05,
1.6400004e-05,
1.7465768e-05,
1.8600792e-05,
1.9809576e-05,
2.1096914e-05,
2.2467911e-05,
2.3928002e-05,
2.5482978e-05,
2.7139006e-05,
2.8902651e-05,
3.0780908e-05,
3.2781225e-05,
3.4911534e-05,
3.7180282e-05,
3.9596466e-05,
4.2169667e-05,
4.4910090e-05,
4.7828601e-05,
5.0936773e-05,
5.4246931e-05,
5.7772202e-05,
6.1526565e-05,
6.5524908e-05,
6.9783085e-05,
7.4317983e-05,
7.9147585e-05,
8.4291040e-05,
8.9768747e-05,
9.5602426e-05,
0.00010181521,
0.00010843174,
0.00011547824,
0.00012298267,
0.00013097477,
0.00013948625,
0.00014855085,
0.00015820453,
0.00016848555,
0.00017943469,
0.00019109536,
0.00020351382,
0.00021673929,
0.00023082423,
0.00024582449,
0.00026179955,
0.00027881276,
0.00029693158,
0.00031622787,
0.00033677814,
0.00035866388,
0.00038197188,
0.00040679456,
0.00043323036,
0.00046138411,
0.00049136745,
0.00052329927,
0.00055730621,
0.00059352311,
0.00063209358,
0.00067317058,
0.00071691700,
0.00076350630,
0.00081312324,
0.00086596457,
0.00092223983,
0.00098217216,
0.0010459992,
0.0011139742,
0.0011863665,
0.0012634633,
0.0013455702,
0.0014330129,
0.0015261382,
0.0016253153,
0.0017309374,
0.0018434235,
0.0019632195,
0.0020908006,
0.0022266726,
0.0023713743,
0.0025254795,
0.0026895994,
0.0028643847,
0.0030505286,
0.0032487691,
0.0034598925,
0.0036847358,
0.0039241906,
0.0041792066,
0.0044507950,
0.0047400328,
0.0050480668,
0.0053761186,
0.0057254891,
0.0060975636,
0.0064938176,
0.0069158225,
0.0073652516,
0.0078438871,
0.0083536271,
0.0088964928,
0.009474637,
0.010090352,
0.010746080,
0.011444421,
0.012188144,
0.012980198,
0.013823725,
0.014722068,
0.015678791,
0.016697687,
0.017782797,
0.018938423,
0.020169149,
0.021479854,
0.022875735,
0.024362330,
0.025945531,
0.027631618,
0.029427276,
0.031339626,
0.033376252,
0.035545228,
0.037855157,
0.040315199,
0.042935108,
0.045725273,
0.048696758,
0.051861348,
0.055231591,
0.058820850,
0.062643361,
0.066714279,
0.071049749,
0.075666962,
0.080584227,
0.085821044,
0.091398179,
0.097337747,
0.10366330,
0.11039993,
0.11757434,
0.12521498,
0.13335215,
0.14201813,
0.15124727,
0.16107617,
0.17154380,
0.18269168,
0.19456402,
0.20720788,
0.22067342,
0.23501402,
0.25028656,
0.26655159,
0.28387361,
0.30232132,
0.32196786,
0.34289114,
0.36517414,
0.38890521,
0.41417847,
0.44109412,
0.46975890,
0.50028648,
0.53279791,
0.56742212,
0.60429640,
0.64356699,
0.68538959,
0.72993007,
0.77736504,
0.82788260,
0.88168307,
0.9389798,
1.0,
];
+5 -2
View File
@@ -1,4 +1,7 @@
//! Vorbis I (feature `vorbis-encode`): the pure-Rust [`encoder`]. Decoding Vorbis goes through
//! [`crate::decode`].
//! Vorbis I: the pure-Rust [`decoder`] (feature `vorbis-decode`) and [`encoder`] (feature
//! `vorbis-encode`). `craft_audio::decode` still reads Vorbis files through symphonia.
#[cfg(feature = "vorbis-decode")]
pub mod decoder;
#[cfg(feature = "vorbis-encode")]
pub mod encoder;
+218
View File
@@ -0,0 +1,218 @@
#![cfg(feature = "g726")]
use craft_audio::g726::{BitPacking, BitRate, Decoder, Encoder, decode, decode_interleaved, encode, encode_interleaved};
use std::process::Command;
fn pearson_correlation(x: &[i16], y: &[i16]) -> f64 {
let n = x.len().min(y.len());
if n == 0 {
return 0.0;
}
let x = &x[..n];
let y = &y[..n];
let mean_x = x.iter().map(|&v| v as f64).sum::<f64>() / (n as f64);
let mean_y = y.iter().map(|&v| v as f64).sum::<f64>() / (n as f64);
let mut num = 0.0;
let mut denom_x = 0.0;
let mut denom_y = 0.0;
for (&a, &b) in x.iter().zip(y) {
let da = (a as f64) - mean_x;
let db = (b as f64) - mean_y;
num += da * db;
denom_x += da * da;
denom_y += db * db;
}
let denom = (denom_x * denom_y).sqrt();
if denom > 0.0 { num / denom } else { 0.0 }
}
fn find_ffmpeg() -> Option<String> {
for cmd in ["ffmpeg", "/opt/homebrew/bin/ffmpeg", "/usr/bin/ffmpeg"] {
if Command::new(cmd).arg("-version").output().is_ok() {
return Some(cmd.to_string());
}
}
None
}
#[test]
fn test_silence_roundtrip() {
let pcm = vec![0i16; 80];
for rate in [BitRate::Rate16k, BitRate::Rate24k, BitRate::Rate32k, BitRate::Rate40k] {
for packing in [BitPacking::BigEndian, BitPacking::LittleEndian] {
let enc = encode(&pcm, rate, packing).expect("encode");
let dec = decode(&enc, rate, packing).expect("decode");
assert_eq!(dec.len(), pcm.len());
for &s in &dec {
assert!(s.abs() <= 16, "silence sample too large: {s}");
}
}
}
}
#[test]
fn test_sine_wave_roundtrip_fidelity() {
let sample_rate = 8000.0;
let freq = 300.0;
let num_samples = 160;
let mut pcm = Vec::with_capacity(num_samples);
for i in 0..num_samples {
let t = i as f64 / sample_rate;
let s = (4000.0 * (2.0 * std::f64::consts::PI * freq * t).sin()) as i16;
pcm.push(s);
}
for rate in [BitRate::Rate16k, BitRate::Rate24k, BitRate::Rate32k, BitRate::Rate40k] {
for packing in [BitPacking::BigEndian, BitPacking::LittleEndian] {
let enc = encode(&pcm, rate, packing).expect("encode sine");
let dec = decode(&enc, rate, packing).expect("decode sine");
let corr = pearson_correlation(&pcm, &dec);
assert!(corr > 0.90, "correlation {corr} too low for rate {rate:?} packing {packing:?}");
}
}
}
#[test]
fn test_encoder_and_decoder_reset() {
let mut enc = Encoder::new(BitRate::Rate32k, BitPacking::BigEndian);
let mut dec = Decoder::new(BitRate::Rate32k, BitPacking::BigEndian);
let pcm = vec![1000i16, 2000, 3000, 2000, 1000, 0, -1000, -2000];
let mut out1 = Vec::new();
enc.encode(&pcm, &mut out1).expect("enc 1");
enc.flush(&mut out1).expect("flush 1");
let mut dec_pcm1 = Vec::new();
dec.decode(&out1, &mut dec_pcm1).expect("dec 1");
enc.reset();
dec.reset();
let mut out2 = Vec::new();
enc.encode(&pcm, &mut out2).expect("enc 2");
enc.flush(&mut out2).expect("flush 2");
let mut dec_pcm2 = Vec::new();
dec.decode(&out2, &mut dec_pcm2).expect("dec 2");
assert_eq!(out1, out2);
assert_eq!(dec_pcm1, dec_pcm2);
}
#[test]
fn test_stereo_multichannel_roundtrip() {
let mut stereo_pcm = Vec::new();
for i in 0..160 {
let left = (3000.0 * (i as f64 * 0.05).sin()) as i16;
let right = (2000.0 * (i as f64 * 0.03).cos()) as i16;
stereo_pcm.push(left);
stereo_pcm.push(right);
}
let enc = encode_interleaved(&stereo_pcm, 2, BitRate::Rate32k, BitPacking::BigEndian).expect("encode stereo");
let dec = decode_interleaved(&enc, 2, BitRate::Rate32k, BitPacking::BigEndian).expect("decode stereo");
assert_eq!(dec.len(), stereo_pcm.len());
let corr = pearson_correlation(&stereo_pcm, &dec);
assert!(corr > 0.90, "stereo correlation {corr} too low");
}
#[test]
fn test_hostile_inputs_do_not_crash() {
let mut dec = Decoder::new(BitRate::Rate32k, BitPacking::BigEndian);
let mut out = Vec::new();
let hostile = vec![0xEE; 1000];
let _ = dec.decode(&hostile, &mut out);
let mut enc = Encoder::new(BitRate::Rate40k, BitPacking::LittleEndian);
let mut out_enc = Vec::new();
let hostile_pcm = vec![i16::MIN, i16::MAX, -1, 1, 0];
let _ = enc.encode(&hostile_pcm, &mut out_enc);
let _ = enc.flush(&mut out_enc);
}
#[test]
fn test_ffmpeg_g726_oracle_encoder_and_decoder() {
let ff = match find_ffmpeg() {
Some(f) => f,
None => return,
};
let sample_rate = 8000;
let num_samples = 240;
let mut pcm = Vec::with_capacity(num_samples);
for i in 0..num_samples {
let t = i as f64 / sample_rate as f64;
let s = (5000.0 * (2.0 * std::f64::consts::PI * 350.0 * t).sin()) as i16;
pcm.push(s);
}
let dir = std::env::temp_dir().join("craft_g726_oracle");
let _ = std::fs::create_dir_all(&dir);
let in_pcm_path = dir.join("input.s16le");
let mut in_bytes = Vec::with_capacity(pcm.len() * 2);
for &s in &pcm {
in_bytes.extend_from_slice(&s.to_le_bytes());
}
std::fs::write(&in_pcm_path, &in_bytes).expect("write input pcm");
for rate in [BitRate::Rate16k, BitRate::Rate24k, BitRate::Rate32k, BitRate::Rate40k] {
for (packing, ff_codec, ff_fmt) in [(BitPacking::BigEndian, "g726", "g726"), (BitPacking::LittleEndian, "g726le", "g726le")] {
let cs = rate.code_size();
let ff_g726_path = dir.join(format!("ff_{}_{}.g726", cs, ff_codec));
// 1. FFmpeg encodes PCM to G.726
let out = Command::new(&ff)
.args(["-y", "-f", "s16le", "-ar", "8000", "-ac", "1", "-i"])
.arg(&in_pcm_path)
.args(["-c:a", ff_codec, "-code_size", &cs.to_string(), "-f", ff_fmt])
.arg(&ff_g726_path)
.output()
.expect("ffmpeg encode");
assert!(out.status.success(), "ffmpeg encode failed: {}", String::from_utf8_lossy(&out.stderr));
let ff_g726_bytes = std::fs::read(&ff_g726_path).expect("read ff g726");
// 2. FFmpeg decodes its own G.726 stream
let ff_pcm_back_path = dir.join(format!("ff_back_{}_{}.s16le", cs, ff_codec));
let out = Command::new(&ff)
.args(["-y", "-f", ff_fmt, "-code_size", &cs.to_string(), "-i"])
.arg(&ff_g726_path)
.args(["-f", "s16le", "-ar", "8000", "-ac", "1"])
.arg(&ff_pcm_back_path)
.output()
.expect("ffmpeg decode");
assert!(out.status.success(), "ffmpeg decode failed: {}", String::from_utf8_lossy(&out.stderr));
let ff_back_bytes = std::fs::read(&ff_pcm_back_path).expect("read ff back pcm");
let ff_back_samples: Vec<i16> = ff_back_bytes.as_chunks::<2>().0.iter().map(|c| i16::from_le_bytes(*c)).collect();
// 3. Our decoder decodes FFmpeg's stream
let our_decoded = decode(&ff_g726_bytes, rate, packing).expect("our decode");
let corr = pearson_correlation(&our_decoded, &ff_back_samples);
assert!(corr > 0.998, "correlation {corr} too low between our decoder and FFmpeg decoder for rate {rate:?} packing {packing:?}");
// 4. Our encoder encodes PCM to G.726
let our_g726_bytes = encode(&pcm, rate, packing).expect("our encode");
let our_g726_path = dir.join(format!("our_{}_{}.g726", cs, ff_codec));
std::fs::write(&our_g726_path, &our_g726_bytes).expect("write our g726");
// 5. FFmpeg decodes our encoded stream
let our_decoded_by_ff_path = dir.join(format!("our_by_ff_{}_{}.s16le", cs, ff_codec));
let out = Command::new(&ff)
.args(["-y", "-f", ff_fmt, "-code_size", &cs.to_string(), "-i"])
.arg(&our_g726_path)
.args(["-f", "s16le", "-ar", "8000", "-ac", "1"])
.arg(&our_decoded_by_ff_path)
.output()
.expect("ffmpeg decode of our stream");
assert!(out.status.success(), "ffmpeg decode of our stream failed: {}", String::from_utf8_lossy(&out.stderr));
let our_by_ff_bytes = std::fs::read(&our_decoded_by_ff_path).expect("read our by ff pcm");
let our_by_ff_samples: Vec<i16> = our_by_ff_bytes.as_chunks::<2>().0.iter().map(|c| i16::from_le_bytes(*c)).collect();
let corr_input = pearson_correlation(&pcm, &our_by_ff_samples);
assert!(
corr_input > 0.90,
"correlation {corr_input} too low for input vs FFmpeg-decoded our stream for rate {rate:?} packing {packing:?}"
);
}
}
}
+230
View File
@@ -0,0 +1,230 @@
//! Hostile input to the Vorbis decoder: damaged headers and packets, truncations, and headers that
//! ask for absurd sizes. Every case is an error or a result, never a panic or a large allocation.
use craft_audio::vorbis::decoder::{Decoder, Error, decode_ogg};
use craft_audio::{AudioBuffer, OggVorbisOptions, encode_ogg_vorbis};
fn xorshift(seed: &mut u32) -> u32 {
*seed ^= *seed << 13;
*seed ^= *seed >> 17;
*seed ^= *seed << 5;
*seed
}
struct Parts {
ogg: Vec<u8>,
headers: Vec<Vec<u8>>,
packets: Vec<Vec<u8>>,
}
fn parts(channels: usize) -> Parts {
let signal: Vec<Vec<f32>> = (0..channels)
.map(|c| (0..20_000).map(|i| ((i as f32 * (0.03 + 0.01 * c as f32)).sin() * 0.4) + ((i * 7 % 13) as f32 - 6.0) / 60.0).collect())
.collect();
let ogg =
encode_ogg_vorbis(&AudioBuffer { sample_rate: 44_100, channels: signal }, &OggVorbisOptions::default().with_quality(4.0)).expect("encode");
let file = craft_media::ogg::open(ogg.as_slice()).expect("open");
let s = &file.streams[0];
let packets = (0..s.packets.len()).map(|i| file.read_packet(ogg.as_slice(), 0, i).expect("packet")).collect();
Parts { headers: s.headers.clone(), packets, ogg }
}
fn decoder(p: &Parts) -> Decoder {
Decoder::new(&p.headers[0], &p.headers[1], &p.headers[2]).expect("valid headers")
}
#[test]
fn every_truncated_header_is_an_error() {
let p = parts(2);
for which in [0usize, 2] {
for n in 0..p.headers[which].len() {
let mut h = p.headers.clone();
h[which].truncate(n);
assert!(Decoder::new(&h[0], &h[1], &h[2]).is_err(), "header {which} cut to {n} bytes");
}
}
// A damaged comment header is not fatal.
let mut h = p.headers.clone();
h[1].truncate(12);
let d = Decoder::new(&h[0], &h[1], &h[2]).expect("comments are best effort");
assert!(d.comments().comments.is_empty());
assert!(Decoder::new(&p.headers[0], &[], &p.headers[2]).is_err(), "the comment header still needs its tag");
}
#[test]
fn damaged_headers_never_panic() {
let p = parts(2);
let mut seed = 99u32;
for which in [0usize, 2] {
let len = p.headers[which].len();
for _ in 0..800 {
let mut h = p.headers.clone();
for _ in 0..1 + xorshift(&mut seed) % 3 {
let at = xorshift(&mut seed) as usize % len;
h[which][at] = xorshift(&mut seed) as u8;
}
if let Ok(mut d) = Decoder::new(&h[0], &h[1], &h[2]) {
// A setup that survives must also survive audio packets.
for pk in p.packets.iter().take(6) {
let _ = d.decode_packet(pk);
}
}
}
}
}
#[test]
fn damaged_packets_never_panic() {
let p = parts(2);
let mut seed = 7u32;
let mut d = decoder(&p);
for pk in &p.packets {
for _ in 0..6 {
let mut bad = pk.clone();
match xorshift(&mut seed) % 3 {
0 => bad.truncate(xorshift(&mut seed) as usize % (bad.len() + 1)),
1 => {
let at = xorshift(&mut seed) as usize % bad.len();
bad[at] = xorshift(&mut seed) as u8;
}
_ => bad = (0..xorshift(&mut seed) % 300).map(|_| xorshift(&mut seed) as u8).collect(),
}
let _ = d.decode_packet(&bad);
}
}
// The decoder still decodes good packets after all that.
d.reset();
assert!(d.decode_packet(&p.packets[3]).is_ok());
}
#[test]
fn header_packets_and_empty_packets_are_not_audio() {
let p = parts(1);
let mut d = decoder(&p);
assert_eq!(d.decode_packet(&p.headers[0]), Err(Error::NotAudio));
assert_eq!(d.decode_packet(&p.headers[2]), Err(Error::NotAudio));
assert!(matches!(d.decode_packet(&[]), Err(Error::Packet(_))));
}
#[test]
fn damaged_files_never_panic() {
let p = parts(2);
let mut seed = 31u32;
for _ in 0..120 {
let mut bad = p.ogg.clone();
for _ in 0..1 + xorshift(&mut seed) % 4 {
let at = xorshift(&mut seed) as usize % bad.len();
bad[at] = xorshift(&mut seed) as u8;
}
let _ = decode_ogg(&bad);
bad.truncate(xorshift(&mut seed) as usize % bad.len());
let _ = decode_ogg(&bad);
}
assert!(matches!(decode_ogg(b"not an ogg file"), Err(Error::Ogg(_))));
assert!(matches!(decode_ogg(&[]), Err(Error::Ogg(_))));
}
/// A bit writer for hand-built setups.
#[derive(Default)]
struct W(Vec<u8>, usize);
impl W {
fn put(&mut self, v: u64, n: u32) {
for i in 0..n {
if self.1.is_multiple_of(8) {
self.0.push(0);
}
let last = self.0.len() - 1;
self.0[last] |= (((v >> i) & 1) as u8) << (self.1 % 8);
self.1 += 1;
}
}
}
fn setup_with_first_book(book: impl FnOnce(&mut W)) -> Vec<u8> {
let mut w = W::default();
w.put(5, 8);
for c in b"vorbis" {
w.put(u64::from(*c), 8);
}
w.put(0, 8); // one book
book(&mut w);
w.0
}
#[test]
fn absurd_codebook_sizes_are_refused_before_allocating() {
let p = parts(1);
let try_setup = |setup: Vec<u8>| Decoder::new(&p.headers[0], &p.headers[1], &setup);
// 2^24 - 1 entries.
assert!(
try_setup(setup_with_first_book(|w| {
w.put(0x56_4342, 24);
w.put(1, 16);
w.put(0xFF_FFFF, 24);
w.put(1, 1); // ordered
w.put(0, 5);
}))
.is_err()
);
// An ordered book whose lengths overrun its entries.
assert!(
try_setup(setup_with_first_book(|w| {
w.put(0x56_4342, 24);
w.put(1, 16);
w.put(8, 24);
w.put(1, 1);
w.put(0, 5);
w.put(0xFF, 4); // asks for more codewords of length 1 than there are entries
}))
.is_err()
);
// A lookup table of 65535 dimensions over a million entries: 69 billion values.
assert!(
try_setup(setup_with_first_book(|w| {
w.put(0x56_4342, 24);
w.put(0xFFFF, 16);
w.put(1 << 20, 24);
w.put(1, 1); // ordered
w.put(19, 5); // length 20
w.put(1 << 20, 21); // all the entries
w.put(2, 4); // lookup type 2
w.put(0, 32);
w.put(0, 32);
w.put(0, 4);
w.put(0, 1);
}))
.is_err()
);
// A million-entry book with a legitimate tree but a lookup table that is not in the packet.
assert!(
try_setup(setup_with_first_book(|w| {
w.put(0x56_4342, 24);
w.put(4, 16);
w.put(1 << 20, 24);
w.put(1, 1);
w.put(19, 5);
w.put(1 << 20, 21);
w.put(1, 4); // lookup type 1
w.put(0, 32);
w.put(0, 32);
w.put(15, 4);
w.put(0, 1);
}))
.is_err()
);
}
#[test]
fn identification_header_limits() {
let p = parts(1);
let mut h = p.headers[0].clone();
let setup = &p.headers[2];
h[11] = 0; // no channels
assert!(matches!(Decoder::new(&h, &p.headers[1], setup), Err(Error::Header(_))));
let mut h = p.headers[0].clone();
h[28] = 0x95; // a short block size of 2^5 is below the 64 Vorbis allows
assert!(matches!(Decoder::new(&h, &p.headers[1], setup), Err(Error::Header(_))));
let mut h = p.headers[0].clone();
h[7] = 1; // version 1
assert!(matches!(Decoder::new(&h, &p.headers[1], setup), Err(Error::Header(_))));
}
+216
View File
@@ -0,0 +1,216 @@
//! The Vorbis decoder against ffmpeg's as the reference (oracle; skipped when ffmpeg is absent, a
//! failure with `CRAFT_MEDIA_REQUIRE_ORACLES=1`). Streams come from two encoders, this crate's and
//! ffmpeg's own, so the setups differ: different floors, residues, block sizes, channel counts and
//! coupling. Both decoders must return the same samples.
use craft_audio::AudioBuffer;
use craft_audio::vorbis::decoder::{Decoder, decode_ogg};
use craft_audio::{OggVorbisOptions, encode_ogg_vorbis};
use std::path::Path;
use std::process::Command;
fn xorshift(seed: &mut u32) -> u32 {
*seed ^= *seed << 13;
*seed ^= *seed >> 17;
*seed ^= *seed << 5;
*seed
}
/// A signal with tones, noise bursts, clicks and silence, different in every channel.
fn signal(rate: u32, channels: usize, frames: usize) -> Vec<Vec<f32>> {
let mut seed = 0x1234_5678u32;
(0..channels)
.map(|c| {
(0..frames)
.map(|i| {
let t = i as f64 / f64::from(rate);
let tone =
(t * (220.0 + 110.0 * c as f64) * std::f64::consts::TAU).sin() * 0.3 + (t * 3100.0 * std::f64::consts::TAU).sin() * 0.05;
let noise = (f64::from(xorshift(&mut seed) % 2000) - 1000.0) / 6000.0;
let s = match (i / (rate as usize / 4)) % 6 {
0 | 1 => tone,
2 => noise,
3 => 0.0,
4 => tone + if i % 3000 < 8 { 0.6 } else { 0.0 },
_ => tone * (t * 4.0).sin(),
};
s as f32
})
.collect()
})
.collect()
}
fn ffmpeg_decode(ffmpeg: &Path, ogg: &Path, channels: usize) -> Vec<Vec<f32>> {
let out = Command::new(ffmpeg)
.args(["-v", "error", "-nostdin", "-i"])
.arg(ogg)
.args(["-f", "f32le", "-acodec", "pcm_f32le", "-"])
.output()
.expect("run ffmpeg");
assert!(out.status.success(), "ffmpeg failed on {}: {}", ogg.display(), String::from_utf8_lossy(&out.stderr));
let all: Vec<f32> = out.stdout.as_chunks::<4>().0.iter().map(|b| f32::from_le_bytes(*b)).collect();
(0..channels).map(|c| all.iter().skip(c).step_by(channels).copied().collect()).collect()
}
/// Each of our channels matches a different one of theirs (ffmpeg reorders the channels of surround
/// streams), to within float rounding.
fn assert_same(name: &str, ours: &[Vec<f32>], theirs: &[Vec<f32>]) {
assert_prefix(name, ours, theirs, true);
}
/// As [`assert_same`], but with `exact` unset ffmpeg may return more samples than we do (it does
/// not always trim a stream to its last granule position); the common part must agree.
fn assert_prefix(name: &str, ours: &[Vec<f32>], theirs: &[Vec<f32>], exact: bool) {
assert_eq!(ours.len(), theirs.len(), "{name}: channel count");
assert!(ours.iter().chain(theirs).flatten().all(|v| v.is_finite()), "{name}: non-finite oracle input or output");
let mut used = vec![false; theirs.len()];
for (c, o) in ours.iter().enumerate() {
let mut best: Option<(usize, f32)> = None;
for (k, t) in theirs.iter().enumerate() {
if used[k] {
continue;
}
if exact {
assert_eq!(o.len(), t.len(), "{name}: channel {c} length {} vs ffmpeg {}", o.len(), t.len());
} else {
assert!(o.len() <= t.len(), "{name}: channel {c} length {} vs ffmpeg {}", o.len(), t.len());
}
let worst = o.iter().zip(t).map(|(a, b)| (a - b).abs()).fold(0f32, f32::max);
if best.is_none_or(|(_, w)| worst < w) {
best = Some((k, worst));
}
}
let (k, worst) = best.expect("a free channel");
used[k] = true;
assert!(worst < 2e-4, "{name}: channel {c} differs from ffmpeg's channel {k} by up to {worst}");
}
}
#[test]
fn ffmpeg_decodes_our_encoders_streams_to_the_same_samples() {
let Some(ffmpeg) = craft_media_testkit::ffmpeg() else { return };
let dir = craft_media_testkit::fixture_subdir("vorbis_decode");
for (rate, channels, quality) in
[(44_100u32, 2usize, 6.0f32), (44_100, 1, 3.0), (48_000, 2, -1.0), (22_050, 1, 10.0), (44_100, 6, 5.0), (32_000, 8, 4.0)]
{
let buf = AudioBuffer { sample_rate: rate, channels: signal(rate, channels, rate as usize * 2 + 777) };
let ogg = encode_ogg_vorbis(&buf, &OggVorbisOptions::default().with_quality(quality)).expect("encode");
let file = dir.join(format!("ours_{rate}_{channels}_{quality}.ogg"));
std::fs::write(&file, &ogg).expect("write fixture");
let decoded = decode_ogg(&ogg).expect("decode");
assert_eq!(decoded.damaged_packets, 0);
assert_eq!(decoded.sample_rate, rate);
assert_same(&format!("our stream {rate} Hz {channels} ch q{quality}"), &decoded.channels, &ffmpeg_decode(&ffmpeg, &file, channels));
}
}
/// ffmpeg's own Vorbis encoder (stereo only) writes setups ours does not: coupled channels, residue
/// type 2, floor classes with subclasses and master books, two block sizes. The reference is
/// Symphonia, an independent decoder. ffmpeg's decoder agrees on the first channel but not on the
/// second of these streams, where it differs from both Symphonia and this decoder; the stream does
/// not show which reading it means (the channel comes back poorly, about 3 dB SNR, in all three),
/// so ffmpeg is compared on the first channel only. The setups ffmpeg's decoder does read the same
/// way as we do are covered, on both channels, in `vorbis_decode_synthetic.rs`.
#[test]
fn we_decode_ffmpegs_encoder_streams_like_symphonia() {
let Some(ffmpeg) = craft_media_testkit::ffmpeg() else { return };
let dir = craft_media_testkit::fixture_subdir("vorbis_decode");
for (rate, bitrate) in [(44_100u32, "128k"), (48_000, "96k"), (22_050, "48k"), (32_000, "64k")] {
let channels = 2usize;
let pcm: Vec<f32> = {
let s = signal(rate, channels, rate as usize * 2 + 301);
(0..s[0].len()).flat_map(|i| s.iter().map(move |c| c[i])).collect()
};
let raw = dir.join(format!("src_{rate}_{channels}.f32"));
std::fs::write(&raw, pcm.iter().flat_map(|v| v.to_le_bytes()).collect::<Vec<u8>>()).expect("write source");
let file = dir.join(format!("ffmpeg_{rate}_{channels}_{bitrate}.ogg"));
let out = Command::new(&ffmpeg)
.args(["-v", "error", "-nostdin", "-y", "-f", "f32le", "-ar"])
.arg(rate.to_string())
.arg("-ac")
.arg(channels.to_string())
.arg("-i")
.arg(&raw)
.args(["-c:a", "vorbis", "-strict", "-2", "-b:a", bitrate])
.arg(&file)
.output()
.expect("run ffmpeg");
assert!(out.status.success(), "ffmpeg could not encode {}: {}", file.display(), String::from_utf8_lossy(&out.stderr));
let ogg = std::fs::read(&file).expect("read");
let ours = decode_ogg(&ogg).expect("decode");
assert_eq!(ours.damaged_packets, 0, "{}", file.display());
let (_, symphonia) = craft_audio::decode(&ogg, Some("ogg")).expect("symphonia");
let name = format!("ffmpeg-encoded {rate} Hz {bitrate}");
assert_prefix(&name, &ours.channels, &symphonia.channels, false);
// The first channel also agrees with ffmpeg's decoder.
let theirs = ffmpeg_decode(&ffmpeg, &file, channels);
assert_prefix(&format!("{name} (channel 0 against ffmpeg)"), &ours.channels[..1], &theirs[..1], true);
}
}
#[test]
fn packets_decode_one_at_a_time_to_the_file_result() {
let buf = AudioBuffer { sample_rate: 44_100, channels: signal(44_100, 2, 30_000) };
let ogg = encode_ogg_vorbis(&buf, &OggVorbisOptions::default().with_quality(5.0)).expect("encode");
let file = craft_media::ogg::open(ogg.as_slice()).expect("open");
let s = &file.streams[file.stream_of(craft_media::ogg::Codec::Vorbis).expect("vorbis")];
let mut dec = Decoder::new(&s.headers[0], &s.headers[1], &s.headers[2]).expect("decoder");
assert_eq!((dec.identification().channels, dec.identification().sample_rate), (2, 44_100));
let mut got: Vec<Vec<f32>> = vec![Vec::new(); 2];
for i in 0..s.packets.len() {
let p = file.read_packet(ogg.as_slice(), 0, i).expect("packet");
for (a, b) in got.iter_mut().zip(dec.decode_packet(&p).expect("packet decodes")) {
a.extend(b);
}
}
let whole = decode_ogg(&ogg).expect("file");
for (a, b) in got.iter().zip(&whole.channels) {
assert!(a.len() >= b.len() && a[..b.len()] == b[..]);
}
// After a reset the next packet only primes the overlap again.
dec.reset();
let p = file.read_packet(ogg.as_slice(), 0, 5).expect("packet");
assert!(dec.decode_packet(&p).expect("decodes").iter().all(Vec::is_empty));
}
#[test]
fn the_round_trip_through_our_encoder_is_faithful() {
let src = signal(44_100, 2, 44_100);
let buf = AudioBuffer { sample_rate: 44_100, channels: src.clone() };
let ogg = encode_ogg_vorbis(&buf, &OggVorbisOptions::default().with_quality(8.0)).expect("encode");
let out = decode_ogg(&ogg).expect("decode");
assert_eq!(out.channels[0].len(), src[0].len());
let (mut sig, mut err) = (0f64, 0f64);
for (a, b) in src[0].iter().zip(&out.channels[0]) {
sig += f64::from(*a) * f64::from(*a);
err += f64::from(a - b) * f64::from(a - b);
}
assert!(10.0 * (sig / err).log10() > 15.0, "SNR {:.1} dB", 10.0 * (sig / err).log10());
}
/// A stream from libvorbis itself (the repository's `tone.ogg` fixture, made with ffmpeg's libvorbis
/// wrapper): mono, 8 kHz, the lowest quality.
#[test]
fn a_libvorbis_stream_decodes_like_ffmpeg() {
let path = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/tone.ogg");
let ogg = std::fs::read(&path).expect("fixture");
let ours = decode_ogg(&ogg).expect("decode");
assert_eq!((ours.sample_rate, ours.channels.len(), ours.damaged_packets), (8000, 1, 0));
assert!(ours.comments.vendor.contains("Lavf") || ours.comments.vendor.contains("libVorbis"), "vendor {:?}", ours.comments.vendor);
assert!(ours.channels[0].iter().any(|s| s.abs() > 0.1), "the fixture is a tone");
let Some(ffmpeg) = craft_media_testkit::ffmpeg() else { return };
assert_prefix("libvorbis tone.ogg", &ours.channels, &ffmpeg_decode(&ffmpeg, &path, 1), false);
}
#[test]
#[should_panic(expected = "non-finite oracle input or output")]
fn oracle_comparison_rejects_nan_output() {
assert_same("NaN output", &[vec![f32::NAN]], &[vec![0.0]]);
}
#[test]
#[should_panic(expected = "non-finite oracle input or output")]
fn oracle_comparison_rejects_nan_reference() {
assert_same("NaN reference", &[vec![0.0]], &[vec![f32::NAN]]);
}
@@ -0,0 +1,384 @@
//! Differential tests on streams built by hand: a setup header written here, with floor 0 and
//! floor 1 side by side, residues 0, 1 and 2, several submaps and channel coupling, followed by
//! random audio packets. Every packet of random bits is decodable, so the two decoders (ours and
//! ffmpeg's, the oracle; skipped when ffmpeg is absent, a failure with
//! `CRAFT_MEDIA_REQUIRE_ORACLES=1`) must return the same samples, and no encoder's habits narrow
//! what is exercised. Floor 0, residue 0 and multi-submap mappings are the parts no encoder here
//! writes.
use craft_audio::vorbis::decoder::{Decoder, decode_ogg};
use craft_media::ogg::{PageWriter, vorbis_comment};
use std::process::Command;
fn xorshift(seed: &mut u32) -> u32 {
*seed ^= *seed << 13;
*seed ^= *seed >> 17;
*seed ^= *seed << 5;
*seed
}
/// A least-significant-bit-first bit writer.
#[derive(Default)]
struct W {
bytes: Vec<u8>,
bits: usize,
}
impl W {
fn put(&mut self, v: u64, n: u32) {
for i in 0..n {
if self.bits.is_multiple_of(8) {
self.bytes.push(0);
}
let last = self.bytes.len() - 1;
self.bytes[last] |= (((v >> i) & 1) as u8) << (self.bits % 8);
self.bits += 1;
}
}
fn flag(&mut self, b: bool) {
self.put(u64::from(b), 1);
}
}
fn float_bits(v: f64) -> u64 {
// The Vorbis float32 packing: sign, 10-bit exponent biased by 788, 21-bit mantissa.
if v == 0.0 {
return 0;
}
let (sign, a) = if v < 0.0 { (1u64 << 31, -v) } else { (0, v) };
let mut exp = 0i32;
let mut m = a;
while m >= (1 << 21) as f64 {
m /= 2.0;
exp += 1;
}
while m < (1 << 20) as f64 {
m *= 2.0;
exp -= 1;
}
sign | (((exp + 788) as u64) << 21) | (m.round() as u64 & 0x1F_FFFF)
}
/// A complete codebook: `entries` codewords all of length `len` (`entries == 2^len`), optionally
/// with a lookup table: `(type, minimum, delta, value_bits, multiplicands)`.
fn book(w: &mut W, dims: u32, entries: u32, len: u32, lookup: Option<(u32, f64, f64, u32, Vec<u32>)>) {
w.put(0x56_4342, 24);
w.put(u64::from(dims), 16);
w.put(u64::from(entries), 24);
w.flag(false); // not ordered
w.flag(false); // not sparse
for _ in 0..entries {
w.put(u64::from(len - 1), 5);
}
match lookup {
None => w.put(0, 4),
Some((ty, min, delta, bits, mult)) => {
w.put(u64::from(ty), 4);
w.put(float_bits(min), 32);
w.put(float_bits(delta), 32);
w.put(u64::from(bits - 1), 4);
w.flag(ty == 1); // sequence_p: cumulative for the floor 0 book
for m in mult {
w.put(u64::from(m), bits);
}
}
}
}
struct Layout {
channels: u8,
/// The residue type of each of the two residues.
residue_types: [u32; 2],
/// The submap of each channel (0 or 1): submap 0 uses floor 0 and residue 0, submap 1 uses
/// floor 1 and residue 1.
mux: Vec<u8>,
coupling: Vec<(u8, u8)>,
}
fn identification(channels: u8) -> Vec<u8> {
let mut w = W::default();
w.put(1, 8);
for c in b"vorbis" {
w.put(u64::from(*c), 8);
}
w.put(0, 32);
w.put(u64::from(channels), 8);
w.put(16_000, 32);
w.put(0, 32);
w.put(0, 32);
w.put(0, 32);
w.put(8, 4); // short blocks: 256
w.put(10, 4); // long blocks: 1024
w.flag(true);
w.bytes
}
fn setup(l: &Layout) -> Vec<u8> {
let mut w = W::default();
w.put(5, 8);
for c in b"vorbis" {
w.put(u64::from(*c), 8);
}
// Books: 0 floor 0 vectors, 1 class book, 2 and 3 residue vectors, 4 floor 1 values.
w.put(4, 8); // five books
book(&mut w, 4, 16, 4, Some((1, 0.7, 0.2, 1, vec![0, 1]))); // lookup1_values(16, 4) = 2
book(&mut w, 1, 4, 2, None);
book(&mut w, 2, 16, 4, Some((1, -0.4, 0.25, 2, vec![0, 1, 2, 3]))); // lookup1_values(16, 2) = 4
book(&mut w, 4, 16, 4, Some((1, -0.3, 0.6, 1, vec![0, 1])));
book(&mut w, 1, 128, 7, None);
// One time domain placeholder.
w.put(0, 6);
w.put(0, 16);
// Floors: 0 and 1.
w.put(1, 6); // two floors
w.put(0, 16);
w.put(8, 8); // order
w.put(16_000, 16); // rate
w.put(64, 16); // bark map size
w.put(8, 6); // amplitude bits
w.put(2, 8); // amplitude offset
w.put(0, 4); // one book
w.put(0, 8);
w.put(1, 16);
w.put(1, 5); // one partition
w.put(0, 4); // class 0
w.put(2, 3); // dimension 3
w.put(0, 2); // no subclasses
w.put(4 + 1, 8); // subclass book: book 4 (stored plus one)
w.put(1, 2); // multiplier 2
w.put(8, 4); // rangebits
for x in [10, 40, 130] {
w.put(x, 8);
}
// Residues.
w.put(1, 6); // two residues
for ty in l.residue_types {
w.put(u64::from(ty), 16);
w.put(0, 24); // begin
w.put(if ty == 2 { 96 } else { 48 }, 24); // end
w.put(7, 24); // partition size 8
w.put(3, 6); // four classifications
w.put(1, 8); // class book
for _ in 0..4 {
w.put(0b011, 3); // passes 0 and 1
w.flag(false);
}
for class in 0..4 {
w.put(2 + (class % 2), 8); // pass 0 uses book 2 or 3
w.put(3 - (class % 2), 8); // pass 1 uses book 3 or 2
}
}
// One mapping, two submaps.
w.put(0, 6);
w.put(0, 16);
w.flag(true);
w.put(1, 4); // two submaps
w.flag(!l.coupling.is_empty());
if !l.coupling.is_empty() {
w.put(l.coupling.len() as u64 - 1, 8);
let bits = if l.channels <= 1 { 0 } else { 8 - (l.channels - 1).leading_zeros() };
for &(m, a) in &l.coupling {
w.put(u64::from(m), bits);
w.put(u64::from(a), bits);
}
}
w.put(0, 2);
for &m in &l.mux {
w.put(u64::from(m), 4);
}
for sub in 0..2u64 {
w.put(0, 8);
w.put(sub, 8); // floor
w.put(sub, 8); // residue
}
// Two modes over the one mapping.
w.put(1, 6);
for long in [false, true] {
w.flag(long);
w.put(0, 16);
w.put(0, 16);
w.put(0, 8);
}
w.flag(true);
w.bytes
}
fn stream(l: &Layout, packets: usize, seed: &mut u32) -> Vec<u8> {
let mut pages = PageWriter::new(7);
let mut out = pages.header(&identification(l.channels));
out.extend(pages.header(&vorbis_comment("synthetic", &[("TITLE", "hand built")])));
out.extend(pages.header(&setup(l)));
// The window flags of a long block say whether its neighbours are long, so choose the block
// sizes first.
let longs: Vec<bool> = (0..packets).map(|_| !xorshift(seed).is_multiple_of(3)).collect();
let mut granule = 0i64;
for i in 0..packets {
let long = longs[i];
// Packet type 0, the mode bit, and for a long block the two neighbour flags.
let prev = i == 0 || longs[i - 1];
let next = i + 1 >= packets || longs[i + 1];
let mut w = W::default();
w.flag(false);
w.flag(long);
if long {
w.flag(prev);
w.flag(next);
}
// Floor 0 needs a codebook number that exists, which random bits would not give, so the
// channels that use it (the first ones) are written properly: an amplitude, book 0 and two
// codewords of its vectors (a full tree of 4-bit codes: the code is the entry number).
for _ in 0..l.mux.iter().take_while(|&&m| m == 0).count() {
w.put(1 + u64::from(xorshift(seed)) % 24, 8);
w.put(0, 1);
for _ in 0..2 {
let entry = xorshift(seed) % 16;
for bit in (0..4).rev() {
w.put(u64::from(entry >> bit) & 1, 1);
}
}
}
// The rest is random bits, enough that no residue runs out of packet.
let mut p = w.bytes;
let fixed = p.len();
p.extend((0..200 + xorshift(seed) as usize % 200).map(|_| xorshift(seed) as u8));
let _ = fixed;
// The bits after the fixed fields start mid-byte: the last fixed byte keeps its own bits and
// takes random ones above them.
let used = w.bits % 8;
if used != 0 {
let last = fixed - 1;
p[last] |= (xorshift(seed) as u8) << used;
}
if i > 0 {
let size = |long: bool| if long { 1024 } else { 256 };
granule += (size(longs[i - 1]) / 4 + size(long) / 4) as i64;
}
out.extend(pages.packet(&p, granule));
}
out.extend(pages.finish(None));
out
}
/// `tolerance`: the largest difference from ffmpeg, as a fraction of the peak output. A floor 0
/// curve is an exponential of a quotient that random coefficients can make very large, so where
/// two floors meet in one stream (ffmpeg computes in single precision, we in double) the results
/// agree only to about a thousandth of the peak.
fn run(name: &str, l: &Layout, seed: u32, tolerance: f32) {
let mut s = seed;
let ogg = stream(l, 40, &mut s);
// Packet by packet our decoder must accept every one.
let file = craft_media::ogg::open(ogg.as_slice()).expect("open");
let st = &file.streams[0];
let mut dec = Decoder::new(&st.headers[0], &st.headers[1], &st.headers[2]).expect("setup decodes");
let mut total = vec![Vec::new(); usize::from(l.channels)];
for i in 0..st.packets.len() {
let p = file.read_packet(ogg.as_slice(), 0, i).expect("packet");
let pcm = dec.decode_packet(&p).unwrap_or_else(|e| panic!("{name}: packet {i}: {e}"));
for (t, c) in total.iter_mut().zip(pcm) {
t.extend(c);
}
}
let expected = usize::try_from(st.last_granule.expect("generated final granule")).expect("nonnegative granule");
for channel in &total {
assert_eq!(channel.len(), expected, "{name}: decoded sample count");
assert!(channel.iter().all(|v| v.is_finite()), "{name}: non-finite output");
}
let whole = decode_ogg(&ogg).expect("whole file");
assert_eq!(whole.channels, total, "{name}: packet and file decoding differ");
let Some(ffmpeg) = craft_media_testkit::ffmpeg() else { return };
let dir = craft_media_testkit::fixture_subdir("vorbis_synthetic");
let path = dir.join(format!("{name}.ogg"));
std::fs::write(&path, &ogg).expect("write fixture");
let out = Command::new(&ffmpeg)
.args(["-v", "error", "-nostdin", "-i"])
.arg(&path)
.args(["-f", "f32le", "-acodec", "pcm_f32le", "-"])
.output()
.expect("run ffmpeg");
assert!(out.status.success(), "{name}: ffmpeg failed: {}", String::from_utf8_lossy(&out.stderr));
let all: Vec<f32> = out.stdout.as_chunks::<4>().0.iter().map(|b| f32::from_le_bytes(*b)).collect();
let ch = usize::from(l.channels);
let theirs: Vec<Vec<f32>> = (0..ch).map(|c| all.iter().skip(c).step_by(ch).copied().collect()).collect();
assert!(theirs.iter().flatten().all(|v| v.is_finite()), "{name}: non-finite ffmpeg output");
for channel in &theirs {
assert!(channel.len() >= expected, "{name}: ffmpeg returned fewer than the expected samples");
}
// ffmpeg puts the channels of surround streams in its own order, so each of ours is matched
// with the channel of theirs it agrees with best, and every one of theirs is used once.
let mut used = vec![false; ch];
#[allow(clippy::needless_range_loop)]
for c in 0..ch {
let mut best: Option<(usize, f32, f32)> = None;
for (k, t) in theirs.iter().enumerate().filter(|(k, _)| !used[*k]) {
let n = total[c].len().min(t.len());
assert!(n > 1000, "{name}: only {n} samples to compare");
let scale = total[c][..n].iter().fold(1e-3f32, |m, v| m.max(v.abs()));
let worst = (0..n).map(|i| (total[c][i] - t[i]).abs()).fold(0f32, f32::max);
if best.is_none_or(|(_, w, sc)| worst / scale < w / sc) {
best = Some((k, worst, scale));
}
}
let (k, worst, scale) = best.expect("a free channel");
used[k] = true;
assert!(worst <= tolerance * scale + 1e-5, "{name}: channel {c} differs from ffmpeg's channel {k} by {worst} (peak {scale})");
}
}
#[test]
fn mono_floor_zero_and_residue_zero() {
run("mono_f0_r0", &Layout { channels: 1, residue_types: [0, 0], mux: vec![0], coupling: vec![] }, 11, 1e-4);
}
#[test]
fn mono_floor_one_and_residue_one() {
run("mono_f1_r1", &Layout { channels: 1, residue_types: [1, 1], mux: vec![1], coupling: vec![] }, 12, 1e-4);
}
#[test]
fn two_submaps_floor_zero_and_one() {
run("stereo_mixed", &Layout { channels: 2, residue_types: [0, 1], mux: vec![0, 1], coupling: vec![] }, 13, 3e-3);
}
#[test]
fn coupled_stereo_residue_two() {
run("stereo_coupled_r2", &Layout { channels: 2, residue_types: [2, 2], mux: vec![1, 1], coupling: vec![(0, 1)] }, 14, 1e-4);
}
#[test]
fn six_channels_with_coupling_and_submaps() {
let l = Layout { channels: 6, residue_types: [1, 2], mux: vec![0, 0, 0, 1, 1, 1], coupling: vec![(0, 3), (1, 2), (4, 5)] };
run("six", &l, 15, 3e-3);
}
#[test]
fn mono_floor_one_and_residue_zero() {
run("mono_f1_r0", &Layout { channels: 1, residue_types: [0, 0], mux: vec![1], coupling: vec![] }, 16, 1e-4);
}
#[test]
fn mono_floor_zero_and_residue_one() {
run("mono_f0_r1", &Layout { channels: 1, residue_types: [1, 1], mux: vec![0], coupling: vec![] }, 17, 1e-4);
}
/// A packet that names a floor 0 codebook that does not exist cannot be decoded (§6.2.2); the
/// decoder says so and the stream goes on with the next packet.
#[test]
fn a_floor_zero_codebook_that_does_not_exist_is_a_packet_error() {
let l = Layout { channels: 1, residue_types: [1, 1], mux: vec![0], coupling: vec![] };
let mut seed = 5u32;
let ogg = stream(&l, 6, &mut seed);
let file = craft_media::ogg::open(ogg.as_slice()).expect("open");
let st = &file.streams[0];
let mut dec = Decoder::new(&st.headers[0], &st.headers[1], &st.headers[2]).expect("decoder");
let good = file.read_packet(ogg.as_slice(), 0, 0).expect("packet");
assert!(dec.decode_packet(&good).is_ok());
// A short block: type 0, mode 0, then the 8 amplitude bits, then book number 1 (only 0 exists).
let mut w = W::default();
w.flag(false);
w.flag(false);
w.put(9, 8);
w.put(1, 1);
w.put(0, 64);
assert!(matches!(dec.decode_packet(&w.bytes), Err(craft_audio::vorbis::decoder::Error::Packet(_))));
assert!(dec.decode_packet(&good).is_ok(), "the decoder carries on");
}
+25
View File
@@ -0,0 +1,25 @@
[package]
name = "craft-cloud-local"
version.workspace = true
edition.workspace = true
license.workspace = true
repository.workspace = true
rust-version.workspace = true
description = "A reference CloudBackend over a local folder, proving the contract needs no specific provider"
[lints]
workspace = true
[dependencies]
# Resolution-time confinement: every path this crate resolves goes through a `Dir` capability
# opened on the configured root, never through an absolute OS path this crate builds itself.
# `craft-font-store` already depends on the lower-level `cap-primitives` for the same family of
# guarantee; this crate uses the ergonomic `cap-std` built on top of it instead of hand-rolling
# the equivalent `Dir`-style API again.
cap-std = "4"
craft-cloud = { workspace = true }
craft-ui = { workspace = true, features = ["cloud"] }
[dev-dependencies]
tempfile = "3"
egui = { workspace = true }
+39
View File
@@ -0,0 +1,39 @@
# craft-cloud-local
A reference `CloudBackend` over one folder on disk, proving that `craft-cloud`'s contract needs
no specific provider behind it.
## No-escape policy
Every path this crate is handed, a `Job::List`/`Read`/`Write` path or a `Job::Create` name, is
confined to the configured root before anything touches a filesystem, in two layers.
A path's segments are accepted only when each one parses as a single `Component::Normal`: no
`..`, no `.`, no empty segment, and no absolute form. A `Create` name goes through a stricter
check of its own, since it is one chosen name and not a path: it additionally refuses a
backslash, a colon, or a Windows reserved device name (`CON`, `COM1`, `CONIN$`, and so on), on
every platform this crate builds for, not only on Windows, because a chosen name reaches this
backend from whatever platform the UI runs on. An existing name that already has one of those
characters, discovered through `list`, stays reachable: `:` and `\` are not path separators on
any platform's filesystem, so refusing to read a real file back because of a byte in its name
(macOS stores a `/` typed in Finder as `:` on disk) would be a usability bug with no matching
security benefit.
The second layer is the one that actually keeps resolution inside the root: every already
segment-checked relative path is handed to a [`cap-std`](https://docs.rs/cap-std) `Dir`
capability opened on the root, never joined onto the root as an absolute OS path this crate
builds itself. `cap-std` resolves one path component at a time against real, already-open
directory handles, so a symlink or (on Windows) a directory junction anywhere under the root,
dangling or already pointing outside, cannot walk resolution above the root. There is no separate
"does this still look like it is inside the root" check with a window for the directory tree to
change underneath it between the check and the actual read or write; the one resolution is the
open.
## Hard links
A hard link is not a symlink, and this backend does not refuse one: from inside the sandbox the
inode it names is an ordinary file. What it protects instead is the shared inode's content: a
write never truncates its target in place. It writes a new sibling file and renames that sibling
over the target, so the target's directory entry is replaced rather than its bytes being
overwritten. A rename only ever changes one name; it does not reach through to any other name,
inside the root or out, that happens to point at the same inode.
+510
View File
@@ -0,0 +1,510 @@
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
//! A reference [`CloudBackend`] that treats one folder on disk as the cloud.
//!
//! [`craft_ui::cloud::CloudBackend`] names a contract and nothing about who
//! answers it. This crate is the proof: a working implementation that reaches
//! no service, a vendor, or a personal repo anybody else has to trust. An app
//! wires [`LocalBackend`] the same way it would wire a provider's own client,
//! by calling [`craft_ui::cloud::CloudFiles::connect`]. Swapping one for the
//! other is the whole point; nothing above the [`CloudBackend`] seam can tell
//! the difference.
//!
//! What this is not: a sync engine, a sharing service, or a place to keep a
//! credential. A folder has no concept of another account, so
//! [`Job::SharedWithMe`] answers empty and [`Job::Conversation`] answers with
//! a [`Fault`], and [`Account::capabilities`] claims neither. Reporting a
//! capability this backend cannot honour would be the broken window the
//! contract's own docs warn about.
//!
//! Every job answers on its own [`std::thread`]. [`CloudBackend::submit`]
//! must return before the file is opened, let alone read, or a twenty
//! megabyte document would freeze the window that called it.
//!
//! # The no-escape policy
//!
//! `Entry::id` and `Entry::path` are the same slash-separated path relative to
//! `root`, always starting with `/` below the root itself (the root's own
//! path is the empty string, matching [`Job::List`]'s contract), and a
//! [`Job::Create`] name is one plain file name, not a path at all. Neither is
//! an OS path string handed to `std::fs`. Confinement happens in two layers:
//!
//! 1. [`confine_segment`] (every path segment) and [`confine_name`] (a
//! `Create` name, stricter) reject `..`, `.`, an empty segment, an
//! absolute form, and anything that is not exactly one
//! [`std::path::Component::Normal`]. `confine_name` additionally refuses a
//! backslash, a colon, and a Windows reserved device name
//! (`CON`, `COM1`, `CONIN$`, ...) on every platform this crate builds for,
//! because a chosen name reaches this backend from whatever platform the
//! UI runs on. Browsing an *existing* name is more lenient: a literal `:`
//! or `\` already on disk (macOS stores a `/` typed in Finder as `:`) is
//! not itself a path separator on any platform's filesystem, so
//! `confine_segment` lets `list`, `read` and `write` keep such a name
//! reachable instead of discovering it and then refusing to open it.
//! 2. [`open_root`] opens `root` as a [`cap_std::fs::Dir`] capability once per
//! job. Every [`list`], [`read`], [`write`] and [`create`] routes its
//! already-segment-checked relative path through that one `Dir`, never
//! through an absolute OS path built by this crate. `cap_std` resolves one
//! component at a time against real, already-open directory handles, so a
//! symlink or a Windows junction anywhere under `root`, dangling or not,
//! cannot walk resolution above `root`: there is no separate "is this
//! still inside the root" check with a window for the directory tree to
//! change underneath it, because the one resolution *is* the open.
//!
//! A hard link is not a symlink and `cap_std` does not refuse one; the inode
//! it names is an ordinary file from inside the sandbox. [`write`] protects
//! it a different way: it always writes a sibling temporary file and renames
//! it over the target, rather than truncating the target in place. A rename
//! replaces one name's directory entry; it never reaches through to another
//! name that happens to point at the same inode outside `root`.
use std::ffi::OsStr;
use std::fs;
use std::io::{Read as _, Write as _};
use std::path::{Component, Path, PathBuf};
use std::sync::atomic::{AtomicU64, Ordering};
use cap_std::fs::{Dir, OpenOptions};
use craft_cloud::{Capability, Entry, EntryKind};
use craft_ui::cloud::{Account, CloudBackend, Event, Fault, Job, Post};
/// Where "the cloud" lives for this backend: one directory on disk.
pub struct LocalBackend {
root: PathBuf,
/// Shown in the account badge. Nothing else reads it.
display: String,
}
impl LocalBackend {
/// Treat `root` as the cloud. The directory is not created here; a
/// listing of a root that does not exist yet answers with a [`Fault`]
/// the same way a real provider's missing folder would. [`Job::Create`]
/// is the one exception, and always has been: it brings a fresh root
/// into existence itself, the same way a first Save As does not require
/// the destination folder to already be there.
pub fn new(root: PathBuf) -> LocalBackend {
LocalBackend { root, display: "Local disk".to_owned() }
}
/// Name shown in the account badge, in place of the default "Local disk".
pub fn named(mut self, display: impl Into<String>) -> LocalBackend {
self.display = display.into();
self
}
}
impl CloudBackend for LocalBackend {
fn submit(&self, job: Job, post: Post) {
let root = self.root.clone();
match job {
// Nothing is saved between runs and nothing was ever signed into,
// so there is nothing to forget. The trait states no answer for
// this job; CloudFiles::sign_out clears its own state regardless
// of whether anything ever arrives.
Job::SignOut => {}
Job::Account | Job::SignIn => {
let display = self.display.clone();
std::thread::spawn(move || post.send(Event::Account(Ok(account(display)))));
}
// There is no browser round trip: a local folder is already
// "linked" the moment it exists. Reporting that immediately,
// rather than inventing a URL nobody needs to visit, is what the
// contract calls honest.
Job::LinkStart | Job::LinkPoll(_) => {
std::thread::spawn(move || post.send(Event::Linked(Ok(true))));
}
Job::List(path) => {
std::thread::spawn(move || post.send(Event::Listed { entries: list(&root, &path), path }));
}
Job::SharedWithMe => {
std::thread::spawn(move || post.send(Event::Shared(Ok(Vec::new()))));
}
Job::Read { entry, into } => {
std::thread::spawn(move || match read(&root, &entry) {
Ok((entry, bytes)) => post.send(Event::Read { entry, into, bytes: Ok(bytes) }),
Err(fault) => post.send(Event::Read { entry, into, bytes: Err(fault) }),
});
}
Job::Write { uid, entry, bytes } => {
std::thread::spawn(move || post.send(Event::Saved { uid, entry: write(&root, &entry, &bytes) }));
}
Job::Create { uid, folder, name, bytes } => {
std::thread::spawn(move || post.send(Event::Saved { uid, entry: create(&root, &folder, &name, &bytes) }));
}
// No sharing, no accounts beyond this one, and so nowhere for a
// conversation to live. A reference backend that invented one
// would be lying about what a folder on disk can do.
Job::Conversation(_) => {
std::thread::spawn(move || {
post.send(Event::Conversation(Err(Fault::Refused("this reference backend has no shared workspace to converse in".to_owned()))))
});
}
}
}
}
fn account(display: String) -> Account {
Account { display, capabilities: vec![Capability::Files, Capability::Etags], provider_linked: true }
}
/// Turns a [`Job`]'s slash-separated path into a path relative to `root`,
/// refusing anything that is not a run of ordinary names. The result is
/// never joined onto `root` as an OS path; every caller hands it to a
/// [`cap_std::fs::Dir`] opened on `root` instead, which is the layer that
/// actually keeps resolution from leaving `root`.
///
/// `.` and repeated or leading slashes are just an empty segment and are
/// skipped, the same way a browser's address bar collapses them. Every other
/// segment goes through [`confine_segment`].
fn relative(path: &str) -> Result<PathBuf, Fault> {
let mut rel = PathBuf::new();
for segment in path.split('/') {
if segment.is_empty() || segment == "." {
continue;
}
rel.push(confine_segment(segment)?);
}
Ok(rel)
}
/// Confines one path segment to a single ordinary file or folder name.
///
/// Accepted only when `segment` parses as exactly one [`Component::Normal`]
/// equal to `segment` itself: no empty name, no `.` or `..`, no absolute
/// form, and nothing left over for a second component to hide in.
///
/// This is deliberately lenient about an ordinary character such as `:` or
/// `\` that is not, on its own, a path separator on any platform's
/// filesystem: `list` discovers real names from disk, and a name containing
/// one of those (macOS stores a `/` typed in Finder as `:` on disk) must stay
/// reachable by a later `read` or `write`, not become visible-but-unopenable.
/// [`confine_name`] is the stricter check for a name being chosen new, where
/// refusing those characters costs nothing and closes a real escape.
fn confine_segment(segment: &str) -> Result<&str, Fault> {
let mut components = Path::new(segment).components();
match (components.next(), components.next()) {
(Some(Component::Normal(name)), None) if name == OsStr::new(segment) => Ok(segment),
_ => Err(Fault::Refused(format!("{segment} is not a plain file name"))),
}
}
/// Windows device names that name a device rather than a file, with or
/// without a trailing extension (`CON`, `con.txt`, ...). Checked on every
/// platform: a name chosen here can end up read on a Windows client too.
const WINDOWS_RESERVED_NAMES: &[&str] = &[
"CON", "PRN", "AUX", "NUL", "CONIN$", "CONOUT$", "COM1", "COM2", "COM3", "COM4", "COM5", "COM6", "COM7", "COM8", "COM9", "LPT1", "LPT2", "LPT3",
"LPT4", "LPT5", "LPT6", "LPT7", "LPT8", "LPT9",
];
fn is_windows_reserved_name(name: &str) -> bool {
let stem = name.split('.').next().unwrap_or(name);
WINDOWS_RESERVED_NAMES.iter().any(|reserved| stem.eq_ignore_ascii_case(reserved))
}
/// Confines a [`Job::Create`] `name` to a single ordinary file name that is
/// safe to choose on any platform, not only the one this crate is built for.
///
/// Stricter than [`confine_segment`]: a backslash or colon is refused
/// outright first, because a Unix build's [`Component`] parser does not
/// treat either as a separator and would wave a Windows drive letter or UNC
/// prefix (`C:`, `\\server\share`) straight through as one ordinary name. A
/// Windows reserved device name is refused the same way. None of this costs
/// anything here, unlike applying it to a name `list` merely discovered.
fn confine_name(name: &str) -> Result<&str, Fault> {
if name.contains('\\') || name.contains(':') {
return Err(Fault::Refused(format!("{name} is not a plain file name")));
}
if is_windows_reserved_name(name) {
return Err(Fault::Refused(format!("{name} is a reserved device name on Windows")));
}
confine_segment(name)
}
/// Opens `root` as the one ambient-authority capability this crate holds.
/// Every other path in this crate is resolved relative to the [`Dir`] this
/// returns, never as an absolute OS path built by string concatenation.
fn open_root(root: &Path) -> Result<Dir, Fault> {
Dir::open_ambient_dir(root, cap_std::ambient_authority()).map_err(|error| io_fault(&error))
}
/// The path a listed child gets, given the folder it was found in.
fn child_path(folder: &str, name: &str) -> String {
format!("{}/{name}", folder.trim_end_matches('/'))
}
/// An opaque version token for a file's current bytes.
///
/// Not a timestamp for a person to read (that would go through `chrono`, per
/// this workspace's rule); just enough to tell "the file I read" from "the
/// file as it is now" apart, which is all a conditional write needs.
fn etag_of(meta: &cap_std::fs::Metadata) -> String {
let stamp =
meta.modified().ok().and_then(|time| time.duration_since(cap_std::time::SystemClock::UNIX_EPOCH).ok()).map(|d| d.as_nanos()).unwrap_or(0);
format!("{}-{stamp}", meta.len())
}
fn io_fault(error: &std::io::Error) -> Fault {
use std::io::ErrorKind;
match error.kind() {
ErrorKind::NotFound => Fault::Refused("no such file or folder".to_owned()),
ErrorKind::PermissionDenied => Fault::Refused("permission denied".to_owned()),
_ => Fault::Unreachable(error.to_string()),
}
}
fn list(root: &Path, path: &str) -> Result<Vec<Entry>, Fault> {
let rel = relative(path)?;
let dir = open_root(root)?;
let read = if rel.as_os_str().is_empty() { dir.entries() } else { dir.read_dir(&rel) };
let read = read.map_err(|error| io_fault(&error))?;
let mut entries = Vec::new();
for item in read {
let item = item.map_err(|error| io_fault(&error))?;
let meta = item.metadata().map_err(|error| io_fault(&error))?;
let name = item.file_name().to_string_lossy().into_owned();
let id = child_path(path, &name);
entries.push(Entry {
room: format!("local:{id}"),
name,
kind: if meta.is_dir() { EntryKind::Folder } else { EntryKind::File },
size: if meta.is_dir() { None } else { Some(meta.len()) },
modified: None,
mime: None,
etag: if meta.is_dir() { None } else { Some(etag_of(&meta)) },
writable: true,
shared_by: None,
path: id.clone(),
id,
});
}
entries.sort_by(|a, b| (a.kind != EntryKind::Folder, &a.name).cmp(&(b.kind != EntryKind::Folder, &b.name)));
Ok(entries)
}
/// Reads a file's bytes together with the entry for the version just read.
///
/// The entry is built from a fresh metadata call here, not handed back
/// unchanged from what the caller already believed. A reload after a conflict
/// hands in the stale entry it is trying to get past; echoing that back would
/// carry the old etag into the reloaded document, so the very next save would
/// conflict again on a copy that is actually current.
///
/// Refuses outright if `entry.id` names a symlink, the same policy `write`
/// uses: this backend never reports the bytes of whatever a link happens to
/// point at, inside `root` or out.
fn read(root: &Path, entry: &Entry) -> Result<(Entry, Vec<u8>), Fault> {
let rel = relative(&entry.id)?;
let dir = open_root(root)?;
// Stats without following only the last component; every component
// before it is still resolved, and sandboxed, normally, so a link
// earlier in `rel` is refused by `open_root`'s `Dir` itself when `open`
// below walks it, never silently followed here either way.
match dir.symlink_metadata(&rel) {
Ok(meta) if meta.is_symlink() => return Err(Fault::Refused("that path is a link, not a plain file".to_owned())),
Ok(_) => {}
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
Err(error) => return Err(io_fault(&error)),
}
let mut file = dir.open(&rel).map_err(|error| io_fault(&error))?;
let mut bytes = Vec::new();
file.read_to_end(&mut bytes).map_err(|error| io_fault(&error))?;
let meta = file.metadata().map_err(|error| io_fault(&error))?;
let fresh = Entry { size: Some(meta.len()), etag: Some(etag_of(&meta)), ..entry.clone() };
Ok((fresh, bytes))
}
/// Saves back over `entry`, refusing the write when the file moved on since
/// `entry.etag` was read. A file that no longer exists has nothing to
/// conflict with, the same way a provider treats a deleted document.
///
/// Never truncates `entry.id`'s current bytes in place. [`write_replacing`]
/// writes a sibling temporary file and renames it over the target instead,
/// so a hard link sharing the target's inode keeps its own, unmodified copy:
/// the rename replaces `entry.id`'s directory entry, and never reaches
/// through to any other name pointing at the same inode.
fn write(root: &Path, entry: &Entry, bytes: &[u8]) -> Result<Entry, Fault> {
let rel = relative(&entry.id)?;
let dir = open_root(root)?;
match dir.symlink_metadata(&rel) {
Ok(meta) if meta.is_symlink() => return Err(Fault::Refused("that path is a link, not a plain file".to_owned())),
Ok(meta) => {
if Some(etag_of(&meta)) != entry.etag {
return Err(Fault::Conflict);
}
}
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
Err(error) => return Err(io_fault(&error)),
}
if let Some(parent) = rel.parent() {
dir.create_dir_all(parent).map_err(|error| io_fault(&error))?;
}
write_replacing(&dir, &rel, bytes)?;
let meta = dir.metadata(&rel).map_err(|error| io_fault(&error))?;
Ok(Entry { size: Some(meta.len()), etag: Some(etag_of(&meta)), ..entry.clone() })
}
/// The next in a run of sibling temporary names, unique within this process.
/// Not meant to resist a second process picking the same name; the create
/// this writes with is exclusive (`create_new`), so a collision there is
/// just retried under a new counter value rather than silently overwritten.
static TEMP_SEQUENCE: AtomicU64 = AtomicU64::new(0);
/// Writes `bytes` to a new sibling of `rel` inside `dir`, then renames that
/// sibling over `rel`. See the module docs for why: a direct write would
/// truncate whatever `rel` currently names in place, and if that happens to
/// be a hard link to a file outside `root`, the outside copy would be
/// corrupted along with it. A rename only ever replaces `rel`'s own
/// directory entry.
fn write_replacing(dir: &Dir, rel: &Path, bytes: &[u8]) -> Result<(), Fault> {
let name = rel.file_name().ok_or_else(|| Fault::Refused("that path names no file".to_owned()))?;
let mut temp_name = OsStr::new(".").to_os_string();
temp_name.push(name);
temp_name.push(format!(".tmp-{}-{}", std::process::id(), TEMP_SEQUENCE.fetch_add(1, Ordering::Relaxed)));
let temp_rel = match rel.parent() {
Some(parent) if !parent.as_os_str().is_empty() => parent.join(&temp_name),
_ => PathBuf::from(&temp_name),
};
let mut options = OpenOptions::new();
options.write(true).create_new(true);
let written = (|| -> Result<(), Fault> {
let mut handle = dir.open_with(&temp_rel, &options).map_err(|error| io_fault(&error))?;
handle.write_all(bytes).map_err(|error| io_fault(&error))?;
handle.sync_all().map_err(|error| io_fault(&error))
})();
if let Err(fault) = written {
let _ = dir.remove_file(&temp_rel);
return Err(fault);
}
if let Err(error) = dir.rename(&temp_rel, dir, rel) {
let _ = dir.remove_file(&temp_rel);
return Err(io_fault(&error));
}
Ok(())
}
fn create(root: &Path, folder: &str, name: &str, bytes: &[u8]) -> Result<Entry, Fault> {
// `name` is never a path: it is one `Job::Create` field that names the
// new file, so it goes through `confine_name` directly rather than
// through `relative`, which only ever splits and confines a slash path.
let name = confine_name(name)?;
// `root` is the operator-configured boundary, never attacker input, so
// creating it directly with `std::fs` is safe; this is also the one
// place that has always brought a fresh root into existence (see
// `LocalBackend::new`'s docs), and `Dir::open_ambient_dir` below needs it
// to already exist.
fs::create_dir_all(root).map_err(|error| io_fault(&error))?;
let dir = open_root(root)?;
let rel_folder = relative(folder)?;
dir.create_dir_all(&rel_folder).map_err(|error| io_fault(&error))?;
let rel = rel_folder.join(name);
// Exclusive creation, not an `exists` check ahead of a separate write: two
// creates of the same name racing both see "not there yet" otherwise, and
// the second write silently replaces the first with neither ever refused.
// `create_new` asks the filesystem to open and create in one step, so the
// loser gets an error instead of a chance to overwrite the winner. It also
// refuses outright when `name` already exists as a symlink, dangling or
// not: `O_EXCL` never follows one to decide whether it is "in the way".
let mut create_options = OpenOptions::new();
create_options.write(true).create_new(true);
let mut handle = match dir.open_with(&rel, &create_options) {
Ok(handle) => handle,
Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {
return Err(Fault::Refused(format!("{name} already exists")));
}
Err(error) => return Err(io_fault(&error)),
};
handle.write_all(bytes).map_err(|error| io_fault(&error))?;
drop(handle);
let meta = dir.metadata(&rel).map_err(|error| io_fault(&error))?;
let id = child_path(folder, name);
Ok(Entry {
room: format!("local:{id}"),
name: name.to_owned(),
kind: EntryKind::File,
size: Some(meta.len()),
modified: None,
mime: None,
etag: Some(etag_of(&meta)),
writable: true,
shared_by: None,
path: id.clone(),
id,
})
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn a_parent_segment_cannot_walk_out_of_root() {
assert!(relative("../secrets").is_err());
assert!(relative("Archive/../../../etc/passwd").is_err());
}
#[test]
fn repeated_and_trailing_slashes_collapse_like_a_browser_address_bar() {
assert_eq!(relative("//Archive//2026/").unwrap(), Path::new("Archive").join("2026"));
assert_eq!(relative("").unwrap(), PathBuf::new());
}
#[test]
fn confine_segment_accepts_only_a_single_ordinary_name() {
for ordinary in ["Minutes.txt", "Archive", "2026", "a name with spaces.docx"] {
assert!(confine_segment(ordinary).is_ok(), "{ordinary} is an ordinary name");
}
}
#[test]
fn confine_segment_refuses_empty_dot_and_dot_dot() {
for bad in ["", ".", ".."] {
assert!(confine_segment(bad).is_err(), "{bad:?} is not a single ordinary name");
}
}
#[test]
fn confine_segment_refuses_a_slash_hidden_inside_a_create_name() {
assert!(confine_segment("folder/evil").is_err());
assert!(confine_segment("../outside.txt").is_err());
assert!(confine_segment("/outside.txt").is_err());
}
/// The fix for the non-blocking finding that `list` made a real macOS
/// name (a literal `/`, stored on disk as `:`) visible but unreadable: an
/// existing name with an ordinary `:` byte is not refused by the lenient
/// per-segment check, only by `confine_name` for a chosen one. A literal
/// `\` is not tested the same way here: on a Windows build it really is
/// a second path component (`std::path::Component` already splits on
/// it), so there is no existing-name case for `confine_segment` to stay
/// lenient about cross-platform the way there is for `:`.
#[test]
fn confine_segment_tolerates_a_colon_already_on_disk() {
assert!(confine_segment("weird:name.txt").is_ok());
assert!(relative("weird:name.txt").is_ok());
}
#[test]
fn confine_name_refuses_backslash_and_colon_on_every_platform() {
for bad in ["..\\outside.txt", "C:\\x", "C:evil", "\\\\server\\share\\x"] {
assert!(confine_name(bad).is_err(), "{bad:?} must be refused on every platform, not only on Windows");
}
}
#[test]
fn confine_name_refuses_windows_reserved_device_names() {
for bad in ["CON", "con", "con.txt", "NUL", "COM1", "lpt9.log", "CONIN$"] {
assert!(confine_name(bad).is_err(), "{bad:?} names a Windows device, not a file");
}
for ordinary in ["console.txt", "NULLABLE.md", "COM10"] {
assert!(confine_name(ordinary).is_ok(), "{ordinary} only resembles a reserved name");
}
}
#[test]
fn a_folder_announces_no_capability_this_backend_cannot_honour() {
let reported = account("Local disk".to_owned()).capabilities;
for absent in [Capability::SharedWithMe, Capability::Sharing, Capability::Conversation, Capability::Contacts] {
assert!(!reported.contains(&absent), "a folder has no {absent:?}");
}
}
}
+244
View File
@@ -0,0 +1,244 @@
//! Drives [`craft_cloud_local::LocalBackend`] the way an app does: through
//! `craft_ui::cloud`'s own public functions, never by calling the backend
//! directly. That is the seam this crate exists to prove, so the tests stay
//! on the application's side of it.
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use craft_cloud::{Entry, EntryKind};
use craft_cloud_local::LocalBackend;
use craft_ui::cloud::{self, Account, CloudApp, CloudFiles, Destination, DocumentKind, Job};
const NOTE: DocumentKind = DocumentKind { extensions: &["txt"], mimes: &["text/plain"], noun: "note", default_extension: "txt" };
/// A document the harness was told to open: its name and its bytes.
type OpenedDocs = Arc<Mutex<Vec<(String, Vec<u8>)>>>;
/// The least a [`CloudApp`] can be. What it was told to open is recorded, so a
/// round trip can be checked by content and not only by the entry metadata
/// `craft_ui` itself remembers.
struct Harness {
cloud: CloudFiles,
share: craft_ui::share::ShareDialog,
presence: craft_ui::presence::PresenceBar,
opened: OpenedDocs,
said: Arc<Mutex<Vec<String>>>,
}
impl Harness {
fn new() -> Harness {
Harness {
cloud: CloudFiles::new(),
share: craft_ui::share::ShareDialog::new(),
presence: craft_ui::presence::PresenceBar::new(),
opened: Arc::new(Mutex::new(Vec::new())),
said: Arc::new(Mutex::new(Vec::new())),
}
}
}
impl CloudApp for Harness {
fn cloud(&self) -> &CloudFiles {
&self.cloud
}
fn cloud_mut(&mut self) -> &mut CloudFiles {
&mut self.cloud
}
fn kind(&self) -> DocumentKind {
NOTE
}
fn active(&self) -> Option<u64> {
None
}
fn title(&self, _uid: u64) -> String {
String::new()
}
fn open_bytes(&mut self, name: &str, bytes: Vec<u8>) -> Result<Option<u64>, String> {
self.opened.lock().unwrap().push((name.to_owned(), bytes));
Ok(Some(1))
}
fn encode(&mut self, _uid: u64, _into: Destination<'_>) -> Result<Vec<u8>, String> {
Ok(Vec::new())
}
fn close(&mut self, _uid: u64) {}
fn sharing(&mut self) -> &mut craft_ui::share::ShareDialog {
&mut self.share
}
fn presence(&mut self) -> &mut craft_ui::presence::PresenceBar {
&mut self.presence
}
fn paint_icon(&self, _painter: &egui::Painter, _rect: egui::Rect, _kind: EntryKind, _colour: egui::Color32) {}
fn open_url(&self, _url: &str) {}
fn say(&mut self, text: String) {
self.said.lock().unwrap().push(text);
}
fn app_name(&self) -> &str {
"Harness"
}
}
fn wired(root: std::path::PathBuf) -> (Harness, egui::Context) {
let mut app = Harness::new();
app.cloud.connect(LocalBackend::new(root));
(app, egui::Context::default())
}
/// Drains whatever the backend's worker threads have sent so far, until
/// `done` says the state the test is waiting for arrived or five seconds pass.
/// Five seconds is far more than a thread doing local file IO ever needs; it
/// only guards against a real bug leaving the test to hang forever.
fn settle(app: &mut Harness, ctx: &egui::Context, mut done: impl FnMut(&Harness) -> bool) {
let start = Instant::now();
loop {
ctx.begin_pass(Default::default());
cloud::pump(app, ctx);
ctx.end_pass().textures_delta.clear();
if done(app) {
return;
}
assert!(start.elapsed() < Duration::from_secs(5), "timed out waiting for the backend's worker thread");
std::thread::sleep(Duration::from_millis(5));
}
}
#[test]
fn account_and_sign_in_report_the_same_fixed_identity_with_no_browser_round_trip() {
let dir = tempfile::tempdir().unwrap();
let (mut app, ctx) = wired(dir.path().to_path_buf());
cloud::start(&mut app);
settle(&mut app, &ctx, |app| app.cloud().account.is_some());
assert_eq!(
app.cloud().account,
Some(Account {
display: "Local disk".to_owned(),
capabilities: vec![craft_cloud::Capability::Files, craft_cloud::Capability::Etags],
provider_linked: true
})
);
// There was never an account to link, so nothing was ever offered.
assert!(app.cloud().link.is_none());
}
#[test]
fn creating_a_document_writes_real_bytes_a_read_gets_back() {
let dir = tempfile::tempdir().unwrap();
let (mut app, ctx) = wired(dir.path().to_path_buf());
cloud::submit(&app, Job::Create { uid: 1, folder: String::new(), name: "Minutes.txt".to_owned(), bytes: b"first draft".to_vec() });
settle(&mut app, &ctx, |app| app.cloud().origins.contains_key(&1));
let entry = app.cloud().origins.get(&1).unwrap().entry.clone();
assert_eq!(entry.name, "Minutes.txt");
assert_eq!(entry.id, "/Minutes.txt");
assert_eq!(fs_read(dir.path(), "Minutes.txt"), b"first draft");
cloud::submit(&app, Job::Read { entry, into: None });
settle(&mut app, &ctx, |app| !app.opened.lock().unwrap().is_empty());
let opened = app.opened.lock().unwrap().clone();
assert_eq!(opened, vec![("Minutes.txt".to_owned(), b"first draft".to_vec())], "the read got back exactly what the create wrote");
}
#[test]
fn a_write_under_a_stale_version_is_refused_and_never_overwrites_the_newer_file() {
let dir = tempfile::tempdir().unwrap();
let (mut app, ctx) = wired(dir.path().to_path_buf());
cloud::submit(&app, Job::Create { uid: 1, folder: String::new(), name: "Minutes.txt".to_owned(), bytes: b"v1".to_vec() });
settle(&mut app, &ctx, |app| app.cloud().origins.contains_key(&1));
let stale = app.cloud().origins.get(&1).unwrap().entry.clone();
// Somebody else saves first, changing the file's etag without going
// through this window at all.
std::fs::write(dir.path().join("Minutes.txt"), b"somebody else's edit").unwrap();
cloud::submit(&app, Job::Write { uid: 1, entry: stale, bytes: b"this window's edit".to_vec() });
settle(&mut app, &ctx, |app| app.cloud().conflict.is_some());
assert_eq!(app.cloud().conflict.as_ref().unwrap().uid, 1);
assert_eq!(fs_read(dir.path(), "Minutes.txt"), b"somebody else's edit", "the refused write never touched the file");
}
#[test]
fn listing_the_root_reflects_a_file_nobody_went_through_this_backend_to_write() {
let dir = tempfile::tempdir().unwrap();
std::fs::write(dir.path().join("Agenda.txt"), b"1. roll call").unwrap();
std::fs::create_dir(dir.path().join("Archive")).unwrap();
let (mut app, ctx) = wired(dir.path().to_path_buf());
cloud::browse(&mut app, "");
settle(&mut app, &ctx, |app| app.cloud().mine.loaded);
let names: Vec<_> = app.cloud().mine.entries.iter().map(|entry| entry.name.clone()).collect();
assert_eq!(names, vec!["Archive".to_owned(), "Agenda.txt".to_owned()], "folders first, then files, each alphabetised");
let folder = app.cloud().mine.entries.iter().find(|entry| entry.name == "Archive").unwrap();
assert_eq!(folder.kind, EntryKind::Folder);
assert!(folder.size.is_none());
}
#[test]
fn nobody_has_shared_anything_with_a_folder() {
let dir = tempfile::tempdir().unwrap();
let (mut app, ctx) = wired(dir.path().to_path_buf());
cloud::load_shared(&mut app);
settle(&mut app, &ctx, |app| app.cloud().shared.loaded);
assert_eq!(app.cloud().shared.entries, Vec::new());
assert!(app.cloud().shared.error.is_none(), "an empty share list is not a failure");
}
#[test]
fn a_conversation_is_a_fault_this_backend_is_honest_about() {
let dir = tempfile::tempdir().unwrap();
let (mut app, ctx) = wired(dir.path().to_path_buf());
cloud::submit(&app, Job::Conversation("/Minutes.txt".to_owned()));
settle(&mut app, &ctx, |app| app.cloud().status.is_some());
assert_eq!(app.cloud().status, Some("this reference backend has no shared workspace to converse in".to_owned()));
}
#[test]
fn a_read_of_a_document_that_is_not_there_is_put_to_the_user_not_silently_dropped() {
let dir = tempfile::tempdir().unwrap();
let (mut app, ctx) = wired(dir.path().to_path_buf());
let ghost = Entry {
id: "/ghost.txt".to_owned(),
room: "local:/ghost.txt".to_owned(),
name: "ghost.txt".to_owned(),
path: "/ghost.txt".to_owned(),
kind: EntryKind::File,
size: None,
modified: None,
mime: None,
etag: None,
writable: true,
shared_by: None,
};
cloud::submit(&app, Job::Read { entry: ghost, into: None });
settle(&mut app, &ctx, |app| !app.said.lock().unwrap().is_empty());
assert!(app.said.lock().unwrap()[0].contains("ghost.txt"), "the message names the document, not just that something failed");
}
fn fs_read(root: &std::path::Path, name: &str) -> Vec<u8> {
std::fs::read(root.join(name)).unwrap()
}
@@ -0,0 +1,431 @@
//! Confinement regressions for `crates/cloud-local`. The first round (FredApps' re-review) found
//! that `resolve` only split on `/` and rejected a segment equal to `..`, with no constraint on a
//! Windows backslash, a drive letter or UNC prefix, or a filesystem link, and that `create`
//! joined `name` onto the resolved folder with no validation of `name` at all. The second round
//! (benandez's re-review) reproduced three escapes the first fix still left open: a dangling
//! symlink, a check-then-use race on a swapped link, and a hard link sharing a file's inode.
//! Driven through `craft_ui::cloud`'s public functions the way an application does, with a
//! sentinel file and directory kept as a sibling of the configured root: every test here must
//! leave both untouched and unread.
use std::path::PathBuf;
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use craft_cloud::{Entry, EntryKind};
use craft_cloud_local::LocalBackend;
use craft_ui::cloud::{self, CloudApp, CloudFiles, Destination, DocumentKind, Job};
const NOTE: DocumentKind = DocumentKind { extensions: &["txt"], mimes: &["text/plain"], noun: "note", default_extension: "txt" };
/// A document the harness was told to open: its name and its bytes.
type OpenedDocs = Arc<Mutex<Vec<(String, Vec<u8>)>>>;
struct Harness {
cloud: CloudFiles,
share: craft_ui::share::ShareDialog,
presence: craft_ui::presence::PresenceBar,
opened: OpenedDocs,
said: Arc<Mutex<Vec<String>>>,
}
impl Harness {
fn new() -> Harness {
Harness {
cloud: CloudFiles::new(),
share: craft_ui::share::ShareDialog::new(),
presence: craft_ui::presence::PresenceBar::new(),
opened: Arc::new(Mutex::new(Vec::new())),
said: Arc::new(Mutex::new(Vec::new())),
}
}
}
impl CloudApp for Harness {
fn cloud(&self) -> &CloudFiles {
&self.cloud
}
fn cloud_mut(&mut self) -> &mut CloudFiles {
&mut self.cloud
}
fn kind(&self) -> DocumentKind {
NOTE
}
fn active(&self) -> Option<u64> {
None
}
fn title(&self, _uid: u64) -> String {
String::new()
}
fn open_bytes(&mut self, name: &str, bytes: Vec<u8>) -> Result<Option<u64>, String> {
self.opened.lock().unwrap().push((name.to_owned(), bytes));
Ok(Some(1))
}
fn encode(&mut self, _uid: u64, _into: Destination<'_>) -> Result<Vec<u8>, String> {
Ok(Vec::new())
}
fn close(&mut self, _uid: u64) {}
fn sharing(&mut self) -> &mut craft_ui::share::ShareDialog {
&mut self.share
}
fn presence(&mut self) -> &mut craft_ui::presence::PresenceBar {
&mut self.presence
}
fn paint_icon(&self, _painter: &egui::Painter, _rect: egui::Rect, _kind: EntryKind, _colour: egui::Color32) {}
fn open_url(&self, _url: &str) {}
fn say(&mut self, text: String) {
self.said.lock().unwrap().push(text);
}
fn app_name(&self) -> &str {
"Harness"
}
}
fn wired(root: PathBuf) -> (Harness, egui::Context) {
let mut app = Harness::new();
app.cloud.connect(LocalBackend::new(root));
(app, egui::Context::default())
}
/// Drains whatever the backend's worker threads have sent so far, until `done` says the state
/// the test is waiting for arrived or five seconds pass. Five seconds is far more than a thread
/// doing local file IO ever needs; it only guards against a real bug leaving the test to hang.
fn settle(app: &mut Harness, ctx: &egui::Context, mut done: impl FnMut(&Harness) -> bool) {
let start = Instant::now();
loop {
ctx.begin_pass(Default::default());
cloud::pump(app, ctx);
ctx.end_pass().textures_delta.clear();
if done(app) {
return;
}
assert!(start.elapsed() < Duration::from_secs(5), "timed out waiting for the backend's worker thread");
std::thread::sleep(Duration::from_millis(5));
}
}
/// Clears both event sinks, then waits for the next job to land in either one. Clearing first
/// matters for a test that submits several jobs in a loop: without it, a stale `said` from two
/// iterations ago would satisfy `settle` immediately, before the new job had actually answered.
fn said(app: &mut Harness, ctx: &egui::Context) -> Vec<String> {
app.opened.lock().unwrap().clear();
app.said.lock().unwrap().clear();
settle(app, ctx, |app| !app.said.lock().unwrap().is_empty() || !app.opened.lock().unwrap().is_empty());
app.said.lock().unwrap().clone()
}
/// A tempdir holding the cloud root, plus a sentinel file and a sentinel directory that sit
/// beside it rather than inside it. Every test below must leave both exactly as they start.
struct Sentinels {
// Kept only so the tempdir outlives every test that borrows the paths below; nothing reads
// it back once the paths it was built from are in hand.
_arena: tempfile::TempDir,
root: PathBuf,
sentinel_file: PathBuf,
sentinel_dir: PathBuf,
}
const SENTINEL_FILE_BYTES: &[u8] = b"a sibling file this backend must never read or write";
const SENTINEL_DIR_FILE_BYTES: &[u8] = b"a sibling directory this backend must never list";
fn sentinels() -> Sentinels {
let arena = tempfile::tempdir().unwrap();
let root = arena.path().join("root");
std::fs::create_dir(&root).unwrap();
let sentinel_file = arena.path().join("sentinel.txt");
std::fs::write(&sentinel_file, SENTINEL_FILE_BYTES).unwrap();
let sentinel_dir = arena.path().join("sentinel-dir");
std::fs::create_dir(&sentinel_dir).unwrap();
std::fs::write(sentinel_dir.join("secret.txt"), SENTINEL_DIR_FILE_BYTES).unwrap();
Sentinels { _arena: arena, root, sentinel_file, sentinel_dir }
}
fn assert_sentinels_untouched(sentinels: &Sentinels) {
assert_eq!(std::fs::read(&sentinels.sentinel_file).unwrap(), SENTINEL_FILE_BYTES, "the sentinel file outside the root was changed");
assert_eq!(
std::fs::read(sentinels.sentinel_dir.join("secret.txt")).unwrap(),
SENTINEL_DIR_FILE_BYTES,
"the sentinel directory's file outside the root was changed"
);
}
/// Every one of these is a [`Job::Create`] `name` that must be refused before any byte reaches
/// disk: the review's own examples (a parent segment, an absolute path, and the three
/// Windows-shaped forms), none of which a Unix build's native parser alone would catch, and a
/// name that already is nothing but a Windows separator or drive prefix on top of that.
fn malicious_create_names() -> Vec<&'static str> {
vec!["../sentinel.txt", "/sentinel.txt", "..\\sentinel.txt", "C:\\x", "C:evil", "\\\\server\\share\\x"]
}
#[test]
fn create_refuses_every_escaping_name_and_the_sentinels_stay_untouched() {
for name in malicious_create_names() {
let sentinels = sentinels();
let (mut app, ctx) = wired(sentinels.root.clone());
cloud::submit(&app, Job::Create { uid: 1, folder: String::new(), name: name.to_owned(), bytes: b"pwned".to_vec() });
let messages = said(&mut app, &ctx);
assert!(cloud::origin(&app).is_none(), "{name:?} must not be reported as created");
assert!(!messages.is_empty(), "{name:?} must be refused, not silently dropped");
assert_sentinels_untouched(&sentinels);
}
}
// ------------------------------------------------------------------- link-based escapes
//
// `escape(&sentinels)` places one link named `escape` inside the cloud root, pointing out to
// `sentinels.sentinel_dir`: a symlink on Unix, a directory junction on Windows, which needs no
// elevated privilege, unlike a Windows symlink. The Linux gate runs the symlink half of every
// test below; CI's windows-latest lanes run the junction half. Either way `root/escape/...`
// reaches into the sentinel directory through the link.
/// Nothing to clean up on Unix: `tempfile`'s own recursive removal of the arena directory
/// removes a plain symlink entry without following it. This exists only so `escape` returns the
/// same shape of RAII handle on every platform, rather than a bare `()` some lint would flag.
#[cfg(unix)]
struct Linked;
#[cfg(unix)]
fn escape(sentinels: &Sentinels) -> Linked {
std::os::unix::fs::symlink(&sentinels.sentinel_dir, sentinels.root.join("escape")).unwrap();
Linked
}
/// A directory junction, removed on drop before the tempdir it sits beside is cleaned up.
#[cfg(windows)]
struct Junction(PathBuf);
#[cfg(windows)]
impl Drop for Junction {
fn drop(&mut self) {
let _ = std::fs::remove_dir(&self.0);
}
}
#[cfg(windows)]
fn escape(sentinels: &Sentinels) -> Junction {
use std::os::windows::process::CommandExt as _;
let link = sentinels.root.join("escape");
let target = &sentinels.sentinel_dir;
assert!(link.is_absolute() && target.is_absolute());
let output = std::process::Command::new("cmd")
.args(["/C", "mklink", "/J"])
.arg(&link)
.arg(target)
.creation_flags(0x0800_0000) // CREATE_NO_WINDOW
.output()
.unwrap();
assert!(output.status.success(), "{}", String::from_utf8_lossy(&output.stderr));
Junction(link)
}
/// An [`Entry`] stub naming `id`, for a [`Job::Read`] or [`Job::Write`] this file submits
/// directly rather than through an entry the backend itself handed back.
fn leaked_entry(id: &str, name: &str) -> Entry {
Entry {
id: id.to_owned(),
room: format!("local:{id}"),
name: name.to_owned(),
path: id.to_owned(),
kind: EntryKind::File,
size: None,
modified: None,
mime: None,
etag: None,
writable: true,
shared_by: None,
}
}
#[cfg(any(unix, windows))]
#[test]
fn listing_a_linked_folder_cannot_see_outside_the_root() {
let sentinels = sentinels();
let (mut app, ctx) = wired(sentinels.root.clone());
let _escape = escape(&sentinels);
cloud::browse(&mut app, "escape");
settle(&mut app, &ctx, |app| app.cloud().mine.loaded);
assert!(app.cloud().mine.error.is_some(), "listing through a symlink or junction out of the root must be refused");
assert!(app.cloud().mine.entries.is_empty());
assert_sentinels_untouched(&sentinels);
}
#[cfg(any(unix, windows))]
#[test]
fn reading_through_a_linked_folder_cannot_return_bytes_from_outside_the_root() {
let sentinels = sentinels();
let (mut app, ctx) = wired(sentinels.root.clone());
let _escape = escape(&sentinels);
cloud::submit(&app, Job::Read { entry: leaked_entry("/escape/secret.txt", "secret.txt"), into: None });
said(&mut app, &ctx);
assert!(app.opened.lock().unwrap().is_empty(), "a read through a symlink or junction out of the root must not deliver bytes");
assert_sentinels_untouched(&sentinels);
}
/// benandez's finding 4: the original version of this test targeted the pre-existing
/// `secret.txt`, with `etag: None`. `write` refused that for an unrelated reason, a version
/// mismatch, before confinement ever ran, so the test passed whether or not confinement worked.
/// Targeting a name that does not exist yet under the link reaches the code that actually
/// matters, and the oracle (the file must never be created) is one confinement failing would
/// actually flip.
#[cfg(any(unix, windows))]
#[test]
fn writing_through_a_linked_folder_cannot_create_a_file_outside_the_root() {
let sentinels = sentinels();
let (mut app, ctx) = wired(sentinels.root.clone());
let _escape = escape(&sentinels);
cloud::submit(&app, Job::Write { uid: 1, entry: leaked_entry("/escape/not-yet-created.txt", "not-yet-created.txt"), bytes: b"pwned".to_vec() });
said(&mut app, &ctx);
assert!(!sentinels.sentinel_dir.join("not-yet-created.txt").exists(), "a write through a link must not create a file outside the root");
assert_sentinels_untouched(&sentinels);
}
/// benandez's finding 5: nothing previously exercised a link in the `folder` a `Create` targets.
#[cfg(any(unix, windows))]
#[test]
fn creating_through_a_linked_folder_cannot_create_a_file_outside_the_root() {
let sentinels = sentinels();
let (mut app, ctx) = wired(sentinels.root.clone());
let _escape = escape(&sentinels);
cloud::submit(&app, Job::Create { uid: 1, folder: "escape".to_owned(), name: "new.txt".to_owned(), bytes: b"pwned".to_vec() });
let messages = said(&mut app, &ctx);
assert!(cloud::origin(&app).is_none(), "a create into a linked folder must not be reported as created");
assert!(!messages.is_empty(), "a create into a linked folder must be refused, not silently dropped");
assert!(!sentinels.sentinel_dir.join("new.txt").exists(), "a create through a link must not create a file outside the root");
assert_sentinels_untouched(&sentinels);
}
// ------------------------------------------------------------------- Unix-only reproductions
//
// Directory junctions cannot be dangling (`mklink /J` requires the target to exist) and the
// race and hard-link reproductions below are exactly benandez's own repros, run on Linux only,
// as stated in that review. The structural fix (resolving through a `cap_std::fs::Dir`, and
// writing through a temporary file and rename) is the same code on every platform; these three
// just do not have a Windows-shaped equivalent worth a separate repro.
/// benandez's finding 1, reproduced exactly: `root/doc.txt` is a symlink to a file that does not
/// exist yet, outside the root. The old code's `confine_to_root` canonicalised the dangling link
/// to `NotFound`, walked up to `root`, accepted it, found no file to conflict with, and `write`
/// created the target for real, outside the root.
#[cfg(unix)]
#[test]
fn writing_through_a_dangling_symlink_cannot_create_a_file_outside_the_root() {
let sentinels = sentinels();
let outside_target = sentinels._arena.path().join("outside-planted.txt");
std::os::unix::fs::symlink(&outside_target, sentinels.root.join("doc.txt")).unwrap();
let (mut app, ctx) = wired(sentinels.root.clone());
cloud::submit(&app, Job::Write { uid: 1, entry: leaked_entry("/doc.txt", "doc.txt"), bytes: b"pwned".to_vec() });
said(&mut app, &ctx);
assert!(!outside_target.exists(), "a write through a dangling symlink must not create its target outside the root");
assert_sentinels_untouched(&sentinels);
}
/// benandez's finding 3, reproduced exactly: `root/hl` is a hard link sharing an inode with a
/// file outside the root. Reading it through the backend is legitimate (it is, from inside the
/// sandbox, an ordinary file), which is also how this test gets a real, matching etag to write
/// with, the same way a real caller would after opening the document. The old code's `write`
/// truncated the shared inode in place; `write_replacing` must instead replace `hl`'s own
/// directory entry, leaving the sentinel file's inode, and its other name, untouched.
#[cfg(unix)]
#[test]
fn writing_through_a_hard_link_replaces_the_entry_without_touching_the_shared_file() {
let sentinels = sentinels();
std::fs::hard_link(&sentinels.sentinel_file, sentinels.root.join("hl")).unwrap();
let (mut app, ctx) = wired(sentinels.root.clone());
cloud::submit(&app, Job::Read { entry: leaked_entry("/hl", "hl"), into: None });
settle(&mut app, &ctx, |app| !app.opened.lock().unwrap().is_empty());
let real_entry = app.cloud().origins.get(&1).expect("the read registered an origin for uid 1").entry.clone();
// A successful write answers through `cloud.status`, not through `said` or `opened`, so it
// needs its own wait condition rather than the `said` helper used everywhere else in this
// file for a refusal. The read above already set `status` once (`Opened hl from the
// cloud.`), so that has to be cleared first or this would observe the read's own status and
// never actually wait for the write to finish.
app.cloud_mut().status = None;
cloud::submit(&app, Job::Write { uid: 1, entry: real_entry, bytes: b"replaced via rename, not truncated in place".to_vec() });
settle(&mut app, &ctx, |app| app.cloud().status.is_some() || app.cloud().conflict.is_some() || !app.said.lock().unwrap().is_empty());
assert!(app.cloud().conflict.is_none(), "the write through the hard-linked name should succeed, not conflict");
assert!(
app.said.lock().unwrap().is_empty(),
"the write through the hard-linked name should succeed, not be refused: {:?}",
app.said.lock().unwrap()
);
assert_eq!(
std::fs::read(sentinels.root.join("hl")).unwrap(),
b"replaced via rename, not truncated in place",
"the write through the hard-linked name should still succeed"
);
assert_sentinels_untouched(&sentinels);
}
/// benandez's finding 2, reproduced at a smaller scale: a background thread swaps `root/d`
/// between a symlink into an inside folder and a symlink to the sentinel directory, as fast as
/// it can, with no coordination with the reads below. The old code's `confine_to_root` checked
/// one path string and then `fs::read` resolved that path again, a second, later lookup a swap
/// could land in between; out of 3000 such reads benandez's own run returned the outside file's
/// bytes 347 times. `cap_std::fs::Dir::open` resolves and opens `d` as one operation with no
/// such second lookup, so this must come back empty every time, not merely most of the time.
#[cfg(unix)]
#[test]
fn reading_through_a_link_being_swapped_never_observes_the_outside_target() {
use std::sync::atomic::{AtomicBool, Ordering};
let sentinels = sentinels();
let inside = sentinels.root.join("inside");
std::fs::create_dir(&inside).unwrap();
std::fs::write(inside.join("secret.txt"), b"inside bytes, always safe to read").unwrap();
let link = sentinels.root.join("d");
std::os::unix::fs::symlink(&inside, &link).unwrap();
let stop = Arc::new(AtomicBool::new(false));
let swapper = {
let stop = stop.clone();
let link = link.clone();
let outside = sentinels.sentinel_dir.clone();
std::thread::spawn(move || {
let temp = link.with_extension("swap");
let mut toward_outside = false;
while !stop.load(Ordering::Relaxed) {
let target = if toward_outside { &outside } else { &inside };
// A stale `temp` from a rename that did not happen (should not occur, but a
// leftover `temp` would otherwise wedge every later iteration's `symlink` call,
// silently stopping the swap rather than failing the test loudly) is cleared
// first so this loop cannot get stuck.
let _ = std::fs::remove_file(&temp);
if std::os::unix::fs::symlink(target, &temp).is_ok() {
let _ = std::fs::rename(&temp, &link);
}
toward_outside = !toward_outside;
}
})
};
let (mut app, ctx) = wired(sentinels.root.clone());
let mut leaked = false;
for _ in 0..200 {
cloud::submit(&app, Job::Read { entry: leaked_entry("/d/secret.txt", "secret.txt"), into: None });
said(&mut app, &ctx);
if let Some((_, bytes)) = app.opened.lock().unwrap().first() {
leaked |= bytes.as_slice() == SENTINEL_DIR_FILE_BYTES;
}
}
stop.store(true, Ordering::Relaxed);
swapper.join().unwrap();
assert!(!leaked, "a read observed the outside target's bytes while the link was being swapped underneath it");
assert_sentinels_untouched(&sentinels);
}
@@ -0,0 +1,160 @@
//! Two save-path defects that need a real backend to show: a reload after a conflict carrying a
//! stale etag into the next save, and two creates of the same name racing past an exists check.
//! Driven through craft_ui::cloud's public functions the way an application does.
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use craft_cloud::{Entry, EntryKind};
use craft_cloud_local::LocalBackend;
use craft_ui::cloud::{self, CloudApp, CloudFiles, Destination, DocumentKind};
const NOTE: DocumentKind = DocumentKind { extensions: &["txt"], mimes: &["text/plain"], noun: "note", default_extension: "txt" };
struct Window {
cloud: CloudFiles,
share: craft_ui::share::ShareDialog,
presence: craft_ui::presence::PresenceBar,
body: Vec<u8>,
said: Arc<Mutex<Vec<String>>>,
}
impl Window {
fn new(root: std::path::PathBuf, body: &[u8]) -> Window {
let mut w = Window {
cloud: CloudFiles::new(),
share: craft_ui::share::ShareDialog::new(),
presence: craft_ui::presence::PresenceBar::new(),
body: body.to_vec(),
said: Arc::default(),
};
w.cloud.connect(LocalBackend::new(root));
w
}
}
impl CloudApp for Window {
fn cloud(&self) -> &CloudFiles {
&self.cloud
}
fn cloud_mut(&mut self) -> &mut CloudFiles {
&mut self.cloud
}
fn kind(&self) -> DocumentKind {
NOTE
}
fn active(&self) -> Option<u64> {
Some(1)
}
fn title(&self, _uid: u64) -> String {
String::new()
}
fn open_bytes(&mut self, _name: &str, bytes: Vec<u8>) -> Result<Option<u64>, String> {
self.body = bytes;
Ok(Some(1))
}
fn encode(&mut self, _uid: u64, _into: Destination<'_>) -> Result<Vec<u8>, String> {
Ok(self.body.clone())
}
fn close(&mut self, _uid: u64) {}
fn sharing(&mut self) -> &mut craft_ui::share::ShareDialog {
&mut self.share
}
fn presence(&mut self) -> &mut craft_ui::presence::PresenceBar {
&mut self.presence
}
fn paint_icon(&self, _p: &egui::Painter, _r: egui::Rect, _k: EntryKind, _c: egui::Color32) {}
fn open_url(&self, _url: &str) {}
fn say(&mut self, text: String) {
self.said.lock().unwrap().push(text);
}
fn app_name(&self) -> &str {
"Review"
}
}
fn settle(app: &mut Window, ctx: &egui::Context, mut done: impl FnMut(&Window) -> bool) {
let start = Instant::now();
loop {
ctx.begin_pass(Default::default());
cloud::pump(app, ctx);
ctx.end_pass().textures_delta.clear();
if done(app) {
return;
}
assert!(start.elapsed() < Duration::from_secs(5), "timed out");
std::thread::sleep(Duration::from_millis(2));
}
}
fn listed(app: &mut Window, ctx: &egui::Context, name: &str) -> Entry {
cloud::browse(app, "");
settle(app, ctx, |a| a.cloud.mine.loaded);
app.cloud.mine.entries.iter().find(|e| e.name == name).cloned().expect("listed")
}
/// Open v1, another writer stores v2, Save conflicts, the user picks "Reload from the cloud"
/// (`reload_active`, what the modal calls). The window now shows v2's bytes, so the next Save
/// must go through rather than conflict again on the etag the reloaded copy was read at.
#[test]
fn reload_after_a_conflict_carries_the_freshly_read_etag_so_the_next_save_succeeds() {
let root = tempfile::tempdir().unwrap();
std::fs::write(root.path().join("note.txt"), b"v1").unwrap();
let ctx = egui::Context::default();
let mut app = Window::new(root.path().to_owned(), b"");
let v1 = listed(&mut app, &ctx, "note.txt");
cloud::open_entry(&mut app, v1.clone(), None);
settle(&mut app, &ctx, |a| a.cloud.busy.is_none() && a.body == b"v1");
std::thread::sleep(Duration::from_millis(20));
std::fs::write(root.path().join("note.txt"), b"v2 by someone else").unwrap();
app.body = b"my edit".to_vec();
cloud::save(&mut app).unwrap();
settle(&mut app, &ctx, |a| a.cloud.busy.is_none());
assert!(app.cloud.conflict.is_some(), "the first save is refused (correct)");
cloud::reload_active(&mut app).unwrap();
settle(&mut app, &ctx, |a| a.cloud.busy.is_none());
assert_eq!(app.body, b"v2 by someone else", "reload shows the stored bytes");
let origin_etag = cloud::report(&app)["origin"]["etag"].clone();
cloud::save(&mut app).unwrap();
settle(&mut app, &ctx, |a| a.cloud.busy.is_none());
let on_disk = std::fs::read(root.path().join("note.txt")).unwrap();
assert!(
app.cloud.conflict.is_none(),
"after reloading v2 the save conflicted again: origin etag after reload = {origin_etag} \
(listing etag was {:?}); file on disk = {:?}",
v1.etag,
String::from_utf8_lossy(&on_disk)
);
}
/// Two windows Save As the same new name at once. At most one create may succeed; the other
/// must be refused ("already exists") rather than both reporting success while one silently
/// replaces the other on disk.
#[test]
fn concurrent_create_of_one_name_refuses_the_loser() {
let rounds = 300;
let (mut both_ok, mut example) = (0, String::new());
for round in 0..rounds {
let root = tempfile::tempdir().unwrap();
let ctx = egui::Context::default();
let mut a = Window::new(root.path().to_owned(), format!("window A round {round}").as_bytes());
let mut b = Window::new(root.path().to_owned(), format!("window B round {round}").as_bytes());
cloud::save_as(&mut a, "", "Report.txt").unwrap();
cloud::save_as(&mut b, "", "Report.txt").unwrap();
settle(&mut a, &ctx, |w| w.cloud.busy.is_none());
settle(&mut b, &ctx, |w| w.cloud.busy.is_none());
let (a_ok, b_ok) = (cloud::origin(&a).is_some(), cloud::origin(&b).is_some());
if a_ok && b_ok {
both_ok += 1;
if example.is_empty() {
let disk = std::fs::read_to_string(root.path().join("Report.txt")).unwrap();
example = format!("round {round}: A status {:?}, B status {:?}, file holds {disk:?}", a.cloud.status, b.cloud.status);
}
}
}
assert_eq!(both_ok, 0, "{both_ok}/{rounds} rounds: both windows were told the new file was created; e.g. {example}");
}
+15
View File
@@ -0,0 +1,15 @@
[package]
name = "craft-cloud"
version.workspace = true
edition.workspace = true
license.workspace = true
repository.workspace = true
rust-version.workspace = true
description = "What a cloud provider offers the Craft apps: files, sharing, contacts, notifications and mail as contracts, with no vendor and no IO"
[lints]
workspace = true
[dependencies]
serde = { workspace = true }
thiserror = { workspace = true }
+68
View File
@@ -0,0 +1,68 @@
//! One error type for every contract.
//!
//! The cases are the ones a caller acts on differently. A UI shows a different
//! thing for "you may not" than for "the server is unreachable", and a save
//! retries on a conflict where it would not on a refusal.
use thiserror::Error;
#[derive(Debug, Error)]
pub enum Error {
/// The provider could not be reached. Worth retrying.
#[error("the provider could not be reached: {0}")]
Unreachable(String),
/// The credential was rejected. The user has to sign in again.
#[error("the provider rejected the credential")]
Unauthenticated,
/// Authenticated, and not allowed to do this. Signing in again will not
/// help, so a UI must not offer it.
#[error("not permitted: {0}")]
Forbidden(String),
/// No such document, folder or share.
#[error("{what} not found: {id}")]
NotFound { what: &'static str, id: String },
/// The document changed since the caller last read it. A save that would
/// have overwritten somebody else's work lands here.
#[error("{0} changed since you opened it")]
Conflict(String),
/// The provider does not offer this at all.
///
/// Distinct from [`Error::Forbidden`]: there is nothing to grant. A UI
/// hides the feature rather than explaining it.
#[error("this provider does not support {0}")]
Unsupported(&'static str),
/// The answer did not match the protocol.
#[error("malformed {what}: {detail}")]
Malformed { what: &'static str, detail: String },
/// A name or path could not be used safely. Carried, never panicked.
#[error("invalid {what}: {detail}")]
Invalid { what: &'static str, detail: String },
}
impl Error {
pub fn not_found(what: &'static str, id: impl Into<String>) -> Error {
Error::NotFound { what, id: id.into() }
}
pub fn malformed(what: &'static str, detail: impl Into<String>) -> Error {
Error::Malformed { what, detail: detail.into() }
}
pub fn invalid(what: &'static str, detail: impl Into<String>) -> Error {
Error::Invalid { what, detail: detail.into() }
}
/// Whether trying again could succeed without the user doing anything.
pub fn is_transient(&self) -> bool {
matches!(self, Error::Unreachable(_))
}
}
pub type Result<T> = std::result::Result<T, Error>;
+189
View File
@@ -0,0 +1,189 @@
//! What a cloud provider has to offer the Craft apps, and nothing about who
//! provides it.
//!
//! Every capability here is an RFC or an ordinary HTTP API that many systems
//! implement. Files are WebDAV. Contacts are CardDAV. Calendars are CalDAV.
//! Sharing and notification are the only genuinely vendor-shaped parts, and
//! they are small. So there is no reason for an app to know which server it is
//! talking to, and no reason to write the same logic again for the next one.
//!
//! Nothing in this crate opens a socket. Each trait is implemented by a
//! provider crate that brings its own transport, which keeps this testable
//! with no server and keeps the contracts honest about what they need.
//!
//! No type here names a vendor. If a field only makes sense for one product,
//! it belongs in that provider's crate.
//!
//! Nothing here says where an implementation runs. A local adapter may
//! implement these traits against a provider directly, and an application may
//! consume them that way. The contracts do not care.
//!
//! CollabCraft chooses otherwise for its own apps. They reach one service, and
//! that service is the single place a provider is configured, a credential is
//! held and a permission is decided. A desktop binary carrying a provider
//! credential leaks one the moment somebody copies the file, which is what
//! that arrangement exists to prevent. It is a deployment policy, and this
//! crate imposes it on nobody.
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
mod error;
mod types;
pub use error::{Error, Result};
pub use types::{
Capability, Contact, Conversation, Entry, EntryKind, Identity, MailMessage, Notification, Permission, Share, ShareOptions, ShareTarget,
};
/// Reading and writing documents.
///
/// `list` and `shared_with_me` are separate because they answer different
/// questions. One is a folder's contents, the other is everything another
/// person has given this user access to, which has no single parent folder.
pub trait Files {
/// One folder's contents. An empty path means the user's root.
fn list(&self, path: &str) -> Result<Vec<Entry>>;
/// Everything shared with this user, wherever it lives.
///
/// A provider with no concept of this returns an empty list rather than an
/// error, because "nobody has shared anything with you" and "this server
/// cannot tell you" look the same to a file picker.
fn shared_with_me(&self) -> Result<Vec<Entry>>;
/// One document's bytes.
fn read(&self, id: &str) -> Result<Vec<u8>>;
/// A byte range of one document.
///
/// Defaulted to a full [`read`] and a slice, so providers that have no
/// ranged GET still satisfy the trait. A provider with real range support
/// overrides this.
fn read_range(&self, id: &str, offset: u64, length: u64) -> Result<Vec<u8>> {
let bytes = self.read(id)?;
let start = usize::try_from(offset).unwrap_or(usize::MAX);
if start >= bytes.len() {
return Ok(Vec::new());
}
let take = usize::try_from(length).unwrap_or(usize::MAX);
let end = start.saturating_add(take).min(bytes.len());
Ok(bytes[start..end].to_vec())
}
/// Replace one document's bytes.
///
/// `if_match` carries the etag the caller last saw. A provider that
/// supports conditional writes uses it to refuse a save that would
/// overwrite somebody else's, returning [`Error::Conflict`]. Passing
/// `None` means "write regardless", which is what a Save As does.
fn write(&self, id: &str, bytes: &[u8], if_match: Option<&str>) -> Result<Entry>;
/// Create a document and return it.
fn create(&self, folder: &str, name: &str, bytes: &[u8]) -> Result<Entry>;
/// One document's metadata, without its bytes.
fn stat(&self, id: &str) -> Result<Entry>;
}
/// Giving other people access to a document.
///
/// Persisting a share through the provider matters more than it looks. A share
/// recorded only by the caller would be invisible in the provider's own file
/// manager, so the two would disagree about who can open what. The provider
/// stays the authority.
pub trait Sharing {
/// Share a document.
///
/// A link share has no recipient, so [`ShareTarget::Link`] is how a caller
/// asks for one. The returned [`Share::url`] is what to hand out.
fn share(&self, id: &str, with: &ShareTarget, permission: Permission, options: &ShareOptions) -> Result<Share>;
/// Who currently has access.
fn shares(&self, id: &str) -> Result<Vec<Share>>;
/// Withdraw one share.
fn unshare(&self, share_id: &str) -> Result<()>;
}
/// Looking people up, for a share dialog's autocomplete.
///
/// This is the only thing the share dialog needs, so it is the only thing the
/// contract exposes. A full address book API would be a bigger surface for no
/// gain.
pub trait Contacts {
/// People matching a partial name or address.
///
/// `limit` exists because this runs on every keystroke. A provider is free
/// to return fewer.
fn search(&self, query: &str, limit: usize) -> Result<Vec<Contact>>;
}
/// Telling somebody something happened.
///
/// Separate from [`Mail`] on purpose. An in-app notification is cheap, arrives
/// instantly and is read in context. An email is none of those and is the
/// right choice for somebody who is not currently in the app. A share usually
/// wants both.
pub trait Notifications {
fn notify(&self, user: &str, notification: &Notification) -> Result<()>;
}
/// Sending mail as the provider.
pub trait Mail {
fn send(&self, message: &MailMessage) -> Result<()>;
}
/// Who the caller is, according to the provider.
pub trait Accounts {
/// Resolve the signed-in user.
fn me(&self) -> Result<Identity>;
}
/// Working together on a document, through whatever the provider calls it.
///
/// Defaulted throughout. A provider without the feature writes an empty
/// `impl` and reports [`Capability::Conversation`] as absent. The caller
/// then draws no control at all.
pub trait Conversations {
/// What this provider calls the feature. `None` where it has none.
///
/// Separate from the capability on purpose. The name is a property of
/// the provider. Whether it is available is a property of the
/// instance, and an administrator decides that one.
fn conversation_name(&self) -> Option<String> {
None
}
/// Start a conversation about a document, or join the one already there.
///
/// Two people pressing the same control have to arrive in the same
/// place, so this is expected to be idempotent for a given document.
///
/// `id` is the provider's own handle and means nothing outside the
/// account it came from. A conversation is therefore keyed on the
/// provider, the account and the document together, and `self` carries
/// the first two. A document copied into another account is a different
/// document to its provider, so it gets a conversation of its own.
/// Nothing here can attach one account's thread to another's.
fn conversation(&self, _id: &str, _title: &str) -> Result<Conversation> {
Err(Error::Unsupported("conversations"))
}
}
/// Everything one provider can do.
///
/// A provider implements the capabilities it has and reports the rest as
/// absent, so a caller can hide a share button on a server with no sharing
/// rather than offering a feature that will fail.
pub trait Provider: Files + Sharing + Contacts + Notifications + Mail + Accounts + Conversations {
/// A stable identifier, such as `nextcloud`. Used in configuration and
/// logs, never shown to a user.
fn id(&self) -> &str;
/// What this provider actually supports.
fn capabilities(&self) -> Vec<Capability>;
fn supports(&self, capability: Capability) -> bool {
self.capabilities().contains(&capability)
}
}
+216
View File
@@ -0,0 +1,216 @@
//! The data the contracts pass around.
//!
//! Every type here is plain and vendor-neutral. A field that only means
//! something to one product belongs in that provider's crate.
use serde::{Deserialize, Serialize};
/// What a provider can do.
///
/// Reported rather than assumed, so a caller hides a share button on a
/// server without sharing instead of offering a feature that fails on use.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Capability {
Files,
SharedWithMe,
Sharing,
Contacts,
Notifications,
Mail,
/// Conditional writes, so a save can refuse to clobber a newer version.
Etags,
/// Public links. Absent on a server that forbids them, and the dialog then
/// offers no Copy link rather than a button that fails.
LinkShares,
/// A key on a public link. This is what most places mean when they talk
/// about sharing with somebody who has no account.
LinkPassword,
/// The server requires that key, so a link cannot be made without one.
LinkPasswordRequired,
/// An expiry date on a share.
ShareExpiry,
/// The provider can tell a recipient that something was shared with them.
///
/// Distinct from [`Capability::Mail`], which is about sending arbitrary
/// mail. A provider commonly has this and not that: it will notify the
/// person a share names without offering any way to send mail in general.
ShareNotify,
/// A conversation about a document, such as a call or a chat room.
///
/// Every provider that has this calls it something else, so the name
/// comes from [`crate::Conversations::conversation_name`]. Reported
/// separately from that name: a provider knows what it would call the
/// feature whether or not an administrator installed it.
Conversation,
}
/// A file or folder.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct Entry {
/// An opaque handle for reading, writing and sharing this entry.
///
/// Only the provider that issued it may interpret it. A caller passes it
/// back and never parses it.
pub id: String,
/// The stable identity used for a collaboration room.
///
/// Separate from `id` on purpose. A provider's read and write handle is
/// often a path, and a path changes when somebody renames or moves the
/// file. A room keyed on that would silently split in two mid-session,
/// with each half seeing only its own edits. This one survives a rename.
pub room: String,
/// What to show the user.
pub name: String,
/// Where it sits, for a breadcrumb. Never a handle.
pub path: String,
pub kind: EntryKind,
/// Bytes, absent for a folder.
pub size: Option<u64>,
/// RFC 3339. Absent when the provider does not say.
pub modified: Option<String>,
/// For matching a document to the app that opens it.
pub mime: Option<String>,
/// The version the caller last saw, for a conditional write.
pub etag: Option<String>,
/// Whether this user may change it. A read-only share still lists.
pub writable: bool,
/// Who shared it, when it came from somebody else.
pub shared_by: Option<String>,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum EntryKind {
File,
Folder,
}
/// A person, as a share dialog needs them.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct Contact {
/// What the provider wants back when sharing with this person.
pub id: String,
/// How to share with them, decided by the provider and never guessed.
///
/// The distinction is not cosmetic. A contact with an account is a user
/// share. One known only by address is an email share. Sharing an address
/// as though it were a user id fails with "not found", because no such
/// user exists, and nothing in `id` alone says which case this is.
pub target: ShareTarget,
/// What to show. A display name where there is one.
pub name: String,
pub email: Option<String>,
/// An avatar, as a URL the client may fetch. Never image bytes: a search
/// returning ten people would otherwise carry ten pictures nobody asked
/// for yet.
pub avatar: Option<String>,
/// A group rather than a person. Shown differently and shared differently.
pub is_group: bool,
}
/// Who to share with.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case", tag = "kind", content = "id")]
pub enum ShareTarget {
User(String),
Group(String),
/// Somebody with no account, reached by address.
Email(String),
/// Anybody holding the link.
Link,
}
/// What a share allows.
///
/// Deliberately coarse. Providers disagree on the fine detail, and a dialog
/// offering a distinction the server cannot keep is worse than one that does
/// not.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum Permission {
/// Open and read. No edits reach the document.
View,
/// Open and edit.
Edit,
}
/// Everything optional about a share.
///
/// A struct rather than more positional arguments. The list grew past the
/// point where a call site reads correctly, and a provider that cannot honour
/// one of these needs to see which it was given.
#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct ShareOptions {
/// Shown to the recipient.
pub note: Option<String>,
/// Send mail where the provider can.
pub notify: bool,
/// A key the recipient must present. Only meaningful for a link, and
/// refused by a server that does not allow one.
pub password: Option<String>,
/// `YYYY-MM-DD`. After this the share stops working.
pub expires: Option<String>,
}
/// Where to go to work together on a document.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct Conversation {
/// Where to join. Handed to a browser, or to the provider's own app.
pub url: String,
/// What the provider calls the feature, for the control's wording. A
/// menu item reading "Work together in Talk" is built from this.
pub name: String,
}
/// A share that exists.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct Share {
pub id: String,
pub target: ShareTarget,
/// What to call the recipient, where the provider says.
///
/// A share carries the handle the server wants back, which for a user is
/// an account id. People are not known to each other by account id. A
/// provider that knows the difference says so here.
pub display: Option<String>,
pub permission: Permission,
/// Present for a link share, absent otherwise.
pub url: Option<String>,
pub note: Option<String>,
/// RFC 3339.
pub created: Option<String>,
}
/// Something worth telling a user about, in the app.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct Notification {
/// A short machine-readable kind, such as `document_shared`. Lets a client
/// decide how to render without parsing prose.
pub kind: String,
pub subject: String,
pub body: Option<String>,
/// Where to go when it is clicked.
pub link: Option<String>,
}
/// An email to send.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct MailMessage {
pub to: Vec<String>,
pub subject: String,
pub body: String,
/// Whether `body` is HTML. Plain text otherwise.
pub html: bool,
}
/// The signed-in user, as the provider sees them.
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct Identity {
/// The provider's identifier for this account.
pub id: String,
/// What to show. Never an identity: two people can share a display name.
pub display: String,
pub email: Option<String>,
pub avatar: Option<String>,
}
+6
View File
@@ -81,6 +81,12 @@ Buffers are checked once per call, before any pixel is written.
the exact and float paths compare each pixel with white; all other colours are left as the
profiles give them. With a profile whose white falls between its nodes (`coated-cmyk`: 0.4 %
C), the last step to white is that much larger than its neighbours.
* Black preservation (opt-in, `BlackPreserving`): for CMYK → CMYK, a colour with exactly zero
C, M and Y comes out as exactly `(0, 0, 0, K')`, where `K'` is the destination black whose
K-only L* equals the source's (both K ramps sampled at 1024 points when the transform is
built, with its intent and BPC). The test is exact zero, so an input with any chromatic ink
takes the ordinary path. Absolute colorimetric, non-CMYK profiles and K ramps that reverse at
one of the 1024 samples are refused (a reversal between samples is not detected).
## Built-in profiles (all CC0-1.0, generated by this crate)
Binary file not shown.
+205
View File
@@ -0,0 +1,205 @@
//! K-only black preservation for CMYK → CMYK transforms.
//!
//! A CMYK → CMYK transform goes through the three-component PCS, so black alone (C = M = Y = 0)
//! comes out as a four-ink mix. ISO 32000-2 §8.6.5.7 notes that this loses fidelity in the black
//! component. ICC defines no black preservation; this is a named policy the caller opts into.
//!
//! [`BlackPreserving`] converts like its [`Transform`], except that a colour with exactly zero
//! cyan, magenta and yellow becomes exactly `(0, 0, 0, K')`. `K'` is the destination black whose
//! K-only L* equals the source's, both measured to Lab with the transform's own intent and BPC
//! setting: both ramps are sampled once, at construction, and the destination ramp is inverted.
//! A source black darker than the destination's maximum black maps to K = 1. The test is exact zero, so the transform is discontinuous at C = M = Y = 0: an input
//! with any chromatic ink takes the ordinary path.
use crate::transform::{Transform, TransformOptions};
use crate::{Builtin, CmsError, ColorSpace, Intent, Profile, invalid_arg};
/// Samples along each K-only ramp. Linear interpolation between samples of a smooth ramp is far
/// below the 16-bit quantum.
const K_SAMPLES: usize = 1024;
/// Slack for the monotonicity checks (L* units, K units).
const MONOTONIC_EPS: f64 = 1e-6;
/// A CMYK → CMYK transform that keeps black-only colours black-only.
#[derive(Debug)]
pub struct BlackPreserving {
transform: Transform,
/// Destination K for source K at `i / (K_SAMPLES - 1)`; `K_SAMPLES` values in `[0, 1]`.
curve: Vec<f32>,
/// Source K from which the source black is at least as dark as the destination's darkest:
/// mapped to exactly 1. Infinite when the destination's black is the darker.
saturate: f32,
/// The mapping at the 256 8-bit source codes, already quantised.
curve8: [u8; 256],
}
impl BlackPreserving {
/// Transform from `src` to `dst` with black preservation.
///
/// Errors with [`CmsError::InvalidArgument`] when either profile is not CMYK or the intent is
/// absolute colorimetric (which reproduces the source's paper and black, so there is no
/// black to map), and with [`CmsError::Unsupported`] when a profile's K-only ramp is not
/// monotonic at its 1024 samples (black that gets lighter as K rises has no equal-lightness
/// inverse; a reversal between samples is not detected). Other errors
/// are those of [`Transform::with_options`].
pub fn new(src: &Profile, dst: &Profile, opts: TransformOptions) -> Result<BlackPreserving, CmsError> {
if src.color_space != ColorSpace::Cmyk || dst.color_space != ColorSpace::Cmyk {
return Err(invalid_arg("black preservation needs CMYK source and destination profiles"));
}
if opts.intent == Intent::AbsoluteColorimetric {
return Err(invalid_arg("black preservation does not apply to absolute colorimetric"));
}
let transform = Transform::with_options(src, dst, opts)?;
let src_l = k_ramp_lightness(src, opts)?;
let dst_l = k_ramp_lightness(dst, opts)?;
let (curve, saturate) = k_curve(&src_l, &dst_l)?;
let mut curve8 = [0u8; 256];
for (code, slot) in curve8.iter_mut().enumerate() {
*slot = quantise(map(&curve, saturate, code as f32 / 255.0), 255.0) as u8;
}
Ok(BlackPreserving { transform, curve, saturate, curve8 })
}
/// The ordinary transform, which every colour with chromatic ink goes through. Converting with
/// it directly does not preserve black.
pub fn transform(&self) -> &Transform {
&self.transform
}
/// Destination K for a black-only source colour of `k` (clamped to `[0, 1]`).
pub fn map_k(&self, k: f32) -> f32 {
map(&self.curve, self.saturate, k)
}
/// Exact evaluation of one colour; slices as in [`Transform::eval`].
pub fn eval(&self, input: &[f32], out: &mut [f32]) -> Result<(), CmsError> {
self.transform.eval(input, out)?;
self.patch_one(input, out);
Ok(())
}
/// Lookup-table evaluation of one colour; slices as in [`Transform::eval`].
pub fn eval_fast(&self, input: &[f32], out: &mut [f32]) -> Result<(), CmsError> {
self.transform.eval_fast(input, out)?;
self.patch_one(input, out);
Ok(())
}
/// Converts float pixels; layout and errors as in [`Transform::convert_f32`].
pub fn convert_f32(&self, src: &[f32], src_stride: usize, dst: &mut [f32], dst_stride: usize, copy_extra: bool) -> Result<(), CmsError> {
self.transform.convert_f32(src, src_stride, dst, dst_stride, copy_extra)?;
for (s, d) in src.chunks_exact(src_stride).zip(dst.chunks_exact_mut(dst_stride)) {
self.patch_one(s, d);
}
Ok(())
}
/// Converts 8-bit pixels; layout and errors as in [`Transform::convert_u8`].
pub fn convert_u8(&self, src: &[u8], src_stride: usize, dst: &mut [u8], dst_stride: usize, copy_extra: bool) -> Result<(), CmsError> {
self.transform.convert_u8(src, src_stride, dst, dst_stride, copy_extra)?;
for (s, d) in src.chunks_exact(src_stride).zip(dst.chunks_exact_mut(dst_stride)) {
if let ([0, 0, 0, k, ..], [c, m, y, kd, ..]) = (s, d) {
(*c, *m, *y) = (0, 0, 0);
*kd = self.curve8[usize::from(*k)];
}
}
Ok(())
}
/// Converts 16-bit pixels; layout and errors as in [`Transform::convert_u16`].
pub fn convert_u16(&self, src: &[u16], src_stride: usize, dst: &mut [u16], dst_stride: usize, copy_extra: bool) -> Result<(), CmsError> {
self.transform.convert_u16(src, src_stride, dst, dst_stride, copy_extra)?;
for (s, d) in src.chunks_exact(src_stride).zip(dst.chunks_exact_mut(dst_stride)) {
if let ([0, 0, 0, k, ..], [c, m, y, kd, ..]) = (s, d) {
(*c, *m, *y) = (0, 0, 0);
*kd = quantise(self.map_k(f32::from(*k) / 65535.0), 65535.0) as u16;
}
}
Ok(())
}
fn patch_one(&self, input: &[f32], out: &mut [f32]) {
if let ([0.0, 0.0, 0.0, k, ..], [c, m, y, kd, ..]) = (input, out) {
(*c, *m, *y) = (0.0, 0.0, 0.0);
*kd = self.map_k(*k);
}
}
}
/// The sampled mapping from the source to the destination K-only ramp, and the source K at
/// which the destination's darkest black is reached (the `saturate` field).
fn k_curve(src_l: &[f64], dst_l: &[f64]) -> Result<(Vec<f32>, f32), CmsError> {
let curve: Vec<f32> = src_l.iter().map(|&l| invert(dst_l, l) as f32).collect();
if !curve.windows(2).all(|w| f64::from(w[0]) - f64::from(w[1]) <= MONOTONIC_EPS) {
return Err(CmsError::Unsupported("black preservation: the K mapping is not monotonic".into()));
}
let saturate = match (src_l.last(), dst_l.last()) {
(Some(&s), Some(&d)) if s < d => invert(src_l, d) as f32,
_ => f32::INFINITY,
};
Ok((curve, saturate))
}
/// L* of `p`'s K-only ramp at `K_SAMPLES` points, light to dark; refused when it is not
/// non-increasing.
fn k_ramp_lightness(p: &Profile, opts: TransformOptions) -> Result<Vec<f64>, CmsError> {
let to_lab = Transform::with_options(p, Builtin::LabD50.profile(), TransformOptions { precise_float: true, ..opts })?;
let mut ramp = Vec::with_capacity(K_SAMPLES);
let mut lab = [0.0f32; 3];
for i in 0..K_SAMPLES {
let k = i as f32 / (K_SAMPLES - 1) as f32;
to_lab.eval(&[0.0, 0.0, 0.0, k], &mut lab)?;
ramp.push(f64::from(lab[0]) * 100.0);
}
if !ramp.iter().all(|l| l.is_finite()) || !ramp.windows(2).all(|w| w[1] - w[0] <= MONOTONIC_EPS) {
return Err(CmsError::Unsupported("black preservation: a K-only ramp is not monotonic".into()));
}
Ok(ramp)
}
/// The K in `[0, 1]` at which the non-increasing `ramp` reaches `target`, clamped to its ends.
fn invert(ramp: &[f64], target: f64) -> f64 {
let (Some(&first), Some(&last)) = (ramp.first(), ramp.last()) else { return 0.0 };
if ramp.len() < 2 || target >= first {
return 0.0;
}
if target <= last {
return 1.0;
}
// ramp[lo] > target >= ramp[hi]
let (mut lo, mut hi) = (0usize, ramp.len() - 1);
while hi - lo > 1 {
let mid = lo + (hi - lo) / 2;
if ramp.get(mid).is_some_and(|&v| v > target) {
lo = mid;
} else {
hi = mid;
}
}
let (a, b) = (ramp.get(lo).copied().unwrap_or(first), ramp.get(hi).copied().unwrap_or(last));
let t = if a - b > MONOTONIC_EPS { (a - target) / (a - b) } else { 0.0 };
(lo as f64 + t) / (ramp.len() - 1) as f64
}
/// Linear interpolation in `curve` (samples over `[0, 1]`) at `k`, clamped; 1 from `saturate`.
fn map(curve: &[f32], saturate: f32, k: f32) -> f32 {
let Some(&last) = curve.last() else { return 0.0 };
let k = if k.is_nan() { 0.0 } else { k.clamp(0.0, 1.0) };
if k >= saturate {
return 1.0;
}
let x = k * (curve.len() - 1) as f32;
let i = x.floor() as usize;
match (curve.get(i), curve.get(i + 1)) {
(Some(&a), Some(&b)) => a + (b - a) * (x - i as f32),
_ => last,
}
}
fn quantise(v: f32, max: f32) -> f32 {
(v.clamp(0.0, 1.0) * max + 0.5).floor()
}
#[cfg(test)]
#[path = "black_tests.rs"]
mod tests;
+149
View File
@@ -0,0 +1,149 @@
use super::*;
fn coated() -> &'static Profile {
Builtin::CoatedCmyk.profile()
}
fn fogra() -> &'static Profile {
Builtin::CoatedFogra39.profile()
}
fn opts() -> TransformOptions {
TransformOptions::default()
}
/// L* of a CMYK colour under `p`, measured as the transform with `o` measures it.
fn lightness(p: &Profile, o: TransformOptions, cmyk: [f32; 4]) -> f32 {
let t = Transform::with_options(p, Builtin::LabD50.profile(), o).unwrap();
let mut lab = [0.0f32; 3];
t.eval(&cmyk, &mut lab).unwrap();
lab[0] * 100.0
}
#[test]
fn the_plain_transform_turns_black_into_four_inks() {
// What the policy is for: without it, black alone gains chromatic ink across presses.
let t = Transform::with_options(coated(), fogra(), opts()).unwrap();
let mut out = [0.0f32; 4];
t.eval(&[0.0, 0.0, 0.0, 0.5], &mut out).unwrap();
assert!(out[..3].iter().any(|v| *v > 0.01), "{out:?}");
}
#[test]
fn black_only_stays_black_only_on_every_path() {
for (src, dst) in [(coated(), fogra()), (fogra(), coated()), (fogra(), fogra())] {
let bp = BlackPreserving::new(src, dst, opts()).unwrap();
for k in [0.0f32, 0.1, 0.5, 0.9, 1.0] {
for fast in [false, true] {
let mut out = [1.0f32; 4];
if fast { bp.eval_fast(&[0.0, 0.0, 0.0, k], &mut out) } else { bp.eval(&[0.0, 0.0, 0.0, k], &mut out) }.unwrap();
assert_eq!(out[..3], [0.0; 3], "k {k} fast {fast}");
assert_eq!(out[3], bp.map_k(k));
}
}
let src8: Vec<u8> = (0..=255u8).flat_map(|k| [0, 0, 0, k, 7]).collect();
let mut dst8 = vec![0u8; src8.len()];
bp.convert_u8(&src8, 5, &mut dst8, 5, true).unwrap();
for (k, px) in dst8.as_chunks::<5>().0.iter().enumerate() {
assert_eq!(px[..3], [0; 3], "u8 k {k}");
assert!((f32::from(px[3]) - bp.map_k(k as f32 / 255.0) * 255.0).abs() <= 0.5, "u8 k {k}: {}", px[3]);
assert_eq!(px[4], 7, "extra channel copied");
}
let src16: Vec<u16> = (0..=64u16).flat_map(|k| [0, 0, 0, k * 1023]).collect();
let mut dst16 = vec![0u16; src16.len()];
bp.convert_u16(&src16, 4, &mut dst16, 4, false).unwrap();
for (px, k) in dst16.as_chunks::<4>().0.iter().zip(src16.as_chunks::<4>().0) {
assert_eq!(px[..3], [0; 3], "u16 {k:?}");
assert!((f32::from(px[3]) - bp.map_k(f32::from(k[3]) / 65535.0) * 65535.0).abs() <= 0.5, "u16 {k:?}: {}", px[3]);
}
let srcf: Vec<f32> = (0..=10).flat_map(|k| [0.0, 0.0, 0.0, k as f32 / 10.0]).collect();
let mut dstf = vec![0.0f32; srcf.len()];
bp.convert_f32(&srcf, 4, &mut dstf, 4, false).unwrap();
for (px, k) in dstf.as_chunks::<4>().0.iter().zip(srcf.as_chunks::<4>().0) {
assert_eq!(*px, [0.0, 0.0, 0.0, bp.map_k(k[3])], "f32 {k:?}");
}
}
}
#[test]
fn the_destination_black_matches_the_source_lightness() {
// Measured worst error over 201 K steps, both directions, both intents, BPC on and off:
// 0.007 L*.
for intent in [Intent::RelativeColorimetric, Intent::Perceptual] {
for bpc in [true, false] {
let o = TransformOptions { intent, bpc, ..opts() };
for (src, dst) in [(coated(), fogra()), (fogra(), coated())] {
let bp = BlackPreserving::new(src, dst, o).unwrap();
let dst_black = lightness(dst, o, [0.0, 0.0, 0.0, 1.0]);
for k in [0.1f32, 0.3, 0.5, 0.7, 0.9, 1.0] {
let want = lightness(src, o, [0.0, 0.0, 0.0, k]);
if want <= dst_black {
assert_eq!(bp.map_k(k), 1.0, "darker than the destination's black clamps to K = 1");
continue;
}
let got = lightness(dst, o, [0.0, 0.0, 0.0, bp.map_k(k)]);
assert!((got - want).abs() < 0.02, "{intent:?} bpc {bpc} k {k}: L* {got} for {want}");
}
assert_eq!(bp.map_k(0.0), 0.0, "paper stays paper");
assert!(bp.map_k(1.0) > 0.95, "full black stays near full black: {}", bp.map_k(1.0));
assert!(bp.map_k(0.25) < bp.map_k(0.75));
}
}
}
let same = BlackPreserving::new(fogra(), fogra(), opts()).unwrap();
for k in [0.0f32, 0.2, 0.6, 1.0] {
assert!((same.map_k(k) - k).abs() < 1e-3, "{k} -> {}", same.map_k(k));
}
}
#[test]
fn colours_with_any_chromatic_ink_take_the_ordinary_path() {
let bp = BlackPreserving::new(coated(), fogra(), opts()).unwrap();
for input in [[1e-6f32, 0.0, 0.0, 0.5], [0.0, 0.2, 0.0, 0.5], [0.3, 0.3, 0.3, 0.0], [0.0, 0.0, 1.0, 1.0]] {
let (mut a, mut b) = ([0.0f32; 4], [0.0f32; 4]);
bp.eval(&input, &mut a).unwrap();
bp.transform().eval(&input, &mut b).unwrap();
assert_eq!(a, b, "{input:?}");
}
let src = [1u8, 0, 0, 128, 0, 1, 0, 128];
let (mut a, mut b) = ([0u8; 8], [0u8; 8]);
bp.convert_u8(&src, 4, &mut a, 4, false).unwrap();
bp.transform().convert_u8(&src, 4, &mut b, 4, false).unwrap();
assert_eq!(a, b);
}
#[test]
fn refusals_and_hostile_values() {
let srgb = Builtin::Srgb.profile();
assert!(matches!(BlackPreserving::new(srgb, fogra(), opts()), Err(CmsError::InvalidArgument(_))));
assert!(matches!(BlackPreserving::new(fogra(), srgb, opts()), Err(CmsError::InvalidArgument(_))));
let abs = TransformOptions { intent: Intent::AbsoluteColorimetric, ..opts() };
assert!(matches!(BlackPreserving::new(fogra(), coated(), abs), Err(CmsError::InvalidArgument(_))));
let bp = BlackPreserving::new(coated(), fogra(), opts()).unwrap();
for k in [f32::NAN, f32::INFINITY, f32::NEG_INFINITY, -1.0, 2.0] {
assert!((0.0..=1.0).contains(&bp.map_k(k)), "{k}");
}
let mut out = [0.0f32; 4];
assert!(bp.eval(&[0.0; 3], &mut out).is_err(), "short input");
assert!(bp.convert_u8(&[0, 0, 0, 9, 0], 4, &mut [0u8; 4], 4, false).is_err(), "partial pixel");
assert_eq!(invert(&[], 50.0), 0.0);
assert_eq!(invert(&[100.0], 50.0), 0.0);
assert_eq!(invert(&[100.0, 0.0], 150.0), 0.0);
assert_eq!(invert(&[100.0, 0.0], -5.0), 1.0);
assert!((invert(&[100.0, 50.0, 0.0], 25.0) - 0.75).abs() < 1e-12);
assert_eq!(map(&[], f32::INFINITY, 0.5), 0.0);
}
#[test]
fn black_beyond_the_destination_maximum_maps_to_full_black_between_samples() {
// Source L* = 100 (1 - K), destination L* = 100 - 50 K: from K = 0.5 the source is darker
// than any destination black, and 0.5 falls between the 1024 samples.
let ramp = |f: fn(f64) -> f64| (0..K_SAMPLES).map(|i| f(i as f64 / (K_SAMPLES - 1) as f64)).collect::<Vec<_>>();
let (src, dst) = (ramp(|k| 100.0 * (1.0 - k)), ramp(|k| 100.0 - 50.0 * k));
let (curve, saturate) = k_curve(&src, &dst).unwrap();
assert!((saturate - 0.5).abs() < 1e-6, "{saturate}");
for k in [0.5001f32, 0.6, 1.0] {
assert_eq!(map(&curve, saturate, k), 1.0, "{k}");
}
assert!((map(&curve, saturate, 0.25) - 0.5).abs() < 1e-4);
let (_, never) = k_curve(&dst, &src).unwrap();
assert_eq!(never, f32::INFINITY, "a lighter source never saturates");
}
+2
View File
@@ -19,6 +19,7 @@
//! CIE 15, and Adobe's published black point compensation paper.
#![forbid(unsafe_code)]
pub mod black;
pub mod builtin;
mod builtin_v2;
pub mod clut;
@@ -39,6 +40,7 @@ pub mod transform;
mod write;
pub mod write_v2;
pub use black::BlackPreserving;
pub use builtin::Builtin;
pub use clut::Clut;
pub use curve::Curve;
+322 -15
View File
@@ -19,6 +19,9 @@
//! L* is scaled onto the device black, chroma is softly compressed against a gamut boundary
//! descriptor sampled from the forward model — then solved as above.
//!
//! That is [`GamutMapping::Legacy`]. [`GamutMapping::Oklab`] maps out-of-gamut colours in Oklab
//! instead and gives saturation its own table; the Coated FOGRA39 built-in uses it.
//!
//! The model is plausible for coated offset, not a measured characterisation: use a real
//! profile from your print provider for production print.
//!
@@ -49,6 +52,11 @@ pub struct CmykParams {
pub k_max: f64,
/// Exponent of the black generation curve above `gcr_start`.
pub gcr_gamma: f64,
/// Black generation as a curve of `[gray component, black]` points with increasing gray
/// components in `[0, 1]`, interpolated linearly and held flat beyond the ends. Empty by
/// default: black is then `k_max` times the ramp of `gcr_start` and `gcr_gamma` (the curve
/// of the shipped Photocraft profile).
pub gcr_curve: Vec<[f64; 2]>,
/// Tone value increase at 50 % for C, M, Y, K.
pub tvi: [f64; 4],
/// Grid points of the AToB (4D) and BToA (3D) tables.
@@ -60,6 +68,32 @@ pub struct CmykParams {
/// encoding and separates to a trace of ink. Off by default, which keeps the bytes of the
/// shipped Photocraft profile.
pub white_on_node: bool,
/// How the BToA tables bring colours into the gamut. [`GamutMapping::Legacy`] by default,
/// which keeps the bytes of the shipped Photocraft profile.
pub mapping: GamutMapping,
}
/// How the BToA tables map colours the printer cannot reach.
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum GamutMapping {
/// The shipped Photocraft profile: out-of-gamut colours land on the closest reachable
/// colour (ΔE76); perceptual scales L* onto the device black and compresses Lab chroma;
/// saturation shares the perceptual table.
Legacy,
/// Gamut mapping in Oklab, whose hue lines stay perceptually constant (a CIELAB hue line
/// from a saturated blue toward grey drifts toward purple; an Oklab one does not):
///
/// * **Colorimetric**: in-gamut colours separate exactly. An out-of-gamut colour takes the
/// closest colour the printer reaches in Oklab, lightness and hue weighted above chroma,
/// and more so the further outside it is; black is generated from the clipped colour, so
/// inks the clip drives to 100 % stay there.
/// * **Perceptual** and **saturation**: the ICC v4 perceptual reference black is mapped
/// onto the device black linearly in XYZ (as black point compensation does) for neutrals,
/// fading out with Oklab chroma, so greys separate as with colorimetric + BPC and colourful
/// colours keep their lightness. Perceptual then compresses Oklab chroma above `knee` ×
/// the gamut boundary into the boundary; saturation scales Oklab chroma by `saturation`
/// and clips as colorimetric does.
Oklab { knee: f64, saturation: f64 },
}
impl Default for CmykParams {
@@ -71,10 +105,12 @@ impl Default for CmykParams {
gcr_start: 0.2,
k_max: 0.95,
gcr_gamma: 1.1,
gcr_curve: Vec::new(),
tvi: [0.14, 0.14, 0.14, 0.17],
grid_a2b: 11,
grid_b2a: 21,
white_on_node: false,
mapping: GamutMapping::Legacy,
}
}
}
@@ -183,6 +219,11 @@ impl Forward for TableForward {
/// Bounded least-squares solve for C, M, Y given K, minimising ΔE76 to `target`, subject to
/// `C + M + Y ≤ limit` (projected Levenberg–Marquardt with a numerical Jacobian).
fn solve_cmy<F: Forward + ?Sized>(model: &F, target: [f64; 3], k: f64, limit: f64, seeds: &[[f64; 3]]) -> ([f64; 3], f64) {
solve_cmy_by(model, k, limit, seeds, |l| [l[0] - target[0], l[1] - target[1], l[2] - target[2]])
}
/// [`solve_cmy`] minimising the norm of `residual(lab)` instead of ΔE76.
fn solve_cmy_by<F: Forward + ?Sized>(model: &F, k: f64, limit: f64, seeds: &[[f64; 3]], residual: impl Fn([f64; 3]) -> [f64; 3]) -> ([f64; 3], f64) {
let project = |mut x: [f64; 3]| {
for _ in 0..4 {
for v in &mut x {
@@ -202,10 +243,7 @@ fn solve_cmy<F: Forward + ?Sized>(model: &F, target: [f64; 3], k: f64, limit: f6
}
x
};
let err = |x: [f64; 3]| {
let l = model.lab([x[0], x[1], x[2], k]);
[l[0] - target[0], l[1] - target[1], l[2] - target[2]]
};
let err = |x: [f64; 3]| residual(model.lab([x[0], x[1], x[2], k]));
let norm2 = |e: [f64; 3]| e[0] * e[0] + e[1] * e[1] + e[2] * e[2];
let mut best = ([0.0; 3], f64::INFINITY);
for seed in seeds {
@@ -275,13 +313,32 @@ fn solve_cmy<F: Forward + ?Sized>(model: &F, target: [f64; 3], k: f64, limit: f6
const SEEDS: [[f64; 3]; 5] = [[0.3, 0.3, 0.3], [0.05, 0.05, 0.05], [0.8, 0.8, 0.8], [0.9, 0.2, 0.2], [0.2, 0.9, 0.9]];
/// Black for a gray component `g` (the smallest of C, M, Y without black).
fn gcr(p: &CmykParams, g: f64) -> f64 {
let Some(first) = p.gcr_curve.first() else {
// The Photocraft profile's curve.
return if g <= p.gcr_start { 0.0 } else { p.k_max * ((g - p.gcr_start) / (1.0 - p.gcr_start)).powf(p.gcr_gamma) };
};
let mut k = first[1];
for w in p.gcr_curve.windows(2) {
let ([g0, k0], [g1, k1]) = (w[0], w[1]);
if g >= g1 {
k = k1;
} else if g > g0 {
k = k0 + (k1 - k0) * (g - g0) / (g1 - g0);
break;
}
}
k.clamp(0.0, 1.0)
}
/// Colorimetric separation of a relative Lab target: GCR black generation then a CMY solve.
pub fn separate<F: Forward + ?Sized>(model: &F, p: &CmykParams, target: [f64; 3], warm: Option<[f64; 4]>) -> ([f64; 4], f64) {
let mut seeds: Vec<[f64; 3]> = warm.map(|w| vec![[w[0], w[1], w[2]]]).unwrap_or_default();
seeds.extend_from_slice(&SEEDS);
let (cmy0, _) = solve_cmy(model, target, 0.0, 3.0, &seeds);
let g = cmy0[0].min(cmy0[1]).min(cmy0[2]);
let k = if g <= p.gcr_start { 0.0 } else { p.k_max * ((g - p.gcr_start) / (1.0 - p.gcr_start)).powf(p.gcr_gamma) };
let k = gcr(p, g);
let mut seeds2 = vec![[(cmy0[0] - k).max(0.0), (cmy0[1] - k).max(0.0), (cmy0[2] - k).max(0.0)], cmy0];
seeds2.extend_from_slice(&seeds);
let (cmy, de) = solve_cmy(model, target, k, (p.tac - k).max(0.0), &seeds2);
@@ -383,6 +440,187 @@ impl Gbd {
}
}
/// ΔE76 under which a separation counts as reaching its target.
const REACH: f64 = 1.5;
/// Oklab chroma beyond any printable colour: the top of the boundary search.
const OKLAB_CHROMA_MAX: f64 = 0.5;
/// Oklab chroma above which the perceptual and saturation tables no longer lift black: they lift
/// neutrals fully, and less the more colourful a colour is.
const LIFT_CHROMA: f64 = 0.1;
/// Weight of lightness and hue against chroma when a colour just outside the gamut is clipped.
const CLIP_WEIGHT: f64 = 1.5;
/// Growth of that weight with the squared share of chroma the clip loses.
const CLIP_WEIGHT_GROWTH: f64 = 200.0;
// Oklab (Björn Ottosson, 2020): XYZ (D65) → cone response → cube root → Lab-like axes.
const OKLAB_M1: math::Mat3 =
[[0.8189330101, 0.3618667424, -0.1288597137], [0.0329845436, 0.9293118715, 0.0361456387], [0.0482003018, 0.2643662691, 0.6338517070]];
const OKLAB_M2: math::Mat3 =
[[0.2104542553, 0.7936177850, -0.0040720468], [1.9779984951, -2.4285922050, 0.4505937099], [0.0259040371, 0.7827717662, -0.8086757660]];
const OKLAB_M1_INV: math::Mat3 =
[[1.2270138511, -0.5577999807, 0.2812561490], [-0.0405801784, 1.1122568696, -0.0716766787], [-0.0763812845, -0.4214819784, 1.5861632204]];
const OKLAB_M2_INV: math::Mat3 = [[1.0, 0.3963377774, 0.2158037573], [1.0, -0.1055613458, -0.0638541728], [1.0, -0.0894841775, -1.2914855480]];
/// The [`GamutMapping::Oklab`] separations of one forward model.
struct OklabMapper<'a, F: Forward + ?Sized> {
model: &'a F,
p: &'a CmykParams,
knee: f64,
saturation: f64,
/// XYZ scale that maps the v4 perceptual reference black onto the device black (the white
/// stays fixed): `v' = a·v + w·(1 − a)`.
black_scale: f64,
to_d65: math::Mat3,
to_d50: math::Mat3,
}
impl<'a, F: Forward + ?Sized> OklabMapper<'a, F> {
fn new(model: &'a F, p: &'a CmykParams, knee: f64, saturation: f64) -> Self {
let d65 = math::xy_to_xyz(math::D65_XY);
// The darkest neutral the printer reaches.
let (black, _) = separate(model, p, [0.0, 0.0, 0.0], None);
let device = math::lab_to_xyz([model.lab(black)[0], 0.0, 0.0], math::D50)[1];
let reference = crate::transform::PERCEPTUAL_BLACK[1];
OklabMapper {
model,
p,
knee,
saturation,
black_scale: (1.0 - device) / (1.0 - reference),
to_d65: math::bradford(math::D50, d65),
to_d50: math::bradford(d65, math::D50),
}
}
fn oklab(&self, lab: [f64; 3]) -> [f64; 3] {
let xyz = math::apply(&self.to_d65, math::lab_to_xyz(lab, math::D50));
math::apply(&OKLAB_M2, math::apply(&OKLAB_M1, xyz).map(f64::cbrt))
}
fn lab(&self, ok: [f64; 3]) -> [f64; 3] {
let lms = math::apply(&OKLAB_M2_INV, ok).map(|v| v * v * v);
math::xyz_to_lab(math::apply(&self.to_d50, math::apply(&OKLAB_M1_INV, lms)), math::D50)
}
/// The most chroma (scale of `ok`'s chroma in `[0, top]`) the printer reaches at `ok`'s
/// lightness and hue, and its separation; `None` when even the neutral is out of reach.
fn boundary(&self, ok: [f64; 3], top: f64, warm: Option<[f64; 4]>) -> Option<(f64, [f64; 4])> {
let (grey, d) = separate(self.model, self.p, self.lab([ok[0], 0.0, 0.0]), warm);
if d > REACH {
return None;
}
let (mut lo, mut hi, mut best) = (0.0, top, grey);
for _ in 0..14 {
let mid = 0.5 * (lo + hi);
let (c, d) = separate(self.model, self.p, self.lab([ok[0], ok[1] * mid, ok[2] * mid]), Some(best));
if d <= REACH {
lo = mid;
best = c;
} else {
hi = mid;
}
}
Some((lo, best))
}
/// Colorimetric: exact in gamut; outside, the closest colour the printer reaches in Oklab,
/// lightness and hue weighted above chroma, and more so the further outside the colour is.
fn clip(&self, lab: [f64; 3], warm: Option<[f64; 4]>) -> [f64; 4] {
let (cmyk, de) = separate(self.model, self.p, lab, warm);
if de <= REACH {
return cmyk;
}
let ok = self.oklab(lab);
let c0 = ok[1].hypot(ok[2]);
// Unit vectors along the colour's chroma and hue directions.
let (u, v) = if c0 > 1e-9 { ([ok[1] / c0, ok[2] / c0], [-ok[2] / c0, ok[1] / c0]) } else { ([1.0, 0.0], [0.0, 1.0]) };
// Lightness and hue errors weighted by `w`, chroma error by 1; scaled by 100 so the
// solver's step tolerances suit it as they suit ΔE76.
let residual = |w: f64| {
move |l: [f64; 3]| {
let o = self.oklab(l);
let (da, db) = (o[1] - ok[1], o[2] - ok[2]);
[100.0 * w * (o[0] - ok[0]), 100.0 * (da * u[0] + db * u[1]), 100.0 * w * (da * v[0] + db * v[1])]
}
};
let solve = |w: f64, k: f64, from: [f64; 3]| {
let mut seeds = vec![from];
seeds.extend_from_slice(&SEEDS);
solve_cmy_by(self.model, k, (self.p.tac - k).max(0.0), &seeds, residual(w)).0
};
// A first clip without black measures how far outside the colour is. A colour just
// outside may give up a little lightness and hue to keep its chroma (a red reaches the
// full magenta and yellow edge); one far outside keeps its lightness and hue and loses
// chroma (a saturated blue does not slide along the gamut surface).
let first = solve(CLIP_WEIGHT, 0.0, [cmyk[0], cmyk[1], cmyk[2]]);
let o = self.oklab(self.model.lab([first[0], first[1], first[2], 0.0]));
let loss = if c0 > 1e-9 { (1.0 - o[1].hypot(o[2]) / c0).clamp(0.0, 1.0) } else { 0.0 };
let w = CLIP_WEIGHT * (1.0 + CLIP_WEIGHT_GROWTH * loss * loss);
let free = solve(w, 0.0, first);
let norm = |c: [f64; 4]| residual(w)(self.model.lab(c)).iter().map(|v| v * v).sum::<f64>();
// Candidates, the closest kept: the separation itself (a colour it nearly reaches stays
// continuous with its in-gamut neighbours), the clip with the separation's black, and
// the clip with black generated from the black-free clip's gray component (inks the
// clip drives to 100 % stay there and black replaces the rest).
let mut best = (norm(cmyk), cmyk);
for k in [cmyk[3], gcr(self.p, free[0].min(free[1]).min(free[2]))] {
for from in [[cmyk[0], cmyk[1], cmyk[2]], free] {
let cmy = solve(w, k, from);
let c = [cmy[0], cmy[1], cmy[2], k];
let n = norm(c);
if n < best.0 {
best = (n, c);
}
}
}
// Near paper white the gamut narrows to a point: the separation's closest colour keeps
// light tints' chroma where holding lightness would drop them toward bare paper, and
// the table nodes there carry the in-gamut tints between them. The inks blend from the
// clip at L* 85 to the separation at L* 95.
let clip_share = ((95.0 - lab[0]) / 10.0).clamp(0.0, 1.0);
std::array::from_fn(|i| clip_share * best.1[i] + (1.0 - clip_share) * cmyk[i])
}
/// Maps the v4 perceptual reference black onto the device black, linearly in XYZ, for
/// neutrals; colourful colours keep their lightness.
fn lift_black(&self, lab: [f64; 3]) -> [f64; 3] {
let ok = self.oklab(lab);
let neutral = (1.0 - ok[1].hypot(ok[2]) / LIFT_CHROMA).clamp(0.0, 1.0);
let a = 1.0 - neutral * (1.0 - self.black_scale);
let xyz = math::lab_to_xyz(lab, math::D50);
math::xyz_to_lab(std::array::from_fn(|i| a * xyz[i] + math::D50[i] * (1.0 - a)), math::D50)
}
fn perceptual(&self, lab: [f64; 3], warm: Option<[f64; 4]>) -> [f64; 4] {
let lab = self.lift_black(lab);
let ok = self.oklab(lab);
let c = ok[1].hypot(ok[2]);
if c < 1e-9 {
return self.clip(lab, warm);
}
let Some((s, edge)) = self.boundary(ok, OKLAB_CHROMA_MAX / c, warm) else {
return self.clip(lab, warm);
};
// Chroma above the knee is compressed smoothly into the boundary chroma `cb`.
let cb = s * c;
let knee = self.knee * cb;
if c <= knee {
return self.clip(lab, warm);
}
let c2 = knee + (cb - knee) * ((c - knee) / (cb - knee).max(1e-9)).tanh();
let f = c2 / c;
if f >= s {
return edge;
}
self.clip(self.lab([ok[0], ok[1] * f, ok[2] * f]), Some(edge))
}
fn saturation(&self, lab: [f64; 3], warm: Option<[f64; 4]>) -> [f64; 4] {
let ok = self.oklab(self.lift_black(lab));
self.clip(self.lab([ok[0], ok[1] * self.saturation, ok[2] * self.saturation]), warm)
}
}
fn lab_node_legacy(i: usize, n: usize) -> f64 {
i as f64 / (n - 1) as f64
}
@@ -419,6 +657,18 @@ fn white_node_curve(l_axis: bool, x: f64) -> f64 {
}
fn b2a_table<F: Forward + ?Sized>(model: &F, p: &CmykParams, gbd: Option<&Gbd>) -> Result<Clut, CmsError> {
b2a_table_with(p, |lab, warm| {
let target = match gbd {
Some(g) => g.compress(lab),
None => lab,
};
separate(model, p, target, warm).0
})
}
/// Samples a BToA table: `node` separates each relative Lab node, warm-started from the
/// previous node along b.
fn b2a_table_with(p: &CmykParams, node: impl Fn([f64; 3], Option<[f64; 4]>) -> [f64; 4] + Sync) -> Result<Clut, CmsError> {
let n = p.grid_b2a.max(2);
let nodes = n * n * n;
let step = (n - 1) as f64;
@@ -434,11 +684,7 @@ fn b2a_table<F: Forward + ?Sized>(model: &F, p: &CmykParams, gbd: Option<&Gbd>)
} else {
decode_v2([lab_node_legacy(li, n), lab_node_legacy(ai, n), lab_node_legacy(bi, n)])
};
let target = match gbd {
Some(g) => g.compress(lab),
None => lab,
};
let (cmyk, _) = if p.white_on_node && lab == [100.0, 0.0, 0.0] { ([0.0; 4], 0.0) } else { separate(model, p, target, warm) };
let cmyk = if p.white_on_node && lab == [100.0, 0.0, 0.0] { [0.0; 4] } else { node(lab, warm) };
warm = Some(cmyk);
out.extend(cmyk.map(|v| v as f32));
}
@@ -460,6 +706,15 @@ fn check_grids(p: &CmykParams) -> Result<(), CmsError> {
if p.white_on_node && p.grid_b2a.is_multiple_of(2) {
return Err(invalid_arg(format!("white on a node needs an odd BToA grid, not {}", p.grid_b2a)));
}
let valid = |v: &[f64; 2]| v.iter().all(|x| (0.0..=1.0).contains(x));
if !p.gcr_curve.iter().all(valid) || p.gcr_curve.windows(2).any(|w| w[1][0] <= w[0][0]) {
return Err(invalid_arg(format!("black generation curve {:?} needs points in [0, 1] with increasing gray components", p.gcr_curve)));
}
if let GamutMapping::Oklab { knee, saturation } = p.mapping
&& !((0.0..1.0).contains(&knee) && saturation.is_finite() && saturation > 0.0)
{
return Err(invalid_arg(format!("Oklab gamut mapping needs a knee in [0, 1) and a positive saturation, not {knee} and {saturation}")));
}
Ok(())
}
@@ -518,11 +773,23 @@ pub fn cmyk_profile_from_model<F: Forward + ?Sized>(model: &F, paper: [f64; 3],
}
fn lut_profile<F: Forward + ?Sized>(model: &F, a2b: Clut, paper: [f64; 3], p: &CmykParams) -> Result<Profile, CmsError> {
let gbd = Gbd::new(model, p);
let colorimetric = b2a_table(model, p, None)?;
let perceptual = b2a_table(model, p, Some(&gbd))?;
let (perceptual, colorimetric, saturation) = match p.mapping {
GamutMapping::Legacy => {
let gbd = Gbd::new(model, p);
let colorimetric = b2a_table(model, p, None)?;
let perceptual = b2a_table(model, p, Some(&gbd))?;
(perceptual.clone(), colorimetric, perceptual)
}
GamutMapping::Oklab { knee, saturation } => {
let m = OklabMapper::new(model, p, knee, saturation);
let colorimetric = b2a_table_with(p, |lab, warm| m.clip(lab, warm))?;
let perceptual = b2a_table_with(p, |lab, warm| m.perceptual(lab, warm))?;
let saturation = b2a_table_with(p, |lab, warm| m.saturation(lab, warm))?;
(perceptual, colorimetric, saturation)
}
};
let a2b = lut16(a2b, false);
let (b0, b1) = (lut16(perceptual, p.white_on_node), lut16(colorimetric, p.white_on_node));
let (b0, b1, b2) = (lut16(perceptual, p.white_on_node), lut16(colorimetric, p.white_on_node), lut16(saturation, p.white_on_node));
Ok(Profile {
version: (4, 0x30),
class: ProfileClass::Output,
@@ -538,7 +805,7 @@ fn lut_profile<F: Forward + ?Sized>(model: &F, a2b: Clut, paper: [f64; 3], p: &C
trc: None,
gray_trc: None,
a2b: [Some(a2b.clone()), Some(a2b.clone()), Some(a2b)],
b2a: [Some(b0.clone()), Some(b1), Some(b0)],
b2a: [Some(b0), Some(b1), Some(b2)],
bytes: None,
hash: Default::default(),
}
@@ -576,6 +843,46 @@ mod tests {
}
}
#[test]
fn oklab_mapping_refuses_bad_parameters() {
for (knee, saturation) in [(1.0, 1.0), (-0.1, 1.0), (f64::NAN, 1.0), (0.5, 0.0), (0.5, f64::INFINITY), (0.5, f64::NAN)] {
let p = CmykParams { grid_a2b: 2, grid_b2a: 2, mapping: GamutMapping::Oklab { knee, saturation }, ..CmykParams::default() };
assert!(cmyk_profile(&p).is_err(), "knee {knee}, saturation {saturation}");
}
}
#[test]
fn black_generation_curve() {
let p = CmykParams { gcr_curve: vec![[0.0, 0.0], [0.5, 0.2], [1.0, 0.9]], ..CmykParams::default() };
for (g, k) in [(-1.0, 0.0), (0.0, 0.0), (0.25, 0.1), (0.5, 0.2), (0.75, 0.55), (1.0, 0.9), (2.0, 0.9)] {
assert!((gcr(&p, g) - k).abs() < 1e-12, "g {g}: {} != {k}", gcr(&p, g));
}
for curve in [vec![[0.5, 0.2], [0.5, 0.3]], vec![[0.5, 0.2], [0.2, 0.3]], vec![[0.0, 1.5]], vec![[f64::NAN, 0.0]]] {
let p = CmykParams { grid_a2b: 2, grid_b2a: 2, gcr_curve: curve.clone(), ..CmykParams::default() };
assert!(cmyk_profile(&p).is_err(), "{curve:?}");
}
}
#[test]
fn oklab_round_trips() {
let m = CmykModel::coated(CmykParams::default().tvi);
let p = CmykParams::default();
let mapper = OklabMapper::new(&m, &p, 0.65, 1.05);
for lab in [[50.0, 0.0, 0.0], [44.0, 63.7, 47.0], [31.9, 32.9, -76.8], [90.0, -5.0, 80.0]] {
let back = mapper.lab(mapper.oklab(lab));
// The published matrices carry 10 digits, so the inverses are exact to about 1e-5 ΔE.
assert!(math::delta_e76(lab, back) < 1e-3, "{lab:?} -> {back:?}");
}
}
#[test]
fn oklab_profile_has_three_tables() {
let p = CmykParams { grid_a2b: 5, grid_b2a: 5, mapping: GamutMapping::Oklab { knee: 0.65, saturation: 1.05 }, ..CmykParams::default() };
let profile = cmyk_profile(&p).unwrap();
let [Some(b0), Some(b1), Some(b2)] = &profile.b2a else { panic!("missing BToA tables") };
assert!(b0 != b1 && b0 != b2, "perceptual shares a table");
}
#[test]
fn separation_hits_in_gamut_targets() {
let m = CmykModel::coated(CmykParams::default().tvi);
+1 -1
View File
@@ -46,7 +46,7 @@ const OUT_TABLE: usize = GpuMatrixShaper::OUTPUT_ENTRIES;
const _: () = assert!(OUT_TABLE.is_power_of_two());
/// ICC v4 perceptual reference medium black (ICC.1:2010 §6.3.4.4), XYZ.
const PERCEPTUAL_BLACK: [f64; 3] = [0.00336, 0.0034731, 0.00287];
pub(crate) const PERCEPTUAL_BLACK: [f64; 3] = [0.00336, 0.0034731, 0.00287];
/// Accelerated representation of a transform.
#[derive(Debug)]
+60 -1
View File
@@ -12,7 +12,7 @@
//! ([`synth::cmyk_profile_from_model`]).
use craft_cms::math::{self, delta_e2000};
use craft_cms::synth::{self, CmykParams, Forward};
use craft_cms::synth::{self, CmykParams, Forward, GamutMapping};
use craft_cms::{Builtin, Intent, Profile, Transform};
const DATA: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/data/FOGRA39.txt");
@@ -346,8 +346,10 @@ fn regen_coated_fogra39() {
description: Builtin::CoatedFogra39.description().into(),
copyright: craft_cms::builtin::COATED_FOGRA39_COPYRIGHT.into(),
tac: 3.3,
gcr_curve: vec![[0.0, 0.0], [0.2, 0.05], [0.5, 0.205], [0.67, 0.5], [1.0, 0.95]],
grid_b2a: 33,
white_on_node: true,
mapping: GamutMapping::Oklab { knee: 0.65, saturation: 1.05 },
..CmykParams::default()
};
let profile = synth::cmyk_profile_from_model(&f, paper(&p), &params).unwrap();
@@ -564,3 +566,60 @@ fn stand_in_separations_respect_the_ink_limit() {
}
}
}
/// sRGB patches and their separations by the reference application (Convert to Profile to
/// Coated FOGRA39, dither on), CMYK % per intent: relative + BPC, relative, perceptual,
/// saturation, absolute. Measured for storytold/photocraft#14267.
const REFERENCE_SEPARATIONS: [([u8; 3], [[f32; 4]; 5]); 3] = [
(
[200, 30, 30],
[[14.5, 97.7, 95.3, 4.3], [13.3, 100.0, 100.0, 5.5], [18.0, 96.9, 92.9, 8.6], [14.9, 100.0, 100.0, 5.9], [8.6, 100.0, 100.0, 2.4]],
),
([30, 60, 200], [[91.8, 75.3, 0.0, 0.0], [94.5, 76.5, 0.0, 0.0], [94.5, 77.7, 0.0, 0.0], [93.7, 76.9, 0.0, 0.0], [92.6, 74.9, 0.0, 0.0]]),
(
[128, 128, 128],
[[49.0, 38.4, 38.4, 20.0], [49.8, 39.2, 38.8, 21.2], [49.0, 38.4, 38.4, 20.0], [49.0, 38.4, 38.4, 20.0], [46.7, 36.9, 38.4, 18.4]],
),
];
const SEPARATION_INTENTS: [(Intent, bool); 5] = [
(Intent::RelativeColorimetric, true),
(Intent::RelativeColorimetric, false),
(Intent::Perceptual, true),
(Intent::Saturation, true),
(Intent::AbsoluteColorimetric, false),
];
fn separation(intent: Intent, bpc: bool, rgb: [u8; 3]) -> [f32; 4] {
let t = Transform::new(Builtin::Srgb.profile(), fogra39(), intent, bpc).unwrap();
let mut o = [0.0f32; 4];
t.eval(&rgb.map(|v| v as f32 / 255.0), &mut o).unwrap();
o.map(|v| v * 100.0)
}
/// sRGB → Coated FOGRA39 matches the reference application's inks on a red, a saturated blue
/// and a mid grey for every intent. Most land within 2 % per ink; red at the magenta–yellow
/// edge lands between table nodes that are and are not clipped, which costs up to 2.2 % with
/// the 33³ BToA tables.
#[test]
fn separates_like_the_reference_application() {
let mut worst = 0.0f32;
for (rgb, want) in REFERENCE_SEPARATIONS {
for ((intent, bpc), want) in SEPARATION_INTENTS.into_iter().zip(want) {
let got = separation(intent, bpc, rgb);
let diff = (0..4).map(|i| (got[i] - want[i]).abs()).fold(0.0f32, f32::max);
eprintln!("{rgb:?} {intent:?} bpc {bpc}: {got:.1?} reference {want:?} (max {diff:.1})");
worst = worst.max(diff);
}
}
assert!(worst <= 2.5, "worst ink difference {worst} %");
}
/// Saturation has its own table: a red separates with more magenta and yellow than
/// perceptual, whose chroma compression takes some back.
#[test]
fn saturation_and_perceptual_separate_differently() {
let red = [200, 30, 30];
let (perceptual, saturation) = (separation(Intent::Perceptual, true, red), separation(Intent::Saturation, true, red));
assert!(saturation[1] > perceptual[1] + 1.0 && saturation[2] > perceptual[2] + 1.0, "perceptual {perceptual:?}, saturation {saturation:?}");
}
+17
View File
@@ -0,0 +1,17 @@
[package]
name = "craft-evm"
description = "Read-only EVM bytecode dispatcher and compiler metadata inspection"
version.workspace = true
edition.workspace = true
license.workspace = true
rust-version.workspace = true
repository.workspace = true
[lints]
workspace = true
[dependencies]
serde = { workspace = true }
serde_json = { workspace = true }
thiserror = { workspace = true }
[dev-dependencies]
craft-testkit = { workspace = true }
+7
View File
@@ -0,0 +1,7 @@
# craft-evm
Read-only EVM bytecode inspection: strict raw/hex admission, opcode listing, dispatcher selector and jump-pattern facts, and Solidity/Vyper CBOR metadata. No VM execution, RPC, wallet, signing or transaction support.
Preserves Craft Anything's parser behavior and limits: 16 MiB decoded bytecode, 4,096 reported functions and 400 listing entries, plus existing bounded CBOR recursion/collection parsing. These are not a total allocator-byte guarantee. Dispatcher facts are heuristics, not a full control-flow or semantic proof. EOF containers remain unsupported.
Application file I/O, evidence policy and selector naming remain outside this crate.
+895
View File
@@ -0,0 +1,895 @@
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
#![forbid(unsafe_code)]
//! EVM bytecode: the public function selectors a contract's dispatcher compares against, where
//! each one jumps, whether it refuses payments, fallback/receive presence, and the compiler
//! metadata (CBOR) appended to the code. Nothing is executed; this is a linear sweep that
//! respects PUSH data plus pattern matching over the instruction stream.
use EvmError as FormatError;
/// Neutral inspection failure; no code is executed.
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
pub enum EvmError {
#[error("truncated input: {0}")]
Truncated(String),
#[error("malformed input: {0}")]
Malformed(String),
#[error("unsupported: {0}")]
Unsupported(String),
#[error("input too large: {0}")]
TooLarge(String),
}
#[cfg(test)]
mod test_support;
use serde::Serialize;
use serde_json::{Map, Value};
use std::collections::BTreeSet;
/// Largest bytecode we analyse (16 MiB; deployed contracts are at most 24 KiB).
pub const MAX_CODE: usize = 16 << 20;
const MAX_FUNCTIONS: usize = 4096;
const LISTING_CAP: usize = 400;
/// How input bytes are encoded.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Encoding {
/// Hex when the bytes are all hex digits (with an optional 0x), raw otherwise.
Auto,
Raw,
Hex,
}
/// Turn input bytes into bytecode. Hex is strict: optional `0x`, even length, only hex digits;
/// surrounding whitespace is ignored.
pub fn decode_input(input: &[u8], enc: Encoding) -> Result<(Vec<u8>, &'static str), FormatError> {
let trimmed = input.trim_ascii();
let body = trimmed.strip_prefix(b"0x").or_else(|| trimmed.strip_prefix(b"0X")).unwrap_or(trimmed);
let looks_hex = !body.is_empty() && body.iter().all(u8::is_ascii_hexdigit);
let hex = match enc {
Encoding::Hex => true,
Encoding::Raw => false,
Encoding::Auto => looks_hex,
};
if !hex {
if input.len() > MAX_CODE {
return Err(FormatError::TooLarge(format!("{} bytes; the limit is {MAX_CODE}", input.len())));
}
return Ok((input.to_vec(), "raw"));
}
if body.is_empty() {
return Err(FormatError::Malformed("no hex digits".into()));
}
if let Some(p) = body.iter().position(|b| !b.is_ascii_hexdigit()) {
return Err(FormatError::Malformed(format!("not a hex digit at position {p}")));
}
if !body.len().is_multiple_of(2) {
return Err(FormatError::Malformed(format!("odd number of hex digits ({})", body.len())));
}
if body.len() / 2 > MAX_CODE {
return Err(FormatError::TooLarge(format!("{} bytes; the limit is {MAX_CODE}", body.len() / 2)));
}
let nib = |c: u8| -> u8 {
match c {
b'0'..=b'9' => c - b'0',
b'a'..=b'f' => c - b'a' + 10,
_ => c.wrapping_sub(b'A').wrapping_add(10) & 0x0f,
}
};
Ok((body.chunks(2).map(|p| (nib(p.first().copied().unwrap_or(b'0')) << 4) | nib(p.get(1).copied().unwrap_or(b'0'))).collect(), "hex"))
}
#[derive(Debug, Clone, Copy)]
struct Ins {
off: usize,
op: u8,
/// Immediate (PUSH data) start and length.
imm: (usize, usize),
}
fn push_len(op: u8) -> usize {
if (0x60..=0x7f).contains(&op) { usize::from(op - 0x5f) } else { 0 }
}
/// The mnemonic of an opcode (`UNKNOWN_0x…` for unassigned ones).
pub fn mnemonic(op: u8) -> String {
let s = match op {
0x00 => "STOP",
0x01 => "ADD",
0x02 => "MUL",
0x03 => "SUB",
0x04 => "DIV",
0x05 => "SDIV",
0x06 => "MOD",
0x07 => "SMOD",
0x08 => "ADDMOD",
0x09 => "MULMOD",
0x0a => "EXP",
0x0b => "SIGNEXTEND",
0x10 => "LT",
0x11 => "GT",
0x12 => "SLT",
0x13 => "SGT",
0x14 => "EQ",
0x15 => "ISZERO",
0x16 => "AND",
0x17 => "OR",
0x18 => "XOR",
0x19 => "NOT",
0x1a => "BYTE",
0x1b => "SHL",
0x1c => "SHR",
0x1d => "SAR",
0x20 => "KECCAK256",
0x30 => "ADDRESS",
0x31 => "BALANCE",
0x32 => "ORIGIN",
0x33 => "CALLER",
0x34 => "CALLVALUE",
0x35 => "CALLDATALOAD",
0x36 => "CALLDATASIZE",
0x37 => "CALLDATACOPY",
0x38 => "CODESIZE",
0x39 => "CODECOPY",
0x3a => "GASPRICE",
0x3b => "EXTCODESIZE",
0x3c => "EXTCODECOPY",
0x3d => "RETURNDATASIZE",
0x3e => "RETURNDATACOPY",
0x3f => "EXTCODEHASH",
0x40 => "BLOCKHASH",
0x41 => "COINBASE",
0x42 => "TIMESTAMP",
0x43 => "NUMBER",
0x44 => "PREVRANDAO",
0x45 => "GASLIMIT",
0x46 => "CHAINID",
0x47 => "SELFBALANCE",
0x48 => "BASEFEE",
0x49 => "BLOBHASH",
0x4a => "BLOBBASEFEE",
0x50 => "POP",
0x51 => "MLOAD",
0x52 => "MSTORE",
0x53 => "MSTORE8",
0x54 => "SLOAD",
0x55 => "SSTORE",
0x56 => "JUMP",
0x57 => "JUMPI",
0x58 => "PC",
0x59 => "MSIZE",
0x5a => "GAS",
0x5b => "JUMPDEST",
0x5c => "TLOAD",
0x5d => "TSTORE",
0x5e => "MCOPY",
0x5f => "PUSH0",
0x60..=0x7f => return format!("PUSH{}", op - 0x5f),
0x80..=0x8f => return format!("DUP{}", op - 0x7f),
0x90..=0x9f => return format!("SWAP{}", op - 0x8f),
0xa0..=0xa4 => return format!("LOG{}", op - 0xa0),
0xf0 => "CREATE",
0xf1 => "CALL",
0xf2 => "CALLCODE",
0xf3 => "RETURN",
0xf4 => "DELEGATECALL",
0xf5 => "CREATE2",
0xfa => "STATICCALL",
0xfd => "REVERT",
0xfe => "INVALID",
0xff => "SELFDESTRUCT",
_ => return format!("UNKNOWN_{op:#04x}"),
};
s.to_string()
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct EvmFunction {
/// `0x` + 8 hex digits.
pub selector: String,
/// Where the dispatcher sends calls with this selector.
pub bytecode_offset: usize,
/// Offset of the comparison in the dispatcher.
pub compare_offset: usize,
pub dispatch: &'static str,
/// `eq-jumpi` (jump on match) or `eq-iszero-fallthrough` (body follows the check).
pub pattern: &'static str,
/// `nonpayable`, `payable` or `unknown` (view/pure cannot be told from the dispatcher).
pub mutability: &'static str,
/// Why the mutability was chosen.
pub mutability_reason: String,
/// False when the destination is not a JUMPDEST (the match is then doubtful).
pub jumpdest_valid: bool,
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct EvmMetadata {
pub offset: usize,
pub length: usize,
pub entries: Value,
pub compiler: Option<String>,
pub ipfs: Option<String>,
pub swarm: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Serialize)]
pub struct EvmInterface {
pub code_size: usize,
/// Bytes analysed after removing the metadata tail.
pub analysed_size: usize,
pub metadata: Option<EvmMetadata>,
pub instructions: usize,
pub jumpdests: usize,
/// How the selector is pulled from calldata: `shr-224`, `div-2^224`, `and-mask`, or none.
pub selector_extraction: Option<&'static str>,
pub dispatcher_offset: Option<usize>,
/// `solc-like`, `vyper-like`, `mixed` or `none`.
pub style: &'static str,
pub functions: Vec<EvmFunction>,
/// Comparisons that split the selector range (binary-search dispatchers).
pub binary_search_splits: usize,
/// A CALLVALUE check that runs before any selector comparison (all functions non-payable).
pub callvalue_check_before_dispatch: bool,
pub short_calldata_check: bool,
/// `present`, `absent` or `unknown`.
pub fallback: &'static str,
/// `present` or `not-detected`.
pub receive: &'static str,
/// The dispatcher region as text (first instructions only).
pub dispatcher_listing: Vec<String>,
pub warnings: Vec<String>,
}
fn sweep(code: &[u8]) -> Vec<Ins> {
let mut out = Vec::with_capacity(code.len().min(1 << 20) / 2);
let mut i = 0usize;
while i < code.len() {
let op = code.get(i).copied().unwrap_or(0);
let n = push_len(op);
out.push(Ins { off: i, op, imm: (i + 1, n) });
i = i.saturating_add(1).saturating_add(n);
}
out
}
fn imm<'a>(code: &'a [u8], ins: &Ins) -> &'a [u8] {
// Truncated PUSH data at the end of code reads as zero-padded; we only use what exists.
let (s, n) = ins.imm;
code.get(s..s.saturating_add(n).min(code.len())).unwrap_or(&[])
}
fn imm_u64(code: &[u8], ins: &Ins) -> Option<u64> {
let b = imm(code, ins);
if ins.imm.1 > 8 || b.len() != ins.imm.1 {
return None;
}
Some(b.iter().fold(0u64, |a, &x| (a << 8) | u64::from(x)))
}
fn is_push(op: u8) -> bool {
(0x60..=0x7f).contains(&op)
}
/// Strip and decode a trailing CBOR metadata block.
fn split_metadata(code: &[u8]) -> (usize, Option<EvmMetadata>) {
let n = code.len();
if n < 4 {
return (n, None);
}
let len = usize::from(u16::from_be_bytes([code.get(n - 2).copied().unwrap_or(0), code.get(n - 1).copied().unwrap_or(0)]));
if len == 0 || len.saturating_add(2) > n {
return (n, None);
}
let start = n - 2 - len;
let Some(cbor) = code.get(start..n - 2) else { return (n, None) };
if !matches!(cbor.first(), Some(0xa1..=0xb8 | 0xbf)) {
return (n, None);
}
let mut pos = 0usize;
let Some(v) = cbor_item(cbor, &mut pos, 0) else { return (n, None) };
if pos != cbor.len() {
return (n, None);
}
let Value::Object(map) = &v else { return (n, None) };
let compiler = map
.get("solc")
.and_then(|s| s.as_str().map(String::from))
.map(|v| format!("solc {v}"))
.or_else(|| map.get("vyper").and_then(Value::as_str).map(|v| format!("vyper {v}")));
let ipfs = map.get("ipfs").and_then(Value::as_str).map(String::from);
let swarm = map.get("bzzr1").or_else(|| map.get("bzzr0")).and_then(Value::as_str).map(String::from);
(start, Some(EvmMetadata { offset: start, length: len + 2, entries: v.clone(), compiler, ipfs, swarm }))
}
fn cbor_arg(b: &[u8], pos: &mut usize, info: u8) -> Option<u64> {
let take = |pos: &mut usize, k: usize| -> Option<u64> {
let s = b.get(*pos..pos.checked_add(k)?)?;
*pos += k;
Some(s.iter().fold(0u64, |a, &x| (a << 8) | u64::from(x)))
};
match info {
0..=23 => Some(u64::from(info)),
24 => take(pos, 1),
25 => take(pos, 2),
26 => take(pos, 4),
27 => take(pos, 8),
_ => None,
}
}
/// Decode one bounded metadata item. An incomplete or unsupported value invalidates the metadata trailer.
fn cbor_item(b: &[u8], pos: &mut usize, depth: u32) -> Option<Value> {
if depth > 4 {
return None;
}
let ib = *b.get(*pos)?;
*pos += 1;
let major = ib >> 5;
let info = ib & 0x1f;
match major {
0 => cbor_arg(b, pos, info).map(Value::from),
2 | 3 => {
let n = usize::try_from(cbor_arg(b, pos, info)?).ok()?;
let s = b.get(*pos..pos.checked_add(n)?)?;
*pos += n;
Some(if major == 3 { Value::from(std::str::from_utf8(s).ok()?) } else { Value::from(bytes_value(s)) })
}
4 => {
let n = usize::try_from(cbor_arg(b, pos, info)?).ok()?;
if n > 64 {
return None;
}
let mut items = Vec::new();
for _ in 0..n {
items.push(cbor_item(b, pos, depth + 1)?);
}
Some(Value::Array(items))
}
5 => {
let n = usize::try_from(cbor_arg(b, pos, info)?).ok()?;
if n > 32 {
return None;
}
let mut m = Map::new();
for _ in 0..n {
// The metadata view uses JSON object keys. Coercing other CBOR key
// types to strings can merge distinct keys and hide bytecode.
if b.get(*pos).map(|byte| byte >> 5) != Some(3) {
return None;
}
let Value::String(k) = cbor_item(b, pos, depth + 1)? else { return None };
if m.contains_key(&k) {
return None;
}
let v = cbor_item(b, pos, depth + 1)?;
let v = match (k.as_str(), &v) {
("solc" | "vyper", Value::Array(items)) if items.len() == 3 && items.iter().all(Value::is_u64) => {
Value::from(items.iter().map(|v| v.to_string()).collect::<Vec<_>>().join("."))
}
("solc", Value::String(s)) if s.starts_with("0x") && s.len() == 8 => {
let d: Vec<u8> = (0..3).filter_map(|i| hex_byte(s.get(2 + 2 * i..4 + 2 * i)?)).collect();
if let [a, bb, c] = d.as_slice() { Value::from(format!("{a}.{bb}.{c}")) } else { v }
}
("ipfs", Value::String(s)) if s.starts_with("0x1220") => Value::from(base58(&hex_decode(s.get(2..).unwrap_or("")))),
_ => v,
};
m.insert(k, v);
}
Some(Value::Object(m))
}
7 => match info {
20 => Some(Value::Bool(false)),
21 => Some(Value::Bool(true)),
22 => Some(Value::Null),
_ => None,
},
_ => None,
}
}
fn bytes_value(s: &[u8]) -> String {
let mut out = String::with_capacity(2 + s.len() * 2);
out.push_str("0x");
for b in s {
out.push_str(&format!("{b:02x}"));
}
out
}
/// One byte from two hex digits; `from_str_radix` alone would also accept a leading `+`.
fn hex_byte(pair: &str) -> Option<u8> {
if pair.bytes().all(|c| c.is_ascii_hexdigit()) { u8::from_str_radix(pair, 16).ok() } else { None }
}
fn hex_decode(s: &str) -> Vec<u8> {
(0..s.len() / 2).filter_map(|i| hex_byte(s.get(2 * i..2 * i + 2)?)).collect()
}
fn base58(b: &[u8]) -> String {
const A: &[u8; 58] = b"123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz";
let zeros = b.iter().take_while(|&&x| x == 0).count();
let mut digits: Vec<u8> = Vec::new();
for &byte in b.iter().take(128) {
let mut carry = u32::from(byte);
for d in digits.iter_mut() {
carry += u32::from(*d) << 8;
*d = (carry % 58) as u8;
carry /= 58;
}
while carry > 0 {
digits.push((carry % 58) as u8);
carry /= 58;
}
}
let mut s: String = std::iter::repeat_n('1', zeros).collect();
s.extend(digits.iter().rev().map(|&d| char::from(*A.get(usize::from(d)).unwrap_or(&b'1'))));
s
}
fn selector_extraction(body: &[u8], ins: &[Ins]) -> Option<(&'static str, usize)> {
let at = |k: usize| ins.get(k);
let op_at = |k: usize| at(k).map(|i| i.op);
// Selector extraction: CALLDATALOAD followed by SHR 224, DIV 2^224 or a 4-byte mask.
let mut extraction: Option<(&'static str, usize)> = None;
for (k, i) in ins.iter().enumerate() {
if i.op != 0x35 {
continue;
}
for d in 1..=6 {
let Some(n) = at(k + d) else { break };
let kind = match n.op {
0x60 if imm(body, n) == [0xe0] && op_at(k + d + 1) == Some(0x1c) => Some("shr-224"),
0x7c if d == 1
&& imm(body, n).first() == Some(&1)
&& imm(body, n).iter().skip(1).all(|&b| b == 0)
&& op_at(k + d + 1) == Some(0x90)
&& op_at(k + d + 2) == Some(0x04) =>
{
Some("div-2^224")
}
0x63 if imm(body, n) == [0xff, 0xff, 0xff, 0xff] && (op_at(k + d + 1) == Some(0x16) || op_at(k + d + 2) == Some(0x16)) => {
Some("and-mask")
}
_ => None,
};
if let Some(kind) = kind {
extraction = Some((kind, k));
break;
}
}
if extraction.is_some() {
break;
}
}
extraction
}
fn short_calldata_path(body: &[u8], ins: &[Ins], disp_start: usize) -> (bool, &'static str) {
let at = |k: usize| ins.get(k);
let op_at = |k: usize| at(k).map(|i| i.op);
// Short calldata path and receive().
let mut short_calldata = false;
let mut receive = "not-detected";
for (k, i) in ins.iter().enumerate().take(disp_start.saturating_add(64)) {
if i.op != 0x36 {
continue;
}
if op_at(k + 1) == Some(0x15) && at(k + 2).is_some_and(|p| is_push(p.op)) && op_at(k + 3) == Some(0x57) {
receive = "present";
}
// LT pops the size first in PUSH 4, CALLDATASIZE, LT.
if k.checked_sub(1).and_then(at).is_some_and(|p| is_push(p.op) && imm_u64(body, p) == Some(4))
&& op_at(k + 1) == Some(0x10)
&& at(k + 2).is_some_and(|p| is_push(p.op))
&& op_at(k + 3) == Some(0x57)
{
short_calldata = true;
}
}
(short_calldata, receive)
}
/// Analyse bytecode (already decoded from hex if needed).
pub fn analyze(code: &[u8]) -> Result<EvmInterface, FormatError> {
if code.is_empty() {
return Err(FormatError::Truncated("no bytecode".into()));
}
if code.len() > MAX_CODE {
return Err(FormatError::TooLarge(format!("{} bytes; the limit is {MAX_CODE}", code.len())));
}
if code.starts_with(&[0xef, 0x00]) {
return Err(FormatError::Unsupported("EVM Object Format (EOF, 0xEF00) containers are not analysed".into()));
}
let mut warnings = Vec::new();
let (end, metadata) = split_metadata(code);
let body = code.get(..end).unwrap_or(code);
let ins = sweep(body);
let jumpdests = ins.iter().filter(|i| i.op == 0x5b).count();
let at = |k: usize| ins.get(k);
let op_at = |k: usize| at(k).map(|i| i.op);
let index_of = |off: usize| ins.binary_search_by_key(&off, |i| i.off).ok();
let extraction = selector_extraction(body, &ins);
let disp_start = extraction.map(|(_, k)| k).unwrap_or(0);
if extraction.is_none() {
warnings.push("no selector extraction (CALLDATALOAD then SHR/DIV/AND) was seen; matches below are less certain".into());
}
// Only a guard on the straight-line entry path can apply to every selector.
let callvalue_before = ins
.iter()
.take(disp_start)
.enumerate()
.take_while(|(_, i)| !matches!(i.op, 0x00 | 0x56 | 0x57 | 0xf3 | 0xfd | 0xfe | 0xff))
.any(|(k, i)| i.op == 0x34 && rejects_value(body, &ins, k, &index_of));
let mut functions: Vec<EvmFunction> = Vec::new();
let mut selectors = BTreeSet::new();
let mut splits = 0usize;
let mut last_jumpi: Option<usize> = None;
let mut solc_like = 0usize;
let mut vyper_like = 0usize;
for k in disp_start..ins.len() {
let Some(i) = at(k) else { break };
let n = push_len(i.op);
if !(1..=4).contains(&n) {
continue;
}
let Some(sel) = imm_u64(body, i) else { continue };
let found = |dk: usize| -> Option<usize> {
let p = at(dk)?;
(is_push(p.op) && p.imm.1 <= 3 && op_at(dk + 1) == Some(0x57)).then_some(dk)
};
// PUSH sel, [DUP2], EQ, PUSH dest, JUMPI
let eq_k = if op_at(k + 1) == Some(0x14) {
Some(k + 1)
} else if matches!(op_at(k + 1), Some(0x80..=0x8f)) && op_at(k + 2) == Some(0x14) {
Some(k + 2)
} else {
None
};
let mut hit: Option<(usize, usize, &'static str)> = None;
if let Some(e) = eq_k {
if let Some(pk) = found(e + 1)
&& let Some(dest) = at(pk).and_then(|p| imm_u64(body, p))
{
hit = Some((usize::try_from(dest).unwrap_or(usize::MAX), pk + 1, "eq-jumpi"));
} else if n == 4 && op_at(e + 1) == Some(0x15) && found(e + 2).is_some() {
// Vyper-style: if selector != x jump to the next check; the body follows.
if let Some(next) = at(e + 4) {
hit = Some((next.off, e + 3, "eq-iszero-fallthrough"));
}
}
} else if n == 4 {
// Vyper also loads the selector from memory: PUSH sel, PUSH1 0, MLOAD, EQ, ISZERO …
let load = at(k + 1).is_some_and(|p| (p.op == 0x5f || is_push(p.op)) && imm_u64(body, p) == Some(0));
if load
&& op_at(k + 2) == Some(0x51)
&& op_at(k + 3) == Some(0x14)
&& op_at(k + 4) == Some(0x15)
&& found(k + 5).is_some()
&& let Some(next) = at(k + 7)
{
hit = Some((next.off, k + 6, "eq-iszero-fallthrough"));
}
let direct = matches!(op_at(k + 1), Some(0x10 | 0x11)) && found(k + 2).is_some();
let via_dup = matches!(op_at(k + 1), Some(0x80..=0x8f)) && matches!(op_at(k + 2), Some(0x10 | 0x11)) && found(k + 3).is_some();
if hit.is_none() && (direct || via_dup) {
splits += 1;
}
}
let Some((dest, jumpi_k, pattern)) = hit else { continue };
if n < 4 && extraction.is_none() {
continue;
}
// Duplicate identities still advance the end of the recognized dispatcher.
// Otherwise the fallback scan starts inside a later comparison, not after it.
last_jumpi = Some(jumpi_k);
if !selectors.insert(sel) {
continue;
}
let selector = format!("0x{sel:08x}");
if pattern == "eq-jumpi" {
solc_like += 1;
} else {
vyper_like += 1;
}
let di = index_of(dest);
let jumpdest_valid = pattern != "eq-jumpi" || di.and_then(at).is_some_and(|x| x.op == 0x5b);
let (mutability, reason) = if !jumpdest_valid {
("unknown", "the destination is not a JUMPDEST".to_owned())
} else if callvalue_before {
("nonpayable", "a CALLVALUE check runs before the dispatcher, so every function refuses payments".to_string())
} else {
match di {
Some(d) => payability(body, &ins, d, &index_of),
None => ("unknown", "the destination is not an instruction boundary".to_string()),
}
};
functions.push(EvmFunction {
selector,
bytecode_offset: dest,
compare_offset: i.off,
dispatch: "abi",
pattern,
mutability,
mutability_reason: reason,
jumpdest_valid,
});
if functions.len() >= MAX_FUNCTIONS {
warnings.push(format!("stopped at {MAX_FUNCTIONS} selectors"));
break;
}
}
if functions.iter().any(|f| !f.jumpdest_valid) {
warnings.push("some destinations are not JUMPDESTs; those matches may be data, not dispatch".into());
}
let (short_calldata, receive) = short_calldata_path(body, &ins, disp_start);
let mut fallback = "unknown";
// At the selector cap the remaining dispatcher has not been scanned.
// Its next comparison is not evidence that a fallback body exists.
if let Some(j) = last_jumpi
&& functions.len() < MAX_FUNCTIONS
&& solc_like > 0
{
fallback = if reverts_soon(body, &ins, j + 1, &index_of) { "absent" } else { "present" };
}
let style = match (solc_like > 0, vyper_like > 0) {
(true, true) => "mixed",
(true, false) => "solc-like",
(false, true) => "vyper-like",
_ => "none",
};
let listing_end = last_jumpi.map(|j| j + 8).unwrap_or(LISTING_CAP).min(LISTING_CAP);
let dispatcher_listing = ins
.iter()
.take(listing_end)
.map(|i| {
let d = imm(body, i);
if d.is_empty() && push_len(i.op) == 0 {
format!("{:#06x}: {}", i.off, mnemonic(i.op))
} else {
format!("{:#06x}: {} {}", i.off, mnemonic(i.op), bytes_value(d))
}
})
.collect();
if functions.is_empty() {
warnings.push(
"no selector dispatcher was recognised (the code may be creation code without runtime, a proxy, a library, or use a jump-table dispatcher)".into(),
);
}
Ok(EvmInterface {
code_size: code.len(),
analysed_size: body.len(),
metadata,
instructions: ins.len(),
jumpdests,
selector_extraction: extraction.map(|(k, _)| k),
dispatcher_offset: extraction.and_then(|(_, k)| at(k)).map(|i| i.off),
style,
functions,
binary_search_splits: splits,
callvalue_check_before_dispatch: callvalue_before,
short_calldata_check: short_calldata,
fallback,
receive,
dispatcher_listing,
warnings,
})
}
/// Resolve the destination of a JUMP to an instruction index, when the PUSH right before it is
/// readable and lands on an instruction boundary.
fn jump_destination(code: &[u8], ins: &[Ins], k: usize, index_of: &dyn Fn(usize) -> Option<usize>) -> Option<usize> {
let push = k.checked_sub(1).and_then(|p| ins.get(p)).filter(|p| is_push(p.op))?;
index_of(usize::try_from(imm_u64(code, push)?).ok()?)
}
/// Follow a function's entry for a few instructions (and up to two unconditional jumps) looking
/// for a CALLVALUE check.
fn payability(code: &[u8], ins: &[Ins], start: usize, index_of: &dyn Fn(usize) -> Option<usize>) -> (&'static str, String) {
let mut k = start;
let mut hops = 0;
for _ in 0..48 {
let Some(i) = ins.get(k) else { break };
match i.op {
0x34 if rejects_value(code, ins, k, index_of) => return ("nonpayable", format!("rejects nonzero CALLVALUE at {:#x}", i.off)),
0x56 => match jump_destination(code, ins, k, index_of) {
Some(t) if hops < 2 && ins.get(t).is_some_and(|target| target.op == 0x5b) => {
hops += 1;
k = t;
continue;
}
_ => return ("unknown", "jumps somewhere we cannot follow before any CALLVALUE check".into()),
},
0x57 => return ("unknown", "branches before any CALLVALUE check".into()),
0xfd | 0xfe => return ("unknown", "halts with an error before any CALLVALUE check".into()),
0x00 | 0xf3 | 0xff | 0x35 | 0x37 | 0x54 | 0x55 | 0xf1 | 0xf4 | 0xfa | 0xa0..=0xa4 => {
return ("payable", format!("reaches {} at {:#x} without a CALLVALUE check", mnemonic(i.op), i.off));
}
_ => {}
}
k += 1;
}
("unknown", "no CALLVALUE check in the first instructions".into())
}
/// CALLVALUE only reads the payment. Recognize a direct test that sends the nonzero path
/// to a rejection, allowing the common DUP1 that preserves the original value for cleanup.
fn rejects_value(code: &[u8], ins: &[Ins], k: usize, index_of: &dyn Fn(usize) -> Option<usize>) -> bool {
let mut next = k + 1;
if ins.get(next).is_some_and(|i| i.op == 0x80) {
next += 1;
}
let jumps_when_zero = ins.get(next).is_some_and(|i| i.op == 0x15);
next += usize::from(jumps_when_zero);
let Some(push) = ins.get(next).filter(|i| is_push(i.op)) else { return false };
if ins.get(next + 1).is_none_or(|i| i.op != 0x57) {
return false;
}
let Some(target) = imm_u64(code, push).and_then(|v| usize::try_from(v).ok()).and_then(index_of) else { return false };
if ins.get(target).is_none_or(|i| i.op != 0x5b) {
return false;
}
reverts_soon(code, ins, if jumps_when_zero { next + 2 } else { target }, index_of)
}
fn reverts_soon(code: &[u8], ins: &[Ins], mut k: usize, index_of: &dyn Fn(usize) -> Option<usize>) -> bool {
let mut hops = 0;
for _ in 0..12 {
let Some(i) = ins.get(k) else { return false };
match i.op {
0xfd | 0xfe => return true,
0x56 => match jump_destination(code, ins, k, index_of) {
Some(t) if hops < 2 => {
hops += 1;
k = t;
continue;
}
_ => return false,
},
0x5b | 0x50 | 0x80..=0x9f | 0x5f | 0x60..=0x7f => {}
_ => return false,
}
k += 1;
}
false
}
#[cfg(test)]
mod tests {
use super::*;
use crate::test_support::formats as fx;
#[test]
fn hex_signs_are_not_hex_digits() {
let solc = |v: &str| {
let mut b = vec![0xa1, 0x64];
b.extend(b"solc");
b.push(0x60 + v.len() as u8);
b.extend(v.as_bytes());
cbor_item(&b, &mut 0, 0).unwrap()["solc"].clone()
};
assert_eq!(solc("0x+1+2+3"), "0x+1+2+3");
assert_eq!(solc("0x000008"), "0.0.8");
assert!(hex_decode("+f").is_empty());
assert_eq!(hex_decode("0f"), vec![15]);
}
/// A plausible trailer length alone must not remove instructions from the analyzed bytecode.
#[test]
fn incomplete_cbor_must_not_shorten_the_analyzed_contract() {
// A nested array declares 65 integers but supplies just 64. The top-level map
// and its length suffix otherwise look complete.
let mut short = vec![0xa1, 0x61, b'x', 0x98, 65];
short.extend([0; 64]);
let mut short_map = vec![0xa1, 0x61, b'x', 0xb8, 33];
for key in 0..32 {
short_map.extend([0x18, key, 0]);
}
for cbor in [short, short_map, vec![0xa1, 0x61, b'x', 0x61, 0xff]] {
let mut code = vec![0x00];
code.extend(&cbor);
code.extend((cbor.len() as u16).to_be_bytes());
assert!(split_metadata(&code).1.is_none());
assert_eq!(analyze(&code).unwrap().analysed_size, code.len());
}
}
#[test]
fn dispatcher_selectors_and_metadata() {
let (code, dests) = fx::evm_runtime();
let r = analyze(&code).unwrap();
assert_eq!(r.selector_extraction, Some("shr-224"));
assert_eq!(r.style, "solc-like");
let sels: Vec<_> = r.functions.iter().map(|f| f.selector.clone()).collect();
assert_eq!(sels, fx::EVM_SELECTORS.iter().map(|s| format!("0x{s:08x}")).collect::<Vec<_>>());
for (f, d) in r.functions.iter().zip(dests) {
assert_eq!(f.bytecode_offset, d);
assert!(f.jumpdest_valid);
}
assert_eq!(r.functions[0].mutability, "nonpayable");
assert_eq!(r.functions[1].mutability, "nonpayable");
assert_eq!(r.functions[2].mutability, "payable");
assert!(r.short_calldata_check);
assert_eq!(r.fallback, "absent");
let m = r.metadata.unwrap();
assert_eq!(m.compiler.as_deref(), Some("solc 0.8.19"));
assert!(m.ipfs.unwrap().starts_with("Qm"));
assert_eq!(r.analysed_size + m.length, code.len());
}
#[test]
fn hex_input_rules() {
let (code, _) = fx::evm_runtime();
let hex = fx::to_hex(&code);
assert_eq!(decode_input(format!(" {hex}\n").as_bytes(), Encoding::Auto).unwrap(), (code.clone(), "hex"));
assert_eq!(decode_input(&code, Encoding::Auto).unwrap().1, "raw");
assert!(decode_input(b"0x123", Encoding::Hex).is_err());
assert!(decode_input(b"0xzz", Encoding::Hex).is_err());
assert!(decode_input(b"", Encoding::Hex).is_err());
assert!(matches!(analyze(&[0xef, 0x00, 0x01]), Err(FormatError::Unsupported(_))));
assert_eq!(base58(&[0, 0, 1]), "112");
}
#[test]
fn vyper_style_fallthrough() {
// PUSH1 0 CALLDATALOAD PUSH1 0xe0 SHR PUSH1 0 MSTORE PUSH4 sel PUSH1 0 MLOAD EQ ISZERO PUSH2 next JUMPI <body: CALLVALUE ...>
let mut c = vec![
0x60, 0x00, 0x35, 0x60, 0xe0, 0x1c, 0x60, 0x00, 0x52, 0x63, 0x12, 0x34, 0x56, 0x78, 0x60, 0x00, 0x51, 0x14, 0x15, 0x61, 0x00, 0x20, 0x57,
];
let body = c.len();
c.extend_from_slice(&[0x34, 0x15, 0x00]);
let r = analyze(&c).unwrap();
assert_eq!(r.style, "vyper-like");
assert_eq!(r.functions[0].selector, "0x12345678");
assert_eq!(r.functions[0].bytecode_offset, body);
assert_eq!(r.functions[0].mutability, "payable", "ISZERO followed by STOP never rejects payment");
}
#[test]
fn nonpayable_needs_a_rejecting_nonzero_branch() {
fn dispatch(prefix: &[u8]) -> EvmInterface {
let mut code = prefix.to_vec();
code.extend([0x5f, 0x35, 0x60, 0xe0, 0x1c, 0x63, 1, 2, 3, 4, 0x14, 0x60]);
code.extend([(code.len() + 3) as u8, 0x57, 0x00, 0x5b, 0x00]);
analyze(&code).unwrap()
}
let read_only = dispatch(&[0x34, 0x50]); // CALLVALUE POP
assert!(!read_only.callvalue_check_before_dispatch);
assert_eq!(read_only.functions[0].mutability, "payable");
let guard = dispatch(&[0x34, 0x80, 0x15, 0x60, 9, 0x57, 0x5f, 0x80, 0xfd, 0x5b, 0x50]);
assert!(guard.callvalue_check_before_dispatch);
assert_eq!(guard.functions[0].mutability, "nonpayable");
// The other common form jumps to REVERT for a nonzero value and otherwise skips it.
let direct = dispatch(&[0x34, 0x60, 7, 0x57, 0x60, 11, 0x56, 0x5b, 0x5f, 0x80, 0xfd, 0x5b]);
assert!(direct.callvalue_check_before_dispatch);
// Reversing that conditional rejects zero-value calls, not payments.
let reversed = dispatch(&[0x34, 0x15, 0x60, 8, 0x57, 0x60, 12, 0x56, 0x5b, 0x5f, 0x80, 0xfd, 0x5b]);
assert!(!reversed.callvalue_check_before_dispatch);
// An unconditional jump bypasses the otherwise valid guard before the dispatcher.
let bypassed = dispatch(&[0x60, 12, 0x56, 0x34, 0x80, 0x15, 0x60, 12, 0x57, 0x5f, 0x80, 0xfd, 0x5b]);
assert!(!bypassed.callvalue_check_before_dispatch);
}
#[test]
fn hostile_inputs_do_not_panic() {
let (code, _) = fx::evm_runtime();
for m in crate::test_support::mutations(&code) {
let _ = analyze(&m);
}
for n in 0..code.len() {
let _ = analyze(&code[..n]);
}
for seed in 0..200 {
let n = crate::test_support::noise(seed, 300);
let _ = analyze(&n);
let _ = decode_input(&n, Encoding::Auto);
}
// A metadata length that claims everything, and nested CBOR.
let _ = analyze(&[0xa1, 0xa1, 0xa1, 0xa1, 0xa1, 0xa1, 0xa1, 0x00, 0x07]);
let _ = analyze(&[0x7f; 40]);
}
}
+63
View File
@@ -0,0 +1,63 @@
pub mod formats {
pub const EVM_SELECTORS: [u32; 3] = [0xa905_9cbb, 0x70a0_8231, 0xd0e3_0db0];
/// Runtime bytecode with a compiler-style dispatcher, a fallback that reverts, two non-payable
/// functions (CALLVALUE check), one payable function, and a CBOR metadata tail (ipfs + solc 0.8.19).
/// Returns (bytecode, jump destinations of the three functions).
pub fn evm_runtime() -> (Vec<u8>, [usize; 3]) {
let mut c: Vec<u8> = vec![0x60, 0x80, 0x60, 0x40, 0x52];
// if calldatasize < 4 goto fallback
c.extend_from_slice(&[0x60, 0x04, 0x36, 0x10, 0x61, 0, 0]);
let fb_patch = c.len() - 2;
c.push(0x57);
c.extend_from_slice(&[0x60, 0x00, 0x35, 0x60, 0xe0, 0x1c]);
let mut patches = Vec::new();
for sel in EVM_SELECTORS {
c.push(0x80);
c.push(0x63);
c.extend_from_slice(&sel.to_be_bytes());
c.extend_from_slice(&[0x14, 0x61, 0, 0]);
patches.push(c.len() - 2);
c.push(0x57);
}
let fallback = c.len();
c.extend_from_slice(&[0x5b, 0x60, 0x00, 0x80, 0xfd]);
let mut dests = [0usize; 3];
for (i, dest) in dests.iter_mut().enumerate() {
*dest = c.len();
if i < 2 {
// JUMPDEST CALLVALUE DUP1 ISZERO PUSH2 ok JUMPI PUSH1 0 DUP1 REVERT JUMPDEST POP STOP
c.extend_from_slice(&[0x5b, 0x34, 0x80, 0x15, 0x61, 0, 0]);
let ok_patch = c.len() - 2;
c.extend_from_slice(&[0x57, 0x60, 0x00, 0x80, 0xfd]);
let ok = c.len();
c.extend_from_slice(&[0x5b, 0x50, 0x00]);
c[ok_patch..ok_patch + 2].copy_from_slice(&(ok as u16).to_be_bytes());
} else {
c.extend_from_slice(&[0x5b, 0x60, 0x01, 0x60, 0x00, 0x55, 0x00]);
}
}
c[fb_patch..fb_patch + 2].copy_from_slice(&(fallback as u16).to_be_bytes());
for (p, d) in patches.iter().zip(dests) {
c[*p..*p + 2].copy_from_slice(&(d as u16).to_be_bytes());
}
c.push(0xfe);
let mut meta = vec![0xa2, 0x64, b'i', b'p', b'f', b's', 0x58, 0x22, 0x12, 0x20];
meta.extend((0..32u8).map(|i| i.wrapping_mul(7)));
meta.extend_from_slice(&[0x64, b's', b'o', b'l', b'c', 0x43, 0x00, 0x08, 0x13]);
let len = meta.len() as u16;
c.extend_from_slice(&meta);
c.extend_from_slice(&len.to_be_bytes());
(c, dests)
}
/// Bytecode as `0x`-prefixed lowercase hex.
pub fn to_hex(b: &[u8]) -> String {
let mut s = String::from("0x");
for x in b {
s.push_str(&format!("{x:02x}"));
}
s
}
}
pub use craft_testkit::sample_damage::{mutations, noise};
+39
View File
@@ -0,0 +1,39 @@
use craft_evm as evm;
use serde_json::json;
/// Unrelated three-integer arrays stay arrays while compiler versions stay readable.
#[test]
fn cbor_arrays_are_not_all_compiler_versions() {
// {"sizes": [1, 2, 3], "vyper": [0, 3, 10]}
for (cbor, compiler) in [
(b"\xa2\x65sizes\x83\x01\x02\x03\x65vyper\x83\x00\x03\x0a".as_slice(), "vyper 0.3.10"),
(b"\xa2\x65sizes\x83\x01\x02\x03\x64solc\x83\x00\x08\x13".as_slice(), "solc 0.8.19"),
] {
let mut code = vec![0x00];
code.extend(cbor);
code.extend((cbor.len() as u16).to_be_bytes());
let result = evm::analyze(&code).unwrap();
let meta = result.metadata.unwrap();
assert_eq!(meta.entries["sizes"], json!([1, 2, 3]));
assert_eq!(meta.compiler.as_deref(), Some(compiler));
assert_eq!(result.analysed_size, 1);
}
}
/// A non-version Vyper array stays typed, and a truncated array stays bytecode.
#[test]
fn invalid_version_shapes_are_not_invented_or_silently_consumed() {
let cbor = b"\xa1\x65vyper\x82\x00\x03";
let mut code = vec![0x00];
code.extend(cbor);
code.extend((cbor.len() as u16).to_be_bytes());
let meta = evm::analyze(&code).unwrap().metadata.unwrap();
assert_eq!(meta.compiler, None);
assert_eq!(meta.entries["vyper"], json!([0, 3]));
let mut truncated = code;
truncated[8] = 0x83; // Declare three items but keep only two.
let result = evm::analyze(&truncated).unwrap();
assert!(result.metadata.is_none());
assert_eq!(result.analysed_size, truncated.len());
}
+93
View File
@@ -0,0 +1,93 @@
use craft_evm::analyze;
fn contract_with_trailer(cbor: &[u8]) -> Vec<u8> {
let mut code = vec![0x5b, 0x00];
code.extend_from_slice(cbor);
code.extend_from_slice(&u16::try_from(cbor.len()).unwrap().to_be_bytes());
code
}
fn assert_unrecognized(cbor: &[u8]) {
let code = contract_with_trailer(cbor);
let result = analyze(&code).unwrap();
assert!(result.metadata.is_none(), "ambiguous map must remain bytecode: {cbor:02x?}");
assert_eq!(result.analysed_size, code.len());
}
#[test]
fn duplicate_compiler_keys_do_not_replace_the_reported_version() {
assert_unrecognized(b"\xa2\x64solc\x43\x00\x08\x13\x64solc\x43\x00\x08\x1a");
}
#[test]
fn identical_duplicate_keys_are_also_ambiguous() {
assert_unrecognized(b"\xa2\x64solc\x43\x00\x08\x13\x64solc\x43\x00\x08\x13");
}
#[test]
fn numeric_keys_cannot_collide_with_text_keys() {
assert_unrecognized(b"\xa2\x01\x00\x61\x31\x01");
}
#[test]
fn metadata_map_keys_must_be_encoded_as_text() {
for key in [vec![0x01], vec![0x41, b'x'], vec![0x83, 0, 1, 2], vec![0xf5], vec![0xf6], vec![0xa0]] {
let mut cbor = vec![0xa1];
cbor.extend(key);
cbor.push(0);
assert_unrecognized(&cbor);
}
}
#[test]
fn nested_maps_do_not_silently_overwrite_keys() {
assert_unrecognized(b"\xa1\x65extra\xa2\x61x\x00\x61x\x01");
assert_unrecognized(b"\xa1\x65extra\xa1\x01\x00");
}
#[test]
fn ordinary_compiler_metadata_and_unique_nested_text_keys_still_decode() {
let code = contract_with_trailer(b"\xa2\x64solc\x43\x00\x08\x13\x65extra\xa2\x61x\x00\x61y\x01");
let result = analyze(&code).unwrap();
assert_eq!(result.analysed_size, 2);
let metadata = result.metadata.unwrap();
assert_eq!(metadata.compiler.as_deref(), Some("solc 0.8.19"));
assert_eq!(metadata.entries["extra"]["x"], 0);
assert_eq!(metadata.entries["extra"]["y"], 1);
assert_eq!(metadata.length, code.len() - 2);
}
#[test]
fn equal_keys_in_separate_maps_are_independent() {
let code = contract_with_trailer(b"\xa2\x61a\xa1\x61x\x00\x61b\xa1\x61x\x01");
let result = analyze(&code).unwrap();
assert_eq!(result.analysed_size, 2);
let entries = result.metadata.unwrap().entries;
assert_eq!(entries["a"]["x"], 0);
assert_eq!(entries["b"]["x"], 1);
}
#[test]
fn extended_length_metadata_maps_respect_the_pair_limit() {
let map = |count: u8| {
let mut cbor = vec![0xb8, count];
for index in 0..count {
let key = format!("k{index:02}");
cbor.push(0x63);
cbor.extend_from_slice(key.as_bytes());
cbor.push(0);
}
cbor
};
for count in [24, 32] {
let code = contract_with_trailer(&map(count));
let report = analyze(&code).unwrap();
assert_eq!(report.analysed_size, 2);
assert_eq!(report.metadata.unwrap().entries.as_object().unwrap().len(), usize::from(count));
}
assert_unrecognized(&map(33));
assert_unrecognized(&[0xb8]);
let mut truncated = map(24);
truncated.pop();
assert_unrecognized(&truncated);
}
+115
View File
@@ -0,0 +1,115 @@
use craft_evm as evm;
fn prefix() -> Vec<u8> {
vec![0x5b, 0x00, 0x60, 0x00, 0x35, 0x60, 0xe0, 0x1c]
}
fn comparison(code: &mut Vec<u8>, selector: u32, width: usize, destination: u8) {
code.push(0x5f + u8::try_from(width).unwrap());
code.extend_from_slice(&selector.to_be_bytes()[4 - width..]);
code.extend([0x14, 0x60, destination, 0x57]);
}
#[test]
fn repeated_comparisons_keep_the_first_destination_and_encounter_order() {
let mut code = prefix();
comparison(&mut code, 0xfeed0002, 4, 0);
comparison(&mut code, 0xfeed0001, 4, 0);
let expected = evm::analyze(&code).unwrap().functions;
for _ in 0..2000 {
comparison(&mut code, 0xfeed0001, 4, 1);
comparison(&mut code, 0xfeed0002, 4, 1);
}
assert_eq!(evm::analyze(&code).unwrap().functions, expected);
}
#[test]
fn different_push_widths_share_the_same_numeric_selector() {
let mut code = prefix();
for width in 1..=4 {
comparison(&mut code, 17, width, if width == 1 { 0 } else { 1 });
}
let result = evm::analyze(&code).unwrap();
assert_eq!(result.functions.len(), 1);
assert_eq!(result.functions[0].selector, "0x00000011");
assert_eq!(result.functions[0].bytecode_offset, 0);
}
#[test]
fn rejected_candidates_do_not_spend_selector_identity() {
let mut code = vec![0x5b, 0x00];
// Short comparisons without selector extraction are not considered functions.
comparison(&mut code, 17, 1, 1);
// A selector-shaped immediate without a comparison is not a function either.
code.extend([0x63, 0, 0, 0, 17, 0x50]);
comparison(&mut code, 17, 4, 0);
let result = evm::analyze(&code).unwrap();
assert_eq!(result.functions.len(), 1);
assert_eq!(result.functions[0].bytecode_offset, 0);
assert!(result.functions[0].jumpdest_valid);
}
#[test]
fn selector_cap_counts_unique_matches() {
let mut code = prefix();
for selector in 0..4095 {
comparison(&mut code, selector, 4, 0);
}
for _ in 0..10000 {
comparison(&mut code, 4094, 4, 1);
}
let below = evm::analyze(&code).unwrap();
assert_eq!(below.functions.len(), 4095);
assert!(!below.warnings.iter().any(|warning| warning.contains("stopped at")));
comparison(&mut code, 4095, 4, 0);
comparison(&mut code, 4096, 4, 0);
let capped = evm::analyze(&code).unwrap();
assert_eq!(capped.functions.len(), 4096);
assert_eq!(capped.functions.last().unwrap().selector, "0x00000fff");
assert!(capped.warnings.iter().any(|warning| warning == "stopped at 4096 selectors"));
}
#[test]
fn repeated_comparisons_keep_fallback_scan_after_the_last_dispatch() {
for (ending, expected) in [(&[0x5f, 0x5f, 0xfd][..], "absent"), (&[0x00][..], "present")] {
let mut code = prefix();
comparison(&mut code, 0xfeed0001, 4, 0);
comparison(&mut code, 0xfeed0001, 4, 1);
code.extend_from_slice(ending);
let report = evm::analyze(&code).unwrap();
assert_eq!(report.functions.len(), 1);
assert_eq!(report.functions[0].bytecode_offset, 0);
assert_eq!(report.fallback, expected);
}
}
#[test]
fn selector_cap_leaves_fallback_unknown() {
for ending in [&[0x5f, 0x5f, 0xfd][..], &[0x00][..]] {
let mut code = prefix();
for selector in 0..4097 {
comparison(&mut code, selector, 4, 0);
}
code.extend_from_slice(ending);
let report = evm::analyze(&code).unwrap();
assert_eq!(report.functions.len(), 4096);
assert_eq!(report.fallback, "unknown");
}
}
#[test]
fn payability_only_follows_jumps_to_jumpdest() {
for (target, expected) in [(&[0x5b, 0x00][..], "payable"), (&[0x00][..], "unknown")] {
let mut code = prefix();
let entry = u8::try_from(code.len() + 10).unwrap();
comparison(&mut code, 0xfeed0001, 4, entry);
code.push(0x00);
assert_eq!(code.len(), usize::from(entry));
code.extend([0x5b, 0x60, entry + 4, 0x56]);
code.extend_from_slice(target);
let report = evm::analyze(&code).unwrap();
assert_eq!(report.functions.len(), 1);
assert!(report.functions[0].jumpdest_valid);
assert_eq!(report.functions[0].mutability, expected);
}
}
+442
View File
@@ -1,5 +1,7 @@
//! Header-only BMP dimensions and pixel-data coverage, without allocating a canvas.
use crate::error::CodecError;
/// Recognize a BMP file header with a supported DIB header kind.
///
/// This is a format-routing check, not full validation: dimensions and pixel data may still
@@ -13,6 +15,115 @@ pub fn is_supported_header(b: &[u8]) -> bool {
false
}
/// Wrap a DIB (a BMP without its 14-byte `BITMAPFILEHEADER`, as the Windows clipboard holds one
/// for `CF_DIB` and `CF_DIBV5`) as a BMP file the decoders here read. The pixel offset is computed
/// from the header with saturating arithmetic, so a hostile header cannot overflow; whether the
/// pixel data exists is left to the decoder. Apply [`dib_layout`] first for clipboard DIBs.
///
/// # Errors
///
/// Returns [`CodecError::LimitExceeded`] when the output (`dib.len() + 14` bytes) cannot be allocated.
pub fn dib_to_bmp(dib: &[u8]) -> Result<Vec<u8>, CodecError> {
let u16_at = |at: usize| dib.get(at..at + 2).and_then(|b| b.try_into().ok()).map(u16::from_le_bytes);
let u32_at = |at: usize| dib.get(at..at + 4).and_then(|b| b.try_into().ok()).map(u32::from_le_bytes);
let bi_size = u32_at(0).unwrap_or(40);
let offset = if bi_size == 12 {
let bpp = u16_at(10).unwrap_or(24);
let palette = if bpp <= 8 { (1u32 << bpp) * 3 } else { 0 };
14 + bi_size + palette
} else {
let bpp = u16_at(14).unwrap_or(24);
let compression = u32_at(16).unwrap_or(0);
let clr_used = u32_at(32).unwrap_or(0);
let palette_entries = if clr_used != 0 {
clr_used
} else if bpp <= 8 {
1u32 << bpp
} else {
0
};
// The masks of a 52- or 56-byte (V2/V3) header are inside the header; only the other
// header kinds carry them as 12 further bytes before the pixels.
let masks = if compression == 3 && bi_size >= 40 && !matches!(bi_size, 52 | 56) { 12 } else { 0 };
14u32.saturating_add(bi_size).saturating_add(masks).saturating_add(palette_entries.saturating_mul(4))
};
let total = dib.len().saturating_add(14);
let file_size = u32::try_from(total).unwrap_or(u32::MAX);
let mut bmp = Vec::new();
bmp.try_reserve_exact(total).map_err(|_| CodecError::LimitExceeded(format!("a {total}-byte BMP")))?;
bmp.extend_from_slice(b"BM");
bmp.extend_from_slice(&file_size.to_le_bytes());
bmp.extend_from_slice(&[0u8; 4]);
bmp.extend_from_slice(&offset.to_le_bytes());
bmp.extend_from_slice(dib);
Ok(bmp)
}
/// Put a clipboard DIB with a V4 or V5 header (108 bytes or more) in the layout the decoder reads.
/// A 32-bit one whose alpha mask is set but whose compression says `BI_RGB` is marked
/// `BI_BITFIELDS` (with the usual colour masks when it gives none), so its alpha is kept as
/// Windows programs mean it; arboard makes the same change. Bit fields then need the colour masks
/// after the header too, where Windows puts a copy and the decoder skips them; a DIB without that
/// copy gets it. Any other DIB comes back unchanged, as does one whose 12 extra bytes cannot be
/// allocated.
pub fn dib_layout(mut dib: Vec<u8>) -> Vec<u8> {
let u32_at = |d: &[u8], at: usize| d.get(at..at + 4).and_then(|b| b.try_into().ok()).map(u32::from_le_bytes);
let Some(size) = u32_at(&dib, 0).filter(|&s| s >= 108).and_then(|s| usize::try_from(s).ok()) else { return dib };
let put = |d: &mut Vec<u8>, at: usize, v: u32| {
if let Some(b) = d.get_mut(at..at + 4) {
b.copy_from_slice(&v.to_le_bytes());
}
};
// Decide everything first and mutate only after the 12-byte reservation succeeded, so a failed
// allocation returns the DIB unchanged.
let promote = dib.get(14..16) == Some(&[32, 0]) && u32_at(&dib, 16) == Some(0) && u32_at(&dib, 52) == Some(0xFF00_0000);
let default_masks = promote && (40..52).step_by(4).all(|at| u32_at(&dib, at) == Some(0));
let mut masks = dib.get(40..52).and_then(|b| <[u8; 12]>::try_from(b).ok());
if default_masks {
let mut m = [0u8; 12];
m[0..4].copy_from_slice(&0x00FF_0000u32.to_le_bytes());
m[4..8].copy_from_slice(&0x0000_FF00u32.to_le_bytes());
m[8..12].copy_from_slice(&0x0000_00FFu32.to_le_bytes());
masks = Some(m);
}
let bitfields = promote || u32_at(&dib, 16) == Some(3);
// Bytes equal to the masks right after the header are a copy only when more bytes follow than
// the pixels need; otherwise they are the pixels (e.g. a 3x1 V5 DIB with pixels equal to the masks).
let has_copy = |m: &[u8; 12]| {
dib.get(size..size.saturating_add(12)) == Some(m.as_slice()) && pixel_bytes(&dib).is_none_or(|need| dib.len().saturating_sub(size) > need)
};
let insert = match masks {
Some(m) if bitfields && size <= dib.len() && !has_copy(&m) => Some(m),
_ => None,
};
if insert.is_some() && dib.try_reserve(12).is_err() {
return dib;
}
if promote {
put(&mut dib, 16, 3);
if default_masks {
put(&mut dib, 40, 0x00FF_0000);
put(&mut dib, 44, 0x0000_FF00);
put(&mut dib, 48, 0x0000_00FF);
}
}
if let Some(m) = insert {
dib.splice(size..size, m);
}
dib
}
/// Bytes of pixel rows (4-byte aligned stride) the info-header dimensions of `dib` call for, or
/// `None` when width, height or bit count cannot be read.
fn pixel_bytes(dib: &[u8]) -> Option<usize> {
let i32_at = |at: usize| dib.get(at..at + 4).and_then(|b| b.try_into().ok()).map(i32::from_le_bytes);
let w = u64::try_from(i32_at(4)?).ok()?;
let h = u64::from(i32_at(8)?.unsigned_abs());
let bpp = u64::from(u16::from_le_bytes(dib.get(14..16)?.try_into().ok()?));
let stride = w.checked_mul(bpp)?.div_ceil(32).checked_mul(4)?;
usize::try_from(stride.checked_mul(h)?).ok()
}
/// Coverage of the declared pixel array. Row byte counts include row padding; RLE coverage
/// counts the pixels drawn by commands and may be smaller than the canvas.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
@@ -174,4 +285,335 @@ mod tests {
b[0] = b'X';
assert!(!is_supported_header(&b));
}
// The WordCraft originals, kept verbatim as the oracle.
fn oracle_dib_to_bmp(dib: &[u8]) -> Vec<u8> {
let u16_at = |at: usize| dib.get(at..at + 2).and_then(|b| b.try_into().ok()).map(u16::from_le_bytes);
let u32_at = |at: usize| dib.get(at..at + 4).and_then(|b| b.try_into().ok()).map(u32::from_le_bytes);
let bi_size = u32_at(0).unwrap_or(40);
let offset = if bi_size == 12 {
let bpp = u16_at(10).unwrap_or(24);
let palette = if bpp <= 8 { (1u32 << bpp) * 3 } else { 0 };
14 + bi_size + palette
} else {
let bpp = u16_at(14).unwrap_or(24);
let compression = u32_at(16).unwrap_or(0);
let clr_used = u32_at(32).unwrap_or(0);
let palette_entries = if clr_used != 0 {
clr_used
} else if bpp <= 8 {
1u32 << bpp
} else {
0
};
let masks = if compression == 3 && bi_size >= 40 { 12 } else { 0 };
14u32.saturating_add(bi_size).saturating_add(masks).saturating_add(palette_entries.saturating_mul(4))
};
let file_size = u32::try_from(14 + dib.len()).unwrap_or(u32::MAX);
let mut bmp = Vec::with_capacity(14 + dib.len());
bmp.extend_from_slice(b"BM");
bmp.extend_from_slice(&file_size.to_le_bytes());
bmp.extend_from_slice(&[0u8; 4]);
bmp.extend_from_slice(&offset.to_le_bytes());
bmp.extend_from_slice(dib);
bmp
}
fn oracle_dib_layout(mut dib: Vec<u8>) -> Vec<u8> {
let u32_at = |d: &[u8], at: usize| d.get(at..at + 4).and_then(|b| b.try_into().ok()).map(u32::from_le_bytes);
let Some(size) = u32_at(&dib, 0).filter(|&s| s >= 108).and_then(|s| usize::try_from(s).ok()) else { return dib };
let put = |d: &mut Vec<u8>, at: usize, v: u32| {
if let Some(b) = d.get_mut(at..at + 4) {
b.copy_from_slice(&v.to_le_bytes());
}
};
if dib.get(14..16) == Some(&[32, 0]) && u32_at(&dib, 16) == Some(0) && u32_at(&dib, 52) == Some(0xFF00_0000) {
put(&mut dib, 16, 3);
if (40..52).step_by(4).all(|at| u32_at(&dib, at) == Some(0)) {
put(&mut dib, 40, 0x00FF_0000);
put(&mut dib, 44, 0x0000_FF00);
put(&mut dib, 48, 0x0000_00FF);
}
}
if u32_at(&dib, 16) == Some(3)
&& size <= dib.len()
&& let Some(masks) = dib.get(40..52).map(<[u8]>::to_vec)
&& dib.get(size..size + 12) != Some(masks.as_slice())
{
dib.splice(size..size, masks);
}
dib
}
fn core(bpp: u16) -> Vec<u8> {
let mut d = Vec::new();
d.extend_from_slice(&12u32.to_le_bytes());
d.extend_from_slice(&2u16.to_le_bytes());
d.extend_from_slice(&2u16.to_le_bytes());
d.extend_from_slice(&1u16.to_le_bytes());
d.extend_from_slice(&bpp.to_le_bytes());
d.resize(12 + 64, 7);
d
}
#[allow(clippy::too_many_arguments)]
fn info(size: u32, bpp: u16, comp: u32, clr_used: u32, masks: [u32; 4], tail: usize) -> Vec<u8> {
let mut d = vec![0u8; size.clamp(40, 124) as usize];
d[0..4].copy_from_slice(&size.to_le_bytes());
d[4..8].copy_from_slice(&2i32.to_le_bytes());
d[8..12].copy_from_slice(&2i32.to_le_bytes());
d[12..14].copy_from_slice(&1u16.to_le_bytes());
d[14..16].copy_from_slice(&bpp.to_le_bytes());
d[16..20].copy_from_slice(&comp.to_le_bytes());
d[32..36].copy_from_slice(&clr_used.to_le_bytes());
for (i, m) in masks.iter().enumerate() {
if let Some(b) = d.get_mut(40 + 4 * i..44 + 4 * i) {
b.copy_from_slice(&m.to_le_bytes());
}
}
d.extend((0..tail).map(|i| (i * 31 + 5) as u8));
d
}
fn corpus() -> Vec<Vec<u8>> {
let mut v = Vec::new();
for bpp in [1, 4, 8, 24, 0, 99] {
v.push(core(bpp));
}
for bpp in [1u16, 4, 8, 16, 24, 32, 0, 255] {
for clr in [0u32, 1, 5, 256, u32::MAX] {
v.push(info(40, bpp, 0, clr, [0; 4], 40));
}
}
for (size, comp, masks) in [
(40, 3, [0xFF0000, 0xFF00, 0xFF, 0]),
(40, 3, [0; 4]),
(52, 3, [0xFF0000, 0xFF00, 0xFF, 0]),
(108, 3, [0xFF0000, 0xFF00, 0xFF, 0xFF00_0000]),
(108, 0, [0xFF0000, 0xFF00, 0xFF, 0xFF00_0000]),
(108, 0, [0, 0, 0, 0xFF00_0000]),
(108, 0, [0; 4]),
(124, 3, [0xFF0000, 0xFF00, 0xFF, 0xFF00_0000]),
(124, 0, [0, 0, 0, 0xFF00_0000]),
(124, 0, [1, 0, 0, 0xFF00_0000]),
(124, 0, [0xFF0000, 0xFF00, 0xFF, 0]),
(124, 1, [0, 0, 0, 0xFF00_0000]),
(124, 3, [1, 2, 3, 4]),
(0, 0, [0; 4]),
(12, 0, [0; 4]),
(u32::MAX, 3, [0; 4]),
(u32::MAX, 0, [0; 4]),
(200, 3, [9; 4]),
] {
for bpp in [8u16, 16, 32] {
for clr in [0u32, 2, u32::MAX] {
v.push(info(size, bpp, comp, clr, masks, 48));
}
}
}
// 52/56-byte bitfields headers with and without a pixel tail, and 3x1 V5 DIBs.
for size in [52, 56] {
v.push(info(size, 32, 3, 0, [0xFF0000, 0xFF00, 0xFF, 0xFF00_0000], 4));
v.push(info(size, 32, 3, 0, [0xFF0000, 0xFF00, 0xFF, 0xFF00_0000], 0));
}
v.push(v5_3x1(false));
v.push(v5_3x1(true));
v
}
/// 3x1 32-bit V5 BI_BITFIELDS whose three pixels equal the colour masks; with `copy` the DIB
/// also holds the real 12-byte mask copy before the pixels.
fn v5_3x1(copy: bool) -> Vec<u8> {
let masks = [0x00FF_0000u32, 0x0000_FF00, 0x0000_00FF];
let mut d = info(124, 32, 3, 0, [masks[0], masks[1], masks[2], 0xFF00_0000], 0);
d[4..8].copy_from_slice(&3i32.to_le_bytes());
d[8..12].copy_from_slice(&1i32.to_le_bytes());
for _ in 0..usize::from(copy) + 1 {
for m in masks {
d.extend_from_slice(&m.to_le_bytes());
}
}
d
}
fn dib_u32(d: &[u8], at: usize) -> Option<u32> {
d.get(at..at + 4).and_then(|b| b.try_into().ok()).map(u32::from_le_bytes)
}
/// Fixed case 1: a 52- or 56-byte BI_BITFIELDS header holds its masks itself, so `dib_to_bmp`
/// no longer adds the 12 appended-mask bytes the WordCraft original added to the pixel offset.
fn bmp_offset_fixed(dib: &[u8]) -> bool {
matches!(dib_u32(dib, 0).unwrap_or(40), 52 | 56) && dib_u32(dib, 16) == Some(3)
}
/// Fixed case 2: the original read 12 pixel bytes equal to the masks (exact-length payload) as a
/// mask copy and inserted nothing. Returns the corrected output when `cut` is such a case.
fn exact_length_fixed(cut: &[u8]) -> Option<Vec<u8>> {
let o = oracle_dib_layout(cut.to_vec());
let size = usize::try_from(dib_u32(&o, 0).filter(|&s| s >= 108)?).ok()?;
let need = pixel_bytes(&o)?;
let same = o.len() == cut.len()
&& dib_u32(&o, 16) == Some(3)
&& size <= o.len()
&& o.get(size..size + 12).is_some()
&& o.get(size..size + 12) == o.get(40..52)
&& o.len() - size <= need;
if !same {
return None;
}
let mut want = o.clone();
want.splice(size..size, o[40..52].to_vec());
Some(want)
}
#[test]
fn dib_helpers_match_the_wordcraft_originals_byte_for_byte() {
let (mut total, mut offset_cases, mut exact_cases) = (0usize, 0usize, 0usize);
for dib in corpus() {
for n in 0..=dib.len() {
let cut = &dib[..n];
total += 1;
let fixed_layout = exact_length_fixed(cut);
match &fixed_layout {
Some(want) => {
exact_cases += 1;
assert_eq!(&dib_layout(cut.to_vec()), want, "exact-length layout len {n}");
}
None => assert_eq!(dib_layout(cut.to_vec()), oracle_dib_layout(cut.to_vec()), "layout len {n}"),
}
if bmp_offset_fixed(cut) {
offset_cases += 1;
let (new, old) = (dib_to_bmp(cut).unwrap(), oracle_dib_to_bmp(cut));
assert_eq!(new[..10], old[..10]);
assert_eq!(new[14..], old[14..]);
let (a, b) = (dib_u32(&new, 10).unwrap(), dib_u32(&old, 10).unwrap());
assert_eq!(a.saturating_add(12), b, "offset len {n}");
} else {
assert_eq!(dib_to_bmp(cut).unwrap(), oracle_dib_to_bmp(cut), "bmp len {n}");
}
let both = dib_layout(cut.to_vec());
let old_both = oracle_dib_layout(cut.to_vec());
if fixed_layout.is_none() && !bmp_offset_fixed(&both) {
assert_eq!(dib_to_bmp(&both).unwrap(), oracle_dib_to_bmp(&old_both));
}
}
}
eprintln!("corpus cases {total}: 52/56-byte bitfields offset {offset_cases}, exact-length payload {exact_cases}");
assert!(offset_cases > 0 && exact_cases > 0);
}
#[test]
fn dib_helpers_survive_pseudo_random_bytes() {
let mut x = 0x9E37_79B9_7F4A_7C15u64;
for len in 0..300usize {
let d: Vec<u8> = (0..len)
.map(|_| {
x ^= x << 13;
x ^= x >> 7;
x ^= x << 17;
x as u8
})
.collect();
let laid = dib_layout(d.clone());
if exact_length_fixed(&d).is_none() && !bmp_offset_fixed(&laid) {
assert_eq!(dib_to_bmp(&laid).unwrap(), oracle_dib_to_bmp(&oracle_dib_layout(d)));
}
}
}
#[test]
fn huge_palette_count_saturates_the_offset() {
let b = dib_to_bmp(&info(40, 8, 0, u32::MAX, [0; 4], 8)).unwrap();
assert_eq!(u32::from_le_bytes([b[10], b[11], b[12], b[13]]), u32::MAX);
let b = dib_to_bmp(&info(u32::MAX, 8, 3, u32::MAX, [0; 4], 8)).unwrap();
assert_eq!(u32::from_le_bytes([b[10], b[11], b[12], b[13]]), u32::MAX);
}
#[test]
fn wrapped_dibs_decode_and_keep_alpha() {
// 2x2 32-bit V5, BI_RGB with an alpha mask, bottom-up BGRA pixels.
let mut d = info(124, 32, 0, 0, [0, 0, 0, 0xFF00_0000], 0);
let px = [[1u8, 2, 3, 255], [4, 5, 6, 128], [7, 8, 9, 64], [10, 11, 12, 0]];
for p in px {
d.extend_from_slice(&p);
}
let bmp = dib_to_bmp(&dib_layout(d)).unwrap();
let img = crate::decode(&bmp).unwrap();
let alphas: Vec<u16> = img.to_rgba8().chunks(4).map(|p| u16::from(p[3])).collect();
assert_eq!(alphas, [64, 0, 255, 128]);
// 24-bit plain DIB and a 12-byte core 8-bit one open too.
let mut d = info(40, 24, 0, 0, [0; 4], 0);
d.extend_from_slice(&[0, 0, 255, 0, 255, 0, 0, 0, 0, 255, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]);
assert!(crate::decode(&dib_to_bmp(&d).unwrap()).is_ok());
let mut c = core(8);
c.truncate(12);
c.extend((0..256 * 3).map(|i| i as u8));
c.extend_from_slice(&[0, 1, 0, 0, 2, 3, 0, 0]);
assert!(crate::decode(&dib_to_bmp(&c).unwrap()).is_ok());
}
/// 1x1 32-bit BI_BITFIELDS DIB, one pixel B=10 G=20 R=30 A=128, with the given header size.
fn one_pixel(size: u32) -> Vec<u8> {
let mut d = info(size, 32, 3, 0, [0xFF0000, 0xFF00, 0xFF, 0xFF00_0000], 0);
d[4..8].copy_from_slice(&1i32.to_le_bytes());
d[8..12].copy_from_slice(&1i32.to_le_bytes());
if size == 40 {
for m in [0xFF0000u32, 0xFF00, 0xFF] {
d.extend_from_slice(&m.to_le_bytes());
}
}
d.extend_from_slice(&[10, 20, 30, 128]);
d
}
#[test]
fn bitfields_pixel_offset_for_each_header_size_decodes() {
for size in [40u32, 52, 56] {
let bmp = dib_to_bmp(&one_pixel(size)).unwrap();
let off = 14 + size + if size == 40 { 12 } else { 0 };
assert_eq!(dib_u32(&bmp, 10), Some(off), "offset for {size}");
let img = crate::decode(&bmp).unwrap_or_else(|e| panic!("{size}-byte header: {e:?}"));
let rgba = img.to_rgba8();
assert_eq!(&rgba[..3], &[30, 20, 10], "size {size}");
if size == 56 {
assert_eq!(rgba[3], 128);
}
}
// V5: the header's masks, a copy inserted by `dib_layout`, then the pixel.
let bmp = dib_to_bmp(&dib_layout(one_pixel(124))).unwrap();
assert_eq!(&crate::decode(&bmp).unwrap().to_rgba8()[..4], &[30, 20, 10, 128]);
}
#[test]
fn exact_length_pixels_equal_to_the_masks_are_not_a_mask_copy() {
let d = v5_3x1(false);
let laid = dib_layout(d.clone());
assert_eq!(laid.len(), d.len() + 12);
assert_eq!(laid[124..136], d[40..52]);
assert_eq!(oracle_dib_layout(d).len(), laid.len() - 12, "the original skipped the insertion");
let img = crate::decode(&dib_to_bmp(&laid).unwrap()).unwrap().to_rgba8();
assert_eq!(img.len(), 12);
// pixel 0 is 0x00FF0000 (blue 0, green 0, red 255 under the masks).
assert_eq!(&img[..3], &[255, 0, 0]);
// A real copy followed by the pixels is left alone.
let real = v5_3x1(true);
assert_eq!(dib_layout(real.clone()), real);
assert!(crate::decode(&dib_to_bmp(&real).unwrap()).is_ok());
}
#[test]
fn layout_decides_before_it_mutates() {
// BI_RGB V5 with an alpha mask and no colour masks: promoted and given a copy together.
let mut d = info(124, 32, 0, 0, [0, 0, 0, 0xFF00_0000], 16);
d[4..8].copy_from_slice(&2i32.to_le_bytes());
let laid = dib_layout(d.clone());
assert_eq!(laid.len(), d.len() + 12);
assert_eq!(dib_u32(&laid, 16), Some(3));
// Unreadable dimensions keep the byte-equality rule.
let mut odd = info(108, 32, 3, 0, [1, 2, 3, 0], 0);
odd[4..8].copy_from_slice(&(-1i32).to_le_bytes());
let copy = odd[40..52].to_vec();
odd.extend_from_slice(&copy);
assert_eq!(dib_layout(odd.clone()), odd);
}
}
+337 -14
View File
@@ -46,6 +46,45 @@ pub(crate) struct JpegMeta {
pub held_coefficients: u64,
}
/// Validate a JPEG by decoding the 8-bit RGBA output used by renderers.
///
/// This deliberately uses the existing zune backend's default decode policy.
/// Unlike general JPEG import it does not fall back to additional JPEG modes,
/// apply orientation, or retain pixels. The caller supplies the admitted pixel count.
pub fn validate_jpeg_rgba8(bytes: &[u8], max_pixels: u64) -> Result<(), CodecError> {
validate_jpeg_rgba8_with_panic_policy(bytes, max_pixels, cfg!(panic = "abort"))
}
fn validate_jpeg_rgba8_with_panic_policy(bytes: &[u8], max_pixels: u64, panic_aborts: bool) -> Result<(), CodecError> {
// The decoder can assert while upsampling these frames. On abort builds the
// unwind guard below cannot turn that assertion into an image omission.
if panic_aborts && first_component_subsampled(bytes) {
return Err(CodecError::malformed(F, "JPEG first component is subsampled below the frame maximum"));
}
let decode = || {
let options = DecoderOptions::default().jpeg_set_out_colorspace(ColorSpace::RGBA);
let mut decoder = zune_jpeg::JpegDecoder::new_with_options(std::io::Cursor::new(bytes), options);
decoder.decode_headers().map_err(|error| CodecError::malformed(F, format_args!("{error:?}")))?;
let info = decoder.info().ok_or_else(|| CodecError::malformed(F, "missing frame dimensions"))?;
let pixels = u64::from(info.width) * u64::from(info.height);
if pixels == 0 {
return Err(CodecError::malformed(F, "empty frame"));
}
if pixels > max_pixels {
return Err(CodecError::LimitExceeded("JPEG exceeds the admitted pixel count".into()));
}
let length = usize::try_from(pixels * 4).map_err(|_| CodecError::LimitExceeded("JPEG RGBA output is too large".into()))?;
if decoder.output_colorspace() != Some(ColorSpace::RGBA) || decoder.output_buffer_size() != Some(length) {
return Err(CodecError::malformed(F, "unexpected RGBA output"));
}
let mut output = Vec::new();
output.try_reserve_exact(length).map_err(|_| CodecError::LimitExceeded("could not allocate JPEG RGBA output".into()))?;
output.resize(length, 0);
decoder.decode_into(&mut output).map_err(|error| CodecError::malformed(F, format_args!("{error:?}")))
};
std::panic::catch_unwind(decode).map_err(|_| CodecError::malformed(F, "JPEG decoder failed on this frame"))?
}
/// The first frame of `b` as [`crate::JpegInfo`], from its markers alone (see [`crate::jpeg_info`]).
pub(crate) fn info(b: &[u8]) -> Option<crate::JpegInfo> {
let m = scan_metadata(b);
@@ -86,6 +125,83 @@ fn held_coefficients(marker: u8, seg: &[u8], scan_components: Option<u8>) -> u64
factors.iter().map(|&(fh, fv)| mcus * fh * fv * 64 * 2).sum()
}
/// Bytes of DCT coefficients to budget for before decoding hostile `b`: the larger of
/// [`crate::JpegInfo::held_coefficients`]'s rule (first frame; SOF2, or SOF0/SOF1 whose first
/// scan lacks components; 8-bit only; sampling factors as written) and a coarser one that
/// keeps a progressive JPEG's coefficients for the whole picture:
/// - every SOF2, SOF6, SOF10 and SOF14 header before the first scan counts, at any precision, and
/// the largest wins; each charges 2 bytes a coefficient: per component, `2 * 64 * h * v` per
/// MCU, with `h` and `v` clamped to 1..=4 and the MCUs covering the frame at the largest `h`, `v`;
/// - a header that can't be followed (a segment that is cut, a length under 2, a frame header
/// without its components, no `0xFF` where a marker belongs) charges the costliest frame,
/// four components sampled 4 × 4, at the first frame's size (as `scan_metadata` reads it), and 0 without one.
///
/// Never less than either rule. Frames `craft-jpeg` decodes (arithmetic, lossless, 12-bit)
/// enforce `max_alloc` on their own buffers, and SOF6/SOF14 (hierarchical) are not decoded at all,
/// so charging them is only a conservative budget. Arithmetic saturates; the scan is linear.
pub(crate) fn worst_case_coefficients(b: &[u8]) -> u64 {
let m = scan_metadata(b);
let mut at = 2usize;
let mut most = 0u64;
let first: Option<(u64, u64)> = m.frame.map(|(w, h, _)| (u64::from(w), u64::from(h)));
let mut broken = false;
while !broken {
if b.get(at) != Some(&0xFF) {
broken = true;
break;
}
while b.get(at) == Some(&0xFF) {
at += 1;
}
let Some(&code) = b.get(at) else {
broken = true;
break;
};
at += 1;
match code {
0xD8 | 0x01 | 0xD0..=0xD7 => continue,
0xD9 | 0xDA => break,
_ => {}
}
let len = match (b.get(at), b.get(at + 1)) {
(Some(&hi), Some(&lo)) => usize::from(u16::from_be_bytes([hi, lo])),
_ => 0,
};
let Some(seg) = (len >= 2).then(|| b.get(at + 2..at + len)).flatten() else {
broken = true;
break;
};
if matches!(code, 0xC2 | 0xC6 | 0xCA | 0xCE) {
match progressive_frame_coefficients(seg) {
Some(held) => most = most.max(held),
None => {
broken = true;
break;
}
}
}
at += len;
}
if broken && let Some((w, h)) = first {
let mcus = w.div_ceil(32).saturating_mul(h.div_ceil(32));
most = most.max(mcus.saturating_mul(2 * 64 * 16 * 4));
}
most.max(m.held_coefficients)
}
/// [`worst_case_coefficients`]'s charge for one frame header `sof` (after its length).
fn progressive_frame_coefficients(sof: &[u8]) -> Option<u64> {
let height = u64::from(u16::from_be_bytes([*sof.get(1)?, *sof.get(2)?]));
let width = u64::from(u16::from_be_bytes([*sof.get(3)?, *sof.get(4)?]));
let n = usize::from(*sof.get(5)?);
let comps = sof.get(6..6usize.checked_add(n.checked_mul(3)?)?)?;
let factors: Vec<(u64, u64)> =
comps.as_chunks::<3>().0.iter().map(|&[_, f, _]| (u64::from(f >> 4).clamp(1, 4), u64::from(f & 15).clamp(1, 4))).collect();
let (hmax, vmax) = factors.iter().fold((1, 1), |(h, v), &(a, b)| (h.max(a), v.max(b)));
let mcus = width.div_ceil(8 * hmax).saturating_mul(height.div_ceil(8 * vmax));
Some(factors.iter().fold(0u64, |sum, &(h, v)| sum.saturating_add(mcus.saturating_mul(2 * 64 * h * v))))
}
/// Walk the marker segments up to SOS and collect metadata.
pub(crate) fn scan_metadata(b: &[u8]) -> JpegMeta {
let mut m = JpegMeta::default();
@@ -340,10 +456,17 @@ fn first_component_subsampled(b: &[u8]) -> bool {
return false;
}
let mut i = 2;
while let (Some(&0xFF), Some(&m)) = (b.get(i), b.get(i + 1)) {
while let Some(&prefix) = b.get(i) {
// The decoder searches forward between segments. Stray bytes and
// stuffed FF 00 pairs must not hide a frame from the abort-build guard.
if prefix != 0xFF {
i += 1;
continue;
}
let Some(&m) = b.get(i + 1) else { return false };
match m {
0xFF => i += 1,
0x01 | 0xD0..=0xD8 => i += 2,
0x00 | 0x01 | 0xD0..=0xD8 => i += 2,
0xD9 | 0xDA => return false,
_ => {
let Some(len) = b.get(i + 2..i + 4).map(|l| usize::from(u16::from_be_bytes([l[0], l[1]]))) else { return false };
@@ -825,6 +948,170 @@ mod tests {
assert_eq!(held(&frame(0xC2, 12, &[0x11, 0x11, 0x11], None)), Some(0));
}
/// The consumer's own estimate (WordCraft's `jpeg_coefficients`), ported as the oracle that
/// `worst_case_coefficients` must never undercut.
fn oracle(bytes: &[u8]) -> u64 {
fn worst(bytes: &[u8]) -> u64 {
let Some((w, h)) = info(bytes).map(|i| (i.width, i.height)) else { return 0 };
let mcus = u64::from(w).div_ceil(32) * u64::from(h).div_ceil(32);
mcus.saturating_mul(2 * 64 * 16 * 4)
}
fn frame(sof: &[u8]) -> Option<u64> {
let height = u64::from(u16::from_be_bytes([*sof.get(1)?, *sof.get(2)?]));
let width = u64::from(u16::from_be_bytes([*sof.get(3)?, *sof.get(4)?]));
let n = usize::from(*sof.get(5)?);
let s: Vec<(u64, u64)> = sof
.get(6..6 + 3 * n)?
.as_chunks::<3>()
.0
.iter()
.map(|&[_, s, _]| (u64::from(s >> 4).clamp(1, 4), u64::from(s & 15).clamp(1, 4)))
.collect();
let (hm, vm) = s.iter().fold((1, 1), |(h, v), &(a, b)| (h.max(a), v.max(b)));
let mcus = width.div_ceil(8 * hm).checked_mul(height.div_ceil(8 * vm))?;
s.iter().try_fold(0u64, |sum, &(h, v)| sum.checked_add(mcus.checked_mul(2 * 64 * h * v)?))
}
let mut at = 2;
let mut most = 0u64;
loop {
if bytes.get(at) != Some(&0xFF) {
return worst(bytes).max(most);
}
while bytes.get(at) == Some(&0xFF) {
at += 1;
}
let Some(&code) = bytes.get(at) else { return worst(bytes).max(most) };
at += 1;
match code {
0xD8 | 0x01 | 0xD0..=0xD7 => continue,
0xD9 | 0xDA => return most,
_ => {}
}
let len = match (bytes.get(at), bytes.get(at + 1)) {
(Some(&hi), Some(&lo)) => usize::from(u16::from_be_bytes([hi, lo])),
_ => 0,
};
let Some(seg) = bytes.get(at + 2..at + len).filter(|_| len >= 2) else { return worst(bytes).max(most) };
if matches!(code, 0xC2 | 0xC6 | 0xCA | 0xCE) {
let Some(h) = frame(seg) else { return worst(bytes).max(most) };
most = most.max(h);
}
at += len;
}
}
/// A JPEG header: each frame is (marker, precision, width, height, sampling bytes), then an
/// optional scan of `scan` components, then EOI.
fn header(frames: &[(u8, u8, u16, u16, &[u8])], scan: Option<u8>) -> Vec<u8> {
let mut b = vec![0xFF, 0xD8];
for &(marker, precision, w, h, f) in frames {
b.extend_from_slice(&[0xFF, marker]);
b.extend_from_slice(&(8 + 3 * f.len() as u16).to_be_bytes());
b.push(precision);
b.extend_from_slice(&h.to_be_bytes());
b.extend_from_slice(&w.to_be_bytes());
b.push(f.len() as u8);
for (i, &s) in f.iter().enumerate() {
b.extend_from_slice(&[i as u8 + 1, s, 0]);
}
}
if let Some(n) = scan {
b.extend_from_slice(&[0xFF, 0xDA, 0, 6 + 2 * n, n]);
(1..=n).for_each(|c| b.extend_from_slice(&[c, 0]));
b.extend_from_slice(&[0, 63, 0]);
}
b.extend_from_slice(&[0xFF, 0xD9]);
b
}
/// Never less than the oracle or `held_coefficients`; equal to the larger on valid headers.
#[test]
fn worst_case_coefficients_covers_both_rule_sets() {
const C3: &[u8] = &[0x22, 0x11, 0x11];
const G: &[u8] = &[0x11];
let cases: Vec<Vec<u8>> = vec![
header(&[(0xC0, 8, 640, 480, C3)], Some(3)),
header(&[(0xC0, 8, 640, 480, G)], Some(1)),
header(&[(0xC0, 8, 640, 480, C3)], Some(1)),
header(&[(0xC1, 8, 640, 480, C3)], None),
header(&[(0xC2, 8, 640, 480, C3)], Some(1)),
header(&[(0xC2, 12, 640, 480, C3)], Some(1)),
header(&[(0xC6, 8, 640, 480, C3)], None),
header(&[(0xCA, 8, 640, 480, C3)], None),
header(&[(0xCE, 8, 640, 480, C3)], None),
header(&[(0xC0, 8, 64, 64, G), (0xC2, 8, 6000, 4000, C3), (0xC2, 8, 100, 100, C3)], Some(1)),
header(&[(0xC2, 8, 640, 480, &[0x00, 0x00, 0x00])], None),
header(&[(0xC2, 8, 640, 480, &[0xFF, 0xFF, 0xFF, 0xFF])], None),
header(&[(0xC2, 8, 65535, 65535, &[0x44; 4])], None),
header(&[(0xC0, 8, 65535, 65535, &[0xFF; 4])], Some(1)),
header(&[], None),
vec![0xFF, 0xD8],
vec![],
b"not a jpeg at all".to_vec(),
];
for b in &cases {
let held = scan_metadata(b).held_coefficients;
let got = worst_case_coefficients(b);
assert!(got >= oracle(b) && got >= held, "{b:02x?}");
assert!(got >= info(b).map_or(0, |i| i.held_coefficients));
}
let at = |i: usize| worst_case_coefficients(&cases[i]);
// Equal to the oracle where the shared rule charges nothing extra.
for i in [0, 1, 6, 7, 8, 9, 10, 12, 14] {
assert_eq!(at(i), oracle(&cases[i]), "case {i}");
}
// Where it does, the shared figure wins (the oracle misses a short first scan).
assert_eq!(oracle(&cases[2]), 0);
assert_eq!(at(2), scan_metadata(&cases[2]).held_coefficients);
// Unclamped factors (15 × 15) charge the shared rule more than the clamped oracle.
assert_eq!(at(11), scan_metadata(&cases[11]).held_coefficients);
assert!(at(11) > oracle(&cases[11]));
assert!(at(2) > 0 && at(3) == 0 && at(0) == 0);
// 12-bit and arithmetic progressive frames are charged though craft-jpeg limits itself.
assert_eq!(at(5), at(4));
// Sampling factors clamp: 0 acts as 1, 15 as 4.
assert_eq!(at(10), 80 * 60 * 3 * 128);
assert_eq!(at(12), 2048 * 2048 * 4 * 2 * 64 * 16);
// The largest of several frames wins.
assert_eq!(at(9), oracle(&header(&[(0xC2, 8, 6000, 4000, C3)], None)));
}
/// A cut header charges the costliest frame at its size; every prefix and random damage stays
/// bounded, never panics and never undercuts the oracle.
#[test]
fn worst_case_coefficients_hostile_input() {
let full = header(&[(0xC2, 8, 6000, 4000, &[0x22, 0x11, 0x11])], Some(1));
let cut = &full[..full.len() - 8];
// Cut inside the component list: a frame the scan can't read, at its own size.
let sof_cut = &full[..14];
assert_eq!(worst_case_coefficients(sof_cut), oracle(sof_cut));
assert_eq!(worst_case_coefficients(cut), oracle(cut));
let mut bases = vec![full.clone(), header(&[(0xC0, 8, 6000, 4000, &[0x11; 3])], Some(1))];
bases.push(header(&[(0xC2, 8, 65535, 65535, &[0xFF; 4])], None));
let mut seed = 0x9E37_79B9_7F4A_7C15u64;
let mut next = move || {
seed = seed.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1_442_695_040_888_963_407);
(seed >> 33) as usize
};
for base in &bases {
for n in 0..=base.len() {
let b = &base[..n];
let got = worst_case_coefficients(b);
assert!(got >= oracle(b) && got < 1 << 50);
}
for _ in 0..2000 {
let mut b = base.clone();
for _ in 0..=next() % 4 {
let i = next() % b.len();
b[i] = next() as u8;
}
b.truncate(next() % (b.len() + 1));
let got = worst_case_coefficients(&b);
assert!(got >= oracle(&b) && got >= scan_metadata(&b).held_coefficients && got < 1 << 50);
}
}
}
/// An APP1 Extended XMP fragment (GUID, total size, offset, data).
fn extended_xmp(offset: u32) -> Vec<u8> {
let mut seg = EXTENDED_XMP_HEADER.to_vec();
@@ -1073,25 +1360,61 @@ mod tests {
(v * 65535.0 / 4095.0).round() as u16
}
fn progressive_frame_header(comps: &[(u8, u8)]) -> Vec<u8> {
let mut s = vec![0xFF, 0xD8, 0xFF, 0xE0, 0, 4, 0, 0, 0xFF, 0xC2, 0, (8 + 3 * comps.len()) as u8, 8, 0, 8, 0, 8, comps.len() as u8];
for (i, &(h, v)) in comps.iter().enumerate() {
s.extend([i as u8 + 1, (h << 4) | v, 0]);
}
s
}
#[test]
fn first_component_subsampling() {
let sof = |comps: &[(u8, u8)]| {
let mut s = vec![0xFF, 0xD8, 0xFF, 0xE0, 0, 4, 0, 0, 0xFF, 0xC2, 0, (8 + 3 * comps.len()) as u8, 8, 0, 8, 0, 8, comps.len() as u8];
for (i, &(h, v)) in comps.iter().enumerate() {
s.extend([i as u8 + 1, (h << 4) | v, 0]);
}
s
};
assert!(first_component_subsampled(&sof(&[(1, 1), (1, 1), (1, 1), (2, 2)])));
assert!(first_component_subsampled(&sof(&[(1, 2), (1, 2), (2, 1)])));
assert!(!first_component_subsampled(&sof(&[(2, 2), (1, 1), (1, 1)])));
assert!(!first_component_subsampled(&sof(&[(1, 1); 4])));
assert!(!first_component_subsampled(&sof(&[])));
assert!(first_component_subsampled(&progressive_frame_header(&[(1, 1), (1, 1), (1, 1), (2, 2)])));
assert!(first_component_subsampled(&progressive_frame_header(&[(1, 2), (1, 2), (2, 1)])));
assert!(!first_component_subsampled(&progressive_frame_header(&[(2, 2), (1, 1), (1, 1)])));
assert!(!first_component_subsampled(&progressive_frame_header(&[(1, 1); 4])));
assert!(!first_component_subsampled(&progressive_frame_header(&[])));
assert!(!first_component_subsampled(&[0xFF, 0xD8, 0xFF, 0xDA, 0, 2]));
assert!(!first_component_subsampled(&[0xFF, 0xD8, 0xFF, 0xC0, 0, 20, 8]));
assert!(!first_component_subsampled(b"not a jpeg"));
}
#[test]
fn renderer_validation_refuses_unsafe_sampling_under_abort_policy() {
// The frame header alone exercises refusal before any pixel decode.
let header = progressive_frame_header(&[(1, 1), (1, 1), (1, 1), (2, 2)]);
let refuses_sampling = |result| {
matches!(
result,
Err(CodecError::Malformed { format: Format::Jpeg, message })
if message == "JPEG first component is subsampled below the frame maximum"
)
};
assert!(refuses_sampling(validate_jpeg_rgba8_with_panic_policy(&header, 64, true)));
assert!(!refuses_sampling(validate_jpeg_rgba8_with_panic_policy(&header, 64, false)));
let mut ordinary = Vec::new();
jpeg_encoder::Encoder::new(&mut ordinary, 90).encode(&[128], 1, 1, jpeg_encoder::ColorType::Luma).unwrap();
validate_jpeg_rgba8_with_panic_policy(&ordinary, 1, true).unwrap();
}
#[test]
fn jpeg_sampling_guard_finds_frame_after_marker_padding() {
let header = progressive_frame_header(&[(1, 1), (1, 1), (1, 1), (2, 2)]);
// Exercise the shared header guard, without a pixel payload.
for split in [2, 8] {
let (before, after) = header.split_at(split);
for padding in [&[0][..], &[0x7F, 0][..], &[0xFF, 0][..], &[0xFF][..]] {
let padded = [before, padding, after].concat();
assert!(first_component_subsampled(&padded), "offset {split}, padding {padding:?}");
let got = validate_jpeg_rgba8_with_panic_policy(&padded, 64, true);
assert!(matches!(got, Err(CodecError::Malformed { message, .. }) if message.contains("frame maximum")));
}
}
assert!(!first_component_subsampled(&[0xFF, 0xD8, 0, 0xFF]));
}
#[test]
fn routing() {
assert!(!uses_craft_jpeg(&[0xFF, 0xD8, 0xFF, 0xC0, 0, 11, 8, 0, 8, 0, 8, 1, 1, 0x11, 0]));
+10
View File
@@ -45,6 +45,7 @@ pub use crate::cicp::ContentLightLevel;
pub use crate::codecs::avif::{HDR_REFERENCE_WHITE_NITS, HdrTransfer, encode_avif_hdr, encode_avif_with_speed};
pub use crate::codecs::exr_cryptomatte::{cryptomatte_id, cryptomatte_key, cryptomatte_preview_color};
pub use crate::codecs::gif_frames::{AnimationFrame, MAX_GIF_FRAMES};
pub use crate::codecs::jpeg::validate_jpeg_rgba8;
pub use crate::codecs::png::{
PngDecodeRows, PngExtras, PngHeader, PngPalette, PngRowSink, PngRows, SharedBytes, encode_indexed as encode_png_indexed,
encode_pq as encode_png_pq, palette as png_palette, stereo as png_stereo,
@@ -307,6 +308,15 @@ pub fn jpeg_info(bytes: &[u8]) -> Option<JpegInfo> {
jpeg::info(bytes)
}
/// Bytes of DCT coefficients to budget for a JPEG before decoding it: never less than
/// [`JpegInfo::held_coefficients`] nor than the whole-picture estimate of every progressive
/// frame header (SOF2, SOF6, SOF10, SOF14 at any precision, sampling clamped to 1..=4), and the
/// costliest frame at the first size read when the header can't be followed. 0 without a
/// readable frame. See the function body's rules for the precise charges.
pub fn jpeg_worst_case_coefficients(bytes: &[u8]) -> u64 {
jpeg::worst_case_coefficients(bytes)
}
/// PCX's dimensions are available from its fixed-size header without decoding the RLE rows.
pub fn pcx_dimensions(bytes: &[u8]) -> Option<(u32, u32)> {
codecs::pcx::dimensions(bytes)
+223
View File
@@ -136,6 +136,93 @@ pub fn color_chunks(b: &[u8]) -> Option<ColorChunks> {
Some(result)
}
/// Most chunks [`pixel_model`] walks before the image data; each chunk is at least 12 bytes.
const MAX_PROBE_CHUNKS: usize = 1 << 16;
/// Pixel model of a PNG, from `IHDR` and the chunk headers before the first `IDAT`.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct PixelModel {
pub width: u32,
pub height: u32,
/// `IHDR` bit depth (1, 2, 4, 8 or 16).
pub bit_depth: u8,
/// `IHDR` colour type (0 grey, 2 RGB, 3 palette, 4 grey+alpha, 6 RGBA).
pub color_type: u8,
/// A `tRNS` chunk precedes the image data; it adds alpha to grey, RGB and palette images.
pub has_trns: bool,
}
impl PixelModel {
/// Channels of the decoded buffer (palette expands to RGB; `tRNS` adds an alpha channel).
pub fn channels(&self) -> u8 {
let base = match self.color_type {
0 => 1,
2 | 3 => 3,
4 => 2,
_ => 4,
};
base + u8::from(self.has_trns && matches!(self.color_type, 0 | 2 | 3))
}
/// Bytes per pixel of the buffer a decoder that expands sub-byte samples to 8 bits gives:
/// `channels * (2 if bit_depth == 16 else 1)`.
pub fn bytes_per_pixel(&self) -> u8 {
self.channels() * if self.bit_depth == 16 { 2 } else { 1 }
}
/// Whether the image has transparency (alpha colour type or `tRNS`).
pub fn has_alpha(&self) -> bool {
matches!(self.color_type, 4 | 6) || self.has_trns
}
}
/// Dimensions, bit depth, colour type and `tRNS` presence without inflating anything (an `iCCP`
/// profile is skipped, not decompressed). Walks at most 65,536 chunk headers and stops at `IDAT`;
/// `IEND` or the end of the input before `IDAT` is an error. Does not validate CRCs.
/// A caller gets its `(width, height, bytes_per_pixel, alpha)` as
/// `(m.width, m.height, u64::from(m.bytes_per_pixel()), m.has_alpha())`.
pub fn pixel_model(b: &[u8]) -> Result<PixelModel, CodecError> {
if !b.starts_with(b"\x89PNG\r\n\x1a\n") {
return Err(err("bad signature"));
}
let be32 = |p: usize| b.get(p..p.checked_add(4)?).and_then(|v| <[u8; 4]>::try_from(v).ok()).map(u32::from_be_bytes);
if be32(8) != Some(13) || b.get(12..16) != Some(b"IHDR") {
return Err(err("missing IHDR"));
}
let ihdr = b.get(16..29).ok_or_else(|| err("truncated IHDR"))?;
let (width, height) = (be32(16).unwrap_or(0), be32(20).unwrap_or(0));
let (bit_depth, color_type) = (ihdr[8], ihdr[9]);
if width == 0 || height == 0 || width > i32::MAX as u32 || height > i32::MAX as u32 {
return Err(err("dimensions out of range"));
}
let depth_ok = match color_type {
0 => matches!(bit_depth, 1 | 2 | 4 | 8 | 16),
3 => matches!(bit_depth, 1 | 2 | 4 | 8),
2 | 4 | 6 => matches!(bit_depth, 8 | 16),
_ => false,
};
if !depth_ok {
return Err(err("invalid IHDR"));
}
let mut has_trns = false;
let mut p = 8usize;
for _ in 0..MAX_PROBE_CHUNKS {
let len = be32(p).ok_or_else(|| err("truncated"))?;
let kind = p.checked_add(4).and_then(|k| b.get(k..k.checked_add(4)?)).ok_or_else(|| err("truncated"))?;
match kind {
b"IDAT" => return Ok(PixelModel { width, height, bit_depth, color_type, has_trns }),
b"IEND" => return Err(err("no image data")),
b"tRNS" => has_trns = true,
_ => {}
}
if len > i32::MAX as u32 {
return Err(err("chunk length out of range"));
}
p = p.checked_add(12).and_then(|q| q.checked_add(len as usize)).ok_or_else(|| err("truncated"))?;
}
Err(CodecError::LimitExceeded("PNG chunk count".into()))
}
#[cfg(test)]
mod tests {
use super::*;
@@ -187,4 +274,140 @@ mod tests {
assert!(!color.srgb);
assert!(color_chunks(b"not png").is_none());
}
/// WordCraft's `png_chunks` (crates/geom/src/picture.rs) as the differential oracle.
fn oracle(bytes: &[u8]) -> Option<(u32, u32, u64, bool)> {
let be = |at: usize| bytes.get(at..at.checked_add(4)?).and_then(|b| <[u8; 4]>::try_from(b).ok()).map(u32::from_be_bytes);
if bytes.get(12..16) != Some(b"IHDR".as_slice()) {
return None;
}
let (width, height) = (be(16)?, be(20)?);
let (depth, color) = (*bytes.get(24)?, *bytes.get(25)?);
let mut trns = false;
let mut at = 8usize;
loop {
let len = usize::try_from(be(at)?).ok()?;
match bytes.get(at.checked_add(4)?..at.checked_add(8)?)? {
b"IDAT" => break,
b"IEND" => return None,
b"tRNS" => trns = true,
_ => {}
}
at = at.checked_add(12)?.checked_add(len)?;
}
let (channels, alpha) = match color {
0 => (1 + u64::from(trns), trns),
2 | 3 => (3 + u64::from(trns), trns),
4 => (2, true),
6 => (4, true),
_ => return None,
};
Some((width, height, channels * if depth == 16 { 2 } else { 1 }, alpha))
}
fn tuple(b: &[u8]) -> Option<(u32, u32, u64, bool)> {
pixel_model(b).ok().map(|m| (m.width, m.height, u64::from(m.bytes_per_pixel()), m.has_alpha()))
}
fn png_with(depth: u8, color: u8, before_idat: &[(&[u8; 4], Vec<u8>)]) -> Vec<u8> {
let mut bytes = b"\x89PNG\r\n\x1a\n".to_vec();
let mut ihdr = Vec::new();
ihdr.extend_from_slice(&5u32.to_be_bytes());
ihdr.extend_from_slice(&7u32.to_be_bytes());
ihdr.extend_from_slice(&[depth, color, 0, 0, 0]);
chunk(&mut bytes, b"IHDR", &ihdr);
for (k, d) in before_idat {
chunk(&mut bytes, k, d);
}
chunk(&mut bytes, b"IDAT", &[1, 2, 3]);
chunk(&mut bytes, b"IEND", &[]);
bytes
}
#[test]
fn pixel_model_matches_oracle_for_every_color_type_depth_and_trns() {
let valid: [(u8, &[u8]); 5] = [(0, &[1, 2, 4, 8, 16]), (2, &[8, 16]), (3, &[1, 2, 4, 8]), (4, &[8, 16]), (6, &[8, 16])];
for (color, depths) in valid {
for &depth in depths {
for trns in [false, true] {
let extra: Vec<(&[u8; 4], Vec<u8>)> = if trns { vec![(b"tRNS", vec![0, 0])] } else { vec![] };
let bytes = png_with(depth, color, &extra);
assert!(tuple(&bytes).is_some(), "color {color} depth {depth}");
assert_eq!(tuple(&bytes), oracle(&bytes), "color {color} depth {depth} trns {trns}");
}
}
}
}
#[test]
fn pixel_model_known_values() {
let m = pixel_model(&png_with(16, 2, &[(b"tRNS", vec![0; 6])])).unwrap();
assert_eq!((m.width, m.height, m.channels(), m.bytes_per_pixel(), m.has_alpha()), (5, 7, 4, 8, true));
let m = pixel_model(&png_with(4, 3, &[])).unwrap();
assert_eq!((m.bytes_per_pixel(), m.has_alpha()), (3, false));
}
#[test]
fn pixel_model_skips_a_huge_iccp_without_inflating() {
let junk = vec![0xAB; 20 << 20];
let bytes = png_with(8, 2, &[(b"iCCP", junk), (b"tRNS", vec![0; 6])]);
assert_eq!(tuple(&bytes), oracle(&bytes));
assert_eq!(tuple(&bytes), Some((5, 7, 4, true)));
}
#[test]
fn pixel_model_malformed_inputs_agree_with_oracle_and_never_panic() {
let full = png_with(8, 6, &[(b"tRNS", vec![0]), (b"gAMA", vec![0; 4])]);
for n in 0..=full.len() {
let cut = &full[..n];
let got = tuple(cut);
// an Err where the oracle answers can only be a stricter header check
if got.is_some() {
assert_eq!(got, oracle(cut), "prefix {n}");
}
}
assert!(pixel_model(&full[..33]).is_err());
// missing IHDR
let mut bad = full.clone();
bad[12..16].copy_from_slice(b"IHDX");
assert!(pixel_model(&bad).is_err() && oracle(&bad).is_none());
// chunk length overflow: a length past the input, then one past i32::MAX
for len in [u32::MAX, i32::MAX as u32, 0x7fff_fff0] {
let mut o = full.clone();
o[33..37].copy_from_slice(&len.to_be_bytes());
assert!(pixel_model(&o).is_err(), "len {len:#x}");
assert_eq!(tuple(&o), oracle(&o));
}
// IEND before IDAT, unknown colour type, zero size, bad signature
let mut iend = png_with(8, 2, &[]);
let at = iend.windows(4).position(|w| w == b"IDAT").unwrap();
iend[at..at + 4].copy_from_slice(b"IEND");
assert!(pixel_model(&iend).is_err() && oracle(&iend).is_none());
assert!(pixel_model(&png_with(8, 5, &[])).is_err());
let mut zero = png_with(8, 2, &[]);
zero[16..20].copy_from_slice(&0u32.to_be_bytes());
assert!(pixel_model(&zero).is_err());
assert!(pixel_model(b"not a png").is_err());
// deterministic pseudo-random corruption
let mut s = 0x9e37_79b9u32;
for _ in 0..2000 {
let mut f = full.clone();
for _ in 0..3 {
s = s.wrapping_mul(1_664_525).wrapping_add(1_013_904_223);
let i = (s >> 8) as usize % f.len();
f[i] = (s >> 24) as u8;
}
let _ = pixel_model(&f);
}
}
#[test]
fn pixel_model_many_tiny_chunks_and_the_chunk_cap() {
let many: Vec<(&[u8; 4], Vec<u8>)> = (0..10_000).map(|_| (b"tEXt", Vec::new())).chain([(b"tRNS", vec![0])]).collect();
let bytes = png_with(8, 0, &many);
assert_eq!(tuple(&bytes), oracle(&bytes));
assert_eq!(tuple(&bytes), Some((5, 7, 2, true)));
let over: Vec<(&[u8; 4], Vec<u8>)> = (0..MAX_PROBE_CHUNKS + 1).map(|_| (b"tEXt", Vec::new())).collect();
assert!(matches!(pixel_model(&png_with(8, 0, &over)), Err(CodecError::LimitExceeded(_))));
}
}
+54
View File
@@ -32,6 +32,25 @@ pub fn header(bytes: &[u8]) -> Result<(u32, u32, u16), CodecError> {
Ok((w, h, orientation))
}
/// Whether a WebP carries transparency, from the first chunk only (nothing is decoded):
/// `VP8X` flag bit 0x10, `VP8L` `alpha_is_used`, plain `VP8` never.
pub fn has_alpha(bytes: &[u8]) -> Result<bool, CodecError> {
let err = |s: &str| CodecError::malformed(Format::WebP, s);
if bytes.get(..4) != Some(b"RIFF") || bytes.get(8..12) != Some(b"WEBP") {
return Err(err("bad signature"));
}
match bytes.get(12..16) {
Some(b"VP8X") => Ok(bytes.get(20).ok_or_else(|| err("truncated VP8X"))? & 0x10 != 0),
// 0x2f signature byte, then 14+14 size bits, then the alpha_is_used bit (bit 28)
Some(b"VP8L") => match bytes.get(20..25) {
Some([0x2f, a, b, c, d]) => Ok(u32::from_le_bytes([*a, *b, *c, *d]) >> 28 & 1 != 0),
_ => Err(err("bad VP8L header")),
},
Some(b"VP8 ") => Ok(false),
_ => Err(err("no image data")),
}
}
#[cfg(test)]
mod tests {
use super::*;
@@ -71,4 +90,39 @@ mod tests {
bytes.push(0);
assert!(header(&bytes).is_err());
}
#[test]
fn has_alpha_reads_vp8x_vp8l_and_plain_vp8() {
for (layout, want) in [(crate::ChannelLayout::Rgb, false), (crate::ChannelLayout::Rgba, true)] {
let n = layout.channels();
let image = crate::Image::from_u8(2, 3, layout, vec![42; 6 * n]).unwrap();
let bytes = crate::encode(&image, Format::WebP, &crate::EncodeOptions::default()).unwrap();
assert_eq!(has_alpha(&bytes).unwrap(), want);
}
let riff = |chunk: &[u8; 4], body: &[u8]| {
let mut v = b"RIFF\0\0\0\0WEBP".to_vec();
v.extend_from_slice(chunk);
v.extend_from_slice(&(body.len() as u32).to_le_bytes());
v.extend_from_slice(body);
v
};
let mut x = [0u8; 10];
assert!(!has_alpha(&riff(b"VP8X", &x)).unwrap());
x[0] = 0x10;
assert!(has_alpha(&riff(b"VP8X", &x)).unwrap());
assert!(!has_alpha(&riff(b"VP8 ", &[0; 10])).unwrap());
assert!(has_alpha(&riff(b"VP8L", &[0x2f, 0, 0, 0, 0x10])).unwrap());
assert!(!has_alpha(&riff(b"VP8L", &[0x2f, 0, 0, 0, 0])).unwrap());
}
#[test]
fn has_alpha_rejects_malformed_prefixes() {
let full = [b"RIFF\0\0\0\0WEBPVP8X\x0a\0\0\0".as_slice(), &[0x10; 10]].concat();
for n in 0..20 {
assert!(has_alpha(&full[..n]).is_err(), "prefix {n}");
}
assert!(has_alpha(&full[..21]).is_ok());
assert!(has_alpha(b"RIFF\0\0\0\0WEBPVP8L\x05\0\0\0\x2e").is_err());
assert!(has_alpha(b"junk").is_err());
}
}
+8 -3
View File
@@ -76,9 +76,11 @@ fn truncated_files_mostly_error() {
#[test]
fn bit_flips_never_panic() {
// CRAFT_HOSTILE_DIVISOR=N (pull requests) runs 1/N of the flips.
let rounds = 150usize.div_ceil(craft_media_testkit::hostile::divisor());
let mut rng = common::Rng::new(42);
for (f, bytes) in samples() {
for _ in 0..150 {
for _ in 0..rounds {
let mut b = bytes.clone();
let flips = 1 + (rng.next_u64() % 4) as usize;
for _ in 0..flips {
@@ -92,8 +94,11 @@ fn bit_flips_never_panic() {
#[test]
fn byte_overwrites_in_header_never_panic() {
for (f, bytes) in samples() {
for i in 0..bytes.len().min(64) {
// CRAFT_HOSTILE_DIVISOR=N (pull requests) overwrites every N-th byte, from a different offset
// for each sample, so the samples together still reach every header position.
let step = craft_media_testkit::hostile::divisor();
for (k, (f, bytes)) in samples().into_iter().enumerate() {
for i in (0..bytes.len().min(64)).filter(|i| (i + k) % step == 0) {
for v in [0x00, 0x7F, 0xFF] {
let mut b = bytes.clone();
b[i] = v;
+7 -4
View File
@@ -820,15 +820,18 @@ fn poke(bytes: &[u8]) {
#[test]
fn zoo_decodes_and_never_panics_when_cut_or_corrupted() {
for f in zoo() {
// CRAFT_HOSTILE_DIVISOR=N (pull requests) runs every N-th cut and flip, from a different offset
// for each file, and 1/N of the random rounds.
let step = craft_media_testkit::hostile::divisor();
for (k, f) in zoo().into_iter().enumerate() {
let info = tiff_info(&f).unwrap();
for i in 0..info.pages.len() {
decode_tiff_page(&f, i, &DecodeOptions::default()).unwrap();
}
for cut in 0..f.len() {
for cut in (0..f.len()).filter(|cut| (cut + k) % step == 0) {
poke(&f[..cut]);
}
for i in 0..f.len() {
for i in (0..f.len()).filter(|i| (i + k) % step == 0) {
for flip in [0xffu8, 0x80, 0x01] {
let mut g = f.clone();
g[i] ^= flip;
@@ -836,7 +839,7 @@ fn zoo_decodes_and_never_panics_when_cut_or_corrupted() {
}
}
let mut r = Rng::new(f.len() as u64);
for _ in 0..300 {
for _ in 0..300usize.div_ceil(step) {
let mut g = f.clone();
for _ in 0..4 {
let i = (r.next_u64() as usize) % g.len();
+15
View File
@@ -0,0 +1,15 @@
[package]
name="craft-java-class"
description="Java class-file inspection and JVM bytecode decoding with retained bounds and partial reports"
version.workspace=true
edition.workspace=true
license.workspace=true
rust-version.workspace=true
repository.workspace=true
[lints]
workspace=true
[dependencies]
serde={workspace=true}
thiserror={workspace=true}
[dev-dependencies]
craft-testkit = { workspace = true }
+51
View File
@@ -0,0 +1,51 @@
//! Checked little/big-endian reads over untrusted bytes. Every read returns `None` when out of range.
#[derive(Debug, Clone, Copy)]
pub struct Reader<'a> {
data: &'a [u8],
}
impl<'a> Reader<'a> {
pub fn new(data: &'a [u8]) -> Self {
Reader { data }
}
pub fn slice(&self, off: usize, len: usize) -> Option<&'a [u8]> {
let end = off.checked_add(len)?;
self.data.get(off..end)
}
pub fn u8(&self, off: usize) -> Option<u8> {
self.data.get(off).copied()
}
fn arr<const N: usize>(&self, off: usize) -> Option<[u8; N]> {
self.slice(off, N)?.try_into().ok()
}
pub fn u16_be(&self, off: usize) -> Option<u16> {
self.arr(off).map(u16::from_be_bytes)
}
pub fn u32_be(&self, off: usize) -> Option<u32> {
self.arr(off).map(u32::from_be_bytes)
}
pub fn u64_be(&self, off: usize) -> Option<u64> {
self.arr(off).map(u64::from_be_bytes)
}
}
/// Decode one modified UTF-8 code unit. Invalid bytes consume only themselves,
/// so a damaged sequence cannot hide valid text or borrow missing bytes.
pub(crate) fn mutf8_unit(bytes: &[u8]) -> (u16, usize) {
match bytes {
[a @ 0x00..=0x7f, ..] => (u16::from(*a), 1),
[0xc0, 0x80, ..] => (0, 2),
[a @ 0xc2..=0xdf, b @ 0x80..=0xbf, ..] => ((u16::from(*a & 0x1f) << 6) | u16::from(*b & 0x3f), 2),
[a @ 0xe0..=0xef, b @ 0x80..=0xbf, c @ 0x80..=0xbf, ..] if *a != 0xe0 || *b >= 0xa0 => {
((u16::from(*a & 0x0f) << 12) | (u16::from(*b & 0x3f) << 6) | u16::from(*c & 0x3f), 3)
}
_ => (0xfffd, 1),
}
}
+19
View File
@@ -0,0 +1,19 @@
// Descriptor display helper retained from the source DEX reader.
pub fn java_name(desc: &str) -> String {
let dims = desc.chars().take_while(|&c| c == '[').count();
let base = desc.get(dims..).unwrap_or("");
let name = match base {
"V" => "void".to_string(),
"Z" => "boolean".to_string(),
"B" => "byte".to_string(),
"S" => "short".to_string(),
"C" => "char".to_string(),
"I" => "int".to_string(),
"J" => "long".to_string(),
"F" => "float".to_string(),
"D" => "double".to_string(),
b if b.starts_with('L') && b.ends_with(';') => b.get(1..b.len() - 1).unwrap_or("").replace('/', "."),
b => b.to_string(),
};
format!("{name}{}", "[]".repeat(dims.min(255)))
}
File diff suppressed because it is too large. Load diff
+711
View File
@@ -0,0 +1,711 @@
//! JVM bytecode: the opcode table (every defined opcode plus the `wide`, `tableswitch` and
//! `lookupswitch` variable-length forms) and a decoder for each instruction. Pure decoding;
//! constant-pool index operands are resolved by the caller (see [`crate::java_class`]).
use serde::Serialize;
/// The shape of an opcode's operands.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
pub enum Operands {
/// No operands.
None,
/// One signed byte immediate (`bipush`).
Byte,
/// One signed short immediate (`sipush`).
Short,
/// A one-byte local-variable index.
Local,
/// `iinc`: a one-byte local index and a signed byte increment.
LocalConst,
/// A two-byte signed branch offset.
Branch2,
/// A four-byte signed branch offset (`goto_w`, `jsr_w`).
Branch4,
/// A one-byte constant-pool index (`ldc`).
Cp1,
/// A two-byte constant-pool index.
Cp2,
/// `newarray`: a one-byte primitive array type.
NewArray,
/// `invokeinterface`: a two-byte index, a count byte and a zero byte.
InvokeInterface,
/// `invokedynamic`: a two-byte index and two zero bytes.
InvokeDynamic,
/// `multianewarray`: a two-byte class index and a dimension byte.
MultiANewArray,
/// `tableswitch` (variable length, 4-byte aligned).
TableSwitch,
/// `lookupswitch` (variable length, 4-byte aligned).
LookupSwitch,
/// `wide`: a prefix that widens the following load/store/ret/iinc.
Wide,
/// A reserved or unknown opcode.
Reserved,
}
/// One opcode slot.
#[derive(Debug, Clone, Copy)]
pub struct Op {
pub name: &'static str,
pub operands: Operands,
}
const fn op(name: &'static str, operands: Operands) -> Op {
Op { name, operands }
}
/// The opcode table entry for `b`.
pub fn opcode(b: u8) -> Op {
use Operands as O;
match b {
0x00 => op("nop", O::None),
0x01 => op("aconst_null", O::None),
0x02 => op("iconst_m1", O::None),
0x03 => op("iconst_0", O::None),
0x04 => op("iconst_1", O::None),
0x05 => op("iconst_2", O::None),
0x06 => op("iconst_3", O::None),
0x07 => op("iconst_4", O::None),
0x08 => op("iconst_5", O::None),
0x09 => op("lconst_0", O::None),
0x0a => op("lconst_1", O::None),
0x0b => op("fconst_0", O::None),
0x0c => op("fconst_1", O::None),
0x0d => op("fconst_2", O::None),
0x0e => op("dconst_0", O::None),
0x0f => op("dconst_1", O::None),
0x10 => op("bipush", O::Byte),
0x11 => op("sipush", O::Short),
0x12 => op("ldc", O::Cp1),
0x13 => op("ldc_w", O::Cp2),
0x14 => op("ldc2_w", O::Cp2),
0x15 => op("iload", O::Local),
0x16 => op("lload", O::Local),
0x17 => op("fload", O::Local),
0x18 => op("dload", O::Local),
0x19 => op("aload", O::Local),
0x1a => op("iload_0", O::None),
0x1b => op("iload_1", O::None),
0x1c => op("iload_2", O::None),
0x1d => op("iload_3", O::None),
0x1e => op("lload_0", O::None),
0x1f => op("lload_1", O::None),
0x20 => op("lload_2", O::None),
0x21 => op("lload_3", O::None),
0x22 => op("fload_0", O::None),
0x23 => op("fload_1", O::None),
0x24 => op("fload_2", O::None),
0x25 => op("fload_3", O::None),
0x26 => op("dload_0", O::None),
0x27 => op("dload_1", O::None),
0x28 => op("dload_2", O::None),
0x29 => op("dload_3", O::None),
0x2a => op("aload_0", O::None),
0x2b => op("aload_1", O::None),
0x2c => op("aload_2", O::None),
0x2d => op("aload_3", O::None),
0x2e => op("iaload", O::None),
0x2f => op("laload", O::None),
0x30 => op("faload", O::None),
0x31 => op("daload", O::None),
0x32 => op("aaload", O::None),
0x33 => op("baload", O::None),
0x34 => op("caload", O::None),
0x35 => op("saload", O::None),
0x36 => op("istore", O::Local),
0x37 => op("lstore", O::Local),
0x38 => op("fstore", O::Local),
0x39 => op("dstore", O::Local),
0x3a => op("astore", O::Local),
0x3b => op("istore_0", O::None),
0x3c => op("istore_1", O::None),
0x3d => op("istore_2", O::None),
0x3e => op("istore_3", O::None),
0x3f => op("lstore_0", O::None),
0x40 => op("lstore_1", O::None),
0x41 => op("lstore_2", O::None),
0x42 => op("lstore_3", O::None),
0x43 => op("fstore_0", O::None),
0x44 => op("fstore_1", O::None),
0x45 => op("fstore_2", O::None),
0x46 => op("fstore_3", O::None),
0x47 => op("dstore_0", O::None),
0x48 => op("dstore_1", O::None),
0x49 => op("dstore_2", O::None),
0x4a => op("dstore_3", O::None),
0x4b => op("astore_0", O::None),
0x4c => op("astore_1", O::None),
0x4d => op("astore_2", O::None),
0x4e => op("astore_3", O::None),
0x4f => op("iastore", O::None),
0x50 => op("lastore", O::None),
0x51 => op("fastore", O::None),
0x52 => op("dastore", O::None),
0x53 => op("aastore", O::None),
0x54 => op("bastore", O::None),
0x55 => op("castore", O::None),
0x56 => op("sastore", O::None),
0x57 => op("pop", O::None),
0x58 => op("pop2", O::None),
0x59 => op("dup", O::None),
0x5a => op("dup_x1", O::None),
0x5b => op("dup_x2", O::None),
0x5c => op("dup2", O::None),
0x5d => op("dup2_x1", O::None),
0x5e => op("dup2_x2", O::None),
0x5f => op("swap", O::None),
0x60 => op("iadd", O::None),
0x61 => op("ladd", O::None),
0x62 => op("fadd", O::None),
0x63 => op("dadd", O::None),
0x64 => op("isub", O::None),
0x65 => op("lsub", O::None),
0x66 => op("fsub", O::None),
0x67 => op("dsub", O::None),
0x68 => op("imul", O::None),
0x69 => op("lmul", O::None),
0x6a => op("fmul", O::None),
0x6b => op("dmul", O::None),
0x6c => op("idiv", O::None),
0x6d => op("ldiv", O::None),
0x6e => op("fdiv", O::None),
0x6f => op("ddiv", O::None),
0x70 => op("irem", O::None),
0x71 => op("lrem", O::None),
0x72 => op("frem", O::None),
0x73 => op("drem", O::None),
0x74 => op("ineg", O::None),
0x75 => op("lneg", O::None),
0x76 => op("fneg", O::None),
0x77 => op("dneg", O::None),
0x78 => op("ishl", O::None),
0x79 => op("lshl", O::None),
0x7a => op("ishr", O::None),
0x7b => op("lshr", O::None),
0x7c => op("iushr", O::None),
0x7d => op("lushr", O::None),
0x7e => op("iand", O::None),
0x7f => op("land", O::None),
0x80 => op("ior", O::None),
0x81 => op("lor", O::None),
0x82 => op("ixor", O::None),
0x83 => op("lxor", O::None),
0x84 => op("iinc", O::LocalConst),
0x85 => op("i2l", O::None),
0x86 => op("i2f", O::None),
0x87 => op("i2d", O::None),
0x88 => op("l2i", O::None),
0x89 => op("l2f", O::None),
0x8a => op("l2d", O::None),
0x8b => op("f2i", O::None),
0x8c => op("f2l", O::None),
0x8d => op("f2d", O::None),
0x8e => op("d2i", O::None),
0x8f => op("d2l", O::None),
0x90 => op("d2f", O::None),
0x91 => op("i2b", O::None),
0x92 => op("i2c", O::None),
0x93 => op("i2s", O::None),
0x94 => op("lcmp", O::None),
0x95 => op("fcmpl", O::None),
0x96 => op("fcmpg", O::None),
0x97 => op("dcmpl", O::None),
0x98 => op("dcmpg", O::None),
0x99 => op("ifeq", O::Branch2),
0x9a => op("ifne", O::Branch2),
0x9b => op("iflt", O::Branch2),
0x9c => op("ifge", O::Branch2),
0x9d => op("ifgt", O::Branch2),
0x9e => op("ifle", O::Branch2),
0x9f => op("if_icmpeq", O::Branch2),
0xa0 => op("if_icmpne", O::Branch2),
0xa1 => op("if_icmplt", O::Branch2),
0xa2 => op("if_icmpge", O::Branch2),
0xa3 => op("if_icmpgt", O::Branch2),
0xa4 => op("if_icmple", O::Branch2),
0xa5 => op("if_acmpeq", O::Branch2),
0xa6 => op("if_acmpne", O::Branch2),
0xa7 => op("goto", O::Branch2),
0xa8 => op("jsr", O::Branch2),
0xa9 => op("ret", O::Local),
0xaa => op("tableswitch", O::TableSwitch),
0xab => op("lookupswitch", O::LookupSwitch),
0xac => op("ireturn", O::None),
0xad => op("lreturn", O::None),
0xae => op("freturn", O::None),
0xaf => op("dreturn", O::None),
0xb0 => op("areturn", O::None),
0xb1 => op("return", O::None),
0xb2 => op("getstatic", O::Cp2),
0xb3 => op("putstatic", O::Cp2),
0xb4 => op("getfield", O::Cp2),
0xb5 => op("putfield", O::Cp2),
0xb6 => op("invokevirtual", O::Cp2),
0xb7 => op("invokespecial", O::Cp2),
0xb8 => op("invokestatic", O::Cp2),
0xb9 => op("invokeinterface", O::InvokeInterface),
0xba => op("invokedynamic", O::InvokeDynamic),
0xbb => op("new", O::Cp2),
0xbc => op("newarray", O::NewArray),
0xbd => op("anewarray", O::Cp2),
0xbe => op("arraylength", O::None),
0xbf => op("athrow", O::None),
0xc0 => op("checkcast", O::Cp2),
0xc1 => op("instanceof", O::Cp2),
0xc2 => op("monitorenter", O::None),
0xc3 => op("monitorexit", O::None),
0xc4 => op("wide", O::Wide),
0xc5 => op("multianewarray", O::MultiANewArray),
0xc6 => op("ifnull", O::Branch2),
0xc7 => op("ifnonnull", O::Branch2),
0xc8 => op("goto_w", O::Branch4),
0xc9 => op("jsr_w", O::Branch4),
0xca => op("breakpoint", O::None),
0xfe => op("impdep1", O::None),
0xff => op("impdep2", O::None),
_ => op("unknown", O::Reserved),
}
}
/// The primitive array type for a `newarray` atype operand.
pub fn array_type(atype: i32) -> &'static str {
match atype {
4 => "boolean",
5 => "char",
6 => "float",
7 => "double",
8 => "byte",
9 => "short",
10 => "int",
11 => "long",
_ => "?",
}
}
/// A decoded `tableswitch`/`lookupswitch`.
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct Switch {
/// Absolute byte offset of the default branch.
pub default: i64,
/// `(key, absolute target)` pairs in listing order.
pub pairs: Vec<(i32, i64)>,
}
/// One decoded instruction.
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct Insn {
/// Byte offset from the start of the method's code.
pub pc: u32,
pub opcode: u8,
pub name: &'static str,
/// Total length in bytes, including operands.
pub len: u32,
/// Local-variable index operand (load/store/ret/iinc).
#[serde(skip_serializing_if = "Option::is_none")]
pub local: Option<u16>,
/// Integer immediate (`bipush`/`sipush`/`iinc` increment/`newarray` type/`multianewarray` dims).
#[serde(skip_serializing_if = "Option::is_none")]
pub int: Option<i32>,
/// Constant-pool index operand (resolved by the caller).
#[serde(skip_serializing_if = "Option::is_none")]
pub cp_index: Option<u16>,
/// Absolute byte offset of a branch target.
#[serde(skip_serializing_if = "Option::is_none")]
pub branch: Option<i64>,
/// A switch table when the opcode is `tableswitch`/`lookupswitch`.
#[serde(skip_serializing_if = "Option::is_none")]
pub switch: Option<Switch>,
/// True when this instruction was prefixed by `wide`.
#[serde(skip_serializing_if = "std::ops::Not::not")]
pub wide: bool,
}
/// Most entries decoded from one switch table.
const MAX_SWITCH: usize = 1 << 20;
fn s8(code: &[u8], at: usize) -> Option<i32> {
code.get(at).map(|&b| i32::from(b as i8))
}
fn u16b(code: &[u8], at: usize) -> Option<u16> {
Some(u16::from(*code.get(at)?) << 8 | u16::from(*code.get(at + 1)?))
}
fn s16(code: &[u8], at: usize) -> Option<i32> {
u16b(code, at).map(|v| i32::from(v as i16))
}
fn s32(code: &[u8], at: usize) -> Option<i32> {
Some(i32::from(*code.get(at)?) << 24 | i32::from(*code.get(at + 1)?) << 16 | i32::from(*code.get(at + 2)?) << 8 | i32::from(*code.get(at + 3)?))
}
fn base(pc: usize, opcode: u8, name: &'static str) -> Insn {
Insn { pc: pc as u32, opcode, name, len: 1, local: None, int: None, cp_index: None, branch: None, switch: None, wide: false }
}
/// Decode one instruction at `pc`, or `None` for invalid or incomplete operands.
pub fn decode_one(code: &[u8], pc: usize) -> Option<Insn> {
let b = *code.get(pc)?;
let o = opcode(b);
let mut insn = base(pc, b, o.name);
let rel = |d: i32| pc as i64 + i64::from(d);
match o.operands {
Operands::None | Operands::Reserved => {}
Operands::Byte => {
insn.int = Some(s8(code, pc + 1)?);
insn.len = 2;
}
Operands::Short => {
insn.int = Some(s16(code, pc + 1)?);
insn.len = 3;
}
Operands::Local => {
insn.local = Some(u16::from(*code.get(pc + 1)?));
insn.len = 2;
}
Operands::LocalConst => {
insn.local = Some(u16::from(*code.get(pc + 1)?));
insn.int = Some(s8(code, pc + 2)?);
insn.len = 3;
}
Operands::Branch2 => {
insn.branch = Some(rel(s16(code, pc + 1)?));
insn.len = 3;
}
Operands::Branch4 => {
insn.branch = Some(rel(s32(code, pc + 1)?));
insn.len = 5;
}
Operands::Cp1 => {
insn.cp_index = Some(u16::from(*code.get(pc + 1)?));
insn.len = 2;
}
Operands::Cp2 => {
insn.cp_index = Some(u16b(code, pc + 1)?);
insn.len = 3;
}
Operands::NewArray => {
let atype = *code.get(pc + 1)?;
if !(4..=11).contains(&atype) {
return None;
}
insn.int = Some(i32::from(atype));
insn.len = 2;
}
Operands::InvokeInterface => {
insn.cp_index = Some(u16b(code, pc + 1)?);
if *code.get(pc + 3)? == 0 || *code.get(pc + 4)? != 0 {
return None;
}
insn.len = 5;
}
Operands::InvokeDynamic => {
insn.cp_index = Some(u16b(code, pc + 1)?);
if *code.get(pc + 3)? != 0 || *code.get(pc + 4)? != 0 {
return None;
}
insn.len = 5;
}
Operands::MultiANewArray => {
insn.cp_index = Some(u16b(code, pc + 1)?);
let dimensions = *code.get(pc + 3)?;
if dimensions == 0 {
return None;
}
insn.int = Some(i32::from(dimensions));
insn.len = 4;
}
Operands::Wide => return decode_wide(code, pc),
Operands::TableSwitch => return decode_tableswitch(code, pc),
Operands::LookupSwitch => return decode_lookupswitch(code, pc),
}
if pc.checked_add(insn.len as usize)? > code.len() {
return None;
}
Some(insn)
}
fn decode_wide(code: &[u8], pc: usize) -> Option<Insn> {
let real = *code.get(pc + 1)?;
if !matches!(real, 0x15..=0x19 | 0x36..=0x3a | 0x84 | 0xa9) {
return None;
}
let o = opcode(real);
let mut insn = base(pc, real, o.name);
insn.wide = true;
if real == 0x84 {
// wide iinc: 2-byte local, 2-byte signed const
insn.local = Some(u16b(code, pc + 2)?);
insn.int = Some(s16(code, pc + 4)?);
insn.len = 6;
} else {
// wide load/store/ret: 2-byte local
insn.local = Some(u16b(code, pc + 2)?);
insn.len = 4;
}
if pc.checked_add(insn.len as usize)? > code.len() {
return None;
}
Some(insn)
}
/// Padding so the operand word after a switch opcode is 4-byte aligned from the code start.
fn switch_pad(pc: usize) -> usize {
(4 - ((pc + 1) % 4)) % 4
}
fn decode_tableswitch(code: &[u8], pc: usize) -> Option<Insn> {
let mut at = pc + 1 + switch_pad(pc);
let default = rel_i64(pc, s32(code, at)?);
let low = s32(code, at + 4)?;
let high = s32(code, at + 8)?;
at += 12;
if high < low {
return None;
}
let count = usize::try_from(i64::from(high) - i64::from(low) + 1).ok()?;
if count > MAX_SWITCH {
return None;
}
let end = at.checked_add(count.checked_mul(4)?)?;
code.get(at..end)?;
let mut pairs = Vec::with_capacity(count);
for i in 0..count {
let off = s32(code, at + i * 4)?;
pairs.push((low.wrapping_add(i as i32), rel_i64(pc, off)));
}
let mut insn = base(pc, 0xaa, "tableswitch");
insn.switch = Some(Switch { default, pairs });
insn.len = (end - pc) as u32;
Some(insn)
}
fn decode_lookupswitch(code: &[u8], pc: usize) -> Option<Insn> {
let mut at = pc + 1 + switch_pad(pc);
let default = rel_i64(pc, s32(code, at)?);
let npairs = s32(code, at + 4)?;
at += 8;
if npairs < 0 {
return None;
}
let count = npairs as usize;
if count > MAX_SWITCH {
return None;
}
let end = at.checked_add(count.checked_mul(8)?)?;
code.get(at..end)?;
let mut pairs = Vec::with_capacity(count);
let mut previous = None;
for i in 0..count {
let key = s32(code, at + i * 8)?;
if previous.is_some_and(|previous| key <= previous) {
return None;
}
previous = Some(key);
let off = s32(code, at + i * 8 + 4)?;
pairs.push((key, rel_i64(pc, off)));
}
let mut insn = base(pc, 0xab, "lookupswitch");
insn.switch = Some(Switch { default, pairs });
insn.len = (end - pc) as u32;
Some(insn)
}
fn rel_i64(pc: usize, off: i32) -> i64 {
pc as i64 + i64::from(off)
}
/// Decode a whole method body, stopping at `max` instructions. `complete` is false when a
/// instruction has invalid/incomplete operands or the cap was hit.
pub fn decode(code: &[u8], max: usize) -> (Vec<Insn>, bool) {
let mut out = Vec::new();
let mut pc = 0usize;
let mut complete = true;
while pc < code.len() {
if out.len() >= max {
complete = false;
break;
}
match decode_one(code, pc) {
Some(i) => {
pc += (i.len as usize).max(1);
out.push(i);
}
None => {
complete = false;
break;
}
}
}
(out, complete)
}
#[cfg(test)]
mod tests {
use super::*;
fn lookup(pc: usize, keys: &[i32]) -> Vec<u8> {
let mut code = vec![0; pc];
code.push(0xab);
while !code.len().is_multiple_of(4) {
code.push(0);
}
code.extend_from_slice(&0i32.to_be_bytes());
code.extend_from_slice(&(keys.len() as i32).to_be_bytes());
for key in keys {
code.extend_from_slice(&key.to_be_bytes());
code.extend_from_slice(&0i32.to_be_bytes());
}
code
}
#[test]
fn lookup_keys_must_increase_strictly() {
for keys in [&[1, 1][..], &[1, 0], &[0, -1], &[i32::MIN, i32::MAX, i32::MIN]] {
for pc in 0..4 {
let mut code = lookup(pc, keys);
code.push(0xb1);
assert!(decode_one(&code, pc).is_none(), "pc {pc}, keys {keys:?}");
let (decoded, complete) = decode(&code, 100);
assert!(!complete);
assert_eq!(decoded.len(), pc, "preserve preceding nops, stop before the bad switch");
}
}
}
#[test]
fn lookup_keys_use_signed_order_at_every_alignment() {
for keys in [&[][..], &[i32::MIN, -1, 0, i32::MAX]] {
for pc in 0..4 {
let code = lookup(pc, keys);
let insn = decode_one(&code, pc).unwrap();
assert_eq!(insn.len as usize, code.len() - pc);
let switch = insn.switch.unwrap();
assert_eq!(switch.default, pc as i64);
assert_eq!(switch.pairs, keys.iter().map(|&key| (key, pc as i64)).collect::<Vec<_>>());
assert!(decode(&code, 100).1);
}
}
}
#[test]
fn switch_payloads_must_be_complete() {
for pc in 0..4 {
let lookup = lookup(pc, &[-1, 1]);
let mut dense = vec![0; pc];
dense.push(0xaa);
while !dense.len().is_multiple_of(4) {
dense.push(0);
}
for word in [0i32, i32::MAX - 1, i32::MAX, 0, 0] {
dense.extend_from_slice(&word.to_be_bytes());
}
for code in [lookup, dense] {
assert!(decode_one(&code, pc).is_some());
for end in pc + 1..code.len() {
assert!(decode_one(&code[..end], pc).is_none(), "pc {pc}, end {end}");
}
}
}
}
#[test]
fn switch_counts_cannot_reserve_an_absent_table() {
let mut lookup = lookup(0, &[]);
lookup[8..12].copy_from_slice(&(MAX_SWITCH as i32).to_be_bytes());
assert!(decode_one(&lookup, 0).is_none());
for (low, high) in [(0i32, MAX_SWITCH as i32 - 1), (i32::MIN, i32::MAX)] {
let mut dense = vec![0xaa, 0, 0, 0];
for word in [0, low, high] {
dense.extend_from_slice(&word.to_be_bytes());
}
assert!(decode_one(&dense, 0).is_none());
}
}
#[test]
fn table_covers_defined_opcodes() {
let mut named = 0;
for b in 0..=255u8 {
if opcode(b).name != "unknown" {
named += 1;
}
}
// 0x00..=0xca is 203 opcodes, plus impdep1/impdep2.
assert_eq!(named, 203 + 2);
assert_eq!(opcode(0xb6).name, "invokevirtual");
assert_eq!(opcode(0x10).name, "bipush");
assert_eq!(opcode(0xc4).name, "wide");
}
#[test]
fn decodes_core_formats() {
// iconst_1; bipush 7; istore_1; goto +3 ; (pad to reach len) return
let code = [0x04, 0x10, 0x07, 0x3c, 0xa7, 0x00, 0x03, 0xb1];
let (ins, complete) = decode(&code, 100);
assert!(complete, "{ins:?}");
assert_eq!(ins[0].name, "iconst_1");
assert_eq!(ins[1].name, "bipush");
assert_eq!(ins[1].int, Some(7));
assert_eq!(ins[2].name, "istore_1");
assert_eq!(ins[3].name, "goto");
assert_eq!(ins[3].branch, Some(7)); // pc 4 + 3
assert_eq!(ins[4].name, "return");
}
#[test]
fn decodes_invoke_and_wide() {
// invokevirtual #5 ; invokeinterface #6 count 2 ; wide iload 300
let code = [0xb6, 0x00, 0x05, 0xb9, 0x00, 0x06, 0x02, 0x00, 0xc4, 0x15, 0x01, 0x2c];
let (ins, complete) = decode(&code, 100);
assert!(complete);
assert_eq!(ins[0].name, "invokevirtual");
assert_eq!(ins[0].cp_index, Some(5));
assert_eq!(ins[0].len, 3);
assert_eq!(ins[1].name, "invokeinterface");
assert_eq!(ins[1].cp_index, Some(6));
assert_eq!(ins[1].len, 5);
assert_eq!(ins[2].name, "iload");
assert!(ins[2].wide);
assert_eq!(ins[2].local, Some(300));
assert_eq!(ins[2].len, 4);
}
#[test]
fn decodes_switches() {
// tableswitch at pc 0: pad 3, default +40, low 1, high 2, offsets +10, +20
let mut code = vec![0xaa, 0, 0, 0];
code.extend_from_slice(&40i32.to_be_bytes());
code.extend_from_slice(&1i32.to_be_bytes());
code.extend_from_slice(&2i32.to_be_bytes());
code.extend_from_slice(&10i32.to_be_bytes());
code.extend_from_slice(&20i32.to_be_bytes());
let i = decode_one(&code, 0).unwrap();
assert_eq!(i.name, "tableswitch");
let sw = i.switch.unwrap();
assert_eq!(sw.default, 40);
assert_eq!(sw.pairs, vec![(1, 10), (2, 20)]);
assert_eq!(i.len as usize, code.len());
// lookupswitch at pc 0: pad 3, default +4, npairs 1, (key 99 -> +8)
let mut lk = vec![0xab, 0, 0, 0];
lk.extend_from_slice(&4i32.to_be_bytes());
lk.extend_from_slice(&1i32.to_be_bytes());
lk.extend_from_slice(&99i32.to_be_bytes());
lk.extend_from_slice(&8i32.to_be_bytes());
let l = decode_one(&lk, 0).unwrap();
let sw = l.switch.unwrap();
assert_eq!(sw.default, 4);
assert_eq!(sw.pairs, vec![(99, 8)]);
}
#[test]
fn hostile() {
for seed in 0..256 {
let n = crate::test_support::noise(seed, 512);
let _ = decode(&n, 10_000);
}
// truncated operands never panic
let _ = decode(&[0xb6, 0x00], 10);
let _ = decode(&[0xaa, 0, 0, 0, 0, 0, 0, 0], 10);
let _ = decode(&[0xab, 0, 0, 0, 0, 0, 0, 0, 0x7f, 0xff, 0xff, 0xff], 10);
let _ = decode(&[0xc4], 10);
}
}
+50
View File
@@ -0,0 +1,50 @@
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
#![forbid(unsafe_code)]
//! Java class-file inspection and JVM bytecode decoding. Never executes bytecode.
//! Retains source item, attribute and annotation-depth limits and partial reports.
mod bytes;
mod descriptor;
pub mod java_class;
pub mod java_ops;
pub use java_class::*;
#[cfg(test)]
mod test_support;
/// File admission errors; damaged optional structures retain partial-report warnings.
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
pub enum JavaClassError {
#[error("the file ends early ({0})")]
Truncated(String),
#[error("the file is malformed ({0})")]
Malformed(String),
#[error("not supported: {0}")]
Unsupported(String),
}
use JavaClassError as FormatError;
#[cfg(test)]
#[path = "regressions/java_array_operands.rs"]
mod java_array_operands;
#[cfg(test)]
#[path = "regressions/java_attribute_names.rs"]
mod java_attribute_names;
#[cfg(test)]
#[path = "regressions/java_constant_pool_slots.rs"]
mod java_constant_pool_slots;
#[cfg(test)]
#[path = "regressions/java_invoke_operands.rs"]
mod java_invoke_operands;
#[cfg(test)]
#[path = "regressions/java_linkage_references.rs"]
mod java_linkage_references;
#[cfg(test)]
#[path = "regressions/java_truncated_attributes.rs"]
mod java_truncated_attributes;
#[cfg(test)]
#[path = "regressions/java_wide_opcodes.rs"]
mod java_wide_opcodes;
@@ -0,0 +1,48 @@
use crate::java_ops::{decode, decode_one};
#[test]
fn invalid_primitive_array_types_stop_before_the_allocation() {
for atype in (0..=u8::MAX).filter(|value| !(4..=11).contains(value)) {
assert!(decode_one(&[0xbc, atype], 0).is_none(), "atype {atype}");
let (instructions, complete) = decode(&[0, 0xbc, atype, 0xb1], 100);
assert!(!complete, "atype {atype}");
assert_eq!(instructions.len(), 1);
assert_eq!(instructions[0].name, "nop");
}
}
#[test]
fn zero_dimension_multidimensional_allocations_are_not_complete_bytecode() {
assert!(decode_one(&[0xc5, 0, 1, 0], 0).is_none());
let (instructions, complete) = decode(&[0, 0xc5, 0, 1, 0, 0xb1], 100);
assert!(!complete);
assert_eq!(instructions.len(), 1);
assert_eq!(instructions[0].name, "nop");
}
#[test]
fn every_defined_primitive_type_and_nonzero_dimension_remains_decodable() {
for atype in 4..=11 {
let (instructions, complete) = decode(&[0xbc, atype, 0xb1], 100);
assert!(complete);
assert_eq!(instructions.len(), 2);
assert_eq!(instructions[0].int, Some(i32::from(atype)));
assert_eq!(instructions[0].len, 2);
}
for dimensions in 1..=u8::MAX {
let instruction = decode_one(&[0, 0xc5, 0x12, 0x34, dimensions], 1).unwrap();
assert_eq!(instruction.pc, 1);
assert_eq!(instruction.cp_index, Some(0x1234));
assert_eq!(instruction.int, Some(i32::from(dimensions)));
assert_eq!(instruction.len, 4);
}
}
#[test]
fn incomplete_array_operands_remain_rejected() {
for instruction in [&[0xbc, 10][..], &[0xc5, 0, 1, 2][..]] {
for end in 1..instruction.len() {
assert!(decode_one(&instruction[..end], 0).is_none());
}
}
}
@@ -0,0 +1,132 @@
use crate::java_class::{self, ClassFile, Const};
#[derive(Clone, Copy, Debug)]
enum Scope {
Class,
Field,
Method,
Code,
}
fn u16b(out: &mut Vec<u8>, n: u16) {
out.extend_from_slice(&n.to_be_bytes());
}
fn u32b(out: &mut Vec<u8>, n: u32) {
out.extend_from_slice(&n.to_be_bytes());
}
fn utf8(out: &mut Vec<u8>, text: &str) {
out.push(1);
u16b(out, text.len() as u16);
out.extend_from_slice(text.as_bytes());
}
fn attr(name: u16, info: &[u8]) -> Vec<u8> {
let mut out = Vec::new();
u16b(&mut out, name);
u32b(&mut out, info.len() as u32);
out.extend_from_slice(info);
out
}
fn attributes(out: &mut Vec<u8>, repeated: u16, known: &[Vec<u8>]) {
u16b(out, repeated + known.len() as u16);
for _ in 0..repeated {
out.extend_from_slice(&attr(5, &[]));
}
for bytes in known {
out.extend_from_slice(bytes);
}
}
// A valid shared CONSTANT_Utf8 is named by every unknown attribute. Known
// attributes follow it, so avoiding name copies must not discard their facts.
fn fixture(scope: Scope, repeated: u16) -> Vec<u8> {
let mut out = vec![0xca, 0xfe, 0xba, 0xbe, 0, 0, 0, 52];
u16b(&mut out, 17);
utf8(&mut out, "AttributeFixture"); // 1
out.push(7);
u16b(&mut out, 1); // 2: this class
utf8(&mut out, "java/lang/Object"); // 3
out.push(7);
u16b(&mut out, 3); // 4: superclass
utf8(&mut out, &format!("org.example.{}", "A".repeat(usize::from(u16::MAX) - 12))); // 5
for text in ["SourceFile", "AttributeFixture.java", "field", "I"] {
utf8(&mut out, text); // 6..9
}
out.push(3);
u32b(&mut out, 42); // 10
for text in ["ConstantValue", "run", "()V", "Code", "LineNumberTable", "Deprecated"] {
utf8(&mut out, text); // 11..16
}
for word in [0x0021, 2, 4, 0, 1, 0x0019, 8, 9] {
u16b(&mut out, word);
}
attributes(&mut out, if matches!(scope, Scope::Field) { repeated } else { 0 }, &[attr(11, &10u16.to_be_bytes()), attr(16, &[])]);
for word in [1, 0x0009, 12, 13] {
u16b(&mut out, word);
}
let mut code = vec![0, 0, 0, 0, 0, 0, 0, 1, 0xb1, 0, 0];
attributes(&mut code, if matches!(scope, Scope::Code) { repeated } else { 0 }, &[attr(15, &[0, 1, 0, 0, 0, 7])]);
attributes(&mut out, if matches!(scope, Scope::Method) { repeated } else { 0 }, &[attr(14, &code)]);
attributes(&mut out, if matches!(scope, Scope::Class) { repeated } else { 0 }, &[attr(6, &7u16.to_be_bytes())]);
out
}
fn check_facts(parsed: &ClassFile) {
assert!(parsed.warnings.is_empty(), "{:?}", parsed.warnings);
assert_eq!(parsed.this_class, "AttributeFixture");
assert_eq!(parsed.super_class.as_deref(), Some("java/lang/Object"));
assert_eq!(parsed.source_file.as_deref(), Some("AttributeFixture.java"));
assert_eq!(parsed.fields.len(), 1);
assert_eq!(parsed.fields[0].name, "field");
assert_eq!(parsed.fields[0].constant_value.as_deref(), Some("42"));
assert!(parsed.fields[0].deprecated);
assert_eq!(parsed.methods.len(), 1);
assert_eq!(parsed.methods[0].name, "run");
let code = parsed.methods[0].code.as_ref().unwrap();
assert_eq!(code.bytecode, [0xb1]);
assert_eq!(code.line_numbers, [(0, 7)]);
assert!(matches!(&parsed.pool[5], Const::Utf8(name) if name.len() == usize::from(u16::MAX)));
}
#[test]
fn known_attributes_survive_repeated_unknown_names_at_every_scope() {
for scope in [Scope::Class, Scope::Field, Scope::Method, Scope::Code] {
check_facts(&java_class::parse(&fixture(scope, 64)).unwrap());
}
}
fn memory_case(scope: Scope) {
let bytes = fixture(scope, 4096);
assert!(bytes.len() < 100_000);
check_facts(&java_class::parse(&bytes).unwrap());
}
// Run each ignored case separately with a 128 MiB process address-space limit.
// The original parser copies 256 MiB of one attribute name; the fixture itself
// is under 100 KiB. These checks deliberately make no timing assertion.
#[test]
#[ignore = "run separately under a process memory limit"]
fn repeated_class_attribute_names_fit_in_bounded_memory() {
memory_case(Scope::Class);
}
#[test]
#[ignore = "run separately under a process memory limit"]
fn repeated_field_attribute_names_fit_in_bounded_memory() {
memory_case(Scope::Field);
}
#[test]
#[ignore = "run separately under a process memory limit"]
fn repeated_method_attribute_names_fit_in_bounded_memory() {
memory_case(Scope::Method);
}
#[test]
#[ignore = "run separately under a process memory limit"]
fn repeated_code_attribute_names_fit_in_bounded_memory() {
memory_case(Scope::Code);
}
@@ -0,0 +1,53 @@
use crate::{FormatError, java_class};
fn class_with_pool(count: u16, pool: &[u8]) -> Vec<u8> {
let mut bytes = vec![0xca, 0xfe, 0xba, 0xbe, 0, 0, 0, 52];
bytes.extend_from_slice(&count.to_be_bytes());
bytes.extend_from_slice(pool);
bytes.extend_from_slice(&[0; 14]);
bytes
}
#[test]
fn a_wide_constant_needs_its_reserved_slot() {
for tag in [5, 6] {
let mut pool = vec![tag];
pool.extend_from_slice(&[0; 8]);
let bytes = class_with_pool(2, &pool);
assert!(matches!(java_class::parse(&bytes), Err(FormatError::Malformed(message)) if message.contains("second slot")));
}
}
#[test]
fn the_largest_pool_cannot_overflow_on_a_trailing_wide_constant() {
for tag in [5, 6] {
let mut pool = [3, 0, 0, 0, 0].repeat(usize::from(u16::MAX) - 2);
pool.push(tag);
pool.extend_from_slice(&[0; 8]);
let bytes = class_with_pool(u16::MAX, &pool);
assert!(matches!(java_class::parse(&bytes), Err(FormatError::Malformed(message)) if message.contains("second slot")));
}
}
#[test]
fn a_wide_constant_with_two_remaining_slots_is_retained() {
for tag in [5, 6] {
for count in [3, u16::MAX] {
let mut pool = [3, 0, 0, 0, 0].repeat(usize::from(count) - 3);
pool.push(tag);
pool.extend_from_slice(&[0; 8]);
let parsed = java_class::parse(&class_with_pool(count, &pool)).unwrap();
assert_eq!(parsed.pool.len(), usize::from(count));
assert!(matches!(parsed.pool.last(), Some(java_class::Const::Unusable)));
assert!(matches!(parsed.pool[usize::from(count) - 2], java_class::Const::Long(_) | java_class::Const::Double(_)));
}
}
}
#[test]
fn ordinary_final_constants_and_existing_classes_still_parse() {
let parsed = java_class::parse(&class_with_pool(2, &[3, 0, 0, 0, 7])).unwrap();
assert_eq!(parsed.pool, vec![java_class::Const::Unusable, java_class::Const::Integer(7)]);
let parsed = java_class::parse(&crate::test_support::java::hello_class()).unwrap();
assert!(!parsed.methods.is_empty());
}
@@ -0,0 +1,76 @@
use crate::java_class;
use crate::java_ops::{decode, decode_one};
#[test]
fn invokeinterface_requires_a_count_and_zero_reserved_byte() {
for code in [[0xb9, 0, 1, 0, 0], [0xb9, 0, 1, 1, 1], [0xb9, 0, 1, 255, 255]] {
assert!(decode_one(&code, 0).is_none(), "{code:?}");
assert!(!decode(&code, 16).1);
}
}
#[test]
fn invokedynamic_requires_both_reserved_bytes_to_be_zero() {
for code in [[0xba, 0, 1, 1, 0], [0xba, 0, 1, 0, 1], [0xba, 0, 1, 255, 255]] {
assert!(decode_one(&code, 0).is_none(), "{code:?}");
assert!(!decode(&code, 16).1);
}
}
#[test]
fn valid_and_truncated_invocations_keep_their_boundaries() {
for count in [1, 2, 255] {
let instruction = decode_one(&[0xb9, 0x12, 0x34, count, 0], 0).unwrap();
assert_eq!(instruction.cp_index, Some(0x1234));
assert_eq!(instruction.len, 5);
}
let instruction = decode_one(&[0xba, 0x12, 0x34, 0, 0], 0).unwrap();
assert_eq!(instruction.cp_index, Some(0x1234));
for code in [[0xb9, 0, 1, 1, 0], [0xba, 0, 1, 0, 0]] {
for len in 1..5 {
assert!(decode_one(&code[..len], 0).is_none());
}
}
let (instructions, complete) = decode(&[0xb9, 0, 1, 1, 0, 0xba, 0, 2, 0, 0, 0xb1], 16);
assert!(complete);
assert_eq!(instructions.len(), 3);
assert_eq!(instructions[2].pc, 10);
}
fn reference_class() -> java_class::ClassFile {
let mut class = java_class::parse(&crate::test_support::java::hello_class()).unwrap();
class.methods.retain(|method| method.name == "run");
class
}
#[test]
fn malformed_invocations_mark_reference_scans_incomplete() {
let valid = reference_class();
let (expected, incomplete) = java_class::references(&valid, "java/io/PrintStream", None, 10);
assert!(!incomplete);
assert_eq!(expected.len(), 1);
for invalid in [[0xb9, 0, 1, 0, 0], [0xb9, 0, 1, 1, 1], [0xba, 0, 1, 1, 0], [0xba, 0, 1, 0, 1]] {
let mut class = valid.clone();
let code = &mut class.methods[0].code.as_mut().unwrap().bytecode;
let later_reference = code.clone();
*code = invalid.to_vec();
code.extend_from_slice(&later_reference);
let (found, incomplete) = java_class::references(&class, "java/io/PrintStream", None, 10);
assert!(found.is_empty());
assert!(incomplete, "invalid invocation must not hide an unfinished reference scan: {invalid:?}");
}
}
#[test]
fn incomplete_scans_keep_earlier_references_and_scan_other_methods() {
let mut class = reference_class();
let mut later_method = class.methods[0].clone();
later_method.name = "later".into();
class.methods[0].code.as_mut().unwrap().bytecode.extend_from_slice(&[0xba, 0, 1, 1, 0]);
class.methods.push(later_method);
let (found, incomplete) = java_class::references(&class, "java/io/PrintStream", None, 10);
assert!(incomplete);
assert_eq!(found.len(), 2);
assert!(found[0].caller.contains(".run:("));
assert!(found[1].caller.contains(".later:("));
}
@@ -0,0 +1,97 @@
use crate::java_class::{parse, references};
/// A method with a chosen load or call, plus a dynamic call site and a dynamic constant with distinct bootstrap signatures.
fn class(code: &[u8]) -> Vec<u8> {
fn utf(s: &str) -> Vec<u8> {
let mut b = vec![1];
b.extend((s.len() as u16).to_be_bytes());
b.extend(s.bytes());
b
}
let pool = [
utf("Uses"),
vec![7, 0, 1],
utf("java/lang/Object"),
vec![7, 0, 3],
utf("run"),
utf("()V"),
utf("Code"),
utf("BootstrapMethods"),
utf("Target"),
vec![7, 0, 9],
utf("apply"),
vec![12, 0, 11, 0, 6],
vec![10, 0, 10, 0, 12],
vec![15, 6, 0, 13],
utf("Factory"),
vec![7, 0, 15],
utf("boot"),
vec![12, 0, 17, 0, 28],
vec![10, 0, 16, 0, 18],
vec![15, 6, 0, 19],
utf("([LTarget;)V"),
vec![16, 0, 21],
vec![18, 0, 0, 0, 12],
utf("value"),
utf("I"),
vec![12, 0, 24, 0, 25],
vec![17, 0, 1, 0, 26],
utf(
"(Ljava/lang/invoke/MethodHandles$Lookup;Ljava/lang/String;Ljava/lang/invoke/MethodType;Ljava/lang/invoke/MethodHandle;)Ljava/lang/invoke/CallSite;",
),
utf("(Ljava/lang/invoke/MethodHandles$Lookup;Ljava/lang/String;Ljava/lang/Class;)I"),
vec![12, 0, 17, 0, 29],
vec![10, 0, 16, 0, 30],
vec![15, 6, 0, 31],
];
let mut bytes = vec![0xca, 0xfe, 0xba, 0xbe, 0, 0, 0, 55];
bytes.extend((pool.len() as u16 + 1).to_be_bytes());
for e in pool {
bytes.extend(e);
}
bytes.extend([0, 0x21, 0, 2, 0, 4, 0, 0, 0, 0, 0, 1, 0, 9, 0, 5, 0, 6, 0, 1, 0, 7]);
bytes.extend(((12 + code.len()) as u32).to_be_bytes());
bytes.extend([0, 3, 0, 0]);
bytes.extend((code.len() as u32).to_be_bytes());
bytes.extend(code);
bytes.extend([0, 0, 0, 0]);
let boots = [0, 2, 0, 20, 0, 1, 0, 14, 0, 32, 0, 0];
bytes.extend([0, 1, 0, 8]);
bytes.extend((boots.len() as u32).to_be_bytes());
bytes.extend(boots);
bytes
}
fn kinds(code: &[u8], owner: &str, method: Option<&str>) -> Vec<&'static str> {
let cf = parse(&class(code)).unwrap();
let (hits, partial) = references(&cf, owner, method, 10);
assert!(!partial);
hits.into_iter().map(|r| r.kind).collect()
}
#[test]
fn class_literal_has_a_use_site() {
assert_eq!(kinds(&[0x12, 10, 0x57, 0xb1], "Target", None), ["type"]);
}
#[test]
fn literal_method_handle_finds_its_member() {
assert_eq!(kinds(&[0x12, 14, 0x57, 0xb1], "Target", Some("apply")), ["method-handle"]);
}
#[test]
fn method_type_mentions_object_array_components() {
assert_eq!(kinds(&[0x12, 22, 0x57, 0xb1], "Target", None), ["method-type"]);
}
#[test]
fn dynamic_call_exposes_factory_dependency() {
assert_eq!(kinds(&[0xba, 0, 23, 0, 0, 0xb1], "Factory", Some("boot")), ["bootstrap-method"]);
}
#[test]
fn dynamic_call_exposes_static_method_handle_argument() {
assert_eq!(kinds(&[0xba, 0, 23, 0, 0, 0xb1], "Target", Some("apply")), ["method-handle"]);
}
#[test]
fn dynamic_constant_exposes_bootstrap_dependency() {
assert_eq!(kinds(&[0x12, 27, 0x57, 0xb1], "Factory", Some("boot")), ["bootstrap-method"]);
}
#[test]
fn unused_bootstrap_does_not_introduce_a_reference() {
assert!(kinds(&[0xb1], "Factory", None).is_empty());
}
@@ -0,0 +1,46 @@
use crate::java_class::parse;
use crate::test_support::java::hello_class;
/// The fixture ends in a class attribute count and a single two-byte SourceFile attribute.
const ATTRIBUTE_TABLE_BYTES: usize = 10;
/// Every cut of the attribute count, header or body leaves a partial class report.
#[test]
fn incomplete_class_attribute_table_reports_a_warning() {
let data = hello_class();
let start = data.len() - ATTRIBUTE_TABLE_BYTES;
for end in start..data.len() {
let class = parse(&data[..end]).unwrap();
assert_eq!(class.methods.len(), 4);
assert!(class.source_file.is_none());
assert!(class.warnings.iter().any(|warning| warning.contains("class attribute table")), "cut at {}", end - start);
}
}
/// A complete attribute stays available when a subsequent attribute is cut short.
#[test]
fn completed_class_attributes_survive_a_truncated_record() {
let mut data = hello_class();
let start = data.len() - ATTRIBUTE_TABLE_BYTES;
data[start..start + 2].copy_from_slice(&2u16.to_be_bytes());
data.extend_from_slice(&[0, 1, 0, 0, 0, 2, 0]);
let class = parse(&data).unwrap();
assert_eq!(class.source_file.as_deref(), Some("Hello.java"));
assert!(class.warnings.iter().any(|warning| warning.contains("class attribute table")));
}
/// Complete, empty and unknown class attributes do not trigger a truncation warning.
#[test]
fn complete_class_attribute_tables_are_accepted() {
let mut data = hello_class();
let start = data.len() - ATTRIBUTE_TABLE_BYTES;
let original = parse(&data).unwrap();
assert_eq!(original.source_file.as_deref(), Some("Hello.java"));
assert!(original.warnings.is_empty());
data[start..start + 2].copy_from_slice(&2u16.to_be_bytes());
data.extend_from_slice(&[0, 1, 0, 0, 0, 1, 0]);
assert!(parse(&data).unwrap().warnings.is_empty());
data.truncate(start + 2);
data[start..].fill(0);
assert!(parse(&data).unwrap().warnings.is_empty());
}
@@ -0,0 +1,33 @@
use crate::java_ops::{decode, decode_one};
#[test]
fn only_defined_wide_opcodes_are_accepted() {
for op in 0..=255u8 {
let valid = matches!(op, 0x15..=0x19 | 0x36..=0x3a | 0x84 | 0xa9);
let code = [0xc4, op, 0x01, 0x2c, 0xff, 0xfe];
assert_eq!(decode_one(&code, 0).is_some(), valid, "opcode {op:#x}");
}
}
#[test]
fn invalid_wide_does_not_invent_a_method_body() {
let (instructions, complete) = decode(&[0x00, 0xc4, 0xb1, 0, 1, 0xb1], 32);
assert!(!complete);
assert_eq!(instructions.len(), 1);
assert_eq!(instructions[0].name, "nop");
}
#[test]
fn wide_locals_and_signed_increments_are_preserved() {
for op in [0x15, 0x16, 0x17, 0x18, 0x19, 0x36, 0x37, 0x38, 0x39, 0x3a, 0xa9] {
let instruction = decode_one(&[0xc4, op, 0xff, 0xff], 0).unwrap();
assert_eq!(instruction.local, Some(u16::MAX));
assert_eq!(instruction.len, 4);
assert!(instruction.wide);
}
let instruction = decode_one(&[0xc4, 0x84, 0x01, 0x2c, 0xff, 0xfe], 0).unwrap();
assert_eq!(instruction.local, Some(300));
assert_eq!(instruction.int, Some(-2));
assert_eq!(instruction.len, 6);
assert!(decode_one(&[0xc4, 0x84, 0x00, 1], 0).is_none());
}
+276
View File
@@ -0,0 +1,276 @@
pub use craft_testkit::sample_damage::{mutations, noise};
pub mod java {
use std::collections::HashMap;
/// A constant-pool builder that interns entries and tracks the double slot after `Long`/`Double`.
#[derive(Default)]
struct Pool {
items: Vec<Vec<u8>>,
/// Next free index (1-based; accounts for long/double taking two slots).
count: u16,
cache: HashMap<String, u16>,
}
impl Pool {
fn new() -> Self {
Pool { items: Vec::new(), count: 1, cache: HashMap::new() }
}
fn add(&mut self, bytes: Vec<u8>, slots: u16) -> u16 {
let idx = self.count;
self.items.push(bytes);
self.count += slots;
idx
}
fn utf8(&mut self, s: &str) -> u16 {
let key = format!("u:{s}");
if let Some(&i) = self.cache.get(&key) {
return i;
}
let mut b = vec![1u8];
b.extend_from_slice(&(s.len() as u16).to_be_bytes());
b.extend_from_slice(s.as_bytes());
let i = self.add(b, 1);
self.cache.insert(key, i);
i
}
fn class(&mut self, name: &str) -> u16 {
let n = self.utf8(name);
let key = format!("c:{name}");
if let Some(&i) = self.cache.get(&key) {
return i;
}
let mut b = vec![7u8];
b.extend_from_slice(&n.to_be_bytes());
let i = self.add(b, 1);
self.cache.insert(key, i);
i
}
fn string(&mut self, s: &str) -> u16 {
let u = self.utf8(s);
let mut b = vec![8u8];
b.extend_from_slice(&u.to_be_bytes());
self.add(b, 1)
}
fn nat(&mut self, name: &str, desc: &str) -> u16 {
let n = self.utf8(name);
let d = self.utf8(desc);
let mut b = vec![12u8];
b.extend_from_slice(&n.to_be_bytes());
b.extend_from_slice(&d.to_be_bytes());
self.add(b, 1)
}
fn reff(&mut self, tag: u8, owner: &str, name: &str, desc: &str) -> u16 {
let c = self.class(owner);
let nt = self.nat(name, desc);
let mut b = vec![tag];
b.extend_from_slice(&c.to_be_bytes());
b.extend_from_slice(&nt.to_be_bytes());
self.add(b, 1)
}
fn long(&mut self, v: i64) -> u16 {
let mut b = vec![5u8];
b.extend_from_slice(&v.to_be_bytes());
self.add(b, 2)
}
/// The serialized constant pool and the count field.
fn finish(&self) -> (Vec<u8>, u16) {
(self.items.concat(), self.count)
}
}
fn u16b(v: &mut Vec<u8>, x: u16) {
v.extend_from_slice(&x.to_be_bytes());
}
fn u32b(v: &mut Vec<u8>, x: u32) {
v.extend_from_slice(&x.to_be_bytes());
}
/// Build an attribute: name_index, length, body.
fn attr(name_idx: u16, body: Vec<u8>) -> Vec<u8> {
let mut v = Vec::new();
u16b(&mut v, name_idx);
u32b(&mut v, body.len() as u32);
v.extend_from_slice(&body);
v
}
/// A method's `Code` attribute body (max_stack, max_locals, code, exception table, no nested attrs).
fn code_body(max_stack: u16, max_locals: u16, code: &[u8], exc: &[(u16, u16, u16, u16)]) -> Vec<u8> {
let mut v = Vec::new();
u16b(&mut v, max_stack);
u16b(&mut v, max_locals);
u32b(&mut v, code.len() as u32);
v.extend_from_slice(code);
u16b(&mut v, exc.len() as u16);
for (s, e, h, t) in exc {
u16b(&mut v, *s);
u16b(&mut v, *e);
u16b(&mut v, *h);
u16b(&mut v, *t);
}
u16b(&mut v, 0); // nested attributes count
v
}
/// A method_info: access, name, descriptor, and one already-built Code attribute.
fn method(p: &mut Pool, access: u16, name: &str, desc: &str, code_attribute: Vec<u8>) -> Vec<u8> {
let name_idx = p.utf8(name);
let desc_idx = p.utf8(desc);
let mut v = Vec::new();
u16b(&mut v, access);
u16b(&mut v, name_idx);
u16b(&mut v, desc_idx);
u16b(&mut v, 1); // one attribute
v.extend_from_slice(&code_attribute);
v
}
/// Assemble a whole class from its parts.
#[allow(clippy::too_many_arguments)]
fn assemble(
pool: &Pool,
access: u16,
this_c: u16,
super_c: u16,
interfaces: &[u16],
fields: &[Vec<u8>],
methods: &[Vec<u8>],
class_attrs: &[Vec<u8>],
) -> Vec<u8> {
let (cp, count) = pool.finish();
let mut v = Vec::new();
v.extend_from_slice(&[0xca, 0xfe, 0xba, 0xbe]);
u16b(&mut v, 0); // minor
u16b(&mut v, 52); // major (Java 8)
u16b(&mut v, count);
v.extend_from_slice(&cp);
u16b(&mut v, access);
u16b(&mut v, this_c);
u16b(&mut v, super_c);
u16b(&mut v, interfaces.len() as u16);
for i in interfaces {
u16b(&mut v, *i);
}
u16b(&mut v, fields.len() as u16);
for f in fields {
v.extend_from_slice(f);
}
u16b(&mut v, methods.len() as u16);
for m in methods {
v.extend_from_slice(m);
}
u16b(&mut v, class_attrs.len() as u16);
for a in class_attrs {
v.extend_from_slice(a);
}
v
}
pub const HELLO: &str = "Hello, Craft!";
/// `public class com.example.Hello implements Runnable` with a `count` field, a `BIG` long
/// constant, `<init>`, `run()` (System.out.println("Hello, Craft!")), `pick(int)` (a tableswitch)
/// and `guarded()` (a try/catch over the println call).
pub fn hello_class() -> Vec<u8> {
let mut p = Pool::new();
let this_c = p.class("com/example/Hello");
let obj_c = p.class("java/lang/Object");
let runnable_c = p.class("java/lang/Runnable");
let code_attr = p.utf8("Code");
// fields: int count; static final long BIG;
let count_name = p.utf8("count");
let i_desc = p.utf8("I");
let big_name = p.utf8("BIG");
let j_desc = p.utf8("J");
let constval_attr = p.utf8("ConstantValue");
let big_val = p.long(0x1122_3344_5566_7788);
// refs used by code
let obj_init = p.reff(10, "java/lang/Object", "<init>", "()V");
let sysout = p.reff(9, "java/lang/System", "out", "Ljava/io/PrintStream;");
let hello = p.string(HELLO);
let println = p.reff(10, "java/io/PrintStream", "println", "(Ljava/lang/String;)V");
let rt_exc = p.class("java/lang/RuntimeException");
// <init>(): aload_0; invokespecial Object.<init>; return
let mut init = vec![0x2a, 0xb7];
init.extend_from_slice(&obj_init.to_be_bytes());
init.push(0xb1);
let init_m = method(&mut p, 0x0001, "<init>", "()V", attr(code_attr, code_body(1, 1, &init, &[])));
// run(): getstatic System.out; ldc "Hello"; invokevirtual println; return
let mut run = vec![0xb2];
run.extend_from_slice(&sysout.to_be_bytes());
run.push(0x12);
run.push(hello as u8); // ldc (1-byte index; pool is small)
run.push(0xb6);
run.extend_from_slice(&println.to_be_bytes());
run.push(0xb1);
let run_m = method(&mut p, 0x0001, "run", "()V", attr(code_attr, code_body(2, 1, &run, &[])));
// pick(int): iload_0; tableswitch{0->L0,1->L1,default->Ldef}; L0: bipush 10; ireturn; L1: bipush 20; ireturn; Ldef: iconst_m1; ireturn
// offsets: iload_1@0; tableswitch@1 (pad 2) ends @24; L0@24; L1@27; Ldef@30
let mut pick = vec![0x1b, 0xaa, 0x00, 0x00];
u32b(&mut pick, 29); // default -> 1+29 = 30
u32b(&mut pick, 0); // low
u32b(&mut pick, 1); // high
u32b(&mut pick, 23); // case 0 -> 1+23 = 24
u32b(&mut pick, 26); // case 1 -> 1+26 = 27
debug_assert_eq!(pick.len(), 24);
pick.extend_from_slice(&[0x10, 10, 0xac]); // @24: bipush 10; ireturn
pick.extend_from_slice(&[0x10, 20, 0xac]); // @27: bipush 20; ireturn
pick.extend_from_slice(&[0x02, 0xac]); // @30: iconst_m1; ireturn
let pick_m = method(&mut p, 0x0001, "pick", "(I)I", attr(code_attr, code_body(1, 1, &pick, &[])));
// guarded(): try { System.out.println("Hello"); } catch (RuntimeException e) {} (then return)
// 0: getstatic(3) ; 3: ldc(2) ; 5: invokevirtual(3) ; 8: goto +5 ->13 ; 11: astore_1 ; 12: return ; 13: return
let mut g = vec![0xb2];
g.extend_from_slice(&sysout.to_be_bytes());
g.push(0x12);
g.push(hello as u8);
g.push(0xb6);
g.extend_from_slice(&println.to_be_bytes());
g.extend_from_slice(&[0xa7, 0x00, 0x05]); // @8 goto -> 13
g.push(0x4c); // @11 astore_1
g.push(0xb1); // @12 return
g.push(0xb1); // @13 return
let guarded_m = method(&mut p, 0x0001, "guarded", "()V", attr(code_attr, code_body(2, 2, &g, &[(0, 8, 11, rt_exc)])));
// field infos
let field = |p: &mut Pool, access: u16, name_idx: u16, desc_idx: u16, attrs: &[Vec<u8>]| -> Vec<u8> {
let _ = p;
let mut v = Vec::new();
u16b(&mut v, access);
u16b(&mut v, name_idx);
u16b(&mut v, desc_idx);
u16b(&mut v, attrs.len() as u16);
for a in attrs {
v.extend_from_slice(a);
}
v
};
let count_f = field(&mut p, 0x0002, count_name, i_desc, &[]); // private int count
let mut cv = Vec::new();
u16b(&mut cv, big_val);
let big_f = field(&mut p, 0x0019, big_name, j_desc, &[attr(constval_attr, cv)]); // public static final long BIG
let source_attr = p.utf8("SourceFile");
let source_val = p.utf8("Hello.java");
let mut sf = Vec::new();
u16b(&mut sf, source_val);
let source = attr(source_attr, sf);
assemble(
&p,
0x0021, // public super
this_c,
obj_c,
&[runnable_c],
&[count_f, big_f],
&[init_m, run_m, pick_m, guarded_m],
&[source],
)
}
}
+105 -22
View File
@@ -419,6 +419,34 @@ pub fn encode_baseline(
sub: ChromaSubsampling,
app_segments: &[(u8, Vec<u8>)],
) -> Result<Vec<u8>, EncodeError> {
validate_encode(data, width, height, channels, app_segments)?;
Ok(encode(data, width, height, channels, quality, sub, app_segments))
}
/// Encode the same optimized baseline JPEG as [`encode_baseline`], trading a second DCT pass
/// for working memory independent of image height. This path is sequential even when the
/// `parallel` feature is enabled. It retains only one MCU row of conversion planes and writes
/// directly into the returned encoded buffer: no image-wide tokens or duplicate coded buffer.
///
/// Scratch consists of at most `24 * width + 8` floats (odd-width 4:2:0), plus fixed-size coding tables.
/// The caller still owns the input pixels and the returned JPEG; this is not a cap on output
/// size or a replacement for the caller's image/admission budget. Geometry, quality, metadata,
/// Huffman tables and restart boundaries match [`encode_baseline`] byte for byte.
#[allow(clippy::too_many_arguments)]
pub fn encode_baseline_low_memory(
data: &[u8],
width: usize,
height: usize,
channels: usize,
quality: u8,
sub: ChromaSubsampling,
app_segments: &[(u8, Vec<u8>)],
) -> Result<Vec<u8>, EncodeError> {
validate_encode(data, width, height, channels, app_segments)?;
Ok(encode_impl(data, width, height, channels, quality, sub, app_segments, true, true))
}
fn validate_encode(data: &[u8], width: usize, height: usize, channels: usize, app_segments: &[(u8, Vec<u8>)]) -> Result<(), EncodeError> {
let fits = width.checked_mul(height).and_then(|n| n.checked_mul(channels)).is_some_and(|n| n <= data.len());
if !fits || width == 0 || height == 0 || width > 65_535 || height > 65_535 || !matches!(channels, 1 | 3 | 4) {
return Err(EncodeError::InvalidInput(format!("{width}x{height}, {channels} channels, {} bytes", data.len())));
@@ -426,7 +454,7 @@ pub fn encode_baseline(
if app_segments.iter().any(|(marker, payload)| !(0xE0..=0xEF).contains(marker) || payload.len() > 65_533) {
return Err(EncodeError::InvalidInput("APP markers must be 0xE0..=0xEF with payloads no larger than 65533 bytes".into()));
}
Ok(encode(data, width, height, channels, quality, sub, app_segments))
Ok(())
}
/// [`encode`], with optimized Huffman tables or (`optimize` false) the Annex K example tables.
@@ -440,6 +468,21 @@ fn encode_with(
sub: ChromaSubsampling,
app_segments: &[(u8, Vec<u8>)],
optimize: bool,
) -> Vec<u8> {
encode_impl(data, width, height, channels, quality, sub, app_segments, optimize, false)
}
#[allow(clippy::too_many_arguments)]
fn encode_impl(
data: &[u8],
width: usize,
height: usize,
channels: usize,
quality: u8,
sub: ChromaSubsampling,
app_segments: &[(u8, Vec<u8>)],
optimize: bool,
low_memory: bool,
) -> Vec<u8> {
// The public entry takes caller-sized dimensions: refuse (an empty output) what would size an allocation from
// them or index past `data`, rather than panic or allocate for them.
@@ -509,6 +552,8 @@ fn encode_with(
planes.push(Plane { w: cw, h: crows.len(), data: cb });
planes.push(Plane { w: cw, h: crows.len(), data: cr });
}
#[cfg(test)]
low_memory_tests::record_planes(planes.iter().map(|p| p.data.len()).sum());
planes
};
@@ -519,45 +564,58 @@ fn encode_with(
let (mx, my) = (width.div_ceil(mcu_w), height.div_ceil(mcu_h));
// restart interval: whole MCU rows per band, ≤ 65535 MCUs, enough bands to keep all cores busy
let rows_per_band = (my / 64).clamp(1, (65_535 / mx.max(1)).max(1));
let bands: Vec<usize> = (0..my.div_ceil(rows_per_band)).collect();
let num_bands = my.div_ceil(rows_per_band);
let bands: Vec<usize> = if low_memory { Vec::new() } else { (0..num_bands).collect() };
// first pass: each band's quantized blocks as symbols, with their counts per table (DC
// prediction restarts per band, as the restart markers say)
let tokenize_band = |band: &usize| -> (Vec<Token>, [Freq; 4]) {
let visit_band = |band: &usize, emit: &mut dyn FnMut(&[i16; 64], &mut i32, bool)| {
let (mut blk, mut q) = ([0f32; 64], [0i16; 64]);
let (mut tokens, mut freq) = (Vec::new(), [[0u64; 256]; 4]);
let mut pred = [0i32; 3];
let (m0, m1) = (band * rows_per_band, ((band + 1) * rows_per_band).min(my));
// the band's planes, its MCU row `m0` at their top (blocks past the image's bottom repeat its last row,
// which is the band's last row then)
let planes = convert(m0 * mcu_h..(m1 * mcu_h).min(height), m0 * 8..(m1 * 8).min(ch));
for mcu_y in m0..m1 {
let mcu_y = mcu_y - m0;
// Convert only the current MCU row. DC predictors still span the original band,
// so its symbols/restart decisions are identical to whole-band conversion.
let planes = convert(mcu_y * mcu_h..((mcu_y + 1) * mcu_h).min(height), mcu_y * 8..((mcu_y + 1) * 8).min(ch));
for mcu_x in 0..mx {
for vy in 0..vs {
for hx in 0..hs {
planes[0].block(mcu_x * hs + hx, mcu_y * vs + vy, &mut blk);
planes[0].block(mcu_x * hs + hx, vy, &mut blk);
fdct_quant(&blk, &rl, &mut q);
tokenize_block(&q, &mut pred[0], false, &mut tokens, &mut freq);
emit(&q, &mut pred[0], false);
}
}
if !gray {
for (plane, pred) in planes.iter().zip(pred.iter_mut()).take(3).skip(1) {
plane.block(mcu_x, mcu_y, &mut blk);
plane.block(mcu_x, 0, &mut blk);
fdct_quant(&blk, &rc, &mut q);
tokenize_block(&q, pred, true, &mut tokens, &mut freq);
emit(&q, pred, true);
}
}
}
}
};
let tokenize_band = |band: &usize| -> (Vec<Token>, [Freq; 4]) {
let (mut tokens, mut freq) = (Vec::new(), [[0u64; 256]; 4]);
visit_band(band, &mut |q, pred, chroma| tokenize_block(q, pred, chroma, &mut tokens, &mut freq));
#[cfg(test)]
low_memory_tests::record_tokens(tokens.len());
(tokens, freq)
};
#[cfg(feature = "parallel")]
let tokenized: Vec<(Vec<Token>, [Freq; 4])> = bands.par_iter().map(tokenize_band).collect();
let tokenized: Vec<(Vec<Token>, [Freq; 4])> = if low_memory { Vec::new() } else { bands.par_iter().map(tokenize_band).collect() };
#[cfg(not(feature = "parallel"))]
let tokenized: Vec<(Vec<Token>, [Freq; 4])> = bands.iter().map(tokenize_band).collect();
let tokenized: Vec<(Vec<Token>, [Freq; 4])> = if low_memory { Vec::new() } else { bands.iter().map(tokenize_band).collect() };
// the tables: DC luma, AC luma, DC chroma, AC chroma
let tables: [([u8; 16], Vec<u8>); 4] = if optimize {
let mut total = [[0u64; 256]; 4];
if low_memory {
for band in 0..num_bands {
visit_band(&band, &mut |q, pred, chroma| {
let base = if chroma { 2 } else { 0 };
block_symbols(q, pred, |ac, symbol, _, _| total[base + ac as usize][symbol & 0xFF] += 1);
});
}
}
for (_, f) in &tokenized {
for (t, f) in total.iter_mut().zip(f) {
for (t, f) in t.iter_mut().zip(f) {
@@ -577,9 +635,9 @@ fn encode_with(
let huff = tables.each_ref().map(|(bits, vals)| Huff::new(bits, vals));
// second pass: code the symbols
#[cfg(feature = "parallel")]
let coded: Vec<Vec<u8>> = tokenized.into_par_iter().map(|(t, _)| write_tokens(&t, &huff)).collect();
let coded: Vec<Vec<u8>> = if low_memory { Vec::new() } else { tokenized.into_par_iter().map(|(t, _)| write_tokens(&t, &huff)).collect() };
#[cfg(not(feature = "parallel"))]
let coded: Vec<Vec<u8>> = tokenized.into_iter().map(|(t, _)| write_tokens(&t, &huff)).collect();
let coded: Vec<Vec<u8>> = if low_memory { Vec::new() } else { tokenized.into_iter().map(|(t, _)| write_tokens(&t, &huff)).collect() };
// headers
let mut out = Vec::with_capacity(coded.iter().map(Vec::len).sum::<usize>() + 4096);
@@ -617,7 +675,7 @@ fn encode_with(
dht(&mut out, 0x11, acb, acv);
}
let interval = (rows_per_band * mx) as u16;
if coded.len() > 1 {
if num_bands > 1 {
segment(&mut out, 0xDD, &interval.to_be_bytes());
}
if gray {
@@ -625,11 +683,32 @@ fn encode_with(
} else {
segment(&mut out, 0xDA, &[3, 1, 0x00, 2, 0x11, 3, 0x11, 0, 63, 0]);
}
let last = coded.len().saturating_sub(1);
for (i, c) in coded.iter().enumerate() {
out.extend_from_slice(c);
if i < last {
out.extend_from_slice(&[0xFF, 0xD0 + (i % 8) as u8]);
if low_memory {
let last = num_bands.saturating_sub(1);
for i in 0..num_bands {
// Continue directly in the final buffer, resetting entropy bits at the same band
// boundaries as the retained-token path. Only the output grows with image height.
let mut bits = Bits { out, acc: 0, n: 0 };
visit_band(&i, &mut |q, pred, chroma| {
let base = if chroma { 2 } else { 0 };
block_symbols(q, pred, |ac, symbol, extra, n| {
let (code, len) = huff[base + ac as usize].code[symbol & 0xFF];
bits.put(code as u32, len as u32);
bits.put(extra & 0xFFFF, n & 0xF);
});
});
out = bits.finish();
if i < last {
out.extend_from_slice(&[0xFF, 0xD0 + (i % 8) as u8]);
}
}
} else {
let last = coded.len().saturating_sub(1);
for (i, c) in coded.iter().enumerate() {
out.extend_from_slice(c);
if i < last {
out.extend_from_slice(&[0xFF, 0xD0 + (i % 8) as u8]);
}
}
}
out.extend_from_slice(&[0xFF, 0xD9]);
@@ -877,3 +956,7 @@ mod tests {
assert!(psnr(&src, &px) > 35.0);
}
}
#[cfg(test)]
#[path = "encode_low_memory_tests.rs"]
mod low_memory_tests;
+155
View File
@@ -0,0 +1,155 @@
use super::*;
use std::cell::Cell;
thread_local! {
static STATS: Cell<(usize, usize)> = const { Cell::new((0, 0)) };
}
pub(super) fn record_planes(samples: usize) {
STATS.with(|s| {
let (peak, tokens) = s.get();
s.set((peak.max(samples), tokens));
});
}
pub(super) fn record_tokens(count: usize) {
STATS.with(|s| {
let (peak, tokens) = s.get();
s.set((peak, tokens + count));
});
}
fn noise(len: usize) -> Vec<u8> {
let mut seed = 0x89ab_cdefu32;
(0..len)
.map(|_| {
seed ^= seed << 13;
seed ^= seed >> 17;
seed ^= seed << 5;
seed as u8
})
.collect()
}
fn compare(src: &[u8], w: usize, h: usize, ch: usize, q: u8, sub: ChromaSubsampling, app: &[(u8, Vec<u8>)]) {
let expected = encode_baseline(src, w, h, ch, q, sub, app).unwrap();
STATS.with(|s| s.set((0, 0)));
let actual = encode_baseline_low_memory(src, w, h, ch, q, sub, app).unwrap();
assert_eq!(actual, expected, "{w}x{h} channels={ch} quality={q} subsampling={sub:?}");
let (peak, tokens) = STATS.with(Cell::get);
assert!(peak <= 24 * w + 8, "MCU-row scratch exceeded: {peak}");
assert_eq!(tokens, 0, "low-memory mode retained entropy tokens");
}
#[test]
fn all_sampling_and_channels_match_retained_tokens_byte_for_byte() {
for (w, h) in [(1, 1), (7, 9), (17, 31), (33, 65), (17, 1025)] {
for ch in [1, 3, 4] {
let src = noise(w * h * ch);
for sub in [ChromaSubsampling::S444, ChromaSubsampling::S422, ChromaSubsampling::S420] {
for q in [0, 1, 25, 50, 90, 100, 255] {
compare(&src, w, h, ch, q, sub, &[]);
}
}
}
}
}
#[test]
fn high_entropy_hd_and_wide_frames_keep_only_one_mcu_row() {
for (w, h) in [(1920, 1080), (65_535, 1)] {
let src = noise(w * h * 3);
compare(&src, w, h, 3, 100, ChromaSubsampling::S444, &[]);
}
}
#[test]
fn metadata_empty_tables_and_extra_source_bytes_keep_exact_output() {
let app = [(0xe0, b"custom density".to_vec()), (0xe2, vec![73; 65_533])];
for ch in [1, 3, 4] {
let src = vec![127; 19 * 23 * ch + 5];
compare(&src, 19, 23, ch, 75, ChromaSubsampling::S420, &app);
}
}
#[test]
fn checked_admission_matches_original_without_panics() {
for (w, h, ch) in [(0, 1, 3), (1, 0, 3), (1, 1, 0), (1, 1, 2), (usize::MAX, 2, 4), (65_536, 1, 1), (1, 65_536, 1), (4, 4, 3)] {
let src = vec![0; if w == 65_536 || h == 65_536 { 65_536 } else { 8 }];
let old = encode_baseline(&src, w, h, ch, 90, ChromaSubsampling::S444, &[]);
let new = encode_baseline_low_memory(&src, w, h, ch, 90, ChromaSubsampling::S444, &[]);
assert!(new.is_err());
assert_eq!(new, old);
}
for app in [vec![(0xdf, vec![])], vec![(0xf0, vec![])], vec![(0xe1, vec![0; 65_534])]] {
assert_eq!(
encode_baseline_low_memory(&[0; 3], 1, 1, 3, 90, ChromaSubsampling::S444, &app),
encode_baseline(&[0; 3], 1, 1, 3, 90, ChromaSubsampling::S444, &app)
);
}
}
// Generated with the immutable pre-change encoder at craft-libs
// 937ac93c5cf3cd4b6a81e195b078fe161eb081b9, not the refactored token-retaining path.
// Odd chroma edges and 1025/2049-row inputs exercise multi-row restart bands.
#[test]
fn prechange_encoded_bytes_remain_pinned_across_band_boundaries() {
let cases = [
(1, 1, 1, 1, 444, 159, 0xe3bd59145381ba57),
(1, 1, 1, 75, 422, 159, 0xcb2b103914bbdbd3),
(1, 1, 1, 100, 420, 160, 0xd9f640613efb6abf),
(1, 1, 3, 1, 444, 284, 0x61d2772b8fd9df17),
(1, 1, 3, 75, 422, 286, 0xa0e03ab7efa23296),
(1, 1, 3, 100, 420, 288, 0x5660667f1abd0a63),
(1, 1, 4, 1, 444, 284, 0x61d2772b8fd9df17),
(1, 1, 4, 75, 422, 286, 0xa0e03ab7efa23296),
(1, 1, 4, 100, 420, 288, 0x5660667f1abd0a63),
(17, 31, 1, 1, 444, 240, 0xf8d277e9499253d4),
(17, 31, 1, 75, 422, 515, 0xa6946d4f144bf243),
(17, 31, 1, 100, 420, 793, 0xbb3db259708d8951),
(17, 31, 3, 1, 444, 363, 0xff2b5b41ffb3f81b),
(17, 31, 3, 75, 422, 877, 0x0374ffb527378dfe),
(17, 31, 3, 100, 420, 1220, 0x2feeedfa90885737),
(17, 31, 4, 1, 444, 366, 0xde7f8d1330e436eb),
(17, 31, 4, 75, 422, 879, 0xb1559fbfc3ee0cf6),
(17, 31, 4, 100, 420, 1214, 0xaf5dcf12d28c8e88),
(33, 65, 1, 1, 444, 388, 0xbee80491e42c8fd2),
(33, 65, 1, 75, 422, 1470, 0x284f4eb78b785ca5),
(33, 65, 1, 100, 420, 2576, 0xa81cc94015f326dc),
(33, 65, 3, 1, 444, 526, 0xf75b4036acc30874),
(33, 65, 3, 75, 422, 2322, 0x6fbc56bd9a1eec70),
(33, 65, 3, 100, 420, 3796, 0xcad25051562ac816),
(33, 65, 4, 1, 444, 527, 0x4bfedc11153de4c1),
(33, 65, 4, 75, 422, 2309, 0x47bf11bf32bac6a4),
(33, 65, 4, 100, 420, 3775, 0x64c2303fde273cd1),
(17, 1025, 1, 1, 444, 2017, 0x242f4a95d8579612),
(17, 1025, 1, 75, 422, 10728, 0xfe49408f379bdffb),
(17, 1025, 1, 100, 420, 19686, 0x1355b995e38d6467),
(17, 1025, 3, 1, 444, 2137, 0x4e319440cb6946c2),
(17, 1025, 3, 75, 422, 17682, 0xf456c63b731dde8f),
(17, 1025, 3, 100, 420, 29015, 0x0631001154b128cf),
(17, 1025, 4, 1, 444, 2155, 0xa2de9c1d77e62aa6),
(17, 1025, 4, 75, 422, 17648, 0x7d85fbdf56061f0c),
(17, 1025, 4, 100, 420, 29010, 0x4848eeb93daf1df2),
(19, 2049, 1, 1, 444, 3531, 0x9b93cf219ff85796),
(19, 2049, 1, 75, 422, 26283, 0xb160523444397608),
(19, 2049, 1, 100, 420, 51883, 0xc5c26836c0a976c6),
(19, 2049, 3, 1, 444, 3303, 0x26eaac2272e1fdb8),
(19, 2049, 3, 75, 422, 42745, 0x38b9843fb54db698),
(19, 2049, 3, 100, 420, 79868, 0xd6a7917af19682f7),
(19, 2049, 4, 1, 444, 3343, 0x6bf577903528692f),
(19, 2049, 4, 75, 422, 42706, 0x0e62366274593a93),
(19, 2049, 4, 100, 420, 79804, 0x867d05cf9d944599),
];
for (w, h, ch, q, sampling, len, hash) in cases {
let sub = match sampling {
444 => ChromaSubsampling::S444,
422 => ChromaSubsampling::S422,
_ => ChromaSubsampling::S420,
};
let src = noise(w * h * ch);
for actual in [encode_baseline(&src, w, h, ch, q, sub, &[]).unwrap(), encode_baseline_low_memory(&src, w, h, ch, q, sub, &[]).unwrap()] {
let actual_hash = actual.iter().fold(0xcbf29ce484222325u64, |h, b| (h ^ u64::from(*b)).wrapping_mul(0x100000001b3));
assert_eq!((actual.len(), actual_hash), (len, hash), "{w}x{h} channels={ch} quality={q} sampling={sampling}");
}
}
}
+1 -1
View File
@@ -1705,7 +1705,7 @@ mod tests;
/// Baseline JPEG encoder (8-bit, Huffman-coded sequential DCT).
mod encode;
pub use encode::{ChromaSubsampling, EncodeError, encode_baseline};
pub use encode::{ChromaSubsampling, EncodeError, encode_baseline, encode_baseline_low_memory};
mod app14;
pub use app14::adobe_transform;
+1 -1
View File
@@ -15,7 +15,7 @@ mod server;
pub use content::{Contents, base64, image_result, text_result};
pub use schema::{Effect, check_arguments, object, string, tool};
pub use server::{Incoming, Info, PROTOCOL_VERSION, Reference, SUPPORTED_VERSIONS, Server, Tools, read_message, response};
pub use server::{Incoming, Info, Options, PROTOCOL_VERSION, Reference, SUPPORTED_VERSIONS, Server, Tools, read_message, response};
#[cfg(test)]
mod tests;
Loaded 100 of 202 files, more files were not shown because too many files have changed in this diff. Show more