From a97b7948a9379fb614c2a5cc52594f6a00bcf8ff Mon Sep 17 00:00:00 2001 From: FredApps Date: Tue, 17 Nov 2026 17:02:07 +0100 Subject: [PATCH 01/16] ci: pilot resource-grouped Linux nextest with doctests --- .config/nextest.toml | 21 +++++- .github/workflows/workspace.yml | 17 ++++- scripts/test-workspace-tests.py | 114 ++++++++++++++++++++++++++++++++ 3 files changed, 146 insertions(+), 6 deletions(-) create mode 100644 scripts/test-workspace-tests.py diff --git a/.config/nextest.toml b/.config/nextest.toml index 1037b2310..5e02e048d 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -1,7 +1,6 @@ # cargo-nextest (https://nexte.st) runs every test in its own process, in parallel across test -# binaries. The Windows leg of the Workspace gate uses it (.github/workflows/workspace.yml); every -# other leg, and local runs, keep `cargo test`. Nextest does not run doctests, so the Windows leg -# also runs `cargo test --doc`. +# binaries. Linux and Windows in the Workspace gate use it (.github/workflows/workspace.yml). +# macOS and local runs keep `cargo test`. Both nextest legs also run `cargo test --doc`. # # cargo nextest run --workspace --profile ci # cargo test --workspace --doc @@ -22,6 +21,8 @@ video-fixtures = { max-threads = 1 } # craft-print: `PRINT_TICKET_PROVIDER` serializes the Windows PrintTicket provider, and the tests # share the one test print queue the gate installs. print = { max-threads = 1 } +# The Linux pilot serializes video integration tests sharing generated fixture directories. +linux-video-fixtures = { max-threads = 1 } [[profile.default.overrides]] filter = 'package(craft-ui) & (binary(visual_regression) | test(/^combo::tests::|selected_tab_erases_a_baseline_centered_on_its_bottom_edge/))' @@ -42,3 +43,17 @@ test-group = 'print' fail-fast = false slow-timeout = { period = "60s" } failure-output = "immediate-final" + +# Linux-only pilot: keep the established Windows profile unchanged. GPU consumers share one +# process-independent slot; the video integration binaries share fixture directories. This +# deliberately starts conservatively rather than relying on in-process mutexes across binaries. +[profile.ci-linux] +inherits = "ci" + +[[profile.ci-linux.overrides]] +filter = 'package(craft-ui) | package(craft-keymap) | package(craft-update)' +test-group = 'gpu' + +[[profile.ci-linux.overrides]] +filter = 'package(craft-video) & kind(test)' +test-group = 'linux-video-fixtures' diff --git a/.github/workflows/workspace.yml b/.github/workflows/workspace.yml index 4d717fed8..96cc9333e 100644 --- a/.github/workflows/workspace.yml +++ b/.github/workflows/workspace.yml @@ -25,7 +25,7 @@ jobs: with: os: '["ubuntu-latest","windows-latest","macos-latest"]' timeout-minutes: 120 - # The Windows leg runs its tests with nextest (.config/nextest.toml). + # Linux and Windows use nextest with explicit doctests (.config/nextest.toml). tools: cargo-nextest@0.9.150 packages: mesa-vulkan-drivers libvulkan1 craft-fonts-ref: b076417b5880bd4154bd22f00695b1d28c6bea83 @@ -49,11 +49,22 @@ jobs: printf 'CRAFT_PDF_TEST_FONT_DIR=%s\n' "$RUNNER_TEMP/craft-pdf-test-fonts" >> "$GITHUB_ENV" command: | python3 scripts/test-workspace-concurrency.py + python3 scripts/test-workspace-tests.py cargo fmt --check cargo clippy --workspace --all-targets --locked -- -D warnings cargo clippy --workspace --all-targets --all-features --locked -- -D warnings - cargo test --workspace --locked - cargo test --workspace --all-features --locked + if [ "$RUNNER_OS" = Linux ]; then + # Keep example compilation as well as test=true example tests selected by nextest. + cargo build --workspace --examples --locked + cargo nextest run --workspace --locked --profile ci-linux + cargo test --workspace --doc --locked + cargo build --workspace --examples --all-features --locked + cargo nextest run --workspace --all-features --locked --profile ci-linux + cargo test --workspace --doc --all-features --locked + else + cargo test --workspace --locked + cargo test --workspace --all-features --locked + fi windows-command: | powershell.exe -NoProfile -Command '$ErrorActionPreference = "Stop"; Start-Service Spooler; Add-PrinterDriver -Name "Generic / Text Only"' cargo fmt --check diff --git a/scripts/test-workspace-tests.py b/scripts/test-workspace-tests.py new file mode 100644 index 000000000..f6e55f968 --- /dev/null +++ b/scripts/test-workspace-tests.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +"""Exercise the native workflow's command selection without compiling Rust.""" +import os +import re +import subprocess +import tomllib +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +WORKFLOW = ROOT / ".github/workflows/workspace.yml" + + +def block(text, name): + match = re.search(r"^ " + re.escape(name) + r": \|\n((?: .*\n|\n)+)", text, re.M) + if match is None: + raise AssertionError(f"missing command block: {name}") + return "\n".join(line[8:] for line in match[1].splitlines()) + + +def commands(os_name, fail=""): + # Functions avoid executing any caller-supplied cargo command or changing the workspace. + # The real bash -euo pipefail semantics still apply, including failures inside a branch. + stub = """ +cargo() { + printf '%s\\n' "$*" + if [ -n "$FAIL" ] && [ "$*" = "$FAIL" ]; then return 17; fi +} +python3() { :; } +""" + return subprocess.run( + ["bash", "-euo", "pipefail", "-c", stub + block(WORKFLOW.read_text(), "command")], + env={**os.environ, "RUNNER_OS": os_name, "FAIL": fail}, + text=True, capture_output=True, check=False, + ) + + +COMMON = [ + "fmt --check", + "clippy --workspace --all-targets --locked -- -D warnings", + "clippy --workspace --all-targets --all-features --locked -- -D warnings", +] +LINUX = [ + "build --workspace --examples --locked", + "nextest run --workspace --locked --profile ci-linux", + "test --workspace --doc --locked", + "build --workspace --examples --all-features --locked", + "nextest run --workspace --all-features --locked --profile ci-linux", + "test --workspace --doc --all-features --locked", +] + + +class WorkspaceTests(unittest.TestCase): + def test_linux_keeps_both_feature_sets_examples_and_doctests(self): + result = commands("Linux") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.splitlines(), COMMON + LINUX) + + def test_macos_keeps_cargo_test(self): + result = commands("macOS") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.splitlines(), COMMON + [ + "test --workspace --locked", "test --workspace --all-features --locked"]) + + def test_each_linux_stage_failure_remains_fatal(self): + for index, command in enumerate(COMMON + LINUX): + with self.subTest(command=command): + result = commands("Linux", command) + self.assertEqual(result.returncode, 17, result.stderr) + self.assertEqual(result.stdout.splitlines(), (COMMON + LINUX)[:index + 1]) + + def test_windows_keeps_its_existing_profile_and_doctests(self): + source = block(WORKFLOW.read_text(), "windows-command") + cargo = [line.strip() for line in source.splitlines() if line.strip().startswith("cargo ")] + self.assertEqual(cargo, ["cargo " + command for command in COMMON + [ + "nextest run --workspace --locked --profile ci", + "test --workspace --doc --locked", + "nextest run --workspace --all-features --locked --profile ci", + "test --workspace --doc --all-features --locked", + ]]) + + def test_linux_inherits_no_retries_no_filtering_and_existing_failure_policy(self): + config = tomllib.loads((ROOT / ".config/nextest.toml").read_text()) + self.assertEqual(config["profile"]["ci-linux"]["inherits"], "ci") + self.assertIs(config["profile"]["ci"]["fail-fast"], False) + for profile in config["profile"].values(): + self.assertNotIn("default-filter", profile) + self.assertNotIn("retries", profile) + self.assertEqual(config["profile"]["ci"]["slow-timeout"], {"period": "60s"}) + + def test_linux_resource_groups_are_serial_and_do_not_change_windows(self): + config = tomllib.loads((ROOT / ".config/nextest.toml").read_text()) + overrides = config["profile"]["ci-linux"]["overrides"] + self.assertEqual(overrides, [ + {"filter": "package(craft-ui) | package(craft-keymap) | package(craft-update)", + "test-group": "gpu"}, + {"filter": "package(craft-video) & kind(test)", "test-group": "linux-video-fixtures"}, + ]) + for override in overrides: + self.assertEqual(config["test-groups"][override["test-group"]]["max-threads"], 1) + self.assertNotIn("overrides", config["profile"]["ci"]) + + def test_workflow_stays_shared_and_keeps_platform_gates(self): + source = WORKFLOW.read_text() + self.assertIn("uses: storytold/craft-common-actions/.github/workflows/gate.yml@main", source) + self.assertIn('os: \'["ubuntu-latest","windows-latest","macos-latest"]\'', source) + self.assertIn(" msrv:", source) + self.assertIn(" wasm:", source) + self.assertIn("python3 scripts/test-workspace-tests.py", block(source, "command")) + + +if __name__ == "__main__": + unittest.main() + From ff502fb75f8e91106b38fd2c867a3f6cfc824e8d Mon Sep 17 00:00:00 2001 From: aiongg Date: Tue, 17 Nov 2026 18:59:57 +0000 Subject: [PATCH 02/16] ci: run a fraction of the fuzz-style tests on pull requests, all of them nightly craft-testkit's hostile hammer reads CRAFT_HOSTILE_DIVISOR=N: the unmodified seed, every N-th truncation and 1/N of the rounds. hostile::divisor() gives the same value to sweeps written without the hammer; craft-image-codecs' TIFF zoo and malformed-header sweeps use it, with a different offset per input so the inputs together still reach every position. Proptests without their own case count read proptest's PROPTEST_CASES. The Workspace gate and the media, audio, video and PDF workflows set CRAFT_HOSTILE_DIVISOR=8 and PROPTEST_CASES=64. The longest of these tests (craft-media's MXF hostile input, 1,310 s on Windows; craft-pdf's crypt proptests, up to 257 s; the image-codecs sweeps, 140-162 s) drop to about an eighth or a quarter. nightly.yml runs the whole workspace on Linux without either variable, so every case still runs once a day. The gate's test-input setup moves to scripts/ci-test-inputs.sh, shared by both workflows; it uses a corpus that is already present. --- .github/workflows/actions.lock | 1 + .github/workflows/audio.yml | 5 ++ .github/workflows/media.yml | 5 ++ .github/workflows/nightly.yml | 31 +++++++ .github/workflows/pdf.yml | 4 + .github/workflows/video.yml | 5 ++ .github/workflows/workspace.yml | 24 ++---- crates/image-codecs/tests/it/malformed.rs | 11 ++- crates/image-codecs/tests/it/tiff_pages.rs | 11 ++- crates/media-testkit/src/hostile.rs | 10 ++- crates/media-testkit/src/lib.rs | 3 +- crates/media-testkit/tests/hostile.rs | 13 +-- crates/testkit/README.md | 10 +++ crates/testkit/src/hostile.rs | 95 +++++++++++++++++++--- scripts/ci-test-inputs.sh | 32 ++++++++ 15 files changed, 215 insertions(+), 45 deletions(-) create mode 100644 .github/workflows/nightly.yml create mode 100755 scripts/ci-test-inputs.sh diff --git a/.github/workflows/actions.lock b/.github/workflows/actions.lock index 02cdf71c8..52c8faf77 100644 --- a/.github/workflows/actions.lock +++ b/.github/workflows/actions.lock @@ -80,6 +80,7 @@ workflows: - 'actions/checkout@v7.0.1' - 'dtolnay/rust-toolchain@stable' - 'swatinem/rust-cache@v2.9.3' + '.github/workflows/nightly.yml': [] '.github/workflows/pcraft-format.yml': - 'actions/checkout@v7.0.1' - 'step-security/dtolnay-rust-toolchain@v1.0.0' diff --git a/.github/workflows/audio.yml b/.github/workflows/audio.yml index d6a7ff5e2..48c1384b2 100644 --- a/.github/workflows/audio.yml +++ b/.github/workflows/audio.yml @@ -11,6 +11,11 @@ on: permissions: contents: read +# A fraction of the fuzz-style cases, as in the Workspace gate; nightly.yml runs them in full. +env: + CRAFT_HOSTILE_DIVISOR: "8" + PROPTEST_CASES: "64" + concurrency: # Pull requests share one group per branch, so a newer push cancels the older run. Every other # run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it diff --git a/.github/workflows/media.yml b/.github/workflows/media.yml index 4814e35ca..f397f5cb3 100644 --- a/.github/workflows/media.yml +++ b/.github/workflows/media.yml @@ -11,6 +11,11 @@ on: permissions: contents: read +# A fraction of the fuzz-style cases, as in the Workspace gate; nightly.yml runs them in full. +env: + CRAFT_HOSTILE_DIVISOR: "8" + PROPTEST_CASES: "64" + concurrency: # Pull requests share one group per branch, so a newer push cancels the older run. Every other # run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 000000000..2ecc68ecb --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,31 @@ +name: Nightly full tests + +# The tests the Workspace gate shortens, in full, once a day: every hostile-input hammer and sweep +# without CRAFT_HOSTILE_DIVISOR, and every proptest at its own case count (no PROPTEST_CASES). The +# rest of the workspace's tests run with them, on Linux, with the gate's setup. +on: + schedule: + - cron: "37 2 * * *" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: nightly-${{ github.ref }} + cancel-in-progress: false + +jobs: + full: + name: Full tests + uses: storytold/craft-common-actions/.github/workflows/gate.yml@main + with: + os: '["ubuntu-latest"]' + timeout-minutes: 120 + tools: cargo-nextest@0.9.150 + packages: mesa-vulkan-drivers libvulkan1 + craft-fonts-ref: b076417b5880bd4154bd22f00695b1d28c6bea83 + linux-env: WGPU_BACKEND=vulkan + env: CRAFT_PRINT_REQUIRE_QUEUE=0 + setup-command: bash scripts/ci-test-inputs.sh + command: cargo nextest run --workspace --all-features --locked --profile ci diff --git a/.github/workflows/pdf.yml b/.github/workflows/pdf.yml index 09ddfabca..96d6f3faa 100644 --- a/.github/workflows/pdf.yml +++ b/.github/workflows/pdf.yml @@ -11,6 +11,10 @@ on: permissions: contents: read +# A fraction of the proptest cases, as in the Workspace gate; nightly.yml runs them in full. +env: + PROPTEST_CASES: "64" + concurrency: # Pull requests share one group per branch, so a newer push cancels the older run. Every other # run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it diff --git a/.github/workflows/video.yml b/.github/workflows/video.yml index fa667bdfc..97e7c0d34 100644 --- a/.github/workflows/video.yml +++ b/.github/workflows/video.yml @@ -11,6 +11,11 @@ on: permissions: contents: read +# A fraction of the fuzz-style cases, as in the Workspace gate; nightly.yml runs them in full. +env: + CRAFT_HOSTILE_DIVISOR: "8" + PROPTEST_CASES: "64" + concurrency: group: shared-video-${{ github.ref }} cancel-in-progress: true diff --git a/.github/workflows/workspace.yml b/.github/workflows/workspace.yml index 96cc9333e..e3295d251 100644 --- a/.github/workflows/workspace.yml +++ b/.github/workflows/workspace.yml @@ -31,22 +31,14 @@ jobs: craft-fonts-ref: b076417b5880bd4154bd22f00695b1d28c6bea83 linux-env: WGPU_BACKEND=vulkan windows-env: CRAFT_PRINT_REQUIRE_QUEUE=1 - env: CRAFT_PRINT_REQUIRE_QUEUE=0 - setup-command: | - set -euo pipefail - corpus_fetcher=.test-inputs/photocraft - mkdir -p "$corpus_fetcher" - git -C "$corpus_fetcher" init - git -C "$corpus_fetcher" remote add origin https://github.com/storytold/PhotoCraft.git - git -C "$corpus_fetcher" fetch --quiet --depth=1 origin 74330157cb601573ae4523a95bb9132822fc3918 - git -C "$corpus_fetcher" checkout --quiet --detach FETCH_HEAD - (cd "$corpus_fetcher" && cargo xtask corpus --psd --psd-tools --atn --exr --heif --pngsuite) - mkdir -p corpus - cp -a "$corpus_fetcher/corpus/." corpus/ - python scripts/test-fetch-pdf-test-fonts.py - python scripts/fetch-pdf-test-fonts.py "$RUNNER_TEMP/craft-pdf-test-fonts" - printf 'CRAFT_PSD_CORPUS=%s\n' "$GITHUB_WORKSPACE/$corpus_fetcher/corpus" >> "$GITHUB_ENV" - printf 'CRAFT_PDF_TEST_FONT_DIR=%s\n' "$RUNNER_TEMP/craft-pdf-test-fonts" >> "$GITHUB_ENV" + # The fuzz-style tests run a fraction of their cases here: every hostile-input hammer and + # sweep 1/8 (craft-testkit's CRAFT_HOSTILE_DIVISOR), every proptest without its own case + # count 64 cases instead of 256. nightly.yml runs them all in full. + env: | + CRAFT_PRINT_REQUIRE_QUEUE=0 + CRAFT_HOSTILE_DIVISOR=8 + PROPTEST_CASES=64 + setup-command: bash scripts/ci-test-inputs.sh command: | python3 scripts/test-workspace-concurrency.py python3 scripts/test-workspace-tests.py diff --git a/crates/image-codecs/tests/it/malformed.rs b/crates/image-codecs/tests/it/malformed.rs index ec06109e0..b247bdfc6 100644 --- a/crates/image-codecs/tests/it/malformed.rs +++ b/crates/image-codecs/tests/it/malformed.rs @@ -76,9 +76,11 @@ fn truncated_files_mostly_error() { #[test] fn bit_flips_never_panic() { + // CRAFT_HOSTILE_DIVISOR=N (pull requests) runs 1/N of the flips. + let rounds = 150usize.div_ceil(craft_media_testkit::hostile::divisor()); let mut rng = common::Rng::new(42); for (f, bytes) in samples() { - for _ in 0..150 { + for _ in 0..rounds { let mut b = bytes.clone(); let flips = 1 + (rng.next_u64() % 4) as usize; for _ in 0..flips { @@ -92,8 +94,11 @@ fn bit_flips_never_panic() { #[test] fn byte_overwrites_in_header_never_panic() { - for (f, bytes) in samples() { - for i in 0..bytes.len().min(64) { + // CRAFT_HOSTILE_DIVISOR=N (pull requests) overwrites every N-th byte, from a different offset + // for each sample, so the samples together still reach every header position. + let step = craft_media_testkit::hostile::divisor(); + for (k, (f, bytes)) in samples().into_iter().enumerate() { + for i in (0..bytes.len().min(64)).filter(|i| (i + k) % step == 0) { for v in [0x00, 0x7F, 0xFF] { let mut b = bytes.clone(); b[i] = v; diff --git a/crates/image-codecs/tests/it/tiff_pages.rs b/crates/image-codecs/tests/it/tiff_pages.rs index 7501be936..7f6708f88 100644 --- a/crates/image-codecs/tests/it/tiff_pages.rs +++ b/crates/image-codecs/tests/it/tiff_pages.rs @@ -820,15 +820,18 @@ fn poke(bytes: &[u8]) { #[test] fn zoo_decodes_and_never_panics_when_cut_or_corrupted() { - for f in zoo() { + // CRAFT_HOSTILE_DIVISOR=N (pull requests) runs every N-th cut and flip, from a different offset + // for each file, and 1/N of the random rounds. + let step = craft_media_testkit::hostile::divisor(); + for (k, f) in zoo().into_iter().enumerate() { let info = tiff_info(&f).unwrap(); for i in 0..info.pages.len() { decode_tiff_page(&f, i, &DecodeOptions::default()).unwrap(); } - for cut in 0..f.len() { + for cut in (0..f.len()).filter(|cut| (cut + k) % step == 0) { poke(&f[..cut]); } - for i in 0..f.len() { + for i in (0..f.len()).filter(|i| (i + k) % step == 0) { for flip in [0xffu8, 0x80, 0x01] { let mut g = f.clone(); g[i] ^= flip; @@ -836,7 +839,7 @@ fn zoo_decodes_and_never_panics_when_cut_or_corrupted() { } } let mut r = Rng::new(f.len() as u64); - for _ in 0..300 { + for _ in 0..300usize.div_ceil(step) { let mut g = f.clone(); for _ in 0..4 { let i = (r.next_u64() as usize) % g.len(); diff --git a/crates/media-testkit/src/hostile.rs b/crates/media-testkit/src/hostile.rs index f40021986..ddfc55f79 100644 --- a/crates/media-testkit/src/hostile.rs +++ b/crates/media-testkit/src/hostile.rs @@ -3,9 +3,9 @@ //! thread with a time limit for each case. [`Rng`], [`mutations`] and the other names here are //! `craft_testkit`'s. +pub use craft_testkit::hostile::{DIVISOR_VARIABLE, MAX_CASES, Mutation, Mutations, Rng, mutations}; #[cfg(not(target_arch = "wasm32"))] -pub use craft_testkit::hostile::{HammerConfig, HammerError}; -pub use craft_testkit::hostile::{MAX_CASES, Mutation, Mutations, Rng, mutations}; +pub use craft_testkit::hostile::{HammerConfig, HammerError, divisor}; /// The variable that multiplies [`hammer`]'s iterations. #[cfg(not(target_arch = "wasm32"))] @@ -17,8 +17,10 @@ const MULTIPLIER_VARIABLE: &str = "CRAFT_MEDIA_HOSTILE_MULT"; /// Panics on the first case that panics or does not finish, naming `name`, the case, its length and /// the panic's message. /// -/// `CRAFT_MEDIA_HOSTILE_MULT=10` runs ten times the iterations (soak runs). A value that is not a -/// whole number above zero panics, naming the variable, before `run` is called. +/// `CRAFT_MEDIA_HOSTILE_MULT=10` runs ten times the iterations (soak runs), and +/// `CRAFT_HOSTILE_DIVISOR=8` every eighth truncation and an eighth of the iterations (pull +/// requests). A value that is not a whole number above zero panics, naming the variable, before +/// `run` is called. #[cfg(not(target_arch = "wasm32"))] #[track_caller] pub fn hammer(name: &str, seed: &[u8], iters: usize, run: impl Fn(&[u8]) + Send + Sync + 'static) { diff --git a/crates/media-testkit/src/lib.rs b/crates/media-testkit/src/lib.rs index 3fa99c54f..db4c844f9 100644 --- a/crates/media-testkit/src/lib.rs +++ b/crates/media-testkit/src/lib.rs @@ -13,7 +13,8 @@ //! //! [`hostile::hammer`] runs a parser over every truncation and mutations of a valid input and fails //! on a panic or a case that does not finish in time; `CRAFT_MEDIA_HOSTILE_MULT=10` runs ten times -//! the iterations. It and [`with_deadline`] call `craft_testkit`. +//! the iterations and `CRAFT_HOSTILE_DIVISOR=8` an eighth of them. It and [`with_deadline`] call +//! `craft_testkit`. //! [`vmd`] writes Sierra VMD fixtures, which ffmpeg can read but not write. //! //! Failing loudly is what a test helper is for: a wrong `CRAFT_MEDIA_*` setting or an unwritable diff --git a/crates/media-testkit/tests/hostile.rs b/crates/media-testkit/tests/hostile.rs index 8e34cd5af..5281f4a41 100644 --- a/crates/media-testkit/tests/hostile.rs +++ b/crates/media-testkit/tests/hostile.rs @@ -5,7 +5,7 @@ use std::process::Command; use std::sync::{Arc, Mutex}; use std::time::Duration; -use craft_media_testkit::hostile::hammer; +use craft_media_testkit::hostile::{divisor, hammer}; /// Set only in the child processes of `run_child`: the value names what `child_hammer` does. const CHILD_VARIABLE: &str = "CRAFT_MEDIA_TESTKIT_HOSTILE_CHILD"; @@ -60,10 +60,11 @@ fn a_bad_multiplier_names_the_variable() { #[test] fn a_panic_keeps_its_message() { - let payload = std::panic::catch_unwind(|| hammer("demo", &[9, 8, 7, 6], 0, |d| assert!(d.len() != 2, "boom at {}", d.len()))).unwrap_err(); + // Truncation to 0 runs under any CRAFT_HOSTILE_DIVISOR. + let payload = std::panic::catch_unwind(|| hammer("demo", &[9, 8, 7, 6], 0, |d| assert!(!d.is_empty(), "boom at {}", d.len()))).unwrap_err(); let message = payload.downcast_ref::().cloned().unwrap_or_default(); - assert!(message.starts_with("demo: truncation to 2 (len 2): the parser panicked: "), "{message}"); - assert!(message.contains("boom at 2"), "{message}"); + assert!(message.starts_with("demo: truncation to 0 (len 0): the parser panicked: "), "{message}"); + assert!(message.contains("boom at 0"), "{message}"); } #[test] @@ -74,8 +75,8 @@ fn every_truncation_of_a_large_seed_runs() { hammer("large", &seed, 0, move |d| { record.lock().unwrap().insert(d.len()); }); - // The seed and its 8,192 truncations. - assert_eq!(lengths.lock().unwrap().len(), 8_193); + // The seed and its 8,192 truncations, every N-th of them under CRAFT_HOSTILE_DIVISOR=N. + assert_eq!(lengths.lock().unwrap().len(), 1 + 8_192usize.div_ceil(divisor())); } #[test] diff --git a/crates/testkit/README.md b/crates/testkit/README.md index 8b8e9b208..1f58a7af9 100644 --- a/crates/testkit/README.md +++ b/crates/testkit/README.md @@ -192,6 +192,16 @@ A product of the iterations and the multiplier that does not fit in a `usize`, a more than `MAX_CASES` (10,000,000) cases in all, and a limit variable that `deadline` would reject also fail the test before `parse` is called. +### Quick runs + +`CRAFT_HOSTILE_DIVISOR=N` runs a fraction of every sweep: the unmodified seed, every N-th truncation +(lengths 0, N, 2N, ...) and the rounds divided by N, rounded up. It takes the same values as the +multiplier. The Workspace gate sets it on every run and the nightly run leaves it unset, so every +case still runs once a day. A sweep written without `hammer` reads it with `hostile::divisor()` and +runs the cases whose index plus an offset is a multiple of it, with a different offset for each input +so that the inputs together still reach every index. A failing case is found again with the same +value. + Changes from `craft-media-testkit`'s `hammer`: - A case that does not finish fails the test after its limit with the case's label. Before, a case diff --git a/crates/testkit/src/hostile.rs b/crates/testkit/src/hostile.rs index 4480d6aaf..2b34819e6 100644 --- a/crates/testkit/src/hostile.rs +++ b/crates/testkit/src/hostile.rs @@ -11,7 +11,10 @@ //! mutations, one case after another on a worker thread, with a time limit for each case. It fails //! the test, naming the case and the input length, when the parser panics or does not finish in //! time. The number of rounds is a count the caller gives, multiplied by a variable the caller -//! names. [`hammer`] and [`HammerConfig`] are not built for `wasm32`. +//! names. [`DIVISOR_VARIABLE`] (`CRAFT_HOSTILE_DIVISOR`) runs a fraction of every sweep for quick +//! runs such as pull requests: every N-th truncation and 1/N of the rounds; [`divisor`] reads it +//! for sweeps written without [`hammer`]. [`hammer`], [`HammerConfig`] and [`divisor`] are not +//! built for `wasm32`. use std::fmt::Write as _; @@ -40,6 +43,12 @@ const WORKER_NAME: &str = "craft-testkit hostile"; /// [`HammerConfig::with_multiplier_variable`] names another. pub const DEFAULT_MULTIPLIER_VARIABLE: &str = "CRAFT_HOSTILE_MULT"; +/// The variable that divides every hostile-input sweep: with `N`, [`hammer`] runs the unmodified +/// seed, every N-th truncation (lengths 0, N, 2N, ...) and 1/N of its rounds, rounded up. Unset, or +/// 1, runs everything. The Workspace gate sets it and the nightly full run leaves it unset; a +/// failing case is found again with the same value. +pub const DIVISOR_VARIABLE: &str = "CRAFT_HOSTILE_DIVISOR"; + /// The default limit for one case of [`hammer`]. The limit's variable, `CRAFT_TEST_TIMEOUT_SECS` /// unless [`HammerConfig::with_timeout`] names another, replaces it. #[cfg(not(target_arch = "wasm32"))] @@ -339,6 +348,36 @@ fn parse_multiplier(value: &str) -> Result { } } +/// The value of [`DIVISOR_VARIABLE`], or 1 when it is not set, for a sweep written without +/// [`hammer`]: run the cases whose index plus an offset is a multiple of it, and give each input of +/// the sweep a different offset so that together they still reach every index. Panics, naming the +/// variable, when the value is not a whole number above zero. +/// +/// ``` +/// let step = craft_testkit::hostile::divisor(); +/// let positions: Vec = (0..64).filter(|i| (i + 3) % step == 0).collect(); +/// assert!(!positions.is_empty()); +/// ``` +// Failing the calling test is what this function is for. +#[cfg(not(target_arch = "wasm32"))] +#[allow(clippy::panic)] +#[track_caller] +pub fn divisor() -> usize { + match divisor_with(read_variable) { + Ok(divisor) => divisor, + Err(error) => panic!("{error}"), + } +} + +/// [`divisor`] with the variable lookup passed in. +#[cfg(not(target_arch = "wasm32"))] +fn divisor_with(lookup: impl FnOnce(&str) -> Option) -> Result { + let Some(raw) = lookup(DIVISOR_VARIABLE) else { return Ok(1) }; + let variable = DIVISOR_VARIABLE.to_owned(); + let value = raw.into_string().map_err(|_| HammerError::NotUnicode { variable: variable.clone() })?; + parse_multiplier(&value).map_err(|reason| HammerError::BadValue { variable, value, reason }) +} + /// Reads the variable `name` from the process environment. #[cfg(not(target_arch = "wasm32"))] fn read_variable(name: &str) -> Option { @@ -388,15 +427,16 @@ where { let label = config.label; let rounds = config.rounds_with(&lookup).map_err(|error| format!("{label}: {error}"))?; - let rounds = if seed.is_empty() { 0 } else { rounds }; - let cases = 1 + seed.len() as u128 + rounds as u128; + let divisor = divisor_with(&lookup).map_err(|error| format!("{label}: {error}"))?; + let rounds = if seed.is_empty() { 0 } else { rounds.div_ceil(divisor) }; + let cases = 1 + seed.len().div_ceil(divisor) as u128 + rounds as u128; if cases > MAX_CASES as u128 { return Err(format!("{label}: {}", HammerError::TooManyCases { cases })); } let start = Instant::now(); let limit = resolve(&config.timeout, &lookup, start).map(Limit::from).map_err(|error| format!("{label}: {error}"))?; let clock: Clock = Arc::new(move || start.elapsed()); - run_cases(label.to_owned(), limit, Arc::from(seed), rounds, parse, clock) + run_cases(label.to_owned(), limit, Arc::from(seed), rounds, divisor, parse, clock) } /// A clock that returns the time since a [`hammer`] call started. @@ -467,7 +507,7 @@ impl Drop for Report { /// Starts the worker that runs every case, then waits for it, failing when a case runs past the /// limit. #[cfg(not(target_arch = "wasm32"))] -fn run_cases(label: String, limit: Limit, seed: Arc<[u8]>, rounds: usize, parse: F, clock: Clock) -> Result<(), String> +fn run_cases(label: String, limit: Limit, seed: Arc<[u8]>, rounds: usize, divisor: usize, parse: F, clock: Clock) -> Result<(), String> where F: Fn(&[u8]) + Send + Sync + 'static, { @@ -480,7 +520,7 @@ where .name(WORKER_NAME.to_owned()) .spawn(move || { let mut report = Report { watch: worker_watch, outcome: Err(format!("{worker_label}: the worker thread stopped without a result")) }; - report.outcome = run_all(&report.watch, &worker_clock, &worker_label, &worker_limit, &seed, rounds, &parse); + report.outcome = run_all(&report.watch, &worker_clock, &worker_label, &worker_limit, &seed, rounds, divisor, &parse); }) .map_err(|error| format!("{label}: could not start the worker thread: {error}"))?; let mut state = watch.lock(); @@ -498,11 +538,21 @@ where } } -/// Runs the unmodified seed, its truncations and its mutations in order, stopping at the first -/// failure. +/// Runs the unmodified seed, every `divisor`-th truncation and its mutations in order, stopping at +/// the first failure. #[cfg(not(target_arch = "wasm32"))] -fn run_all(watch: &Watch, clock: &Clock, label: &str, limit: &Limit, seed: &[u8], rounds: usize, parse: &F) -> Result<(), String> { - for n in std::iter::once(seed.len()).chain(0..seed.len()) { +#[allow(clippy::too_many_arguments)] +fn run_all( + watch: &Watch, + clock: &Clock, + label: &str, + limit: &Limit, + seed: &[u8], + rounds: usize, + divisor: usize, + parse: &F, +) -> Result<(), String> { + for n in std::iter::once(seed.len()).chain((0..seed.len()).step_by(divisor)) { let case = if n == seed.len() { "the unmodified seed".to_owned() } else { format!("truncation to {n}") }; run_one(watch, clock, label, limit, case, seed.get(..n).unwrap_or_default(), parse)?; } @@ -675,7 +725,7 @@ mod tests { Duration::ZERO }); let limit = Limit { limit: Duration::from_secs(5), text: "5s".to_owned() }; - assert_eq!(run_cases("late start".to_owned(), limit, Arc::from(&[1u8][..]), 0, |_: &[u8]| {}, clock), Ok(())); + assert_eq!(run_cases("late start".to_owned(), limit, Arc::from(&[1u8][..]), 0, 1, |_: &[u8]| {}, clock), Ok(())); } #[test] @@ -742,6 +792,29 @@ mod tests { assert_eq!(config.rounds_with(unset), Ok(7)); } + #[test] + fn divisor_runs_every_nth_truncation_and_a_fraction_of_the_rounds() { + let lookup = |name: &str| (name == DIVISOR_VARIABLE).then(|| OsString::from("4")); + let seen = Arc::new(Mutex::new(Vec::new())); + let record = Arc::clone(&seen); + hammer_with(&config("divided", 10), &[0; 10], move |d| record.lock().unwrap().push(d.len()), lookup).unwrap(); + let lengths = seen.lock().unwrap().clone(); + // The seed, truncations to 0, 4 and 8, then 10 / 4 = 3 rounds rounded up. + assert_eq!(lengths.len(), 7, "{lengths:?}"); + assert_eq!(lengths[..4], [10, 0, 4, 8]); + assert_eq!(divisor_with(unset), Ok(1)); + } + + #[test] + fn invalid_divisor_fails_before_any_parse() { + let called = Arc::new(Mutex::new(false)); + let flag = Arc::clone(&called); + let lookup = |name: &str| (name == DIVISOR_VARIABLE).then(|| OsString::from("0")); + let message = hammer_with(&config("bad divisor", 3), &[1, 2], move |_| *flag.lock().unwrap() = true, lookup).unwrap_err(); + assert_eq!(message, r#"bad divisor: CRAFT_HOSTILE_DIVISOR="0" is zero; set it to a whole number above zero"#); + assert!(!*called.lock().unwrap()); + } + #[test] fn invalid_multiplier_names_variable() { for (value, reason) in [ diff --git a/scripts/ci-test-inputs.sh b/scripts/ci-test-inputs.sh new file mode 100755 index 000000000..32af3ea77 --- /dev/null +++ b/scripts/ci-test-inputs.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# The test inputs the Workspace gate and the nightly full run fetch before their tests: the +# PhotoCraft test corpora (PSD, PSD tools, ATN, EXR, HEIF, PngSuite) at a pinned PhotoCraft commit, +# and the licensed PDF test fonts. Exports CRAFT_PSD_CORPUS and CRAFT_PDF_TEST_FONT_DIR through +# $GITHUB_ENV. Run from the repository root. +# +# A corpus already in .test-inputs/photocraft/corpus (restored from the CI cache, which is keyed on +# this file) is used as it is, without the PhotoCraft checkout and its xtask build. +set -euo pipefail + +# The PhotoCraft commit whose `cargo xtask corpus` pins the corpora. Changing it changes this file, +# which starts a new corpus cache entry. +photocraft=74330157cb601573ae4523a95bb9132822fc3918 +fetcher=.test-inputs/photocraft + +if [ -z "$(ls -A "$fetcher/corpus" 2>/dev/null)" ]; then + rm -rf "$fetcher" + mkdir -p "$fetcher" + git -C "$fetcher" init --quiet + git -C "$fetcher" remote add origin https://github.com/storytold/PhotoCraft.git + git -C "$fetcher" fetch --quiet --depth=1 origin "$photocraft" + git -C "$fetcher" checkout --quiet --detach FETCH_HEAD + (cd "$fetcher" && cargo xtask corpus --psd --psd-tools --atn --exr --heif --pngsuite) +fi +mkdir -p corpus +cp -a "$fetcher/corpus/." corpus/ + +python scripts/test-fetch-pdf-test-fonts.py +python scripts/fetch-pdf-test-fonts.py "$RUNNER_TEMP/craft-pdf-test-fonts" + +printf 'CRAFT_PSD_CORPUS=%s\n' "$GITHUB_WORKSPACE/$fetcher/corpus" >> "$GITHUB_ENV" +printf 'CRAFT_PDF_TEST_FONT_DIR=%s\n' "$RUNNER_TEMP/craft-pdf-test-fonts" >> "$GITHUB_ENV" From 6af133786067ad5be087366963aa70af2b486447 Mon Sep 17 00:00:00 2001 From: aiongg Date: Tue, 17 Nov 2026 19:00:41 +0000 Subject: [PATCH 03/16] ci: nextest on every leg, tests with all features once Every Workspace leg runs nextest with --all-features and the doctests with cargo test --doc. The default-feature test run is gone: it added exactly four tests, all in craft-image-codecs and craft-text (feature-off behaviour), so those two crates run their tests with default features as well. The default-feature compile stays covered by the first clippy run. The resource groups from craft-libs#2058's audit move into the ci profile with platform = 'cfg(unix)', so Linux and macOS share one profile and Windows keeps the groups it already passes with. No separate example build: nextest builds examples as cargo test does, and clippy --all-targets checks them. scripts/test-workspace-tests.py follows the single command set. --- .config/nextest.toml | 26 ++++++++-------- .github/workflows/workspace.yml | 26 ++++++---------- scripts/test-workspace-tests.py | 54 +++++++++++++-------------------- 3 files changed, 44 insertions(+), 62 deletions(-) diff --git a/.config/nextest.toml b/.config/nextest.toml index 5e02e048d..0bc679e47 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -1,6 +1,6 @@ # cargo-nextest (https://nexte.st) runs every test in its own process, in parallel across test -# binaries. Linux and Windows in the Workspace gate use it (.github/workflows/workspace.yml). -# macOS and local runs keep `cargo test`. Both nextest legs also run `cargo test --doc`. +# binaries. Every leg of the Workspace gate uses it (.github/workflows/workspace.yml); local runs may +# keep `cargo test`. Nextest does not run doctests, so the gate also runs `cargo test --doc`. # # cargo nextest run --workspace --profile ci # cargo test --workspace --doc @@ -21,8 +21,9 @@ video-fixtures = { max-threads = 1 } # craft-print: `PRINT_TICKET_PROVIDER` serializes the Windows PrintTicket provider, and the tests # share the one test print queue the gate installs. print = { max-threads = 1 } -# The Linux pilot serializes video integration tests sharing generated fixture directories. -linux-video-fixtures = { max-threads = 1 } +# craft-video integration tests on Linux and macOS: the binaries generate and read the same fixture +# directories (see the `ci` overrides below). +video-integration = { max-threads = 1 } [[profile.default.overrides]] filter = 'package(craft-ui) & (binary(visual_regression) | test(/^combo::tests::|selected_tab_erases_a_baseline_centered_on_its_bottom_edge/))' @@ -44,16 +45,17 @@ fail-fast = false slow-timeout = { period = "60s" } failure-output = "immediate-final" -# Linux-only pilot: keep the established Windows profile unchanged. GPU consumers share one -# process-independent slot; the video integration binaries share fixture directories. This -# deliberately starts conservatively rather than relying on in-process mutexes across binaries. -[profile.ci-linux] -inherits = "ci" -[[profile.ci-linux.overrides]] +# Linux and macOS (the resource audit in craft-libs#2058): every GPU consumer shares the one `gpu` +# slot, and the craft-video integration binaries share their fixture directories. Conservative on +# purpose: the in-process mutexes these tests rely on do not reach across nextest's processes. +# Windows keeps the default groups above, which its leg has passed with since it moved to nextest. +[[profile.ci.overrides]] +platform = 'cfg(unix)' filter = 'package(craft-ui) | package(craft-keymap) | package(craft-update)' test-group = 'gpu' -[[profile.ci-linux.overrides]] +[[profile.ci.overrides]] +platform = 'cfg(unix)' filter = 'package(craft-video) & kind(test)' -test-group = 'linux-video-fixtures' +test-group = 'video-integration' diff --git a/.github/workflows/workspace.yml b/.github/workflows/workspace.yml index e3295d251..554e645ae 100644 --- a/.github/workflows/workspace.yml +++ b/.github/workflows/workspace.yml @@ -25,7 +25,7 @@ jobs: with: os: '["ubuntu-latest","windows-latest","macos-latest"]' timeout-minutes: 120 - # Linux and Windows use nextest with explicit doctests (.config/nextest.toml). + # Every leg runs its tests with nextest, and its doctests with cargo test (.config/nextest.toml). tools: cargo-nextest@0.9.150 packages: mesa-vulkan-drivers libvulkan1 craft-fonts-ref: b076417b5880bd4154bd22f00695b1d28c6bea83 @@ -45,29 +45,21 @@ jobs: cargo fmt --check cargo clippy --workspace --all-targets --locked -- -D warnings cargo clippy --workspace --all-targets --all-features --locked -- -D warnings - if [ "$RUNNER_OS" = Linux ]; then - # Keep example compilation as well as test=true example tests selected by nextest. - cargo build --workspace --examples --locked - cargo nextest run --workspace --locked --profile ci-linux - cargo test --workspace --doc --locked - cargo build --workspace --examples --all-features --locked - cargo nextest run --workspace --all-features --locked --profile ci-linux - cargo test --workspace --doc --all-features --locked - else - cargo test --workspace --locked - cargo test --workspace --all-features --locked - fi + # The tests run once, with every feature, in parallel across test binaries; nextest skips + # doctests, so `cargo test --doc` runs them. The default-feature compile is the first + # clippy run's; the second command runs the crates whose tests check behaviour with a + # feature off, which an all-features build cannot reach. + cargo nextest run --workspace --all-features --locked --profile ci + cargo test --workspace --doc --all-features --locked + cargo nextest run -p craft-image-codecs -p craft-text --locked --profile ci windows-command: | powershell.exe -NoProfile -Command '$ErrorActionPreference = "Stop"; Start-Service Spooler; Add-PrinterDriver -Name "Generic / Text Only"' cargo fmt --check cargo clippy --workspace --all-targets --locked -- -D warnings cargo clippy --workspace --all-targets --all-features --locked -- -D warnings - # The same tests as `cargo test`, run in parallel across test binaries instead of one - # binary at a time; nextest skips doctests, so `cargo test --doc` runs them. - cargo nextest run --workspace --locked --profile ci - cargo test --workspace --doc --locked cargo nextest run --workspace --all-features --locked --profile ci cargo test --workspace --doc --all-features --locked + cargo nextest run -p craft-image-codecs -p craft-text --locked --profile ci msrv: name: Rust 1.95 diff --git a/scripts/test-workspace-tests.py b/scripts/test-workspace-tests.py index f6e55f968..4da52d701 100644 --- a/scripts/test-workspace-tests.py +++ b/scripts/test-workspace-tests.py @@ -40,65 +40,53 @@ COMMON = [ "clippy --workspace --all-targets --locked -- -D warnings", "clippy --workspace --all-targets --all-features --locked -- -D warnings", ] -LINUX = [ - "build --workspace --examples --locked", - "nextest run --workspace --locked --profile ci-linux", - "test --workspace --doc --locked", - "build --workspace --examples --all-features --locked", - "nextest run --workspace --all-features --locked --profile ci-linux", +TESTS = [ + "nextest run --workspace --all-features --locked --profile ci", "test --workspace --doc --all-features --locked", + "nextest run -p craft-image-codecs -p craft-text --locked --profile ci", ] class WorkspaceTests(unittest.TestCase): - def test_linux_keeps_both_feature_sets_examples_and_doctests(self): - result = commands("Linux") - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(result.stdout.splitlines(), COMMON + LINUX) + def test_linux_and_macos_run_one_feature_set_with_doctests(self): + for os_name in ("Linux", "macOS"): + with self.subTest(os=os_name): + result = commands(os_name) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.splitlines(), COMMON + TESTS) - def test_macos_keeps_cargo_test(self): - result = commands("macOS") - self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(result.stdout.splitlines(), COMMON + [ - "test --workspace --locked", "test --workspace --all-features --locked"]) - - def test_each_linux_stage_failure_remains_fatal(self): - for index, command in enumerate(COMMON + LINUX): + def test_each_stage_failure_remains_fatal(self): + for index, command in enumerate(COMMON + TESTS): with self.subTest(command=command): result = commands("Linux", command) self.assertEqual(result.returncode, 17, result.stderr) - self.assertEqual(result.stdout.splitlines(), (COMMON + LINUX)[:index + 1]) + self.assertEqual(result.stdout.splitlines(), (COMMON + TESTS)[:index + 1]) - def test_windows_keeps_its_existing_profile_and_doctests(self): + def test_windows_runs_the_same_cargo_commands(self): source = block(WORKFLOW.read_text(), "windows-command") cargo = [line.strip() for line in source.splitlines() if line.strip().startswith("cargo ")] - self.assertEqual(cargo, ["cargo " + command for command in COMMON + [ - "nextest run --workspace --locked --profile ci", - "test --workspace --doc --locked", - "nextest run --workspace --all-features --locked --profile ci", - "test --workspace --doc --all-features --locked", - ]]) + self.assertEqual(cargo, ["cargo " + command for command in COMMON + TESTS]) - def test_linux_inherits_no_retries_no_filtering_and_existing_failure_policy(self): + def test_ci_profile_keeps_its_failure_policy_without_retries_or_filtering(self): config = tomllib.loads((ROOT / ".config/nextest.toml").read_text()) - self.assertEqual(config["profile"]["ci-linux"]["inherits"], "ci") self.assertIs(config["profile"]["ci"]["fail-fast"], False) for profile in config["profile"].values(): self.assertNotIn("default-filter", profile) self.assertNotIn("retries", profile) self.assertEqual(config["profile"]["ci"]["slow-timeout"], {"period": "60s"}) - def test_linux_resource_groups_are_serial_and_do_not_change_windows(self): + def test_unix_resource_groups_are_serial_and_leave_windows_alone(self): config = tomllib.loads((ROOT / ".config/nextest.toml").read_text()) - overrides = config["profile"]["ci-linux"]["overrides"] + overrides = config["profile"]["ci"]["overrides"] self.assertEqual(overrides, [ - {"filter": "package(craft-ui) | package(craft-keymap) | package(craft-update)", + {"platform": "cfg(unix)", + "filter": "package(craft-ui) | package(craft-keymap) | package(craft-update)", "test-group": "gpu"}, - {"filter": "package(craft-video) & kind(test)", "test-group": "linux-video-fixtures"}, + {"platform": "cfg(unix)", "filter": "package(craft-video) & kind(test)", + "test-group": "video-integration"}, ]) for override in overrides: self.assertEqual(config["test-groups"][override["test-group"]]["max-threads"], 1) - self.assertNotIn("overrides", config["profile"]["ci"]) def test_workflow_stays_shared_and_keeps_platform_gates(self): source = WORKFLOW.read_text() From 7aa03b89cc92a355e56e2ad5d5a96a249eea3b90 Mon Sep 17 00:00:00 2001 From: aiongg Date: Tue, 17 Nov 2026 19:00:47 +0000 Subject: [PATCH 04/16] ci: no debuginfo on the Windows leg CARGO_PROFILE_DEV_DEBUG=0 on Windows, as PhotoCraft's Windows legs do: writing PDBs for every test binary is a large part of the Windows build. The gate writes windows-env after its dev-debug default, so the Linux and macOS legs keep line-tables-only. --- .github/workflows/workspace.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/workspace.yml b/.github/workflows/workspace.yml index 554e645ae..7faba8a04 100644 --- a/.github/workflows/workspace.yml +++ b/.github/workflows/workspace.yml @@ -30,7 +30,12 @@ jobs: packages: mesa-vulkan-drivers libvulkan1 craft-fonts-ref: b076417b5880bd4154bd22f00695b1d28c6bea83 linux-env: WGPU_BACKEND=vulkan - windows-env: CRAFT_PRINT_REQUIRE_QUEUE=1 + # No debuginfo on Windows: writing PDBs for every test binary is a large part of its link + # time. A panic still names its file and line; only backtrace frames lose theirs, and the + # Linux and macOS legs keep line tables. + windows-env: | + CRAFT_PRINT_REQUIRE_QUEUE=1 + CARGO_PROFILE_DEV_DEBUG=0 # The fuzz-style tests run a fraction of their cases here: every hostile-input hammer and # sweep 1/8 (craft-testkit's CRAFT_HOSTILE_DIVISOR), every proptest without its own case # count 64 cases instead of 256. nightly.yml runs them all in full. From 899d6e0302eb1cf022f8f8d5122b1945467a25a1 Mon Sep 17 00:00:00 2001 From: aiongg Date: Tue, 17 Nov 2026 19:05:04 +0000 Subject: [PATCH 05/16] ci: per-crate workflows run only what the Workspace gate does not The Workspace gate already runs fmt, both clippy runs and the tests of every crate on Linux, Windows and macOS. The per-crate workflows drop those steps and keep their own checks: wasm32 builds and tests, no-default and single-feature clippy and tests, cross-target clippy, docs, release and sample tests, --ignored tests, ffmpeg oracle tests, the Language-data proofing builds, the PDF Unicode fixtures, read-only contract and vendored backends. All of them run on Linux. docpath.yml, print.yml, i18n.yml and ui.yml had no check of their own and are removed. They also stop caching dependencies: about 25 rust-cache entries no longer compete with the gate's for the repository's 10 GB, which the gate's main run missed on every OS. The data caches (Opus vectors, CRW samples) stay. --- .github/workflows/actions.lock | 28 --------- .github/workflows/ai-core.yml | 13 +---- .github/workflows/atomic.yml | 15 +---- .github/workflows/audio.yml | 38 +----------- .github/workflows/cms.yml | 4 +- .github/workflows/crw.yml | 37 +----------- .github/workflows/dialogs.yml | 3 +- .github/workflows/doc.yml | 15 +---- .github/workflows/docpath.yml | 41 ------------- .github/workflows/ecproj-format.yml | 13 +---- .github/workflows/font-store.yml | 16 +---- .github/workflows/formats.yml | 23 ++------ .github/workflows/i18n.yml | 41 ------------- .github/workflows/jpeg.yml | 11 +--- .github/workflows/keymap.yml | 49 ++-------------- .github/workflows/media.yml | 17 +----- .github/workflows/menu.yml | 11 +--- .github/workflows/pcraft-format.yml | 14 ++--- .github/workflows/pdf.yml | 57 ++---------------- .github/workflows/print.yml | 46 --------------- .github/workflows/proof.yml | 17 ++---- .github/workflows/ui.yml | 74 ------------------------ .github/workflows/update.yml | 16 ++--- .github/workflows/vectorcraft-format.yml | 14 ++--- .github/workflows/video.yml | 25 +++----- .github/workflows/web-api.yml | 6 +- .github/workflows/web-host.yml | 12 +--- .github/workflows/window.yml | 18 ++---- crates/print/README.md | 8 +-- 29 files changed, 85 insertions(+), 597 deletions(-) delete mode 100644 .github/workflows/docpath.yml delete mode 100644 .github/workflows/i18n.yml delete mode 100644 .github/workflows/print.yml delete mode 100644 .github/workflows/ui.yml diff --git a/.github/workflows/actions.lock b/.github/workflows/actions.lock index 52c8faf77..46d7b8d3c 100644 --- a/.github/workflows/actions.lock +++ b/.github/workflows/actions.lock @@ -18,7 +18,6 @@ workflows: - 'actions/checkout@v7.0.1' - 'step-security/dtolnay-rust-toolchain@v1.0.0' - 'step-security/harden-runner@v2.22.4' - - 'step-security/rust-cache@v2.9.2' '.github/workflows/cms.yml': - 'actions/checkout@v4.4.0' - 'dtolnay/rust-toolchain@stable' @@ -27,7 +26,6 @@ workflows: - 'actions/checkout@v7.0.1' - 'step-security/dtolnay-rust-toolchain@v1.0.0' - 'step-security/harden-runner@v2.22.4' - - 'step-security/rust-cache@v2.9.2' '.github/workflows/dialogs.yml': - 'actions/checkout@v7.0.1' - 'step-security/dtolnay-rust-toolchain@v1.0.0' @@ -36,10 +34,6 @@ workflows: - 'actions/checkout@v7.0.1' - 'step-security/dtolnay-rust-toolchain@v1.0.0' - 'step-security/harden-runner@v2.22.4' - '.github/workflows/docpath.yml': - - 'actions/checkout@v7.0.1' - - 'step-security/dtolnay-rust-toolchain@v1.0.0' - - 'step-security/harden-runner@v2.22.4' '.github/workflows/ecproj-format.yml': - 'actions/checkout@v7.0.1' - 'step-security/dtolnay-rust-toolchain@v1.0.0' @@ -52,22 +46,15 @@ workflows: - 'actions/checkout@v7.0.1' - 'dtolnay/rust-toolchain@stable' - 'step-security/harden-runner@v2.22.4' - - 'swatinem/rust-cache@v2.9.3' '.github/workflows/house-rules.yml': [] - '.github/workflows/i18n.yml': - - 'actions/checkout@v7.0.1' - - 'step-security/dtolnay-rust-toolchain@v1.0.0' - - 'step-security/harden-runner@v2.22.4' '.github/workflows/jpeg.yml': - 'actions/checkout@v7.0.1' - 'step-security/dtolnay-rust-toolchain@v1.0.0' - 'step-security/harden-runner@v2.22.4' - - 'step-security/rust-cache@v2.9.2' '.github/workflows/keymap.yml': - 'actions/checkout@v7.0.1' - 'step-security/dtolnay-rust-toolchain@v1.0.0' - 'step-security/harden-runner@v2.22.4' - - 'step-security/rust-cache@v2.9.2' '.github/workflows/lockfile.yml': - 'actions/checkout@v7.0.1' - 'step-security/harden-runner@v2.22.4' @@ -75,11 +62,9 @@ workflows: - 'actions/checkout@v7.0.1' - 'step-security/dtolnay-rust-toolchain@v1.0.0' - 'step-security/harden-runner@v2.22.4' - - 'step-security/rust-cache@v2.9.2' '.github/workflows/menu.yml': - 'actions/checkout@v7.0.1' - 'dtolnay/rust-toolchain@stable' - - 'swatinem/rust-cache@v2.9.3' '.github/workflows/nightly.yml': [] '.github/workflows/pcraft-format.yml': - 'actions/checkout@v7.0.1' @@ -90,23 +75,12 @@ workflows: - 'actions/setup-python@v7.1.0' - 'step-security/dtolnay-rust-toolchain@v1.0.0' - 'step-security/harden-runner@v2.22.4' - - 'step-security/rust-cache@v2.9.2' - 'step-security/taiki-e-install-action@v2.87.32' - '.github/workflows/print.yml': - - 'actions/checkout@v7.0.1' - - 'step-security/dtolnay-rust-toolchain@v1.0.0' - - 'step-security/harden-runner@v2.22.4' '.github/workflows/proof.yml': - 'actions/checkout@v7.0.1' - 'step-security/dtolnay-rust-toolchain@v1.0.0' - 'step-security/harden-runner@v2.22.4' - - 'step-security/rust-cache@v2.9.2' '.github/workflows/sonarcloud.yml': [] - '.github/workflows/ui.yml': - - 'actions/checkout@v7.0.1' - - 'step-security/dtolnay-rust-toolchain@v1.0.0' - - 'step-security/harden-runner@v2.22.4' - - 'step-security/rust-cache@v2.9.2' '.github/workflows/update.yml': - 'actions/checkout@v7.0.1' - 'step-security/dtolnay-rust-toolchain@v1.0.0' @@ -119,7 +93,6 @@ workflows: - 'actions/checkout@v7.0.1' - 'step-security/dtolnay-rust-toolchain@v1.0.0' - 'step-security/harden-runner@v2.22.4' - - 'step-security/rust-cache@v2.9.2' '.github/workflows/web-api.yml': - 'actions/checkout@v7.0.1' - 'step-security/dtolnay-rust-toolchain@v1.0.0' @@ -129,7 +102,6 @@ workflows: - 'actions/checkout@v7.0.1' - 'step-security/dtolnay-rust-toolchain@v1.0.0' - 'step-security/harden-runner@v2.22.4' - - 'step-security/rust-cache@v2.9.2' - 'step-security/taiki-e-install-action@v2.87.32' '.github/workflows/window.yml': - 'actions/checkout@v7.0.1' diff --git a/.github/workflows/ai-core.yml b/.github/workflows/ai-core.yml index 5471bf1ec..4de27650e 100644 --- a/.github/workflows/ai-core.yml +++ b/.github/workflows/ai-core.yml @@ -17,15 +17,12 @@ concurrency: group: ai-core-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} +# fmt, clippy and the tests run in the Workspace gate on every OS; this checks what it does not. jobs: ai-core: + name: AI model WASM timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - # macOS runners are kept for releases: macOS runs only when started by hand. - os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }} - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest steps: - name: Harden runner uses: step-security/harden-runner@v2.22.4 @@ -35,10 +32,6 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: stable - components: rustfmt, clippy targets: wasm32-unknown-unknown - - run: cargo fmt --check - - run: cargo clippy -p craft-ai-core --all-targets --locked -- -D warnings - - run: cargo test -p craft-ai-core --locked # The model promises to compile for the web. - run: cargo check -p craft-ai-core --target wasm32-unknown-unknown --locked diff --git a/.github/workflows/atomic.yml b/.github/workflows/atomic.yml index f72a64f1e..be11c8d89 100644 --- a/.github/workflows/atomic.yml +++ b/.github/workflows/atomic.yml @@ -17,15 +17,12 @@ concurrency: group: atomic-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} +# fmt, clippy and the tests run in the Workspace gate on every OS; this runs what it does not. jobs: atomic: + name: Atomic inherited Linux ACLs timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - # macOS runners are kept for releases: macOS runs only when started by hand. - os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }} - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest steps: - name: Harden runner uses: step-security/harden-runner@v2.22.4 @@ -35,15 +32,9 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: stable - components: rustfmt, clippy - name: Install ACL tools - if: runner.os == 'Linux' run: sudo apt-get update && sudo apt-get install -y acl - - run: cargo fmt --check - - run: cargo clippy -p craft-atomic -p craft-settings --all-targets --locked -- -D warnings - - run: cargo test -p craft-atomic -p craft-settings --locked -- --nocapture - name: Check inherited Linux ACLs - if: runner.os == 'Linux' run: cargo test -p craft-atomic --locked -- --ignored --nocapture wasm: diff --git a/.github/workflows/audio.yml b/.github/workflows/audio.yml index 48c1384b2..d4b2a5202 100644 --- a/.github/workflows/audio.yml +++ b/.github/workflows/audio.yml @@ -26,7 +26,8 @@ concurrency: jobs: lint: timeout-minutes: 15 - name: Audio lint, feature sets, docs, WASM + # fmt, clippy with all features and the minimum Rust version run in the Workspace gate. + name: Audio feature sets, docs, WASM runs-on: ubuntu-latest steps: - name: Harden runner @@ -39,13 +40,8 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: stable - components: rustfmt, clippy + components: clippy targets: wasm32-unknown-unknown - - uses: step-security/rust-cache@v2.9.2 - with: - key: craft-audio-lint - save-if: ${{ github.ref == 'refs/heads/main' }} - - run: cargo fmt -p craft-audio --check # Each feature on its own, so code that needs another feature fails here. - name: Clippy, each feature alone run: | @@ -54,7 +50,6 @@ jobs: cargo clippy -p craft-audio --all-targets --locked --no-default-features ${f:+--features "$f"} -- -D warnings echo "::endgroup::" done - - run: cargo clippy -p craft-audio --all-targets --locked --all-features -- -D warnings # missing_docs is a warning in the crate; -D warnings makes it (and broken links) fail here. - run: cargo doc -p craft-audio --all-features --no-deps --locked env: @@ -68,25 +63,6 @@ jobs: RUSTDOCFLAGS: "-D warnings" - run: cargo check -p craft-audio --all-features --locked --target wasm32-unknown-unknown - run: cargo check -p craft-audio --no-default-features --features lossless-encode --locked --target wasm32-unknown-unknown - msrv: - timeout-minutes: 10 - name: Audio on the minimum Rust version - runs-on: ubuntu-latest - steps: - - name: Harden runner - uses: step-security/harden-runner@v2.22.4 - with: - egress-policy: audit - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - # The workspace's rust-version. - - run: rustup toolchain install 1.95.0 --profile minimal - - uses: step-security/rust-cache@v2.9.2 - with: - key: craft-audio-msrv - save-if: ${{ github.ref == 'refs/heads/main' }} - - run: cargo +1.95.0 check -p craft-audio -p craft-media --all-features --locked features: timeout-minutes: 10 name: Audio tests without symphonia and lossless-encode only @@ -102,10 +78,6 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: stable - - uses: step-security/rust-cache@v2.9.2 - with: - key: craft-audio-features - save-if: ${{ github.ref == 'refs/heads/main' }} - run: cargo test -p craft-audio --no-default-features --features lossless-encode --locked - run: cargo test -p craft-audio --no-default-features --features pcm-encode --locked --lib - run: cargo test -p craft-audio --no-default-features --features flac-encode --locked --lib @@ -127,10 +99,6 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: stable - - uses: step-security/rust-cache@v2.9.2 - with: - key: craft-audio-test - save-if: ${{ github.ref == 'refs/heads/main' }} - uses: actions/cache@v6.1.0 with: path: ${{ runner.temp }}/media-fixtures/opus/vectors diff --git a/.github/workflows/cms.yml b/.github/workflows/cms.yml index 74ce0815f..2040212c2 100644 --- a/.github/workflows/cms.yml +++ b/.github/workflows/cms.yml @@ -11,6 +11,7 @@ on: permissions: contents: read +# clippy and the debug tests run in the Workspace gate; this runs the release build of the tests. jobs: cms: timeout-minutes: 20 @@ -18,9 +19,6 @@ jobs: steps: - uses: actions/checkout@v4.4.0 - uses: dtolnay/rust-toolchain@stable - with: - components: clippy - - run: cargo clippy -p craft-cms --all-targets --all-features --locked -- -D warnings # Release: the FOGRA39 fit and ink-limit tests take about 35 s in debug and 4 s here. Tests that need # installed system profiles print "skipped:" when none are present. - run: cargo test -p craft-cms --all-features --release --locked -- --nocapture diff --git a/.github/workflows/crw.yml b/.github/workflows/crw.yml index 54be7ba3b..f5bb91a38 100644 --- a/.github/workflows/crw.yml +++ b/.github/workflows/crw.yml @@ -18,35 +18,8 @@ concurrency: group: shared-crw-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} +# fmt, clippy and the tests run in the Workspace gate on every OS; this checks what it does not. jobs: - native: - timeout-minutes: 20 - name: CRW lint and tests (${{ matrix.os }}) - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, windows-latest] - runs-on: ${{ matrix.os }} - steps: - - name: Harden runner - uses: step-security/harden-runner@v2.22.4 - with: - egress-policy: audit - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - uses: step-security/dtolnay-rust-toolchain@v1.0.0 - with: - toolchain: stable - components: rustfmt, clippy - - uses: step-security/rust-cache@v2.9.2 - with: - key: craft-crw-${{ matrix.os }} - save-if: ${{ github.ref == 'refs/heads/main' }} - - run: cargo fmt -p craft-crw --check - - run: cargo clippy -p craft-crw --all-targets --locked -- -D warnings - - run: cargo test -p craft-crw --locked - wasm: timeout-minutes: 10 name: CRW WASM @@ -63,10 +36,6 @@ jobs: with: toolchain: stable targets: wasm32-unknown-unknown - - uses: step-security/rust-cache@v2.9.2 - with: - key: craft-crw-wasm - save-if: ${{ github.ref == 'refs/heads/main' }} - run: cargo check -p craft-crw --locked --target wasm32-unknown-unknown samples: @@ -84,10 +53,6 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: stable - - uses: step-security/rust-cache@v2.9.2 - with: - key: craft-crw-samples - save-if: ${{ github.ref == 'refs/heads/main' }} - name: Cache the CC0 sample files uses: actions/cache@v6.1.0 with: diff --git a/.github/workflows/dialogs.yml b/.github/workflows/dialogs.yml index c8d43d725..151a63e36 100644 --- a/.github/workflows/dialogs.yml +++ b/.github/workflows/dialogs.yml @@ -37,6 +37,5 @@ jobs: components: clippy # The macOS code is linted on Linux: it compiles for the target without the SDK. targets: aarch64-apple-darwin - - run: cargo clippy -p craft-dialogs --all-targets --locked -- -D warnings + # Native clippy and the tests run in the Workspace gate. - run: cargo clippy -p craft-dialogs --target aarch64-apple-darwin --locked -- -D warnings - - run: cargo test -p craft-dialogs --locked diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 6caa4f0ee..394cdbdbe 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -17,15 +17,12 @@ concurrency: group: doc-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} +# fmt, clippy and the tests run in the Workspace gate on every OS; this checks what it does not. jobs: doc: + name: Craft documents WASM timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - # macOS runners are kept for releases: macOS runs only when started by hand. - os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }} - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest steps: - name: Harden runner uses: step-security/harden-runner@v2.22.4 @@ -37,11 +34,5 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: stable - components: rustfmt, clippy targets: wasm32-unknown-unknown - - run: cargo fmt --check - - run: cargo clippy -p craft-doc --all-targets --locked -- -D warnings - - run: cargo clippy -p craft-doc --all-targets --all-features --locked -- -D warnings - - run: cargo test -p craft-doc --locked -- --nocapture - - run: cargo test -p craft-doc --all-features --locked -- --nocapture - run: cargo check -p craft-doc --all-features --locked --target wasm32-unknown-unknown diff --git a/.github/workflows/docpath.yml b/.github/workflows/docpath.yml deleted file mode 100644 index e0115f6f5..000000000 --- a/.github/workflows/docpath.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: Document paths - -on: - workflow_dispatch: - push: - branches: [main] - paths: ['crates/docpath/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/docpath.yml'] - pull_request: - paths: ['crates/docpath/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/docpath.yml'] - -permissions: - contents: read - -concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued. - group: docpath-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -jobs: - docpath: - timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - # macOS runners are kept for releases: macOS runs only when started by hand. - os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }} - runs-on: ${{ matrix.os }} - steps: - - name: Harden runner - uses: step-security/harden-runner@v2.22.4 - with: - egress-policy: audit - - uses: actions/checkout@v7.0.1 - - uses: step-security/dtolnay-rust-toolchain@v1.0.0 - with: - toolchain: stable - components: rustfmt, clippy - - run: cargo fmt --check - - run: cargo clippy -p craft-docpath --all-targets --locked -- -D warnings - - run: cargo test -p craft-docpath --locked -- --nocapture diff --git a/.github/workflows/ecproj-format.yml b/.github/workflows/ecproj-format.yml index 24a3350af..eeacf5801 100644 --- a/.github/workflows/ecproj-format.yml +++ b/.github/workflows/ecproj-format.yml @@ -17,15 +17,12 @@ concurrency: group: ecproj-format-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} +# fmt, clippy and the tests run in the Workspace gate on every OS; this checks what it does not. jobs: ecproj-format: + name: EffectCraft project format WASM timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - # macOS runners are kept for releases: macOS runs only when started by hand. - os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }} - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest steps: - name: Harden runner uses: step-security/harden-runner@v2.22.4 @@ -37,9 +34,5 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: stable - components: rustfmt, clippy targets: wasm32-unknown-unknown - - run: cargo fmt --check - - run: cargo clippy -p craft-ecproj-format --all-targets --locked -- -D warnings - - run: cargo test -p craft-ecproj-format --locked -- --nocapture - run: cargo check -p craft-ecproj-format --locked --target wasm32-unknown-unknown diff --git a/.github/workflows/font-store.yml b/.github/workflows/font-store.yml index 0578f18ec..34c7c09a1 100644 --- a/.github/workflows/font-store.yml +++ b/.github/workflows/font-store.yml @@ -17,15 +17,12 @@ concurrency: group: font-store-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} +# fmt, clippy and the tests run in the Workspace gate on every OS; this checks what it does not. jobs: font-store: + name: Font store WASM timeout-minutes: 20 - strategy: - fail-fast: false - matrix: - # macOS runners are kept for releases: macOS runs only when started by hand. - os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }} - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest steps: - name: Harden runner uses: step-security/harden-runner@v2.22.4 @@ -35,13 +32,6 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: stable - components: rustfmt, clippy targets: wasm32-unknown-unknown - - run: cargo fmt --check - - run: cargo clippy -p craft-font-store --all-targets --locked -- -D warnings - - run: cargo test -p craft-font-store --locked - - run: cargo clippy -p craft-font-catalog --all-targets --locked -- -D warnings - - run: cargo test -p craft-font-catalog --locked - name: Check the web build - if: runner.os == 'Linux' run: cargo check -p craft-font-store --target wasm32-unknown-unknown --locked diff --git a/.github/workflows/formats.yml b/.github/workflows/formats.yml index 84d2c0d7e..769271d84 100644 --- a/.github/workflows/formats.yml +++ b/.github/workflows/formats.yml @@ -68,7 +68,9 @@ jobs: - { crate: craft-tiff, features: "" } - { crate: craft-heif, features: "testdata" } - { crate: craft-cms, features: "" } - name: ${{ matrix.crate }} lint, tests, WASM + # fmt, clippy with default and with all features, and the tests run in the Workspace gate on + # every OS; this lints each feature alone and checks the web build. + name: ${{ matrix.crate }} feature sets, WASM runs-on: ubuntu-latest steps: - name: Harden runner @@ -80,32 +82,19 @@ jobs: persist-credentials: false - uses: dtolnay/rust-toolchain@stable with: - components: rustfmt, clippy + components: clippy targets: wasm32-unknown-unknown - - uses: Swatinem/rust-cache@v2.9.3 - with: - key: ${{ matrix.crate }} - save-if: ${{ github.ref == 'refs/heads/main' }} - - run: cargo fmt -p "$CRATE" --check - env: - CRATE: ${{ matrix.crate }} - - name: Clippy, default features and each feature alone + - name: Clippy, each feature alone + if: matrix.features != '' env: CRATE: ${{ matrix.crate }} FEATURES: ${{ matrix.features }} run: | - cargo clippy -p "$CRATE" --all-targets --locked -- -D warnings for f in $FEATURES; do echo "::group::feature: $f" cargo clippy -p "$CRATE" --all-targets --locked --no-default-features --features "$f" -- -D warnings echo "::endgroup::" done - - run: cargo clippy -p "$CRATE" --all-targets --locked --all-features -- -D warnings - env: - CRATE: ${{ matrix.crate }} - - run: cargo test -p "$CRATE" --all-features --locked - env: - CRATE: ${{ matrix.crate }} - run: cargo check -p "$CRATE" --all-features --locked --target wasm32-unknown-unknown env: CRATE: ${{ matrix.crate }} diff --git a/.github/workflows/i18n.yml b/.github/workflows/i18n.yml deleted file mode 100644 index 7e6903188..000000000 --- a/.github/workflows/i18n.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: Translations - -on: - workflow_dispatch: - push: - branches: [main] - paths: ['crates/i18n/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/i18n.yml'] - pull_request: - paths: ['crates/i18n/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/i18n.yml'] - -permissions: - contents: read - -concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued. - group: i18n-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -jobs: - i18n: - timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - # macOS runners are kept for releases: macOS runs only when started by hand. - os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }} - runs-on: ${{ matrix.os }} - steps: - - name: Harden runner - uses: step-security/harden-runner@v2.22.4 - with: - egress-policy: audit - - uses: actions/checkout@v7.0.1 - - uses: step-security/dtolnay-rust-toolchain@v1.0.0 - with: - toolchain: stable - components: rustfmt, clippy - - run: cargo fmt --check - - run: cargo clippy -p craft-i18n --all-targets --features collate --locked -- -D warnings - - run: cargo test -p craft-i18n --features collate --locked -- --nocapture diff --git a/.github/workflows/jpeg.yml b/.github/workflows/jpeg.yml index 21db0ab79..325cc0c02 100644 --- a/.github/workflows/jpeg.yml +++ b/.github/workflows/jpeg.yml @@ -21,7 +21,8 @@ concurrency: jobs: check: timeout-minutes: 10 - name: JPEG lint, tests, WASM + # fmt, clippy and the tests run in the Workspace gate on every OS. + name: JPEG WASM runs-on: ubuntu-latest steps: - name: Harden runner @@ -34,13 +35,5 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: stable - components: rustfmt, clippy targets: wasm32-unknown-unknown - - uses: step-security/rust-cache@v2.9.2 - with: - key: craft-jpeg - save-if: ${{ github.ref == 'refs/heads/main' }} - - run: cargo fmt -p craft-jpeg --check - - run: cargo clippy -p craft-jpeg --all-targets --locked -- -D warnings - - run: cargo test -p craft-jpeg --locked - run: cargo check -p craft-jpeg --locked --target wasm32-unknown-unknown diff --git a/.github/workflows/keymap.yml b/.github/workflows/keymap.yml index 41fcc0502..f85f84935 100644 --- a/.github/workflows/keymap.yml +++ b/.github/workflows/keymap.yml @@ -21,7 +21,9 @@ concurrency: jobs: linux: timeout-minutes: 15 - name: Keymap (Linux) + # fmt, and clippy and the tests with default and with all features, run in the Workspace gate + # on every OS; this runs the feature subsets. + name: Keymap feature sets runs-on: ubuntu-latest env: CRAFT_FONTS_DIR: ${{ github.workspace }}/.test-inputs/craft-fonts @@ -43,12 +45,7 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: stable - components: rustfmt, clippy - - uses: step-security/rust-cache@v2.9.2 - with: - key: craft-keymap-linux - save-if: ${{ github.ref == 'refs/heads/main' }} - - run: cargo fmt -p craft-keymap --check + components: clippy - run: cargo clippy -p craft-keymap --all-targets --no-default-features --locked -- -D warnings # `snapshot` covers `ui` and `egui` and compiles the dialog screenshots, which need a GPU # to run, so the test steps leave them out. @@ -58,40 +55,6 @@ jobs: run: cargo test -p craft-keymap --no-default-features --lib --tests --locked - name: Test egui dispatch and the shortcuts dialog run: cargo test -p craft-keymap --features ui --locked - native: - timeout-minutes: 15 - name: Keymap (${{ matrix.os }}) - strategy: - fail-fast: false - matrix: - # macOS runners are kept for releases: macOS runs only when started by hand. - os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["macos-latest","windows-latest"]') || fromJSON('["windows-latest"]') }} - runs-on: ${{ matrix.os }} - env: - CRAFT_FONTS_DIR: ${{ github.workspace }}/.test-inputs/craft-fonts - steps: - - name: Harden runner - uses: step-security/harden-runner@v2.22.4 - with: - egress-policy: audit - - uses: actions/checkout@v7.0.1 - with: - persist-credentials: false - - name: Checkout test fonts - uses: actions/checkout@v7.0.1 - with: - repository: storytold/craft-fonts - ref: b076417b5880bd4154bd22f00695b1d28c6bea83 - path: .test-inputs/craft-fonts - persist-credentials: false - - uses: step-security/dtolnay-rust-toolchain@v1.0.0 - with: - toolchain: stable - - uses: step-security/rust-cache@v2.9.2 - with: - key: craft-keymap-${{ matrix.os }} - save-if: ${{ github.ref == 'refs/heads/main' }} - - run: cargo test -p craft-keymap --features ui --locked wasm: timeout-minutes: 10 name: Keymap WASM @@ -108,8 +71,4 @@ jobs: with: toolchain: stable targets: wasm32-unknown-unknown - - uses: step-security/rust-cache@v2.9.2 - with: - key: craft-keymap-wasm - save-if: ${{ github.ref == 'refs/heads/main' }} - run: cargo check -p craft-keymap --features ui --locked --target wasm32-unknown-unknown diff --git a/.github/workflows/media.yml b/.github/workflows/media.yml index f397f5cb3..0fbf78b12 100644 --- a/.github/workflows/media.yml +++ b/.github/workflows/media.yml @@ -26,7 +26,8 @@ concurrency: jobs: lint: timeout-minutes: 10 - name: Media lint, feature sets, WASM + # fmt, and clippy with default and with all features, run in the Workspace gate on every OS. + name: Media feature sets, WASM runs-on: ubuntu-latest steps: - name: Harden runner @@ -39,20 +40,12 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: stable - components: rustfmt, clippy + components: clippy targets: wasm32-unknown-unknown - - uses: step-security/rust-cache@v2.9.2 - with: - key: craft-media-lint - save-if: ${{ github.ref == 'refs/heads/main' }} - - run: cargo fmt -p craft-media --check - run: cargo clippy -p craft-media --all-targets --locked --no-default-features -- -D warnings - run: cargo clippy -p craft-media --all-targets --locked --no-default-features --features bitstream -- -D warnings - - run: cargo clippy -p craft-media --all-targets --locked -- -D warnings - run: cargo clippy -p craft-media --all-targets --locked --no-default-features --features matroska -- -D warnings - run: cargo clippy -p craft-media --all-targets --locked --no-default-features --features ogg -- -D warnings - - run: cargo clippy -p craft-media --all-targets --locked --all-features -- -D warnings - - run: cargo clippy -p craft-media-testkit --all-targets --locked -- -D warnings - run: cargo clippy -p craft-media --all-targets --locked --no-default-features --features serde -- -D warnings - run: cargo check -p craft-media --all-features --locked --target wasm32-unknown-unknown test: @@ -70,10 +63,6 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: stable - - uses: step-security/rust-cache@v2.9.2 - with: - key: craft-media-test - save-if: ${{ github.ref == 'refs/heads/main' }} - name: Build tests while installing ffmpeg run: | # The man-db index rebuild after dpkg is slow on hosted runners. diff --git a/.github/workflows/menu.yml b/.github/workflows/menu.yml index 8d1058731..b12c8bcc6 100644 --- a/.github/workflows/menu.yml +++ b/.github/workflows/menu.yml @@ -18,7 +18,8 @@ concurrency: jobs: linux: timeout-minutes: 15 - name: Menu (Linux) + # fmt, native clippy and the tests run in the Workspace gate on every OS. + name: Menu macOS and WASM lint runs-on: ubuntu-latest steps: - uses: actions/checkout@v7.0.1 @@ -26,14 +27,8 @@ jobs: persist-credentials: false - uses: dtolnay/rust-toolchain@stable with: - components: rustfmt, clippy + components: clippy targets: aarch64-apple-darwin, wasm32-unknown-unknown - - uses: Swatinem/rust-cache@v2.9.3 - with: - key: craft-menu-linux - - run: cargo fmt -p craft-menu --check - - run: cargo clippy -p craft-menu --all-targets --locked -- -D warnings # The macOS backend type-checks from Linux; it needs no linker or SDK. - run: cargo clippy -p craft-menu --lib --locked --target aarch64-apple-darwin -- -D warnings - run: cargo clippy -p craft-menu --lib --locked --target wasm32-unknown-unknown -- -D warnings - - run: cargo test -p craft-menu --locked diff --git a/.github/workflows/pcraft-format.yml b/.github/workflows/pcraft-format.yml index 9c66b5a10..0e82ba888 100644 --- a/.github/workflows/pcraft-format.yml +++ b/.github/workflows/pcraft-format.yml @@ -17,15 +17,13 @@ concurrency: group: pcraft-format-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} +# fmt, clippy and the tests run in the Workspace gate on every OS, and its msrv job checks the +# workspace on 1.95; this checks the web build on 1.95. jobs: pcraft-format: + name: PhotoCraft format WASM timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - # macOS runners are kept for releases: macOS runs only when started by hand. - os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }} - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest steps: - name: Harden runner uses: step-security/harden-runner@v2.22.4 @@ -37,9 +35,5 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: "1.95" - components: rustfmt, clippy targets: wasm32-unknown-unknown - - run: cargo fmt --check - - run: cargo clippy -p craft-pcraft-format --all-targets --locked -- -D warnings - - run: cargo test -p craft-pcraft-format --locked -- --nocapture - run: cargo check -p craft-pcraft-format --locked --target wasm32-unknown-unknown diff --git a/.github/workflows/pdf.yml b/.github/workflows/pdf.yml index 96d6f3faa..3be70ad99 100644 --- a/.github/workflows/pdf.yml +++ b/.github/workflows/pdf.yml @@ -24,13 +24,11 @@ concurrency: jobs: native: - name: PDF (${{ matrix.os }}) - strategy: - fail-fast: false - matrix: - # macOS runners are kept for releases: macOS runs only when started by hand. - os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }} - runs-on: ${{ matrix.os }} + # fmt, clippy and the tests run in the Workspace gate on every OS, with the licensed fonts and + # the plate tests; this runs the tests behind craft_pdf_unicode_fixtures and the read-only + # contract, which is its own workspace. + name: PDF Unicode fixtures and read-only contract + runs-on: ubuntu-latest env: RUSTFLAGS: --cfg craft_pdf_unicode_fixtures steps: @@ -42,11 +40,6 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: stable - components: rustfmt, clippy - - uses: step-security/rust-cache@v2.9.2 - with: - key: craft-pdf - save-if: ${{ github.ref == 'refs/heads/main' }} - uses: actions/setup-python@v7.1.0 with: python-version: "3.12" @@ -56,9 +49,7 @@ jobs: python scripts/test-fetch-pdf-test-fonts.py python scripts/fetch-pdf-test-fonts.py "$RUNNER_TEMP/craft-pdf-test-fonts" echo "CRAFT_PDF_TEST_FONT_DIR=$RUNNER_TEMP/craft-pdf-test-fonts" >> "$GITHUB_ENV" - - run: cargo fmt --check - - run: cargo clippy -p craft-pdf --all-targets --all-features --locked -- -D warnings - - run: cargo test -p craft-pdf --all-features --locked + - run: cargo test -p craft-pdf --all-features --locked --test unicode_writing --test upstream_compatibility --test upstream_reading - run: cargo test --manifest-path crates/pdf/read-only-contract/Cargo.toml --locked backends: name: PDF vendored contracts @@ -72,10 +63,6 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: stable - - uses: step-security/rust-cache@v2.9.2 - with: - key: craft-pdf-backends - save-if: ${{ github.ref == 'refs/heads/main' }} - run: bash scripts/test-pdf-backends.sh wasm: name: PDF WASM runtime @@ -93,10 +80,6 @@ jobs: with: toolchain: stable targets: wasm32-unknown-unknown - - uses: step-security/rust-cache@v2.9.2 - with: - key: craft-pdf-wasm - save-if: ${{ github.ref == 'refs/heads/main' }} - uses: actions/setup-python@v7.1.0 with: python-version: "3.12" @@ -121,31 +104,3 @@ jobs: fallback: cargo-install - run: cargo test -p craft-pdf --all-features --locked --target wasm32-unknown-unknown - run: cargo test --manifest-path crates/pdf/read-only-contract/Cargo.toml --locked --target wasm32-unknown-unknown - plate-test: - name: Plate tests and craft-prepress - runs-on: ubuntu-latest - steps: - - name: Harden runner - uses: step-security/harden-runner@v2.22.4 - with: - egress-policy: audit - - uses: actions/checkout@v7.0.1 - - uses: step-security/dtolnay-rust-toolchain@v1.0.0 - with: - toolchain: stable - components: clippy - - uses: step-security/rust-cache@v2.9.2 - with: - key: craft-plate-test - save-if: ${{ github.ref == 'refs/heads/main' }} - - uses: actions/setup-python@v7.1.0 - with: - python-version: "3.12" - - name: Provision licensed PDF test fonts - shell: bash - run: | - python scripts/test-fetch-pdf-test-fonts.py - python scripts/fetch-pdf-test-fonts.py "$RUNNER_TEMP/craft-pdf-test-fonts" - echo "CRAFT_PDF_TEST_FONT_DIR=$RUNNER_TEMP/craft-pdf-test-fonts" >> "$GITHUB_ENV" - - run: cargo clippy -p craft-plate-test -p craft-prepress --all-targets --all-features --locked -- -D warnings - - run: cargo test -p craft-plate-test -p craft-prepress --all-features --locked diff --git a/.github/workflows/print.yml b/.github/workflows/print.yml deleted file mode 100644 index 8e2108e09..000000000 --- a/.github/workflows/print.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: Print spooler -on: - workflow_dispatch: - push: - branches: [main] - paths: ['crates/print/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/print.yml'] - pull_request: - paths: ['crates/print/**', 'Cargo.toml', 'Cargo.lock', '.github/workflows/print.yml'] -permissions: - contents: read -concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued. - group: print-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} -jobs: - print: - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - # macOS runners are kept for releases: macOS runs only when started by hand. - os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }} - runs-on: ${{ matrix.os }} - steps: - - name: Harden runner - uses: step-security/harden-runner@v2.22.4 - with: - egress-policy: audit - - uses: actions/checkout@v7.0.1 - - uses: step-security/dtolnay-rust-toolchain@v1.0.0 - with: - toolchain: stable - components: rustfmt, clippy - - run: cargo fmt --check - - run: cargo clippy -p craft-print --all-targets -- -D warnings - - name: Prepare the Windows test driver - if: runner.os == 'Windows' - shell: powershell - run: | - Start-Service Spooler - Add-PrinterDriver -Name 'Generic / Text Only' - - run: cargo test -p craft-print --locked -- --nocapture - timeout-minutes: 10 - env: - CRAFT_PRINT_REQUIRE_QUEUE: ${{ runner.os == 'Windows' && '1' || '0' }} diff --git a/.github/workflows/proof.yml b/.github/workflows/proof.yml index 10365a230..27bf5b32b 100644 --- a/.github/workflows/proof.yml +++ b/.github/workflows/proof.yml @@ -21,7 +21,8 @@ concurrency: jobs: check: timeout-minutes: 20 - name: Proofing lint, tests with and without Language data, WASM + # fmt, and clippy and the tests without Language data, run in the Workspace gate on every OS. + name: Proofing with Language data, WASM runs-on: ubuntu-latest steps: - name: Harden runner @@ -43,18 +44,10 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: stable - components: rustfmt, clippy + components: clippy targets: wasm32-unknown-unknown - - uses: step-security/rust-cache@v2.9.2 - with: - key: craft-proof - save-if: ${{ github.ref == 'refs/heads/main' }} - - run: cargo fmt -p craft-proof --check - - name: Without Language data (a fork, docs.rs, offline) - run: | - cargo clippy -p craft-proof --all-targets --locked -- -D warnings - cargo test -p craft-proof --locked - cargo check -p craft-proof --locked --target wasm32-unknown-unknown + - name: The web build without Language data (a fork, docs.rs, offline) + run: cargo check -p craft-proof --locked --target wasm32-unknown-unknown - name: A release without Language data fails run: | if CRAFT_I18N_REQUIRED=1 cargo check -p craft-proof --locked 2>build.log; then diff --git a/.github/workflows/ui.yml b/.github/workflows/ui.yml deleted file mode 100644 index f15ef6ddc..000000000 --- a/.github/workflows/ui.yml +++ /dev/null @@ -1,74 +0,0 @@ -name: Shared UI - -on: - push: - branches: [main] - paths: ["crates/ui/**", "crates/i18n/**", "Cargo.toml", "Cargo.lock", ".github/workflows/ui.yml"] - pull_request: - paths: ["crates/ui/**", "crates/i18n/**", "Cargo.toml", "Cargo.lock", ".github/workflows/ui.yml"] - workflow_dispatch: - -permissions: - contents: read - -concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it - # always gets to save the cache. - group: shared-ui-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -jobs: - native: - timeout-minutes: 30 - name: UI (${{ matrix.os }}) - strategy: - fail-fast: false - matrix: - # macOS runners are kept for releases: macOS runs only when started by hand. - os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }} - runs-on: ${{ matrix.os }} - steps: - - name: Harden runner - uses: step-security/harden-runner@v2.22.4 - with: - egress-policy: audit - - uses: actions/checkout@v7.0.1 - - name: Install software Vulkan renderer - if: runner.os == 'Linux' - run: | - sudo apt-get update - sudo apt-get install -y mesa-vulkan-drivers libvulkan1 - echo 'WGPU_BACKEND=vulkan' >> "$GITHUB_ENV" - - uses: step-security/dtolnay-rust-toolchain@v1.0.0 - with: - toolchain: stable - components: rustfmt, clippy - - uses: step-security/rust-cache@v2.9.2 - with: - key: craft-ui - save-if: ${{ github.ref == 'refs/heads/main' }} - - run: cargo fmt --check - - run: cargo clippy -p craft-ui --all-targets --locked -- -D warnings - - run: cargo clippy -p craft-ui --all-targets --all-features --locked -- -D warnings - - run: cargo test -p craft-ui --locked - - run: cargo test -p craft-ui --all-features --locked - wasm: - timeout-minutes: 15 - name: UI WASM - runs-on: ubuntu-latest - steps: - - name: Harden runner - uses: step-security/harden-runner@v2.22.4 - with: - egress-policy: audit - - uses: actions/checkout@v7.0.1 - - uses: step-security/dtolnay-rust-toolchain@v1.0.0 - with: - toolchain: stable - targets: wasm32-unknown-unknown - - uses: step-security/rust-cache@v2.9.2 - with: - key: craft-ui-wasm - save-if: ${{ github.ref == 'refs/heads/main' }} - - run: cargo check -p craft-ui --all-features --locked --target wasm32-unknown-unknown diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml index 9a7139026..cfb14ec9a 100644 --- a/.github/workflows/update.yml +++ b/.github/workflows/update.yml @@ -17,15 +17,13 @@ concurrency: group: update-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} +# fmt, clippy and the tests with default and with all features run in the Workspace gate on every +# OS; this runs the egui feature alone. jobs: update: + name: Update with egui alone timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - # macOS runners are kept for releases: macOS runs only when started by hand. - os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }} - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest steps: - name: Harden runner uses: step-security/harden-runner@v2.22.4 @@ -39,10 +37,6 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: stable - components: rustfmt, clippy - - run: cargo fmt --check - # Default (no egui) and the egui feature both build and lint cleanly. - - run: cargo clippy -p craft-update --all-targets --locked -- -D warnings + components: clippy - run: cargo clippy -p craft-update --all-targets --locked --features egui -- -D warnings - - run: cargo test -p craft-update --locked -- --nocapture - run: cargo test -p craft-update --locked --features egui -- --nocapture diff --git a/.github/workflows/vectorcraft-format.yml b/.github/workflows/vectorcraft-format.yml index 74144f30f..e6cfb7a70 100644 --- a/.github/workflows/vectorcraft-format.yml +++ b/.github/workflows/vectorcraft-format.yml @@ -17,15 +17,13 @@ concurrency: group: vectorcraft-format-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} +# fmt, clippy and the tests run in the Workspace gate on every OS, and its msrv job checks the +# workspace on 1.95; this checks the web build on 1.95. jobs: vectorcraft-format: + name: VectorCraft format WASM timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - # macOS runners are kept for releases: macOS runs only when started by hand. - os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }} - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest steps: - name: Harden runner uses: step-security/harden-runner@v2.22.4 @@ -37,9 +35,5 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: "1.95" - components: rustfmt, clippy targets: wasm32-unknown-unknown - - run: cargo fmt --check - - run: cargo clippy -p craft-vectorcraft-format --all-targets --locked -- -D warnings - - run: cargo test -p craft-vectorcraft-format --locked -- --nocapture - run: cargo check -p craft-vectorcraft-format --locked --target wasm32-unknown-unknown diff --git a/.github/workflows/video.yml b/.github/workflows/video.yml index 97e7c0d34..20afe0d70 100644 --- a/.github/workflows/video.yml +++ b/.github/workflows/video.yml @@ -23,7 +23,8 @@ concurrency: jobs: lint: timeout-minutes: 20 - name: Video lint, feature sets, docs, WASM + # fmt and clippy with all features run in the Workspace gate on every OS. + name: Video feature sets, docs, WASM runs-on: ubuntu-latest steps: - name: Harden runner @@ -36,12 +37,8 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: "1.95" - components: rustfmt, clippy + components: clippy targets: wasm32-unknown-unknown - - uses: step-security/rust-cache@v2.9.2 - with: - key: craft-video-lint - - run: cargo fmt -p craft-video --check # Each feature on its own, so code that needs another feature fails here. - name: Clippy, each feature alone run: | @@ -50,19 +47,15 @@ jobs: cargo clippy -p craft-video --all-targets --locked --no-default-features ${f:+--features "$f"} -- -D warnings echo "::endgroup::" done - - run: cargo clippy -p craft-video --all-targets --locked --all-features -- -D warnings - run: cargo doc -p craft-video --all-features --no-deps --locked env: RUSTDOCFLAGS: "-D warnings" - run: cargo check -p craft-video --all-features --locked --target wasm32-unknown-unknown test: timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, windows-latest] - runs-on: ${{ matrix.os }} - name: Video tests (${{ matrix.os }}) + # The Workspace gate runs these tests on every OS without ffmpeg; this runs them with it. + runs-on: ubuntu-latest + name: Video tests with ffmpeg steps: - name: Harden runner uses: step-security/harden-runner@v2.22.4 @@ -74,11 +67,7 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: "1.95" - - uses: step-security/rust-cache@v2.9.2 - with: - key: craft-video-${{ matrix.os }} - - name: Install ffmpeg (Linux) - if: runner.os == 'Linux' + - name: Install ffmpeg run: | sudo rm -f /var/lib/man-db/auto-update sudo apt-get update -qq && sudo apt-get install -y -qq --no-install-recommends ffmpeg diff --git a/.github/workflows/web-api.yml b/.github/workflows/web-api.yml index 4bf1a30fe..1c9b2c92f 100644 --- a/.github/workflows/web-api.yml +++ b/.github/workflows/web-api.yml @@ -35,7 +35,7 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: stable - components: rustfmt, clippy + components: clippy targets: wasm32-unknown-unknown - name: Read the wasm-bindgen version from Cargo.lock id: wasm-bindgen @@ -50,8 +50,6 @@ jobs: with: tool: wasm-bindgen@${{ steps.wasm-bindgen.outputs.version }} fallback: cargo-install - - run: cargo fmt --check - - run: cargo clippy -p craft-web-api --all-targets --locked -- -D warnings + # fmt, native clippy and the native tests run in the Workspace gate. - run: cargo clippy -p craft-web-api --all-targets --locked --target wasm32-unknown-unknown -- -D warnings - - run: cargo test -p craft-web-api --locked - run: cargo test -p craft-web-api --locked --target wasm32-unknown-unknown diff --git a/.github/workflows/web-host.yml b/.github/workflows/web-host.yml index 76e6701e5..28129a542 100644 --- a/.github/workflows/web-host.yml +++ b/.github/workflows/web-host.yml @@ -21,7 +21,8 @@ concurrency: jobs: check: timeout-minutes: 20 - name: Web host lint, tests, WASM tests + # fmt, native clippy and the native tests run in the Workspace gate. + name: Web host WASM lint and tests runs-on: ubuntu-latest env: CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUNNER: wasm-bindgen-test-runner @@ -36,16 +37,9 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: stable - components: rustfmt, clippy + components: clippy targets: wasm32-unknown-unknown - - uses: step-security/rust-cache@v2.9.2 - with: - key: craft-web-host - save-if: ${{ github.ref == 'refs/heads/main' }} - - run: cargo fmt -p craft-web-host --check - - run: cargo clippy -p craft-web-host --all-targets --locked -- -D warnings - run: cargo clippy -p craft-web-host --all-targets --locked --target wasm32-unknown-unknown -- -D warnings - - run: cargo test -p craft-web-host --locked - name: Read the wasm-bindgen version from Cargo.lock id: wasm-bindgen shell: bash diff --git a/.github/workflows/window.yml b/.github/workflows/window.yml index fdd28e203..bf5f39835 100644 --- a/.github/workflows/window.yml +++ b/.github/workflows/window.yml @@ -17,15 +17,13 @@ concurrency: group: window-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} +# fmt, clippy and the tests with default and with all features (egui and restore together) run in +# the Workspace gate on every OS; this runs each feature alone. jobs: window: + name: Window with each feature alone timeout-minutes: 15 - strategy: - fail-fast: false - matrix: - # macOS runners are kept for releases: macOS runs only when started by hand. - os: ${{ github.event_name == 'workflow_dispatch' && fromJSON('["ubuntu-latest","windows-latest","macos-latest"]') || fromJSON('["ubuntu-latest","windows-latest"]') }} - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest steps: - name: Harden runner uses: step-security/harden-runner@v2.22.4 @@ -39,12 +37,8 @@ jobs: - uses: step-security/dtolnay-rust-toolchain@v1.0.0 with: toolchain: stable - components: rustfmt, clippy - - run: cargo fmt --check - # Default (no egui) and the egui feature both build and lint cleanly. - - run: cargo clippy -p craft-window --all-targets --locked -- -D warnings + components: clippy - run: cargo clippy -p craft-window --all-targets --locked --features egui -- -D warnings - - run: cargo test -p craft-window --locked -- --nocapture - run: cargo test -p craft-window --locked --features egui -- --nocapture - run: cargo clippy -p craft-window --all-targets --locked --features restore -- -D warnings - - run: cargo test -p craft-window --locked --features restore,egui -- --nocapture + - run: cargo test -p craft-window --locked --features restore -- --nocapture diff --git a/crates/print/README.md b/crates/print/README.md index eb7af885e..56ba5ed86 100644 --- a/crates/print/README.md +++ b/crates/print/README.md @@ -199,10 +199,10 @@ job) and `an_installed_printer_prints_a_file` (`CRAFT_PRINT_TEST_PRINTER`, `CRAFT_PRINT_TEST_FILE`, optional `CRAFT_PRINT_TEST_SCALING=shrink` and `CRAFT_PRINT_TEST_DUPLEX=long`). Physical printer fidelity remains a hardware check. -The cross-platform GitHub Actions workflow is defined in `.github/workflows/print.yml`. -It runs formatting, Clippy and this crate's own tests on Windows, Linux and macOS when print -code changes, and supports manual runs. The Windows job executes the spooler regression above; -physical printer fidelity still requires hardware verification. +The Workspace gate (`.github/workflows/workspace.yml`) runs formatting, Clippy and this crate's +tests on Windows, Linux and macOS. Its Windows leg installs the test print queue and runs the +spooler regression above with `CRAFT_PRINT_REQUIRE_QUEUE=1`; physical printer fidelity still +requires hardware verification. Windows discovery reports driver duplex support in `Printer::two_sided` (a two-sided option in the driver's PrintCapabilities); CUPS leaves it unknown. Explicit simplex/duplex choices that the driver From 688c5beba2cc540299bd8f5411e88cadfe802fe6 Mon Sep 17 00:00:00 2001 From: aiongg Date: Tue, 17 Nov 2026 19:05:33 +0000 Subject: [PATCH 06/16] ci: cache the PhotoCraft test corpora The gate restores .test-inputs/photocraft/corpus before its setup, keyed on scripts/ci-test-inputs.sh (which holds the PhotoCraft pin), and the setup uses a restored corpus as it is. A hit skips the PhotoCraft checkout and its xtask build: 2:17 of Windows setup, about 1:00 on Linux and macOS. Uses craft-common-actions' new cache-paths, cache-key and cache-key-files inputs. --- .github/workflows/nightly.yml | 3 +++ .github/workflows/workspace.yml | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 2ecc68ecb..ed9d36127 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -28,4 +28,7 @@ jobs: linux-env: WGPU_BACKEND=vulkan env: CRAFT_PRINT_REQUIRE_QUEUE=0 setup-command: bash scripts/ci-test-inputs.sh + cache-paths: .test-inputs/photocraft/corpus + cache-key: photocraft-corpus + cache-key-files: scripts/ci-test-inputs.sh command: cargo nextest run --workspace --all-features --locked --profile ci diff --git a/.github/workflows/workspace.yml b/.github/workflows/workspace.yml index 7faba8a04..77020820f 100644 --- a/.github/workflows/workspace.yml +++ b/.github/workflows/workspace.yml @@ -44,6 +44,11 @@ jobs: CRAFT_HOSTILE_DIVISOR=8 PROPTEST_CASES=64 setup-command: bash scripts/ci-test-inputs.sh + # The PhotoCraft corpora, keyed on the script that pins them: a hit skips the PhotoCraft + # checkout and its xtask build (1:51 on Windows). + cache-paths: .test-inputs/photocraft/corpus + cache-key: photocraft-corpus + cache-key-files: scripts/ci-test-inputs.sh command: | python3 scripts/test-workspace-concurrency.py python3 scripts/test-workspace-tests.py From 9bd64169668b8638d9f2822ca40c63f954524852 Mon Sep 17 00:00:00 2001 From: aiongg Date: Tue, 17 Nov 2026 19:05:59 +0000 Subject: [PATCH 07/16] ci: clippy and fmt on a lint leg beside each test leg workspace.yml moves fmt, both clippy runs and the workflow script checks into the gate's lint-command, so each OS gets a lint () leg running beside its gate () leg. Clippy (about 6 min on Linux and 11 on Windows) no longer runs before the test build. scripts/test-workspace-tests.py checks both blocks. --- .github/workflows/workspace.yml | 17 +++++++++-------- scripts/test-workspace-tests.py | 30 +++++++++++++++++++----------- 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/.github/workflows/workspace.yml b/.github/workflows/workspace.yml index 77020820f..86b47d398 100644 --- a/.github/workflows/workspace.yml +++ b/.github/workflows/workspace.yml @@ -49,24 +49,25 @@ jobs: cache-paths: .test-inputs/photocraft/corpus cache-key: photocraft-corpus cache-key-files: scripts/ci-test-inputs.sh - command: | - python3 scripts/test-workspace-concurrency.py - python3 scripts/test-workspace-tests.py + # A `lint ()` leg beside each `gate ()` leg, so clippy is off the tests' critical path. + lint-command: | + if [ "$RUNNER_OS" = Linux ]; then + python3 scripts/test-workspace-concurrency.py + python3 scripts/test-workspace-tests.py + fi cargo fmt --check cargo clippy --workspace --all-targets --locked -- -D warnings cargo clippy --workspace --all-targets --all-features --locked -- -D warnings + command: | # The tests run once, with every feature, in parallel across test binaries; nextest skips - # doctests, so `cargo test --doc` runs them. The default-feature compile is the first - # clippy run's; the second command runs the crates whose tests check behaviour with a + # doctests, so `cargo test --doc` runs them. The default-feature compile is the lint leg's + # first clippy run; the last command runs the crates whose tests check behaviour with a # feature off, which an all-features build cannot reach. cargo nextest run --workspace --all-features --locked --profile ci cargo test --workspace --doc --all-features --locked cargo nextest run -p craft-image-codecs -p craft-text --locked --profile ci windows-command: | powershell.exe -NoProfile -Command '$ErrorActionPreference = "Stop"; Start-Service Spooler; Add-PrinterDriver -Name "Generic / Text Only"' - cargo fmt --check - cargo clippy --workspace --all-targets --locked -- -D warnings - cargo clippy --workspace --all-targets --all-features --locked -- -D warnings cargo nextest run --workspace --all-features --locked --profile ci cargo test --workspace --doc --all-features --locked cargo nextest run -p craft-image-codecs -p craft-text --locked --profile ci diff --git a/scripts/test-workspace-tests.py b/scripts/test-workspace-tests.py index 4da52d701..fada6dc9b 100644 --- a/scripts/test-workspace-tests.py +++ b/scripts/test-workspace-tests.py @@ -18,7 +18,7 @@ def block(text, name): return "\n".join(line[8:] for line in match[1].splitlines()) -def commands(os_name, fail=""): +def commands(os_name, fail="", name="command"): # Functions avoid executing any caller-supplied cargo command or changing the workspace. # The real bash -euo pipefail semantics still apply, including failures inside a branch. stub = """ @@ -29,7 +29,7 @@ cargo() { python3() { :; } """ return subprocess.run( - ["bash", "-euo", "pipefail", "-c", stub + block(WORKFLOW.read_text(), "command")], + ["bash", "-euo", "pipefail", "-c", stub + block(WORKFLOW.read_text(), name)], env={**os.environ, "RUNNER_OS": os_name, "FAIL": fail}, text=True, capture_output=True, check=False, ) @@ -48,24 +48,32 @@ TESTS = [ class WorkspaceTests(unittest.TestCase): + def test_every_leg_lints_with_both_feature_sets(self): + for os_name in ("Linux", "macOS", "Windows"): + with self.subTest(os=os_name): + result = commands(os_name, name="lint-command") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.splitlines(), COMMON) + def test_linux_and_macos_run_one_feature_set_with_doctests(self): for os_name in ("Linux", "macOS"): with self.subTest(os=os_name): result = commands(os_name) self.assertEqual(result.returncode, 0, result.stderr) - self.assertEqual(result.stdout.splitlines(), COMMON + TESTS) + self.assertEqual(result.stdout.splitlines(), TESTS) def test_each_stage_failure_remains_fatal(self): - for index, command in enumerate(COMMON + TESTS): - with self.subTest(command=command): - result = commands("Linux", command) - self.assertEqual(result.returncode, 17, result.stderr) - self.assertEqual(result.stdout.splitlines(), (COMMON + TESTS)[:index + 1]) + for name, stages in (("lint-command", COMMON), ("command", TESTS)): + for index, command in enumerate(stages): + with self.subTest(block=name, command=command): + result = commands("Linux", command, name) + self.assertEqual(result.returncode, 17, result.stderr) + self.assertEqual(result.stdout.splitlines(), stages[:index + 1]) - def test_windows_runs_the_same_cargo_commands(self): + def test_windows_runs_the_same_tests(self): source = block(WORKFLOW.read_text(), "windows-command") cargo = [line.strip() for line in source.splitlines() if line.strip().startswith("cargo ")] - self.assertEqual(cargo, ["cargo " + command for command in COMMON + TESTS]) + self.assertEqual(cargo, ["cargo " + command for command in TESTS]) def test_ci_profile_keeps_its_failure_policy_without_retries_or_filtering(self): config = tomllib.loads((ROOT / ".config/nextest.toml").read_text()) @@ -94,7 +102,7 @@ class WorkspaceTests(unittest.TestCase): self.assertIn('os: \'["ubuntu-latest","windows-latest","macos-latest"]\'', source) self.assertIn(" msrv:", source) self.assertIn(" wasm:", source) - self.assertIn("python3 scripts/test-workspace-tests.py", block(source, "command")) + self.assertIn("python3 scripts/test-workspace-tests.py", block(source, "lint-command")) if __name__ == "__main__": From a1398a3b6168ec2540588a92f7f80983382d054f Mon Sep 17 00:00:00 2001 From: aiongg Date: Tue, 17 Nov 2026 19:06:05 +0000 Subject: [PATCH 08/16] ci: macOS and Windows legs on a pull request only when it can change them workspace.yml passes select-os to the gate: a pull request runs the macOS and Windows legs when a changed file is in a crate with platform cfg code or in a crate that depends on one, is part of the build setup (workflows, Cargo.lock, the root Cargo.toml, toolchain, .cargo, .config), or lies outside the crates and is not documentation. 37 of 177 crates have platform code and 57 are affected with their dependents; 39 of the last 80 merged pull requests would have run Linux only, without the macOS queue. The full-ci label runs every leg from the next push or manual run. Pushes to main run every leg. --- .github/workflows/workspace.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/workspace.yml b/.github/workflows/workspace.yml index 86b47d398..aa2e04fe3 100644 --- a/.github/workflows/workspace.yml +++ b/.github/workflows/workspace.yml @@ -24,6 +24,11 @@ jobs: uses: storytold/craft-common-actions/.github/workflows/gate.yml@main with: os: '["ubuntu-latest","windows-latest","macos-latest"]' + # On a pull request, macOS and Windows run only when it can change platform behaviour: a + # crate with platform cfg code or one that depends on it, the build setup, or a file outside + # the crates (craft-common-actions' select-platforms). The full-ci label runs them anyway + # from the next push or manual run; pushes to main always run every leg. + select-os: '["windows-latest","macos-latest"]' timeout-minutes: 120 # Every leg runs its tests with nextest, and its doctests with cargo test (.config/nextest.toml). tools: cargo-nextest@0.9.150 From c0421c7fc6774e5087ea37e7c4fd0882a656b8fc Mon Sep 17 00:00:00 2001 From: aiongg Date: Tue, 17 Nov 2026 19:08:21 +0000 Subject: [PATCH 09/16] ci: cancel superseded main runs; a scheduled full run saves the caches Pushes to main cancel the Workspace run they supersede, as pull requests already do; the newest push's run tests everything before it. On 16 Nov main had 144 pushes, each a three-OS run that could not be cancelled, which is much of what fills the org's macOS runners. Release tags, manual runs and a new scheduled run every 6 hours keep a group of their own and always finish, and the scheduled run saves the caches (the gate already saves on scheduled runs of the default branch). scripts/test-workspace-concurrency.py checks the new policy. The per-crate workflows, which no longer save caches, cancel superseded runs on every branch, main included. --- .github/workflows/ai-core.yml | 8 ++-- .github/workflows/atomic.yml | 8 ++-- .github/workflows/audio.yml | 9 ++--- .github/workflows/cms.yml | 6 +++ .github/workflows/crw.yml | 9 ++--- .github/workflows/dialogs.yml | 8 ++-- .github/workflows/doc.yml | 8 ++-- .github/workflows/ecproj-format.yml | 8 ++-- .github/workflows/font-store.yml | 8 ++-- .github/workflows/formats.yml | 9 ++--- .github/workflows/jpeg.yml | 9 ++--- .github/workflows/keymap.yml | 9 ++--- .github/workflows/media.yml | 9 ++--- .github/workflows/pcraft-format.yml | 8 ++-- .github/workflows/pdf.yml | 9 ++--- .github/workflows/proof.yml | 9 ++--- .github/workflows/update.yml | 8 ++-- .github/workflows/vectorcraft-format.yml | 8 ++-- .github/workflows/web-api.yml | 8 ++-- .github/workflows/web-host.yml | 9 ++--- .github/workflows/window.yml | 8 ++-- .github/workflows/workspace.yml | 16 +++++--- scripts/test-workspace-concurrency.py | 50 ++++++++++++++++-------- 23 files changed, 129 insertions(+), 112 deletions(-) diff --git a/.github/workflows/ai-core.yml b/.github/workflows/ai-core.yml index 4de27650e..2be123b23 100644 --- a/.github/workflows/ai-core.yml +++ b/.github/workflows/ai-core.yml @@ -12,10 +12,10 @@ permissions: contents: read concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued. - group: ai-core-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + # One group per pull request and per branch: a newer push cancels the run it supersedes, on main + # too, since the newest run checks everything before it. + group: ai-core-${{ github.ref }} + cancel-in-progress: true # fmt, clippy and the tests run in the Workspace gate on every OS; this checks what it does not. jobs: diff --git a/.github/workflows/atomic.yml b/.github/workflows/atomic.yml index be11c8d89..23de7f9f4 100644 --- a/.github/workflows/atomic.yml +++ b/.github/workflows/atomic.yml @@ -12,10 +12,10 @@ permissions: contents: read concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued. - group: atomic-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + # One group per pull request and per branch: a newer push cancels the run it supersedes, on main + # too, since the newest run checks everything before it. + group: atomic-${{ github.ref }} + cancel-in-progress: true # fmt, clippy and the tests run in the Workspace gate on every OS; this runs what it does not. jobs: diff --git a/.github/workflows/audio.yml b/.github/workflows/audio.yml index d4b2a5202..5fd971c62 100644 --- a/.github/workflows/audio.yml +++ b/.github/workflows/audio.yml @@ -17,11 +17,10 @@ env: PROPTEST_CASES: "64" concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it - # always gets to save the cache. - group: shared-audio-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + # One group per pull request and per branch: a newer push cancels the run it supersedes, on main + # too, since the newest run checks everything before it. + group: shared-audio-${{ github.ref }} + cancel-in-progress: true jobs: lint: diff --git a/.github/workflows/cms.yml b/.github/workflows/cms.yml index 2040212c2..9220d8e09 100644 --- a/.github/workflows/cms.yml +++ b/.github/workflows/cms.yml @@ -11,6 +11,12 @@ on: permissions: contents: read +concurrency: + # One group per pull request and per branch: a newer push cancels the run it supersedes, on main + # too, since the newest run checks everything before it. + group: cms-${{ github.ref }} + cancel-in-progress: true + # clippy and the debug tests run in the Workspace gate; this runs the release build of the tests. jobs: cms: diff --git a/.github/workflows/crw.yml b/.github/workflows/crw.yml index f5bb91a38..466b6c05d 100644 --- a/.github/workflows/crw.yml +++ b/.github/workflows/crw.yml @@ -12,11 +12,10 @@ permissions: contents: read concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it - # always gets to save the cache. - group: shared-crw-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + # One group per pull request and per branch: a newer push cancels the run it supersedes, on main + # too, since the newest run checks everything before it. + group: shared-crw-${{ github.ref }} + cancel-in-progress: true # fmt, clippy and the tests run in the Workspace gate on every OS; this checks what it does not. jobs: diff --git a/.github/workflows/dialogs.yml b/.github/workflows/dialogs.yml index 151a63e36..23def9feb 100644 --- a/.github/workflows/dialogs.yml +++ b/.github/workflows/dialogs.yml @@ -12,10 +12,10 @@ permissions: contents: read concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued. - group: dialogs-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + # One group per pull request and per branch: a newer push cancels the run it supersedes, on main + # too, since the newest run checks everything before it. + group: dialogs-${{ github.ref }} + cancel-in-progress: true jobs: dialogs: diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 394cdbdbe..627543381 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -12,10 +12,10 @@ permissions: contents: read concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued. - group: doc-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + # One group per pull request and per branch: a newer push cancels the run it supersedes, on main + # too, since the newest run checks everything before it. + group: doc-${{ github.ref }} + cancel-in-progress: true # fmt, clippy and the tests run in the Workspace gate on every OS; this checks what it does not. jobs: diff --git a/.github/workflows/ecproj-format.yml b/.github/workflows/ecproj-format.yml index eeacf5801..048c897b3 100644 --- a/.github/workflows/ecproj-format.yml +++ b/.github/workflows/ecproj-format.yml @@ -12,10 +12,10 @@ permissions: contents: read concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued. - group: ecproj-format-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + # One group per pull request and per branch: a newer push cancels the run it supersedes, on main + # too, since the newest run checks everything before it. + group: ecproj-format-${{ github.ref }} + cancel-in-progress: true # fmt, clippy and the tests run in the Workspace gate on every OS; this checks what it does not. jobs: diff --git a/.github/workflows/font-store.yml b/.github/workflows/font-store.yml index 34c7c09a1..e655697b3 100644 --- a/.github/workflows/font-store.yml +++ b/.github/workflows/font-store.yml @@ -12,10 +12,10 @@ permissions: contents: read concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued. - group: font-store-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + # One group per pull request and per branch: a newer push cancels the run it supersedes, on main + # too, since the newest run checks everything before it. + group: font-store-${{ github.ref }} + cancel-in-progress: true # fmt, clippy and the tests run in the Workspace gate on every OS; this checks what it does not. jobs: diff --git a/.github/workflows/formats.yml b/.github/workflows/formats.yml index 769271d84..12478d2c9 100644 --- a/.github/workflows/formats.yml +++ b/.github/workflows/formats.yml @@ -42,11 +42,10 @@ permissions: contents: read concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it - # always gets to save the cache. - group: shared-formats-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + # One group per pull request and per branch: a newer push cancels the run it supersedes, on main + # too, since the newest run checks everything before it. + group: shared-formats-${{ github.ref }} + cancel-in-progress: true jobs: check: diff --git a/.github/workflows/jpeg.yml b/.github/workflows/jpeg.yml index 325cc0c02..c49c03581 100644 --- a/.github/workflows/jpeg.yml +++ b/.github/workflows/jpeg.yml @@ -12,11 +12,10 @@ permissions: contents: read concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it - # always gets to save the cache. - group: shared-jpeg-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + # One group per pull request and per branch: a newer push cancels the run it supersedes, on main + # too, since the newest run checks everything before it. + group: shared-jpeg-${{ github.ref }} + cancel-in-progress: true jobs: check: diff --git a/.github/workflows/keymap.yml b/.github/workflows/keymap.yml index f85f84935..bd4ef280a 100644 --- a/.github/workflows/keymap.yml +++ b/.github/workflows/keymap.yml @@ -12,11 +12,10 @@ permissions: contents: read concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it - # always gets to save the cache. - group: keymap-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + # One group per pull request and per branch: a newer push cancels the run it supersedes, on main + # too, since the newest run checks everything before it. + group: keymap-${{ github.ref }} + cancel-in-progress: true jobs: linux: diff --git a/.github/workflows/media.yml b/.github/workflows/media.yml index 0fbf78b12..1b5afaebd 100644 --- a/.github/workflows/media.yml +++ b/.github/workflows/media.yml @@ -17,11 +17,10 @@ env: PROPTEST_CASES: "64" concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it - # always gets to save the cache. - group: shared-media-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + # One group per pull request and per branch: a newer push cancels the run it supersedes, on main + # too, since the newest run checks everything before it. + group: shared-media-${{ github.ref }} + cancel-in-progress: true jobs: lint: diff --git a/.github/workflows/pcraft-format.yml b/.github/workflows/pcraft-format.yml index 0e82ba888..0e4feabc9 100644 --- a/.github/workflows/pcraft-format.yml +++ b/.github/workflows/pcraft-format.yml @@ -12,10 +12,10 @@ permissions: contents: read concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued. - group: pcraft-format-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + # One group per pull request and per branch: a newer push cancels the run it supersedes, on main + # too, since the newest run checks everything before it. + group: pcraft-format-${{ github.ref }} + cancel-in-progress: true # fmt, clippy and the tests run in the Workspace gate on every OS, and its msrv job checks the # workspace on 1.95; this checks the web build on 1.95. diff --git a/.github/workflows/pdf.yml b/.github/workflows/pdf.yml index 3be70ad99..7b7397757 100644 --- a/.github/workflows/pdf.yml +++ b/.github/workflows/pdf.yml @@ -16,11 +16,10 @@ env: PROPTEST_CASES: "64" concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it - # always gets to save the cache. - group: shared-pdf-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + # One group per pull request and per branch: a newer push cancels the run it supersedes, on main + # too, since the newest run checks everything before it. + group: shared-pdf-${{ github.ref }} + cancel-in-progress: true jobs: native: diff --git a/.github/workflows/proof.yml b/.github/workflows/proof.yml index 27bf5b32b..ea5edb8a5 100644 --- a/.github/workflows/proof.yml +++ b/.github/workflows/proof.yml @@ -12,11 +12,10 @@ permissions: contents: read concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it - # always gets to save the cache. - group: craft-proof-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + # One group per pull request and per branch: a newer push cancels the run it supersedes, on main + # too, since the newest run checks everything before it. + group: craft-proof-${{ github.ref }} + cancel-in-progress: true jobs: check: diff --git a/.github/workflows/update.yml b/.github/workflows/update.yml index cfb14ec9a..96ab4d6ad 100644 --- a/.github/workflows/update.yml +++ b/.github/workflows/update.yml @@ -12,10 +12,10 @@ permissions: contents: read concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued. - group: update-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + # One group per pull request and per branch: a newer push cancels the run it supersedes, on main + # too, since the newest run checks everything before it. + group: update-${{ github.ref }} + cancel-in-progress: true # fmt, clippy and the tests with default and with all features run in the Workspace gate on every # OS; this runs the egui feature alone. diff --git a/.github/workflows/vectorcraft-format.yml b/.github/workflows/vectorcraft-format.yml index e6cfb7a70..cd66615a5 100644 --- a/.github/workflows/vectorcraft-format.yml +++ b/.github/workflows/vectorcraft-format.yml @@ -12,10 +12,10 @@ permissions: contents: read concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued. - group: vectorcraft-format-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + # One group per pull request and per branch: a newer push cancels the run it supersedes, on main + # too, since the newest run checks everything before it. + group: vectorcraft-format-${{ github.ref }} + cancel-in-progress: true # fmt, clippy and the tests run in the Workspace gate on every OS, and its msrv job checks the # workspace on 1.95; this checks the web build on 1.95. diff --git a/.github/workflows/web-api.yml b/.github/workflows/web-api.yml index 1c9b2c92f..f0f8e22da 100644 --- a/.github/workflows/web-api.yml +++ b/.github/workflows/web-api.yml @@ -12,10 +12,10 @@ permissions: contents: read concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued. - group: web-api-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + # One group per pull request and per branch: a newer push cancels the run it supersedes, on main + # too, since the newest run checks everything before it. + group: web-api-${{ github.ref }} + cancel-in-progress: true jobs: web-api: diff --git a/.github/workflows/web-host.yml b/.github/workflows/web-host.yml index 28129a542..dc0254850 100644 --- a/.github/workflows/web-host.yml +++ b/.github/workflows/web-host.yml @@ -12,11 +12,10 @@ permissions: contents: read concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets its own group: it is never cancelled or replaced while queued, so it - # always gets to save the cache. - group: shared-web-host-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + # One group per pull request and per branch: a newer push cancels the run it supersedes, on main + # too, since the newest run checks everything before it. + group: shared-web-host-${{ github.ref }} + cancel-in-progress: true jobs: check: diff --git a/.github/workflows/window.yml b/.github/workflows/window.yml index bf5f39835..c13b1ac3f 100644 --- a/.github/workflows/window.yml +++ b/.github/workflows/window.yml @@ -12,10 +12,10 @@ permissions: contents: read concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (main, manual) gets a group of its own, so it is neither cancelled nor replaced while queued. - group: window-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + # One group per pull request and per branch: a newer push cancels the run it supersedes, on main + # too, since the newest run checks everything before it. + group: window-${{ github.ref }} + cancel-in-progress: true # fmt, clippy and the tests with default and with all features (egui and restore together) run in # the Workspace gate on every OS; this runs each feature alone. diff --git a/.github/workflows/workspace.yml b/.github/workflows/workspace.yml index aa2e04fe3..af7e04769 100644 --- a/.github/workflows/workspace.yml +++ b/.github/workflows/workspace.yml @@ -6,17 +6,21 @@ on: tags: ["v*"] pull_request: workflow_dispatch: + # A full run on main every 6 hours that is never cancelled, so the caches are saved while pushes + # to main cancel each other. + schedule: + - cron: "17 */6 * * *" permissions: contents: read concurrency: - # Pull requests share one group per branch, so a newer push cancels the older run. Every other - # run (push to main, a release tag, a manual run) gets a group of its own: it is neither - # cancelled nor replaced while queued, so a main run always finishes and saves the dependency - # cache, and a release run is never cancelled. - group: workspace-${{ github.event_name == 'pull_request' && github.ref || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + # A pull request, and main, each have one group, so a newer push cancels the run it supersedes; + # the newest push's run still tests everything before it. Every other run (a release tag, a + # manual run, the scheduled run) gets a group of its own and is neither cancelled nor replaced + # while queued: a release run always finishes, and the scheduled run saves the caches. + group: workspace-${{ (github.event_name == 'pull_request' || github.event_name == 'push' && github.ref == 'refs/heads/main') && github.ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || github.event_name == 'push' && github.ref == 'refs/heads/main' }} jobs: native: diff --git a/scripts/test-workspace-concurrency.py b/scripts/test-workspace-concurrency.py index 96bfae304..db20031e9 100644 --- a/scripts/test-workspace-concurrency.py +++ b/scripts/test-workspace-concurrency.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 -"""Check the Workspace workflow's pull-request-only cancellation policy. +"""Check the Workspace workflow's cancellation policy: pull requests and pushes to main cancel the +run they supersede; release tags, manual runs and scheduled runs are never cancelled. The policy is the top-level `concurrency` block of .github/workflows/workspace.yml. This script reads those fields, requires the documented expression shapes, and checks which group each event @@ -21,16 +22,28 @@ except ImportError: yaml = None DEFAULT_WORKFLOW = Path(__file__).resolve().parent.parent / ".github" / "workflows" / "workspace.yml" -PULL_REQUEST, PUSH, DISPATCH = "pull_request", "push", "workflow_dispatch" +PULL_REQUEST, PUSH, DISPATCH, SCHEDULE = "pull_request", "push", "workflow_dispatch", "schedule" -# The two shapes this policy may use. GitHub reads `a && b || c` as "b, or c when a is false"; these -# patterns are the whole of that expression language this script understands, so a reworded policy -# has to be reworded here too, on purpose. github.ref, never github.head_ref or github.sha. +# The runs that share a group per ref and cancel the run they supersede. GitHub reads `a || b && c` +# as `a || (b && c)`, and `x && github.ref || github.run_id` as "github.ref, or the run id when x is +# false". This exact condition is the whole of the expression language this script understands, so +# a reworded policy has to be reworded here too, on purpose. github.ref, never github.head_ref or +# github.sha. +SUPERSEDED = "github.event_name == 'pull_request' || github.event_name == 'push' && github.ref == 'refs/heads/main'" GROUP_SHAPE = re.compile( - r"(?P[A-Za-z0-9._-]+)-\$\{\{\s*github\.event_name\s*==\s*'(?P[a-z_]+)'\s*" + r"(?P[A-Za-z0-9._-]+)-\$\{\{\s*\(\s*(?P[^()]+?)\s*\)\s*" r"&&\s*github\.ref\s*\|\|\s*github\.run_id\s*\}\}" ) -CANCEL_SHAPE = re.compile(r"\$\{\{\s*github\.event_name\s*==\s*'(?P[a-z_]+)'\s*\}\}") +CANCEL_SHAPE = re.compile(r"\$\{\{\s*(?P[^{}]+?)\s*\}\}") + + +def normal(expression): + return re.sub(r"\s+", " ", expression).strip() + + +def superseded(event_name, ref): + """SUPERSEDED, evaluated.""" + return event_name == PULL_REQUEST or (event_name == PUSH and ref == "refs/heads/main") def read_concurrency(text): @@ -72,22 +85,25 @@ def other_run(event_name, ref, run_id): MAIN, TAG = ("refs/heads/main", "refs/tags/v1.2.3") SAME_GROUP = [ ("a newer commit on the same pull request", pull_request(1316, 43599251877), pull_request(1316, 43600899878)), - ("a rerun of one main run", other_run(PUSH, MAIN, 7001), other_run(PUSH, MAIN, 7001)), + ("two pushes to main", other_run(PUSH, MAIN, 3), other_run(PUSH, MAIN, 4)), ] DIFFERENT_GROUPS = [ ("two pull requests", pull_request(1316, 1), pull_request(1317, 1)), ("the same branch name in two forks", pull_request(1316, 2), pull_request(1400, 2)), - ("two pushes to main", other_run(PUSH, MAIN, 3), other_run(PUSH, MAIN, 4)), ("two release tag runs", other_run(PUSH, TAG, 5), other_run(PUSH, TAG, 6)), ("two manual runs", other_run(DISPATCH, MAIN, 7), other_run(DISPATCH, MAIN, 8)), + ("two scheduled runs", other_run(SCHEDULE, MAIN, 15), other_run(SCHEDULE, MAIN, 16)), + ("a main push and a scheduled run", other_run(PUSH, MAIN, 17), other_run(SCHEDULE, MAIN, 17)), + ("a main push and a manual run of main", other_run(PUSH, MAIN, 18), other_run(DISPATCH, MAIN, 18)), ("a main push and a pull request", other_run(PUSH, MAIN, 9), pull_request(1316, 9)), ("a release tag and a pull request", other_run(PUSH, TAG, 10), pull_request(1316, 10)), ] CANCELLATION = [ ("a push supersedes the previous run of its pull request", pull_request(1316, 11), True), - ("a push to main is never cancelled", other_run(PUSH, MAIN, 12), False), + ("a push to main supersedes the previous main run", other_run(PUSH, MAIN, 12), True), ("a release tag run is never cancelled", other_run(PUSH, TAG, 13), False), ("a manual run is never cancelled", other_run(DISPATCH, MAIN, 14), False), + ("the scheduled run that saves the caches is never cancelled", other_run(SCHEDULE, MAIN, 19), False), ] @@ -103,17 +119,17 @@ class ConcurrencyTests(unittest.TestCase): if not cls.group_shape or not cls.cancel_shape: raise AssertionError(f"the concurrency fields are not the documented shapes: {cls.fields}") - def cancels(self, event_name): - return event_name == self.cancel_shape["event"] + def cancels(self, event_name, ref): + return superseded(event_name, ref) def group(self, event_name, ref, run_id): - return self.group_shape["prefix"] + "-" + (ref if event_name == self.group_shape["event"] else str(run_id)) + return self.group_shape["prefix"] + "-" + (ref if superseded(event_name, ref) else str(run_id)) def test_policy_is_the_two_documented_fields(self): self.assertEqual(sorted(self.fields), ["cancel-in-progress", "group"]) self.assertEqual(self.group_shape["prefix"], "workspace") - self.assertEqual(self.group_shape["event"], PULL_REQUEST) - self.assertEqual(self.cancel_shape["event"], PULL_REQUEST) + self.assertEqual(normal(self.group_shape["condition"]), SUPERSEDED) + self.assertEqual(normal(self.cancel_shape["condition"]), SUPERSEDED) def test_a_scalar_concurrency_value_is_not_read_as_a_mapping(self): cases = { @@ -133,10 +149,10 @@ class ConcurrencyTests(unittest.TestCase): with self.subTest(description): self.assertNotEqual(self.group(*first), self.group(*second), description) - def test_only_a_pull_request_cancels(self): + def test_only_pull_requests_and_main_pushes_cancel(self): for description, context, expected in CANCELLATION: with self.subTest(description): - self.assertEqual(self.cancels(context[0]), expected, description) + self.assertEqual(self.cancels(context[0], context[1]), expected, description) def test_pyyaml_reads_the_same_fields(self): if yaml is None: From e1d883e444fe23a518feeab9f9c6b8a96774731b Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 17 Nov 2026 14:23:06 -0500 Subject: [PATCH 10/16] psd: share legacy guide record prefix decoding Adapt PhotoCraft parse_guides from 16941f7198, retaining signed-position semantics from 1a54384d80. Allocation-free framing preserves complete-prefix recovery. Co-authored-by: Dylan M. Taylor --- crates/psd/src/guides.rs | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/crates/psd/src/guides.rs b/crates/psd/src/guides.rs index 5f100c480..a9d566e21 100644 --- a/crates/psd/src/guides.rs +++ b/crates/psd/src/guides.rs @@ -5,6 +5,29 @@ //! Unknown direction values retain PhotoCraft's vertical interpretation. Grid cycles, reserved //! bytes and trailing data are not interpreted. Resource 1097 remains opaque except its count. +/// One legacy Grid and Guides record (resource 1032). +/// Position is signed 1/32 pixel; direction 1 is horizontal, all others vertical. +/// Adapted from PhotoCraft's reader by Brandon Thomas and writer by aiongg. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct GuideV1 { + /// Signed position in units of 1/32 pixel. + pub location: i32, + /// Raw direction byte, retained without normalization. + pub direction: u8, +} + +/// Read complete legacy records, preserving the valid prefix on truncation. +/// Matches PhotoCraft recovery: version/grid fields are ignored. Declared counts +/// cannot drive allocation or scanning beyond complete records in the input. +pub fn parse_v1_prefix(data: &[u8]) -> impl Iterator + '_ { + let count = be_u32(data, 12).unwrap_or(0); + data.get(16..) + .unwrap_or_default() + .chunks_exact(5) + .zip(0..count) + .map(|(g, _)| GuideV1 { location: i32::from_be_bytes([g[0], g[1], g[2], g[3]]), direction: g[4] }) +} + /// Grid and Guides version 2 resource ID. pub const GUIDES_V2: u16 = 1092; /// Associated guide identifiers resource ID. From d55beb8f612bcedfe2af1907b8d2bef9fc548187 Mon Sep 17 00:00:00 2001 From: aiongg Date: Tue, 17 Nov 2026 14:23:06 -0500 Subject: [PATCH 11/16] psd: share legacy guide header and record encoding Extract PhotoCraft guide framing from 2aa88b0956; leave rounding and document ordering with callers. Add fixed-byte and hostile-prefix coverage. Co-authored-by: Dylan M. Taylor --- crates/psd/src/guides.rs | 45 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/crates/psd/src/guides.rs b/crates/psd/src/guides.rs index a9d566e21..f122c7805 100644 --- a/crates/psd/src/guides.rs +++ b/crates/psd/src/guides.rs @@ -16,6 +16,23 @@ pub struct GuideV1 { pub direction: u8, } +impl GuideV1 { + /// Encode exactly one five-byte record without allocation. + pub fn to_bytes(self) -> [u8; 5] { + let p = self.location.to_be_bytes(); + [p[0], p[1], p[2], p[3], self.direction] + } +} + +/// Encode a legacy resource header. Grid cycles and count are caller supplied. +pub fn header_v1(horizontal_cycle: u32, vertical_cycle: u32, count: u32) -> [u8; 16] { + let mut out = [0; 16]; + for (chunk, value) in out.chunks_exact_mut(4).zip([1u32, horizontal_cycle, vertical_cycle, count]) { + chunk.copy_from_slice(&value.to_be_bytes()); + } + out +} + /// Read complete legacy records, preserving the valid prefix on truncation. /// Matches PhotoCraft recovery: version/grid fields are ignored. Declared counts /// cannot drive allocation or scanning beyond complete records in the input. @@ -124,3 +141,31 @@ mod tests { assert_eq!(ids_count(&[0, 0, 0]), None); } } + +#[cfg(test)] +mod v1_tests { + use super::*; + + #[test] + fn wire_roundtrip_and_signed_unknown_direction() { + let entries = [GuideV1 { location: -33, direction: 1 }, GuideV1 { location: i32::MAX, direction: 255 }]; + let mut data = header_v1(576, 576, 2).to_vec(); + assert_eq!(data, [0, 0, 0, 1, 0, 0, 2, 64, 0, 0, 2, 64, 0, 0, 0, 2]); + for e in entries { + data.extend(e.to_bytes()); + } + assert_eq!(&data[16..21], &[255, 255, 255, 223, 1]); + assert_eq!(parse_v1_prefix(&data).collect::>(), entries); + for end in 0..=data.len() { + assert_eq!(parse_v1_prefix(&data[..end]).count(), end.saturating_sub(16) / 5); + } + data[..4].copy_from_slice(&99u32.to_be_bytes()); + data[12..16].copy_from_slice(&u32::MAX.to_be_bytes()); + data.extend([9, 9, 9, 9]); + assert_eq!(parse_v1_prefix(&data).collect::>(), entries); + data[12..16].copy_from_slice(&1u32.to_be_bytes()); + assert_eq!(parse_v1_prefix(&data).collect::>(), &entries[..1]); + data[12..16].fill(0); + assert_eq!(parse_v1_prefix(&data).count(), 0); + } +} From 8d5c939e203fbbd7cd7543f0d4e79043b9a500a5 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Tue, 17 Nov 2026 14:46:32 -0500 Subject: [PATCH 12/16] psd: use fixed-size guide chunks for strict clippy --- crates/psd/src/guides.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/psd/src/guides.rs b/crates/psd/src/guides.rs index f122c7805..2b654ec8f 100644 --- a/crates/psd/src/guides.rs +++ b/crates/psd/src/guides.rs @@ -27,7 +27,7 @@ impl GuideV1 { /// Encode a legacy resource header. Grid cycles and count are caller supplied. pub fn header_v1(horizontal_cycle: u32, vertical_cycle: u32, count: u32) -> [u8; 16] { let mut out = [0; 16]; - for (chunk, value) in out.chunks_exact_mut(4).zip([1u32, horizontal_cycle, vertical_cycle, count]) { + for (chunk, value) in out.as_chunks_mut::<4>().0.iter_mut().zip([1u32, horizontal_cycle, vertical_cycle, count]) { chunk.copy_from_slice(&value.to_be_bytes()); } out @@ -40,7 +40,9 @@ pub fn parse_v1_prefix(data: &[u8]) -> impl Iterator + '_ { let count = be_u32(data, 12).unwrap_or(0); data.get(16..) .unwrap_or_default() - .chunks_exact(5) + .as_chunks::<5>() + .0 + .iter() .zip(0..count) .map(|(g, _)| GuideV1 { location: i32::from_be_bytes([g[0], g[1], g[2], g[3]]), direction: g[4] }) } From 1d5255e07f2463ea784860505e1f7ce68d6adffd Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Tue, 17 Nov 2026 15:48:26 -0500 Subject: [PATCH 13/16] vectordrawable: bound configurable XML depth with iterative writing --- crates/vectordrawable/src/write.rs | 135 +++++++++++++++++++++++++---- 1 file changed, 116 insertions(+), 19 deletions(-) diff --git a/crates/vectordrawable/src/write.rs b/crates/vectordrawable/src/write.rs index ec14fca97..5a883df77 100644 --- a/crates/vectordrawable/src/write.rs +++ b/crates/vectordrawable/src/write.rs @@ -13,6 +13,10 @@ use crate::model::{Cap, Color, Fill, GradientKind, Group, Item, ItemKind, Join, /// never recurses without bound. Android Studio's own drawables nest a handful of levels. pub const MAX_DEPTH: usize = 256; +/// Hard ceiling for [`document_with_depth_limit`]. A caller may need extra XML groups for +/// transforms and clips around each bounded application layer, but cannot disable the guard. +pub const MAX_CONFIGURED_DEPTH: usize = 1024; + const ANDROID: &str = "http://schemas.android.com/apk/res/android"; const AAPT: &str = "http://schemas.android.com/aapt"; @@ -110,18 +114,37 @@ impl Out { } /// Whether a path that is written (see [`MAX_DEPTH`]) has a gradient fill. -fn uses_gradient(items: &[Item], depth: usize) -> bool { - items.iter().any(|it| match &it.kind { - ItemKind::Path(p) => matches!(p.fill, Some(Fill::Gradient(_))), - ItemKind::Group(g) => depth < MAX_DEPTH && uses_gradient(&g.children, depth + 1), - }) +fn uses_gradient(items: &[Item], max_depth: usize) -> bool { + let mut pending = vec![(1, items.iter())]; + while let Some((depth, items)) = pending.last_mut() { + let depth = *depth; + match items.next().map(|it| &it.kind) { + Some(ItemKind::Path(p)) if matches!(p.fill, Some(Fill::Gradient(_))) => return true, + Some(ItemKind::Group(g)) if depth < max_depth => pending.push((depth + 1, g.children.iter())), + Some(_) => {} + None => { + pending.pop(); + } + } + } + false } /// The whole file: a `width`×`height` dp drawable with a viewport of the same size. pub fn document(width: f64, height: f64, items: &[Item]) -> String { + document_with_depth_limit(width, height, items, MAX_DEPTH) +} + +/// Write with a caller-selected XML group depth cap, clamped to `1..=MAX_CONFIGURED_DEPTH`. +/// +/// A group at the cap keeps its attributes and clip but not its children. The default +/// [`document`] cap is unchanged. Use this when one bounded source layer projects to several +/// XML wrapper groups; the caller must include those wrappers in its depth calculation. +pub fn document_with_depth_limit(width: f64, height: f64, items: &[Item], max_depth: usize) -> String { + let max_depth = max_depth.clamp(1, MAX_CONFIGURED_DEPTH); let mut o = Out { s: String::from("\n") }; let mut attrs = vec![("xmlns:android", ANDROID.to_string())]; - if uses_gradient(items, 1) { + if uses_gradient(items, max_depth) { attrs.push(("xmlns:aapt", AAPT.to_string())); } attrs.extend([ @@ -135,21 +158,35 @@ pub fn document(width: f64, height: f64, items: &[Item]) -> String { // Android refuses a without a ("no path defined"): one empty, unfilled path. o.open(1, "path", &[("android:pathData", "M0,0".to_string())], true); } - for it in items { - item(&mut o, 1, it); - } + write_items(&mut o, items, max_depth); o.close(0, "vector"); o.s } -fn item(o: &mut Out, depth: usize, it: &Item) { - match &it.kind { - ItemKind::Path(p) => path(o, depth, p), - ItemKind::Group(g) => group(o, depth, g), +fn write_items(o: &mut Out, items: &[Item], max_depth: usize) { + // One iterator per open group, not one pending entry per sibling. The configured depth + // bounds this stack, and wide documents do not require a second list of their items. + let mut pending = vec![(1, items.iter())]; + while let Some((depth, items)) = pending.last_mut() { + let depth = *depth; + match items.next().map(|it| &it.kind) { + Some(ItemKind::Path(p)) => path(o, depth, p), + Some(ItemKind::Group(g)) => { + if group(o, depth, g, max_depth) { + pending.push((depth + 1, g.children.iter())); + } + } + None => { + pending.pop(); + if depth > 1 { + o.close(depth - 1, "group"); + } + } + } } } -fn group(o: &mut Out, depth: usize, g: &Group) { +fn group(o: &mut Out, depth: usize, g: &Group, max_depth: usize) -> bool { let mut attrs = Vec::new(); if let Some(n) = &g.name { attrs.push(("android:name", esc(n))); @@ -172,19 +209,21 @@ fn group(o: &mut Out, depth: usize, g: &Group) { attrs.push(("android:translateY", num(x.translate.1))); } } - let children: &[Item] = if depth < MAX_DEPTH { &g.children } else { &[] }; + let children: &[Item] = if depth < max_depth { &g.children } else { &[] }; if children.is_empty() && g.clip.is_none() { o.open(depth, "group", &attrs, true); - return; + return false; } o.open(depth, "group", &attrs, false); if let Some(c) = &g.clip { o.open(depth + 1, "clip-path", &[("android:pathData", path_data(c))], true); } - for c in children { - item(o, depth + 1, c); + if children.is_empty() { + o.close(depth, "group"); + false + } else { + true } - o.close(depth, "group"); } fn stroke_attrs(attrs: &mut Vec<(&str, String)>, s: &Stroke) { @@ -445,3 +484,61 @@ mod tests { assert!(xml.contains(" Item { + let mut item = Item { + bbox: Rect::ZERO, + kind: ItemKind::Path(Path { + name: Some("deep".into()), + data: BezPath::new(), + stroke: None, + even_odd: false, + fill: Some(Fill::Gradient(Gradient { kind: GradientKind::Sweep { center: (0.0, 0.0) }, stops: vec![] })), + }), + }; + for _ in 0..groups { + item = Item { bbox: Rect::ZERO, kind: ItemKind::Group(Group { name: None, transform: None, clip: None, children: vec![item] }) }; + } + item + } + + #[test] + fn expanded_source_layers_keep_their_leaf_and_namespace() { + let items = [nested(514)]; + assert!(!document(1.0, 1.0, &items).contains("aapt")); + let xml = document_with_depth_limit(1.0, 1.0, &items, 516); + assert_eq!(xml.matches(" Date: Tue, 17 Nov 2026 16:00:45 -0500 Subject: [PATCH 14/16] jpeg: add tokenless two-pass baseline encoding --- crates/jpeg/src/encode.rs | 127 ++++++++++++++--- crates/jpeg/src/encode_low_memory_tests.rs | 155 +++++++++++++++++++++ crates/jpeg/src/lib.rs | 2 +- 3 files changed, 261 insertions(+), 23 deletions(-) create mode 100644 crates/jpeg/src/encode_low_memory_tests.rs diff --git a/crates/jpeg/src/encode.rs b/crates/jpeg/src/encode.rs index 548f4c9f9..fb5fe183c 100644 --- a/crates/jpeg/src/encode.rs +++ b/crates/jpeg/src/encode.rs @@ -419,6 +419,34 @@ pub fn encode_baseline( sub: ChromaSubsampling, app_segments: &[(u8, Vec)], ) -> Result, EncodeError> { + validate_encode(data, width, height, channels, app_segments)?; + Ok(encode(data, width, height, channels, quality, sub, app_segments)) +} + +/// Encode the same optimized baseline JPEG as [`encode_baseline`], trading a second DCT pass +/// for working memory independent of image height. This path is sequential even when the +/// `parallel` feature is enabled. It retains only one MCU row of conversion planes and writes +/// directly into the returned encoded buffer: no image-wide tokens or duplicate coded buffer. +/// +/// Scratch consists of at most `24 * width + 8` floats (odd-width 4:2:0), plus fixed-size coding tables. +/// The caller still owns the input pixels and the returned JPEG; this is not a cap on output +/// size or a replacement for the caller's image/admission budget. Geometry, quality, metadata, +/// Huffman tables and restart boundaries match [`encode_baseline`] byte for byte. +#[allow(clippy::too_many_arguments)] +pub fn encode_baseline_low_memory( + data: &[u8], + width: usize, + height: usize, + channels: usize, + quality: u8, + sub: ChromaSubsampling, + app_segments: &[(u8, Vec)], +) -> Result, EncodeError> { + validate_encode(data, width, height, channels, app_segments)?; + Ok(encode_impl(data, width, height, channels, quality, sub, app_segments, true, true)) +} + +fn validate_encode(data: &[u8], width: usize, height: usize, channels: usize, app_segments: &[(u8, Vec)]) -> Result<(), EncodeError> { let fits = width.checked_mul(height).and_then(|n| n.checked_mul(channels)).is_some_and(|n| n <= data.len()); if !fits || width == 0 || height == 0 || width > 65_535 || height > 65_535 || !matches!(channels, 1 | 3 | 4) { return Err(EncodeError::InvalidInput(format!("{width}x{height}, {channels} channels, {} bytes", data.len()))); @@ -426,7 +454,7 @@ pub fn encode_baseline( if app_segments.iter().any(|(marker, payload)| !(0xE0..=0xEF).contains(marker) || payload.len() > 65_533) { return Err(EncodeError::InvalidInput("APP markers must be 0xE0..=0xEF with payloads no larger than 65533 bytes".into())); } - Ok(encode(data, width, height, channels, quality, sub, app_segments)) + Ok(()) } /// [`encode`], with optimized Huffman tables or (`optimize` false) the Annex K example tables. @@ -440,6 +468,21 @@ fn encode_with( sub: ChromaSubsampling, app_segments: &[(u8, Vec)], optimize: bool, +) -> Vec { + encode_impl(data, width, height, channels, quality, sub, app_segments, optimize, false) +} + +#[allow(clippy::too_many_arguments)] +fn encode_impl( + data: &[u8], + width: usize, + height: usize, + channels: usize, + quality: u8, + sub: ChromaSubsampling, + app_segments: &[(u8, Vec)], + optimize: bool, + low_memory: bool, ) -> Vec { // The public entry takes caller-sized dimensions: refuse (an empty output) what would size an allocation from // them or index past `data`, rather than panic or allocate for them. @@ -509,6 +552,8 @@ fn encode_with( planes.push(Plane { w: cw, h: crows.len(), data: cb }); planes.push(Plane { w: cw, h: crows.len(), data: cr }); } + #[cfg(test)] + low_memory_tests::record_planes(planes.iter().map(|p| p.data.len()).sum()); planes }; @@ -519,45 +564,58 @@ fn encode_with( let (mx, my) = (width.div_ceil(mcu_w), height.div_ceil(mcu_h)); // restart interval: whole MCU rows per band, ≤ 65535 MCUs, enough bands to keep all cores busy let rows_per_band = (my / 64).clamp(1, (65_535 / mx.max(1)).max(1)); - let bands: Vec = (0..my.div_ceil(rows_per_band)).collect(); + let num_bands = my.div_ceil(rows_per_band); + let bands: Vec = if low_memory { Vec::new() } else { (0..num_bands).collect() }; // first pass: each band's quantized blocks as symbols, with their counts per table (DC // prediction restarts per band, as the restart markers say) - let tokenize_band = |band: &usize| -> (Vec, [Freq; 4]) { + let visit_band = |band: &usize, emit: &mut dyn FnMut(&[i16; 64], &mut i32, bool)| { let (mut blk, mut q) = ([0f32; 64], [0i16; 64]); - let (mut tokens, mut freq) = (Vec::new(), [[0u64; 256]; 4]); let mut pred = [0i32; 3]; let (m0, m1) = (band * rows_per_band, ((band + 1) * rows_per_band).min(my)); - // the band's planes, its MCU row `m0` at their top (blocks past the image's bottom repeat its last row, - // which is the band's last row then) - let planes = convert(m0 * mcu_h..(m1 * mcu_h).min(height), m0 * 8..(m1 * 8).min(ch)); for mcu_y in m0..m1 { - let mcu_y = mcu_y - m0; + // Convert only the current MCU row. DC predictors still span the original band, + // so its symbols/restart decisions are identical to whole-band conversion. + let planes = convert(mcu_y * mcu_h..((mcu_y + 1) * mcu_h).min(height), mcu_y * 8..((mcu_y + 1) * 8).min(ch)); for mcu_x in 0..mx { for vy in 0..vs { for hx in 0..hs { - planes[0].block(mcu_x * hs + hx, mcu_y * vs + vy, &mut blk); + planes[0].block(mcu_x * hs + hx, vy, &mut blk); fdct_quant(&blk, &rl, &mut q); - tokenize_block(&q, &mut pred[0], false, &mut tokens, &mut freq); + emit(&q, &mut pred[0], false); } } if !gray { for (plane, pred) in planes.iter().zip(pred.iter_mut()).take(3).skip(1) { - plane.block(mcu_x, mcu_y, &mut blk); + plane.block(mcu_x, 0, &mut blk); fdct_quant(&blk, &rc, &mut q); - tokenize_block(&q, pred, true, &mut tokens, &mut freq); + emit(&q, pred, true); } } } } + }; + let tokenize_band = |band: &usize| -> (Vec, [Freq; 4]) { + let (mut tokens, mut freq) = (Vec::new(), [[0u64; 256]; 4]); + visit_band(band, &mut |q, pred, chroma| tokenize_block(q, pred, chroma, &mut tokens, &mut freq)); + #[cfg(test)] + low_memory_tests::record_tokens(tokens.len()); (tokens, freq) }; #[cfg(feature = "parallel")] - let tokenized: Vec<(Vec, [Freq; 4])> = bands.par_iter().map(tokenize_band).collect(); + let tokenized: Vec<(Vec, [Freq; 4])> = if low_memory { Vec::new() } else { bands.par_iter().map(tokenize_band).collect() }; #[cfg(not(feature = "parallel"))] - let tokenized: Vec<(Vec, [Freq; 4])> = bands.iter().map(tokenize_band).collect(); + let tokenized: Vec<(Vec, [Freq; 4])> = if low_memory { Vec::new() } else { bands.iter().map(tokenize_band).collect() }; // the tables: DC luma, AC luma, DC chroma, AC chroma let tables: [([u8; 16], Vec); 4] = if optimize { let mut total = [[0u64; 256]; 4]; + if low_memory { + for band in 0..num_bands { + visit_band(&band, &mut |q, pred, chroma| { + let base = if chroma { 2 } else { 0 }; + block_symbols(q, pred, |ac, symbol, _, _| total[base + ac as usize][symbol & 0xFF] += 1); + }); + } + } for (_, f) in &tokenized { for (t, f) in total.iter_mut().zip(f) { for (t, f) in t.iter_mut().zip(f) { @@ -577,9 +635,9 @@ fn encode_with( let huff = tables.each_ref().map(|(bits, vals)| Huff::new(bits, vals)); // second pass: code the symbols #[cfg(feature = "parallel")] - let coded: Vec> = tokenized.into_par_iter().map(|(t, _)| write_tokens(&t, &huff)).collect(); + let coded: Vec> = if low_memory { Vec::new() } else { tokenized.into_par_iter().map(|(t, _)| write_tokens(&t, &huff)).collect() }; #[cfg(not(feature = "parallel"))] - let coded: Vec> = tokenized.into_iter().map(|(t, _)| write_tokens(&t, &huff)).collect(); + let coded: Vec> = if low_memory { Vec::new() } else { tokenized.into_iter().map(|(t, _)| write_tokens(&t, &huff)).collect() }; // headers let mut out = Vec::with_capacity(coded.iter().map(Vec::len).sum::() + 4096); @@ -617,7 +675,7 @@ fn encode_with( dht(&mut out, 0x11, acb, acv); } let interval = (rows_per_band * mx) as u16; - if coded.len() > 1 { + if num_bands > 1 { segment(&mut out, 0xDD, &interval.to_be_bytes()); } if gray { @@ -625,11 +683,32 @@ fn encode_with( } else { segment(&mut out, 0xDA, &[3, 1, 0x00, 2, 0x11, 3, 0x11, 0, 63, 0]); } - let last = coded.len().saturating_sub(1); - for (i, c) in coded.iter().enumerate() { - out.extend_from_slice(c); - if i < last { - out.extend_from_slice(&[0xFF, 0xD0 + (i % 8) as u8]); + if low_memory { + let last = num_bands.saturating_sub(1); + for i in 0..num_bands { + // Continue directly in the final buffer, resetting entropy bits at the same band + // boundaries as the retained-token path. Only the output grows with image height. + let mut bits = Bits { out, acc: 0, n: 0 }; + visit_band(&i, &mut |q, pred, chroma| { + let base = if chroma { 2 } else { 0 }; + block_symbols(q, pred, |ac, symbol, extra, n| { + let (code, len) = huff[base + ac as usize].code[symbol & 0xFF]; + bits.put(code as u32, len as u32); + bits.put(extra & 0xFFFF, n & 0xF); + }); + }); + out = bits.finish(); + if i < last { + out.extend_from_slice(&[0xFF, 0xD0 + (i % 8) as u8]); + } + } + } else { + let last = coded.len().saturating_sub(1); + for (i, c) in coded.iter().enumerate() { + out.extend_from_slice(c); + if i < last { + out.extend_from_slice(&[0xFF, 0xD0 + (i % 8) as u8]); + } } } out.extend_from_slice(&[0xFF, 0xD9]); @@ -877,3 +956,7 @@ mod tests { assert!(psnr(&src, &px) > 35.0); } } + +#[cfg(test)] +#[path = "encode_low_memory_tests.rs"] +mod low_memory_tests; diff --git a/crates/jpeg/src/encode_low_memory_tests.rs b/crates/jpeg/src/encode_low_memory_tests.rs new file mode 100644 index 000000000..de45f8dfe --- /dev/null +++ b/crates/jpeg/src/encode_low_memory_tests.rs @@ -0,0 +1,155 @@ +use super::*; +use std::cell::Cell; + +thread_local! { + static STATS: Cell<(usize, usize)> = const { Cell::new((0, 0)) }; +} + +pub(super) fn record_planes(samples: usize) { + STATS.with(|s| { + let (peak, tokens) = s.get(); + s.set((peak.max(samples), tokens)); + }); +} +pub(super) fn record_tokens(count: usize) { + STATS.with(|s| { + let (peak, tokens) = s.get(); + s.set((peak, tokens + count)); + }); +} + +fn noise(len: usize) -> Vec { + let mut seed = 0x89ab_cdefu32; + (0..len) + .map(|_| { + seed ^= seed << 13; + seed ^= seed >> 17; + seed ^= seed << 5; + seed as u8 + }) + .collect() +} + +fn compare(src: &[u8], w: usize, h: usize, ch: usize, q: u8, sub: ChromaSubsampling, app: &[(u8, Vec)]) { + let expected = encode_baseline(src, w, h, ch, q, sub, app).unwrap(); + STATS.with(|s| s.set((0, 0))); + let actual = encode_baseline_low_memory(src, w, h, ch, q, sub, app).unwrap(); + assert_eq!(actual, expected, "{w}x{h} channels={ch} quality={q} subsampling={sub:?}"); + let (peak, tokens) = STATS.with(Cell::get); + assert!(peak <= 24 * w + 8, "MCU-row scratch exceeded: {peak}"); + assert_eq!(tokens, 0, "low-memory mode retained entropy tokens"); +} + +#[test] +fn all_sampling_and_channels_match_retained_tokens_byte_for_byte() { + for (w, h) in [(1, 1), (7, 9), (17, 31), (33, 65), (17, 1025)] { + for ch in [1, 3, 4] { + let src = noise(w * h * ch); + for sub in [ChromaSubsampling::S444, ChromaSubsampling::S422, ChromaSubsampling::S420] { + for q in [0, 1, 25, 50, 90, 100, 255] { + compare(&src, w, h, ch, q, sub, &[]); + } + } + } + } +} + +#[test] +fn high_entropy_hd_and_wide_frames_keep_only_one_mcu_row() { + for (w, h) in [(1920, 1080), (65_535, 1)] { + let src = noise(w * h * 3); + compare(&src, w, h, 3, 100, ChromaSubsampling::S444, &[]); + } +} + +#[test] +fn metadata_empty_tables_and_extra_source_bytes_keep_exact_output() { + let app = [(0xe0, b"custom density".to_vec()), (0xe2, vec![73; 65_533])]; + for ch in [1, 3, 4] { + let src = vec![127; 19 * 23 * ch + 5]; + compare(&src, 19, 23, ch, 75, ChromaSubsampling::S420, &app); + } +} + +#[test] +fn checked_admission_matches_original_without_panics() { + for (w, h, ch) in [(0, 1, 3), (1, 0, 3), (1, 1, 0), (1, 1, 2), (usize::MAX, 2, 4), (65_536, 1, 1), (1, 65_536, 1), (4, 4, 3)] { + let src = vec![0; if w == 65_536 || h == 65_536 { 65_536 } else { 8 }]; + let old = encode_baseline(&src, w, h, ch, 90, ChromaSubsampling::S444, &[]); + let new = encode_baseline_low_memory(&src, w, h, ch, 90, ChromaSubsampling::S444, &[]); + assert!(new.is_err()); + assert_eq!(new, old); + } + for app in [vec![(0xdf, vec![])], vec![(0xf0, vec![])], vec![(0xe1, vec![0; 65_534])]] { + assert_eq!( + encode_baseline_low_memory(&[0; 3], 1, 1, 3, 90, ChromaSubsampling::S444, &app), + encode_baseline(&[0; 3], 1, 1, 3, 90, ChromaSubsampling::S444, &app) + ); + } +} + +// Generated with the immutable pre-change encoder at craft-libs +// 937ac93c5cf3cd4b6a81e195b078fe161eb081b9, not the refactored token-retaining path. +// Odd chroma edges and 1025/2049-row inputs exercise multi-row restart bands. +#[test] +fn prechange_encoded_bytes_remain_pinned_across_band_boundaries() { + let cases = [ + (1, 1, 1, 1, 444, 159, 0xe3bd59145381ba57), + (1, 1, 1, 75, 422, 159, 0xcb2b103914bbdbd3), + (1, 1, 1, 100, 420, 160, 0xd9f640613efb6abf), + (1, 1, 3, 1, 444, 284, 0x61d2772b8fd9df17), + (1, 1, 3, 75, 422, 286, 0xa0e03ab7efa23296), + (1, 1, 3, 100, 420, 288, 0x5660667f1abd0a63), + (1, 1, 4, 1, 444, 284, 0x61d2772b8fd9df17), + (1, 1, 4, 75, 422, 286, 0xa0e03ab7efa23296), + (1, 1, 4, 100, 420, 288, 0x5660667f1abd0a63), + (17, 31, 1, 1, 444, 240, 0xf8d277e9499253d4), + (17, 31, 1, 75, 422, 515, 0xa6946d4f144bf243), + (17, 31, 1, 100, 420, 793, 0xbb3db259708d8951), + (17, 31, 3, 1, 444, 363, 0xff2b5b41ffb3f81b), + (17, 31, 3, 75, 422, 877, 0x0374ffb527378dfe), + (17, 31, 3, 100, 420, 1220, 0x2feeedfa90885737), + (17, 31, 4, 1, 444, 366, 0xde7f8d1330e436eb), + (17, 31, 4, 75, 422, 879, 0xb1559fbfc3ee0cf6), + (17, 31, 4, 100, 420, 1214, 0xaf5dcf12d28c8e88), + (33, 65, 1, 1, 444, 388, 0xbee80491e42c8fd2), + (33, 65, 1, 75, 422, 1470, 0x284f4eb78b785ca5), + (33, 65, 1, 100, 420, 2576, 0xa81cc94015f326dc), + (33, 65, 3, 1, 444, 526, 0xf75b4036acc30874), + (33, 65, 3, 75, 422, 2322, 0x6fbc56bd9a1eec70), + (33, 65, 3, 100, 420, 3796, 0xcad25051562ac816), + (33, 65, 4, 1, 444, 527, 0x4bfedc11153de4c1), + (33, 65, 4, 75, 422, 2309, 0x47bf11bf32bac6a4), + (33, 65, 4, 100, 420, 3775, 0x64c2303fde273cd1), + (17, 1025, 1, 1, 444, 2017, 0x242f4a95d8579612), + (17, 1025, 1, 75, 422, 10728, 0xfe49408f379bdffb), + (17, 1025, 1, 100, 420, 19686, 0x1355b995e38d6467), + (17, 1025, 3, 1, 444, 2137, 0x4e319440cb6946c2), + (17, 1025, 3, 75, 422, 17682, 0xf456c63b731dde8f), + (17, 1025, 3, 100, 420, 29015, 0x0631001154b128cf), + (17, 1025, 4, 1, 444, 2155, 0xa2de9c1d77e62aa6), + (17, 1025, 4, 75, 422, 17648, 0x7d85fbdf56061f0c), + (17, 1025, 4, 100, 420, 29010, 0x4848eeb93daf1df2), + (19, 2049, 1, 1, 444, 3531, 0x9b93cf219ff85796), + (19, 2049, 1, 75, 422, 26283, 0xb160523444397608), + (19, 2049, 1, 100, 420, 51883, 0xc5c26836c0a976c6), + (19, 2049, 3, 1, 444, 3303, 0x26eaac2272e1fdb8), + (19, 2049, 3, 75, 422, 42745, 0x38b9843fb54db698), + (19, 2049, 3, 100, 420, 79868, 0xd6a7917af19682f7), + (19, 2049, 4, 1, 444, 3343, 0x6bf577903528692f), + (19, 2049, 4, 75, 422, 42706, 0x0e62366274593a93), + (19, 2049, 4, 100, 420, 79804, 0x867d05cf9d944599), + ]; + for (w, h, ch, q, sampling, len, hash) in cases { + let sub = match sampling { + 444 => ChromaSubsampling::S444, + 422 => ChromaSubsampling::S422, + _ => ChromaSubsampling::S420, + }; + let src = noise(w * h * ch); + for actual in [encode_baseline(&src, w, h, ch, q, sub, &[]).unwrap(), encode_baseline_low_memory(&src, w, h, ch, q, sub, &[]).unwrap()] { + let actual_hash = actual.iter().fold(0xcbf29ce484222325u64, |h, b| (h ^ u64::from(*b)).wrapping_mul(0x100000001b3)); + assert_eq!((actual.len(), actual_hash), (len, hash), "{w}x{h} channels={ch} quality={q} sampling={sampling}"); + } + } +} diff --git a/crates/jpeg/src/lib.rs b/crates/jpeg/src/lib.rs index 23434fdd0..ed2ee1988 100644 --- a/crates/jpeg/src/lib.rs +++ b/crates/jpeg/src/lib.rs @@ -1705,7 +1705,7 @@ mod tests; /// Baseline JPEG encoder (8-bit, Huffman-coded sequential DCT). mod encode; -pub use encode::{ChromaSubsampling, EncodeError, encode_baseline}; +pub use encode::{ChromaSubsampling, EncodeError, encode_baseline, encode_baseline_low_memory}; mod app14; pub use app14::adobe_transform; From 019d54bfffb429b3f10778a8bf7a318a9843455d Mon Sep 17 00:00:00 2001 From: aiongg Date: Tue, 17 Nov 2026 22:07:49 +0000 Subject: [PATCH 15/16] ci(jpeg): the uncached wasm check gets a 20-minute limit --- .github/workflows/jpeg.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/jpeg.yml b/.github/workflows/jpeg.yml index c49c03581..495af8b23 100644 --- a/.github/workflows/jpeg.yml +++ b/.github/workflows/jpeg.yml @@ -19,7 +19,8 @@ concurrency: jobs: check: - timeout-minutes: 10 + # Uncached: the wasm check builds its dependencies on every run (about 10 minutes with setup). + timeout-minutes: 20 # fmt, clippy and the tests run in the Workspace gate on every OS. name: JPEG WASM runs-on: ubuntu-latest From e5f00f5bf658ef2cce7bf96fbbd769a6fff5bc72 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Tue, 17 Nov 2026 17:47:23 -0500 Subject: [PATCH 16/16] Extract bounded XFDF field-value reader (#2056) * Extract bounded XFDF field-value reader * Record retained XFDF source contributor provenance --------- Co-authored-by: Brandon Thomas --- ATTRIBUTION.md | 6 ++ Cargo.lock | 8 ++ Cargo.toml | 1 + crates/xfdf/Cargo.toml | 13 +++ crates/xfdf/README.md | 11 +++ crates/xfdf/src/lib.rs | 177 +++++++++++++++++++++++++++++++++++++++++ 6 files changed, 216 insertions(+) create mode 100644 crates/xfdf/Cargo.toml create mode 100644 crates/xfdf/README.md create mode 100644 crates/xfdf/src/lib.rs diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index ae72ae851..232635e98 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -157,3 +157,9 @@ Sanitization, animation fitting and numeric output policy remain caller-owned. ## HTTP Archive reader `crates/har` preserves Craft Anything `crates/formats/src/har.rs` from `5979f9b13f16dcf1df3c7ae7f8e7735f452866f2` (MIT OR Apache-2.0). Brandon Thomas introduced the typed reader in `f64ae3bf3c8863a982af1304e06de20653803968`; BFlat added UTF-8 BOM compatibility in `cb0bb9d8fa4536ba44ef2c86f5ed759b879cc2d3`. Both source snapshots retain primary author attribution. Shared wiring, explicit byte admission and its tests are subsequent Dylan M. Taylor changes. Redaction and analysis remain app-owned. + +## XFDF field reader + +`crates/xfdf` extracts PDFCraft field reading from #5937 refresh `cb564fd827f572818e9ddb0ea5989a432b1f39fd` (MIT OR Apache-2.0). App-side annotation and form policy remains local; source history is retained in the original repository. + +Retained XFDF source contributors (move-aware blame): Brandon Thomas, Marco Arnulfo, pkibiego and storybored379; module split by aiongg. Subsequent budget/integration work by Dylan M. Taylor. diff --git a/Cargo.lock b/Cargo.lock index cc0965d17..d7561e563 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3458,6 +3458,14 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "craft-xfdf" +version = "0.1.0" +dependencies = [ + "craft-xml", + "thiserror 2.0.21", +] + [[package]] name = "craft-xfl" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index fd9ccf7a6..159c22ad4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,7 @@ craft-java-class = { path = "crates/java-class" } craft-sourcemap = { path = "crates/sourcemap" } craft-pcap = { path = "crates/pcap" } craft-har = { path = "crates/har" } +craft-xfdf = { path = "crates/xfdf" } craft-jsonc = { path = "crates/jsonc" } craft-palette = { path = "crates/palette" } craft-score = { path = "crates/score" } diff --git a/crates/xfdf/Cargo.toml b/crates/xfdf/Cargo.toml new file mode 100644 index 000000000..9b1304d89 --- /dev/null +++ b/crates/xfdf/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "craft-xfdf" +description = "Bounded XFDF field-value reader without PDF or form policy" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true +[lints] +workspace = true +[dependencies] +craft-xml = { workspace = true } +thiserror = { workspace = true } diff --git a/crates/xfdf/README.md b/crates/xfdf/README.md new file mode 100644 index 000000000..ce8c2802e --- /dev/null +++ b/crates/xfdf/README.md @@ -0,0 +1,11 @@ +# craft-xfdf + +Scoped XFDF field-value reader extracted from PDFCraft #5937 refresh cb564fd82. Does not read FDF/PDF, parse annotations or apply field values. PDF application, permissions, calculations and password filtering stay caller-owned. + +Preserves local-name matching, duplicate field order, dotted names, empty leaf clearing and ignored rich-text-only fields, direct split text concatenation and source UTF-8 replacement policy. XML depth1024, field depth64,2million nodes and64MiB charged expanded names/text. Names and values are charged before copying; temporary parents and emitted name copies both count. XML input/tree memory remains governed by craft-xml defaults, not this output budget alone. + +Current strict craft-xml also rejects duplicate raw/expanded attributes and more than1024 active inherited namespace declarations; older app XML accepted those. These deliberate tighter admissions must be disclosed by consumers. No complete XFDF or corpus parity claim. + +Source history remains in PDFCraft. Shared move and preallocation budget reconciliation by Dylan M. Taylor, MIT OR Apache-2.0. No assets are imported. + +Retained source contributors (move-aware blame): Brandon Thomas, Marco Arnulfo, pkibiego and storybored379; module split by aiongg. Subsequent budget/integration work by Dylan M. Taylor. diff --git a/crates/xfdf/src/lib.rs b/crates/xfdf/src/lib.rs new file mode 100644 index 000000000..002376106 --- /dev/null +++ b/crates/xfdf/src/lib.rs @@ -0,0 +1,177 @@ +//! Bounded XFDF field values; PDF annotations and form application remain caller-owned. +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +#![forbid(unsafe_code)] +const MAX_NODES: u32 = 2_000_000; +const MAX_DEPTH: usize = 64; +const MAX_XML_DEPTH: usize = 1024; +const MAX_XFDF_TEXT: usize = 64 << 20; +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +pub enum Error { + #[error("this file isn't XFDF, FDF, XML or text form data")] + UnknownFormat, + #[error("the file can't be read: {0}")] + Malformed(String), +} +/// Read field names and values in file order. This does not apply values to a PDF. +/// UTF-8 replacement and leading BOM/space handling retain the source app policy. +pub fn read_fields(bytes: &[u8]) -> Result)>, Error> { + let text = String::from_utf8_lossy(bytes); + let text = text.trim_start_matches('\u{feff}').trim_start(); + check_form_xml_depth(text)?; + let root = parse_form_xml(text).map_err(|_| Error::UnknownFormat)?; + if root.local() != "xfdf" { + return Err(Error::UnknownFormat); + } + let mut values = Vec::new(); + if let Some(fields) = root.child(craft_xml::Local("fields")) { + walk_xfdf(fields, "", &mut values, 0, &mut 0)?; + } + Ok(values) +} +fn xml_nesting_depth(text: &str) -> usize { + fn find(hay: &[u8], from: usize, needle: &[u8]) -> Option { + (from..hay.len()).find(|&p| hay[p..].starts_with(needle)) + } + let b = text.as_bytes(); + let (mut i, mut depth, mut max) = (0usize, 0usize, 0usize); + while i < b.len() { + if b[i] != b'<' { + i += 1; + } else if b[i..].starts_with(b"").map_or(b.len(), |p| p + 3); + } else if b[i..].starts_with(b"").map_or(b.len(), |p| p + 3); + } else if b[i..].starts_with(b"").map_or(b.len(), |p| p + 2); + } else if b[i..].starts_with(b"").map_or(b.len(), |p| p + 1); + } else if b[i..].starts_with(b"").map_or(b.len(), |p| p + 1); + } else { + depth += 1; + max = max.max(depth); + // Skip the rest of the start tag, over any quoted attribute value, and remember its last + // significant byte: a tag that ends in `/` closes itself, so it is one level deep while it + // is read rather than a level left open. + i += 1; + let mut quote = 0u8; + let mut last = 0u8; + while i < b.len() { + let c = b[i]; + if quote != 0 { + if c == quote { + quote = 0; + } + } else if c == b'"' || c == b'\'' { + quote = c; + } else if c == b'>' { + i += 1; + break; + } else if !c.is_ascii_whitespace() { + last = c; + } + i += 1; + } + if last == b'/' { + depth -= 1; + } + } + } + max +} + +fn check_form_xml_depth(text: &str) -> Result<(), Error> { + let depth = xml_nesting_depth(text); + if depth > MAX_XML_DEPTH { + return Err(Error::Malformed(format!("the file nests XML elements {depth} deep, past the {MAX_XML_DEPTH} the reader accepts"))); + } + Ok(()) +} + +fn parse_form_xml(text: &str) -> Result { + let mut opts = craft_xml::Options { strict: true, doctype: craft_xml::Doctype::Reject, ..Default::default() }; + opts.limits.max_nodes = MAX_NODES as usize; + opts.limits.max_depth = MAX_XML_DEPTH.max(1024); + opts.parse(text.as_bytes()) +} + +fn el_text(el: &craft_xml::El) -> String { + el.children + .iter() + .filter_map(|n| match n { + craft_xml::Node::Text(t) => Some(t.as_str()), + _ => None, + }) + .collect() +} + +fn walk_xfdf(node: &craft_xml::El, prefix: &str, out: &mut Vec<(String, Vec)>, depth: usize, bytes: &mut usize) -> Result<(), Error> { + if depth > MAX_DEPTH { + return Err(Error::Malformed(format!("XFDF fields nest more than {MAX_DEPTH} deep"))); + } + for f in node.children(craft_xml::Local("field")) { + let Some(name) = f.attr("name") else { continue }; + // Charge temporary parent names too, before allocation/recursion can amplify them. + let full_len = if prefix.is_empty() { name.len() } else { prefix.len().saturating_add(1).saturating_add(name.len()) }; + charge_xfdf_text(bytes, full_len)?; + let full = if prefix.is_empty() { name.to_string() } else { format!("{prefix}.{name}") }; + let mut vals = Vec::new(); + for value in f.children(craft_xml::Local("value")) { + let len = value + .children + .iter() + .filter_map(|n| match n { + craft_xml::Node::Text(t) => Some(t.len()), + _ => None, + }) + .fold(0, usize::saturating_add); + charge_xfdf_text(bytes, len)?; + vals.push(el_text(value)); + } + // A leaf with no `` is an empty field (the export writes one that way), so it + // clears the field like FDF's `/V ()`; a field with kids only names their parent. A + // leaf holding only `` is not read yet, so it leaves the field alone. + let names_kids_or_rich = f.elements().any(|c| matches!(c.local(), "field" | "value-richtext")); + if !vals.is_empty() || !names_kids_or_rich { + charge_xfdf_text(bytes, full.len())?; // The emitted name is a second owned copy. + out.push((full.clone(), vals)); + } + walk_xfdf(f, &full, out, depth + 1, bytes)?; + } + Ok(()) +} + +fn charge_xfdf_text(bytes: &mut usize, amount: usize) -> Result<(), Error> { + *bytes = bytes.saturating_add(amount); + if *bytes > MAX_XFDF_TEXT { + return Err(Error::Malformed(format!("the file expands to more than {} MiB of field names and values", MAX_XFDF_TEXT >> 20))); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn field_order_empty_rich_and_split_values() { + let xml = b"abcskiplast"; + assert_eq!( + read_fields(xml).unwrap(), + vec![("parent.child".into(), vec!["ab".into(), "c".into()]), ("empty".into(), vec![]), ("parent.child".into(), vec!["last".into()])] + ); + } + #[test] + fn bounds_and_strict_xml_hardening() { + assert_eq!(read_fields(b""), Err(Error::UnknownFormat)); + assert!(read_fields(b"").is_err()); + assert!(read_fields(b"").is_err()); + let text = format!("{}{}{}", "".repeat(65), "x", "".repeat(65)); + assert!(read_fields(text.as_bytes()).is_err()); + let fields = parse_form_xml("").unwrap(); + let mut out = Vec::new(); + assert!(walk_xfdf(&fields, "", &mut out, 0, &mut (MAX_XFDF_TEXT - 5)).is_err()); + assert!(out.is_empty()); + walk_xfdf(&fields, "", &mut out, 0, &mut (MAX_XFDF_TEXT - 6)).unwrap(); + } +}