From 6589b3c83e69cd60901065de24af3655f270cad3 Mon Sep 17 00:00:00 2001 From: FredApps Date: Tue, 17 Nov 2026 23:51:18 +0100 Subject: [PATCH 1/4] ci: add opt-in uncapped three-platform full validation --- .github/workflows/full-validation.yml | 64 ++++++++++++++++++++++ docs/full-validation.md | 28 ++++++++++ scripts/ci-full-validation.sh | 13 +++++ scripts/test-full-validation.py | 76 +++++++++++++++++++++++++++ 4 files changed, 181 insertions(+) create mode 100644 .github/workflows/full-validation.yml create mode 100644 docs/full-validation.md create mode 100644 scripts/ci-full-validation.sh create mode 100644 scripts/test-full-validation.py diff --git a/.github/workflows/full-validation.yml b/.github/workflows/full-validation.yml new file mode 100644 index 000000000..85ad96f2f --- /dev/null +++ b/.github/workflows/full-validation.yml @@ -0,0 +1,64 @@ +name: Full validation + +# Add full-validation to a PR for uncapped validation of its merge ref. Keep the label across pushes. +# Fast Workspace checks are not substitutes for these six completed native/PDF gate checks. +on: + pull_request: + types: [opened, synchronize, reopened, labeled] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + # An unrelated label event must not cancel an active full run. + group: full-validation-${{ github.event.action == 'labeled' && github.event.label.name != 'full-validation' && github.run_id || github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + native: + if: >- + github.event_name == 'workflow_dispatch' || + (contains(github.event.pull_request.labels.*.name, 'full-validation') && + (github.event.action != 'labeled' || github.event.label.name == 'full-validation')) + name: Full native + uses: storytold/craft-common-actions/.github/workflows/gate.yml@main + with: + os: '["ubuntu-latest","windows-latest","macos-latest"]' + select-os: '[]' + timeout-minutes: 120 + tools: "" + packages: mesa-vulkan-drivers libvulkan1 + craft-fonts-ref: b076417b5880bd4154bd22f00695b1d28c6bea83 + linux-env: WGPU_BACKEND=vulkan + windows-env: CRAFT_PRINT_REQUIRE_QUEUE=1 + env: CRAFT_PRINT_REQUIRE_QUEUE=0 + setup-command: bash scripts/ci-test-inputs.sh + command: | + python scripts/test-full-validation.py + bash scripts/ci-full-validation.sh + + pdf-contracts: + if: >- + github.event_name == 'workflow_dispatch' || + (contains(github.event.pull_request.labels.*.name, 'full-validation') && + (github.event.action != 'labeled' || github.event.label.name == 'full-validation')) + name: Full PDF contracts + uses: storytold/craft-common-actions/.github/workflows/gate.yml@main + with: + os: '["ubuntu-latest","windows-latest","macos-latest"]' + select-os: '[]' + timeout-minutes: 120 + tools: "" + craft-fonts-ref: "" + require-craft-fonts: false + env: RUSTFLAGS=--cfg craft_pdf_unicode_fixtures + setup-command: | + python scripts/test-fetch-pdf-test-fonts.py + python scripts/fetch-pdf-test-fonts.py "$RUNNER_TEMP/craft-pdf-test-fonts" + echo "CRAFT_PDF_TEST_FONT_DIR=$RUNNER_TEMP/craft-pdf-test-fonts" >> "$GITHUB_ENV" + command: | + unset CRAFT_HOSTILE_DIVISOR PROPTEST_CASES + cargo clippy -p craft-pdf --all-targets --all-features --locked -- -D warnings + cargo test -p craft-pdf --all-features --locked + cargo test --manifest-path crates/pdf/read-only-contract/Cargo.toml --locked diff --git a/docs/full-validation.md b/docs/full-validation.md new file mode 100644 index 000000000..dfa3d8a75 --- /dev/null +++ b/docs/full-validation.md @@ -0,0 +1,28 @@ +# Full validation + +The fast Workspace workflow intentionally samples hostile-input/proptest cases and primarily +runs all features. It is not evidence of full default-feature runtime or doctest coverage. + +Add the `full-validation` label to a pull request to start the separate Full validation workflow. +Keep it on the pull request so each new push receives full checks. The workflow checks the PR +merge ref using `pull_request`, never `pull_request_target`, and does not inherit secrets. +A workflow dispatch validates the selected repository ref; it is not evidence for another PR SHA. + +Before treating a candidate as fully validated, verify on its current head/merge ref: + +- All three `Full native / gate` checks actually ran and succeeded (Linux, Windows, macOS). +- Both workspace test configurations ran with `cargo test`: default and all features, including + their doctests and Cargo's ordinary example compilation/linking. Both clippy configurations + and formatting checks also pass. No hostile divisor or proptest case override is set. +- All three `Full PDF contracts / gate` checks ran and succeeded, including the Unicode fixture + cfg and the separate read-only-contract manifest. Linux-only generic tests do not replace them. +- The normal MSRV, WASM and applicable specialized workflow checks pass too. This optional + workflow does not replace those checks, Sonar analysis, independent review, or branch rules. + +An unlabelled PR's skipped Full validation jobs are not a full-validation pass. The label is an +opt-in mechanism, not a branch-protection change or a merge authorization. A new base/head may +need a fresh full run. Do not dispatch another copy when the matching run is already active. + +The existing fast path and nightly workflow are left unchanged. This workflow is separate from +platform-selection and lint/cache optimization work in #2113. Its native commands deliberately +retain the repository's `cargo test --workspace` pre-merge requirement. diff --git a/scripts/ci-full-validation.sh b/scripts/ci-full-validation.sh new file mode 100644 index 000000000..6e9509b5e --- /dev/null +++ b/scripts/ci-full-validation.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Restore both runtime feature configurations, docs and ordinary example linking, without fuzz caps. +set -euo pipefail +unset CRAFT_HOSTILE_DIVISOR PROPTEST_CASES +if [ "${RUNNER_OS:-}" = Windows ]; then + powershell.exe -NoProfile -Command '$ErrorActionPreference = "Stop"; Start-Service Spooler; Add-PrinterDriver -Name "Generic / Text Only"' +fi +cargo fmt --check +cargo clippy --workspace --all-targets --locked -- -D warnings +cargo clippy --workspace --all-targets --all-features --locked -- -D warnings +# cargo test includes docs and ordinary example linking, as required by AGENTS.md. +cargo test --workspace --locked +cargo test --workspace --all-features --locked diff --git a/scripts/test-full-validation.py b/scripts/test-full-validation.py new file mode 100644 index 000000000..c1729dbfb --- /dev/null +++ b/scripts/test-full-validation.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +"""Exercise the full gate's command sequence without building Rust locally.""" +import os +from pathlib import Path +import subprocess +import unittest + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts/ci-full-validation.sh" +COMMANDS = [ + "fmt --check", + "clippy --workspace --all-targets --locked -- -D warnings", + "clippy --workspace --all-targets --all-features --locked -- -D warnings", + "test --workspace --locked", + "test --workspace --all-features --locked", +] + + +def run(os_name, fail=""): + env = dict(os.environ, RUNNER_OS=os_name, FAIL=fail, + CRAFT_HOSTILE_DIVISOR="8", PROPTEST_CASES="64") + mock = r""" +cargo() { + test -z "${CRAFT_HOSTILE_DIVISOR+x}" && test -z "${PROPTEST_CASES+x}" || return 29 + printf '%s\n' "$*" + if [ "$*" = "$FAIL" ]; then return 17; fi +} +powershell.exe() { printf 'print-queue\n'; } +""" + return subprocess.run(["bash", "-c", mock + SCRIPT.read_text()], env=env, + capture_output=True, text=True, check=False) + + +class FullValidation(unittest.TestCase): + def test_every_os_runs_both_full_configurations(self): + for name in ["Linux", "macOS", "Windows"]: + with self.subTest(os=name): + p = run(name) + self.assertEqual(p.returncode, 0, p.stderr) + self.assertEqual(p.stdout.splitlines(), + (["print-queue"] if name == "Windows" else []) + COMMANDS) + + def test_every_failure_stops_the_gate(self): + for i, command in enumerate(COMMANDS): + with self.subTest(command=command): + p = run("Linux", command) + self.assertEqual(p.returncode, 17) + self.assertEqual(p.stdout.splitlines(), COMMANDS[:i + 1]) + + def test_workflow_preserves_fork_and_platform_boundaries(self): + text = (ROOT / ".github/workflows/full-validation.yml").read_text() + self.assertIn("os: '[\"ubuntu-latest\",\"windows-latest\",\"macos-latest\"]'", text) + self.assertIn("select-os: '[]'", text) + self.assertIn("contents: read", text) + self.assertNotIn("pull_request_target", text) + self.assertNotIn("secrets:", text) + self.assertIn("bash scripts/ci-full-validation.sh", text) + self.assertIn("github.event.label.name != 'full-validation' && github.run_id", text) + self.assertIn("github.event.action != 'labeled' || github.event.label.name == 'full-validation'", text) + self.assertNotIn("CRAFT_HOSTILE_DIVISOR=", text) + self.assertNotIn("PROPTEST_CASES=", text) + + def test_pdf_contracts_are_separate_and_uncapped(self): + text = (ROOT / ".github/workflows/full-validation.yml").read_text() + pdf = text.split(" pdf-contracts:", 1)[1] + self.assertIn("RUSTFLAGS=--cfg craft_pdf_unicode_fixtures", pdf) + self.assertIn("unset CRAFT_HOSTILE_DIVISOR PROPTEST_CASES", pdf) + self.assertIn("cargo test -p craft-pdf --all-features --locked", pdf) + self.assertIn("cargo test --manifest-path crates/pdf/read-only-contract/Cargo.toml --locked", pdf) + self.assertIn("select-os: '[]'", pdf) + self.assertIn("windows-latest", pdf) + self.assertIn("macos-latest", pdf) + + +if __name__ == "__main__": + unittest.main() From 373d0e1683adcf9cb26e15bd1d05c16cbda3f73d Mon Sep 17 00:00:00 2001 From: FredApps Date: Wed, 18 Nov 2026 00:07:54 +0100 Subject: [PATCH 2/4] test(ci): use Git Bash explicitly on Windows --- scripts/test-full-validation.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/scripts/test-full-validation.py b/scripts/test-full-validation.py index c1729dbfb..f0e4153af 100644 --- a/scripts/test-full-validation.py +++ b/scripts/test-full-validation.py @@ -3,10 +3,14 @@ import os from pathlib import Path import subprocess +import sys import unittest ROOT = Path(__file__).resolve().parents[1] SCRIPT = ROOT / "scripts/ci-full-validation.sh" +# Actions uses Git Bash on Windows; Python's PATH search can instead select the WSL launcher. +BASH = (str(Path(os.environ.get("ProgramFiles", r"C:\Program Files")) / "Git/bin/bash.exe") + if sys.platform == "win32" else "bash") COMMANDS = [ "fmt --check", "clippy --workspace --all-targets --locked -- -D warnings", @@ -27,7 +31,7 @@ cargo() { } powershell.exe() { printf 'print-queue\n'; } """ - return subprocess.run(["bash", "-c", mock + SCRIPT.read_text()], env=env, + return subprocess.run([BASH, "-c", mock + SCRIPT.read_text()], env=env, capture_output=True, text=True, check=False) @@ -36,7 +40,7 @@ class FullValidation(unittest.TestCase): for name in ["Linux", "macOS", "Windows"]: with self.subTest(os=name): p = run(name) - self.assertEqual(p.returncode, 0, p.stderr) + self.assertEqual(p.returncode, 0, p.stdout + p.stderr) self.assertEqual(p.stdout.splitlines(), (["print-queue"] if name == "Windows" else []) + COMMANDS) @@ -44,7 +48,7 @@ class FullValidation(unittest.TestCase): for i, command in enumerate(COMMANDS): with self.subTest(command=command): p = run("Linux", command) - self.assertEqual(p.returncode, 17) + self.assertEqual(p.returncode, 17, p.stdout + p.stderr) self.assertEqual(p.stdout.splitlines(), COMMANDS[:i + 1]) def test_workflow_preserves_fork_and_platform_boundaries(self): From 29ce80e136b1c58e1c1c50cef77581af8781d710 Mon Sep 17 00:00:00 2001 From: FredApps Date: Wed, 18 Nov 2026 01:44:16 +0100 Subject: [PATCH 3/4] ci: register full validation in generated actions lockfile --- .github/workflows/actions.lock | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/.github/workflows/actions.lock b/.github/workflows/actions.lock index 46d7b8d3c..e78310f7f 100644 --- a/.github/workflows/actions.lock +++ b/.github/workflows/actions.lock @@ -46,6 +46,7 @@ workflows: - 'actions/checkout@v7.0.1' - 'dtolnay/rust-toolchain@stable' - 'step-security/harden-runner@v2.22.4' + '.github/workflows/full-validation.yml': [] '.github/workflows/house-rules.yml': [] '.github/workflows/jpeg.yml': - 'actions/checkout@v7.0.1' @@ -144,18 +145,8 @@ dependencies: commit: 'sha1-1d88384fd173af6ecb012dfd0264a306b711241a' owner_id: 88700172 repo_id: 422287306 - 'step-security/rust-cache@v2.9.2': - ref: 'v2.9.2' - commit: 'sha1-8871c8ba063416da7c32246932ea6ae08700453e' - owner_id: 88700172 - repo_id: 822636713 'step-security/taiki-e-install-action@v2.87.32': ref: 'v2.87.32' commit: 'sha1-76747d22a7567cc4438546a84f11bdf18e670ab8' owner_id: 88700172 repo_id: 1382753260 - 'swatinem/rust-cache@v2.9.3': - ref: 'v2.9.3' - commit: 'sha1-d1527c8f67ef9af357290475d4af4b0fab8f12e0' - owner_id: 580492 - repo_id: 298565987 From a3784a72edfedbca0d968e23febff43f3e265059 Mon Sep 17 00:00:00 2001 From: FredApps Date: Wed, 18 Nov 2026 03:16:31 +0100 Subject: [PATCH 4/4] test(ci): keep full gate source out of shell harness --- scripts/test-full-validation.py | 46 ++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/scripts/test-full-validation.py b/scripts/test-full-validation.py index f0e4153af..658f49f54 100644 --- a/scripts/test-full-validation.py +++ b/scripts/test-full-validation.py @@ -5,6 +5,7 @@ from pathlib import Path import subprocess import sys import unittest +from unittest.mock import patch ROOT = Path(__file__).resolve().parents[1] SCRIPT = ROOT / "scripts/ci-full-validation.sh" @@ -20,7 +21,30 @@ COMMANDS = [ ] +# Validate the real script, but execute only this fixed, reviewed command sequence. +# Repository-file bytes must never become bash -c source in a mock-only harness. +SOURCE = r""" +set -euo pipefail +unset CRAFT_HOSTILE_DIVISOR PROPTEST_CASES +if [ "${RUNNER_OS:-}" = Windows ]; then + powershell.exe -NoProfile -Command '$ErrorActionPreference = "Stop"; Start-Service Spooler; Add-PrinterDriver -Name "Generic / Text Only"' +fi +cargo fmt --check +cargo clippy --workspace --all-targets --locked -- -D warnings +cargo clippy --workspace --all-targets --all-features --locked -- -D warnings +cargo test --workspace --locked +cargo test --workspace --all-features --locked +""" + + +def source_lines(text): + return [line.strip() for line in text.splitlines() + if line.strip() and not line.lstrip().startswith("#")] + + def run(os_name, fail=""): + if source_lines(SCRIPT.read_text()) != source_lines(SOURCE): + raise AssertionError("full validation command sequence changed") env = dict(os.environ, RUNNER_OS=os_name, FAIL=fail, CRAFT_HOSTILE_DIVISOR="8", PROPTEST_CASES="64") mock = r""" @@ -31,11 +55,31 @@ cargo() { } powershell.exe() { printf 'print-queue\n'; } """ - return subprocess.run([BASH, "-c", mock + SCRIPT.read_text()], env=env, + return subprocess.run([BASH, "-c", mock + SOURCE], env=env, capture_output=True, text=True, check=False) class FullValidation(unittest.TestCase): + def test_changed_source_is_rejected_without_execution(self): + for source in [ + SOURCE + "\necho injected", + SOURCE.replace("cargo fmt --check", "cargo fmt --check; echo injected"), + SOURCE.replace("cargo fmt --check", "cargo fmt $(echo injected) --check"), + SOURCE.replace("cargo test --workspace --locked", ""), + SOURCE + "\ncargo test --workspace --locked", + ]: + with self.subTest(source=source): + with patch.object(Path, "read_text", return_value=source), patch.object(subprocess, "run") as execute: + with self.assertRaises(AssertionError): + run("Linux") + execute.assert_not_called() + + def test_comments_and_blank_lines_do_not_change_commands(self): + with patch.object(Path, "read_text", return_value="# harmless comment\n\n" + SOURCE): + result = run("Linux") + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + self.assertEqual(result.stdout.splitlines(), COMMANDS) + def test_every_os_runs_both_full_configurations(self): for name in ["Linux", "macOS", "Windows"]: with self.subTest(os=name):