From 6589b3c83e69cd60901065de24af3655f270cad3 Mon Sep 17 00:00:00 2001 From: FredApps Date: Tue, 17 Nov 2026 23:51:18 +0100 Subject: [PATCH] ci: add opt-in uncapped three-platform full validation --- .github/workflows/full-validation.yml | 64 ++++++++++++++++++++++ docs/full-validation.md | 28 ++++++++++ scripts/ci-full-validation.sh | 13 +++++ scripts/test-full-validation.py | 76 +++++++++++++++++++++++++++ 4 files changed, 181 insertions(+) create mode 100644 .github/workflows/full-validation.yml create mode 100644 docs/full-validation.md create mode 100644 scripts/ci-full-validation.sh create mode 100644 scripts/test-full-validation.py diff --git a/.github/workflows/full-validation.yml b/.github/workflows/full-validation.yml new file mode 100644 index 000000000..85ad96f2f --- /dev/null +++ b/.github/workflows/full-validation.yml @@ -0,0 +1,64 @@ +name: Full validation + +# Add full-validation to a PR for uncapped validation of its merge ref. Keep the label across pushes. +# Fast Workspace checks are not substitutes for these six completed native/PDF gate checks. +on: + pull_request: + types: [opened, synchronize, reopened, labeled] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + # An unrelated label event must not cancel an active full run. + group: full-validation-${{ github.event.action == 'labeled' && github.event.label.name != 'full-validation' && github.run_id || github.event.pull_request.number || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + native: + if: >- + github.event_name == 'workflow_dispatch' || + (contains(github.event.pull_request.labels.*.name, 'full-validation') && + (github.event.action != 'labeled' || github.event.label.name == 'full-validation')) + name: Full native + uses: storytold/craft-common-actions/.github/workflows/gate.yml@main + with: + os: '["ubuntu-latest","windows-latest","macos-latest"]' + select-os: '[]' + timeout-minutes: 120 + tools: "" + packages: mesa-vulkan-drivers libvulkan1 + craft-fonts-ref: b076417b5880bd4154bd22f00695b1d28c6bea83 + linux-env: WGPU_BACKEND=vulkan + windows-env: CRAFT_PRINT_REQUIRE_QUEUE=1 + env: CRAFT_PRINT_REQUIRE_QUEUE=0 + setup-command: bash scripts/ci-test-inputs.sh + command: | + python scripts/test-full-validation.py + bash scripts/ci-full-validation.sh + + pdf-contracts: + if: >- + github.event_name == 'workflow_dispatch' || + (contains(github.event.pull_request.labels.*.name, 'full-validation') && + (github.event.action != 'labeled' || github.event.label.name == 'full-validation')) + name: Full PDF contracts + uses: storytold/craft-common-actions/.github/workflows/gate.yml@main + with: + os: '["ubuntu-latest","windows-latest","macos-latest"]' + select-os: '[]' + timeout-minutes: 120 + tools: "" + craft-fonts-ref: "" + require-craft-fonts: false + env: RUSTFLAGS=--cfg craft_pdf_unicode_fixtures + setup-command: | + python scripts/test-fetch-pdf-test-fonts.py + python scripts/fetch-pdf-test-fonts.py "$RUNNER_TEMP/craft-pdf-test-fonts" + echo "CRAFT_PDF_TEST_FONT_DIR=$RUNNER_TEMP/craft-pdf-test-fonts" >> "$GITHUB_ENV" + command: | + unset CRAFT_HOSTILE_DIVISOR PROPTEST_CASES + cargo clippy -p craft-pdf --all-targets --all-features --locked -- -D warnings + cargo test -p craft-pdf --all-features --locked + cargo test --manifest-path crates/pdf/read-only-contract/Cargo.toml --locked diff --git a/docs/full-validation.md b/docs/full-validation.md new file mode 100644 index 000000000..dfa3d8a75 --- /dev/null +++ b/docs/full-validation.md @@ -0,0 +1,28 @@ +# Full validation + +The fast Workspace workflow intentionally samples hostile-input/proptest cases and primarily +runs all features. It is not evidence of full default-feature runtime or doctest coverage. + +Add the `full-validation` label to a pull request to start the separate Full validation workflow. +Keep it on the pull request so each new push receives full checks. The workflow checks the PR +merge ref using `pull_request`, never `pull_request_target`, and does not inherit secrets. +A workflow dispatch validates the selected repository ref; it is not evidence for another PR SHA. + +Before treating a candidate as fully validated, verify on its current head/merge ref: + +- All three `Full native / gate` checks actually ran and succeeded (Linux, Windows, macOS). +- Both workspace test configurations ran with `cargo test`: default and all features, including + their doctests and Cargo's ordinary example compilation/linking. Both clippy configurations + and formatting checks also pass. No hostile divisor or proptest case override is set. +- All three `Full PDF contracts / gate` checks ran and succeeded, including the Unicode fixture + cfg and the separate read-only-contract manifest. Linux-only generic tests do not replace them. +- The normal MSRV, WASM and applicable specialized workflow checks pass too. This optional + workflow does not replace those checks, Sonar analysis, independent review, or branch rules. + +An unlabelled PR's skipped Full validation jobs are not a full-validation pass. The label is an +opt-in mechanism, not a branch-protection change or a merge authorization. A new base/head may +need a fresh full run. Do not dispatch another copy when the matching run is already active. + +The existing fast path and nightly workflow are left unchanged. This workflow is separate from +platform-selection and lint/cache optimization work in #2113. Its native commands deliberately +retain the repository's `cargo test --workspace` pre-merge requirement. diff --git a/scripts/ci-full-validation.sh b/scripts/ci-full-validation.sh new file mode 100644 index 000000000..6e9509b5e --- /dev/null +++ b/scripts/ci-full-validation.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Restore both runtime feature configurations, docs and ordinary example linking, without fuzz caps. +set -euo pipefail +unset CRAFT_HOSTILE_DIVISOR PROPTEST_CASES +if [ "${RUNNER_OS:-}" = Windows ]; then + powershell.exe -NoProfile -Command '$ErrorActionPreference = "Stop"; Start-Service Spooler; Add-PrinterDriver -Name "Generic / Text Only"' +fi +cargo fmt --check +cargo clippy --workspace --all-targets --locked -- -D warnings +cargo clippy --workspace --all-targets --all-features --locked -- -D warnings +# cargo test includes docs and ordinary example linking, as required by AGENTS.md. +cargo test --workspace --locked +cargo test --workspace --all-features --locked diff --git a/scripts/test-full-validation.py b/scripts/test-full-validation.py new file mode 100644 index 000000000..c1729dbfb --- /dev/null +++ b/scripts/test-full-validation.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +"""Exercise the full gate's command sequence without building Rust locally.""" +import os +from pathlib import Path +import subprocess +import unittest + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts/ci-full-validation.sh" +COMMANDS = [ + "fmt --check", + "clippy --workspace --all-targets --locked -- -D warnings", + "clippy --workspace --all-targets --all-features --locked -- -D warnings", + "test --workspace --locked", + "test --workspace --all-features --locked", +] + + +def run(os_name, fail=""): + env = dict(os.environ, RUNNER_OS=os_name, FAIL=fail, + CRAFT_HOSTILE_DIVISOR="8", PROPTEST_CASES="64") + mock = r""" +cargo() { + test -z "${CRAFT_HOSTILE_DIVISOR+x}" && test -z "${PROPTEST_CASES+x}" || return 29 + printf '%s\n' "$*" + if [ "$*" = "$FAIL" ]; then return 17; fi +} +powershell.exe() { printf 'print-queue\n'; } +""" + return subprocess.run(["bash", "-c", mock + SCRIPT.read_text()], env=env, + capture_output=True, text=True, check=False) + + +class FullValidation(unittest.TestCase): + def test_every_os_runs_both_full_configurations(self): + for name in ["Linux", "macOS", "Windows"]: + with self.subTest(os=name): + p = run(name) + self.assertEqual(p.returncode, 0, p.stderr) + self.assertEqual(p.stdout.splitlines(), + (["print-queue"] if name == "Windows" else []) + COMMANDS) + + def test_every_failure_stops_the_gate(self): + for i, command in enumerate(COMMANDS): + with self.subTest(command=command): + p = run("Linux", command) + self.assertEqual(p.returncode, 17) + self.assertEqual(p.stdout.splitlines(), COMMANDS[:i + 1]) + + def test_workflow_preserves_fork_and_platform_boundaries(self): + text = (ROOT / ".github/workflows/full-validation.yml").read_text() + self.assertIn("os: '[\"ubuntu-latest\",\"windows-latest\",\"macos-latest\"]'", text) + self.assertIn("select-os: '[]'", text) + self.assertIn("contents: read", text) + self.assertNotIn("pull_request_target", text) + self.assertNotIn("secrets:", text) + self.assertIn("bash scripts/ci-full-validation.sh", text) + self.assertIn("github.event.label.name != 'full-validation' && github.run_id", text) + self.assertIn("github.event.action != 'labeled' || github.event.label.name == 'full-validation'", text) + self.assertNotIn("CRAFT_HOSTILE_DIVISOR=", text) + self.assertNotIn("PROPTEST_CASES=", text) + + def test_pdf_contracts_are_separate_and_uncapped(self): + text = (ROOT / ".github/workflows/full-validation.yml").read_text() + pdf = text.split(" pdf-contracts:", 1)[1] + self.assertIn("RUSTFLAGS=--cfg craft_pdf_unicode_fixtures", pdf) + self.assertIn("unset CRAFT_HOSTILE_DIVISOR PROPTEST_CASES", pdf) + self.assertIn("cargo test -p craft-pdf --all-features --locked", pdf) + self.assertIn("cargo test --manifest-path crates/pdf/read-only-contract/Cargo.toml --locked", pdf) + self.assertIn("select-os: '[]'", pdf) + self.assertIn("windows-latest", pdf) + self.assertIn("macos-latest", pdf) + + +if __name__ == "__main__": + unittest.main()