From e10d698d5c951a7abb019f2efa4f5b2346ecd38e Mon Sep 17 00:00:00 2001 From: skylartaylor <26588250+skylartaylor@users.noreply.github.com> Date: Fri, 16 Oct 2026 04:55:29 +0200 Subject: [PATCH] ci: bound shared jobs and preserve caller validation --- .github/workflows/freebsd.yml | 45 +++++-- .github/workflows/packaging-lint.yml | 11 ++ .github/workflows/release.yml | 9 ++ .github/workflows/test-freebsd.yml | 29 +++++ README.md | 56 ++++++++ tests/test_freebsd.py | 186 +++++++++++++++++++++++++++ 6 files changed, 328 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/test-freebsd.yml create mode 100644 tests/test_freebsd.py diff --git a/.github/workflows/freebsd.yml b/.github/workflows/freebsd.yml index 73439e3..a539a04 100644 --- a/.github/workflows/freebsd.yml +++ b/.github/workflows/freebsd.yml @@ -8,6 +8,11 @@ on: type: string required: false default: "" + require-craft-fonts: + description: "Require the supplied craft-fonts checkout during builds and tests" + type: boolean + required: false + default: false freebsd-release: description: "FreeBSD OS release version to run in the VM" type: string @@ -63,24 +68,48 @@ jobs: mem: 12288 copyback: false prepare: | - pkg install -y bash git curl pkgconf cmake python3 libxkbcommon wayland libX11 libXcursor libXrandr libXi libxcb mesa-libs vulkan-loader gtk3 fontconfig freetype2 alsa-lib + pkg install -y bash git curl pkgconf cmake python3 libxkbcommon wayland libX11 libXcursor libXrandr libXi libxcb mesa-libs vulkan-loader gtk3 fontconfig freetype2 alsa-lib dejavu curl -fsSL https://sh.rustup.rs | sh -s -- -y --profile minimal run: | . "$HOME/.cargo/env" git config --global --add safe.directory "*" rustc --version + if [ -d craft-fonts ]; then + export CRAFT_FONTS_DIR="$PWD/craft-fonts" + fi + if [ "${{ inputs.require-craft-fonts }}" = "true" ]; then + if [ ! -d craft-fonts ]; then + echo "error: required craft-fonts checkout is missing" >&2 + exit 1 + fi + export CRAFT_FONTS_REQUIRED=1 + fi CARGO_PROFILE_DEV_DEBUG=0 cargo build --workspace --all-targets --locked ${{ inputs.cargo-build-extra-args }} if [ "${{ inputs.skip-tests }}" != "true" ]; then - if [ -n "${{ inputs.custom-test-command }}" ]; then - ${{ inputs.custom-test-command }} - elif [ -d "craft-fonts" ]; then - CRAFT_FONTS_DIR="$PWD/craft-fonts" CARGO_PROFILE_DEV_DEBUG=0 cargo test --workspace --locked ${{ inputs.cargo-test-extra-args }} + if [ "${{ inputs.custom-test-command != '' }}" = "true" ]; then + # Isolate the command list: an early failure in an && chain must not + # be hidden by a later successful packaging command in this script. + sh -eu <<'CRAFT_CUSTOM_TESTS' || exit "$?" + ${{ inputs.custom-test-command }} + CRAFT_CUSTOM_TESTS else CARGO_PROFILE_DEV_DEBUG=0 cargo test --workspace --locked ${{ inputs.cargo-test-extra-args }} fi fi - if [ "${{ inputs.run-package-check }}" = "true" ] && [ -f packaging/freebsd/package.sh ]; then - rm -rf target/debug || true + if [ "${{ inputs.run-package-check }}" = "true" ]; then + if [ ! -f packaging/freebsd/package.sh ]; then + echo "error: requested FreeBSD packaging script is missing" >&2 + exit 1 + fi + rm -rf target/debug CARGO_BUILD_JOBS=4 bash packaging/freebsd/package.sh - tar -tzf dist/release/*-freebsd-x86_64.tar.gz || true + # Check every produced archive; an unmatched glob or unreadable tarball + # must fail instead of turning a requested package check into a no-op. + for archive in dist/release/*-freebsd-x86_64.tar.gz; do + if [ ! -f "$archive" ]; then + echo "error: no FreeBSD x86_64 package was produced" >&2 + exit 1 + fi + tar -tzf "$archive" + done fi diff --git a/.github/workflows/packaging-lint.yml b/.github/workflows/packaging-lint.yml index c09c4e1..e12e606 100644 --- a/.github/workflows/packaging-lint.yml +++ b/.github/workflows/packaging-lint.yml @@ -8,13 +8,24 @@ on: type: string required: false default: "1.7.12" + extra-checks-script: + description: "Optional repository-local Bash script for app-specific packaging checks" + type: string + required: false + default: "" jobs: lint: runs-on: ubuntu-latest + timeout-minutes: 15 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Run packaging lint uses: storytold/craft-common-actions/actions/packaging-lint@main with: actionlint-version: ${{ inputs.actionlint-version }} + - name: App-specific packaging checks + if: inputs.extra-checks-script != '' + env: + CHECKS_SCRIPT: ${{ inputs.extra-checks-script }} + run: bash -- "$CHECKS_SCRIPT" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b90fa53..e0bfb82 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -102,6 +102,7 @@ permissions: jobs: version: + timeout-minutes: 10 name: version runs-on: ubuntu-latest outputs: @@ -118,6 +119,7 @@ jobs: app-title: ${{ inputs.app-title }} macos: + timeout-minutes: 90 name: macOS universal needs: version runs-on: macos-15-xlarge @@ -243,6 +245,7 @@ jobs: if [ -n "${MACOS_KEYCHAIN:-}" ]; then security delete-keychain "$MACOS_KEYCHAIN" || true; fi windows: + timeout-minutes: 90 name: Windows ${{ matrix.arch }} needs: version runs-on: windows-latest @@ -371,6 +374,7 @@ jobs: if-no-files-found: error linux: + timeout-minutes: 90 name: Linux ${{ matrix.arch }} needs: version runs-on: ${{ matrix.runner }} @@ -472,6 +476,7 @@ jobs: if-no-files-found: error flatpak: + timeout-minutes: 90 name: Flatpak ${{ matrix.arch }} needs: [version, linux] runs-on: ${{ matrix.runner }} @@ -660,6 +665,7 @@ jobs: if-no-files-found: error web: + timeout-minutes: 90 name: Web (wasm) if: inputs.has-web needs: version @@ -750,6 +756,7 @@ jobs: if-no-files-found: error linux-riscv64: + timeout-minutes: 90 name: Linux riscv64 if: inputs.has-riscv64 needs: version @@ -852,6 +859,7 @@ jobs: if-no-files-found: error server: + timeout-minutes: 90 name: Server Linux ${{ matrix.arch }} if: inputs.has-server needs: version @@ -896,6 +904,7 @@ jobs: if-no-files-found: error release: + timeout-minutes: 10 name: Draft GitHub Release if: "!cancelled() && !contains(needs.*.result, 'failure')" needs: [version, macos, windows, linux, linux-riscv64, flatpak, freebsd, web, server] diff --git a/.github/workflows/test-freebsd.yml b/.github/workflows/test-freebsd.yml new file mode 100644 index 0000000..f5f4927 --- /dev/null +++ b/.github/workflows/test-freebsd.yml @@ -0,0 +1,29 @@ +name: Shared FreeBSD workflow tests + +on: + pull_request: + paths: + - .github/workflows/freebsd.yml + - .github/workflows/test-freebsd.yml + - tests/** + push: + branches: [main] + paths: + - .github/workflows/freebsd.yml + - .github/workflows/test-freebsd.yml + - tests/** + workflow_dispatch: + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Install YAML parser + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends python3-yaml + - name: Exercise failing tests, package validation and required fonts + run: /usr/bin/python3 -m unittest discover -s tests -v diff --git a/README.md b/README.md index d3e571a..47b57d3 100644 --- a/README.md +++ b/README.md @@ -144,6 +144,21 @@ jobs: ### 4. `freebsd.yml` Builds and runs workspace tests inside a FreeBSD 15 VM on a Linux runner. +`craft-fonts-ref` makes the checkout available to builds, default tests and custom +tests through `CRAFT_FONTS_DIR`. Set `require-craft-fonts: true` to also export +`CRAFT_FONTS_REQUIRED=1` and fail before building if the checkout is missing. +The application's build script validates its required font faces. + +Use a multiline `custom-test-command` for separate test groups. It runs in a +separate `sh -eu` process whose failure stops the VM script before packaging. +Do not rely on `errexit` to catch nonfinal commands of an `&&` list followed by +other commands; write each required test command on its own line. + +`run-package-check: true` requires `packaging/freebsd/package.sh` to exist and +succeed, then requires at least one readable `*-freebsd-x86_64.tar.gz` in +`dist/release`. Each archive is checked separately. The app's package script +retains responsibility for its required binaries, metadata and licence files. + #### Caller Example: ```yaml name: FreeBSD @@ -190,3 +205,44 @@ Individual actions are also available for composing custom pipelines: ## Storytold Organization Hosting This repository is hosted at `storytold/craft-common-actions` for all Storytold / ArtCraft craftapps. + +## App-specific packaging checks + +Keep app-specific packaging assertions in the caller repository and pass their +Bash script to `packaging-lint.yml` with +`extra-checks-script: packaging/check-app-specific.sh`. It runs after the shared +lint action, with the checkout and lint tools available. A missing or failing +requested script fails the job. Native Windows installer probes remain separate +caller jobs; Linux static lint does not replace them. + +## Job time limits + +Limits live on executable jobs in these reusable workflows, since caller jobs +using `uses` cannot set `timeout-minutes`. + +| Work | Limit (minutes) | +| --- | ---: | +| Packaging lint | 15 | +| Release version / draft publication | 10 | +| Release macOS, Windows, Linux, Flatpak, web, RISC-V and server builds | 90 | +| Release FreeBSD | 180 | +| Standalone FreeBSD build, test and optional package | 300 | +| Windows ARM64 package / install | 90 / 30 | + +The new limits leave headroom over sampled Craft app job timings; the existing +FreeBSD and ARM64 limits are retained. Treat these as initial limits, review +successful cold builds after rollout and lower them when the evidence supports it. +These are per-job limits, not a total workflow duration or spend budget. + +## Testing workflow failure paths + +With Python 3 and PyYAML installed, run: + +```sh +python3 -m unittest discover -s tests -v +``` + +The tests execute the FreeBSD run template under the host's POSIX shell with +fixture build/test commands and real tar archives. They cover failures in each +custom test group, default build/test failures, missing/corrupt packages and font +environment propagation. They do not replace native FreeBSD caller runs. diff --git a/tests/test_freebsd.py b/tests/test_freebsd.py new file mode 100644 index 0000000..1ae98a1 --- /dev/null +++ b/tests/test_freebsd.py @@ -0,0 +1,186 @@ +"""Exercise the VM's actual run template with failing test/package commands. + +Linux /bin/sh exercises the POSIX failure paths without building an app or booting +a VM. Native FreeBSD and Windows acceptance still belongs to the caller PR runs. +Requires PyYAML (python3-yaml on the hosted Ubuntu runner). +""" + +import os +from pathlib import Path +import re +import subprocess +import tempfile +import unittest + +import yaml + + +ROOT = Path(__file__).resolve().parents[1] +WORKFLOW = Path(os.environ.get("FREEBSD_WORKFLOW", ROOT / ".github/workflows/freebsd.yml")) +DATA = yaml.load(WORKFLOW.read_text(), Loader=yaml.BaseLoader) +TEMPLATE = next(step["with"]["run"] for step in DATA["jobs"]["freebsd"]["steps"] if "with" in step and "run" in step["with"]) + + +class FreeBSDRunTests(unittest.TestCase): + def run_vm(self, *, command="", fail_test=0, package="valid", package_check=True, + skip_tests=False, fonts=False, require_fonts=False, fail_build=False): + inputs = {key: value.get("default", "") for key, value in DATA["on"]["workflow_call"]["inputs"].items()} + inputs.update({ + "custom-test-command": command, + "skip-tests": str(skip_tests).lower(), + "run-package-check": str(package_check).lower(), + "require-craft-fonts": str(require_fonts).lower(), + }) + + def replace(match): + key = match.group(1).strip() + if key == "custom-test-command != ''": + return str(bool(command)).lower() + return inputs[key] + + script = re.sub(r"\$\{\{ inputs\.(.*?) \}\}", replace, TEMPLATE) + # The host rustup environment is irrelevant: keep all commands inside the fixture. + script = script.replace('. "$HOME/.cargo/env"', ': # isolated cargo/rustc shims') + with tempfile.TemporaryDirectory(prefix="shared-freebsd-test-") as directory: + work = Path(directory) + tools = work / "tools" + tools.mkdir() + if fonts: + (work / "craft-fonts").mkdir() + cargo = tools / "cargo" + cargo.write_text("""#!/bin/sh +set -eu +printf '%s|%s|%s|%s\\n' "$1" "${CRAFT_FONTS_DIR-}" "${CRAFT_FONTS_REQUIRED-}" "$*" >> "$PWD/cargo.log" +if [ "$1" = build ]; then + [ "$FAIL_BUILD" = false ] || exit 31 +else + count=0 + if [ -f test-count ]; then count=$(cat test-count); fi + count=$((count + 1)) + echo "$count" > test-count + [ "$count" -ne "$FAIL_TEST" ] || exit 42 +fi +""") + cargo.chmod(0o755) + for tool in ("git", "rustc"): + path = tools / tool + path.write_text("#!/bin/sh\nexit 0\n") + path.chmod(0o755) + if package != "missing-script": + path = work / "packaging/freebsd/package.sh" + path.parent.mkdir(parents=True) + path.write_text("""#!/bin/bash +set -euo pipefail +touch package-ran +case "$PACKAGE_MODE" in + failed) exit 43 ;; + absent) exit 0 ;; +esac +mkdir -p dist/release stage/bin +echo fixture > stage/bin/app +tar -czf dist/release/app-freebsd-x86_64.tar.gz stage +if [ "$PACKAGE_MODE" = corrupt ]; then + echo corrupt > dist/release/app-freebsd-x86_64.tar.gz +elif [ "$PACKAGE_MODE" = second-corrupt ]; then + echo corrupt > dist/release/other-freebsd-x86_64.tar.gz +elif [ "$PACKAGE_MODE" = multiple ]; then + cp dist/release/app-freebsd-x86_64.tar.gz dist/release/other-freebsd-x86_64.tar.gz +fi +""") + env = os.environ.copy() + for key in ("CRAFT_FONTS_DIR", "CRAFT_FONTS_REQUIRED"): + env.pop(key, None) + env.update(PATH=str(tools) + os.pathsep + env["PATH"], FAIL_TEST=str(fail_test), + PACKAGE_MODE=package, FAIL_BUILD=str(fail_build).lower()) + result = subprocess.run(["/bin/sh", "-eu", "-c", script], cwd=work, env=env, + text=True, capture_output=True, timeout=15) + log = (work / "cargo.log").read_text().splitlines() if (work / "cargo.log").exists() else [] + outcome = dict(code=result.returncode, packaged=(work / "package-ran").exists(), + log=log, output=result.stdout + result.stderr) + return outcome + + def test_each_failure_in_custom_chain_is_fatal_before_packaging(self): + command = " && ".join("cargo test --group " + str(n) for n in range(1, 5)) + for group in range(1, 5): + with self.subTest(group=group): + result = self.run_vm(command=command, fail_test=group) + self.assertEqual(result["code"], 42, result) + self.assertFalse(result["packaged"], result) + self.assertEqual(len(result["log"]), 1 + group, result) + + def test_each_failure_in_multiline_commands_stops_later_commands(self): + command = "\n".join("cargo test --group " + str(n) for n in range(1, 5)) + for group in range(1, 5): + with self.subTest(group=group): + result = self.run_vm(command=command, fail_test=group) + self.assertEqual(result["code"], 42, result) + self.assertFalse(result["packaged"], result) + self.assertEqual(len(result["log"]), 1 + group, result) + + def test_default_tests_must_pass_before_packaging(self): + result = self.run_vm(fail_test=1) + self.assertEqual(result["code"], 42, result) + self.assertFalse(result["packaged"], result) + + def test_build_failure_is_fatal_before_tests(self): + result = self.run_vm(fail_build=True) + self.assertEqual(result["code"], 31, result) + self.assertEqual(len(result["log"]), 1, result) + self.assertFalse(result["packaged"], result) + + def test_successful_custom_tests_reach_packaging(self): + result = self.run_vm(command="cargo test --lib\ncargo test --doc") + self.assertEqual(result["code"], 0, result) + self.assertEqual(len(result["log"]), 3, result) + self.assertTrue(result["packaged"], result) + + def test_skip_tests_still_checks_requested_package(self): + result = self.run_vm(skip_tests=True, fail_test=1) + self.assertEqual(result["code"], 0, result) + self.assertEqual(len(result["log"]), 1, result) + self.assertTrue(result["packaged"], result) + + def test_packaging_is_optional(self): + result = self.run_vm(package_check=False, package="missing-script") + self.assertEqual(result["code"], 0, result) + self.assertFalse(result["packaged"], result) + + def test_requested_packaging_must_produce_readable_archives(self): + for mode in ("missing-script", "failed", "absent", "corrupt", "second-corrupt"): + with self.subTest(mode=mode): + result = self.run_vm(package=mode) + self.assertNotEqual(result["code"], 0, result) + + def test_all_valid_archives_are_readable(self): + result = self.run_vm(package="multiple") + self.assertEqual(result["code"], 0, result) + self.assertEqual(result["output"].count("stage/bin/app"), 2, result) + + def test_required_fonts_reach_build_and_both_test_paths(self): + for command in ("", "cargo test --lib\ncargo test --doc"): + with self.subTest(command=command): + result = self.run_vm(command=command, fonts=True, require_fonts=True) + self.assertEqual(result["code"], 0, result) + for line in result["log"]: + _, directory, required, _ = line.split("|", 3) + self.assertTrue(directory.endswith("/craft-fonts"), result) + self.assertEqual(required, "1", result) + + def test_required_missing_fonts_fail_before_build(self): + result = self.run_vm(require_fonts=True) + self.assertNotEqual(result["code"], 0, result) + self.assertEqual(result["log"], [], result) + + def test_optional_fonts_remain_optional(self): + for fonts in (True, False): + with self.subTest(fonts=fonts): + result = self.run_vm(fonts=fonts) + self.assertEqual(result["code"], 0, result) + for line in result["log"]: + _, directory, required, _ = line.split("|", 3) + self.assertEqual(bool(directory), fonts, result) + self.assertEqual(required, "", result) + + +if __name__ == "__main__": + unittest.main()