ci: bound shared jobs and preserve caller validation

.github/workflows/freebsd.yml

@@ -8,6 +8,11 @@ on:
         type: string
         required: false
         default: ""
+      require-craft-fonts:
+        description: "Require the supplied craft-fonts checkout during builds and tests"
+        type: boolean
+        required: false
+        default: false
       freebsd-release:
         description: "FreeBSD OS release version to run in the VM"
         type: string
@@ -63,24 +68,48 @@ jobs:
           mem: 12288
           copyback: false
           prepare: |
-            pkg install -y bash git curl pkgconf cmake python3 libxkbcommon wayland libX11 libXcursor libXrandr libXi libxcb mesa-libs vulkan-loader gtk3 fontconfig freetype2 alsa-lib
+            pkg install -y bash git curl pkgconf cmake python3 libxkbcommon wayland libX11 libXcursor libXrandr libXi libxcb mesa-libs vulkan-loader gtk3 fontconfig freetype2 alsa-lib dejavu
             curl -fsSL https://sh.rustup.rs | sh -s -- -y --profile minimal
           run: |
             . "$HOME/.cargo/env"
             git config --global --add safe.directory "*"
             rustc --version
+            if [ -d craft-fonts ]; then
+              export CRAFT_FONTS_DIR="$PWD/craft-fonts"
+            fi
+            if [ "${{ inputs.require-craft-fonts }}" = "true" ]; then
+              if [ ! -d craft-fonts ]; then
+                echo "error: required craft-fonts checkout is missing" >&2
+                exit 1
+              fi
+              export CRAFT_FONTS_REQUIRED=1
+            fi
             CARGO_PROFILE_DEV_DEBUG=0 cargo build --workspace --all-targets --locked ${{ inputs.cargo-build-extra-args }}
             if [ "${{ inputs.skip-tests }}" != "true" ]; then
-              if [ -n "${{ inputs.custom-test-command }}" ]; then
-                ${{ inputs.custom-test-command }}
-              elif [ -d "craft-fonts" ]; then
-                CRAFT_FONTS_DIR="$PWD/craft-fonts" CARGO_PROFILE_DEV_DEBUG=0 cargo test --workspace --locked ${{ inputs.cargo-test-extra-args }}
+              if [ "${{ inputs.custom-test-command != '' }}" = "true" ]; then
+                # Isolate the command list: an early failure in an && chain must not
+                # be hidden by a later successful packaging command in this script.
+                sh -eu <<'CRAFT_CUSTOM_TESTS' || exit "$?"
+            ${{ inputs.custom-test-command }}
+            CRAFT_CUSTOM_TESTS
               else
                 CARGO_PROFILE_DEV_DEBUG=0 cargo test --workspace --locked ${{ inputs.cargo-test-extra-args }}
               fi
             fi
-            if [ "${{ inputs.run-package-check }}" = "true" ] && [ -f packaging/freebsd/package.sh ]; then
-              rm -rf target/debug || true
+            if [ "${{ inputs.run-package-check }}" = "true" ]; then
+              if [ ! -f packaging/freebsd/package.sh ]; then
+                echo "error: requested FreeBSD packaging script is missing" >&2
+                exit 1
+              fi
+              rm -rf target/debug
               CARGO_BUILD_JOBS=4 bash packaging/freebsd/package.sh
-              tar -tzf dist/release/*-freebsd-x86_64.tar.gz || true
+              # Check every produced archive; an unmatched glob or unreadable tarball
+              # must fail instead of turning a requested package check into a no-op.
+              for archive in dist/release/*-freebsd-x86_64.tar.gz; do
+                if [ ! -f "$archive" ]; then
+                  echo "error: no FreeBSD x86_64 package was produced" >&2
+                  exit 1
+                fi
+                tar -tzf "$archive"
+              done
             fi

.github/workflows/packaging-lint.yml

@@ -8,13 +8,24 @@ on:
         type: string
         required: false
         default: "1.7.12"
+      extra-checks-script:
+        description: "Optional repository-local Bash script for app-specific packaging checks"
+        type: string
+        required: false
+        default: ""
 
 jobs:
   lint:
     runs-on: ubuntu-latest
+    timeout-minutes: 15
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - name: Run packaging lint
         uses: storytold/craft-common-actions/actions/packaging-lint@main
         with:
           actionlint-version: ${{ inputs.actionlint-version }}
+      - name: App-specific packaging checks
+        if: inputs.extra-checks-script != ''
+        env:
+          CHECKS_SCRIPT: ${{ inputs.extra-checks-script }}
+        run: bash -- "$CHECKS_SCRIPT"

.github/workflows/release.yml

@@ -102,6 +102,7 @@ permissions:
 
 jobs:
   version:
+    timeout-minutes: 10
     name: version
     runs-on: ubuntu-latest
     outputs:
@@ -118,6 +119,7 @@ jobs:
           app-title: ${{ inputs.app-title }}
 
   macos:
+    timeout-minutes: 90
     name: macOS universal
     needs: version
     runs-on: macos-15-xlarge
@@ -243,6 +245,7 @@ jobs:
           if [ -n "${MACOS_KEYCHAIN:-}" ]; then security delete-keychain "$MACOS_KEYCHAIN" || true; fi
 
   windows:
+    timeout-minutes: 90
     name: Windows ${{ matrix.arch }}
     needs: version
     runs-on: windows-latest
@@ -371,6 +374,7 @@ jobs:
           if-no-files-found: error
 
   linux:
+    timeout-minutes: 90
     name: Linux ${{ matrix.arch }}
     needs: version
     runs-on: ${{ matrix.runner }}
@@ -472,6 +476,7 @@ jobs:
           if-no-files-found: error
 
   flatpak:
+    timeout-minutes: 90
     name: Flatpak ${{ matrix.arch }}
     needs: [version, linux]
     runs-on: ${{ matrix.runner }}
@@ -660,6 +665,7 @@ jobs:
           if-no-files-found: error
 
   web:
+    timeout-minutes: 90
     name: Web (wasm)
     if: inputs.has-web
     needs: version
@@ -750,6 +756,7 @@ jobs:
           if-no-files-found: error
 
   linux-riscv64:
+    timeout-minutes: 90
     name: Linux riscv64
     if: inputs.has-riscv64
     needs: version
@@ -852,6 +859,7 @@ jobs:
           if-no-files-found: error
 
   server:
+    timeout-minutes: 90
     name: Server Linux ${{ matrix.arch }}
     if: inputs.has-server
     needs: version
@@ -896,6 +904,7 @@ jobs:
           if-no-files-found: error
 
   release:
+    timeout-minutes: 10
     name: Draft GitHub Release
     if: "!cancelled() && !contains(needs.*.result, 'failure')"
     needs: [version, macos, windows, linux, linux-riscv64, flatpak, freebsd, web, server]

.github/workflows/test-freebsd.yml

@@ -0,0 +1,29 @@
+name: Shared FreeBSD workflow tests
+
+on:
+  pull_request:
+    paths:
+      - .github/workflows/freebsd.yml
+      - .github/workflows/test-freebsd.yml
+      - tests/**
+  push:
+    branches: [main]
+    paths:
+      - .github/workflows/freebsd.yml
+      - .github/workflows/test-freebsd.yml
+      - tests/**
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+      - name: Install YAML parser
+        run: sudo apt-get update && sudo apt-get install -y --no-install-recommends python3-yaml
+      - name: Exercise failing tests, package validation and required fonts
+        run: /usr/bin/python3 -m unittest discover -s tests -v

README.md

@@ -144,6 +144,21 @@ jobs:
 ### 4. `freebsd.yml`
 Builds and runs workspace tests inside a FreeBSD 15 VM on a Linux runner.
 
+`craft-fonts-ref` makes the checkout available to builds, default tests and custom
+tests through `CRAFT_FONTS_DIR`. Set `require-craft-fonts: true` to also export
+`CRAFT_FONTS_REQUIRED=1` and fail before building if the checkout is missing.
+The application's build script validates its required font faces.
+
+Use a multiline `custom-test-command` for separate test groups. It runs in a
+separate `sh -eu` process whose failure stops the VM script before packaging.
+Do not rely on `errexit` to catch nonfinal commands of an `&&` list followed by
+other commands; write each required test command on its own line.
+
+`run-package-check: true` requires `packaging/freebsd/package.sh` to exist and
+succeed, then requires at least one readable `*-freebsd-x86_64.tar.gz` in
+`dist/release`. Each archive is checked separately. The app's package script
+retains responsibility for its required binaries, metadata and licence files.
+
 #### Caller Example:
 ```yaml
 name: FreeBSD
@@ -190,3 +205,44 @@ Individual actions are also available for composing custom pipelines:
 ## Storytold Organization Hosting
 
 This repository is hosted at `storytold/craft-common-actions` for all Storytold / ArtCraft craftapps.
+
+## App-specific packaging checks
+
+Keep app-specific packaging assertions in the caller repository and pass their
+Bash script to `packaging-lint.yml` with
+`extra-checks-script: packaging/check-app-specific.sh`. It runs after the shared
+lint action, with the checkout and lint tools available. A missing or failing
+requested script fails the job. Native Windows installer probes remain separate
+caller jobs; Linux static lint does not replace them.
+
+## Job time limits
+
+Limits live on executable jobs in these reusable workflows, since caller jobs
+using `uses` cannot set `timeout-minutes`.
+
+| Work | Limit (minutes) |
+| --- | ---: |
+| Packaging lint | 15 |
+| Release version / draft publication | 10 |
+| Release macOS, Windows, Linux, Flatpak, web, RISC-V and server builds | 90 |
+| Release FreeBSD | 180 |
+| Standalone FreeBSD build, test and optional package | 300 |
+| Windows ARM64 package / install | 90 / 30 |
+
+The new limits leave headroom over sampled Craft app job timings; the existing
+FreeBSD and ARM64 limits are retained. Treat these as initial limits, review
+successful cold builds after rollout and lower them when the evidence supports it.
+These are per-job limits, not a total workflow duration or spend budget.
+
+## Testing workflow failure paths
+
+With Python 3 and PyYAML installed, run:
+
+```sh
+python3 -m unittest discover -s tests -v
+```
+
+The tests execute the FreeBSD run template under the host's POSIX shell with
+fixture build/test commands and real tar archives. They cover failures in each
+custom test group, default build/test failures, missing/corrupt packages and font
+environment propagation. They do not replace native FreeBSD caller runs.

tests/test_freebsd.py

@@ -0,0 +1,186 @@
+"""Exercise the VM's actual run template with failing test/package commands.
+
+Linux /bin/sh exercises the POSIX failure paths without building an app or booting
+a VM. Native FreeBSD and Windows acceptance still belongs to the caller PR runs.
+Requires PyYAML (python3-yaml on the hosted Ubuntu runner).
+"""
+
+import os
+from pathlib import Path
+import re
+import subprocess
+import tempfile
+import unittest
+
+import yaml
+
+
+ROOT = Path(__file__).resolve().parents[1]
+WORKFLOW = Path(os.environ.get("FREEBSD_WORKFLOW", ROOT / ".github/workflows/freebsd.yml"))
+DATA = yaml.load(WORKFLOW.read_text(), Loader=yaml.BaseLoader)
+TEMPLATE = next(step["with"]["run"] for step in DATA["jobs"]["freebsd"]["steps"] if "with" in step and "run" in step["with"])
+
+
+class FreeBSDRunTests(unittest.TestCase):
+    def run_vm(self, *, command="", fail_test=0, package="valid", package_check=True,
+               skip_tests=False, fonts=False, require_fonts=False, fail_build=False):
+        inputs = {key: value.get("default", "") for key, value in DATA["on"]["workflow_call"]["inputs"].items()}
+        inputs.update({
+            "custom-test-command": command,
+            "skip-tests": str(skip_tests).lower(),
+            "run-package-check": str(package_check).lower(),
+            "require-craft-fonts": str(require_fonts).lower(),
+        })
+
+        def replace(match):
+            key = match.group(1).strip()
+            if key == "custom-test-command != ''":
+                return str(bool(command)).lower()
+            return inputs[key]
+
+        script = re.sub(r"\$\{\{ inputs\.(.*?) \}\}", replace, TEMPLATE)
+        # The host rustup environment is irrelevant: keep all commands inside the fixture.
+        script = script.replace('. "$HOME/.cargo/env"', ': # isolated cargo/rustc shims')
+        with tempfile.TemporaryDirectory(prefix="shared-freebsd-test-") as directory:
+            work = Path(directory)
+            tools = work / "tools"
+            tools.mkdir()
+            if fonts:
+                (work / "craft-fonts").mkdir()
+            cargo = tools / "cargo"
+            cargo.write_text("""#!/bin/sh
+set -eu
+printf '%s|%s|%s|%s\\n' "$1" "${CRAFT_FONTS_DIR-}" "${CRAFT_FONTS_REQUIRED-}" "$*" >> "$PWD/cargo.log"
+if [ "$1" = build ]; then
+  [ "$FAIL_BUILD" = false ] || exit 31
+else
+  count=0
+  if [ -f test-count ]; then count=$(cat test-count); fi
+  count=$((count + 1))
+  echo "$count" > test-count
+  [ "$count" -ne "$FAIL_TEST" ] || exit 42
+fi
+""")
+            cargo.chmod(0o755)
+            for tool in ("git", "rustc"):
+                path = tools / tool
+                path.write_text("#!/bin/sh\nexit 0\n")
+                path.chmod(0o755)
+            if package != "missing-script":
+                path = work / "packaging/freebsd/package.sh"
+                path.parent.mkdir(parents=True)
+                path.write_text("""#!/bin/bash
+set -euo pipefail
+touch package-ran
+case "$PACKAGE_MODE" in
+  failed) exit 43 ;;
+  absent) exit 0 ;;
+esac
+mkdir -p dist/release stage/bin
+echo fixture > stage/bin/app
+tar -czf dist/release/app-freebsd-x86_64.tar.gz stage
+if [ "$PACKAGE_MODE" = corrupt ]; then
+  echo corrupt > dist/release/app-freebsd-x86_64.tar.gz
+elif [ "$PACKAGE_MODE" = second-corrupt ]; then
+  echo corrupt > dist/release/other-freebsd-x86_64.tar.gz
+elif [ "$PACKAGE_MODE" = multiple ]; then
+  cp dist/release/app-freebsd-x86_64.tar.gz dist/release/other-freebsd-x86_64.tar.gz
+fi
+""")
+            env = os.environ.copy()
+            for key in ("CRAFT_FONTS_DIR", "CRAFT_FONTS_REQUIRED"):
+                env.pop(key, None)
+            env.update(PATH=str(tools) + os.pathsep + env["PATH"], FAIL_TEST=str(fail_test),
+                       PACKAGE_MODE=package, FAIL_BUILD=str(fail_build).lower())
+            result = subprocess.run(["/bin/sh", "-eu", "-c", script], cwd=work, env=env,
+                                    text=True, capture_output=True, timeout=15)
+            log = (work / "cargo.log").read_text().splitlines() if (work / "cargo.log").exists() else []
+            outcome = dict(code=result.returncode, packaged=(work / "package-ran").exists(),
+                           log=log, output=result.stdout + result.stderr)
+            return outcome
+
+    def test_each_failure_in_custom_chain_is_fatal_before_packaging(self):
+        command = " && ".join("cargo test --group " + str(n) for n in range(1, 5))
+        for group in range(1, 5):
+            with self.subTest(group=group):
+                result = self.run_vm(command=command, fail_test=group)
+                self.assertEqual(result["code"], 42, result)
+                self.assertFalse(result["packaged"], result)
+                self.assertEqual(len(result["log"]), 1 + group, result)
+
+    def test_each_failure_in_multiline_commands_stops_later_commands(self):
+        command = "\n".join("cargo test --group " + str(n) for n in range(1, 5))
+        for group in range(1, 5):
+            with self.subTest(group=group):
+                result = self.run_vm(command=command, fail_test=group)
+                self.assertEqual(result["code"], 42, result)
+                self.assertFalse(result["packaged"], result)
+                self.assertEqual(len(result["log"]), 1 + group, result)
+
+    def test_default_tests_must_pass_before_packaging(self):
+        result = self.run_vm(fail_test=1)
+        self.assertEqual(result["code"], 42, result)
+        self.assertFalse(result["packaged"], result)
+
+    def test_build_failure_is_fatal_before_tests(self):
+        result = self.run_vm(fail_build=True)
+        self.assertEqual(result["code"], 31, result)
+        self.assertEqual(len(result["log"]), 1, result)
+        self.assertFalse(result["packaged"], result)
+
+    def test_successful_custom_tests_reach_packaging(self):
+        result = self.run_vm(command="cargo test --lib\ncargo test --doc")
+        self.assertEqual(result["code"], 0, result)
+        self.assertEqual(len(result["log"]), 3, result)
+        self.assertTrue(result["packaged"], result)
+
+    def test_skip_tests_still_checks_requested_package(self):
+        result = self.run_vm(skip_tests=True, fail_test=1)
+        self.assertEqual(result["code"], 0, result)
+        self.assertEqual(len(result["log"]), 1, result)
+        self.assertTrue(result["packaged"], result)
+
+    def test_packaging_is_optional(self):
+        result = self.run_vm(package_check=False, package="missing-script")
+        self.assertEqual(result["code"], 0, result)
+        self.assertFalse(result["packaged"], result)
+
+    def test_requested_packaging_must_produce_readable_archives(self):
+        for mode in ("missing-script", "failed", "absent", "corrupt", "second-corrupt"):
+            with self.subTest(mode=mode):
+                result = self.run_vm(package=mode)
+                self.assertNotEqual(result["code"], 0, result)
+
+    def test_all_valid_archives_are_readable(self):
+        result = self.run_vm(package="multiple")
+        self.assertEqual(result["code"], 0, result)
+        self.assertEqual(result["output"].count("stage/bin/app"), 2, result)
+
+    def test_required_fonts_reach_build_and_both_test_paths(self):
+        for command in ("", "cargo test --lib\ncargo test --doc"):
+            with self.subTest(command=command):
+                result = self.run_vm(command=command, fonts=True, require_fonts=True)
+                self.assertEqual(result["code"], 0, result)
+                for line in result["log"]:
+                    _, directory, required, _ = line.split("|", 3)
+                    self.assertTrue(directory.endswith("/craft-fonts"), result)
+                    self.assertEqual(required, "1", result)
+
+    def test_required_missing_fonts_fail_before_build(self):
+        result = self.run_vm(require_fonts=True)
+        self.assertNotEqual(result["code"], 0, result)
+        self.assertEqual(result["log"], [], result)
+
+    def test_optional_fonts_remain_optional(self):
+        for fonts in (True, False):
+            with self.subTest(fonts=fonts):
+                result = self.run_vm(fonts=fonts)
+                self.assertEqual(result["code"], 0, result)
+                for line in result["log"]:
+                    _, directory, required, _ = line.split("|", 3)
+                    self.assertEqual(bool(directory), fonts, result)
+                    self.assertEqual(required, "", result)
+
+
+if __name__ == "__main__":
+    unittest.main()