daniel543x/craft-common-actions
1
0 Fork 0
mirror of https://github.com/storytold/craft-common-actions.git synced 2026-11-15 15:02:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into ci/opt-in-sccache

Browse Source 
# Conflicts:
#	.github/workflows/gate.yml
#	README.md
This commit is contained in:
FredApps committed 2026-11-13 20:46:39 +01:00
2 parents e59a0c94ab 5008f8cd1b
commit bc67c8a00c
48 files changed
+2732 -129
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.github/workflows/ast-grep.yml
+3
View File
@@ -40,6 +40,9 @@ jobs:
with:
egress-policy: block
allowed-endpoints: >
*.amazontrust.com:80
esm.ubuntu.com:443
motd.ubuntu.com:443
api.github.com:443
github.com:443
objects.githubusercontent.com:443
.github/workflows/docker-build.yml
+8 -8
View File
@@ -142,11 +142,11 @@ jobs:
run: bash --noprofile --norc -eo pipefail -c "$PRE_BUILD_COMMAND"
- name: Set up Buildx
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
uses: step-security/setup-buildx-action@e50058a0fcd8d1a7d626c905502d010a468bf238 # v4.4.1
- name: Log in to Docker Hub (DHI base images)
if: ${{ inputs.dhi-login }}
uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4.7.0
uses: step-security/docker-login-action@8c7b695d8738c5699a679ff5254baf0f19f20a94 # v4.6.0
with:
registry: dhi.io
username: ${{ secrets.DOCKERHUB_USERNAME }}
@@ -154,7 +154,7 @@ jobs:
- name: Log in to GHCR
if: ${{ contains(inputs.images, 'ghcr.io') }}
uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4.7.0
uses: step-security/docker-login-action@8c7b695d8738c5699a679ff5254baf0f19f20a94 # v4.6.0
with:
registry: ghcr.io
username: ${{ github.actor }}
@@ -162,7 +162,7 @@ jobs:
- name: Log in to Docker Hub
if: ${{ contains(inputs.images, 'docker.io/') }}
uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4.7.0
uses: step-security/docker-login-action@8c7b695d8738c5699a679ff5254baf0f19f20a94 # v4.6.0
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -268,11 +268,11 @@ jobs:
merge-multiple: true
- name: Set up Buildx
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
uses: step-security/setup-buildx-action@e50058a0fcd8d1a7d626c905502d010a468bf238 # v4.4.1
- name: Log in to GHCR
if: ${{ contains(inputs.images, 'ghcr.io') }}
uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4.7.0
uses: step-security/docker-login-action@8c7b695d8738c5699a679ff5254baf0f19f20a94 # v4.6.0
with:
registry: ghcr.io
username: ${{ github.actor }}
@@ -280,14 +280,14 @@ jobs:
- name: Log in to Docker Hub
if: ${{ contains(inputs.images, 'docker.io/') }}
uses: docker/login-action@81c6f93d4bc991e510b342a00b419b3bb4efc5c6 # v4.7.0
uses: step-security/docker-login-action@8c7b695d8738c5699a679ff5254baf0f19f20a94 # v4.6.0
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Docker metadata
id: meta
uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0
uses: step-security/metadata-action@21378f5b76c96c69ac34d21cc3873d85e26381b1 # v6.2.0
with:
images: ${{ inputs.images }}
tags: ${{ inputs.tags }}
.github/workflows/flatpak-oci.yml
+3
View File
@@ -47,6 +47,9 @@ jobs:
name: release-linux-${{ matrix.arch }}
path: dist/linux
- name: Configure apt mirrors
uses: storytold/craft-common-actions/actions/configure-apt-mirrors@main
- name: Install Flatpak and Skopeo
run: |
sudo apt-get update
.github/workflows/freebsd.yml
+7 -1
View File
@@ -94,8 +94,14 @@ jobs:
x86_64|aarch64) ;;
*) echo "::error::unsupported architecture '$ARCH' (use x86_64 or aarch64)"; exit 1 ;;
esac
# The VM action apt-installs QEMU on the host: give it the mirrors outside ubuntu.com, and
# drop the image's Microsoft source, which this job never installs from.
- name: Configure apt mirrors
uses: storytold/craft-common-actions/actions/configure-apt-mirrors@main
- name: Drop the Microsoft apt source
run: sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list /etc/apt/sources.list.d/microsoft-prod.sources
- name: Start the FreeBSD VM
uses: vmactions/freebsd-vm@c46abacb49f09938ca4e1702d15d836285d694cc # v1.5.9
uses: step-security/freebsd-vm@a2f5754275f7b0d5cea7f2762c9960727950f128 # v1.5.9
with:
release: ${{ inputs.freebsd-release == '15.1-STABLE' && '15.1' || inputs.freebsd-release }}
# Always an x86_64 guest, which KVM accelerates: aarch64 is cross-compiled in it,
.github/workflows/gate.yml
+25 -2
View File
@@ -17,6 +17,11 @@ on:
type: string
required: false
default: "cargo xtask ci"
setup-command:
description: "Optional setup script, run from the repository root before the gate (for example, fetch SHA-verified corpora)"
type: string
required: false
default: ""
windows-command:
description: "The gate on Windows legs; empty runs `command`"
type: string
@@ -128,6 +133,9 @@ jobs:
with:
egress-policy: audit
allowed-endpoints: >
*.amazontrust.com:80
esm.ubuntu.com:443
motd.ubuntu.com:443
*.crates.io:443
*.github.com:443
*.githubusercontent.com:443
@@ -141,8 +149,13 @@ jobs:
azure.archive.ubuntu.com:80
azure.archive.ubuntu.com:443
crates.io:443
ftp.tu-chemnitz.de:443
github.com:443
index.crates.io:443
*.dotsrc.org:443
*.kernel.org:443
mirrors.dotsrc.org:443
mirrors.edge.kernel.org:443
objects.githubusercontent.com:443
packages.microsoft.com:443
release-assets.githubusercontent.com:443
@@ -237,7 +250,7 @@ jobs:
- name: Install cargo tools
if: inputs.tools != ''
uses: taiki-e/install-action@b80e2bb2635f28eb98b9fce1e2f0887e1535c6b2 # v2.90.0
uses: step-security/taiki-e-install-action@76747d22a7567cc4438546a84f11bdf18e670ab8 # v2.87.32
with:
tool: ${{ inputs.tools }}
@@ -264,7 +277,7 @@ jobs:
# Default-branch writes remain the default. Maintainers can explicitly opt manual
# branch runs into branch-scoped writes; pull-request runs never save a cache.
- uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
- uses: step-security/rust-cache@8871c8ba063416da7c32246932ea6ae08700453e # v2.9.2
with:
prefix-key: ${{ inputs.cache-version }}-rust
key: ${{ steps.cache-profile.outputs.key }}
@@ -298,6 +311,16 @@ jobs:
echo "SCCACHE_GHA_RW_MODE=READ_ONLY" >> "$GITHUB_ENV"
fi
- name: Caller setup
if: inputs.setup-command != ''
shell: bash
env:
SETUP: ${{ inputs.setup-command }}
run: |
set -euo pipefail
printf '%s\n' "$SETUP" > "$RUNNER_TEMP/gate-setup.sh"
bash -euo pipefail "$RUNNER_TEMP/gate-setup.sh"
- name: Gate
shell: bash
env:
.github/workflows/house-rules.yml
+3
View File
@@ -35,6 +35,9 @@ jobs:
with:
egress-policy: block
allowed-endpoints: >
*.amazontrust.com:80
esm.ubuntu.com:443
motd.ubuntu.com:443
api.github.com:443
github.com:443
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
.github/workflows/packaging-lint.yml
+7
View File
@@ -24,6 +24,9 @@ jobs:
with:
egress-policy: block
allowed-endpoints: >
*.amazontrust.com:80
esm.ubuntu.com:443
motd.ubuntu.com:443
*.github.com:443
*.githubusercontent.com:443
*.microsoft.com:443
@@ -35,6 +38,10 @@ jobs:
azure.archive.ubuntu.com:80
azure.archive.ubuntu.com:443
github.com:443
*.dotsrc.org:443
*.kernel.org:443
mirrors.dotsrc.org:443
mirrors.edge.kernel.org:443
objects.githubusercontent.com:443
packages.microsoft.com:443
release-assets.githubusercontent.com:443
.github/workflows/release.yml
+84 -26
View File
@@ -84,6 +84,23 @@ on:
type: boolean
required: false
default: false
publish:
description: >-
Publish the draft when the run is done, instead of leaving it for a maintainer: only on
the release branch, only when no build job failed or was cancelled, and only when the
draft has at least as many assets as the last published release (a skipped job leaves the
run green).
type: boolean
required: false
default: false
dry-run:
description: >-
Build and check without releasing: no signing, no `release` environment, no macOS (its
larger runner is paid for), no Flatpak OCI push and no draft. For a nightly run on main
that finds release-only breakage (riscv64, Flatpak, FreeBSD, packaging) before release day.
type: boolean
required: false
default: false
secrets:
APPLE_CERTIFICATE:
@@ -214,9 +231,10 @@ jobs:
macos:
timeout-minutes: 90
name: macOS universal
if: ${{ !inputs.dry-run }}
needs: [version]
runs-on: macos-15-xlarge
environment: release
environment: ${{ !inputs.dry-run && 'release' || '' }}
env:
CARGO_TERM_COLOR: always
CARGO_INCREMENTAL: 0
@@ -277,11 +295,11 @@ jobs:
ref: ${{ inputs.craft-fonts-ref }}
- name: Check out craft-i18n
uses: storytold/craft-common-actions/actions/checkout-craft-i18n@main
- uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # v1
- uses: step-security/dtolnay-rust-toolchain@8dd607977d06adf4ed49b190381e74c17a9dd25f # v1.0.0
with:
toolchain: stable
targets: aarch64-apple-darwin, x86_64-apple-darwin
- uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
- uses: step-security/rust-cache@8871c8ba063416da7c32246932ea6ae08700453e # v2.9.2
with:
key: release-macos-universal
- name: Apply version override
@@ -335,14 +353,16 @@ jobs:
timeout-minutes: 90
name: Windows ${{ matrix.arch }}
needs: [version]
runs-on: windows-latest
environment: release
# arm64 builds natively on GitHub's Arm runner (free for public repositories). Cross-compiled
# on x64 it was the slowest job of the 11 Nov 2026 releases (30 to 39 minutes).
runs-on: ${{ matrix.runner }}
environment: ${{ !inputs.dry-run && 'release' || '' }}
strategy:
fail-fast: false
matrix:
include:
- { arch: x64, target: x86_64-pc-windows-msvc }
- { arch: arm64, target: aarch64-pc-windows-msvc }
- { arch: x64, target: x86_64-pc-windows-msvc, runner: windows-latest }
- { arch: arm64, target: aarch64-pc-windows-msvc, runner: windows-11-arm }
env:
CARGO_TERM_COLOR: always
CARGO_INCREMENTAL: 0
@@ -405,11 +425,11 @@ jobs:
ref: ${{ inputs.craft-fonts-ref }}
- name: Check out craft-i18n
uses: storytold/craft-common-actions/actions/checkout-craft-i18n@main
- uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # v1
- uses: step-security/dtolnay-rust-toolchain@8dd607977d06adf4ed49b190381e74c17a9dd25f # v1.0.0
with:
toolchain: stable
targets: ${{ matrix.target }}
- uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
- uses: step-security/rust-cache@8871c8ba063416da7c32246932ea6ae08700453e # v2.9.2
with:
key: release-windows-${{ matrix.arch }}
- name: Apply version override
@@ -446,7 +466,7 @@ jobs:
needs: [version]
# The signing key lives in the release environment, as it does for the macOS, Windows and
# flatpak jobs; without this the Linux packages are built unsigned.
environment: release
environment: ${{ !inputs.dry-run && 'release' || '' }}
runs-on: ${{ matrix.runner }}
strategy:
fail-fast: false
@@ -496,10 +516,10 @@ jobs:
with:
deb-arch: ${{ matrix.deb }}
nfpm-version: ${{ inputs.nfpm-version }}
- uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # v1
- uses: step-security/dtolnay-rust-toolchain@8dd607977d06adf4ed49b190381e74c17a9dd25f # v1.0.0
with:
toolchain: stable
- uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
- uses: step-security/rust-cache@8871c8ba063416da7c32246932ea6ae08700453e # v2.9.2
with:
key: release-linux-${{ matrix.arch }}
- name: Apply version override
@@ -543,7 +563,7 @@ jobs:
needs: [version, linux]
runs-on: ${{ matrix.runner }}
# LINUX_SIGNING_KEY comes from the repo's release environment, as the macOS secrets do.
environment: release
environment: ${{ !inputs.dry-run && 'release' || '' }}
permissions:
contents: read
packages: write
@@ -582,6 +602,8 @@ jobs:
with:
name: release-linux-${{ matrix.arch }}
path: dist/linux
- name: Configure apt mirrors
uses: storytold/craft-common-actions/actions/configure-apt-mirrors@main
- name: Install flatpak-builder
run: |
sudo apt-get update
@@ -617,7 +639,7 @@ jobs:
packaging/linux/flatpak-bundle.sh "$tarball"
printf 'tarball=%s\n' "$tarball" >> "$GITHUB_OUTPUT"
- name: Publish Flatpak OCI layout to GHCR and Docker Hub
if: github.event_name != 'pull_request'
if: github.event_name != 'pull_request' && !inputs.dry-run
env:
RELEASE_VERSION: ${{ needs.version.outputs.version }}
ARCH: ${{ matrix.arch }}
@@ -793,7 +815,7 @@ jobs:
- name: Check out craft-i18n
uses: storytold/craft-common-actions/actions/checkout-craft-i18n@main
- name: Start the FreeBSD VM
uses: vmactions/freebsd-vm@c46abacb49f09938ca4e1702d15d836285d694cc # v1.5.9
uses: step-security/freebsd-vm@a2f5754275f7b0d5cea7f2762c9960727950f128 # v1.5.9
with:
release: ${{ inputs.freebsd-release == '15.1-STABLE' && '15.1' || inputs.freebsd-release }}
# Always an x86_64 guest, which KVM accelerates: aarch64 is cross-compiled in it,
@@ -883,7 +905,11 @@ jobs:
uses: storytold/craft-common-actions/actions/freebsd-cargo-cache@main
with:
profile: release
configuration: ${{ toJSON(inputs) }}
# Only what changes the FreeBSD build. Not toJSON(inputs): contributors-artifact names
# this run (contributor-snapshot-<run id>-<attempt>), so no later run could restore the
# entry, and dry-run must share it, so that a dry run warms the release's cache.
configuration: ${{ format('{{"app-name":{0},"arch":{1},"craft-fonts-ref":{2},"freebsd-release":{3}}}', toJSON(inputs.app-name), toJSON(matrix.arch), toJSON(inputs.craft-fonts-ref), toJSON(inputs.freebsd-release)) }}
target: ${{ matrix.arch == 'aarch64' && 'aarch64-unknown-freebsd' || '' }}
source-hash: ${{ hashFiles('**/Cargo.lock', '**/Cargo.toml', '**/build.rs', '.cargo/**', 'rust-toolchain*', 'packaging/**', '.github/workflows/release.yml') }}
- name: Build and package
shell: freebsd {0}
@@ -967,7 +993,11 @@ jobs:
with:
phase: save
profile: release
configuration: ${{ toJSON(inputs) }}
# Only what changes the FreeBSD build. Not toJSON(inputs): contributors-artifact names
# this run (contributor-snapshot-<run id>-<attempt>), so no later run could restore the
# entry, and dry-run must share it, so that a dry run warms the release's cache.
configuration: ${{ format('{{"app-name":{0},"arch":{1},"craft-fonts-ref":{2},"freebsd-release":{3}}}', toJSON(inputs.app-name), toJSON(matrix.arch), toJSON(inputs.craft-fonts-ref), toJSON(inputs.freebsd-release)) }}
target: ${{ matrix.arch == 'aarch64' && 'aarch64-unknown-freebsd' || '' }}
source-hash: ${{ hashFiles('**/Cargo.lock', '**/Cargo.toml', '**/build.rs', '.cargo/**', 'rust-toolchain*', 'packaging/**', '.github/workflows/release.yml') }}
key: ${{ steps.cargo-cache.outputs.key }}
@@ -1009,11 +1039,11 @@ jobs:
app-name: ${{ inputs.app-name }}
version: ${{ needs.version.outputs.version }}
date: ${{ needs.version.outputs.date }}
- uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # v1
- uses: step-security/dtolnay-rust-toolchain@8dd607977d06adf4ed49b190381e74c17a9dd25f # v1.0.0
with:
toolchain: stable
targets: wasm32-unknown-unknown
- uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
- uses: step-security/rust-cache@8871c8ba063416da7c32246932ea6ae08700453e # v2.9.2
with:
key: release-web
- uses: storytold/craft-common-actions/actions/install-trunk@main
@@ -1049,7 +1079,7 @@ jobs:
name: Linux riscv64
if: inputs.has-riscv64
needs: [version]
environment: release
environment: ${{ !inputs.dry-run && 'release' || '' }}
# Not 22.04 like the other Linux jobs: Ubuntu never built jammy's ncurses security update for
# riscv64, so the amd64 and riscv64 libtinfo6 can't match and the cross deps don't install (#44).
runs-on: ubuntu-24.04
@@ -1094,11 +1124,11 @@ jobs:
ref: ${{ inputs.craft-fonts-ref }}
- name: Check out craft-i18n
uses: storytold/craft-common-actions/actions/checkout-craft-i18n@main
- uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # v1
- uses: step-security/dtolnay-rust-toolchain@8dd607977d06adf4ed49b190381e74c17a9dd25f # v1.0.0
with:
toolchain: stable
targets: riscv64gc-unknown-linux-gnu
- uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
- uses: step-security/rust-cache@8871c8ba063416da7c32246932ea6ae08700453e # v2.9.2
with:
key: release-linux-riscv64
- name: Apply version override
@@ -1189,10 +1219,10 @@ jobs:
date: ${{ needs.version.outputs.date }}
- name: Check out craft-i18n
uses: storytold/craft-common-actions/actions/checkout-craft-i18n@main
- uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # v1
- uses: step-security/dtolnay-rust-toolchain@8dd607977d06adf4ed49b190381e74c17a9dd25f # v1.0.0
with:
toolchain: stable
- uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
- uses: step-security/rust-cache@8871c8ba063416da7c32246932ea6ae08700453e # v2.9.2
with:
key: release-server-${{ matrix.arch }}
- name: Apply version override
@@ -1215,10 +1245,10 @@ jobs:
release:
timeout-minutes: 10
name: Draft GitHub Release
if: "!cancelled() && !contains(needs.*.result, 'failure')"
if: "!inputs.dry-run && !cancelled() && !contains(needs.*.result, 'failure')"
needs: [version, macos, windows, linux, linux-riscv64, flatpak, arch, freebsd, web, server]
runs-on: ubuntu-24.04
environment: release
environment: ${{ !inputs.dry-run && 'release' || '' }}
permissions:
contents: write
steps:
@@ -1233,3 +1263,31 @@ jobs:
prerelease: ${{ needs.version.outputs.prerelease }}
github-token: ${{ secrets.GITHUB_TOKEN }}
signing-key: ${{ github.ref == 'refs/heads/release' && secrets.LINUX_SIGNING_KEY || '' }}
- name: Publish the release
if: inputs.publish && !inputs.dry-run && github.ref == 'refs/heads/release'
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GH_REPO: ${{ github.repository }}
TAG: v${{ needs.version.outputs.version }}
CANCELLED: ${{ contains(needs.*.result, 'cancelled') }}
run: |
set -euo pipefail
refuse() {
echo "::error::$1"
echo "Not published: $1" >> "$GITHUB_STEP_SUMMARY"
exit 1
}
if [ "$CANCELLED" != false ]; then
refuse "a build job was cancelled: $TAG stays a draft"
fi
now="$(gh release view "$TAG" --json assets --jq '.assets | length')"
# The most recently published release; gh lists by creation, not publication.
last="$(gh release list --exclude-drafts --exclude-pre-releases --limit 50 --json tagName,publishedAt --jq 'sort_by(.publishedAt) | last | .tagName // ""')"
if [ -n "$last" ] && [ "$last" != "$TAG" ]; then
before="$(gh release view "$last" --json assets --jq '.assets | length')"
if [ "$now" -lt "$before" ]; then
refuse "$TAG has $now assets where $last had $before: it stays a draft"
fi
fi
gh release edit "$TAG" --draft=false
echo "Published $TAG ($now assets)." >> "$GITHUB_STEP_SUMMARY"
.github/workflows/sonarcloud.yml
+194 -28
View File
@@ -97,7 +97,11 @@ jobs:
uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4
with:
egress-policy: block
# Ubuntu Pro APT hooks use esm.ubuntu.com for security-update metadata.
# Keep this explicit alongside the archive hosts instead of allowing all Ubuntu subdomains.
allowed-endpoints: >
*.amazontrust.com:80
motd.ubuntu.com:443
*.amazonaws.com:443
*.crates.io:443
*.s3.amazonaws.com:443
@@ -109,8 +113,6 @@ jobs:
*.pythonhosted.org:443
*.sonarcloud.io:443
*.sonarsource.com:443
*.ubuntu.com:80
*.ubuntu.com:443
api.github.com:443
archive.ubuntu.com:80
archive.ubuntu.com:443
@@ -118,11 +120,16 @@ jobs:
azure.archive.ubuntu.com:443
binaries.sonarsource.com:443
crates.io:443
esm.ubuntu.com:443
files.pythonhosted.org:443
github.com:443
index.crates.io:443
keys.openpgp.org:443
keyserver.ubuntu.com:443
*.dotsrc.org:443
*.kernel.org:443
mirrors.dotsrc.org:443
mirrors.edge.kernel.org:443
objects.githubusercontent.com:443
packages.microsoft.com:443
pypi.org:443
@@ -168,7 +175,11 @@ jobs:
uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4
with:
egress-policy: block
# Ubuntu Pro APT hooks use esm.ubuntu.com for security-update metadata.
# Keep this explicit alongside the archive hosts instead of allowing all Ubuntu subdomains.
allowed-endpoints: >
*.amazontrust.com:80
motd.ubuntu.com:443
*.amazonaws.com:443
*.crates.io:443
*.s3.amazonaws.com:443
@@ -180,8 +191,6 @@ jobs:
*.pythonhosted.org:443
*.sonarcloud.io:443
*.sonarsource.com:443
*.ubuntu.com:80
*.ubuntu.com:443
api.github.com:443
archive.ubuntu.com:80
archive.ubuntu.com:443
@@ -189,11 +198,16 @@ jobs:
azure.archive.ubuntu.com:443
binaries.sonarsource.com:443
crates.io:443
esm.ubuntu.com:443
files.pythonhosted.org:443
github.com:443
index.crates.io:443
keys.openpgp.org:443
keyserver.ubuntu.com:443
*.dotsrc.org:443
*.kernel.org:443
mirrors.dotsrc.org:443
mirrors.edge.kernel.org:443
objects.githubusercontent.com:443
packages.microsoft.com:443
pypi.org:443
@@ -211,6 +225,8 @@ jobs:
uses: storytold/craft-common-actions/actions/checkout-craft-fonts@main
with:
ref: ${{ inputs.fonts-ref }}
- name: Check out craft-i18n
uses: storytold/craft-common-actions/actions/checkout-craft-i18n@main
- name: Require requested craft-fonts
if: inputs.fonts-ref != ''
shell: bash
@@ -234,10 +250,10 @@ jobs:
run: |
rustup toolchain install stable --profile minimal --component llvm-tools-preview
rustup default stable
- uses: taiki-e/install-action@b80e2bb2635f28eb98b9fce1e2f0887e1535c6b2 # v2.90.0
- uses: step-security/taiki-e-install-action@76747d22a7567cc4438546a84f11bdf18e670ab8 # v2.87.32
with:
tool: cargo-llvm-cov,cargo-nextest@0.9.150
- uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
- uses: step-security/rust-cache@8871c8ba063416da7c32246932ea6ae08700453e # v2.9.2
with:
save-if: ${{ github.ref == 'refs/heads/main' }}
- name: Provision pinned PDF test fonts
@@ -291,7 +307,11 @@ jobs:
uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4
with:
egress-policy: block
# Ubuntu Pro APT hooks use esm.ubuntu.com for security-update metadata.
# Keep this explicit alongside the archive hosts instead of allowing all Ubuntu subdomains.
allowed-endpoints: >
*.amazontrust.com:80
motd.ubuntu.com:443
*.amazonaws.com:443
*.crates.io:443
*.s3.amazonaws.com:443
@@ -303,8 +323,6 @@ jobs:
*.pythonhosted.org:443
*.sonarcloud.io:443
*.sonarsource.com:443
*.ubuntu.com:80
*.ubuntu.com:443
api.github.com:443
archive.ubuntu.com:80
archive.ubuntu.com:443
@@ -312,11 +330,16 @@ jobs:
azure.archive.ubuntu.com:443
binaries.sonarsource.com:443
crates.io:443
esm.ubuntu.com:443
files.pythonhosted.org:443
github.com:443
index.crates.io:443
keys.openpgp.org:443
keyserver.ubuntu.com:443
*.dotsrc.org:443
*.kernel.org:443
mirrors.dotsrc.org:443
mirrors.edge.kernel.org:443
objects.githubusercontent.com:443
packages.microsoft.com:443
pypi.org:443
@@ -334,6 +357,8 @@ jobs:
uses: storytold/craft-common-actions/actions/checkout-craft-fonts@main
with:
ref: ${{ inputs.fonts-ref }}
- name: Check out craft-i18n
uses: storytold/craft-common-actions/actions/checkout-craft-i18n@main
- name: Require requested craft-fonts
if: inputs.fonts-ref != ''
shell: bash
@@ -349,7 +374,7 @@ jobs:
pkg-config libxkbcommon-dev libwayland-dev libx11-dev libxcursor-dev libxrandr-dev
libxi-dev libgtk-3-dev libfontconfig1-dev libfreetype-dev libasound2-dev libsoup-3.0-dev libwebkit2gtk-4.1-dev
libgl1-mesa-dev mesa-vulkan-drivers
- uses: taiki-e/install-action@b80e2bb2635f28eb98b9fce1e2f0887e1535c6b2 # v2.90.0
- uses: step-security/taiki-e-install-action@76747d22a7567cc4438546a84f11bdf18e670ab8 # v2.87.32
with:
tool: cargo-nextest@0.9.150
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
@@ -371,8 +396,8 @@ jobs:
run: |
mkdir -p profraw
# nextest exits 100 when tests fail. A red test must not hide the static analysis, so
# that alone keeps the job green. Any other exit means the shard could not run its
# tests at all (for example a test binary nextest cannot list): fail the job, because a
# record that status for the scan job to fail after uploading analysis. Any other exit
# means the shard could not run its tests at all (for example a test binary nextest cannot list): fail the job, because a
# shard that runs nothing would scan an empty coverage report.
rc=0
cargo nextest run --archive-file archive/nextest-archive.tar.zst \
@@ -381,15 +406,73 @@ jobs:
echo "::error::nextest could not run shard $SHARD (exit $rc)"
exit "$rc"
fi
if ! ls profraw/*.profraw >/dev/null 2>&1; then
mkdir -p profdata
printf '%s\n' "$rc" > "profdata/shard-$SHARD.status"
if ls profraw/*.profraw >/dev/null 2>&1; then
profiles=true
else
profiles=false
fi
if [ "$rc" -eq 100 ]; then
echo "::warning::Coverage shard $SHARD/$TOTAL_SHARDS has failing tests; see this job's test log."
if [ "$profiles" = true ]; then
result="**Tests failed.** Coverage profiles are available, so the SonarCloud scan will continue. A green coverage job does not mean the tests passed; see this job's log for failed tests and diagnostics."
else
result='**Tests failed.** This shard produced no coverage profiles, so its tests cannot be included in the SonarCloud scan.'
fi
else
if [ "$profiles" = true ]; then
result='**Tests passed.** Coverage profiles are available for the SonarCloud scan.'
else
result='**Tests passed.** This shard produced no coverage profiles, so its tests cannot be included in the SonarCloud scan.'
fi
fi
printf '### Coverage shard %s/%s\n\n%s\n' "$SHARD" "$TOTAL_SHARDS" "$result" >> "$GITHUB_STEP_SUMMARY"
if [ "$profiles" != true ]; then
echo "::error::shard $SHARD produced no coverage profiles"
exit 1
fi
- name: Install LLVM tools for the archive's toolchain
run: rustup toolchain install "$(cat archive/rustc-release.txt)" --profile minimal --component llvm-tools-preview
- name: Merge this shard's profiles
# Raw profiles of a large workspace run to tens of gigabytes per shard; the scan job cannot
# hold every shard's at once. One sparse indexed profile per shard is a small fraction of that.
shell: bash
env:
SHARD: ${{ matrix.shard }}
run: |
set -euo pipefail
toolchain="$(cat archive/rustc-release.txt)"
llvm_bin="$(rustup run "$toolchain" rustc --print sysroot)/lib/rustlib/$(rustup run "$toolchain" rustc -vV | sed -n 's/^host: //p')/bin"
echo "Raw profiles: $(du -sh profraw | cut -f1) in $(find profraw -name '*.profraw' | wc -l) files"
df -h .
mkdir -p profdata
# failure-mode=any stops on any unreadable input. A profile cut short (a test process that
# crashed while writing it, or a full disk) fails this shard and names the file, instead of
# being uploaded and failing the scan's merge later with no shard to look at.
status=0
"$llvm_bin/llvm-profdata" merge -sparse --failure-mode=any profraw/*.profraw \
-o "profdata/shard-$SHARD.profdata" 2>merge-errors.txt || status=$?
if [ "$status" -ne 0 ]; then
cat merge-errors.txt >&2
sed -n 's/^warning: \(.*\.profraw\): \(.*\)$/\1: \2/p' merge-errors.txt | while IFS= read -r bad; do
echo "::error::shard $SHARD wrote a truncated or corrupt coverage profile: $bad"
done
echo "::error::llvm-profdata could not merge shard $SHARD's profiles (exit $status)"
rm -rf profdata
exit "$status"
fi
rm -rf profraw merge-errors.txt
echo "Merged profile: $(du -sh "profdata/shard-$SHARD.profdata" | cut -f1)"
df -h .
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: profraw-${{ matrix.shard }}
path: profraw/
if-no-files-found: ignore
name: profdata-${{ matrix.shard }}
path: profdata/
if-no-files-found: error
retention-days: 1
sonarcloud:
@@ -404,7 +487,11 @@ jobs:
uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4
with:
egress-policy: block
# Ubuntu Pro APT hooks use esm.ubuntu.com for security-update metadata.
# Keep this explicit alongside the archive hosts instead of allowing all Ubuntu subdomains.
allowed-endpoints: >
*.amazontrust.com:80
motd.ubuntu.com:443
*.amazonaws.com:443
*.crates.io:443
*.s3.amazonaws.com:443
@@ -416,8 +503,6 @@ jobs:
*.pythonhosted.org:443
*.sonarcloud.io:443
*.sonarsource.com:443
*.ubuntu.com:80
*.ubuntu.com:443
api.github.com:443
archive.ubuntu.com:80
archive.ubuntu.com:443
@@ -425,11 +510,16 @@ jobs:
azure.archive.ubuntu.com:443
binaries.sonarsource.com:443
crates.io:443
esm.ubuntu.com:443
files.pythonhosted.org:443
github.com:443
index.crates.io:443
keys.openpgp.org:443
keyserver.ubuntu.com:443
*.dotsrc.org:443
*.kernel.org:443
mirrors.dotsrc.org:443
mirrors.edge.kernel.org:443
objects.githubusercontent.com:443
packages.microsoft.com:443
pypi.org:443
@@ -469,12 +559,31 @@ jobs:
echo "::error::Invalid SonarCloud project key: must start with an alphanumeric character or underscore and contain only letters, digits, dash, underscore, period or colon (max 400 characters)."
exit 1
fi
printf 'key=%s\n' "$PROJECT_KEY" >>"$GITHUB_OUTPUT"
# A -D argument replaces the same property in sonar-project.properties, so the repository's
# own duplication exclusions are appended to the shared one here.
cpd='**/i18n/**'
own=""
if [[ -f sonar-project.properties ]]; then
# Properties syntax: optional blanks around `=` or `:`, and a CRLF file leaves a trailing \r.
own=$(sed -n 's/^[ \t]*sonar\.cpd\.exclusions[ \t]*[=:][ \t]*//p' sonar-project.properties | tail -n 1 | sed 's/[ \t\r]*$//')
fi
if [[ -n "$own" ]]; then
if [[ ! "$own" =~ ^[a-zA-Z0-9_.*?/,-]+$ ]]; then
echo "::error::Invalid sonar.cpd.exclusions in sonar-project.properties: only letters, digits, and _ . * ? / , - are allowed."
exit 1
fi
cpd="$cpd,$own"
fi
printf 'key=%s\ncpd=%s\n' "$PROJECT_KEY" "$cpd" >>"$GITHUB_OUTPUT"
- name: Check out craft-fonts
if: steps.token.outputs.available == 'true' && inputs.shards <= 1
uses: storytold/craft-common-actions/actions/checkout-craft-fonts@main
with:
ref: ${{ inputs.fonts-ref }}
- name: Check out craft-i18n
if: steps.token.outputs.available == 'true' && inputs.shards <= 1
id: i18n
uses: storytold/craft-common-actions/actions/checkout-craft-i18n@main
- name: Require requested craft-fonts
if: inputs.fonts-ref != '' && steps.token.outputs.available == 'true' && inputs.shards <= 1
shell: bash
@@ -504,11 +613,11 @@ jobs:
run: |
rustup toolchain install stable --profile minimal --component llvm-tools-preview
rustup default stable
- uses: taiki-e/install-action@b80e2bb2635f28eb98b9fce1e2f0887e1535c6b2 # v2.90.0
- uses: step-security/taiki-e-install-action@76747d22a7567cc4438546a84f11bdf18e670ab8 # v2.87.32
if: steps.token.outputs.available == 'true' && hashFiles('Cargo.toml') != '' && inputs.shards <= 1
with:
tool: cargo-llvm-cov
- uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
- uses: step-security/rust-cache@8871c8ba063416da7c32246932ea6ae08700453e # v2.9.2
if: steps.token.outputs.available == 'true' && hashFiles('Cargo.toml') != '' && inputs.shards <= 1
with:
save-if: ${{ github.ref == 'refs/heads/main' }}
@@ -518,6 +627,9 @@ jobs:
run: |
python3 scripts/fetch-pdf-test-fonts.py "$RUNNER_TEMP/craft-pdf-test-fonts"
echo "CRAFT_PDF_TEST_FONT_DIR=$RUNNER_TEMP/craft-pdf-test-fonts" >> "$GITHUB_ENV"
- name: Fetch locked Rust dependencies
if: steps.token.outputs.available == 'true' && hashFiles('Cargo.toml') != '' && inputs.shards <= 1
run: cargo fetch --locked
- name: Generate Rust coverage
id: coverage
if: steps.token.outputs.available == 'true' && hashFiles('Cargo.toml') != '' && inputs.shards <= 1
@@ -538,14 +650,23 @@ jobs:
if [ "$SQLX_OFFLINE_INPUT" = true ]; then export SQLX_OFFLINE=true; fi
features=()
if [ -n "$CARGO_FEATURES" ]; then features=(--features "$CARGO_FEATURES"); fi
cargo llvm-cov --workspace "${features[@]}" --ignore-run-fail --lcov --output-path lcov.info \
# Run every instrumented test and remember failures, but still create coverage so
# SonarCloud can publish diagnostics before this workflow reports a red result.
set +e
cargo llvm-cov --workspace "${features[@]}" --no-report --no-fail-fast \
--ignore-filename-regex "$COVERAGE_IGNORE"
test_status=$?
set -e
echo "test_status=$test_status" >> "$GITHUB_OUTPUT"
cargo llvm-cov report --lcov --output-path lcov.info \
--ignore-filename-regex "$COVERAGE_IGNORE"
- name: Download shard profiles and archive
if: steps.token.outputs.available == 'true' && inputs.shards > 1 && hashFiles('Cargo.toml') != ''
uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
with:
pattern: profraw-*
path: profraw
# One merged profile per shard (shard-N.profdata), never the raw profiles.
pattern: profdata-*
path: profdata
merge-multiple: true
- name: Download test archive
if: steps.token.outputs.available == 'true' && inputs.shards > 1 && hashFiles('Cargo.toml') != ''
@@ -564,10 +685,10 @@ jobs:
toolchain="$(cat archive/rustc-release.txt)"
llvm_bin="$(rustup run "$toolchain" rustc --print sysroot)/lib/rustlib/$(rustup run "$toolchain" rustc -vV | sed -n 's/^host: //p')/bin"
df -h . "$RUNNER_TEMP"
# Do not hold every raw profile beside the expanded archive. Retain the raw inputs
# Each shard merged its own raw profiles; combine those. Retain the shard profiles
# until the merge succeeds, then release them before unpacking the test binaries.
"$llvm_bin/llvm-profdata" merge -sparse profraw/*.profraw -o coverage.profdata
rm -f profraw/*.profraw
"$llvm_bin/llvm-profdata" merge -sparse profdata/*.profdata -o coverage.profdata
rm -f profdata/*.profdata
df -h . "$RUNNER_TEMP"
# Unpack outside the checkout: the scanner analyses everything under the workspace, and build
# output (generated sources) there would be counted as duplicated code.
@@ -656,9 +777,18 @@ jobs:
status=0
npm run --silent -- "$NODE_COVERAGE_SCRIPT" || status=$?
echo "status=$status" >>"$GITHUB_OUTPUT"
# The checkout is a build input, not caller source. Moving it after all coverage runs
# prevents an unsharded scan from counting another repository's files as this project.
- name: Move craft-i18n outside scan workspace
if: steps.i18n.outputs.rev != ''
shell: bash
run: |
set -euo pipefail
destination="$(mktemp -d "$RUNNER_TEMP/craft-i18n.XXXXXX")"
mv -- "$GITHUB_WORKSPACE/craft-i18n" "$destination/checkout"
- name: SonarCloud Scan
if: steps.token.outputs.available == 'true'
uses: SonarSource/sonarqube-scan-action@e805d875275f728fac1885b413934be6d7c63f65 # v8.4.1
uses: step-security/sonarqube-scan-action@c2312f8888f2bff4220bd815febb9d4a7cb32433 # v8.2.1
env:
SONAR_TOKEN: ${{ secrets.SONAR_CLOUD_PAT || secrets.SONAR_TOKEN }}
SONAR_HOST_URL: https://sonarcloud.io
@@ -667,7 +797,21 @@ jobs:
"-Dsonar.projectKey=${{ steps.project.outputs.key }}"
-Dsonar.organization=storytold
-Dsonar.rust.lcov.reportPaths=lcov.info
"-Dsonar.cpd.exclusions=**/i18n/**"
"-Dsonar.cpd.exclusions=${{ steps.project.outputs.cpd }}"
- name: Fail on Rust coverage generation failure
if: steps.coverage.outcome == 'failure'
shell: bash
run: |
echo "::error::cargo llvm-cov failed to build the test archive or write lcov.info"
exit 1
- name: Fail on Rust instrumented test failures
if: steps.coverage.outputs.test_status != '' && steps.coverage.outputs.test_status != '0'
shell: bash
env:
RUST_TEST_STATUS: ${{ steps.coverage.outputs.test_status }}
run: |
echo "::error::instrumented Rust tests exited with status $RUST_TEST_STATUS"
exit 1
- name: Fail on JavaScript/TypeScript test failures
if: steps.node-coverage.outputs.status != '' && steps.node-coverage.outputs.status != '0'
env:
@@ -675,3 +819,25 @@ jobs:
run: |
echo "::error::node-coverage-script exited with status $NODE_COVERAGE_STATUS"
exit 1
- name: Fail on Rust shard test failures
if: steps.token.outputs.available == 'true' && inputs.shards > 1 && hashFiles('Cargo.toml') != ''
shell: bash
env:
TOTAL_SHARDS: ${{ inputs.shards }}
run: |
set -euo pipefail
failed=0
for ((shard=1; shard<=TOTAL_SHARDS; shard++)); do
path="profdata/shard-$shard.status"
if [ ! -f "$path" ]; then
echo "::error::Missing test status for coverage shard $shard"
failed=1
continue
fi
status="$(cat "$path")"
if [ "$status" != 0 ]; then
echo "::error::Coverage shard $shard tests failed (exit $status); analysis was still uploaded"
failed=1
fi
done
exit "$failed"
.github/workflows/test-ast-grep.yml
+3
View File
@@ -27,6 +27,9 @@ jobs:
with:
egress-policy: block
allowed-endpoints: >
*.amazontrust.com:80
esm.ubuntu.com:443
motd.ubuntu.com:443
api.github.com:443
github.com:443
objects.githubusercontent.com:443
.github/workflows/test-freebsd.yml
+22
View File
@@ -28,6 +28,9 @@ jobs:
with:
egress-policy: block
allowed-endpoints: >
*.amazontrust.com:80
esm.ubuntu.com:443
motd.ubuntu.com:443
api.github.com:443
github.com:443
archive.ubuntu.com:80
@@ -38,10 +41,29 @@ jobs:
archive.ubuntu.com:443
azure.archive.ubuntu.com:443
security.ubuntu.com:443
*.dotsrc.org:443
*.kernel.org:443
mirrors.dotsrc.org:443
mirrors.edge.kernel.org:443
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Configure apt mirrors
uses: ./actions/configure-apt-mirrors
- name: Install YAML parser
run: sudo apt-get update && sudo apt-get install -y --no-install-recommends python3-yaml tcsh rpm debsigs dpkg-dev skopeo
# On 11 Nov 2026 one run found no candidate for tcsh or debsigs right after an update that
# fetched every index, while the runs before and after passed. These answer whether apt
# read those indexes, and whether this CPU runs the amd64v3 packages they list.
- name: Show why apt found no candidate
if: failure()
run: |
set +e
apt-cache policy tcsh debsigs
apt-config dump | grep -iE 'architecture|variant'
ls -l /var/lib/apt/lists/ | grep -E 'universe.*Packages'
grep -m1 'model name' /proc/cpuinfo
echo "avx2 cores: $(grep -c avx2 /proc/cpuinfo)"
/lib64/ld-linux-x86-64.so.2 --help | grep -A6 'glibc-hwcaps'
- name: Exercise failing tests, package validation and required fonts
run: /usr/bin/python3 -m unittest discover -s tests -v
.github/workflows/test-house-rules.yml
+3
View File
@@ -29,6 +29,9 @@ jobs:
with:
egress-policy: block
allowed-endpoints: >
*.amazontrust.com:80
esm.ubuntu.com:443
motd.ubuntu.com:443
api.github.com:443
github.com:443
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
.github/workflows/test-install-apt-packages.yml
+55 -1
View File
@@ -4,11 +4,13 @@ on:
pull_request:
paths:
- actions/install-apt-packages/**
- actions/configure-apt-mirrors/**
- .github/workflows/test-install-apt-packages.yml
push:
branches: [main]
paths:
- actions/install-apt-packages/**
- actions/configure-apt-mirrors/**
- .github/workflows/test-install-apt-packages.yml
workflow_dispatch:
@@ -17,14 +19,30 @@ permissions:
jobs:
install:
runs-on: ubuntu-24.04
name: install (${{ matrix.runner }}${{ matrix.outage && ', ubuntu.com down' || '' }})
runs-on: ${{ matrix.runner }}
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
include:
- { runner: ubuntu-24.04, outage: false }
- { runner: ubuntu-latest, outage: false }
# The images the apps' jobs use, with no ubuntu.com host reachable.
- { runner: ubuntu-22.04, outage: true }
- { runner: ubuntu-24.04, outage: true }
- { runner: ubuntu-latest, outage: true }
- { runner: ubuntu-22.04-arm, outage: true }
- { runner: ubuntu-24.04-arm, outage: true }
steps:
- name: Harden runner
uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4
with:
egress-policy: block
allowed-endpoints: >
*.amazontrust.com:80
esm.ubuntu.com:443
motd.ubuntu.com:443
api.github.com:443
github.com:443
archive.ubuntu.com:80
@@ -35,18 +53,54 @@ jobs:
archive.ubuntu.com:443
azure.archive.ubuntu.com:443
security.ubuntu.com:443
ports.ubuntu.com:80
ports.ubuntu.com:443
*.dotsrc.org:443
*.kernel.org:443
mirrors.dotsrc.org:443
mirrors.edge.kernel.org:443
ftp.tu-chemnitz.de:443
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Take ubuntu.com offline
if: matrix.outage
# As on 11 Nov 2026, when no ubuntu.com host resolved: the Ubuntu hosts the images use
# point at addresses that never answer, so only the mirrors outside ubuntu.com can serve.
run: |
set -euo pipefail
for host in azure.archive.ubuntu.com archive.ubuntu.com security.ubuntu.com ports.ubuntu.com; do
printf '192.0.2.1 %s\n100::1 %s\n' "$host" "$host"
done | sudo tee -a /etc/hosts
- name: Install packages
uses: ./actions/install-apt-packages
with:
# pkg-config is on every image and needs no download, so the outage legs add hello: a
# package no image has, and a list no cache was saved for (save: never), so it has to
# come from a mirror.
packages: |
pkg-config libxkbcommon-dev
mesa-vulkan-drivers
${{ matrix.outage && 'hello' || '' }}
save: ${{ matrix.outage && 'never' || 'default-branch' }}
- name: Check the packages and the kept downloads
run: |
set -euo pipefail
dpkg -s pkg-config libxkbcommon-dev mesa-vulkan-drivers > /dev/null
ls "$HOME"/.cache/craft-apt-debs/*.deb > /dev/null
test -z "$(find "$HOME/.cache/craft-apt-debs" ! -user "$(id -un)" -print -quit)"
- name: Check apt went around ubuntu.com
if: matrix.outage
run: |
set -euo pipefail
dpkg -s hello > /dev/null
cat /etc/apt/craft-apt-mirrors.txt
# The first mirror apt tries is outside ubuntu.com, and a fresh update of the Ubuntu
# sources gets every index, which apt can only have fetched from there. The image's
# other repositories (sources.list.d) are left out: they are not what this tests.
head -n 1 /etc/apt/craft-apt-mirrors.txt | grep -v 'ubuntu\.com'
parts=$(mktemp -d)
if [ -f /etc/apt/sources.list.d/ubuntu.sources ]; then
cp /etc/apt/sources.list.d/ubuntu.sources "$parts/"
fi
sudo apt-get update -o Dir::Etc::SourceParts="$parts" -o APT::Update::Error-Mode=any
.github/workflows/test-install-riscv64-cross.yml
+59
View File
@@ -0,0 +1,59 @@
name: install-riscv64-cross test
on:
pull_request:
paths:
- actions/install-riscv64-cross/**
- .github/workflows/test-install-riscv64-cross.yml
push:
branches: [main]
paths:
- actions/install-riscv64-cross/**
- .github/workflows/test-install-riscv64-cross.yml
workflow_dispatch:
permissions:
contents: read
jobs:
install:
name: install (${{ matrix.outage && 'ubuntu.com down' || 'ubuntu.com up' }})
# The image release.yml's Linux riscv64 job uses.
runs-on: ubuntu-24.04
timeout-minutes: 20
strategy:
fail-fast: false
matrix:
outage: [false, true]
steps:
- name: Harden runner
uses: step-security/harden-runner@1d88384fd173af6ecb012dfd0264a306b711241a # v2.22.4
with:
egress-policy: audit
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Take ubuntu.com offline
if: matrix.outage
# As on 11 Nov 2026, when no ubuntu.com host resolved on the runners.
run: |
set -euo pipefail
for host in azure.archive.ubuntu.com archive.ubuntu.com security.ubuntu.com ports.ubuntu.com; do
echo "192.0.2.1 $host"
echo "100::1 $host"
done | sudo tee -a /etc/hosts
- name: Install the riscv64 cross toolchain
uses: ./actions/install-riscv64-cross
- name: Check the cross toolchain and the sources it used
run: |
set -euo pipefail
dpkg -s gcc-riscv64-linux-gnu libc6-dev:riscv64 libxkbcommon-dev:riscv64 libasound2-dev:riscv64 > /dev/null
printf 'int main(void) { return 0; }\n' > "$RUNNER_TEMP/t.c"
riscv64-linux-gnu-gcc -o "$RUNNER_TEMP/t" "$RUNNER_TEMP/t.c"
cat /etc/apt/sources.list.d/amd64-primary.list /etc/apt/sources.list.d/riscv64.list
if [ "${{ matrix.outage }}" = true ]; then
# Nothing came from ubuntu.com: every Ubuntu source is a fallback mirror.
if grep -h ubuntu.com /etc/apt/sources.list.d/amd64-*.list /etc/apt/sources.list.d/riscv64.list; then
exit 1
fi
fi
.github/workflows/windows-arm64.yml
+8 -2
View File
@@ -60,6 +60,9 @@ jobs:
with:
egress-policy: audit
allowed-endpoints: >
*.amazontrust.com:80
esm.ubuntu.com:443
motd.ubuntu.com:443
*.crates.io:443
*.github.com:443
*.githubusercontent.com:443
@@ -76,14 +79,14 @@ jobs:
persist-credentials: false
- name: Check out craft-i18n
uses: storytold/craft-common-actions/actions/checkout-craft-i18n@main
- uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # v1
- uses: step-security/dtolnay-rust-toolchain@8dd607977d06adf4ed49b190381e74c17a9dd25f # v1.0.0
with:
toolchain: stable
targets: aarch64-pc-windows-msvc
# The nightly run on main saves; pull requests and `release` restore it. A pull request's
# cache (about 0.7 GB) is visible to that pull request alone, and the apps' caches sit at
# the 10 GB limit, so each one evicted a cache main's gate restores.
- uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3
- uses: step-security/rust-cache@8871c8ba063416da7c32246932ea6ae08700453e # v2.9.2
with:
key: windows-arm64-package
save-if: ${{ github.ref == 'refs/heads/main' }}
@@ -114,6 +117,9 @@ jobs:
with:
egress-policy: audit
allowed-endpoints: >
*.amazontrust.com:80
esm.ubuntu.com:443
motd.ubuntu.com:443
*.crates.io:443
*.github.com:443
*.githubusercontent.com:443
README.md
+57 -7
View File
@@ -26,6 +26,30 @@ Every build job sets `VERSION`, `BUILD_DATE` and `BUILD_SHA`, and the app's own
new app needs no change here; without that line it is `app-name` in capitals (`craft-anything`:
`CRAFT_ANYTHING`).
#### Cutting a release
`scripts/cut_release.py <owner/app>` cuts a release without a version-bump pull request, so a
change of version strings never waits on the full CI of a pull request:
1. It takes the newest commit on main whose `CI` run for a push passed (`--commit` picks another;
`--gate-workflow` names another workflow).
2. It commits the version bump there: `[workspace.package] version`, the internal path
dependencies, and Cargo.lock's entries for the workspace's own packages. A repository with
other files that name the version adds `packaging/set-version.sh <old> <new>`.
3. It merges that into `release` (main's version lines win) and pushes, which starts the app's
`release.yml`.
4. It opens a pull request that brings the same bump to main, with auto-merge on where the
repository allows it. The release doesn't wait for it.
5. With `--publish` it waits for `release.yml` and publishes the draft once every job passed and
the draft has assets. Otherwise it stops after the push.
The version defaults to the next minor after the last published release (`--part patch`, or
`--version 1.2.3-rc.1`). `--dry-run` does everything but push. It needs git, cargo (to check
Cargo.lock) and an authenticated `gh` whose account may update `release`.
`scripts/release_train.py --all --skip figcraft,soundcraft --publish` runs it for many apps at once, in
parallel, with a log per app and a summary of what was published or stopped.
#### Caller Example:
```yaml
name: Release
@@ -368,7 +392,13 @@ action's version prefix.
Cache data is restored on the host and explicitly transferred to the guest's
`/var/tmp/craft-cargo` and `/var/tmp/craft-target`. Only registry/git data and
`debug`/`release` artifacts are saved, excluding incremental state, Cargo
credentials/config and Rust toolchains. Successful non-PR runs save caches on a best-effort basis; PRs
credentials/config and Rust toolchains. A cross build passes its triple as `target`, and
its `target/<triple>/<profile>` output is saved and restored too: without it, the aarch64
release job kept only its build scripts and compiled everything else cold. `release.yml`
keys its cache on the inputs that change the FreeBSD build (app name, architecture,
craft-fonts ref and FreeBSD release), not on all its inputs: `contributors-artifact`
names the run, so a key that held it could never be restored, and a `dry-run` shares
the release's entry, so a dry run warms the cache a release restores. Successful non-PR runs save caches on a best-effort basis; PRs
only restore. Restore, checkpoint and save failures are non-blocking; each phase has a 10-minute
time budget. A failed
restore discards partial guest data before building; a failed checkpoint discards
@@ -483,6 +513,8 @@ The quality gate for an app's pull requests and pushes to main. It runs `command
- the app and craft-fonts (at the suite's pin, with `CRAFT_FONTS_REQUIRED=1`) checked out without
persisted credentials, and craft-i18n when the app has `craft-i18n.rev`;
- `packages` installed through `actions/install-apt-packages` on Linux legs (none when empty);
- an optional `setup-command` run from the repository root before the gate, for fetching pinned
test data such as licensed corpora; its environment changes can be written to `$GITHUB_ENV`;
- the channel from `rust-toolchain.toml`, or `toolchain`, with clippy, rustfmt and `targets`;
- prebuilt cargo tools from `tools` (by default `cargo-nextest@0.9.150`);
- `CARGO_PROFILE_DEV_DEBUG=line-tables-only` (set `dev-debug: ""` to keep the repository's
@@ -521,6 +553,11 @@ check, can be a second call with its own `command`, `os`, `tools` and `targets`,
command: packaging/web/package.sh
```
`setup-command` is also passed through the environment, then run with Bash from the checkout root.
Use it for caller-specific test inputs whose contents are pinned and verified by the caller's
fetcher. For example, a corpus-enabled workspace can fetch its locked corpus before running the
gate and export `CRAFT_PSD_CORPUS` through `$GITHUB_ENV`; a failed fetch stops the job before tests.
#### Caller Example:
```yaml
name: CI
@@ -765,7 +802,8 @@ Individual actions are also available for composing custom pipelines:
with:
packages: pkg-config libxkbcommon-dev libwayland-dev mesa-vulkan-drivers
```
- `actions/install-riscv64-cross`: Installs Ubuntu ports RISC-V multiarch and cross-compilers.
- `actions/configure-apt-mirrors`: Points the runner's Ubuntu archive sources at a mirror list that reaches beyond ubuntu.com, with the mirrors that answer first: the image's own mirrors (azure.archive, archive and security.ubuntu.com, or ports.ubuntu.com on arm), then `mirrors.edge.kernel.org` and `mirrors.dotsrc.org` (amd64) or `mirrors.dotsrc.org` and `ftp.tu-chemnitz.de` (arm). `install-apt-packages` runs it before every download attempt; run it before a plain `apt-get update`. A job whose Harden-Runner policy blocks egress must allow those hosts on port 443.
- `actions/install-riscv64-cross`: Installs Ubuntu ports RISC-V multiarch and cross-compilers. amd64 and riscv64 come from one pair of mirrors serving the same snapshot: Ubuntu's own (archive, security and ports.ubuntu.com), then `mirrors.dotsrc.org` for both, then `mirrors.edge.kernel.org` with `ftp.tu-chemnitz.de` (ports); the image's own Ubuntu sources are disabled in that job. A job whose Harden-Runner policy blocks egress must allow those hosts on port 443.
- `actions/install-trunk`: Installs Trunk for WebAssembly builds.
- `actions/draft-release`: Collects artifacts, signs the Linux packages when given `signing-key`, calculates (and signs) SHA256 checksums, and publishes/updates draft GitHub release.
- `actions/import-signing-key`: Imports an ASCII-armored OpenPGP signing key into a private `GNUPGHOME` and exports `GNUPGHOME`, `SIGNING_FPR` and `SIGNING_PUBKEY`; an empty key does nothing.
@@ -857,6 +895,12 @@ so missing embedded fonts fail explicitly. Callers with `scripts/fetch-pdf-test-
and set `CRAFT_PDF_TEST_FONT_DIR` before archive compilation and either test runner. Fixture provisioning
failures stop that job and prevent the dependent scan. The scan still needs to finish and pass its quality gate.
Sharded Rust coverage continues after test failures when the shard produced coverage profiles, so
SonarCloud can still analyse the code. Each shard reports its test outcome in the job summary;
failing tests also produce a warning annotation pointing to the test log. A green coverage job
therefore does not mean all tests passed. If a shard produces no profiles, its summary records
that the scan cannot include it and the shard fails. Runner errors also fail the shard.
Callers with Python tooling can pass both `python-coverage-source` (a source directory)
and `python-coverage-script` (the test script). The scan job runs that script under
pinned coverage.py in an isolated virtual environment and writes `python-coverage.xml`.
@@ -871,14 +915,20 @@ This step requires the Sonar token, so token-independent CI should also run the
Workspaces run their tests as four parallel `cargo nextest` partitions by
default. One job builds the instrumented test archive; each partition runs from
that archive and uploads its profile data, and the scan job merges the profiles
with the rustup `llvm-profdata` and `llvm-cov` tools. Pass `shards: 1` to keep
the single `cargo llvm-cov` job. Sharded runs skip doctests, because nextest
does not run them.
that archive, merges its own raw profiles into one sparse `.profdata` with the
rustup `llvm-profdata`, and uploads that plus a small test-exit-status file; the scan job merges the shard
profiles and exports them with `llvm-cov`. A raw profile that `llvm-profdata`
cannot read (for example one cut short by a test process that crashed while
writing it) fails its shard and is named in an error annotation, rather than
reaching the scan. Pass `shards: 1` to keep the single `cargo llvm-cov` job.
Sharded runs skip doctests, because nextest does not run them.
Ordinary test failures still allow coverage and static analysis to upload, but then
fail the scan job. Missing shard status also fails closed; a green analysis job
must not conceal failed tests. The single-shard Rust path is unchanged.
The scan job frees unused runner SDKs before either Rust coverage path. The archive build
frees them in the background, so the build starts at once. Sharded
coverage merges and removes raw profiles before extracting the test archive, then
coverage merges and removes the shard profiles before extracting the test archive, then
releases the compressed archive and temporary binaries as they are consumed.
Disk-space snapshots in the logs show headroom before cleanup and aggregation.
actions/configure-apt-mirrors/action.yml
+9
View File
@@ -0,0 +1,9 @@
name: "Configure apt mirrors"
description: "Points the runner's Ubuntu archive sources at a mirror list that reaches beyond ubuntu.com (kernel.org and dotsrc.org for amd64, dotsrc.org and TU Chemnitz for arm), with the mirrors that answer first. Run it before apt-get update; install-apt-packages already does."
runs:
using: "composite"
steps:
- name: Configure apt mirrors
shell: bash
run: sudo "$GITHUB_ACTION_PATH/configure-apt-mirrors.sh"
actions/configure-apt-mirrors/configure-apt-mirrors.sh
Executable
+134
View File
@@ -0,0 +1,134 @@
#!/usr/bin/env bash
# Points the runner's Ubuntu archive sources at a mirror list that does not depend on ubuntu.com
# alone, with the mirrors that answer right now first. Runs as root; safe to run again.
#
# The runner images read the archive from azure.archive, archive and security.ubuntu.com
# (through mirror+file:/etc/apt/apt-mirrors.txt), or on arm from ports.ubuntu.com. All of them
# sit in the ubuntu.com DNS zone: on 11 Nov 2026 none of them resolved for half an hour, apt
# spent each retry on 20-second lookups of the same dead names, and the jobs failed. So the list
# keeps the image's mirrors in the image's order and adds two full mirrors outside ubuntu.com.
# Each mirror is checked first: the ones that serve this release's InRelease keep their order,
# the others move to the end, where apt still falls back to them. Every mirror carries the same
# archive signed with Ubuntu's key, which apt checks wherever the files come from.
#
# Only the image's own source files change. A list another action adds keeps its own URIs:
# install-riscv64-cross pins amd64 and riscv64 to one pair of mirrors so both architectures match.
#
# A job whose Harden-Runner policy blocks egress must allow these hosts on port 443:
# amd64: mirrors.edge.kernel.org, mirrors.dotsrc.org (the workflows allow *.kernel.org, *.dotsrc.org)
# arm64: mirrors.dotsrc.org, ftp.tu-chemnitz.de
# A blocked mirror only fails its check and goes to the end of the list. apt's SRV lookups
# (_https._tcp.<host>) are turned off: Harden-Runner blocks those names, and apt does not need them.
set -euo pipefail
# cloud-init may still be about to write ubuntu.sources; changing it before then is undone.
"$(dirname "${BASH_SOURCE[0]}")/wait-for-cloud-init.sh"
# CRAFT_APT_ROOT and CRAFT_APT_ARCH stand in for / and the dpkg architecture in tests.
root=${CRAFT_APT_ROOT:-}
apt_dir="$root/etc/apt"
list_name=craft-apt-mirrors.txt
list_uri="mirror+file:/etc/apt/$list_name"
arch=${CRAFT_APT_ARCH:-$(dpkg --print-architecture)}
codename=$(sed -n 's/^VERSION_CODENAME=//p' "$root/etc/os-release" | tr -d '"')
if [ -z "$codename" ]; then
echo "::warning::configure-apt-mirrors: no VERSION_CODENAME in /etc/os-release; apt sources left as they are"
exit 0
fi
case "$arch" in
amd64 | i386)
mirrors=(
http://azure.archive.ubuntu.com/ubuntu/
https://archive.ubuntu.com/ubuntu/
https://security.ubuntu.com/ubuntu/
https://mirrors.edge.kernel.org/ubuntu/
https://mirrors.dotsrc.org/ubuntu/
)
;;
*)
mirrors=(
https://ports.ubuntu.com/ubuntu-ports/
https://mirrors.dotsrc.org/ubuntu-ports/
https://ftp.tu-chemnitz.de/pub/linux/ubuntu-ports/
)
;;
esac
# The image's sources: one-line entries in sources.list (22.04) or the deb822 ubuntu.sources
# (24.04 and later).
sources=()
for file in "$apt_dir/sources.list" "$apt_dir/sources.list.d/ubuntu.sources"; do
if [ -f "$file" ]; then
sources+=("$file")
fi
done
if [ "${#sources[@]}" -eq 0 ]; then
echo "::warning::configure-apt-mirrors: no sources.list or ubuntu.sources; apt sources left as they are"
exit 0
fi
# Check every mirror at once, so a dead one costs one timeout rather than one per mirror.
results=$(mktemp -d)
trap 'rm -rf "$results"' EXIT
for i in "${!mirrors[@]}"; do
(
if curl --proto '=http,https' -fsS -o /dev/null -r 0-0 --connect-timeout 5 --max-time 10 \
"${mirrors[$i]}dists/$codename/InRelease" 2> "$results/$i.err"; then
touch "$results/$i.ok"
fi
) &
done
wait
ordered=()
late=()
for i in "${!mirrors[@]}"; do
if [ -e "$results/$i.ok" ]; then
ordered+=("${mirrors[$i]}")
else
late+=("${mirrors[$i]}")
reason=$(tr '\n' ' ' < "$results/$i.err")
echo "::warning::configure-apt-mirrors: ${mirrors[$i]} did not serve $codename/InRelease (${reason:-no answer}); apt tries it last"
fi
done
ordered+=("${late[@]}")
{
for i in "${!ordered[@]}"; do
printf '%s\tpriority:%d\n' "${ordered[$i]}" "$((i + 1))"
done
} > "$results/list"
install -m 0644 "$results/list" "$apt_dir/$list_name"
mkdir -p "$apt_dir/apt.conf.d"
echo 'Acquire::EnableSrvRecords "false";' > "$apt_dir/apt.conf.d/99craft-no-srv"
# Each Ubuntu archive URI (or the image's own mirror list) becomes the new list. A deb822 URIs
# field naming several of them keeps one copy. Brackets, not backslashes: awk -v would eat those.
pattern='^(mirror[+]file:/etc/apt/apt-mirrors[.]txt|https?://(azure[.]archive|([a-z][a-z][.])?archive|security|([a-z][a-z][.])?ports)[.]ubuntu[.]com/ubuntu(-ports)?/?)$'
for file in "${sources[@]}"; do
awk -v pattern="$pattern" -v list="$list_uri" '
/^[[:space:]]*(deb|deb-src)[[:space:]]/ || /^URIs:/ {
out = ""
seen = 0
for (i = 1; i <= NF; i++) {
field = $i
if (field ~ pattern) {
if (seen) continue
field = list
seen = 1
}
out = out (i > 1 ? " " : "") field
}
print out
next
}
{ print }
' "$file" > "$results/source"
if ! cmp -s "$results/source" "$file"; then
cat "$results/source" > "$file"
fi
done
echo "apt mirrors for $codename ($arch), in the order apt tries them:"
sed 's/^/ /' "$apt_dir/$list_name"
actions/configure-apt-mirrors/wait-for-cloud-init.sh
Executable
+30
View File
@@ -0,0 +1,30 @@
#!/usr/bin/env bash
# Waits until cloud-init has finished, so it cannot rewrite the apt sources after we have.
#
# cloud-init's apt_configure module writes /etc/apt/sources.list.d/ubuntu.sources from its
# template once per instance, after looking for a mirror. That usually ends before a job starts.
# On 11 Nov 2026, with no ubuntu.com name resolving, its mirror search sat through two 10-second
# DNS timeouts and the write landed in the middle of jobs: apt-get update read the sources
# configure-apt-mirrors had written, cloud-init then put the image's sources back, and
# apt-get install found no indexes for them, so no Ubuntu package had an installation candidate.
#
# CRAFT_CLOUD_INIT names the cloud-init command (tests use a fake). Without cloud-init, or once
# it has finished, this returns at once. It never fails the job: after CRAFT_CLOUD_INIT_TIMEOUT
# seconds (default 300) it warns and lets the caller go on.
set -uo pipefail
cloud_init=${CRAFT_CLOUD_INIT:-cloud-init}
limit=${CRAFT_CLOUD_INIT_TIMEOUT:-300}
if ! command -v "$cloud_init" > /dev/null 2>&1; then
exit 0
fi
start=$SECONDS
timeout "$limit" "$cloud_init" status --wait > /dev/null 2>&1
status=$?
waited=$((SECONDS - start))
if [ "$status" -eq 124 ]; then
echo "::warning::cloud-init was still running after ${limit}s; it may rewrite the apt sources"
elif [ "$waited" -gt 0 ]; then
echo "Waited ${waited}s for cloud-init to finish before changing the apt sources."
fi
exit 0
actions/freebsd-cargo-cache/action.yml
+17 -2
View File
@@ -16,6 +16,11 @@ inputs:
key:
description: Primary key returned by the restore invocation (package/save)
default: ""
target:
description: >-
Rust target triple of a cross build (e.g. aarch64-unknown-freebsd), whose output Cargo
writes to target/<triple>/<profile> rather than target/<profile>. Empty for a native build.
default: ""
outputs:
key:
description: Primary cache key for the subsequent save invocation
@@ -77,6 +82,7 @@ runs:
env:
CACHE_PHASE: ${{ inputs.phase }}
CACHE_PROFILE: ${{ inputs.profile }}
CACHE_TARGET: ${{ inputs.target }}
# PRs may restore their base branch's cache, but never write it.
CACHE_CAN_SAVE: ${{ github.event_name != 'pull_request' && github.event_name != 'pull_request_target' }}
run: |
@@ -85,6 +91,7 @@ runs:
# host archive outside that tree to avoid a second copy inside the VM.
cache="$HOME/.cache/craft-freebsd-cargo"
case "$CACHE_PROFILE" in debug|release) ;; *) echo 'invalid cache profile' >&2; exit 1 ;; esac
case "$CACHE_TARGET" in *[!a-z0-9_-]*) echo 'invalid cache target' >&2; exit 1 ;; esac
ssh_args='ssh -o ServerAliveInterval=30 -o ServerAliveCountMax=60 -o ConnectTimeout=120'
remote_rsync='env PATH=/usr/local/bin:/usr/bin:/bin rsync'
push() {
@@ -123,6 +130,13 @@ runs:
push git /var/tmp/craft-cargo/git
# Do not load both profiles into the space-constrained VM at once.
push "target/$CACHE_PROFILE" "/var/tmp/craft-target/$CACHE_PROFILE"
# A cross build keeps only its build scripts and proc macros in target/<profile>;
# everything it builds for the target is under target/<triple>/<profile>.
if [ -n "$CACHE_TARGET" ]; then
# shellcheck disable=SC2029
ssh freebsd "mkdir -p /var/tmp/craft-target/$CACHE_TARGET/$CACHE_PROFILE"
push "target/$CACHE_TARGET/$CACHE_PROFILE" "/var/tmp/craft-target/$CACHE_TARGET/$CACHE_PROFILE"
fi
;;
package)
if [ "$CACHE_CAN_SAVE" = true ]; then
@@ -137,8 +151,9 @@ runs:
# excluded. Keep the debug checkpoint when packaging removed it.
pull registry /var/tmp/craft-cargo/registry
pull git /var/tmp/craft-cargo/git
for profile in debug release; do
# profile is one of the two literal directory names above.
for profile in debug release ${CACHE_TARGET:+"$CACHE_TARGET/debug" "$CACHE_TARGET/release"}; do
# profile is one of the literal directory names above, under the validated triple
# for a cross build.
# shellcheck disable=SC2029
if ssh freebsd "test -d /var/tmp/craft-target/$profile"; then
pull "target/$profile" "/var/tmp/craft-target/$profile"
actions/install-apt-packages/action.yml
+6
View File
@@ -57,6 +57,7 @@ runs:
# Seconds one attempt at a mirror download may take, and how many attempts are made.
APT_NETWORK_TIMEOUT: "300"
APT_NETWORK_ATTEMPTS: "3"
CONFIGURE_MIRRORS: ${{ github.action_path }}/../configure-apt-mirrors/configure-apt-mirrors.sh
run: |
set -euo pipefail
debs="$HOME/.cache/craft-apt-debs"
@@ -72,9 +73,14 @@ runs:
# Acquire timeouts don't end a connection that stays open and sends nothing: one such
# stall held a job for 58 minutes until its own timeout. So each mirror download also has
# a wall-clock limit and is retried. Only downloads are limited; dpkg is never killed.
# Before each attempt the mirrors are checked again (configure-apt-mirrors): one that
# stopped answering moves to the end of the list, and mirrors outside ubuntu.com take
# over when no ubuntu.com host resolves. Without the check, every attempt spent its
# time on the same dead names.
network() {
local attempt status
for ((attempt = 1; ; attempt++)); do
sudo "$CONFIGURE_MIRRORS" || echo "::warning::install-apt-packages: configuring the apt mirrors failed; keeping the current sources"
status=0
sudo timeout --kill-after=30 "$APT_NETWORK_TIMEOUT" apt-get "${opts[@]}" "$@" || status=$?
if [ "$status" -eq 0 ] || [ "$attempt" -ge "$APT_NETWORK_ATTEMPTS" ]; then
actions/install-linux-deps/action.yml
+2
View File
@@ -17,6 +17,8 @@ runs:
- name: Install build and packaging tools
shell: bash
run: |
# A mirror list that reaches beyond ubuntu.com, so an outage there does not stop the job.
sudo "$GITHUB_ACTION_PATH/../configure-apt-mirrors/configure-apt-mirrors.sh"
sudo apt-get update
sudo apt-get install -y --no-install-recommends \
libxkbcommon-dev libwayland-dev libx11-dev libxrandr-dev libxi-dev libgl1-mesa-dev \
actions/install-riscv64-cross/action.yml
+3 -23
View File
@@ -7,30 +7,10 @@ runs:
- name: Install cross-compilation dependencies
shell: bash
run: |
. /etc/os-release
sudo dpkg --add-architecture riscv64
# Pin the existing (archive.ubuntu.com) sources to amd64; riscv64 comes from ports.
if [ -f /etc/apt/sources.list.d/ubuntu.sources ]; then
sudo sed -i '/^Types: deb/a Architectures: amd64' /etc/apt/sources.list.d/ubuntu.sources
fi
if [ -f /etc/apt/sources.list ]; then
sudo sed -i 's/^deb /deb [arch=amd64] /g' /etc/apt/sources.list
sudo sed -i 's/^deb-src /deb-src [arch=amd64] /g' /etc/apt/sources.list
fi
for suite in "$VERSION_CODENAME" "$VERSION_CODENAME-updates" "$VERSION_CODENAME-security"; do
echo "deb [arch=riscv64] https://ports.ubuntu.com/ubuntu-ports $suite main universe restricted multiverse"
done | sudo tee /etc/apt/sources.list.d/riscv64.list
# Multi-Arch: same packages (libc6, linux-libc-dev, zlib1g, ...) must match across amd64 and
# riscv64. ports.ubuntu.com publishes a suite as soon as archive.ubuntu.com does, but the
# runner's amd64 mirror (azure.archive.ubuntu.com) lags for hours, leaving apt without a
# matching pair: it reports "held broken packages" and names a dependency such as
# linux-libc-dev:riscv64 that "is not going to be installed". Read amd64 from the primary
# for every suite, security included, so both sides always offer the same versions.
for suite in "$VERSION_CODENAME" "$VERSION_CODENAME-updates"; do
echo "deb [arch=amd64] https://archive.ubuntu.com/ubuntu $suite main universe restricted multiverse"
done | sudo tee /etc/apt/sources.list.d/amd64-primary.list
echo "deb [arch=amd64] https://security.ubuntu.com/ubuntu $VERSION_CODENAME-security main universe restricted multiverse" \
| sudo tee /etc/apt/sources.list.d/amd64-security.list
# amd64 and riscv64 from one pair of mirrors serving the same snapshot, with fallbacks
# outside ubuntu.com; the script explains the choice.
sudo "$GITHUB_ACTION_PATH/select-mirrors.sh"
sudo apt-get update
sudo apt-get install -y --no-install-recommends \
gcc-riscv64-linux-gnu binutils-riscv64-linux-gnu libc6-dev-riscv64-cross libc6-dev:riscv64 qemu-user \
actions/install-riscv64-cross/select-mirrors.sh
Executable
+125
View File
@@ -0,0 +1,125 @@
#!/usr/bin/env bash
# Chooses where install-riscv64-cross reads Ubuntu from, and writes those apt sources.
# Runs as root.
#
# Multi-Arch: same packages (libc6, linux-libc-dev, zlib1g, ...) must have the same version for
# amd64 and riscv64, so both architectures have to come from one snapshot of the archive. The
# runner's amd64 mirror (azure.archive.ubuntu.com) lags ports.ubuntu.com for hours, leaving apt
# without a matching pair: it reports "held broken packages" and names a dependency such as
# linux-libc-dev:riscv64 that "is not going to be installed". So this job reads amd64 and
# riscv64 from one pair of mirrors that serve the same snapshot, and nothing else.
#
# The pairs, in order: Ubuntu's own (archive and security.ubuntu.com, ports.ubuntu.com); dotsrc,
# which carries both trees; kernel.org with TU Chemnitz's ports. On 11 Nov 2026 no ubuntu.com
# name resolved on the runners, so the first pair alone left the job without riscv64 packages.
# A pair is used when both sides serve every suite and their InRelease files carry the same
# Date: Ubuntu signs one Release for archive and ports, so equal dates mean the same snapshot.
# When no pair matches, the first pair that answers is used anyway, with a warning.
#
# The image's own Ubuntu sources are disabled: the lists written here cover every suite the job
# installs from, and the image's mirrors would only add a second snapshot to mix in.
set -euo pipefail
# cloud-init may still be about to write ubuntu.sources (and re-enable it); wait for it first.
"$(dirname "${BASH_SOURCE[0]}")/../configure-apt-mirrors/wait-for-cloud-init.sh"
# CRAFT_APT_ROOT stands in for / in tests.
root=${CRAFT_APT_ROOT:-}
apt_dir="$root/etc/apt"
codename=$(sed -n 's/^VERSION_CODENAME=//p' "$root/etc/os-release" | tr -d '"')
components="main universe restricted multiverse"
suites=("$codename" "$codename-updates" "$codename-security")
# amd64 base, amd64 security pocket, riscv64 (ports) base.
pairs=(
"https://archive.ubuntu.com/ubuntu https://security.ubuntu.com/ubuntu https://ports.ubuntu.com/ubuntu-ports"
"https://mirrors.dotsrc.org/ubuntu https://mirrors.dotsrc.org/ubuntu https://mirrors.dotsrc.org/ubuntu-ports"
"https://mirrors.edge.kernel.org/ubuntu https://mirrors.edge.kernel.org/ubuntu https://ftp.tu-chemnitz.de/pub/linux/ubuntu-ports"
)
tmp=$(mktemp -d)
trap 'rm -rf "$tmp"' EXIT
# Every InRelease of every pair at once, so dead mirrors cost one timeout in all.
for p in "${!pairs[@]}"; do
read -r amd64 security ports <<< "${pairs[$p]}"
for s in "${!suites[@]}"; do
suite=${suites[$s]}
base=$amd64
if [[ "$suite" == *-security ]]; then
base=$security
fi
for side in amd64 riscv64; do
url="$base/dists/$suite/InRelease"
if [ "$side" = riscv64 ]; then
url="$ports/dists/$suite/InRelease"
fi
(
if curl --proto '=https' -fsS --connect-timeout 5 --max-time 30 -o "$tmp/$p-$s-$side" "$url" \
2> "$tmp/$p-$s-$side.err"; then
touch "$tmp/$p-$s-$side.ok"
fi
) &
done
done
done
wait
chosen=""
reachable=""
for p in "${!pairs[@]}"; do
answered=1
same=1
for s in "${!suites[@]}"; do
for side in amd64 riscv64; do
if [ ! -e "$tmp/$p-$s-$side.ok" ]; then
answered=0
echo "::warning::install-riscv64-cross: no ${suites[$s]} InRelease for $side from pair $((p + 1)) ($(tr '\n' ' ' < "$tmp/$p-$s-$side.err"))"
fi
done
if [ "$answered" = 1 ]; then
if ! grep -q 'main/binary-riscv64/Packages' "$tmp/$p-$s-riscv64"; then
answered=0
echo "::warning::install-riscv64-cross: pair $((p + 1)) lists no riscv64 packages for ${suites[$s]}"
elif [ "$(grep -m1 '^Date:' "$tmp/$p-$s-amd64")" != "$(grep -m1 '^Date:' "$tmp/$p-$s-riscv64")" ]; then
same=0
fi
fi
done
if [ "$answered" = 1 ] && [ "$same" = 1 ]; then
chosen=$p
break
fi
if [ "$answered" = 1 ]; then
echo "::warning::install-riscv64-cross: pair $((p + 1)) serves different amd64 and riscv64 snapshots"
if [ -z "$reachable" ]; then
reachable=$p
fi
fi
done
if [ -z "$chosen" ]; then
chosen=${reachable:-0}
echo "::warning::install-riscv64-cross: no mirror pair serves one snapshot for both architectures; using pair $((chosen + 1)), so apt may report held broken packages"
fi
read -r amd64 security ports <<< "${pairs[$chosen]}"
mkdir -p "$apt_dir/sources.list.d"
for suite in "$codename" "$codename-updates"; do
echo "deb [arch=amd64] $amd64 $suite $components"
done > "$apt_dir/sources.list.d/amd64-primary.list"
echo "deb [arch=amd64] $security $codename-security $components" > "$apt_dir/sources.list.d/amd64-security.list"
for suite in "${suites[@]}"; do
echo "deb [arch=riscv64] $ports $suite $components"
done > "$apt_dir/sources.list.d/riscv64.list"
# The image's Ubuntu sources: deb822 stanzas get Enabled: no, one-line entries are commented out.
if [ -f "$apt_dir/sources.list.d/ubuntu.sources" ] && ! grep -q '^Enabled: no' "$apt_dir/sources.list.d/ubuntu.sources"; then
sed -i '/^Types: /a Enabled: no' "$apt_dir/sources.list.d/ubuntu.sources"
fi
if [ -f "$apt_dir/sources.list" ]; then
sed -i -E 's/^(deb|deb-src)([[:space:]])/# install-riscv64-cross: \1\2/' "$apt_dir/sources.list"
fi
echo "Ubuntu sources for amd64 and riscv64 (pair $((chosen + 1))):"
cat "$apt_dir/sources.list.d/amd64-primary.list" "$apt_dir/sources.list.d/amd64-security.list" \
"$apt_dir/sources.list.d/riscv64.list" | sed 's/^/ /'
actions/packaging-lint/action.yml
+1
View File
@@ -20,6 +20,7 @@ runs:
echo "::error::invalid actionlint-version '$version'"
exit 1
fi
sudo "$GITHUB_ACTION_PATH/../configure-apt-mirrors/configure-apt-mirrors.sh"
sudo apt-get update
sudo apt-get install -y --no-install-recommends shellcheck desktop-file-utils appstream libxml2-utils python3-yaml
# GitHub release downloads can return transient 503s before lint starts.
actions/sign-rpms/action.yml
+1
View File
@@ -15,6 +15,7 @@ runs:
run: |
set -euo pipefail
if ! command -v rpm >/dev/null 2>&1; then
sudo "$GITHUB_ACTION_PATH/../configure-apt-mirrors/configure-apt-mirrors.sh"
sudo apt-get update
sudo apt-get install -y --no-install-recommends rpm
fi
scripts/cut_release.py
+339
View File
@@ -0,0 +1,339 @@
#!/usr/bin/env python3
"""Cut a Craft app release without a version-bump pull request.
The release builds from the newest commit on main whose CI passed; the version bump is a commit
on top of it that goes straight to `release`, whose push starts the app's release.yml. The same
bump goes back to main as a pull request that nothing waits for, so main carries the released
version too. With --publish the script waits for release.yml and publishes the draft when every
job passed and the draft has assets.
scripts/cut_release.py storytold/pdfcraft # next minor, from green main
scripts/cut_release.py storytold/pdfcraft --version 0.22.0-rc.1 --publish
scripts/cut_release.py storytold/pdfcraft --dry-run # shows what it would push
Needs git, an authenticated `gh` whose account may update `release`, and (to check Cargo.lock)
cargo. Only the Python standard library.
What the bump changes, as the release commits before it did: `[workspace.package] version` in
the root Cargo.toml, `version = "<old>"` of the internal path dependencies there, and Cargo.lock's
entries for the workspace's own packages. A repository with other files that name the version
adds `packaging/set-version.sh`, run as `packaging/set-version.sh <old> <new>` after the Cargo edits.
"""
from __future__ import annotations
import argparse
import json
import os
import re
import shutil
import subprocess
import sys
import tempfile
import time
from pathlib import Path
SEMVER = re.compile(r"^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?$")
HOOK = "packaging/set-version.sh"
# ------------------------------------------------------------------------------- the bump itself
def next_version(last: str, part: str) -> str:
"""The version after `last` (a release tag with or without its v): the next minor or patch."""
m = SEMVER.match(last.removeprefix("v"))
if not m:
raise ValueError(f"{last!r} is not a version like 1.2.3")
major, minor, patch = int(m[1]), int(m[2]), int(m[3])
if part == "minor":
return f"{major}.{minor + 1}.0"
if part == "patch":
# A prerelease's patch release is the release itself: 0.3.0-rc.1 -> 0.3.0.
return f"{major}.{minor}.{patch}" if m[4] else f"{major}.{minor}.{patch + 1}"
raise ValueError(f"unknown part {part!r}")
def version_key(v: str) -> tuple:
"""Sort key for versions: numbers compared as numbers, a prerelease before its release."""
m = SEMVER.match(v.removeprefix("v"))
if not m:
raise ValueError(f"{v!r} is not a version like 1.2.3")
return (int(m[1]), int(m[2]), int(m[3]), 0 if m[4] else 1, m[4] or "")
REPO = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$")
SHA = re.compile(r"^[0-9a-f]{7,40}$")
def check_args(repo: str, version: str | None, commit: str | None, workflow: str, title: str | None) -> None:
"""Refuse values that git or gh could read as options, or that break a commit message,
before any command runs."""
if not REPO.match(repo):
raise SystemExit(f"{repo!r} is not owner/name")
if version is not None and not SEMVER.match(version):
raise SystemExit(f"{version!r} is not a version like 1.2.3 or 1.2.3-rc.1")
if commit is not None and not SHA.match(commit):
raise SystemExit(f"{commit!r} is not a commit hash")
if not workflow or workflow.startswith("-") or "\n" in workflow:
raise SystemExit(f"{workflow!r} is not a workflow name")
if title is not None and (not title.strip() or title.startswith("-") or "\n" in title):
raise SystemExit(f"{title!r} is not an app title")
def _read(path: Path) -> tuple[str, bool]:
raw = path.read_bytes().decode("utf-8")
return raw.replace("\r\n", "\n"), raw.count("\r\n") > raw.count("\n") // 2
def _write(path: Path, text: str, crlf: bool) -> None:
path.write_bytes((text.replace("\n", "\r\n") if crlf else text).encode("utf-8"))
def workspace_version(root: Path) -> str:
text, _ = _read(root / "Cargo.toml")
section = None
for line in text.split("\n"):
header = re.match(r"^\[([^\]]+)\]", line)
if header:
section = header[1].strip()
continue
if section == "workspace.package":
m = re.match(r'^version\s*=\s*"([^"]+)"', line)
if m:
return m[1]
raise ValueError("Cargo.toml has no [workspace.package] version")
def bump_cargo(root: Path, new: str, members: set[str] | None = None) -> tuple[str, int, int]:
"""Set the workspace version to `new`. Returns (old version, Cargo.toml lines, lock packages).
`members` names the workspace's own packages (`cargo metadata --no-deps`): only their
Cargo.lock entries change, not a local path dependency's outside the workspace that happens
to share the version. Without it every source-less entry at the old version counts as one."""
if not SEMVER.match(new):
raise ValueError(f"{new!r} is not a version like 1.2.3 or 1.2.3-rc.1")
old = workspace_version(root)
if old == new:
raise ValueError(f"the workspace is already at {new}")
text, crlf = _read(root / "Cargo.toml")
lines, section, changed = text.split("\n"), None, 0
for i, line in enumerate(lines):
header = re.match(r"^\[([^\]]+)\]", line)
if header:
section = header[1].strip()
continue
if section == "workspace.package" and re.match(rf'^version\s*=\s*"{re.escape(old)}"', line):
# Only the value: a trailing comment stays.
lines[i] = re.sub(rf'^(version\s*=\s*"){re.escape(old)}"', lambda m: f'{m[1]}{new}"', line)
changed += 1
elif "path = " in line and f'version = "{old}"' in line:
lines[i] = line.replace(f'version = "{old}"', f'version = "{new}"')
changed += 1
_write(root / "Cargo.toml", "\n".join(lines), crlf)
lock_path, packages = root / "Cargo.lock", 0
if lock_path.exists():
lock, lcrlf = _read(lock_path)
blocks = lock.split("\n[[package]]\n")
for b, block in enumerate(blocks[1:], 1):
fields = block.split("\n")
# Registry and git packages carry a source; the workspace's own packages don't.
if any(f.startswith("source = ") for f in fields):
continue
name = next((f[len('name = "'):-1] for f in fields if f.startswith('name = "')), None)
if members is not None and name not in members:
continue
for j, f in enumerate(fields):
if f == f'version = "{old}"':
fields[j] = f'version = "{new}"'
packages += 1
break
blocks[b] = "\n".join(fields)
_write(lock_path, "\n[[package]]\n".join(blocks), lcrlf)
return old, changed, packages
def workspace_members(root: Path) -> set[str]:
"""The workspace's own package names."""
meta = json.loads(sh("cargo", "metadata", "--no-deps", "--format-version", "1", cwd=root))
return {p["name"] for p in meta["packages"]}
def bump(root: Path, new: str, run=subprocess.run, members: set[str] | None = None) -> str:
"""The whole bump: Cargo files, then the repository's own hook. Returns the old version."""
old, _, _ = bump_cargo(root, new, members)
hook = root / HOOK
if hook.exists():
run(["bash", HOOK, old, new], cwd=root, check=True)
return old
# ------------------------------------------------------------------------------------ git and gh
def sh(*cmd: str, cwd: Path | None = None, capture: bool = True) -> str:
out = subprocess.run(cmd, cwd=cwd, check=True, text=True, capture_output=capture)
return (out.stdout or "").strip()
def say(msg: str) -> None:
print(f"{time.strftime('%H:%M:%S')} {msg}", flush=True)
def gh_json(*args: str):
return json.loads(sh("gh", *args) or "null")
def last_release(repo: str) -> tuple[str, str]:
"""(tag, title) of the newest published, non-prerelease release; the title without its version."""
rows = gh_json("release", "list", "-R", repo, "--exclude-drafts", "--exclude-pre-releases",
"--limit", "1", "--json", "tagName,name")
if not rows:
raise SystemExit(f"{repo} has no published release: pass --version and --title")
tag, name = rows[0]["tagName"], rows[0]["name"] or ""
return tag, re.sub(r"\s+v?\d+\.\d+\.\d+\S*$", "", name) or repo.split("/")[-1]
def green_main(repo: str, workflow: str) -> str:
"""The newest commit on main whose `workflow` run for a push passed."""
rows = gh_json("run", "list", "-R", repo, "--workflow", workflow, "--branch", "main", "--event", "push",
"--status", "success", "--limit", "1", "--json", "headSha")
if not rows:
raise SystemExit(f"no passing {workflow!r} run on main in {repo}")
return rows[0]["headSha"]
def cargo_check(root: Path) -> None:
"""Cargo.lock still matches the manifests. Resolving, without --no-deps: only resolution
checks the lock file against the path dependencies' versions."""
sh("cargo", "metadata", "--locked", "--format-version", "1", cwd=root)
def commit(root: Path, message: str) -> str:
sh("git", "commit", "-q", "-am", message, cwd=root)
return sh("git", "rev-parse", "HEAD", cwd=root)
def wait_and_publish(repo: str, sha: str, tag: str, previous: str = "", allow_fewer: bool = False) -> None:
rid = None
for _ in range(40):
rows = gh_json("run", "list", "-R", repo, "--workflow", "release.yml", "--branch", "release",
"--json", "databaseId,headSha", "--limit", "20")
rid = next((r["databaseId"] for r in rows if r["headSha"] == sha), None)
if rid:
break
time.sleep(15)
if not rid:
raise SystemExit(f"no release.yml run started for {sha}")
say(f"release.yml https://github.com/{repo}/actions/runs/{rid}")
while gh_json("run", "view", str(rid), "-R", repo, "--json", "status")["status"] != "completed":
time.sleep(120)
run = gh_json("run", "view", str(rid), "-R", repo, "--json", "conclusion,jobs")
if run["conclusion"] != "success":
bad = [j["name"] for j in run["jobs"] if j["conclusion"] not in ("success", "skipped")]
raise SystemExit(f"release.yml {run['conclusion']}: {', '.join(bad)}. Rerun those jobs, then publish "
f"with: gh release edit {tag} -R {repo} --draft=false")
draft = gh_json("release", "view", tag, "-R", repo, "--json", "isDraft,assets")
if not draft["isDraft"] or not draft["assets"]:
raise SystemExit(f"{tag} is not a draft with assets; not publishing")
# A build job that is skipped (an app turning a platform off, or anything else) leaves the
# run green, so the draft must carry at least as many assets as the last release.
if previous:
before = len(gh_json("release", "view", previous, "-R", repo, "--json", "assets")["assets"])
if len(draft["assets"]) < before and not allow_fewer:
raise SystemExit(f"{tag} has {len(draft['assets'])} assets where {previous} had {before}: not publishing. "
f"Check the run, then publish by hand or rerun with --allow-fewer-assets.")
sh("gh", "release", "edit", tag, "-R", repo, "--draft=false")
say(f"published https://github.com/{repo}/releases/tag/{tag} ({len(draft['assets'])} assets)")
# ------------------------------------------------------------------------------------------ main
def main(argv: list[str] | None = None) -> int:
p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
p.add_argument("repo", help="owner/name, e.g. storytold/pdfcraft")
p.add_argument("--version", help="the version to release (default: next --part of the last release)")
p.add_argument("--part", choices=["minor", "patch"], default="minor")
p.add_argument("--gate-workflow", default="CI", help="workflow whose passing push run marks a releasable commit")
p.add_argument("--commit", help="release this main commit instead of the newest green one")
p.add_argument("--title", help="the app's name in commit and PR titles (default: from the last release)")
p.add_argument("--no-sync", action="store_true", help="don't open the pull request that brings the bump to main")
p.add_argument("--publish", action="store_true", help="wait for release.yml and publish the draft")
p.add_argument("--allow-fewer-assets", action="store_true",
help="publish a draft with fewer assets than the last release")
p.add_argument("--dry-run", action="store_true", help="prepare everything, push nothing")
a = p.parse_args(argv)
check_args(a.repo, a.version, a.commit, a.gate_workflow, a.title)
for tool in ("git", "gh", "cargo"):
if shutil.which(tool) is None:
raise SystemExit(f"{tool} is needed")
last, title = last_release(a.repo) if not (a.version and a.title) else ("", a.title)
title = a.title or title
version = a.version or next_version(last, a.part)
tag = f"v{version}"
sha = a.commit or green_main(a.repo, a.gate_workflow)
say(f"{title} {tag} from main {sha[:10]} (last release {last or '-'})")
work = Path(tempfile.mkdtemp(prefix="cut-release-"))
try:
sh("git", "clone", "-q", "--filter=blob:none", "--no-checkout", f"https://github.com/{a.repo}", str(work))
sh("git", "fetch", "-q", "origin", "main", "release", cwd=work)
if subprocess.run(["git", "merge-base", "--is-ancestor", sha, "origin/main"], cwd=work).returncode:
raise SystemExit(f"{sha} is not on main")
sh("git", "checkout", "-q", "--detach", sha, cwd=work)
old = bump(work, version, members=workspace_members(work))
cargo_check(work)
bumped = commit(work, f"Release: {title} {tag}")
say(f"bumped {old} -> {version}")
sh("git", "checkout", "-q", "-B", "release", "origin/release", cwd=work)
sh("git", "merge", "-q", "--no-edit", "-X", "theirs", "-m", f"Release: {title} {tag} (main {sha[:10]})",
bumped, cwd=work)
if workspace_version(work) != version:
raise SystemExit("release is not at the new version after the merge")
cargo_check(work)
left = sh("git", "diff", "--stat", bumped, "HEAD", cwd=work)
if left:
say(f"release keeps changes main doesn't have:\n{left}")
head = sh("git", "rev-parse", "HEAD", cwd=work)
sync = None
if not a.no_sync:
sh("git", "checkout", "-q", "-B", f"release-sync-{version}", "origin/main", cwd=work)
on_main = workspace_version(work)
if version_key(on_main) >= version_key(version):
say(f"main is at {on_main}, not behind {version}: no sync pull request")
else:
bump(work, version, members=workspace_members(work))
cargo_check(work)
sync = commit(work, f"Release: {title} {tag} on main")
if a.dry_run:
say(f"dry run: would push release {head[:10]}" + (f" and open a sync PR from {sync[:10]}" if sync else ""))
return 0
sh("git", "push", "-q", "origin", "release", cwd=work)
say(f"pushed release {head[:10]}: release.yml builds the {tag} draft")
if sync:
sh("git", "push", "-q", "-f", "origin", f"release-sync-{version}", cwd=work)
url = sh("gh", "pr", "create", "-R", a.repo, "--base", "main", "--head", f"release-sync-{version}",
"--title", f"Release: {title} {tag} on main", "--body",
f"Brings the {tag} version bump to main. The release was cut from {sha} with "
f"`scripts/cut_release.py` (storytold/craft-common-actions); this changes only version strings.")
say(f"sync PR {url}")
merge = subprocess.run(["gh", "pr", "merge", url, "--auto", "--squash"], capture_output=True, text=True)
if merge.returncode == 0:
say("auto-merge is on for the sync PR")
else:
say(f"WARNING: auto-merge could not be turned on ({merge.stderr.strip()}). Merge {url} by hand "
f"once its checks pass, or main stays behind the release.")
if a.publish:
wait_and_publish(a.repo, head, tag, last, a.allow_fewer_assets)
return 0
finally:
shutil.rmtree(work, ignore_errors=True)
if __name__ == "__main__":
sys.exit(main())
scripts/release_train.py
+113
View File
@@ -0,0 +1,113 @@
#!/usr/bin/env python3
"""Release many Craft apps at once: scripts/cut_release.py for each, in parallel.
scripts/release_train.py storytold/pdfcraft storytold/wordcraft ... # next minor each
scripts/release_train.py --all --skip figcraft,soundcraft --publish
scripts/release_train.py --all --dry-run
Each app gets its own log (--logs, default ./release-train-logs/<app>.log) and the train ends with
one line per app: published, pushed, or stopped and why. Apps don't wait for each other, and one
app stopping doesn't stop the rest. Options after `--` go to every cut_release.py run.
"""
from __future__ import annotations
import argparse
import concurrent.futures
import re
import subprocess
import sys
import time
from pathlib import Path
HERE = Path(__file__).resolve().parent
# Every Craft app with a release.yml calling the shared workflow.
APPS = [
"anicraft", "cadcraft", "deckcraft", "designcraft", "effectcraft", "figcraft", "filmcraft", "gridcraft",
"lightcraft", "materialcraft", "pdfcraft", "photocraft", "soundcraft", "texturecraft", "vectorcraft",
"wordcraft",
]
REPO = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$")
# cut_release.py's options a train may pass on, and the values they may take.
FLAGS = {"--publish", "--dry-run", "--no-sync", "--allow-fewer-assets"}
VALUED = {"--part": re.compile(r"^(minor|patch)$"), "--gate-workflow": re.compile(r"^[A-Za-z0-9 ._()-]+$")}
def check(repos: list[str], extra: list[str]) -> None:
"""Only owner/name repos and known cut_release.py options reach the subprocess."""
for repo in repos:
if not REPO.match(repo):
raise SystemExit(f"{repo!r} is not owner/name")
i = 0
while i < len(extra):
opt = extra[i]
if opt in FLAGS:
i += 1
elif opt in VALUED and i + 1 < len(extra) and VALUED[opt].match(extra[i + 1]):
i += 2
else:
raise SystemExit(f"{opt!r} is not an option a train passes to cut_release.py")
def logs_dir(name: str) -> Path:
"""The log folder: a plain relative path inside the current directory, nothing outside it."""
base = Path.cwd().resolve()
if not re.fullmatch(r"[A-Za-z0-9._/-]+", name) or Path(name).is_absolute():
raise SystemExit(f"{name!r}: --logs is a relative folder name inside the current directory")
path = (base / name).resolve()
if path != base and base not in path.parents:
raise SystemExit(f"{name!r}: --logs must stay inside {base}")
return path
def run_app(repo: str, extra: list[str], logs: Path) -> tuple[str, int, str]:
log = logs / f"{repo.split('/')[-1]}.log"
with log.open("w", encoding="utf-8") as out:
rc = subprocess.run([sys.executable, str(HERE / "cut_release.py"), repo, *extra],
stdout=out, stderr=subprocess.STDOUT).returncode
lines = [l for l in log.read_text(encoding="utf-8").splitlines() if l.strip()]
return repo, rc, lines[-1] if lines else "(no output)"
def main(argv: list[str] | None = None) -> int:
argv = list(sys.argv[1:] if argv is None else argv)
extra = argv[argv.index("--") + 1:] if "--" in argv else []
argv = argv[:argv.index("--")] if "--" in argv else argv
p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
p.add_argument("repos", nargs="*", help="owner/name of each app")
p.add_argument("--all", action="store_true", help="every app in the list in this script")
p.add_argument("--skip", default="", help="comma-separated app names to leave out")
p.add_argument("--owner", default="storytold")
p.add_argument("--publish", action="store_true", help="pass --publish: wait and publish each draft")
p.add_argument("--dry-run", action="store_true", help="pass --dry-run: push nothing")
p.add_argument("--logs", default="release-train-logs")
a = p.parse_args(argv)
skip = {s.strip() for s in a.skip.split(",") if s.strip()}
repos = a.repos + ([f"{a.owner}/{app}" for app in APPS] if a.all else [])
repos = [r for r in dict.fromkeys(repos) if r.split("/")[-1] not in skip]
if not repos:
p.error("name the apps, or pass --all")
extra += ["--publish"] * a.publish + ["--dry-run"] * a.dry_run
check(repos, extra)
logs = logs_dir(a.logs)
logs.mkdir(parents=True, exist_ok=True)
start = time.time()
print(f"releasing {len(repos)} apps; logs in {logs}/", flush=True)
with concurrent.futures.ThreadPoolExecutor(max_workers=len(repos)) as pool:
futures = [pool.submit(run_app, r, extra, logs) for r in repos]
results = []
for f in concurrent.futures.as_completed(futures):
repo, rc, last = f.result()
results.append((repo, rc, last))
print(f"{'ok ' if rc == 0 else 'STOP'} {repo}: {last}", flush=True)
stopped = [r for r in results if r[1] != 0]
print(f"\n{len(results) - len(stopped)} of {len(results)} done in {(time.time() - start) / 60:.0f} min"
+ (f"; stopped: {', '.join(r[0].split('/')[-1] for r in stopped)}" if stopped else ""))
return 1 if stopped else 0
if __name__ == "__main__":
sys.exit(main())
tests/test_contributor_refresh.py
+1 -1
View File
@@ -134,7 +134,7 @@ Path('contributors/contributors.json').write_text(json.dumps(credits))
self.assertEqual(settings['path'], 'contributors')
build_steps = [i for i, s in enumerate(steps)
if 'cargo ' in s.get('run', '') or 'packaging/' in s.get('run', '')
or 'vmactions/freebsd-vm@' in s.get('uses', '')]
or 'step-security/freebsd-vm@' in s.get('uses', '')]
self.assertTrue(build_steps)
self.assertTrue(all(download < i for i in build_steps))
self.assertNotIn('continue-on-error', steps[download])
tests/test_cut_release.py
+156
View File
@@ -0,0 +1,156 @@
"""scripts/cut_release.py: the version bump a release commits, without git or gh."""
import importlib.util
import tempfile
import unittest
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
spec = importlib.util.spec_from_file_location("cut_release", ROOT / "scripts/cut_release.py")
cut_release = importlib.util.module_from_spec(spec)
spec.loader.exec_module(cut_release)
CARGO_TOML = """[workspace]
members = ["crates/*", "apps/demo"]
[workspace.package]
version = "0.18.1"
edition = "2024"
[workspace.dependencies]
demo-geom = { path = "crates/geom", version = "0.18.1" }
demo-io = { path = "crates/io", version = "0.18.1" }
serde = { version = "0.18.1" }
craft-update = { git = "https://github.com/storytold/craft-libs", version = "=0.1.0", rev = "abc" }
"""
CARGO_LOCK = """# This file is automatically @generated by Cargo.
version = 4
[[package]]
name = "demo"
version = "0.18.1"
dependencies = [
"demo-geom",
]
[[package]]
name = "demo-geom"
version = "0.18.1"
[[package]]
name = "serde"
version = "0.18.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "00"
[[package]]
name = "xtask"
version = "0.1.0"
"""
class NextVersion(unittest.TestCase):
def test_minor_and_patch(self):
self.assertEqual(cut_release.next_version("v0.20.0", "minor"), "0.21.0")
self.assertEqual(cut_release.next_version("0.23.1", "minor"), "0.24.0")
self.assertEqual(cut_release.next_version("v0.23.1", "patch"), "0.23.2")
def test_a_prerelease_patch_is_the_release_itself(self):
self.assertEqual(cut_release.next_version("v0.3.0-rc.1", "patch"), "0.3.0")
def test_rejects_what_is_not_a_version(self):
with self.assertRaises(ValueError):
cut_release.next_version("nightly", "minor")
class Versions(unittest.TestCase):
def test_compared_as_numbers_with_prereleases_first(self):
key = cut_release.version_key
self.assertGreater(key("0.10.0"), key("0.9.0"))
self.assertGreater(key("v0.22.0"), key("0.21.0"))
self.assertLess(key("0.21.0-rc.1"), key("0.21.0"))
self.assertEqual(key("0.21.0"), key("v0.21.0"))
class Arguments(unittest.TestCase):
def test_what_could_reach_git_or_gh_as_an_option_is_refused(self):
ok = dict(repo="storytold/pdfcraft", version="0.21.0", commit="5ff567a4a2", workflow="CI", title="PdfCraft")
cut_release.check_args(**ok)
for field, bad in [("repo", "--upload-pack=x"), ("repo", "storytold"), ("version", "--force"),
("version", "0.21"), ("commit", "--output=/tmp/x"), ("commit", "HEAD~1"),
("workflow", "-x"), ("title", "-m evil"), ("title", "two\nlines")]:
with self.assertRaises(SystemExit, msg=f"{field}={bad!r}"):
cut_release.check_args(**{**ok, field: bad})
class Bump(unittest.TestCase):
def repo(self, toml=CARGO_TOML, lock=CARGO_LOCK, crlf=False):
d = Path(tempfile.mkdtemp())
nl = "\r\n" if crlf else "\n"
(d / "Cargo.toml").write_bytes(toml.replace("\n", nl).encode())
(d / "Cargo.lock").write_bytes(lock.replace("\n", nl).encode())
return d
def test_bumps_the_workspace_its_path_dependencies_and_its_own_lock_entries(self):
d = self.repo()
old, lines, packages = cut_release.bump_cargo(d, "0.21.0")
self.assertEqual((old, lines, packages), ("0.18.1", 3, 2))
toml = (d / "Cargo.toml").read_text()
self.assertIn('[workspace.package]\nversion = "0.21.0"', toml)
self.assertIn('demo-geom = { path = "crates/geom", version = "0.21.0" }', toml)
# A registry dependency that happens to share the version is not the workspace's.
self.assertIn('serde = { version = "0.18.1" }', toml)
lock = (d / "Cargo.lock").read_text()
self.assertIn('name = "demo"\nversion = "0.21.0"', lock)
self.assertIn('name = "demo-geom"\nversion = "0.21.0"', lock)
self.assertIn('name = "serde"\nversion = "0.18.1"\nsource', lock)
self.assertIn('name = "xtask"\nversion = "0.1.0"', lock)
self.assertEqual(cut_release.workspace_version(d), "0.21.0")
def test_keeps_crlf_files_crlf(self):
d = self.repo(crlf=True)
cut_release.bump_cargo(d, "0.21.0")
for name in ("Cargo.toml", "Cargo.lock"):
raw = (d / name).read_bytes()
self.assertNotIn(b"\n", raw.replace(b"\r\n", b""), name)
def test_refuses_a_bump_to_the_same_version_or_a_bad_one(self):
d = self.repo()
with self.assertRaises(ValueError):
cut_release.bump_cargo(d, "0.18.1")
with self.assertRaises(ValueError):
cut_release.bump_cargo(d, "0.21")
def test_runs_the_repositorys_own_hook_with_old_and_new(self):
d = self.repo()
(d / "packaging").mkdir()
(d / cut_release.HOOK).write_text("#!/bin/bash\n")
calls = []
old = cut_release.bump(d, "0.21.0", run=lambda cmd, **kw: calls.append((cmd, kw["cwd"])))
self.assertEqual(old, "0.18.1")
self.assertEqual(calls, [(["bash", cut_release.HOOK, "0.18.1", "0.21.0"], d)])
def test_a_comment_after_the_workspace_version_stays(self):
d = self.repo(toml=CARGO_TOML.replace('version = "0.18.1"\nedition', 'version = "0.18.1" # the release\nedition'))
cut_release.bump_cargo(d, "0.21.0")
self.assertIn('version = "0.21.0" # the release', (d / "Cargo.toml").read_text())
self.assertEqual(cut_release.workspace_version(d), "0.21.0")
def test_given_the_members_only_their_lock_entries_change(self):
# A local path dependency outside the workspace keeps its version even when it matches.
d = self.repo()
_, _, packages = cut_release.bump_cargo(d, "0.21.0", members={"demo"})
lock = (d / "Cargo.lock").read_text()
self.assertEqual(packages, 1)
self.assertIn('name = "demo"\nversion = "0.21.0"', lock)
self.assertIn('name = "demo-geom"\nversion = "0.18.1"', lock)
def test_no_hook_no_call(self):
d = self.repo()
calls = []
cut_release.bump(d, "0.21.0", run=lambda *a, **kw: calls.append(a))
self.assertEqual(calls, [])
if __name__ == "__main__":
unittest.main()
tests/test_freebsd.py
+1 -1
View File
@@ -120,7 +120,7 @@ fi
for key in ("CRAFT_FONTS_DIR", "CRAFT_FONTS_REQUIRED"):
env.pop(key, None)
# The VM wrapper forwards only the configured SSH SendEnv names.
vm = next(step for step in STEPS if step.get("uses", "").startswith("vmactions/freebsd-vm@"))
vm = next(step for step in STEPS if step.get("uses", "").startswith("step-security/freebsd-vm@"))
forwarded = vm["with"]["envs"].split()
configured = {**DATA["jobs"]["freebsd"].get("env", {}), **BUILD_STEP.get("env", {})}
env.update({key: re.sub(r"\$\{\{ inputs\.(.*?) \}\}", replace, value)
tests/test_freebsd_cache.py
+61 -3
View File
@@ -35,7 +35,7 @@ class CargoCacheTests(unittest.TestCase):
self.env = dict(os.environ, PATH=str(self.tools) + os.pathsep + os.environ["PATH"],
HOME=str(self.root / "home"), RUNNER_TEMP=str(self.root / "host"),
FIXTURE_GUEST=str(self.guest), REAL_RSYNC=shutil.which("rsync"),
CACHE_PHASE="restore", CACHE_PROFILE="debug", CACHE_CAN_SAVE="true",
CACHE_PHASE="restore", CACHE_PROFILE="debug", CACHE_TARGET="", CACHE_CAN_SAVE="true",
CACHE_CONFIGURATION='{"features":"heif","fonts":"abc"}',
CACHE_SOURCE_HASH="lock-and-build-config", GITHUB_SHA="commit1",
GITHUB_WORKSPACE=str(self.root),
@@ -234,6 +234,33 @@ sys.exit(subprocess.run([os.environ['REAL_RSYNC'], *args]).returncode)
self.transfer("save")
self.assertEqual((self.cache / "target/release/app").read_text(), "cached-release")
def test_cross_target_round_trip_keeps_the_triple_directory(self):
triple = "aarch64-unknown-freebsd"
self.write(self.cache, "target/release/build/dependency/output", "host-build-script")
self.write(self.cache, f"target/{triple}/release/deps/libdependency.rlib", "cached-cross")
self.transfer("restore", CACHE_PROFILE="release", CACHE_TARGET=triple)
self.assertEqual((self.guest / "var/tmp/craft-target/release/build/dependency/output").read_text(), "host-build-script")
self.assertEqual((self.guest / f"var/tmp/craft-target/{triple}/release/deps/libdependency.rlib").read_text(), "cached-cross")
self.write(self.guest, f"var/tmp/craft-target/{triple}/release/app", "cross-release")
self.write(self.guest, f"var/tmp/craft-target/{triple}/release/incremental/state", "discard")
self.transfer("save", CACHE_PROFILE="release", CACHE_TARGET=triple)
self.assertEqual((self.cache / f"target/{triple}/release/app").read_text(), "cross-release")
self.assertEqual((self.cache / f"target/{triple}/release/deps/libdependency.rlib").read_text(), "cached-cross")
self.assertFalse((self.cache / f"target/{triple}/release/incremental").exists())
self.assertEqual((self.cache / "target/release/build/dependency/output").read_text(), "host-build-script")
def test_native_build_leaves_cross_directories_alone(self):
self.write(self.cache, "target/aarch64-unknown-freebsd/release/app", "other-arch")
self.transfer("restore", CACHE_PROFILE="release")
self.assertFalse((self.guest / "var/tmp/craft-target/aarch64-unknown-freebsd").exists())
def test_target_must_be_a_plain_triple(self):
for target in ("../../etc", "aarch64 unknown", "a;rm -rf /", "Aarch64/x"):
with self.subTest(target=target):
result = self.run_script(TRANSFER, CACHE_PROFILE="release", CACHE_TARGET=target)
self.assertNotEqual(result.returncode, 0)
self.assertIn("invalid cache target", result.stderr)
def test_transport_failure_is_visible(self):
self.tool("rsync", "import sys\nsys.exit(23)\n")
result = self.run_script(TRANSFER)
@@ -241,6 +268,33 @@ sys.exit(subprocess.run([os.environ['REAL_RSYNC'], *args]).returncode)
class WorkflowCacheIntegrationTests(unittest.TestCase):
# Release inputs that do not change what the FreeBSD job compiles, so they stay out of the
# cache key. contributors-artifact names the run itself, and a key that held it could never
# be restored by a later run; dry-run must share the release's entry so a dry run warms it;
# publish only decides whether the finished draft is published.
RELEASE_INPUTS_OUTSIDE_THE_KEY = {
"app-title", "web-craft-fonts", "version", "contributors-artifact", "has-web", "has-riscv64",
"has-freebsd", "has-arch", "wix-version", "nfpm-version", "trunk-version",
"freebsd-architectures", "has-server", "dry-run", "publish",
}
def test_release_cache_key_names_every_input_that_changes_the_freebsd_build(self):
data = yaml.load((ROOT / ".github/workflows/release.yml").read_text(), Loader=yaml.BaseLoader)
inputs = set(data["on"]["workflow_call"]["inputs"])
steps = data["jobs"]["freebsd"]["steps"]
cache_steps = [step for step in steps if "/freebsd-cargo-cache@" in step.get("uses", "")]
self.assertEqual(len(cache_steps), 2)
for step in cache_steps:
configuration = step["with"]["configuration"]
self.assertNotIn("toJSON(inputs)", configuration)
self.assertIn("matrix.arch", configuration)
keyed = {name for name in inputs if f"inputs.{name})" in configuration}
# A new input must be put in the key or listed above as not affecting the build.
self.assertEqual(keyed | self.RELEASE_INPUTS_OUTSIDE_THE_KEY, inputs)
self.assertFalse(keyed & self.RELEASE_INPUTS_OUTSIDE_THE_KEY)
self.assertEqual(step["with"]["target"],
"${{ matrix.arch == 'aarch64' && 'aarch64-unknown-freebsd' || '' }}")
def test_restore_tries_the_exact_revision_then_a_compatible_one(self):
restore = next(step for step in ACTION["runs"]["steps"]
if step.get("uses", "").startswith("actions/cache/restore@"))
@@ -278,7 +332,7 @@ class WorkflowCacheIntegrationTests(unittest.TestCase):
job = data["jobs"]["freebsd"]
steps = job["steps"]
index = lambda name: next(i for i, step in enumerate(steps) if step.get("name") == name)
vm = next(step for step in steps if step.get("uses", "").startswith("vmactions/freebsd-vm@"))
vm = next(step for step in steps if step.get("uses", "").startswith("step-security/freebsd-vm@"))
self.assertEqual(vm["with"]["copyback"], "false")
for variable in ("CARGO_HOME", "CARGO_TARGET_DIR", "CARGO_INCREMENTAL", "CARGO_BUILD_JOBS"):
self.assertIn(variable, vm["with"]["envs"].split())
@@ -296,7 +350,11 @@ class WorkflowCacheIntegrationTests(unittest.TestCase):
self.assertIn("steps.cargo-cache.outputs.key != ''", save["if"])
self.assertIn("steps.cargo-cache.outcome", build["run"])
self.assertIn("pull_request", save["if"])
self.assertIn("toJSON(inputs)", restore["with"]["configuration"])
if filename == "freebsd.yml":
self.assertIn("toJSON(inputs)", restore["with"]["configuration"])
else:
self.assertEqual(restore["with"]["configuration"], save["with"]["configuration"])
self.assertEqual(restore["with"]["target"], save["with"]["target"])
if filename == "release.yml":
self.assertEqual(job["strategy"]["matrix"]["arch"], "${{ fromJSON(inputs.freebsd-architectures) }}")
self.assertEqual(restore["with"]["profile"], "release")
tests/test_gate.py
+27
View File
@@ -135,6 +135,33 @@ class CallerEnvironmentTests(unittest.TestCase):
self.assertEqual(self.run_step(ALL=f'GOOD=1\n{line}')[0], 1)
class CallerSetupTests(unittest.TestCase):
STEP = step('Caller setup')
def run_step(self, setup):
with tempfile.TemporaryDirectory() as directory:
github_env = Path(directory, 'github_env')
github_env.touch()
result = subprocess.run(
['bash', '--noprofile', '--norc', '-eo', 'pipefail', '-c', self.STEP['run']],
cwd=directory, capture_output=True, text=True,
env={**os.environ, 'SETUP': setup, 'RUNNER_TEMP': directory,
'GITHUB_ENV': str(github_env)})
return result.returncode, github_env.read_text().splitlines(), Path(directory, 'setup-ran').exists()
def test_setup_input_is_passed_through_env_before_the_gate(self):
self.assertEqual(set(self.STEP['env']), {'SETUP'})
self.assertLess(STEPS.index(self.STEP), STEPS.index(step('Gate')))
self.assertNotIn('${{', self.STEP['run'])
def test_setup_can_provision_data_and_export_its_path(self):
setup = 'touch setup-ran\necho "CRAFT_PSD_CORPUS=/corpus" >> "$GITHUB_ENV"'
self.assertEqual(self.run_step(setup), (0, ['CRAFT_PSD_CORPUS=/corpus'], True))
def test_setup_failure_stops_the_gate_job(self):
self.assertEqual(self.run_step('exit 17')[0], 17)
class ProfileCacheTests(unittest.TestCase):
def key(self, manifest):
with tempfile.TemporaryDirectory() as directory:
tests/test_install_apt_packages.py
+283 -4
View File
@@ -1,4 +1,5 @@
"""The cached apt install action: pinned actions, and PR runs that never write the cache."""
"""The cached apt install action: pinned actions, PR runs that never write the cache, and the
mirror list that keeps apt working when ubuntu.com is down."""
import os
from pathlib import Path
import re
@@ -10,6 +11,7 @@ import yaml
ROOT = Path(__file__).resolve().parents[1]
ACTION = ROOT / 'actions/install-apt-packages/action.yml'
MIRRORS = ROOT / 'actions/configure-apt-mirrors/configure-apt-mirrors.sh'
def steps():
@@ -105,10 +107,17 @@ if [ "$op" = "${STALL_OP:-}" ] && [ "$n" -le "${STALLS:-0}" ]; then exec /usr/bi
"""
# Stands in for configure-apt-mirrors: logs each call, and fails when MIRRORS_FAIL is set.
FAKE_MIRRORS = """#!/bin/bash
echo mirrors >> "$APT_LOG"
[ -z "${MIRRORS_FAIL:-}" ]
"""
class StalledMirrorTests(unittest.TestCase):
"""The install step against a fake apt-get, with a one-second limit per mirror download."""
def run_install(self, stall_op='', stalls=0):
def run_install(self, stall_op='', stalls=0, mirrors_fail=False, keep_mirror_calls=False):
install_step = next(s for s in steps() if s.get('name') == 'Install packages')
self.assertEqual(install_step['env']['APT_NETWORK_TIMEOUT'], '300')
with tempfile.TemporaryDirectory() as tmpdir:
@@ -117,6 +126,7 @@ class StalledMirrorTests(unittest.TestCase):
# Keep the action's system-only man-db cleanup inside the fixture too.
for name, body in [
('apt-get', FAKE_APT),
('configure-apt-mirrors', FAKE_MIRRORS),
('sudo', '#!/bin/bash\nexec "$@"\n'),
('sleep', '#!/bin/bash\n'),
('rm', '#!/bin/bash\nprintf "%s\\n" "$*" >> "$RM_LOG"\n[[ "$*" == "-f /var/lib/man-db/auto-update" ]]\n'),
@@ -131,13 +141,18 @@ class StalledMirrorTests(unittest.TestCase):
['bash', '--noprofile', '--norc', '-e', '-o', 'pipefail', '-c', install_step['run']],
env={**os.environ, 'PATH': f"{bin_dir}{os.pathsep}{os.environ['PATH']}", 'HOME': tmpdir,
'PACKAGE_LIST': 'curl git ', 'APT_NETWORK_TIMEOUT': '1', 'APT_NETWORK_ATTEMPTS': '3',
'APT_LOG': str(log), 'RM_LOG': str(rm_log), 'STALL_OP': stall_op, 'STALLS': str(stalls)},
'APT_LOG': str(log), 'RM_LOG': str(rm_log), 'STALL_OP': stall_op, 'STALLS': str(stalls),
'CONFIGURE_MIRRORS': str(bin_dir / 'configure-apt-mirrors'),
'MIRRORS_FAIL': '1' if mirrors_fail else ''},
capture_output=True,
text=True,
timeout=120,
)
self.assertEqual(rm_log.read_text().splitlines(), ['-f /var/lib/man-db/auto-update'])
return res, log.read_text().split()
ops = log.read_text().split()
if not keep_mirror_calls:
ops = [op for op in ops if op != 'mirrors']
return res, ops
def test_a_stalled_update_is_retried(self):
res, ops = self.run_install('update', stalls=1)
@@ -162,10 +177,274 @@ class StalledMirrorTests(unittest.TestCase):
self.assertEqual(ops, ['update', 'download', 'download', 'download'])
self.assertIn('::error::install-apt-packages: downloading the packages failed 3 times', res.stdout)
def test_the_mirrors_are_checked_before_every_download_attempt(self):
res, ops = self.run_install('update', stalls=1, keep_mirror_calls=True)
self.assertEqual(res.returncode, 0, res.stdout + res.stderr)
self.assertEqual(ops, ['mirrors', 'update', 'mirrors', 'update', 'mirrors', 'download',
'install', 'autoclean'])
def test_a_failed_mirror_check_keeps_the_sources_and_installs(self):
res, ops = self.run_install(mirrors_fail=True)
self.assertEqual(res.returncode, 0, res.stdout + res.stderr)
self.assertEqual(ops, ['update', 'download', 'install', 'autoclean'])
self.assertIn('::warning::install-apt-packages: configuring the apt mirrors failed', res.stdout)
def test_the_mirror_script_sits_where_the_action_looks(self):
install_step = next(s for s in steps() if s.get('name') == 'Install packages')
self.assertEqual(install_step['env']['CONFIGURE_MIRRORS'],
'${{ github.action_path }}/../configure-apt-mirrors/configure-apt-mirrors.sh')
self.assertTrue(MIRRORS.is_file())
def test_a_partly_failed_update_is_an_error_so_it_is_retried(self):
install_step = next(s for s in steps() if s.get('name') == 'Install packages')
self.assertIn('network update -o APT::Update::Error-Mode=any', install_step['run'])
# curl that fails, as an unresolvable name does, for the hosts in DOWN, and logs what it fetched.
FAKE_CURL = """#!/bin/bash
url=${@: -1}
echo "$url" >> "$CURL_LOG"
host=${url#*://}
host=${host%%/*}
for down in ${DOWN:-}; do
if [ "$host" = "$down" ]; then
echo "curl: (6) Could not resolve host: $host" >&2
exit 6
fi
done
"""
# cloud-init that, like the real one on 11 Nov 2026, writes the image's ubuntu.sources (from
# IMAGE_SOURCES) just before `status --wait` returns.
FAKE_CLOUD_INIT = """#!/bin/bash
[ "$*" = "status --wait" ] || exit 2
sleep "${CLOUD_INIT_DELAY:-0}"
cp "$IMAGE_SOURCES" "$CRAFT_APT_ROOT/etc/apt/sources.list.d/ubuntu.sources"
"""
UBUNTU_HOSTS = 'azure.archive.ubuntu.com archive.ubuntu.com security.ubuntu.com ports.ubuntu.com'
NOBLE_SOURCES = """Types: deb
URIs: mirror+file:/etc/apt/apt-mirrors.txt
Suites: noble noble-updates noble-backports
Components: main restricted universe multiverse
Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg
Types: deb
URIs: mirror+file:/etc/apt/apt-mirrors.txt
Suites: noble-security
Components: main restricted universe multiverse
Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg
"""
PORTS_SOURCES = """Types: deb
URIs: http://ports.ubuntu.com/ubuntu-ports/
Suites: noble noble-updates noble-backports noble-security
Components: main restricted universe multiverse
Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg
"""
JAMMY_SOURCES_LIST = """# deb-src lines are commented out
deb mirror+file:/etc/apt/apt-mirrors.txt jammy main restricted
deb mirror+file:/etc/apt/apt-mirrors.txt jammy-updates main restricted
deb http://security.ubuntu.com/ubuntu jammy-security main restricted
deb https://esm.ubuntu.com/apps/ubuntu jammy-apps-security main
"""
IMAGE_MIRRORS = """http://azure.archive.ubuntu.com/ubuntu/\tpriority:1
https://archive.ubuntu.com/ubuntu/\tpriority:2
https://security.ubuntu.com/ubuntu/\tpriority:3
"""
# What install-riscv64-cross adds: its amd64 source must stay paired with its riscv64 source.
RISCV_LIST = 'deb [arch=amd64] https://archive.ubuntu.com/ubuntu noble main universe\n'
class ConfigureAptMirrorsTests(unittest.TestCase):
"""configure-apt-mirrors.sh against a fake /etc and a fake curl."""
def configure(self, files, arch='amd64', codename='noble', down='', runs=1, cloud_init=False):
with tempfile.TemporaryDirectory() as tmpdir:
root = Path(tmpdir) / 'root'
files = {'etc/os-release': f'NAME="Ubuntu"\nVERSION_CODENAME={codename}\n', **files}
for name, body in files.items():
(root / name).parent.mkdir(parents=True, exist_ok=True)
(root / name).write_text(body, newline='\n')
bin_dir = Path(tmpdir) / 'bin'
bin_dir.mkdir()
(bin_dir / 'curl').write_text(FAKE_CURL, newline='\n')
(bin_dir / 'curl').chmod(0o755)
curl_log = Path(tmpdir) / 'curl.log'
curl_log.touch()
(bin_dir / 'cloud-init').write_text(FAKE_CLOUD_INIT, newline='\n')
(bin_dir / 'cloud-init').chmod(0o755)
image_sources = Path(tmpdir) / 'image.sources'
image_sources.write_text(NOBLE_SOURCES, newline='\n')
for _ in range(runs):
res = subprocess.run(
['bash', str(MIRRORS)],
env={**os.environ, 'PATH': f"{bin_dir}{os.pathsep}{os.environ['PATH']}",
'CRAFT_APT_ROOT': str(root), 'CRAFT_APT_ARCH': arch, 'DOWN': down,
'CURL_LOG': str(curl_log), 'IMAGE_SOURCES': str(image_sources),
'CRAFT_CLOUD_INIT': str(bin_dir / 'cloud-init') if cloud_init else 'no-cloud-init-here'},
capture_output=True, text=True, timeout=60)
self.assertEqual(res.returncode, 0, res.stdout + res.stderr)
mirror_list = root / 'etc/apt/craft-apt-mirrors.txt'
result = {name: (root / name).read_text() for name in files}
result['list'] = mirror_list.read_text().splitlines() if mirror_list.exists() else None
result['curl'] = curl_log.read_text().split()
srv = root / 'etc/apt/apt.conf.d/99craft-no-srv'
result['srv'] = srv.read_text() if srv.exists() else None
return res, result
def test_healthy_mirrors_keep_the_image_order_then_the_fallbacks(self):
res, out = self.configure({'etc/apt/apt-mirrors.txt': IMAGE_MIRRORS,
'etc/apt/sources.list.d/ubuntu.sources': NOBLE_SOURCES})
self.assertEqual(out['list'], [
'http://azure.archive.ubuntu.com/ubuntu/\tpriority:1',
'https://archive.ubuntu.com/ubuntu/\tpriority:2',
'https://security.ubuntu.com/ubuntu/\tpriority:3',
'https://mirrors.edge.kernel.org/ubuntu/\tpriority:4',
'https://mirrors.dotsrc.org/ubuntu/\tpriority:5',
])
self.assertIn('https://mirrors.edge.kernel.org/ubuntu/dists/noble/InRelease', out['curl'])
self.assertNotIn('::warning::', res.stdout)
self.assertEqual(out['srv'], 'Acquire::EnableSrvRecords "false";\n')
def test_mirrors_outside_ubuntu_com_lead_when_it_is_down(self):
res, out = self.configure({'etc/apt/apt-mirrors.txt': IMAGE_MIRRORS,
'etc/apt/sources.list.d/ubuntu.sources': NOBLE_SOURCES},
down=UBUNTU_HOSTS)
self.assertEqual([line.split('\t')[0] for line in out['list']], [
'https://mirrors.edge.kernel.org/ubuntu/',
'https://mirrors.dotsrc.org/ubuntu/',
'http://azure.archive.ubuntu.com/ubuntu/',
'https://archive.ubuntu.com/ubuntu/',
'https://security.ubuntu.com/ubuntu/',
])
self.assertIn('::warning::configure-apt-mirrors: http://azure.archive.ubuntu.com/ubuntu/ did not serve '
'noble/InRelease (curl: (6) Could not resolve host: azure.archive.ubuntu.com', res.stdout)
def test_cloud_init_rewriting_the_sources_is_waited_out(self):
res, out = self.configure({'etc/apt/apt-mirrors.txt': IMAGE_MIRRORS,
'etc/apt/sources.list.d/ubuntu.sources': NOBLE_SOURCES},
cloud_init=True)
self.assertEqual(out['etc/apt/sources.list.d/ubuntu.sources'], NOBLE_SOURCES.replace(
'mirror+file:/etc/apt/apt-mirrors.txt', 'mirror+file:/etc/apt/craft-apt-mirrors.txt'))
def test_deb822_sources_read_the_new_list_and_keep_their_other_fields(self):
_, out = self.configure({'etc/apt/apt-mirrors.txt': IMAGE_MIRRORS,
'etc/apt/sources.list.d/ubuntu.sources': NOBLE_SOURCES})
self.assertEqual(out['etc/apt/sources.list.d/ubuntu.sources'], NOBLE_SOURCES.replace(
'mirror+file:/etc/apt/apt-mirrors.txt', 'mirror+file:/etc/apt/craft-apt-mirrors.txt'))
def test_one_line_sources_change_only_the_ubuntu_archive(self):
_, out = self.configure({'etc/apt/apt-mirrors.txt': IMAGE_MIRRORS,
'etc/apt/sources.list': JAMMY_SOURCES_LIST}, codename='jammy')
self.assertEqual(out['etc/apt/sources.list'], """# deb-src lines are commented out
deb mirror+file:/etc/apt/craft-apt-mirrors.txt jammy main restricted
deb mirror+file:/etc/apt/craft-apt-mirrors.txt jammy-updates main restricted
deb mirror+file:/etc/apt/craft-apt-mirrors.txt jammy-security main restricted
deb https://esm.ubuntu.com/apps/ubuntu jammy-apps-security main
""")
self.assertIn('https://mirrors.edge.kernel.org/ubuntu/dists/jammy/InRelease', out['curl'])
def test_arm_reads_ports_mirrors(self):
_, out = self.configure({'etc/apt/sources.list.d/ubuntu.sources': PORTS_SOURCES},
arch='arm64', down=UBUNTU_HOSTS)
self.assertEqual([line.split('\t')[0] for line in out['list']], [
'https://mirrors.dotsrc.org/ubuntu-ports/',
'https://ftp.tu-chemnitz.de/pub/linux/ubuntu-ports/',
'https://ports.ubuntu.com/ubuntu-ports/',
])
self.assertIn('URIs: mirror+file:/etc/apt/craft-apt-mirrors.txt\n',
out['etc/apt/sources.list.d/ubuntu.sources'])
def test_several_archive_uris_in_one_field_become_one(self):
sources = PORTS_SOURCES.replace(
'URIs: http://ports.ubuntu.com/ubuntu-ports/',
'URIs: http://ports.ubuntu.com/ubuntu-ports/ https://ports.ubuntu.com/ubuntu-ports')
_, out = self.configure({'etc/apt/sources.list.d/ubuntu.sources': sources}, arch='arm64')
self.assertIn('URIs: mirror+file:/etc/apt/craft-apt-mirrors.txt\n',
out['etc/apt/sources.list.d/ubuntu.sources'])
def test_lists_other_actions_add_keep_their_sources(self):
_, out = self.configure({'etc/apt/apt-mirrors.txt': IMAGE_MIRRORS,
'etc/apt/sources.list.d/ubuntu.sources': NOBLE_SOURCES,
'etc/apt/sources.list.d/amd64-primary.list': RISCV_LIST})
self.assertEqual(out['etc/apt/sources.list.d/amd64-primary.list'], RISCV_LIST)
def test_running_again_checks_anew_and_changes_nothing_else(self):
files = {'etc/apt/apt-mirrors.txt': IMAGE_MIRRORS,
'etc/apt/sources.list.d/ubuntu.sources': NOBLE_SOURCES}
_, once = self.configure(files)
_, twice = self.configure(files, runs=2)
self.assertEqual(once['etc/apt/sources.list.d/ubuntu.sources'],
twice['etc/apt/sources.list.d/ubuntu.sources'])
self.assertEqual(once['list'], twice['list'])
self.assertEqual(len(twice['curl']), 2 * len(once['curl']))
def test_without_the_image_sources_nothing_changes(self):
res, out = self.configure({})
self.assertIsNone(out['list'])
self.assertIn('::warning::configure-apt-mirrors: no sources.list or ubuntu.sources', res.stdout)
class AllowedMirrorTests(unittest.TestCase):
"""Every job that blocks egress and allows the Ubuntu archive also allows the fallbacks."""
def test_block_mode_jobs_allow_the_fallback_mirrors(self):
fallbacks = set(re.findall(r'https://([^/]+)/ubuntu/$', MIRRORS.read_text(), re.M))
self.assertEqual(fallbacks, {'archive.ubuntu.com', 'security.ubuntu.com',
'mirrors.edge.kernel.org', 'mirrors.dotsrc.org'})
fallbacks -= {'archive.ubuntu.com', 'security.ubuntu.com'}
checked = 0
for workflow in sorted((ROOT / '.github/workflows').glob('*.yml')):
jobs = yaml.load(workflow.read_text(), Loader=yaml.BaseLoader).get('jobs', {})
for name, job in jobs.items():
for step in job.get('steps', []):
allowed = step.get('with', {}).get('allowed-endpoints', '').split()
if 'harden-runner' in step.get('uses', '') and 'archive.ubuntu.com:443' in allowed:
checked += 1
for host in sorted(fallbacks):
with self.subTest(workflow=workflow.name, job=name, host=host):
self.assertIn(f'{host}:443', allowed)
self.assertGreaterEqual(checked, 8)
WAIT = ROOT / 'actions/configure-apt-mirrors/wait-for-cloud-init.sh'
class WaitForCloudInitTests(unittest.TestCase):
def wait(self, body, limit='300'):
with tempfile.TemporaryDirectory() as tmpdir:
fake = Path(tmpdir) / 'cloud-init'
fake.write_text('#!/bin/bash\n' + body, newline='\n')
fake.chmod(0o755)
return subprocess.run(['bash', str(WAIT)], capture_output=True, text=True, timeout=60,
env={**os.environ, 'CRAFT_CLOUD_INIT': str(fake),
'CRAFT_CLOUD_INIT_TIMEOUT': limit})
def test_waits_until_cloud_init_is_done(self):
res = self.wait('[ "$*" = "status --wait" ] && sleep 2\n')
self.assertEqual(res.returncode, 0)
self.assertRegex(res.stdout, r'Waited [23]s for cloud-init')
def test_a_cloud_init_that_never_finishes_only_warns(self):
res = self.wait('sleep 30\n', limit='1')
self.assertEqual(res.returncode, 0)
self.assertIn('::warning::cloud-init was still running after 1s', res.stdout)
def test_a_cloud_init_that_reports_an_error_does_not_fail_the_job(self):
res = self.wait('exit 1\n')
self.assertEqual(res.returncode, 0)
self.assertEqual(res.stdout, '')
def test_without_cloud_init_it_returns_at_once(self):
res = subprocess.run(['bash', str(WAIT)], capture_output=True, text=True, timeout=10,
env={**os.environ, 'CRAFT_CLOUD_INIT': 'no-cloud-init-here'})
self.assertEqual((res.returncode, res.stdout), (0, ''))
if __name__ == '__main__':
unittest.main()
tests/test_install_riscv64_cross.py
+169
View File
@@ -0,0 +1,169 @@
"""install-riscv64-cross: amd64 and riscv64 come from one pair of mirrors serving one snapshot."""
import os
from pathlib import Path
import subprocess
import tempfile
import unittest
import yaml
ROOT = Path(__file__).resolve().parents[1]
ACTION = ROOT / 'actions/install-riscv64-cross/action.yml'
SCRIPT = ROOT / 'actions/install-riscv64-cross/select-mirrors.sh'
# curl that fails, as an unresolvable name does, for the hosts in DOWN, and otherwise writes an
# InRelease whose Date is OLD_DATE for the URL prefixes in STALE and NEW_DATE for the rest.
FAKE_CURL = """#!/bin/bash
out=
while [ $# -gt 1 ]; do
if [ "$1" = -o ]; then out=$2; shift; fi
shift
done
url=$1
host=${url#*://}
host=${host%%/*}
for down in ${DOWN:-}; do
if [ "$host" = "$down" ]; then
echo "curl: (6) Could not resolve host: $host" >&2
exit 6
fi
done
date="Wed, 11 Nov 2026 9:35:45 UTC"
for stale in ${STALE:-}; do
case "$url" in "$stale"*) date="Sun, 08 Nov 2026 0:30:00 UTC" ;; esac
done
printf 'Origin: Ubuntu\\nDate: %s\\n %s main/binary-riscv64/Packages.xz\\n' "$date" "$RANDOM" > "$out"
"""
# cloud-init that, like the real one on 11 Nov 2026, writes the image's ubuntu.sources (from
# IMAGE_SOURCES) just before `status --wait` returns.
FAKE_CLOUD_INIT = """#!/bin/bash
[ "$*" = "status --wait" ] || exit 2
sleep "${CLOUD_INIT_DELAY:-0}"
cp "$IMAGE_SOURCES" "$CRAFT_APT_ROOT/etc/apt/sources.list.d/ubuntu.sources"
"""
UBUNTU_HOSTS = 'archive.ubuntu.com security.ubuntu.com ports.ubuntu.com azure.archive.ubuntu.com'
NOBLE_SOURCES = """Types: deb
URIs: mirror+file:/etc/apt/apt-mirrors.txt
Suites: noble noble-updates noble-backports
Components: main restricted universe multiverse
Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg
Types: deb
URIs: mirror+file:/etc/apt/apt-mirrors.txt
Suites: noble-security
Components: main restricted universe multiverse
Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg
"""
COMPONENTS = 'main universe restricted multiverse'
def lists(amd64, security, ports):
return {
'amd64-primary.list': f'deb [arch=amd64] {amd64} noble {COMPONENTS}\n'
f'deb [arch=amd64] {amd64} noble-updates {COMPONENTS}\n',
'amd64-security.list': f'deb [arch=amd64] {security} noble-security {COMPONENTS}\n',
'riscv64.list': f'deb [arch=riscv64] {ports} noble {COMPONENTS}\n'
f'deb [arch=riscv64] {ports} noble-updates {COMPONENTS}\n'
f'deb [arch=riscv64] {ports} noble-security {COMPONENTS}\n',
}
UBUNTU = lists('https://archive.ubuntu.com/ubuntu', 'https://security.ubuntu.com/ubuntu',
'https://ports.ubuntu.com/ubuntu-ports')
DOTSRC = lists('https://mirrors.dotsrc.org/ubuntu', 'https://mirrors.dotsrc.org/ubuntu',
'https://mirrors.dotsrc.org/ubuntu-ports')
KERNEL = lists('https://mirrors.edge.kernel.org/ubuntu', 'https://mirrors.edge.kernel.org/ubuntu',
'https://ftp.tu-chemnitz.de/pub/linux/ubuntu-ports')
class SelectMirrorsTests(unittest.TestCase):
def select(self, files=None, down='', stale='', runs=1, cloud_init=False):
files = {'etc/apt/sources.list.d/ubuntu.sources': NOBLE_SOURCES} if files is None else files
with tempfile.TemporaryDirectory() as tmpdir:
root = Path(tmpdir) / 'root'
files = {'etc/os-release': 'NAME="Ubuntu"\nVERSION_CODENAME=noble\n', **files}
for name, body in files.items():
(root / name).parent.mkdir(parents=True, exist_ok=True)
(root / name).write_text(body, newline='\n')
bin_dir = Path(tmpdir) / 'bin'
bin_dir.mkdir()
(bin_dir / 'curl').write_text(FAKE_CURL, newline='\n')
(bin_dir / 'curl').chmod(0o755)
(bin_dir / 'cloud-init').write_text(FAKE_CLOUD_INIT, newline='\n')
(bin_dir / 'cloud-init').chmod(0o755)
image_sources = Path(tmpdir) / 'image.sources'
image_sources.write_text(NOBLE_SOURCES, newline='\n')
for _ in range(runs):
res = subprocess.run(
['bash', str(SCRIPT)],
env={**os.environ, 'PATH': f"{bin_dir}{os.pathsep}{os.environ['PATH']}",
'CRAFT_APT_ROOT': str(root), 'DOWN': down, 'STALE': stale,
'IMAGE_SOURCES': str(image_sources),
'CRAFT_CLOUD_INIT': str(bin_dir / 'cloud-init') if cloud_init else 'no-cloud-init-here'},
capture_output=True, text=True, timeout=60)
self.assertEqual(res.returncode, 0, res.stdout + res.stderr)
parts = root / 'etc/apt/sources.list.d'
written = {name: (parts / name).read_text() for name in UBUNTU}
image = {name: (root / name).read_text() for name in files}
return res, written, image
def test_ubuntu_itself_when_it_answers(self):
res, written, _ = self.select()
self.assertEqual(written, UBUNTU)
self.assertNotIn('::warning::', res.stdout)
def test_dotsrc_for_both_architectures_when_ubuntu_com_is_down(self):
res, written, _ = self.select(down=UBUNTU_HOSTS)
self.assertEqual(written, DOTSRC)
self.assertIn('::warning::install-riscv64-cross: no noble InRelease for amd64 from pair 1 '
'(curl: (6) Could not resolve host: archive.ubuntu.com', res.stdout)
def test_a_pair_with_mismatched_snapshots_is_passed_over(self):
res, written, _ = self.select(down=UBUNTU_HOSTS, stale='https://mirrors.dotsrc.org/ubuntu-ports/')
self.assertEqual(written, KERNEL)
self.assertIn('::warning::install-riscv64-cross: pair 2 serves different amd64 and riscv64 snapshots',
res.stdout)
def test_without_a_matching_pair_the_first_that_answers_is_used(self):
res, written, _ = self.select(down=UBUNTU_HOSTS, stale='https://mirrors.dotsrc.org/ubuntu-ports/ '
'https://ftp.tu-chemnitz.de/')
self.assertEqual(written, DOTSRC)
self.assertIn('no mirror pair serves one snapshot for both architectures; using pair 2', res.stdout)
def test_with_nothing_answering_ubuntu_itself_is_kept(self):
res, written, _ = self.select(down=UBUNTU_HOSTS + ' mirrors.dotsrc.org mirrors.edge.kernel.org '
'ftp.tu-chemnitz.de')
self.assertEqual(written, UBUNTU)
self.assertIn('using pair 1', res.stdout)
def test_the_image_sources_are_disabled_once(self):
_, _, image = self.select(runs=2)
sources = image['etc/apt/sources.list.d/ubuntu.sources']
self.assertEqual(sources.count('Enabled: no'), 2)
self.assertEqual(sources.replace('Enabled: no\n', ''), NOBLE_SOURCES)
def test_cloud_init_re_enabling_the_image_sources_is_waited_out(self):
_, _, image = self.select(cloud_init=True)
self.assertEqual(image['etc/apt/sources.list.d/ubuntu.sources'].count('Enabled: no'), 2)
def test_one_line_image_sources_are_commented_out(self):
_, _, image = self.select({'etc/apt/sources.list': '# comment\n'
'deb mirror+file:/etc/apt/apt-mirrors.txt noble main\n'
'deb-src mirror+file:/etc/apt/apt-mirrors.txt noble main\n'})
self.assertEqual(image['etc/apt/sources.list'], '# comment\n'
'# install-riscv64-cross: deb mirror+file:/etc/apt/apt-mirrors.txt noble main\n'
'# install-riscv64-cross: deb-src mirror+file:/etc/apt/apt-mirrors.txt noble main\n')
def test_the_action_runs_the_script_before_apt(self):
run = yaml.load(ACTION.read_text(), Loader=yaml.BaseLoader)['runs']['steps'][0]['run']
self.assertLess(run.index('dpkg --add-architecture riscv64'),
run.index('sudo "$GITHUB_ACTION_PATH/select-mirrors.sh"'))
self.assertLess(run.index('select-mirrors.sh'), run.index('apt-get update'))
if __name__ == '__main__':
unittest.main()
tests/test_nextest_versions.py
+3 -3
View File
@@ -99,7 +99,7 @@ class NextestVersionTests(unittest.TestCase):
gate = self.documents[GATE]
self.assertEqual(gate['on']['workflow_call']['inputs']['tools']['default'], PIN)
installers = [step for step in gate['jobs']['gate']['steps']
if step.get('uses', '').startswith('taiki-e/install-action@')]
if step.get('uses', '').startswith('step-security/taiki-e-install-action@')]
self.assertEqual(len(installers), 1)
self.assertEqual(installers[0]['with']['tool'], '${{ inputs.tools }}')
self.assertEqual(installers[0]['if'], "inputs.tools != ''")
@@ -109,7 +109,7 @@ class NextestVersionTests(unittest.TestCase):
for job, expected in [('build', 'cargo-llvm-cov,' + PIN), ('coverage', PIN)]:
with self.subTest(job=job):
installers = [step for step in jobs[job]['steps']
if step.get('uses', '').startswith('taiki-e/install-action@')]
if step.get('uses', '').startswith('step-security/taiki-e-install-action@')]
self.assertEqual(len(installers), 1)
self.assertEqual(installers[0]['with']['tool'], expected)
@@ -137,7 +137,7 @@ class NextestVersionTests(unittest.TestCase):
with self.subTest(tool=tool):
changed = deepcopy(self.documents)
changed[SONAR]['jobs']['coverage']['steps'].append({
'uses': 'taiki-e/install-action@' + '0' * 40,
'uses': 'step-security/taiki-e-install-action@' + '0' * 40,
'with': {'tool': tool},
})
with self.assertRaises(AssertionError):
tests/test_release_dry_run.py
+39
View File
@@ -0,0 +1,39 @@
"""release.yml's dry-run input: a nightly run on main builds and checks but releases nothing."""
from pathlib import Path
import unittest
import yaml
ROOT = Path(__file__).resolve().parents[1]
DATA = yaml.load((ROOT / '.github/workflows/release.yml').read_text(), Loader=yaml.BaseLoader)
JOBS = DATA['jobs']
DRY = '!inputs.dry-run'
class DryRunTests(unittest.TestCase):
def test_the_input_defaults_to_a_real_release(self):
spec = DATA['on']['workflow_call']['inputs']['dry-run']
self.assertEqual((spec['type'], spec['default']), ('boolean', 'false'))
def test_no_job_enters_the_release_environment_on_a_dry_run(self):
# The environment holds the signing secrets and only admits the release branch.
for name, job in JOBS.items():
env = job.get('environment')
if env is not None:
self.assertIn(DRY, env, name)
self.assertNotEqual(env.strip(), 'release', name)
def test_the_draft_and_macos_are_skipped(self):
# The draft is the release; macOS runs on a paid larger runner.
for name in ('release', 'macos'):
self.assertIn(DRY, JOBS[name].get('if', ''), name)
def test_nothing_is_pushed_to_a_registry(self):
for name, job in JOBS.items():
for step in job.get('steps', []):
if 'skopeo copy' in step.get('run', '') or 'docker push' in step.get('run', ''):
self.assertIn(DRY, step.get('if', ''), f"{name}: {step.get('name')}")
if __name__ == '__main__':
unittest.main()
tests/test_release_flatpak_oci.py
+2 -2
View File
@@ -98,8 +98,8 @@ class FlatpakOciPublishTests(unittest.TestCase):
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
self.assertEqual(self.log.read_text(), "")
def test_the_step_never_runs_for_a_pull_request(self):
self.assertEqual(STEP.get("if"), "github.event_name != 'pull_request'")
def test_the_step_never_runs_for_a_pull_request_or_a_dry_run(self):
self.assertEqual(STEP.get("if"), "github.event_name != 'pull_request' && !inputs.dry-run")
def test_dockerhub_credentials_trigger_dual_push_for_stable_release(self):
result = self.publish("1.2.0", "refs/heads/release", DOCKERHUB_USERNAME="user", DOCKERHUB_TOKEN="secret")
tests/test_release_inputs.py
+1 -1
View File
@@ -51,7 +51,7 @@ Path('args.json').write_text(json.dumps(sys.argv[1:]))
configured = {**job.get('env', {}), **step.get('env', {})}
if job_name == 'freebsd':
vm = next(item for item in job['steps']
if item.get('uses', '').startswith('vmactions/freebsd-vm@'))
if item.get('uses', '').startswith('step-security/freebsd-vm@'))
forwarded = vm['with']['envs'].split()
configured = {key: value for key, value in configured.items() if key in forwarded}
env.update({key: value.replace(EXPRESSION, app_name)
tests/test_release_publish.py
+78
View File
@@ -0,0 +1,78 @@
"""release.yml's publish input: the release job publishes its own draft, but only a complete one."""
import os
from pathlib import Path
import subprocess
import sys
import tempfile
import unittest
import yaml
ROOT = Path(__file__).resolve().parents[1]
DATA = yaml.load((ROOT / '.github/workflows/release.yml').read_text(), Loader=yaml.BaseLoader)
STEP = next(s for s in DATA['jobs']['release']['steps'] if s.get('name') == 'Publish the release')
# A gh that answers from env: NOW assets on the draft, BEFORE on the last release (LAST), and
# records an edit in EDITED.
FAKE_GH = '''import os, sys
args = sys.argv[1:]
if args[:2] == ["release", "list"]:
print(os.environ.get("LAST", ""))
elif args[:2] == ["release", "view"]:
print(os.environ["NOW"] if args[2] == os.environ["TAG"] else os.environ["BEFORE"])
elif args[:2] == ["release", "edit"]:
open(os.environ["EDITED"], "w").write(" ".join(args))
else:
sys.exit(2)
'''
class PublishTests(unittest.TestCase):
def run_step(self, now, before, last="v0.20.0", cancelled="false", with_summary=False):
with tempfile.TemporaryDirectory() as d:
tools = Path(d) / 'tools'
tools.mkdir()
gh = tools / 'gh'
gh.write_text(f'#!{sys.executable}\n' + FAKE_GH)
gh.chmod(0o755)
edited, summary = Path(d) / 'edited', Path(d) / 'summary'
env = dict(os.environ, PATH=f"{tools}{os.pathsep}{os.environ['PATH']}", TAG='v0.21.0',
NOW=str(now), BEFORE=str(before), LAST=last, CANCELLED=cancelled,
EDITED=str(edited), GITHUB_STEP_SUMMARY=str(summary))
r = subprocess.run(['bash', '-c', STEP['run']], env=env, capture_output=True, text=True)
out = (r, edited.read_text() if edited.exists() else '')
return out + (summary.read_text() if summary.exists() else '',) if with_summary else out
def test_only_on_the_release_branch_when_asked_and_not_in_a_dry_run(self):
self.assertEqual(STEP['if'], "inputs.publish && !inputs.dry-run && github.ref == 'refs/heads/release'")
spec = DATA['on']['workflow_call']['inputs']['publish']
self.assertEqual((spec['type'], spec['default']), ('boolean', 'false'))
def test_a_complete_draft_is_published(self):
r, edited = self.run_step(now=50, before=50)
self.assertEqual(r.returncode, 0, r.stdout + r.stderr)
self.assertEqual(edited, 'release edit v0.21.0 --draft=false')
def test_fewer_assets_than_the_last_release_stays_a_draft(self):
r, edited, summary = self.run_step(now=44, before=50, with_summary=True)
self.assertNotEqual(r.returncode, 0)
self.assertIn('stays a draft', r.stdout)
self.assertIn('Not published: v0.21.0 has 44 assets where v0.20.0 had 50', summary)
self.assertEqual(edited, '')
def test_the_baseline_is_the_most_recently_published_release(self):
self.assertIn("sort_by(.publishedAt) | last", STEP['run'])
def test_a_cancelled_build_job_stays_a_draft(self):
r, edited = self.run_step(now=50, before=50, cancelled='true')
self.assertNotEqual(r.returncode, 0)
self.assertEqual(edited, '')
def test_a_first_release_has_nothing_to_compare_with(self):
r, edited = self.run_step(now=12, before=0, last='')
self.assertEqual(r.returncode, 0, r.stdout + r.stderr)
self.assertIn('--draft=false', edited)
if __name__ == '__main__':
unittest.main()
tests/test_release_train.py
+65
View File
@@ -0,0 +1,65 @@
"""scripts/release_train.py: which apps a train releases, and what it passes on."""
import importlib.util
import os
import tempfile
import unittest
from pathlib import Path
from unittest import mock
ROOT = Path(__file__).resolve().parents[1]
spec = importlib.util.spec_from_file_location("release_train", ROOT / "scripts/release_train.py")
release_train = importlib.util.module_from_spec(spec)
spec.loader.exec_module(release_train)
class TrainTests(unittest.TestCase):
def run_train(self, argv, rc=0):
calls = []
def fake(repo, extra, logs):
calls.append((repo, extra))
return repo, rc, "done"
cwd = os.getcwd()
os.chdir(tempfile.mkdtemp())
try:
with mock.patch.object(release_train, "run_app", side_effect=fake):
code = release_train.main(["--logs", "logs"] + argv)
finally:
os.chdir(cwd)
return code, calls
def test_all_but_the_skipped_apps_with_the_flags_passed_on(self):
code, calls = self.run_train(["--all", "--skip", "figcraft,soundcraft", "--publish", "--", "--part", "patch"])
repos = sorted(r for r, _ in calls)
self.assertEqual(code, 0)
self.assertEqual(len(repos), len(release_train.APPS) - 2)
self.assertNotIn("storytold/figcraft", repos)
self.assertTrue(all(extra == ["--part", "patch", "--publish"] for _, extra in calls))
def test_only_owner_name_repos_and_known_options_reach_cut_release(self):
release_train.check(["storytold/pdfcraft"], ["--part", "patch", "--publish", "--gate-workflow", "CI"])
for repos, extra in [(["--upload-pack=x"], []), (["pdfcraft"], []),
(["storytold/pdfcraft"], ["--commit", "abc"]), (["storytold/pdfcraft"], ["--part", "major"]),
(["storytold/pdfcraft"], ["--version"])]:
with self.assertRaises(SystemExit, msg=f"{repos} {extra}"):
release_train.check(repos, extra)
def test_the_log_folder_stays_inside_the_current_directory(self):
cwd = os.getcwd()
os.chdir(tempfile.mkdtemp())
try:
self.assertEqual(release_train.logs_dir("logs/today"), Path.cwd().resolve() / "logs" / "today")
for bad in ["/tmp/x", "../outside", "logs/../../x", "C:/Windows", "a b"]:
with self.assertRaises(SystemExit, msg=bad):
release_train.logs_dir(bad)
finally:
os.chdir(cwd)
def test_one_stopped_app_fails_the_train(self):
code, _ = self.run_train(["storytold/pdfcraft"], rc=1)
self.assertEqual(code, 1)
if __name__ == "__main__":
unittest.main()
tests/test_sign_linux.py
+2 -1
View File
@@ -156,7 +156,8 @@ class SigningWiringTests(unittest.TestCase):
def test_the_flatpak_job_imports_the_key_before_building_the_bundle(self):
job = WORKFLOW["jobs"]["flatpak"]
self.assertEqual(job["environment"], "release")
# The release environment holds the signing key; a dry run (release.yml's dry-run) stays out.
self.assertEqual(job["environment"], "${{ !inputs.dry-run && 'release' || '' }}")
uses = [s.get("uses", "") for s in job["steps"]]
names = [s.get("name") for s in job["steps"]]
imported = next(i for i, u in enumerate(uses) if u.startswith("storytold/craft-common-actions/actions/import-signing-key@"))
tests/test_sonarcloud.py
+158 -3
View File
@@ -22,7 +22,7 @@ class SonarCloudTests(unittest.TestCase):
self.steps = workflow['jobs']['sonarcloud']['steps']
self.validation = next(step for step in self.steps if step.get('id') == 'project')
self.scan = next(step for step in self.steps
if step.get('uses', '').startswith('SonarSource/sonarqube-scan-action@'))
if step.get('uses', '').startswith('step-security/sonarqube-scan-action@'))
def run_step(self, step, **env):
with tempfile.TemporaryDirectory() as temp:
@@ -41,7 +41,7 @@ class SonarCloudTests(unittest.TestCase):
with self.subTest(key=key):
result, output = self.run_step(self.validation, PROJECT_KEY=key)
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(output, f'key={key}\n')
self.assertEqual(output, f'key={key}\ncpd=**/i18n/**\n')
def test_invalid_keys_cannot_publish_scanner_arguments(self):
for key in (
@@ -80,16 +80,134 @@ class SonarCloudTests(unittest.TestCase):
self.assertEqual(
self.scan['with']['args'].splitlines()[0],
'"-Dsonar.projectKey=${{ steps.project.outputs.key }}" -Dsonar.organization=storytold '
'-Dsonar.rust.lcov.reportPaths=lcov.info "-Dsonar.cpd.exclusions=**/i18n/**"',
'-Dsonar.rust.lcov.reportPaths=lcov.info '
'"-Dsonar.cpd.exclusions=${{ steps.project.outputs.cpd }}"',
)
def run_validation_with_properties(self, properties):
with tempfile.TemporaryDirectory() as temp:
(Path(temp) / 'sonar-project.properties').write_text(properties)
output = Path(temp) / 'output'
result = subprocess.run(
['bash', '--noprofile', '--norc', '-e', '-o', 'pipefail', '-c', self.validation['run']],
cwd=temp,
env={**os.environ, 'PROJECT_KEY': 'storytold_x', 'LC_ALL': 'C', 'GITHUB_OUTPUT': str(output)},
capture_output=True,
text=True,
)
return result, output.read_text() if output.exists() else ''
def test_repository_duplication_exclusions_are_appended(self):
result, output = self.run_validation_with_properties(
'sonar.projectKey=storytold_x\nsonar.cpd.exclusions=**/tables.rs,crates/core/src/region.rs\n')
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(output, 'key=storytold_x\ncpd=**/i18n/**,**/tables.rs,crates/core/src/region.rs\n')
def test_duplication_exclusions_are_read_as_the_scanner_reads_them(self):
for line in ('sonar.cpd.exclusions = **/table?.rs', 'sonar.cpd.exclusions:**/table?.rs',
' sonar.cpd.exclusions=**/table?.rs ', 'sonar.cpd.exclusions=**/table?.rs\r'):
with self.subTest(line=line):
result, output = self.run_validation_with_properties(f'sonar.organization=x\r\n{line}\n')
self.assertEqual(result.returncode, 0, result.stdout)
self.assertEqual(output, 'key=storytold_x\ncpd=**/i18n/**,**/table?.rs\n')
def test_duplication_exclusions_cannot_publish_scanner_arguments(self):
for value in ('a" -Dsonar.host.url="https://attacker.invalid', 'a b', '$(touch injected)', "a'b"):
with self.subTest(value=value):
result, output = self.run_validation_with_properties(f'sonar.cpd.exclusions={value}\n')
self.assertNotEqual(result.returncode, 0)
self.assertIn('::error::', result.stdout)
self.assertEqual(output, '')
def test_coverage_is_generated_before_the_scan_imports_it(self):
coverage = next(step for step in self.steps if step.get('id') == 'coverage')
fetch = next(step for step in self.steps if step.get('name') == 'Fetch locked Rust dependencies')
self.assertLess(self.steps.index(fetch), self.steps.index(coverage))
self.assertEqual(fetch['run'], 'cargo fetch --locked')
self.assertEqual(
fetch['if'],
"steps.token.outputs.available == 'true' && hashFiles('Cargo.toml') != '' && inputs.shards <= 1",
)
with tempfile.TemporaryDirectory() as temp:
root = Path(temp)
bin_dir = root / 'bin'
bin_dir.mkdir()
cargo = bin_dir / 'cargo'
cargo.write_text('#!/usr/bin/env bash\nprintf "%s\\n" "$*" > "$CARGO_CALLS"\n')
cargo.chmod(0o755)
calls = root / 'calls'
result = subprocess.run(
['bash', '--noprofile', '--norc', '-e', '-o', 'pipefail', '-c', fetch['run']],
cwd=root,
env={**os.environ, 'PATH': f'{bin_dir}:{os.environ["PATH"]}', 'CARGO_CALLS': str(calls)},
capture_output=True,
text=True,
)
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(calls.read_text(), 'fetch --locked\n')
self.assertLess(self.steps.index(coverage), self.steps.index(self.scan))
self.assertIn('cargo llvm-cov', coverage['run'])
self.assertIn('--output-path lcov.info', coverage['run'])
self.assertIn('--no-report --no-fail-fast', coverage['run'])
self.assertNotIn('--ignore-run-fail', coverage['run'])
self.assertIn('echo "test_status=$test_status" >> "$GITHUB_OUTPUT"', coverage['run'])
self.assertEqual(coverage['continue-on-error'], 'true')
def test_instrumented_test_failure_is_saved_while_lcov_is_still_written(self):
coverage = next(step for step in self.steps if step.get('id') == 'coverage')
with tempfile.TemporaryDirectory() as temp:
root = Path(temp)
bin_dir = root / 'bin'
bin_dir.mkdir()
cargo = bin_dir / 'cargo'
cargo.write_text(
'#!/usr/bin/env bash\n'
'printf "%s\\n" "$*" >> "$CARGO_CALLS"\n'
'if [[ "$*" == *"--no-report"* ]]; then exit 101; fi\n'
'printf "TN:\\n" > lcov.info\n'
)
cargo.chmod(0o755)
calls = root / 'calls'
output = root / 'output'
result = subprocess.run(
['bash', '--noprofile', '--norc', '-e', '-o', 'pipefail', '-c', coverage['run']],
cwd=root,
env={**os.environ, **coverage.get('env', {}), 'PATH': f'{bin_dir}:{os.environ["PATH"]}',
'CARGO_CALLS': str(calls), 'GITHUB_OUTPUT': str(output),
'CARGO_FEATURES': '', 'SQLX_OFFLINE_INPUT': 'false', 'COVERAGE_IGNORE': ''},
capture_output=True,
text=True,
)
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual(output.read_text(), 'test_status=101\n')
self.assertTrue((root / 'lcov.info').is_file())
invocations = calls.read_text().splitlines()
self.assertEqual(len(invocations), 2)
self.assertIn('--no-report --no-fail-fast', invocations[0])
self.assertEqual(invocations[1], 'llvm-cov report --lcov --output-path lcov.info --ignore-filename-regex ')
def test_coverage_generation_failure_is_blocking_after_scan(self):
coverage = next(step for step in self.steps if step.get('id') == 'coverage')
failure = next(step for step in self.steps
if step.get('name') == 'Fail on Rust coverage generation failure')
self.assertEqual(failure['if'], 'steps.coverage.outcome == \'failure\'')
self.assertGreater(self.steps.index(failure), self.steps.index(self.scan))
self.assertIn('cargo llvm-cov failed to build the test archive or write lcov.info', failure['run'])
self.assertTrue(coverage['continue-on-error'], 'the scan still runs to publish diagnostics')
def test_instrumented_test_failure_is_blocking_after_scan(self):
coverage = next(step for step in self.steps if step.get('id') == 'coverage')
failure = next(step for step in self.steps
if step.get('name') == 'Fail on Rust instrumented test failures')
self.assertEqual(
failure['if'],
"steps.coverage.outputs.test_status != '' && steps.coverage.outputs.test_status != '0'",
)
self.assertGreater(self.steps.index(failure), self.steps.index(self.scan))
self.assertGreater(self.steps.index(failure), self.steps.index(coverage))
self.assertEqual(failure['env']['RUST_TEST_STATUS'], '${{ steps.coverage.outputs.test_status }}')
self.assertIn('instrumented Rust tests exited with status', failure['run'])
def test_coverage_actions_track_main_and_pin_external_actions(self):
workflow = yaml.load(
(ROOT / '.github/workflows/sonarcloud.yml').read_text(), Loader=yaml.BaseLoader)
@@ -107,6 +225,43 @@ class SonarCloudTests(unittest.TestCase):
else:
self.assertRegex(step['uses'], r'^[^@]+@[0-9a-f]{40}$')
def test_caller_i18n_is_available_before_each_coverage_run(self):
workflow = yaml.load(
(ROOT / '.github/workflows/sonarcloud.yml').read_text(), Loader=yaml.BaseLoader)
for job, runner_name in [('build', 'Build instrumented test archive'),
('coverage', "Run this shard's tests"),
('sonarcloud', 'Generate Rust coverage')]:
steps = workflow['jobs'][job]['steps']
checkout = next(step for step in steps if step.get('name') == 'Check out craft-i18n')
runner = next(step for step in steps if step.get('name') == runner_name)
self.assertEqual(checkout['uses'], 'storytold/craft-common-actions/actions/checkout-craft-i18n@main')
self.assertLess(steps.index(checkout), steps.index(runner))
if job == 'sonarcloud':
self.assertIn('inputs.shards <= 1', checkout['if'])
self.assertIn("steps.token.outputs.available == 'true'", checkout['if'])
self.assertEqual(checkout['id'], 'i18n')
steps = workflow['jobs']['sonarcloud']['steps']
move = next(step for step in steps if step.get('name') == 'Move craft-i18n outside scan workspace')
coverage = next(step for step in steps if step.get('name') == 'Generate Rust coverage')
scan = next(step for step in steps if step.get('name') == 'SonarCloud Scan')
self.assertEqual(move['if'], "steps.i18n.outputs.rev != ''")
self.assertLess(steps.index(coverage), steps.index(move))
self.assertLess(steps.index(move), steps.index(scan))
with tempfile.TemporaryDirectory() as temp:
root = Path(temp)
workspace, runner = root / 'workspace', root / 'runner'
(workspace / 'craft-i18n').mkdir(parents=True)
runner.mkdir()
(workspace / 'craft-i18n' / 'catalog').write_text('pinned')
result = subprocess.run(
['bash', '-e', '-o', 'pipefail', '-c', move['run']],
env={**os.environ, 'GITHUB_WORKSPACE': str(workspace), 'RUNNER_TEMP': str(runner)},
capture_output=True, text=True)
self.assertEqual(result.returncode, 0, result.stderr)
self.assertFalse((workspace / 'craft-i18n').exists())
self.assertEqual([p.read_text() for p in runner.glob('craft-i18n.*/checkout/catalog')], ['pinned'])
def test_archive_build_has_the_same_fonts_as_runtime(self):
workflow = yaml.load(
(ROOT / '.github/workflows/sonarcloud.yml').read_text(), Loader=yaml.BaseLoader)
tests/test_sonarcloud_disk.py
+9 -8
View File
@@ -74,10 +74,10 @@ class SonarCloudDiskTests(unittest.TestCase):
runner_temp.mkdir()
sentinel = runner_temp / 'keep-unrelated.txt'
sentinel.write_text('keep')
profiles = workspace / 'profraw'
profiles = workspace / 'profdata'
profiles.mkdir()
for name in ('one', 'two'):
(profiles / f'{name}.profraw').write_text(name)
for shard in ('1', '2'):
(profiles / f'shard-{shard}.profdata').write_text(shard)
archive = workspace / 'archive'
archive.mkdir()
(archive / 'rustc-release.txt').write_text('1.99.1\n')
@@ -109,13 +109,14 @@ with open(os.environ['CALLS'], 'a') as calls:
if name == 'rustup':
print(os.environ['SYSROOT'] if '--print' in args else 'host: test-host')
elif name == 'llvm-profdata':
assert list(pathlib.Path('profraw').glob('*.profraw'))
assert args[:2] == ['merge', '-sparse'], args
assert sorted(args[2:4]) == ['profdata/shard-1.profdata', 'profdata/shard-2.profdata'], args
if failure == 'merge':
sys.exit(7)
pathlib.Path(args[args.index('-o') + 1]).write_text('merged profile')
elif name == 'tar':
assert pathlib.Path('coverage.profdata').is_file()
assert not list(pathlib.Path('profraw').glob('*.profraw'))
assert not list(pathlib.Path('profdata').glob('*.profdata'))
if failure == 'extract':
(pathlib.Path(args[args.index('-C') + 1]) / 'partial').write_text('partial extract')
sys.exit(8)
@@ -142,7 +143,7 @@ elif name == 'llvm-cov':
return {
'result': result,
'calls': calls.read_text().splitlines(),
'raw_profiles': sorted(path.name for path in profiles.glob('*.profraw')),
'shard_profiles': sorted(path.name for path in profiles.glob('*.profdata')),
'archive_exists': archive_file.exists(),
'merged_exists': (workspace / 'coverage.profdata').exists(),
'extracted': list(runner_temp.glob('coverage-archive*')),
@@ -155,7 +156,7 @@ elif name == 'llvm-cov':
observed = self.run_combine()
self.assertEqual(observed['result'].returncode, 0, observed['result'].stderr)
self.assertLess(observed['calls'].index('llvm-profdata'), observed['calls'].index('tar'))
self.assertEqual(observed['raw_profiles'], [])
self.assertEqual(observed['shard_profiles'], [])
self.assertFalse(observed['archive_exists'])
self.assertFalse(observed['merged_exists'])
self.assertEqual(observed['extracted'], [])
@@ -165,7 +166,7 @@ elif name == 'llvm-cov':
def test_failed_profile_merge_preserves_inputs_and_never_extracts(self):
observed = self.run_combine('merge')
self.assertEqual(observed['result'].returncode, 7, observed['result'].stderr)
self.assertEqual(observed['raw_profiles'], ['one.profraw', 'two.profraw'])
self.assertEqual(observed['shard_profiles'], ['shard-1.profdata', 'shard-2.profdata'])
self.assertTrue(observed['archive_exists'])
self.assertNotIn('tar', observed['calls'])
self.assertEqual(observed['extracted'], [])
tests/test_sonarcloud_shard_merge.py
+189
View File
@@ -0,0 +1,189 @@
"""Exercise the per-shard profile merge in sharded Sonar coverage."""
import os
from pathlib import Path
import shutil
import subprocess
import tempfile
import unittest
import yaml
ROOT = Path(__file__).resolve().parents[1]
CORRUPT = 'invalid instrumentation profile data (file header is corrupt)'
def llvm_release():
"""Return a clang and llvm-profdata from one LLVM release, so the raw profile format matches."""
for bin_dir in sorted(Path('/usr/lib').glob('llvm-*/bin'), reverse=True):
clang, profdata = bin_dir / 'clang', bin_dir / 'llvm-profdata'
if clang.is_file() and profdata.is_file():
return clang, profdata
return None
def load_workflow():
return yaml.load((ROOT / '.github/workflows/sonarcloud.yml').read_text(),
Loader=yaml.BaseLoader)
@unittest.skipUnless(shutil.which('bash'), 'requires bash')
class ShardMergeTests(unittest.TestCase):
def setUp(self):
workflow = load_workflow()
self.shard_steps = workflow['jobs']['coverage']['steps']
self.scan_steps = workflow['jobs']['sonarcloud']['steps']
self.merge = next(step for step in self.shard_steps
if step.get('name') == "Merge this shard's profiles")
def test_shards_upload_one_merged_profile_and_the_scan_merges_those(self):
names = [step.get('name') or step.get('uses', '') for step in self.shard_steps]
tests = names.index("Run this shard's tests")
install = names.index("Install LLVM tools for the archive's toolchain")
merge = names.index("Merge this shard's profiles")
upload = next(index for index, step in enumerate(self.shard_steps)
if step.get('uses', '').startswith('actions/upload-artifact@'))
self.assertLess(tests, install)
self.assertLess(install, merge)
self.assertLess(merge, upload)
self.assertIn('archive/rustc-release.txt', self.shard_steps[install]['run'])
self.assertIn('llvm-tools-preview', self.shard_steps[install]['run'])
self.assertEqual(self.merge['env']['SHARD'], '${{ matrix.shard }}')
uploaded = self.shard_steps[upload]['with']
self.assertEqual(uploaded['name'], 'profdata-${{ matrix.shard }}')
self.assertEqual(uploaded['path'], 'profdata/')
self.assertEqual(uploaded['if-no-files-found'], 'error')
# No job uploads or downloads raw profiles any more.
for job in load_workflow()['jobs'].values():
for step in job['steps']:
with self.subTest(step=step.get('name') or step.get('uses')):
self.assertNotIn('profraw', step.get('with', {}).get('name', ''))
self.assertNotIn('profraw', step.get('with', {}).get('pattern', ''))
self.assertNotIn('profraw', step.get('with', {}).get('path', ''))
download = next(step for step in self.scan_steps
if step.get('name') == 'Download shard profiles and archive')
self.assertEqual(download['with']['pattern'], 'profdata-*')
self.assertEqual(download['with']['path'], 'profdata')
combine = next(step for step in self.scan_steps
if step.get('name') == 'Combine shard coverage')
self.assertIn('merge -sparse profdata/*.profdata -o coverage.profdata', combine['run'])
def run_merge(self, profiles, profdata_tool):
"""Run the merge step with a fake rustup whose sysroot holds the given llvm-profdata."""
temp = tempfile.TemporaryDirectory(prefix='sonar shard ')
self.addCleanup(temp.cleanup)
root = Path(temp.name)
workspace = root / 'workspace'
(workspace / 'archive').mkdir(parents=True)
(workspace / 'archive/rustc-release.txt').write_text('1.99.1\n')
raw = workspace / 'profraw'
raw.mkdir()
for name, data in profiles.items():
(raw / name).write_bytes(data)
sysroot = root / 'toolchain'
llvm = sysroot / 'lib/rustlib/test-host/bin'
llvm.mkdir(parents=True)
(llvm / 'llvm-profdata').symlink_to(profdata_tool)
tools = root / 'tools'
tools.mkdir()
rustup = tools / 'rustup'
rustup.write_text('#!/bin/sh\n'
'case "$*" in *--print*) echo "$SYSROOT" ;; *) echo "host: test-host" ;; esac\n')
rustup.chmod(0o755)
result = subprocess.run(
['bash', '--noprofile', '--norc', '-e', '-o', 'pipefail', '-c', self.merge['run']],
cwd=workspace, capture_output=True, text=True,
env={**os.environ, 'PATH': f'{tools}{os.pathsep}{os.environ["PATH"]}',
'SYSROOT': str(sysroot), 'SHARD': '3', 'CALLS': str(root / 'calls')},
)
return result, workspace
def fake_profdata(self, corrupt=''):
temp = tempfile.TemporaryDirectory(prefix='fake llvm ')
self.addCleanup(temp.cleanup)
tool = Path(temp.name) / 'llvm-profdata'
tool.write_text(f'''#!/usr/bin/env python3
import os, pathlib, sys
args = sys.argv[1:]
with open(os.environ['CALLS'], 'a') as calls:
calls.write(' '.join(args) + '\\n')
corrupt = {corrupt!r}
if corrupt:
print(f'warning: profraw/{{corrupt}}: {CORRUPT}', file=sys.stderr)
print('error: no profile can be merged', file=sys.stderr)
sys.exit(1)
pathlib.Path(args[args.index('-o') + 1]).write_text('merged')
''')
tool.chmod(0o755)
return tool
def test_merge_replaces_raw_profiles_with_one_sparse_profile(self):
result, workspace = self.run_merge(
{'shard-3-1-a.profraw': b'a', 'shard-3-2-b.profraw': b'b'}, self.fake_profdata())
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
calls = (workspace.parent / 'calls').read_text().split()
self.assertEqual(calls[:3], ['merge', '-sparse', '--failure-mode=any'])
self.assertEqual(sorted(calls[3:5]),
['profraw/shard-3-1-a.profraw', 'profraw/shard-3-2-b.profraw'])
self.assertEqual(calls[5:], ['-o', 'profdata/shard-3.profdata'])
self.assertEqual([path.name for path in (workspace / 'profdata').iterdir()],
['shard-3.profdata'])
self.assertFalse((workspace / 'profraw').exists())
self.assertNotIn('::error::', result.stdout)
def test_corrupt_raw_profile_fails_the_shard_and_names_the_file(self):
result, workspace = self.run_merge(
{'shard-3-1-a.profraw': b'a', 'shard-3-2-b.profraw': b'b'},
self.fake_profdata(corrupt='shard-3-2-b.profraw'))
self.assertEqual(result.returncode, 1)
self.assertIn('::error::shard 3 wrote a truncated or corrupt coverage profile: '
f'profraw/shard-3-2-b.profraw: {CORRUPT}', result.stdout)
self.assertIn("::error::llvm-profdata could not merge shard 3's profiles (exit 1)",
result.stdout)
# Nothing is left for the upload step, and the raw profiles stay for inspection.
self.assertFalse((workspace / 'profdata').exists())
self.assertEqual(sorted(path.name for path in (workspace / 'profraw').iterdir()),
['shard-3-1-a.profraw', 'shard-3-2-b.profraw'])
@unittest.skipUnless(llvm_release(), 'requires clang and llvm-profdata from /usr/lib/llvm-*')
def test_real_llvm_profdata_rejects_a_truncated_raw_profile(self):
clang, profdata = llvm_release()
with tempfile.TemporaryDirectory(prefix='sonar real ') as temp:
root = Path(temp)
source = root / 'main.c'
source.write_text('int twice(int x) { return 2 * x; }\n'
'int main(int argc, char **argv) { (void)argv; return twice(argc) - 2; }\n')
binary = root / 'main'
built = subprocess.run([str(clang), '-fprofile-instr-generate', '-fcoverage-mapping',
str(source), '-o', str(binary)], capture_output=True, text=True)
if built.returncode != 0:
# The profile runtime (libclang_rt.profile) is a separate package on some systems.
self.skipTest(f'{clang} cannot build an instrumented binary: {built.stderr}')
profiles = {}
for run in ('one', 'two'):
raw = root / f'{run}.profraw'
subprocess.run([str(binary)], check=True,
env={**os.environ, 'LLVM_PROFILE_FILE': str(raw)})
profiles[f'shard-3-{run}.profraw'] = raw.read_bytes()
result, workspace = self.run_merge(profiles, profdata)
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
merged = workspace / 'profdata/shard-3.profdata'
shown = subprocess.run([str(profdata), 'show', '--function=twice', '--counts', str(merged)],
check=True, capture_output=True, text=True)
# Two runs of one call each, merged.
self.assertIn('Function count: 2', shown.stdout)
self.assertFalse((workspace / 'profraw').exists())
whole = profiles['shard-3-two.profraw']
result, workspace = self.run_merge(
{**profiles, 'shard-3-two.profraw': whole[:len(whole) // 2]}, profdata)
self.assertNotEqual(result.returncode, 0)
self.assertIn('::error::shard 3 wrote a truncated or corrupt coverage profile: '
'profraw/shard-3-two.profraw: ', result.stdout)
self.assertFalse((workspace / 'profdata').exists())
if __name__ == '__main__':
unittest.main()
tests/test_sonarcloud_test_reporting.py
+85
View File
@@ -0,0 +1,85 @@
"""Coverage keeps scanning after test failures, but must report them visibly."""
import os
from pathlib import Path
import subprocess
import tempfile
import unittest
import yaml
ROOT = Path(__file__).resolve().parents[1]
class CoverageTestReportingTests(unittest.TestCase):
def run_shard(self, code, profiles=True):
"""Run the real shard shell with a fake cargo command and optional coverage profile."""
workflow = yaml.load(
(ROOT / '.github/workflows/sonarcloud.yml').read_text(encoding='utf-8'), Loader=yaml.BaseLoader)
step = next(step for step in workflow['jobs']['coverage']['steps']
if step.get('name') == "Run this shard's tests")
script = step['run']
with tempfile.TemporaryDirectory(prefix='coverage reporting ') as temp:
root = Path(temp)
tools = root / 'tools'
tools.mkdir()
cargo = tools / 'cargo'
cargo.write_text('#!/bin/sh\n'
+ ('printf profile > profraw/test.profraw\n' if profiles else '')
+ f'exit {code}\n', encoding='utf-8')
cargo.chmod(0o755)
summary = root / 'summary'
result = subprocess.run(
['bash', '--noprofile', '--norc', '-e', '-o', 'pipefail', '-c', script],
cwd=root, capture_output=True, text=True, encoding='utf-8',
env={**os.environ, 'PATH': f'{tools}{os.pathsep}{os.environ["PATH"]}',
'GITHUB_STEP_SUMMARY': str(summary), 'SHARD': '2', 'TOTAL_SHARDS': '4'},
)
profile = root / 'profraw/test.profraw'
return (result, summary.read_text(encoding='utf-8') if summary.exists() else '',
profile.read_text(encoding='utf-8') if profile.exists() else '')
def test_passing_tests_report_success(self):
result, summary, profile = self.run_shard(0)
self.assertEqual(result.returncode, 0, result.stderr)
self.assertIn('Coverage shard 2/4', summary)
self.assertIn('Tests passed', summary)
self.assertNotIn('Tests failed', summary)
self.assertNotIn('::warning::', result.stdout)
self.assertEqual(profile, 'profile')
def test_failed_tests_warn_and_keep_coverage_available(self):
result, summary, profile = self.run_shard(100)
self.assertEqual(result.returncode, 0, result.stderr)
self.assertIn('::warning::Coverage shard 2/4 has failing tests', result.stdout)
self.assertIn('Tests failed', summary)
self.assertIn('SonarCloud scan will continue', summary)
self.assertIn('does not mean the tests passed', summary)
self.assertNotIn('Tests passed', summary)
self.assertEqual(profile, 'profile')
def test_runner_errors_still_fail(self):
"""A nextest runner error is red and never looks like an ordinary test failure."""
for code in (1, 4, 101, 104, 105):
with self.subTest(code=code):
result, summary, _ = self.run_shard(code)
self.assertEqual(result.returncode, code, result.stderr)
self.assertIn('::error::nextest could not run shard 2', result.stdout)
self.assertNotIn('Tests passed', summary)
def test_missing_profiles_still_fail(self):
"""Report test outcomes before failing a shard that produced no coverage profiles."""
for code in (0, 100):
with self.subTest(code=code):
result, summary, _ = self.run_shard(code, profiles=False)
self.assertNotEqual(result.returncode, 0, result.stderr)
self.assertIn('::error::shard 2 produced no coverage profiles', result.stdout)
self.assertIn('Coverage shard 2/4', summary)
if code == 100:
self.assertIn('Tests failed', summary)
self.assertIn('cannot be included', summary)
self.assertNotIn('SonarCloud scan will continue', summary)
self.assertIn('::warning::Coverage shard 2/4 has failing tests', result.stdout)
else:
self.assertIn('Tests passed', summary)
tests/test_sonarcloud_test_status.py
+79
View File
@@ -0,0 +1,79 @@
"""Keep analysis available while surfacing failed archived Rust tests."""
import os
from pathlib import Path
import subprocess
import tempfile
import unittest
import yaml
ROOT = Path(__file__).resolve().parents[1]
class ShardStatusTests(unittest.TestCase):
def setUp(self):
self.workflow = yaml.load(
(ROOT / '.github/workflows/sonarcloud.yml').read_text(), Loader=yaml.BaseLoader)
self.steps = self.workflow['jobs']['sonarcloud']['steps']
self.check = next(s for s in self.steps if s.get('name') == 'Fail on Rust shard test failures')
def run_check(self, statuses):
with tempfile.TemporaryDirectory() as tmp:
directory = Path(tmp) / 'profdata'
directory.mkdir()
for shard, status in statuses.items():
(directory / f'shard-{shard}.status').write_text(status)
return subprocess.run(['bash', '-e', '-o', 'pipefail', '-c', self.check['run']],
cwd=tmp, env={**os.environ, 'TOTAL_SHARDS': '2'},
capture_output=True, text=True)
def test_success_requires_all_shards(self):
self.assertEqual(self.run_check({1: '0\n', 2: '0\n'}).returncode, 0)
self.assertNotEqual(self.run_check({1: '0\n'}).returncode, 0)
def test_failure_empty_and_malformed_status_fail(self):
for value in ('100\n', '', 'unexpected\n'):
with self.subTest(value=value):
self.assertNotEqual(self.run_check({1: '0\n', 2: value}).returncode, 0)
def test_shard_records_test_exit_code_without_losing_coverage(self):
shard_step = next(s for s in self.workflow['jobs']['coverage']['steps']
if s.get('name') == "Run this shard's tests")
for status in (0, 100, 101):
with self.subTest(status=status), tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
binary = root / 'cargo'
binary.write_text('#!/bin/bash\nmkdir -p profraw\ntouch profraw/test.profraw\nexit "$FAKE_EXIT"\n')
binary.chmod(0o755)
summary = root / 'summary'
result = subprocess.run(['bash', '-e', '-o', 'pipefail', '-c', shard_step['run']],
cwd=tmp, env={**os.environ, 'SHARD': '1', 'TOTAL_SHARDS': '2',
'FAKE_EXIT': str(status), 'PATH': f"{tmp}:{os.environ['PATH']}",
'GITHUB_STEP_SUMMARY': str(summary)},
capture_output=True, text=True)
if status in (0, 100):
self.assertEqual(result.returncode, 0, result.stderr)
self.assertEqual((root / 'profdata/shard-1.status').read_text(), f'{status}\n')
if status == 100:
self.assertIn('Tests failed', summary.read_text())
self.assertIn('::warning::', result.stdout)
else:
self.assertIn('Tests passed', summary.read_text())
else:
self.assertNotEqual(result.returncode, 0)
def test_scan_precedes_failure_and_status_survives_profile_cleanup(self):
scan = next(s for s in self.steps if s.get('name') == 'SonarCloud Scan')
self.assertLess(self.steps.index(scan), self.steps.index(self.check))
self.assertIn("steps.token.outputs.available == 'true'", self.check['if'])
self.assertIn('inputs.shards > 1', self.check['if'])
combine = next(s for s in self.steps if s.get('name') == 'Combine shard coverage')
self.assertIn('rm -f profdata/*.profdata', combine['run'])
shard = next(s for s in self.workflow['jobs']['coverage']['steps']
if s.get('name') == "Run this shard's tests")
self.assertIn('"profdata/shard-$SHARD.status"', shard['run'])
self.assertIn('[ "$rc" -ne 100 ]', shard['run'])
if __name__ == '__main__':
unittest.main()
tests/test_windows_release.py
+3 -2
View File
@@ -22,10 +22,11 @@ class WindowsReleaseWorkflowTests(unittest.TestCase):
windows_job = self.data["jobs"]["windows"]
matrix_include = windows_job["strategy"]["matrix"]["include"]
expected = [
{"arch": "x64", "target": "x86_64-pc-windows-msvc"},
{"arch": "arm64", "target": "aarch64-pc-windows-msvc"},
{"arch": "x64", "target": "x86_64-pc-windows-msvc", "runner": "windows-latest"},
{"arch": "arm64", "target": "aarch64-pc-windows-msvc", "runner": "windows-11-arm"},
]
self.assertEqual(matrix_include, expected)
self.assertEqual(windows_job["runs-on"], "${{ matrix.runner }}")
arches = [item["arch"] for item in matrix_include]
targets = [item["target"] for item in matrix_include]
self.assertNotIn("x86", arches)