From be6bbf3ca5a5de466786908b3986163da7c05314 Mon Sep 17 00:00:00 2001 From: skylar taylor-barrick Date: Sun, 25 Oct 2026 21:37:12 -0600 Subject: [PATCH] Provision fonts and optional Python coverage for Sonar (#88) * Provision coverage fonts for archive builds and test runners * Keep coverage fonts optional unless a pin is requested * Pin nested actions used by coverage jobs * Export the font path only after a requested checkout * Measure optional Python tooling coverage in Sonar * ci: restore craft-common-actions tracking main * ci: share Python coverage configuration with XML reporting --- .github/workflows/sonarcloud.yml | 106 ++++++++++++++++++++++++++++- README.md | 19 +++++- tests/test_python_coverage.py | 113 +++++++++++++++++++++++++++++++ tests/test_sonarcloud.py | 85 +++++++++++++++++++++++ 4 files changed, 320 insertions(+), 3 deletions(-) create mode 100644 tests/test_python_coverage.py diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml index bb3d416..317c294 100644 --- a/.github/workflows/sonarcloud.yml +++ b/.github/workflows/sonarcloud.yml @@ -13,6 +13,21 @@ on: type: string required: false default: "" + pdf-test-fonts: + description: "Provision the caller's pinned PDF test fonts with scripts/fetch-pdf-test-fonts.py" + type: boolean + required: false + default: false + python-coverage-source: + description: "Optional Python source directory to measure; requires python-coverage-script" + type: string + required: false + default: "" + python-coverage-script: + description: "Optional Python test script; requires python-coverage-source" + type: string + required: false + default: "" timeout-minutes: description: "Maximum job runtime, including coverage generation and the scan" type: number @@ -79,6 +94,18 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: Check out craft-fonts + uses: storytold/craft-common-actions/actions/checkout-craft-fonts@main + with: + ref: ${{ inputs.fonts-ref }} + - name: Require requested craft-fonts + if: inputs.fonts-ref != '' + shell: bash + env: + CRAFT_FONTS_PATH: ${{ github.workspace }}/craft-fonts + run: | + echo "CRAFT_FONTS_DIR=$CRAFT_FONTS_PATH" >> "$GITHUB_ENV" + echo "CRAFT_FONTS_REQUIRED=1" >> "$GITHUB_ENV" - name: Free disk space run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL - name: Install Linux build dependencies @@ -98,6 +125,12 @@ jobs: - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: save-if: ${{ github.ref == 'refs/heads/main' }} + - name: Provision pinned PDF test fonts + if: inputs.pdf-test-fonts + shell: bash + run: | + python3 scripts/fetch-pdf-test-fonts.py "$RUNNER_TEMP/craft-pdf-test-fonts" + echo "CRAFT_PDF_TEST_FONT_DIR=$RUNNER_TEMP/craft-pdf-test-fonts" >> "$GITHUB_ENV" - name: Build instrumented test archive env: CARGO_TERM_COLOR: always @@ -136,6 +169,14 @@ jobs: uses: storytold/craft-common-actions/actions/checkout-craft-fonts@main with: ref: ${{ inputs.fonts-ref }} + - name: Require requested craft-fonts + if: inputs.fonts-ref != '' + shell: bash + env: + CRAFT_FONTS_PATH: ${{ github.workspace }}/craft-fonts + run: | + echo "CRAFT_FONTS_DIR=$CRAFT_FONTS_PATH" >> "$GITHUB_ENV" + echo "CRAFT_FONTS_REQUIRED=1" >> "$GITHUB_ENV" - name: Install Linux build dependencies uses: storytold/craft-common-actions/actions/install-apt-packages@main with: @@ -150,9 +191,14 @@ jobs: with: name: nextest-archive path: archive + - name: Provision pinned PDF test fonts + if: inputs.pdf-test-fonts + shell: bash + run: | + python3 scripts/fetch-pdf-test-fonts.py "$RUNNER_TEMP/craft-pdf-test-fonts" + echo "CRAFT_PDF_TEST_FONT_DIR=$RUNNER_TEMP/craft-pdf-test-fonts" >> "$GITHUB_ENV" - name: Run this shard's tests env: - CRAFT_FONTS_DIR: ${{ github.workspace }}/craft-fonts # Each instrumented binary writes its profile here; the scan job merges them. LLVM_PROFILE_FILE: ${{ github.workspace }}/profraw/shard-${{ matrix.shard }}-%p-%m.profraw run: | @@ -221,6 +267,14 @@ jobs: uses: storytold/craft-common-actions/actions/checkout-craft-fonts@main with: ref: ${{ inputs.fonts-ref }} + - name: Require requested craft-fonts + if: inputs.fonts-ref != '' && steps.token.outputs.available == 'true' && inputs.shards <= 1 + shell: bash + env: + CRAFT_FONTS_PATH: ${{ github.workspace }}/craft-fonts + run: | + echo "CRAFT_FONTS_DIR=$CRAFT_FONTS_PATH" >> "$GITHUB_ENV" + echo "CRAFT_FONTS_REQUIRED=1" >> "$GITHUB_ENV" - name: Free disk space # The instrumented workspace build overflows the ~14 GB free on a stock runner. if: steps.token.outputs.available == 'true' && hashFiles('Cargo.toml') != '' && inputs.shards <= 1 @@ -246,6 +300,12 @@ jobs: if: steps.token.outputs.available == 'true' && hashFiles('Cargo.toml') != '' && inputs.shards <= 1 with: save-if: ${{ github.ref == 'refs/heads/main' }} + - name: Provision pinned PDF test fonts + if: inputs.pdf-test-fonts && steps.token.outputs.available == 'true' && hashFiles('Cargo.toml') != '' && inputs.shards <= 1 + shell: bash + run: | + python3 scripts/fetch-pdf-test-fonts.py "$RUNNER_TEMP/craft-pdf-test-fonts" + echo "CRAFT_PDF_TEST_FONT_DIR=$RUNNER_TEMP/craft-pdf-test-fonts" >> "$GITHUB_ENV" - name: Generate Rust coverage id: coverage if: steps.token.outputs.available == 'true' && hashFiles('Cargo.toml') != '' && inputs.shards <= 1 @@ -253,7 +313,6 @@ jobs: continue-on-error: true env: CARGO_TERM_COLOR: always - CRAFT_FONTS_DIR: ${{ github.workspace }}/craft-fonts # Coverage comes from instrumentation, not DWARF; debuginfo is most of target/. CARGO_PROFILE_DEV_DEBUG: "0" run: > @@ -301,6 +360,49 @@ jobs: for object in "${objects[@]:1}"; do extra+=(-object "$object"); done "$llvm_bin/llvm-cov" export -format=lcov -instr-profile=coverage.profdata \ "${objects[0]}" "${extra[@]}" -ignore-filename-regex="$COVERAGE_IGNORE" >lcov.info + - name: Generate Python coverage + if: steps.token.outputs.available == 'true' && (inputs.python-coverage-source != '' || inputs.python-coverage-script != '') + shell: bash + env: + PYTHON_COVERAGE_SOURCE: ${{ inputs.python-coverage-source }} + PYTHON_COVERAGE_SCRIPT: ${{ inputs.python-coverage-script }} + run: | + set -euo pipefail + if [[ -z "$PYTHON_COVERAGE_SOURCE" || -z "$PYTHON_COVERAGE_SCRIPT" ]]; then + echo '::error::Python coverage requires both source and test script inputs' + exit 1 + fi + if [[ ! -d "$PYTHON_COVERAGE_SOURCE" || ! -f "$PYTHON_COVERAGE_SCRIPT" ]]; then + echo '::error::Python coverage source directory or test script is missing' + exit 1 + fi + source_dir=$(realpath -- "$PYTHON_COVERAGE_SOURCE") + test_script=$(realpath -- "$PYTHON_COVERAGE_SCRIPT") + workspace_dir=$(pwd -P) + case "$source_dir" in + "$workspace_dir"|"$workspace_dir"/*) ;; + *) echo '::error::Python coverage source must be inside the checkout'; exit 1 ;; + esac + case "$test_script" in + "$workspace_dir"/*) ;; + *) echo '::error::Python test script must be inside the checkout'; exit 1 ;; + esac + case "$source_dir$test_script" in + *','*|*'['*|*']'*|*'*'*|*'?'*) echo '::error::Python coverage paths must not contain commas or glob characters'; exit 1 ;; + esac + coverage_env=$(mktemp -d "$RUNNER_TEMP/python-coverage.XXXXXX") + python3 -m venv "$coverage_env" + "$coverage_env/bin/python" -m pip install --disable-pip-version-check coverage==7.10.7 + export COVERAGE_FILE="$coverage_env/.coverage" + export PYTHON_COVERAGE_SOURCE_REAL="$source_dir" + export PYTHON_COVERAGE_SCRIPT_REAL="$test_script" + coverage_config="$coverage_env/coverage.ini" + # Environment substitution keeps paths literal and shares selection with reporting. + printf '%s\n' '[run]' 'branch = true' 'source = ${PYTHON_COVERAGE_SOURCE_REAL}' 'omit = ${PYTHON_COVERAGE_SCRIPT_REAL}' > "$coverage_config" + status=0 + "$coverage_env/bin/python" -m coverage run --rcfile="$coverage_config" "$test_script" || status=$? + "$coverage_env/bin/python" -m coverage xml --rcfile="$coverage_config" -o python-coverage.xml + exit "$status" - name: SonarCloud Scan if: steps.token.outputs.available == 'true' uses: SonarSource/sonarqube-scan-action@d209202bc7d53ff1cc128f7f907dac145c9d6ae9 # v8.3.0 diff --git a/README.md b/README.md index 114c22d..76f5f06 100644 --- a/README.md +++ b/README.md @@ -492,7 +492,24 @@ SonarCloud callers whose instrumented workspace tests exceed the 90-minute default can pass `timeout-minutes: 120` under the calling job's `with`. This changes the job's time budget without dropping tests or coverage. Pass `fonts-ref` at the app's tested craft-fonts commit when its tests need those -fonts. The scan still needs to finish and pass its quality gate. +fonts. The pinned fonts are available both when compiling the test archive and +when running tests; a requested font pin also sets `CRAFT_FONTS_REQUIRED=1` +so missing embedded fonts fail explicitly. Callers with `scripts/fetch-pdf-test-fonts.py` can also pass +`pdf-test-fonts: true` to provision their pinned PDF fixtures outside the checkout +and set `CRAFT_PDF_TEST_FONT_DIR` before archive compilation and either test runner. Fixture provisioning +failures stop that job and prevent the dependent scan. The scan still needs to finish and pass its quality gate. + +Callers with Python tooling can pass both `python-coverage-source` (a source directory) +and `python-coverage-script` (the test script). The scan job runs that script under +pinned coverage.py in an isolated virtual environment and writes `python-coverage.xml`. +Both paths must be inside the checkout and contain no commas or glob characters. +The environment contains only coverage.py and the runner’s Python standard library; +caller coverage configuration is ignored so parallel-data settings cannot hide the report. +Set `sonar.python.coverage.reportPaths=python-coverage.xml` in the caller's Sonar +configuration and classify its test files as tests. The script runs in full, but +is omitted from the production coverage report. Missing inputs, failed tests or +report generation stop the job; empty inputs leave existing callers unchanged. +This step requires the Sonar token, so token-independent CI should also run the tests. Workspaces run their tests as four parallel `cargo nextest` partitions by default. One job builds the instrumented test archive; each partition runs from diff --git a/tests/test_python_coverage.py b/tests/test_python_coverage.py new file mode 100644 index 0000000..5f07cdf --- /dev/null +++ b/tests/test_python_coverage.py @@ -0,0 +1,113 @@ +"""Exercise the optional coverage step's shell and failure propagation.""" + +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + +import yaml + + +ROOT = Path(__file__).resolve().parents[1] + + +class PythonCoverageTests(unittest.TestCase): + def setUp(self): + workflow = yaml.load((ROOT / '.github/workflows/sonarcloud.yml').read_text(), Loader=yaml.BaseLoader) + self.inputs = workflow['on']['workflow_call']['inputs'] + self.steps = workflow['jobs']['sonarcloud']['steps'] + self.step = next(s for s in self.steps if s.get('name') == 'Generate Python coverage') + + def run_coverage(self, *, test_status=0, report_status=0, missing=False, nonexistent=False, outside=False): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + source = root / 'source with spaces' + source.mkdir() + script = source / 'test tools.py' + script.write_text('') + binary = root / 'bin' + binary.mkdir() + stub = binary / 'python3' + stub.write_text('''#!/usr/bin/env python3 +import json, os, pathlib, shutil, sys +args = sys.argv[1:] +with open(os.environ['CALL_LOG'], 'a') as handle: + handle.write(json.dumps(args) + '\\n') +if args[:2] == ['-m', 'venv']: + target = pathlib.Path(args[2]) / 'bin' / 'python' + target.parent.mkdir(parents=True) + shutil.copyfile(__file__, target) + target.chmod(0o700) +else: + if args[:3] == ['-m', 'coverage', 'run']: + sys.exit(int(os.environ['TEST_STATUS'])) + if args[:3] == ['-m', 'coverage', 'xml']: + sys.exit(int(os.environ['REPORT_STATUS'])) +'''.replace('#!/usr/bin/env python3', '#!' + os.sys.executable, 1)) + stub.chmod(0o700) + log = root / 'calls.jsonl' + environment = dict(os.environ, PATH=str(binary) + os.pathsep + os.environ['PATH'], + RUNNER_TEMP=str(root), CALL_LOG=str(log), + TEST_STATUS=str(test_status), REPORT_STATUS=str(report_status), + PYTHON_COVERAGE_SOURCE=str(root.parent if outside else source), + PYTHON_COVERAGE_SCRIPT='' if missing else str(source / 'absent.py' if nonexistent else script)) + result = subprocess.run(['bash', '-c', self.step['run']], cwd=root, + env=environment, text=True, capture_output=True) + calls = [json.loads(line) for line in log.read_text().splitlines()] if log.exists() else [] + return result, calls, str(script) + + def test_paths_with_spaces_remain_single_arguments(self): + result, calls, script = self.run_coverage() + self.assertEqual(result.returncode, 0, result.stderr) + run = next(args for args in calls if args[:3] == ['-m', 'coverage', 'run']) + self.assertEqual(run[-1], script) + self.assertEqual(len([arg for arg in run if arg.startswith('--rcfile=')]), 1) + + def test_failed_tests_still_generate_report_but_fail_job(self): + result, calls, _ = self.run_coverage(test_status=7) + self.assertEqual(result.returncode, 7, result.stderr) + self.assertTrue(any(args[:3] == ['-m', 'coverage', 'xml'] for args in calls)) + + def test_failed_report_fails_job(self): + result, _, _ = self.run_coverage(report_status=9) + self.assertEqual(result.returncode, 9, result.stderr) + + def test_partial_configuration_fails_before_installing(self): + result, calls, _ = self.run_coverage(missing=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn('requires both', result.stdout) + self.assertEqual(calls, []) + + def test_invalid_paths_fail_before_running_tools(self): + for options in ({'nonexistent': True}, {'outside': True}): + with self.subTest(options=options): + result, calls, _ = self.run_coverage(**options) + self.assertNotEqual(result.returncode, 0) + self.assertIn('::error::', result.stdout) + self.assertEqual(calls, []) + + def test_opt_in_wiring_and_report_contract(self): + for suffix in ('source', 'script'): + self.assertEqual(self.inputs['python-coverage-' + suffix]['default'], '') + self.assertEqual(self.step['env']['PYTHON_COVERAGE_' + suffix.upper()], '${{ inputs.python-coverage-' + suffix + ' }}') + self.assertEqual(self.step['if'], "steps.token.outputs.available == 'true' && (inputs.python-coverage-source != '' || inputs.python-coverage-script != '')") + result, calls, script = self.run_coverage() + self.assertEqual(result.returncode, 0, result.stderr) + run = next(args for args in calls if args[:3] == ['-m', 'coverage', 'run']) + report = next(args for args in calls if args[:3] == ['-m', 'coverage', 'xml']) + config = next(arg for arg in run if arg.startswith('--rcfile=')) + self.assertNotEqual(config, '--rcfile=/dev/null') + self.assertEqual(report, ['-m', 'coverage', 'xml', config, '-o', 'python-coverage.xml']) + self.assertIn('source = ${PYTHON_COVERAGE_SOURCE_REAL}', self.step['run']) + self.assertIn('omit = ${PYTHON_COVERAGE_SCRIPT_REAL}', self.step['run']) + + def test_failure_is_not_suppressed_and_step_precedes_scan(self): + self.assertNotIn('continue-on-error', self.step) + scan = next(s for s in self.steps if s.get('name') == 'SonarCloud Scan') + self.assertLess(self.steps.index(self.step), self.steps.index(scan)) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_sonarcloud.py b/tests/test_sonarcloud.py index 615e579..d704db7 100644 --- a/tests/test_sonarcloud.py +++ b/tests/test_sonarcloud.py @@ -84,6 +84,91 @@ class SonarCloudTests(unittest.TestCase): self.assertIn('--output-path lcov.info', coverage['run']) self.assertEqual(coverage['continue-on-error'], 'true') + def test_coverage_actions_track_main_and_pin_external_actions(self): + workflow = yaml.load( + (ROOT / '.github/workflows/sonarcloud.yml').read_text(), Loader=yaml.BaseLoader) + step_groups = [job.get('steps', []) for job in workflow['jobs'].values()] + for helper in ('checkout-craft-fonts', 'install-apt-packages'): + action = yaml.load( + (ROOT / 'actions' / helper / 'action.yml').read_text(), Loader=yaml.BaseLoader) + step_groups.append(action['runs']['steps']) + for steps in step_groups: + for step in steps: + if 'uses' in step: + with self.subTest(action=step['uses']): + if step['uses'].startswith('storytold/craft-common-actions/'): + self.assertTrue(step['uses'].endswith('@main')) + else: + self.assertRegex(step['uses'], r'^[^@]+@[0-9a-f]{40}$') + + def test_archive_build_has_the_same_fonts_as_runtime(self): + workflow = yaml.load( + (ROOT / '.github/workflows/sonarcloud.yml').read_text(), Loader=yaml.BaseLoader) + for job, runner_name in [('build', 'Build instrumented test archive'), + ('coverage', "Run this shard's tests"), + ('sonarcloud', 'Generate Rust coverage')]: + steps = workflow['jobs'][job]['steps'] + checkout = next(step for step in steps if step.get('name') == 'Check out craft-fonts') + runner = next(step for step in steps if step.get('name') == runner_name) + self.assertLess(steps.index(checkout), steps.index(runner)) + self.assertEqual(checkout['with']['ref'], '${{ inputs.fonts-ref }}') + self.assertNotIn('CRAFT_FONTS_DIR', runner['env']) + self.assertNotIn('CRAFT_FONTS_REQUIRED', runner['env']) + required = next(step for step in steps if step.get('name') == 'Require requested craft-fonts') + self.assertLess(steps.index(required), steps.index(runner)) + self.assertTrue(required['if'].startswith("inputs.fonts-ref != ''")) + self.assertEqual(required['env']['CRAFT_FONTS_PATH'], '${{ github.workspace }}/craft-fonts') + with tempfile.TemporaryDirectory(prefix='font setup ') as temp: + output = Path(temp) / 'github env' + font_dir = Path(temp) / 'craft fonts' + font_dir.mkdir() + result = subprocess.run( + ['bash', '-e', '-o', 'pipefail', '-c', required['run']], + env={**os.environ, 'CRAFT_FONTS_PATH': str(font_dir), 'GITHUB_ENV': str(output)}, + capture_output=True, text=True) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(output.read_text(), + f'CRAFT_FONTS_DIR={font_dir}\nCRAFT_FONTS_REQUIRED=1\n') + + def test_pdf_fixture_setup_runs_before_both_test_runners(self): + workflow = yaml.load( + (ROOT / '.github/workflows/sonarcloud.yml').read_text(), Loader=yaml.BaseLoader) + self.assertEqual(workflow['on']['workflow_call']['inputs']['pdf-test-fonts']['default'], 'false') + for job, runner_name in [('build', 'Build instrumented test archive'), + ('coverage', "Run this shard's tests"), + ('sonarcloud', 'Generate Rust coverage')]: + steps = workflow['jobs'][job]['steps'] + setup = next(step for step in steps if step.get('name') == 'Provision pinned PDF test fonts') + runner = next(step for step in steps if step.get('name') == runner_name) + self.assertLess(steps.index(setup), steps.index(runner)) + self.assertIn('inputs.pdf-test-fonts', setup['if']) + if job == 'sonarcloud': + self.assertEqual(setup['if'], 'inputs.pdf-test-fonts && ' + runner['if']) + # Execute the actual shell with a stub caller fetcher, including a path with spaces. + for fail in (False, True): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + (root / 'scripts').mkdir() + (root / 'scripts/fetch-pdf-test-fonts.py').write_text( + 'import pathlib, sys\n' + + ('sys.exit(7)\n' if fail else + 'p = pathlib.Path(sys.argv[1]); p.mkdir(parents=True)\n' + '(p / "Inter-Regular.ttf").write_text("fixture")\n')) + env_file = root / 'env' + runner_temp = root / 'runner temp' + result = subprocess.run( + ['bash', '-e', '-o', 'pipefail', '-c', setup['run']], cwd=root, + env={**os.environ, 'RUNNER_TEMP': str(runner_temp), 'GITHUB_ENV': str(env_file)}, + capture_output=True, text=True) + if fail: + self.assertNotEqual(result.returncode, 0) + self.assertFalse(env_file.exists()) + else: + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue((runner_temp / 'craft-pdf-test-fonts/Inter-Regular.ttf').is_file()) + self.assertEqual(env_file.read_text(), + f'CRAFT_PDF_TEST_FONT_DIR={runner_temp}/craft-pdf-test-fonts\n') + def test_sharding_defaults_to_four_and_keeps_the_unsharded_path(self): workflow = yaml.load( (ROOT / '.github/workflows/sonarcloud.yml').read_text(), Loader=yaml.BaseLoader)