diff --git a/.github/workflows/sonarcloud.yml b/.github/workflows/sonarcloud.yml index 96aa434..9ad1a17 100644 --- a/.github/workflows/sonarcloud.yml +++ b/.github/workflows/sonarcloud.yml @@ -398,11 +398,44 @@ jobs: echo "::error::shard $SHARD produced no coverage profiles" exit 1 fi + - name: Install LLVM tools for the archive's toolchain + run: rustup toolchain install "$(cat archive/rustc-release.txt)" --profile minimal --component llvm-tools-preview + - name: Merge this shard's profiles + # Raw profiles of a large workspace run to tens of gigabytes per shard; the scan job cannot + # hold every shard's at once. One sparse indexed profile per shard is a small fraction of that. + shell: bash + env: + SHARD: ${{ matrix.shard }} + run: | + set -euo pipefail + toolchain="$(cat archive/rustc-release.txt)" + llvm_bin="$(rustup run "$toolchain" rustc --print sysroot)/lib/rustlib/$(rustup run "$toolchain" rustc -vV | sed -n 's/^host: //p')/bin" + echo "Raw profiles: $(du -sh profraw | cut -f1) in $(find profraw -name '*.profraw' | wc -l) files" + df -h . + mkdir -p profdata + # failure-mode=any stops on any unreadable input. A profile cut short (a test process that + # crashed while writing it, or a full disk) fails this shard and names the file, instead of + # being uploaded and failing the scan's merge later with no shard to look at. + status=0 + "$llvm_bin/llvm-profdata" merge -sparse --failure-mode=any profraw/*.profraw \ + -o "profdata/shard-$SHARD.profdata" 2>merge-errors.txt || status=$? + if [ "$status" -ne 0 ]; then + cat merge-errors.txt >&2 + sed -n 's/^warning: \(.*\.profraw\): \(.*\)$/\1: \2/p' merge-errors.txt | while IFS= read -r bad; do + echo "::error::shard $SHARD wrote a truncated or corrupt coverage profile: $bad" + done + echo "::error::llvm-profdata could not merge shard $SHARD's profiles (exit $status)" + rm -rf profdata + exit "$status" + fi + rm -rf profraw merge-errors.txt + echo "Merged profile: $(du -sh "profdata/shard-$SHARD.profdata" | cut -f1)" + df -h . - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: profraw-${{ matrix.shard }} - path: profraw/ - if-no-files-found: ignore + name: profdata-${{ matrix.shard }} + path: profdata/ + if-no-files-found: error retention-days: 1 sonarcloud: @@ -577,8 +610,9 @@ jobs: if: steps.token.outputs.available == 'true' && inputs.shards > 1 && hashFiles('Cargo.toml') != '' uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: - pattern: profraw-* - path: profraw + # One merged profile per shard (shard-N.profdata), never the raw profiles. + pattern: profdata-* + path: profdata merge-multiple: true - name: Download test archive if: steps.token.outputs.available == 'true' && inputs.shards > 1 && hashFiles('Cargo.toml') != '' @@ -597,10 +631,10 @@ jobs: toolchain="$(cat archive/rustc-release.txt)" llvm_bin="$(rustup run "$toolchain" rustc --print sysroot)/lib/rustlib/$(rustup run "$toolchain" rustc -vV | sed -n 's/^host: //p')/bin" df -h . "$RUNNER_TEMP" - # Do not hold every raw profile beside the expanded archive. Retain the raw inputs + # Each shard merged its own raw profiles; combine those. Retain the shard profiles # until the merge succeeds, then release them before unpacking the test binaries. - "$llvm_bin/llvm-profdata" merge -sparse profraw/*.profraw -o coverage.profdata - rm -f profraw/*.profraw + "$llvm_bin/llvm-profdata" merge -sparse profdata/*.profdata -o coverage.profdata + rm -f profdata/*.profdata df -h . "$RUNNER_TEMP" # Unpack outside the checkout: the scanner analyses everything under the workspace, and build # output (generated sources) there would be counted as duplicated code. diff --git a/README.md b/README.md index a89231c..f5f24fc 100644 --- a/README.md +++ b/README.md @@ -886,13 +886,16 @@ This step requires the Sonar token, so token-independent CI should also run the Workspaces run their tests as four parallel `cargo nextest` partitions by default. One job builds the instrumented test archive; each partition runs from -that archive and uploads its profile data, and the scan job merges the profiles -with the rustup `llvm-profdata` and `llvm-cov` tools. Pass `shards: 1` to keep -the single `cargo llvm-cov` job. Sharded runs skip doctests, because nextest -does not run them. +that archive, merges its own raw profiles into one sparse `.profdata` with the +rustup `llvm-profdata`, and uploads only that; the scan job merges the shard +profiles and exports them with `llvm-cov`. A raw profile that `llvm-profdata` +cannot read (for example one cut short by a test process that crashed while +writing it) fails its shard and is named in an error annotation, rather than +reaching the scan. Pass `shards: 1` to keep the single `cargo llvm-cov` job. +Sharded runs skip doctests, because nextest does not run them. The scan job frees unused runner SDKs before either Rust coverage path. Sharded -coverage merges and removes raw profiles before extracting the test archive, then +coverage merges and removes the shard profiles before extracting the test archive, then releases the compressed archive and temporary binaries as they are consumed. Disk-space snapshots in the logs show headroom before cleanup and aggregation. diff --git a/tests/test_sonarcloud_disk.py b/tests/test_sonarcloud_disk.py index 26f3b53..370ddc8 100644 --- a/tests/test_sonarcloud_disk.py +++ b/tests/test_sonarcloud_disk.py @@ -74,10 +74,10 @@ class SonarCloudDiskTests(unittest.TestCase): runner_temp.mkdir() sentinel = runner_temp / 'keep-unrelated.txt' sentinel.write_text('keep') - profiles = workspace / 'profraw' + profiles = workspace / 'profdata' profiles.mkdir() - for name in ('one', 'two'): - (profiles / f'{name}.profraw').write_text(name) + for shard in ('1', '2'): + (profiles / f'shard-{shard}.profdata').write_text(shard) archive = workspace / 'archive' archive.mkdir() (archive / 'rustc-release.txt').write_text('1.99.1\n') @@ -109,13 +109,14 @@ with open(os.environ['CALLS'], 'a') as calls: if name == 'rustup': print(os.environ['SYSROOT'] if '--print' in args else 'host: test-host') elif name == 'llvm-profdata': - assert list(pathlib.Path('profraw').glob('*.profraw')) + assert args[:2] == ['merge', '-sparse'], args + assert sorted(args[2:4]) == ['profdata/shard-1.profdata', 'profdata/shard-2.profdata'], args if failure == 'merge': sys.exit(7) pathlib.Path(args[args.index('-o') + 1]).write_text('merged profile') elif name == 'tar': assert pathlib.Path('coverage.profdata').is_file() - assert not list(pathlib.Path('profraw').glob('*.profraw')) + assert not list(pathlib.Path('profdata').glob('*.profdata')) if failure == 'extract': (pathlib.Path(args[args.index('-C') + 1]) / 'partial').write_text('partial extract') sys.exit(8) @@ -142,7 +143,7 @@ elif name == 'llvm-cov': return { 'result': result, 'calls': calls.read_text().splitlines(), - 'raw_profiles': sorted(path.name for path in profiles.glob('*.profraw')), + 'shard_profiles': sorted(path.name for path in profiles.glob('*.profdata')), 'archive_exists': archive_file.exists(), 'merged_exists': (workspace / 'coverage.profdata').exists(), 'extracted': list(runner_temp.glob('coverage-archive*')), @@ -155,7 +156,7 @@ elif name == 'llvm-cov': observed = self.run_combine() self.assertEqual(observed['result'].returncode, 0, observed['result'].stderr) self.assertLess(observed['calls'].index('llvm-profdata'), observed['calls'].index('tar')) - self.assertEqual(observed['raw_profiles'], []) + self.assertEqual(observed['shard_profiles'], []) self.assertFalse(observed['archive_exists']) self.assertFalse(observed['merged_exists']) self.assertEqual(observed['extracted'], []) @@ -165,7 +166,7 @@ elif name == 'llvm-cov': def test_failed_profile_merge_preserves_inputs_and_never_extracts(self): observed = self.run_combine('merge') self.assertEqual(observed['result'].returncode, 7, observed['result'].stderr) - self.assertEqual(observed['raw_profiles'], ['one.profraw', 'two.profraw']) + self.assertEqual(observed['shard_profiles'], ['shard-1.profdata', 'shard-2.profdata']) self.assertTrue(observed['archive_exists']) self.assertNotIn('tar', observed['calls']) self.assertEqual(observed['extracted'], []) diff --git a/tests/test_sonarcloud_shard_merge.py b/tests/test_sonarcloud_shard_merge.py new file mode 100644 index 0000000..79fa66a --- /dev/null +++ b/tests/test_sonarcloud_shard_merge.py @@ -0,0 +1,189 @@ +"""Exercise the per-shard profile merge in sharded Sonar coverage.""" + +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + +import yaml + + +ROOT = Path(__file__).resolve().parents[1] +CORRUPT = 'invalid instrumentation profile data (file header is corrupt)' + + +def llvm_release(): + """Return a clang and llvm-profdata from one LLVM release, so the raw profile format matches.""" + for bin_dir in sorted(Path('/usr/lib').glob('llvm-*/bin'), reverse=True): + clang, profdata = bin_dir / 'clang', bin_dir / 'llvm-profdata' + if clang.is_file() and profdata.is_file(): + return clang, profdata + return None + + +def load_workflow(): + return yaml.load((ROOT / '.github/workflows/sonarcloud.yml').read_text(), + Loader=yaml.BaseLoader) + + +@unittest.skipUnless(shutil.which('bash'), 'requires bash') +class ShardMergeTests(unittest.TestCase): + def setUp(self): + workflow = load_workflow() + self.shard_steps = workflow['jobs']['coverage']['steps'] + self.scan_steps = workflow['jobs']['sonarcloud']['steps'] + self.merge = next(step for step in self.shard_steps + if step.get('name') == "Merge this shard's profiles") + + def test_shards_upload_one_merged_profile_and_the_scan_merges_those(self): + names = [step.get('name') or step.get('uses', '') for step in self.shard_steps] + tests = names.index("Run this shard's tests") + install = names.index("Install LLVM tools for the archive's toolchain") + merge = names.index("Merge this shard's profiles") + upload = next(index for index, step in enumerate(self.shard_steps) + if step.get('uses', '').startswith('actions/upload-artifact@')) + self.assertLess(tests, install) + self.assertLess(install, merge) + self.assertLess(merge, upload) + self.assertIn('archive/rustc-release.txt', self.shard_steps[install]['run']) + self.assertIn('llvm-tools-preview', self.shard_steps[install]['run']) + self.assertEqual(self.merge['env']['SHARD'], '${{ matrix.shard }}') + uploaded = self.shard_steps[upload]['with'] + self.assertEqual(uploaded['name'], 'profdata-${{ matrix.shard }}') + self.assertEqual(uploaded['path'], 'profdata/') + self.assertEqual(uploaded['if-no-files-found'], 'error') + # No job uploads or downloads raw profiles any more. + for job in load_workflow()['jobs'].values(): + for step in job['steps']: + with self.subTest(step=step.get('name') or step.get('uses')): + self.assertNotIn('profraw', step.get('with', {}).get('name', '')) + self.assertNotIn('profraw', step.get('with', {}).get('pattern', '')) + self.assertNotIn('profraw', step.get('with', {}).get('path', '')) + download = next(step for step in self.scan_steps + if step.get('name') == 'Download shard profiles and archive') + self.assertEqual(download['with']['pattern'], 'profdata-*') + self.assertEqual(download['with']['path'], 'profdata') + combine = next(step for step in self.scan_steps + if step.get('name') == 'Combine shard coverage') + self.assertIn('merge -sparse profdata/*.profdata -o coverage.profdata', combine['run']) + + def run_merge(self, profiles, profdata_tool): + """Run the merge step with a fake rustup whose sysroot holds the given llvm-profdata.""" + temp = tempfile.TemporaryDirectory(prefix='sonar shard ') + self.addCleanup(temp.cleanup) + root = Path(temp.name) + workspace = root / 'workspace' + (workspace / 'archive').mkdir(parents=True) + (workspace / 'archive/rustc-release.txt').write_text('1.99.1\n') + raw = workspace / 'profraw' + raw.mkdir() + for name, data in profiles.items(): + (raw / name).write_bytes(data) + sysroot = root / 'toolchain' + llvm = sysroot / 'lib/rustlib/test-host/bin' + llvm.mkdir(parents=True) + (llvm / 'llvm-profdata').symlink_to(profdata_tool) + tools = root / 'tools' + tools.mkdir() + rustup = tools / 'rustup' + rustup.write_text('#!/bin/sh\n' + 'case "$*" in *--print*) echo "$SYSROOT" ;; *) echo "host: test-host" ;; esac\n') + rustup.chmod(0o755) + result = subprocess.run( + ['bash', '--noprofile', '--norc', '-e', '-o', 'pipefail', '-c', self.merge['run']], + cwd=workspace, capture_output=True, text=True, + env={**os.environ, 'PATH': f'{tools}{os.pathsep}{os.environ["PATH"]}', + 'SYSROOT': str(sysroot), 'SHARD': '3', 'CALLS': str(root / 'calls')}, + ) + return result, workspace + + def fake_profdata(self, corrupt=''): + temp = tempfile.TemporaryDirectory(prefix='fake llvm ') + self.addCleanup(temp.cleanup) + tool = Path(temp.name) / 'llvm-profdata' + tool.write_text(f'''#!/usr/bin/env python3 +import os, pathlib, sys +args = sys.argv[1:] +with open(os.environ['CALLS'], 'a') as calls: + calls.write(' '.join(args) + '\\n') +corrupt = {corrupt!r} +if corrupt: + print(f'warning: profraw/{{corrupt}}: {CORRUPT}', file=sys.stderr) + print('error: no profile can be merged', file=sys.stderr) + sys.exit(1) +pathlib.Path(args[args.index('-o') + 1]).write_text('merged') +''') + tool.chmod(0o755) + return tool + + def test_merge_replaces_raw_profiles_with_one_sparse_profile(self): + result, workspace = self.run_merge( + {'shard-3-1-a.profraw': b'a', 'shard-3-2-b.profraw': b'b'}, self.fake_profdata()) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + calls = (workspace.parent / 'calls').read_text().split() + self.assertEqual(calls[:3], ['merge', '-sparse', '--failure-mode=any']) + self.assertEqual(sorted(calls[3:5]), + ['profraw/shard-3-1-a.profraw', 'profraw/shard-3-2-b.profraw']) + self.assertEqual(calls[5:], ['-o', 'profdata/shard-3.profdata']) + self.assertEqual([path.name for path in (workspace / 'profdata').iterdir()], + ['shard-3.profdata']) + self.assertFalse((workspace / 'profraw').exists()) + self.assertNotIn('::error::', result.stdout) + + def test_corrupt_raw_profile_fails_the_shard_and_names_the_file(self): + result, workspace = self.run_merge( + {'shard-3-1-a.profraw': b'a', 'shard-3-2-b.profraw': b'b'}, + self.fake_profdata(corrupt='shard-3-2-b.profraw')) + self.assertEqual(result.returncode, 1) + self.assertIn('::error::shard 3 wrote a truncated or corrupt coverage profile: ' + f'profraw/shard-3-2-b.profraw: {CORRUPT}', result.stdout) + self.assertIn("::error::llvm-profdata could not merge shard 3's profiles (exit 1)", + result.stdout) + # Nothing is left for the upload step, and the raw profiles stay for inspection. + self.assertFalse((workspace / 'profdata').exists()) + self.assertEqual(sorted(path.name for path in (workspace / 'profraw').iterdir()), + ['shard-3-1-a.profraw', 'shard-3-2-b.profraw']) + + @unittest.skipUnless(llvm_release(), 'requires clang and llvm-profdata from /usr/lib/llvm-*') + def test_real_llvm_profdata_rejects_a_truncated_raw_profile(self): + clang, profdata = llvm_release() + with tempfile.TemporaryDirectory(prefix='sonar real ') as temp: + root = Path(temp) + source = root / 'main.c' + source.write_text('int twice(int x) { return 2 * x; }\n' + 'int main(int argc, char **argv) { (void)argv; return twice(argc) - 2; }\n') + binary = root / 'main' + built = subprocess.run([str(clang), '-fprofile-instr-generate', '-fcoverage-mapping', + str(source), '-o', str(binary)], capture_output=True, text=True) + if built.returncode != 0: + # The profile runtime (libclang_rt.profile) is a separate package on some systems. + self.skipTest(f'{clang} cannot build an instrumented binary: {built.stderr}') + profiles = {} + for run in ('one', 'two'): + raw = root / f'{run}.profraw' + subprocess.run([str(binary)], check=True, + env={**os.environ, 'LLVM_PROFILE_FILE': str(raw)}) + profiles[f'shard-3-{run}.profraw'] = raw.read_bytes() + + result, workspace = self.run_merge(profiles, profdata) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + merged = workspace / 'profdata/shard-3.profdata' + shown = subprocess.run([str(profdata), 'show', '--function=twice', '--counts', str(merged)], + check=True, capture_output=True, text=True) + # Two runs of one call each, merged. + self.assertIn('Function count: 2', shown.stdout) + self.assertFalse((workspace / 'profraw').exists()) + + whole = profiles['shard-3-two.profraw'] + result, workspace = self.run_merge( + {**profiles, 'shard-3-two.profraw': whole[:len(whole) // 2]}, profdata) + self.assertNotEqual(result.returncode, 0) + self.assertIn('::error::shard 3 wrote a truncated or corrupt coverage profile: ' + 'profraw/shard-3-two.profraw: ', result.stdout) + self.assertFalse((workspace / 'profdata').exists()) + + +if __name__ == '__main__': + unittest.main()