sonarcloud: merge each shard's profiles before the scan downloads them (#210)

Each coverage shard now merges its own raw profiles into one sparse
.profdata with the archive toolchain's llvm-profdata and uploads only
that. The scan job merges the shard profiles instead of downloading
every raw profile beside the test archive, which on 11 Nov 2026 filled
the runner for vectorcraft (four ~7 GB zipped profraw artifacts).

A raw profile llvm-profdata cannot read, such as one cut short by a
test process that crashed while writing it (lightcraft, 11 Nov 2026),
now fails its shard with an error naming the file, instead of being
uploaded and failing the scan's merge.

Single-shard callers keep the cargo llvm-cov path unchanged.

.github/workflows/sonarcloud.yml

@@ -398,11 +398,44 @@ jobs:
             echo "::error::shard $SHARD produced no coverage profiles"
             exit 1
           fi
+      - name: Install LLVM tools for the archive's toolchain
+        run: rustup toolchain install "$(cat archive/rustc-release.txt)" --profile minimal --component llvm-tools-preview
+      - name: Merge this shard's profiles
+        # Raw profiles of a large workspace run to tens of gigabytes per shard; the scan job cannot
+        # hold every shard's at once. One sparse indexed profile per shard is a small fraction of that.
+        shell: bash
+        env:
+          SHARD: ${{ matrix.shard }}
+        run: |
+          set -euo pipefail
+          toolchain="$(cat archive/rustc-release.txt)"
+          llvm_bin="$(rustup run "$toolchain" rustc --print sysroot)/lib/rustlib/$(rustup run "$toolchain" rustc -vV | sed -n 's/^host: //p')/bin"
+          echo "Raw profiles: $(du -sh profraw | cut -f1) in $(find profraw -name '*.profraw' | wc -l) files"
+          df -h .
+          mkdir -p profdata
+          # failure-mode=any stops on any unreadable input. A profile cut short (a test process that
+          # crashed while writing it, or a full disk) fails this shard and names the file, instead of
+          # being uploaded and failing the scan's merge later with no shard to look at.
+          status=0
+          "$llvm_bin/llvm-profdata" merge -sparse --failure-mode=any profraw/*.profraw \
+            -o "profdata/shard-$SHARD.profdata" 2>merge-errors.txt || status=$?
+          if [ "$status" -ne 0 ]; then
+            cat merge-errors.txt >&2
+            sed -n 's/^warning: \(.*\.profraw\): \(.*\)$/\1: \2/p' merge-errors.txt | while IFS= read -r bad; do
+              echo "::error::shard $SHARD wrote a truncated or corrupt coverage profile: $bad"
+            done
+            echo "::error::llvm-profdata could not merge shard $SHARD's profiles (exit $status)"
+            rm -rf profdata
+            exit "$status"
+          fi
+          rm -rf profraw merge-errors.txt
+          echo "Merged profile: $(du -sh "profdata/shard-$SHARD.profdata" | cut -f1)"
+          df -h .
       - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
         with:
-          name: profraw-${{ matrix.shard }}
-          path: profraw/
-          if-no-files-found: ignore
+          name: profdata-${{ matrix.shard }}
+          path: profdata/
+          if-no-files-found: error
           retention-days: 1
 
   sonarcloud:
@@ -577,8 +610,9 @@ jobs:
         if: steps.token.outputs.available == 'true' && inputs.shards > 1 && hashFiles('Cargo.toml') != ''
         uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
         with:
-          pattern: profraw-*
-          path: profraw
+          # One merged profile per shard (shard-N.profdata), never the raw profiles.
+          pattern: profdata-*
+          path: profdata
           merge-multiple: true
       - name: Download test archive
         if: steps.token.outputs.available == 'true' && inputs.shards > 1 && hashFiles('Cargo.toml') != ''
@@ -597,10 +631,10 @@ jobs:
           toolchain="$(cat archive/rustc-release.txt)"
           llvm_bin="$(rustup run "$toolchain" rustc --print sysroot)/lib/rustlib/$(rustup run "$toolchain" rustc -vV | sed -n 's/^host: //p')/bin"
           df -h . "$RUNNER_TEMP"
-          # Do not hold every raw profile beside the expanded archive. Retain the raw inputs
+          # Each shard merged its own raw profiles; combine those. Retain the shard profiles
           # until the merge succeeds, then release them before unpacking the test binaries.
-          "$llvm_bin/llvm-profdata" merge -sparse profraw/*.profraw -o coverage.profdata
-          rm -f profraw/*.profraw
+          "$llvm_bin/llvm-profdata" merge -sparse profdata/*.profdata -o coverage.profdata
+          rm -f profdata/*.profdata
           df -h . "$RUNNER_TEMP"
           # Unpack outside the checkout: the scanner analyses everything under the workspace, and build
           # output (generated sources) there would be counted as duplicated code.

README.md

@@ -886,13 +886,16 @@ This step requires the Sonar token, so token-independent CI should also run the
 
 Workspaces run their tests as four parallel `cargo nextest` partitions by
 default. One job builds the instrumented test archive; each partition runs from
-that archive and uploads its profile data, and the scan job merges the profiles
-with the rustup `llvm-profdata` and `llvm-cov` tools. Pass `shards: 1` to keep
-the single `cargo llvm-cov` job. Sharded runs skip doctests, because nextest
-does not run them.
+that archive, merges its own raw profiles into one sparse `.profdata` with the
+rustup `llvm-profdata`, and uploads only that; the scan job merges the shard
+profiles and exports them with `llvm-cov`. A raw profile that `llvm-profdata`
+cannot read (for example one cut short by a test process that crashed while
+writing it) fails its shard and is named in an error annotation, rather than
+reaching the scan. Pass `shards: 1` to keep the single `cargo llvm-cov` job.
+Sharded runs skip doctests, because nextest does not run them.
 
 The scan job frees unused runner SDKs before either Rust coverage path. Sharded
-coverage merges and removes raw profiles before extracting the test archive, then
+coverage merges and removes the shard profiles before extracting the test archive, then
 releases the compressed archive and temporary binaries as they are consumed.
 Disk-space snapshots in the logs show headroom before cleanup and aggregation.
 

tests/test_sonarcloud_disk.py

@@ -74,10 +74,10 @@ class SonarCloudDiskTests(unittest.TestCase):
             runner_temp.mkdir()
             sentinel = runner_temp / 'keep-unrelated.txt'
             sentinel.write_text('keep')
-            profiles = workspace / 'profraw'
+            profiles = workspace / 'profdata'
             profiles.mkdir()
-            for name in ('one', 'two'):
-                (profiles / f'{name}.profraw').write_text(name)
+            for shard in ('1', '2'):
+                (profiles / f'shard-{shard}.profdata').write_text(shard)
             archive = workspace / 'archive'
             archive.mkdir()
             (archive / 'rustc-release.txt').write_text('1.99.1\n')
@@ -109,13 +109,14 @@ with open(os.environ['CALLS'], 'a') as calls:
 if name == 'rustup':
     print(os.environ['SYSROOT'] if '--print' in args else 'host: test-host')
 elif name == 'llvm-profdata':
-    assert list(pathlib.Path('profraw').glob('*.profraw'))
+    assert args[:2] == ['merge', '-sparse'], args
+    assert sorted(args[2:4]) == ['profdata/shard-1.profdata', 'profdata/shard-2.profdata'], args
     if failure == 'merge':
         sys.exit(7)
     pathlib.Path(args[args.index('-o') + 1]).write_text('merged profile')
 elif name == 'tar':
     assert pathlib.Path('coverage.profdata').is_file()
-    assert not list(pathlib.Path('profraw').glob('*.profraw'))
+    assert not list(pathlib.Path('profdata').glob('*.profdata'))
     if failure == 'extract':
         (pathlib.Path(args[args.index('-C') + 1]) / 'partial').write_text('partial extract')
         sys.exit(8)
@@ -142,7 +143,7 @@ elif name == 'llvm-cov':
             return {
                 'result': result,
                 'calls': calls.read_text().splitlines(),
-                'raw_profiles': sorted(path.name for path in profiles.glob('*.profraw')),
+                'shard_profiles': sorted(path.name for path in profiles.glob('*.profdata')),
                 'archive_exists': archive_file.exists(),
                 'merged_exists': (workspace / 'coverage.profdata').exists(),
                 'extracted': list(runner_temp.glob('coverage-archive*')),
@@ -155,7 +156,7 @@ elif name == 'llvm-cov':
         observed = self.run_combine()
         self.assertEqual(observed['result'].returncode, 0, observed['result'].stderr)
         self.assertLess(observed['calls'].index('llvm-profdata'), observed['calls'].index('tar'))
-        self.assertEqual(observed['raw_profiles'], [])
+        self.assertEqual(observed['shard_profiles'], [])
         self.assertFalse(observed['archive_exists'])
         self.assertFalse(observed['merged_exists'])
         self.assertEqual(observed['extracted'], [])
@@ -165,7 +166,7 @@ elif name == 'llvm-cov':
     def test_failed_profile_merge_preserves_inputs_and_never_extracts(self):
         observed = self.run_combine('merge')
         self.assertEqual(observed['result'].returncode, 7, observed['result'].stderr)
-        self.assertEqual(observed['raw_profiles'], ['one.profraw', 'two.profraw'])
+        self.assertEqual(observed['shard_profiles'], ['shard-1.profdata', 'shard-2.profdata'])
         self.assertTrue(observed['archive_exists'])
         self.assertNotIn('tar', observed['calls'])
         self.assertEqual(observed['extracted'], [])

tests/test_sonarcloud_shard_merge.py

@@ -0,0 +1,189 @@
+"""Exercise the per-shard profile merge in sharded Sonar coverage."""
+
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import tempfile
+import unittest
+
+import yaml
+
+
+ROOT = Path(__file__).resolve().parents[1]
+CORRUPT = 'invalid instrumentation profile data (file header is corrupt)'
+
+
+def llvm_release():
+    """Return a clang and llvm-profdata from one LLVM release, so the raw profile format matches."""
+    for bin_dir in sorted(Path('/usr/lib').glob('llvm-*/bin'), reverse=True):
+        clang, profdata = bin_dir / 'clang', bin_dir / 'llvm-profdata'
+        if clang.is_file() and profdata.is_file():
+            return clang, profdata
+    return None
+
+
+def load_workflow():
+    return yaml.load((ROOT / '.github/workflows/sonarcloud.yml').read_text(),
+                     Loader=yaml.BaseLoader)
+
+
+@unittest.skipUnless(shutil.which('bash'), 'requires bash')
+class ShardMergeTests(unittest.TestCase):
+    def setUp(self):
+        workflow = load_workflow()
+        self.shard_steps = workflow['jobs']['coverage']['steps']
+        self.scan_steps = workflow['jobs']['sonarcloud']['steps']
+        self.merge = next(step for step in self.shard_steps
+                          if step.get('name') == "Merge this shard's profiles")
+
+    def test_shards_upload_one_merged_profile_and_the_scan_merges_those(self):
+        names = [step.get('name') or step.get('uses', '') for step in self.shard_steps]
+        tests = names.index("Run this shard's tests")
+        install = names.index("Install LLVM tools for the archive's toolchain")
+        merge = names.index("Merge this shard's profiles")
+        upload = next(index for index, step in enumerate(self.shard_steps)
+                      if step.get('uses', '').startswith('actions/upload-artifact@'))
+        self.assertLess(tests, install)
+        self.assertLess(install, merge)
+        self.assertLess(merge, upload)
+        self.assertIn('archive/rustc-release.txt', self.shard_steps[install]['run'])
+        self.assertIn('llvm-tools-preview', self.shard_steps[install]['run'])
+        self.assertEqual(self.merge['env']['SHARD'], '${{ matrix.shard }}')
+        uploaded = self.shard_steps[upload]['with']
+        self.assertEqual(uploaded['name'], 'profdata-${{ matrix.shard }}')
+        self.assertEqual(uploaded['path'], 'profdata/')
+        self.assertEqual(uploaded['if-no-files-found'], 'error')
+        # No job uploads or downloads raw profiles any more.
+        for job in load_workflow()['jobs'].values():
+            for step in job['steps']:
+                with self.subTest(step=step.get('name') or step.get('uses')):
+                    self.assertNotIn('profraw', step.get('with', {}).get('name', ''))
+                    self.assertNotIn('profraw', step.get('with', {}).get('pattern', ''))
+                    self.assertNotIn('profraw', step.get('with', {}).get('path', ''))
+        download = next(step for step in self.scan_steps
+                        if step.get('name') == 'Download shard profiles and archive')
+        self.assertEqual(download['with']['pattern'], 'profdata-*')
+        self.assertEqual(download['with']['path'], 'profdata')
+        combine = next(step for step in self.scan_steps
+                       if step.get('name') == 'Combine shard coverage')
+        self.assertIn('merge -sparse profdata/*.profdata -o coverage.profdata', combine['run'])
+
+    def run_merge(self, profiles, profdata_tool):
+        """Run the merge step with a fake rustup whose sysroot holds the given llvm-profdata."""
+        temp = tempfile.TemporaryDirectory(prefix='sonar shard ')
+        self.addCleanup(temp.cleanup)
+        root = Path(temp.name)
+        workspace = root / 'workspace'
+        (workspace / 'archive').mkdir(parents=True)
+        (workspace / 'archive/rustc-release.txt').write_text('1.99.1\n')
+        raw = workspace / 'profraw'
+        raw.mkdir()
+        for name, data in profiles.items():
+            (raw / name).write_bytes(data)
+        sysroot = root / 'toolchain'
+        llvm = sysroot / 'lib/rustlib/test-host/bin'
+        llvm.mkdir(parents=True)
+        (llvm / 'llvm-profdata').symlink_to(profdata_tool)
+        tools = root / 'tools'
+        tools.mkdir()
+        rustup = tools / 'rustup'
+        rustup.write_text('#!/bin/sh\n'
+                          'case "$*" in *--print*) echo "$SYSROOT" ;; *) echo "host: test-host" ;; esac\n')
+        rustup.chmod(0o755)
+        result = subprocess.run(
+            ['bash', '--noprofile', '--norc', '-e', '-o', 'pipefail', '-c', self.merge['run']],
+            cwd=workspace, capture_output=True, text=True,
+            env={**os.environ, 'PATH': f'{tools}{os.pathsep}{os.environ["PATH"]}',
+                 'SYSROOT': str(sysroot), 'SHARD': '3', 'CALLS': str(root / 'calls')},
+        )
+        return result, workspace
+
+    def fake_profdata(self, corrupt=''):
+        temp = tempfile.TemporaryDirectory(prefix='fake llvm ')
+        self.addCleanup(temp.cleanup)
+        tool = Path(temp.name) / 'llvm-profdata'
+        tool.write_text(f'''#!/usr/bin/env python3
+import os, pathlib, sys
+args = sys.argv[1:]
+with open(os.environ['CALLS'], 'a') as calls:
+    calls.write(' '.join(args) + '\\n')
+corrupt = {corrupt!r}
+if corrupt:
+    print(f'warning: profraw/{{corrupt}}: {CORRUPT}', file=sys.stderr)
+    print('error: no profile can be merged', file=sys.stderr)
+    sys.exit(1)
+pathlib.Path(args[args.index('-o') + 1]).write_text('merged')
+''')
+        tool.chmod(0o755)
+        return tool
+
+    def test_merge_replaces_raw_profiles_with_one_sparse_profile(self):
+        result, workspace = self.run_merge(
+            {'shard-3-1-a.profraw': b'a', 'shard-3-2-b.profraw': b'b'}, self.fake_profdata())
+        self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+        calls = (workspace.parent / 'calls').read_text().split()
+        self.assertEqual(calls[:3], ['merge', '-sparse', '--failure-mode=any'])
+        self.assertEqual(sorted(calls[3:5]),
+                         ['profraw/shard-3-1-a.profraw', 'profraw/shard-3-2-b.profraw'])
+        self.assertEqual(calls[5:], ['-o', 'profdata/shard-3.profdata'])
+        self.assertEqual([path.name for path in (workspace / 'profdata').iterdir()],
+                         ['shard-3.profdata'])
+        self.assertFalse((workspace / 'profraw').exists())
+        self.assertNotIn('::error::', result.stdout)
+
+    def test_corrupt_raw_profile_fails_the_shard_and_names_the_file(self):
+        result, workspace = self.run_merge(
+            {'shard-3-1-a.profraw': b'a', 'shard-3-2-b.profraw': b'b'},
+            self.fake_profdata(corrupt='shard-3-2-b.profraw'))
+        self.assertEqual(result.returncode, 1)
+        self.assertIn('::error::shard 3 wrote a truncated or corrupt coverage profile: '
+                      f'profraw/shard-3-2-b.profraw: {CORRUPT}', result.stdout)
+        self.assertIn("::error::llvm-profdata could not merge shard 3's profiles (exit 1)",
+                      result.stdout)
+        # Nothing is left for the upload step, and the raw profiles stay for inspection.
+        self.assertFalse((workspace / 'profdata').exists())
+        self.assertEqual(sorted(path.name for path in (workspace / 'profraw').iterdir()),
+                         ['shard-3-1-a.profraw', 'shard-3-2-b.profraw'])
+
+    @unittest.skipUnless(llvm_release(), 'requires clang and llvm-profdata from /usr/lib/llvm-*')
+    def test_real_llvm_profdata_rejects_a_truncated_raw_profile(self):
+        clang, profdata = llvm_release()
+        with tempfile.TemporaryDirectory(prefix='sonar real ') as temp:
+            root = Path(temp)
+            source = root / 'main.c'
+            source.write_text('int twice(int x) { return 2 * x; }\n'
+                              'int main(int argc, char **argv) { (void)argv; return twice(argc) - 2; }\n')
+            binary = root / 'main'
+            built = subprocess.run([str(clang), '-fprofile-instr-generate', '-fcoverage-mapping',
+                                    str(source), '-o', str(binary)], capture_output=True, text=True)
+            if built.returncode != 0:
+                # The profile runtime (libclang_rt.profile) is a separate package on some systems.
+                self.skipTest(f'{clang} cannot build an instrumented binary: {built.stderr}')
+            profiles = {}
+            for run in ('one', 'two'):
+                raw = root / f'{run}.profraw'
+                subprocess.run([str(binary)], check=True,
+                               env={**os.environ, 'LLVM_PROFILE_FILE': str(raw)})
+                profiles[f'shard-3-{run}.profraw'] = raw.read_bytes()
+
+        result, workspace = self.run_merge(profiles, profdata)
+        self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+        merged = workspace / 'profdata/shard-3.profdata'
+        shown = subprocess.run([str(profdata), 'show', '--function=twice', '--counts', str(merged)],
+                               check=True, capture_output=True, text=True)
+        # Two runs of one call each, merged.
+        self.assertIn('Function count: 2', shown.stdout)
+        self.assertFalse((workspace / 'profraw').exists())
+
+        whole = profiles['shard-3-two.profraw']
+        result, workspace = self.run_merge(
+            {**profiles, 'shard-3-two.profraw': whole[:len(whole) // 2]}, profdata)
+        self.assertNotEqual(result.returncode, 0)
+        self.assertIn('::error::shard 3 wrote a truncated or corrupt coverage profile: '
+                      'profraw/shard-3-two.profraw: ', result.stdout)
+        self.assertFalse((workspace / 'profdata').exists())
+
+
+if __name__ == '__main__':
+    unittest.main()