apt: wait for cloud-init before rewriting the sources (#204)

During the ubuntu.com outage, cloud-init's apt_configure module (once per
instance, at boot) spent two 10-second DNS timeouts looking for a mirror
and then wrote ubuntu.sources from its template, after the job had
started. On those VMs configure-apt-mirrors had already pointed the
sources at its list: apt-get update fetched every index for that list,
cloud-init put the image's mirror list back, and apt-get install found
no indexes for it, so no Ubuntu package had a candidate (materialcraft's
Flatpak x86_64 job, the FreeBSD tests job on 26.04). A repro on 12
runners failed on 8, every round, and a file watcher caught cloud-init's
write landing mid-update.

wait-for-cloud-init.sh runs `cloud-init status --wait` (at most 300 s,
never failing the job) before configure-apt-mirrors and
install-riscv64-cross's select-mirrors.sh touch anything.

actions/configure-apt-mirrors/configure-apt-mirrors.sh

@@ -21,6 +21,9 @@
 # (_https._tcp.<host>) are turned off: Harden-Runner blocks those names, and apt does not need them.
 set -euo pipefail
 
+# cloud-init may still be about to write ubuntu.sources; changing it before then is undone.
+"$(dirname "${BASH_SOURCE[0]}")/wait-for-cloud-init.sh"
+
 # CRAFT_APT_ROOT and CRAFT_APT_ARCH stand in for / and the dpkg architecture in tests.
 root=${CRAFT_APT_ROOT:-}
 apt_dir="$root/etc/apt"

actions/configure-apt-mirrors/wait-for-cloud-init.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+# Waits until cloud-init has finished, so it cannot rewrite the apt sources after we have.
+#
+# cloud-init's apt_configure module writes /etc/apt/sources.list.d/ubuntu.sources from its
+# template once per instance, after looking for a mirror. That usually ends before a job starts.
+# On 11 Nov 2026, with no ubuntu.com name resolving, its mirror search sat through two 10-second
+# DNS timeouts and the write landed in the middle of jobs: apt-get update read the sources
+# configure-apt-mirrors had written, cloud-init then put the image's sources back, and
+# apt-get install found no indexes for them, so no Ubuntu package had an installation candidate.
+#
+# CRAFT_CLOUD_INIT names the cloud-init command (tests use a fake). Without cloud-init, or once
+# it has finished, this returns at once. It never fails the job: after CRAFT_CLOUD_INIT_TIMEOUT
+# seconds (default 300) it warns and lets the caller go on.
+set -uo pipefail
+
+cloud_init=${CRAFT_CLOUD_INIT:-cloud-init}
+limit=${CRAFT_CLOUD_INIT_TIMEOUT:-300}
+if ! command -v "$cloud_init" > /dev/null 2>&1; then
+  exit 0
+fi
+start=$SECONDS
+timeout "$limit" "$cloud_init" status --wait > /dev/null 2>&1
+status=$?
+waited=$((SECONDS - start))
+if [ "$status" -eq 124 ]; then
+  echo "::warning::cloud-init was still running after ${limit}s; it may rewrite the apt sources"
+elif [ "$waited" -gt 0 ]; then
+  echo "Waited ${waited}s for cloud-init to finish before changing the apt sources."
+fi
+exit 0

actions/install-riscv64-cross/select-mirrors.sh

@@ -20,6 +20,9 @@
 # installs from, and the image's mirrors would only add a second snapshot to mix in.
 set -euo pipefail
 
+# cloud-init may still be about to write ubuntu.sources (and re-enable it); wait for it first.
+"$(dirname "${BASH_SOURCE[0]}")/../configure-apt-mirrors/wait-for-cloud-init.sh"
+
 # CRAFT_APT_ROOT stands in for / in tests.
 root=${CRAFT_APT_ROOT:-}
 apt_dir="$root/etc/apt"

tests/test_install_apt_packages.py

@@ -214,6 +214,14 @@ for down in ${DOWN:-}; do
 done
 """
 
+# cloud-init that, like the real one on 11 Nov 2026, writes the image's ubuntu.sources (from
+# IMAGE_SOURCES) just before `status --wait` returns.
+FAKE_CLOUD_INIT = """#!/bin/bash
+[ "$*" = "status --wait" ] || exit 2
+sleep "${CLOUD_INIT_DELAY:-0}"
+cp "$IMAGE_SOURCES" "$CRAFT_APT_ROOT/etc/apt/sources.list.d/ubuntu.sources"
+"""
+
 UBUNTU_HOSTS = 'azure.archive.ubuntu.com archive.ubuntu.com security.ubuntu.com ports.ubuntu.com'
 
 NOBLE_SOURCES = """Types: deb
@@ -255,7 +263,7 @@ RISCV_LIST = 'deb [arch=amd64] https://archive.ubuntu.com/ubuntu noble main univ
 class ConfigureAptMirrorsTests(unittest.TestCase):
     """configure-apt-mirrors.sh against a fake /etc and a fake curl."""
 
-    def configure(self, files, arch='amd64', codename='noble', down='', runs=1):
+    def configure(self, files, arch='amd64', codename='noble', down='', runs=1, cloud_init=False):
         with tempfile.TemporaryDirectory() as tmpdir:
             root = Path(tmpdir) / 'root'
             files = {'etc/os-release': f'NAME="Ubuntu"\nVERSION_CODENAME={codename}\n', **files}
@@ -268,12 +276,17 @@ class ConfigureAptMirrorsTests(unittest.TestCase):
             (bin_dir / 'curl').chmod(0o755)
             curl_log = Path(tmpdir) / 'curl.log'
             curl_log.touch()
+            (bin_dir / 'cloud-init').write_text(FAKE_CLOUD_INIT, newline='\n')
+            (bin_dir / 'cloud-init').chmod(0o755)
+            image_sources = Path(tmpdir) / 'image.sources'
+            image_sources.write_text(NOBLE_SOURCES, newline='\n')
             for _ in range(runs):
                 res = subprocess.run(
                     ['bash', str(MIRRORS)],
                     env={**os.environ, 'PATH': f"{bin_dir}{os.pathsep}{os.environ['PATH']}",
                          'CRAFT_APT_ROOT': str(root), 'CRAFT_APT_ARCH': arch, 'DOWN': down,
-                         'CURL_LOG': str(curl_log)},
+                         'CURL_LOG': str(curl_log), 'IMAGE_SOURCES': str(image_sources),
+                         'CRAFT_CLOUD_INIT': str(bin_dir / 'cloud-init') if cloud_init else 'no-cloud-init-here'},
                     capture_output=True, text=True, timeout=60)
                 self.assertEqual(res.returncode, 0, res.stdout + res.stderr)
             mirror_list = root / 'etc/apt/craft-apt-mirrors.txt'
@@ -312,6 +325,13 @@ class ConfigureAptMirrorsTests(unittest.TestCase):
         self.assertIn('::warning::configure-apt-mirrors: http://azure.archive.ubuntu.com/ubuntu/ did not serve '
                       'noble/InRelease (curl: (6) Could not resolve host: azure.archive.ubuntu.com', res.stdout)
 
+    def test_cloud_init_rewriting_the_sources_is_waited_out(self):
+        res, out = self.configure({'etc/apt/apt-mirrors.txt': IMAGE_MIRRORS,
+                                   'etc/apt/sources.list.d/ubuntu.sources': NOBLE_SOURCES},
+                                  cloud_init=True)
+        self.assertEqual(out['etc/apt/sources.list.d/ubuntu.sources'], NOBLE_SOURCES.replace(
+            'mirror+file:/etc/apt/apt-mirrors.txt', 'mirror+file:/etc/apt/craft-apt-mirrors.txt'))
+
     def test_deb822_sources_read_the_new_list_and_keep_their_other_fields(self):
         _, out = self.configure({'etc/apt/apt-mirrors.txt': IMAGE_MIRRORS,
                                  'etc/apt/sources.list.d/ubuntu.sources': NOBLE_SOURCES})
@@ -392,5 +412,39 @@ class AllowedMirrorTests(unittest.TestCase):
         self.assertGreaterEqual(checked, 8)
 
 
+WAIT = ROOT / 'actions/configure-apt-mirrors/wait-for-cloud-init.sh'
+
+
+class WaitForCloudInitTests(unittest.TestCase):
+    def wait(self, body, limit='300'):
+        with tempfile.TemporaryDirectory() as tmpdir:
+            fake = Path(tmpdir) / 'cloud-init'
+            fake.write_text('#!/bin/bash\n' + body, newline='\n')
+            fake.chmod(0o755)
+            return subprocess.run(['bash', str(WAIT)], capture_output=True, text=True, timeout=60,
+                                  env={**os.environ, 'CRAFT_CLOUD_INIT': str(fake),
+                                       'CRAFT_CLOUD_INIT_TIMEOUT': limit})
+
+    def test_waits_until_cloud_init_is_done(self):
+        res = self.wait('[ "$*" = "status --wait" ] && sleep 2\n')
+        self.assertEqual(res.returncode, 0)
+        self.assertRegex(res.stdout, r'Waited [23]s for cloud-init')
+
+    def test_a_cloud_init_that_never_finishes_only_warns(self):
+        res = self.wait('sleep 30\n', limit='1')
+        self.assertEqual(res.returncode, 0)
+        self.assertIn('::warning::cloud-init was still running after 1s', res.stdout)
+
+    def test_a_cloud_init_that_reports_an_error_does_not_fail_the_job(self):
+        res = self.wait('exit 1\n')
+        self.assertEqual(res.returncode, 0)
+        self.assertEqual(res.stdout, '')
+
+    def test_without_cloud_init_it_returns_at_once(self):
+        res = subprocess.run(['bash', str(WAIT)], capture_output=True, text=True, timeout=10,
+                             env={**os.environ, 'CRAFT_CLOUD_INIT': 'no-cloud-init-here'})
+        self.assertEqual((res.returncode, res.stdout), (0, ''))
+
+
 if __name__ == '__main__':
     unittest.main()

tests/test_install_riscv64_cross.py

@@ -35,6 +35,14 @@ done
 printf 'Origin: Ubuntu\\nDate: %s\\n %s main/binary-riscv64/Packages.xz\\n' "$date" "$RANDOM" > "$out"
 """
 
+# cloud-init that, like the real one on 11 Nov 2026, writes the image's ubuntu.sources (from
+# IMAGE_SOURCES) just before `status --wait` returns.
+FAKE_CLOUD_INIT = """#!/bin/bash
+[ "$*" = "status --wait" ] || exit 2
+sleep "${CLOUD_INIT_DELAY:-0}"
+cp "$IMAGE_SOURCES" "$CRAFT_APT_ROOT/etc/apt/sources.list.d/ubuntu.sources"
+"""
+
 UBUNTU_HOSTS = 'archive.ubuntu.com security.ubuntu.com ports.ubuntu.com azure.archive.ubuntu.com'
 
 NOBLE_SOURCES = """Types: deb
@@ -73,7 +81,7 @@ KERNEL = lists('https://mirrors.edge.kernel.org/ubuntu', 'https://mirrors.edge.k
 
 
 class SelectMirrorsTests(unittest.TestCase):
-    def select(self, files=None, down='', stale='', runs=1):
+    def select(self, files=None, down='', stale='', runs=1, cloud_init=False):
         files = {'etc/apt/sources.list.d/ubuntu.sources': NOBLE_SOURCES} if files is None else files
         with tempfile.TemporaryDirectory() as tmpdir:
             root = Path(tmpdir) / 'root'
@@ -85,11 +93,17 @@ class SelectMirrorsTests(unittest.TestCase):
             bin_dir.mkdir()
             (bin_dir / 'curl').write_text(FAKE_CURL, newline='\n')
             (bin_dir / 'curl').chmod(0o755)
+            (bin_dir / 'cloud-init').write_text(FAKE_CLOUD_INIT, newline='\n')
+            (bin_dir / 'cloud-init').chmod(0o755)
+            image_sources = Path(tmpdir) / 'image.sources'
+            image_sources.write_text(NOBLE_SOURCES, newline='\n')
             for _ in range(runs):
                 res = subprocess.run(
                     ['bash', str(SCRIPT)],
                     env={**os.environ, 'PATH': f"{bin_dir}{os.pathsep}{os.environ['PATH']}",
-                         'CRAFT_APT_ROOT': str(root), 'DOWN': down, 'STALE': stale},
+                         'CRAFT_APT_ROOT': str(root), 'DOWN': down, 'STALE': stale,
+                         'IMAGE_SOURCES': str(image_sources),
+                         'CRAFT_CLOUD_INIT': str(bin_dir / 'cloud-init') if cloud_init else 'no-cloud-init-here'},
                     capture_output=True, text=True, timeout=60)
                 self.assertEqual(res.returncode, 0, res.stdout + res.stderr)
             parts = root / 'etc/apt/sources.list.d'
@@ -132,6 +146,10 @@ class SelectMirrorsTests(unittest.TestCase):
         self.assertEqual(sources.count('Enabled: no'), 2)
         self.assertEqual(sources.replace('Enabled: no\n', ''), NOBLE_SOURCES)
 
+    def test_cloud_init_re_enabling_the_image_sources_is_waited_out(self):
+        _, _, image = self.select(cloud_init=True)
+        self.assertEqual(image['etc/apt/sources.list.d/ubuntu.sources'].count('Enabled: no'), 2)
+
     def test_one_line_image_sources_are_commented_out(self):
         _, _, image = self.select({'etc/apt/sources.list': '# comment\n'
                                    'deb mirror+file:/etc/apt/apt-mirrors.txt noble main\n'