Dev infra: harden Windows bootstrap after two adversarial debug passes

Pass 1 (correctness sweep): - All native invocations (mysql, mysqld init, winget, curl downloads, cargo run, seed curl) now go through Invoke-Native, so a caller capturing the scripts' output can never turn ordinary stderr (warnings, progress) into a terminating NativeCommandError under Stop preference - Provisioning DDL feeds through a temp file + `source` and prints the mysql output when it fails (with the MYSQL_ROOT_PASSWORD hint) - Get-MySqlCount distinguishes "table empty" (0) from "query failed" (-1); bootstrap and doctor no longer misread a failed COUNT as unseeded - diesel_cli that is present but broken (moved libmysql.dll) is detected via an exit-code probe and reinstalled with --force - Shared Add-BuildToolsToSession maps winget install dirs + LIBCLANG_PATH in bootstrap, doctor, and run_backend_dev (cargo run may recompile); the tools check maps PATH before probing so re-runs skip winget entirely - my.ini values quoted (clone paths with spaces); service-start timeouts point at the .devstack log files Pass 2 (live retest matrix, all green on this machine): - Full re-run with output captured in-PowerShell: exit 0, no NativeCommandError - Wiped-state rebuild (mysql-data/my.ini/redis/secrets deleted): re-init, re-provision, 281 migrations replayed, seeds + secrets regenerated, backend booted, demo user recreated via /v1/create_account, doctor 0 fail / 0 warn - Failure paths: seed without backend and doctor without services both die cleanly with actionable messages and exit 1; stop_dev_services frees both ports gracefully
2026-10-31 07:27:51 +00:00 · 2026-07-28 22:37:03 -07:00
1 parent 345cbe766a
commit f23a6df80b
@@ -99,6 +99,9 @@ function Ensure-CoreTools {
 }

 function Ensure-BuildTools {
+  # Tools installed by a previous run land in dirs this shell may not have on
+  # PATH yet - map them in before probing, so we don't re-invoke winget.
+  Add-BuildToolsToSession
  $missing = @($BuildTools | Where-Object { -not (Get-Command $_.Exe -ErrorAction SilentlyContinue) })
  if ($missing.Count -eq 0) {
    Write-Log "All native build tools present (cmake, perl, nasm, clang)."
@@ -116,7 +119,7 @@ function Ensure-BuildTools {

  foreach ($tool in $missing) {
    Write-Log "winget install $($tool.WingetId)..."
-    winget install --id $tool.WingetId -e --accept-source-agreements --accept-package-agreements --disable-interactivity
+    Invoke-Native "winget install --id $($tool.WingetId) -e --accept-source-agreements --accept-package-agreements --disable-interactivity"
    if ($LASTEXITCODE -ne 0) {
      Write-Warn2 "winget exited with $LASTEXITCODE for $($tool.Name); checking whether the tool landed anyway..."
    }
@@ -156,13 +159,14 @@ function Install-PortableMySql {

  if (-not (Test-Path $MySqlIniPath)) {
    # Minimal config; utf8mb4 collation comes from each migration's DDL.
-    # MySQL's ini parser escape-processes backslashes - use forward slashes.
+    # MySQL's ini parser escape-processes backslashes - use forward slashes,
+    # and quote the values in case the clone path contains spaces.
    $baseForward = $MySqlBaseDir -replace '\\', '/'
    $dataForward = $MySqlDataDir -replace '\\', '/'
    $ini = @"
 [mysqld]
-basedir=$baseForward
-datadir=$dataForward
+basedir="$baseForward"
+datadir="$dataForward"
 port=3306
 bind-address=127.0.0.1
 "@
@@ -185,13 +189,15 @@ function Provision-MySqlDatabase {
  }

  $mysql = Get-MySqlTool "mysql"
+  if (-not $mysql) { Die "mysql client not found (portable install missing and none on PATH)." }
  # Portable install: root has no password. External server: honor MYSQL_ROOT_PASSWORD.
  $prev = $env:MYSQL_PWD
  if ($env:MYSQL_ROOT_PASSWORD) { $env:MYSQL_PWD = $env:MYSQL_ROOT_PASSWORD } else { $env:MYSQL_PWD = "" }
  try {
    # Same DDL as _docs/dev_setup_server.md, made idempotent. The extra
    # '127.0.0.1' account covers hosts where reverse-DNS of 127.0.0.1 doesn't
-    # resolve to 'localhost'.
+    # resolve to 'localhost'. Fed through a temp file + `source` so mysql's
+    # stderr can never become a terminating PowerShell error mid-provision.
    $ddl = @"
 CREATE DATABASE IF NOT EXISTS $DevMySqlDb;
 CREATE USER IF NOT EXISTS '$DevMySqlUser'@'localhost' IDENTIFIED BY '$DevMySqlPassword';
@@ -200,8 +206,13 @@ GRANT ALL PRIVILEGES ON $DevMySqlDb.* TO '$DevMySqlUser'@'localhost';
 GRANT ALL PRIVILEGES ON $DevMySqlDb.* TO '$DevMySqlUser'@'127.0.0.1';
 FLUSH PRIVILEGES;
 "@
-    $ddl | & $mysql -u root -h 127.0.0.1
+    $ddlFile = Join-Path $env:TEMP "artcraft_provision_ddl.sql"
+    Set-Content -Path $ddlFile -Value $ddl -Encoding ascii
+    $ddlForward = $ddlFile -replace '\\', '/'
+    $out = Invoke-Native "`"$mysql`" -u root -h 127.0.0.1 -e `"source $ddlForward`""
+    Remove-Item $ddlFile -Force -Confirm:$false -ErrorAction SilentlyContinue
    if ($LASTEXITCODE -ne 0) {
+      if ($out) { Write-Host ($out -join "`n") }
      Die "Provisioning failed. For an external MySQL server, set MYSQL_ROOT_PASSWORD and re-run."
    }
  } finally {
@@ -236,10 +247,22 @@ function Install-PortableRedis {

 function Ensure-DieselCli {
  Add-MySqlClientToSession
+  $needInstall = $true
+  $forceFlag = ""
  if (Get-Command diesel -ErrorAction SilentlyContinue) {
-    Write-Log "Found $(diesel --version)."
-    return
+    # A diesel.exe built against a client lib that has since moved fails at
+    # load time - probe it rather than trusting its presence.
+    $version = Invoke-Native "diesel --version"
+    if ($LASTEXITCODE -eq 0) {
+      Write-Log "Found $version."
+      $needInstall = $false
+    } else {
+      Write-Warn2 "diesel is on PATH but broken (missing libmysql.dll?) - reinstalling."
+      $forceFlag = " --force"
+    }
  }
+  if (-not $needInstall) { return }
+
  # diesel_cli's mysql feature links against libmysqlclient. The portable MySQL
  # ships its own client lib - point mysqlclient-sys at it instead of requiring
  # vcpkg. sqlite is bundled so no system sqlite is needed.
@@ -249,9 +272,9 @@ function Ensure-DieselCli {
  Write-Log "Installing diesel_cli (compiles from source; a few minutes)..."
  $env:MYSQLCLIENT_LIB_DIR = Join-Path $MySqlBaseDir "lib"
  $env:MYSQLCLIENT_VERSION = $MySqlVersion
-  Invoke-Native "cargo install diesel_cli --no-default-features --features mysql,sqlite-bundled"
+  Invoke-Native "cargo install diesel_cli --no-default-features --features mysql,sqlite-bundled$forceFlag"
  if ($LASTEXITCODE -ne 0) { Die "cargo install diesel_cli failed (exit $LASTEXITCODE)." }
-  Write-Log "Installed $(diesel --version)."
+  Write-Log "Installed $(Invoke-Native 'diesel --version')."
 }

 function Run-Migrations {
@@ -276,7 +299,8 @@ function Seed-RolesAndBadges {
  # Same data as _database/sql/seed/bootstrap_inserts_roles_etc.sh, but guarded
  # so re-runs don't hit duplicate-key errors. The 'user' role is mandatory:
  # account creation hardcodes user_role_slug='user'.
-  $roleCount = [int](Invoke-MySqlApp -Sql "SELECT COUNT(*) FROM user_roles" | Select-Object -First 1)
+  $roleCount = Get-MySqlCount "SELECT COUNT(*) FROM user_roles"
+  if ($roleCount -lt 0) { Die "Could not query user_roles - are migrations applied? (re-run bootstrap)" }
  if ($roleCount -gt 0) {
    Write-Log "user_roles already seeded ($roleCount rows)."
  } else {
@@ -285,7 +309,8 @@ function Seed-RolesAndBadges {
    Write-Log "Inserted system roles (user, mod, admin)."
  }

-  $badgeCount = [int](Invoke-MySqlApp -Sql "SELECT COUNT(*) FROM badges" | Select-Object -First 1)
+  $badgeCount = Get-MySqlCount "SELECT COUNT(*) FROM badges"
+  if ($badgeCount -lt 0) { Die "Could not query badges - are migrations applied? (re-run bootstrap)" }
  if ($badgeCount -gt 0) {
    Write-Log "badges already seeded ($badgeCount rows)."
  } else {
@@ -361,13 +386,7 @@ function Build-Backend {
    Write-Log "Skipped (-SkipRustBuild)."
    return
  }
-  # Freshly-installed tools live in dirs this shell may not have on PATH yet;
-  # bindgen (boring2/aws-lc-sys) additionally wants LIBCLANG_PATH.
-  foreach ($tool in $BuildTools) { Add-SessionPath $tool.PathDirs }
-  if (-not $env:LIBCLANG_PATH) {
-    $clang = Get-Command clang -ErrorAction SilentlyContinue
-    if ($clang) { $env:LIBCLANG_PATH = Split-Path $clang.Source }
-  }
+  Add-BuildToolsToSession
  Push-Location $RootDir
  try {
    # SQLX_OFFLINE only affects compile-time query verification (checked-in
@@ -413,8 +432,11 @@ function Setup-Frontend {
 function Download-File([string]$Url, [string]$Destination) {
  # curl.exe ships with Windows 10+ and handles redirects/resume better than
  # Invoke-WebRequest (which is also painfully slow on large files in PS 5.1).
-  curl.exe -fL --retry 3 -o "$Destination.partial" $Url
+  # -sS: no progress meter (it goes to stderr, which pollutes captured logs),
+  # but errors still print.
+  $out = Invoke-Native "curl.exe -fsSL --retry 3 -o `"$Destination.partial`" `"$Url`""
  if ($LASTEXITCODE -ne 0) {
+    if ($out) { Write-Host ($out -join "`n") }
    Remove-Item "$Destination.partial" -Force -Confirm:$false -ErrorAction SilentlyContinue
    Die "Download failed: $Url"
  }
@@ -92,10 +92,12 @@ function Get-RedisTool([string]$Name) {
  return $null
 }

-# Run SQL as the app user against the dev database. Returns stdout lines.
+# Run SQL as the app user against the dev database. Returns output lines
+# (stdout and stderr merged via Invoke-Native, so a caller capturing our
+# streams can never turn a mysql warning into a terminating error).
 # Password goes via MYSQL_PWD so it stays off the command line.
-# -Quiet merges stderr into the discarded output (for probes that are
-# expected to fail, e.g. Test-MySqlApp before provisioning).
+# -Quiet discards output (for probes that are expected to fail, e.g.
+# Test-MySqlApp before provisioning). Check $LASTEXITCODE for success.
 function Invoke-MySqlApp([string]$Sql, [string]$SourceFile, [switch]$Quiet) {
  $mysql = Get-MySqlTool "mysql"
  if (-not $mysql) { Die "mysql client not found (run bootstrap_dev_stack.ps1 first)." }
@@ -104,17 +106,27 @@ function Invoke-MySqlApp([string]$Sql, [string]$SourceFile, [switch]$Quiet) {
  try {
    if ($SourceFile) {
      $forward = $SourceFile -replace '\\', '/'
-      & $mysql -u $DevMySqlUser -h 127.0.0.1 -D $DevMySqlDb -e "source $forward"
-    } elseif ($Quiet) {
-      & $mysql -u $DevMySqlUser -h 127.0.0.1 -D $DevMySqlDb -N -e $Sql 2>&1 | Out-Null
+      $out = Invoke-Native "`"$mysql`" -u $DevMySqlUser -h 127.0.0.1 -D $DevMySqlDb -e `"source $forward`""
    } else {
-      & $mysql -u $DevMySqlUser -h 127.0.0.1 -D $DevMySqlDb -N -e $Sql
+      $out = Invoke-Native "`"$mysql`" -u $DevMySqlUser -h 127.0.0.1 -D $DevMySqlDb -N -e `"$Sql`""
    }
+    if (-not $Quiet) { return $out }
  } finally {
    $env:MYSQL_PWD = $prev
  }
 }

+# Run a query expected to return a single non-negative integer (e.g. COUNT(*)).
+# Returns -1 if the query failed or produced no numeric line, so callers can
+# distinguish "table is empty" (0) from "query failed" (-1).
+function Get-MySqlCount([string]$Sql) {
+  $out = @(Invoke-MySqlApp -Sql $Sql)
+  if ($LASTEXITCODE -ne 0) { return -1 }
+  $line = $out | Where-Object { "$_" -match '^\d+$' } | Select-Object -First 1
+  if ($null -eq $line) { return -1 }
+  return [int]"$line"
+}
+
 function Test-MySqlApp {
  try {
    Invoke-MySqlApp -Sql "SELECT 1" -Quiet
@@ -128,7 +140,7 @@ function Test-Redis {
  $cli = Get-RedisTool "redis-cli"
  if (-not $cli) { return $false }
  try {
-    $pong = & $cli -h 127.0.0.1 ping 2>$null
+    $pong = Invoke-Native "`"$cli`" -h 127.0.0.1 ping"
    return ("$pong" -match "PONG")
  } catch {
    return $false
@@ -188,3 +200,20 @@ function Add-SessionPath([string[]]$Dirs) {
 function Add-MySqlClientToSession {
  Add-SessionPath @((Join-Path $MySqlBaseDir "bin"), (Join-Path $MySqlBaseDir "lib"))
 }
+
+# Native build tools (cmake/perl/nasm/llvm) land in these dirs when installed
+# via winget; a shell opened before the install won't have them on PATH yet.
+# bindgen (used by boring2 / aws-lc-sys) additionally wants LIBCLANG_PATH.
+function Add-BuildToolsToSession {
+  Add-SessionPath @(
+    "$env:ProgramFiles\CMake\bin",
+    "C:\Strawberry\perl\bin",
+    "$env:ProgramFiles\NASM",
+    "$env:LOCALAPPDATA\bin\NASM",
+    "$env:ProgramFiles\LLVM\bin"
+  )
+  if (-not $env:LIBCLANG_PATH) {
+    $clang = Get-Command clang -ErrorAction SilentlyContinue
+    if ($clang) { $env:LIBCLANG_PATH = Split-Path $clang.Source }
+  }
+}
@@ -22,7 +22,7 @@ Write-Host "--- Toolchains ---"
 Add-SessionPath @((Join-Path $env:USERPROFILE ".cargo\bin"))
 Add-MySqlClientToSession
 # Freshly-installed build tools may not be on this session's PATH yet.
-Add-SessionPath @("$env:ProgramFiles\CMake\bin", "C:\Strawberry\perl\bin", "$env:ProgramFiles\NASM", "$env:ProgramFiles\LLVM\bin")
+Add-BuildToolsToSession

 if (Get-Command cargo -ErrorAction SilentlyContinue) {
  Pass "rust: $(rustc --version)"
@@ -31,11 +31,9 @@ if (Get-Command cargo -ErrorAction SilentlyContinue) {
 }

 if (Get-Command diesel -ErrorAction SilentlyContinue) {
-  try {
-    $dv = diesel --version 2>$null
-    if ($LASTEXITCODE -eq 0) { Pass "diesel_cli: $dv" }
-    else { Fail "diesel_cli: present but broken (libmysql.dll missing from PATH? re-run bootstrap)" }
-  } catch { Fail "diesel_cli: present but broken (libmysql.dll missing from PATH?)" }
+  $dv = Invoke-Native "diesel --version"
+  if ($LASTEXITCODE -eq 0) { Pass "diesel_cli: $dv" }
+  else { Fail "diesel_cli: present but broken (libmysql.dll missing from PATH? re-run bootstrap)" }
 } else {
  Fail "diesel_cli: not installed (bootstrap installs it against the portable MySQL client lib)"
 }
@@ -72,8 +70,7 @@ if (Test-TcpPort 3306) {
    Pass "mysql: reachable as '$DevMySqlUser' on database '$DevMySqlDb'"

    $migrationDirs = (Get-ChildItem -Directory (Join-Path $RootDir "_database\sql\migrations")).Count
-    $applied = 0
-    try { $applied = [int](Invoke-MySqlApp -Sql "SELECT COUNT(*) FROM __diesel_schema_migrations" | Select-Object -First 1) } catch {}
+    $applied = Get-MySqlCount "SELECT COUNT(*) FROM __diesel_schema_migrations"
    if (($applied -ge $migrationDirs) -and ($applied -gt 0)) {
      Pass "migrations: $applied applied ($migrationDirs in _database\sql\migrations)"
    } elseif ($applied -gt 0) {
@@ -82,21 +79,18 @@ if (Test-TcpPort 3306) {
      Fail "migrations: none applied - run 'diesel migration run' (or re-run bootstrap)"
    }

-    $roleCount = 0
-    try { $roleCount = [int](Invoke-MySqlApp -Sql "SELECT COUNT(*) FROM user_roles" | Select-Object -First 1) } catch {}
+    $roleCount = Get-MySqlCount "SELECT COUNT(*) FROM user_roles"
    if ($roleCount -ge 3) {
      Pass "seed: user_roles has $roleCount rows (user/mod/admin present)"
    } else {
      Fail "seed: user_roles has $roleCount rows - account creation needs the 'user' role (re-run bootstrap)"
    }

-    $badgeCount = 0
-    try { $badgeCount = [int](Invoke-MySqlApp -Sql "SELECT COUNT(*) FROM badges" | Select-Object -First 1) } catch {}
+    $badgeCount = Get-MySqlCount "SELECT COUNT(*) FROM badges"
    if ($badgeCount -gt 0) { Pass "seed: badges has $badgeCount rows" }
    else { WarnCheck "seed: badges table is empty (cosmetic; re-run bootstrap to fill)" }

-    $demoCount = 0
-    try { $demoCount = [int](Invoke-MySqlApp -Sql "SELECT COUNT(*) FROM users WHERE username='$DemoUsername'" | Select-Object -First 1) } catch {}
+    $demoCount = Get-MySqlCount "SELECT COUNT(*) FROM users WHERE username='$DemoUsername'"
    if ($demoCount -gt 0) { Pass "demo user: '$DemoUsername' exists" }
    else { WarnCheck "demo user: '$DemoUsername' not created yet (seed_demo_user.ps1, backend must be running)" }
  } else {
@@ -16,6 +16,8 @@ if (-not (Test-MySqlApp)) { Die "MySQL is up but '$DevMySqlUser' cannot reach '$
 if (-not (Test-Path $SecretsEnvFile)) { Die "Missing $SecretsEnvFile. Run bootstrap_dev_stack.ps1." }

 Add-SessionPath @((Join-Path $env:USERPROFILE ".cargo\bin"))
+# cargo run may need to (re)compile, which needs the native build toolchain.
+Add-BuildToolsToSession
 Push-Location $RootDir
 try {
  # SERVER_ENVIRONMENT defaults to Development when unset; dev config files are
@@ -23,17 +23,20 @@ Write-Log "Backend is up."

 # curl.exe + ConvertFrom-Json: Invoke-RestMethod in PS 5.1 throws on non-2xx
 # and hides the response body, which we need for the already-exists case.
+# Invoke-Native keeps curl's stderr from becoming a terminating PS error.
 function Invoke-JsonPost([string]$Url, [hashtable]$Body) {
  $json = $Body | ConvertTo-Json -Compress
  $tmp = Join-Path $env:TEMP "bootstrap_post_body.json"
  Set-Content -Path $tmp -Value $json -Encoding ascii
-  $response = curl.exe -sS -X POST $Url -H "Content-Type: application/json" --data "@$tmp"
+  $response = Invoke-Native "curl.exe -sS -X POST `"$Url`" -H `"Content-Type: application/json`" --data `"@$tmp`""
+  $curlExit = $LASTEXITCODE
  Remove-Item $tmp -Force -Confirm:$false -ErrorAction SilentlyContinue
-  if ($LASTEXITCODE -ne 0) { Die "Request to $Url failed (curl exit $LASTEXITCODE)." }
+  $joined = ($response -join "`n")
+  if ($curlExit -ne 0) { Die "Request to $Url failed (curl exit $curlExit): $joined" }
  try {
-    return $response | ConvertFrom-Json
+    return $joined | ConvertFrom-Json
  } catch {
-    Die "Non-JSON response from ${Url}: $response"
+    Die "Non-JSON response from ${Url}: $joined"
  }
 }

@@ -20,7 +20,7 @@ function Start-DevMySql {
  }
  Write-Log "Starting mysqld (portable, data in .devstack\mysql-data)..."
  Start-DevProcess -Name "mysqld" -Exe $mysqld -Arguments @("--defaults-file=`"$MySqlIniPath`"") | Out-Null
-  Wait-ForCondition { Test-TcpPort 3306 } 60 "MySQL to listen on 3306" | Out-Null
+  Wait-ForCondition { Test-TcpPort 3306 } 60 "MySQL to listen on 3306 (see $LogsDir\mysqld.err.log and the .err file in $MySqlDataDir)" | Out-Null
  Write-Log "MySQL is up."
 }

@@ -42,7 +42,7 @@ function Start-DevRedis {
    "--save", "`"`"",
    "--dir", "`"$RedisDir`""
  ) | Out-Null
-  Wait-ForCondition { Test-TcpPort 6379 } 30 "Redis to listen on 6379" | Out-Null
+  Wait-ForCondition { Test-TcpPort 6379 } 30 "Redis to listen on 6379 (see $LogsDir\redis.err.log)" | Out-Null
  Write-Log "Redis is up."
 }