Files
photocraft/xtask/src/perf.rs
T
FredApps 32722ffcbf Merge approved PhotoCraft PR #9019: fix(xtask): reject unknown performance benchmark selections
Port behavior and regression coverage into current modules; preserve current upstream behavior.
Validate benchmark names before both measurement and baseline-from-files paths, retaining newer baseline modes.
2026-10-29 04:21:14 +01:00

1601 lines
71 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
//! `cargo xtask perf`: run the release benches, merge their `--json` reports into one result keyed
//! by scenario id, and check it against `perf/budgets.toml` and `perf/baseline.json` (#221).
//!
//! ```text
//! cargo xtask perf [--quick] [--update-baseline [--allow-contended]] [--threshold PCT] [--bench NAME]... [--skip-build] [--reuse]
//! cargo xtask perf --update-baseline --from results.json [--from results.json]... [--allow-contended]
//! ```
//!
//! - `--quick`: only the benches with `quick_args` (small synthetic documents; minutes, not tens
//! of minutes). Quick numbers are compared only with a quick baseline, and budgets don't apply
//! (they are set for the full-size documents).
//! - `--update-baseline`: write this run into `perf/baseline.json` (the only way it changes).
//! A contended run (1-minute load at the start above `settings.max_load_per_core` per core) is
//! refused; `--allow-contended` records it, labelled `"contended": true`.
//! - `--from FILE`: with `--update-baseline`, build the baseline from saved `results.json` files
//! (at least `settings.baseline_min_runs`, one machine class and mode, clean trees) instead of
//! running: per scenario the median p50 and the highest (the noise ceiling regressions are
//! judged against). Prints which budgets could be enforced and which enforced ones are flaky.
//! - `--threshold PCT`: the regression threshold (default `settings.regression_pct`, 15 %).
//! - `--bench NAME`: run only these benches (repeatable).
//! - `--skip-build`: don't run `cargo build` first. `--reuse`: don't run the benches either;
//! re-evaluate the reports already in `target/perf/`.
//!
//! Output: `target/perf/results.json` (everything), `target/perf/summary.md` (the table the
//! nightly job posts as its summary) and one `target/perf/<bench>.json` per bench.
//!
//! Exit status: non-zero when an enforced budget breaks (full mode), a scenario's p50 regresses
//! more than the threshold against a baseline from the same machine class and mode (against its
//! noise ceiling when it was built from several runs), or a bench
//! fails. Runs from different machine classes are never compared.
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
use std::process::Command;
use std::time::Instant;
use serde::Deserialize;
use serde_json::{Value, json};
pub const BUDGETS: &str = "perf/budgets.toml";
pub const BASELINE: &str = "perf/baseline.json";
const MISSING_PREFIX: &str = "**missing**: ";
// ---- budgets.toml ----------------------------------------------------------------------------
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Budgets {
#[serde(default)]
pub settings: Settings,
#[serde(default)]
pub bench: Vec<BenchSpec>,
#[serde(default)]
pub scenario: Vec<Scenario>,
}
#[derive(Debug, Deserialize)]
#[serde(default, deny_unknown_fields)]
pub struct Settings {
/// A scenario regresses when its p50 grows by more than this share of the baseline…
pub regression_pct: f64,
/// …and by more than this many milliseconds (timer noise on sub-millisecond rows).
pub regression_min_ms: f64,
/// A run is contended when its 1-minute load at the start exceeds this many per core: its
/// numbers measure the machine's load, so `--update-baseline` refuses it.
pub max_load_per_core: f64,
/// `--update-baseline --from` needs at least this many saved runs.
pub baseline_min_runs: usize,
/// A budget could be enforced when its metric stays within this share of the budget in every run.
pub enforce_margin: f64,
}
impl Default for Settings {
fn default() -> Self {
Self { regression_pct: 15.0, regression_min_ms: 1.0, max_load_per_core: 1.0, baseline_min_runs: 5, enforce_margin: 0.6 }
}
}
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct BenchSpec {
pub name: String,
pub package: String,
/// Example name; defaults to `name`.
pub example: Option<String>,
/// Arguments of a full run (`--json <file>` is appended).
#[serde(default)]
pub args: Vec<String>,
/// Arguments of a `--quick` run; absent = the bench is skipped in quick mode.
pub quick_args: Option<Vec<String>>,
/// Source file that must exist for the bench to run (a bench that lives on another branch):
/// skipped, not failed, while it is missing.
pub requires: Option<String>,
#[serde(default)]
pub note: String,
}
impl BenchSpec {
pub fn example(&self) -> &str {
self.example.as_deref().unwrap_or(&self.name)
}
}
#[derive(Clone, Copy, Debug, Default, Deserialize, PartialEq, Eq)]
#[serde(rename_all = "lowercase")]
pub enum Metric {
P50,
#[default]
P95,
Max,
}
impl Metric {
fn label(self) -> &'static str {
match self {
Metric::P50 => "p50",
Metric::P95 => "p95",
Metric::Max => "max",
}
}
}
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Scenario {
pub id: String,
pub title: String,
pub issue: Option<u32>,
/// Bench and row name that measure it; both absent when `not_measurable` is set.
pub bench: Option<String>,
pub row: Option<String>,
#[serde(default)]
pub metric: Metric,
pub budget_ms: Option<f64>,
/// Free-form target when it isn't a plain time budget.
pub target: Option<String>,
/// A broken budget fails the run. Off for targets the code doesn't meet yet: they are
/// reported ("over budget") but only regressions fail.
#[serde(default = "yes")]
pub enforce: bool,
/// Why this can't be measured yet (the code doesn't support it).
pub not_measurable: Option<String>,
}
fn yes() -> bool {
true
}
impl Scenario {
pub fn target_text(&self) -> String {
if let Some(t) = &self.target {
return t.clone();
}
match self.budget_ms {
Some(b) if b.fract() == 0.0 => format!("≤ {b:.0} ms {}", self.metric.label()),
Some(b) => format!("≤ {b} ms {}", self.metric.label()),
None => "track only".into(),
}
}
}
pub fn parse_budgets(text: &str) -> Result<Budgets, String> {
let b: Budgets = toml::from_str(text).map_err(|e| format!("{BUDGETS}: {e}"))?;
let mut ids = std::collections::BTreeSet::new();
for s in &b.scenario {
if !ids.insert(s.id.as_str()) {
return Err(format!("{BUDGETS}: duplicate scenario id {}", s.id));
}
match (&s.bench, &s.row, &s.not_measurable) {
(Some(bench), Some(_), None) => {
if !b.bench.iter().any(|x| &x.name == bench) {
return Err(format!("{BUDGETS}: scenario {} uses undefined bench `{bench}`", s.id));
}
}
(None, None, Some(_)) => {}
_ => return Err(format!("{BUDGETS}: scenario {} needs either bench + row, or not_measurable", s.id)),
}
if s.budget_ms.is_some_and(|v| !(v.is_finite() && v > 0.0)) {
return Err(format!("{BUDGETS}: scenario {} has an invalid budget_ms", s.id));
}
}
if !(b.settings.regression_pct.is_finite() && b.settings.regression_pct > 0.0) {
return Err(format!("{BUDGETS}: settings.regression_pct must be > 0"));
}
if !(b.settings.regression_min_ms.is_finite() && b.settings.regression_min_ms >= 0.0) {
return Err(format!("{BUDGETS}: settings.regression_min_ms must be finite and >= 0"));
}
if !(b.settings.max_load_per_core.is_finite() && b.settings.max_load_per_core > 0.0) {
return Err(format!("{BUDGETS}: settings.max_load_per_core must be > 0"));
}
if b.settings.baseline_min_runs == 0 {
return Err(format!("{BUDGETS}: settings.baseline_min_runs must be >= 1"));
}
if !(b.settings.enforce_margin.is_finite() && b.settings.enforce_margin > 0.0 && b.settings.enforce_margin <= 1.0) {
return Err(format!("{BUDGETS}: settings.enforce_margin must be in (0, 1]"));
}
Ok(b)
}
// ---- measurements ----------------------------------------------------------------------------
/// One scenario's numbers.
#[derive(Clone, Debug, Default, PartialEq)]
pub struct Stats {
pub n: usize,
pub p50: Option<f64>,
pub p95: Option<f64>,
pub max: Option<f64>,
pub peak_rss: Option<u64>,
pub gpu_bytes: Option<u64>,
/// A baseline built from several runs: the highest p50 among them (its noise ceiling).
pub p50_max: Option<f64>,
}
impl Stats {
pub fn metric(&self, m: Metric) -> Option<f64> {
match m {
Metric::P50 => self.p50,
Metric::P95 => self.p95,
Metric::Max => self.max,
}
}
pub fn to_json(&self) -> Value {
let mut v =
json!({"n": self.n, "p50_ms": self.p50, "p95_ms": self.p95, "max_ms": self.max, "peak_rss_bytes": self.peak_rss, "gpu_bytes": self.gpu_bytes});
if let (Some(m), Some(o)) = (self.p50_max, v.as_object_mut()) {
o.insert("p50_max_ms".into(), json!(m));
}
v
}
pub fn from_json(v: &Value) -> Self {
Self {
n: v.get("n").and_then(Value::as_u64).unwrap_or(0) as usize,
p50: v.get("p50_ms").and_then(Value::as_f64),
p95: v.get("p95_ms").and_then(Value::as_f64),
max: v.get("max_ms").and_then(Value::as_f64),
peak_rss: v.get("peak_rss_bytes").and_then(Value::as_u64),
gpu_bytes: v.get("gpu_bytes").and_then(Value::as_u64),
p50_max: v.get("p50_max_ms").and_then(Value::as_f64),
}
}
}
/// Nearest-rank percentile of sorted, non-empty `v`.
fn rank(v: &[f64], q: f64) -> Option<f64> {
let n = v.len();
if n == 0 {
return None;
}
let k = ((q * n as f64).ceil() as usize).clamp(1, n);
v.get(k - 1).copied()
}
/// A bench row → stats. Rows carry `samples_ms` (preferred: percentiles are recomputed here),
/// or `p50_ms`/`p95_ms`/`max_ms`, or the older `median_ms`/`max_ms`. A p95 is never invented:
/// with neither samples nor a reported p95 it stays empty.
pub fn row_stats(row: &Value) -> Stats {
let mut samples: Vec<f64> =
row.get("samples_ms").and_then(Value::as_array).map(|a| a.iter().filter_map(Value::as_f64).filter(|x| x.is_finite()).collect()).unwrap_or_default();
samples.sort_by(f64::total_cmp);
let f = |k: &str| row.get(k).and_then(Value::as_f64);
let mut s = Stats {
n: samples.len(),
p50: f("p50_ms").or_else(|| f("median_ms")),
p95: f("p95_ms"),
max: f("max_ms"),
peak_rss: row.get("peak_rss_bytes").and_then(Value::as_u64),
gpu_bytes: row.get("gpu_bytes").and_then(Value::as_u64),
p50_max: None,
};
if !samples.is_empty() {
s.p50 = rank(&samples, 0.5);
s.p95 = rank(&samples, 0.95);
s.max = samples.last().copied();
} else if let Some(n) = row.get("n").and_then(Value::as_u64) {
s.n = n as usize;
}
s
}
/// The row named `name` in a bench report (`{"rows": [{"name": …}, …]}`).
pub fn find_row<'a>(report: &'a Value, name: &str) -> Option<&'a Value> {
report.get("rows")?.as_array()?.iter().find(|r| r.get("name").and_then(Value::as_str) == Some(name))
}
/// `cur` regressed against `base` by more than `pct` % and more than `min_ms`.
pub fn regressed(cur: f64, base: f64, pct: f64, min_ms: f64) -> bool {
base > 0.0 && cur.is_finite() && cur > base * (1.0 + pct / 100.0) && cur - base > min_ms
}
pub fn change_pct(cur: f64, base: f64) -> Option<f64> {
(base > 0.0 && cur.is_finite()).then(|| (cur / base - 1.0) * 100.0)
}
#[derive(Clone, Debug, PartialEq)]
pub enum Status {
Pass,
/// Over budget, not enforced (target not met yet).
Over,
/// Over an enforced budget.
Broken,
Regressed,
/// Measured, no budget, no regression.
Tracked,
/// The bench ran but the row is missing or errored.
Missing(String),
NotRun,
NotMeasurable(String),
}
impl Status {
pub fn label(&self) -> String {
match self {
Status::Pass => "pass".into(),
Status::Over => "over budget (target not met yet)".into(),
Status::Broken => "**BUDGET BROKEN**".into(),
Status::Regressed => "**REGRESSED**".into(),
Status::Tracked => "tracked".into(),
Status::Missing(e) => format!("{MISSING_PREFIX}{e}"),
Status::NotRun => "not run".into(),
Status::NotMeasurable(_) => "not measurable yet".into(),
}
}
pub fn fails(&self) -> bool {
matches!(self, Status::Broken | Status::Regressed | Status::Missing(_))
}
}
/// One evaluated scenario.
#[derive(Clone, Debug)]
pub struct Outcome {
pub id: String,
pub title: String,
pub issue: Option<u32>,
pub target: String,
pub stats: Option<Stats>,
pub baseline_p50: Option<f64>,
pub change_pct: Option<f64>,
pub status: Status,
}
/// Budget and regression verdict for one scenario. `budgets_apply` is false in quick mode.
/// A regression is judged against the baseline's noise ceiling (the highest p50 of the runs it was
/// built from), or its p50 when it came from one run.
pub fn verdict(sc: &Scenario, stats: &Stats, base: Option<&Stats>, settings: &Settings, threshold: f64, budgets_apply: bool) -> Status {
if let (Some(cur), Some(b)) = (stats.p50, base.and_then(|b| b.p50_max.or(b.p50)))
&& regressed(cur, b, threshold, settings.regression_min_ms)
{
return Status::Regressed;
}
match (budgets_apply, sc.budget_ms, stats.metric(sc.metric)) {
(true, Some(budget), Some(v)) if v > budget => {
if sc.enforce {
Status::Broken
} else {
Status::Over
}
}
(true, Some(_), Some(_)) => Status::Pass,
(true, Some(_), None) => Status::Missing(format!("no {} in the bench row", sc.metric.label())),
_ => Status::Tracked,
}
}
/// Evaluates every scenario against the bench reports (`bench name → report`, `None` when the
/// bench didn't run in this mode) and the baseline's scenarios for this mode.
pub fn evaluate(
budgets: &Budgets,
reports: &BTreeMap<String, Result<Value, String>>,
baseline: Option<&BTreeMap<String, Stats>>,
threshold: f64,
budgets_apply: bool,
) -> Vec<Outcome> {
budgets
.scenario
.iter()
.map(|sc| {
let base = baseline.and_then(|b| b.get(&sc.id));
let mut o = Outcome {
id: sc.id.clone(),
title: sc.title.clone(),
issue: sc.issue,
target: sc.target_text(),
stats: None,
baseline_p50: base.and_then(|b| b.p50),
change_pct: None,
status: Status::NotRun,
};
if let Some(why) = &sc.not_measurable {
o.status = Status::NotMeasurable(why.clone());
return o;
}
let (Some(bench), Some(row)) = (&sc.bench, &sc.row) else { return o };
match reports.get(bench) {
None => {}
Some(Err(e)) => o.status = Status::Missing(format!("bench `{bench}` failed: {e}")),
Some(Ok(rep)) => match find_row(rep, row) {
None => {
let err = rep
.get("context")
.and_then(|c| c.get("errors"))
.and_then(Value::as_array)
.and_then(|a| a.iter().find(|e| e.get("name").and_then(Value::as_str) == Some(row.as_str())))
.and_then(|e| e.get("error"))
.and_then(Value::as_str)
.map_or_else(|| format!("row `{row}` not in `{bench}`"), str::to_string);
o.status = Status::Missing(err);
}
Some(r) => {
let st = row_stats(r);
o.change_pct = st.p50.zip(o.baseline_p50).and_then(|(c, b)| change_pct(c, b));
o.status = verdict(sc, &st, base, &budgets.settings, threshold, budgets_apply);
o.stats = Some(st);
}
},
}
o
})
.collect()
}
// ---- formatting ------------------------------------------------------------------------------
pub fn fmt_num(v: f64) -> String {
if v >= 100.0 {
format!("{v:.0}")
} else if v >= 10.0 {
format!("{v:.1}")
} else {
format!("{v:.2}")
}
}
fn fmt_ms(v: Option<f64>) -> String {
v.map_or_else(|| "–".into(), fmt_num)
}
pub fn fmt_bytes(v: Option<u64>) -> String {
match v {
None | Some(0) => "–".into(),
Some(b) if b >= 1 << 30 => format!("{:.1} GB", b as f64 / (1u64 << 30) as f64),
Some(b) => format!("{} MB", b >> 20),
}
}
pub fn issue_link(n: Option<u32>) -> String {
n.map_or_else(String::new, |n| format!("[#{n}](https://github.com/storytold/photocraft/issues/{n})"))
}
/// The scenario table (Markdown); `with_baseline` adds the change against the baseline.
/// Not-measurable rows go to a separate list after it.
pub fn render_table(outcomes: &[Outcome], with_baseline: bool) -> String {
let mut s = if with_baseline {
String::from(
"| Id | Scenario | Target | p50 ms | p95 ms | max ms | Peak RSS | GPU | vs baseline | Status | Issue |\n|---|---|---|---:|---:|---:|---:|---:|---:|---|---|\n",
)
} else {
String::from(
"| Id | Scenario | Target | p50 ms | p95 ms | max ms | Peak RSS | GPU | Status | Issue |\n|---|---|---|---:|---:|---:|---:|---:|---|---|\n",
)
};
for o in outcomes.iter().filter(|o| !matches!(o.status, Status::NotMeasurable(_))) {
let st = o.stats.clone().unwrap_or_default();
let vs = if with_baseline { format!(" {} |", o.change_pct.map_or_else(|| "–".into(), |c| format!("{c:+.0} %"))) } else { String::new() };
s.push_str(&format!(
"| {} | {} | {} | {} | {} | {} | {} | {} |{} {} | {} |\n",
o.id,
o.title,
o.target,
fmt_ms(st.p50),
fmt_ms(st.p95),
fmt_ms(st.max),
fmt_bytes(st.peak_rss),
fmt_bytes(st.gpu_bytes),
vs,
o.status.label(),
issue_link(o.issue),
));
}
let nm: Vec<&Outcome> = outcomes.iter().filter(|o| matches!(o.status, Status::NotMeasurable(_))).collect();
if !nm.is_empty() {
s.push_str(
"\n**Not measurable yet** (the code can't do it yet; no number is recorded):\n\n| Id | Scenario | Target | Why | Issue |\n|---|---|---|---|---|\n",
);
for o in nm {
let why = match &o.status {
Status::NotMeasurable(w) => w.as_str(),
_ => "",
};
s.push_str(&format!("| {} | {} | {} | {} | {} |\n", o.id, o.title, o.target, why, issue_link(o.issue)));
}
}
s
}
/// Counts for the scorecard summary: (within budget, over budget or failing, tracked only, not measurable).
/// Not-run scenarios are in none of them; the scorecard counts those separately.
pub fn counts(outcomes: &[Outcome]) -> (usize, usize, usize, usize) {
let mut c = (0, 0, 0, 0);
for o in outcomes {
match o.status {
Status::Pass => c.0 += 1,
Status::Over | Status::Broken | Status::Regressed | Status::Missing(_) => c.1 += 1,
Status::Tracked => c.2 += 1,
Status::NotMeasurable(_) => c.3 += 1,
Status::NotRun => {}
}
}
c
}
// ---- machine ---------------------------------------------------------------------------------
/// Lower-case, `-`-separated, alphanumeric only.
pub fn slug(s: &str) -> String {
let mut out = String::new();
for ch in s.chars() {
if ch.is_ascii_alphanumeric() {
out.push(ch.to_ascii_lowercase());
} else if !out.ends_with('-') && !out.is_empty() {
out.push('-');
}
}
out.trim_end_matches('-').to_string()
}
/// Runs are compared only within one class: OS, architecture, CPU model and GPU adapter.
pub fn machine_class(os: &str, arch: &str, cpu: &str, gpu: &str) -> String {
[os, arch, cpu, gpu].iter().map(|p| slug(p)).filter(|p| !p.is_empty()).collect::<Vec<_>>().join("-")
}
/// The first three numbers in `/proc/loadavg` or `sysctl -n vm.loadavg` output.
pub fn parse_loadavg(s: &str) -> Option<[f64; 3]> {
let v: Vec<f64> = s.split_whitespace().filter_map(|t| t.trim_matches(|c| c == '{' || c == '}').parse().ok()).take(3).collect();
Some([*v.first()?, *v.get(1)?, *v.get(2)?])
}
fn output(cmd: &str, args: &[&str]) -> Option<String> {
let o = Command::new(cmd).args(args).output().ok()?;
o.status.success().then(|| String::from_utf8_lossy(&o.stdout).trim().to_string()).filter(|s| !s.is_empty())
}
pub fn load_avg() -> Option<[f64; 3]> {
if let Ok(s) = std::fs::read_to_string("/proc/loadavg") {
return parse_loadavg(&s);
}
output("sysctl", &["-n", "vm.loadavg"]).and_then(|s| parse_loadavg(&s))
}
fn proc_field(file: &str, key: &str) -> Option<String> {
let s = std::fs::read_to_string(file).ok()?;
s.lines().find(|l| l.starts_with(key)).and_then(|l| l.split_once(':').or_else(|| l.split_once('='))).map(|(_, v)| v.trim().trim_matches('"').to_string())
}
pub fn machine() -> Value {
let (os, arch) = (std::env::consts::OS, std::env::consts::ARCH);
let cores = std::thread::available_parallelism().map(|n| n.get()).ok();
let (cpu, ram, os_version) = match os {
"macos" => (
output("sysctl", &["-n", "machdep.cpu.brand_string"]),
output("sysctl", &["-n", "hw.memsize"]).and_then(|s| s.parse::<u64>().ok()),
output("sw_vers", &["-productVersion"]).map(|v| format!("macOS {v}")),
),
"linux" => (
proc_field("/proc/cpuinfo", "model name").or_else(|| proc_field("/proc/cpuinfo", "CPU part").map(|p| format!("ARM part {p}"))),
proc_field("/proc/meminfo", "MemTotal")
.and_then(|v| v.split_whitespace().next().and_then(|k| k.parse::<u64>().ok()))
.and_then(|k| k.checked_mul(1024)),
proc_field("/etc/os-release", "PRETTY_NAME"),
),
_ => (std::env::var("PROCESSOR_IDENTIFIER").ok(), None, output("cmd", &["/c", "ver"])),
};
json!({"os": os, "os_version": os_version, "arch": arch, "cpu": cpu, "cores": cores, "ram_bytes": ram})
}
/// `YYYY-MM-DDTHH:MM:SSZ` for a Unix time (proleptic Gregorian, UTC). A time past chrono's year
/// range still converts (the calendar repeats every 400 years, so the instant is moved into range
/// by whole cycles and only the year is carried, matching the old hand-rolled `iso_utc`), never a
/// silent reset to the epoch. Accepts the full `u64` range the old code did.
pub fn iso_utc(secs: u64) -> String {
use chrono::Datelike;
// `secs / 86_400` is at most 213_503_982_334_601, so `days` always fits in `i64` (old code too).
let (days, sod) = ((secs / 86_400) as i64, (secs % 86_400) as i64);
let (cycles, rest) = (days.div_euclid(146_097), days.rem_euclid(146_097));
// `rest + 719_163` (1970-01-01 is CE day 719_163) is always a valid CE day, and `sod` < 86_400.
let date = i32::try_from(rest + 719_163).ok().and_then(chrono::NaiveDate::from_num_days_from_ce_opt);
let time = u32::try_from(sod).ok().and_then(|s| chrono::NaiveTime::from_num_seconds_from_midnight_opt(s, 0));
match (date, time) {
(Some(d), Some(t)) => {
let y = i128::from(d.year()) + i128::from(cycles) * 400;
let y = i64::try_from(y).unwrap_or(if y < 0 { i64::MIN } else { i64::MAX });
// `{:04}` rather than `%Y`, which writes a `+` before years past 9999.
format!("{y:04}-{}", d.and_time(t).format("%m-%dT%H:%M:%SZ"))
}
_ => "1970-01-01T00:00:00Z".to_string(), // unreachable
}
}
fn now_iso() -> String {
iso_utc(std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_secs()).unwrap_or(0))
}
fn git(root: &Path, args: &[&str]) -> Option<String> {
let o = Command::new("git").current_dir(root).args(args).output().ok()?;
o.status.success().then(|| String::from_utf8_lossy(&o.stdout).trim().to_string())
}
// ---- baseline --------------------------------------------------------------------------------
/// The baseline's scenarios for `mode`, if it was recorded on `class`. `Err` explains a skip.
pub fn baseline_for(baseline: &Value, class: &str, mode: &str) -> Result<BTreeMap<String, Stats>, String> {
let bc = baseline.get("machine_class").and_then(Value::as_str).unwrap_or("");
if bc != class {
return Err(format!("baseline is from machine class `{bc}`, this is `{class}`: regression check skipped"));
}
let sc = baseline
.get("modes")
.and_then(|m| m.get(mode))
.and_then(|m| m.get("scenarios"))
.and_then(Value::as_object)
.ok_or_else(|| format!("baseline has no `{mode}` run: regression check skipped"))?;
Ok(sc.iter().map(|(k, v)| (k.clone(), Stats::from_json(v))).collect())
}
/// A run is contended when its 1-minute load at the start exceeds `max_per_core` per core:
/// its numbers measure the machine's load, not the code.
/// Without a load (Windows) or a core count, nothing can be judged: not contended.
pub fn contended(load: Option<[f64; 3]>, cores: Option<usize>, max_per_core: f64) -> bool {
match (load, cores) {
(Some([one, _, _]), Some(c)) if c > 0 => one > c as f64 * max_per_core,
_ => false,
}
}
fn load_of(v: Option<&Value>) -> Option<[f64; 3]> {
let a = v?.as_array()?;
Some([a.first()?.as_f64()?, a.get(1)?.as_f64()?, a.get(2)?.as_f64()?])
}
fn cores_of(machine: Option<&Value>) -> Option<usize> {
machine?.get("cores")?.as_u64().map(|c| c as usize)
}
/// Whether a `results.json` (or a baseline mode plus its machine) was contended at the start.
pub fn run_contended(load_avg_start: Option<&Value>, machine: Option<&Value>, max_per_core: f64) -> bool {
contended(load_of(load_avg_start), cores_of(machine), max_per_core)
}
/// What `baseline_from_runs` accepts.
pub struct BaselineRules {
/// Fewer runs are refused.
pub min_runs: usize,
pub max_load_per_core: f64,
/// Accept contended runs (the baseline then records `"contended": true`).
pub allow_contended: bool,
}
fn median(v: &mut [f64]) -> Option<f64> {
v.sort_by(f64::total_cmp);
let n = v.len();
match n {
0 => None,
_ if n % 2 == 1 => v.get(n / 2).copied(),
_ => Some((v.get(n / 2 - 1)? + v.get(n / 2)?) / 2.0),
}
}
/// `old` with one mode replaced by a baseline built from `runs` (saved `results.json` values of
/// one machine class and mode), or a fresh baseline when the machine class changed. Per scenario:
/// `p50_ms` is the median of the runs' p50 and, from two runs on, `p50_max_ms` their maximum (the
/// noise ceiling regressions are judged against); p95 is the median, max and memory the maximum.
/// One run is the plain `--update-baseline` case. Refused: too few runs, mixed machine classes or
/// modes, dirty trees when there are several runs, and contended runs unless allowed.
pub fn baseline_from_runs(old: Option<&Value>, runs: &[Value], rules: &BaselineRules) -> Result<Value, String> {
if runs.len() < rules.min_runs.max(1) {
return Err(format!("a baseline needs at least {} run(s), got {}", rules.min_runs.max(1), runs.len()));
}
let class_of = |r: &Value| r.get("machine").and_then(|m| m.get("class")).cloned().unwrap_or(Value::Null);
let mode_of = |r: &Value| r.get("mode").and_then(Value::as_str).unwrap_or("full").to_string();
let mut runs: Vec<&Value> = runs.iter().collect();
runs.sort_by_key(|r| r.get("date").and_then(Value::as_str).unwrap_or("").to_string());
let Some(last) = runs.last().copied() else { return Err("no runs".into()) };
let (class, mode) = (class_of(last), mode_of(last));
if runs.iter().any(|r| class_of(r) != class) {
return Err("the runs are from different machine classes: a baseline needs one".into());
}
if runs.iter().any(|r| mode_of(r) != mode) {
return Err("the runs mix full and quick modes: a baseline needs one".into());
}
if runs.len() > 1
&& let Some(r) = runs.iter().find(|r| r.get("dirty").and_then(Value::as_bool) == Some(true))
{
return Err(format!("run {} was taken on a dirty tree: a baseline needs committed code", r.get("date").and_then(Value::as_str).unwrap_or("?")));
}
let is_contended = |r: &Value| run_contended(r.get("load_avg_start"), r.get("machine"), rules.max_load_per_core);
let any_contended = runs.iter().any(|r| is_contended(r));
if any_contended && !rules.allow_contended {
let msgs: Vec<String> = runs
.iter()
.filter(|r| is_contended(r))
.map(|r| {
format!(
"load {} on {} cores at the start of {}",
load_of(r.get("load_avg_start")).map_or_else(|| "?".into(), |l| format!("{:.2}", l[0])),
cores_of(r.get("machine")).map_or_else(|| "?".into(), |c| c.to_string()),
r.get("date").and_then(Value::as_str).unwrap_or("the run")
)
})
.collect();
return Err(format!(
"contended run, baseline not written: {} (above {} per core). Its numbers measure machine load; rerun on an idle machine, or pass --allow-contended to record it labelled as contended",
msgs.join("; "),
rules.max_load_per_core
));
}
let mut per: BTreeMap<String, Vec<Stats>> = BTreeMap::new();
let mut failed = serde_json::Map::new();
for r in &runs {
for (k, v) in r.get("scenarios").and_then(Value::as_object).into_iter().flatten() {
if v.get("p50_ms").is_some_and(|x| !x.is_null()) {
per.entry(k.clone()).or_default().push(Stats::from_json(v));
} else if let Some(e) = v.get("status").and_then(Value::as_str).and_then(|s| s.strip_prefix(MISSING_PREFIX)) {
// A crashed row (a crash the bench found): kept so the scorecard says so.
failed.insert(k.clone(), json!(e));
}
}
}
let mut scenarios = serde_json::Map::new();
for (k, st) in &per {
// A row that crashed in any run stays failed, and one missing from some runs is left
// out: no partial aggregate under a full run count.
if failed.contains_key(k) || st.len() != runs.len() {
continue;
}
let mut p50: Vec<f64> = st.iter().filter_map(|s| s.p50).collect();
let mut p95: Vec<f64> = st.iter().filter_map(|s| s.p95).collect();
let fmax = |f: fn(&Stats) -> Option<f64>| st.iter().filter_map(f).reduce(f64::max);
let umax = |f: fn(&Stats) -> Option<u64>| st.iter().filter_map(f).max();
let s = Stats {
n: st.iter().map(|s| s.n).sum(),
p50_max: (runs.len() > 1).then(|| fmax(|s| s.p50)).flatten(),
p50: median(&mut p50),
p95: median(&mut p95),
max: fmax(|s| s.max),
peak_rss: umax(|s| s.peak_rss),
gpu_bytes: umax(|s| s.gpu_bytes),
};
scenarios.insert(k.clone(), s.to_json());
}
let mut modes = old.filter(|o| o.get("machine_class") == Some(&class)).and_then(|o| o.get("modes")).and_then(Value::as_object).cloned().unwrap_or_default();
let mut entry = json!({
"date": last.get("date"),
"commit": last.get("commit"),
"load_avg_start": last.get("load_avg_start"),
"load_avg_end": last.get("load_avg_end"),
"cores": cores_of(last.get("machine")),
"runs": runs.len(),
"scenarios": scenarios,
"failed": failed,
});
if let Some(o) = entry.as_object_mut() {
if any_contended {
o.insert("contended".into(), json!(true));
}
if runs.len() > 1 {
let each: Vec<Value> = runs
.iter()
.map(|r| json!({"date": r.get("date"), "commit": r.get("commit"), "load_avg_start": r.get("load_avg_start"), "contended": is_contended(r)}))
.collect();
o.insert("run_list".into(), json!(each));
}
}
modes.insert(mode, entry);
Ok(json!({"schema": 1, "machine_class": class, "machine": last.get("machine"), "modes": modes}))
}
/// From saved runs: budgets that could be enforced (not enforced, and within `enforce_margin` of
/// the budget in every run) and enforced budgets that went over in some run (flaky).
pub fn enforce_report(budgets: &Budgets, runs: &[Value]) -> (Vec<String>, Vec<String>) {
let (mut could, mut flaky) = (Vec::new(), Vec::new());
let margin = budgets.settings.enforce_margin;
for sc in &budgets.scenario {
let (Some(budget), None) = (sc.budget_ms, &sc.not_measurable) else { continue };
let key = format!("{}_ms", sc.metric.label());
let vals: Vec<f64> = runs.iter().filter_map(|r| r.get("scenarios")?.get(&sc.id)?.get(&key)?.as_f64()).collect();
let Some(worst) = vals.iter().copied().reduce(f64::max) else { continue };
let m = sc.metric.label();
if sc.enforce {
let over = vals.iter().filter(|v| **v > budget).count();
if over > 0 {
flaky.push(format!("{} {}: over {} ms in {over} of {} runs (worst {m} {})", sc.id, sc.title, fmt_num(budget), runs.len(), fmt_num(worst)));
}
} else if vals.len() == runs.len() && worst <= margin * budget {
could.push(format!("{} {}: worst {m} {} ms ≤ {margin} × {} ms in all {} runs", sc.id, sc.title, fmt_num(worst), fmt_num(budget), runs.len()));
}
}
(could, flaky)
}
// ---- running ---------------------------------------------------------------------------------
struct Opts {
quick: bool,
update_baseline: bool,
threshold: Option<f64>,
benches: Vec<String>,
skip_build: bool,
reuse: bool,
/// `--update-baseline --from`: build the baseline from these saved `results.json` files.
from: Vec<PathBuf>,
allow_contended: bool,
}
fn parse_opts(rest: &[&str]) -> Result<Opts, String> {
let mut o = Opts {
quick: false,
update_baseline: false,
threshold: None,
benches: Vec::new(),
skip_build: false,
reuse: false,
from: Vec::new(),
allow_contended: false,
};
let mut it = rest.iter();
while let Some(a) = it.next() {
match *a {
"--quick" => o.quick = true,
"--update-baseline" => o.update_baseline = true,
"--skip-build" => o.skip_build = true,
"--reuse" => {
o.reuse = true;
o.skip_build = true;
}
"--threshold" => {
let v = it.next().and_then(|v| v.parse::<f64>().ok()).filter(|v| v.is_finite() && *v > 0.0).ok_or("--threshold needs a percentage > 0")?;
o.threshold = Some(v);
}
"--bench" => o.benches.push(it.next().ok_or("--bench needs a name")?.to_string()),
"--from" => o.from.push(PathBuf::from(it.next().ok_or("--from needs a results.json path")?)),
"--allow-contended" => o.allow_contended = true,
other => return Err(format!("perf: unknown option `{other}`")),
}
}
if !o.from.is_empty() && !o.update_baseline {
return Err("--from builds a baseline: use it with --update-baseline".into());
}
Ok(o)
}
fn target_dir() -> Result<PathBuf, String> {
let m = crate::metadata()?;
m.get("target_directory").and_then(Value::as_str).map(PathBuf::from).ok_or_else(|| "cargo metadata: no target_directory".into())
}
fn write(path: &Path, text: &str) -> Result<(), String> {
if let Some(dir) = path.parent() {
std::fs::create_dir_all(dir).map_err(|e| format!("create {}: {e}", dir.display()))?;
}
std::fs::write(path, text).map_err(|e| format!("write {}: {e}", path.display()))
}
pub fn run(root: &Path, rest: &[&str]) -> Result<(), String> {
let o = parse_opts(rest)?;
let text = std::fs::read_to_string(root.join(BUDGETS)).map_err(|e| format!("read {BUDGETS}: {e}"))?;
let budgets = parse_budgets(&text)?;
for name in &o.benches {
if !budgets.bench.iter().any(|b| &b.name == name) {
return Err(format!("perf: unknown bench `{name}` (see {BUDGETS})"));
}
}
let baseline_value: Option<Value> = std::fs::read_to_string(root.join(BASELINE)).ok().and_then(|t| serde_json::from_str(&t).ok());
if !o.from.is_empty() {
return baseline_from_files(root, &budgets, baseline_value.as_ref(), &o);
}
let mode = if o.quick { "quick" } else { "full" };
let threshold = o.threshold.unwrap_or(budgets.settings.regression_pct);
let target = target_dir()?;
let out_dir = target.join("perf");
// Which benches run in this mode.
let mut plan: Vec<(&BenchSpec, Vec<String>)> = Vec::new();
let mut skipped: Vec<(String, String)> = Vec::new();
for b in &budgets.bench {
if !o.benches.is_empty() && !o.benches.contains(&b.name) {
continue;
}
let args = if o.quick {
match &b.quick_args {
Some(a) => a.clone(),
None => continue,
}
} else {
b.args.clone()
};
if let Some(req) = &b.requires
&& !root.join(req).exists()
{
skipped.push((b.name.clone(), format!("{req} not on this branch yet")));
continue;
}
plan.push((b, args));
}
// Sampled before the build: the harness's own compile isn't contention. The load after the
// build is still recorded, as the first bench's `load_avg_before`.
let load_start = load_avg();
if !o.skip_build {
let mut by_pkg: BTreeMap<&str, Vec<&str>> = BTreeMap::new();
for (b, _) in &plan {
by_pkg.entry(b.package.as_str()).or_default().push(b.example());
}
for (pkg, examples) in by_pkg {
let mut c = crate::cargo();
c.args(["build", "--release", "-p", pkg]);
for e in &examples {
c.args(["--example", e]);
}
crate::run(c, &format!("cargo build --release -p {pkg} --example {}", examples.join(" --example ")))?;
}
}
let mut reports: BTreeMap<String, Result<Value, String>> = BTreeMap::new();
let mut bench_meta = serde_json::Map::new();
for (b, args) in &plan {
let json_path = out_dir.join(format!("{}.json", b.name));
let before = load_avg();
let t = Instant::now();
let result = if o.reuse {
std::fs::read_to_string(&json_path).map_err(|e| format!("--reuse: {}: {e}", json_path.display()))
} else {
let _ = std::fs::remove_file(&json_path);
if let Some(dir) = json_path.parent() {
std::fs::create_dir_all(dir).map_err(|e| format!("create {}: {e}", dir.display()))?;
}
let exe = target.join("release").join("examples").join(format!("{}{}", b.example(), std::env::consts::EXE_SUFFIX));
let mut c = Command::new(&exe);
c.current_dir(root).args(args).arg("--json").arg(&json_path);
let shown = format!("{} {} --json {}", b.example(), args.join(" "), json_path.display());
crate::run(c, &shown).and_then(|()| std::fs::read_to_string(&json_path).map_err(|e| format!("{}: no JSON report ({e})", b.name)))
};
let parsed = result.and_then(|t| serde_json::from_str::<Value>(&t).map_err(|e| format!("{}: bad JSON: {e}", b.name)));
let after = load_avg();
bench_meta.insert(
b.name.clone(),
json!({
"status": if parsed.is_ok() { "ok" } else { "error" },
"note": b.note,
"error": parsed.as_ref().err(),
"seconds": t.elapsed().as_secs_f64(),
"args": args,
"load_avg_before": before,
"load_avg_after": after,
"context": parsed.as_ref().ok().and_then(|r| r.get("context")).cloned(),
"process_peak_rss_bytes": parsed.as_ref().ok().and_then(|r| r.get("process_peak_rss_bytes")).cloned(),
// Every row, mapped to a scenario or not (new benches show up here first).
"rows": parsed.as_ref().ok().and_then(|r| r.get("rows")).and_then(Value::as_array).map(|rows| {
rows.iter()
.map(|r| {
let mut v = row_stats(r).to_json();
if let Some(o) = v.as_object_mut() {
o.insert("name".into(), r.get("name").cloned().unwrap_or(Value::Null));
}
v
})
.collect::<Vec<_>>()
}),
}),
);
reports.insert(b.name.clone(), parsed);
}
for (name, why) in &skipped {
bench_meta.insert(name.clone(), json!({"status": "skipped", "error": why}));
}
let load_end = load_avg();
let mut m = machine();
let gpu = reports
.values()
.filter_map(|r| r.as_ref().ok())
.find_map(|r| r.get("context").and_then(|c| c.get("gpu_adapter")).and_then(Value::as_str).map(str::to_string));
let gpu_name = gpu.clone().unwrap_or_else(|| "cpu-canvas".into());
let class = machine_class(
m.get("os").and_then(Value::as_str).unwrap_or(""),
m.get("arch").and_then(Value::as_str).unwrap_or(""),
m.get("cpu").and_then(Value::as_str).unwrap_or(""),
&gpu_name,
);
if let Some(obj) = m.as_object_mut() {
obj.insert("gpu_adapter".into(), json!(gpu));
obj.insert("class".into(), json!(class));
}
let (baseline, baseline_note) = match baseline_value.as_ref().map(|b| baseline_for(b, &class, mode)) {
Some(Ok(b)) => (Some(b), None),
Some(Err(e)) => (None, Some(e)),
None => (None, Some(format!("no {BASELINE}: regression check skipped"))),
};
let outcomes = evaluate(&budgets, &reports, baseline.as_ref(), threshold, !o.quick);
let scenarios: serde_json::Map<String, Value> = budgets
.scenario
.iter()
.zip(&outcomes)
.map(|(sc, oc)| {
let mut v = oc.stats.as_ref().map(Stats::to_json).unwrap_or_else(|| json!({}));
if let Some(obj) = v.as_object_mut() {
obj.insert("title".into(), json!(sc.title));
obj.insert("issue".into(), json!(sc.issue));
obj.insert("bench".into(), json!(sc.bench));
obj.insert("row".into(), json!(sc.row));
obj.insert("metric".into(), json!(sc.metric.label()));
obj.insert("budget_ms".into(), json!(sc.budget_ms));
obj.insert("target".into(), json!(sc.target_text()));
obj.insert("enforce".into(), json!(sc.enforce));
obj.insert("baseline_p50_ms".into(), json!(oc.baseline_p50));
obj.insert("change_pct".into(), json!(oc.change_pct));
obj.insert("status".into(), json!(oc.status.label()));
if let Status::NotMeasurable(w) = &oc.status {
obj.insert("not_measurable".into(), json!(w));
}
}
(sc.id.clone(), v)
})
.collect();
let failures: Vec<String> = outcomes.iter().filter(|o| o.status.fails()).map(|o| format!("{} {}: {}", o.id, o.title, o.status.label())).collect();
let bench_failures: Vec<String> = reports.iter().filter_map(|(n, r)| r.as_ref().err().map(|e| format!("bench {n}: {e}"))).collect();
let commit = git(root, &["rev-parse", "HEAD"]);
let dirty = git(root, &["status", "--porcelain"]).is_some_and(|s| !s.is_empty());
let is_contended = contended(load_start, m.get("cores").and_then(Value::as_u64).map(|c| c as usize), budgets.settings.max_load_per_core);
let results = json!({
"schema": 1,
"mode": mode,
"date": now_iso(),
"commit": commit,
"dirty": dirty,
"machine": m,
"load_avg_start": load_start,
"load_avg_end": load_end,
"contended": is_contended,
"threshold_pct": threshold,
"baseline_note": baseline_note,
"benches": bench_meta,
"scenarios": scenarios,
"failures": failures.iter().chain(&bench_failures).collect::<Vec<_>>(),
});
let text = serde_json::to_string_pretty(&results).map_err(|e| format!("encode results: {e}"))?;
write(&out_dir.join("results.json"), &text)?;
let summary = render_summary(&results, &outcomes, baseline_note.as_deref());
write(&out_dir.join("summary.md"), &summary)?;
println!("\n{summary}");
println!("wrote {}", out_dir.join("results.json").display());
let mut errors: Vec<String> = Vec::new();
if o.update_baseline {
// The results are written either way; a contended run never silently becomes the baseline.
let rules = BaselineRules { min_runs: 1, max_load_per_core: budgets.settings.max_load_per_core, allow_contended: o.allow_contended };
match baseline_from_runs(baseline_value.as_ref(), std::slice::from_ref(&results), &rules) {
Ok(nb) => {
write_baseline(root, &nb)?;
println!("updated {BASELINE} ({mode}, machine class {class}{})", if is_contended { ", labelled contended" } else { "" });
}
// A refused baseline still reports this run's perf failures below.
Err(e) => errors.push(e),
}
}
let all: Vec<&String> = failures.iter().chain(&bench_failures).collect();
if !all.is_empty() {
errors.push(format!("{} perf failure(s):\n {}", all.len(), all.iter().map(|s| s.as_str()).collect::<Vec<_>>().join("\n ")));
}
if errors.is_empty() { Ok(()) } else { Err(errors.join("\n")) }
}
fn write_baseline(root: &Path, baseline: &Value) -> Result<(), String> {
let text = serde_json::to_string_pretty(baseline).map_err(|e| format!("encode baseline: {e}"))?;
write(&root.join(BASELINE), &format!("{text}\n"))
}
/// `--update-baseline --from <results.json>...`: no benches run. Prints the enforcement report.
fn baseline_from_files(root: &Path, budgets: &Budgets, old: Option<&Value>, o: &Opts) -> Result<(), String> {
let runs = o
.from
.iter()
.map(|p| {
let t = std::fs::read_to_string(p).map_err(|e| format!("read {}: {e}", p.display()))?;
serde_json::from_str::<Value>(&t).map_err(|e| format!("{}: {e}", p.display()))
})
.collect::<Result<Vec<_>, _>>()?;
let (could, flaky) = enforce_report(budgets, &runs);
let list = |v: &[String]| if v.is_empty() { " none".to_string() } else { v.iter().map(|l| format!(" {l}")).collect::<Vec<_>>().join("\n") };
println!("Budgets that could be enforced (within {} of the budget in every run):\n{}", budgets.settings.enforce_margin, list(&could));
println!("Enforced budgets that went over in some run (flaky):\n{}", list(&flaky));
let rules = BaselineRules {
min_runs: budgets.settings.baseline_min_runs,
max_load_per_core: budgets.settings.max_load_per_core,
allow_contended: o.allow_contended,
};
let nb = baseline_from_runs(old, &runs, &rules)?;
write_baseline(root, &nb)?;
println!("updated {BASELINE} from {} runs (machine class {})", runs.len(), nb.get("machine_class").and_then(Value::as_str).unwrap_or("?"));
Ok(())
}
/// The committed baseline's full-mode numbers judged against the budgets (what the scorecard
/// shows: deterministic, no run needed). Scenarios without a baseline number are "not run".
pub fn outcomes_from_baseline(budgets: &Budgets, baseline: Option<&Value>) -> Vec<Outcome> {
let mode = baseline.and_then(|b| b.get("modes")?.get("full"));
let full: Option<BTreeMap<String, Stats>> =
mode.and_then(|m| m.get("scenarios")?.as_object().cloned()).map(|m| m.iter().map(|(k, v)| (k.clone(), Stats::from_json(v))).collect());
let failed = |id: &str| mode.and_then(|m| m.get("failed")?.get(id)?.as_str()).map(str::to_string);
budgets
.scenario
.iter()
.map(|sc| {
let stats = full.as_ref().and_then(|f| f.get(&sc.id)).cloned();
let status = match (&sc.not_measurable, &stats) {
(Some(w), _) => Status::NotMeasurable(w.clone()),
(None, Some(st)) => verdict(sc, st, None, &budgets.settings, budgets.settings.regression_pct, true),
(None, None) => failed(&sc.id).map_or(Status::NotRun, Status::Missing),
};
Outcome {
id: sc.id.clone(),
title: sc.title.clone(),
issue: sc.issue,
target: sc.target_text(),
stats,
baseline_p50: None,
change_pct: None,
status,
}
})
.collect()
}
pub fn fmt_load(v: Option<&Value>) -> String {
v.and_then(Value::as_array).map_or_else(|| "n/a".into(), |a| a.iter().filter_map(Value::as_f64).map(|x| format!("{x:.2}")).collect::<Vec<_>>().join(" / "))
}
/// The Markdown summary of one run: context lines and the scenario table.
pub fn render_summary(results: &Value, outcomes: &[Outcome], baseline_note: Option<&str>) -> String {
let m = results.get("machine").cloned().unwrap_or(Value::Null);
let s = |v: Option<&Value>| v.and_then(Value::as_str).unwrap_or("unknown").to_string();
let ram = m.get("ram_bytes").and_then(Value::as_u64).map_or_else(|| "?".into(), |b| format!("{} GB", b >> 30));
let mut out = format!(
"## Performance ({} run)\n\n- Machine: {} ({} cores, {ram}), GPU {}, {} {}; class `{}`\n- Commit {}{}, {}\n- Load average (1/5/15 min): {} at start, {} at end\n",
s(results.get("mode")),
s(m.get("cpu")),
m.get("cores").and_then(Value::as_u64).unwrap_or(0),
s(m.get("gpu_adapter")),
s(m.get("os_version")),
s(m.get("arch")),
s(m.get("class")),
s(results.get("commit")),
if results.get("dirty").and_then(Value::as_bool) == Some(true) { " (dirty)" } else { "" },
s(results.get("date")),
fmt_load(results.get("load_avg_start")),
fmt_load(results.get("load_avg_end")),
);
if results.get("contended").and_then(Value::as_bool) == Some(true) {
out.push_str("- **contended run**: numbers measure machine load (the 1-minute load at the start was above `max_load_per_core` per core)\n");
}
if let Some(n) = baseline_note {
out.push_str(&format!("- {n}\n"));
}
if results.get("mode").and_then(Value::as_str) == Some("quick") {
out.push_str("- Quick mode: small documents, budgets not applied (they are set for full-size documents).\n");
}
if let Some(b) = results.get("benches").and_then(Value::as_object) {
for (name, v) in b {
let st = v.get("status").and_then(Value::as_str).unwrap_or("?");
let secs = v.get("seconds").and_then(Value::as_f64).map_or_else(String::new, |x| format!(", {x:.0} s"));
let err = v.get("error").and_then(Value::as_str).map_or_else(String::new, |e| format!(": {e}"));
out.push_str(&format!("- Bench `{name}`: {st}{secs}, load {}{err}\n", fmt_load(v.get("load_avg_before"))));
}
}
out.push('\n');
out.push_str(&render_table(outcomes, true));
out
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn iso_utc_handles_boundaries() {
assert_eq!(iso_utc(0), "1970-01-01T00:00:00Z");
assert_eq!(iso_utc(1_791_446_357), "2026-10-08T07:59:17Z");
// Years past 9999 keep the plain zero-padded year (no `+`).
assert_eq!(iso_utc(253_402_300_800), "10000-01-01T00:00:00Z");
// A value past chrono's range still converts via the 400-year cycle (the old hand-rolled
// `iso_utc` did too): pin that output, never the epoch.
assert_eq!(iso_utc(10_000_000_000_000), "318857-05-20T17:46:40Z");
assert_eq!(iso_utc(u64::MAX), "584554051223-11-09T07:00:15Z");
}
const TOML: &str = r#"
[settings]
regression_pct = 15.0
regression_min_ms = 0.5
[[bench]]
name = "b"
package = "p"
quick_args = ["--quick"]
[[scenario]]
id = "P1"
title = "move"
issue = 209
bench = "b"
row = "move row"
budget_ms = 25
[[scenario]]
id = "P2"
title = "opacity"
bench = "b"
row = "opacity row"
budget_ms = 20
enforce = false
[[scenario]]
id = "P3"
title = "zoom"
target = "≤ 16 ms per frame"
not_measurable = "needs a frame harness"
"#;
fn report(rows: Value) -> Value {
json!({"bench": "b", "rows": rows})
}
#[test]
fn budgets_parse_and_validate() {
let b = parse_budgets(TOML).unwrap();
assert_eq!(b.scenario.len(), 3);
assert_eq!(b.scenario[0].metric, Metric::P95);
assert!(b.scenario[0].enforce);
assert!(!b.scenario[1].enforce);
assert_eq!(b.scenario[0].target_text(), "≤ 25 ms p95");
assert_eq!(b.scenario[2].target_text(), "≤ 16 ms per frame");
assert_eq!(b.settings.regression_min_ms, 0.5);
// Duplicate ids, undefined benches, half-specified rows and bad budgets are rejected.
assert!(parse_budgets(&format!("{TOML}\n[[scenario]]\nid = \"P1\"\ntitle = \"x\"\nnot_measurable = \"y\"\n")).is_err());
assert!(parse_budgets("[[scenario]]\nid = \"P1\"\ntitle = \"x\"\nbench = \"nope\"\nrow = \"r\"\n").is_err());
assert!(parse_budgets("[[bench]]\nname = \"b\"\npackage = \"p\"\n[[scenario]]\nid = \"P1\"\ntitle = \"x\"\nbench = \"b\"\n").is_err());
assert!(
parse_budgets("[[bench]]\nname = \"b\"\npackage = \"p\"\n[[scenario]]\nid = \"P1\"\ntitle = \"x\"\nbench = \"b\"\nrow = \"r\"\nbudget_ms = -1\n")
.is_err()
);
assert!(parse_budgets("[settings]\nregression_pct = 0\n").is_err());
assert!(parse_budgets("[settings]\nregression_min_ms = -1\n").is_err());
assert!(parse_budgets("[settings]\nregression_min_ms = 0\n").is_ok());
assert!(parse_budgets("bogus = 1\n").is_err());
assert!(parse_budgets("not toml [").is_err());
}
#[test]
fn row_stats_prefers_samples_and_never_invents_p95() {
let s = row_stats(&json!({"name": "x", "samples_ms": [5.0, 1.0, 3.0, 2.0, 4.0], "p50_ms": 99.0}));
assert_eq!((s.n, s.p50, s.p95, s.max), (5, Some(3.0), Some(5.0), Some(5.0)));
// Older rows: median + max only. p95 stays empty.
let s = row_stats(&json!({"name": "x", "median_ms": 4.0, "min_ms": 1.0, "max_ms": 9.0}));
assert_eq!((s.p50, s.p95, s.max), (Some(4.0), None, Some(9.0)));
let s = row_stats(&json!({"name": "x", "samples_ms": ["bad", null]}));
assert_eq!(s.p50, None);
}
#[test]
fn regression_detection() {
assert!(!regressed(11.4, 10.0, 15.0, 0.5));
assert!(regressed(11.6, 10.0, 15.0, 0.5));
// Below the absolute noise floor: no regression.
assert!(!regressed(0.30, 0.20, 15.0, 0.5));
assert!(!regressed(5.0, 0.0, 15.0, 0.5));
assert!(!regressed(f64::NAN, 1.0, 15.0, 0.0));
assert_eq!(change_pct(12.0, 10.0).map(|v| v.round()), Some(20.0));
assert_eq!(change_pct(1.0, 0.0), None);
}
#[test]
fn evaluate_budgets_regressions_and_missing_rows() {
let b = parse_budgets(TOML).unwrap();
let mut reports = BTreeMap::new();
reports.insert(
"b".to_string(),
Ok(report(json!([{"name": "move row", "samples_ms": [10.0, 12.0, 30.0]}, {"name": "opacity row", "samples_ms": [40.0]}]))),
);
let out = evaluate(&b, &reports, None, 15.0, true);
assert_eq!(out[0].status, Status::Broken, "p95 30 > 25 and enforced");
assert_eq!(out[1].status, Status::Over, "not enforced: reported, doesn't fail");
assert!(matches!(out[2].status, Status::NotMeasurable(_)));
assert!(out[0].status.fails() && !out[1].status.fails() && !out[2].status.fails());
// Quick mode: no budgets, only regressions.
let mut base = BTreeMap::new();
base.insert("P1".to_string(), Stats { p50: Some(10.0), ..Default::default() });
base.insert("P2".to_string(), Stats { p50: Some(30.0), ..Default::default() });
let out = evaluate(&b, &reports, Some(&base), 15.0, false);
assert_eq!(out[0].status, Status::Regressed, "p50 12 vs 10 is +20 %");
assert_eq!(out[1].status, Status::Regressed, "p50 40 vs 30");
let out = evaluate(&b, &reports, Some(&base), 50.0, false);
assert_eq!(out[0].status, Status::Tracked, "threshold is configurable");
// A row the bench didn't produce fails, with the bench's own error when it has one.
let mut reports = BTreeMap::new();
reports.insert("b".to_string(), Ok(json!({"rows": [], "context": {"errors": [{"name": "move row", "error": "boom"}]}})));
let out = evaluate(&b, &reports, None, 15.0, true);
assert_eq!(out[0].status, Status::Missing("boom".into()));
assert!(matches!(&out[1].status, Status::Missing(e) if e.contains("not in")));
// A bench that didn't run in this mode: not run, not a failure.
let out = evaluate(&b, &BTreeMap::new(), None, 15.0, true);
assert_eq!(out[0].status, Status::NotRun);
assert!(!out[0].status.fails());
assert_eq!(counts(&out), (0, 0, 0, 1), "not-run scenarios are not tracked; the scorecard counts them separately");
}
fn rules(min_runs: usize, allow_contended: bool) -> BaselineRules {
BaselineRules { min_runs, max_load_per_core: 1.0, allow_contended }
}
/// The plain `--update-baseline` case: one run.
fn one(old: Option<&Value>, results: &Value) -> Value {
baseline_from_runs(old, std::slice::from_ref(results), &rules(1, false)).unwrap()
}
/// A saved run of class `c` with one p50 (and p95) per scenario.
fn run_of(date: &str, load: f64, rows: &[(&str, f64)]) -> Value {
let sc: serde_json::Map<String, Value> =
rows.iter().map(|(k, v)| (k.to_string(), json!({"n": 5, "p50_ms": v, "p95_ms": v * 1.5, "max_ms": v * 2.0}))).collect();
json!({"mode": "full", "date": date, "commit": format!("c-{date}"), "dirty": false, "machine": {"class": "c", "cores": 3}, "load_avg_start": [load, load, load], "scenarios": sc})
}
#[test]
fn contention_is_judged_on_start_load_per_core() {
assert!(contended(Some([224.9, 226.31, 196.03]), Some(14), 1.0), "the committed 2026-10-05 baseline");
assert!(!contended(Some([0.5, 3.0, 9.0]), Some(12), 1.0), "only the 1-minute load counts");
assert!(!contended(None, Some(12), 1.0), "no load (Windows): can't judge");
assert!(!contended(Some([50.0, 0.0, 0.0]), None, 1.0));
assert!(!contended(Some([9.5, 0.0, 0.0]), Some(3), 4.0), "the threshold is configurable");
assert!(contended(Some([9.5, 0.0, 0.0]), Some(3), 3.0));
assert!(run_contended(Some(&json!([224.9, 226.31, 196.03])), Some(&json!({"cores": 14})), 1.0));
}
#[test]
fn update_baseline_refuses_contended_run_unless_allowed() {
let mut r = run_of("2026-10-05", 224.9, &[("P1", 3.0)]);
r["machine"]["cores"] = json!(14);
let e = baseline_from_runs(None, std::slice::from_ref(&r), &rules(1, false)).unwrap_err();
assert!(e.contains("224.90") && e.contains("14 cores") && e.contains("--allow-contended"), "{e}");
let b = baseline_from_runs(None, std::slice::from_ref(&r), &rules(1, true)).unwrap();
let m = &b["modes"]["full"];
assert_eq!(m["contended"], json!(true));
assert_eq!(m["cores"], json!(14));
assert_eq!(m["runs"], json!(1));
// A quiet run isn't labelled, and one run has no noise ceiling.
let b = one(None, &run_of("d", 0.5, &[("P1", 3.0)]));
assert!(b["modes"]["full"].get("contended").is_none());
assert!(b["modes"]["full"]["scenarios"]["P1"].get("p50_max_ms").is_none());
}
#[test]
fn baseline_from_runs_takes_median_and_noise_ceiling() {
let runs: Vec<Value> = [(1, 2174.0, 30.5), (2, 2868.0, 45.3), (3, 2000.0, 28.0), (4, 2500.0, 31.0), (5, 2300.0, 40.0)]
.iter()
.map(|(d, a, b)| run_of(&format!("2026-10-2{d}"), 1.0, &[("P44", *a), ("P46", *b)]))
.collect();
let b = baseline_from_runs(None, &runs, &rules(5, false)).unwrap();
let m = &b["modes"]["full"];
assert_eq!(m["runs"], json!(5));
assert_eq!(m["date"], json!("2026-10-25"), "the latest run dates the baseline");
assert_eq!(m["run_list"].as_array().map(Vec::len), Some(5));
assert_eq!(m["run_list"][0]["load_avg_start"], json!([1.0, 1.0, 1.0]));
assert_eq!(m["run_list"][0]["contended"], json!(false));
let got = baseline_for(&b, "c", "full").unwrap();
let p44 = &got["P44"];
assert_eq!((p44.p50, p44.p50_max, p44.n), (Some(2300.0), Some(2868.0), 25));
assert_eq!(got["P46"].p50, Some(31.0));
assert_eq!(median(&mut [4.0, 1.0, 3.0, 2.0]), Some(2.5));
}
#[test]
fn baseline_from_runs_keeps_a_row_failed_in_any_run() {
let mut runs: Vec<Value> = (1..=3).map(|d| run_of(&format!("2026-10-2{d}"), 1.0, &[("P1", 3.0), ("P2", 5.0)])).collect();
runs[1]["scenarios"]["P1"] = json!({"status": format!("{MISSING_PREFIX}bench crashed")});
let b = baseline_from_runs(None, &runs, &rules(3, false)).unwrap();
let m = &b["modes"]["full"];
assert!(m["scenarios"].get("P1").is_none(), "no partial aggregate: {m}");
assert_eq!(m["failed"]["P1"], json!("bench crashed"));
assert_eq!(m["scenarios"]["P2"]["p50_ms"], json!(5.0));
assert!(m["failed"].get("P2").is_none());
}
#[test]
fn baseline_from_runs_leaves_out_a_row_missing_from_some_runs() {
let mut runs: Vec<Value> = (1..=3).map(|d| run_of(&format!("2026-10-2{d}"), 1.0, &[("P1", 3.0), ("P2", 5.0)])).collect();
runs[0]["scenarios"].as_object_mut().unwrap().remove("P1");
let b = baseline_from_runs(None, &runs, &rules(3, false)).unwrap();
let m = &b["modes"]["full"];
assert!(m["scenarios"].get("P1").is_none(), "2 of 3 runs is no baseline: {m}");
assert_eq!(m["scenarios"]["P2"]["p50_ms"], json!(5.0));
assert_eq!(m["runs"], json!(3));
}
#[test]
fn baseline_from_runs_rejects_mixed_classes_modes_dirty_and_too_few() {
let runs: Vec<Value> = (1..=5).map(|d| run_of(&format!("2026-10-2{d}"), 1.0, &[("P1", 3.0)])).collect();
assert!(baseline_from_runs(None, &runs, &rules(5, false)).is_ok());
assert!(baseline_from_runs(None, &runs[..4], &rules(5, false)).unwrap_err().contains("at least 5"));
let mut m = runs.clone();
m[2]["machine"]["class"] = json!("other");
assert!(baseline_from_runs(None, &m, &rules(5, false)).unwrap_err().contains("machine classes"));
let mut m = runs.clone();
m[2]["mode"] = json!("quick");
assert!(baseline_from_runs(None, &m, &rules(5, false)).unwrap_err().contains("modes"));
let mut m = runs.clone();
m[2]["dirty"] = json!(true);
assert!(baseline_from_runs(None, &m, &rules(5, false)).unwrap_err().contains("dirty"));
let mut m = runs.clone();
m[2]["load_avg_start"] = json!([9.5, 9.0, 9.0]);
assert!(baseline_from_runs(None, &m, &rules(5, false)).unwrap_err().contains("contended"));
assert_eq!(baseline_from_runs(None, &m, &rules(5, true)).unwrap()["modes"]["full"]["run_list"][2]["contended"], json!(true));
}
#[test]
fn regression_is_judged_against_the_noise_ceiling() {
// Runner numbers (macos-15, 7 nightlies): median and highest p50 per row.
let b = parse_budgets(TOML).unwrap();
let s = &b.settings;
let sc = &b.scenario[1]; // no enforced budget in play: opacity, budget 20, not enforced
let base = |p50: f64, max: f64| Stats { p50: Some(p50), p50_max: Some(max), ..Default::default() };
let cur = |p50: f64| Stats { p50: Some(p50), ..Default::default() };
assert_eq!(verdict(sc, &cur(3935.0), Some(&base(2174.0, 2868.0)), s, 15.0, false), Status::Regressed, "P44 +81 % (#5694)");
assert_eq!(verdict(sc, &cur(76.9), Some(&base(30.5, 45.3)), s, 15.0, false), Status::Regressed, "P46 +152 % (#5697)");
assert_eq!(verdict(sc, &cur(387.0), Some(&base(258.0, 448.0)), s, 15.0, false), Status::Tracked, "P11 +50 % (#5695) is within runner noise");
// A single-run baseline falls back to its p50.
let single = Stats { p50: Some(258.0), ..Default::default() };
assert_eq!(verdict(sc, &cur(387.0), Some(&single), s, 15.0, false), Status::Regressed);
}
#[test]
fn enforce_report_needs_margin_in_every_run() {
let toml = r#"
[[bench]]
name = "b"
package = "p"
[[scenario]]
id = "P16"
title = "dab"
bench = "b"
row = "r16"
budget_ms = 100
enforce = false
[[scenario]]
id = "P29"
title = "wand"
bench = "b"
row = "r29"
budget_ms = 100
enforce = false
[[scenario]]
id = "P30"
title = "liquify"
bench = "b"
row = "r30"
budget_ms = 100
enforce = false
[[scenario]]
id = "P25"
title = "cancel"
bench = "b"
row = "r25"
budget_ms = 200
[[scenario]]
id = "P99"
title = "track"
bench = "b"
row = "r99"
"#;
let b = parse_budgets(toml).unwrap();
// run_of stores p95 = 1.5 × p50 (the default metric is p95).
let runs: Vec<Value> = [(30.0, 20.0, 30.0, 100.0), (35.2, 22.0, 45.0, 110.0), (20.0, 22.0, 30.0, 199.4)]
.iter()
.enumerate()
.map(|(i, (a, c, d, e))| run_of(&format!("d{i}"), 1.0, &[("P16", *a), ("P29", *c), ("P30", *d), ("P25", *e), ("P99", 1.0)]))
.collect();
let (could, flaky) = enforce_report(&b, &runs);
assert_eq!(could.len(), 2, "{could:?}");
assert!(could[0].starts_with("P16 ") && could[0].contains("52.8"), "{could:?}");
assert!(could[1].starts_with("P29 "));
assert!(!could.iter().any(|l| l.starts_with("P30")), "67.5 > 60 once: not suggested");
assert_eq!(flaky.len(), 1, "{flaky:?}");
assert!(flaky[0].starts_with("P25 ") && flaky[0].contains("in 1 of 3 runs"), "{flaky:?}");
// A row missing from one run can't be vouched for.
let mut gap = runs.clone();
if let Some(o) = gap[1]["scenarios"].as_object_mut() {
o.remove("P29");
}
assert!(!enforce_report(&b, &gap).0.iter().any(|l| l.starts_with("P29")));
}
#[test]
fn baselines_never_cross_machine_classes() {
let results = json!({"mode": "full", "date": "d", "commit": "c", "machine": {"class": "macos-aarch64-m4"}, "scenarios": {"P1": {"p50_ms": 3.0, "p95_ms": 4.0, "max_ms": 5.0, "n": 3}, "P3": {"status": "not measurable yet"}}});
let b = one(None, &results);
let got = baseline_for(&b, "macos-aarch64-m4", "full").unwrap();
assert_eq!(got.get("P1").and_then(|s| s.p50), Some(3.0));
assert!(!got.contains_key("P3"), "rows without numbers are not baselined");
assert!(baseline_for(&b, "linux-x86-64-epyc", "full").is_err());
assert!(baseline_for(&b, "macos-aarch64-m4", "quick").is_err());
// Same class: the other mode is kept. New class: replaced.
let quick = json!({"mode": "quick", "machine": {"class": "macos-aarch64-m4"}, "scenarios": {"P1": {"p50_ms": 1.0}}});
let b2 = one(Some(&b), &quick);
assert!(baseline_for(&b2, "macos-aarch64-m4", "full").is_ok());
assert!(baseline_for(&b2, "macos-aarch64-m4", "quick").is_ok());
// A crashed row is kept as a failure, and the scorecard shows it.
let crashed = json!({"mode": "full", "machine": {"class": "c"}, "scenarios": {"P1": {"status": "**missing**: panicked: boom"}}});
let bc = one(None, &crashed);
let b = parse_budgets(TOML).unwrap();
let o = outcomes_from_baseline(&b, Some(&bc));
assert_eq!(o[0].status, Status::Missing("panicked: boom".into()));
assert_eq!(o[1].status, Status::NotRun);
let other = json!({"mode": "quick", "machine": {"class": "other"}, "scenarios": {}});
let b3 = one(Some(&b2), &other);
assert!(baseline_for(&b3, "other", "full").is_err());
}
#[test]
fn machine_helpers() {
assert_eq!(slug("Apple M4 Pro (Metal)"), "apple-m4-pro-metal");
assert_eq!(machine_class("macos", "aarch64", "Apple M4 Pro", "Apple M4 Pro"), "macos-aarch64-apple-m4-pro-apple-m4-pro");
assert_eq!(parse_loadavg("{ 2.10 1.50 0.75 }"), Some([2.1, 1.5, 0.75]));
assert_eq!(parse_loadavg("0.52 0.58 0.59 1/467 12345\n"), Some([0.52, 0.58, 0.59]));
assert_eq!(parse_loadavg("garbage"), None);
assert_eq!(iso_utc(0), "1970-01-01T00:00:00Z");
assert_eq!(iso_utc(1_791_158_400), "2026-10-05T00:00:00Z");
assert_eq!(iso_utc(951_782_400), "2000-02-29T00:00:00Z");
}
#[test]
fn table_lists_not_measurable_rows_separately() {
let b = parse_budgets(TOML).unwrap();
let mut reports = BTreeMap::new();
reports.insert("b".to_string(), Ok(report(json!([{"name": "move row", "samples_ms": [10.0], "peak_rss_bytes": 3u64 << 30}]))));
let t = render_table(&evaluate(&b, &reports, None, 15.0, true), true);
assert!(t.contains("| P1 | move | ≤ 25 ms p95 | 10.0 | 10.0 | 10.0 | 3.0 GB |"));
assert!(t.contains("**Not measurable yet**"));
assert!(t.contains("| P3 | zoom | ≤ 16 ms per frame | needs a frame harness | |"));
assert_eq!(counts(&evaluate(&b, &reports, None, 15.0, true)), (1, 1, 0, 1), "the row the bench lacks counts as failing");
}
#[test]
fn unknown_bench_is_rejected_before_updating_the_baseline() {
let root = std::env::temp_dir().join(format!("photocraft-perf-unknown-bench-{}", std::process::id()));
std::fs::create_dir(&root).unwrap();
std::fs::create_dir(root.join("perf")).unwrap();
std::fs::write(root.join(BUDGETS), "[[bench]]\nname = \"known\"\npackage = \"unused\"\nrequires = \"missing.rs\"\n").unwrap();
let baseline = "{\"schema\":1,\"sentinel\":true}";
std::fs::write(root.join(BASELINE), baseline).unwrap();
let results = [
run(&root, &["--bench", "unknown", "--reuse", "--update-baseline"]),
run(&root, &["--bench", "known", "--bench", "unknown", "--quick", "--reuse", "--update-baseline"]),
];
let after = std::fs::read_to_string(root.join(BASELINE)).unwrap();
std::fs::remove_dir_all(root).unwrap();
for result in results {
assert_eq!(result, Err(format!("perf: unknown bench `unknown` (see {BUDGETS})")));
}
assert_eq!(after, baseline, "an unknown bench must not update the baseline");
}
#[test]
fn options() {
assert!(parse_opts(&["--bogus"]).is_err());
assert!(parse_opts(&["--threshold", "x"]).is_err());
assert!(parse_opts(&["--threshold", "-3"]).is_err());
let o = parse_opts(&["--quick", "--threshold", "20", "--bench", "a", "--reuse"]).unwrap();
assert!(o.quick && o.reuse && o.skip_build);
assert_eq!(o.threshold, Some(20.0));
assert_eq!(o.benches, ["a"]);
assert!(!o.allow_contended && o.from.is_empty());
let o = parse_opts(&["--update-baseline", "--from", "a.json", "--from", "b.json", "--allow-contended"]).unwrap();
assert!(o.update_baseline && o.allow_contended);
assert_eq!(o.from, [PathBuf::from("a.json"), PathBuf::from("b.json")]);
assert!(parse_opts(&["--from", "a.json"]).is_err(), "--from needs --update-baseline");
assert!(parse_opts(&["--update-baseline", "--from"]).is_err());
}
}