mirror of
https://github.com/storytold/photocraft.git
synced 2026-11-21 01:18:43 +00:00
Port behavior and regression coverage into current modules; preserve current upstream behavior. Validate benchmark names before both measurement and baseline-from-files paths, retaining newer baseline modes.
1601 lines
71 KiB
Rust
1601 lines
71 KiB
Rust
//! `cargo xtask perf`: run the release benches, merge their `--json` reports into one result keyed
|
||
//! by scenario id, and check it against `perf/budgets.toml` and `perf/baseline.json` (#221).
|
||
//!
|
||
//! ```text
|
||
//! cargo xtask perf [--quick] [--update-baseline [--allow-contended]] [--threshold PCT] [--bench NAME]... [--skip-build] [--reuse]
|
||
//! cargo xtask perf --update-baseline --from results.json [--from results.json]... [--allow-contended]
|
||
//! ```
|
||
//!
|
||
//! - `--quick`: only the benches with `quick_args` (small synthetic documents; minutes, not tens
|
||
//! of minutes). Quick numbers are compared only with a quick baseline, and budgets don't apply
|
||
//! (they are set for the full-size documents).
|
||
//! - `--update-baseline`: write this run into `perf/baseline.json` (the only way it changes).
|
||
//! A contended run (1-minute load at the start above `settings.max_load_per_core` per core) is
|
||
//! refused; `--allow-contended` records it, labelled `"contended": true`.
|
||
//! - `--from FILE`: with `--update-baseline`, build the baseline from saved `results.json` files
|
||
//! (at least `settings.baseline_min_runs`, one machine class and mode, clean trees) instead of
|
||
//! running: per scenario the median p50 and the highest (the noise ceiling regressions are
|
||
//! judged against). Prints which budgets could be enforced and which enforced ones are flaky.
|
||
//! - `--threshold PCT`: the regression threshold (default `settings.regression_pct`, 15 %).
|
||
//! - `--bench NAME`: run only these benches (repeatable).
|
||
//! - `--skip-build`: don't run `cargo build` first. `--reuse`: don't run the benches either;
|
||
//! re-evaluate the reports already in `target/perf/`.
|
||
//!
|
||
//! Output: `target/perf/results.json` (everything), `target/perf/summary.md` (the table the
|
||
//! nightly job posts as its summary) and one `target/perf/<bench>.json` per bench.
|
||
//!
|
||
//! Exit status: non-zero when an enforced budget breaks (full mode), a scenario's p50 regresses
|
||
//! more than the threshold against a baseline from the same machine class and mode (against its
|
||
//! noise ceiling when it was built from several runs), or a bench
|
||
//! fails. Runs from different machine classes are never compared.
|
||
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
|
||
|
||
use std::collections::BTreeMap;
|
||
use std::path::{Path, PathBuf};
|
||
use std::process::Command;
|
||
use std::time::Instant;
|
||
|
||
use serde::Deserialize;
|
||
use serde_json::{Value, json};
|
||
|
||
pub const BUDGETS: &str = "perf/budgets.toml";
|
||
pub const BASELINE: &str = "perf/baseline.json";
|
||
const MISSING_PREFIX: &str = "**missing**: ";
|
||
|
||
// ---- budgets.toml ----------------------------------------------------------------------------
|
||
|
||
#[derive(Debug, Deserialize)]
|
||
#[serde(deny_unknown_fields)]
|
||
pub struct Budgets {
|
||
#[serde(default)]
|
||
pub settings: Settings,
|
||
#[serde(default)]
|
||
pub bench: Vec<BenchSpec>,
|
||
#[serde(default)]
|
||
pub scenario: Vec<Scenario>,
|
||
}
|
||
|
||
#[derive(Debug, Deserialize)]
|
||
#[serde(default, deny_unknown_fields)]
|
||
pub struct Settings {
|
||
/// A scenario regresses when its p50 grows by more than this share of the baseline…
|
||
pub regression_pct: f64,
|
||
/// …and by more than this many milliseconds (timer noise on sub-millisecond rows).
|
||
pub regression_min_ms: f64,
|
||
/// A run is contended when its 1-minute load at the start exceeds this many per core: its
|
||
/// numbers measure the machine's load, so `--update-baseline` refuses it.
|
||
pub max_load_per_core: f64,
|
||
/// `--update-baseline --from` needs at least this many saved runs.
|
||
pub baseline_min_runs: usize,
|
||
/// A budget could be enforced when its metric stays within this share of the budget in every run.
|
||
pub enforce_margin: f64,
|
||
}
|
||
|
||
impl Default for Settings {
|
||
fn default() -> Self {
|
||
Self { regression_pct: 15.0, regression_min_ms: 1.0, max_load_per_core: 1.0, baseline_min_runs: 5, enforce_margin: 0.6 }
|
||
}
|
||
}
|
||
|
||
#[derive(Debug, Deserialize)]
|
||
#[serde(deny_unknown_fields)]
|
||
pub struct BenchSpec {
|
||
pub name: String,
|
||
pub package: String,
|
||
/// Example name; defaults to `name`.
|
||
pub example: Option<String>,
|
||
/// Arguments of a full run (`--json <file>` is appended).
|
||
#[serde(default)]
|
||
pub args: Vec<String>,
|
||
/// Arguments of a `--quick` run; absent = the bench is skipped in quick mode.
|
||
pub quick_args: Option<Vec<String>>,
|
||
/// Source file that must exist for the bench to run (a bench that lives on another branch):
|
||
/// skipped, not failed, while it is missing.
|
||
pub requires: Option<String>,
|
||
#[serde(default)]
|
||
pub note: String,
|
||
}
|
||
|
||
impl BenchSpec {
|
||
pub fn example(&self) -> &str {
|
||
self.example.as_deref().unwrap_or(&self.name)
|
||
}
|
||
}
|
||
|
||
#[derive(Clone, Copy, Debug, Default, Deserialize, PartialEq, Eq)]
|
||
#[serde(rename_all = "lowercase")]
|
||
pub enum Metric {
|
||
P50,
|
||
#[default]
|
||
P95,
|
||
Max,
|
||
}
|
||
|
||
impl Metric {
|
||
fn label(self) -> &'static str {
|
||
match self {
|
||
Metric::P50 => "p50",
|
||
Metric::P95 => "p95",
|
||
Metric::Max => "max",
|
||
}
|
||
}
|
||
}
|
||
|
||
#[derive(Debug, Deserialize)]
|
||
#[serde(deny_unknown_fields)]
|
||
pub struct Scenario {
|
||
pub id: String,
|
||
pub title: String,
|
||
pub issue: Option<u32>,
|
||
/// Bench and row name that measure it; both absent when `not_measurable` is set.
|
||
pub bench: Option<String>,
|
||
pub row: Option<String>,
|
||
#[serde(default)]
|
||
pub metric: Metric,
|
||
pub budget_ms: Option<f64>,
|
||
/// Free-form target when it isn't a plain time budget.
|
||
pub target: Option<String>,
|
||
/// A broken budget fails the run. Off for targets the code doesn't meet yet: they are
|
||
/// reported ("over budget") but only regressions fail.
|
||
#[serde(default = "yes")]
|
||
pub enforce: bool,
|
||
/// Why this can't be measured yet (the code doesn't support it).
|
||
pub not_measurable: Option<String>,
|
||
}
|
||
|
||
fn yes() -> bool {
|
||
true
|
||
}
|
||
|
||
impl Scenario {
|
||
pub fn target_text(&self) -> String {
|
||
if let Some(t) = &self.target {
|
||
return t.clone();
|
||
}
|
||
match self.budget_ms {
|
||
Some(b) if b.fract() == 0.0 => format!("≤ {b:.0} ms {}", self.metric.label()),
|
||
Some(b) => format!("≤ {b} ms {}", self.metric.label()),
|
||
None => "track only".into(),
|
||
}
|
||
}
|
||
}
|
||
|
||
pub fn parse_budgets(text: &str) -> Result<Budgets, String> {
|
||
let b: Budgets = toml::from_str(text).map_err(|e| format!("{BUDGETS}: {e}"))?;
|
||
let mut ids = std::collections::BTreeSet::new();
|
||
for s in &b.scenario {
|
||
if !ids.insert(s.id.as_str()) {
|
||
return Err(format!("{BUDGETS}: duplicate scenario id {}", s.id));
|
||
}
|
||
match (&s.bench, &s.row, &s.not_measurable) {
|
||
(Some(bench), Some(_), None) => {
|
||
if !b.bench.iter().any(|x| &x.name == bench) {
|
||
return Err(format!("{BUDGETS}: scenario {} uses undefined bench `{bench}`", s.id));
|
||
}
|
||
}
|
||
(None, None, Some(_)) => {}
|
||
_ => return Err(format!("{BUDGETS}: scenario {} needs either bench + row, or not_measurable", s.id)),
|
||
}
|
||
if s.budget_ms.is_some_and(|v| !(v.is_finite() && v > 0.0)) {
|
||
return Err(format!("{BUDGETS}: scenario {} has an invalid budget_ms", s.id));
|
||
}
|
||
}
|
||
if !(b.settings.regression_pct.is_finite() && b.settings.regression_pct > 0.0) {
|
||
return Err(format!("{BUDGETS}: settings.regression_pct must be > 0"));
|
||
}
|
||
if !(b.settings.regression_min_ms.is_finite() && b.settings.regression_min_ms >= 0.0) {
|
||
return Err(format!("{BUDGETS}: settings.regression_min_ms must be finite and >= 0"));
|
||
}
|
||
if !(b.settings.max_load_per_core.is_finite() && b.settings.max_load_per_core > 0.0) {
|
||
return Err(format!("{BUDGETS}: settings.max_load_per_core must be > 0"));
|
||
}
|
||
if b.settings.baseline_min_runs == 0 {
|
||
return Err(format!("{BUDGETS}: settings.baseline_min_runs must be >= 1"));
|
||
}
|
||
if !(b.settings.enforce_margin.is_finite() && b.settings.enforce_margin > 0.0 && b.settings.enforce_margin <= 1.0) {
|
||
return Err(format!("{BUDGETS}: settings.enforce_margin must be in (0, 1]"));
|
||
}
|
||
Ok(b)
|
||
}
|
||
|
||
// ---- measurements ----------------------------------------------------------------------------
|
||
|
||
/// One scenario's numbers.
|
||
#[derive(Clone, Debug, Default, PartialEq)]
|
||
pub struct Stats {
|
||
pub n: usize,
|
||
pub p50: Option<f64>,
|
||
pub p95: Option<f64>,
|
||
pub max: Option<f64>,
|
||
pub peak_rss: Option<u64>,
|
||
pub gpu_bytes: Option<u64>,
|
||
/// A baseline built from several runs: the highest p50 among them (its noise ceiling).
|
||
pub p50_max: Option<f64>,
|
||
}
|
||
|
||
impl Stats {
|
||
pub fn metric(&self, m: Metric) -> Option<f64> {
|
||
match m {
|
||
Metric::P50 => self.p50,
|
||
Metric::P95 => self.p95,
|
||
Metric::Max => self.max,
|
||
}
|
||
}
|
||
|
||
pub fn to_json(&self) -> Value {
|
||
let mut v =
|
||
json!({"n": self.n, "p50_ms": self.p50, "p95_ms": self.p95, "max_ms": self.max, "peak_rss_bytes": self.peak_rss, "gpu_bytes": self.gpu_bytes});
|
||
if let (Some(m), Some(o)) = (self.p50_max, v.as_object_mut()) {
|
||
o.insert("p50_max_ms".into(), json!(m));
|
||
}
|
||
v
|
||
}
|
||
|
||
pub fn from_json(v: &Value) -> Self {
|
||
Self {
|
||
n: v.get("n").and_then(Value::as_u64).unwrap_or(0) as usize,
|
||
p50: v.get("p50_ms").and_then(Value::as_f64),
|
||
p95: v.get("p95_ms").and_then(Value::as_f64),
|
||
max: v.get("max_ms").and_then(Value::as_f64),
|
||
peak_rss: v.get("peak_rss_bytes").and_then(Value::as_u64),
|
||
gpu_bytes: v.get("gpu_bytes").and_then(Value::as_u64),
|
||
p50_max: v.get("p50_max_ms").and_then(Value::as_f64),
|
||
}
|
||
}
|
||
}
|
||
|
||
/// Nearest-rank percentile of sorted, non-empty `v`.
|
||
fn rank(v: &[f64], q: f64) -> Option<f64> {
|
||
let n = v.len();
|
||
if n == 0 {
|
||
return None;
|
||
}
|
||
let k = ((q * n as f64).ceil() as usize).clamp(1, n);
|
||
v.get(k - 1).copied()
|
||
}
|
||
|
||
/// A bench row → stats. Rows carry `samples_ms` (preferred: percentiles are recomputed here),
|
||
/// or `p50_ms`/`p95_ms`/`max_ms`, or the older `median_ms`/`max_ms`. A p95 is never invented:
|
||
/// with neither samples nor a reported p95 it stays empty.
|
||
pub fn row_stats(row: &Value) -> Stats {
|
||
let mut samples: Vec<f64> =
|
||
row.get("samples_ms").and_then(Value::as_array).map(|a| a.iter().filter_map(Value::as_f64).filter(|x| x.is_finite()).collect()).unwrap_or_default();
|
||
samples.sort_by(f64::total_cmp);
|
||
let f = |k: &str| row.get(k).and_then(Value::as_f64);
|
||
let mut s = Stats {
|
||
n: samples.len(),
|
||
p50: f("p50_ms").or_else(|| f("median_ms")),
|
||
p95: f("p95_ms"),
|
||
max: f("max_ms"),
|
||
peak_rss: row.get("peak_rss_bytes").and_then(Value::as_u64),
|
||
gpu_bytes: row.get("gpu_bytes").and_then(Value::as_u64),
|
||
p50_max: None,
|
||
};
|
||
if !samples.is_empty() {
|
||
s.p50 = rank(&samples, 0.5);
|
||
s.p95 = rank(&samples, 0.95);
|
||
s.max = samples.last().copied();
|
||
} else if let Some(n) = row.get("n").and_then(Value::as_u64) {
|
||
s.n = n as usize;
|
||
}
|
||
s
|
||
}
|
||
|
||
/// The row named `name` in a bench report (`{"rows": [{"name": …}, …]}`).
|
||
pub fn find_row<'a>(report: &'a Value, name: &str) -> Option<&'a Value> {
|
||
report.get("rows")?.as_array()?.iter().find(|r| r.get("name").and_then(Value::as_str) == Some(name))
|
||
}
|
||
|
||
/// `cur` regressed against `base` by more than `pct` % and more than `min_ms`.
|
||
pub fn regressed(cur: f64, base: f64, pct: f64, min_ms: f64) -> bool {
|
||
base > 0.0 && cur.is_finite() && cur > base * (1.0 + pct / 100.0) && cur - base > min_ms
|
||
}
|
||
|
||
pub fn change_pct(cur: f64, base: f64) -> Option<f64> {
|
||
(base > 0.0 && cur.is_finite()).then(|| (cur / base - 1.0) * 100.0)
|
||
}
|
||
|
||
#[derive(Clone, Debug, PartialEq)]
|
||
pub enum Status {
|
||
Pass,
|
||
/// Over budget, not enforced (target not met yet).
|
||
Over,
|
||
/// Over an enforced budget.
|
||
Broken,
|
||
Regressed,
|
||
/// Measured, no budget, no regression.
|
||
Tracked,
|
||
/// The bench ran but the row is missing or errored.
|
||
Missing(String),
|
||
NotRun,
|
||
NotMeasurable(String),
|
||
}
|
||
|
||
impl Status {
|
||
pub fn label(&self) -> String {
|
||
match self {
|
||
Status::Pass => "pass".into(),
|
||
Status::Over => "over budget (target not met yet)".into(),
|
||
Status::Broken => "**BUDGET BROKEN**".into(),
|
||
Status::Regressed => "**REGRESSED**".into(),
|
||
Status::Tracked => "tracked".into(),
|
||
Status::Missing(e) => format!("{MISSING_PREFIX}{e}"),
|
||
Status::NotRun => "not run".into(),
|
||
Status::NotMeasurable(_) => "not measurable yet".into(),
|
||
}
|
||
}
|
||
pub fn fails(&self) -> bool {
|
||
matches!(self, Status::Broken | Status::Regressed | Status::Missing(_))
|
||
}
|
||
}
|
||
|
||
/// One evaluated scenario.
|
||
#[derive(Clone, Debug)]
|
||
pub struct Outcome {
|
||
pub id: String,
|
||
pub title: String,
|
||
pub issue: Option<u32>,
|
||
pub target: String,
|
||
pub stats: Option<Stats>,
|
||
pub baseline_p50: Option<f64>,
|
||
pub change_pct: Option<f64>,
|
||
pub status: Status,
|
||
}
|
||
|
||
/// Budget and regression verdict for one scenario. `budgets_apply` is false in quick mode.
|
||
/// A regression is judged against the baseline's noise ceiling (the highest p50 of the runs it was
|
||
/// built from), or its p50 when it came from one run.
|
||
pub fn verdict(sc: &Scenario, stats: &Stats, base: Option<&Stats>, settings: &Settings, threshold: f64, budgets_apply: bool) -> Status {
|
||
if let (Some(cur), Some(b)) = (stats.p50, base.and_then(|b| b.p50_max.or(b.p50)))
|
||
&& regressed(cur, b, threshold, settings.regression_min_ms)
|
||
{
|
||
return Status::Regressed;
|
||
}
|
||
match (budgets_apply, sc.budget_ms, stats.metric(sc.metric)) {
|
||
(true, Some(budget), Some(v)) if v > budget => {
|
||
if sc.enforce {
|
||
Status::Broken
|
||
} else {
|
||
Status::Over
|
||
}
|
||
}
|
||
(true, Some(_), Some(_)) => Status::Pass,
|
||
(true, Some(_), None) => Status::Missing(format!("no {} in the bench row", sc.metric.label())),
|
||
_ => Status::Tracked,
|
||
}
|
||
}
|
||
|
||
/// Evaluates every scenario against the bench reports (`bench name → report`, `None` when the
|
||
/// bench didn't run in this mode) and the baseline's scenarios for this mode.
|
||
pub fn evaluate(
|
||
budgets: &Budgets,
|
||
reports: &BTreeMap<String, Result<Value, String>>,
|
||
baseline: Option<&BTreeMap<String, Stats>>,
|
||
threshold: f64,
|
||
budgets_apply: bool,
|
||
) -> Vec<Outcome> {
|
||
budgets
|
||
.scenario
|
||
.iter()
|
||
.map(|sc| {
|
||
let base = baseline.and_then(|b| b.get(&sc.id));
|
||
let mut o = Outcome {
|
||
id: sc.id.clone(),
|
||
title: sc.title.clone(),
|
||
issue: sc.issue,
|
||
target: sc.target_text(),
|
||
stats: None,
|
||
baseline_p50: base.and_then(|b| b.p50),
|
||
change_pct: None,
|
||
status: Status::NotRun,
|
||
};
|
||
if let Some(why) = &sc.not_measurable {
|
||
o.status = Status::NotMeasurable(why.clone());
|
||
return o;
|
||
}
|
||
let (Some(bench), Some(row)) = (&sc.bench, &sc.row) else { return o };
|
||
match reports.get(bench) {
|
||
None => {}
|
||
Some(Err(e)) => o.status = Status::Missing(format!("bench `{bench}` failed: {e}")),
|
||
Some(Ok(rep)) => match find_row(rep, row) {
|
||
None => {
|
||
let err = rep
|
||
.get("context")
|
||
.and_then(|c| c.get("errors"))
|
||
.and_then(Value::as_array)
|
||
.and_then(|a| a.iter().find(|e| e.get("name").and_then(Value::as_str) == Some(row.as_str())))
|
||
.and_then(|e| e.get("error"))
|
||
.and_then(Value::as_str)
|
||
.map_or_else(|| format!("row `{row}` not in `{bench}`"), str::to_string);
|
||
o.status = Status::Missing(err);
|
||
}
|
||
Some(r) => {
|
||
let st = row_stats(r);
|
||
o.change_pct = st.p50.zip(o.baseline_p50).and_then(|(c, b)| change_pct(c, b));
|
||
o.status = verdict(sc, &st, base, &budgets.settings, threshold, budgets_apply);
|
||
o.stats = Some(st);
|
||
}
|
||
},
|
||
}
|
||
o
|
||
})
|
||
.collect()
|
||
}
|
||
|
||
// ---- formatting ------------------------------------------------------------------------------
|
||
|
||
pub fn fmt_num(v: f64) -> String {
|
||
if v >= 100.0 {
|
||
format!("{v:.0}")
|
||
} else if v >= 10.0 {
|
||
format!("{v:.1}")
|
||
} else {
|
||
format!("{v:.2}")
|
||
}
|
||
}
|
||
|
||
fn fmt_ms(v: Option<f64>) -> String {
|
||
v.map_or_else(|| "–".into(), fmt_num)
|
||
}
|
||
|
||
pub fn fmt_bytes(v: Option<u64>) -> String {
|
||
match v {
|
||
None | Some(0) => "–".into(),
|
||
Some(b) if b >= 1 << 30 => format!("{:.1} GB", b as f64 / (1u64 << 30) as f64),
|
||
Some(b) => format!("{} MB", b >> 20),
|
||
}
|
||
}
|
||
|
||
pub fn issue_link(n: Option<u32>) -> String {
|
||
n.map_or_else(String::new, |n| format!("[#{n}](https://github.com/storytold/photocraft/issues/{n})"))
|
||
}
|
||
|
||
/// The scenario table (Markdown); `with_baseline` adds the change against the baseline.
|
||
/// Not-measurable rows go to a separate list after it.
|
||
pub fn render_table(outcomes: &[Outcome], with_baseline: bool) -> String {
|
||
let mut s = if with_baseline {
|
||
String::from(
|
||
"| Id | Scenario | Target | p50 ms | p95 ms | max ms | Peak RSS | GPU | vs baseline | Status | Issue |\n|---|---|---|---:|---:|---:|---:|---:|---:|---|---|\n",
|
||
)
|
||
} else {
|
||
String::from(
|
||
"| Id | Scenario | Target | p50 ms | p95 ms | max ms | Peak RSS | GPU | Status | Issue |\n|---|---|---|---:|---:|---:|---:|---:|---|---|\n",
|
||
)
|
||
};
|
||
for o in outcomes.iter().filter(|o| !matches!(o.status, Status::NotMeasurable(_))) {
|
||
let st = o.stats.clone().unwrap_or_default();
|
||
let vs = if with_baseline { format!(" {} |", o.change_pct.map_or_else(|| "–".into(), |c| format!("{c:+.0} %"))) } else { String::new() };
|
||
s.push_str(&format!(
|
||
"| {} | {} | {} | {} | {} | {} | {} | {} |{} {} | {} |\n",
|
||
o.id,
|
||
o.title,
|
||
o.target,
|
||
fmt_ms(st.p50),
|
||
fmt_ms(st.p95),
|
||
fmt_ms(st.max),
|
||
fmt_bytes(st.peak_rss),
|
||
fmt_bytes(st.gpu_bytes),
|
||
vs,
|
||
o.status.label(),
|
||
issue_link(o.issue),
|
||
));
|
||
}
|
||
let nm: Vec<&Outcome> = outcomes.iter().filter(|o| matches!(o.status, Status::NotMeasurable(_))).collect();
|
||
if !nm.is_empty() {
|
||
s.push_str(
|
||
"\n**Not measurable yet** (the code can't do it yet; no number is recorded):\n\n| Id | Scenario | Target | Why | Issue |\n|---|---|---|---|---|\n",
|
||
);
|
||
for o in nm {
|
||
let why = match &o.status {
|
||
Status::NotMeasurable(w) => w.as_str(),
|
||
_ => "",
|
||
};
|
||
s.push_str(&format!("| {} | {} | {} | {} | {} |\n", o.id, o.title, o.target, why, issue_link(o.issue)));
|
||
}
|
||
}
|
||
s
|
||
}
|
||
|
||
/// Counts for the scorecard summary: (within budget, over budget or failing, tracked only, not measurable).
|
||
/// Not-run scenarios are in none of them; the scorecard counts those separately.
|
||
pub fn counts(outcomes: &[Outcome]) -> (usize, usize, usize, usize) {
|
||
let mut c = (0, 0, 0, 0);
|
||
for o in outcomes {
|
||
match o.status {
|
||
Status::Pass => c.0 += 1,
|
||
Status::Over | Status::Broken | Status::Regressed | Status::Missing(_) => c.1 += 1,
|
||
Status::Tracked => c.2 += 1,
|
||
Status::NotMeasurable(_) => c.3 += 1,
|
||
Status::NotRun => {}
|
||
}
|
||
}
|
||
c
|
||
}
|
||
|
||
// ---- machine ---------------------------------------------------------------------------------
|
||
|
||
/// Lower-case, `-`-separated, alphanumeric only.
|
||
pub fn slug(s: &str) -> String {
|
||
let mut out = String::new();
|
||
for ch in s.chars() {
|
||
if ch.is_ascii_alphanumeric() {
|
||
out.push(ch.to_ascii_lowercase());
|
||
} else if !out.ends_with('-') && !out.is_empty() {
|
||
out.push('-');
|
||
}
|
||
}
|
||
out.trim_end_matches('-').to_string()
|
||
}
|
||
|
||
/// Runs are compared only within one class: OS, architecture, CPU model and GPU adapter.
|
||
pub fn machine_class(os: &str, arch: &str, cpu: &str, gpu: &str) -> String {
|
||
[os, arch, cpu, gpu].iter().map(|p| slug(p)).filter(|p| !p.is_empty()).collect::<Vec<_>>().join("-")
|
||
}
|
||
|
||
/// The first three numbers in `/proc/loadavg` or `sysctl -n vm.loadavg` output.
|
||
pub fn parse_loadavg(s: &str) -> Option<[f64; 3]> {
|
||
let v: Vec<f64> = s.split_whitespace().filter_map(|t| t.trim_matches(|c| c == '{' || c == '}').parse().ok()).take(3).collect();
|
||
Some([*v.first()?, *v.get(1)?, *v.get(2)?])
|
||
}
|
||
|
||
fn output(cmd: &str, args: &[&str]) -> Option<String> {
|
||
let o = Command::new(cmd).args(args).output().ok()?;
|
||
o.status.success().then(|| String::from_utf8_lossy(&o.stdout).trim().to_string()).filter(|s| !s.is_empty())
|
||
}
|
||
|
||
pub fn load_avg() -> Option<[f64; 3]> {
|
||
if let Ok(s) = std::fs::read_to_string("/proc/loadavg") {
|
||
return parse_loadavg(&s);
|
||
}
|
||
output("sysctl", &["-n", "vm.loadavg"]).and_then(|s| parse_loadavg(&s))
|
||
}
|
||
|
||
fn proc_field(file: &str, key: &str) -> Option<String> {
|
||
let s = std::fs::read_to_string(file).ok()?;
|
||
s.lines().find(|l| l.starts_with(key)).and_then(|l| l.split_once(':').or_else(|| l.split_once('='))).map(|(_, v)| v.trim().trim_matches('"').to_string())
|
||
}
|
||
|
||
pub fn machine() -> Value {
|
||
let (os, arch) = (std::env::consts::OS, std::env::consts::ARCH);
|
||
let cores = std::thread::available_parallelism().map(|n| n.get()).ok();
|
||
let (cpu, ram, os_version) = match os {
|
||
"macos" => (
|
||
output("sysctl", &["-n", "machdep.cpu.brand_string"]),
|
||
output("sysctl", &["-n", "hw.memsize"]).and_then(|s| s.parse::<u64>().ok()),
|
||
output("sw_vers", &["-productVersion"]).map(|v| format!("macOS {v}")),
|
||
),
|
||
"linux" => (
|
||
proc_field("/proc/cpuinfo", "model name").or_else(|| proc_field("/proc/cpuinfo", "CPU part").map(|p| format!("ARM part {p}"))),
|
||
proc_field("/proc/meminfo", "MemTotal")
|
||
.and_then(|v| v.split_whitespace().next().and_then(|k| k.parse::<u64>().ok()))
|
||
.and_then(|k| k.checked_mul(1024)),
|
||
proc_field("/etc/os-release", "PRETTY_NAME"),
|
||
),
|
||
_ => (std::env::var("PROCESSOR_IDENTIFIER").ok(), None, output("cmd", &["/c", "ver"])),
|
||
};
|
||
json!({"os": os, "os_version": os_version, "arch": arch, "cpu": cpu, "cores": cores, "ram_bytes": ram})
|
||
}
|
||
|
||
/// `YYYY-MM-DDTHH:MM:SSZ` for a Unix time (proleptic Gregorian, UTC). A time past chrono's year
|
||
/// range still converts (the calendar repeats every 400 years, so the instant is moved into range
|
||
/// by whole cycles and only the year is carried, matching the old hand-rolled `iso_utc`), never a
|
||
/// silent reset to the epoch. Accepts the full `u64` range the old code did.
|
||
pub fn iso_utc(secs: u64) -> String {
|
||
use chrono::Datelike;
|
||
// `secs / 86_400` is at most 213_503_982_334_601, so `days` always fits in `i64` (old code too).
|
||
let (days, sod) = ((secs / 86_400) as i64, (secs % 86_400) as i64);
|
||
let (cycles, rest) = (days.div_euclid(146_097), days.rem_euclid(146_097));
|
||
// `rest + 719_163` (1970-01-01 is CE day 719_163) is always a valid CE day, and `sod` < 86_400.
|
||
let date = i32::try_from(rest + 719_163).ok().and_then(chrono::NaiveDate::from_num_days_from_ce_opt);
|
||
let time = u32::try_from(sod).ok().and_then(|s| chrono::NaiveTime::from_num_seconds_from_midnight_opt(s, 0));
|
||
match (date, time) {
|
||
(Some(d), Some(t)) => {
|
||
let y = i128::from(d.year()) + i128::from(cycles) * 400;
|
||
let y = i64::try_from(y).unwrap_or(if y < 0 { i64::MIN } else { i64::MAX });
|
||
// `{:04}` rather than `%Y`, which writes a `+` before years past 9999.
|
||
format!("{y:04}-{}", d.and_time(t).format("%m-%dT%H:%M:%SZ"))
|
||
}
|
||
_ => "1970-01-01T00:00:00Z".to_string(), // unreachable
|
||
}
|
||
}
|
||
|
||
fn now_iso() -> String {
|
||
iso_utc(std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).map(|d| d.as_secs()).unwrap_or(0))
|
||
}
|
||
|
||
fn git(root: &Path, args: &[&str]) -> Option<String> {
|
||
let o = Command::new("git").current_dir(root).args(args).output().ok()?;
|
||
o.status.success().then(|| String::from_utf8_lossy(&o.stdout).trim().to_string())
|
||
}
|
||
|
||
// ---- baseline --------------------------------------------------------------------------------
|
||
|
||
/// The baseline's scenarios for `mode`, if it was recorded on `class`. `Err` explains a skip.
|
||
pub fn baseline_for(baseline: &Value, class: &str, mode: &str) -> Result<BTreeMap<String, Stats>, String> {
|
||
let bc = baseline.get("machine_class").and_then(Value::as_str).unwrap_or("");
|
||
if bc != class {
|
||
return Err(format!("baseline is from machine class `{bc}`, this is `{class}`: regression check skipped"));
|
||
}
|
||
let sc = baseline
|
||
.get("modes")
|
||
.and_then(|m| m.get(mode))
|
||
.and_then(|m| m.get("scenarios"))
|
||
.and_then(Value::as_object)
|
||
.ok_or_else(|| format!("baseline has no `{mode}` run: regression check skipped"))?;
|
||
Ok(sc.iter().map(|(k, v)| (k.clone(), Stats::from_json(v))).collect())
|
||
}
|
||
|
||
/// A run is contended when its 1-minute load at the start exceeds `max_per_core` per core:
|
||
/// its numbers measure the machine's load, not the code.
|
||
/// Without a load (Windows) or a core count, nothing can be judged: not contended.
|
||
pub fn contended(load: Option<[f64; 3]>, cores: Option<usize>, max_per_core: f64) -> bool {
|
||
match (load, cores) {
|
||
(Some([one, _, _]), Some(c)) if c > 0 => one > c as f64 * max_per_core,
|
||
_ => false,
|
||
}
|
||
}
|
||
|
||
fn load_of(v: Option<&Value>) -> Option<[f64; 3]> {
|
||
let a = v?.as_array()?;
|
||
Some([a.first()?.as_f64()?, a.get(1)?.as_f64()?, a.get(2)?.as_f64()?])
|
||
}
|
||
|
||
fn cores_of(machine: Option<&Value>) -> Option<usize> {
|
||
machine?.get("cores")?.as_u64().map(|c| c as usize)
|
||
}
|
||
|
||
/// Whether a `results.json` (or a baseline mode plus its machine) was contended at the start.
|
||
pub fn run_contended(load_avg_start: Option<&Value>, machine: Option<&Value>, max_per_core: f64) -> bool {
|
||
contended(load_of(load_avg_start), cores_of(machine), max_per_core)
|
||
}
|
||
|
||
/// What `baseline_from_runs` accepts.
|
||
pub struct BaselineRules {
|
||
/// Fewer runs are refused.
|
||
pub min_runs: usize,
|
||
pub max_load_per_core: f64,
|
||
/// Accept contended runs (the baseline then records `"contended": true`).
|
||
pub allow_contended: bool,
|
||
}
|
||
|
||
fn median(v: &mut [f64]) -> Option<f64> {
|
||
v.sort_by(f64::total_cmp);
|
||
let n = v.len();
|
||
match n {
|
||
0 => None,
|
||
_ if n % 2 == 1 => v.get(n / 2).copied(),
|
||
_ => Some((v.get(n / 2 - 1)? + v.get(n / 2)?) / 2.0),
|
||
}
|
||
}
|
||
|
||
/// `old` with one mode replaced by a baseline built from `runs` (saved `results.json` values of
|
||
/// one machine class and mode), or a fresh baseline when the machine class changed. Per scenario:
|
||
/// `p50_ms` is the median of the runs' p50 and, from two runs on, `p50_max_ms` their maximum (the
|
||
/// noise ceiling regressions are judged against); p95 is the median, max and memory the maximum.
|
||
/// One run is the plain `--update-baseline` case. Refused: too few runs, mixed machine classes or
|
||
/// modes, dirty trees when there are several runs, and contended runs unless allowed.
|
||
pub fn baseline_from_runs(old: Option<&Value>, runs: &[Value], rules: &BaselineRules) -> Result<Value, String> {
|
||
if runs.len() < rules.min_runs.max(1) {
|
||
return Err(format!("a baseline needs at least {} run(s), got {}", rules.min_runs.max(1), runs.len()));
|
||
}
|
||
let class_of = |r: &Value| r.get("machine").and_then(|m| m.get("class")).cloned().unwrap_or(Value::Null);
|
||
let mode_of = |r: &Value| r.get("mode").and_then(Value::as_str).unwrap_or("full").to_string();
|
||
let mut runs: Vec<&Value> = runs.iter().collect();
|
||
runs.sort_by_key(|r| r.get("date").and_then(Value::as_str).unwrap_or("").to_string());
|
||
let Some(last) = runs.last().copied() else { return Err("no runs".into()) };
|
||
let (class, mode) = (class_of(last), mode_of(last));
|
||
if runs.iter().any(|r| class_of(r) != class) {
|
||
return Err("the runs are from different machine classes: a baseline needs one".into());
|
||
}
|
||
if runs.iter().any(|r| mode_of(r) != mode) {
|
||
return Err("the runs mix full and quick modes: a baseline needs one".into());
|
||
}
|
||
if runs.len() > 1
|
||
&& let Some(r) = runs.iter().find(|r| r.get("dirty").and_then(Value::as_bool) == Some(true))
|
||
{
|
||
return Err(format!("run {} was taken on a dirty tree: a baseline needs committed code", r.get("date").and_then(Value::as_str).unwrap_or("?")));
|
||
}
|
||
let is_contended = |r: &Value| run_contended(r.get("load_avg_start"), r.get("machine"), rules.max_load_per_core);
|
||
let any_contended = runs.iter().any(|r| is_contended(r));
|
||
if any_contended && !rules.allow_contended {
|
||
let msgs: Vec<String> = runs
|
||
.iter()
|
||
.filter(|r| is_contended(r))
|
||
.map(|r| {
|
||
format!(
|
||
"load {} on {} cores at the start of {}",
|
||
load_of(r.get("load_avg_start")).map_or_else(|| "?".into(), |l| format!("{:.2}", l[0])),
|
||
cores_of(r.get("machine")).map_or_else(|| "?".into(), |c| c.to_string()),
|
||
r.get("date").and_then(Value::as_str).unwrap_or("the run")
|
||
)
|
||
})
|
||
.collect();
|
||
return Err(format!(
|
||
"contended run, baseline not written: {} (above {} per core). Its numbers measure machine load; rerun on an idle machine, or pass --allow-contended to record it labelled as contended",
|
||
msgs.join("; "),
|
||
rules.max_load_per_core
|
||
));
|
||
}
|
||
|
||
let mut per: BTreeMap<String, Vec<Stats>> = BTreeMap::new();
|
||
let mut failed = serde_json::Map::new();
|
||
for r in &runs {
|
||
for (k, v) in r.get("scenarios").and_then(Value::as_object).into_iter().flatten() {
|
||
if v.get("p50_ms").is_some_and(|x| !x.is_null()) {
|
||
per.entry(k.clone()).or_default().push(Stats::from_json(v));
|
||
} else if let Some(e) = v.get("status").and_then(Value::as_str).and_then(|s| s.strip_prefix(MISSING_PREFIX)) {
|
||
// A crashed row (a crash the bench found): kept so the scorecard says so.
|
||
failed.insert(k.clone(), json!(e));
|
||
}
|
||
}
|
||
}
|
||
let mut scenarios = serde_json::Map::new();
|
||
for (k, st) in &per {
|
||
// A row that crashed in any run stays failed, and one missing from some runs is left
|
||
// out: no partial aggregate under a full run count.
|
||
if failed.contains_key(k) || st.len() != runs.len() {
|
||
continue;
|
||
}
|
||
let mut p50: Vec<f64> = st.iter().filter_map(|s| s.p50).collect();
|
||
let mut p95: Vec<f64> = st.iter().filter_map(|s| s.p95).collect();
|
||
let fmax = |f: fn(&Stats) -> Option<f64>| st.iter().filter_map(f).reduce(f64::max);
|
||
let umax = |f: fn(&Stats) -> Option<u64>| st.iter().filter_map(f).max();
|
||
let s = Stats {
|
||
n: st.iter().map(|s| s.n).sum(),
|
||
p50_max: (runs.len() > 1).then(|| fmax(|s| s.p50)).flatten(),
|
||
p50: median(&mut p50),
|
||
p95: median(&mut p95),
|
||
max: fmax(|s| s.max),
|
||
peak_rss: umax(|s| s.peak_rss),
|
||
gpu_bytes: umax(|s| s.gpu_bytes),
|
||
};
|
||
scenarios.insert(k.clone(), s.to_json());
|
||
}
|
||
|
||
let mut modes = old.filter(|o| o.get("machine_class") == Some(&class)).and_then(|o| o.get("modes")).and_then(Value::as_object).cloned().unwrap_or_default();
|
||
let mut entry = json!({
|
||
"date": last.get("date"),
|
||
"commit": last.get("commit"),
|
||
"load_avg_start": last.get("load_avg_start"),
|
||
"load_avg_end": last.get("load_avg_end"),
|
||
"cores": cores_of(last.get("machine")),
|
||
"runs": runs.len(),
|
||
"scenarios": scenarios,
|
||
"failed": failed,
|
||
});
|
||
if let Some(o) = entry.as_object_mut() {
|
||
if any_contended {
|
||
o.insert("contended".into(), json!(true));
|
||
}
|
||
if runs.len() > 1 {
|
||
let each: Vec<Value> = runs
|
||
.iter()
|
||
.map(|r| json!({"date": r.get("date"), "commit": r.get("commit"), "load_avg_start": r.get("load_avg_start"), "contended": is_contended(r)}))
|
||
.collect();
|
||
o.insert("run_list".into(), json!(each));
|
||
}
|
||
}
|
||
modes.insert(mode, entry);
|
||
Ok(json!({"schema": 1, "machine_class": class, "machine": last.get("machine"), "modes": modes}))
|
||
}
|
||
|
||
/// From saved runs: budgets that could be enforced (not enforced, and within `enforce_margin` of
|
||
/// the budget in every run) and enforced budgets that went over in some run (flaky).
|
||
pub fn enforce_report(budgets: &Budgets, runs: &[Value]) -> (Vec<String>, Vec<String>) {
|
||
let (mut could, mut flaky) = (Vec::new(), Vec::new());
|
||
let margin = budgets.settings.enforce_margin;
|
||
for sc in &budgets.scenario {
|
||
let (Some(budget), None) = (sc.budget_ms, &sc.not_measurable) else { continue };
|
||
let key = format!("{}_ms", sc.metric.label());
|
||
let vals: Vec<f64> = runs.iter().filter_map(|r| r.get("scenarios")?.get(&sc.id)?.get(&key)?.as_f64()).collect();
|
||
let Some(worst) = vals.iter().copied().reduce(f64::max) else { continue };
|
||
let m = sc.metric.label();
|
||
if sc.enforce {
|
||
let over = vals.iter().filter(|v| **v > budget).count();
|
||
if over > 0 {
|
||
flaky.push(format!("{} {}: over {} ms in {over} of {} runs (worst {m} {})", sc.id, sc.title, fmt_num(budget), runs.len(), fmt_num(worst)));
|
||
}
|
||
} else if vals.len() == runs.len() && worst <= margin * budget {
|
||
could.push(format!("{} {}: worst {m} {} ms ≤ {margin} × {} ms in all {} runs", sc.id, sc.title, fmt_num(worst), fmt_num(budget), runs.len()));
|
||
}
|
||
}
|
||
(could, flaky)
|
||
}
|
||
|
||
// ---- running ---------------------------------------------------------------------------------
|
||
|
||
struct Opts {
|
||
quick: bool,
|
||
update_baseline: bool,
|
||
threshold: Option<f64>,
|
||
benches: Vec<String>,
|
||
skip_build: bool,
|
||
reuse: bool,
|
||
/// `--update-baseline --from`: build the baseline from these saved `results.json` files.
|
||
from: Vec<PathBuf>,
|
||
allow_contended: bool,
|
||
}
|
||
|
||
fn parse_opts(rest: &[&str]) -> Result<Opts, String> {
|
||
let mut o = Opts {
|
||
quick: false,
|
||
update_baseline: false,
|
||
threshold: None,
|
||
benches: Vec::new(),
|
||
skip_build: false,
|
||
reuse: false,
|
||
from: Vec::new(),
|
||
allow_contended: false,
|
||
};
|
||
let mut it = rest.iter();
|
||
while let Some(a) = it.next() {
|
||
match *a {
|
||
"--quick" => o.quick = true,
|
||
"--update-baseline" => o.update_baseline = true,
|
||
"--skip-build" => o.skip_build = true,
|
||
"--reuse" => {
|
||
o.reuse = true;
|
||
o.skip_build = true;
|
||
}
|
||
"--threshold" => {
|
||
let v = it.next().and_then(|v| v.parse::<f64>().ok()).filter(|v| v.is_finite() && *v > 0.0).ok_or("--threshold needs a percentage > 0")?;
|
||
o.threshold = Some(v);
|
||
}
|
||
"--bench" => o.benches.push(it.next().ok_or("--bench needs a name")?.to_string()),
|
||
"--from" => o.from.push(PathBuf::from(it.next().ok_or("--from needs a results.json path")?)),
|
||
"--allow-contended" => o.allow_contended = true,
|
||
other => return Err(format!("perf: unknown option `{other}`")),
|
||
}
|
||
}
|
||
if !o.from.is_empty() && !o.update_baseline {
|
||
return Err("--from builds a baseline: use it with --update-baseline".into());
|
||
}
|
||
Ok(o)
|
||
}
|
||
|
||
fn target_dir() -> Result<PathBuf, String> {
|
||
let m = crate::metadata()?;
|
||
m.get("target_directory").and_then(Value::as_str).map(PathBuf::from).ok_or_else(|| "cargo metadata: no target_directory".into())
|
||
}
|
||
|
||
fn write(path: &Path, text: &str) -> Result<(), String> {
|
||
if let Some(dir) = path.parent() {
|
||
std::fs::create_dir_all(dir).map_err(|e| format!("create {}: {e}", dir.display()))?;
|
||
}
|
||
std::fs::write(path, text).map_err(|e| format!("write {}: {e}", path.display()))
|
||
}
|
||
|
||
pub fn run(root: &Path, rest: &[&str]) -> Result<(), String> {
|
||
let o = parse_opts(rest)?;
|
||
let text = std::fs::read_to_string(root.join(BUDGETS)).map_err(|e| format!("read {BUDGETS}: {e}"))?;
|
||
let budgets = parse_budgets(&text)?;
|
||
for name in &o.benches {
|
||
if !budgets.bench.iter().any(|b| &b.name == name) {
|
||
return Err(format!("perf: unknown bench `{name}` (see {BUDGETS})"));
|
||
}
|
||
}
|
||
let baseline_value: Option<Value> = std::fs::read_to_string(root.join(BASELINE)).ok().and_then(|t| serde_json::from_str(&t).ok());
|
||
if !o.from.is_empty() {
|
||
return baseline_from_files(root, &budgets, baseline_value.as_ref(), &o);
|
||
}
|
||
let mode = if o.quick { "quick" } else { "full" };
|
||
let threshold = o.threshold.unwrap_or(budgets.settings.regression_pct);
|
||
let target = target_dir()?;
|
||
let out_dir = target.join("perf");
|
||
|
||
// Which benches run in this mode.
|
||
let mut plan: Vec<(&BenchSpec, Vec<String>)> = Vec::new();
|
||
let mut skipped: Vec<(String, String)> = Vec::new();
|
||
for b in &budgets.bench {
|
||
if !o.benches.is_empty() && !o.benches.contains(&b.name) {
|
||
continue;
|
||
}
|
||
let args = if o.quick {
|
||
match &b.quick_args {
|
||
Some(a) => a.clone(),
|
||
None => continue,
|
||
}
|
||
} else {
|
||
b.args.clone()
|
||
};
|
||
if let Some(req) = &b.requires
|
||
&& !root.join(req).exists()
|
||
{
|
||
skipped.push((b.name.clone(), format!("{req} not on this branch yet")));
|
||
continue;
|
||
}
|
||
plan.push((b, args));
|
||
}
|
||
|
||
// Sampled before the build: the harness's own compile isn't contention. The load after the
|
||
// build is still recorded, as the first bench's `load_avg_before`.
|
||
let load_start = load_avg();
|
||
if !o.skip_build {
|
||
let mut by_pkg: BTreeMap<&str, Vec<&str>> = BTreeMap::new();
|
||
for (b, _) in &plan {
|
||
by_pkg.entry(b.package.as_str()).or_default().push(b.example());
|
||
}
|
||
for (pkg, examples) in by_pkg {
|
||
let mut c = crate::cargo();
|
||
c.args(["build", "--release", "-p", pkg]);
|
||
for e in &examples {
|
||
c.args(["--example", e]);
|
||
}
|
||
crate::run(c, &format!("cargo build --release -p {pkg} --example {}", examples.join(" --example ")))?;
|
||
}
|
||
}
|
||
|
||
let mut reports: BTreeMap<String, Result<Value, String>> = BTreeMap::new();
|
||
let mut bench_meta = serde_json::Map::new();
|
||
for (b, args) in &plan {
|
||
let json_path = out_dir.join(format!("{}.json", b.name));
|
||
let before = load_avg();
|
||
let t = Instant::now();
|
||
let result = if o.reuse {
|
||
std::fs::read_to_string(&json_path).map_err(|e| format!("--reuse: {}: {e}", json_path.display()))
|
||
} else {
|
||
let _ = std::fs::remove_file(&json_path);
|
||
if let Some(dir) = json_path.parent() {
|
||
std::fs::create_dir_all(dir).map_err(|e| format!("create {}: {e}", dir.display()))?;
|
||
}
|
||
let exe = target.join("release").join("examples").join(format!("{}{}", b.example(), std::env::consts::EXE_SUFFIX));
|
||
let mut c = Command::new(&exe);
|
||
c.current_dir(root).args(args).arg("--json").arg(&json_path);
|
||
let shown = format!("{} {} --json {}", b.example(), args.join(" "), json_path.display());
|
||
crate::run(c, &shown).and_then(|()| std::fs::read_to_string(&json_path).map_err(|e| format!("{}: no JSON report ({e})", b.name)))
|
||
};
|
||
let parsed = result.and_then(|t| serde_json::from_str::<Value>(&t).map_err(|e| format!("{}: bad JSON: {e}", b.name)));
|
||
let after = load_avg();
|
||
bench_meta.insert(
|
||
b.name.clone(),
|
||
json!({
|
||
"status": if parsed.is_ok() { "ok" } else { "error" },
|
||
"note": b.note,
|
||
"error": parsed.as_ref().err(),
|
||
"seconds": t.elapsed().as_secs_f64(),
|
||
"args": args,
|
||
"load_avg_before": before,
|
||
"load_avg_after": after,
|
||
"context": parsed.as_ref().ok().and_then(|r| r.get("context")).cloned(),
|
||
"process_peak_rss_bytes": parsed.as_ref().ok().and_then(|r| r.get("process_peak_rss_bytes")).cloned(),
|
||
// Every row, mapped to a scenario or not (new benches show up here first).
|
||
"rows": parsed.as_ref().ok().and_then(|r| r.get("rows")).and_then(Value::as_array).map(|rows| {
|
||
rows.iter()
|
||
.map(|r| {
|
||
let mut v = row_stats(r).to_json();
|
||
if let Some(o) = v.as_object_mut() {
|
||
o.insert("name".into(), r.get("name").cloned().unwrap_or(Value::Null));
|
||
}
|
||
v
|
||
})
|
||
.collect::<Vec<_>>()
|
||
}),
|
||
}),
|
||
);
|
||
reports.insert(b.name.clone(), parsed);
|
||
}
|
||
for (name, why) in &skipped {
|
||
bench_meta.insert(name.clone(), json!({"status": "skipped", "error": why}));
|
||
}
|
||
let load_end = load_avg();
|
||
|
||
let mut m = machine();
|
||
let gpu = reports
|
||
.values()
|
||
.filter_map(|r| r.as_ref().ok())
|
||
.find_map(|r| r.get("context").and_then(|c| c.get("gpu_adapter")).and_then(Value::as_str).map(str::to_string));
|
||
let gpu_name = gpu.clone().unwrap_or_else(|| "cpu-canvas".into());
|
||
let class = machine_class(
|
||
m.get("os").and_then(Value::as_str).unwrap_or(""),
|
||
m.get("arch").and_then(Value::as_str).unwrap_or(""),
|
||
m.get("cpu").and_then(Value::as_str).unwrap_or(""),
|
||
&gpu_name,
|
||
);
|
||
if let Some(obj) = m.as_object_mut() {
|
||
obj.insert("gpu_adapter".into(), json!(gpu));
|
||
obj.insert("class".into(), json!(class));
|
||
}
|
||
|
||
let (baseline, baseline_note) = match baseline_value.as_ref().map(|b| baseline_for(b, &class, mode)) {
|
||
Some(Ok(b)) => (Some(b), None),
|
||
Some(Err(e)) => (None, Some(e)),
|
||
None => (None, Some(format!("no {BASELINE}: regression check skipped"))),
|
||
};
|
||
let outcomes = evaluate(&budgets, &reports, baseline.as_ref(), threshold, !o.quick);
|
||
|
||
let scenarios: serde_json::Map<String, Value> = budgets
|
||
.scenario
|
||
.iter()
|
||
.zip(&outcomes)
|
||
.map(|(sc, oc)| {
|
||
let mut v = oc.stats.as_ref().map(Stats::to_json).unwrap_or_else(|| json!({}));
|
||
if let Some(obj) = v.as_object_mut() {
|
||
obj.insert("title".into(), json!(sc.title));
|
||
obj.insert("issue".into(), json!(sc.issue));
|
||
obj.insert("bench".into(), json!(sc.bench));
|
||
obj.insert("row".into(), json!(sc.row));
|
||
obj.insert("metric".into(), json!(sc.metric.label()));
|
||
obj.insert("budget_ms".into(), json!(sc.budget_ms));
|
||
obj.insert("target".into(), json!(sc.target_text()));
|
||
obj.insert("enforce".into(), json!(sc.enforce));
|
||
obj.insert("baseline_p50_ms".into(), json!(oc.baseline_p50));
|
||
obj.insert("change_pct".into(), json!(oc.change_pct));
|
||
obj.insert("status".into(), json!(oc.status.label()));
|
||
if let Status::NotMeasurable(w) = &oc.status {
|
||
obj.insert("not_measurable".into(), json!(w));
|
||
}
|
||
}
|
||
(sc.id.clone(), v)
|
||
})
|
||
.collect();
|
||
let failures: Vec<String> = outcomes.iter().filter(|o| o.status.fails()).map(|o| format!("{} {}: {}", o.id, o.title, o.status.label())).collect();
|
||
let bench_failures: Vec<String> = reports.iter().filter_map(|(n, r)| r.as_ref().err().map(|e| format!("bench {n}: {e}"))).collect();
|
||
let commit = git(root, &["rev-parse", "HEAD"]);
|
||
let dirty = git(root, &["status", "--porcelain"]).is_some_and(|s| !s.is_empty());
|
||
let is_contended = contended(load_start, m.get("cores").and_then(Value::as_u64).map(|c| c as usize), budgets.settings.max_load_per_core);
|
||
let results = json!({
|
||
"schema": 1,
|
||
"mode": mode,
|
||
"date": now_iso(),
|
||
"commit": commit,
|
||
"dirty": dirty,
|
||
"machine": m,
|
||
"load_avg_start": load_start,
|
||
"load_avg_end": load_end,
|
||
"contended": is_contended,
|
||
"threshold_pct": threshold,
|
||
"baseline_note": baseline_note,
|
||
"benches": bench_meta,
|
||
"scenarios": scenarios,
|
||
"failures": failures.iter().chain(&bench_failures).collect::<Vec<_>>(),
|
||
});
|
||
let text = serde_json::to_string_pretty(&results).map_err(|e| format!("encode results: {e}"))?;
|
||
write(&out_dir.join("results.json"), &text)?;
|
||
|
||
let summary = render_summary(&results, &outcomes, baseline_note.as_deref());
|
||
write(&out_dir.join("summary.md"), &summary)?;
|
||
println!("\n{summary}");
|
||
println!("wrote {}", out_dir.join("results.json").display());
|
||
|
||
let mut errors: Vec<String> = Vec::new();
|
||
if o.update_baseline {
|
||
// The results are written either way; a contended run never silently becomes the baseline.
|
||
let rules = BaselineRules { min_runs: 1, max_load_per_core: budgets.settings.max_load_per_core, allow_contended: o.allow_contended };
|
||
match baseline_from_runs(baseline_value.as_ref(), std::slice::from_ref(&results), &rules) {
|
||
Ok(nb) => {
|
||
write_baseline(root, &nb)?;
|
||
println!("updated {BASELINE} ({mode}, machine class {class}{})", if is_contended { ", labelled contended" } else { "" });
|
||
}
|
||
// A refused baseline still reports this run's perf failures below.
|
||
Err(e) => errors.push(e),
|
||
}
|
||
}
|
||
let all: Vec<&String> = failures.iter().chain(&bench_failures).collect();
|
||
if !all.is_empty() {
|
||
errors.push(format!("{} perf failure(s):\n {}", all.len(), all.iter().map(|s| s.as_str()).collect::<Vec<_>>().join("\n ")));
|
||
}
|
||
if errors.is_empty() { Ok(()) } else { Err(errors.join("\n")) }
|
||
}
|
||
|
||
fn write_baseline(root: &Path, baseline: &Value) -> Result<(), String> {
|
||
let text = serde_json::to_string_pretty(baseline).map_err(|e| format!("encode baseline: {e}"))?;
|
||
write(&root.join(BASELINE), &format!("{text}\n"))
|
||
}
|
||
|
||
/// `--update-baseline --from <results.json>...`: no benches run. Prints the enforcement report.
|
||
fn baseline_from_files(root: &Path, budgets: &Budgets, old: Option<&Value>, o: &Opts) -> Result<(), String> {
|
||
let runs = o
|
||
.from
|
||
.iter()
|
||
.map(|p| {
|
||
let t = std::fs::read_to_string(p).map_err(|e| format!("read {}: {e}", p.display()))?;
|
||
serde_json::from_str::<Value>(&t).map_err(|e| format!("{}: {e}", p.display()))
|
||
})
|
||
.collect::<Result<Vec<_>, _>>()?;
|
||
let (could, flaky) = enforce_report(budgets, &runs);
|
||
let list = |v: &[String]| if v.is_empty() { " none".to_string() } else { v.iter().map(|l| format!(" {l}")).collect::<Vec<_>>().join("\n") };
|
||
println!("Budgets that could be enforced (within {} of the budget in every run):\n{}", budgets.settings.enforce_margin, list(&could));
|
||
println!("Enforced budgets that went over in some run (flaky):\n{}", list(&flaky));
|
||
let rules = BaselineRules {
|
||
min_runs: budgets.settings.baseline_min_runs,
|
||
max_load_per_core: budgets.settings.max_load_per_core,
|
||
allow_contended: o.allow_contended,
|
||
};
|
||
let nb = baseline_from_runs(old, &runs, &rules)?;
|
||
write_baseline(root, &nb)?;
|
||
println!("updated {BASELINE} from {} runs (machine class {})", runs.len(), nb.get("machine_class").and_then(Value::as_str).unwrap_or("?"));
|
||
Ok(())
|
||
}
|
||
|
||
/// The committed baseline's full-mode numbers judged against the budgets (what the scorecard
|
||
/// shows: deterministic, no run needed). Scenarios without a baseline number are "not run".
|
||
pub fn outcomes_from_baseline(budgets: &Budgets, baseline: Option<&Value>) -> Vec<Outcome> {
|
||
let mode = baseline.and_then(|b| b.get("modes")?.get("full"));
|
||
let full: Option<BTreeMap<String, Stats>> =
|
||
mode.and_then(|m| m.get("scenarios")?.as_object().cloned()).map(|m| m.iter().map(|(k, v)| (k.clone(), Stats::from_json(v))).collect());
|
||
let failed = |id: &str| mode.and_then(|m| m.get("failed")?.get(id)?.as_str()).map(str::to_string);
|
||
budgets
|
||
.scenario
|
||
.iter()
|
||
.map(|sc| {
|
||
let stats = full.as_ref().and_then(|f| f.get(&sc.id)).cloned();
|
||
let status = match (&sc.not_measurable, &stats) {
|
||
(Some(w), _) => Status::NotMeasurable(w.clone()),
|
||
(None, Some(st)) => verdict(sc, st, None, &budgets.settings, budgets.settings.regression_pct, true),
|
||
(None, None) => failed(&sc.id).map_or(Status::NotRun, Status::Missing),
|
||
};
|
||
Outcome {
|
||
id: sc.id.clone(),
|
||
title: sc.title.clone(),
|
||
issue: sc.issue,
|
||
target: sc.target_text(),
|
||
stats,
|
||
baseline_p50: None,
|
||
change_pct: None,
|
||
status,
|
||
}
|
||
})
|
||
.collect()
|
||
}
|
||
|
||
pub fn fmt_load(v: Option<&Value>) -> String {
|
||
v.and_then(Value::as_array).map_or_else(|| "n/a".into(), |a| a.iter().filter_map(Value::as_f64).map(|x| format!("{x:.2}")).collect::<Vec<_>>().join(" / "))
|
||
}
|
||
|
||
/// The Markdown summary of one run: context lines and the scenario table.
|
||
pub fn render_summary(results: &Value, outcomes: &[Outcome], baseline_note: Option<&str>) -> String {
|
||
let m = results.get("machine").cloned().unwrap_or(Value::Null);
|
||
let s = |v: Option<&Value>| v.and_then(Value::as_str).unwrap_or("unknown").to_string();
|
||
let ram = m.get("ram_bytes").and_then(Value::as_u64).map_or_else(|| "?".into(), |b| format!("{} GB", b >> 30));
|
||
let mut out = format!(
|
||
"## Performance ({} run)\n\n- Machine: {} ({} cores, {ram}), GPU {}, {} {}; class `{}`\n- Commit {}{}, {}\n- Load average (1/5/15 min): {} at start, {} at end\n",
|
||
s(results.get("mode")),
|
||
s(m.get("cpu")),
|
||
m.get("cores").and_then(Value::as_u64).unwrap_or(0),
|
||
s(m.get("gpu_adapter")),
|
||
s(m.get("os_version")),
|
||
s(m.get("arch")),
|
||
s(m.get("class")),
|
||
s(results.get("commit")),
|
||
if results.get("dirty").and_then(Value::as_bool) == Some(true) { " (dirty)" } else { "" },
|
||
s(results.get("date")),
|
||
fmt_load(results.get("load_avg_start")),
|
||
fmt_load(results.get("load_avg_end")),
|
||
);
|
||
if results.get("contended").and_then(Value::as_bool) == Some(true) {
|
||
out.push_str("- **contended run**: numbers measure machine load (the 1-minute load at the start was above `max_load_per_core` per core)\n");
|
||
}
|
||
if let Some(n) = baseline_note {
|
||
out.push_str(&format!("- {n}\n"));
|
||
}
|
||
if results.get("mode").and_then(Value::as_str) == Some("quick") {
|
||
out.push_str("- Quick mode: small documents, budgets not applied (they are set for full-size documents).\n");
|
||
}
|
||
if let Some(b) = results.get("benches").and_then(Value::as_object) {
|
||
for (name, v) in b {
|
||
let st = v.get("status").and_then(Value::as_str).unwrap_or("?");
|
||
let secs = v.get("seconds").and_then(Value::as_f64).map_or_else(String::new, |x| format!(", {x:.0} s"));
|
||
let err = v.get("error").and_then(Value::as_str).map_or_else(String::new, |e| format!(": {e}"));
|
||
out.push_str(&format!("- Bench `{name}`: {st}{secs}, load {}{err}\n", fmt_load(v.get("load_avg_before"))));
|
||
}
|
||
}
|
||
out.push('\n');
|
||
out.push_str(&render_table(outcomes, true));
|
||
out
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
|
||
#[test]
|
||
fn iso_utc_handles_boundaries() {
|
||
assert_eq!(iso_utc(0), "1970-01-01T00:00:00Z");
|
||
assert_eq!(iso_utc(1_791_446_357), "2026-10-08T07:59:17Z");
|
||
// Years past 9999 keep the plain zero-padded year (no `+`).
|
||
assert_eq!(iso_utc(253_402_300_800), "10000-01-01T00:00:00Z");
|
||
// A value past chrono's range still converts via the 400-year cycle (the old hand-rolled
|
||
// `iso_utc` did too): pin that output, never the epoch.
|
||
assert_eq!(iso_utc(10_000_000_000_000), "318857-05-20T17:46:40Z");
|
||
assert_eq!(iso_utc(u64::MAX), "584554051223-11-09T07:00:15Z");
|
||
}
|
||
|
||
const TOML: &str = r#"
|
||
[settings]
|
||
regression_pct = 15.0
|
||
regression_min_ms = 0.5
|
||
|
||
[[bench]]
|
||
name = "b"
|
||
package = "p"
|
||
quick_args = ["--quick"]
|
||
|
||
[[scenario]]
|
||
id = "P1"
|
||
title = "move"
|
||
issue = 209
|
||
bench = "b"
|
||
row = "move row"
|
||
budget_ms = 25
|
||
|
||
[[scenario]]
|
||
id = "P2"
|
||
title = "opacity"
|
||
bench = "b"
|
||
row = "opacity row"
|
||
budget_ms = 20
|
||
enforce = false
|
||
|
||
[[scenario]]
|
||
id = "P3"
|
||
title = "zoom"
|
||
target = "≤ 16 ms per frame"
|
||
not_measurable = "needs a frame harness"
|
||
"#;
|
||
|
||
fn report(rows: Value) -> Value {
|
||
json!({"bench": "b", "rows": rows})
|
||
}
|
||
|
||
#[test]
|
||
fn budgets_parse_and_validate() {
|
||
let b = parse_budgets(TOML).unwrap();
|
||
assert_eq!(b.scenario.len(), 3);
|
||
assert_eq!(b.scenario[0].metric, Metric::P95);
|
||
assert!(b.scenario[0].enforce);
|
||
assert!(!b.scenario[1].enforce);
|
||
assert_eq!(b.scenario[0].target_text(), "≤ 25 ms p95");
|
||
assert_eq!(b.scenario[2].target_text(), "≤ 16 ms per frame");
|
||
assert_eq!(b.settings.regression_min_ms, 0.5);
|
||
// Duplicate ids, undefined benches, half-specified rows and bad budgets are rejected.
|
||
assert!(parse_budgets(&format!("{TOML}\n[[scenario]]\nid = \"P1\"\ntitle = \"x\"\nnot_measurable = \"y\"\n")).is_err());
|
||
assert!(parse_budgets("[[scenario]]\nid = \"P1\"\ntitle = \"x\"\nbench = \"nope\"\nrow = \"r\"\n").is_err());
|
||
assert!(parse_budgets("[[bench]]\nname = \"b\"\npackage = \"p\"\n[[scenario]]\nid = \"P1\"\ntitle = \"x\"\nbench = \"b\"\n").is_err());
|
||
assert!(
|
||
parse_budgets("[[bench]]\nname = \"b\"\npackage = \"p\"\n[[scenario]]\nid = \"P1\"\ntitle = \"x\"\nbench = \"b\"\nrow = \"r\"\nbudget_ms = -1\n")
|
||
.is_err()
|
||
);
|
||
assert!(parse_budgets("[settings]\nregression_pct = 0\n").is_err());
|
||
assert!(parse_budgets("[settings]\nregression_min_ms = -1\n").is_err());
|
||
assert!(parse_budgets("[settings]\nregression_min_ms = 0\n").is_ok());
|
||
assert!(parse_budgets("bogus = 1\n").is_err());
|
||
assert!(parse_budgets("not toml [").is_err());
|
||
}
|
||
|
||
#[test]
|
||
fn row_stats_prefers_samples_and_never_invents_p95() {
|
||
let s = row_stats(&json!({"name": "x", "samples_ms": [5.0, 1.0, 3.0, 2.0, 4.0], "p50_ms": 99.0}));
|
||
assert_eq!((s.n, s.p50, s.p95, s.max), (5, Some(3.0), Some(5.0), Some(5.0)));
|
||
// Older rows: median + max only. p95 stays empty.
|
||
let s = row_stats(&json!({"name": "x", "median_ms": 4.0, "min_ms": 1.0, "max_ms": 9.0}));
|
||
assert_eq!((s.p50, s.p95, s.max), (Some(4.0), None, Some(9.0)));
|
||
let s = row_stats(&json!({"name": "x", "samples_ms": ["bad", null]}));
|
||
assert_eq!(s.p50, None);
|
||
}
|
||
|
||
#[test]
|
||
fn regression_detection() {
|
||
assert!(!regressed(11.4, 10.0, 15.0, 0.5));
|
||
assert!(regressed(11.6, 10.0, 15.0, 0.5));
|
||
// Below the absolute noise floor: no regression.
|
||
assert!(!regressed(0.30, 0.20, 15.0, 0.5));
|
||
assert!(!regressed(5.0, 0.0, 15.0, 0.5));
|
||
assert!(!regressed(f64::NAN, 1.0, 15.0, 0.0));
|
||
assert_eq!(change_pct(12.0, 10.0).map(|v| v.round()), Some(20.0));
|
||
assert_eq!(change_pct(1.0, 0.0), None);
|
||
}
|
||
|
||
#[test]
|
||
fn evaluate_budgets_regressions_and_missing_rows() {
|
||
let b = parse_budgets(TOML).unwrap();
|
||
let mut reports = BTreeMap::new();
|
||
reports.insert(
|
||
"b".to_string(),
|
||
Ok(report(json!([{"name": "move row", "samples_ms": [10.0, 12.0, 30.0]}, {"name": "opacity row", "samples_ms": [40.0]}]))),
|
||
);
|
||
let out = evaluate(&b, &reports, None, 15.0, true);
|
||
assert_eq!(out[0].status, Status::Broken, "p95 30 > 25 and enforced");
|
||
assert_eq!(out[1].status, Status::Over, "not enforced: reported, doesn't fail");
|
||
assert!(matches!(out[2].status, Status::NotMeasurable(_)));
|
||
assert!(out[0].status.fails() && !out[1].status.fails() && !out[2].status.fails());
|
||
|
||
// Quick mode: no budgets, only regressions.
|
||
let mut base = BTreeMap::new();
|
||
base.insert("P1".to_string(), Stats { p50: Some(10.0), ..Default::default() });
|
||
base.insert("P2".to_string(), Stats { p50: Some(30.0), ..Default::default() });
|
||
let out = evaluate(&b, &reports, Some(&base), 15.0, false);
|
||
assert_eq!(out[0].status, Status::Regressed, "p50 12 vs 10 is +20 %");
|
||
assert_eq!(out[1].status, Status::Regressed, "p50 40 vs 30");
|
||
let out = evaluate(&b, &reports, Some(&base), 50.0, false);
|
||
assert_eq!(out[0].status, Status::Tracked, "threshold is configurable");
|
||
|
||
// A row the bench didn't produce fails, with the bench's own error when it has one.
|
||
let mut reports = BTreeMap::new();
|
||
reports.insert("b".to_string(), Ok(json!({"rows": [], "context": {"errors": [{"name": "move row", "error": "boom"}]}})));
|
||
let out = evaluate(&b, &reports, None, 15.0, true);
|
||
assert_eq!(out[0].status, Status::Missing("boom".into()));
|
||
assert!(matches!(&out[1].status, Status::Missing(e) if e.contains("not in")));
|
||
// A bench that didn't run in this mode: not run, not a failure.
|
||
let out = evaluate(&b, &BTreeMap::new(), None, 15.0, true);
|
||
assert_eq!(out[0].status, Status::NotRun);
|
||
assert!(!out[0].status.fails());
|
||
assert_eq!(counts(&out), (0, 0, 0, 1), "not-run scenarios are not tracked; the scorecard counts them separately");
|
||
}
|
||
|
||
fn rules(min_runs: usize, allow_contended: bool) -> BaselineRules {
|
||
BaselineRules { min_runs, max_load_per_core: 1.0, allow_contended }
|
||
}
|
||
|
||
/// The plain `--update-baseline` case: one run.
|
||
fn one(old: Option<&Value>, results: &Value) -> Value {
|
||
baseline_from_runs(old, std::slice::from_ref(results), &rules(1, false)).unwrap()
|
||
}
|
||
|
||
/// A saved run of class `c` with one p50 (and p95) per scenario.
|
||
fn run_of(date: &str, load: f64, rows: &[(&str, f64)]) -> Value {
|
||
let sc: serde_json::Map<String, Value> =
|
||
rows.iter().map(|(k, v)| (k.to_string(), json!({"n": 5, "p50_ms": v, "p95_ms": v * 1.5, "max_ms": v * 2.0}))).collect();
|
||
json!({"mode": "full", "date": date, "commit": format!("c-{date}"), "dirty": false, "machine": {"class": "c", "cores": 3}, "load_avg_start": [load, load, load], "scenarios": sc})
|
||
}
|
||
|
||
#[test]
|
||
fn contention_is_judged_on_start_load_per_core() {
|
||
assert!(contended(Some([224.9, 226.31, 196.03]), Some(14), 1.0), "the committed 2026-10-05 baseline");
|
||
assert!(!contended(Some([0.5, 3.0, 9.0]), Some(12), 1.0), "only the 1-minute load counts");
|
||
assert!(!contended(None, Some(12), 1.0), "no load (Windows): can't judge");
|
||
assert!(!contended(Some([50.0, 0.0, 0.0]), None, 1.0));
|
||
assert!(!contended(Some([9.5, 0.0, 0.0]), Some(3), 4.0), "the threshold is configurable");
|
||
assert!(contended(Some([9.5, 0.0, 0.0]), Some(3), 3.0));
|
||
assert!(run_contended(Some(&json!([224.9, 226.31, 196.03])), Some(&json!({"cores": 14})), 1.0));
|
||
}
|
||
|
||
#[test]
|
||
fn update_baseline_refuses_contended_run_unless_allowed() {
|
||
let mut r = run_of("2026-10-05", 224.9, &[("P1", 3.0)]);
|
||
r["machine"]["cores"] = json!(14);
|
||
let e = baseline_from_runs(None, std::slice::from_ref(&r), &rules(1, false)).unwrap_err();
|
||
assert!(e.contains("224.90") && e.contains("14 cores") && e.contains("--allow-contended"), "{e}");
|
||
let b = baseline_from_runs(None, std::slice::from_ref(&r), &rules(1, true)).unwrap();
|
||
let m = &b["modes"]["full"];
|
||
assert_eq!(m["contended"], json!(true));
|
||
assert_eq!(m["cores"], json!(14));
|
||
assert_eq!(m["runs"], json!(1));
|
||
// A quiet run isn't labelled, and one run has no noise ceiling.
|
||
let b = one(None, &run_of("d", 0.5, &[("P1", 3.0)]));
|
||
assert!(b["modes"]["full"].get("contended").is_none());
|
||
assert!(b["modes"]["full"]["scenarios"]["P1"].get("p50_max_ms").is_none());
|
||
}
|
||
|
||
#[test]
|
||
fn baseline_from_runs_takes_median_and_noise_ceiling() {
|
||
let runs: Vec<Value> = [(1, 2174.0, 30.5), (2, 2868.0, 45.3), (3, 2000.0, 28.0), (4, 2500.0, 31.0), (5, 2300.0, 40.0)]
|
||
.iter()
|
||
.map(|(d, a, b)| run_of(&format!("2026-10-2{d}"), 1.0, &[("P44", *a), ("P46", *b)]))
|
||
.collect();
|
||
let b = baseline_from_runs(None, &runs, &rules(5, false)).unwrap();
|
||
let m = &b["modes"]["full"];
|
||
assert_eq!(m["runs"], json!(5));
|
||
assert_eq!(m["date"], json!("2026-10-25"), "the latest run dates the baseline");
|
||
assert_eq!(m["run_list"].as_array().map(Vec::len), Some(5));
|
||
assert_eq!(m["run_list"][0]["load_avg_start"], json!([1.0, 1.0, 1.0]));
|
||
assert_eq!(m["run_list"][0]["contended"], json!(false));
|
||
let got = baseline_for(&b, "c", "full").unwrap();
|
||
let p44 = &got["P44"];
|
||
assert_eq!((p44.p50, p44.p50_max, p44.n), (Some(2300.0), Some(2868.0), 25));
|
||
assert_eq!(got["P46"].p50, Some(31.0));
|
||
assert_eq!(median(&mut [4.0, 1.0, 3.0, 2.0]), Some(2.5));
|
||
}
|
||
|
||
#[test]
|
||
fn baseline_from_runs_keeps_a_row_failed_in_any_run() {
|
||
let mut runs: Vec<Value> = (1..=3).map(|d| run_of(&format!("2026-10-2{d}"), 1.0, &[("P1", 3.0), ("P2", 5.0)])).collect();
|
||
runs[1]["scenarios"]["P1"] = json!({"status": format!("{MISSING_PREFIX}bench crashed")});
|
||
let b = baseline_from_runs(None, &runs, &rules(3, false)).unwrap();
|
||
let m = &b["modes"]["full"];
|
||
assert!(m["scenarios"].get("P1").is_none(), "no partial aggregate: {m}");
|
||
assert_eq!(m["failed"]["P1"], json!("bench crashed"));
|
||
assert_eq!(m["scenarios"]["P2"]["p50_ms"], json!(5.0));
|
||
assert!(m["failed"].get("P2").is_none());
|
||
}
|
||
|
||
#[test]
|
||
fn baseline_from_runs_leaves_out_a_row_missing_from_some_runs() {
|
||
let mut runs: Vec<Value> = (1..=3).map(|d| run_of(&format!("2026-10-2{d}"), 1.0, &[("P1", 3.0), ("P2", 5.0)])).collect();
|
||
runs[0]["scenarios"].as_object_mut().unwrap().remove("P1");
|
||
let b = baseline_from_runs(None, &runs, &rules(3, false)).unwrap();
|
||
let m = &b["modes"]["full"];
|
||
assert!(m["scenarios"].get("P1").is_none(), "2 of 3 runs is no baseline: {m}");
|
||
assert_eq!(m["scenarios"]["P2"]["p50_ms"], json!(5.0));
|
||
assert_eq!(m["runs"], json!(3));
|
||
}
|
||
|
||
#[test]
|
||
fn baseline_from_runs_rejects_mixed_classes_modes_dirty_and_too_few() {
|
||
let runs: Vec<Value> = (1..=5).map(|d| run_of(&format!("2026-10-2{d}"), 1.0, &[("P1", 3.0)])).collect();
|
||
assert!(baseline_from_runs(None, &runs, &rules(5, false)).is_ok());
|
||
assert!(baseline_from_runs(None, &runs[..4], &rules(5, false)).unwrap_err().contains("at least 5"));
|
||
let mut m = runs.clone();
|
||
m[2]["machine"]["class"] = json!("other");
|
||
assert!(baseline_from_runs(None, &m, &rules(5, false)).unwrap_err().contains("machine classes"));
|
||
let mut m = runs.clone();
|
||
m[2]["mode"] = json!("quick");
|
||
assert!(baseline_from_runs(None, &m, &rules(5, false)).unwrap_err().contains("modes"));
|
||
let mut m = runs.clone();
|
||
m[2]["dirty"] = json!(true);
|
||
assert!(baseline_from_runs(None, &m, &rules(5, false)).unwrap_err().contains("dirty"));
|
||
let mut m = runs.clone();
|
||
m[2]["load_avg_start"] = json!([9.5, 9.0, 9.0]);
|
||
assert!(baseline_from_runs(None, &m, &rules(5, false)).unwrap_err().contains("contended"));
|
||
assert_eq!(baseline_from_runs(None, &m, &rules(5, true)).unwrap()["modes"]["full"]["run_list"][2]["contended"], json!(true));
|
||
}
|
||
|
||
#[test]
|
||
fn regression_is_judged_against_the_noise_ceiling() {
|
||
// Runner numbers (macos-15, 7 nightlies): median and highest p50 per row.
|
||
let b = parse_budgets(TOML).unwrap();
|
||
let s = &b.settings;
|
||
let sc = &b.scenario[1]; // no enforced budget in play: opacity, budget 20, not enforced
|
||
let base = |p50: f64, max: f64| Stats { p50: Some(p50), p50_max: Some(max), ..Default::default() };
|
||
let cur = |p50: f64| Stats { p50: Some(p50), ..Default::default() };
|
||
assert_eq!(verdict(sc, &cur(3935.0), Some(&base(2174.0, 2868.0)), s, 15.0, false), Status::Regressed, "P44 +81 % (#5694)");
|
||
assert_eq!(verdict(sc, &cur(76.9), Some(&base(30.5, 45.3)), s, 15.0, false), Status::Regressed, "P46 +152 % (#5697)");
|
||
assert_eq!(verdict(sc, &cur(387.0), Some(&base(258.0, 448.0)), s, 15.0, false), Status::Tracked, "P11 +50 % (#5695) is within runner noise");
|
||
// A single-run baseline falls back to its p50.
|
||
let single = Stats { p50: Some(258.0), ..Default::default() };
|
||
assert_eq!(verdict(sc, &cur(387.0), Some(&single), s, 15.0, false), Status::Regressed);
|
||
}
|
||
|
||
#[test]
|
||
fn enforce_report_needs_margin_in_every_run() {
|
||
let toml = r#"
|
||
[[bench]]
|
||
name = "b"
|
||
package = "p"
|
||
|
||
[[scenario]]
|
||
id = "P16"
|
||
title = "dab"
|
||
bench = "b"
|
||
row = "r16"
|
||
budget_ms = 100
|
||
enforce = false
|
||
|
||
[[scenario]]
|
||
id = "P29"
|
||
title = "wand"
|
||
bench = "b"
|
||
row = "r29"
|
||
budget_ms = 100
|
||
enforce = false
|
||
|
||
[[scenario]]
|
||
id = "P30"
|
||
title = "liquify"
|
||
bench = "b"
|
||
row = "r30"
|
||
budget_ms = 100
|
||
enforce = false
|
||
|
||
[[scenario]]
|
||
id = "P25"
|
||
title = "cancel"
|
||
bench = "b"
|
||
row = "r25"
|
||
budget_ms = 200
|
||
|
||
[[scenario]]
|
||
id = "P99"
|
||
title = "track"
|
||
bench = "b"
|
||
row = "r99"
|
||
"#;
|
||
let b = parse_budgets(toml).unwrap();
|
||
// run_of stores p95 = 1.5 × p50 (the default metric is p95).
|
||
let runs: Vec<Value> = [(30.0, 20.0, 30.0, 100.0), (35.2, 22.0, 45.0, 110.0), (20.0, 22.0, 30.0, 199.4)]
|
||
.iter()
|
||
.enumerate()
|
||
.map(|(i, (a, c, d, e))| run_of(&format!("d{i}"), 1.0, &[("P16", *a), ("P29", *c), ("P30", *d), ("P25", *e), ("P99", 1.0)]))
|
||
.collect();
|
||
let (could, flaky) = enforce_report(&b, &runs);
|
||
assert_eq!(could.len(), 2, "{could:?}");
|
||
assert!(could[0].starts_with("P16 ") && could[0].contains("52.8"), "{could:?}");
|
||
assert!(could[1].starts_with("P29 "));
|
||
assert!(!could.iter().any(|l| l.starts_with("P30")), "67.5 > 60 once: not suggested");
|
||
assert_eq!(flaky.len(), 1, "{flaky:?}");
|
||
assert!(flaky[0].starts_with("P25 ") && flaky[0].contains("in 1 of 3 runs"), "{flaky:?}");
|
||
// A row missing from one run can't be vouched for.
|
||
let mut gap = runs.clone();
|
||
if let Some(o) = gap[1]["scenarios"].as_object_mut() {
|
||
o.remove("P29");
|
||
}
|
||
assert!(!enforce_report(&b, &gap).0.iter().any(|l| l.starts_with("P29")));
|
||
}
|
||
|
||
#[test]
|
||
fn baselines_never_cross_machine_classes() {
|
||
let results = json!({"mode": "full", "date": "d", "commit": "c", "machine": {"class": "macos-aarch64-m4"}, "scenarios": {"P1": {"p50_ms": 3.0, "p95_ms": 4.0, "max_ms": 5.0, "n": 3}, "P3": {"status": "not measurable yet"}}});
|
||
let b = one(None, &results);
|
||
let got = baseline_for(&b, "macos-aarch64-m4", "full").unwrap();
|
||
assert_eq!(got.get("P1").and_then(|s| s.p50), Some(3.0));
|
||
assert!(!got.contains_key("P3"), "rows without numbers are not baselined");
|
||
assert!(baseline_for(&b, "linux-x86-64-epyc", "full").is_err());
|
||
assert!(baseline_for(&b, "macos-aarch64-m4", "quick").is_err());
|
||
// Same class: the other mode is kept. New class: replaced.
|
||
let quick = json!({"mode": "quick", "machine": {"class": "macos-aarch64-m4"}, "scenarios": {"P1": {"p50_ms": 1.0}}});
|
||
let b2 = one(Some(&b), &quick);
|
||
assert!(baseline_for(&b2, "macos-aarch64-m4", "full").is_ok());
|
||
assert!(baseline_for(&b2, "macos-aarch64-m4", "quick").is_ok());
|
||
// A crashed row is kept as a failure, and the scorecard shows it.
|
||
let crashed = json!({"mode": "full", "machine": {"class": "c"}, "scenarios": {"P1": {"status": "**missing**: panicked: boom"}}});
|
||
let bc = one(None, &crashed);
|
||
let b = parse_budgets(TOML).unwrap();
|
||
let o = outcomes_from_baseline(&b, Some(&bc));
|
||
assert_eq!(o[0].status, Status::Missing("panicked: boom".into()));
|
||
assert_eq!(o[1].status, Status::NotRun);
|
||
let other = json!({"mode": "quick", "machine": {"class": "other"}, "scenarios": {}});
|
||
let b3 = one(Some(&b2), &other);
|
||
assert!(baseline_for(&b3, "other", "full").is_err());
|
||
}
|
||
|
||
#[test]
|
||
fn machine_helpers() {
|
||
assert_eq!(slug("Apple M4 Pro (Metal)"), "apple-m4-pro-metal");
|
||
assert_eq!(machine_class("macos", "aarch64", "Apple M4 Pro", "Apple M4 Pro"), "macos-aarch64-apple-m4-pro-apple-m4-pro");
|
||
assert_eq!(parse_loadavg("{ 2.10 1.50 0.75 }"), Some([2.1, 1.5, 0.75]));
|
||
assert_eq!(parse_loadavg("0.52 0.58 0.59 1/467 12345\n"), Some([0.52, 0.58, 0.59]));
|
||
assert_eq!(parse_loadavg("garbage"), None);
|
||
assert_eq!(iso_utc(0), "1970-01-01T00:00:00Z");
|
||
assert_eq!(iso_utc(1_791_158_400), "2026-10-05T00:00:00Z");
|
||
assert_eq!(iso_utc(951_782_400), "2000-02-29T00:00:00Z");
|
||
}
|
||
|
||
#[test]
|
||
fn table_lists_not_measurable_rows_separately() {
|
||
let b = parse_budgets(TOML).unwrap();
|
||
let mut reports = BTreeMap::new();
|
||
reports.insert("b".to_string(), Ok(report(json!([{"name": "move row", "samples_ms": [10.0], "peak_rss_bytes": 3u64 << 30}]))));
|
||
let t = render_table(&evaluate(&b, &reports, None, 15.0, true), true);
|
||
assert!(t.contains("| P1 | move | ≤ 25 ms p95 | 10.0 | 10.0 | 10.0 | 3.0 GB |"));
|
||
assert!(t.contains("**Not measurable yet**"));
|
||
assert!(t.contains("| P3 | zoom | ≤ 16 ms per frame | needs a frame harness | |"));
|
||
assert_eq!(counts(&evaluate(&b, &reports, None, 15.0, true)), (1, 1, 0, 1), "the row the bench lacks counts as failing");
|
||
}
|
||
|
||
#[test]
|
||
fn unknown_bench_is_rejected_before_updating_the_baseline() {
|
||
let root = std::env::temp_dir().join(format!("photocraft-perf-unknown-bench-{}", std::process::id()));
|
||
std::fs::create_dir(&root).unwrap();
|
||
std::fs::create_dir(root.join("perf")).unwrap();
|
||
std::fs::write(root.join(BUDGETS), "[[bench]]\nname = \"known\"\npackage = \"unused\"\nrequires = \"missing.rs\"\n").unwrap();
|
||
let baseline = "{\"schema\":1,\"sentinel\":true}";
|
||
std::fs::write(root.join(BASELINE), baseline).unwrap();
|
||
let results = [
|
||
run(&root, &["--bench", "unknown", "--reuse", "--update-baseline"]),
|
||
run(&root, &["--bench", "known", "--bench", "unknown", "--quick", "--reuse", "--update-baseline"]),
|
||
];
|
||
let after = std::fs::read_to_string(root.join(BASELINE)).unwrap();
|
||
std::fs::remove_dir_all(root).unwrap();
|
||
for result in results {
|
||
assert_eq!(result, Err(format!("perf: unknown bench `unknown` (see {BUDGETS})")));
|
||
}
|
||
assert_eq!(after, baseline, "an unknown bench must not update the baseline");
|
||
}
|
||
|
||
#[test]
|
||
fn options() {
|
||
assert!(parse_opts(&["--bogus"]).is_err());
|
||
assert!(parse_opts(&["--threshold", "x"]).is_err());
|
||
assert!(parse_opts(&["--threshold", "-3"]).is_err());
|
||
let o = parse_opts(&["--quick", "--threshold", "20", "--bench", "a", "--reuse"]).unwrap();
|
||
assert!(o.quick && o.reuse && o.skip_build);
|
||
assert_eq!(o.threshold, Some(20.0));
|
||
assert_eq!(o.benches, ["a"]);
|
||
assert!(!o.allow_contended && o.from.is_empty());
|
||
let o = parse_opts(&["--update-baseline", "--from", "a.json", "--from", "b.json", "--allow-contended"]).unwrap();
|
||
assert!(o.update_baseline && o.allow_contended);
|
||
assert_eq!(o.from, [PathBuf::from("a.json"), PathBuf::from("b.json")]);
|
||
assert!(parse_opts(&["--from", "a.json"]).is_err(), "--from needs --update-baseline");
|
||
assert!(parse_opts(&["--update-baseline", "--from"]).is_err());
|
||
}
|
||
}
|