diff --git a/README.md b/README.md index 3021a76..4590339 100644 --- a/README.md +++ b/README.md @@ -86,14 +86,17 @@ scored. ## Polling and cache -- The app polls every `poll_minutes` (default 5), four repos at a time. Each - repo costs about five GraphQL requests and one REST request: roughly 65 of - the 5,000 GraphQL points GitHub allows per hour per cycle. +- The app polls every `poll_minutes` (default 5), two repos at a time. Each + repo costs about six GraphQL requests and one REST request: roughly 100 of + the 5,000 GraphQL points GitHub allows per hour per cycle. It never uses + search, which GitHub limits far more tightly (to about 30 a minute); PR and + issue counts per window come from walking the recent PRs and issues. - Each repo's icon is `assets/app-icon/hicolor/64x64/apps/*.png` on its default branch. It is downloaded once and again only when the file changes; it's kept in the cache. Repos without one (libraries) get a lettered tile. - If fewer than 150 points remain, polling slows to every 30 minutes. If - GitHub rate-limits the app, it waits three intervals. If every repo fails + GitHub rate-limits the app, it stops that poll at once and waits three + intervals. If every repo fails (offline, or just woke from sleep), it retries after one minute. - Timing follows the wall clock, so a laptop that wakes from sleep polls within about 15 s. diff --git a/craft_status_config.template.toml b/craft_status_config.template.toml index d9ce547..71f5e1a 100644 --- a/craft_status_config.template.toml +++ b/craft_status_config.template.toml @@ -22,7 +22,7 @@ repos = [ ] # How often to poll GitHub (minutes, at least 1). Each poll costs roughly -# 5 GraphQL requests + 1 REST request per repo, far below GitHub's 5000/hour. +# 6 GraphQL requests + 1 REST request per repo, far below GitHub's 5000/hour. poll_minutes = 5.0 # Data older than this is flagged STALE in the window (never less than poll_minutes). diff --git a/crates/craft-github/src/lib.rs b/crates/craft-github/src/lib.rs index f34c30b..15edc74 100644 --- a/crates/craft-github/src/lib.rs +++ b/crates/craft-github/src/lib.rs @@ -1,12 +1,14 @@ //! Fetches one repository's [`RepoStats`] from GitHub. //! //! Per repo: one (usually) GraphQL query for releases + asset download -//! counts; one GraphQL query for counts, dates, every recent-activity window -//! ([`WINDOWS`], plus [`SINCE_RELEASE`], which needs the releases), the app -//! icon's blob id and the open issues to rank; one GraphQL query comparing the -//! latest build's tag with `main`; one or more GraphQL queries for the commit -//! authors since a week ago or the latest build, whichever is earlier, which -//! give each window's people; +//! counts; one GraphQL query for counts, dates, each recent-activity window's +//! commits ([`WINDOWS`], plus [`SINCE_RELEASE`], which needs the releases), +//! the app icon's blob id and the open issues to rank; one GraphQL query +//! comparing the latest build's tag with `main`; one or more GraphQL queries +//! for the commit authors, and one or more for the PRs opened and merged and +//! issues opened, since a week ago or the latest build, whichever is earlier, +//! which give each window's people and PR/issue counts. No search API: GitHub +//! limits searches far more tightly than other queries, even inside GraphQL; //! one (usually) REST call for the contributor list, which GraphQL lacks. //! The icon itself is a REST blob fetch, made only when its blob id changes //! (see [`GitHub::icon`]). Blocking I/O: callers run this on worker threads. @@ -125,12 +127,16 @@ impl GitHub { .get("x-ratelimit-remaining") .and_then(|v| v.to_str().ok()) .map(str::to_string); - let reset = resp - .headers() - .get("x-ratelimit-reset") - .and_then(|v| v.to_str().ok()) - .and_then(|v| v.parse::().ok()) - .and_then(|t| DateTime::from_timestamp(t, 0)); + let header = |name: &str| { + resp.headers() + .get(name) + .and_then(|v| v.to_str().ok()) + .and_then(|v| v.parse::().ok()) + }; + // Secondary (burst) limits say when to retry; the hourly one when it resets. + let reset = header("retry-after") + .map(|secs| Utc::now() + chrono::Duration::seconds(secs)) + .or_else(|| header("x-ratelimit-reset").and_then(|t| DateTime::from_timestamp(t, 0))); let body = resp.text().unwrap_or_default(); if (status.as_u16() == 403 || status.as_u16() == 429) && (remaining.as_deref() == Some("0") || body.contains("rate limit")) @@ -181,16 +187,26 @@ impl GitHub { let releases = self.releases(owner, name)?; let release_at = summarize_releases(&releases).latest_at; let main = self.graphql( - &main_query(repo, now, release_at), + &main_query(now, release_at), json!({ "owner": owner, "name": name }), )?; let since_build = self.since_build(owner, name, &main, &releases)?; - let authors = self.authors(owner, name, now, release_at)?; + let since = activity_start(now, release_at); + let authors = self.authors(owner, name, since)?; + let events = self.events(owner, name, since)?; let contributors = self.contributors(repo)?; let budget = Budget { graphql_remaining: main["rateLimit"]["remaining"].as_u64(), }; - let stats = build_stats(&main, &releases, since_build, &contributors, &authors, now); + let stats = build_stats( + &main, + &releases, + since_build, + &contributors, + &authors, + &events, + now, + ); Ok((stats, budget)) } @@ -210,19 +226,9 @@ impl GitHub { Ok(format!("data:image/png;base64,{b64}")) } - /// `(committed, author)` for each commit on `main` in the longest window, - /// or since the latest build if that's longer. - fn authors( - &self, - owner: &str, - name: &str, - now: DateTime, - release_at: Option>, - ) -> Result> { - let longest = WINDOWS.iter().map(|(_, m)| *m).max().unwrap_or(0); - let since = (now - chrono::Duration::minutes(longest)) - .min(release_at.unwrap_or(now)) - .to_rfc3339_opts(chrono::SecondsFormat::Secs, true); + /// `(committed, author)` for each commit on `main` since `since`. + fn authors(&self, owner: &str, name: &str, since: DateTime) -> Result> { + let since = since.to_rfc3339_opts(chrono::SecondsFormat::Secs, true); let mut out = Vec::new(); let mut after: Option = None; // 10 pages × 100 commits; a busier stretch undercounts its oldest people. @@ -241,6 +247,46 @@ impl GitHub { Ok(out) } + /// When each PR was opened or merged, and each issue opened, since + /// `since`: the three lists walked newest first, each until it passes + /// `since`. Merged PRs are walked by last update, which merging bumps. + fn events(&self, owner: &str, name: &str, since: DateTime) -> Result { + let mut ev = Events::default(); + let mut more = [true; 3]; + let mut after: [Option; 3] = Default::default(); + // 30 pages × 100 of each (photocraft opens ~2000 PRs a week); a busier + // stretch undercounts its oldest windows. + for _ in 0..30 { + if !more.iter().any(|m| *m) { + break; + } + let v = self.graphql( + EVENTS_QUERY, + json!({ + "owner": owner, "name": name, + "opened": more[0], "merged": more[1], "issues": more[2], + "openedAfter": after[0], "mergedAfter": after[1], "issuesAfter": after[2], + }), + )?; + let lists = [ + ("opened", "createdAt", "createdAt", &mut ev.prs_opened), + ("merged", "updatedAt", "mergedAt", &mut ev.prs_merged), + ("issues", "createdAt", "createdAt", &mut ev.issues_opened), + ]; + for (i, (alias, order, field, out)) in lists.into_iter().enumerate() { + if !more[i] { + continue; + } + let conn = &v["repository"][alias]; + let (times, done) = parse_events(conn, order, field, since); + out.extend(times); + more[i] = !done && conn["pageInfo"]["hasNextPage"].as_bool() == Some(true); + after[i] = conn["pageInfo"]["endCursor"].as_str().map(str::to_string); + } + } + Ok(ev) + } + /// Commits on `main` since the latest build; `None` without a build, a /// tagged commit or a default branch. fn since_build( @@ -468,11 +514,42 @@ fn parse_issues(conn: &Value) -> Vec { .unwrap_or_default() } -fn search_count(main: &Value, alias: &str) -> u64 { - main[alias]["issueCount"].as_u64().unwrap_or(0) +/// When PRs were opened and merged and issues opened, since +/// [`activity_start`]; [`recent_activity`] buckets them into windows. +#[derive(Debug, Clone, Default)] +pub struct Events { + pub prs_opened: Vec>, + pub prs_merged: Vec>, + pub issues_opened: Vec>, } -/// Reads the per-window aliases [`main_query`] asked for. A repo with no +/// The start of the longest window, or of the latest build if that's earlier. +fn activity_start(now: DateTime, release_at: Option>) -> DateTime { + let longest = WINDOWS.iter().map(|(_, m)| *m).max().unwrap_or(0); + (now - chrono::Duration::minutes(longest)).min(release_at.unwrap_or(now)) +} + +/// The `field` times at or after `since` from one page of `conn`, whose +/// nodes are sorted newest first by `order`; `true` once the page has reached +/// nodes older than `since`, so later pages can't hold any. +pub fn parse_events( + conn: &Value, + order: &str, + field: &str, + since: DateTime, +) -> (Vec>, bool) { + let mut out = Vec::new(); + for n in conn["nodes"].as_array().into_iter().flatten() { + if time(&n[order]).is_some_and(|t| t < since) { + return (out, true); + } + out.extend(time(&n[field]).filter(|t| *t >= since)); + } + (out, false) +} + +/// Reads the per-window commit counts [`main_query`] asked for and buckets +/// `events` by window. A repo with no /// default branch still gets every window (with zero commits). /// The all-time totals come along as the [`ALL_TIME`] window; its people are /// GitHub's contributors plus any newer `authors`, the others' the distinct @@ -483,6 +560,7 @@ fn recent_activity( main: &Value, contributors: &[String], authors: &[CommitAuthor], + events: &Events, release_at: Option>, unreleased: Option, now: DateTime, @@ -503,11 +581,13 @@ fn recent_activity( .collect::>() .len() as u64, }; + let after = + |times: &[DateTime], since| times.iter().filter(|t| **t >= since).count() as u64; let window = |key: &str, since: DateTime| Activity { commits: count(&head[format!("c{key}")]), - prs_opened: search_count(main, &format!("p{key}")), - prs_merged: search_count(main, &format!("m{key}")), - issues_opened: search_count(main, &format!("i{key}")), + prs_opened: after(&events.prs_opened, since), + prs_merged: after(&events.prs_merged, since), + issues_opened: after(&events.issues_opened, since), people: authors .iter() .filter(|(at, _)| *at >= since) @@ -545,30 +625,16 @@ fn windows( .chain(release_at.map(|at| (SINCE_RELEASE, "r".to_string(), at))) } -/// [`MAIN_QUERY`] with one commit-history count and three searches per -/// window, aliased by window index (`c0`, `p0`, `m0`, `i0`…), plus the -/// [`SINCE_RELEASE`] window (`cr`, `pr`…) when there is a build published at -/// `release_at`. GitHub charges the whole query one rate-limit point however -/// many windows it holds. -pub fn main_query(repo: &str, now: DateTime, release_at: Option>) -> String { +/// [`MAIN_QUERY`] with one commit-history count per window, aliased by +/// window index (`c0`, `c1`…), plus the [`SINCE_RELEASE`] window (`cr`) when +/// there is a build published at `release_at`. +pub fn main_query(now: DateTime, release_at: Option>) -> String { let mut history = String::new(); - let mut searches = String::new(); for (_, i, since) in windows(now, release_at) { let since = since.to_rfc3339_opts(chrono::SecondsFormat::Secs, true); history += &format!("c{i}: history(since: \"{since}\") {{ totalCount }}\n"); - for (alias, filter) in [ - ("p", format!("is:pr created:>={since}")), - ("m", format!("is:pr merged:>={since}")), - ("i", format!("is:issue created:>={since}")), - ] { - let q = Value::from(format!("repo:{repo} {filter}")); - searches += - &format!(" {alias}{i}: search(query: {q}, type: ISSUE) {{ issueCount }}\n"); - } } - MAIN_QUERY - .replace("{HISTORY}", &history) - .replace("{SEARCHES}", &searches) + MAIN_QUERY.replace("{HISTORY}", &history) } /// Pure: turn the GraphQL payloads into the model (unit-tested below). @@ -578,6 +644,7 @@ pub fn build_stats( since_build: Option, contributors: &[String], authors: &[CommitAuthor], + events: &Events, now: DateTime, ) -> RepoStats { let r = &main["repository"]; @@ -598,6 +665,7 @@ pub fn build_stats( main, contributors, authors, + events, rel.latest_at, since_build.as_ref().map(|l| l.commits), now, @@ -644,7 +712,7 @@ query($owner: String!, $name: String!) { } } } -{SEARCHES} rateLimit { remaining resetAt } + rateLimit { remaining resetAt } } fragment I on Issue { number title url createdAt updatedAt authorAssociation @@ -686,6 +754,28 @@ query($owner: String!, $name: String!, $since: GitTimestamp!, $after: String) { } "#; +/// One page each of PRs by creation, merged PRs by last update and issues by +/// creation, newest first; `@include` drops the lists already walked far enough. +const EVENTS_QUERY: &str = r#" +query($owner: String!, $name: String!, $opened: Boolean!, $merged: Boolean!, $issues: Boolean!, + $openedAfter: String, $mergedAfter: String, $issuesAfter: String) { + repository(owner: $owner, name: $name) { + opened: pullRequests(first: 100, after: $openedAfter, orderBy: {field: CREATED_AT, direction: DESC}) @include(if: $opened) { + pageInfo { hasNextPage endCursor } + nodes { createdAt } + } + merged: pullRequests(states: MERGED, first: 100, after: $mergedAfter, orderBy: {field: UPDATED_AT, direction: DESC}) @include(if: $merged) { + pageInfo { hasNextPage endCursor } + nodes { updatedAt mergedAt } + } + issues: issues(first: 100, after: $issuesAfter, orderBy: {field: CREATED_AT, direction: DESC}) @include(if: $issues) { + pageInfo { hasNextPage endCursor } + nodes { createdAt } + } + } +} +"#; + /// The latest build's tag compared with `main`, plus a by-date count for /// when the two share no history (see [`parse_since_build`]). const BUILD_QUERY: &str = r#" @@ -727,10 +817,20 @@ mod tests { "defaultBranchRef": {"target": { "all": {"totalCount": 1234, "nodes": [{"committedDate": "2026-10-08T03:00:00Z"}]}, "c0": {"totalCount": 1}, "c3": {"totalCount": 9}, "cr": {"totalCount": 99}}} - }, - "p3": {"issueCount": 2}, "m3": {"issueCount": 3}, "i3": {"issueCount": 4}, "m6": {"issueCount": 30}, - "pr": {"issueCount": 11}, "mr": {"issueCount": 12}, "ir": {"issueCount": 13} + } }); + let at = |t: &str, n| vec![t.parse::>().unwrap(); n]; + // Inside the last 4 h; inside the last week; before that but after v1.0.0. + let events = Events { + prs_opened: [at("2026-10-08T01:00:00Z", 2), at("2026-10-02T00:00:00Z", 9)].concat(), + prs_merged: [ + at("2026-10-08T01:00:00Z", 3), + at("2026-10-02T00:00:00Z", 27), + at("2026-10-01T02:00:00Z", 1), + ] + .concat(), + issues_opened: [at("2026-10-08T01:00:00Z", 4), at("2026-10-01T02:00:00Z", 9)].concat(), + }; let rel = parse_releases(&json!({"nodes": [ {"tagName": "v1.1.0", "isDraft": true, "isPrerelease": false, "publishedAt": null, "createdAt": "2026-10-08T00:00:00Z", "releaseAssets": {"nodes": [{"name": "a.dmg", "downloadCount": 0}]}}, @@ -754,7 +854,15 @@ mod tests { parse_contributors(&json!([{"login": "ann"}, {"email": "Old@Z", "type": "Anonymous"}])); people.extend((0..10).map(|i| format!("p{i}"))); assert_eq!(people[1], "old@z"); - let s = build_stats(&main, &rel, Some(lag.clone()), &people, &authors, now); + let s = build_stats( + &main, + &rel, + Some(lag.clone()), + &people, + &authors, + &events, + now, + ); assert_eq!((s.open_prs, s.open_issues), (3, 7)); assert_eq!(s.recent.len(), WINDOWS.len() + 2); let a = |id: &str| s.recent[id]; @@ -787,13 +895,13 @@ mod tests { // The Unreleased count, not commits dated after publishing. commits: 5, prs_opened: 11, - prs_merged: 12, + prs_merged: 31, issues_opened: 13, // Cy committed on the 3rd, after v1.0.0 but before the last week. people: 3, } ); - let no_lag = build_stats(&main, &rel, None, &people, &authors, now); + let no_lag = build_stats(&main, &rel, None, &people, &authors, &events, now); assert_eq!(no_lag.recent[SINCE_RELEASE].commits, 99); assert_eq!(s.icon_oid.as_deref(), Some("i1")); assert_eq!(s.latest_build.as_deref(), Some("v1.0.0")); @@ -845,15 +953,42 @@ mod tests { #[test] fn query_asks_for_every_window() { let now: DateTime = "2026-10-08T04:00:00Z".parse().unwrap(); - let q = main_query("o/r", now, None); + let q = main_query(now, None); assert!(q.contains(r#"c3: history(since: "2026-10-08T00:00:00Z")"#)); - assert!(q.contains(r#"m6: search(query: "repo:o/r is:pr merged:>=2026-10-01T04:00:00Z""#)); - assert!(!q.contains("{HISTORY}") && !q.contains("{SEARCHES}")); + assert!(q.contains(r#"c6: history(since: "2026-10-01T04:00:00Z")"#)); + assert!(!q.contains("{HISTORY}") && !q.contains("search(")); assert!(!q.contains("cr:"), "no build, no since-release window"); - let q = main_query("o/r", now, Some("2026-09-20T12:00:00Z".parse().unwrap())); + let q = main_query(now, Some("2026-09-20T12:00:00Z".parse().unwrap())); assert!(q.contains(r#"cr: history(since: "2026-09-20T12:00:00Z")"#)); - assert!( - q.contains(r#"ir: search(query: "repo:o/r is:issue created:>=2026-09-20T12:00:00Z""#) + } + + #[test] + fn events_stop_at_the_window_start() { + let since: DateTime = "2026-10-01T00:00:00Z".parse().unwrap(); + let t = |s: &str| s.parse::>().unwrap(); + let merged = json!({"nodes": [ + {"updatedAt": "2026-10-08T00:00:00Z", "mergedAt": "2026-10-07T00:00:00Z"}, + // Updated lately but merged long ago: skipped, and the walk goes on. + {"updatedAt": "2026-10-06T00:00:00Z", "mergedAt": "2026-09-01T00:00:00Z"}, + {"updatedAt": "2026-10-02T00:00:00Z", "mergedAt": "2026-10-01T00:00:00Z"}, + {"updatedAt": "2026-09-30T00:00:00Z", "mergedAt": "2026-09-30T00:00:00Z"}, + {"updatedAt": "2026-10-05T00:00:00Z", "mergedAt": "2026-10-05T00:00:00Z"} + ]}); + assert_eq!( + parse_events(&merged, "updatedAt", "mergedAt", since), + ( + vec![t("2026-10-07T00:00:00Z"), t("2026-10-01T00:00:00Z")], + true + ) + ); + let opened = json!({"nodes": [{"createdAt": "2026-10-03T00:00:00Z"}]}); + assert_eq!( + parse_events(&opened, "createdAt", "createdAt", since), + (vec![t("2026-10-03T00:00:00Z")], false) + ); + assert_eq!( + parse_events(&Value::Null, "createdAt", "createdAt", since), + (vec![], false) ); } @@ -865,6 +1000,7 @@ mod tests { None, &[], &[], + &Events::default(), Utc::now(), ); assert_eq!(s.recent.len(), WINDOWS.len() + 1); diff --git a/crates/craft-status-app/src/lib.rs b/crates/craft-status-app/src/lib.rs index 9f21ea0..9e3cdc2 100644 --- a/crates/craft-status-app/src/lib.rs +++ b/crates/craft-status-app/src/lib.rs @@ -25,9 +25,9 @@ const TRAY: &str = "main"; const EV_SNAPSHOT: &str = "craft:snapshot"; const EV_FOCUS: &str = "craft:focus"; const EV_META: &str = "craft:meta"; -/// Repositories fetched concurrently. Small on purpose: GitHub frowns on -/// bursts (secondary rate limits) and 12 repos finish in well under 15 s. -const WORKERS: usize = 4; +/// Repositories fetched concurrently. Small on purpose: GitHub's secondary +/// rate limits punish bursts, and a poll has minutes to finish. +const WORKERS: usize = 2; /// How often the poll thread wakes to check the clock, the config file and /// the tray tooltip. Wall-clock based, so a sleeping laptop polls on wake. const TICK: Duration = Duration::from_secs(15); @@ -247,9 +247,14 @@ fn run_cycle(app: &AppHandle) -> i64 { let queue = Mutex::new(cfg.repos.iter().rev().cloned().collect::>()); let failures = Mutex::new(Vec::::new()); + // Once GitHub rate-limits one repo, more requests only prolong it. + let halted = AtomicBool::new(false); std::thread::scope(|scope| { for _ in 0..WORKERS.min(cfg.repos.len()) { scope.spawn(|| loop { + if halted.load(Ordering::Relaxed) { + break; + } let Some(repo) = queue.lock().unwrap().pop() else { break; }; @@ -309,6 +314,9 @@ fn run_cycle(app: &AppHandle) -> i64 { entry.inaccessible = true; } Err(e) => { + if matches!(e, GhError::RateLimited(_)) { + halted.store(true, Ordering::Relaxed); + } entry.error = Some(e.to_string()); failures.lock().unwrap().push(e); } diff --git a/ui/app.js b/ui/app.js index 03f8888..f4c49d4 100644 --- a/ui/app.js +++ b/ui/app.js @@ -136,14 +136,23 @@ function cell(entry, col) { // ───────────────────────────── rendering ───────────────────────────── +/** The group row, built once: rebuilding it would destroy the window + * dropdown, closing it mid-choice whenever a poll or the clock re-renders. */ +let groupRow = null; + function renderHead() { - const groups = el('tr', { class: 'groups' }); - let prev = null; - for (const c of COLUMNS) { - if (c.group === prev) { groups.lastChild.colSpan += 1; continue; } - prev = c.group; - groups.append(el('th', { class: c.group ? 'group-start' : '' }, c.group === RECENT ? windowPicker() : c.group)); + if (!groupRow) { + groupRow = el('tr', { class: 'groups' }); + let prev = null; + for (const c of COLUMNS) { + if (c.group === prev) { groupRow.lastChild.colSpan += 1; continue; } + prev = c.group; + groupRow.append(el('th', { class: c.group ? 'group-start' : '' }, c.group === RECENT ? windowPicker() : c.group)); + } + $('thead').replaceChildren(groupRow); } + const picker = groupRow.querySelector('.window-picker'); + if (picker.value !== state.window) picker.value = state.window; const cols = el('tr', { class: 'cols' }); COLUMNS.forEach((c, i) => { const sorted = state.sort.key === c.key; @@ -160,7 +169,8 @@ function renderHead() { }, }, c.label)); }); - $('thead').replaceChildren(groups, cols); + groupRow.nextSibling?.remove(); + $('thead').append(cols); } /** The dropdown that heads the recent-activity columns. */ @@ -171,8 +181,6 @@ function windowPicker() { title: 'Time window for the activity columns', onchange: (e) => { state.window = e.target.value; save('window', state.window); render(); }, }, WINDOWS.map((w) => el('option', { value: w.id }, w.label))); - sel.value = state.window; - return sel; } function groupStart(i) {