diff --git a/apps/figcraft-server/src/library_assets_tests.rs b/apps/figcraft-server/src/library_assets_tests.rs index 5941e4980..d5d6744e9 100644 --- a/apps/figcraft-server/src/library_assets_tests.rs +++ b/apps/figcraft-server/src/library_assets_tests.rs @@ -5,12 +5,15 @@ use std::time::{Duration, Instant}; -use figcraft_collab::message::{Access, AssetUsage, ClientMsg, LibraryReply, LibraryReq, MAX_ASSET_USAGE, OpenLibrary, PROTOCOL, ServerMsg}; +use figcraft_collab::message::{ + Access, AssetActions, AssetUsage, ClientMsg, LibraryReply, LibraryReq, MAX_ASSET_USAGE, OpenLibrary, PROTOCOL, ServerMsg, Usage, +}; use figcraft_collab_net::Invite; use figcraft_doc::NodeId; use figcraft_engine::Session; use serde_json::{Value, json}; +use crate::store::{MAX_USERS, Store}; use crate::test_util::{Raw, TempDir, config, publish, session}; fn ex(s: &mut Session, cmd: &str, p: Value) -> Value { @@ -185,3 +188,34 @@ fn inserts_and_detaches_count_per_style_and_variable_in_the_period() { assert_eq!(u["variables"][0]["insertions"], 1, "{u}"); server.stop(); } + +#[test] +fn evicting_capped_usage_rows_keeps_asset_peaks_so_a_repeat_report_is_not_counted_twice() { + let dir = TempDir::new("library-asset-peaks-evict"); + let cfg = config(dir.path()); + let mut server = crate::start(&cfg).unwrap(); + let addr = server.addr(); + let mut a = session("Ada"); + let r = rect(&mut a, 0); + ex(&mut a, "style.create", json!({"type": "paint", "name": "Brand", "from": r.0})); + let p = publish(&mut a, dir.path(), addr); + ex(&mut a, "library.publishToServer", json!({})); + let file = Invite::parse(&p.edit).unwrap().file; + server.stop(); + drop(server); + + let mut store = Store::open(dir.path(), cfg.store()).unwrap(); + for i in 0..MAX_USERS { + store.report_usage(&file, Usage { file: format!("Old{i}"), components: Vec::new() }).unwrap(); + } + let actions = AssetActions { file: "Kept".into(), inserted: vec![("Brand".into(), 2)], detached: Vec::new() }; + store.report_asset_actions(&file, actions.clone()).unwrap(); + // One more usage-only file would evict "Kept" if asset activity did not count for retention. + store.report_usage(&file, Usage { file: "Extra".into(), components: Vec::new() }).unwrap(); + store.report_asset_actions(&file, actions).unwrap(); + let weeks = store.library_asset_actions(&file).unwrap(); + let brand: Vec<_> = weeks.iter().filter(|w| w.component == "Brand").collect(); + assert_eq!(brand.len(), 1, "{weeks:?}"); + assert_eq!(brand[0].insertions, 2, "lost peaks would grow the same totals again: {weeks:?}"); + assert_eq!(brand[0].detachments, 0, "{weeks:?}"); +} diff --git a/apps/figcraft-server/src/store.rs b/apps/figcraft-server/src/store.rs index fbf4bc2db..d75a24f1b 100644 --- a/apps/figcraft-server/src/store.rs +++ b/apps/figcraft-server/src/store.rs @@ -103,7 +103,7 @@ pub const NO_PUBLIC_LIBRARY: &str = "on this public server only files published /// Most files whose usage of one library is kept (the least recently changed go first), and most /// components one file's report keeps. -const MAX_USERS: usize = 200; +pub(crate) const MAX_USERS: usize = 200; const MAX_REPORTED: usize = 1_000; /// Largest `usage.json` (and `actions.json`) read. const MAX_USAGE_FILE: u64 = 16 << 20; @@ -325,6 +325,9 @@ struct UsageEntry { variables: Vec<(String, u32)>, #[serde(default, skip_serializing_if = "Vec::is_empty")] asset_peaks: Vec<(String, u32, u32)>, + /// When this file last reported style or variable insertions or detachments (seconds since 1970, UTC). + #[serde(default, skip_serializing_if = "is_zero")] + asset_updated: u64, } fn is_zero(n: &u64) -> bool { diff --git a/apps/figcraft-server/src/store/library.rs b/apps/figcraft-server/src/store/library.rs index 10337a451..4cb32f4e6 100644 --- a/apps/figcraft-server/src/store/library.rs +++ b/apps/figcraft-server/src/store/library.rs @@ -256,7 +256,13 @@ impl Store { let mut grown = Vec::new(); self.report(file, &actions.file, known_assets(counts), |e, counts| { grown = grow(&mut e.asset_peaks, &counts, kind, |_| 0); - !grown.is_empty() + if grown.is_empty() { + return false; + } + // Heard from for retention: evicting the row would drop asset_peaks, the + // baseline grow uses so a later identical report is not counted twice. + e.asset_updated = now(); + true })?; if let (false, Some(u)) = (grown.is_empty(), self.usage.get_mut(file)) { add_to_week(&mut u.asset_weeks, &grown, kind, now()); @@ -329,7 +335,7 @@ impl Store { let at = match u.list.iter().position(|e| e.usage.file == name) { Some(at) => at, None => { - u.list.push(UsageEntry { usage: Usage { file: name, components: Vec::new() }, ..UsageEntry::default() }); + u.list.push(UsageEntry { usage: Usage { file: name.clone(), components: Vec::new() }, ..UsageEntry::default() }); u.list.len() - 1 } }; @@ -338,7 +344,13 @@ impl Store { return Ok(()); } if u.list.len() > MAX_USERS { - u.list.sort_by_key(|e| std::cmp::Reverse(e.updated.max(e.detached_updated))); + // The file just heard from stays, even when several share a timestamp (now() is + // seconds). Asset-action reports use asset_updated so their peaks are not dropped + // in favour of a usage-only row from the same second. + u.list.sort_by_key(|e| { + let recent = e.usage.file == name; + std::cmp::Reverse((e.updated.max(e.detached_updated).max(e.asset_updated), recent)) + }); u.list.truncate(MAX_USERS); } u.dirty = true; diff --git a/crates/engine/src/assets/library_server.rs b/crates/engine/src/assets/library_server.rs index 3d42e7c98..f8ddf0e50 100644 --- a/crates/engine/src/assets/library_server.rs +++ b/crates/engine/src/assets/library_server.rs @@ -124,14 +124,42 @@ pub fn asset_usage(doc: &Document, library: &str) -> Option { } /// How many times `doc` has inserted and detached `library`'s styles and variables, its totals; -/// `None` when it has done neither. +/// `None` when it has done neither. Every count the document keeps is included (up to +/// [`figcraft_doc::MAX_DETACH_COUNTS`]); the wire message is capped at [`MAX_ASSET_USAGE`], so +/// [`asset_action_reports`] splits them into batches an older server still accepts. pub fn asset_actions(doc: &Document, library: &str) -> Option { let counts: Vec<_> = doc.libraries.asset_counts.iter().filter(|c| c.library == library).collect(); - let totals = - |n: fn(&figcraft_doc::AssetCount) -> u32| counts.iter().filter(|c| n(c) > 0).map(|c| (c.key.clone(), n(c))).take(MAX_ASSET_USAGE).collect(); + let totals = |n: fn(&figcraft_doc::AssetCount) -> u32| counts.iter().filter(|c| n(c) > 0).map(|c| (c.key.clone(), n(c))).collect(); (!counts.is_empty()).then(|| AssetActions { file: doc.name.clone(), inserted: totals(|c| c.inserted), detached: totals(|c| c.detached) }) } +/// Reports for the server, each with at most [`MAX_ASSET_USAGE`] inserted and detached counts. +pub fn asset_action_reports(doc: &Document, library: &str) -> Vec { + let Some(full) = asset_actions(doc, library) else { return Vec::new() }; + chunk_asset_actions(full) +} + +fn chunk_asset_actions(actions: AssetActions) -> Vec { + let AssetActions { file, inserted, detached } = actions; + if inserted.is_empty() && detached.is_empty() { + return Vec::new(); + } + let mut out = Vec::new(); + let mut i = 0; + let mut d = 0; + loop { + let ins: Vec<_> = inserted.get(i..).unwrap_or(&[]).iter().take(MAX_ASSET_USAGE).cloned().collect(); + let det: Vec<_> = detached.get(d..).unwrap_or(&[]).iter().take(MAX_ASSET_USAGE).cloned().collect(); + if ins.is_empty() && det.is_empty() { + break; + } + i += ins.len(); + d += det.len(); + out.push(AssetActions { file: file.clone(), inserted: ins, detached: det }); + } + out +} + /// `counts`, the most used first, at most [`MAX_ASSET_USAGE`]. fn most_used(counts: BTreeMap<&str, u32>) -> Vec<(String, u32)> { let mut list: Vec<(String, u32)> = counts.into_iter().map(|(k, n)| (k.to_string(), n)).collect(); @@ -279,10 +307,10 @@ impl Session { detached(doc, &l.id).map(LibraryReq::ReportDetached), inserted(doc, &l.id).map(LibraryReq::ReportInserted), asset_usage(doc, &l.id).map(LibraryReq::ReportAssets), - asset_actions(doc, &l.id).map(LibraryReq::ReportAssetActions), ]; + let asset_reports = asset_action_reports(doc, &l.id).into_iter().map(LibraryReq::ReportAssetActions); let fetch = LibraryReq::Fetch { have, usage: Some(usage(doc, &l.id)) }; - std::iter::once(fetch).chain(reports.into_iter().flatten()).map(|req| (l.path.clone(), req)) + std::iter::once(fetch).chain(reports.into_iter().flatten()).chain(asset_reports).map(|req| (l.path.clone(), req)) }) .collect(); // Confined to automation roots, nothing reaches a server (#1095). diff --git a/crates/engine/src/library_asset_counts_tests.rs b/crates/engine/src/library_asset_counts_tests.rs index e0f4fe456..1eb3a1403 100644 --- a/crates/engine/src/library_asset_counts_tests.rs +++ b/crates/engine/src/library_asset_counts_tests.rs @@ -7,8 +7,9 @@ use serde_json::{Value, json}; use crate::Session; use crate::components::tests::{ex, idv}; -use crate::library_server::asset_actions; -use figcraft_doc::{AssetCount, NodeId, NodeKind, PublishedLibrary}; +use crate::library_server::{asset_action_reports, asset_actions}; +use figcraft_collab::message::MAX_ASSET_USAGE; +use figcraft_doc::{AssetCount, Document, NodeId, NodeKind, PublishedLibrary}; /// A file with a published library holding a paint style, a colour variable and a Card, and a /// second file that uses it: the second session, the library's id and its keys. @@ -152,3 +153,31 @@ fn local_styles_count_nothing_and_undo_takes_a_count_back() { ex(&mut s, "edit.undo", json!({})); assert_eq!(counts(&s).0, (0, 0)); } + +#[test] +fn asset_action_reports_cover_every_count_past_the_message_cap() { + let mut doc = Document::new("File"); + let n = MAX_ASSET_USAGE + 1; + let libraries = std::sync::Arc::make_mut(&mut doc.libraries); + for i in 0..n { + let key = format!("s{i}"); + libraries.count_asset("Lib", &key, false, 1); + libraries.count_asset("Lib", &key, true, 2); + } + // Another library's counts do not belong in these reports. + libraries.count_asset("Other", "x", false, 9); + libraries.count_asset("Other", "x", true, 9); + + let full = asset_actions(&doc, "Lib").unwrap(); + assert_eq!(full.inserted.len(), n, "the document's counts are not truncated"); + assert_eq!(full.detached.len(), n); + + let reports = asset_action_reports(&doc, "Lib"); + assert!(reports.len() >= 2, "one message cannot carry {n} counts"); + assert!(reports.iter().all(|r| r.file == "File" && r.inserted.len() <= MAX_ASSET_USAGE && r.detached.len() <= MAX_ASSET_USAGE)); + let inserted: Vec<_> = reports.iter().flat_map(|r| r.inserted.iter().cloned()).collect(); + let detached: Vec<_> = reports.iter().flat_map(|r| r.detached.iter().cloned()).collect(); + assert_eq!(inserted, full.inserted); + assert_eq!(detached, full.detached); + assert!(asset_action_reports(&doc, "Missing").is_empty()); +}