diff --git a/crates/format/src/build.rs b/crates/format/src/build.rs index ccda9b9b4..c40fa345b 100644 --- a/crates/format/src/build.rs +++ b/crates/format/src/build.rs @@ -44,14 +44,17 @@ pub fn lists(manifest: &Path, families: &[Family]) -> Result { /// A family's `FOLDED` and `PENDING` lists; its folded formats follow on from its base. fn family_lists(family: &Family, root: &Path, out: &mut String, mods: &mut String) -> Result<(), String> { let Family { folder, base, list_prefix } = family; - let mut folded: Vec<(u32, PathBuf)> = std::fs::read_dir(root) - .map_err(|e| format!("{}: {e}", root.display()))? - .filter_map(|entry| { - let path = entry.ok()?.path(); - let version = path.file_name()?.to_str()?.strip_prefix('v')?.parse().ok()?; - path.is_dir().then_some((version, path)) - }) - .collect(); + let mut folded: Vec<(u32, PathBuf)> = Vec::new(); + for entry in std::fs::read_dir(root).map_err(|e| format!("{}: {e}", root.display()))? { + // an entry that can't be read could be a version folder: fail rather than drop it + let path = entry.map_err(|e| format!("{}: {e}", root.display()))?.path(); + let version = path.file_name().and_then(|n| n.to_str()).and_then(|n| n.strip_prefix('v')).and_then(|n| n.parse().ok()); + if let Some(version) = version + && path.is_dir() + { + folded.push((version, path)); + } + } folded.sort(); for (i, (version, _)) in folded.iter().enumerate() { let expected = base + 1 + i as u32; @@ -68,19 +71,42 @@ fn family_lists(family: &Family, root: &Path, out: &mut String, mods: &mut Strin ); for (version, dir) in &folded { let _ = writeln!(out, " ({version}, &["); - parts(dir, &format!("{module}v{version}"), out, mods)?; + parts(dir, &format!("{module}v{version}"), false, out, mods)?; out.push_str(" ]),\n"); } let _ = writeln!(out, "];\n\n/// The changes no release has a number for yet (`{folder}/pending/`).\nconst {list_prefix}PENDING: &[Part] = &["); - parts(&root.join("pending"), &format!("{module}pending"), out, mods)?; + parts(&root.join("pending"), &format!("{module}pending"), true, out, mods)?; out.push_str("];\n\n"); Ok(()) } -/// The `Part`s of the change files in `dir`, in file-name order, and a module for each. -fn parts(dir: &Path, prefix: &str, out: &mut String, mods: &mut String) -> Result<(), String> { - let Ok(entries) = std::fs::read_dir(dir) else { return Ok(()) }; - let mut files: Vec = entries.filter_map(|e| e.ok().map(|e| e.path())).filter(|p| p.extension().is_some_and(|x| x == "rs")).collect(); +/// Whether a change file declares its `step` function (`fn step(`, `pub fn step (` …) on a line of +/// code: a mention in a comment or a note is not a declaration. +fn declares_step(text: &str) -> bool { + text.lines().map(str::trim_start).filter(|line| !line.starts_with("//")).any(|line| { + let line = line.strip_prefix("pub(crate)").or_else(|| line.strip_prefix("pub")).unwrap_or(line).trim_start(); + line.strip_prefix("fn").is_some_and(|rest| { + rest.starts_with(char::is_whitespace) && rest.trim_start().strip_prefix("step").is_some_and(|rest| rest.trim_start().starts_with('(')) + }) + }) +} + +/// The `Part`s of the change files in `dir`, in file-name order, and a module for each. A missing +/// folder is an empty list only where it may be absent (`optional`: `pending/`); any other error +/// reading it fails the build rather than leave a change out of the lists. +fn parts(dir: &Path, prefix: &str, optional: bool, out: &mut String, mods: &mut String) -> Result<(), String> { + let entries = match std::fs::read_dir(dir) { + Ok(entries) => entries, + Err(e) if optional && e.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(e) => return Err(format!("{}: {e}", dir.display())), + }; + let mut files: Vec = Vec::new(); + for entry in entries { + let path = entry.map_err(|e| format!("{}: {e}", dir.display()))?.path(); + if path.extension().is_some_and(|x| x == "rs") { + files.push(path); + } + } files.sort(); for path in files { let name = path.file_stem().and_then(|s| s.to_str()).unwrap_or_default().to_string(); @@ -92,7 +118,7 @@ fn parts(dir: &Path, prefix: &str, out: &mut String, mods: &mut String) -> Resul return Err(format!("{}: start a format change with a `//!` note", path.display())); } let module = format!("{prefix}_{}", name.replace('-', "_")); - let step = if text.contains("fn step(") { format!("Some({module}::step)") } else { "None".into() }; + let step = if declares_step(&text) { format!("Some({module}::step)") } else { "None".into() }; let _ = writeln!(out, " Part {{ name: {name:?}, step: {step} }},"); let _ = writeln!(mods, "#[path = {:?}]\nmod {module};", path.display().to_string()); } diff --git a/crates/format/src/build_tests.rs b/crates/format/src/build_tests.rs index 0c212a7cc..d993a572c 100644 --- a/crates/format/src/build_tests.rs +++ b/crates/format/src/build_tests.rs @@ -73,3 +73,26 @@ fn refuses_gaps_bad_names_missing_notes_and_missing_folders() { assert!(refused(&dir).contains("`//!` note")); let _ = std::fs::remove_dir_all(&dir); } + +#[test] +fn a_step_is_a_declaration_not_a_mention() { + assert!(!declares_step("//! No fn step() is needed.\n")); + assert!(!declares_step("//! a note\n// fn step(v) was removed\nconst X: u8 = 1;\n")); + assert!(declares_step("//! a note\npub fn step (v: &mut Value) -> Result<(), E> { Ok(()) }\n")); + assert!(declares_step("//! a note\npub(crate) fn step(v: &mut Value) -> Result<(), E> { Ok(()) }\n")); + assert!(declares_step("//! a note\n fn step(v: &mut Value) {}\n")); + assert!(!declares_step("//! a note\nfn stepper(v: &mut Value) {}\nfn fnstep() {}\n")); +} + +#[test] +fn a_documentation_only_change_has_no_step_and_a_missing_version_folder_fails() { + let dir = scratch("doc-only"); + write(&dir.join("drawing/v4/doc.rs"), "//! No fn step() is needed.\n"); + let code = lists(&dir, &[DRAWING]).unwrap().code; + assert!(code.contains("Part { name: \"doc\", step: None }"), "{code}"); + // a version folder that can't be listed is an error, not an empty format + let missing = parts(&dir.join("drawing/v9"), "x", false, &mut String::new(), &mut String::new()); + assert!(missing.is_err()); + assert!(parts(&dir.join("drawing/pending"), "x", true, &mut String::new(), &mut String::new()).is_ok()); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/crates/format/src/container.rs b/crates/format/src/container.rs index cfacf20a1..e5955c066 100644 --- a/crates/format/src/container.rs +++ b/crates/format/src/container.rs @@ -37,12 +37,15 @@ pub struct Limits { pub level: fn(&str, usize) -> i64, /// Whether a container of this many bytes may be read, or why not. pub check_size: fn(u64) -> Result<(), String>, - /// Whether the work is cancelled (a save the person stopped): a long part is written a piece - /// at a time and stops before the next piece when this says so. `None`: it cannot be - /// cancelled here. + /// Whether the work is cancelled (a save the person stopped): checked before each part, and + /// a long part is written a piece at a time and stops before the next piece when this says + /// so ([`FormatError::Cancelled`]). `None`: it cannot be cancelled here. pub cancelled: Option bool>, } +/// The most bytes of a part written between two cancellation checks. +const PIECE: usize = 1 << 20; + fn invalid(message: impl std::fmt::Display) -> FormatError { FormatError::Invalid(message.to_string()) } @@ -128,26 +131,54 @@ impl Writer { if bytes.len() > (self.limits.part_limit)(name) { return Err(invalid("part too large")); } - if !self.names.insert(name.to_owned()) { + if self.names.contains(name) { return Ok(()); } - self.expanded = self.expanded.checked_add(bytes.len()).ok_or_else(|| invalid("expanded size overflow"))?; - if self.expanded > (self.limits.max_expanded)() || self.names.len() > self.limits.max_entries { + // Check the counters this part would leave, and commit them only once it is written: a + // refused part is not remembered as written. + let expanded = self.expanded.checked_add(bytes.len()).ok_or_else(|| invalid("expanded size overflow"))?; + if expanded > (self.limits.max_expanded)() || self.names.len().saturating_add(1) > self.limits.max_entries { return Err(invalid("package too large")); } + let cancelled = self.limits.cancelled; + if cancelled.is_some_and(|c| c()) { + return Err(FormatError::Cancelled); + } + let compression = if compressed { craft_zip::Compression::Deflated } else { craft_zip::Compression::Stored }; + match cancelled { + Some(cancelled) if bytes.len() > PIECE => { + let mut stopped = false; + let written = self.zip.add_with(name, compression, |out| { + for piece in bytes.chunks(PIECE) { + if cancelled() { + stopped = true; + return Err(std::io::Error::other("cancelled")); + } + out.write_all(piece)?; + } + Ok(()) + }); + if stopped { + return Err(FormatError::Cancelled); + } + written.map_err(invalid)?; + } + _ => self.zip.add(name, bytes, compression).map_err(invalid)?, + } + self.names.insert(name.to_owned()); + self.expanded = expanded; if let Some((include, parts)) = &mut self.tracked && include(name) { let digest = digest.unwrap_or_else(|| sha256(bytes)); parts.push(PartDigest { name: name.to_owned(), len: bytes.len() as u64, digest }); } - let compression = if compressed { craft_zip::Compression::Deflated } else { craft_zip::Compression::Stored }; - self.zip.add(name, bytes, compression).map_err(invalid)?; Ok(()) } /// Would `entries` more optional parts of `extra` bytes, stored uncompressed, still keep the - /// file within `budget` and its expanded size within the reader's budget? + /// file within `budget` and its expanded size within the reader's budget? The parts already + /// written count at their expanded size (an upper bound on what they take in the file). pub fn room(&self, entries: usize, extra: usize, budget: usize) -> bool { self.names.len().saturating_add(entries) <= self.limits.max_entries && extra <= self.space(entries, budget) } @@ -157,15 +188,14 @@ impl Writer { if self.names.len().saturating_add(entries) > self.limits.max_entries { return 0; } - budget - .saturating_sub(self.names.len().saturating_add(entries).saturating_mul(256).saturating_add(65_536)) - .min((self.limits.max_expanded)().saturating_sub(self.expanded)) + let overhead = self.names.len().saturating_add(entries).saturating_mul(256).saturating_add(65_536); + budget.saturating_sub(overhead.saturating_add(self.expanded)).min((self.limits.max_expanded)().saturating_sub(self.expanded)) } /// Add a payload as a blob, `blobs/.`, and say its name, for the JSON that /// refers to it. pub fn blob(&mut self, bytes: &[u8], extension: &str, compressed: bool) -> Result { - self.resolved = self + let resolved = self .resolved .checked_add(bytes.len()) .filter(|&n| n <= (self.limits.max_expanded)()) @@ -173,6 +203,7 @@ impl Writer { let digest = sha256(bytes); let name = format!("blobs/{}.{extension}", hex(&digest)); self.part_with_digest(&name, bytes, compressed, Some(digest))?; + self.resolved = resolved; Ok(Value::String(name)) } } @@ -194,10 +225,16 @@ impl Reader { let length = source.seek(SeekFrom::End(0)).map_err(invalid)?; (limits.check_size)(length).map_err(invalid)?; - // Read all bytes into memory for the archive. + // Read all bytes into memory for the archive, no more than the length just approved: a file + // that grew since is refused rather than read past its budget. source.seek(SeekFrom::Start(0)).map_err(invalid)?; - let mut bytes = Vec::with_capacity(length as usize); - source.read_to_end(&mut bytes).map_err(invalid)?; + let capacity = usize::try_from(length).map_err(|_| invalid("file too large"))?; + let mut bytes = Vec::new(); + bytes.try_reserve_exact(capacity).map_err(|_| invalid("file too large"))?; + source.take(length.saturating_add(1)).read_to_end(&mut bytes).map_err(invalid)?; + if bytes.len() as u64 != length { + return Err(invalid("the file changed while it was read")); + } let mut archive = craft_zip::Archive::open_owned( bytes, @@ -206,16 +243,21 @@ impl Reader { .map_err(invalid)?; // Validate entry names match format rules: no empty, no backslashes, - // no path components that are "." or "..", and no duplicates. + // no path components that are "." or "..", and no duplicates; and every part's declared + // size within its configured limit. let mut name_set = BTreeSet::new(); for entry_idx in 0..archive.len() { - if let Ok(Some(name)) = archive.entry_name(entry_idx) - && (name.is_empty() + if let Ok(Some(name)) = archive.entry_name(entry_idx) { + if name.is_empty() || name.contains('\\') || name.split('/').any(|c| c.is_empty() || c == "." || c == "..") - || !name_set.insert(name)) - { - return Err(invalid("duplicate or invalid archive entry")); + || !name_set.insert(name.clone()) + { + return Err(invalid("duplicate or invalid archive entry")); + } + if archive.declared_size(&name).is_some_and(|size| size > (limits.part_limit)(&name) as u64) { + return Err(invalid("part too large")); + } } } @@ -243,8 +285,9 @@ impl Reader { self.archive.names().map(|n| n.into_owned()).collect() } - /// The part of this name, at most `limit` bytes. + /// The part of this name, at most `limit` bytes and never more than its configured limit. pub fn limited_part(&mut self, name: &str, limit: usize) -> Result, FormatError> { + let limit = limit.min((self.limits.part_limit)(name)); let expected = self.archive.declared_size(name).ok_or_else(|| invalid("part not found"))?; if expected > limit as u64 { return Err(invalid("part too large")); @@ -329,7 +372,11 @@ impl Thumbnail { let key = serde_json::to_vec(&serde_json::json!({"format": self.format, "app": self.app, "document": digest(document), "payload": digest(png)})) .map_err(invalid)?; - if !writer.room(2, key.len().saturating_add(png.len()), budget) { + let fits = |name: &str, len: usize| len <= (writer.limits.part_limit)(name); + if !fits("cache/thumbnail.json", key.len()) + || !fits("cache/thumbnail.png", png.len()) + || !writer.room(2, key.len().saturating_add(png.len()), budget) + { return Ok(()); } writer.part("cache/thumbnail.json", &key, false)?; diff --git a/crates/format/src/container_tests.rs b/crates/format/src/container_tests.rs index 50bd86715..9f6c8455f 100644 --- a/crates/format/src/container_tests.rs +++ b/crates/format/src/container_tests.rs @@ -66,3 +66,98 @@ fn a_thumbnail_is_keyed_to_its_file() { let mut reader = Reader::new(Cursor::new(writer.finish().unwrap()), LIMITS).unwrap(); assert!(picture.read(&mut reader, 64).is_err(), "stale"); } + +/// A refused part is not remembered as written: once the budget is raised it is written. +#[test] +fn a_refused_part_can_be_written_later() { + static BUDGET: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(4); + let limits = Limits { max_expanded: || BUDGET.load(std::sync::atomic::Ordering::SeqCst), ..LIMITS }; + let mut writer = Writer::new(limits); + assert_eq!(writer.part("a.json", b"12345678", false), Err(FormatError::Invalid("package too large".into()))); + assert!(writer.blob(b"12345678", "bin", false).is_err()); + BUDGET.store(1 << 20, std::sync::atomic::Ordering::SeqCst); + writer.part("a.json", b"12345678", false).unwrap(); + writer.blob(b"12345678", "bin", false).unwrap(); + let reader = Reader::new(Cursor::new(writer.finish().unwrap()), limits).unwrap(); + assert_eq!(reader.part_names().len(), 2, "both parts were written on the retry"); +} + +/// A cancelled save writes no more parts, whether the part is small or written in pieces. +#[test] +fn cancellation_stops_the_save() { + let cancelled = Limits { cancelled: Some(|| true), ..LIMITS }; + let mut writer = Writer::new(cancelled); + assert_eq!(writer.part("a.json", b"a", false), Err(FormatError::Cancelled)); + // cancelled after the first piece of a long part: checks before the part, then per piece + static CHECKS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + let midway = Limits { + cancelled: Some(|| CHECKS.fetch_add(1, std::sync::atomic::Ordering::SeqCst) >= 2), + max_expanded: || 1 << 22, + part_limit: |_| 1 << 22, + ..LIMITS + }; + let mut writer = Writer::new(midway); + assert_eq!(writer.part("big.big", &vec![0; PIECE + 1], true), Err(FormatError::Cancelled)); + assert_eq!(CHECKS.load(std::sync::atomic::Ordering::SeqCst), 3); + let going = Limits { cancelled: Some(|| false), max_expanded: || 1 << 22, part_limit: |_| 1 << 22, ..LIMITS }; + let mut writer = Writer::new(going); + writer.part("big.big", &vec![7; PIECE + 1], true).unwrap(); + let mut reader = Reader::new(Cursor::new(writer.finish().unwrap()), going).unwrap(); + assert_eq!(reader.limited_part("big.big", 1 << 22).unwrap().len(), PIECE + 1); +} + +/// Room counts what is already written, and the reader keeps every part's configured limit. +#[test] +fn room_counts_written_parts_and_reads_keep_part_limits() { + let mut writer = Writer::new(LIMITS); + writer.part("large.big", &[0; 500 * 1024], false).unwrap(); + assert!(!writer.room(1, 200 * 1024, 600 * 1024)); + assert!(writer.room(1, 1024, 1 << 20)); + let wide = Limits { part_limit: |_| 1 << 20, ..LIMITS }; + let mut writer = Writer::new(wide); + writer.part("document.json", &[b' '; 2048], false).unwrap(); + let bytes = writer.finish().unwrap(); + assert!(Reader::new(Cursor::new(bytes.clone()), LIMITS).is_err(), "declared past its configured limit"); + let mut reader = Reader::new(Cursor::new(bytes), wide).unwrap(); + assert!(reader.limited_part("document.json", 4096).is_ok()); +} + +/// A thumbnail past a cache part's own limit is left out, never failing the save. +#[test] +fn a_thumbnail_past_its_part_limit_is_left_out() { + let picture = Thumbnail { format: "app-thumbnail/1", app: "1.0", size: [2, 1], max_bytes: 2048 }; + let mut png = b"\x89PNG\r\n\x1a\n\0\0\0\rIHDR".to_vec(); + png.extend_from_slice(&2u32.to_be_bytes()); + png.extend_from_slice(&1u32.to_be_bytes()); + png.resize(1500, 0); + let mut writer = Writer::new(LIMITS); + picture.write(&mut writer, &png, b"{}", 8 << 20).unwrap(); + let reader = Reader::new(Cursor::new(writer.finish().unwrap()), LIMITS).unwrap(); + assert!(!reader.has_part("cache/thumbnail.json") && !reader.has_part("cache/thumbnail.png")); +} + +/// A source that grows after its length was approved is refused. +#[test] +fn a_source_that_grows_while_read_is_refused() { + struct Growing(Cursor>, bool); + impl Read for Growing { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + if !self.1 { + self.1 = true; + let end = self.0.get_ref().len(); + self.0.get_mut().extend_from_slice(&[0; 16]); + self.0.set_position(self.0.position().min(end as u64)); + } + self.0.read(buf) + } + } + impl Seek for Growing { + fn seek(&mut self, to: SeekFrom) -> std::io::Result { + self.0.seek(to) + } + } + let mut writer = Writer::new(LIMITS); + writer.part("document.json", b"{}", false).unwrap(); + let bytes = writer.finish().unwrap(); + assert!(Reader::new(Growing(Cursor::new(bytes), false), LIMITS).err().is_some_and(|e| e.to_string().contains("changed"))); +} diff --git a/crates/format/src/lib.rs b/crates/format/src/lib.rs index 8eb495417..aadb56fd9 100644 --- a/crates/format/src/lib.rs +++ b/crates/format/src/lib.rs @@ -20,6 +20,8 @@ //! Steps are JSON-level (the file's shape) and return the app's own error type; every error this //! crate makes is a [`FormatError`], which the app's error type takes `From`. +#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))] + use serde_json::Value; pub mod build; @@ -157,7 +159,10 @@ impl Family<'_, E> { /// The version in one of this family's format tags; `None` for any other tag, another /// family's included. pub fn version_of(&self, s: &str) -> Option { - s.strip_prefix(self.prefix).or_else(|| self.legacy_prefix.and_then(|p| s.strip_prefix(p)))?.parse().ok() + // each tag form parses on its own: a legacy prefix may start with the primary one + // (`app/` and `app/design/`) + let parse = |prefix: &str| s.strip_prefix(prefix).and_then(|n| n.parse().ok()); + parse(self.prefix).or_else(|| self.legacy_prefix.and_then(parse)) } /// The change a name a file of format `version` lists stands for: its alias's, or itself. diff --git a/crates/format/src/lib_tests.rs b/crates/format/src/lib_tests.rs index 6874ddbf4..40c5101fb 100644 --- a/crates/format/src/lib_tests.rs +++ b/crates/format/src/lib_tests.rs @@ -132,3 +132,11 @@ fn a_newer_file_is_told_by_its_number_or_an_unknown_change() { assert_eq!(FAMILY.newer(4, Some(&["old-c".to_string()])), Some(Newer { format: 4, reads_up_to: 6, unknown_changes: vec!["old-c".to_string()] })); assert_eq!(FAMILY.newer(2, Some(&["z".to_string()])), None, "a list beside a hand-written format means nothing"); } + +#[test] +fn a_legacy_prefix_that_extends_the_primary_one_is_read() { + let nested = Family { prefix: "app/", legacy_prefix: Some("app/design/"), ..FAMILY }; + assert_eq!(nested.version_of("app/design/3"), Some(3)); + assert_eq!(nested.version_of("app/4"), Some(4)); + assert_eq!(nested.version_of("app/design/x"), None); +} diff --git a/crates/format/src/newer.rs b/crates/format/src/newer.rs index e9cb51eb1..a66ef8674 100644 --- a/crates/format/src/newer.rs +++ b/crates/format/src/newer.rs @@ -12,7 +12,7 @@ //! saved ([`NewerFile::restore_into`]), as they came. What can't be put back because the file was //! changed around it is what `restore_into` returns. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashMap}; use std::sync::Arc; use serde_json::{Map, Value, json}; @@ -118,9 +118,9 @@ impl NewerFile { } /// Whether the next save needs a container to keep what the file holds: plain JSON has no room - /// for parts beside the file's own. + /// for parts beside the file's own, nor for the container's own `container.json`. pub fn needs_container(&self) -> bool { - !self.parts.is_empty() + !self.parts.is_empty() || self.container.is_some() } /// Whether anything kept back names this text (a blob's name in a container). @@ -315,9 +315,10 @@ fn vanished(raw: &Value, known: &Value, path: &str, paths: &mut Vec, pla } (Value::Array(raw), Value::Array(known)) => { let mut items = Vec::new(); + let positions = Positions::of(known); for (at, element) in raw.iter().enumerate() { let anchor = anchor_of(element).unwrap_or(Anchor::Index { at, of: raw.len() }); - let Some(partner) = find(known, &anchor) else { continue }; + let Some(partner) = positions.find(known.len(), &anchor).and_then(|i| known.get(i)) else { continue }; let label = match (element.get("name").and_then(Value::as_str), &anchor) { (Some(name), _) => format!("{path}[{name}]"), (None, Anchor::Id(id)) => format!("{path}[{id}]"), @@ -340,19 +341,34 @@ fn anchor_of(element: &Value) -> Option { element.get("name").and_then(Value::as_str).map(|name| Anchor::Name(name.to_string())) } -fn find<'a>(list: &'a [Value], anchor: &Anchor) -> Option<&'a Value> { - match anchor { - Anchor::Id(id) => list.iter().find(|x| x.get("id").and_then(Value::as_u64) == Some(*id)), - Anchor::Name(name) => list.iter().find(|x| x.get("name").and_then(Value::as_str) == Some(name)), - Anchor::Index { at, of } => (list.len() == *of).then(|| list.get(*at)).flatten(), - } +/// Where each id and name first appears in a list, so matching records is not a scan per record. +struct Positions { + ids: HashMap, + names: HashMap, } -fn find_mut<'a>(list: &'a mut [Value], anchor: &Anchor) -> Option<&'a mut Value> { - match anchor { - Anchor::Id(id) => list.iter_mut().find(|x| x.get("id").and_then(Value::as_u64) == Some(*id)), - Anchor::Name(name) => list.iter_mut().find(|x| x.get("name").and_then(Value::as_str) == Some(name)), - Anchor::Index { at, of } => (list.len() == *of).then(|| list.get_mut(*at)).flatten(), +impl Positions { + fn of(list: &[Value]) -> Positions { + let (mut ids, mut names) = (HashMap::new(), HashMap::new()); + for (i, x) in list.iter().enumerate() { + if let Some(id) = x.get("id").and_then(Value::as_u64) { + ids.entry(id).or_insert(i); + } + if let Some(name) = x.get("name").and_then(Value::as_str) { + names.entry(name.to_string()).or_insert(i); + } + } + Positions { ids, names } + } + + /// The position of the record `anchor` names in a list of `len` records (an index anchor + /// only while the list keeps its length). + fn find(&self, len: usize, anchor: &Anchor) -> Option { + match anchor { + Anchor::Id(id) => self.ids.get(id).copied(), + Anchor::Name(name) => self.names.get(name).copied(), + Anchor::Index { at, of } => (len == *of && at < of).then_some(*at), + } } } @@ -372,7 +388,12 @@ fn apply(kept: &Kept, value: &mut Value, path: &str, lost: &mut Vec) { (Kept::Field(v), None) => { object.insert(key.clone(), v.clone()); } - (Kept::Field(_), Some(_)) => {} + // the edited value stays; a retained one it replaced is reported + (Kept::Field(v), Some(existing)) => { + if existing != v { + lost.push(format!("field `{inner}`")); + } + } (sub, Some(inner_value)) => apply(sub, inner_value, &inner, lost), // The list it was in is empty and not written: its elements were deleted. (Kept::Items(_), None) => {} @@ -385,8 +406,10 @@ fn apply(kept: &Kept, value: &mut Value, path: &str, lost: &mut Vec) { lost.push(format!("field `{path}`")); return; }; + let positions = Positions::of(list); + let len = list.len(); for (anchor, sub) in items { - match (find_mut(list, anchor), anchor) { + match (positions.find(len, anchor).and_then(|i| list.get_mut(i)), anchor) { (Some(element), _) => apply(sub, element, path, lost), (None, Anchor::Index { at, .. }) => lost.push(format!("element {at} of `{path}`")), (None, _) => {} diff --git a/crates/format/src/newer_tests.rs b/crates/format/src/newer_tests.rs index 6419f1b2b..9627daf70 100644 --- a/crates/format/src/newer_tests.rs +++ b/crates/format/src/newer_tests.rs @@ -87,3 +87,42 @@ fn replaced_sections_and_elements_are_reported_lost() { assert!(said.restore_into(&mut fresh).is_empty()); assert_eq!(fresh["shapes"], json!([{"name": "star"}, {"points": 3}])); } + +/// Retained container metadata alone needs a container to survive the save. +#[test] +fn container_metadata_alone_needs_a_container() { + let mut said = NewerFile::new("App", &newer()); + assert!(!said.needs_container()); + said.container = Some(json!({"layout": 2})); + assert!(said.needs_container()); +} + +/// An edit that replaces a retained field keeps the edit and reports the replaced value. +#[test] +fn a_replaced_retained_field_is_reported() { + let mut said = NewerFile::new("App", &newer()); + said.compare(&json!({"extra": 1}), &json!({}), "placeholder", &[]); + let mut edited = json!({"extra": 2}); + assert_eq!(said.restore_into(&mut edited), ["field `extra`"]); + assert_eq!(edited, json!({"extra": 2})); + let mut same = json!({"extra": 1}); + assert!(said.restore_into(&mut same).is_empty()); +} + +/// Records are matched by id and by name across long lists, and an index anchor only while the +/// list keeps its length. +#[test] +fn records_match_by_id_name_or_unchanged_index() { + let records: Vec = (0..2000).map(|i| json!({"id": i, "extra": i})).collect(); + let known: Vec = (0..2000).map(|i| json!({"id": i})).collect(); + let mut said = NewerFile::new("App", &newer()); + said.compare(&json!({"items": records}), &json!({"items": known}), "placeholder", &[]); + let mut saved = json!({"items": (0..2000).rev().map(|i| json!({"id": i})).collect::>()}); + assert!(said.restore_into(&mut saved).is_empty()); + assert_eq!(saved["items"][0], json!({"id": 1999, "extra": 1999})); + let mut by_name = NewerFile::new("App", &newer()); + by_name.compare(&json!({"l": [{"name": "a", "x": 1}, [1, 2]]}), &json!({"l": [{"name": "a"}, [1]]}), "placeholder", &[]); + let mut renamed = json!({"l": [{"name": "a"}]}); + by_name.restore_into(&mut renamed); + assert_eq!(renamed["l"][0], json!({"name": "a", "x": 1})); +}