diff --git a/Cargo.lock b/Cargo.lock index afa50763a..0d62a617c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1849,6 +1849,16 @@ dependencies = [ "craft-fonts-build", ] +[[package]] +name = "craft-format" +version = "0.1.0" +dependencies = [ + "craft-zip", + "proptest", + "serde_json", + "sha2 0.10.9", +] + [[package]] name = "craft-gainmap" version = "0.1.0" @@ -8539,9 +8549,9 @@ checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f" [[package]] name = "rustls-webpki" -version = "0.103.15" +version = "0.103.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" dependencies = [ "aws-lc-rs", "ring", @@ -11964,9 +11974,9 @@ dependencies = [ [[package]] name = "zip" -version = "9.1.0" +version = "9.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2886b00003a72c4ccee3f42cfc34908d33b2508a74d827dc2a4244706475c704" +checksum = "1156b39d142a34b999c2310a50455e0cb3760bf7efad0122f796d813bee26ccb" dependencies = [ "crc32fast", "flate2", diff --git a/crates/format/Cargo.toml b/crates/format/Cargo.toml new file mode 100644 index 000000000..943e0b9b6 --- /dev/null +++ b/crates/format/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "craft-format" +description = "Versioned JSON file formats: the format tag, the registry of versions (hand-written steps, then one folder of changes per version, then the pending changes), the upgrade of an older file and the telling of a newer one" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +serde_json = { workspace = true } +# The container: content-addressed blobs (SHA-256) in a ZIP package. +sha2 = { version = "0.10", default-features = false, features = ["std"] } +craft-zip = { workspace = true } + +[dev-dependencies] +proptest = { workspace = true } diff --git a/crates/format/src/build.rs b/crates/format/src/build.rs new file mode 100644 index 000000000..bbffa98ac --- /dev/null +++ b/crates/format/src/build.rs @@ -0,0 +1,237 @@ +//! For a build script: the generated lists of a family's changes. Each change is one `.rs` file in +//! the family's folder, in `v/` once a release fixed its number, else in `pending/`; the lists +//! are generated so that a change adds a file and never edits a line another change also edits. +//! +//! The generated code, for the app to `include!`, is per family a `FOLDED` list (each folded +//! format and its parts) and a `PENDING` list, both prefixed by the family's `list_prefix`, then a +//! module per change file. It names the app's `Part` type, which the including module has in scope +//! (`type Part = craft_format::Part;`), and a change file's step as `step` (a +//! `pub fn step(v: &mut Value) -> Result<(), AppError>`, when the file has one). + +use std::fmt::Write as _; +use std::path::{Path, PathBuf}; + +/// A family's change files: their folder (relative to the crate's manifest), the last format +/// before them, and the prefix of the family's lists. +pub struct Family { + pub folder: &'static str, + pub base: u32, + pub list_prefix: &'static str, +} + +/// What [`lists`] made: the code to write, and the folders the build script reruns on. +pub struct Lists { + pub code: String, + pub watch: Vec, +} + +/// The `FOLDED` and `PENDING` lists of every family, and a module for each change file. Refuses +/// versions that are not contiguous from a family's base, a change file not named with lowercase +/// letters, digits and dashes, and one that doesn't start with a `//!` note. +pub fn lists(manifest: &Path, families: &[Family]) -> Result { + let mut code = String::new(); + let mut mods = String::new(); + let mut watch = Vec::new(); + for family in families { + let root = manifest.join(family.folder); + family_lists(family, &root, &mut code, &mut mods)?; + watch.push(root); + } + code.push_str(&mods); + Ok(Lists { code, watch }) +} + +/// A family's `FOLDED` and `PENDING` lists; its folded formats follow on from its base. +fn family_lists(family: &Family, root: &Path, out: &mut String, mods: &mut String) -> Result<(), String> { + let Family { folder, base, list_prefix } = family; + let mut folded: Vec<(u32, PathBuf)> = Vec::new(); + for entry in std::fs::read_dir(root).map_err(|e| format!("{}: {e}", root.display()))? { + // an entry that can't be read could be a version folder: fail rather than drop it + let path = entry.map_err(|e| format!("{}: {e}", root.display()))?.path(); + let version = path.file_name().and_then(|n| n.to_str()).and_then(|n| n.strip_prefix('v')).and_then(|n| n.parse().ok()); + if let Some(version) = version + && path.is_dir() + { + folded.push((version, path)); + } + } + folded.sort(); + for (i, (version, _)) in folded.iter().enumerate() { + let expected = base + 1 + i as u32; + if *version != expected { + return Err(format!("{folder}/: expected v{expected}/ next, found v{version}/ (versions are contiguous)")); + } + } + // Module names start with the family's (`drawing_pending_…`), so two families' changes of one + // name never clash. + let module = list_prefix.to_ascii_lowercase(); + let _ = writeln!( + out, + "/// The changes of each format after the base, oldest first (`{folder}/v/`).\nconst {list_prefix}FOLDED: &[(u32, &[Part])] = &[" + ); + for (version, dir) in &folded { + let _ = writeln!(out, " ({version}, &["); + parts(dir, &format!("{module}v{version}"), false, out, mods)?; + out.push_str(" ]),\n"); + } + let _ = writeln!(out, "];\n\n/// The changes no release has a number for yet (`{folder}/pending/`).\nconst {list_prefix}PENDING: &[Part] = &["); + parts(&root.join("pending"), &format!("{module}pending"), true, out, mods)?; + out.push_str("];\n\n"); + Ok(()) +} + +/// Whether a change file declares its `step` function (`fn step(`, `pub fn step (` …) on a line of +/// code: a mention in a comment or a note is not a declaration. +fn declares_step(text: &str) -> bool { + let code = without_comments(text); + let ident = |c: char| c.is_alphanumeric() || c == '_'; + // `fn`, whitespace, `step`, then `(` (or generics), across lines + code.match_indices("fn").any(|(at, _)| { + let before_ok = code[..at].chars().next_back().is_none_or(|c| !ident(c)); + let rest = &code[at + 2..]; + before_ok + && rest.starts_with(char::is_whitespace) + && rest + .trim_start() + .strip_prefix("step") + .is_some_and(|rest| matches!(rest.trim_start().chars().next(), Some('(' | '<')) && !rest.starts_with(ident)) + }) +} + +/// `text` as code only: its `//` and (nested) `/* */` comments become a space, and the contents +/// of its string literals (plain, byte and raw) are dropped, line breaks kept throughout. A comment +/// marker inside a string is not a comment, and `fn/* note */step` still reads as `fn step`. +fn without_comments(text: &str) -> String { + let chars: Vec = text.chars().collect(); + let at = |i: usize| chars.get(i).copied(); + let mut out = String::with_capacity(text.len()); + let (mut i, mut depth) = (0usize, 0usize); + while let Some(c) = at(i) { + let next = at(i + 1); + if depth > 0 { + match (c, next) { + ('/', Some('*')) => (depth, i) = (depth + 1, i + 2), + ('*', Some('/')) => (depth, i) = (depth - 1, i + 2), + _ => { + if c == '\n' { + out.push('\n'); + } + i += 1; + } + } + if depth == 0 { + out.push(' '); + } + continue; + } + let ident_before = i > 0 && at(i - 1).is_some_and(|p| p.is_alphanumeric() || p == '_'); + match (c, next) { + ('/', Some('/')) => { + while at(i).is_some_and(|c| c != '\n') { + i += 1; + } + out.push(' '); + } + ('/', Some('*')) => (depth, i) = (1, i + 2), + ('"', _) => { + out.push('"'); + i += 1; + while let Some(c) = at(i) { + i += 1; + match c { + '\\' => i += 1, + '"' => break, + '\n' => out.push('\n'), + _ => {} + } + } + out.push('"'); + } + // a raw string, `r"…"` or `r#"…"#`, after a byte prefix or on its own + ('r', Some('"' | '#')) + if !ident_before || (i > 0 && at(i - 1) == Some('b') && !(i > 1 && at(i - 2).is_some_and(|p| p.is_alphanumeric() || p == '_'))) => + { + let mut j = i + 1; + let mut hashes = 0; + while at(j) == Some('#') { + hashes += 1; + j += 1; + } + if at(j) != Some('"') { + out.push(c); + i += 1; + continue; + } + out.push('"'); + i = j + 1; + while let Some(c) = at(i) { + if c == '"' && (1..=hashes).all(|k| at(i + k) == Some('#')) { + i += 1 + hashes; + break; + } + if c == '\n' { + out.push('\n'); + } + i += 1; + } + out.push('"'); + } + // a character literal (`'"'`, `'\''`), not a lifetime + ('\'', Some('\\')) => { + i += 2; + while at(i).is_some_and(|c| c != '\'') { + i += 1; + } + i += 1; + out.push_str("' '"); + } + ('\'', Some(_)) if at(i + 2) == Some('\'') => { + i += 3; + out.push_str("' '"); + } + _ => { + out.push(c); + i += 1; + } + } + } + out +} + +/// The `Part`s of the change files in `dir`, in file-name order, and a module for each. A missing +/// folder is an empty list only where it may be absent (`optional`: `pending/`); any other error +/// reading it fails the build rather than leave a change out of the lists. +fn parts(dir: &Path, prefix: &str, optional: bool, out: &mut String, mods: &mut String) -> Result<(), String> { + let entries = match std::fs::read_dir(dir) { + Ok(entries) => entries, + Err(e) if optional && e.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(e) => return Err(format!("{}: {e}", dir.display())), + }; + let mut files: Vec = Vec::new(); + for entry in entries { + let path = entry.map_err(|e| format!("{}: {e}", dir.display()))?.path(); + if path.extension().is_some_and(|x| x == "rs") { + files.push(path); + } + } + files.sort(); + for path in files { + let name = path.file_stem().and_then(|s| s.to_str()).unwrap_or_default().to_string(); + if name.is_empty() || !name.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') { + return Err(format!("{}: name a format change file with lowercase letters, digits and dashes", path.display())); + } + let text = std::fs::read_to_string(&path).map_err(|e| format!("{}: {e}", path.display()))?; + if !text.trim_start().starts_with("//!") { + return Err(format!("{}: start a format change with a `//!` note", path.display())); + } + let module = format!("{prefix}_{}", name.replace('-', "_")); + let step = if declares_step(&text) { format!("Some({module}::step)") } else { "None".into() }; + let _ = writeln!(out, " Part {{ name: {name:?}, step: {step} }},"); + let _ = writeln!(mods, "#[path = {:?}]\nmod {module};", path.display().to_string()); + } + Ok(()) +} + +#[cfg(test)] +#[path = "build_tests.rs"] +mod tests; diff --git a/crates/format/src/build_tests.rs b/crates/format/src/build_tests.rs new file mode 100644 index 000000000..79d3cb385 --- /dev/null +++ b/crates/format/src/build_tests.rs @@ -0,0 +1,115 @@ +#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)] +use super::*; + +/// A fresh, empty folder for one test. +fn scratch(name: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!("craft-format-build-{}-{name}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + dir +} + +fn write(path: &Path, text: &str) { + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, text).unwrap(); +} + +/// The error of a run that must fail. +fn refused(dir: &Path) -> String { + match lists(dir, &[DRAWING]) { + Ok(_) => panic!("expected an error"), + Err(e) => e, + } +} + +const DRAWING: Family = Family { folder: "drawing", base: 3, list_prefix: "DRAWING_" }; + +#[test] +fn lists_folded_and_pending_changes_with_their_modules() { + let dir = scratch("ok"); + write(&dir.join("drawing/v4/add-layers.rs"), "//! Layers.\npub fn step(v: &mut Value) -> Result<(), E> { Ok(()) }\n"); + write(&dir.join("drawing/v4/notes.txt"), "not a change"); + write(&dir.join("drawing/v5/rename.rs"), "\n//! A rename with no step.\n"); + write(&dir.join("drawing/pending/next-1.rs"), "//! Pending.\nfn step(v: &mut Value) {}\n"); + std::fs::create_dir_all(dir.join("drawing/v6x")).unwrap(); + write(&dir.join("drawing/v7"), "a file, not a version folder"); + let lists = lists(&dir, &[DRAWING]).unwrap(); + assert_eq!(lists.watch, vec![dir.join("drawing")]); + let code = &lists.code; + assert!(code.contains("const DRAWING_FOLDED: &[(u32, &[Part])] = &["), "{code}"); + assert!(code.contains(" (4, &[\n Part { name: \"add-layers\", step: Some(drawing_v4_add_layers::step) },\n ]),"), "{code}"); + assert!(code.contains("Part { name: \"rename\", step: None }"), "{code}"); + assert!( + code.contains("const DRAWING_PENDING: &[Part] = &[\n Part { name: \"next-1\", step: Some(drawing_pending_next_1::step) },"), + "{code}" + ); + assert!(code.contains("mod drawing_v5_rename;"), "{code}"); + assert!(!code.contains("notes"), "{code}"); + assert!(code.find("const DRAWING_PENDING").unwrap() < code.find("#[path").unwrap(), "modules follow the lists"); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn an_empty_family_has_empty_lists() { + let dir = scratch("empty"); + std::fs::create_dir_all(dir.join("drawing")).unwrap(); + let code = lists(&dir, &[DRAWING]).unwrap().code; + assert!(code.contains("const DRAWING_FOLDED: &[(u32, &[Part])] = &[\n];"), "{code}"); + assert!(code.contains("const DRAWING_PENDING: &[Part] = &[\n];"), "{code}"); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn refuses_gaps_bad_names_missing_notes_and_missing_folders() { + let dir = scratch("bad"); + assert!(refused(&dir).contains("drawing")); + write(&dir.join("drawing/v5/a.rs"), "//! a"); + assert!(refused(&dir).contains("expected v4/ next, found v5/")); + std::fs::rename(dir.join("drawing/v5"), dir.join("drawing/v4")).unwrap(); + write(&dir.join("drawing/v4/Bad_Name.rs"), "//! b"); + assert!(refused(&dir).contains("lowercase letters")); + std::fs::remove_file(dir.join("drawing/v4/Bad_Name.rs")).unwrap(); + write(&dir.join("drawing/pending/no-note.rs"), "pub fn step() {}"); + assert!(refused(&dir).contains("`//!` note")); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn a_step_is_a_declaration_not_a_mention() { + assert!(!declares_step("//! No fn step() is needed.\n")); + assert!(!declares_step("//! a note\n// fn step(v) was removed\nconst X: u8 = 1;\n")); + assert!(declares_step("//! a note\npub fn step (v: &mut Value) -> Result<(), E> { Ok(()) }\n")); + // a declaration inside a block comment, nested or not, is not one + assert!(!declares_step("//! Documentation only.\n/*\nfn step() {}\n*/\n")); + assert!(!declares_step("/* outer /* inner */\nfn step() {}\n*/\nconst X: u8 = 1;\n")); + assert!(declares_step("/* a note */\nfn step(v: &mut Value) {}\n")); + // comment markers inside strings and character literals are not comments + assert!(declares_step("//! A change with a step.\nconst MARKER: &str = \"/*\";\npub fn step(_v: &mut Value) -> Result { Ok(()) }\n")); + assert!(declares_step("const A: &str = r#\"/* \"quoted\" \"#;\nconst B: &[u8] = br\"/*\";\nfn step(v: &mut Value) {}\n")); + assert!(declares_step("const Q: char = '\"';\nconst E: char = '\\'';\nfn step(v: &mut Value) { let _: &'static str = \"\"; }\n")); + assert!(declares_step("const S: &str = \"a \\\" /* b\";\nfn step(v: &mut Value) {}\n")); + assert!(!declares_step("const S: &str = \"\nfn step(v) \";\n"), "inside a string"); + // a comment between the words is whitespace + assert!(declares_step("pub fn/* migration */step(v: &mut Value) {}\n")); + // across lines + assert!(declares_step("//! A change with a step.\npub fn step\n (_v: &mut Value) -> Result {\n Ok(())\n}\n")); + assert!(declares_step("pub fn\nstep(v: &mut Value) {}\n")); + assert!(!declares_step("fn stepper(v: &mut Value) {}\nfn prefn step() {}\n"), "other names"); + assert!(declares_step("const X: u8 = 1; // fn step() is below\nfn step(v: &mut Value) {}\n")); + assert!(declares_step("//! a note\npub(crate) fn step(v: &mut Value) -> Result<(), E> { Ok(()) }\n")); + assert!(declares_step("//! a note\n fn step(v: &mut Value) {}\n")); + assert!(!declares_step("//! a note\nfn stepper(v: &mut Value) {}\nfn fnstep() {}\n")); +} + +#[test] +fn a_documentation_only_change_has_no_step_and_a_missing_version_folder_fails() { + let dir = scratch("doc-only"); + write(&dir.join("drawing/v4/doc.rs"), "//! No fn step() is needed.\n"); + let code = lists(&dir, &[DRAWING]).unwrap().code; + assert!(code.contains("Part { name: \"doc\", step: None }"), "{code}"); + // a version folder that can't be listed is an error, not an empty format + let missing = parts(&dir.join("drawing/v9"), "x", false, &mut String::new(), &mut String::new()); + assert!(missing.is_err()); + assert!(parts(&dir.join("drawing/pending"), "x", true, &mut String::new(), &mut String::new()).is_ok()); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/crates/format/src/container.rs b/crates/format/src/container.rs new file mode 100644 index 000000000..00377baf5 --- /dev/null +++ b/crates/format/src/container.rs @@ -0,0 +1,440 @@ +//! The file container: a ZIP package of named parts (the file's JSON, binary payloads, optional +//! caches), never extracted to filesystem paths. Payloads are content-addressed blobs, +//! `blobs/.`, stored once however often they are referred to. Every size is +//! bounded by the app's [`Limits`]: the file, each part, all parts expanded, the payloads resolved, +//! and the number of entries; a part is read no further than its declared size (a decompression +//! bomb stops there). What a part holds, and how a payload is encoded, is the app's. + +use std::collections::{BTreeMap, BTreeSet}; +use std::io::{Read, Seek, SeekFrom}; +use std::sync::Arc; + +use serde_json::Value; +use sha2::{Digest, Sha256}; + +use crate::FormatError; + +/// How hard the payloads are compressed: Fast favours interactive file operations; Compact trades +/// CPU time for space. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Compression { + #[default] + Fast, + Compact, +} + +/// An app's budgets for its containers. The functions are read each time they are needed, so a +/// limit the app lets the user raise takes effect at once. +#[derive(Clone, Copy)] +pub struct Limits { + /// The most entries a container holds. + pub max_entries: usize, + /// The most bytes all parts may expand to, and all payloads may resolve to. + pub max_expanded: fn() -> usize, + /// The most bytes a part of this name may expand to. + pub part_limit: fn(&str) -> usize, + /// The Deflate level of a compressed part of this name and size. + pub level: fn(&str, usize) -> i64, + /// Whether a container of this many bytes may be read, or why not. + pub check_size: fn(u64) -> Result<(), String>, + /// Whether the work is cancelled (a save the person stopped): checked before each part, and + /// a long part is written a piece at a time and stops before the next piece when this says + /// so ([`FormatError::Cancelled`]). `None`: it cannot be cancelled here. + pub cancelled: Option bool>, +} + +/// The most bytes of a part written between two cancellation checks. +const PIECE: usize = 1 << 20; + +fn invalid(message: impl std::fmt::Display) -> FormatError { + FormatError::Invalid(message.to_string()) +} + +/// Whether the bytes start as a container does, whatever the file is called. +pub fn is_container(bytes: &[u8]) -> bool { + bytes.starts_with(b"PK\x03\x04") +} + +/// The SHA-256 of the bytes. +pub fn sha256(bytes: &[u8]) -> [u8; 32] { + Sha256::digest(bytes).into() +} + +fn hex(digest: &[u8; 32]) -> String { + use std::fmt::Write as _; + let mut out = String::with_capacity(64); + for byte in digest { + // Writing to a `String` cannot fail. + let _ = write!(&mut out, "{byte:02x}"); + } + out +} + +/// The SHA-256 of the bytes, in lowercase hexadecimal: a blob's name, and how a cache is keyed to +/// the file it was written for. +pub fn digest(bytes: &[u8]) -> String { + hex(&sha256(bytes)) +} + +/// One part a [`Writer`] wrote, with the SHA-256 of its uncompressed content. A caller that signs +/// a container uses these to bind the parts without reading the file back. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct PartDigest { + pub name: String, + pub len: u64, + pub digest: [u8; 32], +} + +/// A container being written. +pub struct Writer { + zip: craft_zip::Writer, + names: BTreeSet, + expanded: usize, + resolved: usize, + limits: Limits, + /// Which written parts to record for a signature, and the records so far. + tracked: Option, +} + +/// A tracking closure and the part digests collected so far. +type TrackedParts = (fn(&str) -> bool, Vec); + +impl Writer { + pub fn new(limits: Limits) -> Writer { + Writer { zip: craft_zip::Writer::new(), names: BTreeSet::new(), expanded: 0, resolved: 0, limits, tracked: None } + } + + /// Record the parts written from now on whose name `include` accepts, with their SHA-256, for + /// [`Writer::take_parts`]. Used by a caller that signs the container. + pub fn track_parts(&mut self, include: fn(&str) -> bool) { + self.tracked = Some((include, Vec::new())); + } + + /// The tracked parts written so far, oldest first, and stop tracking. + pub fn take_parts(&mut self) -> Option> { + self.tracked.take().map(|(_, parts)| parts) + } + + /// The container's bytes. + pub fn finish(self) -> Result, FormatError> { + self.zip.finish().map_err(invalid) + } + + /// Add a part, compressed or stored. A name already written is written once. + pub fn part(&mut self, name: &str, bytes: &[u8], compressed: bool) -> Result<(), FormatError> { + self.part_with_digest(name, bytes, compressed, None) + } + + /// [`Writer::part`] with a digest the caller already worked out (a blob's name is its digest), + /// so a signed save does not hash the same bytes twice. + fn part_with_digest(&mut self, name: &str, bytes: &[u8], compressed: bool, digest: Option<[u8; 32]>) -> Result<(), FormatError> { + if bytes.len() > (self.limits.part_limit)(name) { + return Err(invalid("part too large")); + } + if self.names.contains(name) { + return Ok(()); + } + // Check the counters this part would leave, and commit them only once it is written: a + // refused part is not remembered as written. + let expanded = self.expanded.checked_add(bytes.len()).ok_or_else(|| invalid("expanded size overflow"))?; + if expanded > (self.limits.max_expanded)() || self.names.len().saturating_add(1) > self.limits.max_entries { + return Err(invalid("package too large")); + } + let cancelled = self.limits.cancelled; + if cancelled.is_some_and(|c| c()) { + return Err(FormatError::Cancelled); + } + // the configured level, clamped to the Deflate levels craft-zip writes (1 to 9) + let compression = if compressed { + craft_zip::Compression::Level((self.limits.level)(name, bytes.len()).clamp(1, 9) as u8) + } else { + craft_zip::Compression::Stored + }; + match cancelled { + Some(cancelled) if bytes.len() > PIECE => { + let mut stopped = false; + let written = self.zip.add_with(name, compression, |out| { + for piece in bytes.chunks(PIECE) { + if cancelled() { + stopped = true; + return Err(std::io::Error::other("cancelled")); + } + out.write_all(piece)?; + } + Ok(()) + }); + if stopped { + return Err(FormatError::Cancelled); + } + written.map_err(invalid)?; + } + _ => self.zip.add(name, bytes, compression).map_err(invalid)?, + } + self.names.insert(name.to_owned()); + self.expanded = expanded; + if let Some((include, parts)) = &mut self.tracked + && include(name) + { + let digest = digest.unwrap_or_else(|| sha256(bytes)); + parts.push(PartDigest { name: name.to_owned(), len: bytes.len() as u64, digest }); + } + Ok(()) + } + + /// Would `entries` more optional parts of `extra` bytes, stored uncompressed, still keep the + /// file within `budget` and its expanded size within the reader's budget? The parts already + /// written count at their expanded size (an upper bound on what they take in the file). + pub fn room(&self, entries: usize, extra: usize, budget: usize) -> bool { + self.remaining(entries, budget).is_some_and(|left| extra <= left) + } + + /// The bytes `entries` more optional parts may hold together ([`Writer::room`]). + pub fn space(&self, entries: usize, budget: usize) -> usize { + self.remaining(entries, budget).unwrap_or(0) + } + + /// The bytes left for `entries` more parts, or `None` when the entries don't fit or what is + /// written already is past `budget` or the reader's budget. + fn remaining(&self, entries: usize, budget: usize) -> Option { + let count = self.names.len().checked_add(entries).filter(|&n| n <= self.limits.max_entries)?; + let overhead = count.saturating_mul(256).saturating_add(65_536); + let in_file = budget.checked_sub(overhead.saturating_add(self.expanded))?; + let expanded = (self.limits.max_expanded)().checked_sub(self.expanded)?; + Some(in_file.min(expanded)) + } + + /// Add a payload as a blob, `blobs/.`, and say its name, for the JSON that + /// refers to it. + pub fn blob(&mut self, bytes: &[u8], extension: &str, compressed: bool) -> Result { + let resolved = self + .resolved + .checked_add(bytes.len()) + .filter(|&n| n <= (self.limits.max_expanded)()) + .ok_or_else(|| invalid("resolved payloads too large"))?; + let digest = sha256(bytes); + let name = format!("blobs/{}.{extension}", hex(&digest)); + self.part_with_digest(&name, bytes, compressed, Some(digest))?; + self.resolved = resolved; + Ok(Value::String(name)) + } +} + +/// A container being read. Opening reads all bytes into memory and checks the directory +/// (the entry count, unique and plain names, every part's +/// declared size), not the parts. +pub struct Reader { + archive: craft_zip::Archive<'static>, + blobs: BTreeMap>, + /// The bytes of payloads resolved so far, repeats counted: bounded by the expanded budget. + pub hydrated: usize, + document: Option>, + limits: Limits, + /// Every byte part reads have expanded to, whatever the budget was at the time. + read_total: u64, +} + +impl Reader { + pub fn new(mut source: impl Read + Seek, limits: Limits) -> Result { + let length = source.seek(SeekFrom::End(0)).map_err(invalid)?; + (limits.check_size)(length).map_err(invalid)?; + + // Read all bytes into memory for the archive, no more than the length just approved: a file + // that grew since is refused rather than read past its budget. + source.seek(SeekFrom::Start(0)).map_err(invalid)?; + let capacity = usize::try_from(length).map_err(|_| invalid("file too large"))?; + let mut bytes = Vec::new(); + bytes.try_reserve_exact(capacity).map_err(|_| invalid("file too large"))?; + source.take(length.saturating_add(1)).read_to_end(&mut bytes).map_err(invalid)?; + if bytes.len() as u64 != length { + return Err(invalid("the file changed while it was read")); + } + + let mut archive = craft_zip::Archive::open_owned( + bytes, + craft_zip::Limits { max_entries: limits.max_entries as u64, max_total_bytes: (limits.max_expanded)() as u64 }, + ) + .map_err(invalid)?; + // the directory lists distinct names, so a repeated name is caught here + if archive.has_duplicate_names() { + return Err(invalid("duplicate or invalid archive entry")); + } + + // Validate entry names match format rules: no empty, no backslashes, + // no path components that are "." or "..", and no duplicates; and every part's declared + // size within its configured limit. + let mut name_set = BTreeSet::new(); + for entry_idx in 0..archive.len() { + // an entry whose name can't be read would be left out of the package unnoticed + let Ok(Some(name)) = archive.entry_name(entry_idx) else { + return Err(invalid("duplicate or invalid archive entry")); + }; + { + if name.is_empty() + || name.contains('\\') + || name.split('/').any(|c| c.is_empty() || c == "." || c == "..") + || !name_set.insert(name.clone()) + { + return Err(invalid("duplicate or invalid archive entry")); + } + if archive.declared_size(&name).is_some_and(|size| size > (limits.part_limit)(&name) as u64) { + return Err(invalid("part too large")); + } + } + } + + Ok(Self { archive, blobs: BTreeMap::new(), hydrated: 0, document: None, limits, read_total: 0 }) + } + + /// The file's own part, `document.json`, read once and kept: provisional caches and the + /// authoritative reading are bound to the same bytes, even if another process overwrites the + /// open file in place. Blobs remain hash-checked. + pub fn document(&mut self, limit: usize) -> Result, FormatError> { + if self.document.is_none() { + let bytes = self.limited_part("document.json", limit)?; + self.document = Some(bytes.into()); + } + // a cached document still keeps the current part limit + let limit = limit.min((self.limits.part_limit)("document.json")); + self.document.as_ref().filter(|d| d.len() <= limit).cloned().ok_or_else(|| invalid("metadata budget")) + } + + /// Whether the container has a part of this name. + pub fn has_part(&self, name: &str) -> bool { + self.archive.contains(name) + } + + /// The names of every part, in the container's order. + pub fn part_names(&self) -> Vec { + self.archive.names().map(|n| n.into_owned()).collect() + } + + /// The part of this name, at most `limit` bytes and never more than its configured limit. + pub fn limited_part(&mut self, name: &str, limit: usize) -> Result, FormatError> { + let limit = limit.min((self.limits.part_limit)(name)); + let expected = self.archive.declared_size(name).ok_or_else(|| invalid("part not found"))?; + if expected > limit as u64 { + return Err(invalid("part too large")); + } + // the expanded budget is read each time: a raised (or lowered) one applies at once, the + // bytes already read still counting against it + let budget = self.archive.budget_mut(); + *budget = craft_zip::Budget::new(budget.what(), (self.limits.max_expanded)() as u64); + if budget.charge(self.read_total).is_err() { + return Err(invalid("package too large")); + } + // Read within budget. craft-zip's read() handles decompression bombs via the limit. + let bytes = self.archive.read(name, limit as u64).map_err(invalid)?.ok_or_else(|| invalid("part not found"))?; + if bytes.len() as u64 != expected { + return Err(invalid("part length mismatch")); + } + self.read_total = self.read_total.saturating_add(expected); + Ok(bytes) + } + + /// The blob `value`'s `field` names, of this extension, within its part limit. + pub fn blob(&mut self, value: &Value, field: &str, extension: &str) -> Result<&[u8], FormatError> { + let limit = (self.limits.part_limit)(&format!("blob.{extension}")); + self.limited_blob(value, field, extension, limit) + } + + /// [`Reader::blob`] within `limit` bytes. Its hash is checked when it is first read, and every + /// reference counts against the resolved budget. + pub fn limited_blob(&mut self, value: &Value, field: &str, extension: &str, limit: usize) -> Result<&[u8], FormatError> { + let name = value.get(field).and_then(Value::as_str).ok_or_else(|| invalid(format!("missing `{field}` reference")))?; + // a cached blob still keeps the part's current limit + let limit = limit.min((self.limits.part_limit)(name)); + let hash = name + .strip_prefix("blobs/") + .and_then(|n| n.strip_suffix(&format!(".{extension}"))) + .filter(|h| h.len() == 64 && h.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))) + .ok_or_else(|| invalid("invalid blob reference"))?; + if !self.blobs.contains_key(name) { + let bytes = self.limited_part(name, limit)?; + if digest(&bytes) != hash { + return Err(invalid(format!("blob hash mismatch: {name}"))); + } + self.blobs.insert(name.to_owned(), bytes); + } + let bytes = self.blobs.get(name).ok_or_else(|| invalid("missing blob"))?; + if bytes.len() > limit { + return Err(invalid("blob exceeds its resource budget")); + } + // Count repeated references too: a tiny archive must not expand into unbounded owned + // strings or arrays when the same blob is referenced repeatedly. + self.hydrated = self + .hydrated + .checked_add(bytes.len()) + .filter(|&n| n <= (self.limits.max_expanded)()) + .ok_or_else(|| invalid("resolved payloads too large"))?; + Ok(bytes) + } +} + +/// A small PNG picture of the file, kept in the container as a disposable cache keyed to the +/// exact file it shows: `cache/thumbnail.json` (its key) and `cache/thumbnail.png`. Reading it +/// never resolves a payload; a picture that isn't one, or is stale, is a miss. +#[derive(Clone, Copy, Debug)] +pub struct Thumbnail { + /// The key's format tag. + pub format: &'static str, + /// The build that wrote it: only the same build reads it back. + pub app: &'static str, + /// Its width and height in pixels. + pub size: [usize; 2], + /// The most bytes it may have. + pub max_bytes: usize, +} + +impl Thumbnail { + /// Whether the bytes start as a PNG of this size within the byte budget (not a decode: a + /// consumer decodes with its own dimension and allocation limits). + pub fn valid(&self, bytes: &[u8]) -> bool { + bytes.len() <= self.max_bytes + && bytes.get(..8) == Some(b"\x89PNG\r\n\x1a\n") + && bytes.get(8..16) == Some(b"\0\0\0\rIHDR") + && bytes.get(16..20) == Some((self.size[0] as u32).to_be_bytes().as_slice()) + && bytes.get(20..24) == Some((self.size[1] as u32).to_be_bytes().as_slice()) + } + + /// Add the picture, keyed to `document` (the file's own part), when it is one and there is + /// room for it within `budget`; otherwise leave it out, never failing the save. + pub fn write(&self, writer: &mut Writer, png: &[u8], document: &[u8], budget: usize) -> Result<(), FormatError> { + if !self.valid(png) { + return Ok(()); + } + let key = + serde_json::to_vec(&serde_json::json!({"format": self.format, "app": self.app, "document": digest(document), "payload": digest(png)})) + .map_err(invalid)?; + let fits = |name: &str, len: usize| len <= (writer.limits.part_limit)(name); + if !fits("cache/thumbnail.json", key.len()) + || !fits("cache/thumbnail.png", png.len()) + || !writer.room(2, key.len().saturating_add(png.len()), budget) + { + return Ok(()); + } + writer.part("cache/thumbnail.json", &key, false)?; + writer.part("cache/thumbnail.png", png, false) + } + + /// The picture, when this build wrote it for exactly the file's own part (`document.json`, + /// read within `document_limit`). + pub fn read(&self, reader: &mut Reader, document_limit: usize) -> Result, FormatError> { + let key: Value = serde_json::from_slice(&reader.limited_part("cache/thumbnail.json", 4096)?).map_err(invalid)?; + let document = reader.document(document_limit)?; + if key.get("format").and_then(Value::as_str) != Some(self.format) + || key.get("app").and_then(Value::as_str) != Some(self.app) + || key.get("document").and_then(Value::as_str) != Some(digest(&document).as_str()) + { + return Err(invalid("thumbnail key")); + } + let png = reader.limited_part("cache/thumbnail.png", self.max_bytes)?; + if !self.valid(&png) || key.get("payload").and_then(Value::as_str) != Some(digest(&png).as_str()) { + return Err(invalid("thumbnail payload")); + } + Ok(png) + } +} + +#[cfg(test)] +#[path = "container_tests.rs"] +mod tests; diff --git a/crates/format/src/container_tests.rs b/crates/format/src/container_tests.rs new file mode 100644 index 000000000..6f9c06bbf --- /dev/null +++ b/crates/format/src/container_tests.rs @@ -0,0 +1,260 @@ +use super::*; +use std::io::Cursor; + +const LIMITS: Limits = Limits { + max_entries: 8, + max_expanded: || 1 << 20, + part_limit: |name| if name.ends_with(".big") { 1 << 20 } else { 1024 }, + level: |_, _| 1, + check_size: |length| if length <= 1 << 20 { Ok(()) } else { Err("too large".into()) }, + cancelled: None, +}; + +/// Parts and blobs written are read back as they were; a blob is stored once, named by its hash, +/// and checked against it. +#[test] +fn parts_and_blobs_read_back() { + let mut writer = Writer::new(LIMITS); + writer.part("document.json", br#"{"a":1}"#, true).unwrap(); + let first = writer.blob(b"payload", "bin", false).unwrap(); + let again = writer.blob(b"payload", "bin", true).unwrap(); + assert_eq!(first, again, "the same payload is one blob"); + assert_eq!(first, Value::String(format!("blobs/{}.bin", digest(b"payload")))); + let bytes = writer.finish().unwrap(); + assert!(is_container(&bytes)); + let mut reader = Reader::new(Cursor::new(bytes), LIMITS).unwrap(); + assert_eq!(reader.part_names().len(), 2); + assert!(reader.has_part("document.json") && !reader.has_part("other.json")); + assert_eq!(&*reader.document(64).unwrap(), br#"{"a":1}"#); + let reference = serde_json::json!({"blob": first}); + assert_eq!(reader.blob(&reference, "blob", "bin").unwrap(), b"payload"); + assert!(reader.blob(&reference, "blob", "png").is_err(), "another extension"); + assert_eq!(reader.hydrated, 7); + reader.hydrated = (LIMITS.max_expanded)(); + assert!(reader.blob(&reference, "blob", "bin").unwrap_err().to_string().contains("resolved payloads too large")); +} + +/// A part past its limit, or more entries than the limit, are refused as they are written. +#[test] +fn budgets_are_kept_when_writing() { + let mut writer = Writer::new(LIMITS); + assert_eq!(writer.part("small.json", &[0; 1025], false), Err(FormatError::Invalid("part too large".into()))); + assert!(writer.part("large.big", &[0; 1025], true).is_ok()); + for i in 0..7 { + writer.part(&format!("{i}.json"), b"x", false).unwrap(); + } + assert_eq!(writer.part("one-more.json", b"x", false), Err(FormatError::Invalid("package too large".into()))); + assert!(!writer.room(1, 0, 1 << 20), "no room for an entry past the limit"); +} + +/// A thumbnail is kept only for the file it shows. +#[test] +fn a_thumbnail_is_keyed_to_its_file() { + let picture = Thumbnail { format: "app-thumbnail/1", app: "1.0", size: [2, 1], max_bytes: 1024 }; + let mut png = b"\x89PNG\r\n\x1a\n\0\0\0\rIHDR".to_vec(); + png.extend_from_slice(&2u32.to_be_bytes()); + png.extend_from_slice(&1u32.to_be_bytes()); + assert!(picture.valid(&png)); + let mut writer = Writer::new(LIMITS); + writer.part("document.json", b"{}", false).unwrap(); + picture.write(&mut writer, &png, b"{}", 8 << 20).unwrap(); + let mut reader = Reader::new(Cursor::new(writer.finish().unwrap()), LIMITS).unwrap(); + assert_eq!(picture.read(&mut reader, 64).unwrap(), png); + let mut writer = Writer::new(LIMITS); + writer.part("document.json", b"{\"changed\":1}", false).unwrap(); + picture.write(&mut writer, &png, b"{}", 8 << 20).unwrap(); + let mut reader = Reader::new(Cursor::new(writer.finish().unwrap()), LIMITS).unwrap(); + assert!(picture.read(&mut reader, 64).is_err(), "stale"); +} + +/// A refused part is not remembered as written: once the budget is raised it is written. +#[test] +fn a_refused_part_can_be_written_later() { + static BUDGET: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(4); + let limits = Limits { max_expanded: || BUDGET.load(std::sync::atomic::Ordering::SeqCst), ..LIMITS }; + let mut writer = Writer::new(limits); + assert_eq!(writer.part("a.json", b"12345678", false), Err(FormatError::Invalid("package too large".into()))); + assert!(writer.blob(b"12345678", "bin", false).is_err()); + BUDGET.store(1 << 20, std::sync::atomic::Ordering::SeqCst); + writer.part("a.json", b"12345678", false).unwrap(); + writer.blob(b"12345678", "bin", false).unwrap(); + let reader = Reader::new(Cursor::new(writer.finish().unwrap()), limits).unwrap(); + assert_eq!(reader.part_names().len(), 2, "both parts were written on the retry"); +} + +/// A cancelled save writes no more parts, whether the part is small or written in pieces. +#[test] +fn cancellation_stops_the_save() { + let cancelled = Limits { cancelled: Some(|| true), ..LIMITS }; + let mut writer = Writer::new(cancelled); + assert_eq!(writer.part("a.json", b"a", false), Err(FormatError::Cancelled)); + // cancelled after the first piece of a long part: checks before the part, then per piece + static CHECKS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0); + let midway = Limits { + cancelled: Some(|| CHECKS.fetch_add(1, std::sync::atomic::Ordering::SeqCst) >= 2), + max_expanded: || 1 << 22, + part_limit: |_| 1 << 22, + ..LIMITS + }; + let mut writer = Writer::new(midway); + assert_eq!(writer.part("big.big", &vec![0; PIECE + 1], true), Err(FormatError::Cancelled)); + assert_eq!(CHECKS.load(std::sync::atomic::Ordering::SeqCst), 3); + let going = Limits { cancelled: Some(|| false), max_expanded: || 1 << 22, part_limit: |_| 1 << 22, ..LIMITS }; + let mut writer = Writer::new(going); + writer.part("big.big", &vec![7; PIECE + 1], true).unwrap(); + let mut reader = Reader::new(Cursor::new(writer.finish().unwrap()), going).unwrap(); + assert_eq!(reader.limited_part("big.big", 1 << 22).unwrap().len(), PIECE + 1); +} + +/// Room counts what is already written, and the reader keeps every part's configured limit. +#[test] +fn room_counts_written_parts_and_reads_keep_part_limits() { + let mut writer = Writer::new(LIMITS); + writer.part("large.big", &[0; 500 * 1024], false).unwrap(); + assert!(!writer.room(1, 200 * 1024, 600 * 1024)); + assert!(writer.room(1, 1024, 1 << 20)); + // what is written already is past the budget: not even an empty part fits + assert!(!writer.room(1, 0, 400 * 1024)); + assert_eq!(writer.space(1, 400 * 1024), 0); + let wide = Limits { part_limit: |_| 1 << 20, ..LIMITS }; + let mut writer = Writer::new(wide); + writer.part("document.json", &[b' '; 2048], false).unwrap(); + let bytes = writer.finish().unwrap(); + assert!(Reader::new(Cursor::new(bytes.clone()), LIMITS).is_err(), "declared past its configured limit"); + let mut reader = Reader::new(Cursor::new(bytes), wide).unwrap(); + assert!(reader.limited_part("document.json", 4096).is_ok()); +} + +/// A thumbnail past a cache part's own limit is left out, never failing the save. +#[test] +fn a_thumbnail_past_its_part_limit_is_left_out() { + let picture = Thumbnail { format: "app-thumbnail/1", app: "1.0", size: [2, 1], max_bytes: 2048 }; + let mut png = b"\x89PNG\r\n\x1a\n\0\0\0\rIHDR".to_vec(); + png.extend_from_slice(&2u32.to_be_bytes()); + png.extend_from_slice(&1u32.to_be_bytes()); + png.resize(1500, 0); + let mut writer = Writer::new(LIMITS); + picture.write(&mut writer, &png, b"{}", 8 << 20).unwrap(); + let reader = Reader::new(Cursor::new(writer.finish().unwrap()), LIMITS).unwrap(); + assert!(!reader.has_part("cache/thumbnail.json") && !reader.has_part("cache/thumbnail.png")); +} + +/// A source that grows after its length was approved is refused. +#[test] +fn a_source_that_grows_while_read_is_refused() { + struct Growing(Cursor>, bool); + impl Read for Growing { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + if !self.1 { + self.1 = true; + let end = self.0.get_ref().len(); + self.0.get_mut().extend_from_slice(&[0; 16]); + self.0.set_position(self.0.position().min(end as u64)); + } + self.0.read(buf) + } + } + impl Seek for Growing { + fn seek(&mut self, to: SeekFrom) -> std::io::Result { + self.0.seek(to) + } + } + let mut writer = Writer::new(LIMITS); + writer.part("document.json", b"{}", false).unwrap(); + let bytes = writer.finish().unwrap(); + assert!(Reader::new(Growing(Cursor::new(bytes), false), LIMITS).err().is_some_and(|e| e.to_string().contains("changed"))); +} + +/// Compressed parts use the configured Deflate level. +#[test] +fn compressed_parts_use_the_configured_level() { + // words in a varied order, so the levels compress them differently + let words = ["alpha ", "beta ", "gamma ", "delta ", "epsilon ", "zeta ", "eta ", "theta "]; + let mut seed = 7u32; + let text: Vec = (0..150) + .flat_map(|_| { + seed = seed.wrapping_mul(1_103_515_245).wrapping_add(12_345); + words[(seed >> 16) as usize % words.len()].bytes() + }) + .take(1000) + .collect(); + let size = |level: fn(&str, usize) -> i64| { + let mut writer = Writer::new(Limits { level, ..LIMITS }); + writer.part("document.json", &text, true).unwrap(); + writer.finish().unwrap().len() + }; + assert_ne!(size(|_, _| 1), size(|_, _| 9), "the level is used"); + assert_eq!(size(|_, _| 42), size(|_, _| 9), "past 9 is clamped"); + assert_eq!(size(|_, _| -3), size(|_, _| 1), "below 1 is clamped"); +} + +/// An archive naming one part twice is refused. +#[test] +fn a_repeated_part_name_is_refused() { + let mut writer = Writer::new(LIMITS); + writer.part("document.json", b"{}", false).unwrap(); + writer.part("documenX.json", b"[]", false).unwrap(); + let mut bytes = writer.finish().unwrap(); + let (from, to) = (b"documenX.json", b"document.json"); + let mut at = 0; + while let Some(i) = bytes[at..].windows(from.len()).position(|w| w == from) { + bytes[at + i..at + i + to.len()].copy_from_slice(to); + at += i + to.len(); + } + assert!(Reader::new(Cursor::new(bytes), LIMITS).is_err()); +} + +/// Cached documents and blobs keep the current part limits. +#[test] +fn cached_parts_keep_the_current_part_limits() { + let wide = Limits { part_limit: |_| 1 << 20, ..LIMITS }; + let mut writer = Writer::new(wide); + writer.part("document.json", &[b' '; 512], false).unwrap(); + let blob = writer.blob(&[7; 512], "bin", false).unwrap(); + let bytes = writer.finish().unwrap(); + let mut reader = Reader::new(Cursor::new(bytes), wide).unwrap(); + let reference = serde_json::json!({"blob": blob}); + assert!(reader.document(4096).is_ok()); + assert!(reader.blob(&reference, "blob", "bin").is_ok()); + reader.limits = Limits { part_limit: |_| 128, ..wide }; + assert!(reader.document(4096).is_err()); + assert!(reader.limited_blob(&reference, "blob", "bin", 4096).is_err()); +} + +/// A raised expanded budget applies to the open reader, and what was read still counts. +#[test] +fn a_raised_expanded_budget_applies_at_once() { + static BUDGET: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(600); + let limits = Limits { max_expanded: || BUDGET.load(std::sync::atomic::Ordering::Relaxed), part_limit: |_| 1 << 20, ..LIMITS }; + let wide = Limits { max_expanded: || 1 << 20, ..limits }; + let mut writer = Writer::new(wide); + writer.part("document.json", &[b' '; 256], false).unwrap(); + writer.part("other.big", &[1; 512], false).unwrap(); + let bytes = writer.finish().unwrap(); + BUDGET.store(1 << 20, std::sync::atomic::Ordering::Relaxed); + let mut reader = Reader::new(Cursor::new(bytes), limits).unwrap(); + BUDGET.store(600, std::sync::atomic::Ordering::Relaxed); + assert!(reader.limited_part("document.json", 4096).is_ok()); + assert!(reader.limited_part("other.big", 4096).is_err(), "256 read + 512 is past 600"); + BUDGET.store(1024, std::sync::atomic::Ordering::Relaxed); + assert!(reader.limited_part("other.big", 4096).is_ok(), "768 fits the raised 1024"); + // a lowered budget does not forget what was read: 768 + 256 is past 1000 after a dip to 100 + BUDGET.store(100, std::sync::atomic::Ordering::Relaxed); + assert!(reader.limited_part("document.json", 4096).is_err()); + BUDGET.store(1000, std::sync::atomic::Ordering::Relaxed); + assert!(reader.limited_part("document.json", 4096).is_err(), "1024 is past 1000"); + BUDGET.store(1024, std::sync::atomic::Ordering::Relaxed); + assert!(reader.limited_part("document.json", 4096).is_ok()); +} + +/// An entry whose local header can't be read is refused at open, not skipped. +#[test] +fn an_unreadable_entry_is_refused() { + let mut writer = Writer::new(LIMITS); + writer.part("document.json", b"{}", false).unwrap(); + writer.part("other.json", b"[]", false).unwrap(); + let mut bytes = writer.finish().unwrap(); + let second = bytes.windows(4).enumerate().filter(|(_, w)| *w == b"PK\x03\x04").map(|(i, _)| i).nth(1).unwrap(); + bytes[second + 2] = 0; + assert!(Reader::new(Cursor::new(bytes), LIMITS).is_err()); +} diff --git a/crates/format/src/lib.rs b/crates/format/src/lib.rs new file mode 100644 index 000000000..aadb56fd9 --- /dev/null +++ b/crates/format/src/lib.rs @@ -0,0 +1,257 @@ +//! Versioned file formats: what an app needs to read every file it ever wrote, and to open a file +//! a newer build of it wrote. +//! +//! A file starts with `"format": "N"`. A [`Family`] is one kind of file (an app's designs, +//! its drawings): its tag, its own numbers, and how a file of an older number is brought up to +//! this build's, step by step (N → N+1 → … → current). The app's code only knows the current +//! format; all translation lives in the steps: +//! - the steps written out by hand for the first formats, one per format ([`Formats::hand`]); +//! - then one folder per format, `v/`, one file per change in it ([`Part`]), each with a step +//! when files from before the change are rewritten as they open ([`Formats::folded`]); +//! - then the changes no release has a number for yet, in `pending/` ([`Formats::pending`]): +//! while there are any, the build writes the next number, with their names in the file's +//! `format_pending`, so a later build runs just the steps a file is missing. A release moves +//! them into the next `v/`. [`build`] lists the folders for a build script. +//! +//! A file a newer build wrote says so by a number past this build's or a change name this build +//! doesn't have ([`Family::newer`]); the app then reads what it can and keeps the rest +//! ([`newer`]). A file is saved as JSON, or in a [`container`] with its binary payloads. +//! +//! Steps are JSON-level (the file's shape) and return the app's own error type; every error this +//! crate makes is a [`FormatError`], which the app's error type takes `From`. + +#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))] + +use serde_json::Value; + +pub mod build; +pub mod container; +pub mod newer; + +/// Why a file is refused: its text says what is wrong, as the user reads it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum FormatError { + /// The file isn't one of the family's, or it is damaged. + Invalid(String), + /// The work was cancelled (a save the person stopped): nothing more was written. + Cancelled, +} + +impl std::fmt::Display for FormatError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + FormatError::Invalid(why) => f.write_str(why), + FormatError::Cancelled => f.write_str("cancelled"), + } + } +} + +impl std::error::Error for FormatError {} + +/// One JSON-level upgrade step: rewrites a file in place, or says why it can't. +pub type Step = fn(&mut Value) -> Result<(), E>; + +/// One change to a family's file format: a file in its `v/` or `pending/` folder. +pub struct Part { + /// The file's name, which a file written while the change was pending lists. + pub name: &'static str, + /// How a file from before the change is rewritten as it opens, if it is. + pub step: Option>, +} + +/// Another spelling of a change name that a file of one format may carry (a build that named the +/// change otherwise): read as the change it stands for, never written. +#[derive(Clone, Copy, Debug)] +pub struct Alias { + /// The format whose files may say it. + pub format: u32, + /// The spelling in the file. + pub name: &'static str, + /// The change it stands for. + pub known_as: &'static str, +} + +/// A family's upgrade path: the steps written out by hand, the folded formats after them (each a +/// list of changes) and the pending changes, which make the next format. +pub struct Formats<'a, E> { + /// Entry `i` turns format `i + 1` into `i + 2`. + pub hand: &'a [Step], + /// Each format after the hand-written ones, oldest first, and its changes. + pub folded: &'a [(u32, &'a [Part])], + /// The changes no release has a number for yet. + pub pending: &'a [Part], +} + +/// A kind of file whose format is versioned: its own format tag, its own numbers and its own +/// changes. Reading a file's format, telling a file a newer build saved and the upgrade are the +/// same for every family. +pub struct Family<'a, E> { + /// The app, as messages name it. + pub app: &'static str, + /// What a file of the family is called in messages. + pub noun: &'static str, + /// The tag before the number in a file's `format`. + pub prefix: &'static str, + /// The tag files saved under an older name carry: read, never written. + pub legacy_prefix: Option<&'static str>, + /// The format of a file that names none, if such files exist. + pub untagged: Option, + /// Other spellings of change names that files may carry. + pub aliases: &'a [Alias], + /// The upgrade path. + pub formats: Formats<'a, E>, +} + +/// How a file shows a newer build saved it ([`Family::newer`]). +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Newer { + /// The number the file says. + pub format: u32, + /// The number this build reads up to. + pub reads_up_to: u32, + /// The changes the file names that this build doesn't have. + pub unknown_changes: Vec, +} + +impl Formats<'_, E> { + /// The format this build writes: the last folded one, or the next while changes are pending. + pub const fn current(&self) -> u32 { + self.hand.len() as u32 + 1 + self.folded.len() as u32 + if self.pending.is_empty() { 0 } else { 1 } + } + + /// The changes format `version` is made of, if it is made of changes. + fn parts(&self, version: u32) -> Option<&[Part]> { + let after = version.checked_sub(self.hand.len() as u32 + 2)? as usize; + match self.folded.get(after) { + Some((_, parts)) => Some(parts), + None if after == self.folded.len() && !self.pending.is_empty() => Some(self.pending), + None => None, + } + } + + /// Whether a change of this name is anywhere in this build's history. + fn knows_part_name(&self, name: &str) -> bool { + self.folded.iter().flat_map(|(_, parts)| *parts).chain(self.pending).any(|part| part.name == name) + } +} + +impl Family<'_, E> { + /// The format this build writes: the last folded one, or the next while changes are pending. + pub const fn current(&self) -> u32 { + self.formats.current() + } + + /// What a file of the family is called in messages. + pub const fn noun(&self) -> &'static str { + self.noun + } + + /// The names of the pending changes this build writes into a file (`format_pending`). + pub fn pending_names(&self) -> Vec { + self.formats.pending.iter().map(|p| p.name.to_string()).collect() + } + + /// The format tag of format `n`. + pub fn format_string(&self, n: u32) -> String { + format!("{}{n}", self.prefix) + } + + /// The version in one of this family's format tags; `None` for any other tag, another + /// family's included. + pub fn version_of(&self, s: &str) -> Option { + // each tag form parses on its own: a legacy prefix may start with the primary one + // (`app/` and `app/design/`) + let parse = |prefix: &str| s.strip_prefix(prefix).and_then(|n| n.parse().ok()); + parse(self.prefix).or_else(|| self.legacy_prefix.and_then(parse)) + } + + /// The change a name a file of format `version` lists stands for: its alias's, or itself. + /// An alias counts only while this build has the change it stands for, so recognising it + /// never grants support to another reader. + fn canonical_applied_name<'name>(&self, version: u32, name: &'name str) -> &'name str { + match self.aliases.iter().find(|a| a.format == version && a.name == name) { + Some(alias) if self.formats.knows_part_name(alias.known_as) => alias.known_as, + _ => name, + } + } + + /// What says a file of format `version` with the changes `applied` came from a newer build: + /// a number past this build's, or a change this build doesn't have anywhere in its history + /// (a branch's pending change may have landed at a later number). A list beside a format that + /// isn't made of changes means nothing (a tool or a test set an older number on a newer file). + /// `None` for a file this build fully knows. + pub fn newer(&self, version: u32, applied: Option<&[String]>) -> Option { + let current = self.current(); + let listed = applied.filter(|_| version > current || self.formats.parts(version).is_some()).unwrap_or_default(); + let unknown_changes: Vec = + listed.iter().filter(|name| !self.formats.knows_part_name(self.canonical_applied_name(version, name))).cloned().collect(); + (version > current || !unknown_changes.is_empty()).then_some(Newer { format: version, reads_up_to: current, unknown_changes }) + } +} + +impl> Family<'_, E> { + /// Read a file's format and take its `format_pending` out of its JSON, before anything else + /// in it is read: a file that isn't this family's is refused here. One a newer build saved is + /// not: [`Family::newer`] says so. + pub fn take_version(&self, v: &mut Value) -> Result<(u32, Option>), E> { + let (app, noun) = (self.app, self.noun); + let not_ours = |what: String| E::from(FormatError::Invalid(format!("not a {app} {noun} ({what})"))); + let Some(obj) = v.as_object() else { return Err(not_ours("expected a JSON object".into())) }; + let version = match obj.get("format") { + None => self.untagged.ok_or_else(|| not_ours("no format".into()))?, + Some(Value::String(f)) => self.version_of(f).ok_or_else(|| not_ours(format!("format `{f}`")))?, + Some(_) => return Err(not_ours("format is not a string".into())), + }; + let applied = take_names(v, "format_pending", noun).map_err(E::from)?; + if version == 0 { + return Err(not_ours("format 0".into())); + } + Ok((version, applied)) + } + + /// Bring a file of format `version` up to this build's; `applied` is its `format_pending` + /// (both from [`Family::take_version`]): the changes of its format it already has, when it + /// was written while they were pending. The others' steps run first, and already-applied + /// changes never run again at their later number. A file past this build's number has no + /// steps to run. + pub fn upgrade(&self, v: &mut Value, version: u32, applied: Option<&[String]>) -> Result<(), E> { + let formats = &self.formats; + if version == 0 { + return Err(E::from(FormatError::Invalid(format!("not a {} {} (format 0)", self.app, self.noun)))); + } + // A list beside a format that isn't made of changes is ignored, like any field an upgrade + // drops. + let applied = applied.filter(|_| formats.parts(version).is_some()); + let already_applied = + |part: &Part| applied.is_some_and(|names| names.iter().any(|name| self.canonical_applied_name(version, name) == part.name)); + if applied.is_some() { + for part in formats.parts(version).unwrap_or_default().iter().filter(|part| !already_applied(part)) { + part.step.map_or(Ok(()), |step| step(v))?; + } + } + for from in version..self.current() { + match formats.hand.get(from as usize - 1) { + Some(step) => step(v)?, + None => { + for part in formats.parts(from + 1).unwrap_or_default().iter().filter(|part| !already_applied(part)) { + part.step.map_or(Ok(()), |step| step(v))?; + } + } + } + } + Ok(()) + } +} + +/// Take a short list of names out of a file's JSON: its `format_pending` (the pending changes it +/// has). +fn take_names(v: &mut Value, key: &str, noun: &str) -> Result>, FormatError> { + let Some(list) = v.as_object_mut().and_then(|o| o.remove(key)) else { return Ok(None) }; + let invalid = || FormatError::Invalid(format!("damaged {noun}: `{key}` is not a short list of names")); + let list = list.as_array().filter(|l| l.len() <= 1000).ok_or_else(invalid)?; + list.iter().map(|n| n.as_str().filter(|n| n.len() <= 128).map(str::to_string).ok_or_else(invalid)).collect::>().map(Some) +} + +#[cfg(test)] +#[path = "lib_tests.rs"] +mod tests; diff --git a/crates/format/src/lib_tests.rs b/crates/format/src/lib_tests.rs new file mode 100644 index 000000000..40c5101fb --- /dev/null +++ b/crates/format/src/lib_tests.rs @@ -0,0 +1,142 @@ +use super::*; +use serde_json::json; + +/// The app's error, as an app has one. +#[derive(Debug, PartialEq)] +struct AppError(String); + +impl From for AppError { + fn from(e: FormatError) -> Self { + AppError(e.to_string()) + } +} + +/// Each step appends its name to the file's `log`, so the order steps ran in can be read back. +fn log(v: &mut Value, step: &str) { + if let Some(list) = v.get_mut("log").and_then(Value::as_array_mut) { + list.push(json!(step)); + } +} + +const HAND: [Step; 2] = [ + |v| { + log(v, "1to2"); + Ok(()) + }, + |v| { + log(v, "2to3"); + Ok(()) + }, +]; +const V4: [Part; 2] = [ + Part { + name: "a", + step: Some(|v| { + log(v, "a"); + Ok(()) + }), + }, + Part { name: "b", step: None }, +]; +const V5: [Part; 1] = [Part { + name: "c", + step: Some(|v| { + log(v, "c"); + Ok(()) + }), +}]; +const FOLDED: [(u32, &[Part]); 2] = [(4, &V4), (5, &V5)]; +const PENDING: [Part; 2] = [ + Part { + name: "d", + step: Some(|v| { + log(v, "d"); + Ok(()) + }), + }, + Part { + name: "e", + step: Some(|v| { + log(v, "e"); + Ok(()) + }), + }, +]; +const ALIASES: [Alias; 1] = [Alias { format: 5, name: "old-c", known_as: "c" }]; + +/// Format 1 untagged; hand steps to 3; folded 4 and 5; pending changes make 6. +const FAMILY: Family<'static, AppError> = Family { + app: "App", + noun: "design", + prefix: "app/", + legacy_prefix: Some("old/"), + untagged: Some(1), + aliases: &ALIASES, + formats: Formats { hand: &HAND, folded: &FOLDED, pending: &PENDING }, +}; + +fn steps_run(version: u32, applied: Option<&[String]>) -> Vec { + let mut v = json!({"log": []}); + FAMILY.upgrade(&mut v, version, applied).unwrap(); + v["log"].as_array().unwrap().iter().map(|s| s.as_str().unwrap().to_string()).collect() +} + +#[test] +fn tags_and_numbers() { + assert_eq!(FAMILY.current(), 6); + assert_eq!(FAMILY.format_string(6), "app/6"); + assert_eq!(FAMILY.version_of("app/4"), Some(4)); + assert_eq!(FAMILY.version_of("old/3"), Some(3), "the legacy tag is read"); + assert_eq!(FAMILY.version_of("other/3"), None); + assert_eq!(FAMILY.pending_names(), vec!["d".to_string(), "e".to_string()]); +} + +#[test] +fn a_file_is_taken_apart_or_refused_with_the_apps_words() { + let mut v = json!({"format": "app/6", "format_pending": ["d"], "x": 1}); + assert_eq!(FAMILY.take_version(&mut v), Ok((6, Some(vec!["d".to_string()])))); + assert!(v.get("format_pending").is_none(), "taken out"); + assert_eq!(FAMILY.take_version(&mut json!({})), Ok((1, None)), "untagged"); + let refused = |mut v: Value| FAMILY.take_version(&mut v).unwrap_err().0; + assert_eq!(refused(json!([])), "not a App design (expected a JSON object)"); + assert_eq!(refused(json!({"format": "other/2"})), "not a App design (format `other/2`)"); + assert_eq!(refused(json!({"format": 2})), "not a App design (format is not a string)"); + assert_eq!(refused(json!({"format": "app/0"})), "not a App design (format 0)"); + assert_eq!(refused(json!({"format": "app/2", "format_pending": [1]})), "damaged design: `format_pending` is not a short list of names"); +} + +#[test] +fn every_step_runs_once_in_order() { + assert_eq!(steps_run(1, None), ["1to2", "2to3", "a", "c", "d", "e"]); + assert_eq!(steps_run(4, None), ["c", "d", "e"]); + assert_eq!(steps_run(6, None), Vec::::new()); + // A file written while `d` was pending has it: only `e` runs. + assert_eq!(steps_run(6, Some(&["d".to_string()])), ["e"]); + // A file of format 5 written while `c` was still pending, without it: `c` runs first. + assert_eq!(steps_run(5, Some(&[])), ["c", "d", "e"]); + // Under its other spelling `c` counts as had. + assert_eq!(steps_run(5, Some(&["old-c".to_string()])), ["d", "e"]); + // A list beside a hand-written format means nothing. + assert_eq!(steps_run(2, Some(&["d".to_string()])), ["2to3", "a", "c", "d", "e"]); + let mut v = json!({}); + assert_eq!(FAMILY.upgrade(&mut v, 0, None), Err(AppError("not a App design (format 0)".into()))); +} + +#[test] +fn a_newer_file_is_told_by_its_number_or_an_unknown_change() { + assert_eq!(FAMILY.newer(6, None), None); + assert_eq!(FAMILY.newer(6, Some(&["d".to_string()])), None); + assert_eq!(FAMILY.newer(7, None), Some(Newer { format: 7, reads_up_to: 6, unknown_changes: Vec::new() })); + assert_eq!(FAMILY.newer(6, Some(&["z".to_string()])), Some(Newer { format: 6, reads_up_to: 6, unknown_changes: vec!["z".to_string()] })); + assert_eq!(FAMILY.newer(5, Some(&["old-c".to_string()])), None, "an alias of a known change"); + assert_eq!(FAMILY.newer(4, Some(&["old-c".to_string()])), Some(Newer { format: 4, reads_up_to: 6, unknown_changes: vec!["old-c".to_string()] })); + assert_eq!(FAMILY.newer(2, Some(&["z".to_string()])), None, "a list beside a hand-written format means nothing"); +} + +#[test] +fn a_legacy_prefix_that_extends_the_primary_one_is_read() { + let nested = Family { prefix: "app/", legacy_prefix: Some("app/design/"), ..FAMILY }; + assert_eq!(nested.version_of("app/design/3"), Some(3)); + assert_eq!(nested.version_of("app/4"), Some(4)); + assert_eq!(nested.version_of("app/design/x"), None); +} diff --git a/crates/format/src/newer.rs b/crates/format/src/newer.rs new file mode 100644 index 000000000..0a3141606 --- /dev/null +++ b/crates/format/src/newer.rs @@ -0,0 +1,470 @@ +//! A file a newer build saved: it opens, with a warning, and saving it loses nothing. +//! +//! A newer format number, or a change the file names that this build doesn't have, says a newer +//! build saved the file ([`crate::Family::newer`]). It may use nothing this build lacks, so the app +//! reads it as usual. When part of it can't be read, that part stays out of the way and the rest +//! opens: the app reads the elements of its lists it can ([`NewerFile::keep_element`]), then any +//! other top-level section it can't read is left out whole ([`leave_out_unreadable`], +//! [`read_around`]), and a field it doesn't know, which typed reading would drop, is noticed by +//! comparing the file with what was read ([`NewerFile::compare`]). +//! +//! All of these are named in [`NewerFile::not_understood`] and go back into the file when it is +//! saved ([`NewerFile::restore_into`]), as they came. What can't be put back because the file was +//! changed around it is what `restore_into` returns. + +use std::collections::{BTreeMap, HashMap}; +use std::sync::Arc; + +use serde_json::{Map, Value, json}; + +use crate::Newer; + +/// The most sections left out one at a time to find the one a file can't be read with. +const MAX_SECTION_TRIES: usize = 64; + +/// What the file says and holds that this build doesn't read. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct NewerFile { + /// The app, as the notice names it. + pub app: &'static str, + /// The number the file says. + pub format: u32, + /// The number this build reads up to. + pub reads_up_to: u32, + /// The changes the file names that this build doesn't have. + pub unknown_changes: Vec, + /// What this build couldn't read, by name. + pub not_understood: Vec, + /// Fields known records lost in typed reading. + kept: Option, + /// Top-level sections left out whole. + sections: Vec
, + /// Elements left out of a top-level list: its key and the element. + elements: Vec<(String, Value)>, + /// The parts of a container this build has no use for: name and bytes. A blob only one of the + /// kept pieces refers to is one of them. + pub parts: Vec<(String, Arc<[u8]>)>, + /// The file's `container.json`, when this build doesn't know what it says. + pub container: Option, +} + +#[derive(Clone, Debug, PartialEq)] +struct Section { + key: String, + value: Value, + /// This build's record of the section, without the file's: while it is unchanged, the file's + /// goes back. + empty: Option, +} + +/// The fields of a record that typed reading dropped. +#[derive(Clone, Debug, PartialEq)] +enum Kept { + Field(Value), + Fields(BTreeMap), + Items(Vec<(Anchor, Kept)>), +} + +/// Which element of a list some kept fields belong to. +#[derive(Clone, Debug, PartialEq)] +enum Anchor { + Id(u64), + Name(String), + /// Its place, good only while the list keeps its length. + Index { + at: usize, + of: usize, + }, +} + +impl NewerFile { + /// A file of `app` that `newer` says a newer build saved, nothing left out of it yet. + pub fn new(app: &'static str, newer: &Newer) -> NewerFile { + NewerFile { + app, + format: newer.format, + reads_up_to: newer.reads_up_to, + unknown_changes: newer.unknown_changes.clone(), + ..NewerFile::default() + } + } + + /// What to tell the user, whatever else is said. + pub fn notice(&self) -> String { + let (app, reads, format) = (self.app, self.reads_up_to, self.format); + match self.unknown_changes.first() { + Some(change) if format <= reads => { + format!("Saved by a newer {app} (format {format} with `{change}`, which this version doesn't know). If something's missing, update.") + } + _ => format!("Saved by a newer {app} (format {format}, this version reads up to {reads}). If something's missing, update."), + } + } + + /// Every line to report: the notice, then what couldn't be read, by name. + pub fn warnings(&self) -> Vec { + let unread = self.not_understood.iter().map(|what| format!("Not read: {what}. Saving keeps it.")); + std::iter::once(self.notice()).chain(unread).collect() + } + + /// What a reply says about it. + pub fn report(&self) -> Value { + json!({ + "format": self.format, + "reads_up_to": self.reads_up_to, + "unknown_changes": self.unknown_changes, + "not_understood": self.not_understood, + "notice": self.notice(), + }) + } + + /// Whether the next save needs a container to keep what the file holds: plain JSON has no room + /// for parts beside the file's own, nor for the container's own `container.json`. + pub fn needs_container(&self) -> bool { + !self.parts.is_empty() || self.container.is_some() + } + + /// Whether anything kept back names this text (a blob's name in a container). + pub fn mentions(&self, text: &str) -> bool { + fn kept(k: &Kept, text: &str) -> bool { + match k { + Kept::Field(v) => value_mentions(v, text), + Kept::Fields(m) => m.values().any(|x| kept(x, text)), + Kept::Items(l) => l.iter().any(|(_, x)| kept(x, text)), + } + } + self.kept.as_ref().is_some_and(|k| kept(k, text)) + || self.sections.iter().any(|s| value_mentions(&s.value, text)) + || self.elements.iter().any(|(_, v)| value_mentions(v, text)) + } + + /// Say that a section the reader couldn't finish reading is left out. `empty` is what the + /// app writes for the section while it holds nothing, if anything. + pub fn keep_section(&mut self, key: &str, value: Value, empty: Option, why: &str) { + self.not_understood.push(format!("section `{key}` ({why})")); + self.sections.push(Section { key: key.into(), value, empty }); + } + + /// Say that an element of the top-level list `key` is left out, as `note` names it; it goes + /// back at the end of the list when the file is saved. + pub fn keep_element(&mut self, key: &str, element: Value, note: String) { + self.not_understood.push(note); + self.elements.push((key.into(), element)); + } + + /// Compare the file as it reads (`file`, after the same upgrade as what was read) with what was + /// read of it (`known`, its typed record written out): what `known` lacks is kept and named. + /// An object whose `type` is `placeholder_tag` (an element the app kept whole) is not + /// compared; nor are these top-level fields: those in `container_only` (what a container adds + /// beside the file's own), `format` and `format_pending`, and the sections kept whole (they go + /// back whole). + pub fn compare(&mut self, file: &Value, known: &Value, placeholder_tag: &str, container_only: &[&str]) { + let mut paths = Vec::new(); + let mut top_only: Vec<&str> = container_only.to_vec(); + top_only.extend(["format", "format_pending"]); + top_only.extend(self.sections.iter().map(|s| s.key.as_str())); + // a list typed writing leaves out holds only elements kept on their own: not kept twice + top_only.extend(self.elements.iter().map(|(key, _)| key.as_str()).filter(|key| known.get(*key).is_none())); + let kept = vanished(file, known, "", &mut paths, placeholder_tag, &top_only); + self.kept = kept; + self.not_understood.extend(paths); + for section in &mut self.sections { + section.empty = known.get(§ion.key).cloned(); + } + } + + /// Write back what was kept of the file into `value`, the file as this build writes it. + /// Returns what couldn't go back, because the file was changed around it. + pub fn restore_into(&self, value: &mut Value) -> Vec { + let mut lost = Vec::new(); + if let Some(kept) = &self.kept { + apply(kept, value, "", &mut lost); + } + if let Some(top) = value.as_object_mut() { + for section in &self.sections { + if top.get(§ion.key) == section.empty.as_ref() { + top.insert(section.key.clone(), section.value.clone()); + } else { + lost.push(format!("section `{}`, replaced by what was edited here", section.key)); + } + } + // each list's records indexed once, by the identity records are matched by + let mut indexes: HashMap<&str, Positions> = HashMap::new(); + for (key, element) in &self.elements { + let name = element.get("name"); + let anchor = anchor_of(element); + match top.entry(key.clone()).or_insert_with(|| json!([])) { + Value::Array(list) => { + let index = indexes.entry(key.as_str()).or_insert_with(|| Positions::of(list)); + if anchor.as_ref().is_some_and(|a| index.find(list.len(), a).is_some()) { + lost.push(format!( + "`{key}` element {}, replaced by what was made here", + name.and_then(Value::as_str).unwrap_or("unnamed") + )); + } else { + index.add(element, list.len()); + list.push(element.clone()); + } + } + _ => lost.push(format!("`{key}` element {}, replaced by what was made here", name.and_then(Value::as_str).unwrap_or("unnamed"))), + } + } + } + lost + } +} + +/// Whether a string anywhere in `v` is `text`; iterative, so no nesting depth overflows the stack. +fn value_mentions(v: &Value, text: &str) -> bool { + let mut todo = vec![v]; + while let Some(v) = todo.pop() { + match v { + Value::String(s) if s == text => return true, + Value::Array(a) => todo.extend(a), + Value::Object(o) => todo.extend(o.values()), + _ => {} + } + } + false +} + +/// The first line of a reader's error, without the list of every kind it knows, and without the +/// `" file: "` its reader starts it with. +pub fn short<'a>(why: &'a str, noun: &str) -> &'a str { + let why = why.split(", expected").next().unwrap_or(why); + why.strip_prefix(&format!("{noun} file: ")).unwrap_or(why) +} + +/// Leave a section out and keep it. A section the file needs is left empty. +fn move_out(top: &mut Map, key: &str, why: &str, noun: &str, file: &mut NewerFile) { + let Some(value) = top.remove(key) else { return }; + file.not_understood.push(format!("section `{key}` ({})", short(why, noun))); + file.sections.push(Section { key: key.into(), value, empty: None }); +} + +/// Every top-level section of `top` that doesn't read by itself (`piece`, the app's reading of a +/// file holding just that section) is left out and kept in `file`. `noun` is what a file of the +/// app's is called, which its reader's errors start with. +pub fn leave_out_unreadable( + top: &mut Map, + file: &mut NewerFile, + noun: &str, + piece: impl Fn(&str, &Value) -> Result<(), E>, +) { + let keys: Vec = top.keys().cloned().collect(); + for key in &keys { + let value = top.get(key).cloned().unwrap_or_default(); + if let Err(why) = piece(key, &value) { + move_out(top, key, &why.to_string(), noun, file); + } + } +} + +/// Read `top` with `attempt`, leaving out the sections it can't be read with: if it doesn't read, +/// each section but the `core` ones (which the file can't be read without) is tried left out alone, +/// and as a last resort every one of them is. What is left out is kept in `file`. The error is the +/// first attempt's, when even the core alone doesn't read. +pub fn read_around( + top: &mut Map, + file: &mut NewerFile, + core: &[&str], + noun: &str, + attempt: impl Fn(&Map) -> Result, +) -> Result { + let first = match attempt(top) { + Ok(doc) => return Ok(doc), + Err(why) => why.to_string(), + }; + // Sections that are fine by themselves but not together with the rest. + let keys: Vec = top.keys().filter(|k| !core.contains(&k.as_str())).cloned().collect(); + // Each try reads the whole file: a file of many sections goes straight to the last resort. + for key in keys.iter().filter(|_| keys.len() <= MAX_SECTION_TRIES) { + let mut without = top.clone(); + without.remove(key); + if let Ok(doc) = attempt(&without) { + move_out(top, key, &first, noun, file); + return Ok(doc); + } + } + let mut bare = top.clone(); + bare.retain(|key, _| core.contains(&key.as_str())); + match attempt(&bare) { + Ok(doc) => { + for key in &keys { + move_out(top, key, &first, noun, file); + } + Ok(doc) + } + Err(_) => Err(first), + } +} + +/// A value a writer leaves out without losing anything: not worth telling anyone about. +pub fn looks_default(v: &Value) -> bool { + match v { + Value::Null => true, + Value::Bool(b) => !b, + Value::Number(n) => n.as_f64() == Some(0.0), + Value::String(s) => s.is_empty(), + Value::Array(a) => a.is_empty(), + Value::Object(o) => o.is_empty(), + } +} + +/// What `raw` has that `known`, what was read of it, doesn't. `paths` gets those fields' names. +/// `top_only` are top-level keys left out of the comparison. +fn vanished(raw: &Value, known: &Value, path: &str, paths: &mut Vec, placeholder_tag: &str, top_only: &[&str]) -> Option { + match (raw, known) { + (Value::Object(raw), Value::Object(known)) => { + if known.get("type").and_then(Value::as_str) == Some(placeholder_tag) { + return None; + } + let mut kept = BTreeMap::new(); + for (key, value) in raw { + if path.is_empty() && top_only.contains(&key.as_str()) { + continue; + } + let inner = if path.is_empty() { key.clone() } else { format!("{path}.{key}") }; + match known.get(key) { + None => { + if !looks_default(value) { + paths.push(format!("field `{inner}`")); + } + kept.insert(key.clone(), Kept::Field(value.clone())); + } + Some(known) => { + if let Some(k) = vanished(value, known, &inner, paths, placeholder_tag, top_only) { + kept.insert(key.clone(), k); + } + } + } + } + (!kept.is_empty()).then_some(Kept::Fields(kept)) + } + (Value::Array(raw), Value::Array(known)) => { + let mut items = Vec::new(); + let positions = Positions::of(known); + for (at, element) in raw.iter().enumerate() { + // an identity typed reading kept: an id it dropped falls back to the name + let anchor = surviving_anchor(element, &positions).unwrap_or(Anchor::Index { at, of: raw.len() }); + let Some(partner) = positions.find(known.len(), &anchor).and_then(|i| known.get(i)) else { continue }; + let label = match (element.get("name").and_then(Value::as_str), &anchor) { + (Some(name), _) => format!("{path}[{name}]"), + (None, Anchor::Id(id)) => format!("{path}[{id}]"), + _ => format!("{path}[{at}]"), + }; + if let Some(k) = vanished(element, partner, &label, paths, placeholder_tag, top_only) { + items.push((anchor, k)); + } + } + (!items.is_empty()).then_some(Kept::Items(items)) + } + _ => None, + } +} + +fn anchor_of(element: &Value) -> Option { + if let Some(id) = element.get("id").and_then(Value::as_u64) { + return Some(Anchor::Id(id)); + } + element.get("name").and_then(Value::as_str).map(|name| Anchor::Name(name.to_string())) +} + +/// The identity of `element` that `known`'s records still carry: its id, or else its name. When +/// typed reading kept neither, there is none, and the record is matched by its place. +fn surviving_anchor(element: &Value, known: &Positions) -> Option { + let id = element.get("id").and_then(Value::as_u64).map(Anchor::Id); + let name = element.get("name").and_then(Value::as_str).map(|n| Anchor::Name(n.to_string())); + [id, name].into_iter().flatten().find(|anchor| known.find(usize::MAX, anchor).is_some()) +} + +/// Where each id and name first appears in a list, so matching records is not a scan per record. +struct Positions { + ids: HashMap, + names: HashMap, +} + +impl Positions { + fn of(list: &[Value]) -> Positions { + let (mut ids, mut names) = (HashMap::new(), HashMap::new()); + for (i, x) in list.iter().enumerate() { + if let Some(id) = x.get("id").and_then(Value::as_u64) { + ids.entry(id).or_insert(i); + } + if let Some(name) = x.get("name").and_then(Value::as_str) { + names.entry(name.to_string()).or_insert(i); + } + } + Positions { ids, names } + } + + /// Count `x`, now at `i`, in. + fn add(&mut self, x: &Value, i: usize) { + if let Some(id) = x.get("id").and_then(Value::as_u64) { + self.ids.entry(id).or_insert(i); + } + if let Some(name) = x.get("name").and_then(Value::as_str) { + self.names.entry(name.to_string()).or_insert(i); + } + } + + /// The position of the record `anchor` names in a list of `len` records (an index anchor + /// only while the list keeps its length). + fn find(&self, len: usize, anchor: &Anchor) -> Option { + match anchor { + Anchor::Id(id) => self.ids.get(id).copied(), + Anchor::Name(name) => self.names.get(name).copied(), + Anchor::Index { at, of } => (len == *of && at < of).then_some(*at), + } + } +} + +/// Put the kept fields back. An element that is gone was deleted here, its fields with it; one +/// whose place can't be told any more, and a field whose record is gone, go into `lost`. +fn apply(kept: &Kept, value: &mut Value, path: &str, lost: &mut Vec) { + match kept { + Kept::Field(_) => {} + Kept::Fields(fields) => { + let Some(object) = value.as_object_mut() else { + lost.push(format!("field `{path}`")); + return; + }; + for (key, sub) in fields { + let inner = if path.is_empty() { key.clone() } else { format!("{path}.{key}") }; + match (sub, object.get_mut(key)) { + (Kept::Field(v), None) => { + object.insert(key.clone(), v.clone()); + } + // the edited value stays; a retained one it replaced is reported + (Kept::Field(v), Some(existing)) => { + if existing != v { + lost.push(format!("field `{inner}`")); + } + } + (sub, Some(inner_value)) => apply(sub, inner_value, &inner, lost), + // The list it was in is empty and not written: its elements were deleted. + (Kept::Items(_), None) => {} + (_, None) => lost.push(format!("field `{inner}`")), + } + } + } + Kept::Items(items) => { + let Some(list) = value.as_array_mut() else { + lost.push(format!("field `{path}`")); + return; + }; + let positions = Positions::of(list); + let len = list.len(); + for (anchor, sub) in items { + match (positions.find(len, anchor).and_then(|i| list.get_mut(i)), anchor) { + (Some(element), _) => apply(sub, element, path, lost), + (None, Anchor::Index { at, .. }) => lost.push(format!("element {at} of `{path}`")), + (None, _) => {} + } + } + } + } +} + +#[cfg(test)] +#[path = "newer_tests.rs"] +mod tests; diff --git a/crates/format/src/newer_tests.rs b/crates/format/src/newer_tests.rs new file mode 100644 index 000000000..ae228405b --- /dev/null +++ b/crates/format/src/newer_tests.rs @@ -0,0 +1,218 @@ +use super::*; + +fn newer() -> Newer { + Newer { format: 9, reads_up_to: 8, unknown_changes: Vec::new() } +} + +/// What typed reading dropped is named and goes back on saving, where it was. +#[test] +fn dropped_fields_are_named_and_put_back() { + let file = json!({"name": "A", "format": "app/9", "items": [{"id": 1, "size": 2, "colour": "red"}], "extra": {"x": 1}}); + let known = json!({"name": "A", "items": [{"id": 1, "size": 2}]}); + let mut said = NewerFile::new("App", &newer()); + said.compare(&file, &known, "placeholder", &[]); + assert_eq!(said.not_understood, ["field `extra`", "field `items[1].colour`"]); + let mut saved = json!({"name": "A", "items": [{"id": 1, "size": 3}]}); + assert!(said.restore_into(&mut saved).is_empty()); + assert_eq!(saved, json!({"name": "A", "items": [{"id": 1, "size": 3, "colour": "red"}], "extra": {"x": 1}})); + assert_eq!(said.notice(), "Saved by a newer App (format 9, this version reads up to 8). If something's missing, update."); +} + +/// A section that can't be read is left out, kept, and goes back while it is unchanged here. +#[test] +fn unreadable_sections_are_left_out_and_kept() { + let mut top = match json!({"name": "A", "good": 1, "bad": 2}) { + Value::Object(m) => m, + _ => unreachable!(), + }; + let mut said = NewerFile::new("App", &newer()); + leave_out_unreadable(&mut top, &mut said, "design", |key: &str, _: &Value| if key == "bad" { Err("design file: no") } else { Ok(()) }); + assert!(!top.contains_key("bad")); + assert_eq!(said.not_understood, ["section `bad` (no)"]); + let read = + read_around( + &mut top, + &mut said, + &["name"], + "design", + |top: &Map| { + if top.contains_key("good") { Err("design file: clash") } else { Ok(top.len()) } + }, + ); + assert_eq!(read, Ok(1)); + assert_eq!(said.not_understood, ["section `bad` (no)", "section `good` (clash)"]); + let mut saved = json!({"name": "A"}); + said.compare(&saved.clone(), &saved.clone(), "placeholder", &[]); + assert!(said.restore_into(&mut saved).is_empty()); + assert_eq!(saved, json!({"name": "A", "good": 1, "bad": 2})); +} + +/// Notices, warnings and reports name what was left out, and kept values can be searched. +#[test] +fn warnings_report_and_mentions_cover_everything_kept() { + let mut said = NewerFile::new("App", &Newer { format: 8, reads_up_to: 8, unknown_changes: vec!["curves".into()] }); + assert_eq!(said.notice(), "Saved by a newer App (format 8 with `curves`, which this version doesn't know). If something's missing, update."); + assert!(!said.needs_container()); + said.keep_section("plugins", json!({"tool": "ruler"}), Some(json!({})), "unknown"); + said.keep_element("shapes", json!({"name": "star", "points": ["tip"]}), "shape `star`".into()); + let file = json!({"shapes": [{"name": "a", "tags": ["deep"], "more": {"x": "nested"}}], "n": 1}); + said.compare(&file, &json!({"shapes": [{"name": "a"}], "n": 1}), "placeholder", &[]); + assert_eq!(said.warnings()[0], said.notice()); + assert!(said.warnings().contains(&"Not read: section `plugins` (unknown). Saving keeps it.".to_string())); + let report = said.report(); + assert_eq!(report["format"], 8); + assert_eq!(report["unknown_changes"], json!(["curves"])); + assert_eq!(report["notice"], json!(said.notice())); + for text in ["ruler", "tip", "deep", "nested"] { + assert!(said.mentions(text), "{text}"); + } + assert!(!said.mentions("absent")); + said.parts.push(("blob".into(), Arc::from(&b"x"[..]))); + assert!(said.needs_container()); +} + +/// A kept section or element that was replaced here is reported lost, not written over. +#[test] +fn replaced_sections_and_elements_are_reported_lost() { + let mut said = NewerFile::new("App", &newer()); + said.keep_section("plugins", json!({"tool": "ruler"}), None, "unknown"); + said.keep_element("shapes", json!({"name": "star"}), "shape `star`".into()); + said.keep_element("shapes", json!({"points": 3}), "an unnamed shape".into()); + let mut saved = json!({"plugins": {"edited": true}, "shapes": [{"name": "star"}]}); + let lost = said.restore_into(&mut saved); + assert_eq!(lost, ["section `plugins`, replaced by what was edited here", "`shapes` element star, replaced by what was made here"]); + assert_eq!(saved["shapes"], json!([{"name": "star"}, {"points": 3}])); + let mut fresh = json!({}); + said.sections.clear(); + assert!(said.restore_into(&mut fresh).is_empty()); + assert_eq!(fresh["shapes"], json!([{"name": "star"}, {"points": 3}])); +} + +/// Retained container metadata alone needs a container to survive the save. +#[test] +fn container_metadata_alone_needs_a_container() { + let mut said = NewerFile::new("App", &newer()); + assert!(!said.needs_container()); + said.container = Some(json!({"layout": 2})); + assert!(said.needs_container()); +} + +/// An edit that replaces a retained field keeps the edit and reports the replaced value. +#[test] +fn a_replaced_retained_field_is_reported() { + let mut said = NewerFile::new("App", &newer()); + said.compare(&json!({"extra": 1}), &json!({}), "placeholder", &[]); + let mut edited = json!({"extra": 2}); + assert_eq!(said.restore_into(&mut edited), ["field `extra`"]); + assert_eq!(edited, json!({"extra": 2})); + let mut same = json!({"extra": 1}); + assert!(said.restore_into(&mut same).is_empty()); +} + +/// Records are matched by id and by name across long lists, and an index anchor only while the +/// list keeps its length. +#[test] +fn records_match_by_id_name_or_unchanged_index() { + let records: Vec = (0..2000).map(|i| json!({"id": i, "extra": i})).collect(); + let known: Vec = (0..2000).map(|i| json!({"id": i})).collect(); + let mut said = NewerFile::new("App", &newer()); + said.compare(&json!({"items": records}), &json!({"items": known}), "placeholder", &[]); + let mut saved = json!({"items": (0..2000).rev().map(|i| json!({"id": i})).collect::>()}); + assert!(said.restore_into(&mut saved).is_empty()); + assert_eq!(saved["items"][0], json!({"id": 1999, "extra": 1999})); + let mut by_name = NewerFile::new("App", &newer()); + by_name.compare(&json!({"l": [{"name": "a", "x": 1}, [1, 2]]}), &json!({"l": [{"name": "a"}, [1]]}), "placeholder", &[]); + let mut renamed = json!({"l": [{"name": "a"}]}); + by_name.restore_into(&mut renamed); + assert_eq!(renamed["l"][0], json!({"name": "a", "x": 1})); +} + +/// A section kept whole goes back whole, even when this build's record of it has fields. +#[test] +fn a_section_kept_whole_is_not_compared_field_by_field() { + let mut said = NewerFile::new("App", &newer()); + said.keep_section("plugins", json!({"mode": "future", "payload": 1}), None, "unreadable"); + let file = json!({"name": "A", "plugins": {"mode": "future", "payload": 1}}); + said.compare(&file, &json!({"name": "A", "plugins": {"mode": "none"}}), "placeholder", &[]); + let mut saved = json!({"name": "A", "plugins": {"mode": "none"}}); + assert!(said.restore_into(&mut saved).is_empty()); + assert_eq!(saved, file); +} + +/// A kept element goes back only while no record has its id or name. +#[test] +fn a_kept_element_keeps_record_identity() { + let mut said = NewerFile::new("App", &newer()); + said.keep_element("items", json!({"id": 7, "kind": "future"}), "element 7".into()); + said.keep_element("items", json!({"kind": "anonymous"}), "unnamed".into()); + let mut clash = json!({"items": [{"id": 7, "kind": "made here"}]}); + assert_eq!(said.restore_into(&mut clash).len(), 1); + assert_eq!(clash, json!({"items": [{"id": 7, "kind": "made here"}, {"kind": "anonymous"}]})); + let mut free = json!({"items": []}); + assert!(said.restore_into(&mut free).is_empty()); + assert_eq!(free, json!({"items": [{"id": 7, "kind": "future"}, {"kind": "anonymous"}]})); +} + +/// Container-only fields are left out at the top level only. +#[test] +fn container_only_fields_are_top_level() { + let mut said = NewerFile::new("App", &newer()); + let file = json!({"thumbnail": "t", "layer": {"thumbnail": "nested"}}); + said.compare(&file, &json!({"layer": {}}), "placeholder", &["thumbnail"]); + assert_eq!(said.not_understood, ["field `layer.thumbnail`"]); + let mut saved = json!({"layer": {}}); + assert!(said.restore_into(&mut saved).is_empty()); + assert_eq!(saved, json!({"layer": {"thumbnail": "nested"}})); +} + +/// An element kept on its own is not kept again with its list when typed writing leaves the list out. +#[test] +fn a_kept_element_is_not_restored_twice() { + let mut said = NewerFile::new("App", &newer()); + said.keep_element("items", json!({"kind": "future"}), "unnamed".into()); + said.compare(&json!({"items": [{"kind": "future"}]}), &json!({}), "placeholder", &[]); + let mut saved = json!({}); + assert!(said.restore_into(&mut saved).is_empty()); + assert_eq!(saved, json!({"items": [{"kind": "future"}]})); +} + +/// A record whose id typed reading drops is matched by its name, and its fields go back. +#[test] +fn a_dropped_id_falls_back_to_the_name() { + let mut said = NewerFile::new("App", &newer()); + said.compare(&json!({"items": [{"id": 7, "name": "a", "extra": 1}]}), &json!({"items": [{"name": "a"}]}), "placeholder", &[]); + let mut saved = json!({"items": [{"name": "a"}]}); + assert!(said.restore_into(&mut saved).is_empty()); + assert_eq!(saved, json!({"items": [{"name": "a", "id": 7, "extra": 1}]})); +} + +/// Kept elements sharing a list are checked against one index, the ones put back included. +#[test] +fn kept_elements_check_each_other() { + let mut said = NewerFile::new("App", &newer()); + said.keep_element("items", json!({"id": 1, "v": "first"}), "1".into()); + said.keep_element("items", json!({"id": 1, "v": "again"}), "1 again".into()); + let mut saved = json!({"items": []}); + assert_eq!(said.restore_into(&mut saved).len(), 1); + assert_eq!(saved, json!({"items": [{"id": 1, "v": "first"}]})); + // deep nesting is searched without recursion + let mut deep = json!("needle"); + for _ in 0..20_000 { + deep = Value::Array(vec![deep]); + } + said.keep_section("deep", deep, None, "test"); + assert!(said.mentions("needle")); + // serde_json drops deep values recursively; leak it rather than overflow the test thread + std::mem::forget(said); +} + +/// A record whose whole identity typed reading drops is matched by its place while the list keeps +/// its length, so an unchanged save keeps its fields. +#[test] +fn a_record_without_a_surviving_identity_is_matched_by_place() { + let mut said = NewerFile::new("App", &newer()); + said.compare(&json!({"items": [{"id": 7, "extra": 1}]}), &json!({"items": [{}]}), "placeholder", &[]); + let mut saved = json!({"items": [{}]}); + assert!(said.restore_into(&mut saved).is_empty()); + assert_eq!(saved, json!({"items": [{"id": 7, "extra": 1}]})); +}