mirror of
https://github.com/storytold/craft-libs.git
synced 2026-11-16 23:42:27 +00:00
Merge pull request #1627 from storytold/pr-solidcraft-format
feat: add craft-format (versioned JSON file formats)
This commit is contained in:
10 files changed
+2173
-4
No files matched your search
Generated
+14
-4
@@ -1849,6 +1849,16 @@ dependencies = [
|
||||
"craft-fonts-build",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-format"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"craft-zip",
|
||||
"proptest",
|
||||
"serde_json",
|
||||
"sha2 0.10.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "craft-gainmap"
|
||||
version = "0.1.0"
|
||||
@@ -8539,9 +8549,9 @@ checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f"
|
||||
|
||||
[[package]]
|
||||
name = "rustls-webpki"
|
||||
version = "0.103.15"
|
||||
version = "0.103.16"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2"
|
||||
checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"ring",
|
||||
@@ -11964,9 +11974,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "zip"
|
||||
version = "9.1.0"
|
||||
version = "9.1.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2886b00003a72c4ccee3f42cfc34908d33b2508a74d827dc2a4244706475c704"
|
||||
checksum = "1156b39d142a34b999c2310a50455e0cb3760bf7efad0122f796d813bee26ccb"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"flate2",
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
[package]
|
||||
name = "craft-format"
|
||||
description = "Versioned JSON file formats: the format tag, the registry of versions (hand-written steps, then one folder of changes per version, then the pending changes), the upgrade of an older file and the telling of a newer one"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
rust-version.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
serde_json = { workspace = true }
|
||||
# The container: content-addressed blobs (SHA-256) in a ZIP package.
|
||||
sha2 = { version = "0.10", default-features = false, features = ["std"] }
|
||||
craft-zip = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
proptest = { workspace = true }
|
||||
@@ -0,0 +1,237 @@
|
||||
//! For a build script: the generated lists of a family's changes. Each change is one `.rs` file in
|
||||
//! the family's folder, in `v<N>/` once a release fixed its number, else in `pending/`; the lists
|
||||
//! are generated so that a change adds a file and never edits a line another change also edits.
|
||||
//!
|
||||
//! The generated code, for the app to `include!`, is per family a `FOLDED` list (each folded
|
||||
//! format and its parts) and a `PENDING` list, both prefixed by the family's `list_prefix`, then a
|
||||
//! module per change file. It names the app's `Part` type, which the including module has in scope
|
||||
//! (`type Part = craft_format::Part<AppError>;`), and a change file's step as `step` (a
|
||||
//! `pub fn step(v: &mut Value) -> Result<(), AppError>`, when the file has one).
|
||||
|
||||
use std::fmt::Write as _;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
/// A family's change files: their folder (relative to the crate's manifest), the last format
|
||||
/// before them, and the prefix of the family's lists.
|
||||
pub struct Family {
|
||||
pub folder: &'static str,
|
||||
pub base: u32,
|
||||
pub list_prefix: &'static str,
|
||||
}
|
||||
|
||||
/// What [`lists`] made: the code to write, and the folders the build script reruns on.
|
||||
pub struct Lists {
|
||||
pub code: String,
|
||||
pub watch: Vec<PathBuf>,
|
||||
}
|
||||
|
||||
/// The `FOLDED` and `PENDING` lists of every family, and a module for each change file. Refuses
|
||||
/// versions that are not contiguous from a family's base, a change file not named with lowercase
|
||||
/// letters, digits and dashes, and one that doesn't start with a `//!` note.
|
||||
pub fn lists(manifest: &Path, families: &[Family]) -> Result<Lists, String> {
|
||||
let mut code = String::new();
|
||||
let mut mods = String::new();
|
||||
let mut watch = Vec::new();
|
||||
for family in families {
|
||||
let root = manifest.join(family.folder);
|
||||
family_lists(family, &root, &mut code, &mut mods)?;
|
||||
watch.push(root);
|
||||
}
|
||||
code.push_str(&mods);
|
||||
Ok(Lists { code, watch })
|
||||
}
|
||||
|
||||
/// A family's `FOLDED` and `PENDING` lists; its folded formats follow on from its base.
|
||||
fn family_lists(family: &Family, root: &Path, out: &mut String, mods: &mut String) -> Result<(), String> {
|
||||
let Family { folder, base, list_prefix } = family;
|
||||
let mut folded: Vec<(u32, PathBuf)> = Vec::new();
|
||||
for entry in std::fs::read_dir(root).map_err(|e| format!("{}: {e}", root.display()))? {
|
||||
// an entry that can't be read could be a version folder: fail rather than drop it
|
||||
let path = entry.map_err(|e| format!("{}: {e}", root.display()))?.path();
|
||||
let version = path.file_name().and_then(|n| n.to_str()).and_then(|n| n.strip_prefix('v')).and_then(|n| n.parse().ok());
|
||||
if let Some(version) = version
|
||||
&& path.is_dir()
|
||||
{
|
||||
folded.push((version, path));
|
||||
}
|
||||
}
|
||||
folded.sort();
|
||||
for (i, (version, _)) in folded.iter().enumerate() {
|
||||
let expected = base + 1 + i as u32;
|
||||
if *version != expected {
|
||||
return Err(format!("{folder}/: expected v{expected}/ next, found v{version}/ (versions are contiguous)"));
|
||||
}
|
||||
}
|
||||
// Module names start with the family's (`drawing_pending_…`), so two families' changes of one
|
||||
// name never clash.
|
||||
let module = list_prefix.to_ascii_lowercase();
|
||||
let _ = writeln!(
|
||||
out,
|
||||
"/// The changes of each format after the base, oldest first (`{folder}/v<N>/`).\nconst {list_prefix}FOLDED: &[(u32, &[Part])] = &["
|
||||
);
|
||||
for (version, dir) in &folded {
|
||||
let _ = writeln!(out, " ({version}, &[");
|
||||
parts(dir, &format!("{module}v{version}"), false, out, mods)?;
|
||||
out.push_str(" ]),\n");
|
||||
}
|
||||
let _ = writeln!(out, "];\n\n/// The changes no release has a number for yet (`{folder}/pending/`).\nconst {list_prefix}PENDING: &[Part] = &[");
|
||||
parts(&root.join("pending"), &format!("{module}pending"), true, out, mods)?;
|
||||
out.push_str("];\n\n");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Whether a change file declares its `step` function (`fn step(`, `pub fn step (` …) on a line of
|
||||
/// code: a mention in a comment or a note is not a declaration.
|
||||
fn declares_step(text: &str) -> bool {
|
||||
let code = without_comments(text);
|
||||
let ident = |c: char| c.is_alphanumeric() || c == '_';
|
||||
// `fn`, whitespace, `step`, then `(` (or generics), across lines
|
||||
code.match_indices("fn").any(|(at, _)| {
|
||||
let before_ok = code[..at].chars().next_back().is_none_or(|c| !ident(c));
|
||||
let rest = &code[at + 2..];
|
||||
before_ok
|
||||
&& rest.starts_with(char::is_whitespace)
|
||||
&& rest
|
||||
.trim_start()
|
||||
.strip_prefix("step")
|
||||
.is_some_and(|rest| matches!(rest.trim_start().chars().next(), Some('(' | '<')) && !rest.starts_with(ident))
|
||||
})
|
||||
}
|
||||
|
||||
/// `text` as code only: its `//` and (nested) `/* */` comments become a space, and the contents
|
||||
/// of its string literals (plain, byte and raw) are dropped, line breaks kept throughout. A comment
|
||||
/// marker inside a string is not a comment, and `fn/* note */step` still reads as `fn step`.
|
||||
fn without_comments(text: &str) -> String {
|
||||
let chars: Vec<char> = text.chars().collect();
|
||||
let at = |i: usize| chars.get(i).copied();
|
||||
let mut out = String::with_capacity(text.len());
|
||||
let (mut i, mut depth) = (0usize, 0usize);
|
||||
while let Some(c) = at(i) {
|
||||
let next = at(i + 1);
|
||||
if depth > 0 {
|
||||
match (c, next) {
|
||||
('/', Some('*')) => (depth, i) = (depth + 1, i + 2),
|
||||
('*', Some('/')) => (depth, i) = (depth - 1, i + 2),
|
||||
_ => {
|
||||
if c == '\n' {
|
||||
out.push('\n');
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
if depth == 0 {
|
||||
out.push(' ');
|
||||
}
|
||||
continue;
|
||||
}
|
||||
let ident_before = i > 0 && at(i - 1).is_some_and(|p| p.is_alphanumeric() || p == '_');
|
||||
match (c, next) {
|
||||
('/', Some('/')) => {
|
||||
while at(i).is_some_and(|c| c != '\n') {
|
||||
i += 1;
|
||||
}
|
||||
out.push(' ');
|
||||
}
|
||||
('/', Some('*')) => (depth, i) = (1, i + 2),
|
||||
('"', _) => {
|
||||
out.push('"');
|
||||
i += 1;
|
||||
while let Some(c) = at(i) {
|
||||
i += 1;
|
||||
match c {
|
||||
'\\' => i += 1,
|
||||
'"' => break,
|
||||
'\n' => out.push('\n'),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
out.push('"');
|
||||
}
|
||||
// a raw string, `r"…"` or `r#"…"#`, after a byte prefix or on its own
|
||||
('r', Some('"' | '#'))
|
||||
if !ident_before || (i > 0 && at(i - 1) == Some('b') && !(i > 1 && at(i - 2).is_some_and(|p| p.is_alphanumeric() || p == '_'))) =>
|
||||
{
|
||||
let mut j = i + 1;
|
||||
let mut hashes = 0;
|
||||
while at(j) == Some('#') {
|
||||
hashes += 1;
|
||||
j += 1;
|
||||
}
|
||||
if at(j) != Some('"') {
|
||||
out.push(c);
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
out.push('"');
|
||||
i = j + 1;
|
||||
while let Some(c) = at(i) {
|
||||
if c == '"' && (1..=hashes).all(|k| at(i + k) == Some('#')) {
|
||||
i += 1 + hashes;
|
||||
break;
|
||||
}
|
||||
if c == '\n' {
|
||||
out.push('\n');
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
out.push('"');
|
||||
}
|
||||
// a character literal (`'"'`, `'\''`), not a lifetime
|
||||
('\'', Some('\\')) => {
|
||||
i += 2;
|
||||
while at(i).is_some_and(|c| c != '\'') {
|
||||
i += 1;
|
||||
}
|
||||
i += 1;
|
||||
out.push_str("' '");
|
||||
}
|
||||
('\'', Some(_)) if at(i + 2) == Some('\'') => {
|
||||
i += 3;
|
||||
out.push_str("' '");
|
||||
}
|
||||
_ => {
|
||||
out.push(c);
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// The `Part`s of the change files in `dir`, in file-name order, and a module for each. A missing
|
||||
/// folder is an empty list only where it may be absent (`optional`: `pending/`); any other error
|
||||
/// reading it fails the build rather than leave a change out of the lists.
|
||||
fn parts(dir: &Path, prefix: &str, optional: bool, out: &mut String, mods: &mut String) -> Result<(), String> {
|
||||
let entries = match std::fs::read_dir(dir) {
|
||||
Ok(entries) => entries,
|
||||
Err(e) if optional && e.kind() == std::io::ErrorKind::NotFound => return Ok(()),
|
||||
Err(e) => return Err(format!("{}: {e}", dir.display())),
|
||||
};
|
||||
let mut files: Vec<PathBuf> = Vec::new();
|
||||
for entry in entries {
|
||||
let path = entry.map_err(|e| format!("{}: {e}", dir.display()))?.path();
|
||||
if path.extension().is_some_and(|x| x == "rs") {
|
||||
files.push(path);
|
||||
}
|
||||
}
|
||||
files.sort();
|
||||
for path in files {
|
||||
let name = path.file_stem().and_then(|s| s.to_str()).unwrap_or_default().to_string();
|
||||
if name.is_empty() || !name.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') {
|
||||
return Err(format!("{}: name a format change file with lowercase letters, digits and dashes", path.display()));
|
||||
}
|
||||
let text = std::fs::read_to_string(&path).map_err(|e| format!("{}: {e}", path.display()))?;
|
||||
if !text.trim_start().starts_with("//!") {
|
||||
return Err(format!("{}: start a format change with a `//!` note", path.display()));
|
||||
}
|
||||
let module = format!("{prefix}_{}", name.replace('-', "_"));
|
||||
let step = if declares_step(&text) { format!("Some({module}::step)") } else { "None".into() };
|
||||
let _ = writeln!(out, " Part {{ name: {name:?}, step: {step} }},");
|
||||
let _ = writeln!(mods, "#[path = {:?}]\nmod {module};", path.display().to_string());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "build_tests.rs"]
|
||||
mod tests;
|
||||
@@ -0,0 +1,115 @@
|
||||
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
|
||||
use super::*;
|
||||
|
||||
/// A fresh, empty folder for one test.
|
||||
fn scratch(name: &str) -> PathBuf {
|
||||
let dir = std::env::temp_dir().join(format!("craft-format-build-{}-{name}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
dir
|
||||
}
|
||||
|
||||
fn write(path: &Path, text: &str) {
|
||||
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
|
||||
std::fs::write(path, text).unwrap();
|
||||
}
|
||||
|
||||
/// The error of a run that must fail.
|
||||
fn refused(dir: &Path) -> String {
|
||||
match lists(dir, &[DRAWING]) {
|
||||
Ok(_) => panic!("expected an error"),
|
||||
Err(e) => e,
|
||||
}
|
||||
}
|
||||
|
||||
const DRAWING: Family = Family { folder: "drawing", base: 3, list_prefix: "DRAWING_" };
|
||||
|
||||
#[test]
|
||||
fn lists_folded_and_pending_changes_with_their_modules() {
|
||||
let dir = scratch("ok");
|
||||
write(&dir.join("drawing/v4/add-layers.rs"), "//! Layers.\npub fn step(v: &mut Value) -> Result<(), E> { Ok(()) }\n");
|
||||
write(&dir.join("drawing/v4/notes.txt"), "not a change");
|
||||
write(&dir.join("drawing/v5/rename.rs"), "\n//! A rename with no step.\n");
|
||||
write(&dir.join("drawing/pending/next-1.rs"), "//! Pending.\nfn step(v: &mut Value) {}\n");
|
||||
std::fs::create_dir_all(dir.join("drawing/v6x")).unwrap();
|
||||
write(&dir.join("drawing/v7"), "a file, not a version folder");
|
||||
let lists = lists(&dir, &[DRAWING]).unwrap();
|
||||
assert_eq!(lists.watch, vec![dir.join("drawing")]);
|
||||
let code = &lists.code;
|
||||
assert!(code.contains("const DRAWING_FOLDED: &[(u32, &[Part])] = &["), "{code}");
|
||||
assert!(code.contains(" (4, &[\n Part { name: \"add-layers\", step: Some(drawing_v4_add_layers::step) },\n ]),"), "{code}");
|
||||
assert!(code.contains("Part { name: \"rename\", step: None }"), "{code}");
|
||||
assert!(
|
||||
code.contains("const DRAWING_PENDING: &[Part] = &[\n Part { name: \"next-1\", step: Some(drawing_pending_next_1::step) },"),
|
||||
"{code}"
|
||||
);
|
||||
assert!(code.contains("mod drawing_v5_rename;"), "{code}");
|
||||
assert!(!code.contains("notes"), "{code}");
|
||||
assert!(code.find("const DRAWING_PENDING").unwrap() < code.find("#[path").unwrap(), "modules follow the lists");
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_family_has_empty_lists() {
|
||||
let dir = scratch("empty");
|
||||
std::fs::create_dir_all(dir.join("drawing")).unwrap();
|
||||
let code = lists(&dir, &[DRAWING]).unwrap().code;
|
||||
assert!(code.contains("const DRAWING_FOLDED: &[(u32, &[Part])] = &[\n];"), "{code}");
|
||||
assert!(code.contains("const DRAWING_PENDING: &[Part] = &[\n];"), "{code}");
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn refuses_gaps_bad_names_missing_notes_and_missing_folders() {
|
||||
let dir = scratch("bad");
|
||||
assert!(refused(&dir).contains("drawing"));
|
||||
write(&dir.join("drawing/v5/a.rs"), "//! a");
|
||||
assert!(refused(&dir).contains("expected v4/ next, found v5/"));
|
||||
std::fs::rename(dir.join("drawing/v5"), dir.join("drawing/v4")).unwrap();
|
||||
write(&dir.join("drawing/v4/Bad_Name.rs"), "//! b");
|
||||
assert!(refused(&dir).contains("lowercase letters"));
|
||||
std::fs::remove_file(dir.join("drawing/v4/Bad_Name.rs")).unwrap();
|
||||
write(&dir.join("drawing/pending/no-note.rs"), "pub fn step() {}");
|
||||
assert!(refused(&dir).contains("`//!` note"));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_step_is_a_declaration_not_a_mention() {
|
||||
assert!(!declares_step("//! No fn step() is needed.\n"));
|
||||
assert!(!declares_step("//! a note\n// fn step(v) was removed\nconst X: u8 = 1;\n"));
|
||||
assert!(declares_step("//! a note\npub fn step (v: &mut Value) -> Result<(), E> { Ok(()) }\n"));
|
||||
// a declaration inside a block comment, nested or not, is not one
|
||||
assert!(!declares_step("//! Documentation only.\n/*\nfn step() {}\n*/\n"));
|
||||
assert!(!declares_step("/* outer /* inner */\nfn step() {}\n*/\nconst X: u8 = 1;\n"));
|
||||
assert!(declares_step("/* a note */\nfn step(v: &mut Value) {}\n"));
|
||||
// comment markers inside strings and character literals are not comments
|
||||
assert!(declares_step("//! A change with a step.\nconst MARKER: &str = \"/*\";\npub fn step(_v: &mut Value) -> Result { Ok(()) }\n"));
|
||||
assert!(declares_step("const A: &str = r#\"/* \"quoted\" \"#;\nconst B: &[u8] = br\"/*\";\nfn step(v: &mut Value) {}\n"));
|
||||
assert!(declares_step("const Q: char = '\"';\nconst E: char = '\\'';\nfn step(v: &mut Value) { let _: &'static str = \"\"; }\n"));
|
||||
assert!(declares_step("const S: &str = \"a \\\" /* b\";\nfn step(v: &mut Value) {}\n"));
|
||||
assert!(!declares_step("const S: &str = \"\nfn step(v) \";\n"), "inside a string");
|
||||
// a comment between the words is whitespace
|
||||
assert!(declares_step("pub fn/* migration */step(v: &mut Value) {}\n"));
|
||||
// across lines
|
||||
assert!(declares_step("//! A change with a step.\npub fn step\n (_v: &mut Value) -> Result {\n Ok(())\n}\n"));
|
||||
assert!(declares_step("pub fn\nstep(v: &mut Value) {}\n"));
|
||||
assert!(!declares_step("fn stepper(v: &mut Value) {}\nfn prefn step() {}\n"), "other names");
|
||||
assert!(declares_step("const X: u8 = 1; // fn step() is below\nfn step(v: &mut Value) {}\n"));
|
||||
assert!(declares_step("//! a note\npub(crate) fn step(v: &mut Value) -> Result<(), E> { Ok(()) }\n"));
|
||||
assert!(declares_step("//! a note\n fn step(v: &mut Value) {}\n"));
|
||||
assert!(!declares_step("//! a note\nfn stepper(v: &mut Value) {}\nfn fnstep() {}\n"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_documentation_only_change_has_no_step_and_a_missing_version_folder_fails() {
|
||||
let dir = scratch("doc-only");
|
||||
write(&dir.join("drawing/v4/doc.rs"), "//! No fn step() is needed.\n");
|
||||
let code = lists(&dir, &[DRAWING]).unwrap().code;
|
||||
assert!(code.contains("Part { name: \"doc\", step: None }"), "{code}");
|
||||
// a version folder that can't be listed is an error, not an empty format
|
||||
let missing = parts(&dir.join("drawing/v9"), "x", false, &mut String::new(), &mut String::new());
|
||||
assert!(missing.is_err());
|
||||
assert!(parts(&dir.join("drawing/pending"), "x", true, &mut String::new(), &mut String::new()).is_ok());
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
@@ -0,0 +1,440 @@
|
||||
//! The file container: a ZIP package of named parts (the file's JSON, binary payloads, optional
|
||||
//! caches), never extracted to filesystem paths. Payloads are content-addressed blobs,
|
||||
//! `blobs/<sha256>.<extension>`, stored once however often they are referred to. Every size is
|
||||
//! bounded by the app's [`Limits`]: the file, each part, all parts expanded, the payloads resolved,
|
||||
//! and the number of entries; a part is read no further than its declared size (a decompression
|
||||
//! bomb stops there). What a part holds, and how a payload is encoded, is the app's.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::io::{Read, Seek, SeekFrom};
|
||||
use std::sync::Arc;
|
||||
|
||||
use serde_json::Value;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use crate::FormatError;
|
||||
|
||||
/// How hard the payloads are compressed: Fast favours interactive file operations; Compact trades
|
||||
/// CPU time for space.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub enum Compression {
|
||||
#[default]
|
||||
Fast,
|
||||
Compact,
|
||||
}
|
||||
|
||||
/// An app's budgets for its containers. The functions are read each time they are needed, so a
|
||||
/// limit the app lets the user raise takes effect at once.
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct Limits {
|
||||
/// The most entries a container holds.
|
||||
pub max_entries: usize,
|
||||
/// The most bytes all parts may expand to, and all payloads may resolve to.
|
||||
pub max_expanded: fn() -> usize,
|
||||
/// The most bytes a part of this name may expand to.
|
||||
pub part_limit: fn(&str) -> usize,
|
||||
/// The Deflate level of a compressed part of this name and size.
|
||||
pub level: fn(&str, usize) -> i64,
|
||||
/// Whether a container of this many bytes may be read, or why not.
|
||||
pub check_size: fn(u64) -> Result<(), String>,
|
||||
/// Whether the work is cancelled (a save the person stopped): checked before each part, and
|
||||
/// a long part is written a piece at a time and stops before the next piece when this says
|
||||
/// so ([`FormatError::Cancelled`]). `None`: it cannot be cancelled here.
|
||||
pub cancelled: Option<fn() -> bool>,
|
||||
}
|
||||
|
||||
/// The most bytes of a part written between two cancellation checks.
|
||||
const PIECE: usize = 1 << 20;
|
||||
|
||||
fn invalid(message: impl std::fmt::Display) -> FormatError {
|
||||
FormatError::Invalid(message.to_string())
|
||||
}
|
||||
|
||||
/// Whether the bytes start as a container does, whatever the file is called.
|
||||
pub fn is_container(bytes: &[u8]) -> bool {
|
||||
bytes.starts_with(b"PK\x03\x04")
|
||||
}
|
||||
|
||||
/// The SHA-256 of the bytes.
|
||||
pub fn sha256(bytes: &[u8]) -> [u8; 32] {
|
||||
Sha256::digest(bytes).into()
|
||||
}
|
||||
|
||||
fn hex(digest: &[u8; 32]) -> String {
|
||||
use std::fmt::Write as _;
|
||||
let mut out = String::with_capacity(64);
|
||||
for byte in digest {
|
||||
// Writing to a `String` cannot fail.
|
||||
let _ = write!(&mut out, "{byte:02x}");
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// The SHA-256 of the bytes, in lowercase hexadecimal: a blob's name, and how a cache is keyed to
|
||||
/// the file it was written for.
|
||||
pub fn digest(bytes: &[u8]) -> String {
|
||||
hex(&sha256(bytes))
|
||||
}
|
||||
|
||||
/// One part a [`Writer`] wrote, with the SHA-256 of its uncompressed content. A caller that signs
|
||||
/// a container uses these to bind the parts without reading the file back.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct PartDigest {
|
||||
pub name: String,
|
||||
pub len: u64,
|
||||
pub digest: [u8; 32],
|
||||
}
|
||||
|
||||
/// A container being written.
|
||||
pub struct Writer {
|
||||
zip: craft_zip::Writer,
|
||||
names: BTreeSet<String>,
|
||||
expanded: usize,
|
||||
resolved: usize,
|
||||
limits: Limits,
|
||||
/// Which written parts to record for a signature, and the records so far.
|
||||
tracked: Option<TrackedParts>,
|
||||
}
|
||||
|
||||
/// A tracking closure and the part digests collected so far.
|
||||
type TrackedParts = (fn(&str) -> bool, Vec<PartDigest>);
|
||||
|
||||
impl Writer {
|
||||
pub fn new(limits: Limits) -> Writer {
|
||||
Writer { zip: craft_zip::Writer::new(), names: BTreeSet::new(), expanded: 0, resolved: 0, limits, tracked: None }
|
||||
}
|
||||
|
||||
/// Record the parts written from now on whose name `include` accepts, with their SHA-256, for
|
||||
/// [`Writer::take_parts`]. Used by a caller that signs the container.
|
||||
pub fn track_parts(&mut self, include: fn(&str) -> bool) {
|
||||
self.tracked = Some((include, Vec::new()));
|
||||
}
|
||||
|
||||
/// The tracked parts written so far, oldest first, and stop tracking.
|
||||
pub fn take_parts(&mut self) -> Option<Vec<PartDigest>> {
|
||||
self.tracked.take().map(|(_, parts)| parts)
|
||||
}
|
||||
|
||||
/// The container's bytes.
|
||||
pub fn finish(self) -> Result<Vec<u8>, FormatError> {
|
||||
self.zip.finish().map_err(invalid)
|
||||
}
|
||||
|
||||
/// Add a part, compressed or stored. A name already written is written once.
|
||||
pub fn part(&mut self, name: &str, bytes: &[u8], compressed: bool) -> Result<(), FormatError> {
|
||||
self.part_with_digest(name, bytes, compressed, None)
|
||||
}
|
||||
|
||||
/// [`Writer::part`] with a digest the caller already worked out (a blob's name is its digest),
|
||||
/// so a signed save does not hash the same bytes twice.
|
||||
fn part_with_digest(&mut self, name: &str, bytes: &[u8], compressed: bool, digest: Option<[u8; 32]>) -> Result<(), FormatError> {
|
||||
if bytes.len() > (self.limits.part_limit)(name) {
|
||||
return Err(invalid("part too large"));
|
||||
}
|
||||
if self.names.contains(name) {
|
||||
return Ok(());
|
||||
}
|
||||
// Check the counters this part would leave, and commit them only once it is written: a
|
||||
// refused part is not remembered as written.
|
||||
let expanded = self.expanded.checked_add(bytes.len()).ok_or_else(|| invalid("expanded size overflow"))?;
|
||||
if expanded > (self.limits.max_expanded)() || self.names.len().saturating_add(1) > self.limits.max_entries {
|
||||
return Err(invalid("package too large"));
|
||||
}
|
||||
let cancelled = self.limits.cancelled;
|
||||
if cancelled.is_some_and(|c| c()) {
|
||||
return Err(FormatError::Cancelled);
|
||||
}
|
||||
// the configured level, clamped to the Deflate levels craft-zip writes (1 to 9)
|
||||
let compression = if compressed {
|
||||
craft_zip::Compression::Level((self.limits.level)(name, bytes.len()).clamp(1, 9) as u8)
|
||||
} else {
|
||||
craft_zip::Compression::Stored
|
||||
};
|
||||
match cancelled {
|
||||
Some(cancelled) if bytes.len() > PIECE => {
|
||||
let mut stopped = false;
|
||||
let written = self.zip.add_with(name, compression, |out| {
|
||||
for piece in bytes.chunks(PIECE) {
|
||||
if cancelled() {
|
||||
stopped = true;
|
||||
return Err(std::io::Error::other("cancelled"));
|
||||
}
|
||||
out.write_all(piece)?;
|
||||
}
|
||||
Ok(())
|
||||
});
|
||||
if stopped {
|
||||
return Err(FormatError::Cancelled);
|
||||
}
|
||||
written.map_err(invalid)?;
|
||||
}
|
||||
_ => self.zip.add(name, bytes, compression).map_err(invalid)?,
|
||||
}
|
||||
self.names.insert(name.to_owned());
|
||||
self.expanded = expanded;
|
||||
if let Some((include, parts)) = &mut self.tracked
|
||||
&& include(name)
|
||||
{
|
||||
let digest = digest.unwrap_or_else(|| sha256(bytes));
|
||||
parts.push(PartDigest { name: name.to_owned(), len: bytes.len() as u64, digest });
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Would `entries` more optional parts of `extra` bytes, stored uncompressed, still keep the
|
||||
/// file within `budget` and its expanded size within the reader's budget? The parts already
|
||||
/// written count at their expanded size (an upper bound on what they take in the file).
|
||||
pub fn room(&self, entries: usize, extra: usize, budget: usize) -> bool {
|
||||
self.remaining(entries, budget).is_some_and(|left| extra <= left)
|
||||
}
|
||||
|
||||
/// The bytes `entries` more optional parts may hold together ([`Writer::room`]).
|
||||
pub fn space(&self, entries: usize, budget: usize) -> usize {
|
||||
self.remaining(entries, budget).unwrap_or(0)
|
||||
}
|
||||
|
||||
/// The bytes left for `entries` more parts, or `None` when the entries don't fit or what is
|
||||
/// written already is past `budget` or the reader's budget.
|
||||
fn remaining(&self, entries: usize, budget: usize) -> Option<usize> {
|
||||
let count = self.names.len().checked_add(entries).filter(|&n| n <= self.limits.max_entries)?;
|
||||
let overhead = count.saturating_mul(256).saturating_add(65_536);
|
||||
let in_file = budget.checked_sub(overhead.saturating_add(self.expanded))?;
|
||||
let expanded = (self.limits.max_expanded)().checked_sub(self.expanded)?;
|
||||
Some(in_file.min(expanded))
|
||||
}
|
||||
|
||||
/// Add a payload as a blob, `blobs/<sha256>.<extension>`, and say its name, for the JSON that
|
||||
/// refers to it.
|
||||
pub fn blob(&mut self, bytes: &[u8], extension: &str, compressed: bool) -> Result<Value, FormatError> {
|
||||
let resolved = self
|
||||
.resolved
|
||||
.checked_add(bytes.len())
|
||||
.filter(|&n| n <= (self.limits.max_expanded)())
|
||||
.ok_or_else(|| invalid("resolved payloads too large"))?;
|
||||
let digest = sha256(bytes);
|
||||
let name = format!("blobs/{}.{extension}", hex(&digest));
|
||||
self.part_with_digest(&name, bytes, compressed, Some(digest))?;
|
||||
self.resolved = resolved;
|
||||
Ok(Value::String(name))
|
||||
}
|
||||
}
|
||||
|
||||
/// A container being read. Opening reads all bytes into memory and checks the directory
|
||||
/// (the entry count, unique and plain names, every part's
|
||||
/// declared size), not the parts.
|
||||
pub struct Reader {
|
||||
archive: craft_zip::Archive<'static>,
|
||||
blobs: BTreeMap<String, Vec<u8>>,
|
||||
/// The bytes of payloads resolved so far, repeats counted: bounded by the expanded budget.
|
||||
pub hydrated: usize,
|
||||
document: Option<Arc<[u8]>>,
|
||||
limits: Limits,
|
||||
/// Every byte part reads have expanded to, whatever the budget was at the time.
|
||||
read_total: u64,
|
||||
}
|
||||
|
||||
impl Reader {
|
||||
pub fn new(mut source: impl Read + Seek, limits: Limits) -> Result<Self, FormatError> {
|
||||
let length = source.seek(SeekFrom::End(0)).map_err(invalid)?;
|
||||
(limits.check_size)(length).map_err(invalid)?;
|
||||
|
||||
// Read all bytes into memory for the archive, no more than the length just approved: a file
|
||||
// that grew since is refused rather than read past its budget.
|
||||
source.seek(SeekFrom::Start(0)).map_err(invalid)?;
|
||||
let capacity = usize::try_from(length).map_err(|_| invalid("file too large"))?;
|
||||
let mut bytes = Vec::new();
|
||||
bytes.try_reserve_exact(capacity).map_err(|_| invalid("file too large"))?;
|
||||
source.take(length.saturating_add(1)).read_to_end(&mut bytes).map_err(invalid)?;
|
||||
if bytes.len() as u64 != length {
|
||||
return Err(invalid("the file changed while it was read"));
|
||||
}
|
||||
|
||||
let mut archive = craft_zip::Archive::open_owned(
|
||||
bytes,
|
||||
craft_zip::Limits { max_entries: limits.max_entries as u64, max_total_bytes: (limits.max_expanded)() as u64 },
|
||||
)
|
||||
.map_err(invalid)?;
|
||||
// the directory lists distinct names, so a repeated name is caught here
|
||||
if archive.has_duplicate_names() {
|
||||
return Err(invalid("duplicate or invalid archive entry"));
|
||||
}
|
||||
|
||||
// Validate entry names match format rules: no empty, no backslashes,
|
||||
// no path components that are "." or "..", and no duplicates; and every part's declared
|
||||
// size within its configured limit.
|
||||
let mut name_set = BTreeSet::new();
|
||||
for entry_idx in 0..archive.len() {
|
||||
// an entry whose name can't be read would be left out of the package unnoticed
|
||||
let Ok(Some(name)) = archive.entry_name(entry_idx) else {
|
||||
return Err(invalid("duplicate or invalid archive entry"));
|
||||
};
|
||||
{
|
||||
if name.is_empty()
|
||||
|| name.contains('\\')
|
||||
|| name.split('/').any(|c| c.is_empty() || c == "." || c == "..")
|
||||
|| !name_set.insert(name.clone())
|
||||
{
|
||||
return Err(invalid("duplicate or invalid archive entry"));
|
||||
}
|
||||
if archive.declared_size(&name).is_some_and(|size| size > (limits.part_limit)(&name) as u64) {
|
||||
return Err(invalid("part too large"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(Self { archive, blobs: BTreeMap::new(), hydrated: 0, document: None, limits, read_total: 0 })
|
||||
}
|
||||
|
||||
/// The file's own part, `document.json`, read once and kept: provisional caches and the
|
||||
/// authoritative reading are bound to the same bytes, even if another process overwrites the
|
||||
/// open file in place. Blobs remain hash-checked.
|
||||
pub fn document(&mut self, limit: usize) -> Result<Arc<[u8]>, FormatError> {
|
||||
if self.document.is_none() {
|
||||
let bytes = self.limited_part("document.json", limit)?;
|
||||
self.document = Some(bytes.into());
|
||||
}
|
||||
// a cached document still keeps the current part limit
|
||||
let limit = limit.min((self.limits.part_limit)("document.json"));
|
||||
self.document.as_ref().filter(|d| d.len() <= limit).cloned().ok_or_else(|| invalid("metadata budget"))
|
||||
}
|
||||
|
||||
/// Whether the container has a part of this name.
|
||||
pub fn has_part(&self, name: &str) -> bool {
|
||||
self.archive.contains(name)
|
||||
}
|
||||
|
||||
/// The names of every part, in the container's order.
|
||||
pub fn part_names(&self) -> Vec<String> {
|
||||
self.archive.names().map(|n| n.into_owned()).collect()
|
||||
}
|
||||
|
||||
/// The part of this name, at most `limit` bytes and never more than its configured limit.
|
||||
pub fn limited_part(&mut self, name: &str, limit: usize) -> Result<Vec<u8>, FormatError> {
|
||||
let limit = limit.min((self.limits.part_limit)(name));
|
||||
let expected = self.archive.declared_size(name).ok_or_else(|| invalid("part not found"))?;
|
||||
if expected > limit as u64 {
|
||||
return Err(invalid("part too large"));
|
||||
}
|
||||
// the expanded budget is read each time: a raised (or lowered) one applies at once, the
|
||||
// bytes already read still counting against it
|
||||
let budget = self.archive.budget_mut();
|
||||
*budget = craft_zip::Budget::new(budget.what(), (self.limits.max_expanded)() as u64);
|
||||
if budget.charge(self.read_total).is_err() {
|
||||
return Err(invalid("package too large"));
|
||||
}
|
||||
// Read within budget. craft-zip's read() handles decompression bombs via the limit.
|
||||
let bytes = self.archive.read(name, limit as u64).map_err(invalid)?.ok_or_else(|| invalid("part not found"))?;
|
||||
if bytes.len() as u64 != expected {
|
||||
return Err(invalid("part length mismatch"));
|
||||
}
|
||||
self.read_total = self.read_total.saturating_add(expected);
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
/// The blob `value`'s `field` names, of this extension, within its part limit.
|
||||
pub fn blob(&mut self, value: &Value, field: &str, extension: &str) -> Result<&[u8], FormatError> {
|
||||
let limit = (self.limits.part_limit)(&format!("blob.{extension}"));
|
||||
self.limited_blob(value, field, extension, limit)
|
||||
}
|
||||
|
||||
/// [`Reader::blob`] within `limit` bytes. Its hash is checked when it is first read, and every
|
||||
/// reference counts against the resolved budget.
|
||||
pub fn limited_blob(&mut self, value: &Value, field: &str, extension: &str, limit: usize) -> Result<&[u8], FormatError> {
|
||||
let name = value.get(field).and_then(Value::as_str).ok_or_else(|| invalid(format!("missing `{field}` reference")))?;
|
||||
// a cached blob still keeps the part's current limit
|
||||
let limit = limit.min((self.limits.part_limit)(name));
|
||||
let hash = name
|
||||
.strip_prefix("blobs/")
|
||||
.and_then(|n| n.strip_suffix(&format!(".{extension}")))
|
||||
.filter(|h| h.len() == 64 && h.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)))
|
||||
.ok_or_else(|| invalid("invalid blob reference"))?;
|
||||
if !self.blobs.contains_key(name) {
|
||||
let bytes = self.limited_part(name, limit)?;
|
||||
if digest(&bytes) != hash {
|
||||
return Err(invalid(format!("blob hash mismatch: {name}")));
|
||||
}
|
||||
self.blobs.insert(name.to_owned(), bytes);
|
||||
}
|
||||
let bytes = self.blobs.get(name).ok_or_else(|| invalid("missing blob"))?;
|
||||
if bytes.len() > limit {
|
||||
return Err(invalid("blob exceeds its resource budget"));
|
||||
}
|
||||
// Count repeated references too: a tiny archive must not expand into unbounded owned
|
||||
// strings or arrays when the same blob is referenced repeatedly.
|
||||
self.hydrated = self
|
||||
.hydrated
|
||||
.checked_add(bytes.len())
|
||||
.filter(|&n| n <= (self.limits.max_expanded)())
|
||||
.ok_or_else(|| invalid("resolved payloads too large"))?;
|
||||
Ok(bytes)
|
||||
}
|
||||
}
|
||||
|
||||
/// A small PNG picture of the file, kept in the container as a disposable cache keyed to the
|
||||
/// exact file it shows: `cache/thumbnail.json` (its key) and `cache/thumbnail.png`. Reading it
|
||||
/// never resolves a payload; a picture that isn't one, or is stale, is a miss.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct Thumbnail {
|
||||
/// The key's format tag.
|
||||
pub format: &'static str,
|
||||
/// The build that wrote it: only the same build reads it back.
|
||||
pub app: &'static str,
|
||||
/// Its width and height in pixels.
|
||||
pub size: [usize; 2],
|
||||
/// The most bytes it may have.
|
||||
pub max_bytes: usize,
|
||||
}
|
||||
|
||||
impl Thumbnail {
|
||||
/// Whether the bytes start as a PNG of this size within the byte budget (not a decode: a
|
||||
/// consumer decodes with its own dimension and allocation limits).
|
||||
pub fn valid(&self, bytes: &[u8]) -> bool {
|
||||
bytes.len() <= self.max_bytes
|
||||
&& bytes.get(..8) == Some(b"\x89PNG\r\n\x1a\n")
|
||||
&& bytes.get(8..16) == Some(b"\0\0\0\rIHDR")
|
||||
&& bytes.get(16..20) == Some((self.size[0] as u32).to_be_bytes().as_slice())
|
||||
&& bytes.get(20..24) == Some((self.size[1] as u32).to_be_bytes().as_slice())
|
||||
}
|
||||
|
||||
/// Add the picture, keyed to `document` (the file's own part), when it is one and there is
|
||||
/// room for it within `budget`; otherwise leave it out, never failing the save.
|
||||
pub fn write(&self, writer: &mut Writer, png: &[u8], document: &[u8], budget: usize) -> Result<(), FormatError> {
|
||||
if !self.valid(png) {
|
||||
return Ok(());
|
||||
}
|
||||
let key =
|
||||
serde_json::to_vec(&serde_json::json!({"format": self.format, "app": self.app, "document": digest(document), "payload": digest(png)}))
|
||||
.map_err(invalid)?;
|
||||
let fits = |name: &str, len: usize| len <= (writer.limits.part_limit)(name);
|
||||
if !fits("cache/thumbnail.json", key.len())
|
||||
|| !fits("cache/thumbnail.png", png.len())
|
||||
|| !writer.room(2, key.len().saturating_add(png.len()), budget)
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
writer.part("cache/thumbnail.json", &key, false)?;
|
||||
writer.part("cache/thumbnail.png", png, false)
|
||||
}
|
||||
|
||||
/// The picture, when this build wrote it for exactly the file's own part (`document.json`,
|
||||
/// read within `document_limit`).
|
||||
pub fn read(&self, reader: &mut Reader, document_limit: usize) -> Result<Vec<u8>, FormatError> {
|
||||
let key: Value = serde_json::from_slice(&reader.limited_part("cache/thumbnail.json", 4096)?).map_err(invalid)?;
|
||||
let document = reader.document(document_limit)?;
|
||||
if key.get("format").and_then(Value::as_str) != Some(self.format)
|
||||
|| key.get("app").and_then(Value::as_str) != Some(self.app)
|
||||
|| key.get("document").and_then(Value::as_str) != Some(digest(&document).as_str())
|
||||
{
|
||||
return Err(invalid("thumbnail key"));
|
||||
}
|
||||
let png = reader.limited_part("cache/thumbnail.png", self.max_bytes)?;
|
||||
if !self.valid(&png) || key.get("payload").and_then(Value::as_str) != Some(digest(&png).as_str()) {
|
||||
return Err(invalid("thumbnail payload"));
|
||||
}
|
||||
Ok(png)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "container_tests.rs"]
|
||||
mod tests;
|
||||
@@ -0,0 +1,260 @@
|
||||
use super::*;
|
||||
use std::io::Cursor;
|
||||
|
||||
const LIMITS: Limits = Limits {
|
||||
max_entries: 8,
|
||||
max_expanded: || 1 << 20,
|
||||
part_limit: |name| if name.ends_with(".big") { 1 << 20 } else { 1024 },
|
||||
level: |_, _| 1,
|
||||
check_size: |length| if length <= 1 << 20 { Ok(()) } else { Err("too large".into()) },
|
||||
cancelled: None,
|
||||
};
|
||||
|
||||
/// Parts and blobs written are read back as they were; a blob is stored once, named by its hash,
|
||||
/// and checked against it.
|
||||
#[test]
|
||||
fn parts_and_blobs_read_back() {
|
||||
let mut writer = Writer::new(LIMITS);
|
||||
writer.part("document.json", br#"{"a":1}"#, true).unwrap();
|
||||
let first = writer.blob(b"payload", "bin", false).unwrap();
|
||||
let again = writer.blob(b"payload", "bin", true).unwrap();
|
||||
assert_eq!(first, again, "the same payload is one blob");
|
||||
assert_eq!(first, Value::String(format!("blobs/{}.bin", digest(b"payload"))));
|
||||
let bytes = writer.finish().unwrap();
|
||||
assert!(is_container(&bytes));
|
||||
let mut reader = Reader::new(Cursor::new(bytes), LIMITS).unwrap();
|
||||
assert_eq!(reader.part_names().len(), 2);
|
||||
assert!(reader.has_part("document.json") && !reader.has_part("other.json"));
|
||||
assert_eq!(&*reader.document(64).unwrap(), br#"{"a":1}"#);
|
||||
let reference = serde_json::json!({"blob": first});
|
||||
assert_eq!(reader.blob(&reference, "blob", "bin").unwrap(), b"payload");
|
||||
assert!(reader.blob(&reference, "blob", "png").is_err(), "another extension");
|
||||
assert_eq!(reader.hydrated, 7);
|
||||
reader.hydrated = (LIMITS.max_expanded)();
|
||||
assert!(reader.blob(&reference, "blob", "bin").unwrap_err().to_string().contains("resolved payloads too large"));
|
||||
}
|
||||
|
||||
/// A part past its limit, or more entries than the limit, are refused as they are written.
|
||||
#[test]
|
||||
fn budgets_are_kept_when_writing() {
|
||||
let mut writer = Writer::new(LIMITS);
|
||||
assert_eq!(writer.part("small.json", &[0; 1025], false), Err(FormatError::Invalid("part too large".into())));
|
||||
assert!(writer.part("large.big", &[0; 1025], true).is_ok());
|
||||
for i in 0..7 {
|
||||
writer.part(&format!("{i}.json"), b"x", false).unwrap();
|
||||
}
|
||||
assert_eq!(writer.part("one-more.json", b"x", false), Err(FormatError::Invalid("package too large".into())));
|
||||
assert!(!writer.room(1, 0, 1 << 20), "no room for an entry past the limit");
|
||||
}
|
||||
|
||||
/// A thumbnail is kept only for the file it shows.
|
||||
#[test]
|
||||
fn a_thumbnail_is_keyed_to_its_file() {
|
||||
let picture = Thumbnail { format: "app-thumbnail/1", app: "1.0", size: [2, 1], max_bytes: 1024 };
|
||||
let mut png = b"\x89PNG\r\n\x1a\n\0\0\0\rIHDR".to_vec();
|
||||
png.extend_from_slice(&2u32.to_be_bytes());
|
||||
png.extend_from_slice(&1u32.to_be_bytes());
|
||||
assert!(picture.valid(&png));
|
||||
let mut writer = Writer::new(LIMITS);
|
||||
writer.part("document.json", b"{}", false).unwrap();
|
||||
picture.write(&mut writer, &png, b"{}", 8 << 20).unwrap();
|
||||
let mut reader = Reader::new(Cursor::new(writer.finish().unwrap()), LIMITS).unwrap();
|
||||
assert_eq!(picture.read(&mut reader, 64).unwrap(), png);
|
||||
let mut writer = Writer::new(LIMITS);
|
||||
writer.part("document.json", b"{\"changed\":1}", false).unwrap();
|
||||
picture.write(&mut writer, &png, b"{}", 8 << 20).unwrap();
|
||||
let mut reader = Reader::new(Cursor::new(writer.finish().unwrap()), LIMITS).unwrap();
|
||||
assert!(picture.read(&mut reader, 64).is_err(), "stale");
|
||||
}
|
||||
|
||||
/// A refused part is not remembered as written: once the budget is raised it is written.
|
||||
#[test]
|
||||
fn a_refused_part_can_be_written_later() {
|
||||
static BUDGET: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(4);
|
||||
let limits = Limits { max_expanded: || BUDGET.load(std::sync::atomic::Ordering::SeqCst), ..LIMITS };
|
||||
let mut writer = Writer::new(limits);
|
||||
assert_eq!(writer.part("a.json", b"12345678", false), Err(FormatError::Invalid("package too large".into())));
|
||||
assert!(writer.blob(b"12345678", "bin", false).is_err());
|
||||
BUDGET.store(1 << 20, std::sync::atomic::Ordering::SeqCst);
|
||||
writer.part("a.json", b"12345678", false).unwrap();
|
||||
writer.blob(b"12345678", "bin", false).unwrap();
|
||||
let reader = Reader::new(Cursor::new(writer.finish().unwrap()), limits).unwrap();
|
||||
assert_eq!(reader.part_names().len(), 2, "both parts were written on the retry");
|
||||
}
|
||||
|
||||
/// A cancelled save writes no more parts, whether the part is small or written in pieces.
|
||||
#[test]
|
||||
fn cancellation_stops_the_save() {
|
||||
let cancelled = Limits { cancelled: Some(|| true), ..LIMITS };
|
||||
let mut writer = Writer::new(cancelled);
|
||||
assert_eq!(writer.part("a.json", b"a", false), Err(FormatError::Cancelled));
|
||||
// cancelled after the first piece of a long part: checks before the part, then per piece
|
||||
static CHECKS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
|
||||
let midway = Limits {
|
||||
cancelled: Some(|| CHECKS.fetch_add(1, std::sync::atomic::Ordering::SeqCst) >= 2),
|
||||
max_expanded: || 1 << 22,
|
||||
part_limit: |_| 1 << 22,
|
||||
..LIMITS
|
||||
};
|
||||
let mut writer = Writer::new(midway);
|
||||
assert_eq!(writer.part("big.big", &vec![0; PIECE + 1], true), Err(FormatError::Cancelled));
|
||||
assert_eq!(CHECKS.load(std::sync::atomic::Ordering::SeqCst), 3);
|
||||
let going = Limits { cancelled: Some(|| false), max_expanded: || 1 << 22, part_limit: |_| 1 << 22, ..LIMITS };
|
||||
let mut writer = Writer::new(going);
|
||||
writer.part("big.big", &vec![7; PIECE + 1], true).unwrap();
|
||||
let mut reader = Reader::new(Cursor::new(writer.finish().unwrap()), going).unwrap();
|
||||
assert_eq!(reader.limited_part("big.big", 1 << 22).unwrap().len(), PIECE + 1);
|
||||
}
|
||||
|
||||
/// Room counts what is already written, and the reader keeps every part's configured limit.
|
||||
#[test]
|
||||
fn room_counts_written_parts_and_reads_keep_part_limits() {
|
||||
let mut writer = Writer::new(LIMITS);
|
||||
writer.part("large.big", &[0; 500 * 1024], false).unwrap();
|
||||
assert!(!writer.room(1, 200 * 1024, 600 * 1024));
|
||||
assert!(writer.room(1, 1024, 1 << 20));
|
||||
// what is written already is past the budget: not even an empty part fits
|
||||
assert!(!writer.room(1, 0, 400 * 1024));
|
||||
assert_eq!(writer.space(1, 400 * 1024), 0);
|
||||
let wide = Limits { part_limit: |_| 1 << 20, ..LIMITS };
|
||||
let mut writer = Writer::new(wide);
|
||||
writer.part("document.json", &[b' '; 2048], false).unwrap();
|
||||
let bytes = writer.finish().unwrap();
|
||||
assert!(Reader::new(Cursor::new(bytes.clone()), LIMITS).is_err(), "declared past its configured limit");
|
||||
let mut reader = Reader::new(Cursor::new(bytes), wide).unwrap();
|
||||
assert!(reader.limited_part("document.json", 4096).is_ok());
|
||||
}
|
||||
|
||||
/// A thumbnail past a cache part's own limit is left out, never failing the save.
|
||||
#[test]
|
||||
fn a_thumbnail_past_its_part_limit_is_left_out() {
|
||||
let picture = Thumbnail { format: "app-thumbnail/1", app: "1.0", size: [2, 1], max_bytes: 2048 };
|
||||
let mut png = b"\x89PNG\r\n\x1a\n\0\0\0\rIHDR".to_vec();
|
||||
png.extend_from_slice(&2u32.to_be_bytes());
|
||||
png.extend_from_slice(&1u32.to_be_bytes());
|
||||
png.resize(1500, 0);
|
||||
let mut writer = Writer::new(LIMITS);
|
||||
picture.write(&mut writer, &png, b"{}", 8 << 20).unwrap();
|
||||
let reader = Reader::new(Cursor::new(writer.finish().unwrap()), LIMITS).unwrap();
|
||||
assert!(!reader.has_part("cache/thumbnail.json") && !reader.has_part("cache/thumbnail.png"));
|
||||
}
|
||||
|
||||
/// A source that grows after its length was approved is refused.
|
||||
#[test]
|
||||
fn a_source_that_grows_while_read_is_refused() {
|
||||
struct Growing(Cursor<Vec<u8>>, bool);
|
||||
impl Read for Growing {
|
||||
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
|
||||
if !self.1 {
|
||||
self.1 = true;
|
||||
let end = self.0.get_ref().len();
|
||||
self.0.get_mut().extend_from_slice(&[0; 16]);
|
||||
self.0.set_position(self.0.position().min(end as u64));
|
||||
}
|
||||
self.0.read(buf)
|
||||
}
|
||||
}
|
||||
impl Seek for Growing {
|
||||
fn seek(&mut self, to: SeekFrom) -> std::io::Result<u64> {
|
||||
self.0.seek(to)
|
||||
}
|
||||
}
|
||||
let mut writer = Writer::new(LIMITS);
|
||||
writer.part("document.json", b"{}", false).unwrap();
|
||||
let bytes = writer.finish().unwrap();
|
||||
assert!(Reader::new(Growing(Cursor::new(bytes), false), LIMITS).err().is_some_and(|e| e.to_string().contains("changed")));
|
||||
}
|
||||
|
||||
/// Compressed parts use the configured Deflate level.
|
||||
#[test]
|
||||
fn compressed_parts_use_the_configured_level() {
|
||||
// words in a varied order, so the levels compress them differently
|
||||
let words = ["alpha ", "beta ", "gamma ", "delta ", "epsilon ", "zeta ", "eta ", "theta "];
|
||||
let mut seed = 7u32;
|
||||
let text: Vec<u8> = (0..150)
|
||||
.flat_map(|_| {
|
||||
seed = seed.wrapping_mul(1_103_515_245).wrapping_add(12_345);
|
||||
words[(seed >> 16) as usize % words.len()].bytes()
|
||||
})
|
||||
.take(1000)
|
||||
.collect();
|
||||
let size = |level: fn(&str, usize) -> i64| {
|
||||
let mut writer = Writer::new(Limits { level, ..LIMITS });
|
||||
writer.part("document.json", &text, true).unwrap();
|
||||
writer.finish().unwrap().len()
|
||||
};
|
||||
assert_ne!(size(|_, _| 1), size(|_, _| 9), "the level is used");
|
||||
assert_eq!(size(|_, _| 42), size(|_, _| 9), "past 9 is clamped");
|
||||
assert_eq!(size(|_, _| -3), size(|_, _| 1), "below 1 is clamped");
|
||||
}
|
||||
|
||||
/// An archive naming one part twice is refused.
|
||||
#[test]
|
||||
fn a_repeated_part_name_is_refused() {
|
||||
let mut writer = Writer::new(LIMITS);
|
||||
writer.part("document.json", b"{}", false).unwrap();
|
||||
writer.part("documenX.json", b"[]", false).unwrap();
|
||||
let mut bytes = writer.finish().unwrap();
|
||||
let (from, to) = (b"documenX.json", b"document.json");
|
||||
let mut at = 0;
|
||||
while let Some(i) = bytes[at..].windows(from.len()).position(|w| w == from) {
|
||||
bytes[at + i..at + i + to.len()].copy_from_slice(to);
|
||||
at += i + to.len();
|
||||
}
|
||||
assert!(Reader::new(Cursor::new(bytes), LIMITS).is_err());
|
||||
}
|
||||
|
||||
/// Cached documents and blobs keep the current part limits.
|
||||
#[test]
|
||||
fn cached_parts_keep_the_current_part_limits() {
|
||||
let wide = Limits { part_limit: |_| 1 << 20, ..LIMITS };
|
||||
let mut writer = Writer::new(wide);
|
||||
writer.part("document.json", &[b' '; 512], false).unwrap();
|
||||
let blob = writer.blob(&[7; 512], "bin", false).unwrap();
|
||||
let bytes = writer.finish().unwrap();
|
||||
let mut reader = Reader::new(Cursor::new(bytes), wide).unwrap();
|
||||
let reference = serde_json::json!({"blob": blob});
|
||||
assert!(reader.document(4096).is_ok());
|
||||
assert!(reader.blob(&reference, "blob", "bin").is_ok());
|
||||
reader.limits = Limits { part_limit: |_| 128, ..wide };
|
||||
assert!(reader.document(4096).is_err());
|
||||
assert!(reader.limited_blob(&reference, "blob", "bin", 4096).is_err());
|
||||
}
|
||||
|
||||
/// A raised expanded budget applies to the open reader, and what was read still counts.
|
||||
#[test]
|
||||
fn a_raised_expanded_budget_applies_at_once() {
|
||||
static BUDGET: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(600);
|
||||
let limits = Limits { max_expanded: || BUDGET.load(std::sync::atomic::Ordering::Relaxed), part_limit: |_| 1 << 20, ..LIMITS };
|
||||
let wide = Limits { max_expanded: || 1 << 20, ..limits };
|
||||
let mut writer = Writer::new(wide);
|
||||
writer.part("document.json", &[b' '; 256], false).unwrap();
|
||||
writer.part("other.big", &[1; 512], false).unwrap();
|
||||
let bytes = writer.finish().unwrap();
|
||||
BUDGET.store(1 << 20, std::sync::atomic::Ordering::Relaxed);
|
||||
let mut reader = Reader::new(Cursor::new(bytes), limits).unwrap();
|
||||
BUDGET.store(600, std::sync::atomic::Ordering::Relaxed);
|
||||
assert!(reader.limited_part("document.json", 4096).is_ok());
|
||||
assert!(reader.limited_part("other.big", 4096).is_err(), "256 read + 512 is past 600");
|
||||
BUDGET.store(1024, std::sync::atomic::Ordering::Relaxed);
|
||||
assert!(reader.limited_part("other.big", 4096).is_ok(), "768 fits the raised 1024");
|
||||
// a lowered budget does not forget what was read: 768 + 256 is past 1000 after a dip to 100
|
||||
BUDGET.store(100, std::sync::atomic::Ordering::Relaxed);
|
||||
assert!(reader.limited_part("document.json", 4096).is_err());
|
||||
BUDGET.store(1000, std::sync::atomic::Ordering::Relaxed);
|
||||
assert!(reader.limited_part("document.json", 4096).is_err(), "1024 is past 1000");
|
||||
BUDGET.store(1024, std::sync::atomic::Ordering::Relaxed);
|
||||
assert!(reader.limited_part("document.json", 4096).is_ok());
|
||||
}
|
||||
|
||||
/// An entry whose local header can't be read is refused at open, not skipped.
|
||||
#[test]
|
||||
fn an_unreadable_entry_is_refused() {
|
||||
let mut writer = Writer::new(LIMITS);
|
||||
writer.part("document.json", b"{}", false).unwrap();
|
||||
writer.part("other.json", b"[]", false).unwrap();
|
||||
let mut bytes = writer.finish().unwrap();
|
||||
let second = bytes.windows(4).enumerate().filter(|(_, w)| *w == b"PK\x03\x04").map(|(i, _)| i).nth(1).unwrap();
|
||||
bytes[second + 2] = 0;
|
||||
assert!(Reader::new(Cursor::new(bytes), LIMITS).is_err());
|
||||
}
|
||||
@@ -0,0 +1,257 @@
|
||||
//! Versioned file formats: what an app needs to read every file it ever wrote, and to open a file
|
||||
//! a newer build of it wrote.
|
||||
//!
|
||||
//! A file starts with `"format": "<prefix>N"`. A [`Family`] is one kind of file (an app's designs,
|
||||
//! its drawings): its tag, its own numbers, and how a file of an older number is brought up to
|
||||
//! this build's, step by step (N → N+1 → … → current). The app's code only knows the current
|
||||
//! format; all translation lives in the steps:
|
||||
//! - the steps written out by hand for the first formats, one per format ([`Formats::hand`]);
|
||||
//! - then one folder per format, `v<N>/`, one file per change in it ([`Part`]), each with a step
|
||||
//! when files from before the change are rewritten as they open ([`Formats::folded`]);
|
||||
//! - then the changes no release has a number for yet, in `pending/` ([`Formats::pending`]):
|
||||
//! while there are any, the build writes the next number, with their names in the file's
|
||||
//! `format_pending`, so a later build runs just the steps a file is missing. A release moves
|
||||
//! them into the next `v<N>/`. [`build`] lists the folders for a build script.
|
||||
//!
|
||||
//! A file a newer build wrote says so by a number past this build's or a change name this build
|
||||
//! doesn't have ([`Family::newer`]); the app then reads what it can and keeps the rest
|
||||
//! ([`newer`]). A file is saved as JSON, or in a [`container`] with its binary payloads.
|
||||
//!
|
||||
//! Steps are JSON-level (the file's shape) and return the app's own error type; every error this
|
||||
//! crate makes is a [`FormatError`], which the app's error type takes `From`.
|
||||
|
||||
#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))]
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
pub mod build;
|
||||
pub mod container;
|
||||
pub mod newer;
|
||||
|
||||
/// Why a file is refused: its text says what is wrong, as the user reads it.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum FormatError {
|
||||
/// The file isn't one of the family's, or it is damaged.
|
||||
Invalid(String),
|
||||
/// The work was cancelled (a save the person stopped): nothing more was written.
|
||||
Cancelled,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for FormatError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
FormatError::Invalid(why) => f.write_str(why),
|
||||
FormatError::Cancelled => f.write_str("cancelled"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for FormatError {}
|
||||
|
||||
/// One JSON-level upgrade step: rewrites a file in place, or says why it can't.
|
||||
pub type Step<E> = fn(&mut Value) -> Result<(), E>;
|
||||
|
||||
/// One change to a family's file format: a file in its `v<N>/` or `pending/` folder.
|
||||
pub struct Part<E> {
|
||||
/// The file's name, which a file written while the change was pending lists.
|
||||
pub name: &'static str,
|
||||
/// How a file from before the change is rewritten as it opens, if it is.
|
||||
pub step: Option<Step<E>>,
|
||||
}
|
||||
|
||||
/// Another spelling of a change name that a file of one format may carry (a build that named the
|
||||
/// change otherwise): read as the change it stands for, never written.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct Alias {
|
||||
/// The format whose files may say it.
|
||||
pub format: u32,
|
||||
/// The spelling in the file.
|
||||
pub name: &'static str,
|
||||
/// The change it stands for.
|
||||
pub known_as: &'static str,
|
||||
}
|
||||
|
||||
/// A family's upgrade path: the steps written out by hand, the folded formats after them (each a
|
||||
/// list of changes) and the pending changes, which make the next format.
|
||||
pub struct Formats<'a, E> {
|
||||
/// Entry `i` turns format `i + 1` into `i + 2`.
|
||||
pub hand: &'a [Step<E>],
|
||||
/// Each format after the hand-written ones, oldest first, and its changes.
|
||||
pub folded: &'a [(u32, &'a [Part<E>])],
|
||||
/// The changes no release has a number for yet.
|
||||
pub pending: &'a [Part<E>],
|
||||
}
|
||||
|
||||
/// A kind of file whose format is versioned: its own format tag, its own numbers and its own
|
||||
/// changes. Reading a file's format, telling a file a newer build saved and the upgrade are the
|
||||
/// same for every family.
|
||||
pub struct Family<'a, E> {
|
||||
/// The app, as messages name it.
|
||||
pub app: &'static str,
|
||||
/// What a file of the family is called in messages.
|
||||
pub noun: &'static str,
|
||||
/// The tag before the number in a file's `format`.
|
||||
pub prefix: &'static str,
|
||||
/// The tag files saved under an older name carry: read, never written.
|
||||
pub legacy_prefix: Option<&'static str>,
|
||||
/// The format of a file that names none, if such files exist.
|
||||
pub untagged: Option<u32>,
|
||||
/// Other spellings of change names that files may carry.
|
||||
pub aliases: &'a [Alias],
|
||||
/// The upgrade path.
|
||||
pub formats: Formats<'a, E>,
|
||||
}
|
||||
|
||||
/// How a file shows a newer build saved it ([`Family::newer`]).
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct Newer {
|
||||
/// The number the file says.
|
||||
pub format: u32,
|
||||
/// The number this build reads up to.
|
||||
pub reads_up_to: u32,
|
||||
/// The changes the file names that this build doesn't have.
|
||||
pub unknown_changes: Vec<String>,
|
||||
}
|
||||
|
||||
impl<E> Formats<'_, E> {
|
||||
/// The format this build writes: the last folded one, or the next while changes are pending.
|
||||
pub const fn current(&self) -> u32 {
|
||||
self.hand.len() as u32 + 1 + self.folded.len() as u32 + if self.pending.is_empty() { 0 } else { 1 }
|
||||
}
|
||||
|
||||
/// The changes format `version` is made of, if it is made of changes.
|
||||
fn parts(&self, version: u32) -> Option<&[Part<E>]> {
|
||||
let after = version.checked_sub(self.hand.len() as u32 + 2)? as usize;
|
||||
match self.folded.get(after) {
|
||||
Some((_, parts)) => Some(parts),
|
||||
None if after == self.folded.len() && !self.pending.is_empty() => Some(self.pending),
|
||||
None => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether a change of this name is anywhere in this build's history.
|
||||
fn knows_part_name(&self, name: &str) -> bool {
|
||||
self.folded.iter().flat_map(|(_, parts)| *parts).chain(self.pending).any(|part| part.name == name)
|
||||
}
|
||||
}
|
||||
|
||||
impl<E> Family<'_, E> {
|
||||
/// The format this build writes: the last folded one, or the next while changes are pending.
|
||||
pub const fn current(&self) -> u32 {
|
||||
self.formats.current()
|
||||
}
|
||||
|
||||
/// What a file of the family is called in messages.
|
||||
pub const fn noun(&self) -> &'static str {
|
||||
self.noun
|
||||
}
|
||||
|
||||
/// The names of the pending changes this build writes into a file (`format_pending`).
|
||||
pub fn pending_names(&self) -> Vec<String> {
|
||||
self.formats.pending.iter().map(|p| p.name.to_string()).collect()
|
||||
}
|
||||
|
||||
/// The format tag of format `n`.
|
||||
pub fn format_string(&self, n: u32) -> String {
|
||||
format!("{}{n}", self.prefix)
|
||||
}
|
||||
|
||||
/// The version in one of this family's format tags; `None` for any other tag, another
|
||||
/// family's included.
|
||||
pub fn version_of(&self, s: &str) -> Option<u32> {
|
||||
// each tag form parses on its own: a legacy prefix may start with the primary one
|
||||
// (`app/` and `app/design/`)
|
||||
let parse = |prefix: &str| s.strip_prefix(prefix).and_then(|n| n.parse().ok());
|
||||
parse(self.prefix).or_else(|| self.legacy_prefix.and_then(parse))
|
||||
}
|
||||
|
||||
/// The change a name a file of format `version` lists stands for: its alias's, or itself.
|
||||
/// An alias counts only while this build has the change it stands for, so recognising it
|
||||
/// never grants support to another reader.
|
||||
fn canonical_applied_name<'name>(&self, version: u32, name: &'name str) -> &'name str {
|
||||
match self.aliases.iter().find(|a| a.format == version && a.name == name) {
|
||||
Some(alias) if self.formats.knows_part_name(alias.known_as) => alias.known_as,
|
||||
_ => name,
|
||||
}
|
||||
}
|
||||
|
||||
/// What says a file of format `version` with the changes `applied` came from a newer build:
|
||||
/// a number past this build's, or a change this build doesn't have anywhere in its history
|
||||
/// (a branch's pending change may have landed at a later number). A list beside a format that
|
||||
/// isn't made of changes means nothing (a tool or a test set an older number on a newer file).
|
||||
/// `None` for a file this build fully knows.
|
||||
pub fn newer(&self, version: u32, applied: Option<&[String]>) -> Option<Newer> {
|
||||
let current = self.current();
|
||||
let listed = applied.filter(|_| version > current || self.formats.parts(version).is_some()).unwrap_or_default();
|
||||
let unknown_changes: Vec<String> =
|
||||
listed.iter().filter(|name| !self.formats.knows_part_name(self.canonical_applied_name(version, name))).cloned().collect();
|
||||
(version > current || !unknown_changes.is_empty()).then_some(Newer { format: version, reads_up_to: current, unknown_changes })
|
||||
}
|
||||
}
|
||||
|
||||
impl<E: From<FormatError>> Family<'_, E> {
|
||||
/// Read a file's format and take its `format_pending` out of its JSON, before anything else
|
||||
/// in it is read: a file that isn't this family's is refused here. One a newer build saved is
|
||||
/// not: [`Family::newer`] says so.
|
||||
pub fn take_version(&self, v: &mut Value) -> Result<(u32, Option<Vec<String>>), E> {
|
||||
let (app, noun) = (self.app, self.noun);
|
||||
let not_ours = |what: String| E::from(FormatError::Invalid(format!("not a {app} {noun} ({what})")));
|
||||
let Some(obj) = v.as_object() else { return Err(not_ours("expected a JSON object".into())) };
|
||||
let version = match obj.get("format") {
|
||||
None => self.untagged.ok_or_else(|| not_ours("no format".into()))?,
|
||||
Some(Value::String(f)) => self.version_of(f).ok_or_else(|| not_ours(format!("format `{f}`")))?,
|
||||
Some(_) => return Err(not_ours("format is not a string".into())),
|
||||
};
|
||||
let applied = take_names(v, "format_pending", noun).map_err(E::from)?;
|
||||
if version == 0 {
|
||||
return Err(not_ours("format 0".into()));
|
||||
}
|
||||
Ok((version, applied))
|
||||
}
|
||||
|
||||
/// Bring a file of format `version` up to this build's; `applied` is its `format_pending`
|
||||
/// (both from [`Family::take_version`]): the changes of its format it already has, when it
|
||||
/// was written while they were pending. The others' steps run first, and already-applied
|
||||
/// changes never run again at their later number. A file past this build's number has no
|
||||
/// steps to run.
|
||||
pub fn upgrade(&self, v: &mut Value, version: u32, applied: Option<&[String]>) -> Result<(), E> {
|
||||
let formats = &self.formats;
|
||||
if version == 0 {
|
||||
return Err(E::from(FormatError::Invalid(format!("not a {} {} (format 0)", self.app, self.noun))));
|
||||
}
|
||||
// A list beside a format that isn't made of changes is ignored, like any field an upgrade
|
||||
// drops.
|
||||
let applied = applied.filter(|_| formats.parts(version).is_some());
|
||||
let already_applied =
|
||||
|part: &Part<E>| applied.is_some_and(|names| names.iter().any(|name| self.canonical_applied_name(version, name) == part.name));
|
||||
if applied.is_some() {
|
||||
for part in formats.parts(version).unwrap_or_default().iter().filter(|part| !already_applied(part)) {
|
||||
part.step.map_or(Ok(()), |step| step(v))?;
|
||||
}
|
||||
}
|
||||
for from in version..self.current() {
|
||||
match formats.hand.get(from as usize - 1) {
|
||||
Some(step) => step(v)?,
|
||||
None => {
|
||||
for part in formats.parts(from + 1).unwrap_or_default().iter().filter(|part| !already_applied(part)) {
|
||||
part.step.map_or(Ok(()), |step| step(v))?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Take a short list of names out of a file's JSON: its `format_pending` (the pending changes it
|
||||
/// has).
|
||||
fn take_names(v: &mut Value, key: &str, noun: &str) -> Result<Option<Vec<String>>, FormatError> {
|
||||
let Some(list) = v.as_object_mut().and_then(|o| o.remove(key)) else { return Ok(None) };
|
||||
let invalid = || FormatError::Invalid(format!("damaged {noun}: `{key}` is not a short list of names"));
|
||||
let list = list.as_array().filter(|l| l.len() <= 1000).ok_or_else(invalid)?;
|
||||
list.iter().map(|n| n.as_str().filter(|n| n.len() <= 128).map(str::to_string).ok_or_else(invalid)).collect::<Result<_, _>>().map(Some)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "lib_tests.rs"]
|
||||
mod tests;
|
||||
@@ -0,0 +1,142 @@
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
/// The app's error, as an app has one.
|
||||
#[derive(Debug, PartialEq)]
|
||||
struct AppError(String);
|
||||
|
||||
impl From<FormatError> for AppError {
|
||||
fn from(e: FormatError) -> Self {
|
||||
AppError(e.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
/// Each step appends its name to the file's `log`, so the order steps ran in can be read back.
|
||||
fn log(v: &mut Value, step: &str) {
|
||||
if let Some(list) = v.get_mut("log").and_then(Value::as_array_mut) {
|
||||
list.push(json!(step));
|
||||
}
|
||||
}
|
||||
|
||||
const HAND: [Step<AppError>; 2] = [
|
||||
|v| {
|
||||
log(v, "1to2");
|
||||
Ok(())
|
||||
},
|
||||
|v| {
|
||||
log(v, "2to3");
|
||||
Ok(())
|
||||
},
|
||||
];
|
||||
const V4: [Part<AppError>; 2] = [
|
||||
Part {
|
||||
name: "a",
|
||||
step: Some(|v| {
|
||||
log(v, "a");
|
||||
Ok(())
|
||||
}),
|
||||
},
|
||||
Part { name: "b", step: None },
|
||||
];
|
||||
const V5: [Part<AppError>; 1] = [Part {
|
||||
name: "c",
|
||||
step: Some(|v| {
|
||||
log(v, "c");
|
||||
Ok(())
|
||||
}),
|
||||
}];
|
||||
const FOLDED: [(u32, &[Part<AppError>]); 2] = [(4, &V4), (5, &V5)];
|
||||
const PENDING: [Part<AppError>; 2] = [
|
||||
Part {
|
||||
name: "d",
|
||||
step: Some(|v| {
|
||||
log(v, "d");
|
||||
Ok(())
|
||||
}),
|
||||
},
|
||||
Part {
|
||||
name: "e",
|
||||
step: Some(|v| {
|
||||
log(v, "e");
|
||||
Ok(())
|
||||
}),
|
||||
},
|
||||
];
|
||||
const ALIASES: [Alias; 1] = [Alias { format: 5, name: "old-c", known_as: "c" }];
|
||||
|
||||
/// Format 1 untagged; hand steps to 3; folded 4 and 5; pending changes make 6.
|
||||
const FAMILY: Family<'static, AppError> = Family {
|
||||
app: "App",
|
||||
noun: "design",
|
||||
prefix: "app/",
|
||||
legacy_prefix: Some("old/"),
|
||||
untagged: Some(1),
|
||||
aliases: &ALIASES,
|
||||
formats: Formats { hand: &HAND, folded: &FOLDED, pending: &PENDING },
|
||||
};
|
||||
|
||||
fn steps_run(version: u32, applied: Option<&[String]>) -> Vec<String> {
|
||||
let mut v = json!({"log": []});
|
||||
FAMILY.upgrade(&mut v, version, applied).unwrap();
|
||||
v["log"].as_array().unwrap().iter().map(|s| s.as_str().unwrap().to_string()).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tags_and_numbers() {
|
||||
assert_eq!(FAMILY.current(), 6);
|
||||
assert_eq!(FAMILY.format_string(6), "app/6");
|
||||
assert_eq!(FAMILY.version_of("app/4"), Some(4));
|
||||
assert_eq!(FAMILY.version_of("old/3"), Some(3), "the legacy tag is read");
|
||||
assert_eq!(FAMILY.version_of("other/3"), None);
|
||||
assert_eq!(FAMILY.pending_names(), vec!["d".to_string(), "e".to_string()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_file_is_taken_apart_or_refused_with_the_apps_words() {
|
||||
let mut v = json!({"format": "app/6", "format_pending": ["d"], "x": 1});
|
||||
assert_eq!(FAMILY.take_version(&mut v), Ok((6, Some(vec!["d".to_string()]))));
|
||||
assert!(v.get("format_pending").is_none(), "taken out");
|
||||
assert_eq!(FAMILY.take_version(&mut json!({})), Ok((1, None)), "untagged");
|
||||
let refused = |mut v: Value| FAMILY.take_version(&mut v).unwrap_err().0;
|
||||
assert_eq!(refused(json!([])), "not a App design (expected a JSON object)");
|
||||
assert_eq!(refused(json!({"format": "other/2"})), "not a App design (format `other/2`)");
|
||||
assert_eq!(refused(json!({"format": 2})), "not a App design (format is not a string)");
|
||||
assert_eq!(refused(json!({"format": "app/0"})), "not a App design (format 0)");
|
||||
assert_eq!(refused(json!({"format": "app/2", "format_pending": [1]})), "damaged design: `format_pending` is not a short list of names");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_step_runs_once_in_order() {
|
||||
assert_eq!(steps_run(1, None), ["1to2", "2to3", "a", "c", "d", "e"]);
|
||||
assert_eq!(steps_run(4, None), ["c", "d", "e"]);
|
||||
assert_eq!(steps_run(6, None), Vec::<String>::new());
|
||||
// A file written while `d` was pending has it: only `e` runs.
|
||||
assert_eq!(steps_run(6, Some(&["d".to_string()])), ["e"]);
|
||||
// A file of format 5 written while `c` was still pending, without it: `c` runs first.
|
||||
assert_eq!(steps_run(5, Some(&[])), ["c", "d", "e"]);
|
||||
// Under its other spelling `c` counts as had.
|
||||
assert_eq!(steps_run(5, Some(&["old-c".to_string()])), ["d", "e"]);
|
||||
// A list beside a hand-written format means nothing.
|
||||
assert_eq!(steps_run(2, Some(&["d".to_string()])), ["2to3", "a", "c", "d", "e"]);
|
||||
let mut v = json!({});
|
||||
assert_eq!(FAMILY.upgrade(&mut v, 0, None), Err(AppError("not a App design (format 0)".into())));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_newer_file_is_told_by_its_number_or_an_unknown_change() {
|
||||
assert_eq!(FAMILY.newer(6, None), None);
|
||||
assert_eq!(FAMILY.newer(6, Some(&["d".to_string()])), None);
|
||||
assert_eq!(FAMILY.newer(7, None), Some(Newer { format: 7, reads_up_to: 6, unknown_changes: Vec::new() }));
|
||||
assert_eq!(FAMILY.newer(6, Some(&["z".to_string()])), Some(Newer { format: 6, reads_up_to: 6, unknown_changes: vec!["z".to_string()] }));
|
||||
assert_eq!(FAMILY.newer(5, Some(&["old-c".to_string()])), None, "an alias of a known change");
|
||||
assert_eq!(FAMILY.newer(4, Some(&["old-c".to_string()])), Some(Newer { format: 4, reads_up_to: 6, unknown_changes: vec!["old-c".to_string()] }));
|
||||
assert_eq!(FAMILY.newer(2, Some(&["z".to_string()])), None, "a list beside a hand-written format means nothing");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_legacy_prefix_that_extends_the_primary_one_is_read() {
|
||||
let nested = Family { prefix: "app/", legacy_prefix: Some("app/design/"), ..FAMILY };
|
||||
assert_eq!(nested.version_of("app/design/3"), Some(3));
|
||||
assert_eq!(nested.version_of("app/4"), Some(4));
|
||||
assert_eq!(nested.version_of("app/design/x"), None);
|
||||
}
|
||||
@@ -0,0 +1,470 @@
|
||||
//! A file a newer build saved: it opens, with a warning, and saving it loses nothing.
|
||||
//!
|
||||
//! A newer format number, or a change the file names that this build doesn't have, says a newer
|
||||
//! build saved the file ([`crate::Family::newer`]). It may use nothing this build lacks, so the app
|
||||
//! reads it as usual. When part of it can't be read, that part stays out of the way and the rest
|
||||
//! opens: the app reads the elements of its lists it can ([`NewerFile::keep_element`]), then any
|
||||
//! other top-level section it can't read is left out whole ([`leave_out_unreadable`],
|
||||
//! [`read_around`]), and a field it doesn't know, which typed reading would drop, is noticed by
|
||||
//! comparing the file with what was read ([`NewerFile::compare`]).
|
||||
//!
|
||||
//! All of these are named in [`NewerFile::not_understood`] and go back into the file when it is
|
||||
//! saved ([`NewerFile::restore_into`]), as they came. What can't be put back because the file was
|
||||
//! changed around it is what `restore_into` returns.
|
||||
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
use std::sync::Arc;
|
||||
|
||||
use serde_json::{Map, Value, json};
|
||||
|
||||
use crate::Newer;
|
||||
|
||||
/// The most sections left out one at a time to find the one a file can't be read with.
|
||||
const MAX_SECTION_TRIES: usize = 64;
|
||||
|
||||
/// What the file says and holds that this build doesn't read.
|
||||
#[derive(Clone, Debug, Default, PartialEq)]
|
||||
pub struct NewerFile {
|
||||
/// The app, as the notice names it.
|
||||
pub app: &'static str,
|
||||
/// The number the file says.
|
||||
pub format: u32,
|
||||
/// The number this build reads up to.
|
||||
pub reads_up_to: u32,
|
||||
/// The changes the file names that this build doesn't have.
|
||||
pub unknown_changes: Vec<String>,
|
||||
/// What this build couldn't read, by name.
|
||||
pub not_understood: Vec<String>,
|
||||
/// Fields known records lost in typed reading.
|
||||
kept: Option<Kept>,
|
||||
/// Top-level sections left out whole.
|
||||
sections: Vec<Section>,
|
||||
/// Elements left out of a top-level list: its key and the element.
|
||||
elements: Vec<(String, Value)>,
|
||||
/// The parts of a container this build has no use for: name and bytes. A blob only one of the
|
||||
/// kept pieces refers to is one of them.
|
||||
pub parts: Vec<(String, Arc<[u8]>)>,
|
||||
/// The file's `container.json`, when this build doesn't know what it says.
|
||||
pub container: Option<Value>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
struct Section {
|
||||
key: String,
|
||||
value: Value,
|
||||
/// This build's record of the section, without the file's: while it is unchanged, the file's
|
||||
/// goes back.
|
||||
empty: Option<Value>,
|
||||
}
|
||||
|
||||
/// The fields of a record that typed reading dropped.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
enum Kept {
|
||||
Field(Value),
|
||||
Fields(BTreeMap<String, Kept>),
|
||||
Items(Vec<(Anchor, Kept)>),
|
||||
}
|
||||
|
||||
/// Which element of a list some kept fields belong to.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
enum Anchor {
|
||||
Id(u64),
|
||||
Name(String),
|
||||
/// Its place, good only while the list keeps its length.
|
||||
Index {
|
||||
at: usize,
|
||||
of: usize,
|
||||
},
|
||||
}
|
||||
|
||||
impl NewerFile {
|
||||
/// A file of `app` that `newer` says a newer build saved, nothing left out of it yet.
|
||||
pub fn new(app: &'static str, newer: &Newer) -> NewerFile {
|
||||
NewerFile {
|
||||
app,
|
||||
format: newer.format,
|
||||
reads_up_to: newer.reads_up_to,
|
||||
unknown_changes: newer.unknown_changes.clone(),
|
||||
..NewerFile::default()
|
||||
}
|
||||
}
|
||||
|
||||
/// What to tell the user, whatever else is said.
|
||||
pub fn notice(&self) -> String {
|
||||
let (app, reads, format) = (self.app, self.reads_up_to, self.format);
|
||||
match self.unknown_changes.first() {
|
||||
Some(change) if format <= reads => {
|
||||
format!("Saved by a newer {app} (format {format} with `{change}`, which this version doesn't know). If something's missing, update.")
|
||||
}
|
||||
_ => format!("Saved by a newer {app} (format {format}, this version reads up to {reads}). If something's missing, update."),
|
||||
}
|
||||
}
|
||||
|
||||
/// Every line to report: the notice, then what couldn't be read, by name.
|
||||
pub fn warnings(&self) -> Vec<String> {
|
||||
let unread = self.not_understood.iter().map(|what| format!("Not read: {what}. Saving keeps it."));
|
||||
std::iter::once(self.notice()).chain(unread).collect()
|
||||
}
|
||||
|
||||
/// What a reply says about it.
|
||||
pub fn report(&self) -> Value {
|
||||
json!({
|
||||
"format": self.format,
|
||||
"reads_up_to": self.reads_up_to,
|
||||
"unknown_changes": self.unknown_changes,
|
||||
"not_understood": self.not_understood,
|
||||
"notice": self.notice(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Whether the next save needs a container to keep what the file holds: plain JSON has no room
|
||||
/// for parts beside the file's own, nor for the container's own `container.json`.
|
||||
pub fn needs_container(&self) -> bool {
|
||||
!self.parts.is_empty() || self.container.is_some()
|
||||
}
|
||||
|
||||
/// Whether anything kept back names this text (a blob's name in a container).
|
||||
pub fn mentions(&self, text: &str) -> bool {
|
||||
fn kept(k: &Kept, text: &str) -> bool {
|
||||
match k {
|
||||
Kept::Field(v) => value_mentions(v, text),
|
||||
Kept::Fields(m) => m.values().any(|x| kept(x, text)),
|
||||
Kept::Items(l) => l.iter().any(|(_, x)| kept(x, text)),
|
||||
}
|
||||
}
|
||||
self.kept.as_ref().is_some_and(|k| kept(k, text))
|
||||
|| self.sections.iter().any(|s| value_mentions(&s.value, text))
|
||||
|| self.elements.iter().any(|(_, v)| value_mentions(v, text))
|
||||
}
|
||||
|
||||
/// Say that a section the reader couldn't finish reading is left out. `empty` is what the
|
||||
/// app writes for the section while it holds nothing, if anything.
|
||||
pub fn keep_section(&mut self, key: &str, value: Value, empty: Option<Value>, why: &str) {
|
||||
self.not_understood.push(format!("section `{key}` ({why})"));
|
||||
self.sections.push(Section { key: key.into(), value, empty });
|
||||
}
|
||||
|
||||
/// Say that an element of the top-level list `key` is left out, as `note` names it; it goes
|
||||
/// back at the end of the list when the file is saved.
|
||||
pub fn keep_element(&mut self, key: &str, element: Value, note: String) {
|
||||
self.not_understood.push(note);
|
||||
self.elements.push((key.into(), element));
|
||||
}
|
||||
|
||||
/// Compare the file as it reads (`file`, after the same upgrade as what was read) with what was
|
||||
/// read of it (`known`, its typed record written out): what `known` lacks is kept and named.
|
||||
/// An object whose `type` is `placeholder_tag` (an element the app kept whole) is not
|
||||
/// compared; nor are these top-level fields: those in `container_only` (what a container adds
|
||||
/// beside the file's own), `format` and `format_pending`, and the sections kept whole (they go
|
||||
/// back whole).
|
||||
pub fn compare(&mut self, file: &Value, known: &Value, placeholder_tag: &str, container_only: &[&str]) {
|
||||
let mut paths = Vec::new();
|
||||
let mut top_only: Vec<&str> = container_only.to_vec();
|
||||
top_only.extend(["format", "format_pending"]);
|
||||
top_only.extend(self.sections.iter().map(|s| s.key.as_str()));
|
||||
// a list typed writing leaves out holds only elements kept on their own: not kept twice
|
||||
top_only.extend(self.elements.iter().map(|(key, _)| key.as_str()).filter(|key| known.get(*key).is_none()));
|
||||
let kept = vanished(file, known, "", &mut paths, placeholder_tag, &top_only);
|
||||
self.kept = kept;
|
||||
self.not_understood.extend(paths);
|
||||
for section in &mut self.sections {
|
||||
section.empty = known.get(§ion.key).cloned();
|
||||
}
|
||||
}
|
||||
|
||||
/// Write back what was kept of the file into `value`, the file as this build writes it.
|
||||
/// Returns what couldn't go back, because the file was changed around it.
|
||||
pub fn restore_into(&self, value: &mut Value) -> Vec<String> {
|
||||
let mut lost = Vec::new();
|
||||
if let Some(kept) = &self.kept {
|
||||
apply(kept, value, "", &mut lost);
|
||||
}
|
||||
if let Some(top) = value.as_object_mut() {
|
||||
for section in &self.sections {
|
||||
if top.get(§ion.key) == section.empty.as_ref() {
|
||||
top.insert(section.key.clone(), section.value.clone());
|
||||
} else {
|
||||
lost.push(format!("section `{}`, replaced by what was edited here", section.key));
|
||||
}
|
||||
}
|
||||
// each list's records indexed once, by the identity records are matched by
|
||||
let mut indexes: HashMap<&str, Positions> = HashMap::new();
|
||||
for (key, element) in &self.elements {
|
||||
let name = element.get("name");
|
||||
let anchor = anchor_of(element);
|
||||
match top.entry(key.clone()).or_insert_with(|| json!([])) {
|
||||
Value::Array(list) => {
|
||||
let index = indexes.entry(key.as_str()).or_insert_with(|| Positions::of(list));
|
||||
if anchor.as_ref().is_some_and(|a| index.find(list.len(), a).is_some()) {
|
||||
lost.push(format!(
|
||||
"`{key}` element {}, replaced by what was made here",
|
||||
name.and_then(Value::as_str).unwrap_or("unnamed")
|
||||
));
|
||||
} else {
|
||||
index.add(element, list.len());
|
||||
list.push(element.clone());
|
||||
}
|
||||
}
|
||||
_ => lost.push(format!("`{key}` element {}, replaced by what was made here", name.and_then(Value::as_str).unwrap_or("unnamed"))),
|
||||
}
|
||||
}
|
||||
}
|
||||
lost
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether a string anywhere in `v` is `text`; iterative, so no nesting depth overflows the stack.
|
||||
fn value_mentions(v: &Value, text: &str) -> bool {
|
||||
let mut todo = vec![v];
|
||||
while let Some(v) = todo.pop() {
|
||||
match v {
|
||||
Value::String(s) if s == text => return true,
|
||||
Value::Array(a) => todo.extend(a),
|
||||
Value::Object(o) => todo.extend(o.values()),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// The first line of a reader's error, without the list of every kind it knows, and without the
|
||||
/// `"<noun> file: "` its reader starts it with.
|
||||
pub fn short<'a>(why: &'a str, noun: &str) -> &'a str {
|
||||
let why = why.split(", expected").next().unwrap_or(why);
|
||||
why.strip_prefix(&format!("{noun} file: ")).unwrap_or(why)
|
||||
}
|
||||
|
||||
/// Leave a section out and keep it. A section the file needs is left empty.
|
||||
fn move_out(top: &mut Map<String, Value>, key: &str, why: &str, noun: &str, file: &mut NewerFile) {
|
||||
let Some(value) = top.remove(key) else { return };
|
||||
file.not_understood.push(format!("section `{key}` ({})", short(why, noun)));
|
||||
file.sections.push(Section { key: key.into(), value, empty: None });
|
||||
}
|
||||
|
||||
/// Every top-level section of `top` that doesn't read by itself (`piece`, the app's reading of a
|
||||
/// file holding just that section) is left out and kept in `file`. `noun` is what a file of the
|
||||
/// app's is called, which its reader's errors start with.
|
||||
pub fn leave_out_unreadable<E: std::fmt::Display>(
|
||||
top: &mut Map<String, Value>,
|
||||
file: &mut NewerFile,
|
||||
noun: &str,
|
||||
piece: impl Fn(&str, &Value) -> Result<(), E>,
|
||||
) {
|
||||
let keys: Vec<String> = top.keys().cloned().collect();
|
||||
for key in &keys {
|
||||
let value = top.get(key).cloned().unwrap_or_default();
|
||||
if let Err(why) = piece(key, &value) {
|
||||
move_out(top, key, &why.to_string(), noun, file);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Read `top` with `attempt`, leaving out the sections it can't be read with: if it doesn't read,
|
||||
/// each section but the `core` ones (which the file can't be read without) is tried left out alone,
|
||||
/// and as a last resort every one of them is. What is left out is kept in `file`. The error is the
|
||||
/// first attempt's, when even the core alone doesn't read.
|
||||
pub fn read_around<D, E: std::fmt::Display>(
|
||||
top: &mut Map<String, Value>,
|
||||
file: &mut NewerFile,
|
||||
core: &[&str],
|
||||
noun: &str,
|
||||
attempt: impl Fn(&Map<String, Value>) -> Result<D, E>,
|
||||
) -> Result<D, String> {
|
||||
let first = match attempt(top) {
|
||||
Ok(doc) => return Ok(doc),
|
||||
Err(why) => why.to_string(),
|
||||
};
|
||||
// Sections that are fine by themselves but not together with the rest.
|
||||
let keys: Vec<String> = top.keys().filter(|k| !core.contains(&k.as_str())).cloned().collect();
|
||||
// Each try reads the whole file: a file of many sections goes straight to the last resort.
|
||||
for key in keys.iter().filter(|_| keys.len() <= MAX_SECTION_TRIES) {
|
||||
let mut without = top.clone();
|
||||
without.remove(key);
|
||||
if let Ok(doc) = attempt(&without) {
|
||||
move_out(top, key, &first, noun, file);
|
||||
return Ok(doc);
|
||||
}
|
||||
}
|
||||
let mut bare = top.clone();
|
||||
bare.retain(|key, _| core.contains(&key.as_str()));
|
||||
match attempt(&bare) {
|
||||
Ok(doc) => {
|
||||
for key in &keys {
|
||||
move_out(top, key, &first, noun, file);
|
||||
}
|
||||
Ok(doc)
|
||||
}
|
||||
Err(_) => Err(first),
|
||||
}
|
||||
}
|
||||
|
||||
/// A value a writer leaves out without losing anything: not worth telling anyone about.
|
||||
pub fn looks_default(v: &Value) -> bool {
|
||||
match v {
|
||||
Value::Null => true,
|
||||
Value::Bool(b) => !b,
|
||||
Value::Number(n) => n.as_f64() == Some(0.0),
|
||||
Value::String(s) => s.is_empty(),
|
||||
Value::Array(a) => a.is_empty(),
|
||||
Value::Object(o) => o.is_empty(),
|
||||
}
|
||||
}
|
||||
|
||||
/// What `raw` has that `known`, what was read of it, doesn't. `paths` gets those fields' names.
|
||||
/// `top_only` are top-level keys left out of the comparison.
|
||||
fn vanished(raw: &Value, known: &Value, path: &str, paths: &mut Vec<String>, placeholder_tag: &str, top_only: &[&str]) -> Option<Kept> {
|
||||
match (raw, known) {
|
||||
(Value::Object(raw), Value::Object(known)) => {
|
||||
if known.get("type").and_then(Value::as_str) == Some(placeholder_tag) {
|
||||
return None;
|
||||
}
|
||||
let mut kept = BTreeMap::new();
|
||||
for (key, value) in raw {
|
||||
if path.is_empty() && top_only.contains(&key.as_str()) {
|
||||
continue;
|
||||
}
|
||||
let inner = if path.is_empty() { key.clone() } else { format!("{path}.{key}") };
|
||||
match known.get(key) {
|
||||
None => {
|
||||
if !looks_default(value) {
|
||||
paths.push(format!("field `{inner}`"));
|
||||
}
|
||||
kept.insert(key.clone(), Kept::Field(value.clone()));
|
||||
}
|
||||
Some(known) => {
|
||||
if let Some(k) = vanished(value, known, &inner, paths, placeholder_tag, top_only) {
|
||||
kept.insert(key.clone(), k);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
(!kept.is_empty()).then_some(Kept::Fields(kept))
|
||||
}
|
||||
(Value::Array(raw), Value::Array(known)) => {
|
||||
let mut items = Vec::new();
|
||||
let positions = Positions::of(known);
|
||||
for (at, element) in raw.iter().enumerate() {
|
||||
// an identity typed reading kept: an id it dropped falls back to the name
|
||||
let anchor = surviving_anchor(element, &positions).unwrap_or(Anchor::Index { at, of: raw.len() });
|
||||
let Some(partner) = positions.find(known.len(), &anchor).and_then(|i| known.get(i)) else { continue };
|
||||
let label = match (element.get("name").and_then(Value::as_str), &anchor) {
|
||||
(Some(name), _) => format!("{path}[{name}]"),
|
||||
(None, Anchor::Id(id)) => format!("{path}[{id}]"),
|
||||
_ => format!("{path}[{at}]"),
|
||||
};
|
||||
if let Some(k) = vanished(element, partner, &label, paths, placeholder_tag, top_only) {
|
||||
items.push((anchor, k));
|
||||
}
|
||||
}
|
||||
(!items.is_empty()).then_some(Kept::Items(items))
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn anchor_of(element: &Value) -> Option<Anchor> {
|
||||
if let Some(id) = element.get("id").and_then(Value::as_u64) {
|
||||
return Some(Anchor::Id(id));
|
||||
}
|
||||
element.get("name").and_then(Value::as_str).map(|name| Anchor::Name(name.to_string()))
|
||||
}
|
||||
|
||||
/// The identity of `element` that `known`'s records still carry: its id, or else its name. When
|
||||
/// typed reading kept neither, there is none, and the record is matched by its place.
|
||||
fn surviving_anchor(element: &Value, known: &Positions) -> Option<Anchor> {
|
||||
let id = element.get("id").and_then(Value::as_u64).map(Anchor::Id);
|
||||
let name = element.get("name").and_then(Value::as_str).map(|n| Anchor::Name(n.to_string()));
|
||||
[id, name].into_iter().flatten().find(|anchor| known.find(usize::MAX, anchor).is_some())
|
||||
}
|
||||
|
||||
/// Where each id and name first appears in a list, so matching records is not a scan per record.
|
||||
struct Positions {
|
||||
ids: HashMap<u64, usize>,
|
||||
names: HashMap<String, usize>,
|
||||
}
|
||||
|
||||
impl Positions {
|
||||
fn of(list: &[Value]) -> Positions {
|
||||
let (mut ids, mut names) = (HashMap::new(), HashMap::new());
|
||||
for (i, x) in list.iter().enumerate() {
|
||||
if let Some(id) = x.get("id").and_then(Value::as_u64) {
|
||||
ids.entry(id).or_insert(i);
|
||||
}
|
||||
if let Some(name) = x.get("name").and_then(Value::as_str) {
|
||||
names.entry(name.to_string()).or_insert(i);
|
||||
}
|
||||
}
|
||||
Positions { ids, names }
|
||||
}
|
||||
|
||||
/// Count `x`, now at `i`, in.
|
||||
fn add(&mut self, x: &Value, i: usize) {
|
||||
if let Some(id) = x.get("id").and_then(Value::as_u64) {
|
||||
self.ids.entry(id).or_insert(i);
|
||||
}
|
||||
if let Some(name) = x.get("name").and_then(Value::as_str) {
|
||||
self.names.entry(name.to_string()).or_insert(i);
|
||||
}
|
||||
}
|
||||
|
||||
/// The position of the record `anchor` names in a list of `len` records (an index anchor
|
||||
/// only while the list keeps its length).
|
||||
fn find(&self, len: usize, anchor: &Anchor) -> Option<usize> {
|
||||
match anchor {
|
||||
Anchor::Id(id) => self.ids.get(id).copied(),
|
||||
Anchor::Name(name) => self.names.get(name).copied(),
|
||||
Anchor::Index { at, of } => (len == *of && at < of).then_some(*at),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Put the kept fields back. An element that is gone was deleted here, its fields with it; one
|
||||
/// whose place can't be told any more, and a field whose record is gone, go into `lost`.
|
||||
fn apply(kept: &Kept, value: &mut Value, path: &str, lost: &mut Vec<String>) {
|
||||
match kept {
|
||||
Kept::Field(_) => {}
|
||||
Kept::Fields(fields) => {
|
||||
let Some(object) = value.as_object_mut() else {
|
||||
lost.push(format!("field `{path}`"));
|
||||
return;
|
||||
};
|
||||
for (key, sub) in fields {
|
||||
let inner = if path.is_empty() { key.clone() } else { format!("{path}.{key}") };
|
||||
match (sub, object.get_mut(key)) {
|
||||
(Kept::Field(v), None) => {
|
||||
object.insert(key.clone(), v.clone());
|
||||
}
|
||||
// the edited value stays; a retained one it replaced is reported
|
||||
(Kept::Field(v), Some(existing)) => {
|
||||
if existing != v {
|
||||
lost.push(format!("field `{inner}`"));
|
||||
}
|
||||
}
|
||||
(sub, Some(inner_value)) => apply(sub, inner_value, &inner, lost),
|
||||
// The list it was in is empty and not written: its elements were deleted.
|
||||
(Kept::Items(_), None) => {}
|
||||
(_, None) => lost.push(format!("field `{inner}`")),
|
||||
}
|
||||
}
|
||||
}
|
||||
Kept::Items(items) => {
|
||||
let Some(list) = value.as_array_mut() else {
|
||||
lost.push(format!("field `{path}`"));
|
||||
return;
|
||||
};
|
||||
let positions = Positions::of(list);
|
||||
let len = list.len();
|
||||
for (anchor, sub) in items {
|
||||
match (positions.find(len, anchor).and_then(|i| list.get_mut(i)), anchor) {
|
||||
(Some(element), _) => apply(sub, element, path, lost),
|
||||
(None, Anchor::Index { at, .. }) => lost.push(format!("element {at} of `{path}`")),
|
||||
(None, _) => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "newer_tests.rs"]
|
||||
mod tests;
|
||||
@@ -0,0 +1,218 @@
|
||||
use super::*;
|
||||
|
||||
fn newer() -> Newer {
|
||||
Newer { format: 9, reads_up_to: 8, unknown_changes: Vec::new() }
|
||||
}
|
||||
|
||||
/// What typed reading dropped is named and goes back on saving, where it was.
|
||||
#[test]
|
||||
fn dropped_fields_are_named_and_put_back() {
|
||||
let file = json!({"name": "A", "format": "app/9", "items": [{"id": 1, "size": 2, "colour": "red"}], "extra": {"x": 1}});
|
||||
let known = json!({"name": "A", "items": [{"id": 1, "size": 2}]});
|
||||
let mut said = NewerFile::new("App", &newer());
|
||||
said.compare(&file, &known, "placeholder", &[]);
|
||||
assert_eq!(said.not_understood, ["field `extra`", "field `items[1].colour`"]);
|
||||
let mut saved = json!({"name": "A", "items": [{"id": 1, "size": 3}]});
|
||||
assert!(said.restore_into(&mut saved).is_empty());
|
||||
assert_eq!(saved, json!({"name": "A", "items": [{"id": 1, "size": 3, "colour": "red"}], "extra": {"x": 1}}));
|
||||
assert_eq!(said.notice(), "Saved by a newer App (format 9, this version reads up to 8). If something's missing, update.");
|
||||
}
|
||||
|
||||
/// A section that can't be read is left out, kept, and goes back while it is unchanged here.
|
||||
#[test]
|
||||
fn unreadable_sections_are_left_out_and_kept() {
|
||||
let mut top = match json!({"name": "A", "good": 1, "bad": 2}) {
|
||||
Value::Object(m) => m,
|
||||
_ => unreachable!(),
|
||||
};
|
||||
let mut said = NewerFile::new("App", &newer());
|
||||
leave_out_unreadable(&mut top, &mut said, "design", |key: &str, _: &Value| if key == "bad" { Err("design file: no") } else { Ok(()) });
|
||||
assert!(!top.contains_key("bad"));
|
||||
assert_eq!(said.not_understood, ["section `bad` (no)"]);
|
||||
let read =
|
||||
read_around(
|
||||
&mut top,
|
||||
&mut said,
|
||||
&["name"],
|
||||
"design",
|
||||
|top: &Map<String, Value>| {
|
||||
if top.contains_key("good") { Err("design file: clash") } else { Ok(top.len()) }
|
||||
},
|
||||
);
|
||||
assert_eq!(read, Ok(1));
|
||||
assert_eq!(said.not_understood, ["section `bad` (no)", "section `good` (clash)"]);
|
||||
let mut saved = json!({"name": "A"});
|
||||
said.compare(&saved.clone(), &saved.clone(), "placeholder", &[]);
|
||||
assert!(said.restore_into(&mut saved).is_empty());
|
||||
assert_eq!(saved, json!({"name": "A", "good": 1, "bad": 2}));
|
||||
}
|
||||
|
||||
/// Notices, warnings and reports name what was left out, and kept values can be searched.
|
||||
#[test]
|
||||
fn warnings_report_and_mentions_cover_everything_kept() {
|
||||
let mut said = NewerFile::new("App", &Newer { format: 8, reads_up_to: 8, unknown_changes: vec!["curves".into()] });
|
||||
assert_eq!(said.notice(), "Saved by a newer App (format 8 with `curves`, which this version doesn't know). If something's missing, update.");
|
||||
assert!(!said.needs_container());
|
||||
said.keep_section("plugins", json!({"tool": "ruler"}), Some(json!({})), "unknown");
|
||||
said.keep_element("shapes", json!({"name": "star", "points": ["tip"]}), "shape `star`".into());
|
||||
let file = json!({"shapes": [{"name": "a", "tags": ["deep"], "more": {"x": "nested"}}], "n": 1});
|
||||
said.compare(&file, &json!({"shapes": [{"name": "a"}], "n": 1}), "placeholder", &[]);
|
||||
assert_eq!(said.warnings()[0], said.notice());
|
||||
assert!(said.warnings().contains(&"Not read: section `plugins` (unknown). Saving keeps it.".to_string()));
|
||||
let report = said.report();
|
||||
assert_eq!(report["format"], 8);
|
||||
assert_eq!(report["unknown_changes"], json!(["curves"]));
|
||||
assert_eq!(report["notice"], json!(said.notice()));
|
||||
for text in ["ruler", "tip", "deep", "nested"] {
|
||||
assert!(said.mentions(text), "{text}");
|
||||
}
|
||||
assert!(!said.mentions("absent"));
|
||||
said.parts.push(("blob".into(), Arc::from(&b"x"[..])));
|
||||
assert!(said.needs_container());
|
||||
}
|
||||
|
||||
/// A kept section or element that was replaced here is reported lost, not written over.
|
||||
#[test]
|
||||
fn replaced_sections_and_elements_are_reported_lost() {
|
||||
let mut said = NewerFile::new("App", &newer());
|
||||
said.keep_section("plugins", json!({"tool": "ruler"}), None, "unknown");
|
||||
said.keep_element("shapes", json!({"name": "star"}), "shape `star`".into());
|
||||
said.keep_element("shapes", json!({"points": 3}), "an unnamed shape".into());
|
||||
let mut saved = json!({"plugins": {"edited": true}, "shapes": [{"name": "star"}]});
|
||||
let lost = said.restore_into(&mut saved);
|
||||
assert_eq!(lost, ["section `plugins`, replaced by what was edited here", "`shapes` element star, replaced by what was made here"]);
|
||||
assert_eq!(saved["shapes"], json!([{"name": "star"}, {"points": 3}]));
|
||||
let mut fresh = json!({});
|
||||
said.sections.clear();
|
||||
assert!(said.restore_into(&mut fresh).is_empty());
|
||||
assert_eq!(fresh["shapes"], json!([{"name": "star"}, {"points": 3}]));
|
||||
}
|
||||
|
||||
/// Retained container metadata alone needs a container to survive the save.
|
||||
#[test]
|
||||
fn container_metadata_alone_needs_a_container() {
|
||||
let mut said = NewerFile::new("App", &newer());
|
||||
assert!(!said.needs_container());
|
||||
said.container = Some(json!({"layout": 2}));
|
||||
assert!(said.needs_container());
|
||||
}
|
||||
|
||||
/// An edit that replaces a retained field keeps the edit and reports the replaced value.
|
||||
#[test]
|
||||
fn a_replaced_retained_field_is_reported() {
|
||||
let mut said = NewerFile::new("App", &newer());
|
||||
said.compare(&json!({"extra": 1}), &json!({}), "placeholder", &[]);
|
||||
let mut edited = json!({"extra": 2});
|
||||
assert_eq!(said.restore_into(&mut edited), ["field `extra`"]);
|
||||
assert_eq!(edited, json!({"extra": 2}));
|
||||
let mut same = json!({"extra": 1});
|
||||
assert!(said.restore_into(&mut same).is_empty());
|
||||
}
|
||||
|
||||
/// Records are matched by id and by name across long lists, and an index anchor only while the
|
||||
/// list keeps its length.
|
||||
#[test]
|
||||
fn records_match_by_id_name_or_unchanged_index() {
|
||||
let records: Vec<Value> = (0..2000).map(|i| json!({"id": i, "extra": i})).collect();
|
||||
let known: Vec<Value> = (0..2000).map(|i| json!({"id": i})).collect();
|
||||
let mut said = NewerFile::new("App", &newer());
|
||||
said.compare(&json!({"items": records}), &json!({"items": known}), "placeholder", &[]);
|
||||
let mut saved = json!({"items": (0..2000).rev().map(|i| json!({"id": i})).collect::<Vec<_>>()});
|
||||
assert!(said.restore_into(&mut saved).is_empty());
|
||||
assert_eq!(saved["items"][0], json!({"id": 1999, "extra": 1999}));
|
||||
let mut by_name = NewerFile::new("App", &newer());
|
||||
by_name.compare(&json!({"l": [{"name": "a", "x": 1}, [1, 2]]}), &json!({"l": [{"name": "a"}, [1]]}), "placeholder", &[]);
|
||||
let mut renamed = json!({"l": [{"name": "a"}]});
|
||||
by_name.restore_into(&mut renamed);
|
||||
assert_eq!(renamed["l"][0], json!({"name": "a", "x": 1}));
|
||||
}
|
||||
|
||||
/// A section kept whole goes back whole, even when this build's record of it has fields.
|
||||
#[test]
|
||||
fn a_section_kept_whole_is_not_compared_field_by_field() {
|
||||
let mut said = NewerFile::new("App", &newer());
|
||||
said.keep_section("plugins", json!({"mode": "future", "payload": 1}), None, "unreadable");
|
||||
let file = json!({"name": "A", "plugins": {"mode": "future", "payload": 1}});
|
||||
said.compare(&file, &json!({"name": "A", "plugins": {"mode": "none"}}), "placeholder", &[]);
|
||||
let mut saved = json!({"name": "A", "plugins": {"mode": "none"}});
|
||||
assert!(said.restore_into(&mut saved).is_empty());
|
||||
assert_eq!(saved, file);
|
||||
}
|
||||
|
||||
/// A kept element goes back only while no record has its id or name.
|
||||
#[test]
|
||||
fn a_kept_element_keeps_record_identity() {
|
||||
let mut said = NewerFile::new("App", &newer());
|
||||
said.keep_element("items", json!({"id": 7, "kind": "future"}), "element 7".into());
|
||||
said.keep_element("items", json!({"kind": "anonymous"}), "unnamed".into());
|
||||
let mut clash = json!({"items": [{"id": 7, "kind": "made here"}]});
|
||||
assert_eq!(said.restore_into(&mut clash).len(), 1);
|
||||
assert_eq!(clash, json!({"items": [{"id": 7, "kind": "made here"}, {"kind": "anonymous"}]}));
|
||||
let mut free = json!({"items": []});
|
||||
assert!(said.restore_into(&mut free).is_empty());
|
||||
assert_eq!(free, json!({"items": [{"id": 7, "kind": "future"}, {"kind": "anonymous"}]}));
|
||||
}
|
||||
|
||||
/// Container-only fields are left out at the top level only.
|
||||
#[test]
|
||||
fn container_only_fields_are_top_level() {
|
||||
let mut said = NewerFile::new("App", &newer());
|
||||
let file = json!({"thumbnail": "t", "layer": {"thumbnail": "nested"}});
|
||||
said.compare(&file, &json!({"layer": {}}), "placeholder", &["thumbnail"]);
|
||||
assert_eq!(said.not_understood, ["field `layer.thumbnail`"]);
|
||||
let mut saved = json!({"layer": {}});
|
||||
assert!(said.restore_into(&mut saved).is_empty());
|
||||
assert_eq!(saved, json!({"layer": {"thumbnail": "nested"}}));
|
||||
}
|
||||
|
||||
/// An element kept on its own is not kept again with its list when typed writing leaves the list out.
|
||||
#[test]
|
||||
fn a_kept_element_is_not_restored_twice() {
|
||||
let mut said = NewerFile::new("App", &newer());
|
||||
said.keep_element("items", json!({"kind": "future"}), "unnamed".into());
|
||||
said.compare(&json!({"items": [{"kind": "future"}]}), &json!({}), "placeholder", &[]);
|
||||
let mut saved = json!({});
|
||||
assert!(said.restore_into(&mut saved).is_empty());
|
||||
assert_eq!(saved, json!({"items": [{"kind": "future"}]}));
|
||||
}
|
||||
|
||||
/// A record whose id typed reading drops is matched by its name, and its fields go back.
|
||||
#[test]
|
||||
fn a_dropped_id_falls_back_to_the_name() {
|
||||
let mut said = NewerFile::new("App", &newer());
|
||||
said.compare(&json!({"items": [{"id": 7, "name": "a", "extra": 1}]}), &json!({"items": [{"name": "a"}]}), "placeholder", &[]);
|
||||
let mut saved = json!({"items": [{"name": "a"}]});
|
||||
assert!(said.restore_into(&mut saved).is_empty());
|
||||
assert_eq!(saved, json!({"items": [{"name": "a", "id": 7, "extra": 1}]}));
|
||||
}
|
||||
|
||||
/// Kept elements sharing a list are checked against one index, the ones put back included.
|
||||
#[test]
|
||||
fn kept_elements_check_each_other() {
|
||||
let mut said = NewerFile::new("App", &newer());
|
||||
said.keep_element("items", json!({"id": 1, "v": "first"}), "1".into());
|
||||
said.keep_element("items", json!({"id": 1, "v": "again"}), "1 again".into());
|
||||
let mut saved = json!({"items": []});
|
||||
assert_eq!(said.restore_into(&mut saved).len(), 1);
|
||||
assert_eq!(saved, json!({"items": [{"id": 1, "v": "first"}]}));
|
||||
// deep nesting is searched without recursion
|
||||
let mut deep = json!("needle");
|
||||
for _ in 0..20_000 {
|
||||
deep = Value::Array(vec![deep]);
|
||||
}
|
||||
said.keep_section("deep", deep, None, "test");
|
||||
assert!(said.mentions("needle"));
|
||||
// serde_json drops deep values recursively; leak it rather than overflow the test thread
|
||||
std::mem::forget(said);
|
||||
}
|
||||
|
||||
/// A record whose whole identity typed reading drops is matched by its place while the list keeps
|
||||
/// its length, so an unchanged save keeps its fields.
|
||||
#[test]
|
||||
fn a_record_without_a_surviving_identity_is_matched_by_place() {
|
||||
let mut said = NewerFile::new("App", &newer());
|
||||
said.compare(&json!({"items": [{"id": 7, "extra": 1}]}), &json!({"items": [{}]}), "placeholder", &[]);
|
||||
let mut saved = json!({"items": [{}]});
|
||||
assert!(said.restore_into(&mut saved).is_empty());
|
||||
assert_eq!(saved, json!({"items": [{"id": 7, "extra": 1}]}));
|
||||
}
|
||||
Reference in new issue
Block a user