Merge pull request #1627 from storytold/pr-solidcraft-format

feat: add craft-format (versioned JSON file formats)
This commit is contained in:
FredApps authored and GitHub committed 2026-11-16 06:55:33 +01:00
commit 51b074a35a
10 files changed
+2173 -4

No files matched your search

Generated
+14 -4
View File
@@ -1849,6 +1849,16 @@ dependencies = [
"craft-fonts-build",
]
[[package]]
name = "craft-format"
version = "0.1.0"
dependencies = [
"craft-zip",
"proptest",
"serde_json",
"sha2 0.10.9",
]
[[package]]
name = "craft-gainmap"
version = "0.1.0"
@@ -8539,9 +8549,9 @@ checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f"
[[package]]
name = "rustls-webpki"
version = "0.103.15"
version = "0.103.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2"
checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281"
dependencies = [
"aws-lc-rs",
"ring",
@@ -11964,9 +11974,9 @@ dependencies = [
[[package]]
name = "zip"
version = "9.1.0"
version = "9.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2886b00003a72c4ccee3f42cfc34908d33b2508a74d827dc2a4244706475c704"
checksum = "1156b39d142a34b999c2310a50455e0cb3760bf7efad0122f796d813bee26ccb"
dependencies = [
"crc32fast",
"flate2",
+20
View File
@@ -0,0 +1,20 @@
[package]
name = "craft-format"
description = "Versioned JSON file formats: the format tag, the registry of versions (hand-written steps, then one folder of changes per version, then the pending changes), the upgrade of an older file and the telling of a newer one"
version.workspace = true
edition.workspace = true
license.workspace = true
rust-version.workspace = true
repository.workspace = true
[lints]
workspace = true
[dependencies]
serde_json = { workspace = true }
# The container: content-addressed blobs (SHA-256) in a ZIP package.
sha2 = { version = "0.10", default-features = false, features = ["std"] }
craft-zip = { workspace = true }
[dev-dependencies]
proptest = { workspace = true }
+237
View File
@@ -0,0 +1,237 @@
//! For a build script: the generated lists of a family's changes. Each change is one `.rs` file in
//! the family's folder, in `v<N>/` once a release fixed its number, else in `pending/`; the lists
//! are generated so that a change adds a file and never edits a line another change also edits.
//!
//! The generated code, for the app to `include!`, is per family a `FOLDED` list (each folded
//! format and its parts) and a `PENDING` list, both prefixed by the family's `list_prefix`, then a
//! module per change file. It names the app's `Part` type, which the including module has in scope
//! (`type Part = craft_format::Part<AppError>;`), and a change file's step as `step` (a
//! `pub fn step(v: &mut Value) -> Result<(), AppError>`, when the file has one).
use std::fmt::Write as _;
use std::path::{Path, PathBuf};
/// A family's change files: their folder (relative to the crate's manifest), the last format
/// before them, and the prefix of the family's lists.
pub struct Family {
pub folder: &'static str,
pub base: u32,
pub list_prefix: &'static str,
}
/// What [`lists`] made: the code to write, and the folders the build script reruns on.
pub struct Lists {
pub code: String,
pub watch: Vec<PathBuf>,
}
/// The `FOLDED` and `PENDING` lists of every family, and a module for each change file. Refuses
/// versions that are not contiguous from a family's base, a change file not named with lowercase
/// letters, digits and dashes, and one that doesn't start with a `//!` note.
pub fn lists(manifest: &Path, families: &[Family]) -> Result<Lists, String> {
let mut code = String::new();
let mut mods = String::new();
let mut watch = Vec::new();
for family in families {
let root = manifest.join(family.folder);
family_lists(family, &root, &mut code, &mut mods)?;
watch.push(root);
}
code.push_str(&mods);
Ok(Lists { code, watch })
}
/// A family's `FOLDED` and `PENDING` lists; its folded formats follow on from its base.
fn family_lists(family: &Family, root: &Path, out: &mut String, mods: &mut String) -> Result<(), String> {
let Family { folder, base, list_prefix } = family;
let mut folded: Vec<(u32, PathBuf)> = Vec::new();
for entry in std::fs::read_dir(root).map_err(|e| format!("{}: {e}", root.display()))? {
// an entry that can't be read could be a version folder: fail rather than drop it
let path = entry.map_err(|e| format!("{}: {e}", root.display()))?.path();
let version = path.file_name().and_then(|n| n.to_str()).and_then(|n| n.strip_prefix('v')).and_then(|n| n.parse().ok());
if let Some(version) = version
&& path.is_dir()
{
folded.push((version, path));
}
}
folded.sort();
for (i, (version, _)) in folded.iter().enumerate() {
let expected = base + 1 + i as u32;
if *version != expected {
return Err(format!("{folder}/: expected v{expected}/ next, found v{version}/ (versions are contiguous)"));
}
}
// Module names start with the family's (`drawing_pending_…`), so two families' changes of one
// name never clash.
let module = list_prefix.to_ascii_lowercase();
let _ = writeln!(
out,
"/// The changes of each format after the base, oldest first (`{folder}/v<N>/`).\nconst {list_prefix}FOLDED: &[(u32, &[Part])] = &["
);
for (version, dir) in &folded {
let _ = writeln!(out, " ({version}, &[");
parts(dir, &format!("{module}v{version}"), false, out, mods)?;
out.push_str(" ]),\n");
}
let _ = writeln!(out, "];\n\n/// The changes no release has a number for yet (`{folder}/pending/`).\nconst {list_prefix}PENDING: &[Part] = &[");
parts(&root.join("pending"), &format!("{module}pending"), true, out, mods)?;
out.push_str("];\n\n");
Ok(())
}
/// Whether a change file declares its `step` function (`fn step(`, `pub fn step (` …) on a line of
/// code: a mention in a comment or a note is not a declaration.
fn declares_step(text: &str) -> bool {
let code = without_comments(text);
let ident = |c: char| c.is_alphanumeric() || c == '_';
// `fn`, whitespace, `step`, then `(` (or generics), across lines
code.match_indices("fn").any(|(at, _)| {
let before_ok = code[..at].chars().next_back().is_none_or(|c| !ident(c));
let rest = &code[at + 2..];
before_ok
&& rest.starts_with(char::is_whitespace)
&& rest
.trim_start()
.strip_prefix("step")
.is_some_and(|rest| matches!(rest.trim_start().chars().next(), Some('(' | '<')) && !rest.starts_with(ident))
})
}
/// `text` as code only: its `//` and (nested) `/* */` comments become a space, and the contents
/// of its string literals (plain, byte and raw) are dropped, line breaks kept throughout. A comment
/// marker inside a string is not a comment, and `fn/* note */step` still reads as `fn step`.
fn without_comments(text: &str) -> String {
let chars: Vec<char> = text.chars().collect();
let at = |i: usize| chars.get(i).copied();
let mut out = String::with_capacity(text.len());
let (mut i, mut depth) = (0usize, 0usize);
while let Some(c) = at(i) {
let next = at(i + 1);
if depth > 0 {
match (c, next) {
('/', Some('*')) => (depth, i) = (depth + 1, i + 2),
('*', Some('/')) => (depth, i) = (depth - 1, i + 2),
_ => {
if c == '\n' {
out.push('\n');
}
i += 1;
}
}
if depth == 0 {
out.push(' ');
}
continue;
}
let ident_before = i > 0 && at(i - 1).is_some_and(|p| p.is_alphanumeric() || p == '_');
match (c, next) {
('/', Some('/')) => {
while at(i).is_some_and(|c| c != '\n') {
i += 1;
}
out.push(' ');
}
('/', Some('*')) => (depth, i) = (1, i + 2),
('"', _) => {
out.push('"');
i += 1;
while let Some(c) = at(i) {
i += 1;
match c {
'\\' => i += 1,
'"' => break,
'\n' => out.push('\n'),
_ => {}
}
}
out.push('"');
}
// a raw string, `r"…"` or `r#"…"#`, after a byte prefix or on its own
('r', Some('"' | '#'))
if !ident_before || (i > 0 && at(i - 1) == Some('b') && !(i > 1 && at(i - 2).is_some_and(|p| p.is_alphanumeric() || p == '_'))) =>
{
let mut j = i + 1;
let mut hashes = 0;
while at(j) == Some('#') {
hashes += 1;
j += 1;
}
if at(j) != Some('"') {
out.push(c);
i += 1;
continue;
}
out.push('"');
i = j + 1;
while let Some(c) = at(i) {
if c == '"' && (1..=hashes).all(|k| at(i + k) == Some('#')) {
i += 1 + hashes;
break;
}
if c == '\n' {
out.push('\n');
}
i += 1;
}
out.push('"');
}
// a character literal (`'"'`, `'\''`), not a lifetime
('\'', Some('\\')) => {
i += 2;
while at(i).is_some_and(|c| c != '\'') {
i += 1;
}
i += 1;
out.push_str("' '");
}
('\'', Some(_)) if at(i + 2) == Some('\'') => {
i += 3;
out.push_str("' '");
}
_ => {
out.push(c);
i += 1;
}
}
}
out
}
/// The `Part`s of the change files in `dir`, in file-name order, and a module for each. A missing
/// folder is an empty list only where it may be absent (`optional`: `pending/`); any other error
/// reading it fails the build rather than leave a change out of the lists.
fn parts(dir: &Path, prefix: &str, optional: bool, out: &mut String, mods: &mut String) -> Result<(), String> {
let entries = match std::fs::read_dir(dir) {
Ok(entries) => entries,
Err(e) if optional && e.kind() == std::io::ErrorKind::NotFound => return Ok(()),
Err(e) => return Err(format!("{}: {e}", dir.display())),
};
let mut files: Vec<PathBuf> = Vec::new();
for entry in entries {
let path = entry.map_err(|e| format!("{}: {e}", dir.display()))?.path();
if path.extension().is_some_and(|x| x == "rs") {
files.push(path);
}
}
files.sort();
for path in files {
let name = path.file_stem().and_then(|s| s.to_str()).unwrap_or_default().to_string();
if name.is_empty() || !name.bytes().all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'-') {
return Err(format!("{}: name a format change file with lowercase letters, digits and dashes", path.display()));
}
let text = std::fs::read_to_string(&path).map_err(|e| format!("{}: {e}", path.display()))?;
if !text.trim_start().starts_with("//!") {
return Err(format!("{}: start a format change with a `//!` note", path.display()));
}
let module = format!("{prefix}_{}", name.replace('-', "_"));
let step = if declares_step(&text) { format!("Some({module}::step)") } else { "None".into() };
let _ = writeln!(out, " Part {{ name: {name:?}, step: {step} }},");
let _ = writeln!(mods, "#[path = {:?}]\nmod {module};", path.display().to_string());
}
Ok(())
}
#[cfg(test)]
#[path = "build_tests.rs"]
mod tests;
+115
View File
@@ -0,0 +1,115 @@
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
/// A fresh, empty folder for one test.
fn scratch(name: &str) -> PathBuf {
let dir = std::env::temp_dir().join(format!("craft-format-build-{}-{name}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).unwrap();
dir
}
fn write(path: &Path, text: &str) {
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
std::fs::write(path, text).unwrap();
}
/// The error of a run that must fail.
fn refused(dir: &Path) -> String {
match lists(dir, &[DRAWING]) {
Ok(_) => panic!("expected an error"),
Err(e) => e,
}
}
const DRAWING: Family = Family { folder: "drawing", base: 3, list_prefix: "DRAWING_" };
#[test]
fn lists_folded_and_pending_changes_with_their_modules() {
let dir = scratch("ok");
write(&dir.join("drawing/v4/add-layers.rs"), "//! Layers.\npub fn step(v: &mut Value) -> Result<(), E> { Ok(()) }\n");
write(&dir.join("drawing/v4/notes.txt"), "not a change");
write(&dir.join("drawing/v5/rename.rs"), "\n//! A rename with no step.\n");
write(&dir.join("drawing/pending/next-1.rs"), "//! Pending.\nfn step(v: &mut Value) {}\n");
std::fs::create_dir_all(dir.join("drawing/v6x")).unwrap();
write(&dir.join("drawing/v7"), "a file, not a version folder");
let lists = lists(&dir, &[DRAWING]).unwrap();
assert_eq!(lists.watch, vec![dir.join("drawing")]);
let code = &lists.code;
assert!(code.contains("const DRAWING_FOLDED: &[(u32, &[Part])] = &["), "{code}");
assert!(code.contains(" (4, &[\n Part { name: \"add-layers\", step: Some(drawing_v4_add_layers::step) },\n ]),"), "{code}");
assert!(code.contains("Part { name: \"rename\", step: None }"), "{code}");
assert!(
code.contains("const DRAWING_PENDING: &[Part] = &[\n Part { name: \"next-1\", step: Some(drawing_pending_next_1::step) },"),
"{code}"
);
assert!(code.contains("mod drawing_v5_rename;"), "{code}");
assert!(!code.contains("notes"), "{code}");
assert!(code.find("const DRAWING_PENDING").unwrap() < code.find("#[path").unwrap(), "modules follow the lists");
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn an_empty_family_has_empty_lists() {
let dir = scratch("empty");
std::fs::create_dir_all(dir.join("drawing")).unwrap();
let code = lists(&dir, &[DRAWING]).unwrap().code;
assert!(code.contains("const DRAWING_FOLDED: &[(u32, &[Part])] = &[\n];"), "{code}");
assert!(code.contains("const DRAWING_PENDING: &[Part] = &[\n];"), "{code}");
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn refuses_gaps_bad_names_missing_notes_and_missing_folders() {
let dir = scratch("bad");
assert!(refused(&dir).contains("drawing"));
write(&dir.join("drawing/v5/a.rs"), "//! a");
assert!(refused(&dir).contains("expected v4/ next, found v5/"));
std::fs::rename(dir.join("drawing/v5"), dir.join("drawing/v4")).unwrap();
write(&dir.join("drawing/v4/Bad_Name.rs"), "//! b");
assert!(refused(&dir).contains("lowercase letters"));
std::fs::remove_file(dir.join("drawing/v4/Bad_Name.rs")).unwrap();
write(&dir.join("drawing/pending/no-note.rs"), "pub fn step() {}");
assert!(refused(&dir).contains("`//!` note"));
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn a_step_is_a_declaration_not_a_mention() {
assert!(!declares_step("//! No fn step() is needed.\n"));
assert!(!declares_step("//! a note\n// fn step(v) was removed\nconst X: u8 = 1;\n"));
assert!(declares_step("//! a note\npub fn step (v: &mut Value) -> Result<(), E> { Ok(()) }\n"));
// a declaration inside a block comment, nested or not, is not one
assert!(!declares_step("//! Documentation only.\n/*\nfn step() {}\n*/\n"));
assert!(!declares_step("/* outer /* inner */\nfn step() {}\n*/\nconst X: u8 = 1;\n"));
assert!(declares_step("/* a note */\nfn step(v: &mut Value) {}\n"));
// comment markers inside strings and character literals are not comments
assert!(declares_step("//! A change with a step.\nconst MARKER: &str = \"/*\";\npub fn step(_v: &mut Value) -> Result { Ok(()) }\n"));
assert!(declares_step("const A: &str = r#\"/* \"quoted\" \"#;\nconst B: &[u8] = br\"/*\";\nfn step(v: &mut Value) {}\n"));
assert!(declares_step("const Q: char = '\"';\nconst E: char = '\\'';\nfn step(v: &mut Value) { let _: &'static str = \"\"; }\n"));
assert!(declares_step("const S: &str = \"a \\\" /* b\";\nfn step(v: &mut Value) {}\n"));
assert!(!declares_step("const S: &str = \"\nfn step(v) \";\n"), "inside a string");
// a comment between the words is whitespace
assert!(declares_step("pub fn/* migration */step(v: &mut Value) {}\n"));
// across lines
assert!(declares_step("//! A change with a step.\npub fn step\n (_v: &mut Value) -> Result {\n Ok(())\n}\n"));
assert!(declares_step("pub fn\nstep(v: &mut Value) {}\n"));
assert!(!declares_step("fn stepper(v: &mut Value) {}\nfn prefn step() {}\n"), "other names");
assert!(declares_step("const X: u8 = 1; // fn step() is below\nfn step(v: &mut Value) {}\n"));
assert!(declares_step("//! a note\npub(crate) fn step(v: &mut Value) -> Result<(), E> { Ok(()) }\n"));
assert!(declares_step("//! a note\n fn step(v: &mut Value) {}\n"));
assert!(!declares_step("//! a note\nfn stepper(v: &mut Value) {}\nfn fnstep() {}\n"));
}
#[test]
fn a_documentation_only_change_has_no_step_and_a_missing_version_folder_fails() {
let dir = scratch("doc-only");
write(&dir.join("drawing/v4/doc.rs"), "//! No fn step() is needed.\n");
let code = lists(&dir, &[DRAWING]).unwrap().code;
assert!(code.contains("Part { name: \"doc\", step: None }"), "{code}");
// a version folder that can't be listed is an error, not an empty format
let missing = parts(&dir.join("drawing/v9"), "x", false, &mut String::new(), &mut String::new());
assert!(missing.is_err());
assert!(parts(&dir.join("drawing/pending"), "x", true, &mut String::new(), &mut String::new()).is_ok());
let _ = std::fs::remove_dir_all(&dir);
}
+440
View File
@@ -0,0 +1,440 @@
//! The file container: a ZIP package of named parts (the file's JSON, binary payloads, optional
//! caches), never extracted to filesystem paths. Payloads are content-addressed blobs,
//! `blobs/<sha256>.<extension>`, stored once however often they are referred to. Every size is
//! bounded by the app's [`Limits`]: the file, each part, all parts expanded, the payloads resolved,
//! and the number of entries; a part is read no further than its declared size (a decompression
//! bomb stops there). What a part holds, and how a payload is encoded, is the app's.
use std::collections::{BTreeMap, BTreeSet};
use std::io::{Read, Seek, SeekFrom};
use std::sync::Arc;
use serde_json::Value;
use sha2::{Digest, Sha256};
use crate::FormatError;
/// How hard the payloads are compressed: Fast favours interactive file operations; Compact trades
/// CPU time for space.
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum Compression {
#[default]
Fast,
Compact,
}
/// An app's budgets for its containers. The functions are read each time they are needed, so a
/// limit the app lets the user raise takes effect at once.
#[derive(Clone, Copy)]
pub struct Limits {
/// The most entries a container holds.
pub max_entries: usize,
/// The most bytes all parts may expand to, and all payloads may resolve to.
pub max_expanded: fn() -> usize,
/// The most bytes a part of this name may expand to.
pub part_limit: fn(&str) -> usize,
/// The Deflate level of a compressed part of this name and size.
pub level: fn(&str, usize) -> i64,
/// Whether a container of this many bytes may be read, or why not.
pub check_size: fn(u64) -> Result<(), String>,
/// Whether the work is cancelled (a save the person stopped): checked before each part, and
/// a long part is written a piece at a time and stops before the next piece when this says
/// so ([`FormatError::Cancelled`]). `None`: it cannot be cancelled here.
pub cancelled: Option<fn() -> bool>,
}
/// The most bytes of a part written between two cancellation checks.
const PIECE: usize = 1 << 20;
fn invalid(message: impl std::fmt::Display) -> FormatError {
FormatError::Invalid(message.to_string())
}
/// Whether the bytes start as a container does, whatever the file is called.
pub fn is_container(bytes: &[u8]) -> bool {
bytes.starts_with(b"PK\x03\x04")
}
/// The SHA-256 of the bytes.
pub fn sha256(bytes: &[u8]) -> [u8; 32] {
Sha256::digest(bytes).into()
}
fn hex(digest: &[u8; 32]) -> String {
use std::fmt::Write as _;
let mut out = String::with_capacity(64);
for byte in digest {
// Writing to a `String` cannot fail.
let _ = write!(&mut out, "{byte:02x}");
}
out
}
/// The SHA-256 of the bytes, in lowercase hexadecimal: a blob's name, and how a cache is keyed to
/// the file it was written for.
pub fn digest(bytes: &[u8]) -> String {
hex(&sha256(bytes))
}
/// One part a [`Writer`] wrote, with the SHA-256 of its uncompressed content. A caller that signs
/// a container uses these to bind the parts without reading the file back.
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct PartDigest {
pub name: String,
pub len: u64,
pub digest: [u8; 32],
}
/// A container being written.
pub struct Writer {
zip: craft_zip::Writer,
names: BTreeSet<String>,
expanded: usize,
resolved: usize,
limits: Limits,
/// Which written parts to record for a signature, and the records so far.
tracked: Option<TrackedParts>,
}
/// A tracking closure and the part digests collected so far.
type TrackedParts = (fn(&str) -> bool, Vec<PartDigest>);
impl Writer {
pub fn new(limits: Limits) -> Writer {
Writer { zip: craft_zip::Writer::new(), names: BTreeSet::new(), expanded: 0, resolved: 0, limits, tracked: None }
}
/// Record the parts written from now on whose name `include` accepts, with their SHA-256, for
/// [`Writer::take_parts`]. Used by a caller that signs the container.
pub fn track_parts(&mut self, include: fn(&str) -> bool) {
self.tracked = Some((include, Vec::new()));
}
/// The tracked parts written so far, oldest first, and stop tracking.
pub fn take_parts(&mut self) -> Option<Vec<PartDigest>> {
self.tracked.take().map(|(_, parts)| parts)
}
/// The container's bytes.
pub fn finish(self) -> Result<Vec<u8>, FormatError> {
self.zip.finish().map_err(invalid)
}
/// Add a part, compressed or stored. A name already written is written once.
pub fn part(&mut self, name: &str, bytes: &[u8], compressed: bool) -> Result<(), FormatError> {
self.part_with_digest(name, bytes, compressed, None)
}
/// [`Writer::part`] with a digest the caller already worked out (a blob's name is its digest),
/// so a signed save does not hash the same bytes twice.
fn part_with_digest(&mut self, name: &str, bytes: &[u8], compressed: bool, digest: Option<[u8; 32]>) -> Result<(), FormatError> {
if bytes.len() > (self.limits.part_limit)(name) {
return Err(invalid("part too large"));
}
if self.names.contains(name) {
return Ok(());
}
// Check the counters this part would leave, and commit them only once it is written: a
// refused part is not remembered as written.
let expanded = self.expanded.checked_add(bytes.len()).ok_or_else(|| invalid("expanded size overflow"))?;
if expanded > (self.limits.max_expanded)() || self.names.len().saturating_add(1) > self.limits.max_entries {
return Err(invalid("package too large"));
}
let cancelled = self.limits.cancelled;
if cancelled.is_some_and(|c| c()) {
return Err(FormatError::Cancelled);
}
// the configured level, clamped to the Deflate levels craft-zip writes (1 to 9)
let compression = if compressed {
craft_zip::Compression::Level((self.limits.level)(name, bytes.len()).clamp(1, 9) as u8)
} else {
craft_zip::Compression::Stored
};
match cancelled {
Some(cancelled) if bytes.len() > PIECE => {
let mut stopped = false;
let written = self.zip.add_with(name, compression, |out| {
for piece in bytes.chunks(PIECE) {
if cancelled() {
stopped = true;
return Err(std::io::Error::other("cancelled"));
}
out.write_all(piece)?;
}
Ok(())
});
if stopped {
return Err(FormatError::Cancelled);
}
written.map_err(invalid)?;
}
_ => self.zip.add(name, bytes, compression).map_err(invalid)?,
}
self.names.insert(name.to_owned());
self.expanded = expanded;
if let Some((include, parts)) = &mut self.tracked
&& include(name)
{
let digest = digest.unwrap_or_else(|| sha256(bytes));
parts.push(PartDigest { name: name.to_owned(), len: bytes.len() as u64, digest });
}
Ok(())
}
/// Would `entries` more optional parts of `extra` bytes, stored uncompressed, still keep the
/// file within `budget` and its expanded size within the reader's budget? The parts already
/// written count at their expanded size (an upper bound on what they take in the file).
pub fn room(&self, entries: usize, extra: usize, budget: usize) -> bool {
self.remaining(entries, budget).is_some_and(|left| extra <= left)
}
/// The bytes `entries` more optional parts may hold together ([`Writer::room`]).
pub fn space(&self, entries: usize, budget: usize) -> usize {
self.remaining(entries, budget).unwrap_or(0)
}
/// The bytes left for `entries` more parts, or `None` when the entries don't fit or what is
/// written already is past `budget` or the reader's budget.
fn remaining(&self, entries: usize, budget: usize) -> Option<usize> {
let count = self.names.len().checked_add(entries).filter(|&n| n <= self.limits.max_entries)?;
let overhead = count.saturating_mul(256).saturating_add(65_536);
let in_file = budget.checked_sub(overhead.saturating_add(self.expanded))?;
let expanded = (self.limits.max_expanded)().checked_sub(self.expanded)?;
Some(in_file.min(expanded))
}
/// Add a payload as a blob, `blobs/<sha256>.<extension>`, and say its name, for the JSON that
/// refers to it.
pub fn blob(&mut self, bytes: &[u8], extension: &str, compressed: bool) -> Result<Value, FormatError> {
let resolved = self
.resolved
.checked_add(bytes.len())
.filter(|&n| n <= (self.limits.max_expanded)())
.ok_or_else(|| invalid("resolved payloads too large"))?;
let digest = sha256(bytes);
let name = format!("blobs/{}.{extension}", hex(&digest));
self.part_with_digest(&name, bytes, compressed, Some(digest))?;
self.resolved = resolved;
Ok(Value::String(name))
}
}
/// A container being read. Opening reads all bytes into memory and checks the directory
/// (the entry count, unique and plain names, every part's
/// declared size), not the parts.
pub struct Reader {
archive: craft_zip::Archive<'static>,
blobs: BTreeMap<String, Vec<u8>>,
/// The bytes of payloads resolved so far, repeats counted: bounded by the expanded budget.
pub hydrated: usize,
document: Option<Arc<[u8]>>,
limits: Limits,
/// Every byte part reads have expanded to, whatever the budget was at the time.
read_total: u64,
}
impl Reader {
pub fn new(mut source: impl Read + Seek, limits: Limits) -> Result<Self, FormatError> {
let length = source.seek(SeekFrom::End(0)).map_err(invalid)?;
(limits.check_size)(length).map_err(invalid)?;
// Read all bytes into memory for the archive, no more than the length just approved: a file
// that grew since is refused rather than read past its budget.
source.seek(SeekFrom::Start(0)).map_err(invalid)?;
let capacity = usize::try_from(length).map_err(|_| invalid("file too large"))?;
let mut bytes = Vec::new();
bytes.try_reserve_exact(capacity).map_err(|_| invalid("file too large"))?;
source.take(length.saturating_add(1)).read_to_end(&mut bytes).map_err(invalid)?;
if bytes.len() as u64 != length {
return Err(invalid("the file changed while it was read"));
}
let mut archive = craft_zip::Archive::open_owned(
bytes,
craft_zip::Limits { max_entries: limits.max_entries as u64, max_total_bytes: (limits.max_expanded)() as u64 },
)
.map_err(invalid)?;
// the directory lists distinct names, so a repeated name is caught here
if archive.has_duplicate_names() {
return Err(invalid("duplicate or invalid archive entry"));
}
// Validate entry names match format rules: no empty, no backslashes,
// no path components that are "." or "..", and no duplicates; and every part's declared
// size within its configured limit.
let mut name_set = BTreeSet::new();
for entry_idx in 0..archive.len() {
// an entry whose name can't be read would be left out of the package unnoticed
let Ok(Some(name)) = archive.entry_name(entry_idx) else {
return Err(invalid("duplicate or invalid archive entry"));
};
{
if name.is_empty()
|| name.contains('\\')
|| name.split('/').any(|c| c.is_empty() || c == "." || c == "..")
|| !name_set.insert(name.clone())
{
return Err(invalid("duplicate or invalid archive entry"));
}
if archive.declared_size(&name).is_some_and(|size| size > (limits.part_limit)(&name) as u64) {
return Err(invalid("part too large"));
}
}
}
Ok(Self { archive, blobs: BTreeMap::new(), hydrated: 0, document: None, limits, read_total: 0 })
}
/// The file's own part, `document.json`, read once and kept: provisional caches and the
/// authoritative reading are bound to the same bytes, even if another process overwrites the
/// open file in place. Blobs remain hash-checked.
pub fn document(&mut self, limit: usize) -> Result<Arc<[u8]>, FormatError> {
if self.document.is_none() {
let bytes = self.limited_part("document.json", limit)?;
self.document = Some(bytes.into());
}
// a cached document still keeps the current part limit
let limit = limit.min((self.limits.part_limit)("document.json"));
self.document.as_ref().filter(|d| d.len() <= limit).cloned().ok_or_else(|| invalid("metadata budget"))
}
/// Whether the container has a part of this name.
pub fn has_part(&self, name: &str) -> bool {
self.archive.contains(name)
}
/// The names of every part, in the container's order.
pub fn part_names(&self) -> Vec<String> {
self.archive.names().map(|n| n.into_owned()).collect()
}
/// The part of this name, at most `limit` bytes and never more than its configured limit.
pub fn limited_part(&mut self, name: &str, limit: usize) -> Result<Vec<u8>, FormatError> {
let limit = limit.min((self.limits.part_limit)(name));
let expected = self.archive.declared_size(name).ok_or_else(|| invalid("part not found"))?;
if expected > limit as u64 {
return Err(invalid("part too large"));
}
// the expanded budget is read each time: a raised (or lowered) one applies at once, the
// bytes already read still counting against it
let budget = self.archive.budget_mut();
*budget = craft_zip::Budget::new(budget.what(), (self.limits.max_expanded)() as u64);
if budget.charge(self.read_total).is_err() {
return Err(invalid("package too large"));
}
// Read within budget. craft-zip's read() handles decompression bombs via the limit.
let bytes = self.archive.read(name, limit as u64).map_err(invalid)?.ok_or_else(|| invalid("part not found"))?;
if bytes.len() as u64 != expected {
return Err(invalid("part length mismatch"));
}
self.read_total = self.read_total.saturating_add(expected);
Ok(bytes)
}
/// The blob `value`'s `field` names, of this extension, within its part limit.
pub fn blob(&mut self, value: &Value, field: &str, extension: &str) -> Result<&[u8], FormatError> {
let limit = (self.limits.part_limit)(&format!("blob.{extension}"));
self.limited_blob(value, field, extension, limit)
}
/// [`Reader::blob`] within `limit` bytes. Its hash is checked when it is first read, and every
/// reference counts against the resolved budget.
pub fn limited_blob(&mut self, value: &Value, field: &str, extension: &str, limit: usize) -> Result<&[u8], FormatError> {
let name = value.get(field).and_then(Value::as_str).ok_or_else(|| invalid(format!("missing `{field}` reference")))?;
// a cached blob still keeps the part's current limit
let limit = limit.min((self.limits.part_limit)(name));
let hash = name
.strip_prefix("blobs/")
.and_then(|n| n.strip_suffix(&format!(".{extension}")))
.filter(|h| h.len() == 64 && h.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)))
.ok_or_else(|| invalid("invalid blob reference"))?;
if !self.blobs.contains_key(name) {
let bytes = self.limited_part(name, limit)?;
if digest(&bytes) != hash {
return Err(invalid(format!("blob hash mismatch: {name}")));
}
self.blobs.insert(name.to_owned(), bytes);
}
let bytes = self.blobs.get(name).ok_or_else(|| invalid("missing blob"))?;
if bytes.len() > limit {
return Err(invalid("blob exceeds its resource budget"));
}
// Count repeated references too: a tiny archive must not expand into unbounded owned
// strings or arrays when the same blob is referenced repeatedly.
self.hydrated = self
.hydrated
.checked_add(bytes.len())
.filter(|&n| n <= (self.limits.max_expanded)())
.ok_or_else(|| invalid("resolved payloads too large"))?;
Ok(bytes)
}
}
/// A small PNG picture of the file, kept in the container as a disposable cache keyed to the
/// exact file it shows: `cache/thumbnail.json` (its key) and `cache/thumbnail.png`. Reading it
/// never resolves a payload; a picture that isn't one, or is stale, is a miss.
#[derive(Clone, Copy, Debug)]
pub struct Thumbnail {
/// The key's format tag.
pub format: &'static str,
/// The build that wrote it: only the same build reads it back.
pub app: &'static str,
/// Its width and height in pixels.
pub size: [usize; 2],
/// The most bytes it may have.
pub max_bytes: usize,
}
impl Thumbnail {
/// Whether the bytes start as a PNG of this size within the byte budget (not a decode: a
/// consumer decodes with its own dimension and allocation limits).
pub fn valid(&self, bytes: &[u8]) -> bool {
bytes.len() <= self.max_bytes
&& bytes.get(..8) == Some(b"\x89PNG\r\n\x1a\n")
&& bytes.get(8..16) == Some(b"\0\0\0\rIHDR")
&& bytes.get(16..20) == Some((self.size[0] as u32).to_be_bytes().as_slice())
&& bytes.get(20..24) == Some((self.size[1] as u32).to_be_bytes().as_slice())
}
/// Add the picture, keyed to `document` (the file's own part), when it is one and there is
/// room for it within `budget`; otherwise leave it out, never failing the save.
pub fn write(&self, writer: &mut Writer, png: &[u8], document: &[u8], budget: usize) -> Result<(), FormatError> {
if !self.valid(png) {
return Ok(());
}
let key =
serde_json::to_vec(&serde_json::json!({"format": self.format, "app": self.app, "document": digest(document), "payload": digest(png)}))
.map_err(invalid)?;
let fits = |name: &str, len: usize| len <= (writer.limits.part_limit)(name);
if !fits("cache/thumbnail.json", key.len())
|| !fits("cache/thumbnail.png", png.len())
|| !writer.room(2, key.len().saturating_add(png.len()), budget)
{
return Ok(());
}
writer.part("cache/thumbnail.json", &key, false)?;
writer.part("cache/thumbnail.png", png, false)
}
/// The picture, when this build wrote it for exactly the file's own part (`document.json`,
/// read within `document_limit`).
pub fn read(&self, reader: &mut Reader, document_limit: usize) -> Result<Vec<u8>, FormatError> {
let key: Value = serde_json::from_slice(&reader.limited_part("cache/thumbnail.json", 4096)?).map_err(invalid)?;
let document = reader.document(document_limit)?;
if key.get("format").and_then(Value::as_str) != Some(self.format)
|| key.get("app").and_then(Value::as_str) != Some(self.app)
|| key.get("document").and_then(Value::as_str) != Some(digest(&document).as_str())
{
return Err(invalid("thumbnail key"));
}
let png = reader.limited_part("cache/thumbnail.png", self.max_bytes)?;
if !self.valid(&png) || key.get("payload").and_then(Value::as_str) != Some(digest(&png).as_str()) {
return Err(invalid("thumbnail payload"));
}
Ok(png)
}
}
#[cfg(test)]
#[path = "container_tests.rs"]
mod tests;
+260
View File
@@ -0,0 +1,260 @@
use super::*;
use std::io::Cursor;
const LIMITS: Limits = Limits {
max_entries: 8,
max_expanded: || 1 << 20,
part_limit: |name| if name.ends_with(".big") { 1 << 20 } else { 1024 },
level: |_, _| 1,
check_size: |length| if length <= 1 << 20 { Ok(()) } else { Err("too large".into()) },
cancelled: None,
};
/// Parts and blobs written are read back as they were; a blob is stored once, named by its hash,
/// and checked against it.
#[test]
fn parts_and_blobs_read_back() {
let mut writer = Writer::new(LIMITS);
writer.part("document.json", br#"{"a":1}"#, true).unwrap();
let first = writer.blob(b"payload", "bin", false).unwrap();
let again = writer.blob(b"payload", "bin", true).unwrap();
assert_eq!(first, again, "the same payload is one blob");
assert_eq!(first, Value::String(format!("blobs/{}.bin", digest(b"payload"))));
let bytes = writer.finish().unwrap();
assert!(is_container(&bytes));
let mut reader = Reader::new(Cursor::new(bytes), LIMITS).unwrap();
assert_eq!(reader.part_names().len(), 2);
assert!(reader.has_part("document.json") && !reader.has_part("other.json"));
assert_eq!(&*reader.document(64).unwrap(), br#"{"a":1}"#);
let reference = serde_json::json!({"blob": first});
assert_eq!(reader.blob(&reference, "blob", "bin").unwrap(), b"payload");
assert!(reader.blob(&reference, "blob", "png").is_err(), "another extension");
assert_eq!(reader.hydrated, 7);
reader.hydrated = (LIMITS.max_expanded)();
assert!(reader.blob(&reference, "blob", "bin").unwrap_err().to_string().contains("resolved payloads too large"));
}
/// A part past its limit, or more entries than the limit, are refused as they are written.
#[test]
fn budgets_are_kept_when_writing() {
let mut writer = Writer::new(LIMITS);
assert_eq!(writer.part("small.json", &[0; 1025], false), Err(FormatError::Invalid("part too large".into())));
assert!(writer.part("large.big", &[0; 1025], true).is_ok());
for i in 0..7 {
writer.part(&format!("{i}.json"), b"x", false).unwrap();
}
assert_eq!(writer.part("one-more.json", b"x", false), Err(FormatError::Invalid("package too large".into())));
assert!(!writer.room(1, 0, 1 << 20), "no room for an entry past the limit");
}
/// A thumbnail is kept only for the file it shows.
#[test]
fn a_thumbnail_is_keyed_to_its_file() {
let picture = Thumbnail { format: "app-thumbnail/1", app: "1.0", size: [2, 1], max_bytes: 1024 };
let mut png = b"\x89PNG\r\n\x1a\n\0\0\0\rIHDR".to_vec();
png.extend_from_slice(&2u32.to_be_bytes());
png.extend_from_slice(&1u32.to_be_bytes());
assert!(picture.valid(&png));
let mut writer = Writer::new(LIMITS);
writer.part("document.json", b"{}", false).unwrap();
picture.write(&mut writer, &png, b"{}", 8 << 20).unwrap();
let mut reader = Reader::new(Cursor::new(writer.finish().unwrap()), LIMITS).unwrap();
assert_eq!(picture.read(&mut reader, 64).unwrap(), png);
let mut writer = Writer::new(LIMITS);
writer.part("document.json", b"{\"changed\":1}", false).unwrap();
picture.write(&mut writer, &png, b"{}", 8 << 20).unwrap();
let mut reader = Reader::new(Cursor::new(writer.finish().unwrap()), LIMITS).unwrap();
assert!(picture.read(&mut reader, 64).is_err(), "stale");
}
/// A refused part is not remembered as written: once the budget is raised it is written.
#[test]
fn a_refused_part_can_be_written_later() {
static BUDGET: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(4);
let limits = Limits { max_expanded: || BUDGET.load(std::sync::atomic::Ordering::SeqCst), ..LIMITS };
let mut writer = Writer::new(limits);
assert_eq!(writer.part("a.json", b"12345678", false), Err(FormatError::Invalid("package too large".into())));
assert!(writer.blob(b"12345678", "bin", false).is_err());
BUDGET.store(1 << 20, std::sync::atomic::Ordering::SeqCst);
writer.part("a.json", b"12345678", false).unwrap();
writer.blob(b"12345678", "bin", false).unwrap();
let reader = Reader::new(Cursor::new(writer.finish().unwrap()), limits).unwrap();
assert_eq!(reader.part_names().len(), 2, "both parts were written on the retry");
}
/// A cancelled save writes no more parts, whether the part is small or written in pieces.
#[test]
fn cancellation_stops_the_save() {
let cancelled = Limits { cancelled: Some(|| true), ..LIMITS };
let mut writer = Writer::new(cancelled);
assert_eq!(writer.part("a.json", b"a", false), Err(FormatError::Cancelled));
// cancelled after the first piece of a long part: checks before the part, then per piece
static CHECKS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
let midway = Limits {
cancelled: Some(|| CHECKS.fetch_add(1, std::sync::atomic::Ordering::SeqCst) >= 2),
max_expanded: || 1 << 22,
part_limit: |_| 1 << 22,
..LIMITS
};
let mut writer = Writer::new(midway);
assert_eq!(writer.part("big.big", &vec![0; PIECE + 1], true), Err(FormatError::Cancelled));
assert_eq!(CHECKS.load(std::sync::atomic::Ordering::SeqCst), 3);
let going = Limits { cancelled: Some(|| false), max_expanded: || 1 << 22, part_limit: |_| 1 << 22, ..LIMITS };
let mut writer = Writer::new(going);
writer.part("big.big", &vec![7; PIECE + 1], true).unwrap();
let mut reader = Reader::new(Cursor::new(writer.finish().unwrap()), going).unwrap();
assert_eq!(reader.limited_part("big.big", 1 << 22).unwrap().len(), PIECE + 1);
}
/// Room counts what is already written, and the reader keeps every part's configured limit.
#[test]
fn room_counts_written_parts_and_reads_keep_part_limits() {
let mut writer = Writer::new(LIMITS);
writer.part("large.big", &[0; 500 * 1024], false).unwrap();
assert!(!writer.room(1, 200 * 1024, 600 * 1024));
assert!(writer.room(1, 1024, 1 << 20));
// what is written already is past the budget: not even an empty part fits
assert!(!writer.room(1, 0, 400 * 1024));
assert_eq!(writer.space(1, 400 * 1024), 0);
let wide = Limits { part_limit: |_| 1 << 20, ..LIMITS };
let mut writer = Writer::new(wide);
writer.part("document.json", &[b' '; 2048], false).unwrap();
let bytes = writer.finish().unwrap();
assert!(Reader::new(Cursor::new(bytes.clone()), LIMITS).is_err(), "declared past its configured limit");
let mut reader = Reader::new(Cursor::new(bytes), wide).unwrap();
assert!(reader.limited_part("document.json", 4096).is_ok());
}
/// A thumbnail past a cache part's own limit is left out, never failing the save.
#[test]
fn a_thumbnail_past_its_part_limit_is_left_out() {
let picture = Thumbnail { format: "app-thumbnail/1", app: "1.0", size: [2, 1], max_bytes: 2048 };
let mut png = b"\x89PNG\r\n\x1a\n\0\0\0\rIHDR".to_vec();
png.extend_from_slice(&2u32.to_be_bytes());
png.extend_from_slice(&1u32.to_be_bytes());
png.resize(1500, 0);
let mut writer = Writer::new(LIMITS);
picture.write(&mut writer, &png, b"{}", 8 << 20).unwrap();
let reader = Reader::new(Cursor::new(writer.finish().unwrap()), LIMITS).unwrap();
assert!(!reader.has_part("cache/thumbnail.json") && !reader.has_part("cache/thumbnail.png"));
}
/// A source that grows after its length was approved is refused.
#[test]
fn a_source_that_grows_while_read_is_refused() {
struct Growing(Cursor<Vec<u8>>, bool);
impl Read for Growing {
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
if !self.1 {
self.1 = true;
let end = self.0.get_ref().len();
self.0.get_mut().extend_from_slice(&[0; 16]);
self.0.set_position(self.0.position().min(end as u64));
}
self.0.read(buf)
}
}
impl Seek for Growing {
fn seek(&mut self, to: SeekFrom) -> std::io::Result<u64> {
self.0.seek(to)
}
}
let mut writer = Writer::new(LIMITS);
writer.part("document.json", b"{}", false).unwrap();
let bytes = writer.finish().unwrap();
assert!(Reader::new(Growing(Cursor::new(bytes), false), LIMITS).err().is_some_and(|e| e.to_string().contains("changed")));
}
/// Compressed parts use the configured Deflate level.
#[test]
fn compressed_parts_use_the_configured_level() {
// words in a varied order, so the levels compress them differently
let words = ["alpha ", "beta ", "gamma ", "delta ", "epsilon ", "zeta ", "eta ", "theta "];
let mut seed = 7u32;
let text: Vec<u8> = (0..150)
.flat_map(|_| {
seed = seed.wrapping_mul(1_103_515_245).wrapping_add(12_345);
words[(seed >> 16) as usize % words.len()].bytes()
})
.take(1000)
.collect();
let size = |level: fn(&str, usize) -> i64| {
let mut writer = Writer::new(Limits { level, ..LIMITS });
writer.part("document.json", &text, true).unwrap();
writer.finish().unwrap().len()
};
assert_ne!(size(|_, _| 1), size(|_, _| 9), "the level is used");
assert_eq!(size(|_, _| 42), size(|_, _| 9), "past 9 is clamped");
assert_eq!(size(|_, _| -3), size(|_, _| 1), "below 1 is clamped");
}
/// An archive naming one part twice is refused.
#[test]
fn a_repeated_part_name_is_refused() {
let mut writer = Writer::new(LIMITS);
writer.part("document.json", b"{}", false).unwrap();
writer.part("documenX.json", b"[]", false).unwrap();
let mut bytes = writer.finish().unwrap();
let (from, to) = (b"documenX.json", b"document.json");
let mut at = 0;
while let Some(i) = bytes[at..].windows(from.len()).position(|w| w == from) {
bytes[at + i..at + i + to.len()].copy_from_slice(to);
at += i + to.len();
}
assert!(Reader::new(Cursor::new(bytes), LIMITS).is_err());
}
/// Cached documents and blobs keep the current part limits.
#[test]
fn cached_parts_keep_the_current_part_limits() {
let wide = Limits { part_limit: |_| 1 << 20, ..LIMITS };
let mut writer = Writer::new(wide);
writer.part("document.json", &[b' '; 512], false).unwrap();
let blob = writer.blob(&[7; 512], "bin", false).unwrap();
let bytes = writer.finish().unwrap();
let mut reader = Reader::new(Cursor::new(bytes), wide).unwrap();
let reference = serde_json::json!({"blob": blob});
assert!(reader.document(4096).is_ok());
assert!(reader.blob(&reference, "blob", "bin").is_ok());
reader.limits = Limits { part_limit: |_| 128, ..wide };
assert!(reader.document(4096).is_err());
assert!(reader.limited_blob(&reference, "blob", "bin", 4096).is_err());
}
/// A raised expanded budget applies to the open reader, and what was read still counts.
#[test]
fn a_raised_expanded_budget_applies_at_once() {
static BUDGET: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(600);
let limits = Limits { max_expanded: || BUDGET.load(std::sync::atomic::Ordering::Relaxed), part_limit: |_| 1 << 20, ..LIMITS };
let wide = Limits { max_expanded: || 1 << 20, ..limits };
let mut writer = Writer::new(wide);
writer.part("document.json", &[b' '; 256], false).unwrap();
writer.part("other.big", &[1; 512], false).unwrap();
let bytes = writer.finish().unwrap();
BUDGET.store(1 << 20, std::sync::atomic::Ordering::Relaxed);
let mut reader = Reader::new(Cursor::new(bytes), limits).unwrap();
BUDGET.store(600, std::sync::atomic::Ordering::Relaxed);
assert!(reader.limited_part("document.json", 4096).is_ok());
assert!(reader.limited_part("other.big", 4096).is_err(), "256 read + 512 is past 600");
BUDGET.store(1024, std::sync::atomic::Ordering::Relaxed);
assert!(reader.limited_part("other.big", 4096).is_ok(), "768 fits the raised 1024");
// a lowered budget does not forget what was read: 768 + 256 is past 1000 after a dip to 100
BUDGET.store(100, std::sync::atomic::Ordering::Relaxed);
assert!(reader.limited_part("document.json", 4096).is_err());
BUDGET.store(1000, std::sync::atomic::Ordering::Relaxed);
assert!(reader.limited_part("document.json", 4096).is_err(), "1024 is past 1000");
BUDGET.store(1024, std::sync::atomic::Ordering::Relaxed);
assert!(reader.limited_part("document.json", 4096).is_ok());
}
/// An entry whose local header can't be read is refused at open, not skipped.
#[test]
fn an_unreadable_entry_is_refused() {
let mut writer = Writer::new(LIMITS);
writer.part("document.json", b"{}", false).unwrap();
writer.part("other.json", b"[]", false).unwrap();
let mut bytes = writer.finish().unwrap();
let second = bytes.windows(4).enumerate().filter(|(_, w)| *w == b"PK\x03\x04").map(|(i, _)| i).nth(1).unwrap();
bytes[second + 2] = 0;
assert!(Reader::new(Cursor::new(bytes), LIMITS).is_err());
}
+257
View File
@@ -0,0 +1,257 @@
//! Versioned file formats: what an app needs to read every file it ever wrote, and to open a file
//! a newer build of it wrote.
//!
//! A file starts with `"format": "<prefix>N"`. A [`Family`] is one kind of file (an app's designs,
//! its drawings): its tag, its own numbers, and how a file of an older number is brought up to
//! this build's, step by step (N → N+1 → … → current). The app's code only knows the current
//! format; all translation lives in the steps:
//! - the steps written out by hand for the first formats, one per format ([`Formats::hand`]);
//! - then one folder per format, `v<N>/`, one file per change in it ([`Part`]), each with a step
//! when files from before the change are rewritten as they open ([`Formats::folded`]);
//! - then the changes no release has a number for yet, in `pending/` ([`Formats::pending`]):
//! while there are any, the build writes the next number, with their names in the file's
//! `format_pending`, so a later build runs just the steps a file is missing. A release moves
//! them into the next `v<N>/`. [`build`] lists the folders for a build script.
//!
//! A file a newer build wrote says so by a number past this build's or a change name this build
//! doesn't have ([`Family::newer`]); the app then reads what it can and keeps the rest
//! ([`newer`]). A file is saved as JSON, or in a [`container`] with its binary payloads.
//!
//! Steps are JSON-level (the file's shape) and return the app's own error type; every error this
//! crate makes is a [`FormatError`], which the app's error type takes `From`.
#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))]
use serde_json::Value;
pub mod build;
pub mod container;
pub mod newer;
/// Why a file is refused: its text says what is wrong, as the user reads it.
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum FormatError {
/// The file isn't one of the family's, or it is damaged.
Invalid(String),
/// The work was cancelled (a save the person stopped): nothing more was written.
Cancelled,
}
impl std::fmt::Display for FormatError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
FormatError::Invalid(why) => f.write_str(why),
FormatError::Cancelled => f.write_str("cancelled"),
}
}
}
impl std::error::Error for FormatError {}
/// One JSON-level upgrade step: rewrites a file in place, or says why it can't.
pub type Step<E> = fn(&mut Value) -> Result<(), E>;
/// One change to a family's file format: a file in its `v<N>/` or `pending/` folder.
pub struct Part<E> {
/// The file's name, which a file written while the change was pending lists.
pub name: &'static str,
/// How a file from before the change is rewritten as it opens, if it is.
pub step: Option<Step<E>>,
}
/// Another spelling of a change name that a file of one format may carry (a build that named the
/// change otherwise): read as the change it stands for, never written.
#[derive(Clone, Copy, Debug)]
pub struct Alias {
/// The format whose files may say it.
pub format: u32,
/// The spelling in the file.
pub name: &'static str,
/// The change it stands for.
pub known_as: &'static str,
}
/// A family's upgrade path: the steps written out by hand, the folded formats after them (each a
/// list of changes) and the pending changes, which make the next format.
pub struct Formats<'a, E> {
/// Entry `i` turns format `i + 1` into `i + 2`.
pub hand: &'a [Step<E>],
/// Each format after the hand-written ones, oldest first, and its changes.
pub folded: &'a [(u32, &'a [Part<E>])],
/// The changes no release has a number for yet.
pub pending: &'a [Part<E>],
}
/// A kind of file whose format is versioned: its own format tag, its own numbers and its own
/// changes. Reading a file's format, telling a file a newer build saved and the upgrade are the
/// same for every family.
pub struct Family<'a, E> {
/// The app, as messages name it.
pub app: &'static str,
/// What a file of the family is called in messages.
pub noun: &'static str,
/// The tag before the number in a file's `format`.
pub prefix: &'static str,
/// The tag files saved under an older name carry: read, never written.
pub legacy_prefix: Option<&'static str>,
/// The format of a file that names none, if such files exist.
pub untagged: Option<u32>,
/// Other spellings of change names that files may carry.
pub aliases: &'a [Alias],
/// The upgrade path.
pub formats: Formats<'a, E>,
}
/// How a file shows a newer build saved it ([`Family::newer`]).
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct Newer {
/// The number the file says.
pub format: u32,
/// The number this build reads up to.
pub reads_up_to: u32,
/// The changes the file names that this build doesn't have.
pub unknown_changes: Vec<String>,
}
impl<E> Formats<'_, E> {
/// The format this build writes: the last folded one, or the next while changes are pending.
pub const fn current(&self) -> u32 {
self.hand.len() as u32 + 1 + self.folded.len() as u32 + if self.pending.is_empty() { 0 } else { 1 }
}
/// The changes format `version` is made of, if it is made of changes.
fn parts(&self, version: u32) -> Option<&[Part<E>]> {
let after = version.checked_sub(self.hand.len() as u32 + 2)? as usize;
match self.folded.get(after) {
Some((_, parts)) => Some(parts),
None if after == self.folded.len() && !self.pending.is_empty() => Some(self.pending),
None => None,
}
}
/// Whether a change of this name is anywhere in this build's history.
fn knows_part_name(&self, name: &str) -> bool {
self.folded.iter().flat_map(|(_, parts)| *parts).chain(self.pending).any(|part| part.name == name)
}
}
impl<E> Family<'_, E> {
/// The format this build writes: the last folded one, or the next while changes are pending.
pub const fn current(&self) -> u32 {
self.formats.current()
}
/// What a file of the family is called in messages.
pub const fn noun(&self) -> &'static str {
self.noun
}
/// The names of the pending changes this build writes into a file (`format_pending`).
pub fn pending_names(&self) -> Vec<String> {
self.formats.pending.iter().map(|p| p.name.to_string()).collect()
}
/// The format tag of format `n`.
pub fn format_string(&self, n: u32) -> String {
format!("{}{n}", self.prefix)
}
/// The version in one of this family's format tags; `None` for any other tag, another
/// family's included.
pub fn version_of(&self, s: &str) -> Option<u32> {
// each tag form parses on its own: a legacy prefix may start with the primary one
// (`app/` and `app/design/`)
let parse = |prefix: &str| s.strip_prefix(prefix).and_then(|n| n.parse().ok());
parse(self.prefix).or_else(|| self.legacy_prefix.and_then(parse))
}
/// The change a name a file of format `version` lists stands for: its alias's, or itself.
/// An alias counts only while this build has the change it stands for, so recognising it
/// never grants support to another reader.
fn canonical_applied_name<'name>(&self, version: u32, name: &'name str) -> &'name str {
match self.aliases.iter().find(|a| a.format == version && a.name == name) {
Some(alias) if self.formats.knows_part_name(alias.known_as) => alias.known_as,
_ => name,
}
}
/// What says a file of format `version` with the changes `applied` came from a newer build:
/// a number past this build's, or a change this build doesn't have anywhere in its history
/// (a branch's pending change may have landed at a later number). A list beside a format that
/// isn't made of changes means nothing (a tool or a test set an older number on a newer file).
/// `None` for a file this build fully knows.
pub fn newer(&self, version: u32, applied: Option<&[String]>) -> Option<Newer> {
let current = self.current();
let listed = applied.filter(|_| version > current || self.formats.parts(version).is_some()).unwrap_or_default();
let unknown_changes: Vec<String> =
listed.iter().filter(|name| !self.formats.knows_part_name(self.canonical_applied_name(version, name))).cloned().collect();
(version > current || !unknown_changes.is_empty()).then_some(Newer { format: version, reads_up_to: current, unknown_changes })
}
}
impl<E: From<FormatError>> Family<'_, E> {
/// Read a file's format and take its `format_pending` out of its JSON, before anything else
/// in it is read: a file that isn't this family's is refused here. One a newer build saved is
/// not: [`Family::newer`] says so.
pub fn take_version(&self, v: &mut Value) -> Result<(u32, Option<Vec<String>>), E> {
let (app, noun) = (self.app, self.noun);
let not_ours = |what: String| E::from(FormatError::Invalid(format!("not a {app} {noun} ({what})")));
let Some(obj) = v.as_object() else { return Err(not_ours("expected a JSON object".into())) };
let version = match obj.get("format") {
None => self.untagged.ok_or_else(|| not_ours("no format".into()))?,
Some(Value::String(f)) => self.version_of(f).ok_or_else(|| not_ours(format!("format `{f}`")))?,
Some(_) => return Err(not_ours("format is not a string".into())),
};
let applied = take_names(v, "format_pending", noun).map_err(E::from)?;
if version == 0 {
return Err(not_ours("format 0".into()));
}
Ok((version, applied))
}
/// Bring a file of format `version` up to this build's; `applied` is its `format_pending`
/// (both from [`Family::take_version`]): the changes of its format it already has, when it
/// was written while they were pending. The others' steps run first, and already-applied
/// changes never run again at their later number. A file past this build's number has no
/// steps to run.
pub fn upgrade(&self, v: &mut Value, version: u32, applied: Option<&[String]>) -> Result<(), E> {
let formats = &self.formats;
if version == 0 {
return Err(E::from(FormatError::Invalid(format!("not a {} {} (format 0)", self.app, self.noun))));
}
// A list beside a format that isn't made of changes is ignored, like any field an upgrade
// drops.
let applied = applied.filter(|_| formats.parts(version).is_some());
let already_applied =
|part: &Part<E>| applied.is_some_and(|names| names.iter().any(|name| self.canonical_applied_name(version, name) == part.name));
if applied.is_some() {
for part in formats.parts(version).unwrap_or_default().iter().filter(|part| !already_applied(part)) {
part.step.map_or(Ok(()), |step| step(v))?;
}
}
for from in version..self.current() {
match formats.hand.get(from as usize - 1) {
Some(step) => step(v)?,
None => {
for part in formats.parts(from + 1).unwrap_or_default().iter().filter(|part| !already_applied(part)) {
part.step.map_or(Ok(()), |step| step(v))?;
}
}
}
}
Ok(())
}
}
/// Take a short list of names out of a file's JSON: its `format_pending` (the pending changes it
/// has).
fn take_names(v: &mut Value, key: &str, noun: &str) -> Result<Option<Vec<String>>, FormatError> {
let Some(list) = v.as_object_mut().and_then(|o| o.remove(key)) else { return Ok(None) };
let invalid = || FormatError::Invalid(format!("damaged {noun}: `{key}` is not a short list of names"));
let list = list.as_array().filter(|l| l.len() <= 1000).ok_or_else(invalid)?;
list.iter().map(|n| n.as_str().filter(|n| n.len() <= 128).map(str::to_string).ok_or_else(invalid)).collect::<Result<_, _>>().map(Some)
}
#[cfg(test)]
#[path = "lib_tests.rs"]
mod tests;
+142
View File
@@ -0,0 +1,142 @@
use super::*;
use serde_json::json;
/// The app's error, as an app has one.
#[derive(Debug, PartialEq)]
struct AppError(String);
impl From<FormatError> for AppError {
fn from(e: FormatError) -> Self {
AppError(e.to_string())
}
}
/// Each step appends its name to the file's `log`, so the order steps ran in can be read back.
fn log(v: &mut Value, step: &str) {
if let Some(list) = v.get_mut("log").and_then(Value::as_array_mut) {
list.push(json!(step));
}
}
const HAND: [Step<AppError>; 2] = [
|v| {
log(v, "1to2");
Ok(())
},
|v| {
log(v, "2to3");
Ok(())
},
];
const V4: [Part<AppError>; 2] = [
Part {
name: "a",
step: Some(|v| {
log(v, "a");
Ok(())
}),
},
Part { name: "b", step: None },
];
const V5: [Part<AppError>; 1] = [Part {
name: "c",
step: Some(|v| {
log(v, "c");
Ok(())
}),
}];
const FOLDED: [(u32, &[Part<AppError>]); 2] = [(4, &V4), (5, &V5)];
const PENDING: [Part<AppError>; 2] = [
Part {
name: "d",
step: Some(|v| {
log(v, "d");
Ok(())
}),
},
Part {
name: "e",
step: Some(|v| {
log(v, "e");
Ok(())
}),
},
];
const ALIASES: [Alias; 1] = [Alias { format: 5, name: "old-c", known_as: "c" }];
/// Format 1 untagged; hand steps to 3; folded 4 and 5; pending changes make 6.
const FAMILY: Family<'static, AppError> = Family {
app: "App",
noun: "design",
prefix: "app/",
legacy_prefix: Some("old/"),
untagged: Some(1),
aliases: &ALIASES,
formats: Formats { hand: &HAND, folded: &FOLDED, pending: &PENDING },
};
fn steps_run(version: u32, applied: Option<&[String]>) -> Vec<String> {
let mut v = json!({"log": []});
FAMILY.upgrade(&mut v, version, applied).unwrap();
v["log"].as_array().unwrap().iter().map(|s| s.as_str().unwrap().to_string()).collect()
}
#[test]
fn tags_and_numbers() {
assert_eq!(FAMILY.current(), 6);
assert_eq!(FAMILY.format_string(6), "app/6");
assert_eq!(FAMILY.version_of("app/4"), Some(4));
assert_eq!(FAMILY.version_of("old/3"), Some(3), "the legacy tag is read");
assert_eq!(FAMILY.version_of("other/3"), None);
assert_eq!(FAMILY.pending_names(), vec!["d".to_string(), "e".to_string()]);
}
#[test]
fn a_file_is_taken_apart_or_refused_with_the_apps_words() {
let mut v = json!({"format": "app/6", "format_pending": ["d"], "x": 1});
assert_eq!(FAMILY.take_version(&mut v), Ok((6, Some(vec!["d".to_string()]))));
assert!(v.get("format_pending").is_none(), "taken out");
assert_eq!(FAMILY.take_version(&mut json!({})), Ok((1, None)), "untagged");
let refused = |mut v: Value| FAMILY.take_version(&mut v).unwrap_err().0;
assert_eq!(refused(json!([])), "not a App design (expected a JSON object)");
assert_eq!(refused(json!({"format": "other/2"})), "not a App design (format `other/2`)");
assert_eq!(refused(json!({"format": 2})), "not a App design (format is not a string)");
assert_eq!(refused(json!({"format": "app/0"})), "not a App design (format 0)");
assert_eq!(refused(json!({"format": "app/2", "format_pending": [1]})), "damaged design: `format_pending` is not a short list of names");
}
#[test]
fn every_step_runs_once_in_order() {
assert_eq!(steps_run(1, None), ["1to2", "2to3", "a", "c", "d", "e"]);
assert_eq!(steps_run(4, None), ["c", "d", "e"]);
assert_eq!(steps_run(6, None), Vec::<String>::new());
// A file written while `d` was pending has it: only `e` runs.
assert_eq!(steps_run(6, Some(&["d".to_string()])), ["e"]);
// A file of format 5 written while `c` was still pending, without it: `c` runs first.
assert_eq!(steps_run(5, Some(&[])), ["c", "d", "e"]);
// Under its other spelling `c` counts as had.
assert_eq!(steps_run(5, Some(&["old-c".to_string()])), ["d", "e"]);
// A list beside a hand-written format means nothing.
assert_eq!(steps_run(2, Some(&["d".to_string()])), ["2to3", "a", "c", "d", "e"]);
let mut v = json!({});
assert_eq!(FAMILY.upgrade(&mut v, 0, None), Err(AppError("not a App design (format 0)".into())));
}
#[test]
fn a_newer_file_is_told_by_its_number_or_an_unknown_change() {
assert_eq!(FAMILY.newer(6, None), None);
assert_eq!(FAMILY.newer(6, Some(&["d".to_string()])), None);
assert_eq!(FAMILY.newer(7, None), Some(Newer { format: 7, reads_up_to: 6, unknown_changes: Vec::new() }));
assert_eq!(FAMILY.newer(6, Some(&["z".to_string()])), Some(Newer { format: 6, reads_up_to: 6, unknown_changes: vec!["z".to_string()] }));
assert_eq!(FAMILY.newer(5, Some(&["old-c".to_string()])), None, "an alias of a known change");
assert_eq!(FAMILY.newer(4, Some(&["old-c".to_string()])), Some(Newer { format: 4, reads_up_to: 6, unknown_changes: vec!["old-c".to_string()] }));
assert_eq!(FAMILY.newer(2, Some(&["z".to_string()])), None, "a list beside a hand-written format means nothing");
}
#[test]
fn a_legacy_prefix_that_extends_the_primary_one_is_read() {
let nested = Family { prefix: "app/", legacy_prefix: Some("app/design/"), ..FAMILY };
assert_eq!(nested.version_of("app/design/3"), Some(3));
assert_eq!(nested.version_of("app/4"), Some(4));
assert_eq!(nested.version_of("app/design/x"), None);
}
+470
View File
@@ -0,0 +1,470 @@
//! A file a newer build saved: it opens, with a warning, and saving it loses nothing.
//!
//! A newer format number, or a change the file names that this build doesn't have, says a newer
//! build saved the file ([`crate::Family::newer`]). It may use nothing this build lacks, so the app
//! reads it as usual. When part of it can't be read, that part stays out of the way and the rest
//! opens: the app reads the elements of its lists it can ([`NewerFile::keep_element`]), then any
//! other top-level section it can't read is left out whole ([`leave_out_unreadable`],
//! [`read_around`]), and a field it doesn't know, which typed reading would drop, is noticed by
//! comparing the file with what was read ([`NewerFile::compare`]).
//!
//! All of these are named in [`NewerFile::not_understood`] and go back into the file when it is
//! saved ([`NewerFile::restore_into`]), as they came. What can't be put back because the file was
//! changed around it is what `restore_into` returns.
use std::collections::{BTreeMap, HashMap};
use std::sync::Arc;
use serde_json::{Map, Value, json};
use crate::Newer;
/// The most sections left out one at a time to find the one a file can't be read with.
const MAX_SECTION_TRIES: usize = 64;
/// What the file says and holds that this build doesn't read.
#[derive(Clone, Debug, Default, PartialEq)]
pub struct NewerFile {
/// The app, as the notice names it.
pub app: &'static str,
/// The number the file says.
pub format: u32,
/// The number this build reads up to.
pub reads_up_to: u32,
/// The changes the file names that this build doesn't have.
pub unknown_changes: Vec<String>,
/// What this build couldn't read, by name.
pub not_understood: Vec<String>,
/// Fields known records lost in typed reading.
kept: Option<Kept>,
/// Top-level sections left out whole.
sections: Vec<Section>,
/// Elements left out of a top-level list: its key and the element.
elements: Vec<(String, Value)>,
/// The parts of a container this build has no use for: name and bytes. A blob only one of the
/// kept pieces refers to is one of them.
pub parts: Vec<(String, Arc<[u8]>)>,
/// The file's `container.json`, when this build doesn't know what it says.
pub container: Option<Value>,
}
#[derive(Clone, Debug, PartialEq)]
struct Section {
key: String,
value: Value,
/// This build's record of the section, without the file's: while it is unchanged, the file's
/// goes back.
empty: Option<Value>,
}
/// The fields of a record that typed reading dropped.
#[derive(Clone, Debug, PartialEq)]
enum Kept {
Field(Value),
Fields(BTreeMap<String, Kept>),
Items(Vec<(Anchor, Kept)>),
}
/// Which element of a list some kept fields belong to.
#[derive(Clone, Debug, PartialEq)]
enum Anchor {
Id(u64),
Name(String),
/// Its place, good only while the list keeps its length.
Index {
at: usize,
of: usize,
},
}
impl NewerFile {
/// A file of `app` that `newer` says a newer build saved, nothing left out of it yet.
pub fn new(app: &'static str, newer: &Newer) -> NewerFile {
NewerFile {
app,
format: newer.format,
reads_up_to: newer.reads_up_to,
unknown_changes: newer.unknown_changes.clone(),
..NewerFile::default()
}
}
/// What to tell the user, whatever else is said.
pub fn notice(&self) -> String {
let (app, reads, format) = (self.app, self.reads_up_to, self.format);
match self.unknown_changes.first() {
Some(change) if format <= reads => {
format!("Saved by a newer {app} (format {format} with `{change}`, which this version doesn't know). If something's missing, update.")
}
_ => format!("Saved by a newer {app} (format {format}, this version reads up to {reads}). If something's missing, update."),
}
}
/// Every line to report: the notice, then what couldn't be read, by name.
pub fn warnings(&self) -> Vec<String> {
let unread = self.not_understood.iter().map(|what| format!("Not read: {what}. Saving keeps it."));
std::iter::once(self.notice()).chain(unread).collect()
}
/// What a reply says about it.
pub fn report(&self) -> Value {
json!({
"format": self.format,
"reads_up_to": self.reads_up_to,
"unknown_changes": self.unknown_changes,
"not_understood": self.not_understood,
"notice": self.notice(),
})
}
/// Whether the next save needs a container to keep what the file holds: plain JSON has no room
/// for parts beside the file's own, nor for the container's own `container.json`.
pub fn needs_container(&self) -> bool {
!self.parts.is_empty() || self.container.is_some()
}
/// Whether anything kept back names this text (a blob's name in a container).
pub fn mentions(&self, text: &str) -> bool {
fn kept(k: &Kept, text: &str) -> bool {
match k {
Kept::Field(v) => value_mentions(v, text),
Kept::Fields(m) => m.values().any(|x| kept(x, text)),
Kept::Items(l) => l.iter().any(|(_, x)| kept(x, text)),
}
}
self.kept.as_ref().is_some_and(|k| kept(k, text))
|| self.sections.iter().any(|s| value_mentions(&s.value, text))
|| self.elements.iter().any(|(_, v)| value_mentions(v, text))
}
/// Say that a section the reader couldn't finish reading is left out. `empty` is what the
/// app writes for the section while it holds nothing, if anything.
pub fn keep_section(&mut self, key: &str, value: Value, empty: Option<Value>, why: &str) {
self.not_understood.push(format!("section `{key}` ({why})"));
self.sections.push(Section { key: key.into(), value, empty });
}
/// Say that an element of the top-level list `key` is left out, as `note` names it; it goes
/// back at the end of the list when the file is saved.
pub fn keep_element(&mut self, key: &str, element: Value, note: String) {
self.not_understood.push(note);
self.elements.push((key.into(), element));
}
/// Compare the file as it reads (`file`, after the same upgrade as what was read) with what was
/// read of it (`known`, its typed record written out): what `known` lacks is kept and named.
/// An object whose `type` is `placeholder_tag` (an element the app kept whole) is not
/// compared; nor are these top-level fields: those in `container_only` (what a container adds
/// beside the file's own), `format` and `format_pending`, and the sections kept whole (they go
/// back whole).
pub fn compare(&mut self, file: &Value, known: &Value, placeholder_tag: &str, container_only: &[&str]) {
let mut paths = Vec::new();
let mut top_only: Vec<&str> = container_only.to_vec();
top_only.extend(["format", "format_pending"]);
top_only.extend(self.sections.iter().map(|s| s.key.as_str()));
// a list typed writing leaves out holds only elements kept on their own: not kept twice
top_only.extend(self.elements.iter().map(|(key, _)| key.as_str()).filter(|key| known.get(*key).is_none()));
let kept = vanished(file, known, "", &mut paths, placeholder_tag, &top_only);
self.kept = kept;
self.not_understood.extend(paths);
for section in &mut self.sections {
section.empty = known.get(&section.key).cloned();
}
}
/// Write back what was kept of the file into `value`, the file as this build writes it.
/// Returns what couldn't go back, because the file was changed around it.
pub fn restore_into(&self, value: &mut Value) -> Vec<String> {
let mut lost = Vec::new();
if let Some(kept) = &self.kept {
apply(kept, value, "", &mut lost);
}
if let Some(top) = value.as_object_mut() {
for section in &self.sections {
if top.get(&section.key) == section.empty.as_ref() {
top.insert(section.key.clone(), section.value.clone());
} else {
lost.push(format!("section `{}`, replaced by what was edited here", section.key));
}
}
// each list's records indexed once, by the identity records are matched by
let mut indexes: HashMap<&str, Positions> = HashMap::new();
for (key, element) in &self.elements {
let name = element.get("name");
let anchor = anchor_of(element);
match top.entry(key.clone()).or_insert_with(|| json!([])) {
Value::Array(list) => {
let index = indexes.entry(key.as_str()).or_insert_with(|| Positions::of(list));
if anchor.as_ref().is_some_and(|a| index.find(list.len(), a).is_some()) {
lost.push(format!(
"`{key}` element {}, replaced by what was made here",
name.and_then(Value::as_str).unwrap_or("unnamed")
));
} else {
index.add(element, list.len());
list.push(element.clone());
}
}
_ => lost.push(format!("`{key}` element {}, replaced by what was made here", name.and_then(Value::as_str).unwrap_or("unnamed"))),
}
}
}
lost
}
}
/// Whether a string anywhere in `v` is `text`; iterative, so no nesting depth overflows the stack.
fn value_mentions(v: &Value, text: &str) -> bool {
let mut todo = vec![v];
while let Some(v) = todo.pop() {
match v {
Value::String(s) if s == text => return true,
Value::Array(a) => todo.extend(a),
Value::Object(o) => todo.extend(o.values()),
_ => {}
}
}
false
}
/// The first line of a reader's error, without the list of every kind it knows, and without the
/// `"<noun> file: "` its reader starts it with.
pub fn short<'a>(why: &'a str, noun: &str) -> &'a str {
let why = why.split(", expected").next().unwrap_or(why);
why.strip_prefix(&format!("{noun} file: ")).unwrap_or(why)
}
/// Leave a section out and keep it. A section the file needs is left empty.
fn move_out(top: &mut Map<String, Value>, key: &str, why: &str, noun: &str, file: &mut NewerFile) {
let Some(value) = top.remove(key) else { return };
file.not_understood.push(format!("section `{key}` ({})", short(why, noun)));
file.sections.push(Section { key: key.into(), value, empty: None });
}
/// Every top-level section of `top` that doesn't read by itself (`piece`, the app's reading of a
/// file holding just that section) is left out and kept in `file`. `noun` is what a file of the
/// app's is called, which its reader's errors start with.
pub fn leave_out_unreadable<E: std::fmt::Display>(
top: &mut Map<String, Value>,
file: &mut NewerFile,
noun: &str,
piece: impl Fn(&str, &Value) -> Result<(), E>,
) {
let keys: Vec<String> = top.keys().cloned().collect();
for key in &keys {
let value = top.get(key).cloned().unwrap_or_default();
if let Err(why) = piece(key, &value) {
move_out(top, key, &why.to_string(), noun, file);
}
}
}
/// Read `top` with `attempt`, leaving out the sections it can't be read with: if it doesn't read,
/// each section but the `core` ones (which the file can't be read without) is tried left out alone,
/// and as a last resort every one of them is. What is left out is kept in `file`. The error is the
/// first attempt's, when even the core alone doesn't read.
pub fn read_around<D, E: std::fmt::Display>(
top: &mut Map<String, Value>,
file: &mut NewerFile,
core: &[&str],
noun: &str,
attempt: impl Fn(&Map<String, Value>) -> Result<D, E>,
) -> Result<D, String> {
let first = match attempt(top) {
Ok(doc) => return Ok(doc),
Err(why) => why.to_string(),
};
// Sections that are fine by themselves but not together with the rest.
let keys: Vec<String> = top.keys().filter(|k| !core.contains(&k.as_str())).cloned().collect();
// Each try reads the whole file: a file of many sections goes straight to the last resort.
for key in keys.iter().filter(|_| keys.len() <= MAX_SECTION_TRIES) {
let mut without = top.clone();
without.remove(key);
if let Ok(doc) = attempt(&without) {
move_out(top, key, &first, noun, file);
return Ok(doc);
}
}
let mut bare = top.clone();
bare.retain(|key, _| core.contains(&key.as_str()));
match attempt(&bare) {
Ok(doc) => {
for key in &keys {
move_out(top, key, &first, noun, file);
}
Ok(doc)
}
Err(_) => Err(first),
}
}
/// A value a writer leaves out without losing anything: not worth telling anyone about.
pub fn looks_default(v: &Value) -> bool {
match v {
Value::Null => true,
Value::Bool(b) => !b,
Value::Number(n) => n.as_f64() == Some(0.0),
Value::String(s) => s.is_empty(),
Value::Array(a) => a.is_empty(),
Value::Object(o) => o.is_empty(),
}
}
/// What `raw` has that `known`, what was read of it, doesn't. `paths` gets those fields' names.
/// `top_only` are top-level keys left out of the comparison.
fn vanished(raw: &Value, known: &Value, path: &str, paths: &mut Vec<String>, placeholder_tag: &str, top_only: &[&str]) -> Option<Kept> {
match (raw, known) {
(Value::Object(raw), Value::Object(known)) => {
if known.get("type").and_then(Value::as_str) == Some(placeholder_tag) {
return None;
}
let mut kept = BTreeMap::new();
for (key, value) in raw {
if path.is_empty() && top_only.contains(&key.as_str()) {
continue;
}
let inner = if path.is_empty() { key.clone() } else { format!("{path}.{key}") };
match known.get(key) {
None => {
if !looks_default(value) {
paths.push(format!("field `{inner}`"));
}
kept.insert(key.clone(), Kept::Field(value.clone()));
}
Some(known) => {
if let Some(k) = vanished(value, known, &inner, paths, placeholder_tag, top_only) {
kept.insert(key.clone(), k);
}
}
}
}
(!kept.is_empty()).then_some(Kept::Fields(kept))
}
(Value::Array(raw), Value::Array(known)) => {
let mut items = Vec::new();
let positions = Positions::of(known);
for (at, element) in raw.iter().enumerate() {
// an identity typed reading kept: an id it dropped falls back to the name
let anchor = surviving_anchor(element, &positions).unwrap_or(Anchor::Index { at, of: raw.len() });
let Some(partner) = positions.find(known.len(), &anchor).and_then(|i| known.get(i)) else { continue };
let label = match (element.get("name").and_then(Value::as_str), &anchor) {
(Some(name), _) => format!("{path}[{name}]"),
(None, Anchor::Id(id)) => format!("{path}[{id}]"),
_ => format!("{path}[{at}]"),
};
if let Some(k) = vanished(element, partner, &label, paths, placeholder_tag, top_only) {
items.push((anchor, k));
}
}
(!items.is_empty()).then_some(Kept::Items(items))
}
_ => None,
}
}
fn anchor_of(element: &Value) -> Option<Anchor> {
if let Some(id) = element.get("id").and_then(Value::as_u64) {
return Some(Anchor::Id(id));
}
element.get("name").and_then(Value::as_str).map(|name| Anchor::Name(name.to_string()))
}
/// The identity of `element` that `known`'s records still carry: its id, or else its name. When
/// typed reading kept neither, there is none, and the record is matched by its place.
fn surviving_anchor(element: &Value, known: &Positions) -> Option<Anchor> {
let id = element.get("id").and_then(Value::as_u64).map(Anchor::Id);
let name = element.get("name").and_then(Value::as_str).map(|n| Anchor::Name(n.to_string()));
[id, name].into_iter().flatten().find(|anchor| known.find(usize::MAX, anchor).is_some())
}
/// Where each id and name first appears in a list, so matching records is not a scan per record.
struct Positions {
ids: HashMap<u64, usize>,
names: HashMap<String, usize>,
}
impl Positions {
fn of(list: &[Value]) -> Positions {
let (mut ids, mut names) = (HashMap::new(), HashMap::new());
for (i, x) in list.iter().enumerate() {
if let Some(id) = x.get("id").and_then(Value::as_u64) {
ids.entry(id).or_insert(i);
}
if let Some(name) = x.get("name").and_then(Value::as_str) {
names.entry(name.to_string()).or_insert(i);
}
}
Positions { ids, names }
}
/// Count `x`, now at `i`, in.
fn add(&mut self, x: &Value, i: usize) {
if let Some(id) = x.get("id").and_then(Value::as_u64) {
self.ids.entry(id).or_insert(i);
}
if let Some(name) = x.get("name").and_then(Value::as_str) {
self.names.entry(name.to_string()).or_insert(i);
}
}
/// The position of the record `anchor` names in a list of `len` records (an index anchor
/// only while the list keeps its length).
fn find(&self, len: usize, anchor: &Anchor) -> Option<usize> {
match anchor {
Anchor::Id(id) => self.ids.get(id).copied(),
Anchor::Name(name) => self.names.get(name).copied(),
Anchor::Index { at, of } => (len == *of && at < of).then_some(*at),
}
}
}
/// Put the kept fields back. An element that is gone was deleted here, its fields with it; one
/// whose place can't be told any more, and a field whose record is gone, go into `lost`.
fn apply(kept: &Kept, value: &mut Value, path: &str, lost: &mut Vec<String>) {
match kept {
Kept::Field(_) => {}
Kept::Fields(fields) => {
let Some(object) = value.as_object_mut() else {
lost.push(format!("field `{path}`"));
return;
};
for (key, sub) in fields {
let inner = if path.is_empty() { key.clone() } else { format!("{path}.{key}") };
match (sub, object.get_mut(key)) {
(Kept::Field(v), None) => {
object.insert(key.clone(), v.clone());
}
// the edited value stays; a retained one it replaced is reported
(Kept::Field(v), Some(existing)) => {
if existing != v {
lost.push(format!("field `{inner}`"));
}
}
(sub, Some(inner_value)) => apply(sub, inner_value, &inner, lost),
// The list it was in is empty and not written: its elements were deleted.
(Kept::Items(_), None) => {}
(_, None) => lost.push(format!("field `{inner}`")),
}
}
}
Kept::Items(items) => {
let Some(list) = value.as_array_mut() else {
lost.push(format!("field `{path}`"));
return;
};
let positions = Positions::of(list);
let len = list.len();
for (anchor, sub) in items {
match (positions.find(len, anchor).and_then(|i| list.get_mut(i)), anchor) {
(Some(element), _) => apply(sub, element, path, lost),
(None, Anchor::Index { at, .. }) => lost.push(format!("element {at} of `{path}`")),
(None, _) => {}
}
}
}
}
}
#[cfg(test)]
#[path = "newer_tests.rs"]
mod tests;
+218
View File
@@ -0,0 +1,218 @@
use super::*;
fn newer() -> Newer {
Newer { format: 9, reads_up_to: 8, unknown_changes: Vec::new() }
}
/// What typed reading dropped is named and goes back on saving, where it was.
#[test]
fn dropped_fields_are_named_and_put_back() {
let file = json!({"name": "A", "format": "app/9", "items": [{"id": 1, "size": 2, "colour": "red"}], "extra": {"x": 1}});
let known = json!({"name": "A", "items": [{"id": 1, "size": 2}]});
let mut said = NewerFile::new("App", &newer());
said.compare(&file, &known, "placeholder", &[]);
assert_eq!(said.not_understood, ["field `extra`", "field `items[1].colour`"]);
let mut saved = json!({"name": "A", "items": [{"id": 1, "size": 3}]});
assert!(said.restore_into(&mut saved).is_empty());
assert_eq!(saved, json!({"name": "A", "items": [{"id": 1, "size": 3, "colour": "red"}], "extra": {"x": 1}}));
assert_eq!(said.notice(), "Saved by a newer App (format 9, this version reads up to 8). If something's missing, update.");
}
/// A section that can't be read is left out, kept, and goes back while it is unchanged here.
#[test]
fn unreadable_sections_are_left_out_and_kept() {
let mut top = match json!({"name": "A", "good": 1, "bad": 2}) {
Value::Object(m) => m,
_ => unreachable!(),
};
let mut said = NewerFile::new("App", &newer());
leave_out_unreadable(&mut top, &mut said, "design", |key: &str, _: &Value| if key == "bad" { Err("design file: no") } else { Ok(()) });
assert!(!top.contains_key("bad"));
assert_eq!(said.not_understood, ["section `bad` (no)"]);
let read =
read_around(
&mut top,
&mut said,
&["name"],
"design",
|top: &Map<String, Value>| {
if top.contains_key("good") { Err("design file: clash") } else { Ok(top.len()) }
},
);
assert_eq!(read, Ok(1));
assert_eq!(said.not_understood, ["section `bad` (no)", "section `good` (clash)"]);
let mut saved = json!({"name": "A"});
said.compare(&saved.clone(), &saved.clone(), "placeholder", &[]);
assert!(said.restore_into(&mut saved).is_empty());
assert_eq!(saved, json!({"name": "A", "good": 1, "bad": 2}));
}
/// Notices, warnings and reports name what was left out, and kept values can be searched.
#[test]
fn warnings_report_and_mentions_cover_everything_kept() {
let mut said = NewerFile::new("App", &Newer { format: 8, reads_up_to: 8, unknown_changes: vec!["curves".into()] });
assert_eq!(said.notice(), "Saved by a newer App (format 8 with `curves`, which this version doesn't know). If something's missing, update.");
assert!(!said.needs_container());
said.keep_section("plugins", json!({"tool": "ruler"}), Some(json!({})), "unknown");
said.keep_element("shapes", json!({"name": "star", "points": ["tip"]}), "shape `star`".into());
let file = json!({"shapes": [{"name": "a", "tags": ["deep"], "more": {"x": "nested"}}], "n": 1});
said.compare(&file, &json!({"shapes": [{"name": "a"}], "n": 1}), "placeholder", &[]);
assert_eq!(said.warnings()[0], said.notice());
assert!(said.warnings().contains(&"Not read: section `plugins` (unknown). Saving keeps it.".to_string()));
let report = said.report();
assert_eq!(report["format"], 8);
assert_eq!(report["unknown_changes"], json!(["curves"]));
assert_eq!(report["notice"], json!(said.notice()));
for text in ["ruler", "tip", "deep", "nested"] {
assert!(said.mentions(text), "{text}");
}
assert!(!said.mentions("absent"));
said.parts.push(("blob".into(), Arc::from(&b"x"[..])));
assert!(said.needs_container());
}
/// A kept section or element that was replaced here is reported lost, not written over.
#[test]
fn replaced_sections_and_elements_are_reported_lost() {
let mut said = NewerFile::new("App", &newer());
said.keep_section("plugins", json!({"tool": "ruler"}), None, "unknown");
said.keep_element("shapes", json!({"name": "star"}), "shape `star`".into());
said.keep_element("shapes", json!({"points": 3}), "an unnamed shape".into());
let mut saved = json!({"plugins": {"edited": true}, "shapes": [{"name": "star"}]});
let lost = said.restore_into(&mut saved);
assert_eq!(lost, ["section `plugins`, replaced by what was edited here", "`shapes` element star, replaced by what was made here"]);
assert_eq!(saved["shapes"], json!([{"name": "star"}, {"points": 3}]));
let mut fresh = json!({});
said.sections.clear();
assert!(said.restore_into(&mut fresh).is_empty());
assert_eq!(fresh["shapes"], json!([{"name": "star"}, {"points": 3}]));
}
/// Retained container metadata alone needs a container to survive the save.
#[test]
fn container_metadata_alone_needs_a_container() {
let mut said = NewerFile::new("App", &newer());
assert!(!said.needs_container());
said.container = Some(json!({"layout": 2}));
assert!(said.needs_container());
}
/// An edit that replaces a retained field keeps the edit and reports the replaced value.
#[test]
fn a_replaced_retained_field_is_reported() {
let mut said = NewerFile::new("App", &newer());
said.compare(&json!({"extra": 1}), &json!({}), "placeholder", &[]);
let mut edited = json!({"extra": 2});
assert_eq!(said.restore_into(&mut edited), ["field `extra`"]);
assert_eq!(edited, json!({"extra": 2}));
let mut same = json!({"extra": 1});
assert!(said.restore_into(&mut same).is_empty());
}
/// Records are matched by id and by name across long lists, and an index anchor only while the
/// list keeps its length.
#[test]
fn records_match_by_id_name_or_unchanged_index() {
let records: Vec<Value> = (0..2000).map(|i| json!({"id": i, "extra": i})).collect();
let known: Vec<Value> = (0..2000).map(|i| json!({"id": i})).collect();
let mut said = NewerFile::new("App", &newer());
said.compare(&json!({"items": records}), &json!({"items": known}), "placeholder", &[]);
let mut saved = json!({"items": (0..2000).rev().map(|i| json!({"id": i})).collect::<Vec<_>>()});
assert!(said.restore_into(&mut saved).is_empty());
assert_eq!(saved["items"][0], json!({"id": 1999, "extra": 1999}));
let mut by_name = NewerFile::new("App", &newer());
by_name.compare(&json!({"l": [{"name": "a", "x": 1}, [1, 2]]}), &json!({"l": [{"name": "a"}, [1]]}), "placeholder", &[]);
let mut renamed = json!({"l": [{"name": "a"}]});
by_name.restore_into(&mut renamed);
assert_eq!(renamed["l"][0], json!({"name": "a", "x": 1}));
}
/// A section kept whole goes back whole, even when this build's record of it has fields.
#[test]
fn a_section_kept_whole_is_not_compared_field_by_field() {
let mut said = NewerFile::new("App", &newer());
said.keep_section("plugins", json!({"mode": "future", "payload": 1}), None, "unreadable");
let file = json!({"name": "A", "plugins": {"mode": "future", "payload": 1}});
said.compare(&file, &json!({"name": "A", "plugins": {"mode": "none"}}), "placeholder", &[]);
let mut saved = json!({"name": "A", "plugins": {"mode": "none"}});
assert!(said.restore_into(&mut saved).is_empty());
assert_eq!(saved, file);
}
/// A kept element goes back only while no record has its id or name.
#[test]
fn a_kept_element_keeps_record_identity() {
let mut said = NewerFile::new("App", &newer());
said.keep_element("items", json!({"id": 7, "kind": "future"}), "element 7".into());
said.keep_element("items", json!({"kind": "anonymous"}), "unnamed".into());
let mut clash = json!({"items": [{"id": 7, "kind": "made here"}]});
assert_eq!(said.restore_into(&mut clash).len(), 1);
assert_eq!(clash, json!({"items": [{"id": 7, "kind": "made here"}, {"kind": "anonymous"}]}));
let mut free = json!({"items": []});
assert!(said.restore_into(&mut free).is_empty());
assert_eq!(free, json!({"items": [{"id": 7, "kind": "future"}, {"kind": "anonymous"}]}));
}
/// Container-only fields are left out at the top level only.
#[test]
fn container_only_fields_are_top_level() {
let mut said = NewerFile::new("App", &newer());
let file = json!({"thumbnail": "t", "layer": {"thumbnail": "nested"}});
said.compare(&file, &json!({"layer": {}}), "placeholder", &["thumbnail"]);
assert_eq!(said.not_understood, ["field `layer.thumbnail`"]);
let mut saved = json!({"layer": {}});
assert!(said.restore_into(&mut saved).is_empty());
assert_eq!(saved, json!({"layer": {"thumbnail": "nested"}}));
}
/// An element kept on its own is not kept again with its list when typed writing leaves the list out.
#[test]
fn a_kept_element_is_not_restored_twice() {
let mut said = NewerFile::new("App", &newer());
said.keep_element("items", json!({"kind": "future"}), "unnamed".into());
said.compare(&json!({"items": [{"kind": "future"}]}), &json!({}), "placeholder", &[]);
let mut saved = json!({});
assert!(said.restore_into(&mut saved).is_empty());
assert_eq!(saved, json!({"items": [{"kind": "future"}]}));
}
/// A record whose id typed reading drops is matched by its name, and its fields go back.
#[test]
fn a_dropped_id_falls_back_to_the_name() {
let mut said = NewerFile::new("App", &newer());
said.compare(&json!({"items": [{"id": 7, "name": "a", "extra": 1}]}), &json!({"items": [{"name": "a"}]}), "placeholder", &[]);
let mut saved = json!({"items": [{"name": "a"}]});
assert!(said.restore_into(&mut saved).is_empty());
assert_eq!(saved, json!({"items": [{"name": "a", "id": 7, "extra": 1}]}));
}
/// Kept elements sharing a list are checked against one index, the ones put back included.
#[test]
fn kept_elements_check_each_other() {
let mut said = NewerFile::new("App", &newer());
said.keep_element("items", json!({"id": 1, "v": "first"}), "1".into());
said.keep_element("items", json!({"id": 1, "v": "again"}), "1 again".into());
let mut saved = json!({"items": []});
assert_eq!(said.restore_into(&mut saved).len(), 1);
assert_eq!(saved, json!({"items": [{"id": 1, "v": "first"}]}));
// deep nesting is searched without recursion
let mut deep = json!("needle");
for _ in 0..20_000 {
deep = Value::Array(vec![deep]);
}
said.keep_section("deep", deep, None, "test");
assert!(said.mentions("needle"));
// serde_json drops deep values recursively; leak it rather than overflow the test thread
std::mem::forget(said);
}
/// A record whose whole identity typed reading drops is matched by its place while the list keeps
/// its length, so an unchanged save keeps its fields.
#[test]
fn a_record_without_a_surviving_identity_is_matched_by_place() {
let mut said = NewerFile::new("App", &newer());
said.compare(&json!({"items": [{"id": 7, "extra": 1}]}), &json!({"items": [{}]}), "placeholder", &[]);
let mut saved = json!({"items": [{}]});
assert!(said.restore_into(&mut saved).is_empty());
assert_eq!(saved, json!({"items": [{"id": 7, "extra": 1}]}));
}