mirror of
https://github.com/storytold/craft-libs.git
synced 2026-11-16 23:42:27 +00:00
format: address the CodeRabbit review
- build: fail on unreadable folders/entries (only a missing pending/ is empty); detect a step by its declaration, not a substring - container: commit part state only after the write; honour cancellation before each part and between pieces of a long one; count written parts in room(); bound the initial read to the approved length; enforce per-part limits when reading; check both thumbnail part limits before writing - lib: crate-root deny lints; parse the legacy prefix on its own - newer: retained container.json needs a container; index list anchors; report a retained field an edit replaced
This commit is contained in:
1 parent
462fe40be1
commit
f28df9c471
8 files changed
+323
-57
No files matched your search
+41
-15
@@ -44,14 +44,17 @@ pub fn lists(manifest: &Path, families: &[Family]) -> Result<Lists, String> {
|
||||
/// A family's `FOLDED` and `PENDING` lists; its folded formats follow on from its base.
|
||||
fn family_lists(family: &Family, root: &Path, out: &mut String, mods: &mut String) -> Result<(), String> {
|
||||
let Family { folder, base, list_prefix } = family;
|
||||
let mut folded: Vec<(u32, PathBuf)> = std::fs::read_dir(root)
|
||||
.map_err(|e| format!("{}: {e}", root.display()))?
|
||||
.filter_map(|entry| {
|
||||
let path = entry.ok()?.path();
|
||||
let version = path.file_name()?.to_str()?.strip_prefix('v')?.parse().ok()?;
|
||||
path.is_dir().then_some((version, path))
|
||||
})
|
||||
.collect();
|
||||
let mut folded: Vec<(u32, PathBuf)> = Vec::new();
|
||||
for entry in std::fs::read_dir(root).map_err(|e| format!("{}: {e}", root.display()))? {
|
||||
// an entry that can't be read could be a version folder: fail rather than drop it
|
||||
let path = entry.map_err(|e| format!("{}: {e}", root.display()))?.path();
|
||||
let version = path.file_name().and_then(|n| n.to_str()).and_then(|n| n.strip_prefix('v')).and_then(|n| n.parse().ok());
|
||||
if let Some(version) = version
|
||||
&& path.is_dir()
|
||||
{
|
||||
folded.push((version, path));
|
||||
}
|
||||
}
|
||||
folded.sort();
|
||||
for (i, (version, _)) in folded.iter().enumerate() {
|
||||
let expected = base + 1 + i as u32;
|
||||
@@ -68,19 +71,42 @@ fn family_lists(family: &Family, root: &Path, out: &mut String, mods: &mut Strin
|
||||
);
|
||||
for (version, dir) in &folded {
|
||||
let _ = writeln!(out, " ({version}, &[");
|
||||
parts(dir, &format!("{module}v{version}"), out, mods)?;
|
||||
parts(dir, &format!("{module}v{version}"), false, out, mods)?;
|
||||
out.push_str(" ]),\n");
|
||||
}
|
||||
let _ = writeln!(out, "];\n\n/// The changes no release has a number for yet (`{folder}/pending/`).\nconst {list_prefix}PENDING: &[Part] = &[");
|
||||
parts(&root.join("pending"), &format!("{module}pending"), out, mods)?;
|
||||
parts(&root.join("pending"), &format!("{module}pending"), true, out, mods)?;
|
||||
out.push_str("];\n\n");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The `Part`s of the change files in `dir`, in file-name order, and a module for each.
|
||||
fn parts(dir: &Path, prefix: &str, out: &mut String, mods: &mut String) -> Result<(), String> {
|
||||
let Ok(entries) = std::fs::read_dir(dir) else { return Ok(()) };
|
||||
let mut files: Vec<PathBuf> = entries.filter_map(|e| e.ok().map(|e| e.path())).filter(|p| p.extension().is_some_and(|x| x == "rs")).collect();
|
||||
/// Whether a change file declares its `step` function (`fn step(`, `pub fn step (` …) on a line of
|
||||
/// code: a mention in a comment or a note is not a declaration.
|
||||
fn declares_step(text: &str) -> bool {
|
||||
text.lines().map(str::trim_start).filter(|line| !line.starts_with("//")).any(|line| {
|
||||
let line = line.strip_prefix("pub(crate)").or_else(|| line.strip_prefix("pub")).unwrap_or(line).trim_start();
|
||||
line.strip_prefix("fn").is_some_and(|rest| {
|
||||
rest.starts_with(char::is_whitespace) && rest.trim_start().strip_prefix("step").is_some_and(|rest| rest.trim_start().starts_with('('))
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
/// The `Part`s of the change files in `dir`, in file-name order, and a module for each. A missing
|
||||
/// folder is an empty list only where it may be absent (`optional`: `pending/`); any other error
|
||||
/// reading it fails the build rather than leave a change out of the lists.
|
||||
fn parts(dir: &Path, prefix: &str, optional: bool, out: &mut String, mods: &mut String) -> Result<(), String> {
|
||||
let entries = match std::fs::read_dir(dir) {
|
||||
Ok(entries) => entries,
|
||||
Err(e) if optional && e.kind() == std::io::ErrorKind::NotFound => return Ok(()),
|
||||
Err(e) => return Err(format!("{}: {e}", dir.display())),
|
||||
};
|
||||
let mut files: Vec<PathBuf> = Vec::new();
|
||||
for entry in entries {
|
||||
let path = entry.map_err(|e| format!("{}: {e}", dir.display()))?.path();
|
||||
if path.extension().is_some_and(|x| x == "rs") {
|
||||
files.push(path);
|
||||
}
|
||||
}
|
||||
files.sort();
|
||||
for path in files {
|
||||
let name = path.file_stem().and_then(|s| s.to_str()).unwrap_or_default().to_string();
|
||||
@@ -92,7 +118,7 @@ fn parts(dir: &Path, prefix: &str, out: &mut String, mods: &mut String) -> Resul
|
||||
return Err(format!("{}: start a format change with a `//!` note", path.display()));
|
||||
}
|
||||
let module = format!("{prefix}_{}", name.replace('-', "_"));
|
||||
let step = if text.contains("fn step(") { format!("Some({module}::step)") } else { "None".into() };
|
||||
let step = if declares_step(&text) { format!("Some({module}::step)") } else { "None".into() };
|
||||
let _ = writeln!(out, " Part {{ name: {name:?}, step: {step} }},");
|
||||
let _ = writeln!(mods, "#[path = {:?}]\nmod {module};", path.display().to_string());
|
||||
}
|
||||
|
||||
@@ -73,3 +73,26 @@ fn refuses_gaps_bad_names_missing_notes_and_missing_folders() {
|
||||
assert!(refused(&dir).contains("`//!` note"));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_step_is_a_declaration_not_a_mention() {
|
||||
assert!(!declares_step("//! No fn step() is needed.\n"));
|
||||
assert!(!declares_step("//! a note\n// fn step(v) was removed\nconst X: u8 = 1;\n"));
|
||||
assert!(declares_step("//! a note\npub fn step (v: &mut Value) -> Result<(), E> { Ok(()) }\n"));
|
||||
assert!(declares_step("//! a note\npub(crate) fn step(v: &mut Value) -> Result<(), E> { Ok(()) }\n"));
|
||||
assert!(declares_step("//! a note\n fn step(v: &mut Value) {}\n"));
|
||||
assert!(!declares_step("//! a note\nfn stepper(v: &mut Value) {}\nfn fnstep() {}\n"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_documentation_only_change_has_no_step_and_a_missing_version_folder_fails() {
|
||||
let dir = scratch("doc-only");
|
||||
write(&dir.join("drawing/v4/doc.rs"), "//! No fn step() is needed.\n");
|
||||
let code = lists(&dir, &[DRAWING]).unwrap().code;
|
||||
assert!(code.contains("Part { name: \"doc\", step: None }"), "{code}");
|
||||
// a version folder that can't be listed is an error, not an empty format
|
||||
let missing = parts(&dir.join("drawing/v9"), "x", false, &mut String::new(), &mut String::new());
|
||||
assert!(missing.is_err());
|
||||
assert!(parts(&dir.join("drawing/pending"), "x", true, &mut String::new(), &mut String::new()).is_ok());
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
@@ -37,12 +37,15 @@ pub struct Limits {
|
||||
pub level: fn(&str, usize) -> i64,
|
||||
/// Whether a container of this many bytes may be read, or why not.
|
||||
pub check_size: fn(u64) -> Result<(), String>,
|
||||
/// Whether the work is cancelled (a save the person stopped): a long part is written a piece
|
||||
/// at a time and stops before the next piece when this says so. `None`: it cannot be
|
||||
/// cancelled here.
|
||||
/// Whether the work is cancelled (a save the person stopped): checked before each part, and
|
||||
/// a long part is written a piece at a time and stops before the next piece when this says
|
||||
/// so ([`FormatError::Cancelled`]). `None`: it cannot be cancelled here.
|
||||
pub cancelled: Option<fn() -> bool>,
|
||||
}
|
||||
|
||||
/// The most bytes of a part written between two cancellation checks.
|
||||
const PIECE: usize = 1 << 20;
|
||||
|
||||
fn invalid(message: impl std::fmt::Display) -> FormatError {
|
||||
FormatError::Invalid(message.to_string())
|
||||
}
|
||||
@@ -128,26 +131,54 @@ impl Writer {
|
||||
if bytes.len() > (self.limits.part_limit)(name) {
|
||||
return Err(invalid("part too large"));
|
||||
}
|
||||
if !self.names.insert(name.to_owned()) {
|
||||
if self.names.contains(name) {
|
||||
return Ok(());
|
||||
}
|
||||
self.expanded = self.expanded.checked_add(bytes.len()).ok_or_else(|| invalid("expanded size overflow"))?;
|
||||
if self.expanded > (self.limits.max_expanded)() || self.names.len() > self.limits.max_entries {
|
||||
// Check the counters this part would leave, and commit them only once it is written: a
|
||||
// refused part is not remembered as written.
|
||||
let expanded = self.expanded.checked_add(bytes.len()).ok_or_else(|| invalid("expanded size overflow"))?;
|
||||
if expanded > (self.limits.max_expanded)() || self.names.len().saturating_add(1) > self.limits.max_entries {
|
||||
return Err(invalid("package too large"));
|
||||
}
|
||||
let cancelled = self.limits.cancelled;
|
||||
if cancelled.is_some_and(|c| c()) {
|
||||
return Err(FormatError::Cancelled);
|
||||
}
|
||||
let compression = if compressed { craft_zip::Compression::Deflated } else { craft_zip::Compression::Stored };
|
||||
match cancelled {
|
||||
Some(cancelled) if bytes.len() > PIECE => {
|
||||
let mut stopped = false;
|
||||
let written = self.zip.add_with(name, compression, |out| {
|
||||
for piece in bytes.chunks(PIECE) {
|
||||
if cancelled() {
|
||||
stopped = true;
|
||||
return Err(std::io::Error::other("cancelled"));
|
||||
}
|
||||
out.write_all(piece)?;
|
||||
}
|
||||
Ok(())
|
||||
});
|
||||
if stopped {
|
||||
return Err(FormatError::Cancelled);
|
||||
}
|
||||
written.map_err(invalid)?;
|
||||
}
|
||||
_ => self.zip.add(name, bytes, compression).map_err(invalid)?,
|
||||
}
|
||||
self.names.insert(name.to_owned());
|
||||
self.expanded = expanded;
|
||||
if let Some((include, parts)) = &mut self.tracked
|
||||
&& include(name)
|
||||
{
|
||||
let digest = digest.unwrap_or_else(|| sha256(bytes));
|
||||
parts.push(PartDigest { name: name.to_owned(), len: bytes.len() as u64, digest });
|
||||
}
|
||||
let compression = if compressed { craft_zip::Compression::Deflated } else { craft_zip::Compression::Stored };
|
||||
self.zip.add(name, bytes, compression).map_err(invalid)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Would `entries` more optional parts of `extra` bytes, stored uncompressed, still keep the
|
||||
/// file within `budget` and its expanded size within the reader's budget?
|
||||
/// file within `budget` and its expanded size within the reader's budget? The parts already
|
||||
/// written count at their expanded size (an upper bound on what they take in the file).
|
||||
pub fn room(&self, entries: usize, extra: usize, budget: usize) -> bool {
|
||||
self.names.len().saturating_add(entries) <= self.limits.max_entries && extra <= self.space(entries, budget)
|
||||
}
|
||||
@@ -157,15 +188,14 @@ impl Writer {
|
||||
if self.names.len().saturating_add(entries) > self.limits.max_entries {
|
||||
return 0;
|
||||
}
|
||||
budget
|
||||
.saturating_sub(self.names.len().saturating_add(entries).saturating_mul(256).saturating_add(65_536))
|
||||
.min((self.limits.max_expanded)().saturating_sub(self.expanded))
|
||||
let overhead = self.names.len().saturating_add(entries).saturating_mul(256).saturating_add(65_536);
|
||||
budget.saturating_sub(overhead.saturating_add(self.expanded)).min((self.limits.max_expanded)().saturating_sub(self.expanded))
|
||||
}
|
||||
|
||||
/// Add a payload as a blob, `blobs/<sha256>.<extension>`, and say its name, for the JSON that
|
||||
/// refers to it.
|
||||
pub fn blob(&mut self, bytes: &[u8], extension: &str, compressed: bool) -> Result<Value, FormatError> {
|
||||
self.resolved = self
|
||||
let resolved = self
|
||||
.resolved
|
||||
.checked_add(bytes.len())
|
||||
.filter(|&n| n <= (self.limits.max_expanded)())
|
||||
@@ -173,6 +203,7 @@ impl Writer {
|
||||
let digest = sha256(bytes);
|
||||
let name = format!("blobs/{}.{extension}", hex(&digest));
|
||||
self.part_with_digest(&name, bytes, compressed, Some(digest))?;
|
||||
self.resolved = resolved;
|
||||
Ok(Value::String(name))
|
||||
}
|
||||
}
|
||||
@@ -194,10 +225,16 @@ impl Reader {
|
||||
let length = source.seek(SeekFrom::End(0)).map_err(invalid)?;
|
||||
(limits.check_size)(length).map_err(invalid)?;
|
||||
|
||||
// Read all bytes into memory for the archive.
|
||||
// Read all bytes into memory for the archive, no more than the length just approved: a file
|
||||
// that grew since is refused rather than read past its budget.
|
||||
source.seek(SeekFrom::Start(0)).map_err(invalid)?;
|
||||
let mut bytes = Vec::with_capacity(length as usize);
|
||||
source.read_to_end(&mut bytes).map_err(invalid)?;
|
||||
let capacity = usize::try_from(length).map_err(|_| invalid("file too large"))?;
|
||||
let mut bytes = Vec::new();
|
||||
bytes.try_reserve_exact(capacity).map_err(|_| invalid("file too large"))?;
|
||||
source.take(length.saturating_add(1)).read_to_end(&mut bytes).map_err(invalid)?;
|
||||
if bytes.len() as u64 != length {
|
||||
return Err(invalid("the file changed while it was read"));
|
||||
}
|
||||
|
||||
let mut archive = craft_zip::Archive::open_owned(
|
||||
bytes,
|
||||
@@ -206,16 +243,21 @@ impl Reader {
|
||||
.map_err(invalid)?;
|
||||
|
||||
// Validate entry names match format rules: no empty, no backslashes,
|
||||
// no path components that are "." or "..", and no duplicates.
|
||||
// no path components that are "." or "..", and no duplicates; and every part's declared
|
||||
// size within its configured limit.
|
||||
let mut name_set = BTreeSet::new();
|
||||
for entry_idx in 0..archive.len() {
|
||||
if let Ok(Some(name)) = archive.entry_name(entry_idx)
|
||||
&& (name.is_empty()
|
||||
if let Ok(Some(name)) = archive.entry_name(entry_idx) {
|
||||
if name.is_empty()
|
||||
|| name.contains('\\')
|
||||
|| name.split('/').any(|c| c.is_empty() || c == "." || c == "..")
|
||||
|| !name_set.insert(name))
|
||||
{
|
||||
return Err(invalid("duplicate or invalid archive entry"));
|
||||
|| !name_set.insert(name.clone())
|
||||
{
|
||||
return Err(invalid("duplicate or invalid archive entry"));
|
||||
}
|
||||
if archive.declared_size(&name).is_some_and(|size| size > (limits.part_limit)(&name) as u64) {
|
||||
return Err(invalid("part too large"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -243,8 +285,9 @@ impl Reader {
|
||||
self.archive.names().map(|n| n.into_owned()).collect()
|
||||
}
|
||||
|
||||
/// The part of this name, at most `limit` bytes.
|
||||
/// The part of this name, at most `limit` bytes and never more than its configured limit.
|
||||
pub fn limited_part(&mut self, name: &str, limit: usize) -> Result<Vec<u8>, FormatError> {
|
||||
let limit = limit.min((self.limits.part_limit)(name));
|
||||
let expected = self.archive.declared_size(name).ok_or_else(|| invalid("part not found"))?;
|
||||
if expected > limit as u64 {
|
||||
return Err(invalid("part too large"));
|
||||
@@ -329,7 +372,11 @@ impl Thumbnail {
|
||||
let key =
|
||||
serde_json::to_vec(&serde_json::json!({"format": self.format, "app": self.app, "document": digest(document), "payload": digest(png)}))
|
||||
.map_err(invalid)?;
|
||||
if !writer.room(2, key.len().saturating_add(png.len()), budget) {
|
||||
let fits = |name: &str, len: usize| len <= (writer.limits.part_limit)(name);
|
||||
if !fits("cache/thumbnail.json", key.len())
|
||||
|| !fits("cache/thumbnail.png", png.len())
|
||||
|| !writer.room(2, key.len().saturating_add(png.len()), budget)
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
writer.part("cache/thumbnail.json", &key, false)?;
|
||||
|
||||
@@ -66,3 +66,98 @@ fn a_thumbnail_is_keyed_to_its_file() {
|
||||
let mut reader = Reader::new(Cursor::new(writer.finish().unwrap()), LIMITS).unwrap();
|
||||
assert!(picture.read(&mut reader, 64).is_err(), "stale");
|
||||
}
|
||||
|
||||
/// A refused part is not remembered as written: once the budget is raised it is written.
|
||||
#[test]
|
||||
fn a_refused_part_can_be_written_later() {
|
||||
static BUDGET: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(4);
|
||||
let limits = Limits { max_expanded: || BUDGET.load(std::sync::atomic::Ordering::SeqCst), ..LIMITS };
|
||||
let mut writer = Writer::new(limits);
|
||||
assert_eq!(writer.part("a.json", b"12345678", false), Err(FormatError::Invalid("package too large".into())));
|
||||
assert!(writer.blob(b"12345678", "bin", false).is_err());
|
||||
BUDGET.store(1 << 20, std::sync::atomic::Ordering::SeqCst);
|
||||
writer.part("a.json", b"12345678", false).unwrap();
|
||||
writer.blob(b"12345678", "bin", false).unwrap();
|
||||
let reader = Reader::new(Cursor::new(writer.finish().unwrap()), limits).unwrap();
|
||||
assert_eq!(reader.part_names().len(), 2, "both parts were written on the retry");
|
||||
}
|
||||
|
||||
/// A cancelled save writes no more parts, whether the part is small or written in pieces.
|
||||
#[test]
|
||||
fn cancellation_stops_the_save() {
|
||||
let cancelled = Limits { cancelled: Some(|| true), ..LIMITS };
|
||||
let mut writer = Writer::new(cancelled);
|
||||
assert_eq!(writer.part("a.json", b"a", false), Err(FormatError::Cancelled));
|
||||
// cancelled after the first piece of a long part: checks before the part, then per piece
|
||||
static CHECKS: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(0);
|
||||
let midway = Limits {
|
||||
cancelled: Some(|| CHECKS.fetch_add(1, std::sync::atomic::Ordering::SeqCst) >= 2),
|
||||
max_expanded: || 1 << 22,
|
||||
part_limit: |_| 1 << 22,
|
||||
..LIMITS
|
||||
};
|
||||
let mut writer = Writer::new(midway);
|
||||
assert_eq!(writer.part("big.big", &vec![0; PIECE + 1], true), Err(FormatError::Cancelled));
|
||||
assert_eq!(CHECKS.load(std::sync::atomic::Ordering::SeqCst), 3);
|
||||
let going = Limits { cancelled: Some(|| false), max_expanded: || 1 << 22, part_limit: |_| 1 << 22, ..LIMITS };
|
||||
let mut writer = Writer::new(going);
|
||||
writer.part("big.big", &vec![7; PIECE + 1], true).unwrap();
|
||||
let mut reader = Reader::new(Cursor::new(writer.finish().unwrap()), going).unwrap();
|
||||
assert_eq!(reader.limited_part("big.big", 1 << 22).unwrap().len(), PIECE + 1);
|
||||
}
|
||||
|
||||
/// Room counts what is already written, and the reader keeps every part's configured limit.
|
||||
#[test]
|
||||
fn room_counts_written_parts_and_reads_keep_part_limits() {
|
||||
let mut writer = Writer::new(LIMITS);
|
||||
writer.part("large.big", &[0; 500 * 1024], false).unwrap();
|
||||
assert!(!writer.room(1, 200 * 1024, 600 * 1024));
|
||||
assert!(writer.room(1, 1024, 1 << 20));
|
||||
let wide = Limits { part_limit: |_| 1 << 20, ..LIMITS };
|
||||
let mut writer = Writer::new(wide);
|
||||
writer.part("document.json", &[b' '; 2048], false).unwrap();
|
||||
let bytes = writer.finish().unwrap();
|
||||
assert!(Reader::new(Cursor::new(bytes.clone()), LIMITS).is_err(), "declared past its configured limit");
|
||||
let mut reader = Reader::new(Cursor::new(bytes), wide).unwrap();
|
||||
assert!(reader.limited_part("document.json", 4096).is_ok());
|
||||
}
|
||||
|
||||
/// A thumbnail past a cache part's own limit is left out, never failing the save.
|
||||
#[test]
|
||||
fn a_thumbnail_past_its_part_limit_is_left_out() {
|
||||
let picture = Thumbnail { format: "app-thumbnail/1", app: "1.0", size: [2, 1], max_bytes: 2048 };
|
||||
let mut png = b"\x89PNG\r\n\x1a\n\0\0\0\rIHDR".to_vec();
|
||||
png.extend_from_slice(&2u32.to_be_bytes());
|
||||
png.extend_from_slice(&1u32.to_be_bytes());
|
||||
png.resize(1500, 0);
|
||||
let mut writer = Writer::new(LIMITS);
|
||||
picture.write(&mut writer, &png, b"{}", 8 << 20).unwrap();
|
||||
let reader = Reader::new(Cursor::new(writer.finish().unwrap()), LIMITS).unwrap();
|
||||
assert!(!reader.has_part("cache/thumbnail.json") && !reader.has_part("cache/thumbnail.png"));
|
||||
}
|
||||
|
||||
/// A source that grows after its length was approved is refused.
|
||||
#[test]
|
||||
fn a_source_that_grows_while_read_is_refused() {
|
||||
struct Growing(Cursor<Vec<u8>>, bool);
|
||||
impl Read for Growing {
|
||||
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
|
||||
if !self.1 {
|
||||
self.1 = true;
|
||||
let end = self.0.get_ref().len();
|
||||
self.0.get_mut().extend_from_slice(&[0; 16]);
|
||||
self.0.set_position(self.0.position().min(end as u64));
|
||||
}
|
||||
self.0.read(buf)
|
||||
}
|
||||
}
|
||||
impl Seek for Growing {
|
||||
fn seek(&mut self, to: SeekFrom) -> std::io::Result<u64> {
|
||||
self.0.seek(to)
|
||||
}
|
||||
}
|
||||
let mut writer = Writer::new(LIMITS);
|
||||
writer.part("document.json", b"{}", false).unwrap();
|
||||
let bytes = writer.finish().unwrap();
|
||||
assert!(Reader::new(Growing(Cursor::new(bytes), false), LIMITS).err().is_some_and(|e| e.to_string().contains("changed")));
|
||||
}
|
||||
@@ -20,6 +20,8 @@
|
||||
//! Steps are JSON-level (the file's shape) and return the app's own error type; every error this
|
||||
//! crate makes is a [`FormatError`], which the app's error type takes `From`.
|
||||
|
||||
#![cfg_attr(not(test), deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable))]
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
pub mod build;
|
||||
@@ -157,7 +159,10 @@ impl<E> Family<'_, E> {
|
||||
/// The version in one of this family's format tags; `None` for any other tag, another
|
||||
/// family's included.
|
||||
pub fn version_of(&self, s: &str) -> Option<u32> {
|
||||
s.strip_prefix(self.prefix).or_else(|| self.legacy_prefix.and_then(|p| s.strip_prefix(p)))?.parse().ok()
|
||||
// each tag form parses on its own: a legacy prefix may start with the primary one
|
||||
// (`app/` and `app/design/`)
|
||||
let parse = |prefix: &str| s.strip_prefix(prefix).and_then(|n| n.parse().ok());
|
||||
parse(self.prefix).or_else(|| self.legacy_prefix.and_then(parse))
|
||||
}
|
||||
|
||||
/// The change a name a file of format `version` lists stands for: its alias's, or itself.
|
||||
|
||||
@@ -132,3 +132,11 @@ fn a_newer_file_is_told_by_its_number_or_an_unknown_change() {
|
||||
assert_eq!(FAMILY.newer(4, Some(&["old-c".to_string()])), Some(Newer { format: 4, reads_up_to: 6, unknown_changes: vec!["old-c".to_string()] }));
|
||||
assert_eq!(FAMILY.newer(2, Some(&["z".to_string()])), None, "a list beside a hand-written format means nothing");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_legacy_prefix_that_extends_the_primary_one_is_read() {
|
||||
let nested = Family { prefix: "app/", legacy_prefix: Some("app/design/"), ..FAMILY };
|
||||
assert_eq!(nested.version_of("app/design/3"), Some(3));
|
||||
assert_eq!(nested.version_of("app/4"), Some(4));
|
||||
assert_eq!(nested.version_of("app/design/x"), None);
|
||||
}
|
||||
+40
-17
@@ -12,7 +12,7 @@
|
||||
//! saved ([`NewerFile::restore_into`]), as they came. What can't be put back because the file was
|
||||
//! changed around it is what `restore_into` returns.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::collections::{BTreeMap, HashMap};
|
||||
use std::sync::Arc;
|
||||
|
||||
use serde_json::{Map, Value, json};
|
||||
@@ -118,9 +118,9 @@ impl NewerFile {
|
||||
}
|
||||
|
||||
/// Whether the next save needs a container to keep what the file holds: plain JSON has no room
|
||||
/// for parts beside the file's own.
|
||||
/// for parts beside the file's own, nor for the container's own `container.json`.
|
||||
pub fn needs_container(&self) -> bool {
|
||||
!self.parts.is_empty()
|
||||
!self.parts.is_empty() || self.container.is_some()
|
||||
}
|
||||
|
||||
/// Whether anything kept back names this text (a blob's name in a container).
|
||||
@@ -315,9 +315,10 @@ fn vanished(raw: &Value, known: &Value, path: &str, paths: &mut Vec<String>, pla
|
||||
}
|
||||
(Value::Array(raw), Value::Array(known)) => {
|
||||
let mut items = Vec::new();
|
||||
let positions = Positions::of(known);
|
||||
for (at, element) in raw.iter().enumerate() {
|
||||
let anchor = anchor_of(element).unwrap_or(Anchor::Index { at, of: raw.len() });
|
||||
let Some(partner) = find(known, &anchor) else { continue };
|
||||
let Some(partner) = positions.find(known.len(), &anchor).and_then(|i| known.get(i)) else { continue };
|
||||
let label = match (element.get("name").and_then(Value::as_str), &anchor) {
|
||||
(Some(name), _) => format!("{path}[{name}]"),
|
||||
(None, Anchor::Id(id)) => format!("{path}[{id}]"),
|
||||
@@ -340,19 +341,34 @@ fn anchor_of(element: &Value) -> Option<Anchor> {
|
||||
element.get("name").and_then(Value::as_str).map(|name| Anchor::Name(name.to_string()))
|
||||
}
|
||||
|
||||
fn find<'a>(list: &'a [Value], anchor: &Anchor) -> Option<&'a Value> {
|
||||
match anchor {
|
||||
Anchor::Id(id) => list.iter().find(|x| x.get("id").and_then(Value::as_u64) == Some(*id)),
|
||||
Anchor::Name(name) => list.iter().find(|x| x.get("name").and_then(Value::as_str) == Some(name)),
|
||||
Anchor::Index { at, of } => (list.len() == *of).then(|| list.get(*at)).flatten(),
|
||||
}
|
||||
/// Where each id and name first appears in a list, so matching records is not a scan per record.
|
||||
struct Positions {
|
||||
ids: HashMap<u64, usize>,
|
||||
names: HashMap<String, usize>,
|
||||
}
|
||||
|
||||
fn find_mut<'a>(list: &'a mut [Value], anchor: &Anchor) -> Option<&'a mut Value> {
|
||||
match anchor {
|
||||
Anchor::Id(id) => list.iter_mut().find(|x| x.get("id").and_then(Value::as_u64) == Some(*id)),
|
||||
Anchor::Name(name) => list.iter_mut().find(|x| x.get("name").and_then(Value::as_str) == Some(name)),
|
||||
Anchor::Index { at, of } => (list.len() == *of).then(|| list.get_mut(*at)).flatten(),
|
||||
impl Positions {
|
||||
fn of(list: &[Value]) -> Positions {
|
||||
let (mut ids, mut names) = (HashMap::new(), HashMap::new());
|
||||
for (i, x) in list.iter().enumerate() {
|
||||
if let Some(id) = x.get("id").and_then(Value::as_u64) {
|
||||
ids.entry(id).or_insert(i);
|
||||
}
|
||||
if let Some(name) = x.get("name").and_then(Value::as_str) {
|
||||
names.entry(name.to_string()).or_insert(i);
|
||||
}
|
||||
}
|
||||
Positions { ids, names }
|
||||
}
|
||||
|
||||
/// The position of the record `anchor` names in a list of `len` records (an index anchor
|
||||
/// only while the list keeps its length).
|
||||
fn find(&self, len: usize, anchor: &Anchor) -> Option<usize> {
|
||||
match anchor {
|
||||
Anchor::Id(id) => self.ids.get(id).copied(),
|
||||
Anchor::Name(name) => self.names.get(name).copied(),
|
||||
Anchor::Index { at, of } => (len == *of && at < of).then_some(*at),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -372,7 +388,12 @@ fn apply(kept: &Kept, value: &mut Value, path: &str, lost: &mut Vec<String>) {
|
||||
(Kept::Field(v), None) => {
|
||||
object.insert(key.clone(), v.clone());
|
||||
}
|
||||
(Kept::Field(_), Some(_)) => {}
|
||||
// the edited value stays; a retained one it replaced is reported
|
||||
(Kept::Field(v), Some(existing)) => {
|
||||
if existing != v {
|
||||
lost.push(format!("field `{inner}`"));
|
||||
}
|
||||
}
|
||||
(sub, Some(inner_value)) => apply(sub, inner_value, &inner, lost),
|
||||
// The list it was in is empty and not written: its elements were deleted.
|
||||
(Kept::Items(_), None) => {}
|
||||
@@ -385,8 +406,10 @@ fn apply(kept: &Kept, value: &mut Value, path: &str, lost: &mut Vec<String>) {
|
||||
lost.push(format!("field `{path}`"));
|
||||
return;
|
||||
};
|
||||
let positions = Positions::of(list);
|
||||
let len = list.len();
|
||||
for (anchor, sub) in items {
|
||||
match (find_mut(list, anchor), anchor) {
|
||||
match (positions.find(len, anchor).and_then(|i| list.get_mut(i)), anchor) {
|
||||
(Some(element), _) => apply(sub, element, path, lost),
|
||||
(None, Anchor::Index { at, .. }) => lost.push(format!("element {at} of `{path}`")),
|
||||
(None, _) => {}
|
||||
|
||||
@@ -87,3 +87,42 @@ fn replaced_sections_and_elements_are_reported_lost() {
|
||||
assert!(said.restore_into(&mut fresh).is_empty());
|
||||
assert_eq!(fresh["shapes"], json!([{"name": "star"}, {"points": 3}]));
|
||||
}
|
||||
|
||||
/// Retained container metadata alone needs a container to survive the save.
|
||||
#[test]
|
||||
fn container_metadata_alone_needs_a_container() {
|
||||
let mut said = NewerFile::new("App", &newer());
|
||||
assert!(!said.needs_container());
|
||||
said.container = Some(json!({"layout": 2}));
|
||||
assert!(said.needs_container());
|
||||
}
|
||||
|
||||
/// An edit that replaces a retained field keeps the edit and reports the replaced value.
|
||||
#[test]
|
||||
fn a_replaced_retained_field_is_reported() {
|
||||
let mut said = NewerFile::new("App", &newer());
|
||||
said.compare(&json!({"extra": 1}), &json!({}), "placeholder", &[]);
|
||||
let mut edited = json!({"extra": 2});
|
||||
assert_eq!(said.restore_into(&mut edited), ["field `extra`"]);
|
||||
assert_eq!(edited, json!({"extra": 2}));
|
||||
let mut same = json!({"extra": 1});
|
||||
assert!(said.restore_into(&mut same).is_empty());
|
||||
}
|
||||
|
||||
/// Records are matched by id and by name across long lists, and an index anchor only while the
|
||||
/// list keeps its length.
|
||||
#[test]
|
||||
fn records_match_by_id_name_or_unchanged_index() {
|
||||
let records: Vec<Value> = (0..2000).map(|i| json!({"id": i, "extra": i})).collect();
|
||||
let known: Vec<Value> = (0..2000).map(|i| json!({"id": i})).collect();
|
||||
let mut said = NewerFile::new("App", &newer());
|
||||
said.compare(&json!({"items": records}), &json!({"items": known}), "placeholder", &[]);
|
||||
let mut saved = json!({"items": (0..2000).rev().map(|i| json!({"id": i})).collect::<Vec<_>>()});
|
||||
assert!(said.restore_into(&mut saved).is_empty());
|
||||
assert_eq!(saved["items"][0], json!({"id": 1999, "extra": 1999}));
|
||||
let mut by_name = NewerFile::new("App", &newer());
|
||||
by_name.compare(&json!({"l": [{"name": "a", "x": 1}, [1, 2]]}), &json!({"l": [{"name": "a"}, [1]]}), "placeholder", &[]);
|
||||
let mut renamed = json!({"l": [{"name": "a"}]});
|
||||
by_name.restore_into(&mut renamed);
|
||||
assert_eq!(renamed["l"][0], json!({"name": "a", "x": 1}));
|
||||
}
|
||||
Reference in new issue
Block a user