From f299664d7ce250df668c6452a6b169d1997f7089 Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Mon, 16 Nov 2026 05:07:48 +0100 Subject: [PATCH] craft-format: string-aware step detection, live expanded budget, unreadable names refused, retention identity fixes --- crates/format/src/build.rs | 108 ++++++++++++++++++++++----- crates/format/src/build_tests.rs | 8 ++ crates/format/src/container.rs | 12 ++- crates/format/src/container_tests.rs | 19 +++++ crates/format/src/newer.rs | 58 +++++++++++--- crates/format/src/newer_tests.rs | 41 ++++++++++ 6 files changed, 216 insertions(+), 30 deletions(-) diff --git a/crates/format/src/build.rs b/crates/format/src/build.rs index 8b4432c1a..e922d35d3 100644 --- a/crates/format/src/build.rs +++ b/crates/format/src/build.rs @@ -91,33 +91,101 @@ fn declares_step(text: &str) -> bool { }) } -/// `text` with its `//` and (nested) `/* */` comments taken out; a comment's line breaks stay, so -/// what follows a block comment is still on its own line. +/// `text` as code only: its `//` and (nested) `/* */` comments become a space, and the contents +/// of its string literals (plain, byte and raw) are dropped, line breaks kept throughout. A comment +/// marker inside a string is not a comment, and `fn/* note */step` still reads as `fn step`. fn without_comments(text: &str) -> String { + let chars: Vec = text.chars().collect(); + let at = |i: usize| chars.get(i).copied(); let mut out = String::with_capacity(text.len()); - let mut chars = text.chars().peekable(); - let mut depth = 0usize; - while let Some(c) = chars.next() { - match (c, chars.peek().copied()) { - ('/', Some('*')) => { - chars.next(); - depth += 1; - } - ('*', Some('/')) if depth > 0 => { - chars.next(); - depth -= 1; - } - ('/', Some('/')) if depth == 0 => { - for c in chars.by_ref() { + let (mut i, mut depth) = (0usize, 0usize); + while let Some(c) = at(i) { + let next = at(i + 1); + if depth > 0 { + match (c, next) { + ('/', Some('*')) => (depth, i) = (depth + 1, i + 2), + ('*', Some('/')) => (depth, i) = (depth - 1, i + 2), + _ => { if c == '\n' { out.push('\n'); - break; } + i += 1; } } - ('\n', _) => out.push('\n'), - (c, _) if depth == 0 => out.push(c), - _ => {} + if depth == 0 { + out.push(' '); + } + continue; + } + let ident_before = i > 0 && at(i - 1).is_some_and(|p| p.is_alphanumeric() || p == '_'); + match (c, next) { + ('/', Some('/')) => { + while at(i).is_some_and(|c| c != '\n') { + i += 1; + } + out.push(' '); + } + ('/', Some('*')) => (depth, i) = (1, i + 2), + ('"', _) => { + out.push('"'); + i += 1; + while let Some(c) = at(i) { + i += 1; + match c { + '\\' => i += 1, + '"' => break, + '\n' => out.push('\n'), + _ => {} + } + } + out.push('"'); + } + // a raw string, `r"…"` or `r#"…"#`, after a byte prefix or on its own + ('r', Some('"' | '#')) + if !ident_before || (i > 0 && at(i - 1) == Some('b') && !(i > 1 && at(i - 2).is_some_and(|p| p.is_alphanumeric() || p == '_'))) => + { + let mut j = i + 1; + let mut hashes = 0; + while at(j) == Some('#') { + hashes += 1; + j += 1; + } + if at(j) != Some('"') { + out.push(c); + i += 1; + continue; + } + out.push('"'); + i = j + 1; + while let Some(c) = at(i) { + if c == '"' && (1..=hashes).all(|k| at(i + k) == Some('#')) { + i += 1 + hashes; + break; + } + if c == '\n' { + out.push('\n'); + } + i += 1; + } + out.push('"'); + } + // a character literal (`'"'`, `'\''`), not a lifetime + ('\'', Some('\\')) => { + i += 2; + while at(i).is_some_and(|c| c != '\'') { + i += 1; + } + i += 1; + out.push_str("' '"); + } + ('\'', Some(_)) if at(i + 2) == Some('\'') => { + i += 3; + out.push_str("' '"); + } + _ => { + out.push(c); + i += 1; + } } } out diff --git a/crates/format/src/build_tests.rs b/crates/format/src/build_tests.rs index 0045c7baf..b97fc9ca9 100644 --- a/crates/format/src/build_tests.rs +++ b/crates/format/src/build_tests.rs @@ -83,6 +83,14 @@ fn a_step_is_a_declaration_not_a_mention() { assert!(!declares_step("//! Documentation only.\n/*\nfn step() {}\n*/\n")); assert!(!declares_step("/* outer /* inner */\nfn step() {}\n*/\nconst X: u8 = 1;\n")); assert!(declares_step("/* a note */\nfn step(v: &mut Value) {}\n")); + // comment markers inside strings and character literals are not comments + assert!(declares_step("//! A change with a step.\nconst MARKER: &str = \"/*\";\npub fn step(_v: &mut Value) -> Result { Ok(()) }\n")); + assert!(declares_step("const A: &str = r#\"/* \"quoted\" \"#;\nconst B: &[u8] = br\"/*\";\nfn step(v: &mut Value) {}\n")); + assert!(declares_step("const Q: char = '\"';\nconst E: char = '\\'';\nfn step(v: &mut Value) { let _: &'static str = \"\"; }\n")); + assert!(declares_step("const S: &str = \"a \\\" /* b\";\nfn step(v: &mut Value) {}\n")); + assert!(!declares_step("const S: &str = \"\nfn step(v) \";\n"), "inside a string"); + // a comment between the words is whitespace + assert!(declares_step("pub fn/* migration */step(v: &mut Value) {}\n")); assert!(declares_step("const X: u8 = 1; // fn step() is below\nfn step(v: &mut Value) {}\n")); assert!(declares_step("//! a note\npub(crate) fn step(v: &mut Value) -> Result<(), E> { Ok(()) }\n")); assert!(declares_step("//! a note\n fn step(v: &mut Value) {}\n")); diff --git a/crates/format/src/container.rs b/crates/format/src/container.rs index e578c59aa..b1e4c1f34 100644 --- a/crates/format/src/container.rs +++ b/crates/format/src/container.rs @@ -262,7 +262,11 @@ impl Reader { // size within its configured limit. let mut name_set = BTreeSet::new(); for entry_idx in 0..archive.len() { - if let Ok(Some(name)) = archive.entry_name(entry_idx) { + // an entry whose name can't be read would be left out of the package unnoticed + let Ok(Some(name)) = archive.entry_name(entry_idx) else { + return Err(invalid("duplicate or invalid archive entry")); + }; + { if name.is_empty() || name.contains('\\') || name.split('/').any(|c| c.is_empty() || c == "." || c == "..") @@ -309,6 +313,12 @@ impl Reader { if expected > limit as u64 { return Err(invalid("part too large")); } + // the expanded budget is read each time: a raised (or lowered) one applies at once, the + // bytes already read still counting against it + let used = self.archive.budget().used(); + let budget = self.archive.budget_mut(); + *budget = craft_zip::Budget::new(budget.what(), (self.limits.max_expanded)() as u64); + let _ = budget.charge(used.min(budget.limit())); // Read within budget. craft-zip's read() handles decompression bombs via the limit. let bytes = self.archive.read(name, limit as u64).map_err(invalid)?.ok_or_else(|| invalid("part not found"))?; if bytes.len() as u64 != expected { diff --git a/crates/format/src/container_tests.rs b/crates/format/src/container_tests.rs index 7fbabf155..a5f8fd773 100644 --- a/crates/format/src/container_tests.rs +++ b/crates/format/src/container_tests.rs @@ -220,3 +220,22 @@ fn cached_parts_keep_the_current_part_limits() { assert!(reader.document(4096).is_err()); assert!(reader.limited_blob(&reference, "blob", "bin", 4096).is_err()); } + +/// A raised expanded budget applies to the open reader, and what was read still counts. +#[test] +fn a_raised_expanded_budget_applies_at_once() { + static BUDGET: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(600); + let limits = Limits { max_expanded: || BUDGET.load(std::sync::atomic::Ordering::Relaxed), part_limit: |_| 1 << 20, ..LIMITS }; + let wide = Limits { max_expanded: || 1 << 20, ..limits }; + let mut writer = Writer::new(wide); + writer.part("document.json", &[b' '; 256], false).unwrap(); + writer.part("other.big", &[1; 512], false).unwrap(); + let bytes = writer.finish().unwrap(); + BUDGET.store(1 << 20, std::sync::atomic::Ordering::Relaxed); + let mut reader = Reader::new(Cursor::new(bytes), limits).unwrap(); + BUDGET.store(600, std::sync::atomic::Ordering::Relaxed); + assert!(reader.limited_part("document.json", 4096).is_ok()); + assert!(reader.limited_part("other.big", 4096).is_err(), "256 read + 512 is past 600"); + BUDGET.store(1024, std::sync::atomic::Ordering::Relaxed); + assert!(reader.limited_part("other.big", 4096).is_ok(), "768 fits the raised 1024"); +} diff --git a/crates/format/src/newer.rs b/crates/format/src/newer.rs index a35426f5d..e09d4f418 100644 --- a/crates/format/src/newer.rs +++ b/crates/format/src/newer.rs @@ -162,6 +162,8 @@ impl NewerFile { let mut top_only: Vec<&str> = container_only.to_vec(); top_only.extend(["format", "format_pending"]); top_only.extend(self.sections.iter().map(|s| s.key.as_str())); + // a list typed writing leaves out holds only elements kept on their own: not kept twice + top_only.extend(self.elements.iter().map(|(key, _)| key.as_str()).filter(|key| known.get(*key).is_none())); let kept = vanished(file, known, "", &mut paths, placeholder_tag, &top_only); self.kept = kept; self.not_understood.extend(paths); @@ -185,13 +187,24 @@ impl NewerFile { lost.push(format!("section `{}`, replaced by what was edited here", section.key)); } } + // each list's records indexed once, by the identity records are matched by + let mut indexes: HashMap<&str, Positions> = HashMap::new(); for (key, element) in &self.elements { let name = element.get("name"); - // the identity records are matched by: an id first, else a name let anchor = anchor_of(element); - let taken = |list: &[Value]| anchor.as_ref().is_some_and(|a| Positions::of(list).find(list.len(), a).is_some()); match top.entry(key.clone()).or_insert_with(|| json!([])) { - Value::Array(list) if !taken(list) => list.push(element.clone()), + Value::Array(list) => { + let index = indexes.entry(key.as_str()).or_insert_with(|| Positions::of(list)); + if anchor.as_ref().is_some_and(|a| index.find(list.len(), a).is_some()) { + lost.push(format!( + "`{key}` element {}, replaced by what was made here", + name.and_then(Value::as_str).unwrap_or("unnamed") + )); + } else { + index.add(element, list.len()); + list.push(element.clone()); + } + } _ => lost.push(format!("`{key}` element {}, replaced by what was made here", name.and_then(Value::as_str).unwrap_or("unnamed"))), } } @@ -200,13 +213,18 @@ impl NewerFile { } } +/// Whether a string anywhere in `v` is `text`; iterative, so no nesting depth overflows the stack. fn value_mentions(v: &Value, text: &str) -> bool { - match v { - Value::String(s) => s == text, - Value::Array(a) => a.iter().any(|x| value_mentions(x, text)), - Value::Object(o) => o.values().any(|x| value_mentions(x, text)), - _ => false, + let mut todo = vec![v]; + while let Some(v) = todo.pop() { + match v { + Value::String(s) if s == text => return true, + Value::Array(a) => todo.extend(a), + Value::Object(o) => todo.extend(o.values()), + _ => {} + } } + false } /// The first line of a reader's error, without the list of every kind it knows, and without the @@ -326,7 +344,8 @@ fn vanished(raw: &Value, known: &Value, path: &str, paths: &mut Vec, pla let mut items = Vec::new(); let positions = Positions::of(known); for (at, element) in raw.iter().enumerate() { - let anchor = anchor_of(element).unwrap_or(Anchor::Index { at, of: raw.len() }); + // an identity typed reading kept: an id it dropped falls back to the name + let anchor = surviving_anchor(element, &positions).unwrap_or(Anchor::Index { at, of: raw.len() }); let Some(partner) = positions.find(known.len(), &anchor).and_then(|i| known.get(i)) else { continue }; let label = match (element.get("name").and_then(Value::as_str), &anchor) { (Some(name), _) => format!("{path}[{name}]"), @@ -350,6 +369,17 @@ fn anchor_of(element: &Value) -> Option { element.get("name").and_then(Value::as_str).map(|name| Anchor::Name(name.to_string())) } +/// The identity of `element` that `known`'s records still carry: its id, or else its name. An +/// element with an identity neither of which survived has none. +fn surviving_anchor(element: &Value, known: &Positions) -> Option { + let id = element.get("id").and_then(Value::as_u64).map(Anchor::Id); + let name = element.get("name").and_then(Value::as_str).map(|n| Anchor::Name(n.to_string())); + match (id, name) { + (Some(id), Some(name)) => Some(if known.find(usize::MAX, &id).is_some() { id } else { name }), + (id, name) => id.or(name), + } +} + /// Where each id and name first appears in a list, so matching records is not a scan per record. struct Positions { ids: HashMap, @@ -370,6 +400,16 @@ impl Positions { Positions { ids, names } } + /// Count `x`, now at `i`, in. + fn add(&mut self, x: &Value, i: usize) { + if let Some(id) = x.get("id").and_then(Value::as_u64) { + self.ids.entry(id).or_insert(i); + } + if let Some(name) = x.get("name").and_then(Value::as_str) { + self.names.entry(name.to_string()).or_insert(i); + } + } + /// The position of the record `anchor` names in a list of `len` records (an index anchor /// only while the list keeps its length). fn find(&self, len: usize, anchor: &Anchor) -> Option { diff --git a/crates/format/src/newer_tests.rs b/crates/format/src/newer_tests.rs index d4578dce7..bfbff081b 100644 --- a/crates/format/src/newer_tests.rs +++ b/crates/format/src/newer_tests.rs @@ -164,3 +164,44 @@ fn container_only_fields_are_top_level() { assert!(said.restore_into(&mut saved).is_empty()); assert_eq!(saved, json!({"layer": {"thumbnail": "nested"}})); } + +/// An element kept on its own is not kept again with its list when typed writing leaves the list out. +#[test] +fn a_kept_element_is_not_restored_twice() { + let mut said = NewerFile::new("App", &newer()); + said.keep_element("items", json!({"kind": "future"}), "unnamed".into()); + said.compare(&json!({"items": [{"kind": "future"}]}), &json!({}), "placeholder", &[]); + let mut saved = json!({}); + assert!(said.restore_into(&mut saved).is_empty()); + assert_eq!(saved, json!({"items": [{"kind": "future"}]})); +} + +/// A record whose id typed reading drops is matched by its name, and its fields go back. +#[test] +fn a_dropped_id_falls_back_to_the_name() { + let mut said = NewerFile::new("App", &newer()); + said.compare(&json!({"items": [{"id": 7, "name": "a", "extra": 1}]}), &json!({"items": [{"name": "a"}]}), "placeholder", &[]); + let mut saved = json!({"items": [{"name": "a"}]}); + assert!(said.restore_into(&mut saved).is_empty()); + assert_eq!(saved, json!({"items": [{"name": "a", "id": 7, "extra": 1}]})); +} + +/// Kept elements sharing a list are checked against one index, the ones put back included. +#[test] +fn kept_elements_check_each_other() { + let mut said = NewerFile::new("App", &newer()); + said.keep_element("items", json!({"id": 1, "v": "first"}), "1".into()); + said.keep_element("items", json!({"id": 1, "v": "again"}), "1 again".into()); + let mut saved = json!({"items": []}); + assert_eq!(said.restore_into(&mut saved).len(), 1); + assert_eq!(saved, json!({"items": [{"id": 1, "v": "first"}]})); + // deep nesting is searched without recursion + let mut deep = json!("needle"); + for _ in 0..20_000 { + deep = Value::Array(vec![deep]); + } + said.keep_section("deep", deep, None, "test"); + assert!(said.mentions("needle")); + // serde_json drops deep values recursively; leak it rather than overflow the test thread + std::mem::forget(said); +}