craft-format: string-aware step detection, live expanded budget, unreadable names refused, retention identity fixes

This commit is contained in:
FredApps committed 2026-11-16 05:07:48 +01:00
1 parent b9636d757a
commit f299664d7c
6 files changed
+216 -30

No files matched your search

+88 -20
View File
@@ -91,33 +91,101 @@ fn declares_step(text: &str) -> bool {
})
}
/// `text` with its `//` and (nested) `/* */` comments taken out; a comment's line breaks stay, so
/// what follows a block comment is still on its own line.
/// `text` as code only: its `//` and (nested) `/* */` comments become a space, and the contents
/// of its string literals (plain, byte and raw) are dropped, line breaks kept throughout. A comment
/// marker inside a string is not a comment, and `fn/* note */step` still reads as `fn step`.
fn without_comments(text: &str) -> String {
let chars: Vec<char> = text.chars().collect();
let at = |i: usize| chars.get(i).copied();
let mut out = String::with_capacity(text.len());
let mut chars = text.chars().peekable();
let mut depth = 0usize;
while let Some(c) = chars.next() {
match (c, chars.peek().copied()) {
('/', Some('*')) => {
chars.next();
depth += 1;
}
('*', Some('/')) if depth > 0 => {
chars.next();
depth -= 1;
}
('/', Some('/')) if depth == 0 => {
for c in chars.by_ref() {
let (mut i, mut depth) = (0usize, 0usize);
while let Some(c) = at(i) {
let next = at(i + 1);
if depth > 0 {
match (c, next) {
('/', Some('*')) => (depth, i) = (depth + 1, i + 2),
('*', Some('/')) => (depth, i) = (depth - 1, i + 2),
_ => {
if c == '\n' {
out.push('\n');
break;
}
i += 1;
}
}
('\n', _) => out.push('\n'),
(c, _) if depth == 0 => out.push(c),
_ => {}
if depth == 0 {
out.push(' ');
}
continue;
}
let ident_before = i > 0 && at(i - 1).is_some_and(|p| p.is_alphanumeric() || p == '_');
match (c, next) {
('/', Some('/')) => {
while at(i).is_some_and(|c| c != '\n') {
i += 1;
}
out.push(' ');
}
('/', Some('*')) => (depth, i) = (1, i + 2),
('"', _) => {
out.push('"');
i += 1;
while let Some(c) = at(i) {
i += 1;
match c {
'\\' => i += 1,
'"' => break,
'\n' => out.push('\n'),
_ => {}
}
}
out.push('"');
}
// a raw string, `r"…"` or `r#"…"#`, after a byte prefix or on its own
('r', Some('"' | '#'))
if !ident_before || (i > 0 && at(i - 1) == Some('b') && !(i > 1 && at(i - 2).is_some_and(|p| p.is_alphanumeric() || p == '_'))) =>
{
let mut j = i + 1;
let mut hashes = 0;
while at(j) == Some('#') {
hashes += 1;
j += 1;
}
if at(j) != Some('"') {
out.push(c);
i += 1;
continue;
}
out.push('"');
i = j + 1;
while let Some(c) = at(i) {
if c == '"' && (1..=hashes).all(|k| at(i + k) == Some('#')) {
i += 1 + hashes;
break;
}
if c == '\n' {
out.push('\n');
}
i += 1;
}
out.push('"');
}
// a character literal (`'"'`, `'\''`), not a lifetime
('\'', Some('\\')) => {
i += 2;
while at(i).is_some_and(|c| c != '\'') {
i += 1;
}
i += 1;
out.push_str("' '");
}
('\'', Some(_)) if at(i + 2) == Some('\'') => {
i += 3;
out.push_str("' '");
}
_ => {
out.push(c);
i += 1;
}
}
}
out
+8
View File
@@ -83,6 +83,14 @@ fn a_step_is_a_declaration_not_a_mention() {
assert!(!declares_step("//! Documentation only.\n/*\nfn step() {}\n*/\n"));
assert!(!declares_step("/* outer /* inner */\nfn step() {}\n*/\nconst X: u8 = 1;\n"));
assert!(declares_step("/* a note */\nfn step(v: &mut Value) {}\n"));
// comment markers inside strings and character literals are not comments
assert!(declares_step("//! A change with a step.\nconst MARKER: &str = \"/*\";\npub fn step(_v: &mut Value) -> Result { Ok(()) }\n"));
assert!(declares_step("const A: &str = r#\"/* \"quoted\" \"#;\nconst B: &[u8] = br\"/*\";\nfn step(v: &mut Value) {}\n"));
assert!(declares_step("const Q: char = '\"';\nconst E: char = '\\'';\nfn step(v: &mut Value) { let _: &'static str = \"\"; }\n"));
assert!(declares_step("const S: &str = \"a \\\" /* b\";\nfn step(v: &mut Value) {}\n"));
assert!(!declares_step("const S: &str = \"\nfn step(v) \";\n"), "inside a string");
// a comment between the words is whitespace
assert!(declares_step("pub fn/* migration */step(v: &mut Value) {}\n"));
assert!(declares_step("const X: u8 = 1; // fn step() is below\nfn step(v: &mut Value) {}\n"));
assert!(declares_step("//! a note\npub(crate) fn step(v: &mut Value) -> Result<(), E> { Ok(()) }\n"));
assert!(declares_step("//! a note\n fn step(v: &mut Value) {}\n"));
+11 -1
View File
@@ -262,7 +262,11 @@ impl Reader {
// size within its configured limit.
let mut name_set = BTreeSet::new();
for entry_idx in 0..archive.len() {
if let Ok(Some(name)) = archive.entry_name(entry_idx) {
// an entry whose name can't be read would be left out of the package unnoticed
let Ok(Some(name)) = archive.entry_name(entry_idx) else {
return Err(invalid("duplicate or invalid archive entry"));
};
{
if name.is_empty()
|| name.contains('\\')
|| name.split('/').any(|c| c.is_empty() || c == "." || c == "..")
@@ -309,6 +313,12 @@ impl Reader {
if expected > limit as u64 {
return Err(invalid("part too large"));
}
// the expanded budget is read each time: a raised (or lowered) one applies at once, the
// bytes already read still counting against it
let used = self.archive.budget().used();
let budget = self.archive.budget_mut();
*budget = craft_zip::Budget::new(budget.what(), (self.limits.max_expanded)() as u64);
let _ = budget.charge(used.min(budget.limit()));
// Read within budget. craft-zip's read() handles decompression bombs via the limit.
let bytes = self.archive.read(name, limit as u64).map_err(invalid)?.ok_or_else(|| invalid("part not found"))?;
if bytes.len() as u64 != expected {
+19
View File
@@ -220,3 +220,22 @@ fn cached_parts_keep_the_current_part_limits() {
assert!(reader.document(4096).is_err());
assert!(reader.limited_blob(&reference, "blob", "bin", 4096).is_err());
}
/// A raised expanded budget applies to the open reader, and what was read still counts.
#[test]
fn a_raised_expanded_budget_applies_at_once() {
static BUDGET: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(600);
let limits = Limits { max_expanded: || BUDGET.load(std::sync::atomic::Ordering::Relaxed), part_limit: |_| 1 << 20, ..LIMITS };
let wide = Limits { max_expanded: || 1 << 20, ..limits };
let mut writer = Writer::new(wide);
writer.part("document.json", &[b' '; 256], false).unwrap();
writer.part("other.big", &[1; 512], false).unwrap();
let bytes = writer.finish().unwrap();
BUDGET.store(1 << 20, std::sync::atomic::Ordering::Relaxed);
let mut reader = Reader::new(Cursor::new(bytes), limits).unwrap();
BUDGET.store(600, std::sync::atomic::Ordering::Relaxed);
assert!(reader.limited_part("document.json", 4096).is_ok());
assert!(reader.limited_part("other.big", 4096).is_err(), "256 read + 512 is past 600");
BUDGET.store(1024, std::sync::atomic::Ordering::Relaxed);
assert!(reader.limited_part("other.big", 4096).is_ok(), "768 fits the raised 1024");
}
+49 -9
View File
@@ -162,6 +162,8 @@ impl NewerFile {
let mut top_only: Vec<&str> = container_only.to_vec();
top_only.extend(["format", "format_pending"]);
top_only.extend(self.sections.iter().map(|s| s.key.as_str()));
// a list typed writing leaves out holds only elements kept on their own: not kept twice
top_only.extend(self.elements.iter().map(|(key, _)| key.as_str()).filter(|key| known.get(*key).is_none()));
let kept = vanished(file, known, "", &mut paths, placeholder_tag, &top_only);
self.kept = kept;
self.not_understood.extend(paths);
@@ -185,13 +187,24 @@ impl NewerFile {
lost.push(format!("section `{}`, replaced by what was edited here", section.key));
}
}
// each list's records indexed once, by the identity records are matched by
let mut indexes: HashMap<&str, Positions> = HashMap::new();
for (key, element) in &self.elements {
let name = element.get("name");
// the identity records are matched by: an id first, else a name
let anchor = anchor_of(element);
let taken = |list: &[Value]| anchor.as_ref().is_some_and(|a| Positions::of(list).find(list.len(), a).is_some());
match top.entry(key.clone()).or_insert_with(|| json!([])) {
Value::Array(list) if !taken(list) => list.push(element.clone()),
Value::Array(list) => {
let index = indexes.entry(key.as_str()).or_insert_with(|| Positions::of(list));
if anchor.as_ref().is_some_and(|a| index.find(list.len(), a).is_some()) {
lost.push(format!(
"`{key}` element {}, replaced by what was made here",
name.and_then(Value::as_str).unwrap_or("unnamed")
));
} else {
index.add(element, list.len());
list.push(element.clone());
}
}
_ => lost.push(format!("`{key}` element {}, replaced by what was made here", name.and_then(Value::as_str).unwrap_or("unnamed"))),
}
}
@@ -200,13 +213,18 @@ impl NewerFile {
}
}
/// Whether a string anywhere in `v` is `text`; iterative, so no nesting depth overflows the stack.
fn value_mentions(v: &Value, text: &str) -> bool {
match v {
Value::String(s) => s == text,
Value::Array(a) => a.iter().any(|x| value_mentions(x, text)),
Value::Object(o) => o.values().any(|x| value_mentions(x, text)),
_ => false,
let mut todo = vec![v];
while let Some(v) = todo.pop() {
match v {
Value::String(s) if s == text => return true,
Value::Array(a) => todo.extend(a),
Value::Object(o) => todo.extend(o.values()),
_ => {}
}
}
false
}
/// The first line of a reader's error, without the list of every kind it knows, and without the
@@ -326,7 +344,8 @@ fn vanished(raw: &Value, known: &Value, path: &str, paths: &mut Vec<String>, pla
let mut items = Vec::new();
let positions = Positions::of(known);
for (at, element) in raw.iter().enumerate() {
let anchor = anchor_of(element).unwrap_or(Anchor::Index { at, of: raw.len() });
// an identity typed reading kept: an id it dropped falls back to the name
let anchor = surviving_anchor(element, &positions).unwrap_or(Anchor::Index { at, of: raw.len() });
let Some(partner) = positions.find(known.len(), &anchor).and_then(|i| known.get(i)) else { continue };
let label = match (element.get("name").and_then(Value::as_str), &anchor) {
(Some(name), _) => format!("{path}[{name}]"),
@@ -350,6 +369,17 @@ fn anchor_of(element: &Value) -> Option<Anchor> {
element.get("name").and_then(Value::as_str).map(|name| Anchor::Name(name.to_string()))
}
/// The identity of `element` that `known`'s records still carry: its id, or else its name. An
/// element with an identity neither of which survived has none.
fn surviving_anchor(element: &Value, known: &Positions) -> Option<Anchor> {
let id = element.get("id").and_then(Value::as_u64).map(Anchor::Id);
let name = element.get("name").and_then(Value::as_str).map(|n| Anchor::Name(n.to_string()));
match (id, name) {
(Some(id), Some(name)) => Some(if known.find(usize::MAX, &id).is_some() { id } else { name }),
(id, name) => id.or(name),
}
}
/// Where each id and name first appears in a list, so matching records is not a scan per record.
struct Positions {
ids: HashMap<u64, usize>,
@@ -370,6 +400,16 @@ impl Positions {
Positions { ids, names }
}
/// Count `x`, now at `i`, in.
fn add(&mut self, x: &Value, i: usize) {
if let Some(id) = x.get("id").and_then(Value::as_u64) {
self.ids.entry(id).or_insert(i);
}
if let Some(name) = x.get("name").and_then(Value::as_str) {
self.names.entry(name.to_string()).or_insert(i);
}
}
/// The position of the record `anchor` names in a list of `len` records (an index anchor
/// only while the list keeps its length).
fn find(&self, len: usize, anchor: &Anchor) -> Option<usize> {
+41
View File
@@ -164,3 +164,44 @@ fn container_only_fields_are_top_level() {
assert!(said.restore_into(&mut saved).is_empty());
assert_eq!(saved, json!({"layer": {"thumbnail": "nested"}}));
}
/// An element kept on its own is not kept again with its list when typed writing leaves the list out.
#[test]
fn a_kept_element_is_not_restored_twice() {
let mut said = NewerFile::new("App", &newer());
said.keep_element("items", json!({"kind": "future"}), "unnamed".into());
said.compare(&json!({"items": [{"kind": "future"}]}), &json!({}), "placeholder", &[]);
let mut saved = json!({});
assert!(said.restore_into(&mut saved).is_empty());
assert_eq!(saved, json!({"items": [{"kind": "future"}]}));
}
/// A record whose id typed reading drops is matched by its name, and its fields go back.
#[test]
fn a_dropped_id_falls_back_to_the_name() {
let mut said = NewerFile::new("App", &newer());
said.compare(&json!({"items": [{"id": 7, "name": "a", "extra": 1}]}), &json!({"items": [{"name": "a"}]}), "placeholder", &[]);
let mut saved = json!({"items": [{"name": "a"}]});
assert!(said.restore_into(&mut saved).is_empty());
assert_eq!(saved, json!({"items": [{"name": "a", "id": 7, "extra": 1}]}));
}
/// Kept elements sharing a list are checked against one index, the ones put back included.
#[test]
fn kept_elements_check_each_other() {
let mut said = NewerFile::new("App", &newer());
said.keep_element("items", json!({"id": 1, "v": "first"}), "1".into());
said.keep_element("items", json!({"id": 1, "v": "again"}), "1 again".into());
let mut saved = json!({"items": []});
assert_eq!(said.restore_into(&mut saved).len(), 1);
assert_eq!(saved, json!({"items": [{"id": 1, "v": "first"}]}));
// deep nesting is searched without recursion
let mut deep = json!("needle");
for _ in 0..20_000 {
deep = Value::Array(vec![deep]);
}
said.keep_section("deep", deep, None, "test");
assert!(said.mentions("needle"));
// serde_json drops deep values recursively; leak it rather than overflow the test thread
std::mem::forget(said);
}