mirror of
https://github.com/storytold/craft-libs.git
synced 2026-11-16 23:42:27 +00:00
craft-format: string-aware step detection, live expanded budget, unreadable names refused, retention identity fixes
This commit is contained in:
1 parent
b9636d757a
commit
f299664d7c
6 files changed
+216
-30
No files matched your search
+88
-20
@@ -91,33 +91,101 @@ fn declares_step(text: &str) -> bool {
|
||||
})
|
||||
}
|
||||
|
||||
/// `text` with its `//` and (nested) `/* */` comments taken out; a comment's line breaks stay, so
|
||||
/// what follows a block comment is still on its own line.
|
||||
/// `text` as code only: its `//` and (nested) `/* */` comments become a space, and the contents
|
||||
/// of its string literals (plain, byte and raw) are dropped, line breaks kept throughout. A comment
|
||||
/// marker inside a string is not a comment, and `fn/* note */step` still reads as `fn step`.
|
||||
fn without_comments(text: &str) -> String {
|
||||
let chars: Vec<char> = text.chars().collect();
|
||||
let at = |i: usize| chars.get(i).copied();
|
||||
let mut out = String::with_capacity(text.len());
|
||||
let mut chars = text.chars().peekable();
|
||||
let mut depth = 0usize;
|
||||
while let Some(c) = chars.next() {
|
||||
match (c, chars.peek().copied()) {
|
||||
('/', Some('*')) => {
|
||||
chars.next();
|
||||
depth += 1;
|
||||
}
|
||||
('*', Some('/')) if depth > 0 => {
|
||||
chars.next();
|
||||
depth -= 1;
|
||||
}
|
||||
('/', Some('/')) if depth == 0 => {
|
||||
for c in chars.by_ref() {
|
||||
let (mut i, mut depth) = (0usize, 0usize);
|
||||
while let Some(c) = at(i) {
|
||||
let next = at(i + 1);
|
||||
if depth > 0 {
|
||||
match (c, next) {
|
||||
('/', Some('*')) => (depth, i) = (depth + 1, i + 2),
|
||||
('*', Some('/')) => (depth, i) = (depth - 1, i + 2),
|
||||
_ => {
|
||||
if c == '\n' {
|
||||
out.push('\n');
|
||||
break;
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
('\n', _) => out.push('\n'),
|
||||
(c, _) if depth == 0 => out.push(c),
|
||||
_ => {}
|
||||
if depth == 0 {
|
||||
out.push(' ');
|
||||
}
|
||||
continue;
|
||||
}
|
||||
let ident_before = i > 0 && at(i - 1).is_some_and(|p| p.is_alphanumeric() || p == '_');
|
||||
match (c, next) {
|
||||
('/', Some('/')) => {
|
||||
while at(i).is_some_and(|c| c != '\n') {
|
||||
i += 1;
|
||||
}
|
||||
out.push(' ');
|
||||
}
|
||||
('/', Some('*')) => (depth, i) = (1, i + 2),
|
||||
('"', _) => {
|
||||
out.push('"');
|
||||
i += 1;
|
||||
while let Some(c) = at(i) {
|
||||
i += 1;
|
||||
match c {
|
||||
'\\' => i += 1,
|
||||
'"' => break,
|
||||
'\n' => out.push('\n'),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
out.push('"');
|
||||
}
|
||||
// a raw string, `r"…"` or `r#"…"#`, after a byte prefix or on its own
|
||||
('r', Some('"' | '#'))
|
||||
if !ident_before || (i > 0 && at(i - 1) == Some('b') && !(i > 1 && at(i - 2).is_some_and(|p| p.is_alphanumeric() || p == '_'))) =>
|
||||
{
|
||||
let mut j = i + 1;
|
||||
let mut hashes = 0;
|
||||
while at(j) == Some('#') {
|
||||
hashes += 1;
|
||||
j += 1;
|
||||
}
|
||||
if at(j) != Some('"') {
|
||||
out.push(c);
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
out.push('"');
|
||||
i = j + 1;
|
||||
while let Some(c) = at(i) {
|
||||
if c == '"' && (1..=hashes).all(|k| at(i + k) == Some('#')) {
|
||||
i += 1 + hashes;
|
||||
break;
|
||||
}
|
||||
if c == '\n' {
|
||||
out.push('\n');
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
out.push('"');
|
||||
}
|
||||
// a character literal (`'"'`, `'\''`), not a lifetime
|
||||
('\'', Some('\\')) => {
|
||||
i += 2;
|
||||
while at(i).is_some_and(|c| c != '\'') {
|
||||
i += 1;
|
||||
}
|
||||
i += 1;
|
||||
out.push_str("' '");
|
||||
}
|
||||
('\'', Some(_)) if at(i + 2) == Some('\'') => {
|
||||
i += 3;
|
||||
out.push_str("' '");
|
||||
}
|
||||
_ => {
|
||||
out.push(c);
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
|
||||
@@ -83,6 +83,14 @@ fn a_step_is_a_declaration_not_a_mention() {
|
||||
assert!(!declares_step("//! Documentation only.\n/*\nfn step() {}\n*/\n"));
|
||||
assert!(!declares_step("/* outer /* inner */\nfn step() {}\n*/\nconst X: u8 = 1;\n"));
|
||||
assert!(declares_step("/* a note */\nfn step(v: &mut Value) {}\n"));
|
||||
// comment markers inside strings and character literals are not comments
|
||||
assert!(declares_step("//! A change with a step.\nconst MARKER: &str = \"/*\";\npub fn step(_v: &mut Value) -> Result { Ok(()) }\n"));
|
||||
assert!(declares_step("const A: &str = r#\"/* \"quoted\" \"#;\nconst B: &[u8] = br\"/*\";\nfn step(v: &mut Value) {}\n"));
|
||||
assert!(declares_step("const Q: char = '\"';\nconst E: char = '\\'';\nfn step(v: &mut Value) { let _: &'static str = \"\"; }\n"));
|
||||
assert!(declares_step("const S: &str = \"a \\\" /* b\";\nfn step(v: &mut Value) {}\n"));
|
||||
assert!(!declares_step("const S: &str = \"\nfn step(v) \";\n"), "inside a string");
|
||||
// a comment between the words is whitespace
|
||||
assert!(declares_step("pub fn/* migration */step(v: &mut Value) {}\n"));
|
||||
assert!(declares_step("const X: u8 = 1; // fn step() is below\nfn step(v: &mut Value) {}\n"));
|
||||
assert!(declares_step("//! a note\npub(crate) fn step(v: &mut Value) -> Result<(), E> { Ok(()) }\n"));
|
||||
assert!(declares_step("//! a note\n fn step(v: &mut Value) {}\n"));
|
||||
|
||||
@@ -262,7 +262,11 @@ impl Reader {
|
||||
// size within its configured limit.
|
||||
let mut name_set = BTreeSet::new();
|
||||
for entry_idx in 0..archive.len() {
|
||||
if let Ok(Some(name)) = archive.entry_name(entry_idx) {
|
||||
// an entry whose name can't be read would be left out of the package unnoticed
|
||||
let Ok(Some(name)) = archive.entry_name(entry_idx) else {
|
||||
return Err(invalid("duplicate or invalid archive entry"));
|
||||
};
|
||||
{
|
||||
if name.is_empty()
|
||||
|| name.contains('\\')
|
||||
|| name.split('/').any(|c| c.is_empty() || c == "." || c == "..")
|
||||
@@ -309,6 +313,12 @@ impl Reader {
|
||||
if expected > limit as u64 {
|
||||
return Err(invalid("part too large"));
|
||||
}
|
||||
// the expanded budget is read each time: a raised (or lowered) one applies at once, the
|
||||
// bytes already read still counting against it
|
||||
let used = self.archive.budget().used();
|
||||
let budget = self.archive.budget_mut();
|
||||
*budget = craft_zip::Budget::new(budget.what(), (self.limits.max_expanded)() as u64);
|
||||
let _ = budget.charge(used.min(budget.limit()));
|
||||
// Read within budget. craft-zip's read() handles decompression bombs via the limit.
|
||||
let bytes = self.archive.read(name, limit as u64).map_err(invalid)?.ok_or_else(|| invalid("part not found"))?;
|
||||
if bytes.len() as u64 != expected {
|
||||
|
||||
@@ -220,3 +220,22 @@ fn cached_parts_keep_the_current_part_limits() {
|
||||
assert!(reader.document(4096).is_err());
|
||||
assert!(reader.limited_blob(&reference, "blob", "bin", 4096).is_err());
|
||||
}
|
||||
|
||||
/// A raised expanded budget applies to the open reader, and what was read still counts.
|
||||
#[test]
|
||||
fn a_raised_expanded_budget_applies_at_once() {
|
||||
static BUDGET: std::sync::atomic::AtomicUsize = std::sync::atomic::AtomicUsize::new(600);
|
||||
let limits = Limits { max_expanded: || BUDGET.load(std::sync::atomic::Ordering::Relaxed), part_limit: |_| 1 << 20, ..LIMITS };
|
||||
let wide = Limits { max_expanded: || 1 << 20, ..limits };
|
||||
let mut writer = Writer::new(wide);
|
||||
writer.part("document.json", &[b' '; 256], false).unwrap();
|
||||
writer.part("other.big", &[1; 512], false).unwrap();
|
||||
let bytes = writer.finish().unwrap();
|
||||
BUDGET.store(1 << 20, std::sync::atomic::Ordering::Relaxed);
|
||||
let mut reader = Reader::new(Cursor::new(bytes), limits).unwrap();
|
||||
BUDGET.store(600, std::sync::atomic::Ordering::Relaxed);
|
||||
assert!(reader.limited_part("document.json", 4096).is_ok());
|
||||
assert!(reader.limited_part("other.big", 4096).is_err(), "256 read + 512 is past 600");
|
||||
BUDGET.store(1024, std::sync::atomic::Ordering::Relaxed);
|
||||
assert!(reader.limited_part("other.big", 4096).is_ok(), "768 fits the raised 1024");
|
||||
}
|
||||
@@ -162,6 +162,8 @@ impl NewerFile {
|
||||
let mut top_only: Vec<&str> = container_only.to_vec();
|
||||
top_only.extend(["format", "format_pending"]);
|
||||
top_only.extend(self.sections.iter().map(|s| s.key.as_str()));
|
||||
// a list typed writing leaves out holds only elements kept on their own: not kept twice
|
||||
top_only.extend(self.elements.iter().map(|(key, _)| key.as_str()).filter(|key| known.get(*key).is_none()));
|
||||
let kept = vanished(file, known, "", &mut paths, placeholder_tag, &top_only);
|
||||
self.kept = kept;
|
||||
self.not_understood.extend(paths);
|
||||
@@ -185,13 +187,24 @@ impl NewerFile {
|
||||
lost.push(format!("section `{}`, replaced by what was edited here", section.key));
|
||||
}
|
||||
}
|
||||
// each list's records indexed once, by the identity records are matched by
|
||||
let mut indexes: HashMap<&str, Positions> = HashMap::new();
|
||||
for (key, element) in &self.elements {
|
||||
let name = element.get("name");
|
||||
// the identity records are matched by: an id first, else a name
|
||||
let anchor = anchor_of(element);
|
||||
let taken = |list: &[Value]| anchor.as_ref().is_some_and(|a| Positions::of(list).find(list.len(), a).is_some());
|
||||
match top.entry(key.clone()).or_insert_with(|| json!([])) {
|
||||
Value::Array(list) if !taken(list) => list.push(element.clone()),
|
||||
Value::Array(list) => {
|
||||
let index = indexes.entry(key.as_str()).or_insert_with(|| Positions::of(list));
|
||||
if anchor.as_ref().is_some_and(|a| index.find(list.len(), a).is_some()) {
|
||||
lost.push(format!(
|
||||
"`{key}` element {}, replaced by what was made here",
|
||||
name.and_then(Value::as_str).unwrap_or("unnamed")
|
||||
));
|
||||
} else {
|
||||
index.add(element, list.len());
|
||||
list.push(element.clone());
|
||||
}
|
||||
}
|
||||
_ => lost.push(format!("`{key}` element {}, replaced by what was made here", name.and_then(Value::as_str).unwrap_or("unnamed"))),
|
||||
}
|
||||
}
|
||||
@@ -200,13 +213,18 @@ impl NewerFile {
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether a string anywhere in `v` is `text`; iterative, so no nesting depth overflows the stack.
|
||||
fn value_mentions(v: &Value, text: &str) -> bool {
|
||||
match v {
|
||||
Value::String(s) => s == text,
|
||||
Value::Array(a) => a.iter().any(|x| value_mentions(x, text)),
|
||||
Value::Object(o) => o.values().any(|x| value_mentions(x, text)),
|
||||
_ => false,
|
||||
let mut todo = vec![v];
|
||||
while let Some(v) = todo.pop() {
|
||||
match v {
|
||||
Value::String(s) if s == text => return true,
|
||||
Value::Array(a) => todo.extend(a),
|
||||
Value::Object(o) => todo.extend(o.values()),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// The first line of a reader's error, without the list of every kind it knows, and without the
|
||||
@@ -326,7 +344,8 @@ fn vanished(raw: &Value, known: &Value, path: &str, paths: &mut Vec<String>, pla
|
||||
let mut items = Vec::new();
|
||||
let positions = Positions::of(known);
|
||||
for (at, element) in raw.iter().enumerate() {
|
||||
let anchor = anchor_of(element).unwrap_or(Anchor::Index { at, of: raw.len() });
|
||||
// an identity typed reading kept: an id it dropped falls back to the name
|
||||
let anchor = surviving_anchor(element, &positions).unwrap_or(Anchor::Index { at, of: raw.len() });
|
||||
let Some(partner) = positions.find(known.len(), &anchor).and_then(|i| known.get(i)) else { continue };
|
||||
let label = match (element.get("name").and_then(Value::as_str), &anchor) {
|
||||
(Some(name), _) => format!("{path}[{name}]"),
|
||||
@@ -350,6 +369,17 @@ fn anchor_of(element: &Value) -> Option<Anchor> {
|
||||
element.get("name").and_then(Value::as_str).map(|name| Anchor::Name(name.to_string()))
|
||||
}
|
||||
|
||||
/// The identity of `element` that `known`'s records still carry: its id, or else its name. An
|
||||
/// element with an identity neither of which survived has none.
|
||||
fn surviving_anchor(element: &Value, known: &Positions) -> Option<Anchor> {
|
||||
let id = element.get("id").and_then(Value::as_u64).map(Anchor::Id);
|
||||
let name = element.get("name").and_then(Value::as_str).map(|n| Anchor::Name(n.to_string()));
|
||||
match (id, name) {
|
||||
(Some(id), Some(name)) => Some(if known.find(usize::MAX, &id).is_some() { id } else { name }),
|
||||
(id, name) => id.or(name),
|
||||
}
|
||||
}
|
||||
|
||||
/// Where each id and name first appears in a list, so matching records is not a scan per record.
|
||||
struct Positions {
|
||||
ids: HashMap<u64, usize>,
|
||||
@@ -370,6 +400,16 @@ impl Positions {
|
||||
Positions { ids, names }
|
||||
}
|
||||
|
||||
/// Count `x`, now at `i`, in.
|
||||
fn add(&mut self, x: &Value, i: usize) {
|
||||
if let Some(id) = x.get("id").and_then(Value::as_u64) {
|
||||
self.ids.entry(id).or_insert(i);
|
||||
}
|
||||
if let Some(name) = x.get("name").and_then(Value::as_str) {
|
||||
self.names.entry(name.to_string()).or_insert(i);
|
||||
}
|
||||
}
|
||||
|
||||
/// The position of the record `anchor` names in a list of `len` records (an index anchor
|
||||
/// only while the list keeps its length).
|
||||
fn find(&self, len: usize, anchor: &Anchor) -> Option<usize> {
|
||||
|
||||
@@ -164,3 +164,44 @@ fn container_only_fields_are_top_level() {
|
||||
assert!(said.restore_into(&mut saved).is_empty());
|
||||
assert_eq!(saved, json!({"layer": {"thumbnail": "nested"}}));
|
||||
}
|
||||
|
||||
/// An element kept on its own is not kept again with its list when typed writing leaves the list out.
|
||||
#[test]
|
||||
fn a_kept_element_is_not_restored_twice() {
|
||||
let mut said = NewerFile::new("App", &newer());
|
||||
said.keep_element("items", json!({"kind": "future"}), "unnamed".into());
|
||||
said.compare(&json!({"items": [{"kind": "future"}]}), &json!({}), "placeholder", &[]);
|
||||
let mut saved = json!({});
|
||||
assert!(said.restore_into(&mut saved).is_empty());
|
||||
assert_eq!(saved, json!({"items": [{"kind": "future"}]}));
|
||||
}
|
||||
|
||||
/// A record whose id typed reading drops is matched by its name, and its fields go back.
|
||||
#[test]
|
||||
fn a_dropped_id_falls_back_to_the_name() {
|
||||
let mut said = NewerFile::new("App", &newer());
|
||||
said.compare(&json!({"items": [{"id": 7, "name": "a", "extra": 1}]}), &json!({"items": [{"name": "a"}]}), "placeholder", &[]);
|
||||
let mut saved = json!({"items": [{"name": "a"}]});
|
||||
assert!(said.restore_into(&mut saved).is_empty());
|
||||
assert_eq!(saved, json!({"items": [{"name": "a", "id": 7, "extra": 1}]}));
|
||||
}
|
||||
|
||||
/// Kept elements sharing a list are checked against one index, the ones put back included.
|
||||
#[test]
|
||||
fn kept_elements_check_each_other() {
|
||||
let mut said = NewerFile::new("App", &newer());
|
||||
said.keep_element("items", json!({"id": 1, "v": "first"}), "1".into());
|
||||
said.keep_element("items", json!({"id": 1, "v": "again"}), "1 again".into());
|
||||
let mut saved = json!({"items": []});
|
||||
assert_eq!(said.restore_into(&mut saved).len(), 1);
|
||||
assert_eq!(saved, json!({"items": [{"id": 1, "v": "first"}]}));
|
||||
// deep nesting is searched without recursion
|
||||
let mut deep = json!("needle");
|
||||
for _ in 0..20_000 {
|
||||
deep = Value::Array(vec![deep]);
|
||||
}
|
||||
said.keep_section("deep", deep, None, "test");
|
||||
assert!(said.mentions("needle"));
|
||||
// serde_json drops deep values recursively; leak it rather than overflow the test thread
|
||||
std::mem::forget(said);
|
||||
}
|
||||
Reference in new issue
Block a user