Preserve quoted multiline variable CSV records (#2242)

This commit is contained in:
Harrison O'Connor-Hoover authored and GitHub committed 2026-10-10 02:17:09 -04:00
1 parent 26d40dbd0f
commit 2db1b26d9d
2 files changed
+102 -4

No files matched your search

+26 -4
View File
@@ -272,15 +272,15 @@ fn import_data_sets(s: &mut Session, p: &Value) -> Result<Value> {
let delim = p.get("delimiter").and_then(Value::as_str).and_then(|d| d.bytes().next()).unwrap_or(b',');
let text = std::fs::read_to_string(path).map_err(|e| EngineError::Other(format!("read `{path}`: {e}")))?;
let vars = s.active().ok_or(EngineError::NoDocument)?.doc.variables.clone();
let mut lines = text.lines().filter(|l| !l.trim().is_empty());
let header: Vec<String> = lines.next().map(|h| split_csv(h, delim)).unwrap_or_default();
let mut records = csv_records(&text).filter(|r| !r.trim().is_empty());
let header: Vec<String> = records.next().map(|h| split_csv(h, delim)).unwrap_or_default();
if header.is_empty() {
return Err(bad("file.import.variableDataSets", "empty CSV"));
}
let named = header[0].trim().is_empty() || header[0].eq_ignore_ascii_case("dataset");
let mut sets = Vec::new();
for (row, line) in lines.enumerate() {
let cells = split_csv(line, delim);
for (row, record) in records.enumerate() {
let cells = split_csv(record, delim);
let name = if named {
cells.first().cloned().filter(|c| !c.trim().is_empty()).unwrap_or_else(|| format!("Data Set {}", row + 1))
} else {
@@ -305,6 +305,28 @@ fn import_data_sets(s: &mut Session, p: &Value) -> Result<Value> {
Ok(json!({"imported": n}))
}
/// Split logical CSV records without changing line endings inside quoted fields.
fn csv_records(text: &str) -> impl Iterator<Item = &str> {
let mut chars = text.char_indices();
let mut start = 0;
std::iter::from_fn(move || {
let mut quoted = false;
for (end, c) in chars.by_ref() {
if c == '"' {
// An escaped quote pair toggles twice, leaving the surrounding state intact.
quoted = !quoted;
} else if c == '\n' && !quoted {
let record = text.get(start..end)?;
start = end.saturating_add(1);
return Some(record.strip_suffix('\r').unwrap_or(record));
}
}
let record = text.get(start..)?;
start = text.len();
(!record.is_empty()).then_some(record)
})
}
/// Minimal CSV field splitter with double-quote support.
fn split_csv(line: &str, delim: u8) -> Vec<String> {
let mut out = Vec::new();
+76
View File
@@ -174,6 +174,82 @@ fn csv_import_preserves_unicode_headers_names_and_text() {
std::fs::remove_dir_all(dir).unwrap();
}
#[test]
fn csv_multiline_values_apply_persist_and_export_as_two_data_sets() {
for (ending, suffix) in [("\n", "lf"), ("\r\n", "crlf")] {
let (mut s, _photo, badge, title) = session();
s.execute(
"image.variables.define",
json!({"defs": [
{"name": "showBadge", "layer": badge.0, "type": "visibility"},
{"name": "headline", "layer": title.0, "type": "textReplacement"},
]}),
)
.unwrap();
let first = format!("Café, \"quoted\"{ending}{ending}日本語 👩‍🎨");
let second = format!("Fin{ending}e\u{301}");
let dir = tmp(&format!("multiline-{suffix}"));
let csv = format!("{dir}/sets.csv");
// Blank physical lines inside a quoted field are data; blank records between rows are
// ignored. The last quoted field ends at EOF, without a final record terminator.
std::fs::write(&csv, format!("DataSet,showBadge,headline{ending}one,true,\"{}\"{ending}{ending}two,false,\"{second}\"", first.replace('"', "\"\"")))
.unwrap();
let result = s.execute("file.import.variableDataSets", json!({"path": csv})).unwrap();
assert_eq!(result["imported"], 2, "{suffix}");
let rows = s.execute("variables.list", json!({})).unwrap();
assert_eq!(rows["dataSets"][0]["values"][1]["value"], first);
assert_eq!(rows["dataSets"][1]["values"][1]["value"], second);
for (name, text, visible) in [("one", &first, true), ("two", &second, false)] {
s.execute("image.applyDataSet", json!({"name": name})).unwrap();
assert_eq!(text_of(&s, title), *text);
assert_eq!(doc(&s).layer(badge).unwrap().visible, visible);
}
let native = format!("{dir}/multiline.pcraft");
crate::file_cmds::save_doc(doc(&s), &native, None).unwrap();
let mut reopened = Session::new();
crate::file_cmds::open_bytes_as(&mut reopened, "multiline.pcraft", &std::fs::read(&native).unwrap(), None, Some(native)).unwrap();
assert_eq!(doc(&reopened).variables, doc(&s).variables);
assert_eq!(text_of(&reopened, title), second);
reopened.execute("image.applyDataSet", json!({"name": "one"})).unwrap();
assert_eq!(text_of(&reopened, title), first);
let before_export = doc(&reopened).clone();
let out = reopened.execute("file.export.dataSetsAsFiles", json!({"dir": format!("{dir}/out"), "format": "png"})).unwrap();
assert_eq!(out["count"], 2);
let files = out["files"].as_array().unwrap();
assert_eq!(files.len(), 2);
let images: Vec<_> = files
.iter()
.map(|path| {
let path = path.as_str().unwrap();
photocraft_io::import(path, &std::fs::read(path).unwrap()).unwrap().document
})
.collect();
assert!(images.iter().all(|image| image.size == doc(&reopened).size));
assert_ne!(images[0].layers[0].surface(), images[1].layers[0].surface(), "the exported rows have different content");
assert_eq!(*doc(&reopened), before_export, "export keeps the live document unchanged");
std::fs::remove_dir_all(dir).unwrap();
}
}
#[test]
fn csv_multiline_default_names_count_logical_nonblank_records() {
let dir = tmp("multiline-default-names");
let csv = format!("{dir}/sets.csv");
for (header, first, second) in [("headline", "\"first\n\nline\"", "\"\""), ("DataSet,headline", ",\"first\n\nline\"", ",\"\"")] {
let (mut s, _, _, title) = session();
s.execute("image.variables.define", json!({"defs": [{"name": "headline", "layer": title.0, "type": "textReplacement"}]})).unwrap();
std::fs::write(&csv, format!("\n \t\n{header}\n\n{first}\n \n{second}")).unwrap();
assert_eq!(s.execute("file.import.variableDataSets", json!({"path": csv})).unwrap()["imported"], 2);
let rows = s.execute("variables.list", json!({})).unwrap();
assert_eq!(rows["dataSets"][0]["name"], "Data Set 1");
assert_eq!(rows["dataSets"][1]["name"], "Data Set 2");
assert_eq!(rows["dataSets"][0]["values"][0]["value"], "first\n\nline");
assert_eq!(rows["dataSets"][1]["values"][0]["value"], "", "a quoted empty field is a record");
}
std::fs::remove_dir_all(dir).unwrap();
}
#[test]
fn csv_import_preserves_unicode_pixel_paths() {
let (mut s, photo, _badge, _title) = session();