mirror of
https://github.com/storytold/designcraft.git
synced 2026-10-13 23:17:42 +00:00
Data merge: review fixes (bounded reads, no truncation, safe links, docs)
- Remove the docs/superpowers planning spec. - textimport: cap decompressed zip parts at 64 MiB (protects Place and xlsx data sources); oversized part is an error. Regression test. - datamerge: refuse data files over 256 MiB and skip image candidates over 256 MiB by metadata before reading. - datamerge: assign_page uses get_mut instead of indexing spreads. - datamerge: one/limit over u32 are errors, not truncated. - select_records dedupes with a HashSet (was O(n^2)), order kept. - Hyperlink cells: only http, https, ftp and mailto schemes link. - Document data.* commands and data.merge creating a new document. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Brandon ThomasandClaude Opus 5.5 committed
1 parent
a2078ece67
commit
2531f039fe
9 files changed
+145
-232
No files matched your search
@@ -324,7 +324,14 @@ fn place_image(
|
||||
report: &mut FillReport,
|
||||
) {
|
||||
let cands = image_candidates(cell, data_dir, doc_dir);
|
||||
let found = cands.iter().find_map(|p| std::fs::read(p).ok().filter(|b| !b.is_empty()).map(|b| (p.clone(), b)));
|
||||
let found = cands.iter().find_map(|p| {
|
||||
// Refuse huge files before reading them; a too-large image counts as missing.
|
||||
let len = std::fs::metadata(p).ok()?.len();
|
||||
if len == 0 || len > super::MAX_IMAGE_FILE_BYTES {
|
||||
return None;
|
||||
}
|
||||
std::fs::read(p).ok().filter(|b| !b.is_empty()).map(|b| (p.clone(), b))
|
||||
});
|
||||
let Some((path, bytes)) = found else {
|
||||
if let Some(it) = doc.item_mut(item) {
|
||||
it.content = Content::Unassigned;
|
||||
@@ -414,6 +421,15 @@ fn classify_link(cell: &str) -> Option<HyperlinkDest> {
|
||||
if value.is_empty() {
|
||||
return None;
|
||||
}
|
||||
// An explicit scheme must be a web or mail one; `javascript:`, `file:` and the like stay text.
|
||||
if let Some((scheme, _)) = value.split_once(':') {
|
||||
let is_scheme = scheme.chars().next().is_some_and(|c| c.is_ascii_alphabetic())
|
||||
&& scheme.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '-' | '.'));
|
||||
if is_scheme {
|
||||
let scheme = scheme.to_ascii_lowercase();
|
||||
return matches!(scheme.as_str(), "http" | "https" | "ftp" | "mailto").then(|| HyperlinkDest::Url(value.to_string()));
|
||||
}
|
||||
}
|
||||
if value.contains("://") || value.chars().any(char::is_whitespace) {
|
||||
return Some(HyperlinkDest::Url(value.to_string()));
|
||||
}
|
||||
@@ -436,3 +452,19 @@ fn anchor_is_parent(doc: &designcraft_doc::Document, anchor: &PlaceholderAnchor)
|
||||
fn cell_of<'a>(fields: &[DataField], row: &'a [String], name: &str) -> &'a str {
|
||||
fields.iter().position(|f| f.name == name).and_then(|i| row.get(i)).map(String::as_str).unwrap_or("")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod link_tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn only_web_and_mail_schemes_become_links() {
|
||||
for ok in ["https://example.com", "HTTP://example.com", "ftp://files.example.com", "mailto:a@example.com", "www.example.com"] {
|
||||
assert!(matches!(classify_link(ok), Some(HyperlinkDest::Url(_))), "{ok}");
|
||||
}
|
||||
assert!(matches!(classify_link("a@example.com"), Some(HyperlinkDest::Email(_))));
|
||||
for bad in ["javascript:alert(1)", "file:///etc/passwd", "data:text/html,hi", "vbscript:x", ""] {
|
||||
assert!(classify_link(bad).is_none(), "{bad}");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -413,7 +413,8 @@ fn block_of(doc: &Document) -> (Rect, Vec<ItemId>) {
|
||||
|
||||
fn assign_page(dst: &mut Document, abs: usize, from: &Page) {
|
||||
let Some((si, pi)) = dst.page_loc(abs) else { return };
|
||||
let sp = Arc::make_mut(&mut dst.spreads[si]);
|
||||
let Some(sp) = dst.spreads.get_mut(si) else { return };
|
||||
let sp = Arc::make_mut(sp);
|
||||
let Some(p) = sp.pages.get_mut(pi) else { return };
|
||||
let size_changed = (p.width - from.width).abs() > 0.01 || (p.height - from.height).abs() > 0.01;
|
||||
p.guides = from.guides.clone();
|
||||
@@ -757,7 +758,21 @@ fn parse_file(bytes: &[u8], name: &str, delimiter: Option<Delimiter>, sheet: Opt
|
||||
table_from_bytes(bytes, name, delimiter)
|
||||
}
|
||||
|
||||
/// Largest data source file read (bytes); bigger files are refused before reading.
|
||||
pub(crate) const MAX_DATA_FILE_BYTES: u64 = 256 * 1024 * 1024;
|
||||
/// Largest image file a merge places (bytes); bigger candidates are skipped before reading.
|
||||
pub(crate) const MAX_IMAGE_FILE_BYTES: u64 = 256 * 1024 * 1024;
|
||||
|
||||
fn read_path(path: &str) -> std::result::Result<Vec<u8>, String> {
|
||||
read_path_capped(path, MAX_DATA_FILE_BYTES)
|
||||
}
|
||||
|
||||
fn read_path_capped(path: &str, cap: u64) -> std::result::Result<Vec<u8>, String> {
|
||||
match std::fs::metadata(path) {
|
||||
Ok(m) if m.len() > cap => return Err(format!("{path}: the data file is larger than {} MiB", cap / (1024 * 1024))),
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Err(format!("{path}: the data file is missing")),
|
||||
_ => {}
|
||||
}
|
||||
match std::fs::read(path) {
|
||||
Ok(b) => Ok(b),
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => Err(format!("{path}: the data file is missing")),
|
||||
@@ -794,7 +809,7 @@ fn apply_options(opt: &mut MergeOptions, p: &Value) -> std::result::Result<(), S
|
||||
opt.records = v.to_string();
|
||||
}
|
||||
if let Some(n) = p.get("one").and_then(Value::as_u64) {
|
||||
opt.one = n as u32;
|
||||
opt.one = u32::try_from(n).map_err(|_| "one is too large".to_string())?;
|
||||
} else if p.get("one").is_some() {
|
||||
return Err("one must be a record number".into());
|
||||
}
|
||||
@@ -837,7 +852,7 @@ fn apply_options(opt: &mut MergeOptions, p: &Value) -> std::result::Result<(), S
|
||||
if v.is_null() {
|
||||
opt.limit = None;
|
||||
} else if let Some(n) = v.as_u64() {
|
||||
opt.limit = Some(n as u32);
|
||||
opt.limit = Some(u32::try_from(n).map_err(|_| "limit is too large".to_string())?);
|
||||
} else {
|
||||
return Err("limit must be a number".into());
|
||||
}
|
||||
|
||||
@@ -152,6 +152,7 @@ fn cell_string(v: Option<&Value>) -> Result<String, String> {
|
||||
/// 0-based indexes into `count` rows, in first-seen order.
|
||||
pub fn select_records(count: usize, records: &str, one: u32, range: &str, limit: Option<u32>) -> Result<Vec<usize>, String> {
|
||||
let mut idx = Vec::new();
|
||||
let mut seen = std::collections::HashSet::new();
|
||||
match records {
|
||||
"all" => idx.extend(0..count),
|
||||
"one" => {
|
||||
@@ -176,7 +177,7 @@ pub fn select_records(count: usize, records: &str, one: u32, range: &str, limit:
|
||||
let mut n = a;
|
||||
while n <= b {
|
||||
let i = record_index(n, count)?;
|
||||
if !idx.contains(&i) {
|
||||
if seen.insert(i) {
|
||||
idx.push(i);
|
||||
}
|
||||
n = n.saturating_add(1);
|
||||
@@ -184,7 +185,7 @@ pub fn select_records(count: usize, records: &str, one: u32, range: &str, limit:
|
||||
} else {
|
||||
let n = parse_record_no(token)?;
|
||||
let i = record_index(n, count)?;
|
||||
if !idx.contains(&i) {
|
||||
if seen.insert(i) {
|
||||
idx.push(i);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@ use designcraft_doc::{Content, DataField, DataFieldKind, DataSource, Delimiter,
|
||||
use serde_json::{Value, json};
|
||||
|
||||
use super::parse::{self, parse_csv};
|
||||
use super::{relative_between, resolve_sources_on_open};
|
||||
use super::{read_path_capped, relative_between, resolve_sources_on_open};
|
||||
use crate::Session;
|
||||
|
||||
fn scratch(name: &str) -> PathBuf {
|
||||
@@ -634,6 +634,11 @@ fn update_remove_preview_inline_rows_and_old_files() {
|
||||
assert!(msg.contains("reversed"), "{msg}");
|
||||
let msg = err(&mut s, "data.merge", json!({"csv": "Name\nAda\n", "limit": 0}));
|
||||
assert!(msg.contains("limit"), "{msg}");
|
||||
// Numbers past u32 are an error, not a silent truncation to a small record number.
|
||||
let msg = err(&mut s, "data.merge", json!({"csv": "Name\nAda\n", "records": "one", "one": 4_294_967_297u64}));
|
||||
assert!(msg.contains("too large"), "{msg}");
|
||||
let msg = err(&mut s, "data.merge", json!({"csv": "Name\nAda\n", "limit": 4_294_967_296u64}));
|
||||
assert!(msg.contains("too large"), "{msg}");
|
||||
assert_eq!(s.documents().len(), before);
|
||||
|
||||
let fresh = dir.join("fresh.designcraft");
|
||||
@@ -785,3 +790,21 @@ fn tile_slots_match_the_insets() {
|
||||
let slots: Vec<_> = plan.hits.iter().map(|h| (h.col, h.row)).collect();
|
||||
assert_eq!(slots, vec![(0, 0), (1, 0), (2, 0), (0, 1), (1, 1), (2, 1)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn select_records_dedupes_large_ranges_in_first_seen_order() {
|
||||
let idx = parse::select_records(100_000, "range", 1, "5,1-100000,3", None).unwrap();
|
||||
assert_eq!(idx.len(), 100_000);
|
||||
assert_eq!(&idx[..3], &[4, 0, 1]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oversized_data_file_is_refused_before_reading() {
|
||||
let dir = scratch("oversized");
|
||||
let path = dir.join("big.csv");
|
||||
write(&path, &"x".repeat(2048));
|
||||
let path = path.to_string_lossy().to_string();
|
||||
let msg = read_path_capped(&path, 1024).unwrap_err();
|
||||
assert!(msg.contains("larger than"), "{msg}");
|
||||
assert_eq!(read_path_capped(&path, 4096).unwrap().len(), 2048);
|
||||
}
|
||||
@@ -125,11 +125,26 @@ pub(crate) fn parse(xml: &str) -> Result<El, ImportError> {
|
||||
Ok(root.kids.drain(..).find_map(|n| if let Node::El(e) = n { Some(e) } else { None }).unwrap_or_default())
|
||||
}
|
||||
|
||||
pub(crate) fn part(zip: &mut zip::ZipArchive<std::io::Cursor<&[u8]>>, name: &str) -> Option<String> {
|
||||
let mut f = zip.by_name(name).ok()?;
|
||||
/// Largest decompressed zip part read (a small file can inflate to gigabytes).
|
||||
pub(crate) const PART_LIMIT: u64 = 64 * 1024 * 1024;
|
||||
|
||||
/// A zip part as text. `Ok(None)` when the part is missing or isn't text; an error when it
|
||||
/// decompresses to more than [`PART_LIMIT`].
|
||||
pub(crate) fn part(zip: &mut zip::ZipArchive<std::io::Cursor<&[u8]>>, name: &str) -> Result<Option<String>, ImportError> {
|
||||
part_capped(zip, name, PART_LIMIT)
|
||||
}
|
||||
|
||||
pub(crate) fn part_capped(zip: &mut zip::ZipArchive<std::io::Cursor<&[u8]>>, name: &str, limit: u64) -> Result<Option<String>, ImportError> {
|
||||
let Ok(f) = zip.by_name(name) else { return Ok(None) };
|
||||
let mut s = String::new();
|
||||
f.read_to_string(&mut s).ok()?;
|
||||
Some(s)
|
||||
// Read one byte past the limit to tell "exactly at the limit" from "over it".
|
||||
if f.take(limit.saturating_add(1)).read_to_string(&mut s).is_err() {
|
||||
return Ok(None);
|
||||
}
|
||||
if s.len() as u64 > limit {
|
||||
return Err(ImportError::Corrupt(format!("{name} is larger than {} MiB uncompressed", limit / (1024 * 1024))));
|
||||
}
|
||||
Ok(Some(s))
|
||||
}
|
||||
|
||||
/// Character attributes from `<w:rPr>`.
|
||||
@@ -313,13 +328,13 @@ impl Ctx {
|
||||
|
||||
pub fn import(bytes: &[u8]) -> Result<Imported, ImportError> {
|
||||
let mut zip = zip::ZipArchive::new(std::io::Cursor::new(bytes)).map_err(|e| ImportError::Corrupt(e.to_string()))?;
|
||||
let doc = part(&mut zip, "word/document.xml").ok_or_else(|| ImportError::Corrupt("no word/document.xml (not a Word document)".into()))?;
|
||||
let doc = part(&mut zip, "word/document.xml")?.ok_or_else(|| ImportError::Corrupt("no word/document.xml (not a Word document)".into()))?;
|
||||
let doc = parse(&doc)?;
|
||||
// Styles.
|
||||
let mut ctx = Ctx { para_names: HashMap::new(), char_names: HashMap::new(), footnotes: HashMap::new(), warnings: vec![], next_table: 0 };
|
||||
let mut para_styles = Vec::new();
|
||||
let mut char_styles = Vec::new();
|
||||
if let Some(styles) = part(&mut zip, "word/styles.xml").map(|s| parse(&s)).transpose()? {
|
||||
if let Some(styles) = part(&mut zip, "word/styles.xml")?.map(|s| parse(&s)).transpose()? {
|
||||
let ids: HashMap<String, String> =
|
||||
styles.els().filter(|s| s.name == "style").filter_map(|s| Some((s.attr("styleId")?.to_string(), s.val("name")?))).collect();
|
||||
for s in styles.els().filter(|s| s.name == "style") {
|
||||
@@ -351,7 +366,7 @@ pub fn import(bytes: &[u8]) -> Result<Imported, ImportError> {
|
||||
}
|
||||
}
|
||||
// Footnotes (ids -1 / 0 are the separators).
|
||||
if let Some(f) = part(&mut zip, "word/footnotes.xml").map(|s| parse(&s)).transpose()? {
|
||||
if let Some(f) = part(&mut zip, "word/footnotes.xml")?.map(|s| parse(&s)).transpose()? {
|
||||
for n in f.els().filter(|n| n.name == "footnote") {
|
||||
if let Some(id) = n.attr("id").filter(|id| id.parse::<i64>().is_ok_and(|v| v > 0)) {
|
||||
ctx.footnotes.insert(id.to_string(), n.clone());
|
||||
@@ -399,6 +414,17 @@ mod tests {
|
||||
|
||||
const W: &str = r#"xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main""#;
|
||||
|
||||
#[test]
|
||||
fn oversized_part_is_an_error_not_a_huge_read() {
|
||||
// A highly compressible part: 64 KiB of spaces inflates past a 4 KiB cap.
|
||||
let big = docx(&" ".repeat(64 * 1024), "", "");
|
||||
let mut zip = zip::ZipArchive::new(std::io::Cursor::new(big.as_slice())).unwrap();
|
||||
assert!(matches!(part_capped(&mut zip, "word/document.xml", 4096), Err(ImportError::Corrupt(_))));
|
||||
// At or under the cap reads fine; a missing part is `None`.
|
||||
assert_eq!(part_capped(&mut zip, "word/document.xml", 64 * 1024).unwrap().map(|s| s.len()), Some(64 * 1024));
|
||||
assert_eq!(part_capped(&mut zip, "word/missing.xml", 4096).unwrap(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn imports_styles_runs_footnotes_and_tables() {
|
||||
let styles = format!(
|
||||
|
||||
@@ -21,10 +21,16 @@ fn cell_ref(r: &str) -> Option<(usize, usize)> {
|
||||
}
|
||||
|
||||
/// The worksheet part of the first sheet (workbook order), via the workbook relationships.
|
||||
fn first_sheet(zip: &mut zip::ZipArchive<std::io::Cursor<&[u8]>>) -> Option<String> {
|
||||
let wb = parse(&part(zip, "xl/workbook.xml")?).ok()?;
|
||||
fn first_sheet(zip: &mut zip::ZipArchive<std::io::Cursor<&[u8]>>) -> Result<Option<String>, ImportError> {
|
||||
let Some(wb) = part(zip, "xl/workbook.xml")? else { return Ok(None) };
|
||||
let Some(rels) = part(zip, "xl/_rels/workbook.xml.rels")? else { return Ok(None) };
|
||||
Ok(first_sheet_target(&wb, &rels))
|
||||
}
|
||||
|
||||
fn first_sheet_target(wb: &str, rels: &str) -> Option<String> {
|
||||
let wb = parse(wb).ok()?;
|
||||
let rid = wb.child("sheets")?.els().find(|e| e.name == "sheet")?.attr("id")?.to_string();
|
||||
let rels = parse(&part(zip, "xl/_rels/workbook.xml.rels")?).ok()?;
|
||||
let rels = parse(rels).ok()?;
|
||||
let target = rels.els().find(|r| r.attr("Id") == Some(rid.as_str()))?.attr("Target")?.trim_start_matches('/').to_string();
|
||||
Some(if target.starts_with("xl/") { target } else { format!("xl/{target}") })
|
||||
}
|
||||
@@ -60,8 +66,8 @@ fn value(c: &El, shared: &[String]) -> String {
|
||||
pub fn records(bytes: &[u8], sheet: Option<&str>) -> Result<Vec<Vec<String>>, ImportError> {
|
||||
let mut zip = zip::ZipArchive::new(std::io::Cursor::new(bytes)).map_err(|e| ImportError::Corrupt(e.to_string()))?;
|
||||
let sheet_part = sheet_target(&mut zip, sheet)?;
|
||||
let sheet_xml = parse(&part(&mut zip, &sheet_part).ok_or_else(|| ImportError::Corrupt(format!("no {sheet_part} (not an Excel workbook)")))?)?;
|
||||
let shared: Vec<String> = part(&mut zip, "xl/sharedStrings.xml")
|
||||
let sheet_xml = parse(&part(&mut zip, &sheet_part)?.ok_or_else(|| ImportError::Corrupt(format!("no {sheet_part} (not an Excel workbook)")))?)?;
|
||||
let shared: Vec<String> = part(&mut zip, "xl/sharedStrings.xml")?
|
||||
.map(|x| parse(&x))
|
||||
.transpose()?
|
||||
.map(|sst| sst.els().filter(|e| e.name == "si").map(El::text).collect())
|
||||
@@ -146,7 +152,7 @@ pub fn records(bytes: &[u8], sheet: Option<&str>) -> Result<Vec<Vec<String>>, Im
|
||||
|
||||
/// Workbook order, or the sheet named `wanted`.
|
||||
fn sheet_target(zip: &mut zip::ZipArchive<std::io::Cursor<&[u8]>>, wanted: Option<&str>) -> Result<String, ImportError> {
|
||||
let wb = parse(&part(zip, "xl/workbook.xml").ok_or_else(|| ImportError::Corrupt("no xl/workbook.xml (not an Excel workbook)".into()))?)?;
|
||||
let wb = parse(&part(zip, "xl/workbook.xml")?.ok_or_else(|| ImportError::Corrupt("no xl/workbook.xml (not an Excel workbook)".into()))?)?;
|
||||
let sheets: Vec<(String, String)> = wb
|
||||
.child("sheets")
|
||||
.map(|s| s.els().filter(|e| e.name == "sheet").filter_map(|e| Some((e.attr("name")?.to_string(), e.attr("id")?.to_string()))).collect())
|
||||
@@ -155,7 +161,7 @@ fn sheet_target(zip: &mut zip::ZipArchive<std::io::Cursor<&[u8]>>, wanted: Optio
|
||||
None => sheets.first().cloned().ok_or_else(|| ImportError::Corrupt("the workbook has no sheets".into()))?,
|
||||
Some(w) => sheets.into_iter().find(|(n, _)| n == w).ok_or_else(|| ImportError::Corrupt(format!("no worksheet \"{w}\"")))?,
|
||||
};
|
||||
let rels = parse(&part(zip, "xl/_rels/workbook.xml.rels").ok_or_else(|| ImportError::Corrupt("no workbook relationships".into()))?)?;
|
||||
let rels = parse(&part(zip, "xl/_rels/workbook.xml.rels")?.ok_or_else(|| ImportError::Corrupt("no workbook relationships".into()))?)?;
|
||||
let target = rels
|
||||
.els()
|
||||
.find(|r| r.attr("Id") == Some(rid.as_str()))
|
||||
@@ -167,9 +173,9 @@ fn sheet_target(zip: &mut zip::ZipArchive<std::io::Cursor<&[u8]>>, wanted: Optio
|
||||
|
||||
pub fn import(bytes: &[u8]) -> Result<Imported, ImportError> {
|
||||
let mut zip = zip::ZipArchive::new(std::io::Cursor::new(bytes)).map_err(|e| ImportError::Corrupt(e.to_string()))?;
|
||||
let sheet_part = first_sheet(&mut zip).unwrap_or_else(|| "xl/worksheets/sheet1.xml".into());
|
||||
let sheet = parse(&part(&mut zip, &sheet_part).ok_or_else(|| ImportError::Corrupt(format!("no {sheet_part} (not an Excel workbook)")))?)?;
|
||||
let shared: Vec<String> = part(&mut zip, "xl/sharedStrings.xml")
|
||||
let sheet_part = first_sheet(&mut zip)?.unwrap_or_else(|| "xl/worksheets/sheet1.xml".into());
|
||||
let sheet = parse(&part(&mut zip, &sheet_part)?.ok_or_else(|| ImportError::Corrupt(format!("no {sheet_part} (not an Excel workbook)")))?)?;
|
||||
let shared: Vec<String> = part(&mut zip, "xl/sharedStrings.xml")?
|
||||
.map(|x| parse(&x))
|
||||
.transpose()?
|
||||
.map(|sst| sst.els().filter(|e| e.name == "si").map(El::text).collect())
|
||||
|
||||
@@ -41,6 +41,20 @@ The result is JSON: `{"completed": n, "results": [...]}`, plus `failedIndex` / `
|
||||
fails (the exit status is non-zero; `--keep-going` records errors and continues). `run --cmd ID=JSON` accepts the
|
||||
same references, and MCP's `batch` tool takes `commands` or `script` text with them.
|
||||
|
||||
## Data merge
|
||||
|
||||
`data.source.select {path, sheet?}` links a CSV/TSV/`.xlsx` file to the document (`data.source.update` re-reads it,
|
||||
`data.source.remove` drops it); `data.fields` lists its fields. `data.placeholder.add {field, role?, story?, at?, end?, item?}`
|
||||
marks text or a frame as a field (`role`: text, image, qr, hyperlink); `data.options` sets records, tiling and image
|
||||
fitting; `data.preview {record}` / `data.preview.stop` show one record without saving it.
|
||||
|
||||
`data.merge` now **creates a new document** and leaves the template unchanged (the old `spread` parameter, which
|
||||
appended pages to the template, is an error). It uses the linked source, or inline `csv` / `rows` / `path` / `bytes`:
|
||||
|
||||
```sh
|
||||
echo 'data.merge {"path": "people.csv", "records": "range", "range": "1-20"}' | designcraft-cli script - --in template.designcraft --save merged.designcraft
|
||||
```
|
||||
|
||||
## The running app
|
||||
|
||||
```sh
|
||||
|
||||
@@ -27,6 +27,10 @@ Start the app with `--control <port>` (or `DESIGNCRAFT_CONTROL_PORT`). The serve
|
||||
| `ui.render` | `{path?, page?, scale?, bleed?}` | Render a page headlessly (PNG; base64 if no path) |
|
||||
| `app.open` / `app.save` / `app.export` / `app.quit` | | Files |
|
||||
|
||||
Data merge runs through `engine.execute` like any command: `data.source.select`, `data.fields`, `data.placeholder.add` /
|
||||
`.remove`, `data.options`, `data.preview` / `data.preview.stop`, `data.merge`. `data.merge` creates and activates a new
|
||||
merged document; the template stays as it was ([agents.md](agents.md#data-merge)).
|
||||
|
||||
Headless window screenshots (locked screen, hidden window): `cargo run -p designcraft-ui-egui --example ui_shot -- script.jsonl`, where each line is one of the requests above, `{"shot": "/abs/out.png"}` or `{"steps": n}` (renders the whole UI offscreen with wgpu).
|
||||
|
||||
The MCP server (`designcraft-cli mcp`) wraps the same methods for Claude and other agents.
|
||||
@@ -1,208 +0,0 @@
|
||||
# Data merge
|
||||
|
||||
Design for phase 1 of data merge. Approved in conversation on 2026-10-07.
|
||||
|
||||
Phase 1 extends the existing `data.merge` command. One template, one linked table, a temporary preview, then a new document. Each record gets a copy of the template, or the items on a single page are tiled.
|
||||
|
||||
Phase 2 is not part of this spec. It adds more than one source, JSON, filter and sort, and a drawn grid. Phase 1 stores a source id on every placeholder so that work can land later without rewriting placeholders. Tiling stays when the grid arrives.
|
||||
|
||||
## What exists today
|
||||
|
||||
`crates/engine/src/cmd/datamerge.rs` takes an inline `csv` string or `rows` array, copies one template spread once per record inside the same document, and replaces `<<Field>>` in story text. One undo step. There is no file on disk, no Excel, no pictures, no QR codes, no hyperlinks, no preview, and no tiling.
|
||||
|
||||
`crates/textimport` already reads the first worksheet of an `.xlsx` file as a table of cached cell values. Place uses that path. Data merge needs rows with empty cells preserved, so place keeps its own reader.
|
||||
|
||||
Pictures go through `file.place` and the Links panel. Hyperlinks are `hyperlink.create` (`url`, `email`, or `page`). QR codes are `object.qrCode` (vector path, types `url`, `text`, `sms`, `email`, `vcard`). Frame fitting is `Fitting` in `crates/doc`: `none`, `fillProportionally`, `fitProportionally`, `fitContentToFrame`, `centerContent`.
|
||||
|
||||
The engine `Session` holds a list of documents and an active index. A merge adds a document. It does not replace the template.
|
||||
|
||||
## Document model
|
||||
|
||||
New fields on `Document` use serde defaults so older files open. Empty data merge state is omitted from the save.
|
||||
|
||||
`DataMerge` on the document:
|
||||
|
||||
- `sources`: a list. Phase 1 has zero or one entry.
|
||||
- `placeholders`: the list below.
|
||||
- `options`: the last merge options, saved with the document.
|
||||
|
||||
`DataSource`:
|
||||
|
||||
- `id`
|
||||
- `path`: absolute path, if the source is a file
|
||||
- `relative_path`: path relative to the document file, set when the document has been saved
|
||||
- `name`: file name, used when the source was passed as bytes and has no path
|
||||
- `delimiter`: `comma`, `tab`, or `semicolon` (text only)
|
||||
- `sheet`: worksheet name, or absent for the first sheet (Excel only)
|
||||
- `fields`: `{ name, kind }` where `kind` is `text`, `image`, or `qr`
|
||||
- `rows`: cached rows, each the same length as `fields`. Missing cells are `""`
|
||||
- `fingerprint`: file size and modification time of the last successful read, if there is a path
|
||||
- `status`: `ok`, `missing`, or `modified`
|
||||
|
||||
`Placeholder`:
|
||||
|
||||
- `id`
|
||||
- `source_id`
|
||||
- `field`: the name, without `@` or `#`
|
||||
- `role`: `text`, `image`, `qr`, or `hyperlink`
|
||||
- `anchor`: either `{ story, start, end }` in Unicode scalar offsets, matching `HyperlinkSource::Text`, or `{ item }` for a frame
|
||||
|
||||
`MergeOptions`, with these defaults:
|
||||
|
||||
- `records`: `all`. Also `one` with a 1-based record number, or `range` with a range string
|
||||
- `per_page`: `single` or `multiple`. Default `single`
|
||||
- `arrange`: `rows` or `columns`. Default `rows`. Used only when `per_page` is `multiple`
|
||||
- `insets`: top, right, bottom, left, in points. Default 36, 36, 36, 36. Used only when tiling
|
||||
- `column_spacing` and `row_spacing`: points. Default 0. Used only when tiling
|
||||
- `fitting`: `fitProportionally` (default), `fillProportionally`, `fitContentToFrame`, or `none`
|
||||
- `center`: bool, default false. Applied after fitting, by moving the graphic to the middle of the frame
|
||||
- `link_images`: bool, default true
|
||||
- `limit`: absent, or a positive count taken after the record selection
|
||||
|
||||
Preview is session state on the open document, not a field of `Document`. It stores the 1-based record being shown, or nothing. Saving writes the unfilled template. `data.preview.stop`, closing the document, and a successful merge all clear it.
|
||||
|
||||
## Reading a source
|
||||
|
||||
`data.source.select` accepts `path`, or `bytes` plus `name`. The name's extension selects the reader. `delimiter` (`comma`, `tab`, `semicolon`) overrides sniffing for text. `sheet` names an Excel worksheet. The first sheet in workbook order is the default.
|
||||
|
||||
Text is UTF-8. A leading byte-order mark is stripped. Any other encoding is an error that names the file. The command does not guess a legacy encoding.
|
||||
|
||||
Delimiter sniff, only when the command does not set one:
|
||||
|
||||
- `.tsv` or `.tab`: tab
|
||||
- `.csv`: comma
|
||||
- `.txt`: tab if the header line contains a tab; otherwise semicolon if that line contains a semicolon and no comma; otherwise comma
|
||||
- anything else with text contents: the same sniff as `.txt`
|
||||
|
||||
Excel (`.xlsx` only):
|
||||
|
||||
- Cached values. The reader does not evaluate formulas. A formula cell with no cached value is empty.
|
||||
- A date is the stored number, not a formatted date.
|
||||
- Booleans are `TRUE` and `FALSE`. Whole numbers drop a trailing `.0`.
|
||||
- Empty cells inside the header's width are `""`.
|
||||
- `.xlsm` and `.xls` are errors.
|
||||
|
||||
Rows:
|
||||
|
||||
- The first row is field names, trimmed. An empty name is an error. A duplicate name is an error.
|
||||
- A leading apostrophe on a header is ignored. Then a leading `@` sets `image` and a leading `#` sets `qr`. The mark is not part of the name. Any other field is `text`.
|
||||
- Each later row is one record. A short row is padded with `""`. A row whose cells are all empty is skipped. Cells past the header are ignored, and the result's `warnings` names the row.
|
||||
- More than 100,000 data rows or 200 columns is an error. The command does not allocate them.
|
||||
|
||||
Inline `csv` on `data.merge` and `data.fields` uses this same text parser and does not create a source. Inline `rows` is an array of objects. The field list is the key order of the first object. Later objects are read by those keys. A missing key is `""`. An extra key is ignored, with a warning. `@` and `#` on a key work the same way as a header cell. Non-strings are stringified with JSON's string form for numbers and bools, and rejected for arrays and objects.
|
||||
|
||||
`data.source.select` returns `{ fields, records, warnings }`. `fields` is `{ name, kind }`. `records` is the data-row count. After a file read, status is `ok` and the fingerprint is stored. On open, and before a merge, if the path is set: missing file sets `missing`; a different size or modification time sets `modified`. A merge uses the cached rows either way and adds a warning for `missing` or `modified`. `data.source.update` re-reads. With no path, update is an error. `data.source.remove` drops the source and leaves placeholders in place.
|
||||
|
||||
When the document is saved, `relative_path` is refreshed from the document's path. On open, a missing absolute path is tried again as `relative_path` beside the document.
|
||||
|
||||
## Placeholders
|
||||
|
||||
`data.placeholder.add` takes `field`, `role`, and a target. The target is the text caret (`story` plus `at`) or a frame (`item`). The field must exist on the document's source. Phase 1 has one source, and the placeholder stores that source id.
|
||||
|
||||
Defaults the panel uses, which the command may also set by omitting `role`:
|
||||
|
||||
- Caret in a story: `text`
|
||||
- A selected frame and an `image` field: `image`
|
||||
- A selected frame and a `qr` field: `qr`
|
||||
- A plain `text` field on a frame is `image` only when `role` is `image`. The panel calls that Bind to frame.
|
||||
- `hyperlink` is always explicit.
|
||||
|
||||
Inserting `text` writes `<<Name>>` at the caret, using the character style already at that point, and anchors the placeholder to that range. `<<` and `>>` are ASCII 0x3C and 0x3E. Inserting `image` or `qr` anchors the placeholder to the frame and shows `<<Name>>` as the frame's label. It does not place a picture yet. Pictures and QR codes are frames, not inline in a story. Inserting `hyperlink` anchors a text range or a frame and does not change the visible text.
|
||||
|
||||
`data.placeholder.remove` deletes the object. A text placeholder also deletes the `<<Name>>` characters it owns. Deleting those characters by editing deletes the placeholder. The role on the placeholder wins over the column kind. An `image` column inserted as `text` merges as the path string. A `text` column with role `image` merges as a picture. A `qr` column inserted as `text` merges as the payload string.
|
||||
|
||||
A typed `<<Name>>` that is not covered by a text placeholder is still replaced with the field's text at preview and at merge. It never becomes a picture, a QR code, or a hyperlink. An unknown `<<Nope>>` is left as typed.
|
||||
|
||||
## Filling one record
|
||||
|
||||
Used by preview and by the merge. Record numbers are 1-based positions in the cached rows after empty rows are skipped.
|
||||
|
||||
- `text`: replace the anchored characters, or the typed `<<Name>>`, with the cell. Keep the character style of the first character of that range.
|
||||
- `image`: resolve the cell as a file path. The first existing file wins: the path as given if it is absolute, then the path beside the data file, then the path beside the saved document. `http` and `https` are not downloaded. They are missing images. Place into that frame. `link_images` true creates a normal link. False embeds the bytes in the document assets and creates no external link. Fitting and `center` come from the merge options. A missing or unreadable file leaves the frame empty, adds `{ record, field, path }` to `missingImages`, and continues.
|
||||
- `qr`: build a vector code with the same encoder as `object.qrCode` type `text`, using the cell as `content`, inside the frame, black. An empty cell leaves the frame empty and adds a warning.
|
||||
- `hyperlink`: do not change visible text. If this same range is also a text placeholder, the text rule still runs. Trim the cell. Empty adds no link. If the value contains `://` or whitespace, the destination is a URL. Otherwise, if it contains exactly one `@` and no `:`, the destination is an email. Otherwise it is a URL. Create one hyperlink on that anchor for this record. A failed create adds a warning and continues.
|
||||
|
||||
A placeholder whose frame or story lives on a parent spread is not filled. The merge copies it onto the new document's parent still showing `<<Name>>`, and adds one warning.
|
||||
|
||||
## Preview
|
||||
|
||||
`data.preview` takes `record`. The record stepper clamps to the cached rows. Preview fills the live template, and the session keeps a stash of the unfilled document. Save, `data.preview.stop`, switching away, removing the source, and `data.merge` restore that stash before they continue. Any other edit stops preview first, so the edit applies to the unfilled template. The file on disk never contains the preview text.
|
||||
|
||||
## Merge
|
||||
|
||||
`data.merge` reads the linked source when `path`, `bytes`, `csv`, and `rows` are all absent. Passing one of those reads it for that run only and does not attach a source. Attaching is `data.source.select`.
|
||||
|
||||
The template document is not modified. Its undo stack is unchanged. Preview is cleared before the copy is made, so the new document is filled from the unfilled template, not from the preview. The command builds a new document, appends it to the session, and makes it active. The new document is a normal document: edit it and save it, or close it and discard the run.
|
||||
|
||||
Record selection, applied to the cached rows:
|
||||
|
||||
- `all`: every row
|
||||
- `one`: that 1-based row. Out of range is an error
|
||||
- `range`: a comma-separated list of tokens. A token is `N` or `A-B`, both ends inclusive, optional whitespace. `A` greater than `B` is an error. A number outside `1..=row count` is an error. Duplicates are dropped. Order is the order of first appearance
|
||||
- Then `limit`, if set, keeps the first N selected records. `0` is an error
|
||||
|
||||
No selected records is an error. No document is created.
|
||||
|
||||
`per_page: single` copies the whole template once per record. A 3-page template writes 3 pages per record, all filled with that record. Pages, parent pages, styles, swatches, layers, and sections are copied. Document-page placeholders are filled. Parent placeholders stay unfilled.
|
||||
|
||||
`per_page: multiple` is tiling. It is an error, and no document is created, when facing pages are on or the template has more than one page. The block is every item on the document page whose bounds intersect the page, except ruler guides. Pasteboard items that miss the page are drawn on the first merged page only. Ruler guides are drawn on every merged page and are not stepped.
|
||||
|
||||
The block's width and height are the union of those items. `step_x` is that width plus `column_spacing`. `step_y` is that height plus `row_spacing`. The first record is placed at the block's original position, even when that position crosses an inset. Later records use whole steps `(col, row)` from that origin, with `col` and `row` starting at 0.
|
||||
|
||||
Rows first increases `col` until the candidate's right edge would pass `page_width - right_inset`, then sets `col` to 0 and increases `row`. Columns first increases `row` until the candidate's bottom edge would pass `page_height - bottom_inset`, then sets `row` to 0 and increases `col`. A candidate that fails the other inset (`row` against the bottom, or `col` against the right) starts a new page, and that record is `(0, 0)` on the new page. The left and top insets do not move the origin. If a second copy cannot be placed on the first page at all, every record gets its own page, with one warning.
|
||||
|
||||
Each merged page has the template page's size, margins, and guides. Pages after the first repeat the block and the guides, not the pasteboard-only items. The last page may be partly empty. No empty records are added to fill a grid.
|
||||
|
||||
The command returns:
|
||||
|
||||
- `records`: how many records were written
|
||||
- `pages`: pages in the new document
|
||||
- `missingImages`: `{ record, field, path }`
|
||||
- `oversetStories`: how many stories in the new document are overset
|
||||
- `warnings`: strings
|
||||
|
||||
Overset text does not fail the merge. The old `spread` parameter is removed.
|
||||
|
||||
## Panel
|
||||
|
||||
Window > Utilities > Data Merge opens the panel. It does not merge by itself. The panel reads engine state and calls `app.run`.
|
||||
|
||||
The panel shows the source file name, or "No data source", plus Update and Remove. The field list shows a kind icon drawn in code (a T, a picture mark, a module grid), the field name, and how many placeholders use that field. Preview is a checkbox. The stepper is first, previous, a record number, next, and last. It is disabled until preview is on. Create Merged Document runs `data.merge` with the saved options. Those options are edited in that dialog: which records, records per page, and, when multiple is selected, arrange, insets, and spacing. Image options are fitting, center, and link images, plus the optional limit.
|
||||
|
||||
Colors come from `theme::Tokens`. New strings go through the existing i18n table. No Adobe icons.
|
||||
|
||||
On the web, and from the control channel, the caller passes `csv`, `rows`, or `bytes` plus `name`. There is no filesystem dialog.
|
||||
|
||||
## Errors
|
||||
|
||||
Every command returns `Result`. A bad path, a bad sheet name, a bad encoding, a bad header, a bad range, a tiling refusal, and the row or column cap are errors and create no document. A missing image, an empty QR cell, a failed hyperlink, a modified source, an ignored extra cell, and a parent placeholder are warnings on a successful merge.
|
||||
|
||||
No `unwrap`, `expect`, `panic`, or `unsafe` in the new code. Sizes and offsets use checked arithmetic. The row and column caps bound the allocation. Locks follow the crate's poison policy.
|
||||
|
||||
## Tests
|
||||
|
||||
Engine tests, with synthetic strings and tiny files in memory:
|
||||
|
||||
- Quoted commas, doubled quotes, and a quoted newline stay one field. The existing comma parser already covers the first two.
|
||||
- Tab and semicolon files, and `.txt` sniffing for each.
|
||||
- Excel: first sheet, a named sheet, a cached number, an empty formula cell, a boolean, and a date serial left as digits.
|
||||
- A non-UTF-8 file, an empty header, and a duplicate header are errors.
|
||||
- `@Photo` and `#Code` become fields `Photo`/`image` and `Code`/`qr`.
|
||||
- Typed `<<Title>>` is replaced. Typed `<<Nope>>` stays. A text placeholder keeps its character style.
|
||||
- Image resolution: absolute, beside the data file, beside the document. A missing file continues, with `missingImages` set and an empty frame. An `https` value is missing and is not fetched.
|
||||
- A `text` field with role `image` places a picture. An `image` field with role `text` inserts the path.
|
||||
- A QR placeholder encodes the cell text. An empty cell warns.
|
||||
- Hyperlink: `a@b.c` is email, `https://example.com` is a URL, empty adds no link.
|
||||
- Single-record merge of a 2-page template and 3 records writes 6 pages. The template's page count and undo length are unchanged. The new document is active.
|
||||
- Tiling a single non-facing page, six records, rows first, produces the measured grid: three across, then the next row, step equal to the block plus spacing. A block that crosses the bottom inset starts a new page.
|
||||
- Tiling a facing-page document, and tiling a two-page document, each return an error and add no document.
|
||||
- Update re-reads a changed fingerprint. Merge with `modified` warns and uses the cache. Remove leaves placeholders.
|
||||
- 100,001 rows is an error.
|
||||
- Preview of record 2 shows that row. Stop restores `<<Name>>`. The saved document bytes do not contain the preview text.
|
||||
- Inline `rows` still merges, including a non-string number.
|
||||
|
||||
UI: the panel lists fields from `data.fields`, and Create Merged Document calls `data.merge`. A headless `designcraft-cli` run covers the command. Wasm still accepts `rows`.
|
||||
|
||||
## Out of scope
|
||||
|
||||
Multiple sources, enabling and disabling sources, JSON, filter rules, sort, skip-on-warning, source filename and merge index fields, a drawn grid (rows, columns, gutter, record offset, record advance, start corner), QR codes from an unmarked column via the QR tool, inline pictures in a story, per-record parent-page fields, downloading remote images, evaluating Excel formulas, formatting Excel dates, `.xls`, and merging straight to PDF.
|
||||
Reference in new issue
Block a user