From 26d5c8941c4d8b7dd6e013ce02c4aef407f360f2 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Thu, 12 Nov 2026 06:27:41 -0500 Subject: [PATCH] IDML: story export and object-style overlay in linear time, embedded image budget * idml: story runs and marker offsets found once, object-style overlay by position, embedded image budget * address review: saturating embedded-bytes counter * address review: IDML export past the embed limit succeeds and warns for each image written as a link * address review: an unlinked image is always embedded, only linked ones fall back to their link --- crates/engine/src/cmd/interchange.rs | 14 +++-- crates/idml/src/export.rs | 26 ++++++++- crates/idml/src/export/item.rs | 12 +++- crates/idml/src/export/story.rs | 76 +++++++++++++++++++++----- crates/idml/src/import/object_style.rs | 30 ++++++---- crates/idml/src/lib.rs | 2 +- crates/idml/src/tests/assets.rs | 39 +++++++++++++ 7 files changed, 165 insertions(+), 34 deletions(-) diff --git a/crates/engine/src/cmd/interchange.rs b/crates/engine/src/cmd/interchange.rs index 47b36b776..e84c28103 100644 --- a/crates/engine/src/cmd/interchange.rs +++ b/crates/engine/src/cmd/interchange.rs @@ -17,7 +17,7 @@ use crate::{DocState, EngineError, Result, Session}; pub fn specs() -> Vec { vec![ cmd!(query "file.exportIdml", "Export IDML…", [], None, - "{path?, embedImages?: true} — writes an IDML package to `path`, or returns {base64} without a path", + "{path?, embedImages?: true} — writes an IDML package to `path`, or returns {base64} without a path; `warnings` names each image written as a link because the embedded-image limit was reached", has_doc, export_idml), cmd!(noundo "file.openIdml", "Open IDML", [], None, "{path | base64, name?} — opens an IDML package, or an InDesign document or template (.indd, .indt: converted to IDML first; its conversion warnings join `warnings`), as a new document (linked images are read next to the file or from its Links/ folder; the fonts in a `Document Fonts` folder beside it load first) → {index, documentFonts, warnings: conversion warnings, package parts missing, font files skipped, GREP styles whose pattern can't compile}", @@ -29,15 +29,19 @@ fn export_idml(s: &mut Session, p: &Value) -> Result { let st = s.doc()?; let opts = designcraft_idml::ExportOptions { embed_images: p.get("embedImages").and_then(Value::as_bool).unwrap_or(true) }; let layer_doc = with_pdf_layer_variants(&st.doc, opts.embed_images)?; - let bytes = designcraft_idml::export_idml_with(layer_doc.as_ref().unwrap_or(&st.doc), &opts); - match str_param(p, "path") { + let (bytes, warnings) = designcraft_idml::export_idml_report(layer_doc.as_ref().unwrap_or(&st.doc), &opts); + let mut out = match str_param(p, "path") { Some(path) => { #[cfg(not(target_arch = "wasm32"))] crate::io::write_file(path, &bytes).map_err(EngineError::Other)?; - Ok(json!({"path": path, "bytes": bytes.len()})) + json!({"path": path, "bytes": bytes.len()}) } - None => Ok(json!({"base64": base64_encode(&bytes), "bytes": bytes.len()})), + None => json!({"base64": base64_encode(&bytes), "bytes": bytes.len()}), + }; + if !warnings.is_empty() { + out["warnings"] = json!(warnings); } + Ok(out) } /// Whether a placement holds an Object Layer Options choice an export can still apply: a graphic diff --git a/crates/idml/src/export.rs b/crates/idml/src/export.rs index 717bb5d6c..fccbc7c2f 100644 --- a/crates/idml/src/export.rs +++ b/crates/idml/src/export.rs @@ -65,9 +65,20 @@ pub fn export_idml(doc: &Document) -> Vec { /// Export with options, using the index sort-key conflict policy of [`export_idml`]. pub fn export_idml_with(doc: &Document, opts: &ExportOptions) -> Vec { + export_idml_report(doc, opts).0 +} + +/// [`export_idml_with`] and its warnings: an image written as a link only because the export's +/// embedded-image limit was reached is named in one, so the package is not silently incomplete. +pub fn export_idml_report(doc: &Document, opts: &ExportOptions) -> (Vec, Vec) { + export_with_limit(doc, opts, MAX_EMBEDDED_BYTES) +} + +pub(crate) fn export_with_limit(doc: &Document, opts: &ExportOptions, embed_limit: usize) -> (Vec, Vec) { let mut ex = Ex::new(doc, opts); + ex.embed_limit = embed_limit; let parts = ex.build(); - zip_parts(&parts) + (zip_parts(&parts), ex.warnings) } fn zip_parts(parts: &[(String, Vec)]) -> Vec { @@ -106,8 +117,18 @@ struct Ex<'a> { /// Index topic path → first serialized sort key; See / See also cross-references. index_topics: BTreeMap, String>, index_xrefs: Vec<(Vec, &'static str, String)>, + /// Bytes of image data written into `` so far (see [`MAX_EMBEDDED_BYTES`]). + embedded: usize, + embed_limit: usize, + /// Images written as links because the embed limit was reached (one line per placement). + warnings: Vec, } +/// Image bytes one export embeds, counting each placement (IDML holds a copy per placement): past +/// it a graphic that has a link of its own is written as that link without its contents (an +/// image with no link is always embedded: it has nowhere else to live). +const MAX_EMBEDDED_BYTES: usize = 512 * 1024 * 1024; + impl<'a> Ex<'a> { fn new(d: &'a Document, opts: &'a ExportOptions) -> Self { let mut ex = Ex { @@ -121,6 +142,9 @@ impl<'a> Ex<'a> { link_text_sources: Vec::new(), index_topics: BTreeMap::new(), index_xrefs: Vec::new(), + embedded: 0, + embed_limit: MAX_EMBEDDED_BYTES, + warnings: Vec::new(), }; ex.assign_swatch_ids(); for s in d.stories.values() { diff --git a/crates/idml/src/export/item.rs b/crates/idml/src/export/item.rs index 457172253..ca27a8ca0 100644 --- a/crates/idml/src/export/item.rs +++ b/crates/idml/src/export/item.rs @@ -268,9 +268,19 @@ impl<'a> Ex<'a> { el.set("FillColor", id.clone()); el.set("FillTint", pct(g.fill.tint as f64)); } - let embed = !asset.data.is_empty() && (self.opts.embed_images || asset.link.is_none()); + let wanted = !asset.data.is_empty() && (self.opts.embed_images || asset.link.is_none()); + // An image with no link of its own has nowhere else to live, so it is always embedded + // (dropping its data would lose it); only an image with a real link can fall back to it. + let embed = wanted && (asset.link.is_none() || self.embedded.saturating_add(asset.data.len()) <= self.embed_limit); + if wanted && !embed { + self.warnings.push(format!( + "image {} is written as a link, not embedded: the package already holds {} bytes of images (limit {})", + asset.name, self.embedded, self.embed_limit + )); + } let mut props = Vec::new(); if embed { + self.embedded = self.embedded.saturating_add(asset.data.len()); let mut c = El::new("Contents"); c.children.push(Node::CData(base64_encode(&asset.data))); props.push(c); diff --git a/crates/idml/src/export/story.rs b/crates/idml/src/export/story.rs index 437376190..7ca41e231 100644 --- a/crates/idml/src/export/story.rs +++ b/crates/idml/src/export/story.rs @@ -278,7 +278,14 @@ impl<'a> Ex<'a> { None } - fn apply_tracked_changes(&mut self, s: &Story, f: &CharFormat, csrs: &mut [El], para_br: bool, para_end: usize) { + fn apply_tracked_changes( + &mut self, + f: &CharFormat, + csrs: &mut [El], + para_br: bool, + runs: &[(std::ops::Range, &CharFormat)], + para_end: usize, + ) { // Tracked changes wrap the text they mark. if let Some(kind) = change_type(f.over.change) { for c in csrs.iter_mut().filter(|c| c.name == "CharacterStyleRange" && !c.children.is_empty()) { @@ -288,7 +295,10 @@ impl<'a> Ex<'a> { } } if para_br && let Some(c) = csrs.last_mut() { - let nl = s.runs().find(|(rr, _)| rr.contains(¶_end)).and_then(|(_, nf)| change_type(nf.over.change)); + let nl = runs + .get(runs.partition_point(|(rr, _)| rr.end <= para_end)) + .filter(|(rr, _)| rr.start <= para_end) + .and_then(|(_, nf)| change_type(nf.over.change)); match nl { None => c.push(El::new("Br")), Some(kind) => match c.children.last_mut() { @@ -307,17 +317,27 @@ impl<'a> Ex<'a> { pub(super) fn story_paras(&mut self, s: &Story) -> Vec { let ranges = s.para_ranges(); let n = ranges.len(); - ranges.into_iter().enumerate().map(|(pi, r)| self.story_para(s, pi, r, pi + 1 == n)).collect() + // The runs and the marker offsets, once: each paragraph (and each marker) searched them from the start. + let runs: Vec<(std::ops::Range, &CharFormat)> = s.runs().collect(); + let marks = Marks::of(s); + ranges.into_iter().enumerate().map(|(pi, r)| self.story_para(s, pi, r, pi + 1 == n, &runs, &marks)).collect() } - fn paragraph_runs(s: &Story, r: &std::ops::Range) -> Vec<(std::ops::Range, CharFormat)> { - // Character runs intersecting the paragraph. + fn paragraph_runs( + s: &Story, + r: &std::ops::Range, + runs: &[(std::ops::Range, &CharFormat)], + ) -> Vec<(std::ops::Range, CharFormat)> { + // Character runs intersecting the paragraph (they ascend: the first is a search away). let mut segs: Vec<(std::ops::Range, CharFormat)> = Vec::new(); - for (rr, f) in s.runs() { + for (rr, f) in &runs[runs.partition_point(|(rr, _)| rr.end <= r.start)..] { + if rr.start >= r.end { + break; + } let a = rr.start.max(r.start); let b = rr.end.min(r.end); if a < b { - segs.push((a..b, f.clone())); + segs.push((a..b, (*f).clone())); } } if segs.is_empty() { @@ -354,11 +374,19 @@ impl<'a> Ex<'a> { } } - fn story_para(&mut self, s: &Story, pi: usize, r: std::ops::Range, last: bool) -> El { + fn story_para( + &mut self, + s: &Story, + pi: usize, + r: std::ops::Range, + last: bool, + runs: &[(std::ops::Range, &CharFormat)], + marks: &Marks, + ) -> El { let pf = s.paras.get(pi).cloned().unwrap_or_default(); let mut psr = self.psr_el(&pf); let ends_with_break = s.text.get(r.clone()).and_then(|t| t.chars().next_back()).is_some_and(st::is_break_char); - let segs = Self::paragraph_runs(s, &r); + let segs = Self::paragraph_runs(s, &r, runs); // A hyperlink on a run of text is written as a `HyperlinkTextSource` around the // ranges it covers, so the runs are cut at its edges first (#1446). let (links, segs) = self.cut_runs_at_links(s, &r, segs); @@ -414,7 +442,7 @@ impl<'a> Ex<'a> { } else if ch == designcraft_doc::ANCHOR_MARK { flush_text(&mut cur, &mut pending); flush_content(&mut pending, &mut out); - let k = s.text[..seg_start + ci].matches(designcraft_doc::ANCHOR_MARK).count(); + let k = marks.anchor.partition_point(|&at| at < seg_start + ci); if let Some(a) = s.anchors.get(k) { out.push(Node::El( El::new("HyperlinkTextDestination") @@ -428,14 +456,14 @@ impl<'a> Ex<'a> { // An anchored object: the item inside the range with its anchoring settings. flush_text(&mut cur, &mut pending); flush_content(&mut pending, &mut out); - let k = s.text[..seg_start + ci].matches(designcraft_doc::OBJECT_MARK).count(); + let k = marks.object.partition_point(|&at| at < seg_start + ci); if let Some(el) = self.anchored_object(s, k) { out.push(Node::El(el)); } } else if ch == designcraft_doc::INDEX_MARK { flush_text(&mut cur, &mut pending); flush_content(&mut pending, &mut out); - let k = s.text[..seg_start + ci].matches(designcraft_doc::INDEX_MARK).count(); + let k = marks.index.partition_point(|&at| at < seg_start + ci); if let Some(el) = self.index_reference(s, k) { out.push(Node::El(el)); } @@ -448,7 +476,7 @@ impl<'a> Ex<'a> { c.children.append(&mut out); csrs.push(c); } - let k = s.text[..seg_start + ci].matches(designcraft_doc::XREF_MARK).count(); + let k = marks.xref.partition_point(|&at| at < seg_start + ci); if let Some(x) = s.xrefs.get(k) { let me = self.fresh(); let fmt = self.d.xref_formats.iter().position(|ff| ff.name == x.format).map_or("n".to_string(), xref_format_self); @@ -525,7 +553,7 @@ impl<'a> Ex<'a> { c.children.extend(out); csrs.push(c); } - self.apply_tracked_changes(s, &f, &mut csrs, para_br, r.end); + self.apply_tracked_changes(&f, &mut csrs, para_br, runs, r.end); let link = links.iter().find(|(_, l)| l.contains(&seg_start) || (l.start == seg_start && l.start == l.end)).map(|(i, _)| *i); for c in csrs { psr_csrs.push((link, c)); @@ -655,3 +683,23 @@ pub(super) fn with_list_tab(mut a: ParaAttrs, resolved: impl FnOnce() -> (design a.tabs = Some(tabs); a } + +/// Byte offsets of a story's anchor, object, index and cross-reference markers, in text order. +pub(super) struct Marks { + anchor: Vec, + object: Vec, + index: Vec, + xref: Vec, +} + +impl Marks { + fn of(s: &Story) -> Self { + let at = |c: char| s.text.match_indices(c).map(|(i, _)| i).collect::>(); + Marks { + anchor: at(designcraft_doc::ANCHOR_MARK), + object: at(designcraft_doc::OBJECT_MARK), + index: at(designcraft_doc::INDEX_MARK), + xref: at(designcraft_doc::XREF_MARK), + } + } +} diff --git a/crates/idml/src/import/object_style.rs b/crates/idml/src/import/object_style.rs index 343e67211..6af6f845e 100644 --- a/crates/idml/src/import/object_style.rs +++ b/crates/idml/src/import/object_style.rs @@ -242,9 +242,11 @@ fn overlay(base: &mut El, top: &El, depth: usize) { *base = top.clone(); return; } + // Where `base`'s Properties child is, once (a search per attribute otherwise). + let props = base.children.iter().position(|n| matches!(n, Node::El(p) if p.local() == "Properties")); for (k, v) in &top.attrs { base.set(k, v); - if let Some(Node::El(p)) = base.children.iter_mut().find(|n| matches!(n, Node::El(p) if p.local() == "Properties")) { + if let Some(Node::El(p)) = props.and_then(|at| base.children.get_mut(at)) { p.children.retain(|n| !matches!(n, Node::El(x) if x.local() == k)); } } @@ -259,22 +261,26 @@ fn overlay(base: &mut El, top: &El, depth: usize) { // The k-th child of a name merges with the k-th child of that name in `base` (repeated elements // such as gradient stops pair up in order). let mut seen: HashMap<&str, usize> = HashMap::new(); + // Positions in `base.children` of its elements, by name, in order: the k-th is a lookup, not a + // scan from the first child (many repeated names made the scans quadratic). + let mut at: HashMap> = HashMap::new(); + for (i, b) in base.children.iter().enumerate() { + if let Node::El(b) = b { + at.entry(b.local().to_string()).or_default().push(i); + } + } for n in &top.children { match n { Node::El(c) => { let k = seen.entry(c.local()).or_insert(0); - let slot = base - .children - .iter_mut() - .filter_map(|b| match b { - Node::El(b) if b.local() == c.local() => Some(b), - _ => None, - }) - .nth(*k); + let slot = at.get(c.local()).and_then(|v| v.get(*k)).copied(); *k += 1; - match slot { - Some(b) => overlay(b, c, depth + 1), - None => base.children.push(n.clone()), + match slot.and_then(|i| base.children.get_mut(i)) { + Some(Node::El(b)) => overlay(b, c, depth + 1), + _ => { + at.entry(c.local().to_string()).or_default().push(base.children.len()); + base.children.push(n.clone()); + } } } Node::Text(_) | Node::CData(_) if has_text => base.children.push(n.clone()), diff --git a/crates/idml/src/lib.rs b/crates/idml/src/lib.rs index 8f7b22ff9..0c024dfab 100644 --- a/crates/idml/src/lib.rs +++ b/crates/idml/src/lib.rs @@ -57,7 +57,7 @@ mod xml; #[cfg(test)] mod tests; -pub use export::{ExportOptions, export_idml, export_idml_with}; +pub use export::{ExportOptions, export_idml, export_idml_report, export_idml_with}; pub use import::{Imported, import_idml, import_idml_report, import_idml_report_with_previews, import_idml_with}; /// The IDML package mimetype (content of the first, stored `mimetype` entry). diff --git a/crates/idml/src/tests/assets.rs b/crates/idml/src/tests/assets.rs index 48875c256..265183540 100644 --- a/crates/idml/src/tests/assets.rs +++ b/crates/idml/src/tests/assets.rs @@ -393,3 +393,42 @@ fn the_swatch_applied_to_an_image_is_kept() { assert!(!images.iter().any(|e| e.contains("Swatch/None")), "{images:?}"); assert_eq!(fills(&import_idml_with(&out, &reader).unwrap()), want); } + +/// Past the embedded-image limit an export still succeeds: the images that no longer fit are +/// written as links, and each is named in a warning. +#[test] +fn export_past_the_embed_limit_warns_for_each_image_written_as_a_link() { + let bytes = vec![42u8; 300]; + let reader = |p: &str| p.ends_with("couple.jpg").then(|| bytes.clone()); + let frames = vec![ + frame(1, 0.0, 1.0, &image(1, PHOTO, None)), + frame(2, 100.0, 2.0, &image(2, PHOTO, None)), + frame(3, 200.0, 3.0, &image(3, PHOTO, None)), + frame(4, 300.0, 4.0, &image(4, "file:/elsewhere/copy.jpg", Some(&bytes))), + ]; + let d = import_idml_with(&package(&frames), &reader).unwrap(); + let opts = ExportOptions { embed_images: true }; + // Four placements of 300 bytes each; the limit holds two. + let (out, warnings) = crate::export::export_with_limit(&d, &opts, 700); + assert_eq!(warnings.len(), 2, "{warnings:?}"); + assert!(warnings.iter().all(|w| w.contains("written as a link")), "{warnings:?}"); + assert_eq!(import_idml_with(&out, &reader).unwrap().assets.len(), 2); + assert!(crate::export::export_with_limit(&d, &opts, 1200).1.is_empty()); +} + +/// An image with no link of its own is embedded even past the limit: it has no other home, and +/// dropping its data would lose it. +#[test] +fn an_unlinked_image_is_embedded_past_the_embed_limit() { + let bytes = vec![42u8; 300]; + let reader = |p: &str| p.ends_with("couple.jpg").then(|| bytes.clone()); + let frames = vec![frame(1, 0.0, 1.0, &image(1, PHOTO, None)), frame(2, 100.0, 2.0, &image(2, PHOTO, None))]; + let mut d = import_idml_with(&package(&frames), &reader).unwrap(); + for a in d.assets.values_mut() { + std::sync::Arc::make_mut(a).link = None; + } + let (out, warnings) = crate::export::export_with_limit(&d, &ExportOptions { embed_images: true }, 100); + assert!(warnings.is_empty(), "{warnings:?}"); + let back = import_idml_with(&out, &|_: &str| None).unwrap(); + assert!(!back.assets.is_empty() && back.assets.values().all(|a| a.data.as_slice() == bytes.as_slice())); +}