feat(assets): deduplicate files and omit unused assets on save

Share duplicate file data while keeping distinct link paths independent, and leave unused assets out of saved documents.

Validation: all six cargo xtask ci gates passed on the merge candidate; 1,170 native tests passed and all 15 WASM packages checked. Independent regression checks cover updates, relinking, embedding, undo and save/reopen.

Task: #420
This commit is contained in:
aiongg authored and GitHub committed 2026-10-11 19:23:58 -06:00
1 parent ca1a8a12bc
commit 223dedfe5a
14 files changed
+812 -82

No files matched your search

+351
View File
@@ -0,0 +1,351 @@
//! Placed files (assets): one asset per distinct file, the assets a document uses, and the
//! compaction a save applies (duplicates merged, unused assets left out).
//!
//! An asset holds what is shared by every placement of a file (bytes, link, type, the PDF page
//! shown); each placement's frame data (transform, crop, fitting) lives on its [`Graphic`].
use std::collections::{BTreeMap, BTreeSet, HashMap};
use std::sync::Arc;
use crate::story::Story;
use crate::{Asset, AssetId, Content, Document, Item};
/// Nesting limit for groups, anchored objects, table cells and notes. A document nested deeper
/// is treated as using every asset, so nothing it references is merged away or pruned.
const MAX_DEPTH: usize = 64;
impl Asset {
/// Whether `other` holds the same file: the same type, page, link and bytes. Files at
/// different paths stay apart even with identical bytes, since each can be updated or
/// relinked on its own. Assets without bytes (a link that couldn't be read) are the same only
/// when they link to the same path.
pub fn same_file(&self, other: &Asset) -> bool {
if self.mime != other.mime || self.page != other.page {
return false;
}
if self.link != other.link {
return false;
}
match (self.data.is_empty(), other.data.is_empty()) {
(false, false) => self.same_bytes(other),
(true, true) => self.link.is_some(),
_ => false,
}
}
fn same_bytes(&self, other: &Asset) -> bool {
Arc::ptr_eq(&self.data, &other.data) || (self.data.len() == other.data.len() && *self.data == *other.data)
}
}
/// The asset in `assets` holding the same file as `asset`.
pub fn find_same_asset(assets: &BTreeMap<AssetId, Arc<Asset>>, asset: &Asset) -> Option<AssetId> {
assets.values().find(|a| a.same_file(asset)).map(|a| a.id)
}
/// Add `asset` to `assets` unless they already hold the same file; returns the id placements
/// should reference. A new asset with the bytes of an existing one shares its buffer.
pub fn insert_asset(assets: &mut BTreeMap<AssetId, Arc<Asset>>, mut asset: Asset) -> AssetId {
if let Some(id) = find_same_asset(assets, &asset) {
return id;
}
if !asset.data.is_empty()
&& let Some(a) = assets.values().find(|a| a.same_bytes(&asset))
{
asset.data = Arc::clone(&a.data);
}
let id = asset.id;
assets.insert(id, Arc::new(asset));
id
}
/// Calls `f` with every asset reference in `it` and its children; false when nesting exceeded
/// [`MAX_DEPTH`] (some references weren't visited).
fn item_refs(it: &Item, f: &mut impl FnMut(AssetId), depth: usize) -> bool {
if depth > MAX_DEPTH {
return false;
}
if let Content::Graphic(g) = &it.content {
f(g.asset);
}
if let Some(p) = it.media.as_ref().and_then(|m| m.poster) {
f(p);
}
it.children().iter().all(|c| item_refs(c, f, depth + 1))
}
/// Asset references in a story's anchored objects, graphic cells and notes (recursively).
fn story_refs(st: &Story, f: &mut impl FnMut(AssetId), depth: usize) -> bool {
if depth > MAX_DEPTH {
return false;
}
let mut complete = st.objects.iter().all(|o| item_refs(&o.item, f, depth + 1));
for t in st.tables.values() {
for c in &t.cells {
if let Some(g) = &c.graphic {
f(g.asset);
}
complete &= story_refs(&c.text, f, depth + 1);
}
}
for n in st.notes.iter().chain(&st.endnotes) {
complete &= story_refs(&n.text, f, depth + 1);
}
complete
}
/// Every asset reference in a story (its anchored objects, graphic cells and notes).
pub fn story_asset_refs(st: &Story) -> Vec<AssetId> {
let mut out = Vec::new();
story_refs(st, &mut |a| out.push(a), 0);
out
}
/// Point a story's references to a key of `map` at its value.
pub fn remap_story_assets(st: &mut Story, map: &HashMap<AssetId, AssetId>) {
if !map.is_empty() {
remap_story(st, map, 0);
}
}
fn item_uses(it: &Item, map: &HashMap<AssetId, AssetId>) -> bool {
let mut hit = false;
item_refs(it, &mut |a| hit |= map.contains_key(&a), 0);
hit
}
fn story_uses(st: &Story, map: &HashMap<AssetId, AssetId>) -> bool {
let mut hit = false;
story_refs(st, &mut |a| hit |= map.contains_key(&a), 0);
hit
}
fn remap_id(id: &mut AssetId, map: &HashMap<AssetId, AssetId>) {
if let Some(n) = map.get(id) {
*id = *n;
}
}
/// Rewrites the references in `it`, copying only the shared items that change.
fn remap_item(it: &mut Arc<Item>, map: &HashMap<AssetId, AssetId>, depth: usize) {
if depth > MAX_DEPTH || !item_uses(it, map) {
return;
}
remap_item_in(Arc::make_mut(it), map, depth);
}
fn remap_item_in(it: &mut Item, map: &HashMap<AssetId, AssetId>, depth: usize) {
if let Content::Graphic(g) = &mut it.content {
remap_id(&mut g.asset, map);
}
if let Some(p) = it.media.as_mut().and_then(|m| m.poster.as_mut()) {
remap_id(p, map);
}
if let Some(kids) = it.children_mut() {
for k in kids {
remap_item(k, map, depth + 1);
}
}
}
fn remap_story(st: &mut Story, map: &HashMap<AssetId, AssetId>, depth: usize) {
if depth > MAX_DEPTH {
return;
}
for o in &mut st.objects {
if item_uses(&o.item, map) {
remap_item_in(&mut Arc::make_mut(o).item, map, depth + 1);
}
}
for t in st.tables.values_mut() {
if !t.cells.iter().any(|c| c.graphic.as_ref().is_some_and(|g| map.contains_key(&g.asset)) || story_uses(&c.text, map)) {
continue;
}
for c in &mut Arc::make_mut(t).cells {
if let Some(g) = &mut c.graphic {
remap_id(&mut g.asset, map);
}
remap_story(&mut c.text, map, depth + 1);
}
}
for n in st.notes.iter_mut().chain(st.endnotes.iter_mut()) {
if story_uses(&n.text, map) {
remap_story(&mut Arc::make_mut(n).text, map, depth + 1);
}
}
}
impl Document {
/// Add a placed file, reusing the asset of an identical file already in the document;
/// returns the id to reference.
pub fn add_asset(&mut self, asset: Asset) -> AssetId {
insert_asset(&mut self.assets, asset)
}
/// Copy asset `id` of `src` into this document (or reuse the identical asset it already
/// has); returns the id to reference here, `None` when `src` has no such asset.
pub fn adopt_asset(&mut self, src: &Document, id: AssetId) -> Option<AssetId> {
let a = src.assets.get(&id)?;
if let Some(same) = find_same_asset(&self.assets, a) {
return Some(same);
}
let fresh = AssetId(self.alloc());
Some(self.add_asset(Asset { id: fresh, ..(**a).clone() }))
}
/// Calls `f` with every asset reference: graphics on document and parent spreads (in
/// groups and multi-state objects too), media posters, anchored and inline objects, graphic
/// table cells and notes. False when nesting exceeded the depth limit.
fn asset_refs(&self, f: &mut impl FnMut(AssetId)) -> bool {
let mut complete = true;
for sp in self.spreads.iter().chain(&self.parents) {
for it in &sp.items {
complete &= item_refs(it, f, 0);
}
}
for st in self.stories.values() {
complete &= story_refs(st, f, 0);
}
complete
}
/// The assets something in the document references. When the document nests deeper than
/// the traversal limit, every asset counts as used.
pub fn used_assets(&self) -> BTreeSet<AssetId> {
let mut used = BTreeSet::new();
if !self.asset_refs(&mut |a| {
used.insert(a);
}) {
return self.assets.keys().copied().collect();
}
used
}
/// Point every reference to a key of `map` at its value.
pub fn remap_assets(&mut self, map: &HashMap<AssetId, AssetId>) {
if map.is_empty() {
return;
}
for sp in self.spreads.iter_mut().chain(self.parents.iter_mut()) {
if sp.items.iter().any(|it| item_uses(it, map)) {
for it in &mut Arc::make_mut(sp).items {
remap_item(it, map, 0);
}
}
}
for st in self.stories.values_mut() {
if story_uses(st, map) {
remap_story(Arc::make_mut(st), map, 0);
}
}
}
/// Merge assets holding the same file into the first of them (lowest id), rewriting the
/// references; returns how many assets were removed.
pub fn merge_duplicate_assets(&mut self) -> usize {
if !self.asset_refs(&mut |_| {}) {
return 0;
}
let mut reps: HashMap<(usize, &str, u32), Vec<&Arc<Asset>>> = HashMap::new();
let mut map = HashMap::new();
for a in self.assets.values() {
let bucket = reps.entry((a.data.len(), a.mime.as_str(), a.page)).or_default();
match bucket.iter().find(|r| r.same_file(a)) {
Some(r) => {
map.insert(a.id, r.id);
}
None => bucket.push(a),
}
}
self.remap_assets(&map);
self.assets.retain(|id, _| !map.contains_key(id));
map.len()
}
/// Merge duplicate assets and drop the ones nothing references (what a saved file holds).
pub fn compact_assets(&mut self) {
self.merge_duplicate_assets();
let used = self.used_assets();
self.assets.retain(|id, _| used.contains(id));
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{Graphic, ItemId, LayerId, Shape, StoryId};
use designcraft_geom::{Affine, Rect, shapes};
fn asset(id: u64, data: &[u8]) -> Asset {
Asset { id: AssetId(id), name: "p.png".into(), mime: "image/png".into(), link: None, data: Arc::new(data.to_vec()), pixels: None, page: 0 }
}
fn graphic_item(id: u64, asset: u64, crop: f64) -> Item {
let mut it = Item::new(ItemId(id), LayerId(1), Shape::Rectangle, shapes::rectangle(Rect::new(0.0, 0.0, 10.0, 10.0)));
it.content = Content::Graphic(Graphic {
asset: AssetId(asset),
size: (10.0, 10.0),
xf: Affine::translate((crop, 0.0)),
auto_fit: Default::default(),
fit_align: 4,
crop: [crop; 4],
});
it
}
#[test]
fn identical_files_share_one_asset() {
let mut d = Document::new(&Default::default());
let a = d.add_asset(asset(1, b"same"));
let b = d.add_asset(asset(2, b"same"));
let c = d.add_asset(asset(3, b"diff"));
let p = d.add_asset(Asset { page: 1, ..asset(4, b"same") });
assert_eq!(a, b);
assert_ne!(a, c);
assert_ne!(a, p);
assert_eq!(d.assets.len(), 3);
// Unreadable links match by path only.
let m1 = d.add_asset(Asset { link: Some("/m/1.png".into()), ..asset(5, b"") });
let m2 = d.add_asset(Asset { link: Some("/m/2.png".into()), ..asset(6, b"") });
let m3 = d.add_asset(asset(7, b""));
let m4 = d.add_asset(asset(8, b""));
assert_ne!(m1, m2);
assert_ne!(m3, m4);
}
#[test]
fn files_at_different_paths_stay_separate_assets() {
let mut d = Document::new(&Default::default());
let linked = |id, path: &str| Asset { link: Some(path.into()), ..asset(id, b"same") };
let embedded = d.add_asset(asset(1, b"same"));
let a = d.add_asset(linked(2, "/p/a.png"));
let b = d.add_asset(linked(3, "/q/b.png"));
assert_eq!(d.add_asset(linked(4, "/p/a.png")), a, "the same path placed again");
assert_eq!(d.assets.len(), 3);
assert!(embedded != a && a != b && embedded != b);
assert_eq!(d.assets[&embedded].link, None);
assert_eq!(d.assets[&b].link.as_deref(), Some("/q/b.png"));
assert!(Arc::ptr_eq(&d.assets[&a].data, &d.assets[&b].data), "identical bytes share one buffer");
}
#[test]
fn compaction_merges_duplicates_and_drops_unused_assets() {
let mut d = Document::new(&Default::default());
for (id, data) in [(1, b"one"), (2, b"one"), (3, b"two"), (4, b"old")] {
d.assets.insert(AssetId(id), Arc::new(asset(id, data)));
}
let sp = Arc::make_mut(d.spreads.first_mut().unwrap());
sp.items.push(Arc::new(graphic_item(10, 1, 1.0)));
sp.items.push(Arc::new(graphic_item(11, 2, 2.0)));
// A graphic cell and an anchored object in a story.
let mut st = Story::new(StoryId(20));
st.objects.push(Arc::new(crate::AnchoredObject::new(graphic_item(12, 3, 3.0), Default::default())));
d.stories.insert(StoryId(20), Arc::new(st));
assert_eq!(d.used_assets(), [1, 2, 3].map(AssetId).into_iter().collect());
d.compact_assets();
assert_eq!(d.assets.keys().copied().collect::<Vec<_>>(), vec![AssetId(1), AssetId(3)]);
let crops: Vec<(AssetId, f64)> = d.spreads[0].items.iter().filter_map(|i| i.graphic().map(|g| (g.asset, g.crop[0]))).collect();
assert_eq!(crops, vec![(AssetId(1), 1.0), (AssetId(1), 2.0)]);
}
}
+2
View File
@@ -12,6 +12,7 @@
pub mod anchored;
pub mod arabic;
pub mod arrow;
pub mod assets;
pub mod attrs;
pub mod build;
pub mod cjk;
@@ -36,6 +37,7 @@ use std::collections::BTreeMap;
use std::sync::Arc;
pub use anchored::{AnchorPosition, AnchoredObject, OBJECT_MARK};
pub use assets::{find_same_asset, insert_asset, remap_story_assets, story_asset_refs};
pub use attrs::*;
pub use datamerge::{
DataField, DataFieldKind, DataMerge, DataSource, Delimiter, Fingerprint, MergeOptions, Placeholder, PlaceholderAnchor, PlaceholderRole,
+26 -15
View File
@@ -75,6 +75,11 @@ fn renumber(dst: &mut designcraft_doc::Document, src: &designcraft_doc::Document
let new_id = ItemId(dst.alloc());
maps.items.insert(old, new_id);
it.id = new_id;
if let Some(m) = it.media.as_mut()
&& let Some(p) = m.poster
{
m.poster = dst.adopt_asset(src, p).or(Some(p));
}
match &mut it.content {
Content::Text(tf) => {
let new_sid = if let Some(n) = maps.stories.get(&tf.story) {
@@ -83,6 +88,15 @@ fn renumber(dst: &mut designcraft_doc::Document, src: &designcraft_doc::Document
let n = StoryId(dst.alloc());
let mut st = src.story(tf.story).cloned().unwrap_or_else(|| Story::new(n));
st.id = n;
let mut assets = HashMap::new();
for a in designcraft_doc::story_asset_refs(&st) {
if let Some(na) = dst.adopt_asset(src, a)
&& na != a
{
assets.insert(a, na);
}
}
designcraft_doc::remap_story_assets(&mut st, &assets);
st.frames.clear();
dst.stories.insert(n, Arc::new(st));
maps.stories.insert(tf.story, n);
@@ -94,8 +108,8 @@ fn renumber(dst: &mut designcraft_doc::Document, src: &designcraft_doc::Document
}
}
Content::Graphic(g) => {
if let Some(a) = src.assets.get(&g.asset) {
dst.assets.entry(g.asset).or_insert_with(|| a.clone());
if let Some(a) = dst.adopt_asset(src, g.asset) {
g.asset = a;
}
}
Content::Group { items } => {
@@ -355,21 +369,18 @@ fn place_image(
};
let (nw, nh) = (f64::from(pw), f64::from(ph));
let mode = fitting_of(&options.fitting);
let aid = AssetId(doc.alloc());
let fresh = AssetId(doc.alloc());
let name = path.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_else(|| field.to_string());
let link = options.link_images.then(|| path.to_string_lossy().into_owned());
doc.assets.insert(
aid,
Arc::new(Asset {
id: aid,
name,
mime: designcraft_render::image_mime(&bytes).to_string(),
link,
data: Arc::new(bytes),
pixels: Some((pw, ph)),
page: 0,
}),
);
let aid = doc.add_asset(Asset {
id: fresh,
name,
mime: designcraft_render::image_mime(&bytes).to_string(),
link,
data: Arc::new(bytes),
pixels: Some((pw, ph)),
page: 0,
});
let mut g = Graphic { asset: aid, size: (nw, nh), xf: Affine::translate((inner.x0, inner.y0)), auto_fit: mode, fit_align: 0, crop: [0.0; 4] };
if let Some(xf) = g.fitted(inner, mode) {
g.xf = xf;
+4 -8
View File
@@ -337,9 +337,9 @@ fn file_place(s: &mut Session, p: &Value) -> Result<Value> {
let py = p.get("y").and_then(Value::as_f64);
let want_w = p.get("width").and_then(Value::as_f64);
s.edit(|d, sel| {
let aid = AssetId(d.alloc());
let fresh = AssetId(d.alloc());
let mime = designcraft_render::image_mime(&bytes).to_string();
d.assets.insert(aid, Arc::new(Asset { page: pdf_page, id: aid, name, mime, link, data: Arc::new(bytes), pixels: Some((pw, ph)) }));
let aid = d.add_asset(Asset { page: pdf_page, id: fresh, name, mime, link, data: Arc::new(bytes), pixels: Some((pw, ph)) });
// The width the whole page (or image) gets; the height follows the proportions.
let w = match want_w {
Some(w) => w,
@@ -443,13 +443,9 @@ fn place_load(s: &mut Session, p: &Value) -> Result<Value> {
let (pw, ph) = designcraft_render::image_size(&bytes).ok_or_else(|| bad("place.load", "unsupported or corrupt image"))?;
let natural = designcraft_render::natural_size(&bytes, 0).unwrap_or((pw as f64, ph as f64));
let aid = s.edit(|d, _| {
let aid = AssetId(d.alloc());
let fresh = AssetId(d.alloc());
let mime = designcraft_render::image_mime(&bytes).to_string();
d.assets.insert(
aid,
Arc::new(Asset { page: 0, id: aid, name: name.clone(), mime, link: link.clone(), data: Arc::new(bytes), pixels: Some((pw, ph)) }),
);
Ok(aid)
Ok(d.add_asset(Asset { page: 0, id: fresh, name: name.clone(), mime, link: link.clone(), data: Arc::new(bytes), pixels: Some((pw, ph)) }))
})?;
s.loaded = Some((aid, natural));
s.set_tool("placeGun");
+117
View File
@@ -112,8 +112,10 @@ pub fn status(a: &designcraft_doc::Asset) -> &'static str {
}
fn list(d: &Document) -> Vec<Value> {
let used = d.used_assets();
d.assets
.values()
.filter(|a| used.contains(&a.id))
.map(|a| {
let uses: Vec<Value> = uses(d, a.id)
.into_iter()
@@ -311,3 +313,118 @@ mod relink_folder_tests {
let _ = std::fs::remove_dir_all(&dir);
}
}
#[cfg(test)]
mod shared_asset_tests {
use super::*;
fn png(w: u32, h: u32) -> Vec<u8> {
designcraft_render::Rendered { width: w, height: h, pixels: vec![200u8; (w * h * 4) as usize] }.to_png()
}
#[test]
fn placements_of_one_file_share_an_asset_and_saves_drop_replaced_ones() {
let mut s = Session::new();
s.execute("file.new", &json!({})).unwrap();
let a = super::super::file::base64_encode(&png(10, 10));
s.execute("file.place", &json!({"base64": a, "name": "a.png", "x": 72, "y": 72})).unwrap();
s.execute("edit.deselectAll", &json!({})).unwrap();
s.execute("file.place", &json!({"base64": a, "name": "a.png", "x": 200, "y": 72})).unwrap();
let l = s.execute("links.list", &json!({})).unwrap();
assert_eq!(l.as_array().unwrap().len(), 1);
assert_eq!(l[0]["uses"].as_array().unwrap().len(), 2);
assert_eq!(s.doc().unwrap().doc.assets.len(), 1);
// A new image into a frame: the old asset stays in memory (undo) but not in the file.
let f = s.execute("frame.create", &json!({"rect": [100, 300, 200, 400]})).unwrap()["id"].as_u64().unwrap();
let b = super::super::file::base64_encode(&png(20, 10));
let c = super::super::file::base64_encode(&png(30, 10));
s.execute("file.place", &json!({"base64": b, "name": "b.png", "frame": f})).unwrap();
s.execute("file.place", &json!({"base64": c, "name": "c.png", "frame": f})).unwrap();
let d = &s.doc().unwrap().doc;
assert_eq!(d.assets.len(), 3);
assert_eq!(s.execute("links.list", &json!({})).unwrap().as_array().unwrap().len(), 2);
let back = designcraft_format::load(&designcraft_format::save(&s.doc().unwrap().doc).unwrap()).unwrap();
let mut names: Vec<&str> = back.assets.values().map(|a| a.name.as_str()).collect();
names.sort();
assert_eq!(names, ["a.png", "c.png"]);
}
#[test]
fn files_with_identical_bytes_at_different_paths_keep_their_own_links() {
let dir = std::env::temp_dir().join(format!("dc-same-bytes-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).unwrap();
let (a, b) = (dir.join("a.png"), dir.join("b.png"));
std::fs::write(&a, png(10, 10)).unwrap();
std::fs::write(&b, png(10, 10)).unwrap();
let mut s = Session::new();
s.execute("file.new", &json!({})).unwrap();
for (path, x) in [(&a, 72), (&b, 200)] {
s.execute("edit.deselectAll", &json!({})).unwrap();
s.execute("file.place", &json!({"path": path.to_string_lossy(), "x": x, "y": 72})).unwrap();
}
assert_eq!(s.doc().unwrap().doc.assets.len(), 2);
let rows = |s: &mut Session| -> Vec<(String, Value)> {
let l = s.execute("links.list", &json!({})).unwrap();
l.as_array().unwrap().iter().map(|r| (r["path"].as_str().unwrap().to_string(), r["pixels"].clone())).collect()
};
let paths = |r: &[(String, Value)]| r.iter().map(|(p, _)| p.clone()).collect::<Vec<_>>();
let before = rows(&mut s);
assert_eq!(paths(&before), [a.to_string_lossy(), b.to_string_lossy()]);
// b.png changes on disk: updating it leaves a.png's placement alone.
std::fs::write(&b, png(20, 10)).unwrap();
assert_eq!(s.execute("links.update", &json!({})).unwrap()["updated"], 1);
let after = rows(&mut s);
assert_eq!(paths(&after), paths(&before));
assert_eq!((after[0].1.clone(), after[1].1.clone()), (json!([10, 10]), json!([20, 10])));
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn pasted_graphics_reuse_the_identical_asset_of_the_target() {
use designcraft_doc::build::NewDocument;
use designcraft_doc::{Asset, Graphic, Item, ItemId, Shape, SpreadRef};
let asset = |id: u64, data: Vec<u8>| Asset {
id: AssetId(id),
name: "p.png".into(),
mime: "image/png".into(),
link: None,
data: Arc::new(data),
pixels: None,
page: 0,
};
let mut src = Document::new(&NewDocument::default());
let mut dst = Document::new(&NewDocument::default());
let (pa, pb) = (png(4, 4), png(8, 4));
// The source's asset id is taken in the target by a different file.
let sid = AssetId(src.alloc());
src.assets.insert(sid, Arc::new(asset(sid.0, pa.clone())));
dst.assets.insert(sid, Arc::new(asset(sid.0, pb)));
dst.alloc();
let same = AssetId(dst.alloc());
assert_ne!(same, sid);
dst.assets.insert(same, Arc::new(asset(same.0, pa)));
let iid = ItemId(src.alloc());
let mut it = Item::new(
iid,
src.default_layer(),
Shape::Rectangle,
designcraft_geom::shapes::rectangle(designcraft_geom::Rect::new(0.0, 0.0, 9.0, 9.0)),
);
it.content = Content::Graphic(Graphic {
asset: sid,
size: (4.0, 4.0),
xf: designcraft_geom::Affine::IDENTITY,
auto_fit: Default::default(),
fit_align: 4,
crop: [1.0; 4],
});
src.insert_item(SpreadRef::Doc(0), it, None).unwrap();
let ids = super::super::object::duplicate_from(&mut dst, &src, &[iid], SpreadRef::Doc(0), designcraft_geom::Vec2::ZERO).unwrap();
let g = dst.item(ids[0]).and_then(|i| i.graphic()).unwrap();
assert_eq!((g.asset, g.crop), (same, [1.0; 4]));
assert_eq!(dst.assets.len(), 2);
}
}
+10 -5
View File
@@ -13,11 +13,16 @@ use crate::{Result, Session};
fn math_item(d: &mut designcraft_doc::Document, latex: &str, size: f64, lid: designcraft_doc::LayerId, at: (f64, f64)) -> Item {
let (svg, (w, h)) = crate::math::to_svg(latex, size);
let aid = AssetId(d.alloc());
d.assets.insert(
aid,
Arc::new(Asset { page: 0, id: aid, name: "math.svg".into(), mime: "image/svg+xml".into(), link: None, data: Arc::new(svg), pixels: None }),
);
let fresh = AssetId(d.alloc());
let aid = d.add_asset(Asset {
page: 0,
id: fresh,
name: "math.svg".into(),
mime: "image/svg+xml".into(),
link: None,
data: Arc::new(svg),
pixels: None,
});
let id = ItemId(d.alloc());
let r = Rect::new(at.0, at.1, at.0 + w, at.1 + h);
let mut it = Item::new(id, lid, Shape::Rectangle, shapes::rectangle(r));
+4 -5
View File
@@ -25,8 +25,8 @@ pub(crate) fn place_media(s: &mut Session, p: &Value, name: String, mime: &str,
let (px, py) = (p.get("x").and_then(Value::as_f64), p.get("y").and_then(Value::as_f64));
let mime = mime.to_string();
s.edit(|d, sel| {
let aid = AssetId(d.alloc());
d.assets.insert(aid, Arc::new(Asset { page: 0, id: aid, name, mime, link, data: Arc::new(bytes), pixels: None }));
let fresh = AssetId(d.alloc());
let aid = d.add_asset(Asset { page: 0, id: fresh, name, mime, link, data: Arc::new(bytes), pixels: None });
let id = match target {
Some(fid) => {
let it = d.item_mut(fid).ok_or(designcraft_doc::DocError::NoItem(fid))?;
@@ -90,10 +90,9 @@ pub fn specs() -> Vec<CommandSpec> {
s.edit(|d, _| {
let poster_id = match poster {
Some(Some((bytes, name, link, px))) => {
let aid = AssetId(d.alloc());
let fresh = AssetId(d.alloc());
let mime = designcraft_render::image_mime(&bytes).to_string();
d.assets.insert(aid, Arc::new(Asset { page: 0, id: aid, name, mime, link, data: Arc::new(bytes), pixels: Some(px) }));
Some(Some(aid))
Some(Some(d.add_asset(Asset { page: 0, id: fresh, name, mime, link, data: Arc::new(bytes), pixels: Some(px) })))
}
Some(None) => Some(None),
None => None,
+16 -4
View File
@@ -900,6 +900,11 @@ pub fn duplicate_from(dst: &mut Document, src: &Document, ids: &[ItemId], to: Sp
fn renumber(dst: &mut Document, src: &Document, it: &mut Item, stories: &mut HashMap<StoryId, StoryId>) {
it.id = ItemId(dst.alloc());
let my_id = it.id;
if let Some(m) = it.media.as_mut()
&& let Some(p) = m.poster
{
m.poster = dst.adopt_asset(src, p).or(Some(p));
}
match &mut it.content {
Content::Text(tf) => {
let new_sid = match stories.get(&tf.story) {
@@ -908,6 +913,15 @@ fn renumber(dst: &mut Document, src: &Document, it: &mut Item, stories: &mut Has
let n = StoryId(dst.alloc());
let mut st = src.story(tf.story).cloned().unwrap_or_else(|| Story::new(n));
st.id = n;
let mut assets = HashMap::new();
for a in designcraft_doc::story_asset_refs(&st) {
if let Some(na) = dst.adopt_asset(src, a)
&& na != a
{
assets.insert(a, na);
}
}
designcraft_doc::remap_story_assets(&mut st, &assets);
st.frames.clear();
dst.stories.insert(n, Arc::new(st));
stories.insert(tf.story, n);
@@ -920,10 +934,8 @@ fn renumber(dst: &mut Document, src: &Document, it: &mut Item, stories: &mut Has
}
}
Content::Graphic(g) => {
if let Some(a) = src.assets.get(&g.asset)
&& !dst.assets.contains_key(&g.asset)
{
dst.assets.insert(g.asset, a.clone());
if let Some(a) = dst.adopt_asset(src, g.asset) {
g.asset = a;
}
}
Content::Group { items } => {
+3 -3
View File
@@ -285,13 +285,13 @@ fn write_links(d: &Document, root: &OutputRoot, dir: &Path, only: Option<&[Asset
let mut out = d.clone();
let mut taken = Vec::new();
let mut written = Vec::new();
let mut ids: Vec<AssetId> = d.assets.keys().copied().collect();
ids.sort_by_key(|i| i.0);
// Only files something places; the saved document leaves the others out.
let ids = d.used_assets();
for id in ids {
if only.is_some_and(|o| !o.contains(&id)) {
continue;
}
let a = &d.assets[&id];
let Some(a) = d.assets.get(&id) else { continue };
let name = link_name(a.link.as_deref().unwrap_or(&a.name), id, &mut taken);
let path = dir.join(&name);
root.write(&path, a.data.as_slice())?;
+3 -13
View File
@@ -1402,20 +1402,10 @@ fn place_graphic(s: &mut Session, p: &Value) -> Result<Value> {
let (pw, ph) = designcraft_render::image_size(&bytes).ok_or_else(|| bad(ID, "unsupported or corrupt image"))?;
let fill = p.get("fit").and_then(Value::as_str) == Some("fill");
s.edit(|d, sel| {
let aid = designcraft_doc::AssetId(d.alloc());
let fresh = designcraft_doc::AssetId(d.alloc());
let mime = designcraft_render::image_mime(&bytes).to_string();
d.assets.insert(
aid,
std::sync::Arc::new(designcraft_doc::Asset {
page: 0,
id: aid,
name,
mime,
link,
data: std::sync::Arc::new(bytes),
pixels: Some((pw, ph)),
}),
);
let aid =
d.add_asset(designcraft_doc::Asset { page: 0, id: fresh, name, mime, link, data: std::sync::Arc::new(bytes), pixels: Some((pw, ph)) });
let st = d.story_mut(g.story).ok_or_else(|| bad(ID, "no story"))?;
let t = st.table_mut(g.table).ok_or_else(|| bad(ID, "no table"))?;
let (r, c) = t.owner(g.range.r0, g.range.c0);
+136 -17
View File
@@ -2,14 +2,15 @@
//!
//! - `mimetype` (stored first, uncompressed): `application/vnd.designcraft+zip`
//! - `document.json`: the [`Document`] (pretty JSON, documented by the serde model in `designcraft-doc`)
//! - `assets/<id>.<ext>`: embedded placed files, byte-for-byte
//! - `assets/<id>.<ext>`: embedded placed files, byte-for-byte: one per distinct file, and only
//! the ones the document references (see [`Document::compact_assets`])
//! - `meta.json`: format version and generator
//!
//! Older single-file JSON documents (assets as base64 in `assetData`) still open.
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
#![forbid(unsafe_code)]
use std::collections::HashSet;
use std::collections::{HashMap, HashSet};
use std::io::{Cursor, Read, Write};
use std::sync::Arc;
@@ -42,8 +43,12 @@ fn ext_for(mime: &str) -> &'static str {
}
}
/// Serialize a document to `.designcraft` bytes.
/// Serialize a document to `.designcraft` bytes. The file holds each distinct placed file once
/// and leaves out assets nothing references; `doc` itself keeps them (undo can bring them back).
pub fn save(doc: &Document) -> Result<Vec<u8>, FormatError> {
let mut compact = doc.clone();
compact.compact_assets();
let doc = &compact;
let io = |e: zip::result::ZipError| FormatError::Io(e.to_string());
let mut buf = Cursor::new(Vec::new());
{
@@ -100,13 +105,26 @@ fn load_zip(bytes: &[u8]) -> Result<Document, FormatError> {
upgrade(&mut v);
let mut doc: Document = serde_json::from_value(v).map_err(|e| FormatError::NotOurs(e.to_string()))?;
let names: Vec<String> = z.file_names().map(str::to_string).collect();
// Files saved with one copy per placement: identical copies share one buffer as they are
// read, so memory holds each distinct file once.
let mut loaded: HashMap<usize, Vec<Arc<Vec<u8>>>> = HashMap::new();
for name in names {
let Some(stem) = name.strip_prefix("assets/") else { continue };
let Some(id) = stem.split('.').next().and_then(|s| s.parse::<u64>().ok()) else { continue };
if let (Some(data), Some(a)) = (read(&mut z, &name), doc.assets.get_mut(&AssetId(id))) {
Arc::make_mut(a).data = Arc::new(data);
let same = loaded.entry(data.len()).or_default();
let data = match same.iter().find(|d| ***d == data) {
Some(d) => d.clone(),
None => {
let d = Arc::new(data);
same.push(d.clone());
d
}
};
Arc::make_mut(a).data = data;
}
}
doc.merge_duplicate_assets();
doc.check().map_err(|e| FormatError::NotOurs(e.to_string()))?;
Ok(doc)
}
@@ -124,6 +142,7 @@ fn load_legacy_json(bytes: &[u8]) -> Result<Document, FormatError> {
}
}
}
doc.merge_duplicate_assets();
doc.check().map_err(|e| FormatError::NotOurs(e.to_string()))?;
Ok(doc)
}
@@ -205,24 +224,67 @@ mod tests {
use designcraft_doc::geom::Rect;
use designcraft_doc::{Asset, ParaFormat, SpreadRef};
fn png_asset(d: &mut Document, data: &[u8]) -> AssetId {
let aid = AssetId(d.alloc());
let a = Asset {
page: 0,
id: aid,
name: "x.png".into(),
mime: "image/png".into(),
link: None,
data: Arc::new(data.to_vec()),
pixels: Some((1, 1)),
};
d.assets.insert(aid, Arc::new(a));
aid
}
fn place(d: &mut Document, asset: AssetId, crop: f64) -> designcraft_doc::ItemId {
use designcraft_doc::geom::{Affine, shapes};
use designcraft_doc::{Content, Graphic, Item, ItemId, Shape};
let id = ItemId(d.alloc());
let mut it = Item::new(id, d.default_layer(), Shape::Rectangle, shapes::rectangle(Rect::new(0.0, 0.0, 50.0, 50.0)));
it.content = Content::Graphic(Graphic {
asset,
size: (1.0, 1.0),
xf: Affine::translate((crop, crop)),
auto_fit: Default::default(),
fit_align: 4,
crop: [crop; 4],
});
d.insert_item(SpreadRef::Doc(0), it, None).unwrap();
id
}
fn asset_entries(bytes: &[u8]) -> usize {
let z = zip::ZipArchive::new(Cursor::new(bytes)).unwrap();
z.file_names().filter(|n| n.starts_with("assets/")).count()
}
/// Zip `d` the way files were saved with one asset per placement (nothing merged or pruned).
fn save_uncompacted(d: &Document) -> Vec<u8> {
let mut buf = Cursor::new(Vec::new());
let mut z = zip::ZipWriter::new(&mut buf);
let opts = zip::write::SimpleFileOptions::default();
z.start_file("mimetype", opts).unwrap();
z.write_all(MIME.as_bytes()).unwrap();
z.start_file("document.json", opts).unwrap();
z.write_all(&serde_json::to_vec(d).unwrap()).unwrap();
for (id, a) in &d.assets {
z.start_file(format!("assets/{}.png", id.0), opts).unwrap();
z.write_all(&a.data).unwrap();
}
z.finish().unwrap();
buf.into_inner()
}
#[test]
fn roundtrip_with_assets() {
let mut d = Document::new(&NewDocument { pages: 3, ..Default::default() });
let lid = d.default_layer();
d.add_text_frame(SpreadRef::Doc(0), Rect::new(10.0, 10.0, 200.0, 200.0), lid, "héllo\nworld", ParaFormat::default()).unwrap();
let aid = AssetId(d.alloc());
d.assets.insert(
aid,
Arc::new(Asset {
page: 0,
id: aid,
name: "x.png".into(),
mime: "image/png".into(),
link: None,
data: Arc::new(vec![1, 2, 3, 4]),
pixels: Some((1, 1)),
}),
);
let aid = png_asset(&mut d, &[1, 2, 3, 4]);
place(&mut d, aid, 0.0);
let bytes = save(&d).unwrap();
assert!(bytes.starts_with(b"PK"));
let back = load(&bytes).unwrap();
@@ -231,6 +293,63 @@ mod tests {
assert_eq!(back.stories, d.stories);
}
#[test]
fn saves_hold_each_used_file_once() {
let mut d = Document::new(&NewDocument::default());
let first = png_asset(&mut d, &[7; 64]);
let copy = png_asset(&mut d, &[7; 64]);
let replaced = png_asset(&mut d, &[9; 64]);
let a = place(&mut d, first, 1.0);
let b = place(&mut d, copy, 2.0);
assert_eq!(d.assets.len(), 3);
// Duplicates merge into the first asset, the unreferenced one is left out, and every
// placement keeps its own crop and transform.
let bytes = save(&d).unwrap();
assert_eq!(asset_entries(&bytes), 1);
assert_eq!(d.assets.len(), 3, "the open document keeps its assets");
let back = load(&bytes).unwrap();
assert_eq!(back.assets.keys().copied().collect::<Vec<_>>(), vec![first]);
assert!(!back.assets.contains_key(&replaced));
for (id, crop) in [(a, 1.0), (b, 2.0)] {
let g = back.item(id).and_then(|i| i.graphic()).unwrap();
assert_eq!((g.asset, g.crop, g.xf), (first, [crop; 4], d.item(id).unwrap().graphic().unwrap().xf));
}
}
#[test]
fn files_with_identical_bytes_at_different_paths_keep_their_paths() {
let mut d = Document::new(&NewDocument::default());
let ids: Vec<AssetId> = (0..3).map(|_| png_asset(&mut d, &[3; 32])).collect();
for (id, link) in ids.iter().zip(["/p/a.png", "/q/b.png"]) {
if let Some(a) = d.assets.get_mut(id) {
Arc::make_mut(a).link = Some(link.into());
}
}
let items: Vec<_> = ids.iter().enumerate().map(|(k, id)| place(&mut d, *id, k as f64)).collect();
for bytes in [save(&d).unwrap(), save_uncompacted(&d)] {
let back = load(&bytes).unwrap();
assert_eq!(back.assets.len(), 3);
let links: Vec<Option<&str>> = items
.iter()
.map(|i| back.item(*i).and_then(|i| i.graphic()).and_then(|g| back.assets.get(&g.asset)).unwrap().link.as_deref())
.collect();
assert_eq!(links, [Some("/p/a.png"), Some("/q/b.png"), None]);
}
}
#[test]
fn older_files_with_one_asset_per_placement_open_with_one() {
let mut d = Document::new(&NewDocument::default());
let ids: Vec<AssetId> = (0..3).map(|_| png_asset(&mut d, &[5; 32])).collect();
for (k, id) in ids.iter().enumerate() {
place(&mut d, *id, k as f64);
}
let back = load(&save_uncompacted(&d)).unwrap();
assert_eq!(back.assets.len(), 1);
assert_eq!(asset_entries(&save(&back).unwrap()), 1);
}
/// The object of the item `id` in a document's JSON.
fn item_json(v: &mut Value, id: u64) -> Option<&mut Value> {
if v.get("id").and_then(Value::as_u64) == Some(id) && v.get("content").is_some() {
+29 -12
View File
@@ -132,6 +132,8 @@ struct Importer<'r> {
page_index: HashMap<String, usize>,
item_ids: HashMap<String, ItemId>,
assets: BTreeMap<AssetId, Arc<Asset>>,
/// Assets read from a link path, so later placements of that path reuse them unread.
linked_assets: HashMap<String, AssetId>,
ctx: ItemCtx,
sections: Vec<Section>,
footnote_options: designcraft_doc::FootnoteOptions,
@@ -217,6 +219,7 @@ impl<'r> Importer<'r> {
page_index: HashMap::new(),
item_ids: HashMap::new(),
assets: BTreeMap::new(),
linked_assets: HashMap::new(),
ctx: ItemCtx { threads: Vec::new() },
sections: Vec::new(),
footnote_options: Default::default(),
@@ -2205,12 +2208,29 @@ impl<'r> Importer<'r> {
let size = ((r - l).abs(), (b - t).abs());
let link = g.find("Link");
let uri = link.and_then(|k| k.get("LinkResourceURI")).map(uri_to_path);
let graphic = |asset| {
Content::Graphic(designcraft_doc::Graphic {
asset,
size,
xf: gxf * Affine::translate((l, t)),
auto_fit: Default::default(),
fit_align: 4,
crop: [0.0; 4],
})
};
let mut data = g.prop_el("Contents").map(|c| base64_decode(&c.text_content())).unwrap_or_default();
let mut read_from = None;
if data.is_empty()
&& let Some(p) = &uri
&& let Some(d) = (self.read_link)(p)
{
data = d;
// Every placement of a linked file shares one asset; read the file once.
if let Some(id) = self.linked_assets.get(p) {
return graphic(*id);
}
if let Some(d) = (self.read_link)(p) {
data = d;
read_from = Some(p.clone());
}
}
let (mime, px) = sniff_image(&data);
let mime = mime.map(str::to_string).unwrap_or_else(|| match link.and_then(|k| k.get("LinkResourceFormat")).unwrap_or("") {
@@ -2226,17 +2246,14 @@ impl<'r> Importer<'r> {
});
let name =
uri.as_deref().and_then(|p| p.rsplit(['/', '\\']).next()).filter(|n| !n.is_empty()).map(str::to_string).unwrap_or_else(|| "image".into());
let id = AssetId(self.alloc());
let fresh = AssetId(self.alloc());
let link_path = uri.filter(|p| p.contains('/') || p.contains('\\'));
self.assets.insert(id, Arc::new(Asset { page: 0, id, name, mime, link: link_path, data: Arc::new(data), pixels }));
Content::Graphic(designcraft_doc::Graphic {
asset: id,
size,
xf: gxf * Affine::translate((l, t)),
auto_fit: Default::default(),
fit_align: 4,
crop: [0.0; 4],
})
let id =
designcraft_doc::insert_asset(&mut self.assets, Asset { page: 0, id: fresh, name, mime, link: link_path, data: Arc::new(data), pixels });
if let Some(p) = read_from {
self.linked_assets.insert(p, id);
}
graphic(id)
}
fn link_threads(&mut self) {
+1
View File
@@ -7,6 +7,7 @@ use designcraft_geom::{Rect, shapes};
use super::*;
mod assets;
mod decorations;
fn zip_files(files: &[(&str, &str)]) -> Vec<u8> {
+110
View File
@@ -0,0 +1,110 @@
//! Placed files: every placement of one file shares an asset.
use std::cell::Cell;
use designcraft_doc::{AssetId, Content};
use super::*;
const PHOTO: &str = "file:/photos/wedding%20day/couple.jpg";
/// A graphic frame at `x` showing `image` (an `<Image>` element) with `crop` on every side.
fn frame(n: usize, x: f64, crop: f64, image: &str) -> String {
format!(
r#"<Rectangle Self="r{n}" ItemLayer="L1" ItemTransform="1 0 0 1 {x} 0" ContentType="GraphicType">
<Properties><PathGeometry><GeometryPathType PathOpen="false"><PathPointArray>
<PathPointType Anchor="0 0"/><PathPointType Anchor="0 30"/><PathPointType Anchor="40 30"/><PathPointType Anchor="40 0"/>
</PathPointArray></GeometryPathType></PathGeometry></Properties>
<FrameFittingOption TopCrop="{crop}" LeftCrop="{crop}" BottomCrop="{crop}" RightCrop="{crop}"/>
{image}
</Rectangle>"#
)
}
fn image(n: usize, uri: &str, contents: Option<&[u8]>) -> String {
let contents = contents.map(|c| format!("<Contents><![CDATA[{}]]></Contents>", base64_encode(c))).unwrap_or_default();
format!(
r#"<Image Self="im{n}" ItemTransform="1 0 0 1 {n} 0">
<Properties>{contents}<GraphicBounds Left="0" Top="0" Right="40" Bottom="30"/></Properties>
<Link Self="lk{n}" LinkResourceURI="{uri}" LinkResourceFormat="$ID/JPEG" StoredState="Normal"/>
</Image>"#
)
}
fn package(frames: &[String]) -> Vec<u8> {
let spread = format!(
r#"<?xml version="1.0" encoding="UTF-8"?>
<idPkg:Spread xmlns:idPkg="http://ns.adobe.com/AdobeInDesign/idml/1.0/packaging" DOMVersion="16.0">
<Spread Self="sp1" PageCount="1" ItemTransform="1 0 0 1 0 0">
<Page Self="p1" AppliedMaster="m1" GeometricBounds="0 0 700 500" ItemTransform="1 0 0 1 0 0"/>
{}
</Spread>
</idPkg:Spread>"#,
frames.join("\n")
);
zip_files(&[
("designmap.xml", DESIGNMAP),
("Resources/Graphic.xml", GRAPHIC),
("Resources/Styles.xml", STYLES),
("Resources/Preferences.xml", PREFS),
("MasterSpreads/MasterSpread_m1.xml", MASTER),
("Spreads/Spread_sp1.xml", &spread),
("Stories/Story_s1.xml", STORY),
])
}
/// (asset, crop, transform) of each graphic, left to right.
fn placements(d: &Document) -> Vec<(AssetId, [f64; 4], designcraft_geom::Affine)> {
let mut out: Vec<(f64, AssetId, [f64; 4], designcraft_geom::Affine)> = d.spreads[0]
.items
.iter()
.filter_map(|i| match &i.content {
Content::Graphic(g) => Some((i.xf.translation().x, g.asset, g.crop, g.xf)),
_ => None,
})
.collect();
out.sort_by(|a, b| a.0.total_cmp(&b.0));
out.into_iter().map(|(_, a, c, x)| (a, c, x)).collect()
}
#[test]
fn one_file_placed_several_times_imports_as_one_asset() {
let bytes = vec![42u8; 300];
let reads = Cell::new(0);
let reader = |p: &str| {
reads.set(reads.get() + 1);
p.ends_with("couple.jpg").then(|| bytes.clone())
};
// Three placements linking the file, and one embedding the same bytes under another path:
// that one keeps its own asset and path.
let frames = vec![
frame(1, 0.0, 1.0, &image(1, PHOTO, None)),
frame(2, 100.0, 2.0, &image(2, PHOTO, None)),
frame(3, 200.0, 3.0, &image(3, PHOTO, None)),
frame(4, 300.0, 4.0, &image(4, "file:/elsewhere/copy.jpg", Some(&bytes))),
];
let d = import_idml_with(&package(&frames), &reader).unwrap();
assert_eq!(reads.get(), 1, "the linked file is read once");
assert_eq!(d.assets.len(), 2);
let p = placements(&d);
assert_eq!(p.iter().map(|(_, c, _)| c[0]).collect::<Vec<_>>(), vec![1.0, 2.0, 3.0, 4.0]);
let link = |id| d.assets.get(id).and_then(|a| a.link.as_deref());
let links: Vec<_> = p.iter().map(|(id, _, _)| link(id)).collect();
assert_eq!(links[..3], [Some("/photos/wedding day/couple.jpg"); 3]);
assert_eq!(links[3], Some("/elsewhere/copy.jpg"));
assert!(p[..3].iter().all(|(id, _, _)| *id == p[0].0));
// Exported linked or embedded, the round trip keeps the two assets and each placement's crop
// and transform.
for embed_images in [false, true] {
let out = export_idml_with(&d, &ExportOptions { embed_images });
let back = import_idml_with(&out, &reader).unwrap();
assert_eq!(back.assets.len(), 2, "embed_images: {embed_images}");
let q = placements(&back);
assert_eq!(q.len(), 4);
for ((_, c0, x0), (_, c1, x1)) in p.iter().zip(&q) {
assert_eq!(c0, c1);
assert!(x0.as_coeffs().iter().zip(x1.as_coeffs()).all(|(a, b)| (a - b).abs() < 1e-6), "{x0:?} {x1:?}");
}
}
}