DES-10 T4: native format 5 with linked assets and proxies

Cargo.lock

@@ -241,7 +241,7 @@ dependencies = [
  "objc2-foundation 0.3.2",
  "parking_lot",
  "percent-encoding",
- "windows-sys 0.61.2",
+ "windows-sys 0.52.0",
  "wl-clipboard-rs",
  "x11rb",
 ]
@@ -1093,10 +1093,10 @@ dependencies = [
 [[package]]
 name = "craft-designcraft-format"
 version = "0.1.0"
-source = "git+https://github.com/storytold/craft-libs?rev=93d0998383aa150fa50a2ffbf49ef37786e37eca#93d0998383aa150fa50a2ffbf49ef37786e37eca"
+source = "git+https://github.com/aiongg/craft-libs?rev=808ec65380fb8fe0d71816aeadb9b2c5dc0142a4#808ec65380fb8fe0d71816aeadb9b2c5dc0142a4"
 dependencies = [
  "craft-document-bundle",
- "craft-eps 0.1.0 (git+https://github.com/storytold/craft-libs?rev=93d0998383aa150fa50a2ffbf49ef37786e37eca)",
+ "craft-eps 0.1.0 (git+https://github.com/aiongg/craft-libs?rev=808ec65380fb8fe0d71816aeadb9b2c5dc0142a4)",
  "serde",
  "serde_json",
  "thiserror 2.0.21",
@@ -1123,9 +1123,9 @@ source = "git+https://github.com/storytold/craft-libs?rev=92d392cff132eaf53e13af
 [[package]]
 name = "craft-document-bundle"
 version = "0.1.0"
-source = "git+https://github.com/storytold/craft-libs?rev=93d0998383aa150fa50a2ffbf49ef37786e37eca#93d0998383aa150fa50a2ffbf49ef37786e37eca"
+source = "git+https://github.com/aiongg/craft-libs?rev=808ec65380fb8fe0d71816aeadb9b2c5dc0142a4#808ec65380fb8fe0d71816aeadb9b2c5dc0142a4"
 dependencies = [
- "craft-zip 0.1.0 (git+https://github.com/storytold/craft-libs?rev=93d0998383aa150fa50a2ffbf49ef37786e37eca)",
+ "craft-zip 0.1.0 (git+https://github.com/aiongg/craft-libs?rev=808ec65380fb8fe0d71816aeadb9b2c5dc0142a4)",
  "serde_json",
  "thiserror 2.0.21",
  "zip 9.1.1",
@@ -1187,7 +1187,7 @@ dependencies = [
 [[package]]
 name = "craft-eps"
 version = "0.1.0"
-source = "git+https://github.com/storytold/craft-libs?rev=93d0998383aa150fa50a2ffbf49ef37786e37eca#93d0998383aa150fa50a2ffbf49ef37786e37eca"
+source = "git+https://github.com/aiongg/craft-libs?rev=808ec65380fb8fe0d71816aeadb9b2c5dc0142a4#808ec65380fb8fe0d71816aeadb9b2c5dc0142a4"
 
 [[package]]
 name = "craft-eps"
@@ -1771,7 +1771,7 @@ dependencies = [
 [[package]]
 name = "craft-zip"
 version = "0.1.0"
-source = "git+https://github.com/storytold/craft-libs?rev=8abfa9d0f14bafbfc3449fc552ffeafb9ca6a551#8abfa9d0f14bafbfc3449fc552ffeafb9ca6a551"
+source = "git+https://github.com/aiongg/craft-libs?rev=808ec65380fb8fe0d71816aeadb9b2c5dc0142a4#808ec65380fb8fe0d71816aeadb9b2c5dc0142a4"
 dependencies = [
  "flate2",
  "thiserror 2.0.21",
@@ -1781,7 +1781,7 @@ dependencies = [
 [[package]]
 name = "craft-zip"
 version = "0.1.0"
-source = "git+https://github.com/storytold/craft-libs?rev=93d0998383aa150fa50a2ffbf49ef37786e37eca#93d0998383aa150fa50a2ffbf49ef37786e37eca"
+source = "git+https://github.com/storytold/craft-libs?rev=8abfa9d0f14bafbfc3449fc552ffeafb9ca6a551#8abfa9d0f14bafbfc3449fc552ffeafb9ca6a551"
 dependencies = [
  "flate2",
  "thiserror 2.0.21",
@@ -2714,7 +2714,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
 dependencies = [
  "libc",
- "windows-sys 0.61.2",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
@@ -4526,7 +4526,7 @@ version = "0.50.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
 dependencies = [
- "windows-sys 0.61.2",
+ "windows-sys 0.59.0",
 ]
 
 [[package]]
@@ -4951,7 +4951,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7d8fae84b431384b68627d0f9b3b1245fcf9f46f6c0e3dc902e9dce64edd1967"
 dependencies = [
  "libc",
- "windows-sys 0.61.2",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
@@ -5513,7 +5513,7 @@ dependencies = [
  "once_cell",
  "socket2",
  "tracing",
- "windows-sys 0.61.2",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
@@ -5920,7 +5920,7 @@ dependencies = [
  "errno",
  "libc",
  "linux-raw-sys 0.12.1",
- "windows-sys 0.61.2",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
@@ -5979,7 +5979,7 @@ dependencies = [
  "security-framework",
  "security-framework-sys",
  "webpki-root-certs",
- "windows-sys 0.61.2",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
@@ -6831,10 +6831,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
 dependencies = [
  "fastrand",
- "getrandom 0.4.3",
+ "getrandom 0.3.4",
  "once_cell",
  "rustix 1.1.5",
- "windows-sys 0.61.2",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]
@@ -8073,7 +8073,7 @@ version = "0.1.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
 dependencies = [
- "windows-sys 0.61.2",
+ "windows-sys 0.52.0",
 ]
 
 [[package]]

Cargo.toml

@@ -13,7 +13,8 @@ repository = "https://github.com/storytold/designcraft"
 [workspace.dependencies]
 # Shared reader and writer; publish the immutable revision before this consumer.
 craft-pdf = { git = "https://github.com/storytold/craft-libs", rev = "40525913633942f84b06d4837a8d90534a55d63f", version = "=0.1.0", default-features = false }
-craft-designcraft-format = { git = "https://github.com/storytold/craft-libs", rev = "93d0998383aa150fa50a2ffbf49ef37786e37eca", version = "=0.1.0" }
+# TODO repin to storytold/craft-libs once #1264 merges
+craft-designcraft-format = { git = "https://github.com/aiongg/craft-libs", rev = "808ec65380fb8fe0d71816aeadb9b2c5dc0142a4", version = "=0.1.0" }
 craft-eps = { git = "https://github.com/storytold/craft-libs", rev = "ba24fe2435adb3c81ad6057faf36365fbba736cf", version = "=0.1.0" }
 craft-epub = { git = "https://github.com/storytold/craft-libs", rev = "ba24fe2435adb3c81ad6057faf36365fbba736cf", version = "=0.1.0" }
 craft-idml = { git = "https://github.com/storytold/craft-libs", rev = "1ad5f35bf", version = "=0.1.0" }

crates/format/src/lib.rs

@@ -6,7 +6,7 @@ use std::collections::HashMap;
 use std::hash::BuildHasher;
 use std::sync::Arc;
 
-use designcraft_doc::{AssetData, AssetId, Document};
+use designcraft_doc::{AssetData, AssetId, Document, Storage};
 #[cfg(test)]
 use serde_json::Value;
 #[cfg(test)]
@@ -26,31 +26,35 @@ pub fn save(doc: &Document) -> Result<Vec<u8>, FormatError> {
 
 /// [`save`], plus a PNG preview of the first page.
 pub fn save_with_preview(doc: &Document, preview: Option<&[u8]>) -> Result<Vec<u8>, FormatError> {
+    let mut out = std::io::Cursor::new(Vec::new());
+    write(&mut out, doc, preview)?;
+    Ok(out.into_inner())
+}
+
+/// [`save_with_preview`] into `out`, so a writer that stops on cancellation stops the write at
+/// its next chunk. Linked assets are stored as their record and proxy only.
+pub fn write<W: std::io::Write + std::io::Seek>(out: &mut W, doc: &Document, preview: Option<&[u8]>) -> Result<(), FormatError> {
     if let Some(id) = doc.non_finite_item() {
         return Err(FormatError::Unsaveable(format!("object {} has a position or size that is not a finite number", id.0)));
     }
     let mut compact = doc.clone();
     compact.compact_assets();
-    let owned_assets = compact
-        .assets
-        .values()
-        .map(|asset| {
-            let data = asset.data.load().map_err(|e| FormatError::Unsaveable(format!("image {}: {e}", asset.name)))?;
-            Ok((asset.id.0.to_string(), asset.mime.clone(), data))
-        })
-        .collect::<Result<Vec<_>, FormatError>>()?;
-    let assets = owned_assets.iter().map(|(id, mime, data)| craft_designcraft_format::Asset { id, mime, data }).collect::<Vec<_>>();
-    craft_designcraft_format::save(&compact, concat!("DesignCraft ", env!("CARGO_PKG_VERSION")), preview, &assets)
-}
-
-/// [`save_with_preview`] into `out`, in chunks, so a writer that stops on cancellation stops the
-/// write at its next chunk.
-pub fn write<W: std::io::Write>(out: &mut W, doc: &Document, preview: Option<&[u8]>) -> Result<(), FormatError> {
-    let bytes = save_with_preview(doc, preview)?;
-    for chunk in bytes.chunks(1 << 20) {
-        out.write_all(chunk).map_err(|e| FormatError::Io(e.to_string()))?;
+    let mut owned_assets = Vec::new();
+    let mut owned_proxies = Vec::new();
+    for asset in compact.assets.values() {
+        let id = asset.id.0.to_string();
+        if let Some(proxy) = &asset.proxy {
+            owned_proxies.push((id.clone(), proxy.mime.clone(), proxy.data.clone()));
+        }
+        if asset.storage == Storage::Linked {
+            continue;
+        }
+        let data = asset.data.load().map_err(|e| FormatError::Unsaveable(format!("image {}: {e}", asset.name)))?;
+        owned_assets.push((id, asset.mime.clone(), data));
     }
-    Ok(())
+    let assets = owned_assets.iter().map(|(id, mime, data)| craft_designcraft_format::Asset { id, mime, data }).collect::<Vec<_>>();
+    let proxies = owned_proxies.iter().map(|(id, mime, data)| craft_designcraft_format::Asset { id, mime, data }).collect::<Vec<_>>();
+    craft_designcraft_format::write_to(out, &compact, concat!("DesignCraft ", env!("CARGO_PKG_VERSION")), preview, &assets, &proxies)
 }
 
 /// Required native format version for a document. Kept for app-side compatibility; inference is
@@ -85,6 +89,19 @@ pub fn load(bytes: &[u8]) -> Result<Document, FormatError> {
         };
         Arc::make_mut(asset).data = data;
     }
+    for (raw_id, bytes) in loaded.proxies {
+        let Ok(id) = raw_id.parse::<u64>() else { continue };
+        let Some(asset) = doc.assets.get_mut(&AssetId(id)) else { continue };
+        if asset.proxy.is_some() {
+            Arc::make_mut(asset).proxy.iter_mut().for_each(|proxy| proxy.data = Arc::new(bytes.clone()));
+        }
+    }
+    for asset in doc.assets.values_mut() {
+        if asset.storage == Storage::Linked {
+            // T3/T7: the loader (and the linked file's length) attach here.
+            Arc::make_mut(asset).data = AssetData::linked(0);
+        }
+    }
     doc.merge_duplicate_assets();
     doc.mark_differing_margins_local();
     doc.renumber_duplicate_items();

crates/format/src/tests.rs

@@ -887,3 +887,51 @@ fn duplicate_ids_at_the_top_of_the_range_do_not_overflow() {
     assert_eq!(all_item_ids(&d), vec![u64::MAX, u64::MAX]);
     assert!(d.check().is_err());
 }
+
+fn zip_version(bytes: &[u8]) -> u32 {
+    let mut z = zip::ZipArchive::new(Cursor::new(bytes)).unwrap();
+    let mut s = String::new();
+    z.by_name("meta.json").map(|mut f| f.read_to_string(&mut s)).ok();
+    serde_json::from_str::<Value>(&s).ok().and_then(|v| v.get("version").and_then(Value::as_u64)).unwrap_or(0) as u32
+}
+
+#[test]
+fn linked_asset_roundtrips_as_record_and_proxy_without_original() {
+    let mut d = Document::new(&NewDocument::default());
+    let aid = AssetId(d.alloc());
+    let asset = Asset {
+        id: aid,
+        name: "big.tif".into(),
+        mime: "image/tiff".into(),
+        link: Some("/pics/big.tif".into()),
+        storage: Storage::Linked,
+        proxy: Some(designcraft_doc::Proxy { mime: "image/png".into(), pixels: (2, 2), data: Arc::new(vec![9, 8, 7]) }),
+        data: AssetData::linked(1 << 30),
+        pixels: Some((4000, 3000)),
+        ..Default::default()
+    };
+    d.assets.insert(aid, Arc::new(asset));
+    place(&mut d, aid, 0.0);
+    let bytes = save(&d).unwrap();
+    assert_eq!(asset_entries(&bytes), 0, "no original is stored");
+    assert_eq!(required_version(&d), 5);
+    let back = load(&bytes).unwrap();
+    let a = &back.assets[&aid];
+    assert_eq!(a.storage, Storage::Linked);
+    assert_eq!(a.link.as_deref(), Some("/pics/big.tif"));
+    assert!(a.data.is_linked());
+    let proxy = a.proxy.as_ref().unwrap();
+    assert_eq!((proxy.mime.as_str(), proxy.pixels, proxy.data.as_slice()), ("image/png", (2, 2), &[9u8, 8, 7][..]));
+}
+
+#[test]
+fn embedded_documents_keep_their_older_version_and_version_5_loads() {
+    let mut d = Document::new(&NewDocument::default());
+    assert_eq!(zip_version(&save(&d).unwrap()), 2);
+    let aid = png_asset(&mut d, &[1, 2, 3]);
+    place(&mut d, aid, 0.0);
+    assert_eq!(zip_version(&save(&d).unwrap()), 2);
+    assert_eq!(VERSION, 5);
+    let loaded = load(&save(&d).unwrap()).unwrap();
+    assert_eq!(loaded.assets.len(), d.assets.len());
+}