From b1b4602a71d19e09ad33f7752a10d6598cf39b2b Mon Sep 17 00:00:00 2001 From: aiongg <67543387+aiongg@users.noreply.github.com> Date: Sat, 14 Nov 2026 02:12:23 -0500 Subject: [PATCH] designcraft-format: linked assets, proxy previews, streaming write (format 5) (#1264) * designcraft-format v5: linked assets, proxy previews, streaming write_to * Skip bytes of linked assets, latch writer errors, charge failed proxy reads --- crates/designcraft-format/src/lib.rs | 57 +++++++- crates/designcraft-format/src/tests.rs | 103 +++++++++++++- crates/document-bundle/src/lib.rs | 186 ++++++++++++++++++++++--- 3 files changed, 313 insertions(+), 33 deletions(-) diff --git a/crates/designcraft-format/src/lib.rs b/crates/designcraft-format/src/lib.rs index 6256eab1c..eab431eab 100644 --- a/crates/designcraft-format/src/lib.rs +++ b/crates/designcraft-format/src/lib.rs @@ -6,7 +6,7 @@ #![forbid(unsafe_code)] use std::collections::{HashMap, HashSet}; -use std::io::{Read, Seek}; +use std::io::{Cursor, Read, Seek, Write}; pub use craft_document_bundle::Asset; use craft_document_bundle::Spec; @@ -14,9 +14,10 @@ use serde::{Serialize, de::DeserializeOwned}; use serde_json::Value; pub const MIME: &str = "application/vnd.designcraft+zip"; -pub const VERSION: u32 = 4; +pub const VERSION: u32 = 5; pub const EXTENSION: &str = "designcraft"; pub const PREVIEW: &str = "preview.png"; +pub const PROXIES_DIRECTORY: &str = "proxies"; pub const MAX_PREVIEW_BYTES: u64 = 4 << 20; const BUNDLE: Spec<'static> = Spec { @@ -26,6 +27,7 @@ const BUNDLE: Spec<'static> = Spec { document_entry: "document.json", preview_entry: PREVIEW, assets_directory: "assets", + proxies_directory: PROXIES_DIRECTORY, format_name: "designcraft", legacy_assets_field: Some("assetData"), }; @@ -57,23 +59,53 @@ impl From for Error { pub struct Loaded { pub document: T, pub assets: Vec<(String, Vec)>, + /// Proxy previews (JPEG or PNG bytes) of linked assets, keyed by asset id. + pub proxies: Vec<(String, Vec)>, } /// The reader capability required by `document`, inferred from its neutral JSON schema and -/// embedded assets. Traversal beyond 64 levels conservatively requires version 4. +/// embedded assets. Traversal beyond 64 levels conservatively requires version 4. A linked +/// asset (`/assets//storage == "linked"`) requires version 5. pub fn required_version(document: &Value, assets: &[Asset<'_>]) -> u32 { + if has_linked_asset(document) { + return 5; + } if has_version_four_feature(document) { return 4; } if assets.iter().any(|asset| craft_eps::is_eps(asset.data)) { 3 } else { 2 } } -/// Serialize a document and its external assets as a `.designcraft` ZIP bundle. -pub fn save(document: &T, generator: &str, preview: Option<&[u8]>, assets: &[Asset<'_>]) -> Result, Error> { +/// Serialize a document, its embedded assets and the proxy previews of its linked assets as a +/// `.designcraft` ZIP bundle. Linked assets have a record in the document and no `assets` entry. +pub fn save( + document: &T, + generator: &str, + preview: Option<&[u8]>, + assets: &[Asset<'_>], + proxies: &[Asset<'_>], +) -> Result, Error> { + let mut out = Cursor::new(Vec::new()); + write_to(&mut out, document, generator, preview, assets, proxies)?; + Ok(out.into_inner()) +} + +/// Like [`save`], streaming the ZIP into `out`. A writer that returns an error (to cancel a +/// long save, say) stops the write and the error is returned as [`Error::Io`]. +pub fn write_to( + out: W, + document: &T, + generator: &str, + preview: Option<&[u8]>, + assets: &[Asset<'_>], + proxies: &[Asset<'_>], +) -> Result<(), Error> { let value = serde_json::to_value(document).map_err(|error| Error::Io(error.to_string()))?; + let embedded: Vec> = assets.iter().filter(|asset| !is_linked(&value, asset.id)).copied().collect(); + let assets = embedded.as_slice(); let version = required_version(&value, assets); let json = serde_json::to_vec_pretty(&value).map_err(|error| Error::Io(error.to_string()))?; - craft_document_bundle::write(&json, version, generator, preview, assets, BUNDLE).map_err(Into::into) + craft_document_bundle::write_to(out, &json, version, generator, preview, assets, proxies, BUNDLE).map_err(Into::into) } /// Read a `.designcraft` ZIP or legacy single-file JSON, applying all versioned schema migrations @@ -82,7 +114,7 @@ pub fn load(bytes: &[u8]) -> Result, Error> { let mut package = craft_document_bundle::read(bytes, VERSION, BUNDLE)?; upgrade_legacy_json(&mut package.document, package.version); let document = serde_json::from_value(package.document).map_err(|error| Error::NotOurs(error.to_string()))?; - Ok(Loaded { document, assets: package.assets }) + Ok(Loaded { document, assets: package.assets, proxies: package.proxies }) } /// Whether `bytes` are a native DesignCraft ZIP package. @@ -100,6 +132,17 @@ pub fn read_preview(reader: R) -> Option> { craft_document_bundle::read_preview_from(reader, PREVIEW) } +fn is_linked(document: &Value, id: &str) -> bool { + document.get("assets").and_then(|assets| assets.get(id)).and_then(|asset| asset.get("storage")).and_then(Value::as_str) == Some("linked") +} + +fn has_linked_asset(document: &Value) -> bool { + document + .get("assets") + .and_then(Value::as_object) + .is_some_and(|assets| assets.values().any(|asset| asset.get("storage").and_then(Value::as_str) == Some("linked"))) +} + fn has_version_four_feature(document: &Value) -> bool { let mut stack = vec![(document, 0usize)]; let mut visited = 0usize; diff --git a/crates/designcraft-format/src/tests.rs b/crates/designcraft-format/src/tests.rs index 0ef2ca191..38cf1c34f 100644 --- a/crates/designcraft-format/src/tests.rs +++ b/crates/designcraft-format/src/tests.rs @@ -7,7 +7,7 @@ fn native_bundle_roundtrips_document_assets_and_preview() -> Result<(), Box Result<(), Box Result<(), Box> { let oversized = vec![0u8; usize::try_from(MAX_PREVIEW_BYTES + 1)?]; - let bytes = save(&json!({}), "DesignCraft test", Some(&oversized), &[])?; + let bytes = save(&json!({}), "DesignCraft test", Some(&oversized), &[], &[])?; assert_eq!(read_preview(std::io::Cursor::new(bytes)), None); Ok(()) } @@ -65,8 +65,105 @@ fn legacy_json_extracts_assets_and_applies_v1_migrations() -> Result<(), Box Result<(), Box> { - let bytes = craft_document_bundle::write(b"{", VERSION + 1, "future", None, &[], BUNDLE)?; + let bytes = craft_document_bundle::write(b"{", VERSION + 1, "future", None, &[], &[], BUNDLE)?; assert!(matches!(load::(&bytes), Err(Error::TooNew(version)) if version == u64::from(VERSION + 1))); assert!(matches!(load::(b"not a package"), Err(Error::NotOurs(_)))); Ok(()) } + +fn linked_doc() -> Value { + json!({"assets":{"7":{"storage":"linked","mime":"image/png"},"8":{"mime":"image/png"}}}) +} + +#[test] +fn embedded_documents_keep_their_version() { + assert_eq!(required_version(&json!({"assets":{"7":{"storage":"embedded"},"8":{}}}), &[]), 2); +} + +#[test] +fn a_linked_asset_requires_version_five_and_old_readers_refuse_it() -> Result<(), Box> { + assert_eq!(required_version(&linked_doc(), &[]), 5); + let bytes = save(&linked_doc(), "t", None, &[], &[])?; + assert_eq!(craft_document_bundle::read(&bytes, 4, BUNDLE), Err(craft_document_bundle::Error::TooNew(5))); + let loaded = load::(&bytes)?; + assert_eq!(loaded.document, linked_doc()); + assert!(loaded.assets.is_empty()); + Ok(()) +} + +#[test] +fn proxies_round_trip_and_unknown_or_oversized_ones_are_skipped() -> Result<(), Box> { + let big = vec![0u8; usize::try_from(craft_document_bundle::PROXY_LIMIT)? + 1]; + let proxies = [ + Asset { id: "7", mime: "image/jpeg", data: b"jpeg" }, + Asset { id: "8", mime: "image/png", data: &big }, + Asset { id: "99", mime: "image/png", data: b"unknown" }, + ]; + let bytes = save(&linked_doc(), "t", None, &[], &proxies)?; + assert_eq!(load::(&bytes)?.proxies, [("7".to_owned(), b"jpeg".to_vec())]); + Ok(()) +} + +#[test] +fn proxy_paths_must_be_exactly_proxies_id_ext() -> Result<(), Box> { + let proxies = [ + Asset { id: "../x", mime: "image/png", data: b"a" }, + Asset { id: "a/b", mime: "image/png", data: b"b" }, + Asset { id: "7/8", mime: "image/png", data: b"c" }, + ]; + let bytes = save(&linked_doc(), "t", None, &[], &proxies)?; + assert!(load::(&bytes)?.proxies.is_empty()); + Ok(()) +} + +#[test] +fn write_to_stops_when_the_writer_fails() { + struct Failing { + inner: Cursor>, + budget: usize, + failed: bool, + calls_after_failure: std::rc::Rc>, + } + impl Write for Failing { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + if self.failed { + self.calls_after_failure.set(self.calls_after_failure.get() + 1); + } + if self.budget < buf.len() { + self.failed = true; + return Err(std::io::Error::other("cancelled")); + } + self.budget -= buf.len(); + self.inner.write(buf) + } + fn flush(&mut self) -> std::io::Result<()> { + if self.failed { + self.calls_after_failure.set(self.calls_after_failure.get() + 1); + } + self.inner.flush() + } + } + impl Seek for Failing { + fn seek(&mut self, pos: std::io::SeekFrom) -> std::io::Result { + if self.failed { + self.calls_after_failure.set(self.calls_after_failure.get() + 1); + } + self.inner.seek(pos) + } + } + let data = vec![1u8; 4096]; + let assets = [Asset { id: "8", mime: "image/png", data: &data }]; + let calls = std::rc::Rc::new(std::cell::Cell::new(0)); + let out = Failing { inner: Cursor::new(Vec::new()), budget: 1024, failed: false, calls_after_failure: calls.clone() }; + let result = write_to(out, &linked_doc(), "t", None, &assets, &[]); + assert_eq!(calls.get(), 0, "no write, flush or seek reaches the writer after it failed"); + assert!(matches!(result, Err(Error::Io(message)) if message.contains("cancelled"))); +} + +#[test] +fn bytes_supplied_for_a_linked_asset_are_not_written() -> Result<(), Box> { + let assets = [Asset { id: "7", mime: "image/png", data: b"linked bytes" }, Asset { id: "8", mime: "image/png", data: b"kept" }]; + let bytes = save(&linked_doc(), "t", None, &assets, &[])?; + assert_eq!(load::(&bytes)?.assets, [("8".to_owned(), b"kept".to_vec())]); + Ok(()) +} diff --git a/crates/document-bundle/src/lib.rs b/crates/document-bundle/src/lib.rs index 18e40a3ff..70a960a1f 100644 --- a/crates/document-bundle/src/lib.rs +++ b/crates/document-bundle/src/lib.rs @@ -6,15 +6,17 @@ //! bounded reads and extraction of legacy base64 asset maps. Entry names and MIME type are //! supplied by the format using the bundle. -use std::io::{Read, Seek}; +use std::io::{Cursor, Read, Seek, Write}; -use craft_zip::{Archive, Compression, Limits, Writer}; +use craft_zip::{Archive, Limits}; use serde_json::Value; const MAX_ENTRIES: u64 = 100_000; const JSON_LIMIT: u64 = 256 << 20; const ASSET_LIMIT: u64 = 1 << 30; const EXPANSION_LIMIT: u64 = 4 << 30; +/// Largest proxy preview read from a bundle; larger entries are skipped. +pub const PROXY_LIMIT: u64 = 1 << 20; /// Per-entry and total-expansion limits for an untrusted document bundle. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -46,6 +48,8 @@ pub struct Spec<'a> { pub preview_entry: &'a str, /// Directory prefix for embedded assets, without a trailing slash. pub assets_directory: &'a str, + /// Directory prefix for proxy previews of linked assets, without a trailing slash. + pub proxies_directory: &'a str, /// Metadata value written in the `format` field. pub format_name: &'a str, /// Old single-file JSON field containing base64 assets keyed by decimal id. @@ -72,6 +76,8 @@ pub struct Package { pub version: u64, /// Embedded asset bytes keyed by decimal asset id. pub assets: Vec<(String, Vec)>, + /// Proxy preview bytes (JPEG or PNG) keyed by decimal asset id, for ids the document lists. + pub proxies: Vec<(String, Vec)>, } /// Why a document bundle could not be read or written. @@ -116,24 +122,104 @@ pub fn starts_with_mimetype(prefix: &[u8], spec: Spec<'_>) -> bool { craft_zip::stored_mimetype(prefix).is_some_and(|mime| mime == spec.mimetype.as_bytes()) } -/// Write a ZIP document bundle containing the serialized document, metadata, preview and assets. -pub fn write(document: &[u8], version: u32, generator: &str, preview: Option<&[u8]>, assets: &[Asset<'_>], spec: Spec<'_>) -> Result, Error> { - let mut writer = Writer::new(); - writer.add(spec.mimetype_entry, spec.mimetype.as_bytes(), Compression::Stored).map_err(|e| Error::Write(e.to_string()))?; - let metadata = serde_json::json!({"format":spec.format_name, "version":version, "generator":generator}); - let metadata = serde_json::to_vec(&metadata).map_err(|e| Error::Write(e.to_string()))?; - writer.add(spec.metadata_entry, &metadata, Compression::Deflated).map_err(|e| Error::Write(e.to_string()))?; - if let Some(preview) = preview { - writer.add(spec.preview_entry, preview, Compression::Stored).map_err(|e| Error::Write(e.to_string()))?; +/// Refuses every write and seek after the first failure of the wrapped writer. Dropping a +/// `ZipWriter` after an error finalizes the archive, which must not touch a cancelled target. +struct Latch { + inner: W, + failed: bool, +} + +impl Latch { + fn latched() -> std::io::Error { + std::io::Error::other("writer already failed") } - writer.add(spec.document_entry, document, Compression::Deflated).map_err(|e| Error::Write(e.to_string()))?; +} + +impl Write for Latch { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + if self.failed { + return Err(Self::latched()); + } + self.inner.write(buf).inspect_err(|_| self.failed = true) + } + fn flush(&mut self) -> std::io::Result<()> { + if self.failed { + return Err(Self::latched()); + } + self.inner.flush().inspect_err(|_| self.failed = true) + } +} + +impl Seek for Latch { + fn seek(&mut self, pos: std::io::SeekFrom) -> std::io::Result { + if self.failed { + return Err(Self::latched()); + } + self.inner.seek(pos).inspect_err(|_| self.failed = true) + } +} + +fn write_err(e: impl std::fmt::Display) -> Error { + Error::Write(e.to_string()) +} + +/// Write a ZIP document bundle containing the serialized document, metadata, preview, assets and +/// proxy previews. +pub fn write( + document: &[u8], + version: u32, + generator: &str, + preview: Option<&[u8]>, + assets: &[Asset<'_>], + proxies: &[Asset<'_>], + spec: Spec<'_>, +) -> Result, Error> { + let mut out = Cursor::new(Vec::new()); + write_to(&mut out, document, version, generator, preview, assets, proxies, spec)?; + Ok(out.into_inner()) +} + +/// Like [`write`], streaming the archive into `out`. A writer that fails (for example to cancel +/// a long save) stops the write at the next member or buffer and the error is returned. +#[allow(clippy::too_many_arguments)] +pub fn write_to( + out: W, + document: &[u8], + version: u32, + generator: &str, + preview: Option<&[u8]>, + assets: &[Asset<'_>], + proxies: &[Asset<'_>], + spec: Spec<'_>, +) -> Result<(), Error> { + use zip::CompressionMethod::{Deflated, Stored}; + let mut zip = zip::ZipWriter::new(Latch { inner: out, failed: false }); + let mut add = |name: &str, data: &[u8], method: zip::CompressionMethod| -> Result<(), Error> { + let options = zip::write::SimpleFileOptions::default() + .compression_method(method) + .compression_level(if method == Deflated { Some(6) } else { None }) + .last_modified_time(zip::DateTime::default()) + .large_file(u64::try_from(data.len()).unwrap_or(u64::MAX) >= u64::from(u32::MAX) / 2); + zip.start_file(name, options).map_err(|e| Error::Write(format!("{name}: {e}")))?; + zip.write_all(data).map_err(|e| Error::Write(format!("{name}: {e}"))) + }; + add(spec.mimetype_entry, spec.mimetype.as_bytes(), Stored)?; + let metadata = serde_json::json!({"format":spec.format_name, "version":version, "generator":generator}); + add(spec.metadata_entry, &serde_json::to_vec(&metadata).map_err(write_err)?, Deflated)?; + if let Some(preview) = preview { + add(spec.preview_entry, preview, Stored)?; + } + add(spec.document_entry, document, Deflated)?; for asset in assets.iter().filter(|asset| !asset.data.is_empty()) { let path = format!("{}/{}.{}", spec.assets_directory, asset.id, extension(asset.mime)); - let compression = - if matches!(asset.mime, "image/png" | "image/jpeg" | "image/gif" | "image/webp") { Compression::Stored } else { Compression::Deflated }; - writer.add(&path, asset.data, compression).map_err(|e| Error::Write(e.to_string()))?; + let method = if matches!(asset.mime, "image/png" | "image/jpeg" | "image/gif" | "image/webp") { Stored } else { Deflated }; + add(&path, asset.data, method)?; } - writer.finish().map_err(|e| Error::Write(e.to_string())) + for proxy in proxies.iter().filter(|proxy| !proxy.data.is_empty()) { + let ext = if proxy.mime == "image/jpeg" { "jpg" } else { "png" }; + add(&format!("{}/{}.{ext}", spec.proxies_directory, proxy.id), proxy.data, Stored)?; + } + zip.finish().map(drop).map_err(write_err) } /// Read a ZIP bundle or legacy JSON document within bounded member sizes. @@ -186,7 +272,40 @@ pub fn read_with_limits(bytes: &[u8], supported_version: u32, spec: Spec<'_>, li embedded.push((id.to_owned(), data)); } } - Ok(Package { document, version, assets: embedded }) + let proxies = read_proxies(&mut archive, &wanted, spec); + Ok(Package { document, version, assets: embedded, proxies }) +} + +/// Proxy previews for known asset ids. Only `/.jpg|png` is accepted; entries over +/// [`PROXY_LIMIT`] or that fail to read are skipped. +fn read_proxies(archive: &mut Archive<'_>, wanted: &std::collections::HashSet, spec: Spec<'_>) -> Vec<(String, Vec)> { + let prefix = format!("{}/", spec.proxies_directory); + let mut proxies = Vec::new(); + for path in archive.names().map(|name| name.into_owned()).collect::>() { + let Some(member) = path.strip_prefix(&prefix) else { continue }; + let Some((id, ext)) = member.split_once('.') else { continue }; + if !matches!(ext, "jpg" | "png") || !wanted.contains(id) || id.parse::().is_err() { + continue; + } + if archive.declared_size(&path).is_none_or(|size| size > PROXY_LIMIT) { + continue; + } + let Some(size) = archive.declared_size(&path) else { continue }; + if archive.budget().remaining() < size { + break; + } + match archive.read(&path, PROXY_LIMIT) { + Ok(Some(data)) => proxies.push((id.to_owned(), data)), + Ok(None) => {} + // A failed read may have decompressed up to `size` bytes without being charged. + Err(_) => { + if archive.budget_mut().charge(size).is_err() { + break; + } + } + } + } + proxies } /// Read a preview with a fixed maximum size without decompressing the full bundle. @@ -235,7 +354,7 @@ fn read_legacy(bytes: &[u8], spec: Spec<'_>) -> Result { object.remove(field); } } - Ok(Package { document, version: 1, assets }) + Ok(Package { document, version: 1, assets, proxies: Vec::new() }) } fn decode_base64(input: &str) -> Vec { @@ -271,6 +390,7 @@ mod tests { document_entry: "document.json", preview_entry: "preview.png", assets_directory: "assets", + proxies_directory: "proxies", format_name: "test", legacy_assets_field: Some("assetData"), }; @@ -278,8 +398,8 @@ mod tests { #[test] fn writes_and_reads_bundle_members() { let payload = [1u8, 2, 3]; - let bytes = - write(br#"{"assets":{"7":{}}}"#, 4, "Test 0.1", Some(b"png"), &[Asset { id: "7", mime: "image/png", data: &payload }], SPEC).unwrap(); + let bytes = write(br#"{"assets":{"7":{}}}"#, 4, "Test 0.1", Some(b"png"), &[Asset { id: "7", mime: "image/png", data: &payload }], &[], SPEC) + .unwrap(); assert!(has_document(&bytes, SPEC)); assert!(starts_with_mimetype(&bytes, SPEC)); let loaded = read(&bytes, 4, SPEC).unwrap(); @@ -298,7 +418,7 @@ mod tests { #[test] fn refuses_newer_versions() { - let bytes = write(b"{}", 9, "future", None, &[], SPEC).unwrap(); + let bytes = write(b"{}", 9, "future", None, &[], &[], SPEC).unwrap(); assert_eq!(read(&bytes, 4, SPEC), Err(Error::TooNew(9))); } @@ -307,11 +427,31 @@ mod tests { let bytes = vec![7u8; 128]; let document = br#"{"assets":{"7":{}}}"#; let limits = ReadLimits { json_bytes: 1 << 20, compressed_asset_bytes: 8, expansion_bytes: 1 << 20 }; - let stored = write(document, 4, "test", None, &[Asset { id: "7", mime: "image/png", data: &bytes }], SPEC).unwrap(); + let stored = write(document, 4, "test", None, &[Asset { id: "7", mime: "image/png", data: &bytes }], &[], SPEC).unwrap(); let package = read_with_limits(&stored, 4, SPEC, limits).unwrap(); assert_eq!(package.assets, [("7".to_owned(), bytes.clone())]); - let compressed = write(document, 4, "test", None, &[Asset { id: "7", mime: "application/octet-stream", data: &bytes }], SPEC).unwrap(); + let compressed = write(document, 4, "test", None, &[Asset { id: "7", mime: "application/octet-stream", data: &bytes }], &[], SPEC).unwrap(); assert!(read_with_limits(&compressed, 4, SPEC, limits).is_err()); } + + #[test] + fn failed_proxy_reads_are_charged_and_stop_reading_when_the_budget_is_spent() { + let pad = " ".repeat(40_000); + let document = format!(r#"{{"assets":{{"1":{{}},"2":{{}},"3":{{}},"4":{{}}}},"pad":"{pad}"}}"#); + let blobs: Vec> = (1u8..=4).map(|n| vec![n.wrapping_mul(50); 1000]).collect(); + let ids = ["1", "2", "3", "4"]; + let proxies: Vec> = ids.iter().zip(&blobs).map(|(id, data)| Asset { id, mime: "image/png", data }).collect(); + let mut bytes = write(document.as_bytes(), 4, "t", None, &[], &proxies, SPEC).unwrap(); + // Corrupt proxies 1 to 3 so their reads fail their checksum. + for blob in &blobs[..3] { + let at = bytes.windows(blob.len()).position(|w| w == blob.as_slice()).unwrap(); + bytes[at + 500] ^= 0xFF; + } + let source = bytes.len() as u64; + let all = read(&bytes, 4, SPEC).unwrap(); + assert_eq!(all.proxies.len(), 1); + let tight = ReadLimits { json_bytes: 1 << 20, compressed_asset_bytes: 1 << 20, expansion_bytes: 40_100 + 2_500 - source }; + assert!(read_with_limits(&bytes, 4, SPEC, tight).unwrap().proxies.is_empty()); + } }