From fc85dc9853d12a65c054cce37b080b0dafffe80e Mon Sep 17 00:00:00 2001 From: aiongg Date: Sat, 14 Nov 2026 05:15:03 +0000 Subject: [PATCH] DES-10 T2: AssetData and SourceKey hold asset bytes in the document model Asset.data becomes AssetData (embedded bytes or a linked length, each under a process-unique SourceKey), with the link_relative, link_hash, storage and proxy fields. AlphaOutline names its source by SourceKey. Dedup in assets.rs compares embedded bytes as before. --- crates/doc/src/anchored.rs | 2 +- crates/doc/src/asset_data.rs | 206 +++++++++++++++++++++++++++++++++++ crates/doc/src/assets.rs | 45 ++++++-- crates/doc/src/lib.rs | 60 ++++++++-- 4 files changed, 296 insertions(+), 17 deletions(-) create mode 100644 crates/doc/src/asset_data.rs diff --git a/crates/doc/src/anchored.rs b/crates/doc/src/anchored.rs index 524e2bfc5..f5c4b856c 100644 --- a/crates/doc/src/anchored.rs +++ b/crates/doc/src/anchored.rs @@ -749,7 +749,7 @@ mod tests { id, name: name.into(), mime: "application/pdf".into(), - data: Arc::new(format!("%PDF-1.7 {name}").into_bytes()), + data: crate::AssetData::embedded(format!("%PDF-1.7 {name}").into_bytes()), ..Default::default() }; let (mut a, host) = doc(); diff --git a/crates/doc/src/asset_data.rs b/crates/doc/src/asset_data.rs new file mode 100644 index 000000000..670625203 --- /dev/null +++ b/crates/doc/src/asset_data.rs @@ -0,0 +1,206 @@ +//! The bytes of a placed file ([`crate::Asset::data`]): held in memory (embedded) or read from +//! the linked file when needed. Caches of decoded images key by the value's [`SourceKey`], +//! which outlives the bytes: a decoded image stays usable after the encoded bytes are dropped. + +use std::fmt; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Weak}; + +/// Names one version of an asset's bytes. Every [`AssetData`] made from new bytes gets a key no +/// other value in this process has had; clones share it. Keys are never reused, so a cache +/// entry under a key can't be mistaken for different bytes allocated later. Counting starts at +/// 1: `SourceKey(0)` names no bytes. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct SourceKey(pub u64); + +impl SourceKey { + fn next() -> Self { + static NEXT: AtomicU64 = AtomicU64::new(1); + SourceKey(NEXT.fetch_add(1, Ordering::Relaxed)) + } +} + +/// Why an asset's bytes can't be had. +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +pub enum Unavailable { + #[error("the linked file is missing")] + Missing, + #[error("the linked file has changed")] + Modified, + #[error("the linked file is too large to load")] + TooLarge, + #[error("no loader for linked files")] + NoLoader, +} + +#[derive(Clone)] +enum Repr { + Embedded(Arc>), + Linked { len: u64 }, +} + +/// An asset's bytes: embedded in the document, or linked (read from the file on demand). +/// Cloning is cheap and keeps the [`SourceKey`]. +#[derive(Clone)] +pub struct AssetData { + key: SourceKey, + repr: Repr, +} + +impl AssetData { + /// Embedded bytes, under a new key. + pub fn embedded(bytes: impl Into>>) -> Self { + AssetData { key: SourceKey::next(), repr: Repr::Embedded(bytes.into()) } + } + + /// The bytes of a linked file of `len` bytes, under a new key. + pub fn linked(len: u64) -> Self { + AssetData { key: SourceKey::next(), repr: Repr::Linked { len } } + } + + /// The bytes, when they are embedded. + pub fn embedded_bytes(&self) -> Option<&Arc>> { + match &self.repr { + Repr::Embedded(b) => Some(b), + Repr::Linked { .. } => None, + } + } + + /// The bytes: embedded ones, or the linked file's. + pub fn load(&self) -> Result>, Unavailable> { + match &self.repr { + Repr::Embedded(b) => Ok(Arc::clone(b)), + Repr::Linked { .. } => Err(Unavailable::NoLoader), + } + } + + pub fn key(&self) -> SourceKey { + self.key + } + + /// The size of the bytes. + pub fn len(&self) -> u64 { + match &self.repr { + Repr::Embedded(b) => b.len() as u64, + Repr::Linked { len } => *len, + } + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + pub fn is_linked(&self) -> bool { + matches!(self.repr, Repr::Linked { .. }) + } + + /// A handle that doesn't keep embedded bytes alive, and upgrades to this value (same key) + /// while anything else does. + pub fn downgrade(&self) -> WeakAssetData { + let repr = match &self.repr { + Repr::Embedded(b) => WeakRepr::Embedded(Arc::downgrade(b)), + Repr::Linked { len } => WeakRepr::Linked { len: *len }, + }; + WeakAssetData { key: self.key, repr } + } +} + +#[derive(Clone)] +enum WeakRepr { + Embedded(Weak>), + Linked { len: u64 }, +} + +/// [`AssetData::downgrade`]. +#[derive(Clone)] +pub struct WeakAssetData { + key: SourceKey, + repr: WeakRepr, +} + +impl WeakAssetData { + /// The value, while its embedded bytes are held elsewhere. + pub fn upgrade(&self) -> Option { + let repr = match &self.repr { + WeakRepr::Embedded(w) => Repr::Embedded(w.upgrade()?), + WeakRepr::Linked { len } => Repr::Linked { len: *len }, + }; + Some(AssetData { key: self.key, repr }) + } + + pub fn key(&self) -> SourceKey { + self.key + } +} + +impl fmt::Debug for WeakAssetData { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("WeakAssetData").field("key", &self.key.0).finish() + } +} + +/// New embedded bytes ([`AssetData::embedded`]). +impl From> for AssetData { + fn from(bytes: Vec) -> Self { + AssetData::embedded(bytes) + } +} + +impl Default for AssetData { + fn default() -> Self { + AssetData::embedded(Vec::new()) + } +} + +/// Embedded bytes compare by content; linked ones are equal only as the same version. +impl PartialEq for AssetData { + fn eq(&self, other: &Self) -> bool { + match (&self.repr, &other.repr) { + (Repr::Embedded(a), Repr::Embedded(b)) => a == b, + (Repr::Linked { len: a }, Repr::Linked { len: b }) => self.key == other.key && a == b, + _ => false, + } + } +} + +impl fmt::Debug for AssetData { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let kind = if self.is_linked() { "Linked" } else { "Embedded" }; + f.debug_struct(kind).field("key", &self.key.0).field("len", &self.len()).finish() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn clones_keep_the_key_and_new_bytes_get_a_new_one() { + let a = AssetData::embedded(vec![1, 2, 3]); + let b = a.clone(); + assert_eq!(a.key(), b.key()); + let c = AssetData::embedded(vec![1, 2, 3]); + assert_ne!(a.key(), c.key()); + assert_eq!(a, c, "embedded bytes compare by content"); + let shared = AssetData::embedded(Arc::clone(a.embedded_bytes().unwrap())); + assert_ne!(a.key(), shared.key(), "wrapping bytes again names a new version"); + assert_eq!(a.load().unwrap().as_slice(), [1, 2, 3]); + assert_eq!(a.len(), 3); + let weak = a.downgrade(); + assert_eq!(weak.upgrade().map(|d| d.key()), Some(a.key())); + drop((a, b)); + assert!(weak.upgrade().is_some(), "`shared` still holds the bytes"); + drop(shared); + assert!(weak.upgrade().is_none()); + } + + #[test] + fn linked_bytes_need_a_loader() { + let l = AssetData::linked(10); + assert!(l.is_linked()); + assert_eq!(l.len(), 10); + assert!(l.embedded_bytes().is_none()); + assert_eq!(l.load(), Err(Unavailable::NoLoader)); + assert!(AssetData::default().is_empty()); + } +} diff --git a/crates/doc/src/assets.rs b/crates/doc/src/assets.rs index 0d100cee7..18f07326f 100644 --- a/crates/doc/src/assets.rs +++ b/crates/doc/src/assets.rs @@ -108,8 +108,23 @@ impl Asset { } } + /// The same bytes: the same version, or equal embedded bytes. Linked bytes aren't read here. fn same_bytes(&self, other: &Asset) -> bool { - Arc::ptr_eq(&self.data, &other.data) || (self.data.len() == other.data.len() && *self.data == *other.data) + if self.data.key() == other.data.key() { + return true; + } + match (self.data.embedded_bytes(), other.data.embedded_bytes()) { + (Some(a), Some(b)) => Arc::ptr_eq(a, b) || (a.len() == b.len() && **a == **b), + _ => false, + } + } + + /// A hash that is equal for assets with [`Asset::same_bytes`]. + fn bytes_hash(&self, state: &std::collections::hash_map::RandomState) -> u64 { + match self.data.embedded_bytes() { + Some(b) => state.hash_one(b.as_slice()), + None => state.hash_one(self.data.key()), + } } } @@ -132,7 +147,7 @@ pub fn insert_asset(assets: &mut BTreeMap>, mut asset: Asset if !asset.data.is_empty() && let Some(a) = assets.values().find(|a| a.same_bytes(&asset)) { - asset.data = Arc::clone(&a.data); + asset.data = a.data.clone(); } let id = asset.id; assets.insert(id, Arc::new(asset)); @@ -165,7 +180,7 @@ impl AssetIndex { assets.insert(id, Arc::new(asset)); return id; } - let hash = self.fingerprint.hash_one(asset.data.as_slice()); + let hash = asset.bytes_hash(&self.fingerprint); let key = (hash, asset.mime.clone(), asset.page, asset.link.clone()); if let Some(id) = self.files.get(&key).and_then(|ids| ids.iter().copied().find(|id| assets.get(id).is_some_and(|a| a.same_file(&asset)))) { return id; @@ -173,7 +188,7 @@ impl AssetIndex { if !asset.data.is_empty() { let bucket = self.bytes.entry(hash).or_default(); match bucket.iter().find_map(|id| assets.get(id).filter(|a| a.same_bytes(&asset))) { - Some(a) => asset.data = Arc::clone(&a.data), + Some(a) => asset.data = a.data.clone(), None => bucket.push(asset.id), } } @@ -381,7 +396,7 @@ impl Document { if a.data.is_empty() && a.link.is_none() { continue; } - let hash = *hashes.entry(Arc::as_ptr(&a.data)).or_insert_with(|| fingerprint.hash_one(a.data.as_slice())); + let hash = *hashes.entry(a.data.key()).or_insert_with(|| a.bytes_hash(&fingerprint)); let bucket = reps.entry((hash, a.mime.as_str(), a.page, a.link.as_deref())).or_default(); // A fingerprint only selects candidates; equality still checks the actual file. match bucket.iter().find(|r| r.same_file(a)) { @@ -411,7 +426,13 @@ mod tests { use designcraft_geom::{Affine, Rect, shapes}; fn asset(id: u64, data: &[u8]) -> Asset { - Asset { id: AssetId(id), name: "p.png".into(), mime: "image/png".into(), data: Arc::new(data.to_vec()), ..Default::default() } + Asset { + id: AssetId(id), + name: "p.png".into(), + mime: "image/png".into(), + data: crate::AssetData::embedded(data.to_vec()), + ..Default::default() + } } #[test] @@ -426,7 +447,7 @@ mod tests { assert_eq!(d.merge_duplicate_assets(), 256); assert_eq!(d.assets.len(), 256); for id in 1..=256u64 { - assert_eq!(d.assets[&AssetId(id)].data[1016..], id.to_le_bytes()); + assert_eq!(d.assets[&AssetId(id)].data.embedded_bytes().unwrap()[1016..], id.to_le_bytes()); } } @@ -547,7 +568,10 @@ mod tests { assert_eq!(s, i); } assert_eq!(scanned.keys().collect::>(), indexed.keys().collect::>()); - assert!(Arc::ptr_eq(&indexed[&AssetId(5)].data, &indexed[&AssetId(6)].data), "identical bytes share one buffer"); + assert!( + Arc::ptr_eq(indexed[&AssetId(5)].data.embedded_bytes().unwrap(), indexed[&AssetId(6)].data.embedded_bytes().unwrap()), + "identical bytes share one buffer" + ); } /// Many empty unlinked placements (graphics with neither bytes nor a link) insert without @@ -581,7 +605,10 @@ mod tests { assert!(embedded != a && a != b && embedded != b); assert_eq!(d.assets[&embedded].link, None); assert_eq!(d.assets[&b].link.as_deref(), Some("/q/b.png")); - assert!(Arc::ptr_eq(&d.assets[&a].data, &d.assets[&b].data), "identical bytes share one buffer"); + assert!( + Arc::ptr_eq(d.assets[&a].data.embedded_bytes().unwrap(), d.assets[&b].data.embedded_bytes().unwrap()), + "identical bytes share one buffer" + ); } #[test] diff --git a/crates/doc/src/lib.rs b/crates/doc/src/lib.rs index 329235be9..1de2cc742 100644 --- a/crates/doc/src/lib.rs +++ b/crates/doc/src/lib.rs @@ -12,6 +12,7 @@ pub mod anchored; pub mod arabic; pub mod arrow; +pub mod asset_data; pub mod assets; pub mod attrs; pub mod build; @@ -44,6 +45,7 @@ use std::collections::BTreeMap; use std::sync::Arc; pub use anchored::{AnchorPosition, AnchoredObject, OBJECT_MARK}; +pub use asset_data::{AssetData, SourceKey, Unavailable, WeakAssetData}; pub use assets::{AssetIndex, find_same_asset, insert_asset, remap_story_assets, story_asset_refs}; pub use attrs::*; pub use datamerge::{ @@ -620,8 +622,20 @@ pub struct Asset { /// clears them. #[serde(default, skip_serializing_if = "Option::is_none")] pub link_fingerprint: Option, + /// `link` relative to the document's folder, for finding the file after both moved. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub link_relative: Option, + /// The linked file's content hash (`sha256:` and hex digits). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub link_hash: Option, + /// Whether a save writes the bytes into the document or only the link to them. + #[serde(default, skip_serializing_if = "Storage::is_embedded")] + pub storage: Storage, + /// A small preview drawn while the full file can't be read. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub proxy: Option, #[serde(skip)] - pub data: Arc>, + pub data: AssetData, /// Pixel size for raster images. #[serde(default, skip_serializing_if = "Option::is_none")] pub pixels: Option<(u32, u32)>, @@ -660,6 +674,39 @@ impl Asset { } } +/// Where a save keeps an asset's bytes. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum Storage { + /// In the document. + #[default] + Embedded, + /// In the linked file only. + Linked, +} + +impl Storage { + fn is_embedded(&self) -> bool { + *self == Storage::Embedded + } +} + +/// A linked asset's preview image. Its bytes are stored next to the JSON, like asset bytes. +#[derive(Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Proxy { + pub mime: String, + pub pixels: (u32, u32), + #[serde(skip)] + pub data: Arc>, +} + +impl std::fmt::Debug for Proxy { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Proxy").field("mime", &self.mime).field("pixels", &self.pixels).field("len", &self.data.len()).finish() + } +} + /// Image Import Options › Crop to: which box of a PDF page a placed PDF shows. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] @@ -717,9 +764,8 @@ pub struct AlphaOutline { /// The boundaries of transparent holes those areas enclose (wound the other way). pub holes: designcraft_geom::PathData, /// The image data it was traced from: a relink or link update replaces the asset's data - /// under the same id, and the outline is traced again. (Weak, so it doesn't keep old image - /// data alive, and its address can't be reused by new data while it's held.) - pub source: std::sync::Weak>, + /// under the same id, and the outline is traced again. + pub source: SourceKey, /// A number no other outline traced in this process has ([`AlphaOutline::next_serial`]): /// composition caches compare it to tell a new outline from the one they used. pub serial: u64, @@ -733,8 +779,8 @@ impl AlphaOutline { } /// Traced from `data`, the asset's current image data. - pub fn traced_from(&self, data: &Arc>) -> bool { - std::ptr::eq(self.source.as_ptr(), Arc::as_ptr(data)) + pub fn traced_from(&self, data: &AssetData) -> bool { + self.source == data.key() } } @@ -743,7 +789,7 @@ impl PartialEq for AlphaOutline { (self.asset, self.threshold, self.width, self.height) == (other.asset, other.threshold, other.width, other.height) && self.outer == other.outer && self.holes == other.holes - && self.source.ptr_eq(&other.source) + && self.source == other.source } }