From ef01d69f7d2f05bd185431f300998ddd9b3b8557 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Thu, 12 Nov 2026 19:11:55 -0500 Subject: [PATCH 01/36] Shared document bundle package IO --- Cargo.lock | 10 + Cargo.toml | 1 + crates/document-bundle/Cargo.toml | 17 ++ crates/document-bundle/src/lib.rs | 292 ++++++++++++++++++++++++++++++ crates/zip/src/archive.rs | 7 + 5 files changed, 327 insertions(+) create mode 100644 crates/document-bundle/Cargo.toml create mode 100644 crates/document-bundle/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index cf0e0bd47..174984c1d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1336,6 +1336,16 @@ dependencies = [ name = "craft-docpath" version = "0.1.0" +[[package]] +name = "craft-document-bundle" +version = "0.1.0" +dependencies = [ + "craft-zip", + "serde_json", + "thiserror 2.0.21", + "zip", +] + [[package]] name = "craft-docx" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 629bab64c..4735dba39 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,6 +39,7 @@ craft-dav = { path = "crates/dav" } craft-dialogs = { path = "crates/dialogs" } craft-doc-binary = { path = "crates/doc-binary" } craft-docx = { path = "crates/docx" } +craft-document-bundle = { path = "crates/document-bundle" } craft-dxf = { path = "crates/dxf" } craft-drawingml = { path = "crates/drawingml" } craft-eps = { path = "crates/eps" } diff --git a/crates/document-bundle/Cargo.toml b/crates/document-bundle/Cargo.toml new file mode 100644 index 000000000..705ac7db5 --- /dev/null +++ b/crates/document-bundle/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "craft-document-bundle" +description = "A reusable JSON document bundle envelope over ZIP: metadata, previews, asset entries and legacy inline asset extraction." +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +craft-zip = { workspace = true } +serde_json = { workspace = true } +thiserror = { workspace = true } +zip = { version = "9", default-features = false, features = ["deflate-flate2"] } diff --git a/crates/document-bundle/src/lib.rs b/crates/document-bundle/src/lib.rs new file mode 100644 index 000000000..97acba322 --- /dev/null +++ b/crates/document-bundle/src/lib.rs @@ -0,0 +1,292 @@ +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +//! A JSON document bundle envelope over ZIP. +//! +//! Applications keep their document models. This crate owns the reusable container mechanics: +//! ordered mimetype, metadata, JSON model, previews, embedded asset paths, compression policy, +//! bounded reads and extraction of legacy base64 asset maps. Entry names and MIME type are +//! supplied by the format using the bundle. + +use std::io::{Read, Seek}; + +use craft_zip::{Archive, Compression, Limits, Writer}; +use serde_json::Value; + +const MAX_ENTRIES: u64 = 100_000; +const JSON_LIMIT: u64 = 256 << 20; +const ASSET_LIMIT: u64 = 1 << 30; +const EXPANSION_LIMIT: u64 = 4 << 30; + +/// Per-entry and total-expansion limits for an untrusted document bundle. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ReadLimits { + /// Maximum expanded size of metadata and document JSON. + pub json_bytes: u64, + /// Maximum expanded size of one compressed asset. Stored assets are bounded by archive size. + pub compressed_asset_bytes: u64, + /// Maximum expansion beyond the bytes in the source archive. + pub expansion_bytes: u64, +} + +impl Default for ReadLimits { + fn default() -> Self { + Self { json_bytes: JSON_LIMIT, compressed_asset_bytes: ASSET_LIMIT, expansion_bytes: EXPANSION_LIMIT } + } +} + +/// Names and identity used by one JSON document bundle format. +#[derive(Clone, Copy, Debug)] +pub struct Spec<'a> { + /// Stored first entry and its payload. + pub mimetype_entry: &'a str, + pub mimetype: &'a str, + /// Metadata and document JSON member names. + pub metadata_entry: &'a str, + pub document_entry: &'a str, + /// Optional preview entry name. + pub preview_entry: &'a str, + /// Directory prefix for embedded assets, without a trailing slash. + pub assets_directory: &'a str, + /// Metadata value written in the `format` field. + pub format_name: &'a str, + /// Old single-file JSON field containing base64 assets keyed by decimal id. + pub legacy_assets_field: Option<&'a str>, +} + +/// An embedded asset passed to [`write`]. +#[derive(Clone, Copy, Debug)] +pub struct Asset<'a> { + /// Decimal asset id used in the archive entry path. + pub id: &'a str, + /// MIME type used to select the extension and compression policy. + pub mime: &'a str, + /// File bytes retained byte-for-byte. + pub data: &'a [u8], +} + +/// A document and embedded payloads returned by [`read`]. +#[derive(Clone, Debug, PartialEq)] +pub struct Package { + /// Parsed model JSON, after extracting legacy inline assets. + pub document: Value, + /// Format version declared by metadata, or 1 when absent. + pub version: u64, + /// Embedded asset bytes keyed by decimal asset id. + pub assets: Vec<(String, Vec)>, +} + +/// Why a document bundle could not be read or written. +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +pub enum Error { + /// The bytes are not a bundle or legacy JSON document. + #[error("not a document bundle: {0}")] + NotBundle(String), + /// The bundle requires a newer reader. + #[error("document format {0} is newer than this reader supports")] + TooNew(u64), + /// A member could not be written. + #[error("could not write document bundle: {0}")] + Write(String), + /// A member exceeded a read limit or could not be read. + #[error("could not read document bundle: {0}")] + Read(String), +} + +fn extension(mime: &str) -> &'static str { + match mime { + "image/png" => "png", + "image/jpeg" => "jpg", + "image/gif" => "gif", + "image/webp" => "webp", + "image/tiff" => "tif", + "application/pdf" => "pdf", + "application/postscript" => "eps", + "image/svg+xml" => "svg", + _ => "bin", + } +} + +/// Whether `bytes` are a ZIP bundle that contains the configured document member. +pub fn has_document(bytes: &[u8], spec: Spec<'_>) -> bool { + let Ok(archive) = Archive::open(bytes, Limits { max_entries: MAX_ENTRIES, max_total_bytes: 0 }) else { return false }; + archive.contains(spec.document_entry) +} + +/// Whether `prefix` starts with the configured stored mimetype entry. +pub fn starts_with_mimetype(prefix: &[u8], spec: Spec<'_>) -> bool { + craft_zip::stored_mimetype(prefix).is_some_and(|mime| mime == spec.mimetype.as_bytes()) +} + +/// Write a ZIP document bundle containing the serialized document, metadata, preview and assets. +pub fn write(document: &[u8], version: u32, generator: &str, preview: Option<&[u8]>, assets: &[Asset<'_>], spec: Spec<'_>) -> Result, Error> { + let mut writer = Writer::new(); + writer.add(spec.mimetype_entry, spec.mimetype.as_bytes(), Compression::Stored).map_err(|e| Error::Write(e.to_string()))?; + let metadata = serde_json::json!({"format":spec.format_name, "version":version, "generator":generator}); + let metadata = serde_json::to_vec(&metadata).map_err(|e| Error::Write(e.to_string()))?; + writer.add(spec.metadata_entry, &metadata, Compression::Deflated).map_err(|e| Error::Write(e.to_string()))?; + if let Some(preview) = preview { + writer.add(spec.preview_entry, preview, Compression::Stored).map_err(|e| Error::Write(e.to_string()))?; + } + writer.add(spec.document_entry, document, Compression::Deflated).map_err(|e| Error::Write(e.to_string()))?; + for asset in assets.iter().filter(|asset| !asset.data.is_empty()) { + let path = format!("{}/{}.{}", spec.assets_directory, asset.id, extension(asset.mime)); + let compression = + if matches!(asset.mime, "image/png" | "image/jpeg" | "image/gif" | "image/webp") { Compression::Stored } else { Compression::Deflated }; + writer.add(&path, asset.data, compression).map_err(|e| Error::Write(e.to_string()))?; + } + writer.finish().map_err(|e| Error::Write(e.to_string())) +} + +/// Read a ZIP bundle or legacy JSON document within bounded member sizes. +/// +/// Members under the configured assets directory whose id does not occur in the document's +/// `assets` object are never decompressed. Unreferenced data cannot consume the caller's memory. +pub fn read(bytes: &[u8], supported_version: u32, spec: Spec<'_>) -> Result { + read_with_limits(bytes, supported_version, spec, ReadLimits::default()) +} + +/// Like [`read`], with caller-selected bounded-read limits. +pub fn read_with_limits(bytes: &[u8], supported_version: u32, spec: Spec<'_>, limits: ReadLimits) -> Result { + if !bytes.starts_with(b"PK") { + return read_legacy(bytes, spec); + } + let source_size = u64::try_from(bytes.len()).unwrap_or(u64::MAX); + let mut archive = Archive::open(bytes, Limits { max_entries: MAX_ENTRIES, max_total_bytes: source_size.saturating_add(limits.expansion_bytes) }) + .map_err(|e| Error::NotBundle(e.to_string()))?; + let version = archive + .read(spec.metadata_entry, limits.json_bytes) + .map_err(|e| Error::Read(e.to_string()))? + .and_then(|metadata| serde_json::from_slice::(&metadata).ok()) + .and_then(|metadata| metadata.get("version").and_then(Value::as_u64)) + .unwrap_or(1); + if version > u64::from(supported_version) { + return Err(Error::TooNew(version)); + } + let document_bytes = archive + .read(spec.document_entry, limits.json_bytes) + .map_err(|e| Error::Read(e.to_string()))? + .ok_or_else(|| Error::NotBundle(format!("missing {}", spec.document_entry)))?; + let document: Value = serde_json::from_slice(&document_bytes).map_err(|e| Error::NotBundle(e.to_string()))?; + let wanted: std::collections::HashSet = + document.get("assets").and_then(Value::as_object).into_iter().flat_map(|assets| assets.keys().cloned()).collect(); + let prefix = format!("{}/", spec.assets_directory); + let mut embedded = Vec::new(); + for path in archive.names().map(|name| name.into_owned()).collect::>() { + let Some(member) = path.strip_prefix(&prefix) else { continue }; + let Some(id) = member.split('.').next() else { continue }; + if !wanted.contains(id) || id.parse::().is_err() { + continue; + } + let stored = archive.is_stored(&path).unwrap_or(false); + let limit = if stored { source_size } else { limits.compressed_asset_bytes }; + let data = archive.read(&path, limit).map_err(|e| Error::Read(e.to_string()))?; + if let Some(data) = data { + embedded.push((id.to_owned(), data)); + } + } + Ok(Package { document, version, assets: embedded }) +} + +/// Read a preview with a fixed maximum size without decompressing the full bundle. +pub fn read_preview(bytes: &[u8], preview_entry: &str) -> Option> { + const PREVIEW_LIMIT: u64 = 4 << 20; + let mut archive = Archive::open(bytes, Limits { max_entries: MAX_ENTRIES, max_total_bytes: PREVIEW_LIMIT }).ok()?; + archive.read(preview_entry, PREVIEW_LIMIT).ok().flatten() +} + +/// Read a bounded preview from a seekable file without loading the full bundle into memory. +pub fn read_preview_from(reader: R, preview_entry: &str) -> Option> { + const PREVIEW_LIMIT: u64 = 4 << 20; + let mut zip = zip::ZipArchive::new(reader).ok()?; + let entry = zip.by_name(preview_entry).ok()?; + if entry.size() > PREVIEW_LIMIT { + return None; + } + let mut preview = Vec::new(); + entry.take(PREVIEW_LIMIT.saturating_add(1)).read_to_end(&mut preview).ok()?; + (u64::try_from(preview.len()).ok()? <= PREVIEW_LIMIT).then_some(preview) +} + +fn read_legacy(bytes: &[u8], spec: Spec<'_>) -> Result { + let mut document: Value = serde_json::from_slice(bytes).map_err(|e| Error::NotBundle(e.to_string()))?; + let mut assets = Vec::new(); + if let Some(field) = spec.legacy_assets_field { + if let Some(inline) = document.get(field).and_then(Value::as_object) { + for (id, value) in inline { + if id.parse::().is_ok() + && let Some(encoded) = value.as_str() + { + assets.push((id.clone(), decode_base64(encoded))); + } + } + } + if let Some(object) = document.as_object_mut() { + object.remove(field); + } + } + Ok(Package { document, version: 1, assets }) +} + +fn decode_base64(input: &str) -> Vec { + let mut out = Vec::with_capacity(input.len().saturating_mul(3) / 4); + let (mut buffer, mut bits) = (0u32, 0u32); + for byte in input.bytes() { + let value = match byte { + b'A'..=b'Z' => byte - b'A', + b'a'..=b'z' => byte - b'a' + 26, + b'0'..=b'9' => byte - b'0' + 52, + b'+' | b'-' => 62, + b'/' | b'_' => 63, + _ => continue, + }; + buffer = (buffer << 6) | u32::from(value); + bits += 6; + if bits >= 8 { + bits -= 8; + out.push((buffer >> bits) as u8); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + const SPEC: Spec<'static> = Spec { + mimetype_entry: "mimetype", + mimetype: "application/vnd.test+zip", + metadata_entry: "meta.json", + document_entry: "document.json", + preview_entry: "preview.png", + assets_directory: "assets", + format_name: "test", + legacy_assets_field: Some("assetData"), + }; + + #[test] + fn writes_and_reads_bundle_members() { + let payload = [1u8, 2, 3]; + let bytes = + write(br#"{"assets":{"7":{}}}"#, 4, "Test 0.1", Some(b"png"), &[Asset { id: "7", mime: "image/png", data: &payload }], SPEC).unwrap(); + assert!(has_document(&bytes, SPEC)); + assert!(starts_with_mimetype(&bytes, SPEC)); + let loaded = read(&bytes, 4, SPEC).unwrap(); + assert_eq!(loaded.document, serde_json::json!({"assets":{"7":{}}})); + assert_eq!(loaded.version, 4); + assert_eq!(loaded.assets, [("7".to_owned(), payload.to_vec())]); + assert_eq!(read_preview(&bytes, SPEC.preview_entry).as_deref(), Some(&b"png"[..])); + } + + #[test] + fn extracts_legacy_inline_assets() { + let loaded = read(br#"{"assetData":{"7":"AQID"},"name":"x"}"#, 4, SPEC).unwrap(); + assert_eq!(loaded.document, serde_json::json!({"name":"x"})); + assert_eq!(loaded.assets, [("7".to_owned(), vec![1, 2, 3])]); + } + + #[test] + fn refuses_newer_versions() { + let bytes = write(b"{}", 9, "future", None, &[], SPEC).unwrap(); + assert_eq!(read(&bytes, 4, SPEC), Err(Error::TooNew(9))); + } +} diff --git a/crates/zip/src/archive.rs b/crates/zip/src/archive.rs index bdcf64cd7..448e2220b 100644 --- a/crates/zip/src/archive.rs +++ b/crates/zip/src/archive.rs @@ -97,6 +97,13 @@ impl<'a> Archive<'a> { self.zip.by_index_raw(index).ok().map(|f| f.compressed_size()) } + /// Is `name` stored without compression? The answer lets a format with in-memory source bytes + /// apply an output-size cap to compressed entries while bounding stored reads by source size. + pub fn is_stored(&mut self, name: &str) -> Option { + let index = self.zip.index_for_name(name)?; + self.zip.by_index_raw(index).ok().map(|f| f.compression() == zip::CompressionMethod::Stored) + } + /// The first `n` bytes of entry `name` (all of it when it is shorter), or `None` when there /// is no such entry: for telling a file by its signature without inflating it. The bytes are /// charged to the total; a total with less left than the bytes wanted refuses before reading. From dc20b58f0157df05f6059bd408d6f14ab130f343 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Thu, 12 Nov 2026 19:19:53 -0500 Subject: [PATCH 02/36] Bound document bundle asset reads --- crates/document-bundle/src/lib.rs | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/crates/document-bundle/src/lib.rs b/crates/document-bundle/src/lib.rs index 97acba322..18e40a3ff 100644 --- a/crates/document-bundle/src/lib.rs +++ b/crates/document-bundle/src/lib.rs @@ -152,6 +152,9 @@ pub fn read_with_limits(bytes: &[u8], supported_version: u32, spec: Spec<'_>, li let source_size = u64::try_from(bytes.len()).unwrap_or(u64::MAX); let mut archive = Archive::open(bytes, Limits { max_entries: MAX_ENTRIES, max_total_bytes: source_size.saturating_add(limits.expansion_bytes) }) .map_err(|e| Error::NotBundle(e.to_string()))?; + if archive.has_duplicate_names() { + return Err(Error::NotBundle("the archive has duplicate member names".into())); + } let version = archive .read(spec.metadata_entry, limits.json_bytes) .map_err(|e| Error::Read(e.to_string()))? @@ -209,12 +212,21 @@ pub fn read_preview_from(reader: R, preview_entry: &str) -> Opti fn read_legacy(bytes: &[u8], spec: Spec<'_>) -> Result { let mut document: Value = serde_json::from_slice(bytes).map_err(|e| Error::NotBundle(e.to_string()))?; let mut assets = Vec::new(); + let mut total = 0u64; if let Some(field) = spec.legacy_assets_field { if let Some(inline) = document.get(field).and_then(Value::as_object) { for (id, value) in inline { if id.parse::().is_ok() && let Some(encoded) = value.as_str() { + let decoded_size = u64::try_from(encoded.len()).unwrap_or(u64::MAX).saturating_mul(3) / 4; + if decoded_size > ASSET_LIMIT { + return Err(Error::Read(format!("legacy asset {id} exceeds the {ASSET_LIMIT}-byte limit"))); + } + total = total.saturating_add(decoded_size); + if total > EXPANSION_LIMIT { + return Err(Error::Read(format!("legacy assets exceed the {EXPANSION_LIMIT}-byte total limit"))); + } assets.push((id.clone(), decode_base64(encoded))); } } @@ -289,4 +301,17 @@ mod tests { let bytes = write(b"{}", 9, "future", None, &[], SPEC).unwrap(); assert_eq!(read(&bytes, 4, SPEC), Err(Error::TooNew(9))); } + + #[test] + fn stored_assets_use_source_size_and_deflated_assets_use_their_limit() { + let bytes = vec![7u8; 128]; + let document = br#"{"assets":{"7":{}}}"#; + let limits = ReadLimits { json_bytes: 1 << 20, compressed_asset_bytes: 8, expansion_bytes: 1 << 20 }; + let stored = write(document, 4, "test", None, &[Asset { id: "7", mime: "image/png", data: &bytes }], SPEC).unwrap(); + let package = read_with_limits(&stored, 4, SPEC, limits).unwrap(); + assert_eq!(package.assets, [("7".to_owned(), bytes.clone())]); + + let compressed = write(document, 4, "test", None, &[Asset { id: "7", mime: "application/octet-stream", data: &bytes }], SPEC).unwrap(); + assert!(read_with_limits(&compressed, 4, SPEC, limits).is_err()); + } } From 409b0036f86f7faa5a1f2d9663eb8f692c97ea5a Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 00:37:31 +0000 Subject: [PATCH 03/36] video: bound H.264 picture number arithmetic --- crates/video/src/h264/decode/dpb.rs | 89 +++++++++++++++++++++++++++-- 1 file changed, 84 insertions(+), 5 deletions(-) diff --git a/crates/video/src/h264/decode/dpb.rs b/crates/video/src/h264/decode/dpb.rs index 67713d9d3..2615698a4 100644 --- a/crates/video/src/h264/decode/dpb.rs +++ b/crates/video/src/h264/decode/dpb.rs @@ -176,6 +176,7 @@ impl Dpb { return Ok(lists); } let cur_fn = sh.frame_num; + ensure!(max_frame_num > 0 && max_frame_num <= (1 << 16) && cur_fn < max_frame_num, "frame number out of range"); let pic_num = |e: &DpbEntry| Self::frame_num_wrap(e, cur_fn, max_frame_num); let short: Vec<&DpbEntry> = self.entries.iter().filter(|e| e.mark == RefMark::Short).collect(); let mut long: Vec<&DpbEntry> = self.entries.iter().filter(|e| e.mark == RefMark::Long).collect(); @@ -212,6 +213,7 @@ impl Dpb { let mut ref_idx = 0usize; for m in &sh.ref_pic_list_mod[l] { ensure!(ref_idx < n, "too many ref_pic_list_modification entries"); + ensure!(m.idc >= 2 || m.value < max_frame_num, "abs_diff_pic_num_minus1 out of range"); list.push(None); // temporarily one longer let (target, is_long): (Option<&DpbEntry>, bool) = match m.idc { 0 | 1 => { @@ -272,13 +274,13 @@ impl Dpb { /// Adaptive memory control (8.2.5.4). Returns true if MMCO 5 was executed. fn apply_mmcos(&mut self, mmcos: &[Mmco], cur_frame_num: u32, max_frame_num: u32, cur_long: &mut Option) -> Result { let mut had5 = false; - let curr_pic_num = cur_frame_num as i32; + let curr_pic_num = i64::from(cur_frame_num); for m in mmcos { match m.op { 1 => { - let pn = curr_pic_num - (m.difference_of_pic_nums_minus1 as i32 + 1); + let pn = curr_pic_num - (i64::from(m.difference_of_pic_nums_minus1) + 1); for e in &mut self.entries { - if e.mark == RefMark::Short && Self::frame_num_wrap(e, cur_frame_num, max_frame_num) == pn { + if e.mark == RefMark::Short && i64::from(Self::frame_num_wrap(e, cur_frame_num, max_frame_num)) == pn { e.mark = RefMark::Unused; } } @@ -295,14 +297,14 @@ impl Dpb { self.max_long_term_frame_idx.is_some_and(|max| m.long_term_frame_idx <= max), "MMCO long-term index exceeds the configured limit" ); - let pn = curr_pic_num - (m.difference_of_pic_nums_minus1 as i32 + 1); + let pn = curr_pic_num - (i64::from(m.difference_of_pic_nums_minus1) + 1); for e in &mut self.entries { if e.mark == RefMark::Long && e.long_term_frame_idx == m.long_term_frame_idx { e.mark = RefMark::Unused; } } for e in &mut self.entries { - if e.mark == RefMark::Short && Self::frame_num_wrap(e, cur_frame_num, max_frame_num) == pn { + if e.mark == RefMark::Short && i64::from(Self::frame_num_wrap(e, cur_frame_num, max_frame_num)) == pn { e.mark = RefMark::Long; e.long_term_frame_idx = m.long_term_frame_idx; } @@ -461,6 +463,83 @@ mod tests { }) } + fn reference_slice() -> SliceHeader { + SliceHeader { + nal_ref_idc: 1, + nal_unit_type: 1, + idr: false, + first_mb_in_slice: 0, + slice_type_raw: 0, + slice_type: SliceType::P, + pps_id: 0, + colour_plane_id: 0, + frame_num: 0, + field_pic: false, + bottom_field: false, + idr_pic_id: 0, + pic_order_cnt_lsb: 0, + delta_pic_order_cnt_bottom: 0, + delta_pic_order_cnt: [0; 2], + redundant_pic_cnt: 0, + direct_spatial_mv_pred: false, + num_ref_idx_active: [1, 0], + ref_pic_list_mod: [Vec::new(), Vec::new()], + pred_weight_table: None, + no_output_of_prior_pics: false, + long_term_reference: false, + adaptive_ref_pic_marking: false, + mmcos: Vec::new(), + cabac_init_idc: 0, + slice_qp_delta: 0, + sp_for_switch: false, + slice_qs_delta: 0, + disable_deblocking_filter_idc: 0, + slice_alpha_c0_offset_div2: 0, + slice_beta_offset_div2: 0, + slice_group_change_cycle: 0, + header_bits: 0, + } + } + + #[test] + fn large_picture_number_differences_do_not_overflow() { + for op in [1, 3] { + for difference in [i32::MAX as u32, 1 << 31, u32::MAX] { + let mut dpb: Dpb = Dpb::new(); + dpb.max_long_term_frame_idx = Some(0); + let op = Mmco { + op, + difference_of_pic_nums_minus1: difference, + long_term_pic_num: 0, + long_term_frame_idx: 0, + max_long_term_frame_idx_plus1: 0, + }; + assert!(dpb.apply_mmcos(&[op], 5, 16, &mut None).is_ok()); + } + } + } + + #[test] + fn invalid_reference_list_differences_are_rejected() { + use crate::h264::decode::slice::RefPicListMod; + for idc in [0, 1] { + for value in [0, 15] { + let mut sh = reference_slice(); + sh.frame_num = 5; + sh.ref_pic_list_mod[0].push(RefPicListMod { idc, value }); + let dpb: Dpb = Dpb::new(); + assert!(dpb.build_ref_lists(&sh, 0, 16, |entry| entry.frame).is_ok()); + } + for value in [16, i32::MAX as u32, 1 << 31, u32::MAX] { + let mut sh = reference_slice(); + sh.frame_num = 5; + sh.ref_pic_list_mod[0].push(RefPicListMod { idc, value }); + let dpb: Dpb = Dpb::new(); + assert!(dpb.build_ref_lists(&sh, 0, 16, |entry| entry.frame).is_err()); + } + } + } + /// A huge frame_num jump must only materialise the frames the sliding window keeps (#2779). #[test] fn frame_num_gap_materialises_at_most_max_num_ref_frames() { From 6fa8323991b54e7ee62d2d2d9d8f71e719f65a21 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 00:44:21 +0000 Subject: [PATCH 04/36] video: saturate extreme H.264 picture order values --- crates/video/src/h264/decode/slice.rs | 42 ++++++++++++++++----------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/crates/video/src/h264/decode/slice.rs b/crates/video/src/h264/decode/slice.rs index 75b6c4f8e..61670f059 100644 --- a/crates/video/src/h264/decode/slice.rs +++ b/crates/video/src/h264/decode/slice.rs @@ -462,21 +462,21 @@ impl PocState { let (prev_msb, prev_lsb) = if sh.idr { (0, 0) } else { (self.prev_ref_msb, self.prev_ref_lsb) }; let max_lsb = 1i32 << sps.log2_max_poc_lsb; let lsb = sh.pic_order_cnt_lsb as i32; - let msb = if lsb < prev_lsb && prev_lsb.wrapping_sub(lsb) >= max_lsb / 2 { - prev_msb.wrapping_add(max_lsb) - } else if lsb > prev_lsb && lsb.wrapping_sub(prev_lsb) > max_lsb / 2 { - prev_msb.wrapping_sub(max_lsb) + let msb = if lsb < prev_lsb && prev_lsb.saturating_sub(lsb) >= max_lsb / 2 { + prev_msb.saturating_add(max_lsb) + } else if lsb > prev_lsb && lsb.saturating_sub(prev_lsb) > max_lsb / 2 { + prev_msb.saturating_sub(max_lsb) } else { prev_msb }; - let top = msb.wrapping_add(lsb); - let bottom = top.wrapping_add(sh.delta_pic_order_cnt_bottom); + let top = msb.saturating_add(lsb); + let bottom = top.saturating_add(sh.delta_pic_order_cnt_bottom); Poc { top, bottom, msb, frame_num_offset: 0 } } 1 => { let frame_num_offset = self.frame_num_offset(sh.idr, sh.frame_num, sps); let n = sps.offset_for_ref_frame.len() as i32; - let mut abs_frame_num = if n != 0 { frame_num_offset.wrapping_add(sh.frame_num as i32) } else { 0 }; + let mut abs_frame_num = if n != 0 { frame_num_offset.saturating_add(sh.frame_num as i32) } else { 0 }; if sh.nal_ref_idc == 0 && abs_frame_num > 0 { abs_frame_num -= 1; } @@ -484,17 +484,17 @@ impl PocState { if abs_frame_num > 0 { let cycle_cnt = (abs_frame_num - 1) / n; let in_cycle = (abs_frame_num - 1) % n; - let delta_cycle = sps.offset_for_ref_frame.iter().fold(0i32, |a, &o| a.wrapping_add(o)); - expected = cycle_cnt.wrapping_mul(delta_cycle); + let delta_cycle = sps.offset_for_ref_frame.iter().fold(0i32, |a, &o| a.saturating_add(o)); + expected = cycle_cnt.saturating_mul(delta_cycle); for &o in &sps.offset_for_ref_frame[..=in_cycle as usize] { - expected = expected.wrapping_add(o); + expected = expected.saturating_add(o); } } if sh.nal_ref_idc == 0 { - expected = expected.wrapping_add(sps.offset_for_non_ref_pic); + expected = expected.saturating_add(sps.offset_for_non_ref_pic); } - let top = expected.wrapping_add(sh.delta_pic_order_cnt[0]); - let bottom = top.wrapping_add(sps.offset_for_top_to_bottom_field).wrapping_add(sh.delta_pic_order_cnt[1]); + let top = expected.saturating_add(sh.delta_pic_order_cnt[0]); + let bottom = top.saturating_add(sps.offset_for_top_to_bottom_field).saturating_add(sh.delta_pic_order_cnt[1]); Poc { top, bottom, msb: 0, frame_num_offset } } _ => { @@ -502,9 +502,9 @@ impl PocState { let temp = if sh.idr { 0 } else if sh.nal_ref_idc == 0 { - 2i32.wrapping_mul(frame_num_offset.wrapping_add(sh.frame_num as i32)).wrapping_sub(1) + 2i32.saturating_mul(frame_num_offset.saturating_add(sh.frame_num as i32)).saturating_sub(1) } else { - 2i32.wrapping_mul(frame_num_offset.wrapping_add(sh.frame_num as i32)) + 2i32.saturating_mul(frame_num_offset.saturating_add(sh.frame_num as i32)) }; Poc { top: temp, bottom: temp, msb: 0, frame_num_offset } } @@ -515,7 +515,7 @@ impl PocState { if idr { 0 } else if self.prev_frame_num > frame_num { - self.prev_frame_num_offset.wrapping_add(sps.max_frame_num() as i32) + self.prev_frame_num_offset.saturating_add(sps.max_frame_num() as i32) } else { self.prev_frame_num_offset } @@ -528,7 +528,7 @@ impl PocState { if sh.nal_ref_idc != 0 { if mmco5 { self.prev_ref_msb = 0; - self.prev_ref_lsb = poc.top.wrapping_sub(poc.frame()); + self.prev_ref_lsb = poc.top.saturating_sub(poc.frame()); } else { self.prev_ref_msb = poc.msb; self.prev_ref_lsb = sh.pic_order_cnt_lsb as i32; @@ -624,6 +624,14 @@ mod poc_tests { } } + #[test] + fn extreme_bottom_delta_saturates_instead_of_reversing_picture_order() { + let sps = sps(0); + let sh = header(0, 1, i32::MAX, [0, 0], 1); + let poc = PocState::default().compute(&sh, &sps); + assert_eq!((poc.top, poc.bottom), (1, i32::MAX)); + } + /// Offsets and deltas are se(v) values that a stream may set anywhere in the i32 range. #[test] fn poc_does_not_overflow_on_extreme_deltas() { From 4cca9be2abc7aeb2e3b55c94673f8413b905bf5e Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 01:10:32 +0000 Subject: [PATCH 05/36] video: configure bounded decoder frame limits --- crates/video/src/h264/decode/README.md | 3 +- crates/video/src/h264/decode/decoder.rs | 13 ++++- crates/video/src/h264/decode/params.rs | 33 +++++++++++- crates/video/src/prores/README.md | 7 +++ crates/video/src/prores/decode.rs | 70 ++++++++++++++++++++++++- crates/video/src/prores/mod.rs | 4 +- 6 files changed, 123 insertions(+), 7 deletions(-) diff --git a/crates/video/src/h264/decode/README.md b/crates/video/src/h264/decode/README.md index a73b8f2f4..4c352fd00 100644 --- a/crates/video/src/h264/decode/README.md +++ b/crates/video/src/h264/decode/README.md @@ -99,7 +99,8 @@ A picture is refused when the pictures the decoder would hold do not fit 1 GiB on 32-bit targets): the DPB (the larger of its slots at the new size and what it holds now), the picture being decoded, and what is in flight or waiting in the output queue. Pictures already held count at their own size, and a buffer shared by the DPB and the queue counts once. -The per-picture size cap applies as well. +The default per-picture luma cap is 2^25 samples; applications with a different bounded frame +policy can raise it with `Decoder::set_max_picture_luma_samples`. `configure_avcc` checks every entry of the record before it handles any: the first list holds SPS NAL units and the second PPS NAL units. A record that carries a picture slice is an error and diff --git a/crates/video/src/h264/decode/decoder.rs b/crates/video/src/h264/decode/decoder.rs index 6d5fe3e5d..01a130b35 100644 --- a/crates/video/src/h264/decode/decoder.rs +++ b/crates/video/src/h264/decode/decoder.rs @@ -115,6 +115,8 @@ pub struct Decoder { /// Bytes the picture store may take ([`crate::budget`]). memory_budget: u64, threads: usize, + /// Maximum reconstructed luma samples in one picture. + max_luma_samples: u64, /// Draft mode: skip deblocking of non-reference pictures ([`Decoder::set_draft`]). draft: bool, } @@ -179,6 +181,7 @@ impl Decoder { max_in_flight: threads.max(1) + 2, memory_budget: crate::budget::DECODE_MEMORY_BUDGET, threads: workers, + max_luma_samples: super::params::MAX_LUMA_SAMPLES, draft: false, } } @@ -391,7 +394,7 @@ impl Decoder { // Before the header fields that depend on the SPS: they are validated for the streams // we decode (a 10-bit slice may have a QP below 0), so a stream we cannot decode must // fail as unsupported here, not as invalid there (#626). - sps.check_supported()?; + sps.check_supported_with_limit(self.max_luma_samples)?; found = Some((pps.clone(), sps.clone())); Ok((&**pps, &**sps)) })?; @@ -468,6 +471,14 @@ impl Decoder { self.memory_budget = bytes; } + /// Set the maximum luma sample count allowed in a single decoded picture. + /// + /// The default is 2^25 samples. This is a per-picture geometry guard; the separate + /// memory budget controls aggregate decoded picture storage. + pub fn set_max_picture_luma_samples(&mut self, samples: u64) { + self.max_luma_samples = samples; + } + /// Refuse an SPS whose pictures would not fit the memory budget, before anything is allocated. fn check_budget(&self, sps: &Sps) -> Result<()> { let chroma = sps.chroma_array_type(); diff --git a/crates/video/src/h264/decode/params.rs b/crates/video/src/h264/decode/params.rs index 62ca789eb..b919efb7d 100644 --- a/crates/video/src/h264/decode/params.rs +++ b/crates/video/src/h264/decode/params.rs @@ -6,7 +6,7 @@ use craft_media::bitstream::BitReader; /// Maximum supported reconstructed luma area. 8K UHD fits comfortably while /// hostile SPS dimensions cannot allocate full pictures and decoder work state. -const MAX_LUMA_SAMPLES: u64 = 1 << 25; +pub(super) const MAX_LUMA_SAMPLES: u64 = 1 << 25; /// Scaling matrices after applying fall-back rules, stored in raster order (weightScale). #[derive(Clone, Debug, PartialEq, Eq)] @@ -527,7 +527,11 @@ impl Sps { } pub fn check_supported(&self) -> Result<()> { - if u64::from(self.width()).saturating_mul(u64::from(self.height())) > MAX_LUMA_SAMPLES { + self.check_supported_with_limit(MAX_LUMA_SAMPLES) + } + + pub(super) fn check_supported_with_limit(&self, max_luma_samples: u64) -> Result<()> { + if u64::from(self.width()).saturating_mul(u64::from(self.height())) > max_luma_samples { return unsupported("picture dimensions exceed the decoder memory limit"); } if !self.frame_mbs_only { @@ -744,6 +748,31 @@ mod tests { assert_eq!(sps.pic_order_cnt_type, 2); assert_eq!(sps.crop_rect(), (0, 0, 176, 144)); } + + #[test] + fn caller_can_raise_single_picture_sample_limit() { + let mut w = BitWriter::new(); + w.write_bits(66, 8); + w.write_bits(0xC0, 8); + w.write_bits(30, 8); + w.write_ue(0); + w.write_ue(0); + w.write_ue(2); + w.write_ue(1); + w.write_bit(false); + w.write_ue(511); // 8192 luma samples wide + w.write_ue(511); // 8192 luma samples high + w.write_bit(true); + w.write_bit(true); + w.write_bit(false); + w.write_bit(false); + w.rbsp_trailing(); + + let sps = Sps::parse(&w.finish()).unwrap(); + assert_eq!((sps.width(), sps.height()), (8192, 8192)); + assert!(sps.check_supported().is_err()); + assert!(sps.check_supported_with_limit(1 << 26).is_ok()); + } /// Baseline SPS with the given `ue(v)` overrides for the fields hostile streams can max out. fn hostile_sps(profile: u32, bit_depth: u32, log2_frame_num: u32, w: u32, h: u32, crop: u32) -> Vec { let mut bw = BitWriter::new(); diff --git a/crates/video/src/prores/README.md b/crates/video/src/prores/README.md index e3a9ea989..9831e3282 100644 --- a/crates/video/src/prores/README.md +++ b/crates/video/src/prores/README.md @@ -16,6 +16,8 @@ use craft_video::prores::*; let frame: Frame = decode_frame(&sample)?; // 10-bit 4:2:2 / 12-bit 4:4:4 let f10 = decode_frame_with(&sample, &DecodeOptions { bit_depth: Some(10), threads: true })?; decode_frame_into(&sample, &DecodeOptions::default(), &mut frame_buf)?; // reuses planes +let limits = DecodeLimits { max_pixels: 8192 * 8192, max_frame_bytes: 512 * 1024 * 1024 }; +let bounded = decode_frame_with_limits(&sample, &DecodeOptions::default(), &limits)?; let hdr: FrameHeader = probe(&sample)?; // header only // Encode (progressive). @@ -28,6 +30,11 @@ alpha: Option>, interlace: Interlace, color: ColorInfo { primaries, tra aspect_ratio, frame_rate_code }` holds planar samples with no row padding (`chroma_width()` is `ceil(width/2)` for 4:2:2). The colour codes are H.273 code points straight from the frame header. +`DecodeLimits` bounds both frame area and aggregate decoded plane storage before allocation. +`decode_frame_with` uses an aggregate limit of 300 MiB and does not impose an additional area +limit; use `decode_frame_with_limits` when the application needs a stricter pixel policy or a +different finite memory ceiling. + `Profile` = `Proxy` (`apco`), `Lt` (`apcs`), `Standard` (`apcn`), `Hq` (`apch`), `P4444` (`ap4h`), `P4444Xq` (`ap4x`). The frame itself doesn't say which profile it is (only the container fourcc does), so the decoder doesn't need to know. diff --git a/crates/video/src/prores/decode.rs b/crates/video/src/prores/decode.rs index f4670af60..7125d4641 100644 --- a/crates/video/src/prores/decode.rs +++ b/crates/video/src/prores/decode.rs @@ -11,6 +11,21 @@ use crate::prores::{AlphaType, ChromaFormat, Error, Frame, Interlace, Result}; /// Maximum aggregate decoded plane storage for one ProRes frame (300 MiB; 8K ProRes 4444 with alpha is ~265 MiB). const MAX_FRAME_BYTES: usize = 300 * 1024 * 1024; +/// Resource limits for decoding one ProRes frame. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DecodeLimits { + /// Maximum frame width × height. Checked before any plane allocation. + pub max_pixels: u64, + /// Maximum aggregate output plane storage in bytes. + pub max_frame_bytes: usize, +} + +impl Default for DecodeLimits { + fn default() -> Self { + Self { max_pixels: u64::MAX, max_frame_bytes: MAX_FRAME_BYTES } + } +} + /// Decoder options. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct DecodeOptions { @@ -105,15 +120,28 @@ impl Scratch { /// Decode one ProRes frame. pub fn decode_frame_with(data: &[u8], opts: &DecodeOptions) -> Result { + decode_frame_with_limits(data, opts, &DecodeLimits::default()) +} + +/// Decode one ProRes frame with explicit geometry and aggregate-memory limits. +pub fn decode_frame_with_limits(data: &[u8], opts: &DecodeOptions, limits: &DecodeLimits) -> Result { let mut f = Frame::new(0, 0, ChromaFormat::Yuv422, 10, false)?; - decode_frame_into(data, opts, &mut f)?; + decode_frame_into_with_limits(data, opts, limits, &mut f)?; Ok(f) } /// Decode one ProRes frame into `out`, reusing its plane allocations (the fastest way to decode /// a sequence). On error the contents of `out` are unspecified. pub fn decode_frame_into(data: &[u8], opts: &DecodeOptions, out: &mut Frame) -> Result<()> { + decode_frame_into_with_limits(data, opts, &DecodeLimits::default(), out) +} + +/// Decode one ProRes frame into `out` under explicit resource limits. +pub fn decode_frame_into_with_limits(data: &[u8], opts: &DecodeOptions, limits: &DecodeLimits, out: &mut Frame) -> Result<()> { let (hdr, mut off, frame_size) = FrameHeader::parse(data)?; + if u64::from(hdr.width) * u64::from(hdr.height) > limits.max_pixels { + return Err(Error::Unsupported("frame size")); + } let data = &data[..frame_size]; let depth = opts.bit_depth.unwrap_or(match hdr.chroma { ChromaFormat::Yuv422 => 10, @@ -131,7 +159,7 @@ pub fn decode_frame_into(data: &[u8], opts: &DecodeOptions, out: &mut Frame) -> .checked_add(chroma.checked_mul(2).ok_or(Error::Invalid("chroma plane count overflows"))?) .and_then(|n| if hdr.alpha == AlphaType::None { Some(n) } else { n.checked_add(luma) }) .ok_or(Error::Invalid("frame plane count overflows"))?; - if samples.checked_mul(std::mem::size_of::()).is_none_or(|n| n > MAX_FRAME_BYTES) { + if samples.checked_mul(std::mem::size_of::()).is_none_or(|n| n > limits.max_frame_bytes) { return Err(Error::Invalid("frame dimensions exceed the decoder memory limit")); } let interlaced = hdr.interlace != Interlace::Progressive; @@ -564,6 +592,44 @@ mod alpha_decode_tests { assert_eq!(slice_quantiser(224).unwrap(), qscale(224)); } + #[test] + fn custom_limits_preserve_large_4444_geometry_before_allocating() { + let hdr = FrameHeader { + header_size: 20, + bitstream_version: 1, + encoder_id: *b"test", + width: 8192, + height: 8192, + chroma: ChromaFormat::Yuv444, + interlace: Interlace::Progressive, + aspect_ratio: 0, + frame_rate_code: 0, + color: crate::prores::ColorInfo::default(), + alpha: AlphaType::Bits16, + qmat_luma: [4; 64], + qmat_chroma: [4; 64], + luma_matrix_present: false, + chroma_matrix_present: false, + }; + let header = hdr.write(); + let frame_size = 8 + header.len(); + let mut data = Vec::with_capacity(frame_size); + data.extend_from_slice(&(frame_size as u32).to_be_bytes()); + data.extend_from_slice(&crate::prores::header::FRAME_ID); + data.extend_from_slice(&header); + + let defaults = DecodeLimits { max_pixels: 8192 * 8192, ..DecodeLimits::default() }; + assert!(matches!( + decode_frame_with_limits(&data, &DecodeOptions::default(), &defaults), + Err(Error::Invalid("frame dimensions exceed the decoder memory limit")) + )); + + let soundcraft = DecodeLimits { max_pixels: 8192 * 8192, max_frame_bytes: 512 * 1024 * 1024 }; + assert!(matches!(decode_frame_with_limits(&data, &DecodeOptions::default(), &soundcraft), Err(Error::Truncated))); + let too_few_pixels = DecodeLimits { max_pixels: 8192 * 8192 - 1, ..soundcraft }; + assert!(matches!(decode_frame_with_limits(&data, &DecodeOptions::default(), &too_few_pixels), Err(Error::Unsupported("frame size")))); + } + #[test] fn alpha_decoder_requires_complete_bounded_runs() { let mut out = [0u16; 1]; diff --git a/crates/video/src/prores/mod.rs b/crates/video/src/prores/mod.rs index 4af49f913..5979c19af 100644 --- a/crates/video/src/prores/mod.rs +++ b/crates/video/src/prores/mod.rs @@ -18,7 +18,9 @@ mod encode; pub mod header; mod tables; -pub use decode::{DecodeOptions, decode_frame, decode_frame_into, decode_frame_with, probe}; +pub use decode::{ + DecodeLimits, DecodeOptions, decode_frame, decode_frame_into, decode_frame_into_with_limits, decode_frame_with, decode_frame_with_limits, probe, +}; pub use encode::{Encoder, EncoderConfig}; pub use header::{FrameHeader, PictureHeader}; From 198697cfc8ef5477f9b8954572978a352010c6db Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 01:18:59 +0000 Subject: [PATCH 06/36] fix(cms): keep clippy compatible with workspace gate --- crates/cms/src/synth.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/cms/src/synth.rs b/crates/cms/src/synth.rs index a59058f08..b1535f78c 100644 --- a/crates/cms/src/synth.rs +++ b/crates/cms/src/synth.rs @@ -111,9 +111,9 @@ impl CmykModel { // Overprints with black: multiplicative in reflectance above a first-surface floor. let floor = 0.006; let mut prim = [[0.0; 3]; 16]; - for m in 0..16 { + for (m, slot) in prim.iter_mut().enumerate() { let p = base[m & 7]; - prim[m] = if m & 8 == 0 { + *slot = if m & 8 == 0 { p } else { std::array::from_fn(|i| { @@ -133,7 +133,7 @@ impl CmykModel { (v + 4.0 * self.tvi[i] * v * (1.0 - v)).clamp(0.0, 1.0) }); let mut s = [0.0; 3]; - for m in 0..16 { + for (m, prim) in self.prim_n.iter().enumerate() { let mut w = 1.0; for (i, ai) in a.iter().enumerate() { w *= if m & (1 << i) != 0 { *ai } else { 1.0 - ai }; @@ -141,7 +141,7 @@ impl CmykModel { if w == 0.0 { continue; } - for (acc, p) in s.iter_mut().zip(&self.prim_n[m]) { + for (acc, p) in s.iter_mut().zip(prim) { *acc += w * p; } } From 92d1bba5dea565afc0cbe15ab4915b594dec6a4f Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 01:28:46 +0000 Subject: [PATCH 07/36] fix(proof): satisfy workspace clippy gate --- crates/proof/src/tr/mod.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/crates/proof/src/tr/mod.rs b/crates/proof/src/tr/mod.rs index 5472fbe21..c728d5872 100644 --- a/crates/proof/src/tr/mod.rs +++ b/crates/proof/src/tr/mod.rs @@ -100,11 +100,7 @@ fn check(d: &TrDict, word: &str, opts: &crate::Options) -> bool { } // Compound words separated by hyphens (e.g., `afyon-karahisar`) - if w.contains('-') && w.split('-').all(|part| part.is_empty() || check(d, part, opts)) { - return true; - } - - false + w.contains('-') && w.split('-').all(|part| part.is_empty() || check(d, part, opts)) } #[cfg(test)] From 2253894e2f54f8117a7292cf17552eb470e9ecd7 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 01:39:26 +0000 Subject: [PATCH 08/36] fix: satisfy Rust 1.100 workspace clippy --- crates/latex/src/tests.rs | 12 ++++++------ crates/video/src/hevc/decode/filter.rs | 5 +---- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/crates/latex/src/tests.rs b/crates/latex/src/tests.rs index 55bde2489..ccf349c80 100644 --- a/crates/latex/src/tests.rs +++ b/crates/latex/src/tests.rs @@ -353,9 +353,9 @@ fn table_borders_and_borderless_round_trip() { let mut doc = Document::new(); let mut t_borderless = Table::new(2, 2, 400.0); t_borderless.props.style = None; - for r in 0..2 { - for c in 0..2 { - t_borderless.rows[r].cells[c].blocks = vec![para_block(Paragraph::with_text("cell", CharProps::default()))]; + for row in &mut t_borderless.rows { + for cell in &mut row.cells { + cell.blocks = vec![para_block(Paragraph::with_text("cell", CharProps::default()))]; } } doc.body = vec![Arc::new(Block::Table(t_borderless))]; @@ -369,9 +369,9 @@ fn table_borders_and_borderless_round_trip() { let mut doc_bordered = Document::new(); let mut t_bordered = Table::new(2, 2, 400.0); t_bordered.props.style = Some("TableGrid".to_string()); - for r in 0..2 { - for c in 0..2 { - t_bordered.rows[r].cells[c].blocks = vec![para_block(Paragraph::with_text("cell", CharProps::default()))]; + for row in &mut t_bordered.rows { + for cell in &mut row.cells { + cell.blocks = vec![para_block(Paragraph::with_text("cell", CharProps::default()))]; } } doc_bordered.body = vec![Arc::new(Block::Table(t_bordered))]; diff --git a/crates/video/src/hevc/decode/filter.rs b/crates/video/src/hevc/decode/filter.rs index d5b94161d..69f936bc1 100644 --- a/crates/video/src/hevc/decode/filter.rs +++ b/crates/video/src/hevc/decode/filter.rs @@ -522,10 +522,7 @@ fn sao_edge_ctb( return false; } } - if !pic.pps.loop_filter_across_tiles && pic.layout.tile_id[pic.layout.rs_to_ts[nrs] as usize] != pic.layout.tile_id[cur_ts as usize] { - return false; - } - true + pic.pps.loop_filter_across_tiles || pic.layout.tile_id[pic.layout.rs_to_ts[nrs] as usize] == pic.layout.tile_id[cur_ts as usize] }; // avail[dy][dx]: the CTB at (rx + dx - 1, ry + dy - 1) may be used. let (rx, ry) = (rs % pic.wctb, rs / pic.wctb); From d316f0fc2d9685ca7b8fdfbaa4bbed1eb49ac496 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 02:02:45 +0000 Subject: [PATCH 09/36] fix(xml): walk deep KRA and ORA trees iteratively --- crates/kra/src/lib.rs | 43 ++++++++++++++++++++++++------ crates/ora/src/lib.rs | 61 ++++++++++++++++++++++++++++++++++--------- 2 files changed, 83 insertions(+), 21 deletions(-) diff --git a/crates/kra/src/lib.rs b/crates/kra/src/lib.rs index 7485b9dd5..12ae3468b 100644 --- a/crates/kra/src/lib.rs +++ b/crates/kra/src/lib.rs @@ -161,6 +161,16 @@ fn children<'a>(el: &'a El, container: &'static str, tag: &'static str) -> impl } fn node(el: &El, is_mask: bool, count: &mut usize) -> Result { + let mut out = node_fields(el, is_mask)?; + if !is_mask { + out.layers = layers(el, count)?; + out.masks = children(el, "masks", "mask").map(|m| node(m, true, count)).collect::>()?; + } + Ok(out) +} + +/// A node's own attributes, without its children. +fn node_fields(el: &El, is_mask: bool) -> Result { let owned = |name: &str| attr(el, name).map(str::to_string); let node_type = attr(el, "nodetype").unwrap_or("").to_string(); // Only the kinds whose pixels are read need sound numbers; a node an application will leave @@ -168,7 +178,7 @@ fn node(el: &El, is_mask: bool, count: &mut usize) -> Result { let read = matches!(node_type.as_str(), "paintlayer" | "grouplayer" | "transparencymask"); let lenient = |v: Result, default: f64| if read { v } else { Ok(v.unwrap_or(default)) }; let (x, y) = if is_mask && !read { (offset(el, "x").unwrap_or(0), offset(el, "y").unwrap_or(0)) } else { (offset(el, "x")?, offset(el, "y")?) }; - let mut out = Node { + let out = Node { node_type, name: owned("name"), visible: attr(el, "visible") != Some("0"), @@ -187,22 +197,39 @@ fn node(el: &El, is_mask: bool, count: &mut usize) -> Result { layers: Vec::new(), masks: Vec::new(), }; - if !is_mask { - out.layers = layers(el, count)?; - out.masks = children(el, "masks", "mask").map(|m| node(m, true, count)).collect::>()?; - } Ok(out) } -/// The `` of `el`. Recursion is bounded by [`MAX_XML_DEPTH`]. +/// The `` of `el`, walked with an explicit stack: groups can nest up to [`MAX_XML_DEPTH`] +/// deep, more than a small thread stack can recurse. fn layers(el: &El, count: &mut usize) -> Result> { + struct Frame<'a> { + node: Node, + rest: std::vec::IntoIter<&'a El>, + } + let mut root = children(el, "layers", "layer").collect::>().into_iter(); + let mut stack: Vec = Vec::new(); let mut out = Vec::new(); - for child in children(el, "layers", "layer") { + loop { + let next = match stack.last_mut() { + Some(frame) => frame.rest.next(), + None => root.next(), + }; + let Some(child) = next else { + let Some(frame) = stack.pop() else { break }; + match stack.last_mut() { + Some(parent) => parent.node.layers.push(frame.node), + None => out.push(frame.node), + } + continue; + }; *count += 1; if *count > MAX_NODES { return Err(invalid(format!("more than {MAX_NODES} layers"))); } - out.push(node(child, false, count)?); + let mut built = node_fields(child, false)?; + built.masks = children(child, "masks", "mask").map(|m| node(m, true, count)).collect::>()?; + stack.push(Frame { node: built, rest: children(child, "layers", "layer").collect::>().into_iter() }); } Ok(out) } diff --git a/crates/ora/src/lib.rs b/crates/ora/src/lib.rs index b2a91cc40..69d494d0d 100644 --- a/crates/ora/src/lib.rs +++ b/crates/ora/src/lib.rs @@ -189,28 +189,63 @@ fn props(el: &El) -> Result { }) } -/// The elements of `stack`. Recursion is bounded by [`MAX_XML_DEPTH`]. +/// The elements of `stack`, walked with an explicit stack: a stack can nest up to +/// [`MAX_XML_DEPTH`] deep, more than a small thread stack can recurse. fn stack(el: &El, count: &mut usize) -> Result> { + struct Frame<'a> { + props: Props, + isolation: Option, + children: Vec, + rest: std::vec::IntoIter<&'a El>, + } + let mut root = el.elements().collect::>().into_iter(); + let mut frames: Vec = Vec::new(); let mut out = Vec::new(); - for child in el.elements() { + loop { + let next = match frames.last_mut() { + Some(frame) => frame.rest.next(), + None => root.next(), + }; + let Some(child) = next else { + let Some(frame) = frames.pop() else { break }; + let node = Node::Stack(Stack { props: frame.props, isolation: frame.isolation, children: frame.children }); + match frames.last_mut() { + Some(parent) => parent.children.push(node), + None => out.push(node), + } + continue; + }; *count += 1; if *count > MAX_NODES { return Err(invalid(format!("more than {MAX_NODES} layers"))); } - out.push(match child.name.as_str() { - "layer" => Node::Layer(Layer { - props: props(child)?, - x: int(child, "x")?, - y: int(child, "y")?, - src: child.attr("src").map(|s| s.trim().to_string()), - }), - "stack" => Node::Stack(Stack { + match child.name.as_str() { + "layer" => { + let node = Node::Layer(Layer { + props: props(child)?, + x: int(child, "x")?, + y: int(child, "y")?, + src: child.attr("src").map(|s| s.trim().to_string()), + }); + match frames.last_mut() { + Some(parent) => parent.children.push(node), + None => out.push(node), + } + } + "stack" => frames.push(Frame { props: props(child)?, isolation: child.attr("isolation").map(|s| s.trim().to_string()), - children: stack(child, count)?, + children: Vec::new(), + rest: child.elements().collect::>().into_iter(), }), - other => Node::Other { tag: other.to_string(), name: child.attr("name").map(str::to_string) }, - }); + other => { + let node = Node::Other { tag: other.to_string(), name: child.attr("name").map(str::to_string) }; + match frames.last_mut() { + Some(parent) => parent.children.push(node), + None => out.push(node), + } + } + } } Ok(out) } From 6ef2334920b94fc91ab9f9fa37428138597c7499 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Thu, 12 Nov 2026 23:58:12 +0000 Subject: [PATCH 10/36] plotter: expect registration offset command --- crates/plotter/tests/golden.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/plotter/tests/golden.rs b/crates/plotter/tests/golden.rs index 26aaae554..4d5029010 100644 --- a/crates/plotter/tests/golden.rs +++ b/crates/plotter/tests/golden.rs @@ -135,7 +135,7 @@ fn silhouette_registration_marks() { let p = silhouette::encode(&pro, &j).unwrap(); let (head, rest) = p.split_at_registration(); let text = |c: &[Command]| c.iter().map(Command::to_ascii).collect::>().join(" "); - assert!(text(head).ends_with(" FC18,1,1 TB50,0 TB99 TB52,2 TB51,400 TB53,10 TB55,1 TB123,4600,3600,200,100")); + assert!(text(head).ends_with(" FC18,1,1 TB50,0 TB99 TB52,2 TB51,400 TB53,10 TB54,0,0 TB55,1 TB123,4600,3600,200,100")); // The design is shifted so the first mark is the origin. assert_eq!(text(rest), "M100,100 D100,300 M100,0 SO0"); assert_eq!(p.registration_wait, Some(head.len())); @@ -148,7 +148,7 @@ fn silhouette_registration_marks() { reg.origin_right_mm = 0.0; assert!(ascii(®).ends_with(" TB124,4600,3600,0,0")); reg.search = false; - assert_eq!(ascii(®), "TB50,0 TB99 TB52,2 TB51,400 TB53,10 TB55,1 TB23,4600,3600"); + assert_eq!(ascii(®), "TB50,0 TB99 TB52,2 TB51,400 TB53,10 TB54,0,0 TB55,1 TB23,4600,3600"); } #[test] From 3afade51630fcdb02662c761e0805bc6147bb0ac Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 00:13:31 -0500 Subject: [PATCH 11/36] Expose renderer-neutral SVG preflight --- crates/svg/src/lib.rs | 1 + crates/svg/src/preflight.rs | 91 +++++++++++++++++++++++++++++++++++++ crates/svg/src/tree.rs | 34 +++++++------- 3 files changed, 109 insertions(+), 17 deletions(-) create mode 100644 crates/svg/src/preflight.rs diff --git a/crates/svg/src/lib.rs b/crates/svg/src/lib.rs index fd6ca685e..982e4f82b 100644 --- a/crates/svg/src/lib.rs +++ b/crates/svg/src/lib.rs @@ -20,6 +20,7 @@ pub mod color; pub mod guard; pub mod paths; +pub mod preflight; pub mod sniff; pub mod svgz; #[cfg(feature = "tree")] diff --git a/crates/svg/src/preflight.rs b/crates/svg/src/preflight.rs new file mode 100644 index 000000000..3ab381ff3 --- /dev/null +++ b/crates/svg/src/preflight.rs @@ -0,0 +1,91 @@ +//! Decode and validate SVG/SVGZ text before a caller hands it to a renderer. +//! +//! This boundary has no dependency on an SVG renderer. It decodes the file's text encoding and +//! bounded gzip container, then checks entity expansion, attribute work, XML structure, node +//! count and nesting depth. + +use std::borrow::Cow; + +/// Limits for decoding and validating one SVG document. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Limits { + /// Bytes of source text, also the largest unpacked SVGZ. + pub bytes: usize, + /// Deepest XML element nesting (the root alone is 1). + pub depth: usize, + /// Total bytes entities may expand to. + pub entity_bytes: usize, + /// Refuse a doctype entity containing markup. + pub refuse_entity_markup: bool, + /// Attributes on one element. + pub attributes: usize, + /// Sum of squared attribute counts across the document. + pub attribute_work: usize, + /// Maximum XML nodes after entity expansion. + pub xml_nodes: u32, +} + +impl Default for Limits { + fn default() -> Self { + Limits { + bytes: 64 << 20, + depth: 256, + entity_bytes: crate::guard::MAX_ENTITY_EXPANSION, + refuse_entity_markup: false, + attributes: crate::guard::MAX_ATTRIBUTES, + attribute_work: crate::guard::MAX_ATTRIBUTE_WORK, + xml_nodes: 4_000_000, + } + } +} + +/// Why an SVG's text or XML could not be read within `Limits`. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum Error { + /// The bytes are too large, corrupt SVGZ, nested SVGZ, or invalid text encoding. + #[error(transparent)] + Text(#[from] crate::svgz::Error), + /// Entity references expand past the configured limit. + #[error("the SVG's entities expand to too much text")] + Entities, + /// A doctype entity contains markup and is refused by the caller's policy. + #[error("the SVG's document type declares an entity holding markup, which isn't supported")] + EntityMarkup, + /// XML element nesting exceeds the configured limit. + #[error("the SVG's elements are nested more than {0} levels deep")] + TooDeep(usize), + /// Attribute count or total attribute work exceeds the configured limit. + #[error("an element of the SVG has more than {0} attributes, or its elements have too many in all")] + TooWide(usize), + /// The XML is malformed or exceeds the node limit. + #[error("invalid SVG XML: {0}")] + Xml(String), +} + +/// Decodes SVG or SVGZ text and checks the XML against `limits`. +pub fn text<'a>(bytes: &'a [u8], limits: &Limits) -> Result, Error> { + let text = crate::svgz::text(bytes, limits.bytes)?; + parse_text(&text, limits)?; + Ok(text) +} + +/// Parses already-decoded SVG text and checks it against `limits`. +pub fn parse_text<'a>(svg: &'a str, limits: &Limits) -> Result, Error> { + if svg.len() > limits.bytes { + return Err(Error::Text(crate::svgz::Error::TooLarge(limits.bytes))); + } + if limits.refuse_entity_markup && crate::guard::entity_holds_markup(svg) { + return Err(Error::EntityMarkup); + } + if crate::guard::entities_expand_past(svg, limits.entity_bytes) { + return Err(Error::Entities); + } + if crate::guard::too_wide(svg, limits.attributes, limits.attribute_work) { + return Err(Error::TooWide(limits.attributes)); + } + if crate::guard::deeper_than(svg, limits.depth) { + return Err(Error::TooDeep(limits.depth)); + } + let options = roxmltree::ParsingOptions { allow_dtd: true, nodes_limit: limits.xml_nodes, ..Default::default() }; + roxmltree::Document::parse_with_options(svg, options).map_err(|e| Error::Xml(e.to_string())) +} diff --git a/crates/svg/src/tree.rs b/crates/svg/src/tree.rs index 6c79c33a7..dd14be619 100644 --- a/crates/svg/src/tree.rs +++ b/crates/svg/src/tree.rs @@ -125,23 +125,23 @@ pub fn parse_text(svg: &str, limits: &Limits, options: &usvg::Options<'_>) -> Re /// Every check on SVG text, cheapest first; the XML it parses to on the way. fn checked<'s>(svg: &'s str, limits: &Limits) -> Result, Error> { - if svg.len() > limits.bytes { - return Err(Error::Text(svgz::Error::TooLarge(limits.bytes))); - } - if limits.refuse_entity_markup && guard::entity_holds_markup(svg) { - return Err(Error::EntityMarkup); - } - if guard::entities_expand_past(svg, limits.entity_bytes) { - return Err(Error::Entities); - } - if guard::too_wide(svg, limits.attributes, limits.attribute_work) { - return Err(Error::TooWide(limits.attributes)); - } - if guard::deeper_than(svg, limits.depth) { - return Err(Error::TooDeep(limits.depth)); - } - let options = roxmltree::ParsingOptions { allow_dtd: true, nodes_limit: limits.xml_nodes, ..Default::default() }; - let xml = roxmltree::Document::parse_with_options(svg, options).map_err(|e| Error::Xml(e.to_string()))?; + let preflight_limits = crate::preflight::Limits { + bytes: limits.bytes, + depth: limits.depth, + entity_bytes: limits.entity_bytes, + refuse_entity_markup: limits.refuse_entity_markup, + attributes: limits.attributes, + attribute_work: limits.attribute_work, + xml_nodes: limits.xml_nodes, + }; + let xml = crate::preflight::parse_text(svg, &preflight_limits).map_err(|e| match e { + crate::preflight::Error::Text(e) => Error::Text(e), + crate::preflight::Error::Entities => Error::Entities, + crate::preflight::Error::EntityMarkup => Error::EntityMarkup, + crate::preflight::Error::TooDeep(depth) => Error::TooDeep(depth), + crate::preflight::Error::TooWide(attributes) => Error::TooWide(attributes), + crate::preflight::Error::Xml(msg) => Error::Xml(msg), + })?; if height(&xml, limits.depth)?.is_none() { return Err(Error::TooDeep(limits.depth)); } From 79c2003ac9c665bfd6a0dbc00c256d42cc3ad1e3 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 00:29:48 -0500 Subject: [PATCH 12/36] Add shared file format detection --- Cargo.lock | 7 +++ Cargo.toml | 1 + crates/file-formats/Cargo.toml | 14 +++++ crates/file-formats/src/lib.rs | 93 ++++++++++++++++++++++++++++++++++ 4 files changed, 115 insertions(+) create mode 100644 crates/file-formats/Cargo.toml create mode 100644 crates/file-formats/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index cf0e0bd47..5e004a7ce 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1454,6 +1454,13 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "craft-file-formats" +version = "0.1.0" +dependencies = [ + "craft-eps", +] + [[package]] name = "craft-flags" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 629bab64c..4af6f586a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -42,6 +42,7 @@ craft-docx = { path = "crates/docx" } craft-dxf = { path = "crates/dxf" } craft-drawingml = { path = "crates/drawingml" } craft-eps = { path = "crates/eps" } +craft-file-formats = { path = "crates/file-formats" } craft-doc = { path = "crates/doc" } craft-docpath = { path = "crates/docpath" } craft-dpx = { path = "crates/dpx" } diff --git a/crates/file-formats/Cargo.toml b/crates/file-formats/Cargo.toml new file mode 100644 index 000000000..0a5be5e76 --- /dev/null +++ b/crates/file-formats/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "craft-file-formats" +description = "Shared bounded file signature and extension resolution for Craft applications" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true + +[dependencies] +craft-eps = { workspace = true } + +[lints] +workspace = true diff --git a/crates/file-formats/src/lib.rs b/crates/file-formats/src/lib.rs new file mode 100644 index 000000000..b51fc399c --- /dev/null +++ b/crates/file-formats/src/lib.rs @@ -0,0 +1,93 @@ +//! Bounded file kind detection and extension resolution shared by Craft applications. +//! +//! This crate identifies a format; it does not parse, validate, or decode the file. +//! Callers remain responsible for choosing which identified formats their product supports. +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +#![forbid(unsafe_code)] + +/// A supported file family, independent of the application model that consumes it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Format { + Pdf, + Eps, + PostScript, + Illustrator, + Dxf, + Svg, + Raster, + Studio, + StencilCraft, +} + +/// Resolve a name and file bytes to a format. Strong signatures take precedence over +/// extension hints. Only format-specific bounded probes and extension tables live here. +pub fn detect(name: &str, bytes: &[u8]) -> Option { + if let Some(found) = craft_eps::sniff(bytes) { + return Some(match (found.kind, found.illustrator) { + (_, true) => Format::Illustrator, + (craft_eps::Kind::Pdf, _) => Format::Pdf, + (craft_eps::Kind::Eps, _) => Format::Eps, + (craft_eps::Kind::PostScript, _) => Format::PostScript, + }); + } + if raster_signature(bytes) { + return Some(Format::Raster); + } + if bytes.starts_with(b"silhouette04;") || bytes.starts_with(b"silhouette05;") { + return Some(Format::Studio); + } + if let Some(ext) = extension(name) { + if let Some(format) = from_extension(ext) { + return Some(format); + } + } + let head = bytes.get(..4096).unwrap_or(bytes); + let xml = head.trim_ascii_start(); + if xml.starts_with(b" Option { + match ext.trim_start_matches('.').to_ascii_lowercase().as_str() { + "pdf" => Some(Format::Pdf), + "eps" => Some(Format::Eps), + "ps" => Some(Format::PostScript), + "ai" => Some(Format::Illustrator), + "dxf" => Some(Format::Dxf), + "svg" | "svgz" => Some(Format::Svg), + "png" | "jpg" | "jpeg" | "bmp" | "gif" | "tif" | "tiff" | "webp" | "heic" | "heif" => Some(Format::Raster), + "studio" | "studio3" => Some(Format::Studio), + "stencilcraft" => Some(Format::StencilCraft), + _ => None, + } +} + +fn extension(name: &str) -> Option<&str> { + Some(name.rsplit(['/', '\\']).next()?.rsplit_once('.')?.1) +} + +fn dxf_ascii_prefix(bytes: &[u8]) -> bool { + let text = String::from_utf8_lossy(bytes); + let mut lines = text.lines().map(str::trim); + matches!((lines.next(), lines.next(), lines.next(), lines.next()), (Some("0"), Some("SECTION"), Some("2"), Some(section)) if matches!(section, "HEADER" | "CLASSES" | "TABLES" | "BLOCKS" | "ENTITIES" | "OBJECTS")) +} + +fn raster_signature(bytes: &[u8]) -> bool { + let head = bytes.get(..16).unwrap_or(bytes); + head.starts_with(b"\x89PNG\r\n\x1a\n") + || head.starts_with(b"\xff\xd8\xff") + || head.starts_with(b"BM") + || head.starts_with(b"GIF87a") + || head.starts_with(b"GIF89a") + || head.starts_with(b"II*\0") + || head.starts_with(b"MM\0*") + || head.starts_with(b"II+\0") + || head.starts_with(b"MM\0+") + || (head.starts_with(b"RIFF") && head.get(8..12) == Some(b"WEBP")) +} From 8d3c753b5e7422d4484f5550b6e84bc93f2ab280 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 00:34:11 -0500 Subject: [PATCH 13/36] Recognize G-code filename extensions --- crates/file-formats/src/lib.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/file-formats/src/lib.rs b/crates/file-formats/src/lib.rs index b51fc399c..c5a2164de 100644 --- a/crates/file-formats/src/lib.rs +++ b/crates/file-formats/src/lib.rs @@ -15,6 +15,7 @@ pub enum Format { Dxf, Svg, Raster, + Gcode, Studio, StencilCraft, } @@ -64,6 +65,7 @@ pub fn from_extension(ext: &str) -> Option { "png" | "jpg" | "jpeg" | "bmp" | "gif" | "tif" | "tiff" | "webp" | "heic" | "heif" => Some(Format::Raster), "studio" | "studio3" => Some(Format::Studio), "stencilcraft" => Some(Format::StencilCraft), + "gcode" | "nc" => Some(Format::Gcode), _ => None, } } From c1a1b976d0decf496d132bf71eecaec0de198675 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 00:34:48 -0500 Subject: [PATCH 14/36] Resolve raster subformats centrally --- crates/file-formats/src/lib.rs | 51 ++++++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 15 deletions(-) diff --git a/crates/file-formats/src/lib.rs b/crates/file-formats/src/lib.rs index c5a2164de..29bf92a3f 100644 --- a/crates/file-formats/src/lib.rs +++ b/crates/file-formats/src/lib.rs @@ -14,7 +14,13 @@ pub enum Format { Illustrator, Dxf, Svg, - Raster, + Png, + Jpeg, + Bmp, + Gif, + Tiff, + WebP, + Heif, Gcode, Studio, StencilCraft, @@ -31,8 +37,8 @@ pub fn detect(name: &str, bytes: &[u8]) -> Option { (craft_eps::Kind::PostScript, _) => Format::PostScript, }); } - if raster_signature(bytes) { - return Some(Format::Raster); + if let Some(raster) = raster_signature(bytes) { + return Some(raster); } if bytes.starts_with(b"silhouette04;") || bytes.starts_with(b"silhouette05;") { return Some(Format::Studio); @@ -62,7 +68,13 @@ pub fn from_extension(ext: &str) -> Option { "ai" => Some(Format::Illustrator), "dxf" => Some(Format::Dxf), "svg" | "svgz" => Some(Format::Svg), - "png" | "jpg" | "jpeg" | "bmp" | "gif" | "tif" | "tiff" | "webp" | "heic" | "heif" => Some(Format::Raster), + "png" => Some(Format::Png), + "jpg" | "jpeg" => Some(Format::Jpeg), + "bmp" => Some(Format::Bmp), + "gif" => Some(Format::Gif), + "tif" | "tiff" => Some(Format::Tiff), + "webp" => Some(Format::WebP), + "heic" | "heif" => Some(Format::Heif), "studio" | "studio3" => Some(Format::Studio), "stencilcraft" => Some(Format::StencilCraft), "gcode" | "nc" => Some(Format::Gcode), @@ -80,16 +92,25 @@ fn dxf_ascii_prefix(bytes: &[u8]) -> bool { matches!((lines.next(), lines.next(), lines.next(), lines.next()), (Some("0"), Some("SECTION"), Some("2"), Some(section)) if matches!(section, "HEADER" | "CLASSES" | "TABLES" | "BLOCKS" | "ENTITIES" | "OBJECTS")) } -fn raster_signature(bytes: &[u8]) -> bool { +fn raster_signature(bytes: &[u8]) -> Option { let head = bytes.get(..16).unwrap_or(bytes); - head.starts_with(b"\x89PNG\r\n\x1a\n") - || head.starts_with(b"\xff\xd8\xff") - || head.starts_with(b"BM") - || head.starts_with(b"GIF87a") - || head.starts_with(b"GIF89a") - || head.starts_with(b"II*\0") - || head.starts_with(b"MM\0*") - || head.starts_with(b"II+\0") - || head.starts_with(b"MM\0+") - || (head.starts_with(b"RIFF") && head.get(8..12) == Some(b"WEBP")) + if head.starts_with(b"\x89PNG\r\n\x1a\n") { + return Some(Format::Png); + } + if head.starts_with(b"\xff\xd8\xff") { + return Some(Format::Jpeg); + } + if head.starts_with(b"BM") { + return Some(Format::Bmp); + } + if head.starts_with(b"GIF87a") || head.starts_with(b"GIF89a") { + return Some(Format::Gif); + } + if head.starts_with(b"II*\0") || head.starts_with(b"MM\0*") || head.starts_with(b"II+\0") || head.starts_with(b"MM\0+") { + return Some(Format::Tiff); + } + if head.starts_with(b"RIFF") && head.get(8..12) == Some(b"WEBP") { + return Some(Format::WebP); + } + None } From ba24fe2435adb3c81ad6057faf36365fbba736cf Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 00:39:37 -0500 Subject: [PATCH 15/36] Move bounded EPS rectangle parsing to craft-eps --- crates/eps/src/lib.rs | 2 + crates/eps/src/rectangles.rs | 136 +++++++++++++++++++++++++++++++++++ 2 files changed, 138 insertions(+) create mode 100644 crates/eps/src/rectangles.rs diff --git a/crates/eps/src/lib.rs b/crates/eps/src/lib.rs index acffdfcf2..e48532349 100644 --- a/crates/eps/src/lib.rs +++ b/crates/eps/src/lib.rs @@ -19,10 +19,12 @@ pub mod ascii85; mod container; mod dsc; mod preview; +mod rectangles; pub use container::{DOS_HEADER, DOS_MAGIC, Sections, sections, wrap}; pub use dsc::{Comments, MAX_HEADER, MAX_TRAILER}; pub use preview::{Bitmap, MAX_EPSI, epsi}; +pub use rectangles::{MAX_SIMPLE_RECTANGLE_SAVE_DEPTH, MAX_SIMPLE_RECTANGLE_TOKENS, MAX_SIMPLE_RECTANGLES, Rectangle, simple_rectangles}; use std::fmt; diff --git a/crates/eps/src/rectangles.rs b/crates/eps/src/rectangles.rs new file mode 100644 index 000000000..d1c871762 --- /dev/null +++ b/crates/eps/src/rectangles.rs @@ -0,0 +1,136 @@ +//! A bounded subset of PostScript used for simple EPS previews. + +use std::collections::HashMap; + +/// A solid rectangle drawn by a simple EPS program. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Rectangle { + /// Left edge in PostScript points. + pub x: f64, + /// Bottom edge in PostScript points. + pub y: f64, + /// Width in PostScript points. + pub width: f64, + /// Height in PostScript points. + pub height: f64, + /// Straight sRGB RGBA. + pub color: [u8; 4], +} + +pub const MAX_SIMPLE_RECTANGLES: usize = 200_000; +pub const MAX_SIMPLE_RECTANGLE_TOKENS: usize = 2_000_000; +pub const MAX_SIMPLE_RECTANGLE_SAVE_DEPTH: usize = 64; + +/// Simple PostScript tokens without comments, with braces kept as individual tokens. +fn tokens(text: &str) -> impl Iterator { + text.split(['\n', '\r']).flat_map(|line| { + line.split('%').next().unwrap_or("").split_whitespace().flat_map(|word| word.split_inclusive(['{', '}'])).flat_map(|piece| { + let (body, brace) = match piece.char_indices().last() { + Some((at, '{' | '}')) => (piece.get(..at).unwrap_or(""), piece.get(at..)), + _ => (piece, None), + }; + [Some(body).filter(|body| !body.is_empty()), brace].into_iter().flatten() + }) + }) +} + +/// Read solid rectangle fills and their sRGB colours from the restricted PostScript subset used +/// by simple EPS previews. None means the program uses an unsupported operator or exceeds a +/// fixed work bound. The function never executes PostScript. +pub fn simple_rectangles(ps: &[u8]) -> Option> { + let text = std::str::from_utf8(ps).ok()?; + let mut aliases: HashMap = HashMap::new(); + let mut stack: Vec = Vec::new(); + let mut color = [0u8, 0, 0, 255]; + let mut saved: Vec<[u8; 4]> = Vec::new(); + let mut rectangles = Vec::new(); + let mut input = tokens(text); + let mut count = 0usize; + while let Some(token) = input.next() { + count += 1; + if count > MAX_SIMPLE_RECTANGLE_TOKENS { + return None; + } + if let Ok(value) = token.parse::() { + if !value.is_finite() { + return None; + } + stack.push(value); + continue; + } + if let Some(name) = token.strip_prefix('/') { + let mut body: Vec<&str> = Vec::new(); + while body.len() < 5 { + count += 1; + if count > MAX_SIMPLE_RECTANGLE_TOKENS { + return None; + } + match input.next() { + Some("def") | None => break, + Some(token) => body.push(token), + } + } + let op = match body.as_slice() { + ["{", op, "}"] | ["{", op, "}", _] => (*op).to_string(), + _ => return None, + }; + aliases.insert(name.to_string(), op); + continue; + } + let op = aliases.get(token).map_or(token, String::as_str); + let mut pop = |n: usize| -> Option> { (stack.len() >= n).then(|| stack.split_off(stack.len() - n)) }; + match op { + "setcmykcolor" => { + let values = pop(4)?; + let black = values[3].clamp(0.0, 1.0); + let channel = |value: f64| ((1.0 - value.clamp(0.0, 1.0)) * (1.0 - black) * 255.0).round() as u8; + color = [channel(values[0]), channel(values[1]), channel(values[2]), 255]; + } + "setrgbcolor" => { + let values = pop(3)?; + let channel = |value: f64| (value.clamp(0.0, 1.0) * 255.0).round() as u8; + color = [channel(values[0]), channel(values[1]), channel(values[2]), 255]; + } + "setgray" => { + let values = pop(1)?; + let gray = (values[0].clamp(0.0, 1.0) * 255.0).round() as u8; + color = [gray, gray, gray, 255]; + } + "rectfill" => { + let values = pop(4)?; + if rectangles.len() >= MAX_SIMPLE_RECTANGLES { + return None; + } + let x1 = values[0] + values[2]; + let y1 = values[1] + values[3]; + if !x1.is_finite() || !y1.is_finite() { + return None; + } + rectangles.push(Rectangle { + x: values[0].min(x1), + y: values[1].min(y1), + width: (x1 - values[0]).abs(), + height: (y1 - values[1]).abs(), + color, + }); + } + "gsave" | "save" => { + if saved.len() >= MAX_SIMPLE_RECTANGLE_SAVE_DEPTH { + return None; + } + saved.push(color); + } + "grestore" | "restore" => { + if let Some(previous) = saved.pop() { + color = previous; + } + } + "pop" => { + pop(1)?; + } + "bind" | "newpath" | "showpage" | "end" | "begin" => {} + _ => return None, + } + } + (!rectangles.is_empty()).then_some(rectangles) +} From d8cc88a7720755e0054038ebc0e152b2e0dcfa00 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 00:39:31 -0500 Subject: [PATCH 16/36] Add image document formats to shared registry --- crates/file-formats/src/lib.rs | 35 ++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/crates/file-formats/src/lib.rs b/crates/file-formats/src/lib.rs index 29bf92a3f..9d3a3c438 100644 --- a/crates/file-formats/src/lib.rs +++ b/crates/file-formats/src/lib.rs @@ -21,6 +21,14 @@ pub enum Format { Tiff, WebP, Heif, + Psd, + Affinity, + Ora, + Kra, + Xcf, + Pdn, + Raw, + Pcraft, Gcode, Studio, StencilCraft, @@ -37,6 +45,18 @@ pub fn detect(name: &str, bytes: &[u8]) -> Option { (craft_eps::Kind::PostScript, _) => Format::PostScript, }); } + if bytes.starts_with(b"8BPS") { + return Some(Format::Psd); + } + if bytes.starts_with(b"PDN3") { + return Some(Format::Pdn); + } + if bytes.starts_with(b"gimp xcf ") || gzipped_xcf(name, bytes) { + return Some(Format::Xcf); + } + if bytes.starts_with(b"\0\xffKA") { + return Some(Format::Affinity); + } if let Some(raster) = raster_signature(bytes) { return Some(raster); } @@ -75,6 +95,16 @@ pub fn from_extension(ext: &str) -> Option { "tif" | "tiff" => Some(Format::Tiff), "webp" => Some(Format::WebP), "heic" | "heif" => Some(Format::Heif), + "psd" | "psb" => Some(Format::Psd), + "afdesign" | "afphoto" | "afpub" => Some(Format::Affinity), + "ora" => Some(Format::Ora), + "kra" => Some(Format::Kra), + "xcf" => Some(Format::Xcf), + "pdn" => Some(Format::Pdn), + "dng" | "3fr" | "ari" | "arw" | "bay" | "cap" | "cr2" | "cr3" | "dcs" | "dcr" | "drf" | "eip" | "erf" | "fff" | "gpr" | "iiq" | "k25" + | "kdc" | "mdc" | "mef" | "mos" | "mrw" | "nef" | "nrw" | "obm" | "orf" | "pef" | "ptx" | "pxn" | "r3d" | "raf" | "raw" | "rwl" | "rw2" + | "rwz" | "sr2" | "srf" | "sti" | "x3f" => Some(Format::Raw), + "pcraft" => Some(Format::Pcraft), "studio" | "studio3" => Some(Format::Studio), "stencilcraft" => Some(Format::StencilCraft), "gcode" | "nc" => Some(Format::Gcode), @@ -86,6 +116,11 @@ fn extension(name: &str) -> Option<&str> { Some(name.rsplit(['/', '\\']).next()?.rsplit_once('.')?.1) } +fn gzipped_xcf(name: &str, bytes: &[u8]) -> bool { + let lower = name.rsplit(['/', '\\']).next().unwrap_or(name).to_ascii_lowercase(); + bytes.starts_with(&[0x1f, 0x8b]) && (lower.ends_with(".xcf.gz") || lower.ends_with(".xcfgz")) +} + fn dxf_ascii_prefix(bytes: &[u8]) -> bool { let text = String::from_utf8_lossy(bytes); let mut lines = text.lines().map(str::trim); From fc9ecb40e72bb3d539ae16d158b84ebed4c65024 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 01:22:58 -0500 Subject: [PATCH 17/36] Extend shared file-format registry --- Cargo.lock | 1 + crates/file-formats/Cargo.toml | 1 + crates/file-formats/src/lib.rs | 39 +++++++++++++++++++++++++++++----- 3 files changed, 36 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5e004a7ce..87ef135e7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1459,6 +1459,7 @@ name = "craft-file-formats" version = "0.1.0" dependencies = [ "craft-eps", + "craft-svg", ] [[package]] diff --git a/crates/file-formats/Cargo.toml b/crates/file-formats/Cargo.toml index 0a5be5e76..00d3b388a 100644 --- a/crates/file-formats/Cargo.toml +++ b/crates/file-formats/Cargo.toml @@ -9,6 +9,7 @@ rust-version.workspace = true [dependencies] craft-eps = { workspace = true } +craft-svg = { workspace = true } [lints] workspace = true diff --git a/crates/file-formats/src/lib.rs b/crates/file-formats/src/lib.rs index 9d3a3c438..9df7fe16a 100644 --- a/crates/file-formats/src/lib.rs +++ b/crates/file-formats/src/lib.rs @@ -14,6 +14,8 @@ pub enum Format { Illustrator, Dxf, Svg, + /// Gzip-compressed SVG. Kept distinct so writers do not place plain SVG under `.svgz`. + Svgz, Png, Jpeg, Bmp, @@ -29,6 +31,13 @@ pub enum Format { Pdn, Raw, Pcraft, + Idml, + DesignCraft, + Docx, + Rtf, + Xlsx, + TaggedText, + Pptx, Gcode, Studio, StencilCraft, @@ -63,16 +72,29 @@ pub fn detect(name: &str, bytes: &[u8]) -> Option { if bytes.starts_with(b"silhouette04;") || bytes.starts_with(b"silhouette05;") { return Some(Format::Studio); } + let tagged = bytes.strip_prefix(b"\xef\xbb\xbf").unwrap_or(bytes).trim_ascii_start(); + if tagged.starts_with(b" Option { "ps" => Some(Format::PostScript), "ai" => Some(Format::Illustrator), "dxf" => Some(Format::Dxf), - "svg" | "svgz" => Some(Format::Svg), + "svg" => Some(Format::Svg), + "svgz" => Some(Format::Svgz), "png" => Some(Format::Png), "jpg" | "jpeg" => Some(Format::Jpeg), "bmp" => Some(Format::Bmp), @@ -105,6 +128,12 @@ pub fn from_extension(ext: &str) -> Option { | "kdc" | "mdc" | "mef" | "mos" | "mrw" | "nef" | "nrw" | "obm" | "orf" | "pef" | "ptx" | "pxn" | "r3d" | "raf" | "raw" | "rwl" | "rw2" | "rwz" | "sr2" | "srf" | "sti" | "x3f" => Some(Format::Raw), "pcraft" => Some(Format::Pcraft), + "idml" => Some(Format::Idml), + "designcraft" | "dcdoc" => Some(Format::DesignCraft), + "docx" => Some(Format::Docx), + "rtf" => Some(Format::Rtf), + "xlsx" => Some(Format::Xlsx), + "pptx" => Some(Format::Pptx), "studio" | "studio3" => Some(Format::Studio), "stencilcraft" => Some(Format::StencilCraft), "gcode" | "nc" => Some(Format::Gcode), From 32529b94e8bfcc2febc08ef9d00dc97187e8e7e2 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 01:24:40 -0500 Subject: [PATCH 18/36] Resolve plain text format extensions --- crates/file-formats/src/lib.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/file-formats/src/lib.rs b/crates/file-formats/src/lib.rs index 9df7fe16a..6029757de 100644 --- a/crates/file-formats/src/lib.rs +++ b/crates/file-formats/src/lib.rs @@ -38,6 +38,7 @@ pub enum Format { Xlsx, TaggedText, Pptx, + PlainText, Gcode, Studio, StencilCraft, @@ -134,6 +135,7 @@ pub fn from_extension(ext: &str) -> Option { "rtf" => Some(Format::Rtf), "xlsx" => Some(Format::Xlsx), "pptx" => Some(Format::Pptx), + "txt" | "text" | "md" => Some(Format::PlainText), "studio" | "studio3" => Some(Format::Studio), "stencilcraft" => Some(Format::StencilCraft), "gcode" | "nc" => Some(Format::Gcode), From 1f9c271f7d0924a50b84032dcde90b259ae3ccf9 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 01:26:09 -0500 Subject: [PATCH 19/36] Recognize legacy Word extension in shared registry --- crates/file-formats/src/lib.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/file-formats/src/lib.rs b/crates/file-formats/src/lib.rs index 6029757de..3c6001673 100644 --- a/crates/file-formats/src/lib.rs +++ b/crates/file-formats/src/lib.rs @@ -34,6 +34,8 @@ pub enum Format { Idml, DesignCraft, Docx, + /// Legacy OLE Compound File Word document; identified by its extension only. + LegacyDoc, Rtf, Xlsx, TaggedText, @@ -132,6 +134,7 @@ pub fn from_extension(ext: &str) -> Option { "idml" => Some(Format::Idml), "designcraft" | "dcdoc" => Some(Format::DesignCraft), "docx" => Some(Format::Docx), + "doc" => Some(Format::LegacyDoc), "rtf" => Some(Format::Rtf), "xlsx" => Some(Format::Xlsx), "pptx" => Some(Format::Pptx), From 6ba1ca33ca21c33de55ef155141db08ca6048e8d Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 01:27:11 -0500 Subject: [PATCH 20/36] Recognize InDesign files in shared registry --- crates/file-formats/src/lib.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/crates/file-formats/src/lib.rs b/crates/file-formats/src/lib.rs index 3c6001673..1e45ad447 100644 --- a/crates/file-formats/src/lib.rs +++ b/crates/file-formats/src/lib.rs @@ -33,6 +33,7 @@ pub enum Format { Pcraft, Idml, DesignCraft, + Indd, Docx, /// Legacy OLE Compound File Word document; identified by its extension only. LegacyDoc, @@ -49,6 +50,9 @@ pub enum Format { /// Resolve a name and file bytes to a format. Strong signatures take precedence over /// extension hints. Only format-specific bounded probes and extension tables live here. pub fn detect(name: &str, bytes: &[u8]) -> Option { + if bytes.starts_with(&INDD_SIGNATURE) { + return Some(Format::Indd); + } if let Some(found) = craft_eps::sniff(bytes) { return Some(match (found.kind, found.illustrator) { (_, true) => Format::Illustrator, @@ -133,6 +137,7 @@ pub fn from_extension(ext: &str) -> Option { "pcraft" => Some(Format::Pcraft), "idml" => Some(Format::Idml), "designcraft" | "dcdoc" => Some(Format::DesignCraft), + "indd" | "indt" => Some(Format::Indd), "docx" => Some(Format::Docx), "doc" => Some(Format::LegacyDoc), "rtf" => Some(Format::Rtf), @@ -146,6 +151,8 @@ pub fn from_extension(ext: &str) -> Option { } } +const INDD_SIGNATURE: [u8; 16] = [0x06, 0x06, 0xED, 0xF5, 0xD8, 0x1D, 0x46, 0xE5, 0xBD, 0x31, 0xEF, 0xE7, 0xFE, 0x74, 0xB7, 0x1D]; + fn extension(name: &str) -> Option<&str> { Some(name.rsplit(['/', '\\']).next()?.rsplit_once('.')?.1) } From 96c848798408370cba30b9930a89b6a73d1b9aa8 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 01:30:54 -0500 Subject: [PATCH 21/36] Add shared InDesign conversion facade --- Cargo.lock | 12 ++++++++++++ Cargo.toml | 2 ++ crates/indd/Cargo.toml | 14 ++++++++++++++ crates/indd/src/lib.rs | 16 ++++++++++++++++ 4 files changed, 44 insertions(+) create mode 100644 crates/indd/Cargo.toml create mode 100644 crates/indd/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 174984c1d..03a7e15f9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1638,6 +1638,13 @@ dependencies = [ "zip", ] +[[package]] +name = "craft-indd" +version = "0.1.0" +dependencies = [ + "indd", +] + [[package]] name = "craft-jmap" version = "0.1.0" @@ -4738,6 +4745,11 @@ version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "65b27460c2c92b037f3f94c538ed9a3342f3fdf923606781629ccb35f82d042a" +[[package]] +name = "indd" +version = "0.1.0" +source = "git+https://github.com/storytold/indd-utils?rev=7f3060e5bce78d21ea70e0d5eacb96edce15dfbe#7f3060e5bce78d21ea70e0d5eacb96edce15dfbe" + [[package]] name = "indexmap" version = "2.14.2" diff --git a/Cargo.toml b/Cargo.toml index 4735dba39..bc06a403b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -65,6 +65,7 @@ craft-fig = { path = "crates/fig" } craft-fonts-build = { path = "crates/fonts-build" } craft-i18n = { path = "crates/i18n" } craft-idml = { path = "crates/idml" } +craft-indd = { path = "crates/indd" } craft-jmap = { path = "crates/jmap" } craft-ical = { path = "crates/ical" } craft-jpeg = { path = "crates/jpeg" } @@ -126,6 +127,7 @@ craft-vectordrawable = { path = "crates/vectordrawable" } craft-xls = { path = "crates/xls" } craft-wordmodel = { path = "crates/wordmodel" } craft-xlsx = { path = "crates/xlsx" } +indd = { git = "https://github.com/storytold/indd-utils", rev = "7f3060e5bce78d21ea70e0d5eacb96edce15dfbe" } craft-xcf = { path = "crates/xcf" } craft-xml = { path = "crates/xml" } craft-zip = { path = "crates/zip" } diff --git a/crates/indd/Cargo.toml b/crates/indd/Cargo.toml new file mode 100644 index 000000000..874ccb0d6 --- /dev/null +++ b/crates/indd/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "craft-indd" +description = "Shared InDesign document to IDML conversion facade for Craft applications" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true + +[dependencies] +indd = { workspace = true } + +[lints] +workspace = true diff --git a/crates/indd/src/lib.rs b/crates/indd/src/lib.rs new file mode 100644 index 000000000..2cce05c0d --- /dev/null +++ b/crates/indd/src/lib.rs @@ -0,0 +1,16 @@ +//! Shared InDesign document conversion boundary. +//! +//! InDesign's binary document model is converted to IDML here. Applications can import the +//! resulting package through [`craft_idml`] while retaining only their own model and UI mapping. + +pub use indd::{Conversion, Error, Preview, Warning}; + +/// Whether `bytes` begins with the fixed signature of an InDesign document. +pub fn is_indd(bytes: &[u8]) -> bool { + bytes.starts_with(&indd::header::SIGNATURE) +} + +/// Convert an InDesign document to IDML. `name` is the file name stored in the package. +pub fn convert(bytes: &[u8], name: &str) -> Result { + indd::convert(bytes, name) +} From 47bf5650b0d503b97bfb5e841746c4957897d11f Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 01:31:26 -0500 Subject: [PATCH 22/36] Expose shared InDesign signature --- crates/indd/src/lib.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/indd/src/lib.rs b/crates/indd/src/lib.rs index 2cce05c0d..657b417fc 100644 --- a/crates/indd/src/lib.rs +++ b/crates/indd/src/lib.rs @@ -3,11 +3,12 @@ //! InDesign's binary document model is converted to IDML here. Applications can import the //! resulting package through [`craft_idml`] while retaining only their own model and UI mapping. +pub use indd::header::SIGNATURE; pub use indd::{Conversion, Error, Preview, Warning}; /// Whether `bytes` begins with the fixed signature of an InDesign document. pub fn is_indd(bytes: &[u8]) -> bool { - bytes.starts_with(&indd::header::SIGNATURE) + bytes.starts_with(&SIGNATURE) } /// Convert an InDesign document to IDML. `name` is the file name stored in the package. From b4ada28f19067d4693e902e0e5f32eb292241e3a Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 01:37:48 -0500 Subject: [PATCH 23/36] Fix format registry clippy and add ASE support --- crates/file-formats/src/lib.rs | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/crates/file-formats/src/lib.rs b/crates/file-formats/src/lib.rs index 1e45ad447..c6ac77540 100644 --- a/crates/file-formats/src/lib.rs +++ b/crates/file-formats/src/lib.rs @@ -23,6 +23,7 @@ pub enum Format { Tiff, WebP, Heif, + Ase, Psd, Affinity, Ora, @@ -64,6 +65,9 @@ pub fn detect(name: &str, bytes: &[u8]) -> Option { if bytes.starts_with(b"8BPS") { return Some(Format::Psd); } + if bytes.starts_with(b"ASEF") { + return Some(Format::Ase); + } if bytes.starts_with(b"PDN3") { return Some(Format::Pdn); } @@ -86,20 +90,19 @@ pub fn detect(name: &str, bytes: &[u8]) -> Option { if bytes.trim_ascii_start().starts_with(b"{\\rtf") { return Some(Format::Rtf); } - if craft_svg::sniff::is_svgz(bytes) { - if let Ok(text) = craft_svg::svgz::text(bytes, 64 * 1024 * 1024) - && craft_svg::sniff::is_svg(text.as_bytes()) - { - return Some(Format::Svgz); - } + if craft_svg::sniff::is_svgz(bytes) + && let Ok(text) = craft_svg::svgz::text(bytes, 64 * 1024 * 1024) + && craft_svg::sniff::is_svg(text.as_bytes()) + { + return Some(Format::Svgz); } if craft_svg::sniff::is_svg(bytes) { return Some(Format::Svg); } - if let Some(ext) = extension(name) { - if let Some(format) = from_extension(ext) { - return Some(format); - } + if let Some(ext) = extension(name) + && let Some(format) = from_extension(ext) + { + return Some(format); } let head = bytes.get(..4096).unwrap_or(bytes); if dxf_ascii_prefix(head) || head.starts_with(b"AutoCAD Binary DXF\r\n\x1a\0") { @@ -125,6 +128,7 @@ pub fn from_extension(ext: &str) -> Option { "tif" | "tiff" => Some(Format::Tiff), "webp" => Some(Format::WebP), "heic" | "heif" => Some(Format::Heif), + "ase" => Some(Format::Ase), "psd" | "psb" => Some(Format::Psd), "afdesign" | "afphoto" | "afpub" => Some(Format::Affinity), "ora" => Some(Format::Ora), From fccc0082711b8ae4e19824dd20339a8473d47309 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 01:44:49 -0500 Subject: [PATCH 24/36] Detect DesignCraft and IDML document content --- Cargo.lock | 4 +++ crates/file-formats/Cargo.toml | 4 +++ crates/file-formats/src/lib.rs | 66 ++++++++++++++++++++++++++++++++++ 3 files changed, 74 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 87ef135e7..8f13c4412 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1459,7 +1459,11 @@ name = "craft-file-formats" version = "0.1.0" dependencies = [ "craft-eps", + "craft-idml", "craft-svg", + "craft-zip", + "serde", + "serde_json", ] [[package]] diff --git a/crates/file-formats/Cargo.toml b/crates/file-formats/Cargo.toml index 00d3b388a..a1a605918 100644 --- a/crates/file-formats/Cargo.toml +++ b/crates/file-formats/Cargo.toml @@ -9,7 +9,11 @@ rust-version.workspace = true [dependencies] craft-eps = { workspace = true } +craft-idml = { workspace = true } craft-svg = { workspace = true } +craft-zip = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } [lints] workspace = true diff --git a/crates/file-formats/src/lib.rs b/crates/file-formats/src/lib.rs index c6ac77540..1bdfc3b5c 100644 --- a/crates/file-formats/src/lib.rs +++ b/crates/file-formats/src/lib.rs @@ -34,6 +34,8 @@ pub enum Format { Pcraft, Idml, DesignCraft, + /// Legacy DesignCraft documents serialized as one JSON object. + DesignCraftJson, Indd, Docx, /// Legacy OLE Compound File Word document; identified by its extension only. @@ -51,6 +53,15 @@ pub enum Format { /// Resolve a name and file bytes to a format. Strong signatures take precedence over /// extension hints. Only format-specific bounded probes and extension tables live here. pub fn detect(name: &str, bytes: &[u8]) -> Option { + if craft_idml::is_idml(bytes) { + return Some(Format::Idml); + } + if craft_zip::stored_mimetype(bytes) == Some(b"application/vnd.designcraft+zip") || designcraft_zip_has_document(bytes) { + return Some(Format::DesignCraft); + } + if is_designcraft_json(bytes) { + return Some(Format::DesignCraftJson); + } if bytes.starts_with(&INDD_SIGNATURE) { return Some(Format::Indd); } @@ -111,6 +122,61 @@ pub fn detect(name: &str, bytes: &[u8]) -> Option { None } +/// DesignCraft's older ZIPs predate their leading `mimetype`; they are identified by the +/// `document.json` member. The directory is bounded before it is indexed and no entry is inflated. +fn designcraft_zip_has_document(bytes: &[u8]) -> bool { + if !bytes.starts_with(b"PK") { + return false; + } + craft_zip::Archive::open(bytes, craft_zip::Limits { max_entries: 100_000, max_total_bytes: 0 }) + .is_ok_and(|archive| archive.contains("document.json")) +} + +/// Read only a bounded prefix, stopping as soon as the model's required top-level keys have +/// appeared. This distinguishes legacy DesignCraft JSON from arbitrary JSON without decoding the +/// document or walking its arrays and assets. +fn is_designcraft_json(bytes: &[u8]) -> bool { + const LIMIT: usize = 1 << 20; + const REQUIRED: u16 = (1 << 10) - 1; + const KEYS: [&str; 10] = ["title", "settings", "spreads", "parents", "layers", "stories", "styles", "swatches", "sections", "next_id"]; + + struct Probe; + impl<'de> serde::de::Visitor<'de> for Probe { + type Value = bool; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("a DesignCraft document object") + } + + fn visit_map>(self, mut map: A) -> Result { + let mut found = 0u16; + while let Some(key) = map.next_key::()? { + if let Some(index) = KEYS.iter().position(|candidate| *candidate == key) { + found |= 1 << index; + } + map.next_value::()?; + if found == REQUIRED { + return Ok(true); + } + } + Ok(false) + } + } + + use serde::de::DeserializeSeed; + struct Seed; + impl<'de> DeserializeSeed<'de> for Seed { + type Value = bool; + fn deserialize>(self, deserializer: D) -> Result { + deserializer.deserialize_map(Probe) + } + } + + let prefix = bytes.get(..bytes.len().min(LIMIT)).unwrap_or(bytes); + let mut deserializer = serde_json::Deserializer::from_slice(prefix); + Seed.deserialize(&mut deserializer).unwrap_or(false) +} + /// Resolve a file extension to a family, case-insensitively. pub fn from_extension(ext: &str) -> Option { match ext.trim_start_matches('.').to_ascii_lowercase().as_str() { From 71337f6772b57a61e0f171c941f695822eea19a5 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 01:51:08 -0500 Subject: [PATCH 25/36] Add shared CSV and TSV file formats --- crates/file-formats/src/lib.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/file-formats/src/lib.rs b/crates/file-formats/src/lib.rs index 1bdfc3b5c..a570f0af4 100644 --- a/crates/file-formats/src/lib.rs +++ b/crates/file-formats/src/lib.rs @@ -45,6 +45,8 @@ pub enum Format { TaggedText, Pptx, PlainText, + Csv, + Tsv, Gcode, Studio, StencilCraft, @@ -214,6 +216,8 @@ pub fn from_extension(ext: &str) -> Option { "xlsx" => Some(Format::Xlsx), "pptx" => Some(Format::Pptx), "txt" | "text" | "md" => Some(Format::PlainText), + "csv" => Some(Format::Csv), + "tsv" | "tab" => Some(Format::Tsv), "studio" | "studio3" => Some(Format::Studio), "stencilcraft" => Some(Format::StencilCraft), "gcode" | "nc" => Some(Format::Gcode), From 8abfa9d0f14bafbfc3449fc552ffeafb9ca6a551 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 01:53:37 -0500 Subject: [PATCH 26/36] Classify legacy Excel extensions in shared registry --- crates/file-formats/src/lib.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/file-formats/src/lib.rs b/crates/file-formats/src/lib.rs index a570f0af4..7859bf539 100644 --- a/crates/file-formats/src/lib.rs +++ b/crates/file-formats/src/lib.rs @@ -47,6 +47,8 @@ pub enum Format { PlainText, Csv, Tsv, + /// Legacy binary Excel workbooks, identified by extension only. + LegacyExcel, Gcode, Studio, StencilCraft, @@ -218,6 +220,7 @@ pub fn from_extension(ext: &str) -> Option { "txt" | "text" | "md" => Some(Format::PlainText), "csv" => Some(Format::Csv), "tsv" | "tab" => Some(Format::Tsv), + "xls" | "xlsm" => Some(Format::LegacyExcel), "studio" | "studio3" => Some(Format::Studio), "stencilcraft" => Some(Format::StencilCraft), "gcode" | "nc" => Some(Format::Gcode), From 56a1f2b7c44d2e9ae4c121bb4bdb76f366e9ed77 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 07:11:12 +0000 Subject: [PATCH 27/36] Avoid premature saturation in H264 type 1 POC --- crates/video/src/h264/decode/slice.rs | 28 ++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/crates/video/src/h264/decode/slice.rs b/crates/video/src/h264/decode/slice.rs index 61670f059..32366ccf9 100644 --- a/crates/video/src/h264/decode/slice.rs +++ b/crates/video/src/h264/decode/slice.rs @@ -475,26 +475,27 @@ impl PocState { } 1 => { let frame_num_offset = self.frame_num_offset(sh.idr, sh.frame_num, sps); - let n = sps.offset_for_ref_frame.len() as i32; - let mut abs_frame_num = if n != 0 { frame_num_offset.saturating_add(sh.frame_num as i32) } else { 0 }; + let n = sps.offset_for_ref_frame.len() as i64; + let mut abs_frame_num = if n != 0 { i64::from(frame_num_offset) + i64::from(sh.frame_num) } else { 0 }; if sh.nal_ref_idc == 0 && abs_frame_num > 0 { abs_frame_num -= 1; } - let mut expected = 0i32; + let mut expected = 0i64; if abs_frame_num > 0 { let cycle_cnt = (abs_frame_num - 1) / n; let in_cycle = (abs_frame_num - 1) % n; - let delta_cycle = sps.offset_for_ref_frame.iter().fold(0i32, |a, &o| a.saturating_add(o)); - expected = cycle_cnt.saturating_mul(delta_cycle); + let delta_cycle = sps.offset_for_ref_frame.iter().map(|&o| i64::from(o)).sum::(); + expected = cycle_cnt * delta_cycle; for &o in &sps.offset_for_ref_frame[..=in_cycle as usize] { - expected = expected.saturating_add(o); + expected += i64::from(o); } } if sh.nal_ref_idc == 0 { - expected = expected.saturating_add(sps.offset_for_non_ref_pic); + expected += i64::from(sps.offset_for_non_ref_pic); } - let top = expected.saturating_add(sh.delta_pic_order_cnt[0]); - let bottom = top.saturating_add(sps.offset_for_top_to_bottom_field).saturating_add(sh.delta_pic_order_cnt[1]); + let top = (expected + i64::from(sh.delta_pic_order_cnt[0])).clamp(i64::from(i32::MIN), i64::from(i32::MAX)) as i32; + let bottom = (i64::from(top) + i64::from(sps.offset_for_top_to_bottom_field) + i64::from(sh.delta_pic_order_cnt[1])) + .clamp(i64::from(i32::MIN), i64::from(i32::MAX)) as i32; Poc { top, bottom, msb: 0, frame_num_offset } } _ => { @@ -662,4 +663,13 @@ mod poc_tests { let _ = poc.frame(); state.update(&sh, &poc); } + + #[test] + fn poc_type1_cancellation_happens_before_clamping() { + let sps = sps(1); + let sh = header(0, 0, 0, [-i32::MAX, 0], 1); + let state = PocState { prev_frame_num_offset: 2, ..PocState::default() }; + let poc = state.compute(&sh, &sps); + assert_eq!(poc.top, i32::MAX); + } } From fd5133174b1f24dd6b5de7f7b9f005ccbd3eee19 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 02:07:34 -0500 Subject: [PATCH 28/36] Add shared pcraft bundle storage --- Cargo.lock | 12 + Cargo.toml | 1 + crates/pcraft-bundle/Cargo.toml | 19 ++ crates/pcraft-bundle/src/lib.rs | 421 ++++++++++++++++++++++++++++++++ 4 files changed, 453 insertions(+) create mode 100644 crates/pcraft-bundle/Cargo.toml create mode 100644 crates/pcraft-bundle/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 8fb3d4dd7..336fa8b0c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1893,6 +1893,18 @@ dependencies = [ "zip", ] +[[package]] +name = "craft-pcraft-bundle" +version = "0.1.0" +dependencies = [ + "blake3", + "craft-atomic", + "craft-pcraft-format", + "craft-zip", + "ruzstd", + "thiserror 2.0.21", +] + [[package]] name = "craft-pcraft-format" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index 170ad8641..2b642f4fe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -95,6 +95,7 @@ craft-ora = { path = "crates/ora" } craft-pptx = { path = "crates/pptx" } craft-penpot = { path = "crates/penpot" } craft-pcraft-format = { path = "crates/pcraft-format" } +craft-pcraft-bundle = { path = "crates/pcraft-bundle" } craft-pkcs7 = { path = "crates/pkcs7" } craft-plotter = { path = "crates/plotter" } craft-prepress = { path = "crates/prepress" } diff --git a/crates/pcraft-bundle/Cargo.toml b/crates/pcraft-bundle/Cargo.toml new file mode 100644 index 000000000..7521ad494 --- /dev/null +++ b/crates/pcraft-bundle/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "craft-pcraft-bundle" +description = "Bounded ZIP and directory storage for .pcraft manifests, previews and content-addressed objects" +version.workspace = true +edition.workspace = true +license.workspace = true +rust-version.workspace = true +repository.workspace = true + +[lints] +workspace = true + +[dependencies] +craft-pcraft-format = { workspace = true } +craft-atomic = { workspace = true } +craft-zip = { workspace = true } +thiserror = { workspace = true } +ruzstd = { version = "0.9.1", default-features = false, features = ["std"] } +blake3 = { version = "1.8", default-features = false, features = ["pure"] } diff --git a/crates/pcraft-bundle/src/lib.rs b/crates/pcraft-bundle/src/lib.rs new file mode 100644 index 000000000..e44c23044 --- /dev/null +++ b/crates/pcraft-bundle/src/lib.rs @@ -0,0 +1,421 @@ +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +//! Container and content-addressed object storage for `.pcraft` bundles. +//! +//! Document schemas stay with the application. This crate owns the wire-level paths, ZIP +//! envelope, zstd object frames and content hashes. A caller supplies serialized manifest bytes +//! and opaque tile/blob bytes; it receives their stable paths to put in its manifest. + +use std::collections::BTreeMap; +use std::fs::{self, OpenOptions}; +use std::io::Read; +use std::path::{Path, PathBuf}; + +use craft_zip::{Archive, Compression, Writer}; + +/// Default bounds for opening an untrusted `.pcraft` ZIP. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Limits { + /// Maximum number of ZIP entries. + pub max_entries: u64, + /// Maximum total uncompressed ZIP bytes read. + pub max_total_bytes: u64, + /// Maximum manifest size. + pub max_manifest_bytes: u64, + /// Maximum compressed object bytes. + pub max_object_bytes: u64, + /// Maximum decoded object bytes. + pub max_decoded_object_bytes: u64, +} + +impl Default for Limits { + fn default() -> Self { + let limits = craft_pcraft_format::Limits::default(); + Self { + max_entries: limits.max_items as u64 + 4, + max_total_bytes: limits.max_total_bytes, + max_manifest_bytes: limits.max_manifest_bytes as u64, + max_object_bytes: limits.max_blob_bytes as u64, + max_decoded_object_bytes: limits.max_blob_bytes as u64, + } + } +} + +/// Why a `.pcraft` container or object could not be read or written. +#[derive(Debug, thiserror::Error)] +pub enum Error { + /// A path is not one of the bundle's canonical entry names. + #[error("invalid .pcraft entry path: {0}")] + InvalidPath(String), + /// An entry is missing, duplicated or violates a bundle invariant. + #[error("invalid .pcraft bundle: {0}")] + InvalidBundle(String), + /// An entry exceeds its caller-supplied bound. + #[error("{entry} exceeds the {limit}-byte limit")] + LimitExceeded { entry: String, limit: u64 }, + /// The object's content hash does not match its entry path. + #[error("content hash mismatch for {0}")] + HashMismatch(String), + /// ZIP container error. + #[error(transparent)] + Zip(#[from] craft_zip::Error), + /// I/O or zstd decoding error. + #[error("{0}")] + Io(String), +} + +pub type Result = std::result::Result; + +/// Whether a ZIP archive declares a `manifest.json` entry. +pub fn is_pcraft_zip(bytes: &[u8]) -> bool { + if !bytes.starts_with(b"PK\x03\x04") { + return false; + } + let limits = Limits { max_entries: 1_000_000, ..Limits::default() }; + Archive::open(bytes, craft_zip::Limits { max_entries: limits.max_entries, max_total_bytes: 0 }) + .is_ok_and(|archive| !archive.has_duplicate_names() && archive.contains(craft_pcraft_format::MANIFEST)) +} + +/// Whether an entry path is a canonical manifest, preview, tile or blob path. +pub fn valid_entry_path(path: &str) -> bool { + if matches!( + path, + craft_pcraft_format::MANIFEST | craft_pcraft_format::THUMB | craft_pcraft_format::COMPOSITE | craft_pcraft_format::FULL_COMPOSITE + ) { + return true; + } + let Some((kind, name)) = path.split_once('/') else { return false }; + let Some(hash) = name.strip_suffix(craft_pcraft_format::OBJECT_SUFFIX) else { return false }; + craft_pcraft_format::is_valid_hash(hash) && (kind == craft_pcraft_format::TILES_DIR || kind == craft_pcraft_format::BLOBS_DIR) +} + +/// Content-addressed, compressed object bytes and their canonical bundle path. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct StoredObject { + /// `tiles/.zst` or `blobs/.zst`. + pub path: String, + /// BLAKE3 of the uncompressed bytes. + pub hash: String, + /// Zstandard frame containing the uncompressed bytes. + pub compressed: Vec, +} + +/// The kind of content-addressed object being stored. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ObjectKind { + /// Pixel tile samples. + Tile, + /// ICC, metadata, masks, or other opaque document data. + Blob, +} + +/// Hashes and compresses an object, returning the path to record in the manifest. +pub fn encode_object(kind: ObjectKind, bytes: &[u8]) -> StoredObject { + let hash = blake3::hash(bytes).to_hex().to_string(); + let path = match kind { + ObjectKind::Tile => craft_pcraft_format::tile_path(&hash), + ObjectKind::Blob => craft_pcraft_format::blob_path(&hash), + } + .unwrap_or_default(); + let compressed = ruzstd::encoding::compress_to_vec(bytes, ruzstd::encoding::CompressionLevel::Fastest); + StoredObject { path, hash, compressed } +} + +/// Decompresses a stored object within `max_decoded_bytes` and checks its content hash. +pub fn decode_object(path: &str, compressed: &[u8], max_decoded_bytes: u64) -> Result> { + if !valid_object_path(path) { + return Err(Error::InvalidPath(path.to_owned())); + } + let max = usize::try_from(max_decoded_bytes).unwrap_or(usize::MAX); + let decoder = ruzstd::decoding::StreamingDecoder::new(compressed).map_err(|e| Error::Io(e.to_string()))?; + let mut bytes = Vec::new(); + decoder.take(max_decoded_bytes.saturating_add(1)).read_to_end(&mut bytes).map_err(|e| Error::Io(e.to_string()))?; + if bytes.len() > max { + return Err(Error::LimitExceeded { entry: path.to_owned(), limit: max_decoded_bytes }); + } + let expected = path.rsplit('/').next().unwrap_or_default().trim_end_matches(craft_pcraft_format::OBJECT_SUFFIX); + if blake3::hash(&bytes).to_hex().as_str() != expected { + return Err(Error::HashMismatch(path.to_owned())); + } + Ok(bytes) +} + +fn valid_object_path(path: &str) -> bool { + let Some((kind, _)) = path.split_once('/') else { return false }; + (kind == craft_pcraft_format::TILES_DIR || kind == craft_pcraft_format::BLOBS_DIR) && valid_entry_path(path) +} + +/// An in-memory `.pcraft` ZIP reader with bounded entry and total reads. +pub struct ZipBundle<'a> { + archive: Archive<'a>, + limits: Limits, +} + +impl<'a> ZipBundle<'a> { + /// Opens a bundle, rejecting duplicate entry names and archives without a manifest. + pub fn open(bytes: &'a [u8], limits: Limits) -> Result { + let archive = Archive::open(bytes, craft_zip::Limits { max_entries: limits.max_entries, max_total_bytes: limits.max_total_bytes })?; + if archive.has_duplicate_names() || !archive.contains(craft_pcraft_format::MANIFEST) { + return Err(Error::InvalidBundle("missing manifest or duplicate entry names".into())); + } + Ok(Self { archive, limits }) + } + + /// Reads the serialized document manifest. + pub fn manifest(&mut self) -> Result> { + self.read(craft_pcraft_format::MANIFEST, self.limits.max_manifest_bytes) + } + + /// Reads an optional preview entry. + pub fn preview(&mut self, path: &str) -> Result>> { + if !matches!(path, craft_pcraft_format::THUMB | craft_pcraft_format::COMPOSITE | craft_pcraft_format::FULL_COMPOSITE) { + return Err(Error::InvalidPath(path.to_owned())); + } + self.archive.read(path, self.limits.max_object_bytes).map_err(Into::into) + } + + /// Reads and verifies a compressed content-addressed object. + pub fn object(&mut self, path: &str) -> Result> { + let compressed = self.read(path, self.limits.max_object_bytes)?; + decode_object(path, &compressed, self.limits.max_decoded_object_bytes) + } + + fn read(&mut self, path: &str, limit: u64) -> Result> { + if !valid_entry_path(path) { + return Err(Error::InvalidPath(path.to_owned())); + } + self.archive.read(path, limit)?.ok_or_else(|| Error::InvalidBundle(format!("missing {path}"))) + } +} + +/// Builds a deterministic STORE ZIP bundle from format-neutral entries. +#[derive(Default)] +pub struct ZipBundleWriter { + entries: BTreeMap>, +} + +impl ZipBundleWriter { + /// Adds or replaces the manifest bytes. + pub fn set_manifest(&mut self, manifest: Vec) -> Result<()> { + if manifest.is_empty() { + return Err(Error::InvalidBundle("manifest is empty".into())); + } + self.entries.insert(craft_pcraft_format::MANIFEST.into(), manifest); + Ok(()) + } + + /// Adds an optional preview. + pub fn set_preview(&mut self, path: &str, bytes: Vec) -> Result<()> { + if !matches!(path, craft_pcraft_format::THUMB | craft_pcraft_format::COMPOSITE | craft_pcraft_format::FULL_COMPOSITE) { + return Err(Error::InvalidPath(path.to_owned())); + } + self.entries.insert(path.to_owned(), bytes); + Ok(()) + } + + /// Adds a compressed content-addressed object, returning its manifest path. + pub fn add_object(&mut self, kind: ObjectKind, bytes: &[u8]) -> Result { + let object = encode_object(kind, bytes); + if object.path.is_empty() { + return Err(Error::InvalidBundle("could not construct object path".into())); + } + self.entries.entry(object.path.clone()).or_insert(object.compressed); + Ok(object.path) + } + + /// Adds an already-compressed object after checking its path/hash. + pub fn add_stored_object(&mut self, path: &str, compressed: Vec, max_decoded_bytes: u64) -> Result<()> { + let _ = decode_object(path, &compressed, max_decoded_bytes)?; + self.entries.insert(path.to_owned(), compressed); + Ok(()) + } + + /// Writes entries in canonical path order, with every ZIP member stored without another layer + /// of compression (object payloads already use zstd). + pub fn finish(self) -> Result> { + if !self.entries.contains_key(craft_pcraft_format::MANIFEST) { + return Err(Error::InvalidBundle("manifest not set".into())); + } + let mut writer = Writer::new(); + let mut entries = self.entries; + let manifest = entries.remove(craft_pcraft_format::MANIFEST).ok_or_else(|| Error::InvalidBundle("manifest not set".into()))?; + writer.add(craft_pcraft_format::MANIFEST, &manifest, Compression::Stored)?; + for (path, bytes) in entries { + if !valid_entry_path(&path) { + return Err(Error::InvalidPath(path)); + } + writer.add(&path, &bytes, Compression::Stored)?; + } + writer.finish().map_err(Into::into) + } + + /// Saves the same entries as a directory bundle. Object files are published before the + /// manifest, the bundle lock serializes writers, and unreferenced canonical objects are + /// collected after the new manifest is durable. + pub fn save_directory(self, root: &Path) -> Result { + if !self.entries.contains_key(craft_pcraft_format::MANIFEST) { + return Err(Error::InvalidBundle("manifest not set".into())); + } + for path in self.entries.keys() { + if !valid_entry_path(path) { + return Err(Error::InvalidPath(path.clone())); + } + } + fs::create_dir_all(root).map_err(io_error)?; + let root = fs::canonicalize(root).map_err(io_error)?; + let lock_path = root.join(craft_pcraft_format::SAVE_LOCK); + refuse_symlink(&lock_path)?; + let lock = OpenOptions::new().read(true).write(true).create(true).truncate(false).open(&lock_path).map_err(io_error)?; + lock.lock().map_err(io_error)?; + + let mut written = 0usize; + for (path, bytes) in self.entries.iter().filter(|(path, _)| path.as_str() != craft_pcraft_format::MANIFEST) { + let target = safe_target(&root, path)?; + if let Some(parent) = target.parent() { + ensure_directory(parent)?; + } + refuse_symlink(&target)?; + craft_atomic::atomic_write(&target, bytes).map_err(io_error)?; + written = written.saturating_add(1); + } + let manifest = self.entries.get(craft_pcraft_format::MANIFEST).ok_or_else(|| Error::InvalidBundle("manifest not set".into()))?; + let manifest_path = root.join(craft_pcraft_format::MANIFEST); + refuse_symlink(&manifest_path)?; + craft_atomic::atomic_write(&manifest_path, manifest).map_err(io_error)?; + written = written.saturating_add(1); + + let mut removed = 0usize; + for kind in [craft_pcraft_format::TILES_DIR, craft_pcraft_format::BLOBS_DIR] { + let dir = root.join(kind); + if !dir.exists() { + continue; + } + refuse_symlink(&dir)?; + for entry in fs::read_dir(&dir).map_err(io_error)? { + let entry = entry.map_err(io_error)?; + let name = entry.file_name(); + let name = name.to_string_lossy(); + let path = format!("{kind}/{name}"); + if valid_entry_path(&path) && !self.entries.contains_key(&path) { + let meta = fs::symlink_metadata(entry.path()).map_err(io_error)?; + if meta.file_type().is_symlink() || !meta.is_file() { + return Err(Error::InvalidBundle(format!("refusing to collect non-file object {path}"))); + } + fs::remove_file(entry.path()).map_err(io_error)?; + removed = removed.saturating_add(1); + } + } + } + Ok(DirectoryStats { written, removed }) + } +} + +/// Work performed by a directory bundle save. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct DirectoryStats { + /// Entries atomically written, including `manifest.json`. + pub written: usize, + /// Stale content-addressed objects collected after manifest publication. + pub removed: usize, +} + +fn io_error(error: std::io::Error) -> Error { + Error::Io(error.to_string()) +} + +fn refuse_symlink(path: &Path) -> Result<()> { + match fs::symlink_metadata(path) { + Ok(metadata) if metadata.file_type().is_symlink() => Err(Error::InvalidBundle(format!("refusing symlink {}", path.display()))), + Ok(_) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(io_error(error)), + } +} + +fn ensure_directory(path: &Path) -> Result<()> { + refuse_symlink(path)?; + match fs::symlink_metadata(path) { + Ok(metadata) if metadata.is_dir() => Ok(()), + Ok(_) => Err(Error::InvalidBundle(format!("{} is not a directory", path.display()))), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + fs::create_dir(path).map_err(io_error)?; + Ok(()) + } + Err(error) => Err(io_error(error)), + } +} + +fn safe_target(root: &Path, relative: &str) -> Result { + if !valid_entry_path(relative) { + return Err(Error::InvalidPath(relative.to_owned())); + } + let mut target = root.to_path_buf(); + let components = relative.split('/').collect::>(); + for (index, component) in components.iter().enumerate() { + target.push(component); + if index + 1 < components.len() { + ensure_directory(&target)?; + } + } + Ok(target) +} + +/// Opens a directory bundle and reads canonical entries with caller-provided limits. +pub struct DirectoryBundle { + root: PathBuf, + limits: Limits, +} + +impl DirectoryBundle { + /// Opens a bundle directory containing a regular `manifest.json`. + pub fn open(root: &Path, limits: Limits) -> Result { + let root = fs::canonicalize(root).map_err(io_error)?; + let manifest = root.join(craft_pcraft_format::MANIFEST); + refuse_symlink(&manifest)?; + let metadata = fs::metadata(&manifest).map_err(io_error)?; + if !metadata.is_file() || metadata.len() > limits.max_manifest_bytes { + return Err(Error::LimitExceeded { entry: craft_pcraft_format::MANIFEST.into(), limit: limits.max_manifest_bytes }); + } + Ok(Self { root, limits }) + } + + /// Reads the serialized document manifest. + pub fn manifest(&self) -> Result> { + self.read(craft_pcraft_format::MANIFEST, self.limits.max_manifest_bytes) + } + + /// Reads an optional preview entry. + pub fn preview(&self, path: &str) -> Result>> { + if !matches!(path, craft_pcraft_format::THUMB | craft_pcraft_format::COMPOSITE | craft_pcraft_format::FULL_COMPOSITE) { + return Err(Error::InvalidPath(path.to_owned())); + } + self.read_optional(path, self.limits.max_object_bytes) + } + + /// Reads and verifies a compressed content-addressed object. + pub fn object(&self, path: &str) -> Result> { + let compressed = self.read(path, self.limits.max_object_bytes)?; + decode_object(path, &compressed, self.limits.max_decoded_object_bytes) + } + + fn read(&self, path: &str, limit: u64) -> Result> { + self.read_optional(path, limit)?.ok_or_else(|| Error::InvalidBundle(format!("missing {path}"))) + } + + fn read_optional(&self, path: &str, limit: u64) -> Result>> { + let target = safe_target(&self.root, path)?; + refuse_symlink(&target)?; + let metadata = match fs::metadata(&target) { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(io_error(error)), + }; + if !metadata.is_file() || metadata.len() > limit { + return Err(Error::LimitExceeded { entry: path.to_owned(), limit }); + } + let bytes = fs::read(target).map_err(io_error)?; + if bytes.len() as u64 > limit { + return Err(Error::LimitExceeded { entry: path.to_owned(), limit }); + } + Ok(Some(bytes)) + } +} From b0284aad097235afca67a82de2596e0ba1d428f6 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 02:08:04 -0500 Subject: [PATCH 29/36] Expose bounded pcraft entry reads --- crates/pcraft-bundle/src/lib.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crates/pcraft-bundle/src/lib.rs b/crates/pcraft-bundle/src/lib.rs index e44c23044..9b2932523 100644 --- a/crates/pcraft-bundle/src/lib.rs +++ b/crates/pcraft-bundle/src/lib.rs @@ -165,6 +165,14 @@ impl<'a> ZipBundle<'a> { self.read(craft_pcraft_format::MANIFEST, self.limits.max_manifest_bytes) } + /// Reads any canonical bundle entry, including the stored zstd bytes of an object. + pub fn entry(&mut self, path: &str, max_bytes: u64) -> Result>> { + if !valid_entry_path(path) { + return Err(Error::InvalidPath(path.to_owned())); + } + self.archive.read(path, max_bytes).map_err(Into::into) + } + /// Reads an optional preview entry. pub fn preview(&mut self, path: &str) -> Result>> { if !matches!(path, craft_pcraft_format::THUMB | craft_pcraft_format::COMPOSITE | craft_pcraft_format::FULL_COMPOSITE) { From de087a45244e6581bcc5b8f42edb5c7bc5b8f758 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 02:12:14 -0500 Subject: [PATCH 30/36] Expose bounded directory bundle entries --- crates/pcraft-bundle/src/lib.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crates/pcraft-bundle/src/lib.rs b/crates/pcraft-bundle/src/lib.rs index 9b2932523..fabf73963 100644 --- a/crates/pcraft-bundle/src/lib.rs +++ b/crates/pcraft-bundle/src/lib.rs @@ -391,6 +391,11 @@ impl DirectoryBundle { self.read(craft_pcraft_format::MANIFEST, self.limits.max_manifest_bytes) } + /// Reads any canonical bundle entry. + pub fn entry(&self, path: &str, max_bytes: u64) -> Result>> { + self.read_optional(path, max_bytes) + } + /// Reads an optional preview entry. pub fn preview(&self, path: &str) -> Result>> { if !matches!(path, craft_pcraft_format::THUMB | craft_pcraft_format::COMPOSITE | craft_pcraft_format::FULL_COMPOSITE) { From 6b8e79294d2eb4ed6ceab6ed18c8d31b2c41ec25 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 02:16:39 -0500 Subject: [PATCH 31/36] Expose bounded pcraft object codecs --- crates/pcraft-bundle/src/lib.rs | 40 ++++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/crates/pcraft-bundle/src/lib.rs b/crates/pcraft-bundle/src/lib.rs index fabf73963..1abd4fa9b 100644 --- a/crates/pcraft-bundle/src/lib.rs +++ b/crates/pcraft-bundle/src/lib.rs @@ -110,16 +110,50 @@ pub enum ObjectKind { /// Hashes and compresses an object, returning the path to record in the manifest. pub fn encode_object(kind: ObjectKind, bytes: &[u8]) -> StoredObject { - let hash = blake3::hash(bytes).to_hex().to_string(); + let hash = hash_content(bytes); let path = match kind { ObjectKind::Tile => craft_pcraft_format::tile_path(&hash), ObjectKind::Blob => craft_pcraft_format::blob_path(&hash), } .unwrap_or_default(); - let compressed = ruzstd::encoding::compress_to_vec(bytes, ruzstd::encoding::CompressionLevel::Fastest); + let compressed = encode_frame(bytes); StoredObject { path, hash, compressed } } +/// BLAKE3 hash of object bytes, in the canonical lowercase hexadecimal form used by entry paths. +pub fn hash_content(bytes: &[u8]) -> String { + blake3::hash(bytes).to_hex().to_string() +} + +/// Encodes one bounded zstd frame at the format's fixed fastest compression level. +pub fn encode_frame(bytes: &[u8]) -> Vec { + ruzstd::encoding::compress_to_vec(bytes, ruzstd::encoding::CompressionLevel::Fastest) +} + +/// Decodes one zstd frame under a byte limit and optional exact output length. +pub fn decode_frame(data: &[u8], max_bytes: usize, exact_bytes: Option, name: &str) -> Result> { + let window = u64::try_from(exact_bytes.unwrap_or(max_bytes)).unwrap_or(u64::MAX).max(8 << 20); + let mut decoder = match ruzstd::decoding::StreamingDecoder::new_with_max_window_size(data, window) { + Ok(decoder) => decoder, + Err(ruzstd::decoding::errors::FrameDecoderError::WindowSizeTooBig { requested, .. }) => { + return Err(Error::LimitExceeded { entry: format!("{name} zstd window ({requested})"), limit: window }); + } + Err(error) => return Err(Error::Io(format!("{name}: zstd: {error}"))), + }; + let mut out = Vec::new(); + let read_limit = u64::try_from(max_bytes).unwrap_or(u64::MAX).saturating_add(1); + (&mut decoder).take(read_limit).read_to_end(&mut out).map_err(|error| Error::Io(format!("{name}: zstd: {error}")))?; + if out.len() > max_bytes { + return Err(Error::LimitExceeded { entry: name.to_owned(), limit: max_bytes as u64 }); + } + if let Some(expected) = exact_bytes + && out.len() != expected + { + return Err(Error::InvalidBundle(format!("{name}: expected {expected} bytes, got {}", out.len()))); + } + Ok(out) +} + /// Decompresses a stored object within `max_decoded_bytes` and checks its content hash. pub fn decode_object(path: &str, compressed: &[u8], max_decoded_bytes: u64) -> Result> { if !valid_object_path(path) { @@ -133,7 +167,7 @@ pub fn decode_object(path: &str, compressed: &[u8], max_decoded_bytes: u64) -> R return Err(Error::LimitExceeded { entry: path.to_owned(), limit: max_decoded_bytes }); } let expected = path.rsplit('/').next().unwrap_or_default().trim_end_matches(craft_pcraft_format::OBJECT_SUFFIX); - if blake3::hash(&bytes).to_hex().as_str() != expected { + if hash_content(&bytes) != expected { return Err(Error::HashMismatch(path.to_owned())); } Ok(bytes) From 98144696139aafad86f63a776e147e989731f43e Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 07:24:13 +0000 Subject: [PATCH 32/36] Bound extreme H264 POC cycle arithmetic --- crates/video/src/h264/decode/slice.rs | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/crates/video/src/h264/decode/slice.rs b/crates/video/src/h264/decode/slice.rs index 32366ccf9..874c575ca 100644 --- a/crates/video/src/h264/decode/slice.rs +++ b/crates/video/src/h264/decode/slice.rs @@ -485,16 +485,18 @@ impl PocState { let cycle_cnt = (abs_frame_num - 1) / n; let in_cycle = (abs_frame_num - 1) % n; let delta_cycle = sps.offset_for_ref_frame.iter().map(|&o| i64::from(o)).sum::(); - expected = cycle_cnt * delta_cycle; + expected = cycle_cnt.saturating_mul(delta_cycle); for &o in &sps.offset_for_ref_frame[..=in_cycle as usize] { - expected += i64::from(o); + expected = expected.saturating_add(i64::from(o)); } } if sh.nal_ref_idc == 0 { - expected += i64::from(sps.offset_for_non_ref_pic); + expected = expected.saturating_add(i64::from(sps.offset_for_non_ref_pic)); } - let top = (expected + i64::from(sh.delta_pic_order_cnt[0])).clamp(i64::from(i32::MIN), i64::from(i32::MAX)) as i32; - let bottom = (i64::from(top) + i64::from(sps.offset_for_top_to_bottom_field) + i64::from(sh.delta_pic_order_cnt[1])) + let top = expected.saturating_add(i64::from(sh.delta_pic_order_cnt[0])).clamp(i64::from(i32::MIN), i64::from(i32::MAX)) as i32; + let bottom = i64::from(top) + .saturating_add(i64::from(sps.offset_for_top_to_bottom_field)) + .saturating_add(i64::from(sh.delta_pic_order_cnt[1])) .clamp(i64::from(i32::MIN), i64::from(i32::MAX)) as i32; Poc { top, bottom, msb: 0, frame_num_offset } } @@ -672,4 +674,14 @@ mod poc_tests { let poc = state.compute(&sh, &sps); assert_eq!(poc.top, i32::MAX); } + + #[test] + fn poc_type1_saturates_only_after_wide_cycle_arithmetic_exceeds_i64() { + let mut sps = sps(1); + sps.offset_for_ref_frame = vec![i32::MAX; 255]; + let sh = header(u32::MAX, 0, 0, [0, 0], 1); + let state = PocState { prev_frame_num_offset: i32::MAX, ..PocState::default() }; + let poc = state.compute(&sh, &sps); + assert_eq!(poc.top, i32::MAX); + } } From c0df5be07dcc61047ddb005dca43df983ea99682 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 02:26:51 -0500 Subject: [PATCH 33/36] Reuse verified objects in directory bundles --- crates/pcraft-bundle/src/lib.rs | 211 ++++++++++++++++++++++++++++---- 1 file changed, 188 insertions(+), 23 deletions(-) diff --git a/crates/pcraft-bundle/src/lib.rs b/crates/pcraft-bundle/src/lib.rs index 1abd4fa9b..422776fd9 100644 --- a/crates/pcraft-bundle/src/lib.rs +++ b/crates/pcraft-bundle/src/lib.rs @@ -5,10 +5,11 @@ //! envelope, zstd object frames and content hashes. A caller supplies serialized manifest bytes //! and opaque tile/blob bytes; it receives their stable paths to put in its manifest. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::fs::{self, OpenOptions}; use std::io::Read; use std::path::{Path, PathBuf}; +use std::time::SystemTime; use craft_zip::{Archive, Compression, Writer}; @@ -267,6 +268,15 @@ impl ZipBundleWriter { /// Adds an already-compressed object after checking its path/hash. pub fn add_stored_object(&mut self, path: &str, compressed: Vec, max_decoded_bytes: u64) -> Result<()> { let _ = decode_object(path, &compressed, max_decoded_bytes)?; + self.stage_compressed_object(path, compressed) + } + + /// Stages an already-compressed object for a bundle write. Its path is checked now and its + /// frame/hash are verified by [`Self::finish`] or [`DirectoryBundleWriter::save_directory`]. + pub fn stage_compressed_object(&mut self, path: &str, compressed: Vec) -> Result<()> { + if !valid_object_path(path) { + return Err(Error::InvalidPath(path.to_owned())); + } self.entries.insert(path.to_owned(), compressed); Ok(()) } @@ -285,6 +295,9 @@ impl ZipBundleWriter { if !valid_entry_path(&path) { return Err(Error::InvalidPath(path)); } + if is_object_path(&path) { + let _ = decode_object(&path, &bytes, 1 << 30)?; + } writer.add(&path, &bytes, Compression::Stored)?; } writer.finish().map_err(Into::into) @@ -294,38 +307,133 @@ impl ZipBundleWriter { /// manifest, the bundle lock serializes writers, and unreferenced canonical objects are /// collected after the new manifest is durable. pub fn save_directory(self, root: &Path) -> Result { - if !self.entries.contains_key(craft_pcraft_format::MANIFEST) { + DirectoryBundleWriter::default().save_directory(root, self) + } +} + +/// Work performed by a directory bundle save. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct DirectoryStats { + /// Entries atomically written, including `manifest.json`. + pub written: usize, + /// Stale content-addressed objects collected after manifest publication. + pub removed: usize, + /// Existing, verified content-addressed objects reused without writing. + pub reused: usize, + /// Canonical object paths written during this save. + pub written_objects: Vec, + /// Canonical object paths reused after verification or a matching cached signature. + pub reused_objects: Vec, +} + +/// Incremental directory-bundle persistence with bounded rolling object verification. +#[derive(Default)] +pub struct DirectoryBundleWriter { + root: Option, + generation: u64, + reverify_budget: u64, + verified: HashMap, +} + +#[derive(Clone, Copy)] +struct CachedFile { + signature: FileSignature, + checked_at: u64, +} + +#[derive(Clone, Copy, PartialEq, Eq)] +struct FileSignature { + len: u64, + modified: SystemTime, + #[cfg(unix)] + unix: (u64, u64, i64, i64), +} + +impl DirectoryBundleWriter { + /// Compressed bytes of already-verified objects to re-read on each save. + /// `0` trusts unchanged signatures; `u64::MAX` rechecks all referenced objects. + pub fn set_reverify_budget(&mut self, bytes: u64) { + self.reverify_budget = bytes; + } + + /// Saves an entry set incrementally. Existing objects are reused only after a hash check; + /// unchanged file signatures may skip checks until they enter the rolling verification + /// window. The manifest is atomically published last, then stale previews and objects are + /// collected while the bundle lock is held. + pub fn save_directory(&mut self, root: &Path, bundle: ZipBundleWriter) -> Result { + if !bundle.entries.contains_key(craft_pcraft_format::MANIFEST) { return Err(Error::InvalidBundle("manifest not set".into())); } - for path in self.entries.keys() { + for path in bundle.entries.keys() { if !valid_entry_path(path) { return Err(Error::InvalidPath(path.clone())); } } fs::create_dir_all(root).map_err(io_error)?; let root = fs::canonicalize(root).map_err(io_error)?; + if self.root.as_ref() != Some(&root) { + self.root = Some(root.clone()); + self.verified.clear(); + self.generation = 0; + } let lock_path = root.join(craft_pcraft_format::SAVE_LOCK); refuse_symlink(&lock_path)?; let lock = OpenOptions::new().read(true).write(true).create(true).truncate(false).open(&lock_path).map_err(io_error)?; lock.lock().map_err(io_error)?; + self.generation = self.generation.saturating_add(1); + let generation = self.generation; + let recheck = self.rolling_recheck(&bundle.entries); + let mut stats = DirectoryStats::default(); - let mut written = 0usize; - for (path, bytes) in self.entries.iter().filter(|(path, _)| path.as_str() != craft_pcraft_format::MANIFEST) { + for (path, bytes) in bundle.entries.iter().filter(|(path, _)| is_object_path(path)) { + let target = safe_target(&root, path)?; + if let Some(parent) = target.parent() { + ensure_directory(parent)?; + } + refuse_symlink(&target)?; + if let Some(signature) = file_signature(&target)? { + let cached = self.verified.get(path).is_some_and(|entry| entry.signature == signature); + if cached && !recheck.contains(path) { + stats.reused = stats.reused.saturating_add(1); + stats.reused_objects.push(path.clone()); + continue; + } + let existing = read_bounded(&target, 1 << 30)?; + if decode_object(path, &existing, 1 << 30).is_ok() { + if let Some(signature) = file_signature(&target)? { + self.verified.insert(path.clone(), CachedFile { signature, checked_at: generation }); + } + stats.reused = stats.reused.saturating_add(1); + stats.reused_objects.push(path.clone()); + continue; + } + } + let _ = decode_object(path, bytes, 1 << 30)?; + craft_atomic::atomic_write(&target, bytes).map_err(io_error)?; + if let Some(signature) = file_signature(&target)? { + self.verified.insert(path.clone(), CachedFile { signature, checked_at: generation }); + } + stats.written = stats.written.saturating_add(1); + stats.written_objects.push(path.clone()); + } + + for (path, bytes) in bundle.entries.iter().filter(|(path, _)| !is_object_path(path) && path.as_str() != craft_pcraft_format::MANIFEST) { let target = safe_target(&root, path)?; if let Some(parent) = target.parent() { ensure_directory(parent)?; } refuse_symlink(&target)?; craft_atomic::atomic_write(&target, bytes).map_err(io_error)?; - written = written.saturating_add(1); + stats.written = stats.written.saturating_add(1); } - let manifest = self.entries.get(craft_pcraft_format::MANIFEST).ok_or_else(|| Error::InvalidBundle("manifest not set".into()))?; + + let manifest = bundle.entries.get(craft_pcraft_format::MANIFEST).ok_or_else(|| Error::InvalidBundle("manifest not set".into()))?; let manifest_path = root.join(craft_pcraft_format::MANIFEST); refuse_symlink(&manifest_path)?; craft_atomic::atomic_write(&manifest_path, manifest).map_err(io_error)?; - written = written.saturating_add(1); + stats.written = stats.written.saturating_add(1); - let mut removed = 0usize; + let mut removed_paths = Vec::new(); for kind in [craft_pcraft_format::TILES_DIR, craft_pcraft_format::BLOBS_DIR] { let dir = root.join(kind); if !dir.exists() { @@ -335,29 +443,86 @@ impl ZipBundleWriter { for entry in fs::read_dir(&dir).map_err(io_error)? { let entry = entry.map_err(io_error)?; let name = entry.file_name(); - let name = name.to_string_lossy(); - let path = format!("{kind}/{name}"); - if valid_entry_path(&path) && !self.entries.contains_key(&path) { - let meta = fs::symlink_metadata(entry.path()).map_err(io_error)?; - if meta.file_type().is_symlink() || !meta.is_file() { + let path = format!("{kind}/{}", name.to_string_lossy()); + if is_object_path(&path) && !bundle.entries.contains_key(&path) { + let metadata = fs::symlink_metadata(entry.path()).map_err(io_error)?; + if metadata.file_type().is_symlink() || !metadata.is_file() { return Err(Error::InvalidBundle(format!("refusing to collect non-file object {path}"))); } fs::remove_file(entry.path()).map_err(io_error)?; - removed = removed.saturating_add(1); + removed_paths.push(path); + stats.removed = stats.removed.saturating_add(1); } } } - Ok(DirectoryStats { written, removed }) + for path in [craft_pcraft_format::THUMB, craft_pcraft_format::COMPOSITE, craft_pcraft_format::FULL_COMPOSITE] { + if !bundle.entries.contains_key(path) { + let target = root.join(path); + refuse_symlink(&target)?; + if fs::symlink_metadata(&target).is_ok() { + fs::remove_file(&target).map_err(io_error)?; + } + } + } + self.verified.retain(|path, _| bundle.entries.contains_key(path) && !removed_paths.contains(path)); + Ok(stats) + } + + fn rolling_recheck(&self, entries: &BTreeMap>) -> HashSet { + if self.reverify_budget == 0 { + return HashSet::new(); + } + let mut cached: Vec<(&String, &CachedFile)> = self.verified.iter().filter(|(path, _)| entries.contains_key(*path)).collect(); + cached.sort_by(|a, b| a.1.checked_at.cmp(&b.1.checked_at).then_with(|| a.0.cmp(b.0))); + let mut out = HashSet::new(); + let mut spent = 0u64; + for (path, value) in cached { + let next = spent.saturating_add(value.signature.len); + if !out.is_empty() && next > self.reverify_budget { + break; + } + spent = next; + out.insert(path.clone()); + } + out } } -/// Work performed by a directory bundle save. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub struct DirectoryStats { - /// Entries atomically written, including `manifest.json`. - pub written: usize, - /// Stale content-addressed objects collected after manifest publication. - pub removed: usize, +fn is_object_path(path: &str) -> bool { + path.starts_with("tiles/") || path.starts_with("blobs/") +} + +fn file_signature(path: &Path) -> Result> { + let metadata = match fs::symlink_metadata(path) { + Ok(metadata) if metadata.is_file() && !metadata.file_type().is_symlink() => metadata, + Ok(_) => return Err(Error::InvalidBundle(format!("{} is not a regular file", path.display()))), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(io_error(error)), + }; + let modified = metadata.modified().unwrap_or(SystemTime::UNIX_EPOCH); + #[cfg(unix)] + let unix = { + use std::os::unix::fs::MetadataExt; + (metadata.dev(), metadata.ino(), metadata.ctime(), metadata.ctime_nsec()) + }; + Ok(Some(FileSignature { + len: metadata.len(), + modified, + #[cfg(unix)] + unix, + })) +} + +fn read_bounded(path: &Path, max: u64) -> Result> { + let metadata = fs::metadata(path).map_err(io_error)?; + if metadata.len() > max { + return Err(Error::LimitExceeded { entry: path.display().to_string(), limit: max }); + } + let bytes = fs::read(path).map_err(io_error)?; + if bytes.len() as u64 > max { + return Err(Error::LimitExceeded { entry: path.display().to_string(), limit: max }); + } + Ok(bytes) } fn io_error(error: std::io::Error) -> Error { From de724aa188bb4131570cc052ef2be43da313da22 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 02:27:44 -0500 Subject: [PATCH 34/36] Configure bundle directory save locks --- crates/pcraft-bundle/src/lib.rs | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/crates/pcraft-bundle/src/lib.rs b/crates/pcraft-bundle/src/lib.rs index 422776fd9..acbe739b6 100644 --- a/crates/pcraft-bundle/src/lib.rs +++ b/crates/pcraft-bundle/src/lib.rs @@ -9,7 +9,7 @@ use std::collections::{BTreeMap, HashMap, HashSet}; use std::fs::{self, OpenOptions}; use std::io::Read; use std::path::{Path, PathBuf}; -use std::time::SystemTime; +use std::time::{Duration, Instant, SystemTime}; use craft_zip::{Archive, Compression, Writer}; @@ -327,14 +327,20 @@ pub struct DirectoryStats { } /// Incremental directory-bundle persistence with bounded rolling object verification. -#[derive(Default)] pub struct DirectoryBundleWriter { root: Option, generation: u64, reverify_budget: u64, + lock_wait: Duration, verified: HashMap, } +impl Default for DirectoryBundleWriter { + fn default() -> Self { + Self { root: None, generation: 0, reverify_budget: 16 << 20, lock_wait: Duration::from_secs(10), verified: HashMap::new() } + } +} + #[derive(Clone, Copy)] struct CachedFile { signature: FileSignature, @@ -356,6 +362,11 @@ impl DirectoryBundleWriter { self.reverify_budget = bytes; } + /// How long to wait for another writer to release the bundle lock. + pub fn set_lock_wait(&mut self, wait: Duration) { + self.lock_wait = wait; + } + /// Saves an entry set incrementally. Existing objects are reused only after a hash check; /// unchanged file signatures may skip checks until they enter the rolling verification /// window. The manifest is atomically published last, then stale previews and objects are @@ -379,7 +390,17 @@ impl DirectoryBundleWriter { let lock_path = root.join(craft_pcraft_format::SAVE_LOCK); refuse_symlink(&lock_path)?; let lock = OpenOptions::new().read(true).write(true).create(true).truncate(false).open(&lock_path).map_err(io_error)?; - lock.lock().map_err(io_error)?; + let start = Instant::now(); + let _lock = loop { + match lock.try_lock() { + Ok(()) => break Some(lock), + Err(std::fs::TryLockError::WouldBlock) if start.elapsed() < self.lock_wait => std::thread::sleep(Duration::from_millis(20)), + Err(std::fs::TryLockError::WouldBlock) => { + return Err(Error::Io("another save of this bundle is in progress".into())); + } + Err(std::fs::TryLockError::Error(_)) => break None, + } + }; self.generation = self.generation.saturating_add(1); let generation = self.generation; let recheck = self.rolling_recheck(&bundle.entries); From 55f6235c5b50b6e987ef2886ddca84070e1ca541 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 02:54:31 -0500 Subject: [PATCH 35/36] Simplify gridline no-fill check --- crates/drawingml/src/chart/read.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/drawingml/src/chart/read.rs b/crates/drawingml/src/chart/read.rs index 9919433e9..38ce9f605 100644 --- a/crates/drawingml/src/chart/read.rs +++ b/crates/drawingml/src/chart/read.rs @@ -443,7 +443,7 @@ fn read_axes<'a>(c: &mut Chart, pa: &'a El, primary_val: &Option) -> (Op // A gridline whose `c:spPr/a:ln` has `a:noFill` is not drawn, though the element is there. let grid = |a: &El, k: &str| { a.child(Local(k)) - .is_some_and(|g| !g.child(Local("spPr")).and_then(|p| p.child(Local("ln"))).is_some_and(|l| l.child(Local("noFill")).is_some())) + .is_some_and(|g| g.child(Local("spPr")).and_then(|p| p.child(Local("ln"))).is_none_or(|l| l.child(Local("noFill")).is_none())) }; if c.kind == ChartType::Combo && c.series.iter().any(|s| s.secondary_axis) { let primary_id = val_ax.and_then(|a| a.child(Local("axId")).and_then(|c| c.attr_local("val"))); From f6cab1980bbe422cc00c4b19d9f0f9ba337e1316 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 02:54:31 -0500 Subject: [PATCH 36/36] Preserve SVG picture fallbacks in the Presentation model --- crates/drawingml/src/read.rs | 6 ++++++ crates/drawingml/src/style.rs | 3 +++ crates/drawingml/src/write.rs | 8 ++++++++ 3 files changed, 17 insertions(+) diff --git a/crates/drawingml/src/read.rs b/crates/drawingml/src/read.rs index 496870774..9e9f8ff3e 100644 --- a/crates/drawingml/src/read.rs +++ b/crates/drawingml/src/read.rs @@ -166,6 +166,12 @@ pub fn blip_fill(src: &mut dyn Source, bf: &El) -> Option { { pf.artistic = Some(crate::style::Artistic { effect, original }); } + pf.svg = blip.and_then(|b| b.child(Local("extLst"))).and_then(|extensions| { + extensions + .elements() + .filter(|extension| extension.local() == "ext" && extension.attr_local("uri") == Some("{96DAC541-7B7A-43D3-8B79-37D633B846F1}")) + .find_map(|extension| extension.child(Local("svgBlip")).and_then(|svg| media_ref(src, svg))) + }); if let Some(b) = blip { for e in b.elements() { match e.local() { diff --git a/crates/drawingml/src/style.rs b/crates/drawingml/src/style.rs index 291ee4e76..36b5a6922 100644 --- a/crates/drawingml/src/style.rs +++ b/crates/drawingml/src/style.rs @@ -184,6 +184,9 @@ impl Default for PictureMode { #[serde(default, rename_all = "camelCase")] pub struct PictureFill { pub media: MediaId, + /// Optional vector SVG version of `media`; the regular media remains the bitmap fallback. + #[serde(skip_serializing_if = "Option::is_none")] + pub svg: Option, /// Source crop as fractions of the image (left, top, right, bottom); negative values pad. pub crop: [f64; 4], pub mode: PictureMode, diff --git a/crates/drawingml/src/write.rs b/crates/drawingml/src/write.rs index 0d9b83b1a..25bfb89db 100644 --- a/crates/drawingml/src/write.rs +++ b/crates/drawingml/src/write.rs @@ -107,6 +107,7 @@ pub fn blip_fill(w: &mut Writer, out: &mut dyn Sink, tag: &str, pf: &PictureFill w.tag(tag).attr("rotWithShape", if pf.upright { "0" } else { "1" }).open(); let adj = &pf.adjust; let has_kids = pf.alpha.is_some() + || pf.svg.is_some() || adj.brightness != 0.0 || adj.contrast != 0.0 || adj.grayscale @@ -168,6 +169,13 @@ pub fn blip_fill(w: &mut Writer, out: &mut dyn Sink, tag: &str, pf: &PictureFill w.close(); w.close(); } + if let Some(svg) = pf.svg.and_then(|id| out.media_rel(id, "image")) { + w.open("a:extLst", &[]); + w.tag("a:ext").attr("uri", "{96DAC541-7B7A-43D3-8B79-37D633B846F1}").open(); + w.tag("asvg:svgBlip").attr("xmlns:asvg", "http://schemas.microsoft.com/office/drawing/2016/SVG/main").attr("r:embed", &svg).empty(); + w.close(); + w.close(); + } w.close(); } else { w.tag("a:blip").list(a).empty();