From 30710f5b8ce0dc85782b77eecbe83e2c4d118f39 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 06:50:58 +0000 Subject: [PATCH 1/8] feat(format): use shared native document JSON contract --- Cargo.lock | 24 +++++++++- Cargo.toml | 1 + crates/format/Cargo.toml | 1 + crates/format/src/lib.rs | 97 ++++++++-------------------------------- 4 files changed, 43 insertions(+), 80 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2544abc87..58cc4b56f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1018,7 +1018,7 @@ source = "git+https://github.com/storytold/craft-libs?rev=d13ac58a15c45d618f2591 dependencies = [ "craft-figma-rest", "craft-kiwi", - "craft-zip", + "craft-zip 0.1.0 (git+https://github.com/storytold/craft-libs?rev=d13ac58a15c45d618f25910772502e38c4ce1d4c)", "flate2", "kurbo", "ruzstd", @@ -1026,6 +1026,17 @@ dependencies = [ "thiserror 2.0.21", ] +[[package]] +name = "craft-figcraft-format" +version = "0.1.0" +source = "git+https://github.com/storytold/craft-libs?rev=111b522d0#111b522d05bcc75bb4f15eaa91496eb5f9c6a54f" +dependencies = [ + "craft-zip 0.1.0 (git+https://github.com/storytold/craft-libs?rev=111b522d0)", + "serde", + "serde_json", + "thiserror 2.0.21", +] + [[package]] name = "craft-figma-rest" version = "0.1.0" @@ -1216,6 +1227,16 @@ dependencies = [ "wayland-client", ] +[[package]] +name = "craft-zip" +version = "0.1.0" +source = "git+https://github.com/storytold/craft-libs?rev=111b522d0#111b522d05bcc75bb4f15eaa91496eb5f9c6a54f" +dependencies = [ + "flate2", + "thiserror 2.0.21", + "zip 9.1.1", +] + [[package]] name = "craft-zip" version = "0.1.0" @@ -1976,6 +1997,7 @@ name = "figcraft-format" version = "0.18.1" dependencies = [ "craft-fig", + "craft-figcraft-format", "craft-figma-rest", "figcraft-doc", "figcraft-layout", diff --git a/Cargo.toml b/Cargo.toml index cea4e7c9d..ee45efdab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,6 +60,7 @@ craft-settings = { git = "https://github.com/storytold/craft-libs", rev = "e11f9 # a .fig message onto the REST file JSON, and that JSON's shared helpers. These crates share # types, so they stay on one rev; they share none with the crates above. craft-fig = { git = "https://github.com/storytold/craft-libs", rev = "d13ac58a15c45d618f25910772502e38c4ce1d4c" } +craft-figcraft-format = { git = "https://github.com/storytold/craft-libs", rev = "111b522d0" } craft-figma-rest = { git = "https://github.com/storytold/craft-libs", rev = "d13ac58a15c45d618f25910772502e38c4ce1d4c" } # foundation diff --git a/crates/format/Cargo.toml b/crates/format/Cargo.toml index a548b7051..9424b95e0 100644 --- a/crates/format/Cargo.toml +++ b/crates/format/Cargo.toml @@ -21,4 +21,5 @@ sha1 = { workspace = true } sha2 = { workspace = true } image = { workspace = true } craft-fig = { workspace = true } +craft-figcraft-format = { workspace = true } craft-figma-rest = { workspace = true } diff --git a/crates/format/src/lib.rs b/crates/format/src/lib.rs index 3fb3049ae..6abf855cb 100644 --- a/crates/format/src/lib.rs +++ b/crates/format/src/lib.rs @@ -35,14 +35,11 @@ use std::io::{Read, Seek, Write}; use std::sync::Arc; use figcraft_doc::{Document, ImageData}; -use serde::de::{Deserializer, IgnoredAny, MapAccess, Visitor}; use serde::{Deserialize, Serialize, Serializer}; use serde_json::{Value, json}; use sha2::Digest; -pub const EXTENSION: &str = "figcraft"; -pub const FORMAT: &str = "figcraft"; -pub const VERSION: u32 = 1; +pub use craft_figcraft_format::{EXTENSION, FORMAT, VERSION}; /// Largest archive member we read (guards against zip bombs). const MAX_MEMBER: u64 = 512 * 1024 * 1024; /// `document.json` may unpack to [`DOCUMENT_RATIO`] times what it takes in the archive, within @@ -243,33 +240,14 @@ pub(crate) fn image_data(data: Vec) -> ImageData { ImageData { bytes: Arc::new(data), width: w, height: h } } -/// `document.json`: the format header, the image index and the document's own fields, written -/// straight from the document (no intermediate JSON tree). -#[derive(Serialize)] -struct DocumentJson<'a> { - format: &'static str, - version: u32, - #[serde(rename = "imageIndex", serialize_with = "image_index")] - images: &'a BTreeMap, - #[serde(flatten)] - doc: &'a Document, -} +/// The image hashes a `document.json` lists in its `imageIndex` (`None` without one). +type ImageIndex = Option>; -impl<'a> DocumentJson<'a> { - fn new(doc: &'a Document) -> Self { - Self { format: FORMAT, version: VERSION, images: &doc.images, doc } - } -} - -/// The `imageIndex`: `[{hash, width, height}]` for every image the document holds. -fn image_index(images: &&BTreeMap, s: S) -> Result { - #[derive(Serialize)] - struct Entry<'a> { - hash: &'a str, - width: u32, - height: u32, - } - s.collect_seq(images.iter().map(|(hash, i)| Entry { hash, width: i.width, height: i.height })) +fn native_image_index(doc: &Document) -> Vec { + doc.images + .iter() + .map(|(hash, image)| craft_figcraft_format::ImageIndexEntry { hash: hash.clone(), width: image.width, height: image.height }) + .collect() } /// A serializer adapter that writes `0` in place of every NaN or infinite float. JSON has no such @@ -452,13 +430,17 @@ impl serde::ser::SerializeStructVariant f /// Serialize the document (without images) to the JSON value stored in `document.json`. /// NaN and infinite numbers are written as `0`. pub fn to_json(doc: &Document) -> Result { - DocumentJson::new(doc).serialize(Finite(serde_json::value::Serializer)).map_err(io_err) + { + let mut bytes = Vec::new(); + craft_figcraft_format::write_document_json(&FiniteValue(doc), &native_image_index(doc), &mut bytes).map_err(io_err)?; + serde_json::from_slice(&bytes).map_err(io_err) + } } /// Write the document (without images) as `document.json` content to `w`. /// NaN and infinite numbers are written as `0`. pub fn write_json(doc: &Document, w: impl Write) -> Result<(), FormatError> { - DocumentJson::new(doc).serialize(Finite(&mut serde_json::Serializer::new(w))).map_err(io_err) + craft_figcraft_format::write_document_json(&FiniteValue(doc), &native_image_index(doc), w).map_err(io_err) } /// Parse `document.json` content. @@ -467,64 +449,21 @@ pub fn from_json(text: &str) -> Result<(Document, Vec), FormatError> { } /// The image hashes a `document.json` lists in its `imageIndex` (`None` without one). -type ImageIndex = Option>; - -/// The header of a `document.json` (its format version and image index), read in a pass that -/// skips over everything else without building it. -#[derive(Default)] -struct Header { - version: Option, - images: ImageIndex, -} - -impl<'de> Deserialize<'de> for Header { - fn deserialize>(d: D) -> Result { - struct HeaderVisitor; - impl<'de> Visitor<'de> for HeaderVisitor { - type Value = Header; - fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - f.write_str("a FigCraft document object") - } - fn visit_map>(self, mut map: A) -> Result { - let mut h = Header::default(); - while let Some(key) = map.next_key::()? { - match key.as_str() { - "version" => h.version = map.next_value::()?.as_u64(), - "imageIndex" => { - h.images = map - .next_value::()? - .as_array() - .map(|a| a.iter().filter_map(|i| i.get("hash").and_then(Value::as_str).map(str::to_string)).collect()); - } - _ => { - map.next_value::()?; - } - } - } - Ok(h) - } - } - d.deserialize_map(HeaderVisitor) - } -} /// Parse `document.json` content; also returns its [`ImageIndex`]. The header is checked first /// (a newer file is refused before its document is read), then the document is deserialized /// straight from the text. fn parse_json(text: &str) -> Result<(Document, Vec, ImageIndex), FormatError> { let corrupt = |e: serde_json::Error| FormatError::Corrupt(e.to_string()); - let header: Header = serde_json::from_str(text).map_err(corrupt)?; - if let Some(ver) = header.version - && ver > u64::from(VERSION) - { - return Err(FormatError::Newer(u32::try_from(ver).unwrap_or(u32::MAX))); - } + let header = craft_figcraft_format::read_header(text).map_err(corrupt)?; + craft_figcraft_format::check_version(&header).map_err(|e| FormatError::Newer(e.0))?; + let images = header.images.map(|hashes| hashes.into_iter().collect()); let mut doc: Document = serde_json::from_str(text).map_err(corrupt)?; let issues = doc.validate(); if !issues.is_empty() { doc.repair(); } - Ok((doc, issues, header.images)) + Ok((doc, issues, images)) } pub(crate) fn io_err(e: impl std::fmt::Display) -> FormatError { From d14728285193b3ecefd1c0051d3bf45ac9a1dfc0 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 08:08:26 +0000 Subject: [PATCH 2/8] move FigCraft history indexes into shared format --- Cargo.lock | 7 +- Cargo.toml | 2 +- crates/format/src/lib.rs | 354 +++++++++------------- crates/format/src/library.rs | 22 +- crates/format/src/unpacked_total_tests.rs | 4 +- 5 files changed, 168 insertions(+), 221 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 58cc4b56f..b9cf7574b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1029,12 +1029,13 @@ dependencies = [ [[package]] name = "craft-figcraft-format" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=111b522d0#111b522d05bcc75bb4f15eaa91496eb5f9c6a54f" +source = "git+https://github.com/storytold/craft-libs?rev=a75b9df240d3e2eb98898fcbf99403ae5a1c73bb#a75b9df240d3e2eb98898fcbf99403ae5a1c73bb" dependencies = [ - "craft-zip 0.1.0 (git+https://github.com/storytold/craft-libs?rev=111b522d0)", + "craft-zip 0.1.0 (git+https://github.com/storytold/craft-libs?rev=a75b9df240d3e2eb98898fcbf99403ae5a1c73bb)", "serde", "serde_json", "thiserror 2.0.21", + "zip 9.1.1", ] [[package]] @@ -1230,7 +1231,7 @@ dependencies = [ [[package]] name = "craft-zip" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=111b522d0#111b522d05bcc75bb4f15eaa91496eb5f9c6a54f" +source = "git+https://github.com/storytold/craft-libs?rev=a75b9df240d3e2eb98898fcbf99403ae5a1c73bb#a75b9df240d3e2eb98898fcbf99403ae5a1c73bb" dependencies = [ "flate2", "thiserror 2.0.21", diff --git a/Cargo.toml b/Cargo.toml index ee45efdab..6929ac8a8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,7 +60,7 @@ craft-settings = { git = "https://github.com/storytold/craft-libs", rev = "e11f9 # a .fig message onto the REST file JSON, and that JSON's shared helpers. These crates share # types, so they stay on one rev; they share none with the crates above. craft-fig = { git = "https://github.com/storytold/craft-libs", rev = "d13ac58a15c45d618f25910772502e38c4ce1d4c" } -craft-figcraft-format = { git = "https://github.com/storytold/craft-libs", rev = "111b522d0" } +craft-figcraft-format = { git = "https://github.com/storytold/craft-libs", rev = "a75b9df240d3e2eb98898fcbf99403ae5a1c73bb" } craft-figma-rest = { git = "https://github.com/storytold/craft-libs", rev = "d13ac58a15c45d618f25910772502e38c4ce1d4c" } # foundation diff --git a/crates/format/src/lib.rs b/crates/format/src/lib.rs index 6abf855cb..c53160728 100644 --- a/crates/format/src/lib.rs +++ b/crates/format/src/lib.rs @@ -31,73 +31,26 @@ pub use library::{is_valid_library_id, read_library, read_library_within}; mod density; use std::collections::{BTreeMap, BTreeSet}; -use std::io::{Read, Seek, Write}; +use std::io::{Read, Write}; use std::sync::Arc; use figcraft_doc::{Document, ImageData}; use serde::{Deserialize, Serialize, Serializer}; -use serde_json::{Value, json}; +use serde_json::Value; +#[cfg(test)] +use serde_json::json; use sha2::Digest; -pub use craft_figcraft_format::{EXTENSION, FORMAT, VERSION}; -/// Largest archive member we read (guards against zip bombs). -const MAX_MEMBER: u64 = 512 * 1024 * 1024; -/// `document.json` may unpack to [`DOCUMENT_RATIO`] times what it takes in the archive, within -/// [`MAX_MEMBER`] and [`MAX_DOCUMENT`]: a large document saved by us (a 900 000-layer file packs -/// 1.48 GB into 244 MB) opens again, while a small archive still can't unpack into gigabytes. -/// 32-bit targets (wasm32) have a 4 GB address space in all, so their ceiling is lower. -const DOCUMENT_RATIO: u64 = 16; -#[cfg(not(target_pointer_width = "32"))] -const MAX_DOCUMENT: u64 = 4 * 1024 * 1024 * 1024; -#[cfg(target_pointer_width = "32")] -const MAX_DOCUMENT: u64 = 1024 * 1024 * 1024; -/// Most versions a file keeps: saving more is refused, extra entries in a file are skipped. -pub const MAX_VERSIONS: usize = 100; +pub use craft_figcraft_format::{ + BRANCH_INDEX, BranchEntry as NativeBranchEntry, BranchIndex as NativeBranchIndex, BranchState as NativeBranchState, DOCUMENT_RATIO, EXTENSION, + FORMAT, FormatError, IMAGE_BUDGET, ImageBudget, MAX_BRANCHES, MAX_DOCUMENT, MAX_MEMBER, MAX_UNPACKED, MAX_VERSION_BYTES, MAX_VERSION_DESCRIPTION, + MAX_VERSION_INDEX, MAX_VERSION_NAME, MAX_VERSIONS, VERSION, VersionEntry as NativeVersionEntry, VersionIndex as NativeVersionIndex, clip, + document_limit, is_valid_version_id, +}; +const JSON_LEVEL: i64 = 3; /// The export warning for colour calculations that resolve to nothing (an operand missing, or more /// than 32 variables to read), shared by the Penpot and `.fig` exports. pub(crate) const CALCULATIONS_LEFT_OUT: &str = "colour calculations that don't resolve were left out"; -/// Longest version name and description kept (chars). -pub const MAX_VERSION_NAME: usize = 200; -pub const MAX_VERSION_DESCRIPTION: usize = 2000; -/// Largest version index and total size of the version snapshots read from one file. -const MAX_VERSION_INDEX: u64 = 16 * 1024 * 1024; -const MAX_VERSION_BYTES: u64 = 1024 * 1024 * 1024; -/// Most images, and image bytes together, read from one file (members past it are left out). -const IMAGE_BUDGET: ImageBudget = ImageBudget { images: 20_000, bytes: 4 * 1024 * 1024 * 1024 }; -/// Most bytes one open of a `.figcraft` file unpacks, every area together: its document, images, -/// published library, version history and branches ([`Unpacked`], #1079). Each area keeps its own -/// cap; their sum was about 10 GB. Read in that order, the document (a 900 000-layer one unpacks -/// to 1.5 GB) and the images up to their own 4 GiB cap fit, so a file whose images pass that cap -/// (#1313: an 84 MiB document with 7.3 GiB of images) keeps as many as before; the history and -/// branches share what is left. 32-bit targets (wasm32) have 4 GB of address space in all. -#[cfg(not(target_pointer_width = "32"))] -const MAX_UNPACKED: u64 = 6 * 1024 * 1024 * 1024; -#[cfg(target_pointer_width = "32")] -const MAX_UNPACKED: u64 = 2 * 1024 * 1024 * 1024; -const BRANCH_INDEX: &str = "branches/index.json"; -/// Most branches a file keeps: making more is refused, extra entries in a file are skipped. -pub const MAX_BRANCHES: usize = 100; -/// Deflate level of JSON members (see [`json_options`]). -const JSON_LEVEL: i64 = 3; - -#[derive(Debug)] -pub enum FormatError { - Io(String), - Corrupt(String), - Newer(u32), -} - -impl std::fmt::Display for FormatError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - FormatError::Io(e) => write!(f, "{e}"), - FormatError::Corrupt(e) => write!(f, "not a valid FigCraft file: {e}"), - FormatError::Newer(v) => write!(f, "this file was saved by a newer FigCraft (format version {v})"), - } - } -} - -impl std::error::Error for FormatError {} /// One entry of a file's version history: a named snapshot of the document. #[derive(Clone, Debug)] @@ -188,16 +141,6 @@ pub struct Branch { pub versions: Vec, } -/// Whether `id` can name a version (and its archive entry). -pub fn is_valid_version_id(id: &str) -> bool { - (1..=64).contains(&id.len()) && id.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_') -} - -/// `s` cut to at most `max` chars. -pub fn clip(s: &str, max: usize) -> String { - s.chars().take(max).collect() -} - /// A file as read: the document, the repairs made (and versions skipped), and its version /// history, oldest first. #[derive(Debug)] @@ -470,44 +413,35 @@ pub(crate) fn io_err(e: impl std::fmt::Display) -> FormatError { FormatError::Io(e.to_string()) } -/// Add one member to the archive. -pub(crate) fn put( - z: &mut zip::ZipWriter, +/// JSON compression for Penpot and external ZIP exporters. +pub(crate) fn json_options() -> zip::write::SimpleFileOptions { + zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Deflated).compression_level(Some(JSON_LEVEL)) +} + +/// Add one member to the native archive. +pub(crate) fn put( + z: &mut craft_figcraft_format::ArchiveWriter, name: &str, - opts: zip::write::SimpleFileOptions, + compression: craft_figcraft_format::Compression, bytes: &[u8], ) -> Result<(), FormatError> { - z.start_file(name, opts).map_err(io_err)?; - z.write_all(bytes).map_err(io_err) + z.add(name, bytes, compression).map_err(io_err) } /// `files` (name, bytes; a `/` in a name makes a folder) as one zip archive: several exports /// delivered as a single download (#969). pub fn zip_files<'a>(files: impl IntoIterator) -> Result, FormatError> { - let mut buf = std::io::Cursor::new(Vec::new()); - { - let mut z = zip::ZipWriter::new(&mut buf); - for (name, bytes) in files { - put(&mut z, name, zip::write::SimpleFileOptions::default(), bytes)?; - } - z.finish().map_err(io_err)?; - } - Ok(buf.into_inner()) + craft_figcraft_format::write_archive(files.into_iter().map(|(name, bytes)| craft_figcraft_format::Member { + name, + bytes, + compression: craft_figcraft_format::Compression::Deflated, + })) + .map_err(io_err) } /// Add `doc` as a `document.json`-style member, serialized straight into the compressor. -fn put_json(z: &mut zip::ZipWriter, name: &str, doc: &Document) -> Result<(), FormatError> { - z.start_file(name, json_options()).map_err(io_err)?; - // The serializer writes many small pieces: batch them before they reach the compressor. - let mut w = std::io::BufWriter::with_capacity(1 << 16, z); - write_json(doc, &mut w)?; - w.flush().map_err(io_err) -} - -/// How JSON members are compressed. Document JSON is very repetitive: a fast level compresses it -/// nearly as well as the default one (about 15% larger) in half the time. -pub(crate) fn json_options() -> zip::write::SimpleFileOptions { - zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Deflated).compression_level(Some(JSON_LEVEL)) +fn put_json(z: &mut craft_figcraft_format::ArchiveWriter, name: &str, doc: &Document) -> Result<(), FormatError> { + z.add_document_json(name, &FiniteValue(doc), &native_image_index(doc)).map_err(io_err) } /// Write a `.figcraft` archive to bytes. @@ -527,50 +461,62 @@ pub fn save_bytes_with_history( versions: &[Version], branches: &[Branch], ) -> Result, FormatError> { - let mut buf = std::io::Cursor::new(Vec::new()); - { - let mut z = zip::ZipWriter::new(&mut buf); - let stored = zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); - put_json(&mut z, "document.json", doc)?; - // Every image once, whether the document, a version or a branch uses it. - let mut written = BTreeSet::new(); - let branch_images = branches.iter().flat_map(|b| [&b.base.images, &b.head.images].into_iter().chain(b.versions.iter().map(Version::images))); - for (hash, img) in std::iter::once(&doc.images).chain(versions.iter().map(Version::images)).chain(branch_images).flatten() { - if written.insert(hash) { - put(&mut z, &format!("images/{hash}"), stored, &img.bytes)?; - } + let mut z = craft_figcraft_format::ArchiveWriter::new(); + let stored = craft_figcraft_format::Compression::Stored; + put_json(&mut z, "document.json", doc)?; + // Every image once, whether the document, a version or a branch uses it. + let mut written = BTreeSet::new(); + let branch_images = branches.iter().flat_map(|b| [&b.base.images, &b.head.images].into_iter().chain(b.versions.iter().map(Version::images))); + for (hash, img) in std::iter::once(&doc.images).chain(versions.iter().map(Version::images)).chain(branch_images).flatten() { + if written.insert(hash) { + put(&mut z, &format!("images/{hash}"), stored, img.bytes.as_slice())?; } - if let Some(t) = thumbnail_png { - put(&mut z, "thumbnail.png", stored, t)?; - } - put_versions(&mut z, "versions", versions)?; - if !branches.is_empty() { - let mut ids = BTreeSet::new(); - let mut index = Vec::with_capacity(branches.len()); - for b in branches { - if !is_valid_version_id(&b.id) || !ids.insert(b.id.as_str()) { - return Err(FormatError::Io(format!("invalid or duplicate branch id `{}`", b.id))); - } - put_json(&mut z, &format!("branches/{}/base.json", b.id), &b.base)?; - put_json(&mut z, &format!("branches/{}/head.json", b.id), &b.head)?; - put_versions(&mut z, &format!("branches/{}/versions", b.id), &b.versions)?; - let images = |d: &Document| d.images.keys().cloned().collect::>(); - index.push(json!({"id": b.id, "name": b.name, "time": b.time, "client": b.client, "state": b.state, - "baseImages": images(&b.base), "headImages": images(&b.head)})); - } - put(&mut z, BRANCH_INDEX, json_options(), &serde_json::to_vec(&json!({"branches": index})).map_err(io_err)?)?; - } - if let Some(lib) = &doc.published { - library::write(&mut z, lib, &written)?; - } - z.finish().map_err(io_err)?; } - Ok(buf.into_inner()) + if let Some(t) = thumbnail_png { + put(&mut z, "thumbnail.png", stored, t)?; + } + put_versions(&mut z, "versions", versions)?; + if !branches.is_empty() { + let mut ids = BTreeSet::new(); + let mut index = Vec::with_capacity(branches.len()); + for b in branches { + if !is_valid_version_id(&b.id) || !ids.insert(b.id.as_str()) { + return Err(FormatError::Io(format!("invalid or duplicate branch id `{}`", b.id))); + } + put_json(&mut z, &format!("branches/{}/base.json", b.id), &b.base)?; + put_json(&mut z, &format!("branches/{}/head.json", b.id), &b.head)?; + put_versions(&mut z, &format!("branches/{}/versions", b.id), &b.versions)?; + let images = |d: &Document| d.images.keys().cloned().collect::>(); + index.push(NativeBranchEntry { + id: b.id.clone(), + name: b.name.clone(), + time: b.time, + client: b.client, + state: match b.state { + BranchState::Active => NativeBranchState::Active, + BranchState::Merged => NativeBranchState::Merged, + BranchState::Archived => NativeBranchState::Archived, + }, + base_images: images(&b.base), + head_images: images(&b.head), + }); + } + put( + &mut z, + BRANCH_INDEX, + craft_figcraft_format::Compression::Level(3), + &serde_json::to_vec(&NativeBranchIndex { branches: index }).map_err(io_err)?, + )?; + } + if let Some(lib) = &doc.published { + library::write(&mut z, lib, &written)?; + } + z.finish().map_err(io_err) } /// Write version history `versions` (oldest first) under `dir`: `/index.json` and one /// `/.json` snapshot per version. Nothing when there are none. -fn put_versions(z: &mut zip::ZipWriter, dir: &str, versions: &[Version]) -> Result<(), FormatError> { +fn put_versions(z: &mut craft_figcraft_format::ArchiveWriter, dir: &str, versions: &[Version]) -> Result<(), FormatError> { if versions.is_empty() { return Ok(()); } @@ -583,16 +529,23 @@ fn put_versions(z: &mut zip::ZipWriter, dir: &str, versions: let name = format!("{dir}/{}.json", v.id); match &v.snapshot { Snapshot::Doc(d) => put_json(z, &name, d)?, - Snapshot::Stored { json, .. } => put(z, &name, json_options(), json)?, + Snapshot::Stored { json, .. } => put(z, &name, craft_figcraft_format::Compression::Level(3), json)?, } - let images: Vec<&String> = v.images().keys().collect(); - let mut entry = json!({"id": v.id, "name": v.name, "description": v.description, "time": v.time, "images": images}); - if v.auto { - entry["auto"] = json!(true); - } - index.push(entry); + index.push(NativeVersionEntry { + id: v.id.clone(), + name: v.name.clone(), + description: v.description.clone(), + time: v.time, + auto: v.auto, + images: v.images().keys().cloned().collect(), + }); } - put(z, &format!("{dir}/index.json"), json_options(), &serde_json::to_vec(&json!({"versions": index})).map_err(io_err)?)?; + put( + z, + &format!("{dir}/index.json"), + craft_figcraft_format::Compression::Level(3), + &serde_json::to_vec(&NativeVersionIndex { versions: index }).map_err(io_err)?, + )?; Ok(()) } @@ -602,16 +555,25 @@ pub fn load_bytes(bytes: &[u8]) -> Result<(Document, Vec), FormatError> } /// Read archive member `name`, refusing members larger than `limit` bytes. -pub(crate) fn read_member(z: &mut zip::ZipArchive, name: &str, limit: u64) -> Result, FormatError> { +fn read_native_member(z: &mut impl craft_figcraft_format::NativeArchive, name: &str, limit: u64) -> Result, FormatError> { + match z.read_member(name, limit) { + Ok(Some(bytes)) => Ok(bytes), + Ok(None) => Err(FormatError::Corrupt(format!("{name}: not found"))), + Err(error) if error.is_limit() => Err(FormatError::Corrupt(format!("{name} is too large"))), + Err(error) => Err(FormatError::Corrupt(format!("{name}: {error}"))), + } +} + +/// Read a member from a foreign ZIP archive, refusing entries larger than `limit`. +pub(crate) fn read_member(z: &mut zip::ZipArchive, name: &str, limit: u64) -> Result, FormatError> { let f = z.by_name(name).map_err(|e| FormatError::Corrupt(format!("{name}: {e}")))?; - let too_large = || FormatError::Corrupt(format!("{name} is too large")); if f.size() > limit { - return Err(too_large()); + return Err(FormatError::Corrupt(format!("{name} is too large"))); } let mut data = Vec::new(); f.take(limit.saturating_add(1)).read_to_end(&mut data).map_err(|e| FormatError::Corrupt(format!("{name}: {e}")))?; if data.len() as u64 > limit { - return Err(too_large()); + return Err(FormatError::Corrupt(format!("{name} is too large"))); } Ok(data) } @@ -636,19 +598,19 @@ impl Unpacked { } /// Whether member `name` (by its recorded size) needs more than is left. - fn exceeded(&self, z: &mut zip::ZipArchive, name: &str) -> bool { - z.by_name(name).is_ok_and(|f| f.size() > self.left) + fn exceeded(&self, z: &mut impl craft_figcraft_format::NativeArchive, name: &str) -> bool { + z.declared_size(name).is_some_and(|size| size > self.left) } /// [`read_member`] within `limit` (the area's own cap) and what is left of the total, which the /// member then uses up. - pub(crate) fn read(&mut self, z: &mut zip::ZipArchive, name: &str, limit: u64) -> Result, FormatError> { - if self.exceeded(z, name) && z.by_name(name).is_ok_and(|f| f.size() <= limit) { + pub(crate) fn read(&mut self, z: &mut impl craft_figcraft_format::NativeArchive, name: &str, limit: u64) -> Result, FormatError> { + if self.exceeded(z, name) && z.declared_size(name).is_some_and(|size| size <= limit) { return Err(FormatError::Corrupt(format!("{name}: {}", self.passed()))); } let cap = limit.min(self.left); - let size = z.by_name(name).map_or(0, |f| f.size()); - let data = read_member(z, name, cap); + let size = z.declared_size(name).unwrap_or(0); + let data = read_native_member(z, name, cap); // A read that failed part way (a bad checksum at the end, say) did its unpacking all the // same: it is charged in full, so an index naming that member again can't repeat it. One // refused for its recorded size before any unpacking costs nothing. @@ -659,18 +621,6 @@ impl Unpacked { } } -/// Largest `document.json` read from an archive of `len` bytes in which it takes `packed`. -pub(crate) fn document_limit(packed: u64, len: u64) -> u64 { - packed.min(len).saturating_mul(DOCUMENT_RATIO).clamp(MAX_MEMBER, MAX_DOCUMENT) -} - -/// How many images, and how many bytes of them together, one file may give. -#[derive(Clone, Copy, Debug)] -pub(crate) struct ImageBudget { - pub images: usize, - pub bytes: u64, -} - /// Read a `.figcraft` archive (or bare JSON) with its version history. pub fn load_bytes_with_versions(bytes: &[u8]) -> Result { load_within(bytes, IMAGE_BUDGET) @@ -689,8 +639,10 @@ pub(crate) fn load_limited(bytes: &[u8], budget: ImageBudget, total: u64) -> Res let (doc, issues) = from_json(text)?; return Ok(Loaded { issues, ..Loaded::from(doc) }); } - let mut z = zip::ZipArchive::new(std::io::Cursor::new(bytes)).map_err(|e| FormatError::Corrupt(e.to_string()))?; - let packed = z.by_name("document.json").map(|f| f.compressed_size()).unwrap_or(0); + let mut z = + craft_figcraft_format::Archive::open(bytes, craft_figcraft_format::ArchiveLimits { max_entries: u64::MAX, max_total_bytes: u64::MAX }) + .map_err(|e| FormatError::Corrupt(e.to_string()))?; + let packed = z.compressed_size("document.json").unwrap_or(0); let limit = document_limit(packed, bytes.len() as u64); let mut unpacked = Unpacked::new(total); let text = String::from_utf8(unpacked.read(&mut z, "document.json", limit)?).map_err(|e| FormatError::Corrupt(e.to_string()))?; @@ -721,13 +673,14 @@ pub(crate) fn load_limited(bytes: &[u8], budget: ImageBudget, total: u64) -> Res /// The archive's `images/` members by hash, within `budget`, and the ones that couldn't be read /// with why. The document's own images (`own`) are read first, so a file past the budget loses /// the images only its history uses before its own. -fn read_images( - z: &mut zip::ZipArchive, +fn read_images( + z: &mut impl craft_figcraft_format::NativeArchive, own: Option<&BTreeSet>, budget: ImageBudget, unpacked: &mut Unpacked, ) -> ImagesRead { - let mut hashes: Vec = z.file_names().filter_map(|n| n.strip_prefix("images/")).filter(|h| !h.is_empty()).map(str::to_string).collect(); + let mut hashes: Vec = + z.names().into_iter().filter_map(|n| n.strip_prefix("images/").map(str::to_owned)).filter(|h| !h.is_empty()).collect(); hashes.sort_by_key(|h| !own.is_some_and(|o| o.contains(h))); let mut images = BTreeMap::new(); let mut unread = Vec::new(); @@ -748,7 +701,7 @@ fn read_images( unread.push((hash, FormatError::Corrupt(format!("the file has more than {} images", budget.images)))); continue; } - if z.by_name(&name).is_ok_and(|f| f.size() > left) { + if z.declared_size(&name).is_some_and(|size| size > left) { let mb = budget.bytes / (1024 * 1024); past_budget.insert(hash.clone()); unread.push((hash, FormatError::Corrupt(format!("the file's images together pass {mb} MB")))); @@ -819,8 +772,8 @@ fn report_lost_images( /// The version history kept under `dir` in an archive (none when it has no index), reading at /// most `budget` bytes of snapshots. Damaged entries are skipped and reported in `issues`; nothing /// here fails the load. -fn read_versions( - z: &mut zip::ZipArchive, +fn read_versions( + z: &mut impl craft_figcraft_format::NativeArchive, dir: &str, images: &BTreeMap, issues: &mut Vec, @@ -828,12 +781,12 @@ fn read_versions( unpacked: &mut Unpacked, ) -> Vec { let index_name = format!("{dir}/index.json"); - if z.index_for_name(&index_name).is_none() { + if !z.contains(&index_name) { return Vec::new(); } let index = match unpacked .read(z, &index_name, MAX_VERSION_INDEX) - .and_then(|b| serde_json::from_slice::(&b).map_err(|e| FormatError::Corrupt(e.to_string()))) + .and_then(|b| serde_json::from_slice::(&b).map_err(|e| FormatError::Corrupt(e.to_string()))) { Ok(v) => v, Err(e) => { @@ -841,14 +794,13 @@ fn read_versions( return Vec::new(); } }; - let entries = index.get("versions").and_then(Value::as_array).map(Vec::as_slice).unwrap_or_default(); let mut out: Vec = Vec::new(); - for e in entries { + for e in index.versions { if out.len() >= MAX_VERSIONS { issues.push(format!("only the first {MAX_VERSIONS} versions were kept")); break; } - let id = e.get("id").and_then(Value::as_str).unwrap_or_default(); + let id = e.id.as_str(); if !is_valid_version_id(id) || out.iter().any(|v| v.id == id) { issues.push("a version with an invalid id was skipped".into()); continue; @@ -861,21 +813,13 @@ fn read_versions( } }; *budget = budget.saturating_sub(json.len() as u64); - let text = |k: &str, max: usize| e.get(k).and_then(Value::as_str).map(|s| clip(s, max)).unwrap_or_default(); - let images = e - .get("images") - .and_then(Value::as_array) - .into_iter() - .flatten() - .filter_map(Value::as_str) - .filter_map(|h| images.get(h).map(|i| (h.to_string(), i.clone()))) - .collect(); + let images = e.images.iter().filter_map(|h| images.get(h).map(|i| (h.clone(), i.clone()))).collect(); out.push(Version { id: id.to_string(), - name: text("name", MAX_VERSION_NAME), - description: text("description", MAX_VERSION_DESCRIPTION), - time: e.get("time").and_then(Value::as_u64).unwrap_or(0), - auto: e.get("auto").and_then(Value::as_bool).unwrap_or(false), + name: clip(&e.name, MAX_VERSION_NAME), + description: clip(&e.description, MAX_VERSION_DESCRIPTION), + time: e.time, + auto: e.auto, snapshot: Snapshot::Stored { json: Arc::new(json), images }, }); } @@ -884,18 +828,18 @@ fn read_versions( /// The branches of an archive (none when it has no index), parsed. Damaged entries are skipped and /// reported in `issues`; nothing here fails the load. -fn read_branches( - z: &mut zip::ZipArchive, +fn read_branches( + z: &mut impl craft_figcraft_format::NativeArchive, images: &BTreeMap, issues: &mut Vec, unpacked: &mut Unpacked, ) -> Vec { - if z.index_for_name(BRANCH_INDEX).is_none() { + if !z.contains(BRANCH_INDEX) { return Vec::new(); } let index = match unpacked .read(z, BRANCH_INDEX, MAX_VERSION_INDEX) - .and_then(|b| serde_json::from_slice::(&b).map_err(|e| FormatError::Corrupt(e.to_string()))) + .and_then(|b| serde_json::from_slice::(&b).map_err(|e| FormatError::Corrupt(e.to_string()))) { Ok(v) => v, Err(e) => { @@ -903,28 +847,26 @@ fn read_branches( return Vec::new(); } }; - let entries = index.get("branches").and_then(Value::as_array).map(Vec::as_slice).unwrap_or_default(); let mut out: Vec = Vec::new(); let mut budget = MAX_VERSION_BYTES; - for e in entries { + for e in index.branches { if out.len() >= MAX_BRANCHES { issues.push(format!("only the first {MAX_BRANCHES} branches were kept")); break; } - let id = e.get("id").and_then(Value::as_str).unwrap_or_default(); + let id = e.id.as_str(); if !is_valid_version_id(id) || out.iter().any(|b| b.id == id) { issues.push("a branch with an invalid id was skipped".into()); continue; } - let mut snapshot = |part: &str, list: &str| -> Result, FormatError> { + let mut snapshot = |part: &str, hashes: &[String]| -> Result, FormatError> { let json = unpacked.read(z, &format!("branches/{id}/{part}.json"), budget.min(MAX_MEMBER))?; budget = budget.saturating_sub(json.len() as u64); let (mut doc, _) = from_json(std::str::from_utf8(&json).map_err(|e| FormatError::Corrupt(e.to_string()))?)?; - let hashes = e.get(list).and_then(Value::as_array).into_iter().flatten().filter_map(Value::as_str); - doc.images = hashes.filter_map(|h| images.get(h).map(|i| (h.to_string(), i.clone()))).collect(); + doc.images = hashes.iter().filter_map(|h| images.get(h).map(|i| (h.clone(), i.clone()))).collect(); Ok(Arc::new(doc)) }; - let (base, head) = match (snapshot("base", "baseImages"), snapshot("head", "headImages")) { + let (base, head) = match (snapshot("base", &e.base_images), snapshot("head", &e.head_images)) { (Ok(b), Ok(h)) => (b, h), (Err(err), _) | (_, Err(err)) => { issues.push(format!("branch `{id}` was skipped: {err}")); @@ -933,10 +875,14 @@ fn read_branches( }; out.push(Branch { id: id.to_string(), - name: e.get("name").and_then(Value::as_str).map(|s| clip(s, MAX_VERSION_NAME)).unwrap_or_default(), - time: e.get("time").and_then(Value::as_u64).unwrap_or(0), - client: e.get("client").and_then(Value::as_u64).and_then(|c| u32::try_from(c).ok()).unwrap_or(0), - state: e.get("state").and_then(|s| BranchState::deserialize(s).ok()).unwrap_or(BranchState::Active), + name: clip(&e.name, MAX_VERSION_NAME), + time: e.time, + client: e.client, + state: match e.state { + NativeBranchState::Active => BranchState::Active, + NativeBranchState::Merged => BranchState::Merged, + NativeBranchState::Archived => BranchState::Archived, + }, versions: read_versions(z, &format!("branches/{id}/versions"), images, issues, &mut budget, unpacked), base, head, diff --git a/crates/format/src/library.rs b/crates/format/src/library.rs index 99047f2be..a40f6d718 100644 --- a/crates/format/src/library.rs +++ b/crates/format/src/library.rs @@ -10,12 +10,12 @@ //! so a missing, damaged or oversized library file is an error, never a crash. use std::collections::{BTreeMap, BTreeSet}; -use std::io::{Read, Seek, Write}; +use std::io::{Read, Seek}; use std::sync::Arc; use figcraft_doc::{ImageData, LibraryManifest, NodeKind, PublishedLibrary, used_images}; -use crate::{FormatError, MAX_MEMBER, Unpacked, from_json, image_data, io_err, json_options, put, put_json}; +use crate::{FormatError, MAX_MEMBER, Unpacked, from_json, image_data, io_err, put, put_json}; const MANIFEST: &str = "library/manifest.json"; const SNAPSHOT: &str = "library/document.json"; @@ -39,13 +39,13 @@ pub fn is_valid_library_id(id: &str) -> bool { } /// Write the published library's entries. `written` are the images already in `images/`. -pub(crate) fn write(z: &mut zip::ZipWriter, lib: &PublishedLibrary, written: &BTreeSet<&String>) -> Result<(), FormatError> { - let stored = zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); - put(z, MANIFEST, json_options(), &serde_json::to_vec(&lib.manifest).map_err(io_err)?)?; +pub(crate) fn write(z: &mut craft_figcraft_format::ArchiveWriter, lib: &PublishedLibrary, written: &BTreeSet<&String>) -> Result<(), FormatError> { + let stored = craft_figcraft_format::Compression::Stored; + put(z, MANIFEST, craft_figcraft_format::Compression::Level(3), &serde_json::to_vec(&lib.manifest).map_err(io_err)?)?; put_json(z, SNAPSHOT, &lib.doc)?; for (hash, img) in &lib.doc.images { if !written.contains(hash) && ImageData::is_valid_key(hash) { - put(z, &format!("{IMAGES}{hash}"), stored, &img.bytes)?; + put(z, &format!("{IMAGES}{hash}"), stored, img.bytes.as_slice())?; } } Ok(()) @@ -54,12 +54,12 @@ pub(crate) fn write(z: &mut zip::ZipWriter, lib: &PublishedL /// The published library in an archive (`None` when the file was never published). `own` are /// the images already read from `images/`; what it unpacks comes out of `unpacked` (beyond the /// per-part caps). -pub(crate) fn read( - z: &mut zip::ZipArchive, +pub(crate) fn read( + z: &mut impl craft_figcraft_format::NativeArchive, own: &BTreeMap, unpacked: &mut Unpacked, ) -> Result, FormatError> { - if z.index_for_name(MANIFEST).is_none() { + if !z.contains(MANIFEST) { return Ok(None); } let mut manifest: LibraryManifest = serde_json::from_slice(&unpacked.read(z, MANIFEST, MAX_MANIFEST)?).map_err(corrupt)?; @@ -77,7 +77,7 @@ pub(crate) fn read( doc.images.insert(hash, img.clone()); continue; } - let name = [format!("{IMAGES}{hash}"), format!("images/{hash}")].into_iter().find(|n| z.index_for_name(n).is_some()); + let name = [format!("{IMAGES}{hash}"), format!("images/{hash}")].into_iter().find(|n| z.contains(n)); // A missing or oversized image shows as a placeholder, like in any other file. let Some(data) = name.and_then(|n| unpacked.read(z, &n, budget.min(MAX_MEMBER)).ok()) else { continue }; budget = budget.saturating_sub(data.len() as u64); @@ -107,7 +107,7 @@ pub fn read_library(r: R) -> Result, Fo /// archive that would unpack to more is refused, or loses the images past it), for libraries from /// people the reader doesn't trust (a server). pub fn read_library_within(r: R, max_bytes: u64) -> Result, FormatError> { - let mut z = zip::ZipArchive::new(r).map_err(corrupt)?; + let mut z = craft_figcraft_format::SeekArchive::open(r, u64::MAX).map_err(corrupt)?; read(&mut z, &BTreeMap::new(), &mut Unpacked::new(max_bytes)) } diff --git a/crates/format/src/unpacked_total_tests.rs b/crates/format/src/unpacked_total_tests.rs index 92db654e5..1aa47c5b3 100644 --- a/crates/format/src/unpacked_total_tests.rs +++ b/crates/format/src/unpacked_total_tests.rs @@ -108,8 +108,8 @@ fn a_read_that_fails_part_way_still_spends_the_total() { let mut bytes = buf.into_inner(); let at = 30 + "snapshots/v1.json".len() + body.len() / 2; bytes[at] ^= 0xff; - let mut z = zip::ZipArchive::new(std::io::Cursor::new(bytes)).unwrap(); - let size = z.by_name("snapshots/v1.json").unwrap().size(); + let mut z = craft_figcraft_format::SeekArchive::open(std::io::Cursor::new(bytes), u64::MAX).unwrap(); + let size = z.declared_size("snapshots/v1.json").unwrap(); assert_eq!(size, body.len() as u64); let mut total = crate::Unpacked::new(size + size / 2); assert!(total.read(&mut z, "snapshots/v1.json", 1 << 20).is_err(), "the damaged member fails"); From 69d7de8510afcf57220e34880d428f04e6c6425b Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 08:21:07 +0000 Subject: [PATCH 3/8] use shared FigCraft archive budget and seek reader --- Cargo.lock | 24 ++++----- Cargo.toml | 2 +- crates/format/src/lib.rs | 105 +++++++++++++++------------------------ 3 files changed, 53 insertions(+), 78 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b9cf7574b..95d0c1aff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -253,7 +253,7 @@ dependencies = [ "objc2-foundation 0.3.2", "parking_lot", "percent-encoding", - "windows-sys 0.61.2", + "windows-sys 0.52.0", "wl-clipboard-rs", "x11rb", ] @@ -1029,9 +1029,9 @@ dependencies = [ [[package]] name = "craft-figcraft-format" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=a75b9df240d3e2eb98898fcbf99403ae5a1c73bb#a75b9df240d3e2eb98898fcbf99403ae5a1c73bb" +source = "git+https://github.com/storytold/craft-libs?rev=1592923e6db6ed1458f7a4f2982b6e9db4057224#1592923e6db6ed1458f7a4f2982b6e9db4057224" dependencies = [ - "craft-zip 0.1.0 (git+https://github.com/storytold/craft-libs?rev=a75b9df240d3e2eb98898fcbf99403ae5a1c73bb)", + "craft-zip 0.1.0 (git+https://github.com/storytold/craft-libs?rev=1592923e6db6ed1458f7a4f2982b6e9db4057224)", "serde", "serde_json", "thiserror 2.0.21", @@ -1231,7 +1231,7 @@ dependencies = [ [[package]] name = "craft-zip" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=a75b9df240d3e2eb98898fcbf99403ae5a1c73bb#a75b9df240d3e2eb98898fcbf99403ae5a1c73bb" +source = "git+https://github.com/storytold/craft-libs?rev=1592923e6db6ed1458f7a4f2982b6e9db4057224#1592923e6db6ed1458f7a4f2982b6e9db4057224" dependencies = [ "flate2", "thiserror 2.0.21", @@ -1438,7 +1438,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -1780,7 +1780,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -3950,7 +3950,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d8fae84b431384b68627d0f9b3b1245fcf9f46f6c0e3dc902e9dce64edd1967" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -4646,7 +4646,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -4703,7 +4703,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -5272,10 +5272,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix 1.1.5", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -6399,7 +6399,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 6929ac8a8..fe891ff98 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,7 +60,7 @@ craft-settings = { git = "https://github.com/storytold/craft-libs", rev = "e11f9 # a .fig message onto the REST file JSON, and that JSON's shared helpers. These crates share # types, so they stay on one rev; they share none with the crates above. craft-fig = { git = "https://github.com/storytold/craft-libs", rev = "d13ac58a15c45d618f25910772502e38c4ce1d4c" } -craft-figcraft-format = { git = "https://github.com/storytold/craft-libs", rev = "a75b9df240d3e2eb98898fcbf99403ae5a1c73bb" } +craft-figcraft-format = { git = "https://github.com/storytold/craft-libs", rev = "1592923e6db6ed1458f7a4f2982b6e9db4057224" } craft-figma-rest = { git = "https://github.com/storytold/craft-libs", rev = "d13ac58a15c45d618f25910772502e38c4ce1d4c" } # foundation diff --git a/crates/format/src/lib.rs b/crates/format/src/lib.rs index c53160728..eaddbf8ca 100644 --- a/crates/format/src/lib.rs +++ b/crates/format/src/lib.rs @@ -31,7 +31,7 @@ pub use library::{is_valid_library_id, read_library, read_library_within}; mod density; use std::collections::{BTreeMap, BTreeSet}; -use std::io::{Read, Write}; +use std::io::{Read, Seek, SeekFrom, Write}; use std::sync::Arc; use figcraft_doc::{Document, ImageData}; @@ -44,8 +44,8 @@ use sha2::Digest; pub use craft_figcraft_format::{ BRANCH_INDEX, BranchEntry as NativeBranchEntry, BranchIndex as NativeBranchIndex, BranchState as NativeBranchState, DOCUMENT_RATIO, EXTENSION, FORMAT, FormatError, IMAGE_BUDGET, ImageBudget, MAX_BRANCHES, MAX_DOCUMENT, MAX_MEMBER, MAX_UNPACKED, MAX_VERSION_BYTES, MAX_VERSION_DESCRIPTION, - MAX_VERSION_INDEX, MAX_VERSION_NAME, MAX_VERSIONS, VERSION, VersionEntry as NativeVersionEntry, VersionIndex as NativeVersionIndex, clip, - document_limit, is_valid_version_id, + MAX_VERSION_INDEX, MAX_VERSION_NAME, MAX_VERSIONS, ReadBudget, VERSION, VersionEntry as NativeVersionEntry, VersionIndex as NativeVersionIndex, + clip, document_limit, is_valid_version_id, }; const JSON_LEVEL: i64 = 3; /// The export warning for colour calculations that resolve to nothing (an operand missing, or more @@ -554,16 +554,6 @@ pub fn load_bytes(bytes: &[u8]) -> Result<(Document, Vec), FormatError> load_bytes_with_versions(bytes).map(|l| (l.doc, l.issues)) } -/// Read archive member `name`, refusing members larger than `limit` bytes. -fn read_native_member(z: &mut impl craft_figcraft_format::NativeArchive, name: &str, limit: u64) -> Result, FormatError> { - match z.read_member(name, limit) { - Ok(Some(bytes)) => Ok(bytes), - Ok(None) => Err(FormatError::Corrupt(format!("{name}: not found"))), - Err(error) if error.is_limit() => Err(FormatError::Corrupt(format!("{name} is too large"))), - Err(error) => Err(FormatError::Corrupt(format!("{name}: {error}"))), - } -} - /// Read a member from a foreign ZIP archive, refusing entries larger than `limit`. pub(crate) fn read_member(z: &mut zip::ZipArchive, name: &str, limit: u64) -> Result, FormatError> { let f = z.by_name(name).map_err(|e| FormatError::Corrupt(format!("{name}: {e}")))?; @@ -578,48 +568,8 @@ pub(crate) fn read_member(z: &mut zip::ZipArchive, n Ok(data) } -/// What one file open may still unpack, every area together ([`MAX_UNPACKED`]). -pub(crate) struct Unpacked { - left: u64, - total: u64, -} - -impl Unpacked { - pub(crate) fn new(total: u64) -> Self { - Self { left: total, total } - } - - /// Why a member is left out once the total is spent. - fn passed(&self) -> String { - match self.total / (1024 * 1024) { - 0 => format!("too large: the file's contents together pass {} bytes", self.total), - mb => format!("too large: the file's contents together pass {mb} MB"), - } - } - - /// Whether member `name` (by its recorded size) needs more than is left. - fn exceeded(&self, z: &mut impl craft_figcraft_format::NativeArchive, name: &str) -> bool { - z.declared_size(name).is_some_and(|size| size > self.left) - } - - /// [`read_member`] within `limit` (the area's own cap) and what is left of the total, which the - /// member then uses up. - pub(crate) fn read(&mut self, z: &mut impl craft_figcraft_format::NativeArchive, name: &str, limit: u64) -> Result, FormatError> { - if self.exceeded(z, name) && z.declared_size(name).is_some_and(|size| size <= limit) { - return Err(FormatError::Corrupt(format!("{name}: {}", self.passed()))); - } - let cap = limit.min(self.left); - let size = z.declared_size(name).unwrap_or(0); - let data = read_native_member(z, name, cap); - // A read that failed part way (a bad checksum at the end, say) did its unpacking all the - // same: it is charged in full, so an index naming that member again can't repeat it. One - // refused for its recorded size before any unpacking costs nothing. - if size <= cap { - self.left = self.left.saturating_sub(data.as_ref().map_or(size, |d| d.len() as u64)); - } - data - } -} +/// The remaining archive-wide expansion budget while the app maps native records. +pub(crate) type Unpacked = ReadBudget; /// Read a `.figcraft` archive (or bare JSON) with its version history. pub fn load_bytes_with_versions(bytes: &[u8]) -> Result { @@ -642,21 +592,25 @@ pub(crate) fn load_limited(bytes: &[u8], budget: ImageBudget, total: u64) -> Res let mut z = craft_figcraft_format::Archive::open(bytes, craft_figcraft_format::ArchiveLimits { max_entries: u64::MAX, max_total_bytes: u64::MAX }) .map_err(|e| FormatError::Corrupt(e.to_string()))?; + load_archive(&mut z, budget, total, bytes.len() as u64) +} + +fn load_archive(z: &mut impl craft_figcraft_format::NativeArchive, budget: ImageBudget, total: u64, archive_len: u64) -> Result { let packed = z.compressed_size("document.json").unwrap_or(0); - let limit = document_limit(packed, bytes.len() as u64); + let limit = document_limit(packed, archive_len); let mut unpacked = Unpacked::new(total); - let text = String::from_utf8(unpacked.read(&mut z, "document.json", limit)?).map_err(|e| FormatError::Corrupt(e.to_string()))?; + let text = String::from_utf8(unpacked.read(z, "document.json", limit)?).map_err(|e| FormatError::Corrupt(e.to_string()))?; let (mut doc, mut issues, own_images) = parse_json(&text)?; - let ImagesRead { images, unread, past_budget } = read_images(&mut z, own_images.as_ref(), budget, &mut unpacked); + let ImagesRead { images, unread, past_budget } = read_images(z, own_images.as_ref(), budget, &mut unpacked); report_lost_images(&mut issues, &unread, &past_budget, own_images.as_ref(), &images); let images_past_budget = past_budget.len(); - match library::read(&mut z, &images, &mut unpacked) { + match library::read(z, &images, &mut unpacked) { Ok(lib) => doc.published = lib.map(Arc::new), Err(e) => issues.push(format!("the published library was skipped: {e}")), } let mut budget = MAX_VERSION_BYTES; - let versions = read_versions(&mut z, "versions", &images, &mut issues, &mut budget, &mut unpacked); - let branches = read_branches(&mut z, &images, &mut issues, &mut unpacked); + let versions = read_versions(z, "versions", &images, &mut issues, &mut budget, &mut unpacked); + let branches = read_branches(z, &images, &mut issues, &mut unpacked); // Images only the version history or branches use stay with them. let branch_images = branches.iter().flat_map(|b| b.base.images.keys().chain(b.head.images.keys()).chain(b.versions.iter().flat_map(|v| v.images().keys()))); @@ -709,9 +663,9 @@ fn read_images( } // Past the file's total, an image is left out as one past the images' own budget: it stays // in the file, which can't then be saved over without losing it (#1313). - if unpacked.exceeded(z, &name) { + if unpacked.exceeds_remaining(z, &name) { past_budget.insert(hash.clone()); - unread.push((hash, FormatError::Corrupt(unpacked.passed()))); + unread.push((hash, FormatError::Corrupt(unpacked.exhausted_message()))); continue; } match unpacked.read(z, &name, left.min(MAX_MEMBER)) { @@ -947,8 +901,29 @@ pub fn write_file_atomic(path: &std::path::Path, bytes: &[u8]) -> std::io::Resul #[cfg(not(target_arch = "wasm32"))] pub fn load_file(path: &std::path::Path) -> Result<(Document, Vec), FormatError> { - let bytes = std::fs::read(path).map_err(|e| FormatError::Io(format!("{}: {e}", path.display())))?; - load_bytes(&bytes) + let mut file = std::fs::File::open(path).map_err(|e| FormatError::Io(format!("{}: {e}", path.display())))?; + let archive_len = file.metadata().map_err(|e| FormatError::Io(format!("{}: {e}", path.display())))?.len(); + let mut first = None; + let mut chunk = [0u8; 8192]; + loop { + let count = file.read(&mut chunk).map_err(|e| FormatError::Io(format!("{}: {e}", path.display())))?; + if count == 0 { + break; + } + if let Some(byte) = chunk[..count].iter().copied().find(|byte| !byte.is_ascii_whitespace()) { + first = Some(byte); + break; + } + } + file.seek(SeekFrom::Start(0)).map_err(|e| FormatError::Io(format!("{}: {e}", path.display())))?; + if first == Some(b'{') { + let mut bytes = Vec::new(); + file.read_to_end(&mut bytes).map_err(|e| FormatError::Io(format!("{}: {e}", path.display())))?; + return load_bytes(&bytes); + } + let mut archive = craft_figcraft_format::SeekArchive::open(file, u64::MAX).map_err(|e| FormatError::Corrupt(e.to_string()))?; + let loaded = load_archive(&mut archive, IMAGE_BUDGET, MAX_UNPACKED, archive_len)?; + Ok((loaded.doc, loaded.issues)) } #[cfg(test)] From aa460b6f3351f29cf6493c7a713a7f5964b7c666 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 10:53:52 +0000 Subject: [PATCH 4/8] Move native library resources into shared format --- Cargo.lock | 6 +- Cargo.toml | 2 +- crates/format/src/lib.rs | 2 - crates/format/src/library.rs | 141 +++++++++++++++++++++++++---------- 4 files changed, 104 insertions(+), 47 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 95d0c1aff..cf89d0f1d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1029,9 +1029,9 @@ dependencies = [ [[package]] name = "craft-figcraft-format" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=1592923e6db6ed1458f7a4f2982b6e9db4057224#1592923e6db6ed1458f7a4f2982b6e9db4057224" +source = "git+https://github.com/storytold/craft-libs?rev=46bc8d7a0#46bc8d7a0ac906b59ce0c94b074367fd98d84d84" dependencies = [ - "craft-zip 0.1.0 (git+https://github.com/storytold/craft-libs?rev=1592923e6db6ed1458f7a4f2982b6e9db4057224)", + "craft-zip 0.1.0 (git+https://github.com/storytold/craft-libs?rev=46bc8d7a0)", "serde", "serde_json", "thiserror 2.0.21", @@ -1231,7 +1231,7 @@ dependencies = [ [[package]] name = "craft-zip" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=1592923e6db6ed1458f7a4f2982b6e9db4057224#1592923e6db6ed1458f7a4f2982b6e9db4057224" +source = "git+https://github.com/storytold/craft-libs?rev=46bc8d7a0#46bc8d7a0ac906b59ce0c94b074367fd98d84d84" dependencies = [ "flate2", "thiserror 2.0.21", diff --git a/Cargo.toml b/Cargo.toml index fe891ff98..04b3fe6b9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,7 +60,7 @@ craft-settings = { git = "https://github.com/storytold/craft-libs", rev = "e11f9 # a .fig message onto the REST file JSON, and that JSON's shared helpers. These crates share # types, so they stay on one rev; they share none with the crates above. craft-fig = { git = "https://github.com/storytold/craft-libs", rev = "d13ac58a15c45d618f25910772502e38c4ce1d4c" } -craft-figcraft-format = { git = "https://github.com/storytold/craft-libs", rev = "1592923e6db6ed1458f7a4f2982b6e9db4057224" } +craft-figcraft-format = { git = "https://github.com/storytold/craft-libs", rev = "46bc8d7a0" } craft-figma-rest = { git = "https://github.com/storytold/craft-libs", rev = "d13ac58a15c45d618f25910772502e38c4ce1d4c" } # foundation diff --git a/crates/format/src/lib.rs b/crates/format/src/lib.rs index eaddbf8ca..90a24ce64 100644 --- a/crates/format/src/lib.rs +++ b/crates/format/src/lib.rs @@ -391,8 +391,6 @@ pub fn from_json(text: &str) -> Result<(Document, Vec), FormatError> { parse_json(text).map(|(doc, issues, _)| (doc, issues)) } -/// The image hashes a `document.json` lists in its `imageIndex` (`None` without one). - /// Parse `document.json` content; also returns its [`ImageIndex`]. The header is checked first /// (a newer file is refused before its document is read), then the document is deserialized /// straight from the text. diff --git a/crates/format/src/library.rs b/crates/format/src/library.rs index a40f6d718..5c09e19a3 100644 --- a/crates/format/src/library.rs +++ b/crates/format/src/library.rs @@ -13,21 +13,19 @@ use std::collections::{BTreeMap, BTreeSet}; use std::io::{Read, Seek}; use std::sync::Arc; -use figcraft_doc::{ImageData, LibraryManifest, NodeKind, PublishedLibrary, used_images}; +use figcraft_doc::{ + ImageData, LibraryAsset, LibraryCollection, LibraryManifest, LibraryStyle, LibraryVariable, NodeId, NodeKind, PublishedLibrary, used_images, +}; -use crate::{FormatError, MAX_MEMBER, Unpacked, from_json, image_data, io_err, put, put_json}; +use crate::{FormatError, MAX_MEMBER, Unpacked, from_json, image_data, io_err}; -const MANIFEST: &str = "library/manifest.json"; -const SNAPSHOT: &str = "library/document.json"; -const IMAGES: &str = "library/images/"; -/// Largest manifest read. -const MAX_MANIFEST: u64 = 16 * 1024 * 1024; -/// Most bytes of images read for one library. -const MAX_IMAGE_BYTES: u64 = 1024 * 1024 * 1024; -/// Longest library id and name kept. -const MAX_ID: usize = 64; -const MAX_NAME: usize = 200; -const MAX_DESCRIPTION: usize = 2000; +use craft_figcraft_format::library as native; + +// Test helpers use these names when constructing deliberately bounded archive entries. +#[cfg(test)] +const MANIFEST: &str = native::MANIFEST; +#[cfg(test)] +const MAX_MANIFEST: u64 = native::MAX_MANIFEST_BYTES; fn corrupt(e: impl std::fmt::Display) -> FormatError { FormatError::Corrupt(e.to_string()) @@ -35,20 +33,89 @@ fn corrupt(e: impl std::fmt::Display) -> FormatError { /// Whether `id` can be a library id: 1–64 ASCII letters, digits, `-` and `_`. pub fn is_valid_library_id(id: &str) -> bool { - (1..=MAX_ID).contains(&id.len()) && id.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_') + native::is_valid_id(id) +} + +fn to_native_manifest(m: &LibraryManifest) -> native::Manifest { + native::Manifest { + id: m.id.clone(), + name: m.name.clone(), + version: m.version, + description: m.description.clone(), + time: m.time, + components: m + .components + .iter() + .map(|a| native::Component { id: a.id.0, name: a.name.clone(), set: a.set, version: a.version, published: a.published }) + .collect(), + styles: m + .styles + .iter() + .map(|s| native::Style { id: s.id.clone(), name: s.name.clone(), kind: s.kind.clone(), version: s.version, published: s.published }) + .collect(), + collections: m + .collections + .iter() + .map(|c| native::Collection { id: c.id.clone(), name: c.name.clone(), version: c.version, published: c.published }) + .collect(), + variables: m + .variables + .iter() + .map(|v| native::Variable { + id: v.id.clone(), + name: v.name.clone(), + collection: v.collection.clone(), + kind: v.kind.clone(), + version: v.version, + published: v.published, + }) + .collect(), + } +} + +fn from_native_manifest(m: native::Manifest) -> LibraryManifest { + LibraryManifest { + id: m.id, + name: m.name, + version: m.version, + description: m.description, + time: m.time, + components: m + .components + .into_iter() + .map(|a| LibraryAsset { id: NodeId(a.id), name: a.name, set: a.set, version: a.version, published: a.published }) + .collect(), + styles: m + .styles + .into_iter() + .map(|s| LibraryStyle { id: s.id, name: s.name, kind: s.kind, version: s.version, published: s.published }) + .collect(), + collections: m + .collections + .into_iter() + .map(|c| LibraryCollection { id: c.id, name: c.name, version: c.version, published: c.published }) + .collect(), + variables: m + .variables + .into_iter() + .map(|v| LibraryVariable { id: v.id, name: v.name, collection: v.collection, kind: v.kind, version: v.version, published: v.published }) + .collect(), + } } /// Write the published library's entries. `written` are the images already in `images/`. pub(crate) fn write(z: &mut craft_figcraft_format::ArchiveWriter, lib: &PublishedLibrary, written: &BTreeSet<&String>) -> Result<(), FormatError> { - let stored = craft_figcraft_format::Compression::Stored; - put(z, MANIFEST, craft_figcraft_format::Compression::Level(3), &serde_json::to_vec(&lib.manifest).map_err(io_err)?)?; - put_json(z, SNAPSHOT, &lib.doc)?; - for (hash, img) in &lib.doc.images { - if !written.contains(hash) && ImageData::is_valid_key(hash) { - put(z, &format!("{IMAGES}{hash}"), stored, img.bytes.as_slice())?; - } - } - Ok(()) + let already_written = written.iter().map(|hash| (*hash).clone()).collect(); + let images = lib.doc.images.iter().map(|(hash, image)| (hash.as_str(), image.bytes.as_slice())); + native::write_members( + z, + &to_native_manifest(&lib.manifest), + &crate::FiniteValue(&lib.doc), + &crate::native_image_index(&lib.doc), + &already_written, + images, + ) + .map_err(io_err) } /// The published library in an archive (`None` when the file was never published). `own` are @@ -59,28 +126,20 @@ pub(crate) fn read( own: &BTreeMap, unpacked: &mut Unpacked, ) -> Result, FormatError> { - if !z.contains(MANIFEST) { + if !z.contains(native::MANIFEST) { return Ok(None); } - let mut manifest: LibraryManifest = serde_json::from_slice(&unpacked.read(z, MANIFEST, MAX_MANIFEST)?).map_err(corrupt)?; + let wire_manifest: native::Manifest = + serde_json::from_slice(&unpacked.read(z, native::MANIFEST, native::MAX_MANIFEST_BYTES)?).map_err(corrupt)?; + let mut manifest = from_native_manifest(wire_manifest); if !is_valid_library_id(&manifest.id) { return Err(corrupt("the library has an invalid id")); } - let text = String::from_utf8(unpacked.read(z, SNAPSHOT, MAX_MEMBER)?).map_err(corrupt)?; - let mut budget = MAX_IMAGE_BYTES; + let text = String::from_utf8(unpacked.read(z, native::DOCUMENT, MAX_MEMBER)?).map_err(corrupt)?; let (mut doc, _) = from_json(&text)?; - for hash in used_images(&doc) { - if !ImageData::is_valid_key(&hash) { - continue; - } - if let Some(img) = own.get(&hash) { - doc.images.insert(hash, img.clone()); - continue; - } - let name = [format!("{IMAGES}{hash}"), format!("images/{hash}")].into_iter().find(|n| z.contains(n)); - // A missing or oversized image shows as a placeholder, like in any other file. - let Some(data) = name.and_then(|n| unpacked.read(z, &n, budget.min(MAX_MEMBER)).ok()) else { continue }; - budget = budget.saturating_sub(data.len() as u64); + let available: BTreeSet = own.keys().cloned().collect(); + let images = native::read_image_resources(z, used_images(&doc), &available, unpacked); + for (hash, data) in images { doc.images.insert(hash, image_data(data)); } // Keep what the snapshot really holds; clip what people see. @@ -92,8 +151,8 @@ pub(crate) fn read( manifest.styles.retain(|s| styles.contains(s.id.as_str())); manifest.collections.retain(|c| collections.contains(c.id.as_str())); manifest.variables.retain(|v| variables.contains(&(v.id.as_str(), v.collection.as_str()))); - manifest.name = manifest.name.chars().take(MAX_NAME).collect(); - manifest.description = manifest.description.chars().take(MAX_DESCRIPTION).collect(); + manifest.name = native::clip_display(&manifest.name, native::MAX_NAME_CHARS); + manifest.description = native::clip_display(&manifest.description, native::MAX_DESCRIPTION_CHARS); Ok(Some(PublishedLibrary { manifest, doc: Arc::new(doc) })) } From 821f9a8631f192b2f554a281760e502262bcf2ba Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 11:05:37 +0000 Subject: [PATCH 5/8] Use shared FigCraft history index codec --- Cargo.lock | 6 +++--- Cargo.toml | 2 +- crates/format/src/lib.rs | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cf89d0f1d..4e56ccd07 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1029,9 +1029,9 @@ dependencies = [ [[package]] name = "craft-figcraft-format" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=46bc8d7a0#46bc8d7a0ac906b59ce0c94b074367fd98d84d84" +source = "git+https://github.com/storytold/craft-libs?rev=c1d198925#c1d198925f10a640c4623cf9c1c0f775ff2d972e" dependencies = [ - "craft-zip 0.1.0 (git+https://github.com/storytold/craft-libs?rev=46bc8d7a0)", + "craft-zip 0.1.0 (git+https://github.com/storytold/craft-libs?rev=c1d198925)", "serde", "serde_json", "thiserror 2.0.21", @@ -1231,7 +1231,7 @@ dependencies = [ [[package]] name = "craft-zip" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=46bc8d7a0#46bc8d7a0ac906b59ce0c94b074367fd98d84d84" +source = "git+https://github.com/storytold/craft-libs?rev=c1d198925#c1d198925f10a640c4623cf9c1c0f775ff2d972e" dependencies = [ "flate2", "thiserror 2.0.21", diff --git a/Cargo.toml b/Cargo.toml index 04b3fe6b9..1e51bcecd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,7 +60,7 @@ craft-settings = { git = "https://github.com/storytold/craft-libs", rev = "e11f9 # a .fig message onto the REST file JSON, and that JSON's shared helpers. These crates share # types, so they stay on one rev; they share none with the crates above. craft-fig = { git = "https://github.com/storytold/craft-libs", rev = "d13ac58a15c45d618f25910772502e38c4ce1d4c" } -craft-figcraft-format = { git = "https://github.com/storytold/craft-libs", rev = "46bc8d7a0" } +craft-figcraft-format = { git = "https://github.com/storytold/craft-libs", rev = "c1d198925" } craft-figma-rest = { git = "https://github.com/storytold/craft-libs", rev = "d13ac58a15c45d618f25910772502e38c4ce1d4c" } # foundation diff --git a/crates/format/src/lib.rs b/crates/format/src/lib.rs index 90a24ce64..011ffff74 100644 --- a/crates/format/src/lib.rs +++ b/crates/format/src/lib.rs @@ -503,7 +503,7 @@ pub fn save_bytes_with_history( &mut z, BRANCH_INDEX, craft_figcraft_format::Compression::Level(3), - &serde_json::to_vec(&NativeBranchIndex { branches: index }).map_err(io_err)?, + &craft_figcraft_format::write_branch_index(&NativeBranchIndex { branches: index }).map_err(io_err)?, )?; } if let Some(lib) = &doc.published { @@ -542,7 +542,7 @@ fn put_versions(z: &mut craft_figcraft_format::ArchiveWriter, dir: &str, version z, &format!("{dir}/index.json"), craft_figcraft_format::Compression::Level(3), - &serde_json::to_vec(&NativeVersionIndex { versions: index }).map_err(io_err)?, + &craft_figcraft_format::write_version_index(&NativeVersionIndex { versions: index }).map_err(io_err)?, )?; Ok(()) } @@ -738,7 +738,7 @@ fn read_versions( } let index = match unpacked .read(z, &index_name, MAX_VERSION_INDEX) - .and_then(|b| serde_json::from_slice::(&b).map_err(|e| FormatError::Corrupt(e.to_string()))) + .and_then(|b| craft_figcraft_format::read_version_index(&b).map_err(|e| FormatError::Corrupt(e.to_string()))) { Ok(v) => v, Err(e) => { @@ -791,7 +791,7 @@ fn read_branches( } let index = match unpacked .read(z, BRANCH_INDEX, MAX_VERSION_INDEX) - .and_then(|b| serde_json::from_slice::(&b).map_err(|e| FormatError::Corrupt(e.to_string()))) + .and_then(|b| craft_figcraft_format::read_branch_index(&b).map_err(|e| FormatError::Corrupt(e.to_string()))) { Ok(v) => v, Err(e) => { From 6ad04aa4d2696767ec9c026055612722ea259105 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 11:54:50 +0000 Subject: [PATCH 6/8] Finish FigCraft native shared format cutover --- Cargo.lock | 6 +-- Cargo.toml | 2 +- crates/format/src/lib.rs | 10 +++-- crates/format/src/library.rs | 3 +- crates/format/src/streaming_tests.rs | 64 ++++++++++++++++++++++++++++ crates/format/src/thumbnail.rs | 22 +++------- 6 files changed, 81 insertions(+), 26 deletions(-) create mode 100644 crates/format/src/streaming_tests.rs diff --git a/Cargo.lock b/Cargo.lock index 4e56ccd07..bf72b5ac4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1029,9 +1029,9 @@ dependencies = [ [[package]] name = "craft-figcraft-format" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=c1d198925#c1d198925f10a640c4623cf9c1c0f775ff2d972e" +source = "git+https://github.com/storytold/craft-libs?rev=9aac9301a#9aac9301a92ec7f8db16760b6de0f0576f4bd52c" dependencies = [ - "craft-zip 0.1.0 (git+https://github.com/storytold/craft-libs?rev=c1d198925)", + "craft-zip 0.1.0 (git+https://github.com/storytold/craft-libs?rev=9aac9301a)", "serde", "serde_json", "thiserror 2.0.21", @@ -1231,7 +1231,7 @@ dependencies = [ [[package]] name = "craft-zip" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=c1d198925#c1d198925f10a640c4623cf9c1c0f775ff2d972e" +source = "git+https://github.com/storytold/craft-libs?rev=9aac9301a#9aac9301a92ec7f8db16760b6de0f0576f4bd52c" dependencies = [ "flate2", "thiserror 2.0.21", diff --git a/Cargo.toml b/Cargo.toml index 1e51bcecd..de9e5c937 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,7 +60,7 @@ craft-settings = { git = "https://github.com/storytold/craft-libs", rev = "e11f9 # a .fig message onto the REST file JSON, and that JSON's shared helpers. These crates share # types, so they stay on one rev; they share none with the crates above. craft-fig = { git = "https://github.com/storytold/craft-libs", rev = "d13ac58a15c45d618f25910772502e38c4ce1d4c" } -craft-figcraft-format = { git = "https://github.com/storytold/craft-libs", rev = "c1d198925" } +craft-figcraft-format = { git = "https://github.com/storytold/craft-libs", rev = "9aac9301a" } craft-figma-rest = { git = "https://github.com/storytold/craft-libs", rev = "d13ac58a15c45d618f25910772502e38c4ce1d4c" } # foundation diff --git a/crates/format/src/lib.rs b/crates/format/src/lib.rs index 011ffff74..167444c62 100644 --- a/crates/format/src/lib.rs +++ b/crates/format/src/lib.rs @@ -395,11 +395,11 @@ pub fn from_json(text: &str) -> Result<(Document, Vec), FormatError> { /// (a newer file is refused before its document is read), then the document is deserialized /// straight from the text. fn parse_json(text: &str) -> Result<(Document, Vec, ImageIndex), FormatError> { - let corrupt = |e: serde_json::Error| FormatError::Corrupt(e.to_string()); - let header = craft_figcraft_format::read_header(text).map_err(corrupt)?; - craft_figcraft_format::check_version(&header).map_err(|e| FormatError::Newer(e.0))?; + let (mut doc, header) = craft_figcraft_format::read_document_json::(text).map_err(|e| match e { + craft_figcraft_format::ReadDocumentError::Json(error) => FormatError::Corrupt(error.to_string()), + craft_figcraft_format::ReadDocumentError::NewerVersion(error) => FormatError::Newer(error.0), + })?; let images = header.images.map(|hashes| hashes.into_iter().collect()); - let mut doc: Document = serde_json::from_str(text).map_err(corrupt)?; let issues = doc.validate(); if !issues.is_empty() { doc.repair(); @@ -939,6 +939,8 @@ mod large_tests; #[cfg(test)] mod staging_tests; #[cfg(test)] +mod streaming_tests; +#[cfg(test)] mod thumbnail_tests; #[cfg(test)] mod unpacked_total_tests; diff --git a/crates/format/src/library.rs b/crates/format/src/library.rs index 5c09e19a3..8ed11fedb 100644 --- a/crates/format/src/library.rs +++ b/crates/format/src/library.rs @@ -129,8 +129,7 @@ pub(crate) fn read( if !z.contains(native::MANIFEST) { return Ok(None); } - let wire_manifest: native::Manifest = - serde_json::from_slice(&unpacked.read(z, native::MANIFEST, native::MAX_MANIFEST_BYTES)?).map_err(corrupt)?; + let wire_manifest = native::read_manifest(&unpacked.read(z, native::MANIFEST, native::MAX_MANIFEST_BYTES)?).map_err(corrupt)?; let mut manifest = from_native_manifest(wire_manifest); if !is_valid_library_id(&manifest.id) { return Err(corrupt("the library has an invalid id")); diff --git a/crates/format/src/streaming_tests.rs b/crates/format/src/streaming_tests.rs new file mode 100644 index 000000000..daf7ecc18 --- /dev/null +++ b/crates/format/src/streaming_tests.rs @@ -0,0 +1,64 @@ +//! The seekable on-disk reader uses the same format and recovery behavior as the byte reader. + +#![cfg(not(target_arch = "wasm32"))] + +use std::sync::Arc; + +use figcraft_doc::{Document, NodeKind}; + +use super::*; + +fn png(red: u8) -> Vec { + let image = image::RgbaImage::from_pixel(2, 2, image::Rgba([red, 0, 0, 255])); + let mut bytes = Vec::new(); + image.write_to(&mut std::io::Cursor::new(&mut bytes), image::ImageFormat::Png).unwrap(); + bytes +} + +#[test] +fn seekable_reader_matches_byte_reader_for_history_and_branch_archives() { + let mut document = Document::new("Current"); + let current_image = add_image(&mut document, png(10)).unwrap().0; + + let mut older = Document::new("Older"); + let old_image = add_image(&mut older, png(20)).unwrap().0; + let version = Version::new("v1", "Before", "", 1, Arc::new(older.clone())); + + let mut head = document.clone(); + let page = head.first_page().unwrap(); + head.create(page, NodeKind::Rectangle, "Branch shape", 1.0, 2.0, 3.0, 4.0).unwrap(); + add_image(&mut head, png(30)).unwrap(); + let branch = Branch { + id: "b1".into(), + name: "Explore".into(), + time: 2, + client: 4, + state: BranchState::Active, + base: Arc::new(document.clone()), + versions: vec![Version::new("v2", "Branch base", "", 3, Arc::new(document.clone()))], + head: Arc::new(head), + }; + + let bytes = save_bytes_with_history(&document, None, &[version], &[branch]).unwrap(); + let from_bytes = load_bytes_with_versions(&bytes).unwrap(); + assert_eq!(from_bytes.versions.len(), 1); + assert_eq!(from_bytes.branches.len(), 1); + assert!(from_bytes.doc.images.contains_key(¤t_image)); + assert!(from_bytes.versions[0].images().contains_key(&old_image)); + + let path = std::env::temp_dir().join(format!("figcraft-stream-reader-{}.figcraft", std::process::id())); + std::fs::write(&path, &bytes).unwrap(); + let from_file = load_file(&path).unwrap(); + let _ = std::fs::remove_file(&path); + assert_eq!(from_file, (from_bytes.doc, from_bytes.issues)); +} + +#[test] +fn seekable_reader_keeps_newer_document_error_precedence() { + let bytes = crate::zip_files([("document.json", br#"{"version":999,"document":["malformed"]}"#.as_slice())]).unwrap(); + let path = std::env::temp_dir().join(format!("figcraft-stream-newer-{}.figcraft", std::process::id())); + std::fs::write(&path, bytes).unwrap(); + let result = load_file(&path); + let _ = std::fs::remove_file(&path); + assert!(matches!(result, Err(FormatError::Newer(999)))); +} diff --git a/crates/format/src/thumbnail.rs b/crates/format/src/thumbnail.rs index 530038095..dc1f4eaff 100644 --- a/crates/format/src/thumbnail.rs +++ b/crates/format/src/thumbnail.rs @@ -160,23 +160,13 @@ pub fn archive_preview(archive: R, fit: u32) -> Preview { /// The first page's background in a `.figcraft` `document.json`, and the file's kind. fn figcraft_page(json: &[u8]) -> (Option<[f32; 4]>, figcraft_doc::FileKind) { - #[derive(serde::Deserialize)] - struct Head { - #[serde(default)] - pages: Vec, - #[serde(default)] - nodes: std::collections::BTreeMap, - #[serde(default)] - kind: figcraft_doc::FileKind, - } - #[derive(serde::Deserialize)] - struct Node { - #[serde(default)] - background: Option, - } - let Ok(head) = serde_json::from_slice::(json) else { return (None, figcraft_doc::FileKind::Design) }; + let Ok(head) = craft_figcraft_format::read_preview_document(json) else { return (None, figcraft_doc::FileKind::Design) }; let page = head.pages.first().and_then(|first| head.nodes.get(&first.to_string())).and_then(|n| n.background.as_ref()).and_then(rgba); - (page, head.kind) + let kind = match head.kind { + craft_figcraft_format::PreviewKind::Design => figcraft_doc::FileKind::Design, + craft_figcraft_format::PreviewKind::Board => figcraft_doc::FileKind::Board, + }; + (page, kind) } /// The canvas colour a `.fig` file's `meta.json` gives (`client_meta.background_color`). From c607ec686addc02060aff2b0c0e475c47d2b532e Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 13:56:22 +0000 Subject: [PATCH 7/8] Pin FigCraft format to merged shared codec --- Cargo.lock | 6 +++--- Cargo.toml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bf72b5ac4..311555615 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1029,9 +1029,9 @@ dependencies = [ [[package]] name = "craft-figcraft-format" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=9aac9301a#9aac9301a92ec7f8db16760b6de0f0576f4bd52c" +source = "git+https://github.com/storytold/craft-libs?rev=0e2a717c644bcfaae1a27cd5e2f6b4c0a6b7798b#0e2a717c644bcfaae1a27cd5e2f6b4c0a6b7798b" dependencies = [ - "craft-zip 0.1.0 (git+https://github.com/storytold/craft-libs?rev=9aac9301a)", + "craft-zip 0.1.0 (git+https://github.com/storytold/craft-libs?rev=0e2a717c644bcfaae1a27cd5e2f6b4c0a6b7798b)", "serde", "serde_json", "thiserror 2.0.21", @@ -1231,7 +1231,7 @@ dependencies = [ [[package]] name = "craft-zip" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=9aac9301a#9aac9301a92ec7f8db16760b6de0f0576f4bd52c" +source = "git+https://github.com/storytold/craft-libs?rev=0e2a717c644bcfaae1a27cd5e2f6b4c0a6b7798b#0e2a717c644bcfaae1a27cd5e2f6b4c0a6b7798b" dependencies = [ "flate2", "thiserror 2.0.21", diff --git a/Cargo.toml b/Cargo.toml index de9e5c937..60d916329 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,7 +60,7 @@ craft-settings = { git = "https://github.com/storytold/craft-libs", rev = "e11f9 # a .fig message onto the REST file JSON, and that JSON's shared helpers. These crates share # types, so they stay on one rev; they share none with the crates above. craft-fig = { git = "https://github.com/storytold/craft-libs", rev = "d13ac58a15c45d618f25910772502e38c4ce1d4c" } -craft-figcraft-format = { git = "https://github.com/storytold/craft-libs", rev = "9aac9301a" } +craft-figcraft-format = { git = "https://github.com/storytold/craft-libs", rev = "0e2a717c644bcfaae1a27cd5e2f6b4c0a6b7798b" } craft-figma-rest = { git = "https://github.com/storytold/craft-libs", rev = "d13ac58a15c45d618f25910772502e38c4ce1d4c" } # foundation From 6081e41a7080ba9b02ac25bb1562ba0f5cc18cd9 Mon Sep 17 00:00:00 2001 From: "Dylan M. Taylor" Date: Fri, 13 Nov 2026 16:07:45 +0000 Subject: [PATCH 8/8] Use shared FigCraft preview reader --- Cargo.lock | 6 +++--- Cargo.toml | 2 +- crates/format/src/thumbnail.rs | 23 +++++++++++------------ 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 50567d97d..94a6987b0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1029,9 +1029,9 @@ dependencies = [ [[package]] name = "craft-figcraft-format" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=0e2a717c644bcfaae1a27cd5e2f6b4c0a6b7798b#0e2a717c644bcfaae1a27cd5e2f6b4c0a6b7798b" +source = "git+https://github.com/storytold/craft-libs?rev=976eb51c59edb0a37f24105850f52d921c8518ee#976eb51c59edb0a37f24105850f52d921c8518ee" dependencies = [ - "craft-zip 0.1.0 (git+https://github.com/storytold/craft-libs?rev=0e2a717c644bcfaae1a27cd5e2f6b4c0a6b7798b)", + "craft-zip 0.1.0 (git+https://github.com/storytold/craft-libs?rev=976eb51c59edb0a37f24105850f52d921c8518ee)", "serde", "serde_json", "thiserror 2.0.21", @@ -1231,7 +1231,7 @@ dependencies = [ [[package]] name = "craft-zip" version = "0.1.0" -source = "git+https://github.com/storytold/craft-libs?rev=0e2a717c644bcfaae1a27cd5e2f6b4c0a6b7798b#0e2a717c644bcfaae1a27cd5e2f6b4c0a6b7798b" +source = "git+https://github.com/storytold/craft-libs?rev=976eb51c59edb0a37f24105850f52d921c8518ee#976eb51c59edb0a37f24105850f52d921c8518ee" dependencies = [ "flate2", "thiserror 2.0.21", diff --git a/Cargo.toml b/Cargo.toml index 37df323f2..0b39eee4c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -60,7 +60,7 @@ craft-settings = { git = "https://github.com/storytold/craft-libs", rev = "e11f9 # a .fig message onto the REST file JSON, and that JSON's shared helpers. These crates share # types, so they stay on one rev; they share none with the crates above. craft-fig = { git = "https://github.com/storytold/craft-libs", rev = "d13ac58a15c45d618f25910772502e38c4ce1d4c" } -craft-figcraft-format = { git = "https://github.com/storytold/craft-libs", rev = "0e2a717c644bcfaae1a27cd5e2f6b4c0a6b7798b" } +craft-figcraft-format = { git = "https://github.com/storytold/craft-libs", rev = "976eb51c59edb0a37f24105850f52d921c8518ee" } craft-figma-rest = { git = "https://github.com/storytold/craft-libs", rev = "d13ac58a15c45d618f25910772502e38c4ce1d4c" } # foundation diff --git a/crates/format/src/thumbnail.rs b/crates/format/src/thumbnail.rs index dc1f4eaff..d60cd8883 100644 --- a/crates/format/src/thumbnail.rs +++ b/crates/format/src/thumbnail.rs @@ -7,13 +7,12 @@ use std::io::{Read, Seek}; use std::path::Path; /// Largest `thumbnail.png` read from an archive. -pub const MAX_THUMBNAIL_BYTES: u64 = 8 * 1024 * 1024; +pub const MAX_THUMBNAIL_BYTES: u64 = craft_figcraft_format::MAX_PREVIEW_THUMBNAIL_BYTES; /// Largest image file read as its own picture. pub const MAX_IMAGE_FILE_BYTES: u64 = 32 * 1024 * 1024; /// Widest or tallest picture decoded (pixels); a larger one gives none. pub const MAX_THUMBNAIL_SIDE: u32 = 8192; -/// Largest `document.json` read for its page colour, and `.fig` `meta.json`. -const MAX_DOCUMENT_BYTES: u64 = 64 * 1024 * 1024; +/// Largest `.fig` `meta.json` read for its page colour. const MAX_META_BYTES: u64 = 1024 * 1024; /// Most memory one decode may take. const MAX_DECODE_ALLOC: u64 = 256 * 1024 * 1024; @@ -29,8 +28,7 @@ pub struct Thumbnail { /// The bytes of `thumbnail.png` in a zip archive (`.figcraft`, `.fig`); `None` when it isn't a zip, /// has no thumbnail, or the thumbnail is larger than [`MAX_THUMBNAIL_BYTES`]. pub fn thumbnail_png(archive: R) -> Option> { - let mut z = zip::ZipArchive::new(archive).ok()?; - crate::read_member(&mut z, "thumbnail.png", MAX_THUMBNAIL_BYTES).ok() + craft_figcraft_format::read_preview_thumbnail(archive).ok().flatten() } /// The bytes of `previews/preview.png` in a `.sketch` zip; `None` as [`thumbnail_png`] gives it. @@ -148,19 +146,20 @@ pub fn file_preview(path: &Path, fit: u32) -> Preview { /// The [`Preview`] of a `.figcraft` or `.fig` archive: `thumbnail.png`, and the first page's colour /// from `document.json` (`.figcraft`) or `meta.json` (`.fig`). pub fn archive_preview(archive: R, fit: u32) -> Preview { - let Ok(mut z) = zip::ZipArchive::new(archive) else { return Preview::default() }; - let picture = crate::read_member(&mut z, "thumbnail.png", MAX_THUMBNAIL_BYTES).ok().and_then(|png| decode_thumbnail(&png, fit)); - let (page, kind) = if z.file_names().any(|n| n == "document.json") { - crate::read_member(&mut z, "document.json", MAX_DOCUMENT_BYTES).ok().map(|json| figcraft_page(&json)).unwrap_or_default() + let Ok(mut z) = craft_figcraft_format::SeekArchive::open(archive, u64::MAX) else { return Preview::default() }; + let native = craft_figcraft_format::read_preview_archive_from(&mut z); + let picture = native.thumbnail_png.and_then(|png| decode_thumbnail(&png, fit)); + let (page, kind) = if native.has_document { + native.document.as_ref().map(figcraft_page).unwrap_or_default() } else { - (crate::read_member(&mut z, "meta.json", MAX_META_BYTES).ok().and_then(|json| fig_page(&json)), figcraft_doc::FileKind::Design) + let meta = z.read("meta.json", MAX_META_BYTES).ok().flatten(); + (meta.as_deref().and_then(fig_page), figcraft_doc::FileKind::Design) }; Preview { page, picture, kind } } /// The first page's background in a `.figcraft` `document.json`, and the file's kind. -fn figcraft_page(json: &[u8]) -> (Option<[f32; 4]>, figcraft_doc::FileKind) { - let Ok(head) = craft_figcraft_format::read_preview_document(json) else { return (None, figcraft_doc::FileKind::Design) }; +fn figcraft_page(head: &craft_figcraft_format::PreviewDocument) -> (Option<[f32; 4]>, figcraft_doc::FileKind) { let page = head.pages.first().and_then(|first| head.nodes.get(&first.to_string())).and_then(|n| n.background.as_ref()).and_then(rgba); let kind = match head.kind { craft_figcraft_format::PreviewKind::Design => figcraft_doc::FileKind::Design,