From 89ed4cb28414a173126e97d1853e957343c5ba98 Mon Sep 17 00:00:00 2001 From: vistaero Date: Sat, 10 Oct 2026 07:08:54 +0200 Subject: [PATCH 1/3] Stream native PSD and EXR imports through bounded seekable bands --- Cargo.lock | 1 + apps/photocraft/src/services.rs | 6 + crates/automation/Cargo.toml | 1 + crates/automation/src/files.rs | 22 +- crates/automation/src/headless.rs | 40 +- crates/automation/src/workspace.rs | 105 ++++- crates/codecs/examples/exr_region_probe.rs | 74 +++ crates/codecs/src/exr_stream.rs | 292 ++++++++++++ crates/codecs/src/lib.rs | 6 +- crates/codecs/tests/exr_stream.rs | 161 +++++++ crates/engine/src/file_cmds.rs | 30 +- crates/engine/src/jobs.rs | 31 +- crates/format/src/atomic.rs | 60 +++ crates/format/src/atomic/tests.rs | 84 ++++ crates/io/examples/scratch_psb_probe.rs | 36 ++ crates/io/src/exr_native.rs | 76 ++++ crates/io/src/lib.rs | 165 +++++++ crates/io/src/psd_export.rs | 85 +++- crates/io/src/psd_import.rs | 19 +- crates/io/src/psd_stream.rs | 395 ++++++++++++++++ crates/io/tests/streaming_exr.rs | 94 ++++ crates/io/tests/streaming_psd.rs | 136 ++++++ crates/io/tests/svg.rs | 17 + crates/psd/src/layer.rs | 4 +- crates/psd/src/lib.rs | 1 + crates/psd/src/stream.rs | 499 +++++++++++++++++++++ crates/ui-egui/src/control.rs | 50 +++ crates/ui-egui/src/file_open.rs | 26 +- crates/ui-egui/src/jobs_ui.rs | 45 ++ crates/ui-egui/src/lib.rs | 3 + crates/ui-egui/tests/exr_open.rs | 44 ++ docs/roadmap.md | 2 + docs/scorecard.md | 2 +- docs/scratch-disks.md | 5 +- docs/streaming-io.md | 36 ++ scorecard/reliability.toml | 2 +- 36 files changed, 2587 insertions(+), 68 deletions(-) create mode 100644 crates/codecs/examples/exr_region_probe.rs create mode 100644 crates/codecs/src/exr_stream.rs create mode 100644 crates/codecs/tests/exr_stream.rs create mode 100644 crates/io/examples/scratch_psb_probe.rs create mode 100644 crates/io/src/exr_native.rs create mode 100644 crates/io/src/psd_stream.rs create mode 100644 crates/io/tests/streaming_exr.rs create mode 100644 crates/io/tests/streaming_psd.rs create mode 100644 crates/psd/src/stream.rs create mode 100644 crates/ui-egui/tests/exr_open.rs create mode 100644 docs/streaming-io.md diff --git a/Cargo.lock b/Cargo.lock index 79b80aae04..9dfd9946f6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4295,6 +4295,7 @@ dependencies = [ "photocraft-format", "photocraft-geom", "photocraft-io", + "photocraft-raster", "rmcp", "schemars", "serde", diff --git a/apps/photocraft/src/services.rs b/apps/photocraft/src/services.rs index d4a3869f31..3cae110ea3 100644 --- a/apps/photocraft/src/services.rs +++ b/apps/photocraft/src/services.rs @@ -406,6 +406,11 @@ fn shell_word(s: &str) -> String { pub fn native(automation: Option) -> Services { let clip: Rc>> = Rc::default(); + let automation_import_path = automation.clone().map(|workspace| { + std::sync::Arc::new(move |path: &str, ctl: &photocraft_raster::Interrupt<'_>| { + crate::crash_guard::guard("Open", || workspace.read_document(path, ctl).map_err(|e| e.to_string())) + }) as photocraft_ui_egui::AutomationImportPathFn + }); let automation_read = automation.clone().map(|workspace| { Box::new(move |path: &str| { let bytes = workspace.read(path).map_err(|error| error.to_string())?; @@ -449,6 +454,7 @@ pub fn native(automation: Option) -> file_dialog: Some(Box::new(show_file_dialog)), write: Some(Box::new(|path: &str, bytes: &[u8]| write_atomic(Path::new(path), bytes))), automation_read, + automation_import_path, automation_write, automation_open_pcraft, automation_save, diff --git a/crates/automation/Cargo.toml b/crates/automation/Cargo.toml index 0738140a41..f15f500e05 100644 --- a/crates/automation/Cargo.toml +++ b/crates/automation/Cargo.toml @@ -15,6 +15,7 @@ photocraft-doc = { workspace = true } photocraft-compose = { workspace = true } photocraft-codecs = { workspace = true } photocraft-io = { workspace = true } +photocraft-raster = { workspace = true } photocraft-format = { path = "../format" } photocraft-engine = { workspace = true } serde = { workspace = true } diff --git a/crates/automation/src/files.rs b/crates/automation/src/files.rs index cd34ab3410..61c6ba246a 100644 --- a/crates/automation/src/files.rs +++ b/crates/automation/src/files.rs @@ -31,9 +31,17 @@ pub fn open(path: &Path) -> Result { if path.is_dir() { return Ok(Opened { document: photocraft_format::load_path(path)?, warnings: Vec::new(), source_read_only: false }); } - let bytes = photocraft_format::read_file(path).map_err(|e| AutomationError::Io(format!("{}: {e}", path.display())))?; let name = path.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(); - open_bytes(&name, &bytes) + #[cfg(not(target_arch = "wasm32"))] + { + let r = photocraft_io::import_path_with(&name, path, &photocraft_raster::Interrupt::NONE)?; + Ok(Opened { document: r.document, warnings: r.warnings, source_read_only: r.source_read_only }) + } + #[cfg(target_arch = "wasm32")] + { + let bytes = photocraft_format::read_file(path).map_err(|e| AutomationError::Io(format!("{}: {e}", path.display())))?; + open_bytes(&name, &bytes) + } } /// Previews for a `.pcraft` bundle. @@ -73,6 +81,16 @@ pub fn save( w.save_path(&named, path, &previews(&named))?; return Ok(Vec::new()); } + #[cfg(not(target_arch = "wasm32"))] + if ext == "psb" || ext == "psd" && (opts.force_psb || doc.size.width > 30_000 || doc.size.height > 30_000) { + let mut warnings = Vec::new(); + photocraft_format::atomic::atomic_write_stream(path, |file| { + warnings = photocraft_io::export_psb_into(doc, file, &photocraft_raster::Interrupt::NONE).map_err(|e| std::io::Error::other(e.to_string()))?; + Ok(()) + }) + .map_err(|e| AutomationError::Io(e.to_string()))?; + return Ok(warnings); + } let r = photocraft_io::export(doc, &ext, opts)?; // Crash-safe: a failed write never destroys the previous file. photocraft_format::atomic_write(path, &r.bytes).map_err(|e| AutomationError::Io(e.to_string()))?; diff --git a/crates/automation/src/headless.rs b/crates/automation/src/headless.rs index d6b3c17102..3a61ffa4dc 100644 --- a/crates/automation/src/headless.rs +++ b/crates/automation/src/headless.rs @@ -79,9 +79,17 @@ impl Headless { Filesystem::Denied => return Err(AutomationError::BadRequest("automation filesystem access is not granted: read authority is absent".into())), Filesystem::TrustedLocal => files::open(path)?, Filesystem::Workspace(workspace) => { - let bytes = workspace.read(requested)?; - let name = path.file_name().and_then(|name| name.to_str()).unwrap_or(requested); - files::open_bytes(name, &bytes)? + #[cfg(not(target_arch = "wasm32"))] + { + let r = workspace.read_document(requested, &photocraft_raster::Interrupt::NONE)?; + files::Opened { document: r.document, warnings: r.warnings, source_read_only: r.source_read_only } + } + #[cfg(target_arch = "wasm32")] + { + let bytes = workspace.read(requested)?; + let name = path.file_name().and_then(|name| name.to_str()).unwrap_or(requested); + files::open_bytes(name, &bytes)? + } } }; let path = match file_cmds::template_name(&self.session, requested) { @@ -251,9 +259,29 @@ impl Headless { files::save(&doc, &target, format, opts, Some(writer))? } Filesystem::Workspace(workspace) => { - let (bytes, warnings) = files::save_bytes(&doc, target_text, format, opts)?; - workspace.write(target_text, &bytes)?; - warnings + #[cfg(not(target_arch = "wasm32"))] + if format + .map(|f| f.trim_start_matches('.').eq_ignore_ascii_case("psb")) + .unwrap_or_else(|| target.extension().is_some_and(|ext| ext.eq_ignore_ascii_case("psb"))) + { + let mut warnings = Vec::new(); + workspace.write_stream(target_text, |file| { + warnings = photocraft_io::export_psb_into(&doc, file, &photocraft_raster::Interrupt::NONE) + .map_err(|e| std::io::Error::other(e.to_string()))?; + Ok(()) + })?; + warnings + } else { + let (bytes, warnings) = files::save_bytes(&doc, target_text, format, opts)?; + workspace.write(target_text, &bytes)?; + warnings + } + #[cfg(target_arch = "wasm32")] + { + let (bytes, warnings) = files::save_bytes(&doc, target_text, format, opts)?; + workspace.write(target_text, &bytes)?; + warnings + } } }; warnings.extend(photocraft_engine::type_extra_cmds::missing_font_warnings(&doc)); diff --git a/crates/automation/src/workspace.rs b/crates/automation/src/workspace.rs index cde3a3a702..6321bc241f 100644 --- a/crates/automation/src/workspace.rs +++ b/crates/automation/src/workspace.rs @@ -68,6 +68,24 @@ impl AuthorizedWorkspace { let source = DirHandle { dir }; Ok(Some(photocraft_format::load_bytes_source(&source, &photocraft_format::LoadOptions::default())?)) } + /// Open a large document by rows through the held read capability, without a file-sized + /// byte array. Independent channel cursors remain rooted beneath the authorized directory. + #[cfg(not(target_arch = "wasm32"))] + pub fn read_document(&self, path: &str, ctl: &photocraft_raster::Interrupt<'_>) -> Result { + let relative = request_path(path, self.read.as_ref())?; + let root = self.read.as_ref().ok_or_else(|| AutomationError::BadRequest(format!("{DENIED}: read authority is absent")))?; + let original = root.dir.metadata(&relative).map_err(|e| file_error("read", path, e))?; + if !original.is_file() { + return Err(AutomationError::BadRequest("automation read path is not a regular file".into())); + } + let name = relative.file_name().and_then(|n| n.to_str()).unwrap_or(path); + let result = photocraft_io::import_seekable_with(name, &|| root.dir.open(&relative), ctl)?; + let current = root.dir.metadata(&relative).map_err(|e| file_error("read", path, e))?; + if current.len() != original.len() || current.modified().ok() != original.modified().ok() { + return Err(AutomationError::Io("source changed during import; open it again".into())); + } + Ok(result) + } /// Create or replace one file below the configured write root, crash-safely: the bytes go to /// a temporary file beside the target, which is synced and renamed over it (the same steps @@ -78,18 +96,30 @@ impl AuthorizedWorkspace { /// resolution and file creation relative to the held directory handle, so /// a non-existent final target is supported without ambient path access. pub fn write(&self, path: &str, bytes: &[u8]) -> Result<(), AutomationError> { + self.write_stream(path, |file| file.write_all(bytes)) + } + + /// Atomically publish a seekable encoder through the held write capability. No ambient + /// paths are exposed to the encoder, including for temporary-file creation and rename. + pub fn write_stream(&self, path: &str, encode: impl FnOnce(&mut cap_std::fs::File) -> std::io::Result<()>) -> Result<(), AutomationError> { + struct Cleanup(Arc, PathBuf); + impl Drop for Cleanup { + fn drop(&mut self) { + let _ = self.0.remove_file(&self.1); + } + } let relative = request_path(path, self.write.as_ref())?; let root = self.write.as_ref().ok_or_else(|| AutomationError::BadRequest(format!("{DENIED}: write authority is absent")))?; let leaf = relative.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(); let parent = relative.parent().map(Path::to_path_buf).unwrap_or_default(); let tmp = parent.join(photocraft_format::atomic::temp_name(&leaf)); let mut options = OpenOptions::new(); - options.write(true).create_new(true); - let written = root.dir.open_with(&tmp, &options).and_then(|mut file| { - file.write_all(bytes)?; - // Falls back to a plain fsync on shares that refuse a full flush (#1336). - photocraft_format::atomic::sync_file(&file.into_std()) - }); + options.read(true).write(true).create_new(true); + let file = root.dir.open_with(&tmp, &options).map_err(|e| file_error("write", path, e))?; + let _cleanup = Cleanup(root.dir.clone(), tmp.clone()); + // This binding drops before the cleanup guard if an encoder unwinds. + let mut file = file; + let written = encode(&mut file).and_then(|()| photocraft_format::atomic::sync_file(&file.into_std())); let renamed = written.and_then(|()| photocraft_format::atomic::retry_rename(RenameRetry::platform(), || root.dir.rename(&tmp, &root.dir, &relative))); if let Err(e) = renamed { let _ = root.dir.remove_file(&tmp); @@ -446,6 +476,69 @@ impl Drop for ExportSequence { mod tests { use super::*; + #[cfg(not(target_arch = "wasm32"))] + #[test] + fn seekable_document_import_stays_within_read_capability() { + let (_, _, workspace) = roots("stream-read"); + let mut session = photocraft_engine::Session::new(); + session.execute("file.new", serde_json::json!({"width": 32, "height": 16})).unwrap(); + session.execute("layer.new.layer", serde_json::json!({})).unwrap(); + let doc = &session.active().unwrap().doc; + workspace + .write_stream("small.psb", |file| { + photocraft_io::export_psb_into(doc, file, &photocraft_raster::Interrupt::NONE).map_err(|e| std::io::Error::other(e.to_string()))?; + Ok(()) + }) + .unwrap(); + let opened = workspace.read_document("small.psb", &photocraft_raster::Interrupt::NONE).unwrap(); + assert_eq!(opened.document.size, doc.size); + assert_eq!(opened.document.layers.len(), doc.layers.len()); + assert!(workspace.read_document("../small.psb", &photocraft_raster::Interrupt::NONE).is_err()); + assert!(workspace.read_document("C:/small.psb", &photocraft_raster::Interrupt::NONE).is_err()); + } + + #[test] + fn seekable_capability_writer_rejects_escape_and_preserves_failed_destination() { + use std::io::{Seek, SeekFrom}; + let (inside, outside, workspace) = roots("stream"); + std::fs::write(inside.join("kept.psb"), b"original").unwrap(); + assert!(workspace.write_stream("../escaped.psb", |_| Ok(())).is_err()); + assert!(!outside.join("escaped.psb").exists()); + assert!( + workspace + .write_stream("kept.psb", |file| { + file.write_all(b"partial")?; + Err(std::io::Error::new(std::io::ErrorKind::StorageFull, "injected full disk")) + }) + .is_err() + ); + assert_eq!(std::fs::read(inside.join("kept.psb")).unwrap(), b"original"); + assert!(temp_files(&inside).is_empty()); + workspace + .write_stream("kept.psb", |file| { + file.write_all(b"0000payload")?; + file.seek(SeekFrom::Start(0))?; + file.write_all(b"0007") + }) + .unwrap(); + assert_eq!(std::fs::read(inside.join("kept.psb")).unwrap(), b"0007payload"); + } + + #[test] + fn seekable_capability_unwind_removes_temporary_after_closing_handle() { + let (inside, _, workspace) = roots("stream-unwind"); + std::fs::write(inside.join("kept.psb"), b"original").unwrap(); + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _ = workspace.write_stream("kept.psb", |file| { + file.write_all(b"partial")?; + panic!("injected encoder unwind"); + }); + })); + assert!(result.is_err()); + assert_eq!(std::fs::read(inside.join("kept.psb")).unwrap(), b"original"); + assert!(temp_files(&inside).is_empty()); + } + fn roots(name: &str) -> (PathBuf, PathBuf, AuthorizedWorkspace) { let base = std::env::temp_dir().join(format!("photocraft-workspace-{}-{name}", std::process::id())); let inside = base.join("inside"); diff --git a/crates/codecs/examples/exr_region_probe.rs b/crates/codecs/examples/exr_region_probe.rs new file mode 100644 index 0000000000..4b57e803aa --- /dev/null +++ b/crates/codecs/examples/exr_region_probe.rs @@ -0,0 +1,74 @@ +//! Read-only independent EXR sample oracle: `exr_region_probe file.exr x y [x y ...]`. +//! Uses the dependency's filtered block reader, independently of our band conversion. +use exr::{block::reader::ChunksReader, meta::attribute::SampleType}; +use std::io::BufReader; + +fn run() -> Result<(), Box> { + let args: Vec = std::env::args().skip(1).collect(); + let path = args.first().ok_or("usage: exr_region_probe file.exr x y [x y ...]")?; + let coordinates = args.get(1..).ok_or("missing coordinates")?; + if coordinates.is_empty() || !coordinates.len().is_multiple_of(2) || coordinates.len() > 512 { + return Err("provide 1 to 256 coordinate pairs".into()); + } + let points: Vec<(usize, usize)> = coordinates + .chunks_exact(2) + .map(|p| Ok((p.first().ok_or("x")?.parse()?, p.get(1).ok_or("y")?.parse()?))) + .collect::>>()?; + // Bound metadata, dimensions and block sizes before using the oracle reader. + let info = photocraft_codecs::exr_stream::Decoder::new(std::fs::File::open(path)?, &|| false)?.info(); + if points.iter().any(|&(x, y)| x >= info.width as usize || y >= info.height as usize) { + return Err("coordinate outside the data window".into()); + } + let reader = exr::block::read(BufReader::new(std::fs::File::open(path)?), false)?; + let mut blocks = reader + .filter_chunks(false, |_, _, block| points.iter().any(|&(_, y)| y >= block.pixel_position.1 && y < block.pixel_position.1 + block.pixel_size.1))? + .sequential_decompressor(false); + let channels = blocks.meta_data().headers.first().ok_or("missing header")?.channels.clone(); + let mut samples = vec![[0_f32, 0.0, 0.0, 1.0]; points.len()]; + let mut seen = vec![[false; 3]; points.len()]; + while let Some(block) = blocks.decompress_next_block() { + let block = block?; + for line in block.lines(&channels) { + let channel = channels.list.get(line.location.channel).ok_or("channel")?; + let name = channel.name.to_string(); + let component = match name.rsplit('.').next() { + Some("R") => 0, + Some("G") => 1, + Some("B") => 2, + _ => continue, + }; + for (index, &(x, y)) in points.iter().enumerate().filter(|(_, p)| p.1 == line.location.position.1) { + let value = match channel.sample_type { + SampleType::F16 => line.read_samples::().nth(x).ok_or("sample")??.to_f32(), + SampleType::F32 => line.read_samples::().nth(x).ok_or("sample")??, + SampleType::U32 => line.read_samples::().nth(x).ok_or("sample")?? as f32, + }; + if !value.is_finite() { + return Err("non-finite sample cannot be represented in JSON".into()); + } + *samples.get_mut(index).and_then(|s| s.get_mut(component)).ok_or("sample index")? = value; + *seen.get_mut(index).and_then(|s| s.get_mut(component)).ok_or("sample index")? = true; + let _ = y; + } + } + } + if seen.iter().flatten().any(|v| !v) { + return Err("missing RGB samples".into()); + } + println!("{{\"width\":{},\"height\":{},\"samples\":[", info.width, info.height); + for (index, (&(x, y), sample)) in points.iter().zip(samples).enumerate() { + println!("{}{{\"x\":{x},\"y\":{y},\"rgba\":{:?}}}", if index == 0 { "" } else { "," }, sample); + } + println!("]}}"); + Ok(()) +} + +fn main() -> std::process::ExitCode { + match run() { + Ok(()) => std::process::ExitCode::SUCCESS, + Err(error) => { + eprintln!("{error}"); + std::process::ExitCode::FAILURE + } + } +} diff --git a/crates/codecs/src/exr_stream.rs b/crates/codecs/src/exr_stream.rs new file mode 100644 index 0000000000..a1d5818a16 --- /dev/null +++ b/crates/codecs/src/exr_stream.rs @@ -0,0 +1,292 @@ +//! Bounded, seekable single-part scanline OpenEXR decoding. Total pixel count does not +//! size a decoded image buffer: only one compression block is returned at a time. +use std::io::{Cursor, Read, Seek, SeekFrom}; + +use crate::{CodecError, Format}; +use exr::{ + block::{ + UncompressedBlock, + chunk::{Chunk, CompressedBlock, CompressedScanLineBlock}, + lines::LineIndex, + }, + meta::{BlockDescription, MetaData, attribute::SampleType}, +}; + +const META_BYTES: usize = 8 << 20; +const BLOCK_BYTES: usize = 64 << 20; +const MAX_CHUNKS: usize = 1 << 20; + +fn invalid(e: impl std::fmt::Display) -> CodecError { + CodecError::malformed(Format::OpenExr, e) +} +fn limit(s: &str) -> CodecError { + CodecError::LimitExceeded(s.into()) +} +fn unsupported(s: &str) -> CodecError { + CodecError::unsupported(Format::OpenExr, s) +} + +fn read(r: &mut impl Read) -> Result<[u8; N], CodecError> { + let mut bytes = [0; N]; + r.read_exact(&mut bytes).map_err(invalid)?; + Ok(bytes) +} + +fn string(r: &mut impl Read, prefix: &mut Vec) -> Result { + for i in 0..256 { + let b = read::<1>(r)?; + prefix.extend(b); + if b == [0] { + return Ok(i != 0); + } + } + Err(limit("EXR attribute name exceeds 255 bytes")) +} + +fn metadata(r: &mut impl Read, cancel: &dyn Fn() -> bool) -> Result, CodecError> { + let start = read::<8>(r)?; + if start.get(..4) != Some(&[0x76, 0x2f, 0x31, 0x01]) { + return Err(invalid("EXR signature")); + } + let version = u32::from_le_bytes(start.get(4..).ok_or_else(|| invalid("version"))?.try_into().map_err(invalid)?); + if version & (0x200 | 0x800 | 0x1000) != 0 { + return Err(unsupported("streaming requires a single flat scanline part")); + } + let mut prefix = start.to_vec(); + let mut attributes = 0; + loop { + if cancel() { + return Err(invalid("EXR import cancelled")); + } + if !string(r, &mut prefix)? { + break; + } + attributes += 1; + if attributes > 4096 { + return Err(limit("EXR metadata exceeds 4096 attributes")); + } + string(r, &mut prefix)?; + let length = read::<4>(r)?; + prefix.extend(length); + let n = usize::try_from(i32::from_le_bytes(length)).map_err(invalid)?; + if n > META_BYTES || prefix.len().checked_add(n).is_none_or(|v| v > META_BYTES) { + return Err(limit("EXR metadata exceeds 8 MiB")); + } + let begin = prefix.len(); + prefix.try_reserve_exact(n).map_err(|_| limit("EXR metadata allocation"))?; + prefix.resize(begin + n, 0); + for bytes in prefix.get_mut(begin..).ok_or_else(|| invalid("attribute size"))?.chunks_mut(64 << 10) { + if cancel() { + return Err(invalid("EXR import cancelled")); + } + r.read_exact(bytes).map_err(invalid)?; + } + } + Ok(prefix) +} + +/// Header facts used by the document owner to admit tile and working memory. +#[derive(Clone, Copy, Debug)] +pub struct Info { + pub width: u32, + pub height: u32, + pub gray: bool, + pub has_alpha: bool, + pub all_half: bool, + pub working_bytes: u64, + pub blocks: usize, +} + +/// Interleaved little-endian F32, GrayA or RGBA, at source resolution. +pub struct Band { + pub x: u32, + pub y: u32, + pub width: u32, + pub height: u32, + pub data: Vec, +} + +pub struct Decoder { + source: R, + meta: MetaData, + info: Info, + offsets: Vec, + seen: Vec, + channels: Vec>, + next: usize, + file_len: u64, + chunks_start: u64, + raw_limit: usize, + rows: usize, +} + +impl Decoder { + /// Read only bounded metadata and offsets; allocate no full-image or channel buffer. + pub fn new(mut source: R, cancel: &dyn Fn() -> bool) -> Result { + let file_len = source.seek(SeekFrom::End(0)).map_err(invalid)?; + source.seek(SeekFrom::Start(0)).map_err(invalid)?; + let prefix = metadata(&mut source, cancel)?; + let meta = std::panic::catch_unwind(|| MetaData::read_from_buffered(Cursor::new(&prefix), false)) + .map_err(|_| invalid("invalid EXR metadata"))? + .map_err(invalid)?; + let header = meta.headers.first().ok_or_else(|| invalid("missing header"))?; + if meta.headers.len() != 1 || header.deep || !matches!(header.blocks, BlockDescription::ScanLines) { + return Err(unsupported("streaming requires a single flat scanline part")); + } + let (w, h) = (header.layer_size.0, header.layer_size.1); + if w == 0 || h == 0 || w > 1 << 18 || h > 1 << 18 { + return Err(limit("EXR dimensions exceed the streaming limit of 262144")); + } + let list = &header.channels.list; + if list.is_empty() || list.len() > 64 || list.iter().any(|c| c.sampling.0 != 1 || c.sampling.1 != 1) { + return Err(unsupported("EXR channel count or subsampling")); + } + let names: Vec = list.iter().map(|c| c.name.to_string()).collect(); + let find = |name| names.iter().position(|n| n.rsplit('.').next() == Some(name)); + let (selected, gray) = if let (Some(r), Some(g), Some(b)) = (find("R"), find("G"), find("B")) { + (vec![Some(r), Some(g), Some(b), find("A")], false) + } else if let Some(y) = find("Y").or_else(|| (list.len() == 1).then_some(0)) { + (vec![Some(y), find("A")], true) + } else { + return Err(unsupported("no R/G/B or Y channels")); + }; + // The document path currently tags Rec. 709 linear samples. Keep custom + // primaries on the existing decoder until their profiles can be mapped. + if !gray && header.shared_attributes.chromaticities.is_some() { + return Err(unsupported("streaming custom EXR chromaticities requires a color profile")); + } + let all_half = selected.iter().flatten().all(|i| list.get(*i).is_some_and(|c| c.sample_type == SampleType::F16)); + let has_alpha = selected.last().copied().flatten().is_some(); + let channels = (0..list.len()).map(|i| selected.iter().position(|s| *s == Some(i))).collect(); + let rows = header.max_block_pixel_size().1; + let expected = h.div_ceil(rows.max(1)); + if rows == 0 || header.chunk_count != expected || expected > MAX_CHUNKS { + return Err(limit("EXR block count exceeds the streaming budget")); + } + let raw_limit = w + .checked_mul(rows) + .and_then(|n| n.checked_mul(header.channels.bytes_per_pixel)) + .filter(|n| *n <= BLOCK_BYTES) + .ok_or_else(|| limit("EXR compression block exceeds 64 MiB"))?; + let output = (w as u64).saturating_mul(rows as u64).saturating_mul(selected.len() as u64 * 4); + let info = Info { + width: w as u32, + height: h as u32, + gray, + has_alpha, + all_half, + blocks: expected, + working_bytes: (raw_limit as u64).saturating_mul(8).saturating_add(output.saturating_mul(2)).saturating_add(1 << 20), + }; + let chunks_start = source + .stream_position() + .map_err(invalid)? + .checked_add(expected as u64 * 8) + .filter(|v| *v <= file_len) + .ok_or_else(|| invalid("truncated offset table"))?; + let mut offsets = Vec::new(); + offsets.try_reserve_exact(expected).map_err(|_| limit("EXR offset allocation"))?; + for _ in 0..expected { + if cancel() { + return Err(invalid("EXR import cancelled")); + } + let offset = u64::from_le_bytes(read(&mut source)?); + if offset < chunks_start || offset.checked_add(8).is_none_or(|v| v > file_len) { + return Err(invalid("EXR chunk offset")); + } + offsets.push(offset); + } + Ok(Self { source, meta, info, offsets, seen: vec![false; expected], channels, next: 0, file_len, chunks_start, raw_limit, rows }) + } + + pub fn info(&self) -> Info { + self.info + } + + /// Decode one block. Check cancellation while reading and between channel rows. + pub fn next_band(&mut self, cancel: &dyn Fn() -> bool) -> Result, CodecError> { + let Some(offset) = self.offsets.get(self.next).copied() else { + return Ok(None); + }; + if cancel() { + return Err(invalid("EXR import cancelled")); + } + self.source.seek(SeekFrom::Start(offset)).map_err(invalid)?; + let y = i32::from_le_bytes(read(&mut self.source)?); + let n = usize::try_from(i32::from_le_bytes(read(&mut self.source)?)).map_err(invalid)?; + if n > self.raw_limit.saturating_mul(2).saturating_add(64 << 10) + || offset < self.chunks_start + || offset.checked_add(8).and_then(|v| v.checked_add(n as u64)).is_none_or(|v| v > self.file_len) + { + return Err(invalid("EXR compressed block length")); + } + let header = self.meta.headers.first().ok_or_else(|| invalid("missing header"))?; + let row = i64::from(y) - i64::from(header.own_attributes.layer_position.1); + if row < 0 || row as u64 >= u64::from(self.info.height) || !(row as usize).is_multiple_of(self.rows) { + return Err(invalid("EXR block coordinate")); + } + let seen = self.seen.get_mut(row as usize / self.rows).ok_or_else(|| invalid("EXR block coordinate"))?; + if *seen { + return Err(invalid("duplicate EXR block")); + } + *seen = true; + let mut packed = Vec::new(); + packed.try_reserve_exact(n).map_err(|_| limit("EXR block allocation"))?; + packed.resize(n, 0); + for bytes in packed.chunks_mut(64 << 10) { + if cancel() { + return Err(invalid("EXR import cancelled")); + } + self.source.read_exact(bytes).map_err(invalid)?; + } + let chunk = + Chunk { layer_index: 0, compressed_block: CompressedBlock::ScanLine(CompressedScanLineBlock { y_coordinate: y, compressed_pixels_le: packed }) }; + let block = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| UncompressedBlock::decompress_chunk(chunk, &self.meta, false))) + .map_err(|_| invalid("invalid EXR compression block"))? + .map_err(invalid)?; + let (w, h) = (block.index.pixel_size.0, block.index.pixel_size.1); + if w != self.info.width as usize + || block.index.pixel_position.0 != 0 + || block.index.pixel_position.1 != row as usize + || h != self.rows.min(self.info.height as usize - row as usize) + { + return Err(invalid("EXR block shape")); + } + let expected = w.checked_mul(h).and_then(|n| n.checked_mul(header.channels.bytes_per_pixel)).ok_or_else(|| invalid("EXR block size"))?; + if block.data.len() != expected { + return Err(invalid("EXR block sample count")); + } + let nc = if self.info.gray { 2 } else { 4 }; + let count = w.checked_mul(h).and_then(|n| n.checked_mul(nc * 4)).ok_or_else(|| limit("EXR output block size"))?; + let mut data = Vec::new(); + data.try_reserve_exact(count).map_err(|_| limit("EXR output allocation"))?; + data.resize(count, 0); + if !self.info.has_alpha { + for pixel in data.chunks_exact_mut(nc * 4) { + pixel.get_mut((nc - 1) * 4..).ok_or_else(|| invalid("alpha sample"))?.copy_from_slice(&1_f32.to_le_bytes()); + } + } + for (range, line) in LineIndex::lines_in_block(block.index, &header.channels) { + if cancel() { + return Err(invalid("EXR import cancelled")); + } + let Some(channel) = self.channels.get(line.channel).copied().flatten() else { + continue; + }; + let sample = header.channels.list.get(line.channel).ok_or_else(|| invalid("EXR channel"))?.sample_type; + let input = block.data.get(range).ok_or_else(|| invalid("EXR row size"))?; + let y = line.position.1.checked_sub(block.index.pixel_position.1).ok_or_else(|| invalid("EXR row coordinate"))?; + let output = data.get_mut(y * w * nc * 4..(y + 1) * w * nc * 4).ok_or_else(|| invalid("EXR row coordinate"))?; + for (bytes, pixel) in input.chunks_exact(sample.bytes_per_sample()).zip(output.chunks_exact_mut(nc * 4)) { + let value = match sample { + SampleType::F16 => half::f16::from_bits(u16::from_ne_bytes(bytes.try_into().map_err(invalid)?)).to_f32(), + SampleType::F32 => f32::from_ne_bytes(bytes.try_into().map_err(invalid)?), + SampleType::U32 => u32::from_ne_bytes(bytes.try_into().map_err(invalid)?) as f32, + }; + pixel.get_mut(channel * 4..channel * 4 + 4).ok_or_else(|| invalid("EXR channel"))?.copy_from_slice(&value.to_le_bytes()); + } + } + self.next += 1; + Ok(Some(Band { x: 0, y: row as u32, width: w as u32, height: h as u32, data })) + } +} diff --git a/crates/codecs/src/lib.rs b/crates/codecs/src/lib.rs index 438e1857e4..2f5422e558 100644 --- a/crates/codecs/src/lib.rs +++ b/crates/codecs/src/lib.rs @@ -7,8 +7,9 @@ //! metadata (EXIF, XMP, DPI, text) are preserved where the format allows. //! * [`caps`] declares per-format capabilities and [`fidelity_warnings`] //! tells the UI exactly what an export will lose. -//! * The API is I/O free (`&[u8]` in, `Vec` out) and builds for -//! `wasm32-unknown-unknown`. Decoders enforce [`Limits`] before allocating. +//! * APIs use bytes or caller-provided readers, with no ambient filesystem access, +//! and build for `wasm32-unknown-unknown`. Byte decoders enforce [`Limits`]; +//! [`exr_stream`] instead bounds metadata and one decoded compression block. #![forbid(unsafe_code)] #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] @@ -16,6 +17,7 @@ mod codecs; mod error; pub mod exif_color; +pub mod exr_stream; mod fidelity; mod format; mod image; diff --git a/crates/codecs/tests/exr_stream.rs b/crates/codecs/tests/exr_stream.rs new file mode 100644 index 0000000000..acf03e8ae1 --- /dev/null +++ b/crates/codecs/tests/exr_stream.rs @@ -0,0 +1,161 @@ +use photocraft_codecs::{exr_stream::Decoder, *}; +use std::io::Cursor; + +fn fixture(layout: ChannelLayout, sample: SampleType, compression: ExrCompression) -> Vec { + let n = 31 * 35 * layout.channels(); + let values: Vec = (0..n) + .map(|i| match i % 5 { + 0 => -0.25, + 1 => 3.5, + 2 => 0.0, + 3 => 0.0001, + _ => 0.5, + }) + .collect(); + let image = Image::from_f32(31, 35, layout, &values).unwrap().convert(layout, sample); + encode(&image, Format::OpenExr, &EncodeOptions { exr_compression: compression, ..Default::default() }).unwrap() +} + +#[test] +fn scanline_blocks_match_whole_decoder_at_each_layout_depth_and_compression() { + for layout in [ChannelLayout::Gray, ChannelLayout::GrayA, ChannelLayout::Rgb, ChannelLayout::Rgba] { + for sample in [SampleType::F16, SampleType::F32] { + for compression in [ExrCompression::None, ExrCompression::Rle, ExrCompression::Zip1, ExrCompression::Zip16, ExrCompression::Piz] { + let bytes = fixture(layout, sample, compression); + let reference = decode(&bytes).unwrap(); + let target = if layout.is_gray() { ChannelLayout::GrayA } else { ChannelLayout::Rgba }; + let reference = reference.convert(target, SampleType::F32); + let mut stream = Decoder::new(Cursor::new(&bytes), &|| false).unwrap(); + assert_eq!(stream.info().all_half, sample == SampleType::F16); + let row = 31 * target.channels() * 4; + let mut actual = vec![0; reference.data().len()]; + let mut blocks = 0; + while let Some(band) = stream.next_band(&|| false).unwrap() { + actual[band.y as usize * row..(band.y + band.height) as usize * row].copy_from_slice(&band.data); + blocks += 1; + } + assert_eq!(blocks, stream.info().blocks); + assert_eq!(actual, reference.data(), "{layout:?}/{sample:?}/{compression:?}"); + } + } + } +} + +fn change_window(bytes: &mut [u8], width: i32, height: i32) { + let needle = b"dataWindow\0box2i\0\x10\0\0\0"; + let at = bytes.windows(needle.len()).position(|v| v == needle).unwrap() + needle.len(); + bytes[at + 8..at + 12].copy_from_slice(&(width - 1).to_le_bytes()); + bytes[at + 12..at + 16].copy_from_slice(&(height - 1).to_le_bytes()); +} + +#[test] +fn enormous_header_is_not_a_whole_image_allocation() { + let mut bytes = fixture(ChannelLayout::Rgb, SampleType::F16, ExrCompression::Zip16); + change_window(&mut bytes, 65536, 32768); + assert!(matches!(decode(&bytes), Err(CodecError::LimitExceeded(ref message)) if message.contains("2147483648 pixels"))); + // Streaming gets as far as the now-truncated offset table, without a pixel allocation. + assert!(matches!(Decoder::new(Cursor::new(bytes), &|| false), Err(CodecError::Malformed { .. }))); +} + +#[test] +fn cancellation_truncation_duplicate_offsets_and_forged_lengths_fail() { + let bytes = fixture(ChannelLayout::Rgb, SampleType::F16, ExrCompression::Zip16); + assert!(Decoder::new(Cursor::new(&bytes), &|| true).is_err()); + let mut decoder = Decoder::new(Cursor::new(&bytes), &|| false).unwrap(); + assert!(decoder.next_band(&|| true).is_err()); + for end in [0, 8, bytes.len() / 2] { + assert!(Decoder::new(Cursor::new(&bytes[..end]), &|| false).and_then(|mut r| r.next_band(&|| false)).is_err()); + } + // Locate the offset table by finding the validated header end. + let mut input = Cursor::new(&bytes); + let _ = exr::meta::MetaData::read_from_buffered(&mut input, false).unwrap(); + let at = input.position() as usize; + let offset = u64::from_le_bytes(bytes[at..at + 8].try_into().unwrap()) as usize; + let mut corrupt = bytes.clone(); + corrupt[offset + 4..offset + 8].copy_from_slice(&i32::MAX.to_le_bytes()); + assert!(Decoder::new(Cursor::new(corrupt), &|| false).unwrap().next_band(&|| false).is_err()); + let mut duplicate = bytes.clone(); + duplicate[at + 8..at + 16].copy_from_slice(&bytes[at..at + 8]); + let mut decoder = Decoder::new(Cursor::new(duplicate), &|| false).unwrap(); + assert!(decoder.next_band(&|| false).unwrap().is_some()); + assert!(decoder.next_band(&|| false).is_err()); +} + +#[test] +fn hostile_metadata_is_bounded_and_mutated_inputs_never_panic() { + let bytes = fixture(ChannelLayout::Rgb, SampleType::F16, ExrCompression::Zip16); + let mut large = bytes.clone(); + let needle = b"channels\0chlist\0"; + let length = bytes.windows(needle.len()).position(|v| v == needle).unwrap() + needle.len(); + large[length..length + 4].copy_from_slice(&i32::MAX.to_le_bytes()); + assert!(matches!(Decoder::new(Cursor::new(large), &|| false), Err(CodecError::LimitExceeded(_)))); + let mut many = bytes[..8].to_vec(); + for _ in 0..4097 { + many.extend_from_slice(b"z\0string\0\0\0\0\0"); + } + many.push(0); + assert!(matches!(Decoder::new(Cursor::new(many), &|| false), Err(CodecError::LimitExceeded(_)))); + // Small deterministic mutations cover headers, offsets and compressed payloads. + for i in 0..256 { + let mut corrupt = bytes.clone(); + let at = i * (bytes.len() - 1) / 255; + corrupt[at] ^= 0xff; + assert!( + std::panic::catch_unwind(|| { + if let Ok(mut decoder) = Decoder::new(Cursor::new(corrupt), &|| false) { + while let Ok(Some(_)) = decoder.next_band(&|| false) {} + } + }) + .is_ok(), + "mutation at byte {at}" + ); + } +} + +#[test] +fn mixed_samples_and_offset_data_window_keep_coordinates_and_values() { + use exr::prelude::{AnyChannel, AnyChannels, Blocks, Compression, Encoding, FlatSamples, Layer, LayerAttributes, LineOrder, WritableImage}; + let channels = AnyChannels::sort( + vec![ + AnyChannel::new("R", FlatSamples::F16(vec![f16::from_f32(-0.25); 31 * 35])), + AnyChannel::new("G", FlatSamples::F32(vec![3.5; 31 * 35])), + AnyChannel::new("B", FlatSamples::U32(vec![42; 31 * 35])), + ] + .into_iter() + .collect(), + ); + let attributes = LayerAttributes { layer_position: exr::math::Vec2(-15, 40), ..Default::default() }; + let layer = + Layer::new((31, 35), attributes, Encoding { compression: Compression::ZIP16, blocks: Blocks::ScanLines, line_order: LineOrder::Decreasing }, channels); + let mut bytes = Cursor::new(Vec::new()); + exr::image::Image::from_layer(layer).write().to_buffered(&mut bytes).unwrap(); + let mut decoder = Decoder::new(bytes, &|| false).unwrap(); + assert!(!decoder.info().all_half); + let mut rows = 0; + while let Some(band) = decoder.next_band(&|| false).unwrap() { + assert_eq!(band.y, rows); + rows += band.height; + for p in band.data.chunks_exact(16) { + let v: Vec = p.chunks_exact(4).map(|s| f32::from_le_bytes(s.try_into().unwrap())).collect(); + assert_eq!(v, [-0.25, 3.5, 42.0, 1.0]); + } + } + assert_eq!(rows, 35); +} + +#[test] +fn unsupported_variants_stay_on_the_guarded_legacy_path() { + let bytes = fixture(ChannelLayout::Rgb, SampleType::F16, ExrCompression::Zip16); + for flag in [0x200_u32, 0x800, 0x1000] { + let mut changed = bytes.clone(); + changed[4..8].copy_from_slice(&(2 | flag).to_le_bytes()); + assert!(matches!(Decoder::new(Cursor::new(changed), &|| false), Err(CodecError::Unsupported { .. }))); + } + let mut custom = bytes[..8].to_vec(); + custom.extend_from_slice(b"chromaticities\0chromaticities\0\x20\0\0\0"); + for v in [0.64_f32, 0.33, 0.3, 0.6, 0.15, 0.06, 0.3127, 0.329] { + custom.extend_from_slice(&v.to_le_bytes()); + } + custom.extend_from_slice(&bytes[8..]); + assert!(matches!(Decoder::new(Cursor::new(custom), &|| false), Err(CodecError::Unsupported { .. }))); +} diff --git a/crates/engine/src/file_cmds.rs b/crates/engine/src/file_cmds.rs index 7689f39679..58c4e6ca7f 100644 --- a/crates/engine/src/file_cmds.rs +++ b/crates/engine/src/file_cmds.rs @@ -357,9 +357,25 @@ pub(crate) fn encode(doc: &Document, path: &str, save: impl Into) -> R } pub(crate) fn save_doc(doc: &Document, path: &str, save: impl Into) -> Result> { - let (bytes, warnings) = encode(doc, path, save)?; - write_file(path, &bytes)?; - Ok(warnings) + #[cfg(not(target_arch = "wasm32"))] + { + let save = save.into(); + let mut opts = photocraft_io::ExportOptions { tiff_layers: save.tiff_layers, ..Default::default() }; + if let Some(q) = save.quality { + let q = (q.clamp(0.0, 12.0) / 12.0 * 99.0 + 1.0).round() as u8; + opts.encode.jpeg_quality = q; + opts.encode.webp_quality = q; + opts.encode.webp_lossless = false; + } + photocraft_io::export_path_with(doc, std::path::Path::new(path), &opts, &photocraft_raster::Interrupt::NONE) + .map_err(|e| EngineError::Other(format!("{path}: {e}"))) + } + #[cfg(target_arch = "wasm32")] + { + let (bytes, warnings) = encode(doc, path, save)?; + write_file(path, &bytes)?; + Ok(warnings) + } } pub(crate) fn str_param<'a>(p: &'a Value, key: &str, cmd: &str) -> Result<&'a str> { @@ -426,8 +442,12 @@ fn close_others(s: &mut Session, p: &Value) -> Result { /// "Revert" history state); the document is clean afterwards. fn revert(s: &mut Session) -> Result { let path = s.active().and_then(|d| d.path.clone()).ok_or(EngineError::Other("the document has never been saved".into()))?; - let bytes = read_file(&path)?; - let fresh = import(&file_name(&path), &bytes)?; + #[cfg(not(target_arch = "wasm32"))] + let fresh = photocraft_io::import_path_with(&file_name(&path), std::path::Path::new(&path), &photocraft_raster::Interrupt::NONE) + .map_err(|e| EngineError::Other(e.to_string()))? + .document; + #[cfg(target_arch = "wasm32")] + let fresh = import(&file_name(&path), &read_file(&path)?)?; s.edit("Revert", |doc, active| { let (id, name) = (doc.id, doc.name.clone()); *doc = fresh; diff --git a/crates/engine/src/jobs.rs b/crates/engine/src/jobs.rs index 1fc75ead29..374300ad06 100644 --- a/crates/engine/src/jobs.rs +++ b/crates/engine/src/jobs.rs @@ -258,17 +258,6 @@ fn independent_of_document(id: &str) -> bool { PREFIXES.iter().any(|p| id.starts_with(p)) || IDS.contains(&id) } -#[cfg(not(target_arch = "wasm32"))] -fn read_path(path: &str) -> Result> { - // Bounded reads, and a clear error for a file larger than memory (#375). - photocraft_format::read_file(std::path::Path::new(path)).map_err(|e| EngineError::Other(format!("{path}: {e}"))) -} - -#[cfg(target_arch = "wasm32")] -fn read_path(path: &str) -> Result> { - Err(EngineError::Other(format!("cannot open paths on the web: {path}"))) -} - /// Run a job-capable command's heavy part. `work` runs on a worker thread when the command was /// started with [`Session::start`] (inline otherwise, and always on wasm); `apply` then runs on /// the calling thread with its output and returns the command's result. With `lock_document`, @@ -436,17 +425,19 @@ impl Session { false, move |ctx| { ctx.progress(0.0, "Reading"); - let bytes = match source { - OpenSource::Bytes(b) => b, - OpenSource::Path(p) => Arc::new(read_path(&p)?), - }; ctx.check()?; ctx.progress(0.02, "Decoding"); - ctx.stage(0.02, 1.0, "Decoding", |ctl| photocraft_io::import_with_svg_group_depth_and_interrupt(&name_w, &bytes, max_svg_group_depth, ctl)) - .map_err(|e| match e { - photocraft_io::IoError::Cancelled => EngineError::Cancelled, - e => EngineError::Other(e.to_string()), - }) + ctx.stage(0.02, 1.0, "Decoding", |ctl| match source { + OpenSource::Bytes(b) => photocraft_io::import_with_svg_group_depth_and_interrupt(&name_w, &b, max_svg_group_depth, ctl), + #[cfg(not(target_arch = "wasm32"))] + OpenSource::Path(p) => photocraft_io::import_path_with_svg_group_depth(&name_w, std::path::Path::new(&p), max_svg_group_depth, ctl), + #[cfg(target_arch = "wasm32")] + OpenSource::Path(_) => Err(photocraft_io::IoError::Unsupported("paths cannot be opened on the web".into())), + }) + .map_err(|e| match e { + photocraft_io::IoError::Cancelled => EngineError::Cancelled, + e => EngineError::Other(e.to_string()), + }) }, move |s, r: photocraft_io::ImportResult| { // Ask When Opening: held back until the prompt is answered (`take_held_open`). A diff --git a/crates/format/src/atomic.rs b/crates/format/src/atomic.rs index bf39fe2761..fb86d9eb38 100644 --- a/crates/format/src/atomic.rs +++ b/crates/format/src/atomic.rs @@ -155,6 +155,66 @@ pub fn atomic_write_progress(path: &Path, bytes: &[u8], progress: &dyn Fn(f32)) atomic_write_with_progress(&RealFs, RenameRetry::platform(), path, bytes, progress) } +/// Stream into a new seekable temporary file and publish it only after successful encoding +/// and fsync. A failed encoder, cancellation or full disk leaves the destination intact. +#[cfg(not(target_arch = "wasm32"))] +pub fn atomic_write_stream(path: &Path, encode: impl FnOnce(&mut std::fs::File) -> io::Result<()>) -> io::Result<()> { + atomic_write_stream_with_commit(path, encode, || Ok(())) +} + +/// Like [`atomic_write_stream`], with a final approval immediately before replacement. +/// The encoder and flush remain cancellable; after `commit` succeeds publication runs to +/// completion. A rejected commit or an unwinding callback removes the temporary file. +#[cfg(not(target_arch = "wasm32"))] +pub fn atomic_write_stream_with_commit( + path: &Path, + encode: impl FnOnce(&mut std::fs::File) -> io::Result<()>, + commit: impl FnOnce() -> io::Result<()>, +) -> io::Result<()> { + struct Cleanup(PathBuf); + impl Drop for Cleanup { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.0); + } + } + let target = resolve_symlink(path); + let name = target.file_name().ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "not a file path"))?; + let dir = target.parent().filter(|p| !p.as_os_str().is_empty()).unwrap_or(Path::new(".")); + std::fs::create_dir_all(dir).map_err(|e| write_error(e, path, dir))?; + let mut temp = None; + for _ in 0..8 { + let candidate = dir.join(temp_name(&name.to_string_lossy())); + match std::fs::OpenOptions::new().read(true).write(true).create_new(true).open(&candidate) { + Ok(file) => { + temp = Some((candidate, file)); + break; + } + Err(e) if e.kind() == io::ErrorKind::AlreadyExists => continue, + Err(e) => return Err(write_error(e, path, dir)), + } + } + let (tmp, file) = temp.ok_or_else(|| io::Error::new(io::ErrorKind::AlreadyExists, "cannot create a save temporary"))?; + let _cleanup = Cleanup(tmp.clone()); + // Close the handle before cleanup runs during unwinding, including on Windows. + let mut file = file; + let result = encode(&mut file).and_then(|()| sync_file(&file)); + drop(file); + if let Err(e) = result { + let _ = std::fs::remove_file(&tmp); + return Err(write_error(e, path, dir)); + } + if target.exists() { + let _ = RealFs.copy_permissions(&target, &tmp); + } + commit()?; + if let Err(e) = retry_rename(RenameRetry::platform(), || RealFs.rename(&tmp, &target)) { + let _ = RealFs.remove(&tmp); + return Err(rename_error(e, path)); + } + let _ = RealFs.sync_dir(dir); + Ok(()) +} + /// [`atomic_write`] through an explicit file system and retry policy (the failure-injection seam). pub fn atomic_write_with(fs: &dyn AtomicFs, retry: RenameRetry, path: &Path, bytes: &[u8]) -> io::Result<()> { atomic_write_with_progress(fs, retry, path, bytes, &|_| {}) diff --git a/crates/format/src/atomic/tests.rs b/crates/format/src/atomic/tests.rs index 8e71bc7ce3..eef149d476 100644 --- a/crates/format/src/atomic/tests.rs +++ b/crates/format/src/atomic/tests.rs @@ -31,6 +31,74 @@ fn assert_no_temp(dir: &Path) { const ORIGINAL: &[u8] = b"the user's precious original document"; +#[test] +fn streaming_commit_runs_after_encoding_and_before_replacement() { + let dir = TempDir::new("stream-commit"); + let path = dir.0.join("document.psb"); + std::fs::write(&path, ORIGINAL).unwrap(); + let encoded = Cell::new(false); + let result = atomic_write_stream_with_commit( + &path, + |file| { + file.write_all(b"complete")?; + encoded.set(true); + Ok(()) + }, + || { + assert!(encoded.get()); + assert_eq!(std::fs::read(&path).unwrap(), ORIGINAL); + Err(io::Error::new(io::ErrorKind::Interrupted, "cancelled before commit")) + }, + ); + assert_eq!(result.unwrap_err().kind(), io::ErrorKind::Interrupted); + assert_eq!(std::fs::read(&path).unwrap(), ORIGINAL); + assert_no_temp(&dir.0); + + atomic_write_stream_with_commit(&path, |file| file.write_all(b"complete"), || Ok(())).unwrap(); + assert_eq!(std::fs::read(&path).unwrap(), b"complete"); + assert_no_temp(&dir.0); +} + +#[test] +fn streaming_commit_unwind_removes_temporary_and_keeps_original() { + let dir = TempDir::new("stream-commit-unwind"); + let path = dir.0.join("document.psb"); + std::fs::write(&path, ORIGINAL).unwrap(); + let result = std::panic::catch_unwind(|| atomic_write_stream_with_commit(&path, |file| file.write_all(b"complete"), || panic!("injected commit panic"))); + assert!(result.is_err()); + assert_eq!(std::fs::read(&path).unwrap(), ORIGINAL); + assert_no_temp(&dir.0); +} + +#[test] +fn seekable_encoder_failure_keeps_original_and_cleans_temp() { + use std::io::{Seek, SeekFrom}; + let dir = TempDir::new("stream"); + let path = dir.0.join("document.psb"); + std::fs::write(&path, ORIGINAL).unwrap(); + for kind in [io::ErrorKind::StorageFull, io::ErrorKind::Interrupted] { + assert!( + atomic_write_stream(&path, |file| { + file.write_all(b"partial")?; + file.seek(SeekFrom::Start(0))?; + file.write_all(b"PATCH")?; + Err(io::Error::new(kind, "injected encoder failure")) + }) + .is_err() + ); + assert_eq!(std::fs::read(&path).unwrap(), ORIGINAL); + assert_no_temp(&dir.0); + } + atomic_write_stream(&path, |file| { + file.write_all(b"0000payload")?; + file.seek(SeekFrom::Start(0))?; + file.write_all(b"0007") + }) + .unwrap(); + assert_eq!(std::fs::read(&path).unwrap(), b"0007payload"); + assert_no_temp(&dir.0); +} + /// Real file system with failures injected at chosen steps. #[derive(Default)] struct Faulty { @@ -265,3 +333,19 @@ fn sync_file_flushes_a_local_file() { let file = std::fs::OpenOptions::new().write(true).create_new(true).open(&path).unwrap(); sync_file(&file).unwrap(); } + +#[test] +fn streaming_encoder_unwind_closes_and_removes_temporary_file() { + let d = TempDir::new("stream-unwind"); + let p = d.0.join("doc.psb"); + std::fs::write(&p, ORIGINAL).unwrap(); + let result = std::panic::catch_unwind(|| { + let _ = atomic_write_stream(&p, |file| { + file.write_all(b"partial output")?; + panic!("injected encoder unwind"); + }); + }); + assert!(result.is_err()); + assert_eq!(std::fs::read(&p).unwrap(), ORIGINAL); + assert_no_temp(&d.0); +} diff --git a/crates/io/examples/scratch_psb_probe.rs b/crates/io/examples/scratch_psb_probe.rs new file mode 100644 index 0000000000..99c210aae0 --- /dev/null +++ b/crates/io/examples/scratch_psb_probe.rs @@ -0,0 +1,36 @@ +//! Read-only large-document import probe. Never exports or changes the input file. +use photocraft_raster::spill; +use std::time::Instant; + +fn main() -> Result<(), String> { + let mut args = std::env::args().skip(1); + let path = args.next().ok_or("usage: scratch_psb_probe ")?; + let dir = args.next().ok_or("missing scratch directory")?; + spill::configure(Some(std::path::Path::new(&dir)), 1 << 30)?; + let start = Instant::now(); + let result = photocraft_io::import_path_with(&path, std::path::Path::new(&path), &photocraft_raster::Interrupt::NONE).map_err(|e| e.to_string())?; + spill::settle(); + let d = &result.document; + println!( + "import_seconds={:.3} size={}x{} depth={:?} mode={:?} warnings={:?}", + start.elapsed().as_secs_f64(), + d.size.width, + d.size.height, + d.depth, + d.mode, + result.warnings + ); + for (_, _, layer) in d.walk() { + if let Some(surface) = layer.surface() { + // Access source pixels directly, without creating a GPU canvas or flattening. + let bounds = surface.tile_bounds(); + let x = bounds.x0.saturating_add(bounds.width() as i32 / 2); + let y = bounds.y0.saturating_add(bounds.height() as i32 / 2); + println!("layer={:?} tiles={} bounds={bounds:?} sample={:?}", layer.name, surface.tile_count(), surface.pixel(x, y)); + } + } + spill::check_integrity()?; + println!("scratch={}", serde_json::to_string(&spill::stats()).map_err(|e| e.to_string())?); + drop(result); + spill::configure(None, 0) +} diff --git a/crates/io/src/exr_native.rs b/crates/io/src/exr_native.rs new file mode 100644 index 0000000000..0d7555bd7b --- /dev/null +++ b/crates/io/src/exr_native.rs @@ -0,0 +1,76 @@ +//! Native OpenEXR blocks go straight into original-depth tiled storage. +use crate::{ImportResult, IoError}; +use photocraft_color::{ColorMode, PixelFormat, SampleType}; +use photocraft_doc::{Document, Layer, LayerContent}; +use photocraft_geom::{Rect, Size, TILE_SIZE}; +use photocraft_raster::{Interrupt, Surface}; +use std::io::{Read, Seek}; + +pub(crate) fn import(source: R, ctl: &Interrupt<'_>) -> Result, IoError> { + let _metadata = photocraft_raster::memory::try_reserve(32 << 20).map_err(IoError::Unsupported)?; + let mut decoder = match photocraft_codecs::exr_stream::Decoder::new(source, &|| ctl.cancelled()) { + Ok(decoder) => decoder, + Err(_) if ctl.cancelled() => return Err(IoError::Cancelled), + Err(photocraft_codecs::CodecError::Unsupported { .. }) => return Ok(None), + Err(e) => return Err(e.into()), + }; + let info = decoder.info(); + let mode = if info.gray { ColorMode::Grayscale } else { ColorMode::Rgb }; + let fmt = PixelFormat::new(mode, SampleType::F32, true); + let scratch_expected = photocraft_raster::spill::enabled(); + if !scratch_expected { + let budget = photocraft_raster::memory::refresh(); + let side = TILE_SIZE as u64; + let tiles = u64::from(info.width).div_ceil(side).saturating_mul(u64::from(info.height).div_ceil(side)); + crate::psd_stream::ram_tile_capacity( + false, + photocraft_raster::spill::stats().resident_bytes as u64, + budget.tile_bytes, + tiles.saturating_mul(side * side).saturating_mul(fmt.bytes_per_pixel() as u64), + )?; + } + let _working = photocraft_raster::memory::try_reserve(info.working_bytes).map_err(IoError::Unsupported)?; + let mut surface = Surface::new(fmt); + let mut done = 0; + loop { + ctl.check().map_err(|_| IoError::Cancelled)?; + crate::psd_stream::require_scratch(scratch_expected)?; + let band = decoder.next_band(&|| ctl.cancelled()).map_err(|e| if ctl.cancelled() { IoError::Cancelled } else { e.into() })?; + let Some(band) = band else { + break; + }; + let rect = Rect::from_xywh(band.x as i32, band.y as i32, band.width, band.height); + if !photocraft_raster::spill::enabled() { + let missing = rect.tiles().filter(|c| surface.tile(*c).is_none()).count() as u64; + crate::psd_stream::ram_tile_capacity( + false, + photocraft_raster::spill::stats().resident_bytes as u64, + photocraft_raster::memory::stats().1.tile_bytes, + missing.saturating_mul((TILE_SIZE as u64).pow(2)).saturating_mul(fmt.bytes_per_pixel() as u64), + )?; + } + surface.write_interleaved(rect, &band.data); + crate::psd_stream::require_scratch(scratch_expected)?; + done += 1; + ctl.progress(0.05 + 0.9 * done as f32 / info.blocks.max(1) as f32); + } + // Missing alpha is filled with one in every allocated tile. Pruning these opaque + // tiles cannot remove any and would unnecessarily reload the entire scratch store. + if info.has_alpha { + surface.prune(); + } + crate::psd_stream::require_scratch(scratch_expected)?; + ctl.check().map_err(|_| IoError::Cancelled)?; + let mut document = Document::new("", Size::new(info.width, info.height), mode, SampleType::F32); + let mut layer = Layer::new("Background", LayerContent::Raster(surface)); + if !info.has_alpha { + layer.locks.transparency = true; + layer.locks.position = true; + } + document.layers.push(layer); + document.icc_profile = + Some(if mode == ColorMode::Rgb { photocraft_cms::Builtin::LinearSrgb.profile().to_bytes() } else { photocraft_cms::builtin::linear_gray().to_bytes() }); + let warnings = if info.all_half { vec!["16-bit float samples are stored as 32-bit float".into()] } else { Vec::new() }; + ctl.progress(1.0); + Ok(Some(ImportResult { document, warnings, source_read_only: false, preview_only: false })) +} diff --git a/crates/io/src/lib.rs b/crates/io/src/lib.rs index 936bab32b9..0cb2b481f9 100644 --- a/crates/io/src/lib.rs +++ b/crates/io/src/lib.rs @@ -41,6 +41,8 @@ pub mod camera_raw_xmp; mod channel_map; pub mod comps_map; pub mod effects_map; +#[cfg(not(target_arch = "wasm32"))] +mod exr_native; pub mod exr_parts; mod flat; mod gradient_bake; @@ -55,6 +57,8 @@ mod pdn; pub mod pixels; mod psd_export; mod psd_import; +#[cfg(not(target_arch = "wasm32"))] +mod psd_stream; pub mod psd_thumbnail; pub mod raw; pub mod slices_map; @@ -241,6 +245,120 @@ pub fn import(name: &str, bytes: &[u8]) -> Result { import_with_svg_group_depth(name, bytes, photocraft_doc::MAX_GROUP_DEPTH) } +/// Native seekable import. Layered PSD/PSB and flat scanline EXR decode bounded bands directly into tiles; +/// the bytes API remains available for web and for formats without a seekable decoder. +#[cfg(not(target_arch = "wasm32"))] +pub fn import_path_with(name: &str, path: &std::path::Path, ctl: &photocraft_raster::Interrupt<'_>) -> Result { + import_path_with_svg_group_depth(name, path, photocraft_doc::MAX_GROUP_DEPTH, ctl) +} + +/// Seekable import preserving the caller's SVG rasterization preference on fallback. +#[cfg(not(target_arch = "wasm32"))] +pub fn import_path_with_svg_group_depth( + name: &str, + path: &std::path::Path, + max_svg_group_depth: usize, + ctl: &photocraft_raster::Interrupt<'_>, +) -> Result { + use std::io::Read; + ctl.check().map_err(|_| IoError::Cancelled)?; + photocraft_compose::discard_unreadable_pixel_caches(); + photocraft_raster::spill::check_integrity().map_err(IoError::Unsupported)?; + let before = photocraft_raster::spill::read_error_generation(); + let mut source = std::fs::File::open(path).map_err(|e| IoError::Unsupported(e.to_string()))?; + let original = source.metadata().map_err(|e| IoError::Unsupported(e.to_string()))?; + let size = original.len(); + let mut magic = [0; 4]; + let n = source.read(&mut magic).map_err(|e| IoError::Unsupported(e.to_string()))?; + if n == 4 && is_psd(&magic) { + let budget = (photocraft_raster::memory::stats().1.transient_bytes / 8).min(128 << 20); + let _metadata = photocraft_raster::memory::try_reserve(budget.saturating_mul(3)).map_err(IoError::Unsupported)?; + let index = photocraft_psd::stream::Index::read(&mut source, budget, &|| ctl.cancelled()) + .map_err(|e| if ctl.cancelled() { IoError::Cancelled } else { e.into() })?; + if psd_stream::eligible(&index) { + if psd_import::group_depth(&index.metadata) > photocraft_doc::MAX_GROUP_DEPTH { + return Err(IoError::Unsupported("PSD layer nesting exceeds the document limit".into())); + } + let mut result = psd_stream::import(&|| std::fs::File::open(path), &index, *ctl)?; + let current = std::fs::metadata(path).map_err(|e| IoError::Unsupported(e.to_string()))?; + if current.len() != original.len() || current.modified().ok() != original.modified().ok() { + return Err(IoError::Unsupported("the source file changed while it was being opened; open it again".into())); + } + result.document.name = name.into(); + photocraft_raster::spill::check_since(before).map_err(IoError::Unsupported)?; + ctl.check().map_err(|_| IoError::Cancelled)?; + return Ok(result); + } + } + if n == 4 + && photocraft_codecs::detect(&magic) == Some(photocraft_codecs::Format::OpenExr) + && let Some(mut result) = exr_native::import(&mut source, ctl)? + { + let current = std::fs::metadata(path).map_err(|e| IoError::Unsupported(e.to_string()))?; + if current.len() != original.len() || current.modified().ok() != original.modified().ok() { + return Err(IoError::Unsupported("the source file changed while it was being opened; open it again".into())); + } + result.document.name = name.into(); + photocraft_raster::spill::check_since(before).map_err(IoError::Unsupported)?; + ctl.check().map_err(|_| IoError::Cancelled)?; + return Ok(result); + } + drop(source); + let _bytes = photocraft_raster::memory::try_reserve(size) + .map_err(|_| IoError::Unsupported("this format requires a whole-file decoder and exceeds the working-memory budget".into()))?; + let bytes = photocraft_format::read_file(path).map_err(|e| IoError::Unsupported(e.to_string()))?; + import_with_svg_group_depth_and_interrupt(name, &bytes, max_svg_group_depth, ctl) +} + +/// Read from handles already authorized by the caller. Independent opens prevent channel +/// readers sharing a seek cursor, and no ambient filesystem access occurs in this function. +#[cfg(not(target_arch = "wasm32"))] +pub fn import_seekable_with( + name: &str, + open: &dyn Fn() -> std::io::Result, + ctl: &photocraft_raster::Interrupt<'_>, +) -> Result { + use std::io::SeekFrom; + ctl.check().map_err(|_| IoError::Cancelled)?; + photocraft_compose::discard_unreadable_pixel_caches(); + photocraft_raster::spill::check_integrity().map_err(IoError::Unsupported)?; + let before = photocraft_raster::spill::read_error_generation(); + let mut source = open().map_err(|e| IoError::Unsupported(e.to_string()))?; + let size = source.seek(SeekFrom::End(0)).map_err(|e| IoError::Unsupported(e.to_string()))?; + source.seek(SeekFrom::Start(0)).map_err(|e| IoError::Unsupported(e.to_string()))?; + let mut magic = [0; 4]; + let n = source.read(&mut magic).map_err(|e| IoError::Unsupported(e.to_string()))?; + if n == 4 && is_psd(&magic) { + let budget = (photocraft_raster::memory::stats().1.transient_bytes / 8).min(128 << 20); + let _metadata = photocraft_raster::memory::try_reserve(budget.saturating_mul(3)).map_err(IoError::Unsupported)?; + let index = photocraft_psd::stream::Index::read(&mut source, budget, &|| ctl.cancelled()) + .map_err(|e| if ctl.cancelled() { IoError::Cancelled } else { e.into() })?; + if psd_stream::eligible(&index) { + if psd_import::group_depth(&index.metadata) > photocraft_doc::MAX_GROUP_DEPTH { + return Err(IoError::Unsupported("PSD layer nesting exceeds the document limit".into())); + } + let mut result = psd_stream::import(open, &index, *ctl)?; + result.document.name = name.into(); + photocraft_raster::spill::check_since(before).map_err(IoError::Unsupported)?; + ctl.check().map_err(|_| IoError::Cancelled)?; + return Ok(result); + } + } + if n == 4 + && photocraft_codecs::detect(&magic) == Some(photocraft_codecs::Format::OpenExr) + && let Some(mut result) = exr_native::import(&mut source, ctl)? + { + result.document.name = name.into(); + photocraft_raster::spill::check_since(before).map_err(IoError::Unsupported)?; + ctl.check().map_err(|_| IoError::Cancelled)?; + return Ok(result); + } + source.seek(SeekFrom::Start(0)).map_err(|e| IoError::Unsupported(e.to_string()))?; + let _working = photocraft_raster::memory::try_reserve_operation(size).map_err(IoError::Unsupported)?; + let bytes = photocraft_format::read::read_all(&mut source, size).map_err(|e| IoError::Unsupported(e.to_string()))?; + import_with(name, &bytes, ctl) +} + /// [`import`] for a background open: checks `ctl` between stages (and per layer for PSD/PSB) and /// reports progress. A cancelled import fails with [`IoError::Cancelled`]. pub fn import_with(name: &str, bytes: &[u8], ctl: &photocraft_raster::Interrupt) -> Result { @@ -359,6 +477,17 @@ pub fn export(doc: &Document, name_or_ext: &str, opts: &ExportOptions) -> Result /// export fails with [`IoError::Cancelled`]; otherwise the bytes are exactly [`export`]'s. pub fn export_with(doc: &Document, name_or_ext: &str, opts: &ExportOptions, ctl: &photocraft_raster::Interrupt) -> Result { ctl.check().map_err(|_| IoError::Cancelled)?; + // The byte API has to retain its complete output. Native large saves use the seekable + // path below; reject an inadmissible legacy working set before building channel arrays. + let tiles: u64 = doc + .walk() + .iter() + .filter_map(|(_, _, l)| l.surface()) + .map(|s| (s.tile_count() as u64).saturating_mul(256 * 256).saturating_mul(s.format().bytes_per_pixel() as u64)) + .sum(); + let estimate = doc.size.area().saturating_mul(32).saturating_add(tiles.saturating_mul(3)); + let _working = photocraft_raster::memory::try_reserve_operation(estimate) + .map_err(|_| IoError::Unsupported("export exceeds the working-memory budget; use native PSB streaming or export a smaller region".into()))?; photocraft_compose::discard_unreadable_pixel_caches(); photocraft_raster::spill::check_integrity().map_err(IoError::Unsupported)?; let scratch_generation = photocraft_raster::spill::read_error_generation(); @@ -369,6 +498,42 @@ pub fn export_with(doc: &Document, name_or_ext: &str, opts: &ExportOptions, ctl: Ok(r) } +/// Native atomic save. PSB writes metadata and bounded original-pixel bands directly to disk; +/// formats with only a bytes encoder use the checked working-set admission in [`export`]. +#[cfg(not(target_arch = "wasm32"))] +pub fn export_path_with(doc: &Document, path: &std::path::Path, opts: &ExportOptions, ctl: &photocraft_raster::Interrupt<'_>) -> Result, IoError> { + ctl.check().map_err(|_| IoError::Cancelled)?; + let ext = path.extension().and_then(|s| s.to_str()).unwrap_or("").to_ascii_lowercase(); + if ext == "psb" || ext == "psd" && (opts.force_psb || doc.size.width > 30_000 || doc.size.height > 30_000) { + photocraft_raster::spill::check_integrity().map_err(IoError::Unsupported)?; + let mut warnings = Vec::new(); + let result = photocraft_format::atomic::atomic_write_stream_with_commit( + path, + |out| { + warnings = psd_stream::export_into(doc, out, *ctl).map_err(|e| std::io::Error::other(e.to_string()))?; + Ok(()) + }, + || ctl.check().map_err(|e| std::io::Error::new(std::io::ErrorKind::Interrupted, e)), + ); + if result.is_err() && ctl.cancelled() { + return Err(IoError::Cancelled); + } + result.map_err(|e| IoError::Unsupported(e.to_string()))?; + return Ok(warnings); + } + let result = export(doc, &path.to_string_lossy(), opts)?; + ctl.check().map_err(|_| IoError::Cancelled)?; + photocraft_format::atomic_write(path, &result.bytes).map_err(|e| IoError::Unsupported(e.to_string()))?; + Ok(result.warnings) +} + +/// Write a native PSB to an already authorized seekable output. The caller owns atomic +/// publication and cleanup; this never opens an ambient path. +#[cfg(not(target_arch = "wasm32"))] +pub fn export_psb_into(doc: &Document, out: &mut W, ctl: &photocraft_raster::Interrupt<'_>) -> Result, IoError> { + psd_stream::export_into(doc, out, *ctl) +} + /// Run `f` with an interrupt whose progress `0..=1` fills `lo..=hi` of `ctl`'s (same cancellation). pub(crate) fn with_span(ctl: &photocraft_raster::Interrupt, lo: f32, hi: f32, f: impl FnOnce(&photocraft_raster::Interrupt) -> R) -> R { let cancelled = || ctl.cancelled(); diff --git a/crates/io/src/psd_export.rs b/crates/io/src/psd_export.rs index 9db3d4799f..e2117b237e 100644 --- a/crates/io/src/psd_export.rs +++ b/crates/io/src/psd_export.rs @@ -81,6 +81,8 @@ struct Ex { patterns: Vec, /// The layer written as Photoshop's Background (see [`psd_background`]). background: Option, + deferred: bool, + sources: StreamSources, } /// The layer Photoshop should open as its Background: the bottom top-level layer when it is a @@ -103,10 +105,14 @@ fn psd_background(doc: &Document) -> Option { return None; } let n = s.channels(); - let opaque = s.read_region(canvas).chunks_exact(n).all(|p| p.last().is_some_and(|a| *a >= 1.0)); + let opaque = canvas + .tiles() + .all(|tc| s.try_read_region(tc.rect().intersect(&canvas)).is_ok_and(|pixels| pixels.chunks_exact(n).all(|p| p.last().is_some_and(|a| *a >= 1.0)))); opaque.then_some(l.id) } +pub(crate) type StreamSources = std::collections::HashMap, Option)>; + /// How deep embedded documents are exported inside each other before a smart object is written /// as pixels instead. const MAX_NESTING: u32 = 8; @@ -336,10 +342,13 @@ impl Ex { } fn empty_channels(&self) -> Vec { + if self.deferred { + return (-1..self.cc as i16).map(|id| ChannelData { id, compression: Some(Compression::Raw), data: Vec::new() }).collect(); + } (-1..self.cc as i16).map(|id| self.encode(id, &[], 0, 0)).collect() } - fn pixel_channels(&self, s: &Surface, name: &str) -> (PsdRect, Vec, Vec) { + fn pixel_channels(&mut self, s: &Surface, name: &str) -> (PsdRect, Vec, Vec, Option) { let mut warnings = Vec::new(); let s = if s.format() != self.fmt { warnings.push(format!("layer \"{name}\": pixels converted from {:?} to {:?}", s.format(), self.fmt)); @@ -349,7 +358,11 @@ impl Ex { }; let r = s.content_bounds(); if r.is_empty() { - return (PsdRect::default(), self.empty_channels(), warnings); + return (PsdRect::default(), self.empty_channels(), warnings, None); + } + if self.deferred { + let channels = (-1..self.cc as i16).map(|id| ChannelData { id, compression: Some(Compression::Raw), data: Vec::new() }).collect(); + return (to_psd_rect(r), channels, warnings, Some(s)); } // The interleaved gather is the sequential pass for full-canvas layers, so it runs // in row bands on all cores and the band results concatenate in order. @@ -373,15 +386,15 @@ impl Ex { let order: Vec<(i16, &Vec)> = std::iter::once((-1, &planes[self.cc])).chain(planes.iter().take(self.cc).enumerate().map(|(c, p)| (c as i16, p))).collect(); let ch = crate::pixels::par_map(order, |(id, plane)| self.encode(id, plane, w, h)); - (to_psd_rect(r), ch, warnings) + (to_psd_rect(r), ch, warnings, None) } - fn mask(&self, m: &LayerMask, vector: Option<(f32, f32)>) -> (MaskData, Option) { + fn mask(&self, m: &LayerMask, vector: Option<(f32, f32)>) -> (MaskData, Option, Option) { let s = if m.surface.format() != self.mask_fmt { m.surface.convert(self.mask_fmt) } else { m.surface.clone() }; let default = s.default_pixel().first().copied().unwrap_or(0.0); let r = s.content_bounds(); let (w, h) = (r.width() as usize, r.height() as usize); - let plane = if r.is_empty() { Vec::new() } else { deinterleave(&s.to_interleaved(r), 1, self.mask_fmt.sample, &[false]).remove(0) }; + let plane = if self.deferred || r.is_empty() { Vec::new() } else { deinterleave(&s.to_interleaved(r), 1, self.mask_fmt.sample, &[false]).remove(0) }; let mut flags = 0u8; if !m.linked { flags |= PsdMask::FLAG_RELATIVE; @@ -402,7 +415,11 @@ impl Ex { real: None, real_first: true, }; - (MaskData::Mask(pm), Some(self.encode(-2, &plane, w, h))) + if self.deferred { + (MaskData::Mask(pm), Some(ChannelData { id: -2, compression: Some(Compression::Raw), data: Vec::new() }), Some(s)) + } else { + (MaskData::Mask(pm), Some(self.encode(-2, &plane, w, h)), None) + } } /// Prefer numbers above imported layer IDs, wrapping safely at u32::MAX. @@ -424,16 +441,18 @@ impl Ex { fn record(&mut self, l: &Layer, extra: Vec, pixels: Option<&Surface>) -> LayerRecord { let id = self.layer_id(l); - let (rect, mut channels, warnings) = match pixels { + let (rect, mut channels, warnings, source) = match pixels { Some(s) => self.pixel_channels(s, &l.name), - None => (PsdRect::default(), self.empty_channels(), Vec::new()), + None => (PsdRect::default(), self.empty_channels(), Vec::new(), None), }; self.warnings.extend(warnings); let mut mask = MaskData::None; + let mut mask_source = None; // Vector mask density/feather live in the mask parameters, with or without a user mask. let vector = l.vector_mask.as_ref().map(|vm| (vm.density, vm.feather)); if let Some(m) = &l.mask { - let (md, ch) = self.mask(m, vector); + let (md, ch, s) = self.mask(m, vector); + mask_source = s; mask = md; channels.extend(ch); } else if let Some(parameters) = mask_parameters(None, vector) { @@ -448,6 +467,9 @@ impl Ex { real_first: true, }); } + if self.deferred { + self.sources.insert(id, (source, mask_source)); + } // A Background carries no transparency channel: that is how Photoshop tells it apart. if self.background == Some(l.id) { channels.retain(|c| c.id != -1); @@ -978,7 +1000,7 @@ impl Ex { } let doc = photocraft_format::load_from_bytes(bytes).map_err(|e| format!("its contents can't be read: {e}"))?; let (file, warnings) = - document_to_psd_nested(&doc, &PsdExportOptions { force_psb: true, ..Default::default() }, self.smart.depth + 1, &Interrupt::NONE); + document_to_psd_nested(&doc, &PsdExportOptions { force_psb: true, ..Default::default() }, self.smart.depth + 1, &Interrupt::NONE, None); let data = file.to_bytes().map_err(|e| format!("its contents can't be written: {e}"))?; let stem = file_name.rsplit_once('.').map_or(file_name, |(a, _)| a); self.warnings.extend(warnings.into_iter().map(|w| format!("smart object {stem}: {w}"))); @@ -1343,14 +1365,21 @@ pub fn document_to_psd(doc: &Document) -> PsdFile { /// anything that could not be represented. The merged composite is rendered /// with `photocraft_compose::flatten`. pub fn document_to_psd_with(doc: &Document, opts: &PsdExportOptions) -> (PsdFile, Vec) { - document_to_psd_nested(doc, opts, 0, &Interrupt::NONE) + document_to_psd_nested(doc, opts, 0, &Interrupt::NONE, None) +} + +#[cfg(not(target_arch = "wasm32"))] +pub(crate) fn stream_metadata(doc: &Document, opts: &PsdExportOptions) -> (PsdFile, Vec, StreamSources) { + let mut sources = StreamSources::new(); + let (file, warnings) = document_to_psd_nested(doc, opts, 0, &Interrupt::NONE, Some(&mut sources)); + (file, warnings, sources) } /// [`document_to_psd_with`] for a background save (#3389): reports progress through `ctl` after /// each layer record and each band of the merged composite. Once `ctl` is cancelled it stops /// early and returns an **incomplete** model: callers check `ctl` and discard it. pub(crate) fn document_to_psd_interruptible(doc: &Document, opts: &PsdExportOptions, ctl: &Interrupt) -> (PsdFile, Vec) { - document_to_psd_nested(doc, opts, 0, ctl) + document_to_psd_nested(doc, opts, 0, ctl, None) } /// How much of a PSD conversion the merged composite weighs, in layer records: rendering it @@ -1480,7 +1509,13 @@ pub(crate) fn psd_size_exceeds_limit(doc: &Document) -> bool { } /// [`document_to_psd_with`] for a document embedded `depth` smart objects deep. -fn document_to_psd_nested(doc: &Document, opts: &PsdExportOptions, depth: u32, ctl: &Interrupt) -> (PsdFile, Vec) { +fn document_to_psd_nested( + doc: &Document, + opts: &PsdExportOptions, + depth: u32, + ctl: &Interrupt, + deferred: Option<&mut StreamSources>, +) -> (PsdFile, Vec) { if doc.mode == ColorMode::Multichannel { return crate::multichannel_map::document_to_psd(doc, opts.force_psb); } @@ -1510,6 +1545,8 @@ fn document_to_psd_nested(doc: &Document, opts: &PsdExportOptions, depth: u32, c smart: SmartOut::new(doc, depth), patterns: doc.patterns.clone(), background: psd_background(doc), + deferred: deferred.is_some(), + sources: Default::default(), }; if big && !opts.force_psb { ex.warnings.push("document exceeds 30000 px; written as PSB".into()); @@ -1584,7 +1621,9 @@ fn document_to_psd_nested(doc: &Document, opts: &PsdExportOptions, depth: u32, c // isn't rendered at all. A document without layer records keeps the real image: its pixels // live nowhere else. let real_merged = opts.maximize_compatibility || ex.records.is_empty(); - let (mut planes, has_alpha) = if real_merged { + let (mut planes, has_alpha) = if ex.deferred { + (Vec::new(), true) + } else if real_merged { // Merged composite, rendered and encoded in bands (no full-size float composite). Matting // against white only changes pixels with alpha < 1; if some are slightly translucent but // all round to opaque (so no alpha channel is written), encode once more without the matte. @@ -1622,6 +1661,9 @@ fn document_to_psd_nested(doc: &Document, opts: &PsdExportOptions, depth: u32, c None => None, }; for a in &extra { + if ex.deferred { + continue; + } let s = if a.surface.format() != ex.mask_fmt { a.surface.convert(ex.mask_fmt) } else { a.surface.clone() }; let bytes = s.to_interleaved(canvas); planes.extend(deinterleave(&bytes, 1, sample, &[false]).remove(0)); @@ -1629,9 +1671,13 @@ fn document_to_psd_nested(doc: &Document, opts: &PsdExportOptions, depth: u32, c let channels = (cc + usize::from(has_alpha) + extra.len()) as u16; let header = Header::new(version, doc.size.width, doc.size.height, channels, psd_depth(sample), psd_mode(fmt.mode)); let mcomp = if sample == SampleType::F32 { Compression::Raw } else { opts.merged_compression }; - let image_data = ImageData::encode(mcomp, &planes, &header) - .or_else(|_| ImageData::encode(Compression::Raw, &planes, &header)) - .unwrap_or(ImageData { compression: Compression::Raw, data: planes }); + let image_data = if ex.deferred { + ImageData { compression: Compression::Raw, data: Vec::new() } + } else { + ImageData::encode(mcomp, &planes, &header) + .or_else(|_| ImageData::encode(Compression::Raw, &planes, &header)) + .unwrap_or(ImageData { compression: Compression::Raw, data: planes }) + }; ctl.progress(1.0); // Resources. @@ -1777,6 +1823,9 @@ fn document_to_psd_nested(doc: &Document, opts: &PsdExportOptions, depth: u32, c layer_mask_trailing: Vec::new(), image_data, }; + if let Some(out) = deferred { + *out = ex.sources; + } (file, ex.warnings) } diff --git a/crates/io/src/psd_import.rs b/crates/io/src/psd_import.rs index 444c2d6e5b..82814f3762 100644 --- a/crates/io/src/psd_import.rs +++ b/crates/io/src/psd_import.rs @@ -66,6 +66,7 @@ pub(crate) struct Ctx<'a> { /// Layer records decoded so far, out of `total` (progress). pub done: usize, pub total: usize, + pub streamed: Option<&'a std::collections::HashMap<(usize, i16), Surface>>, } fn doc_mode(m: PsdMode) -> Option { @@ -171,6 +172,9 @@ impl Ctx<'_> { } fn record_surface(&mut self, rec: &LayerRecord, name: &str) -> Surface { + if let Some(streamed) = self.streamed { + return streamed.get(&(rec as *const LayerRecord as usize, i16::MIN)).cloned().unwrap_or_else(|| Surface::new(self.fmt)); + } let r = rec.rect; if r.is_empty() || r.size().is_err() { return Surface::new(self.fmt); @@ -231,7 +235,11 @@ impl Ctx<'_> { } }; let mut surface = Surface::with_default(self.mask_fmt, &[f32::from(default) / 255.0]); - if !rect.is_empty() + if let Some(streamed) = self.streamed { + if let Some(decoded) = streamed.get(&(rec as *const LayerRecord as usize, id)) { + surface = decoded.clone(); + } + } else if !rect.is_empty() && let Ok((w, h)) = rect.size() && let Some(plane) = self.channel_plane(rec, id, name) { @@ -657,6 +665,14 @@ pub fn psd_to_document(file: &PsdFile) -> (Document, Vec) { /// [`psd_to_document`] for a background open: checks `ctl` per layer record and reports /// progress. `None` when cancelled. pub fn psd_to_document_with(file: &PsdFile, ctl: &photocraft_raster::Interrupt) -> Option<(Document, Vec)> { + psd_to_document_decoded(file, ctl, None) +} + +pub(crate) fn psd_to_document_decoded<'a>( + file: &'a PsdFile, + ctl: &photocraft_raster::Interrupt<'a>, + streamed: Option<&'a std::collections::HashMap<(usize, i16), Surface>>, +) -> Option<(Document, Vec)> { let h = &file.header; let mut warnings = Vec::new(); let mode = doc_mode(h.color_mode).unwrap_or_else(|| { @@ -754,6 +770,7 @@ pub fn psd_to_document_with(file: &PsdFile, ctl: &photocraft_raster::Interrupt) ctl: *ctl, done: 0, total: file.layers().len(), + streamed, }; for b in file.global_blocks.iter().filter(|b| matches!(&b.key, b"FEid" | b"FXid")) { // Smart-filter caches can be large: a cancelled open stops between blocks. diff --git a/crates/io/src/psd_stream.rs b/crates/io/src/psd_stream.rs new file mode 100644 index 0000000000..c11a9e593b --- /dev/null +++ b/crates/io/src/psd_stream.rs @@ -0,0 +1,395 @@ +//! Native import without a whole PSB byte buffer or whole decoded channel buffers. +use crate::{ + ImportResult, IoError, + pixels::{interleave, max_sample, zero_sample}, +}; +use photocraft_color::{ColorMode, PixelFormat, SampleType}; +use photocraft_geom::Rect; +use photocraft_psd::{ + LayerRecord, + stream::{Index, Rows}, +}; +use photocraft_raster::{Interrupt, Surface}; +use std::{ + collections::HashMap, + io::{BufReader, Read, Seek, SeekFrom, Write}, +}; + +pub(crate) fn eligible(index: &Index) -> bool { + let file = &index.metadata; + matches!( + file.header.color_mode, + photocraft_psd::ColorMode::Rgb | photocraft_psd::ColorMode::Grayscale | photocraft_psd::ColorMode::Cmyk | photocraft_psd::ColorMode::Lab + ) && matches!(file.header.depth, 8 | 16 | 32) + && !file.layers().is_empty() + && usize::from(file.header.channels) <= usize::from(file.header.color_mode.color_channels().unwrap_or(0)) + usize::from(file.merged_has_alpha()) +} + +struct Decode<'a, R> { + open: &'a dyn Fn() -> std::io::Result, + index: &'a Index, + ctl: Interrupt<'a>, + done: u64, + total: u64, + scratch_expected: bool, +} + +fn tile_footprint(r: photocraft_psd::Rect, bytes_per_pixel: usize) -> u64 { + if r.is_empty() { + return 0; + } + let side = i64::from(photocraft_geom::TILE_SIZE); + let columns = ((i64::from(r.right) - 1).div_euclid(side) - i64::from(r.left).div_euclid(side) + 1) as u64; + let rows = ((i64::from(r.bottom) - 1).div_euclid(side) - i64::from(r.top).div_euclid(side) + 1) as u64; + columns.saturating_mul(rows).saturating_mul(side as u64 * side as u64).saturating_mul(bytes_per_pixel as u64) +} + +pub(super) fn ram_tile_capacity(scratch: bool, resident: u64, budget: u64, growth: u64) -> Result<(), IoError> { + if !scratch && resident.checked_add(growth).is_none_or(|required| required > budget) { + return Err(IoError::Unsupported("decoded image tiles exceed the available RAM budget; enable a scratch disk or close other documents".into())); + } + Ok(()) +} + +pub(super) fn require_scratch(expected: bool) -> Result<(), IoError> { + if expected && !photocraft_raster::spill::enabled() { + return Err(IoError::Unsupported( + photocraft_raster::spill::last_error().unwrap_or_else(|| "scratch storage was disabled while loading; retry the import".into()), + )); + } + Ok(()) +} + +impl Decode<'_, R> { + fn surface(&mut self, layer: usize, rec: &LayerRecord, ids: &[i16], fmt: PixelFormat, default: &[f32]) -> Result { + let r = if ids.len() == 1 { rec.channel_rect(*ids.first().ok_or_else(|| IoError::Unsupported("empty channel set".into()))?) } else { rec.rect }; + if r.is_empty() { + return Ok(Surface::with_default(fmt, default)); + } + let (width, height) = r.size()?; + let working = + (width as u64).saturating_mul(64).saturating_mul(fmt.bytes_per_pixel() as u64).saturating_mul(3).saturating_add(ids.len() as u64 * (3 << 20)); + let _working = photocraft_raster::memory::try_reserve(working).map_err(IoError::Unsupported)?; + let ranges = self.index.channels.get(layer).ok_or_else(|| IoError::Unsupported("missing indexed layer".into()))?; + let mut readers = Vec::new(); + for id in ids { + readers.push(if let Some(range) = ranges.iter().find(|c| c.id == *id) { + Some(Rows::new( + BufReader::with_capacity(1 << 20, (self.open)().map_err(|e| IoError::Unsupported(e.to_string()))?), + *range, + width, + height, + self.index.metadata.header.depth, + self.index.metadata.header.version, + )?) + } else { + None + }); + } + let mut out = Surface::with_default(fmt, default); + let fill: Vec> = ids.iter().map(|id| if *id == -1 { max_sample(fmt.sample) } else { zero_sample(fmt.sample) }).collect(); + let invert: Vec = ids.iter().map(|id| fmt.mode == ColorMode::Cmyk && *id >= 0).collect(); + let mut planes: Vec> = ids.iter().map(|_| Vec::new()).collect(); + let mut row = Vec::new(); + let mut y = 0usize; + while y < height { + self.ctl.check().map_err(|_| IoError::Cancelled)?; + let rows = (height - y).min(64); + for (reader, plane) in readers.iter_mut().zip(&mut planes) { + plane.clear(); + if let Some(reader) = reader { + for _ in 0..rows { + self.ctl.check().map_err(|_| IoError::Cancelled)?; + reader.read_row(&mut row)?; + plane.extend_from_slice(&row); + } + } + } + let refs: Vec> = readers.iter().zip(&planes).map(|(r, p)| r.as_ref().map(|_| p.as_slice())).collect(); + let mut bytes = interleave(&refs, &fill, width * rows, fmt.sample, &invert); + if fmt.mode == ColorMode::Lab && fmt.sample == SampleType::U16 { + crate::pixels::lab16_chroma(&mut bytes, ids.len(), true); + } + let top = i32::try_from(i64::from(r.top) + y as i64).map_err(|_| IoError::Unsupported("layer coordinates overflow".into()))?; + let bottom = top.checked_add(rows as i32).ok_or_else(|| IoError::Unsupported("layer coordinates overflow".into()))?; + let band = Rect::new(r.left, top, r.right, bottom); + let scratch = photocraft_raster::spill::enabled(); + require_scratch(self.scratch_expected)?; + if !scratch { + let missing = band.tiles().filter(|c| out.tile(*c).is_none()).count() as u64; + let growth = missing.saturating_mul((photocraft_geom::TILE_SIZE as u64).pow(2)).saturating_mul(fmt.bytes_per_pixel() as u64); + ram_tile_capacity(false, photocraft_raster::spill::stats().resident_bytes as u64, photocraft_raster::memory::stats().1.tile_bytes, growth)?; + } + out.write_interleaved(band, &bytes); + // A final band can trigger a scratch write failure too. + require_scratch(self.scratch_expected)?; + y += rows; + self.done = self.done.saturating_add(rows as u64 * ids.len() as u64); + self.ctl.progress(0.05 + 0.85 * self.done as f32 / self.total.max(1) as f32); + } + for reader in readers.into_iter().flatten() { + reader.finish()?; + } + out.prune(); + require_scratch(self.scratch_expected)?; + Ok(out) + } +} + +pub(crate) fn import(open: &dyn Fn() -> std::io::Result, index: &Index, ctl: Interrupt<'_>) -> Result { + let file = &index.metadata; + let mode = match file.header.color_mode { + photocraft_psd::ColorMode::Rgb => ColorMode::Rgb, + photocraft_psd::ColorMode::Grayscale => ColorMode::Grayscale, + photocraft_psd::ColorMode::Cmyk => ColorMode::Cmyk, + photocraft_psd::ColorMode::Lab => ColorMode::Lab, + _ => return Err(IoError::Unsupported("stream color mode".into())), + }; + let sample = crate::pixels::sample_for_depth(file.header.depth); + let fmt = PixelFormat::new(mode, sample, true); + let scratch_expected = photocraft_raster::spill::enabled(); + if !scratch_expected { + let budget = photocraft_raster::memory::refresh(); + let required = file.layers().iter().fold(0_u64, |sum, rec| { + let mut bytes = tile_footprint(rec.rect, fmt.bytes_per_pixel()); + for id in [-2, -3] { + if rec.channel(id).is_some() { + bytes = bytes.saturating_add(tile_footprint(rec.channel_rect(id), sample.bytes())); + } + } + sum.saturating_add(bytes) + }); + ram_tile_capacity(false, photocraft_raster::spill::stats().resident_bytes as u64, budget.tile_bytes, required)?; + } + let total = file.layers().iter().map(|r| r.channel_rect(0).size().map_or(0, |(_, h)| h as u64 * (mode.color_channels() as u64 + 1))).sum(); + let mut decoder = Decode { open, index, ctl, done: 0, total, scratch_expected }; + let mut surfaces = HashMap::new(); + for (i, rec) in file.layers().iter().enumerate() { + ctl.check().map_err(|_| IoError::Cancelled)?; + let key = rec as *const LayerRecord as usize; + let ids: Vec = (0..mode.color_channels() as i16).chain([-1]).collect(); + surfaces.insert((key, i16::MIN), decoder.surface(i, rec, &ids, fmt, &vec![0.0; fmt.channels()])?); + for id in [-2, -3] { + if rec.channel(id).is_some() { + let default = rec.layer_mask().map_or(0, |m| if id == -3 { m.real.map_or(m.default_color, |m| m.background) } else { m.default_color }); + let fmt = PixelFormat::new(ColorMode::Grayscale, sample, false); + surfaces.insert((key, id), decoder.surface(i, rec, &[id], fmt, &[f32::from(default) / 255.0])?); + } + } + } + let (document, warnings) = crate::psd_import::psd_to_document_decoded(file, &ctl, Some(&surfaces)).ok_or(IoError::Cancelled)?; + Ok(ImportResult { document, warnings, source_read_only: false, preview_only: false }) +} + +fn sink_error(e: std::io::Error) -> photocraft_psd::PsdError { + photocraft_psd::PsdError::Invalid(format!("stream output: {e}")) +} + +pub(crate) fn export_into(doc: &photocraft_doc::Document, out: &mut W, ctl: Interrupt<'_>) -> Result, IoError> { + if !matches!(doc.mode, ColorMode::Rgb | ColorMode::Grayscale | ColorMode::Cmyk | ColorMode::Lab) { + return Err(IoError::Unsupported("seekable PSB export requires RGB, Gray, CMYK or Lab".into())); + } + ctl.check().map_err(|_| IoError::Cancelled)?; + let before = photocraft_raster::spill::read_error_generation(); + let fmt = doc.pixel_format(); + let metadata: u64 = doc + .metadata + .psd_resources + .iter() + .map(|(_, _, bytes)| bytes.len() as u64) + .chain(doc.metadata.psd_global_blocks.iter().map(|(_, _, bytes)| bytes.len() as u64)) + .chain(doc.walk().iter().flat_map(|(_, _, l)| l.psd_blocks.iter().map(|(_, b)| b.len() as u64))) + .fold(0_u64, u64::saturating_add); + let embedded = doc + .walk() + .iter() + .filter_map(|(_, _, l)| match &l.content { + photocraft_doc::LayerContent::Smart(sm) => match &sm.source { + photocraft_doc::SmartSource::Embedded { bytes, .. } => Some(bytes.len() as u64), + _ => None, + }, + _ => None, + }) + .fold(0_u64, u64::saturating_add); + let working = u64::from(doc.size.width) + .saturating_mul(64) + .saturating_mul(128) + .saturating_add(64 << 20) + .saturating_add(metadata.saturating_mul(4)) + .saturating_add(embedded.saturating_mul(4)); + let _working = photocraft_raster::memory::try_reserve(working).map_err(IoError::Unsupported)?; + let opts = crate::psd_export::PsdExportOptions { force_psb: true, ..Default::default() }; + let (file, warnings, sources) = crate::psd_export::stream_metadata(doc, &opts); + let check = || -> photocraft_psd::Result<()> { + if ctl.cancelled() { + return Err(photocraft_psd::PsdError::Invalid("stream export cancelled".into())); + } + photocraft_raster::spill::check_since(before).map_err(photocraft_psd::PsdError::Invalid) + }; + let result = photocraft_psd::stream::write_layered( + &file, + out, + |i, id, out| { + check()?; + let rec = file.layers().get(i).ok_or_else(|| photocraft_psd::PsdError::Invalid("missing output record".into()))?; + let source = rec.layer_id().and_then(|id| sources.get(&id)).and_then(|(pixels, mask)| if id == -2 { mask.as_ref() } else { pixels.as_ref() }); + let r = rec.channel_rect(id); + if r.is_empty() { + return Ok(()); + } + let s = source.ok_or_else(|| photocraft_psd::PsdError::Invalid("missing output pixel source".into()))?; + let r = Rect::new(r.left, r.top, r.right, r.bottom); + let channels = s.format().channels(); + let channel = if id == -1 { channels.saturating_sub(1) } else { id.max(0) as usize }; + let mut invert = vec![s.format().mode == ColorMode::Cmyk; channels]; + if (id < 0 || s.format().alpha) + && let Some(a) = invert.last_mut() + { + *a = false; + } + let mut y = r.y0; + while y < r.y1 { + check()?; + let band = Rect::new(r.x0, y, r.x1, y.saturating_add(64).min(r.y1)); + let mut bytes = s.to_interleaved(band); + if s.format().mode == ColorMode::Lab && s.format().sample == SampleType::U16 { + crate::pixels::lab16_chroma(&mut bytes, channels, false); + } + let planes = crate::pixels::deinterleave(&bytes, channels, s.format().sample, &invert); + out.write_all(planes.get(channel).ok_or_else(|| photocraft_psd::PsdError::Invalid("invalid output channel".into()))?).map_err(sink_error)?; + y = band.y1; + } + Ok(()) + }, + |out| { + check()?; + let start = out.stream_position().map_err(sink_error)?; + let row_bytes = u64::from(doc.size.width) * fmt.sample.bytes() as u64; + let plane_bytes = + row_bytes.checked_mul(u64::from(doc.size.height)).ok_or_else(|| photocraft_psd::PsdError::Invalid("output size overflow".into()))?; + let cc = fmt.mode.color_channels(); + let white = photocraft_raster::from_rgba(&fmt, [1.0; 4]); + let space = photocraft_compose::cmyk_space(doc); + photocraft_compose::render_bands(doc, doc.bounds(), 64, |band| -> photocraft_psd::Result<()> { + check()?; + let mut planes: Vec> = (0..=cc).map(|_| Vec::with_capacity(band.px.len() * fmt.sample.bytes())).collect(); + photocraft_color::convert::with_cmyk_space(space.as_ref(), || { + let mut v = [0.0; 5]; + for p in &band.px { + photocraft_raster::from_rgba_into(&fmt, *p, &mut v); + for (c, plane) in planes.iter_mut().enumerate() { + let value = v.get(c).copied().unwrap_or(0.0); + let value = if c < cc { + crate::pixels::matte(value, v.get(cc).copied().unwrap_or(0.0), white.get(c).copied().unwrap_or(1.0)) + } else { + value + }; + let value = if fmt.mode == ColorMode::Cmyk && c < cc { 1.0 - value } else { value }; + let value = if fmt.mode == ColorMode::Lab && fmt.sample == SampleType::U16 && matches!(c, 1 | 2) { + value * crate::pixels::LAB16_CHROMA_MAX / 65535.0 + } else { + value + }; + crate::pixels::encode_be(value, fmt.sample, plane); + } + } + }); + for (c, bytes) in planes.iter().enumerate() { + let offset = start + .checked_add(c as u64 * plane_bytes) + .and_then(|v| v.checked_add(band.rect.y0 as u64 * row_bytes)) + .ok_or_else(|| photocraft_psd::PsdError::Invalid("output offset overflow".into()))?; + out.seek(SeekFrom::Start(offset)).map_err(sink_error)?; + out.write_all(bytes).map_err(sink_error)?; + } + ctl.progress(0.5 + 0.5 * band.rect.y1 as f32 / doc.size.height.max(1) as f32); + Ok(()) + })?; + for (i, channel) in doc.channels.iter().chain(doc.quick_mask.iter()).take(usize::from(file.header.channels).saturating_sub(cc + 1)).enumerate() { + let index = cc + 1 + i; + let mask_fmt = photocraft_color::PixelFormat::new(ColorMode::Grayscale, fmt.sample, false); + let converted = (channel.surface.format() != mask_fmt).then(|| channel.surface.convert(mask_fmt)); + let surface = converted.as_ref().unwrap_or(&channel.surface); + let mut y = 0; + while y < doc.size.height { + check()?; + let end = y.saturating_add(64).min(doc.size.height); + let r = Rect::from_xywh(0, y as i32, doc.size.width, end - y); + let bytes = surface.to_interleaved(r); + let plane = crate::pixels::deinterleave(&bytes, 1, surface.format().sample, &[false]); + out.seek(SeekFrom::Start(start + index as u64 * plane_bytes + u64::from(y) * row_bytes)).map_err(sink_error)?; + out.write_all(plane.first().ok_or_else(|| photocraft_psd::PsdError::Invalid("missing extra output plane".into()))?).map_err(sink_error)?; + y = end; + } + } + let end = start + .checked_add(u64::from(file.header.channels).saturating_mul(plane_bytes)) + .ok_or_else(|| photocraft_psd::PsdError::Invalid("output size overflow".into()))?; + out.seek(SeekFrom::Start(end)).map_err(sink_error)?; + check() + }, + ); + if ctl.cancelled() { + return Err(IoError::Cancelled); + } + result?; + photocraft_raster::spill::check_since(before).map_err(IoError::Unsupported)?; + Ok(warnings) +} + +#[cfg(test)] +mod memory_tests { + use super::*; + use photocraft_psd::{Compression, Version}; + use std::io::Cursor; + + #[test] + fn tile_footprint_counts_partial_negative_tiles_at_each_depth() { + let side = photocraft_geom::TILE_SIZE as u64; + for bpp in [1, 2, 4, 8, 16, 20] { + let rect = photocraft_psd::Rect::from_xywh(-1, -1, 2, 2); + assert_eq!(tile_footprint(rect, bpp), 4 * side * side * bpp as u64); + let aligned = photocraft_psd::Rect::from_xywh(-(side as i32), 0, side as u32, side as u32); + assert_eq!(tile_footprint(aligned, bpp), side * side * bpp as u64); + } + assert_eq!(tile_footprint(photocraft_psd::Rect::default(), 16), 0); + let hostile = photocraft_psd::Rect { top: i32::MIN, left: i32::MIN, bottom: i32::MAX, right: i32::MAX }; + assert_eq!(tile_footprint(hostile, 16), u64::MAX); + } + + #[test] + fn ram_only_admission_accounts_for_live_tiles_and_overflow() { + assert!(ram_tile_capacity(false, 60, 100, 40).is_ok()); + assert!(ram_tile_capacity(false, 60, 100, 41).is_err()); + assert!(ram_tile_capacity(false, u64::MAX, u64::MAX, 1).is_err()); + assert!(ram_tile_capacity(true, 60, 100, 1000).is_ok()); + } + + #[test] + fn oversized_ram_only_import_is_rejected_before_opening_channels() { + assert!(!photocraft_raster::spill::enabled()); + let file = photocraft_psd::testgen::layered(Version::Psb, photocraft_psd::ColorMode::Rgb, 32, Compression::Raw); + let bytes = file.to_bytes().unwrap(); + let mut index = Index::read(&mut Cursor::new(&bytes), 128 << 20, &|| false).unwrap(); + // Valid dimensions would need over a terabyte; the source remains a tiny fixture. + index.metadata.layers_mut()[0].rect = photocraft_psd::Rect::from_xywh(0, 0, 300_000, 300_000); + let opened = std::cell::Cell::new(0); + let result = import( + &|| { + opened.set(opened.get() + 1); + Ok(Cursor::new(&bytes)) + }, + &index, + Interrupt::NONE, + ); + assert!(matches!(result, Err(IoError::Unsupported(ref message)) if message.contains("enable a scratch disk"))); + assert_eq!(opened.get(), 0); + } + + #[test] + fn missing_scratch_is_not_silently_treated_as_ram_only() { + assert!(!photocraft_raster::spill::enabled()); + assert!(require_scratch(false).is_ok()); + assert!(require_scratch(true).is_err()); + } +} diff --git a/crates/io/tests/streaming_exr.rs b/crates/io/tests/streaming_exr.rs new file mode 100644 index 0000000000..c9a6dae4fe --- /dev/null +++ b/crates/io/tests/streaming_exr.rs @@ -0,0 +1,94 @@ +#![cfg(not(target_arch = "wasm32"))] +use photocraft_codecs::{ChannelLayout, EncodeOptions, ExrCompression, Format, Image, SampleType}; +use photocraft_doc::LayerContent; +use photocraft_raster::Interrupt; +use std::{ + io::Cursor, + sync::atomic::{AtomicBool, Ordering}, +}; + +fn fixture(layout: ChannelLayout, sample: SampleType, compression: ExrCompression) -> Vec { + let values: Vec = (0..31 * 35 * layout.channels()).map(|i| [-0.25, 4.0, 0.0, 0.0001, 0.5][i % 5]).collect(); + let image = Image::from_f32(31, 35, layout, &values).unwrap().convert(layout, sample); + photocraft_codecs::encode(&image, Format::OpenExr, &EncodeOptions { exr_compression: compression, ..Default::default() }).unwrap() +} + +// One test keeps these imports from competing for the global working allowance. +#[test] +fn native_scanline_import_preserves_hdr_document_and_cancels_without_publication() { + let path = std::env::temp_dir().join(format!("photocraft-exr-stream-{}.exr", std::process::id())); + for layout in [ChannelLayout::Gray, ChannelLayout::GrayA, ChannelLayout::Rgb, ChannelLayout::Rgba] { + for sample in [SampleType::F16, SampleType::F32] { + for compression in [ExrCompression::None, ExrCompression::Rle, ExrCompression::Zip1, ExrCompression::Zip16, ExrCompression::Piz] { + let bytes = fixture(layout, sample, compression); + std::fs::write(&path, &bytes).unwrap(); + let reference = photocraft_io::import("HDR", &bytes).unwrap(); + let native = photocraft_io::import_path_with("HDR", &path, &Interrupt::NONE).unwrap(); + let authorized = photocraft_io::import_seekable_with("HDR", &|| Ok(Cursor::new(&bytes)), &Interrupt::NONE).unwrap(); + for actual in [native, authorized] { + let a = &actual.document; + let r = &reference.document; + assert_eq!(actual.warnings, reference.warnings); + assert_eq!(a.name, r.name); + assert_eq!(a.size, r.size); + assert_eq!(a.mode, r.mode); + assert_eq!(a.depth, photocraft_color::SampleType::F32); + assert_eq!(a.icc_profile, r.icc_profile); + assert_eq!(a.layers[0].locks, r.layers[0].locks); + let (LayerContent::Raster(a), LayerContent::Raster(r)) = (&a.layers[0].content, &r.layers[0].content) else { + panic!("raster background required") + }; + assert_eq!( + a.to_interleaved(reference.document.bounds()), + r.to_interleaved(reference.document.bounds()), + "{layout:?}/{sample:?}/{compression:?}" + ); + } + } + } + } + let bytes = fixture(ChannelLayout::Rgb, SampleType::F16, ExrCompression::Zip16); + std::fs::write(&path, &bytes).unwrap(); + let cancelled = AtomicBool::new(false); + let cancel = || cancelled.load(Ordering::Relaxed); + let progress = |p| { + if p > 0.2 { + cancelled.store(true, Ordering::Relaxed); + } + }; + assert!(matches!(photocraft_io::import_path_with("HDR", &path, &Interrupt::new(&cancel, &progress)), Err(photocraft_io::IoError::Cancelled))); + std::fs::write(&path, &bytes[..bytes.len() / 2]).unwrap(); + assert!(photocraft_io::import_path_with("HDR", &path, &Interrupt::NONE).is_err()); + // A syntactically complete huge header/table must be refused before reading + // its pixel blocks when scratch is unavailable. No giant fixture is needed. + assert!(!photocraft_raster::spill::enabled()); + let mut header_end = 8; + loop { + if bytes[header_end] == 0 { + header_end += 1; + break; + } + for _ in 0..2 { + while bytes[header_end] != 0 { + header_end += 1; + } + header_end += 1; + } + let length = i32::from_le_bytes(bytes[header_end..header_end + 4].try_into().unwrap()) as usize; + header_end += 4 + length; + } + let mut huge = bytes[..header_end].to_vec(); + let needle = b"dataWindow\0box2i\0\x10\0\0\0"; + let at = huge.windows(needle.len()).position(|v| v == needle).unwrap() + needle.len(); + huge[at + 8..at + 12].copy_from_slice(&65535_i32.to_le_bytes()); + huge[at + 12..at + 16].copy_from_slice(&32767_i32.to_le_bytes()); + let offset = (header_end + 2048 * 8) as u64; + for _ in 0..2048 { + huge.extend_from_slice(&offset.to_le_bytes()); + } + huge.extend_from_slice(&[0; 8]); + assert!( + matches!(photocraft_io::import_seekable_with("huge", &|| Ok(Cursor::new(&huge)), &Interrupt::NONE), Err(photocraft_io::IoError::Unsupported(ref message)) if message.contains("decoded image tiles")) + ); + std::fs::remove_file(path).unwrap(); +} diff --git a/crates/io/tests/streaming_psd.rs b/crates/io/tests/streaming_psd.rs new file mode 100644 index 0000000000..0a8038b3ac --- /dev/null +++ b/crates/io/tests/streaming_psd.rs @@ -0,0 +1,136 @@ +#![cfg(not(target_arch = "wasm32"))] +use photocraft_psd::{ColorMode, Compression, Version}; +use photocraft_raster::Interrupt; + +// Native imports and exports reserve the same process-wide working-memory allowance. +// Keep fidelity fixtures from rejecting each other solely because the test runner is parallel. +static WORKING_MEMORY_TESTS: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +#[test] +fn seekable_import_matches_byte_import_pixels_masks_and_metadata() { + let _test = WORKING_MEMORY_TESTS.lock().unwrap(); + let path = std::env::temp_dir().join(format!("photocraft-stream-parity-{}.psb", std::process::id())); + for version in [Version::Psd, Version::Psb] { + for mode in [ColorMode::Rgb, ColorMode::Grayscale, ColorMode::Cmyk, ColorMode::Lab] { + for depth in [8, 16, 32] { + for compression in Compression::ALL { + let mut file = photocraft_psd::testgen::layered(version, mode, depth, compression); + // The streaming layer path need not consume the stored merged image. + file.header.channels = mode.color_channels().unwrap() + u16::from(file.merged_has_alpha()); + let bytes = file.to_bytes().unwrap(); + std::fs::write(&path, &bytes).unwrap(); + let a = photocraft_io::import("d", &bytes).unwrap(); + let b = photocraft_io::import_path_with("d", &path, &Interrupt::NONE).unwrap(); + let authorized = photocraft_io::import_seekable_with("d", &|| std::fs::File::open(&path), &Interrupt::NONE).unwrap(); + assert_eq!( + photocraft_compose::render(&authorized.document, authorized.document.bounds()).px, + photocraft_compose::render(&b.document, b.document.bounds()).px + ); + assert_eq!(a.warnings, b.warnings, "{version:?}/{mode:?}/{depth}/{compression:?}"); + let aa = photocraft_compose::render(&a.document, a.document.bounds()); + let bb = photocraft_compose::render(&b.document, b.document.bounds()); + assert_eq!(aa.px, bb.px, "{version:?}/{mode:?}/{depth}/{compression:?}"); + let options = photocraft_io::PsdExportOptions::default(); + let (a, _) = photocraft_io::document_to_psd_with(&a.document, &options); + let (b, _) = photocraft_io::document_to_psd_with(&b.document, &options); + assert_eq!(a, b, "layer, mask and unknown-block round trip: {version:?}/{mode:?}/{depth}/{compression:?}"); + } + } + } + } + std::fs::remove_file(path).unwrap(); +} + +#[test] +fn cancelled_and_truncated_imports_publish_no_document() { + let _test = WORKING_MEMORY_TESTS.lock().unwrap(); + let path = std::env::temp_dir().join(format!("photocraft-stream-cancel-{}.psb", std::process::id())); + let mut file = photocraft_psd::testgen::layered(Version::Psb, ColorMode::Rgb, 8, Compression::Zip); + file.header.channels = 3 + u16::from(file.merged_has_alpha()); + let bytes = file.to_bytes().unwrap(); + std::fs::write(&path, &bytes).unwrap(); + assert!(matches!(photocraft_io::import_path_with("d", &path, &Interrupt::cancel_only(&|| true)), Err(photocraft_io::IoError::Cancelled))); + std::fs::write(&path, &bytes[..bytes.len() / 2]).unwrap(); + assert!(photocraft_io::import_path_with("d", &path, &Interrupt::NONE).is_err()); + std::fs::remove_file(path).unwrap(); +} + +#[test] +fn seekable_export_preserves_merged_layers_masks_and_unknown_metadata() { + let _test = WORKING_MEMORY_TESTS.lock().unwrap(); + let path = std::env::temp_dir().join(format!("photocraft-export-parity-{}.psb", std::process::id())); + for mode in [ColorMode::Rgb, ColorMode::Grayscale, ColorMode::Cmyk, ColorMode::Lab] { + for depth in [8, 16, 32] { + let input = photocraft_psd::testgen::layered(Version::Psb, mode, depth, Compression::ZipPrediction); + let doc = photocraft_io::import("input", &input.to_bytes().unwrap()).unwrap().document; + let (reference, _) = photocraft_io::document_to_psd_with(&doc, &photocraft_io::PsdExportOptions { force_psb: true, ..Default::default() }); + photocraft_io::export_path_with(&doc, &path, &Default::default(), &Interrupt::NONE).unwrap(); + let output = photocraft_psd::PsdFile::from_bytes(&std::fs::read(&path).unwrap()).unwrap(); + assert_eq!(output.header, reference.header, "{mode:?}/{depth}"); + assert_eq!(output.resources, reference.resources); + let normalized_reference = photocraft_psd::PsdFile::from_bytes(&reference.to_bytes().unwrap()).unwrap(); + assert_eq!(output.global_blocks, normalized_reference.global_blocks); + assert_eq!(output.image_data.decode(&output.header).unwrap(), reference.image_data.decode(&reference.header).unwrap(), "merged {mode:?}/{depth}"); + let loaded = photocraft_io::import_path_with("output", &path, &Interrupt::NONE).unwrap(); + let (loaded, _) = photocraft_io::document_to_psd_with(&loaded.document, &photocraft_io::PsdExportOptions { force_psb: true, ..Default::default() }); + assert_eq!(loaded.to_bytes().unwrap(), reference.to_bytes().unwrap(), "layer/mask/metadata {mode:?}/{depth}"); + } + } + std::fs::remove_file(path).unwrap(); +} + +#[test] +fn cancelled_stream_save_keeps_destination_and_removes_temporary_output() { + let _test = WORKING_MEMORY_TESTS.lock().unwrap(); + let dir = std::env::temp_dir().join(format!("photocraft-export-cancel-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("kept.psb"); + std::fs::write(&path, b"precious original").unwrap(); + let input = photocraft_psd::testgen::layered(Version::Psb, ColorMode::Rgb, 8, Compression::Raw); + let doc = photocraft_io::import("input", &input.to_bytes().unwrap()).unwrap().document; + let calls = std::sync::atomic::AtomicUsize::new(0); + let cancel = || calls.fetch_add(1, std::sync::atomic::Ordering::Relaxed) > 5; + assert!(matches!( + photocraft_io::export_path_with(&doc, &path, &Default::default(), &Interrupt::cancel_only(&cancel)), + Err(photocraft_io::IoError::Cancelled) + )); + assert_eq!(std::fs::read(&path).unwrap(), b"precious original"); + assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 1); + std::fs::remove_dir_all(dir).unwrap(); +} + +#[cfg(feature = "corpus")] +#[test] +fn native_path_import_matches_byte_import_across_real_psd_corpus() { + let _test = WORKING_MEMORY_TESTS.lock().unwrap(); + fn collect(p: &std::path::Path, out: &mut Vec) { + for e in std::fs::read_dir(p).unwrap().flatten() { + let p = e.path(); + if p.is_dir() { + collect(&p, out); + } else if p.extension().is_some_and(|x| x.eq_ignore_ascii_case("psd") || x.eq_ignore_ascii_case("psb")) { + out.push(p); + } + } + } + let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../corpus/psd"); + assert!(root.is_dir(), "run cargo xtask corpus --all"); + let mut files = Vec::new(); + collect(&root, &mut files); + let mut compared = 0; + for path in files { + let bytes = std::fs::read(&path).unwrap(); + let Ok(a) = photocraft_io::import("d", &bytes) else { continue }; + let b = photocraft_io::import_path_with("d", &path, &Interrupt::NONE).unwrap_or_else(|e| panic!("{}: {e}", path.display())); + assert_eq!(a.warnings, b.warnings, "{}", path.display()); + assert_eq!( + photocraft_compose::render(&a.document, a.document.bounds()).px, + photocraft_compose::render(&b.document, b.document.bounds()).px, + "{}", + path.display() + ); + compared += 1; + } + assert!(compared >= 20, "{compared} corpus files compared"); + eprintln!("seekable corpus parity: {compared} files"); +} diff --git a/crates/io/tests/svg.rs b/crates/io/tests/svg.rs index d1baf418fa..fd56d35b86 100644 --- a/crates/io/tests/svg.rs +++ b/crates/io/tests/svg.rs @@ -93,6 +93,23 @@ fn svg_group_rasterization_threshold_is_configurable_and_clamped() { let LayerContent::Group(outer) = &unlimited.document.layers[0].content else { panic!("outer group should remain editable") }; assert!(outer.children.iter().any(|child| matches!(child.content, LayerContent::Group(_))), "zero disables the preference threshold"); + #[cfg(not(target_arch = "wasm32"))] + { + let path = std::env::temp_dir().join(format!("photocraft-svg-depth-{}.svg", std::process::id())); + std::fs::write(&path, source.as_bytes()).unwrap(); + for (depth, reference) in [(1, &shallow), (0, &unlimited)] { + let seekable = photocraft_io::import_path_with_svg_group_depth("nested.svg", &path, depth, &photocraft_raster::Interrupt::NONE).unwrap(); + assert_eq!(seekable.warnings, reference.warnings); + let LayerContent::Group(outer) = &seekable.document.layers[0].content else { panic!("outer group should remain editable") }; + assert!(outer.children.iter().any(|child| if depth == 1 { + matches!(child.content, LayerContent::Raster(_)) + } else { + matches!(child.content, LayerContent::Group(_)) + })); + } + std::fs::remove_file(path).unwrap(); + } + let clamped = import_with_svg_group_depth("nested.svg", source.as_bytes(), usize::MAX).unwrap(); assert!(matches!(clamped.document.layers[0].content, LayerContent::Group(_)), "out-of-range settings clamp to the document's hard group limit"); } diff --git a/crates/psd/src/layer.rs b/crates/psd/src/layer.rs index 5450ed7e67..42036a994b 100644 --- a/crates/psd/src/layer.rs +++ b/crates/psd/src/layer.rs @@ -544,7 +544,7 @@ impl LayerRecord { ch.decode(w, h, depth, version) } - fn read(r: &mut Reader<'_>, version: Version) -> Result<(Self, Vec)> { + pub(crate) fn read(r: &mut Reader<'_>, version: Version) -> Result<(Self, Vec)> { let rect = Rect::read(r)?; let nch = r.u16()?; if nch > MAX_LAYER_CHANNELS { @@ -586,7 +586,7 @@ impl LayerRecord { Ok((LayerRecord { rect, channels, blend_mode, opacity, clipping, flags, filler, mask, blending_ranges, name, blocks, extra_trailing }, lens)) } - fn write(&self, out: &mut Vec, version: Version) -> Result<()> { + pub(crate) fn write(&self, out: &mut Vec, version: Version) -> Result<()> { self.rect.write(out); let nch = u16::try_from(self.channels.len()).map_err(|_| PsdError::LimitExceeded("too many channels"))?; out.put_u16(nch); diff --git a/crates/psd/src/lib.rs b/crates/psd/src/lib.rs index 00d73c4003..32efb0267c 100644 --- a/crates/psd/src/lib.rs +++ b/crates/psd/src/lib.rs @@ -64,6 +64,7 @@ pub mod pixels; pub mod resources; pub mod shc; pub mod slices; +pub mod stream; pub mod tagged; #[cfg(any(test, feature = "testgen"))] // Test-data generator (tests and fuzz seeds only): failing loudly is the point. diff --git a/crates/psd/src/stream.rs b/crates/psd/src/stream.rs new file mode 100644 index 0000000000..3023b5750a --- /dev/null +++ b/crates/psd/src/stream.rs @@ -0,0 +1,499 @@ +//! Seekable PSD/PSB metadata and row decoding. Pixel ranges stay in the source; no whole-file +//! or whole-channel buffer is created. `metadata` deliberately contains no encoded pixels. +use std::io::{Read, Seek, SeekFrom, Take, Write}; + +use crate::image_data::ImageData; +use crate::io::Reader; +use crate::{Compression, GlobalLayerMask, Header, LayerInfo, LayerInfoPlacement, LayerRecord, PlaneLayout, PsdError, PsdFile, Result, TaggedBlock, Version}; + +fn io(e: std::io::Error) -> PsdError { + PsdError::invalid(format!("stream I/O: {e}")) +} +fn invalid() -> PsdError { + PsdError::invalid("invalid stream range") +} +fn array(r: &mut impl Read) -> Result<[u8; N]> { + let mut b = [0; N]; + r.read_exact(&mut b).map_err(io)?; + Ok(b) +} +fn length(r: &mut impl Read, long: bool) -> Result { + Ok(if long { u64::from_be_bytes(array(r)?) } else { u64::from(u32::from_be_bytes(array(r)?)) }) +} + +#[derive(Clone, Copy, Debug)] +/// One checked encoded layer-channel range, excluding its compression marker. +pub struct ChannelRange { + /// Photoshop channel id (negative for transparency and masks). + pub id: i16, + /// Absolute byte offset in the source. + pub offset: u64, + /// Encoded byte count. + pub length: u64, + /// Channel codec. + pub compression: Compression, +} + +/// Metadata and independent source ranges; encoded pixels are never retained in this model. +pub struct Index { + /// Parsed metadata only. Do not serialize this model: pixel channels are in `channels`. + pub metadata: PsdFile, + /// Ranges per layer record, in stored order. + pub channels: Vec>, +} + +struct Scan<'a, R> { + reader: &'a mut R, + allowance: u64, + cancelled: &'a dyn Fn() -> bool, +} +impl Scan<'_, R> { + fn position(&mut self) -> Result { + self.reader.stream_position().map_err(io) + } + fn end(&mut self, length: u64, boundary: u64) -> Result { + self.position()?.checked_add(length).filter(|end| *end <= boundary).ok_or_else(invalid) + } + fn bytes(&mut self, n: u64, boundary: u64) -> Result> { + self.end(n, boundary)?; + self.allowance = self.allowance.checked_sub(n).ok_or(PsdError::LimitExceeded("stream metadata exceeds working-memory budget"))?; + let n = usize::try_from(n).map_err(|_| invalid())?; + let mut out = Vec::new(); + out.try_reserve_exact(n).map_err(|_| PsdError::LimitExceeded("stream metadata allocation"))?; + out.resize(n, 0); + for chunk in out.chunks_mut(64 << 10) { + if (self.cancelled)() { + return Err(PsdError::invalid("stream cancelled")); + } + self.reader.read_exact(chunk).map_err(io)?; + } + Ok(out) + } + fn layers(&mut self, end: u64, version: Version) -> Result<(LayerInfo, Vec>)> { + let start = self.position()?; + let count = i16::from_be_bytes(array(self.reader)?); + let n = usize::from(count.unsigned_abs()); + if (n as u64).saturating_mul(34) > end.saturating_sub(self.position()?) { + return Err(invalid()); + } + let mut layers = Vec::with_capacity(n); + let mut lengths = Vec::with_capacity(n); + for _ in 0..n { + let mut record = self.bytes(18, end)?; + let nch = u16::from_be_bytes(record.get(16..18).ok_or_else(invalid)?.try_into().map_err(|_| invalid())?); + if nch > crate::layer::MAX_LAYER_CHANNELS { + return Err(PsdError::LimitExceeded("too many layer channels")); + } + let fixed = u64::from(nch) * if version.is_psb() { 10 } else { 6 } + 16; + record.extend(self.bytes(fixed, end)?); + let extra = record.get(record.len().saturating_sub(4)..).ok_or_else(invalid)?; + let extra = u32::from_be_bytes(extra.try_into().map_err(|_| invalid())?); + record.extend(self.bytes(u64::from(extra), end)?); + let (rec, lens) = LayerRecord::read(&mut Reader::new(&record), version)?; + layers.push(rec); + lengths.push(lens); + } + let mut ranges = Vec::with_capacity(n); + for (rec, lens) in layers.iter_mut().zip(lengths) { + let mut channels = Vec::new(); + for (ch, len) in rec.channels.iter_mut().zip(lens) { + if len == 0 { + continue; + } + if len < 2 { + return Err(invalid()); + } + let channel_end = self.end(len, end)?; + let compression = Compression::from_u16(u16::from_be_bytes(array(self.reader)?)); + ch.compression = Some(compression); + channels.push(ChannelRange { id: ch.id, offset: self.position()?, length: len - 2, compression }); + self.reader.seek(SeekFrom::Start(channel_end)).map_err(io)?; + } + ranges.push(channels); + } + let body = self.position()?.saturating_sub(start); + let rest = end.saturating_sub(self.position()?); + let pad = self.bytes(rest, end)?; + let padding = if pad.len() as u64 == body % 2 && pad.iter().all(|b| *b == 0) { None } else { Some(pad) }; + Ok((LayerInfo { merged_alpha: count < 0, layers, padding }, ranges)) + } +} + +impl Index { + /// Index a seekable file with an aggregate metadata cap and cooperative cancellation. + pub fn read(reader: &mut R, metadata_budget: u64, cancelled: &dyn Fn() -> bool) -> Result { + let end = reader.seek(SeekFrom::End(0)).map_err(io)?; + reader.seek(SeekFrom::Start(0)).map_err(io)?; + let header = Header::read(&mut Reader::new(&array::<26>(reader)?))?; + let mut s = Scan { reader, allowance: metadata_budget, cancelled }; + let n = length(s.reader, false)?; + let color_mode_data = s.bytes(n, end)?; + let n = length(s.reader, false)?; + let resources = s.bytes(n, end)?; + let mut resource_bytes = (n as u32).to_be_bytes().to_vec(); + resource_bytes.extend(resources); + let resources = crate::resources::read_section(&mut Reader::new(&resource_bytes))?; + drop(resource_bytes); + let lm_len = length(s.reader, header.version.is_psb())?; + let lm_end = s.end(lm_len, end)?; + let mut layer_info = None; + let mut channels = Vec::new(); + let mut global_layer_mask = None; + let mut global_blocks = Vec::new(); + let mut layer_mask_trailing = Vec::new(); + let mut layer_info_placement = LayerInfoPlacement::Section; + if lm_len > 0 { + let n = length(s.reader, header.version.is_psb())?; + let li_end = s.end(n, lm_end)?; + if n > 0 { + let (info, c) = s.layers(li_end, header.version)?; + layer_info = Some(info); + channels = c; + } + if lm_end.saturating_sub(s.position()?) >= 4 { + let n = length(s.reader, false)?; + global_layer_mask = Some(GlobalLayerMask { data: s.bytes(n, lm_end)? }); + while lm_end.saturating_sub(s.position()?) >= 12 { + let start = s.position()?; + let signature = array::<4>(s.reader)?; + if signature != *b"8BIM" && signature != *b"8B64" { + s.reader.seek(SeekFrom::Start(start)).map_err(io)?; + break; + } + let key = array::<4>(s.reader)?; + let n = length(s.reader, crate::tagged::uses_long_length(header.version, &key))?; + let block_end = s.end(n, lm_end)?; + let is_layers = layer_info.is_none() && matches!(&key, b"Lr16" | b"Lr32" | b"Layr"); + let data = if is_layers { + let (info, c) = s.layers(block_end, header.version)?; + layer_info = Some(info); + channels = c; + Vec::new() + } else { + s.bytes(n, lm_end)? + }; + // Same padding detector as the byte parser, without reading the next block. + let at = s.position()?; + let rest = lm_end.saturating_sub(at); + let peek = s.bytes(rest.min(7), lm_end)?; + s.reader.seek(SeekFrom::Start(at)).map_err(io)?; + let pad = (0..=3usize) + .find(|k| *k as u64 == rest || peek.get(*k..).is_some_and(|b| b.starts_with(b"8BIM") || b.starts_with(b"8B64"))) + .unwrap_or(0); + let bytes = s.bytes(pad as u64, lm_end)?; + let padding = if bytes.len() as u64 == (4 - n % 4) % 4 && bytes.iter().all(|b| *b == 0) { None } else { Some(bytes) }; + if is_layers { + layer_info_placement = LayerInfoPlacement::GlobalBlock { index: global_blocks.len(), signature, key, padding }; + } else { + global_blocks.push(TaggedBlock { signature, key, data, padding }); + } + } + } + let left = lm_end.saturating_sub(s.position()?); + layer_mask_trailing = s.bytes(left, lm_end)?; + } + // The merged image is intentionally not read: layered imports below consume only layers. + let compression = Compression::from_u16(u16::from_be_bytes(array(s.reader)?)); + Ok(Self { + metadata: PsdFile { + header, + color_mode_data, + resources, + layer_info, + layer_info_placement, + global_layer_mask, + global_blocks, + layer_mask_trailing, + image_data: ImageData { compression, data: Vec::new() }, + }, + channels, + }) + } +} + +enum Input { + Raw(Take), + Rle(Take, Vec), + Zip(flate2::read::ZlibDecoder>), +} +/// A sequential channel decoder with bounded row buffers, including ZIP prediction. +pub struct Rows { + input: Input, + layout: PlaneLayout, + prediction: bool, + row: usize, + encoded: Vec, +} +impl Rows { + /// Open a checked range on an independent reader. Clone file handles with shared seek + /// positions must not be used for concurrent channels. + pub fn new(mut reader: R, range: ChannelRange, width: usize, height: usize, depth: u16, version: Version) -> Result { + if width > 300_000 || height > 300_000 || !matches!(depth, 1 | 8 | 16 | 32) { + return Err(PsdError::LimitExceeded("stream channel dimensions")); + } + reader.seek(SeekFrom::Start(range.offset)).map_err(io)?; + let mut source = reader.take(range.length); + let layout = PlaneLayout { planes: 1, width, height, depth, version }; + let input = match range.compression { + Compression::Raw => Input::Raw(source), + Compression::Rle => { + let mut counts = Vec::with_capacity(height); + let mut total = (height as u64).checked_mul(if version.is_psb() { 4 } else { 2 }).ok_or_else(invalid)?; + for _ in 0..height { + let n = + if version.is_psb() { u32::from_be_bytes(array(&mut source)?) as usize } else { usize::from(u16::from_be_bytes(array(&mut source)?)) }; + if n > layout.row_bytes().saturating_mul(2).saturating_add(1024) { + return Err(PsdError::LimitExceeded("encoded RLE row")); + } + total = total.checked_add(n as u64).ok_or_else(invalid)?; + counts.push(n); + } + if total > range.length { + return Err(invalid()); + } + Input::Rle(source, counts) + } + Compression::Zip | Compression::ZipPrediction => Input::Zip(flate2::read::ZlibDecoder::new(source)), + Compression::Unknown(n) => return Err(PsdError::Unsupported(format!("stream compression {n}"))), + }; + Ok(Self { input, layout, prediction: range.compression == Compression::ZipPrediction, row: 0, encoded: Vec::new() }) + } + /// Decode the next native big-endian sample row, reusing `out`. + pub fn read_row(&mut self, out: &mut Vec) -> Result<()> { + if self.row >= self.layout.height { + return Err(invalid()); + } + let bytes = self.layout.row_bytes(); + out.resize(bytes, 0); + match &mut self.input { + Input::Raw(r) => r.read_exact(out).map_err(io)?, + Input::Zip(r) => r.read_exact(out).map_err(io)?, + Input::Rle(r, counts) => { + out.clear(); + self.encoded.resize(*counts.get(self.row).ok_or_else(invalid)?, 0); + r.read_exact(&mut self.encoded).map_err(io)?; + crate::compression::packbits::decode_into(&self.encoded, bytes, out)?; + } + } + if self.prediction { + let row = PlaneLayout { height: 1, ..self.layout }; + crate::compression::unpredict(out, &row)?; + } + self.row += 1; + Ok(()) + } + /// Checks the zlib checksum and rejects extra decoded samples after the expected last row. + pub fn finish(mut self) -> Result<()> { + if self.row != self.layout.height { + return Err(invalid()); + } + let mut extra = [0; 1]; + let n = match &mut self.input { + Input::Raw(r) | Input::Rle(r, _) => r.read(&mut extra), + Input::Zip(r) => r.read(&mut extra), + } + .map_err(io)?; + if n != 0 { + return Err(PsdError::invalid("extra stream channel samples")); + } + Ok(()) + } +} + +fn patch(out: &mut W, at: u64, value: u64) -> Result<()> { + let end = out.stream_position().map_err(io)?; + out.seek(SeekFrom::Start(at)).map_err(io)?; + out.write_all(&value.to_be_bytes()).map_err(io)?; + out.seek(SeekFrom::Start(end)).map_err(io)?; + Ok(()) +} + +fn write_layers(out: &mut W, info: &LayerInfo, pixels: &mut impl FnMut(usize, i16, &mut W) -> Result<()>) -> Result<()> { + let start = out.stream_position().map_err(io)?; + let count = i16::try_from(info.layers.len()).map_err(|_| PsdError::LimitExceeded("too many streamed layers"))?; + out.write_all(&(if info.merged_alpha { -count } else { count }).to_be_bytes()).map_err(io)?; + let mut records = Vec::with_capacity(info.layers.len()); + for rec in &info.layers { + records.push(out.stream_position().map_err(io)?); + let mut encoded = Vec::new(); + rec.write(&mut encoded, Version::Psb)?; + out.write_all(&encoded).map_err(io)?; + } + for (i, rec) in info.layers.iter().enumerate() { + for (c, channel) in rec.channels.iter().enumerate() { + let begin = out.stream_position().map_err(io)?; + if let Some(compression) = channel.compression { + out.write_all(&compression.as_u16().to_be_bytes()).map_err(io)?; + pixels(i, channel.id, out)?; + } + let len = out.stream_position().map_err(io)?.saturating_sub(begin); + let at = records.get(i).copied().ok_or_else(invalid)?.checked_add(20 + c as u64 * 10).ok_or_else(invalid)?; + patch(out, at, len)?; + } + } + let length = out.stream_position().map_err(io)?.saturating_sub(start); + let pad = (4 - length % 4) % 4; + for _ in 0..pad { + out.write_all(&[0]).map_err(io)?; + } + Ok(()) +} + +/// Write a PSB directly to a seekable sink, patching channel and section lengths after each +/// bounded encoded stream. Callbacks supply encoded layer channels and the merged image; this +/// function never buffers their data. Publication/atomic replacement belongs to the caller. +pub fn write_layered( + file: &PsdFile, + out: &mut W, + mut pixels: impl FnMut(usize, i16, &mut W) -> Result<()>, + merged: impl FnOnce(&mut W) -> Result<()>, +) -> Result<()> { + if file.header.version != Version::Psb { + return Err(PsdError::Unsupported("seekable writer requires PSB".into())); + } + file.header.validate()?; + let mut prefix = Vec::new(); + file.header.write(&mut prefix); + prefix.extend(u32::try_from(file.color_mode_data.len()).map_err(|_| invalid())?.to_be_bytes()); + prefix.extend(&file.color_mode_data); + crate::resources::write_section(&mut prefix, &file.resources)?; + out.write_all(&prefix).map_err(io)?; + let lm_at = out.stream_position().map_err(io)?; + out.write_all(&0u64.to_be_bytes()).map_err(io)?; + let start = out.stream_position().map_err(io)?; + let li_at = start; + out.write_all(&0u64.to_be_bytes()).map_err(io)?; + if let (Some(info), LayerInfoPlacement::Section) = (&file.layer_info, &file.layer_info_placement) { + let begin = out.stream_position().map_err(io)?; + write_layers(out, info, &mut pixels)?; + let n = out.stream_position().map_err(io)?.saturating_sub(begin); + patch(out, li_at, n)?; + } + let mask = file.global_layer_mask.as_ref().map_or(&[][..], |m| m.data.as_slice()); + out.write_all(&u32::try_from(mask.len()).map_err(|_| invalid())?.to_be_bytes()).map_err(io)?; + out.write_all(mask).map_err(io)?; + for i in 0..=file.global_blocks.len() { + if let (Some(info), LayerInfoPlacement::GlobalBlock { index, signature, key, padding }) = (&file.layer_info, &file.layer_info_placement) + && i == (*index).min(file.global_blocks.len()) + { + out.write_all(signature).map_err(io)?; + out.write_all(key).map_err(io)?; + let at = out.stream_position().map_err(io)?; + out.write_all(&0u64.to_be_bytes()).map_err(io)?; + let begin = out.stream_position().map_err(io)?; + write_layers(out, info, &mut pixels)?; + let n = out.stream_position().map_err(io)?.saturating_sub(begin); + patch(out, at, n)?; + match padding { + Some(bytes) => out.write_all(bytes).map_err(io)?, + None => out.write_all(&[0u8; 3][..((4 - n % 4) % 4) as usize]).map_err(io)?, + } + } + if let Some(block) = file.global_blocks.get(i) { + let mut bytes = Vec::new(); + crate::tagged::write_blocks(&mut bytes, std::slice::from_ref(block), Version::Psb, 4)?; + out.write_all(&bytes).map_err(io)?; + } + } + out.write_all(&file.layer_mask_trailing).map_err(io)?; + let n = out.stream_position().map_err(io)?.saturating_sub(start); + patch(out, lm_at, n)?; + out.write_all(&file.image_data.compression.as_u16().to_be_bytes()).map_err(io)?; + merged(out) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Cursor; + + #[test] + fn indexed_channels_match_byte_decoder_at_every_depth_and_compression() { + for version in [Version::Psd, Version::Psb] { + for depth in [8, 16, 32] { + for compression in Compression::ALL { + let file = crate::testgen::layered(version, crate::ColorMode::Rgb, depth, compression); + let bytes = file.to_bytes().unwrap(); + let index = Index::read(&mut Cursor::new(&bytes), 32 << 20, &|| false).unwrap(); + assert_eq!(index.metadata.resources, file.resources); + assert_eq!(index.metadata.global_blocks, file.global_blocks); + for (rec, ranges) in index.metadata.layers().iter().zip(&index.channels) { + for range in ranges { + let (w, h) = rec.channel_rect(range.id).size().unwrap(); + let mut rows = Rows::new(Cursor::new(&bytes), *range, w, h, depth, version).unwrap(); + let mut decoded = Vec::new(); + let mut row = Vec::new(); + for _ in 0..h { + rows.read_row(&mut row).unwrap(); + decoded.extend_from_slice(&row); + } + rows.finish().unwrap(); + let original = file.layers().iter().find(|l| l.name == rec.name).unwrap(); + assert_eq!(decoded, original.decode_channel(range.id, depth, version).unwrap()); + } + } + } + } + } + } + + #[test] + fn bounded_index_and_cancel_do_not_allocate_declared_pixel_bytes() { + let file = crate::testgen::layered(Version::Psb, crate::ColorMode::Rgb, 8, Compression::Rle); + let bytes = file.to_bytes().unwrap(); + assert!(Index::read(&mut Cursor::new(&bytes), 2, &|| false).is_err()); + assert!(Index::read(&mut Cursor::new(&bytes), 32 << 20, &|| true).is_err()); + for n in [0, 25, 30, bytes.len() / 2] { + assert!(Index::read(&mut Cursor::new(&bytes[..n]), 32 << 20, &|| false).is_err()); + } + } + + #[test] + fn rows_reject_corrupt_checksum_and_truncated_data() { + let layout = PlaneLayout { planes: 1, width: 10, height: 4, depth: 8, version: Version::Psb }; + let encoded = crate::compression::encode_planes(Compression::Zip, &[1; 40], &layout).unwrap(); + for corrupt in [false, true] { + let mut bytes = encoded.clone(); + if corrupt { + let n = bytes.len(); + bytes[n - 1] ^= 0xff; + } else { + bytes.truncate(bytes.len() - 3); + } + let range = ChannelRange { id: 0, offset: 0, length: bytes.len() as u64, compression: Compression::Zip }; + let mut rows = Rows::new(Cursor::new(&bytes), range, 10, 4, 8, Version::Psb).unwrap(); + let mut row = Vec::new(); + let mut error = false; + for _ in 0..4 { + if rows.read_row(&mut row).is_err() { + error = true; + break; + } + } + assert!(error || rows.finish().is_err(), "truncated or corrupt zlib must fail"); + } + } + + #[test] + fn seekable_writer_patches_layer_channel_and_global_block_lengths() { + for depth in [8, 16, 32] { + for compression in Compression::ALL { + let file = crate::testgen::layered(Version::Psb, crate::ColorMode::Rgb, depth, compression); + let mut out = Cursor::new(Vec::new()); + write_layered( + &file, + &mut out, + |layer, id, out| out.write_all(&file.layers()[layer].channel(id).unwrap().data).map_err(io), + |out| out.write_all(&file.image_data.data).map_err(io), + ) + .unwrap(); + let read = PsdFile::from_bytes(&out.into_inner()).unwrap(); + assert_eq!(read.header, file.header); + assert_eq!(read.global_blocks, file.global_blocks); + assert_eq!(read.resources, file.resources); + assert_eq!(read.image_data, file.image_data); + assert_eq!(read.layers(), file.layers()); + } + } + } +} diff --git a/crates/ui-egui/src/control.rs b/crates/ui-egui/src/control.rs index 64f9c31556..69dc5ddc34 100644 --- a/crates/ui-egui/src/control.rs +++ b/crates/ui-egui/src/control.rs @@ -1219,6 +1219,11 @@ fn dispatch(app: &mut PhotocraftApp, ctx: &egui::Context, req: &ControlRequest) Err(e) => return err(e), } } + if matches!(path.rsplit('.').next().map(|s| s.to_ascii_lowercase()).as_deref(), Some("psd" | "psb" | "psdt" | "exr")) + && let Some(import) = app.services.automation_import_path.clone() + { + return run_waiting(app, wait, |app| crate::jobs_ui::open_authorized(app, path.to_string(), import)); + } let opened = match app.services.automation_read.as_mut() { Some(read) => read(path), None => Err("automation read authority is not configured".into()), @@ -2327,6 +2332,51 @@ mod tests { assert_eq!(written.borrow().len(), 3, "a bad tgaBits writes nothing"); } + #[test] + fn automation_exr_open_uses_authorized_seekable_import() { + use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }; + for source_read_only in [false, true] { + let reads = Arc::new(AtomicUsize::new(0)); + let imported = reads.clone(); + let services = crate::Services { + automation_import_path: Some(Arc::new(move |path, _| { + assert_eq!(path, "in/starmap.EXR"); + imported.fetch_add(1, Ordering::Relaxed); + Ok(photocraft_io::ImportResult { + document: photocraft_doc::Document::new( + "starmap.EXR", + photocraft_geom::Size::new(65536, 32768), + photocraft_color::ColorMode::Rgb, + photocraft_color::SampleType::F32, + ), + warnings: Vec::new(), + source_read_only, + preview_only: source_read_only, + }) + })), + automation_read: Some(Box::new(|_| Err("whole-file read must not run".into()))), + ..Default::default() + }; + let mut app = PhotocraftApp::new(photocraft_engine::Session::new(), services); + let response = call(&mut app, &egui::Context::default(), "app.open", json!({"path": "in/starmap.EXR", "wait": false})); + assert_eq!(response["ok"], true, "{response}"); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + while !app.session.jobs().is_empty() && std::time::Instant::now() < deadline { + app.session.poll_jobs(); + std::thread::sleep(std::time::Duration::from_millis(1)); + } + assert!(app.session.jobs().is_empty(), "EXR import did not finish"); + assert_eq!(reads.load(Ordering::Relaxed), 1); + assert_eq!(app.session.active().unwrap().doc.size, photocraft_geom::Size::new(65536, 32768)); + let st = app.session.active().unwrap(); + assert_eq!(st.source_read_only, source_read_only); + assert_eq!(st.path.is_none(), source_read_only, "read-only imports must not keep a source save path"); + } + } + #[test] fn automation_open_and_save_use_the_roots_and_reply_with_warnings() { use photocraft_color::{ColorMode, SampleType}; diff --git a/crates/ui-egui/src/file_open.rs b/crates/ui-egui/src/file_open.rs index d67e56dbf0..1cec43e5df 100644 --- a/crates/ui-egui/src/file_open.rs +++ b/crates/ui-egui/src/file_open.rs @@ -112,8 +112,30 @@ impl PhotocraftApp { crate::jobs_ui::start_open(self, &display_name(path), Some(path.to_string()), photocraft_engine::jobs::OpenSource::Path(path.to_string()))?; return Ok(Vec::new()); } - let bytes = photocraft_format::read_file(std::path::Path::new(path)).map_err(|e| format!("{path}: {e}"))?; - self.open_file(path, &bytes) + let seekable = ext == "exr" || matches!(ext.as_str(), "psd" | "psb" | "psdt") && std::fs::metadata(path).is_ok_and(|m| m.len() >= 64 << 20); + if !seekable { + let bytes = photocraft_format::read_file(std::path::Path::new(path)).map_err(|e| format!("{path}: {e}"))?; + return self.open_file(path, &bytes); + } + let name = self.open_name(&display_name(path)); + let imported = photocraft_io::import_path_with(&name, std::path::Path::new(path), &photocraft_raster::Interrupt::NONE).map_err(|e| e.to_string())?; + let warnings = with_missing_fonts_of(&imported.document, imported.warnings); + let question = self.session.color.open_question(&imported.document); + crate::open_prompt::arrive( + self, + crate::open_prompt::Held { + doc: imported.document, + read_only: imported.source_read_only, + name, + path: Some(path.to_string()), + warnings: warnings.clone(), + bytes: None, + slot: None, + question, + policy_applied: false, + }, + ); + Ok(warnings) } #[cfg(target_arch = "wasm32")] diff --git a/crates/ui-egui/src/jobs_ui.rs b/crates/ui-egui/src/jobs_ui.rs index 29da41d481..5fc586fe5b 100644 --- a/crates/ui-egui/src/jobs_ui.rs +++ b/crates/ui-egui/src/jobs_ui.rs @@ -249,6 +249,51 @@ pub(crate) fn finish_open_tab(app: &mut PhotocraftApp, tab: &OpenTab, v: &Value) r } +pub(crate) fn open_authorized(app: &mut PhotocraftApp, path: String, import: crate::AutomationImportPathFn) -> Result { + let name = app.open_name(&crate::file_open::display_name(&path)); + let worker_path = path.clone(); + let apply_name = name.clone(); + let apply_path = path.clone(); + match app + .session + .start_job( + OPEN_JOB, + json!({"path": path}), + "Opening", + false, + move |ctx| { + ctx.stage(0.0, 1.0, "Decoding", |ctl| import(&worker_path, ctl)) + .map_err(|e| if ctx.cancelled() { photocraft_engine::EngineError::Cancelled } else { photocraft_engine::EngineError::Other(e) }) + }, + move |session, imported| { + let source_read_only = imported.source_read_only; + let (mut doc, warnings) = (imported.document, imported.warnings); + doc.name = apply_name.clone(); + let document_path = (!source_read_only && !photocraft_engine::file_cmds::is_template(&apply_path)).then(|| apply_path.clone()); + let (index, color) = session.open_document(doc, document_path); + if let Some(st) = session.active_mut() { + st.source_read_only = source_read_only; + } + Ok(json!({"document": index, "name": apply_name, "warnings": warnings, "color": color, "path": apply_path, "automation": true})) + }, + ) + .map_err(|e| e.to_string())? + { + Started::Done(v) => { + finish_open(app, &name, Some(&path), &v, None)?; + Ok(v) + } + Started::Job(job) => { + app.jobs.opens.push(OpenTab { job, name, path: Some(path), slot: None, order: None }); + app.jobs.focus = Some(job); + app.jobs.last_started = Some(job); + app.ui.chrome.home = None; + app.ui.status_error = false; + Ok(json!({"job": job.0, "pending": true})) + } + } +} + /// Save the active document to `path` in the background: `save` encodes and writes it on a /// worker while the window keeps drawing and shows the progress dialog. The document is locked /// until the job ends, which [`tick`] records like a synchronous save. `Ok(None)`: started; diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 04027176b1..a62ab4c161 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -325,6 +325,8 @@ pub(crate) enum SaveKind { /// Automation: flat formats stay copies (#2579) and hooks never fire. Automation, } +/// Decode a document through handles beneath the desktop control session's authorized root. +pub type AutomationImportPathFn = std::sync::Arc) -> Result + Send + Sync>; /// Read bytes through the desktop control session's authorized read root. pub type AutomationReadFn = Box Result<(String, Vec), String>>; /// Write bytes through the desktop control session's authorized write root. @@ -415,6 +417,7 @@ pub struct Services { /// File access used only by control/MCP requests. Interactive dialogs keep /// using `file_dialog` and `write` with the user's authority. pub automation_read: Option, + pub automation_import_path: Option, pub automation_write: Option, pub automation_open_pcraft: Option, pub automation_command: Option, diff --git a/crates/ui-egui/tests/exr_open.rs b/crates/ui-egui/tests/exr_open.rs new file mode 100644 index 0000000000..ae3b737bec --- /dev/null +++ b/crates/ui-egui/tests/exr_open.rs @@ -0,0 +1,44 @@ +#![cfg(not(target_arch = "wasm32"))] +use photocraft_codecs::{ChannelLayout, EncodeOptions, Format, Image}; +use photocraft_ui_egui::PhotocraftApp; + +#[test] +fn synchronous_native_exr_open_uses_tile_admission_before_huge_pixel_decode() { + let image = Image::from_f32(1, 1, ChannelLayout::Rgb, &[0.5, 2.0, -0.25]).unwrap(); + let bytes = photocraft_codecs::encode(&image, Format::OpenExr, &EncodeOptions::default()).unwrap(); + let mut header_end = 8; + loop { + if bytes[header_end] == 0 { + header_end += 1; + break; + } + for _ in 0..2 { + while bytes[header_end] != 0 { + header_end += 1; + } + header_end += 1; + } + let length = i32::from_le_bytes(bytes[header_end..header_end + 4].try_into().unwrap()) as usize; + header_end += 4 + length; + } + let mut huge = bytes[..header_end].to_vec(); + let needle = b"dataWindow\0box2i\0\x10\0\0\0"; + let at = huge.windows(needle.len()).position(|v| v == needle).unwrap() + needle.len(); + huge[at + 8..at + 12].copy_from_slice(&65535_i32.to_le_bytes()); + huge[at + 12..at + 16].copy_from_slice(&32767_i32.to_le_bytes()); + let offset = (header_end + 2048 * 8) as u64; + for _ in 0..2048 { + huge.extend_from_slice(&offset.to_le_bytes()); + } + huge.extend_from_slice(&[0; 8]); + let path = std::env::temp_dir().join(format!("photocraft-exr-ui-{}.exr", std::process::id())); + std::fs::write(&path, huge).unwrap(); + let mut app = PhotocraftApp::new(photocraft_engine::Session::new(), Default::default()); + app.background_jobs = false; + photocraft_raster::spill::configure(None, 0).unwrap(); + photocraft_raster::memory::configure(photocraft_raster::memory::Policy { percent: None, absolute_bytes: 1 << 30 }); + let error = app.open_path(path.to_str().unwrap()).unwrap_err(); + assert!(error.contains("decoded image tiles"), "{error}"); + assert!(app.session.documents().is_empty()); + std::fs::remove_file(path).unwrap(); +} diff --git a/docs/roadmap.md b/docs/roadmap.md index ccdd5b8291..5b03188984 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -124,3 +124,5 @@ Each milestone has a **definition of done (DoD)** and must leave `main` green on | 2026-10-05 | major | Honest parity assessment added | 2026-10-13: native scratch storage pages current and history tiles through compressed, CRC-checked files. Fresh RAM budgets default to 70%; engine-owned memory supervision stops with its last session. Scratch errors block edit/save commits and permit retry. Unix files are private and stale files swept at startup; Windows files use delete-on-close. See [scratch disks](scratch-disks.md). Streaming import and viewport rendering follow separately. + +2026-10-13: seekable native PSD/PSB and supported scanline EXR imports decode bounded bands at the original resolution. Authorized opens retain read-only source protection. See [native streaming I/O](streaming-io.md). diff --git a/docs/scorecard.md b/docs/scorecard.md index 3b7b0f40e8..f7d7d5f29e 100644 --- a/docs/scorecard.md +++ b/docs/scorecard.md @@ -248,7 +248,7 @@ Never-crash attribute on 28 of 28 crate roots. | REL-2 | Every workspace crate carries the never-crash deny(clippy::unwrap_used, ...) attribute | done | | Counted automatically above (crate roots under crates/ and apps/); testkit gained it in #221 | | REL-3 | The app catches escaped panics around command dispatch and import/export | done | | a_panic_in_a_command_... runs the test-only `test.panic` spec through Session::execute and asserts Err plus an unchanged document (engine jobs_tests.rs); the worker guard is covered by a_panic_in_the_worker_...; crash_guard.rs guard_turns_a_panic_into_an_error covers import/export (apps/photocraft) | | REL-4 | Atomic save (temp, fsync, rename) for every document write | done | [#203](https://github.com/storytold/photocraft/issues/203) | photocraft_format::atomic_write (format/src/atomic.rs, #230): same-dir temp, sync_all, rename with Windows PermissionDenied retry, Unix dir fsync; every document write site routes through it and format/src/atomic/tests.rs failure-injects write, rename and read-only-dir errors | -| REL-212-1 | Fallible allocation for paste/erase/fill/transform/save/import, clear out-of-memory error | partial | [#212](https://github.com/storytold/photocraft/issues/212) | Raster region/tile buffers, paste/erase/fill/transform and warp output use fallible reservations; filter and lens working memory is admitted before edits, and current/history tiles spill to CRC-checked scratch. Save/import boundaries preserve state; remaining encoder/decoder allocations still need broader audit. | +| REL-212-1 | Fallible allocation for paste/erase/fill/transform/save/import, clear out-of-memory error | partial | [#212](https://github.com/storytold/photocraft/issues/212) | Raster region/tile buffers, paste/erase/fill/transform and warp output use fallible reservations; filter and lens working memory is admitted before edits, and current/history tiles spill to CRC-checked scratch. Native layered PSD/PSB and supported scanline EXR imports decode bounded bands; save/import boundaries preserve state; remaining encoder/decoder allocations still need broader audit. | | REL-212-2 | Export switches to PSB when the estimated file size exceeds 2 GB | done | [#212](https://github.com/storytold/photocraft/issues/212) | PSD export uses checked size estimates for raster, masks, channels, and retained metadata | | REL-212-3 | Tests: an allocation-limit injection makes each command Err with the document unchanged | partial | [#212](https://github.com/storytold/photocraft/issues/212) | Engine failpoint tests cover paste, erase, fill, transform, import and save transaction boundaries; codec-level allocator failure coverage remains open | diff --git a/docs/scratch-disks.md b/docs/scratch-disks.md index a99cc7b30d..f5cda32305 100644 --- a/docs/scratch-disks.md +++ b/docs/scratch-disks.md @@ -35,5 +35,6 @@ A failed read retains its backing extent and blocks editing, cache admission and until the tile is readable again. Recovery can retry the original bytes after the storage becomes available; zeros from a failed display read must never become saved pixels. -Scratch storage alone does not make every format, filter or renderer streamable. The native -PSD/EXR import and viewport streaming changes build on these guards and memory reservations. +Scratch storage alone does not make every format, filter or renderer streamable. See +[native PSD/EXR streaming](streaming-io.md) for the seekable import and output APIs. Viewport +rendering is a separate change built on these guards and memory reservations. diff --git a/docs/streaming-io.md b/docs/streaming-io.md new file mode 100644 index 0000000000..3cc4d3a674 --- /dev/null +++ b/docs/streaming-io.md @@ -0,0 +1,36 @@ +# Native PSD/EXR streaming + +Native path opens inspect a seekable source before allocating decoded pixels. Supported layered +PSD/PSB files and flat scanline EXR files decode bounded bands directly into guarded raster +tiles. Their document dimensions and samples remain at the source resolution; scratch residency +can move cold tiles to disk. The byte decoder remains available for web and unsupported layouts, +subject to working-memory admission rather than a file-sized allocation without a budget. + +PSD indexing retains bounded metadata and channel offsets without copying all channel data or +decoding the stored merged image. Independent seekable channel handles feed the existing document +mapping, preserving layers, masks, groups, metadata, colour models and depth. Unsupported nesting +and metadata limits produce errors. The native PSB writer uses the same metadata mapper with +deferred pixel sources and writes original-depth channel bands to a seekable output. + +EXR block decoding supports the flat scanline layouts accepted by the streaming decoder, including +the RGB HALF ZIP test image `65536 × 32768`. HALF samples are stored as F32, with a warning; this +does not reduce the document dimensions. Unsupported EXR layouts fall back to the normal codec +when its limits allow them. Chunk sizes, source bounds and transient buffers are checked before +allocation. Cancellation is checked between bands, and progress describes completed work. + +Control and CLI opens use the same importer. Authorized control opens create independent handles +under the held read-directory capability, including channel reopens. They do not gain ambient +filesystem access. Path opens check that file size and modification time remain unchanged. Read-only +and preview import flags survive the native and authorized routes, and templates keep an untitled +document without a source save path. + +The seekable output API accepts a caller-owned writer; atomic path output encodes into a temporary +file beside the destination, syncs it using the shared platform helper, then renames it. Encoder +failure, cancellation or unwinding removes the temporary and preserves the old destination. Its +integration with the desktop background-save service is a separate change, using upstream's +`SaveCtl` and document-lock/saved-revision contract. + +Synthetic tests compare streamed import with the byte decoder across PSD/PSB versions, depths, +colour modes and compression, and verify masks, metadata, cancellation, EXR chunk handling, +source protection and capability bounds. Large-file runtime checks use the public fixtures at +[vistaero.es/photocraft](https://vistaero.es/photocraft/). diff --git a/scorecard/reliability.toml b/scorecard/reliability.toml index d5fc1fb5a7..7bc708e548 100644 --- a/scorecard/reliability.toml +++ b/scorecard/reliability.toml @@ -35,7 +35,7 @@ id = "REL-212-1" target = "Fallible allocation for paste/erase/fill/transform/save/import, clear out-of-memory error" status = "partial" issue = 212 -note = "Raster region/tile buffers, paste/erase/fill/transform and warp output use fallible reservations; filter and lens working memory is admitted before edits, and current/history tiles spill to CRC-checked scratch. Save/import boundaries preserve state; remaining encoder/decoder allocations still need broader audit." +note = "Raster region/tile buffers, paste/erase/fill/transform and warp output use fallible reservations; filter and lens working memory is admitted before edits, and current/history tiles spill to CRC-checked scratch. Native layered PSD/PSB and supported scanline EXR imports decode bounded bands; save/import boundaries preserve state; remaining encoder/decoder allocations still need broader audit." [[item]] id = "REL-212-2" From f7b7e7f6187b965516f35e5479317871ca42f8c8 Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Tue, 3 Nov 2026 09:06:31 +0100 Subject: [PATCH 2/3] Rescope to EXR streaming: drop the separate PSD/PSB reader, seekable outputs and export pre-check Main already opens PSD/PSB from the file in bands (4915f0970, #5106) through psd_to_document_owned, which keeps damaged-file flagging, failed-layer drops, type-layer setup and the PSD pixel budget; PSD limits are handled in #10778. This drops the second PSD/PSB stream reader (psd::stream, io::psd_stream), the seekable PSB writer and atomic stream helpers, the authorized-workspace seekable open plumbing and the working-memory pre-check that refused ordinary exports. The flat scanline EXR band decoder and its tests stay and are wired into main's path open after merging main. --- Cargo.lock | 1 - apps/photocraft/src/services.rs | 6 - crates/automation/Cargo.toml | 1 - crates/automation/src/files.rs | 22 +- crates/automation/src/headless.rs | 40 +- crates/automation/src/workspace.rs | 105 +---- crates/codecs/examples/exr_region_probe.rs | 74 --- crates/engine/src/file_cmds.rs | 30 +- crates/engine/src/jobs.rs | 31 +- crates/format/src/atomic.rs | 60 --- crates/format/src/atomic/tests.rs | 84 ---- crates/io/examples/scratch_psb_probe.rs | 36 -- crates/io/src/lib.rs | 165 ------- crates/io/src/psd_export.rs | 85 +--- crates/io/src/psd_import.rs | 19 +- crates/io/src/psd_stream.rs | 395 ---------------- crates/io/tests/streaming_psd.rs | 136 ------ crates/io/tests/svg.rs | 17 - crates/psd/src/layer.rs | 4 +- crates/psd/src/lib.rs | 1 - crates/psd/src/stream.rs | 499 --------------------- crates/ui-egui/src/control.rs | 50 --- crates/ui-egui/src/file_open.rs | 26 +- crates/ui-egui/src/jobs_ui.rs | 45 -- crates/ui-egui/src/lib.rs | 3 - 25 files changed, 62 insertions(+), 1873 deletions(-) delete mode 100644 crates/codecs/examples/exr_region_probe.rs delete mode 100644 crates/io/examples/scratch_psb_probe.rs delete mode 100644 crates/io/src/psd_stream.rs delete mode 100644 crates/io/tests/streaming_psd.rs delete mode 100644 crates/psd/src/stream.rs diff --git a/Cargo.lock b/Cargo.lock index 9dfd9946f6..79b80aae04 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4295,7 +4295,6 @@ dependencies = [ "photocraft-format", "photocraft-geom", "photocraft-io", - "photocraft-raster", "rmcp", "schemars", "serde", diff --git a/apps/photocraft/src/services.rs b/apps/photocraft/src/services.rs index 3cae110ea3..d4a3869f31 100644 --- a/apps/photocraft/src/services.rs +++ b/apps/photocraft/src/services.rs @@ -406,11 +406,6 @@ fn shell_word(s: &str) -> String { pub fn native(automation: Option) -> Services { let clip: Rc>> = Rc::default(); - let automation_import_path = automation.clone().map(|workspace| { - std::sync::Arc::new(move |path: &str, ctl: &photocraft_raster::Interrupt<'_>| { - crate::crash_guard::guard("Open", || workspace.read_document(path, ctl).map_err(|e| e.to_string())) - }) as photocraft_ui_egui::AutomationImportPathFn - }); let automation_read = automation.clone().map(|workspace| { Box::new(move |path: &str| { let bytes = workspace.read(path).map_err(|error| error.to_string())?; @@ -454,7 +449,6 @@ pub fn native(automation: Option) -> file_dialog: Some(Box::new(show_file_dialog)), write: Some(Box::new(|path: &str, bytes: &[u8]| write_atomic(Path::new(path), bytes))), automation_read, - automation_import_path, automation_write, automation_open_pcraft, automation_save, diff --git a/crates/automation/Cargo.toml b/crates/automation/Cargo.toml index f15f500e05..0738140a41 100644 --- a/crates/automation/Cargo.toml +++ b/crates/automation/Cargo.toml @@ -15,7 +15,6 @@ photocraft-doc = { workspace = true } photocraft-compose = { workspace = true } photocraft-codecs = { workspace = true } photocraft-io = { workspace = true } -photocraft-raster = { workspace = true } photocraft-format = { path = "../format" } photocraft-engine = { workspace = true } serde = { workspace = true } diff --git a/crates/automation/src/files.rs b/crates/automation/src/files.rs index 61c6ba246a..cd34ab3410 100644 --- a/crates/automation/src/files.rs +++ b/crates/automation/src/files.rs @@ -31,17 +31,9 @@ pub fn open(path: &Path) -> Result { if path.is_dir() { return Ok(Opened { document: photocraft_format::load_path(path)?, warnings: Vec::new(), source_read_only: false }); } + let bytes = photocraft_format::read_file(path).map_err(|e| AutomationError::Io(format!("{}: {e}", path.display())))?; let name = path.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(); - #[cfg(not(target_arch = "wasm32"))] - { - let r = photocraft_io::import_path_with(&name, path, &photocraft_raster::Interrupt::NONE)?; - Ok(Opened { document: r.document, warnings: r.warnings, source_read_only: r.source_read_only }) - } - #[cfg(target_arch = "wasm32")] - { - let bytes = photocraft_format::read_file(path).map_err(|e| AutomationError::Io(format!("{}: {e}", path.display())))?; - open_bytes(&name, &bytes) - } + open_bytes(&name, &bytes) } /// Previews for a `.pcraft` bundle. @@ -81,16 +73,6 @@ pub fn save( w.save_path(&named, path, &previews(&named))?; return Ok(Vec::new()); } - #[cfg(not(target_arch = "wasm32"))] - if ext == "psb" || ext == "psd" && (opts.force_psb || doc.size.width > 30_000 || doc.size.height > 30_000) { - let mut warnings = Vec::new(); - photocraft_format::atomic::atomic_write_stream(path, |file| { - warnings = photocraft_io::export_psb_into(doc, file, &photocraft_raster::Interrupt::NONE).map_err(|e| std::io::Error::other(e.to_string()))?; - Ok(()) - }) - .map_err(|e| AutomationError::Io(e.to_string()))?; - return Ok(warnings); - } let r = photocraft_io::export(doc, &ext, opts)?; // Crash-safe: a failed write never destroys the previous file. photocraft_format::atomic_write(path, &r.bytes).map_err(|e| AutomationError::Io(e.to_string()))?; diff --git a/crates/automation/src/headless.rs b/crates/automation/src/headless.rs index 3a61ffa4dc..d6b3c17102 100644 --- a/crates/automation/src/headless.rs +++ b/crates/automation/src/headless.rs @@ -79,17 +79,9 @@ impl Headless { Filesystem::Denied => return Err(AutomationError::BadRequest("automation filesystem access is not granted: read authority is absent".into())), Filesystem::TrustedLocal => files::open(path)?, Filesystem::Workspace(workspace) => { - #[cfg(not(target_arch = "wasm32"))] - { - let r = workspace.read_document(requested, &photocraft_raster::Interrupt::NONE)?; - files::Opened { document: r.document, warnings: r.warnings, source_read_only: r.source_read_only } - } - #[cfg(target_arch = "wasm32")] - { - let bytes = workspace.read(requested)?; - let name = path.file_name().and_then(|name| name.to_str()).unwrap_or(requested); - files::open_bytes(name, &bytes)? - } + let bytes = workspace.read(requested)?; + let name = path.file_name().and_then(|name| name.to_str()).unwrap_or(requested); + files::open_bytes(name, &bytes)? } }; let path = match file_cmds::template_name(&self.session, requested) { @@ -259,29 +251,9 @@ impl Headless { files::save(&doc, &target, format, opts, Some(writer))? } Filesystem::Workspace(workspace) => { - #[cfg(not(target_arch = "wasm32"))] - if format - .map(|f| f.trim_start_matches('.').eq_ignore_ascii_case("psb")) - .unwrap_or_else(|| target.extension().is_some_and(|ext| ext.eq_ignore_ascii_case("psb"))) - { - let mut warnings = Vec::new(); - workspace.write_stream(target_text, |file| { - warnings = photocraft_io::export_psb_into(&doc, file, &photocraft_raster::Interrupt::NONE) - .map_err(|e| std::io::Error::other(e.to_string()))?; - Ok(()) - })?; - warnings - } else { - let (bytes, warnings) = files::save_bytes(&doc, target_text, format, opts)?; - workspace.write(target_text, &bytes)?; - warnings - } - #[cfg(target_arch = "wasm32")] - { - let (bytes, warnings) = files::save_bytes(&doc, target_text, format, opts)?; - workspace.write(target_text, &bytes)?; - warnings - } + let (bytes, warnings) = files::save_bytes(&doc, target_text, format, opts)?; + workspace.write(target_text, &bytes)?; + warnings } }; warnings.extend(photocraft_engine::type_extra_cmds::missing_font_warnings(&doc)); diff --git a/crates/automation/src/workspace.rs b/crates/automation/src/workspace.rs index 6321bc241f..cde3a3a702 100644 --- a/crates/automation/src/workspace.rs +++ b/crates/automation/src/workspace.rs @@ -68,24 +68,6 @@ impl AuthorizedWorkspace { let source = DirHandle { dir }; Ok(Some(photocraft_format::load_bytes_source(&source, &photocraft_format::LoadOptions::default())?)) } - /// Open a large document by rows through the held read capability, without a file-sized - /// byte array. Independent channel cursors remain rooted beneath the authorized directory. - #[cfg(not(target_arch = "wasm32"))] - pub fn read_document(&self, path: &str, ctl: &photocraft_raster::Interrupt<'_>) -> Result { - let relative = request_path(path, self.read.as_ref())?; - let root = self.read.as_ref().ok_or_else(|| AutomationError::BadRequest(format!("{DENIED}: read authority is absent")))?; - let original = root.dir.metadata(&relative).map_err(|e| file_error("read", path, e))?; - if !original.is_file() { - return Err(AutomationError::BadRequest("automation read path is not a regular file".into())); - } - let name = relative.file_name().and_then(|n| n.to_str()).unwrap_or(path); - let result = photocraft_io::import_seekable_with(name, &|| root.dir.open(&relative), ctl)?; - let current = root.dir.metadata(&relative).map_err(|e| file_error("read", path, e))?; - if current.len() != original.len() || current.modified().ok() != original.modified().ok() { - return Err(AutomationError::Io("source changed during import; open it again".into())); - } - Ok(result) - } /// Create or replace one file below the configured write root, crash-safely: the bytes go to /// a temporary file beside the target, which is synced and renamed over it (the same steps @@ -96,30 +78,18 @@ impl AuthorizedWorkspace { /// resolution and file creation relative to the held directory handle, so /// a non-existent final target is supported without ambient path access. pub fn write(&self, path: &str, bytes: &[u8]) -> Result<(), AutomationError> { - self.write_stream(path, |file| file.write_all(bytes)) - } - - /// Atomically publish a seekable encoder through the held write capability. No ambient - /// paths are exposed to the encoder, including for temporary-file creation and rename. - pub fn write_stream(&self, path: &str, encode: impl FnOnce(&mut cap_std::fs::File) -> std::io::Result<()>) -> Result<(), AutomationError> { - struct Cleanup(Arc, PathBuf); - impl Drop for Cleanup { - fn drop(&mut self) { - let _ = self.0.remove_file(&self.1); - } - } let relative = request_path(path, self.write.as_ref())?; let root = self.write.as_ref().ok_or_else(|| AutomationError::BadRequest(format!("{DENIED}: write authority is absent")))?; let leaf = relative.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(); let parent = relative.parent().map(Path::to_path_buf).unwrap_or_default(); let tmp = parent.join(photocraft_format::atomic::temp_name(&leaf)); let mut options = OpenOptions::new(); - options.read(true).write(true).create_new(true); - let file = root.dir.open_with(&tmp, &options).map_err(|e| file_error("write", path, e))?; - let _cleanup = Cleanup(root.dir.clone(), tmp.clone()); - // This binding drops before the cleanup guard if an encoder unwinds. - let mut file = file; - let written = encode(&mut file).and_then(|()| photocraft_format::atomic::sync_file(&file.into_std())); + options.write(true).create_new(true); + let written = root.dir.open_with(&tmp, &options).and_then(|mut file| { + file.write_all(bytes)?; + // Falls back to a plain fsync on shares that refuse a full flush (#1336). + photocraft_format::atomic::sync_file(&file.into_std()) + }); let renamed = written.and_then(|()| photocraft_format::atomic::retry_rename(RenameRetry::platform(), || root.dir.rename(&tmp, &root.dir, &relative))); if let Err(e) = renamed { let _ = root.dir.remove_file(&tmp); @@ -476,69 +446,6 @@ impl Drop for ExportSequence { mod tests { use super::*; - #[cfg(not(target_arch = "wasm32"))] - #[test] - fn seekable_document_import_stays_within_read_capability() { - let (_, _, workspace) = roots("stream-read"); - let mut session = photocraft_engine::Session::new(); - session.execute("file.new", serde_json::json!({"width": 32, "height": 16})).unwrap(); - session.execute("layer.new.layer", serde_json::json!({})).unwrap(); - let doc = &session.active().unwrap().doc; - workspace - .write_stream("small.psb", |file| { - photocraft_io::export_psb_into(doc, file, &photocraft_raster::Interrupt::NONE).map_err(|e| std::io::Error::other(e.to_string()))?; - Ok(()) - }) - .unwrap(); - let opened = workspace.read_document("small.psb", &photocraft_raster::Interrupt::NONE).unwrap(); - assert_eq!(opened.document.size, doc.size); - assert_eq!(opened.document.layers.len(), doc.layers.len()); - assert!(workspace.read_document("../small.psb", &photocraft_raster::Interrupt::NONE).is_err()); - assert!(workspace.read_document("C:/small.psb", &photocraft_raster::Interrupt::NONE).is_err()); - } - - #[test] - fn seekable_capability_writer_rejects_escape_and_preserves_failed_destination() { - use std::io::{Seek, SeekFrom}; - let (inside, outside, workspace) = roots("stream"); - std::fs::write(inside.join("kept.psb"), b"original").unwrap(); - assert!(workspace.write_stream("../escaped.psb", |_| Ok(())).is_err()); - assert!(!outside.join("escaped.psb").exists()); - assert!( - workspace - .write_stream("kept.psb", |file| { - file.write_all(b"partial")?; - Err(std::io::Error::new(std::io::ErrorKind::StorageFull, "injected full disk")) - }) - .is_err() - ); - assert_eq!(std::fs::read(inside.join("kept.psb")).unwrap(), b"original"); - assert!(temp_files(&inside).is_empty()); - workspace - .write_stream("kept.psb", |file| { - file.write_all(b"0000payload")?; - file.seek(SeekFrom::Start(0))?; - file.write_all(b"0007") - }) - .unwrap(); - assert_eq!(std::fs::read(inside.join("kept.psb")).unwrap(), b"0007payload"); - } - - #[test] - fn seekable_capability_unwind_removes_temporary_after_closing_handle() { - let (inside, _, workspace) = roots("stream-unwind"); - std::fs::write(inside.join("kept.psb"), b"original").unwrap(); - let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - let _ = workspace.write_stream("kept.psb", |file| { - file.write_all(b"partial")?; - panic!("injected encoder unwind"); - }); - })); - assert!(result.is_err()); - assert_eq!(std::fs::read(inside.join("kept.psb")).unwrap(), b"original"); - assert!(temp_files(&inside).is_empty()); - } - fn roots(name: &str) -> (PathBuf, PathBuf, AuthorizedWorkspace) { let base = std::env::temp_dir().join(format!("photocraft-workspace-{}-{name}", std::process::id())); let inside = base.join("inside"); diff --git a/crates/codecs/examples/exr_region_probe.rs b/crates/codecs/examples/exr_region_probe.rs deleted file mode 100644 index 4b57e803aa..0000000000 --- a/crates/codecs/examples/exr_region_probe.rs +++ /dev/null @@ -1,74 +0,0 @@ -//! Read-only independent EXR sample oracle: `exr_region_probe file.exr x y [x y ...]`. -//! Uses the dependency's filtered block reader, independently of our band conversion. -use exr::{block::reader::ChunksReader, meta::attribute::SampleType}; -use std::io::BufReader; - -fn run() -> Result<(), Box> { - let args: Vec = std::env::args().skip(1).collect(); - let path = args.first().ok_or("usage: exr_region_probe file.exr x y [x y ...]")?; - let coordinates = args.get(1..).ok_or("missing coordinates")?; - if coordinates.is_empty() || !coordinates.len().is_multiple_of(2) || coordinates.len() > 512 { - return Err("provide 1 to 256 coordinate pairs".into()); - } - let points: Vec<(usize, usize)> = coordinates - .chunks_exact(2) - .map(|p| Ok((p.first().ok_or("x")?.parse()?, p.get(1).ok_or("y")?.parse()?))) - .collect::>>()?; - // Bound metadata, dimensions and block sizes before using the oracle reader. - let info = photocraft_codecs::exr_stream::Decoder::new(std::fs::File::open(path)?, &|| false)?.info(); - if points.iter().any(|&(x, y)| x >= info.width as usize || y >= info.height as usize) { - return Err("coordinate outside the data window".into()); - } - let reader = exr::block::read(BufReader::new(std::fs::File::open(path)?), false)?; - let mut blocks = reader - .filter_chunks(false, |_, _, block| points.iter().any(|&(_, y)| y >= block.pixel_position.1 && y < block.pixel_position.1 + block.pixel_size.1))? - .sequential_decompressor(false); - let channels = blocks.meta_data().headers.first().ok_or("missing header")?.channels.clone(); - let mut samples = vec![[0_f32, 0.0, 0.0, 1.0]; points.len()]; - let mut seen = vec![[false; 3]; points.len()]; - while let Some(block) = blocks.decompress_next_block() { - let block = block?; - for line in block.lines(&channels) { - let channel = channels.list.get(line.location.channel).ok_or("channel")?; - let name = channel.name.to_string(); - let component = match name.rsplit('.').next() { - Some("R") => 0, - Some("G") => 1, - Some("B") => 2, - _ => continue, - }; - for (index, &(x, y)) in points.iter().enumerate().filter(|(_, p)| p.1 == line.location.position.1) { - let value = match channel.sample_type { - SampleType::F16 => line.read_samples::().nth(x).ok_or("sample")??.to_f32(), - SampleType::F32 => line.read_samples::().nth(x).ok_or("sample")??, - SampleType::U32 => line.read_samples::().nth(x).ok_or("sample")?? as f32, - }; - if !value.is_finite() { - return Err("non-finite sample cannot be represented in JSON".into()); - } - *samples.get_mut(index).and_then(|s| s.get_mut(component)).ok_or("sample index")? = value; - *seen.get_mut(index).and_then(|s| s.get_mut(component)).ok_or("sample index")? = true; - let _ = y; - } - } - } - if seen.iter().flatten().any(|v| !v) { - return Err("missing RGB samples".into()); - } - println!("{{\"width\":{},\"height\":{},\"samples\":[", info.width, info.height); - for (index, (&(x, y), sample)) in points.iter().zip(samples).enumerate() { - println!("{}{{\"x\":{x},\"y\":{y},\"rgba\":{:?}}}", if index == 0 { "" } else { "," }, sample); - } - println!("]}}"); - Ok(()) -} - -fn main() -> std::process::ExitCode { - match run() { - Ok(()) => std::process::ExitCode::SUCCESS, - Err(error) => { - eprintln!("{error}"); - std::process::ExitCode::FAILURE - } - } -} diff --git a/crates/engine/src/file_cmds.rs b/crates/engine/src/file_cmds.rs index 58c4e6ca7f..7689f39679 100644 --- a/crates/engine/src/file_cmds.rs +++ b/crates/engine/src/file_cmds.rs @@ -357,25 +357,9 @@ pub(crate) fn encode(doc: &Document, path: &str, save: impl Into) -> R } pub(crate) fn save_doc(doc: &Document, path: &str, save: impl Into) -> Result> { - #[cfg(not(target_arch = "wasm32"))] - { - let save = save.into(); - let mut opts = photocraft_io::ExportOptions { tiff_layers: save.tiff_layers, ..Default::default() }; - if let Some(q) = save.quality { - let q = (q.clamp(0.0, 12.0) / 12.0 * 99.0 + 1.0).round() as u8; - opts.encode.jpeg_quality = q; - opts.encode.webp_quality = q; - opts.encode.webp_lossless = false; - } - photocraft_io::export_path_with(doc, std::path::Path::new(path), &opts, &photocraft_raster::Interrupt::NONE) - .map_err(|e| EngineError::Other(format!("{path}: {e}"))) - } - #[cfg(target_arch = "wasm32")] - { - let (bytes, warnings) = encode(doc, path, save)?; - write_file(path, &bytes)?; - Ok(warnings) - } + let (bytes, warnings) = encode(doc, path, save)?; + write_file(path, &bytes)?; + Ok(warnings) } pub(crate) fn str_param<'a>(p: &'a Value, key: &str, cmd: &str) -> Result<&'a str> { @@ -442,12 +426,8 @@ fn close_others(s: &mut Session, p: &Value) -> Result { /// "Revert" history state); the document is clean afterwards. fn revert(s: &mut Session) -> Result { let path = s.active().and_then(|d| d.path.clone()).ok_or(EngineError::Other("the document has never been saved".into()))?; - #[cfg(not(target_arch = "wasm32"))] - let fresh = photocraft_io::import_path_with(&file_name(&path), std::path::Path::new(&path), &photocraft_raster::Interrupt::NONE) - .map_err(|e| EngineError::Other(e.to_string()))? - .document; - #[cfg(target_arch = "wasm32")] - let fresh = import(&file_name(&path), &read_file(&path)?)?; + let bytes = read_file(&path)?; + let fresh = import(&file_name(&path), &bytes)?; s.edit("Revert", |doc, active| { let (id, name) = (doc.id, doc.name.clone()); *doc = fresh; diff --git a/crates/engine/src/jobs.rs b/crates/engine/src/jobs.rs index 374300ad06..1fc75ead29 100644 --- a/crates/engine/src/jobs.rs +++ b/crates/engine/src/jobs.rs @@ -258,6 +258,17 @@ fn independent_of_document(id: &str) -> bool { PREFIXES.iter().any(|p| id.starts_with(p)) || IDS.contains(&id) } +#[cfg(not(target_arch = "wasm32"))] +fn read_path(path: &str) -> Result> { + // Bounded reads, and a clear error for a file larger than memory (#375). + photocraft_format::read_file(std::path::Path::new(path)).map_err(|e| EngineError::Other(format!("{path}: {e}"))) +} + +#[cfg(target_arch = "wasm32")] +fn read_path(path: &str) -> Result> { + Err(EngineError::Other(format!("cannot open paths on the web: {path}"))) +} + /// Run a job-capable command's heavy part. `work` runs on a worker thread when the command was /// started with [`Session::start`] (inline otherwise, and always on wasm); `apply` then runs on /// the calling thread with its output and returns the command's result. With `lock_document`, @@ -425,19 +436,17 @@ impl Session { false, move |ctx| { ctx.progress(0.0, "Reading"); + let bytes = match source { + OpenSource::Bytes(b) => b, + OpenSource::Path(p) => Arc::new(read_path(&p)?), + }; ctx.check()?; ctx.progress(0.02, "Decoding"); - ctx.stage(0.02, 1.0, "Decoding", |ctl| match source { - OpenSource::Bytes(b) => photocraft_io::import_with_svg_group_depth_and_interrupt(&name_w, &b, max_svg_group_depth, ctl), - #[cfg(not(target_arch = "wasm32"))] - OpenSource::Path(p) => photocraft_io::import_path_with_svg_group_depth(&name_w, std::path::Path::new(&p), max_svg_group_depth, ctl), - #[cfg(target_arch = "wasm32")] - OpenSource::Path(_) => Err(photocraft_io::IoError::Unsupported("paths cannot be opened on the web".into())), - }) - .map_err(|e| match e { - photocraft_io::IoError::Cancelled => EngineError::Cancelled, - e => EngineError::Other(e.to_string()), - }) + ctx.stage(0.02, 1.0, "Decoding", |ctl| photocraft_io::import_with_svg_group_depth_and_interrupt(&name_w, &bytes, max_svg_group_depth, ctl)) + .map_err(|e| match e { + photocraft_io::IoError::Cancelled => EngineError::Cancelled, + e => EngineError::Other(e.to_string()), + }) }, move |s, r: photocraft_io::ImportResult| { // Ask When Opening: held back until the prompt is answered (`take_held_open`). A diff --git a/crates/format/src/atomic.rs b/crates/format/src/atomic.rs index fb86d9eb38..bf39fe2761 100644 --- a/crates/format/src/atomic.rs +++ b/crates/format/src/atomic.rs @@ -155,66 +155,6 @@ pub fn atomic_write_progress(path: &Path, bytes: &[u8], progress: &dyn Fn(f32)) atomic_write_with_progress(&RealFs, RenameRetry::platform(), path, bytes, progress) } -/// Stream into a new seekable temporary file and publish it only after successful encoding -/// and fsync. A failed encoder, cancellation or full disk leaves the destination intact. -#[cfg(not(target_arch = "wasm32"))] -pub fn atomic_write_stream(path: &Path, encode: impl FnOnce(&mut std::fs::File) -> io::Result<()>) -> io::Result<()> { - atomic_write_stream_with_commit(path, encode, || Ok(())) -} - -/// Like [`atomic_write_stream`], with a final approval immediately before replacement. -/// The encoder and flush remain cancellable; after `commit` succeeds publication runs to -/// completion. A rejected commit or an unwinding callback removes the temporary file. -#[cfg(not(target_arch = "wasm32"))] -pub fn atomic_write_stream_with_commit( - path: &Path, - encode: impl FnOnce(&mut std::fs::File) -> io::Result<()>, - commit: impl FnOnce() -> io::Result<()>, -) -> io::Result<()> { - struct Cleanup(PathBuf); - impl Drop for Cleanup { - fn drop(&mut self) { - let _ = std::fs::remove_file(&self.0); - } - } - let target = resolve_symlink(path); - let name = target.file_name().ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "not a file path"))?; - let dir = target.parent().filter(|p| !p.as_os_str().is_empty()).unwrap_or(Path::new(".")); - std::fs::create_dir_all(dir).map_err(|e| write_error(e, path, dir))?; - let mut temp = None; - for _ in 0..8 { - let candidate = dir.join(temp_name(&name.to_string_lossy())); - match std::fs::OpenOptions::new().read(true).write(true).create_new(true).open(&candidate) { - Ok(file) => { - temp = Some((candidate, file)); - break; - } - Err(e) if e.kind() == io::ErrorKind::AlreadyExists => continue, - Err(e) => return Err(write_error(e, path, dir)), - } - } - let (tmp, file) = temp.ok_or_else(|| io::Error::new(io::ErrorKind::AlreadyExists, "cannot create a save temporary"))?; - let _cleanup = Cleanup(tmp.clone()); - // Close the handle before cleanup runs during unwinding, including on Windows. - let mut file = file; - let result = encode(&mut file).and_then(|()| sync_file(&file)); - drop(file); - if let Err(e) = result { - let _ = std::fs::remove_file(&tmp); - return Err(write_error(e, path, dir)); - } - if target.exists() { - let _ = RealFs.copy_permissions(&target, &tmp); - } - commit()?; - if let Err(e) = retry_rename(RenameRetry::platform(), || RealFs.rename(&tmp, &target)) { - let _ = RealFs.remove(&tmp); - return Err(rename_error(e, path)); - } - let _ = RealFs.sync_dir(dir); - Ok(()) -} - /// [`atomic_write`] through an explicit file system and retry policy (the failure-injection seam). pub fn atomic_write_with(fs: &dyn AtomicFs, retry: RenameRetry, path: &Path, bytes: &[u8]) -> io::Result<()> { atomic_write_with_progress(fs, retry, path, bytes, &|_| {}) diff --git a/crates/format/src/atomic/tests.rs b/crates/format/src/atomic/tests.rs index eef149d476..8e71bc7ce3 100644 --- a/crates/format/src/atomic/tests.rs +++ b/crates/format/src/atomic/tests.rs @@ -31,74 +31,6 @@ fn assert_no_temp(dir: &Path) { const ORIGINAL: &[u8] = b"the user's precious original document"; -#[test] -fn streaming_commit_runs_after_encoding_and_before_replacement() { - let dir = TempDir::new("stream-commit"); - let path = dir.0.join("document.psb"); - std::fs::write(&path, ORIGINAL).unwrap(); - let encoded = Cell::new(false); - let result = atomic_write_stream_with_commit( - &path, - |file| { - file.write_all(b"complete")?; - encoded.set(true); - Ok(()) - }, - || { - assert!(encoded.get()); - assert_eq!(std::fs::read(&path).unwrap(), ORIGINAL); - Err(io::Error::new(io::ErrorKind::Interrupted, "cancelled before commit")) - }, - ); - assert_eq!(result.unwrap_err().kind(), io::ErrorKind::Interrupted); - assert_eq!(std::fs::read(&path).unwrap(), ORIGINAL); - assert_no_temp(&dir.0); - - atomic_write_stream_with_commit(&path, |file| file.write_all(b"complete"), || Ok(())).unwrap(); - assert_eq!(std::fs::read(&path).unwrap(), b"complete"); - assert_no_temp(&dir.0); -} - -#[test] -fn streaming_commit_unwind_removes_temporary_and_keeps_original() { - let dir = TempDir::new("stream-commit-unwind"); - let path = dir.0.join("document.psb"); - std::fs::write(&path, ORIGINAL).unwrap(); - let result = std::panic::catch_unwind(|| atomic_write_stream_with_commit(&path, |file| file.write_all(b"complete"), || panic!("injected commit panic"))); - assert!(result.is_err()); - assert_eq!(std::fs::read(&path).unwrap(), ORIGINAL); - assert_no_temp(&dir.0); -} - -#[test] -fn seekable_encoder_failure_keeps_original_and_cleans_temp() { - use std::io::{Seek, SeekFrom}; - let dir = TempDir::new("stream"); - let path = dir.0.join("document.psb"); - std::fs::write(&path, ORIGINAL).unwrap(); - for kind in [io::ErrorKind::StorageFull, io::ErrorKind::Interrupted] { - assert!( - atomic_write_stream(&path, |file| { - file.write_all(b"partial")?; - file.seek(SeekFrom::Start(0))?; - file.write_all(b"PATCH")?; - Err(io::Error::new(kind, "injected encoder failure")) - }) - .is_err() - ); - assert_eq!(std::fs::read(&path).unwrap(), ORIGINAL); - assert_no_temp(&dir.0); - } - atomic_write_stream(&path, |file| { - file.write_all(b"0000payload")?; - file.seek(SeekFrom::Start(0))?; - file.write_all(b"0007") - }) - .unwrap(); - assert_eq!(std::fs::read(&path).unwrap(), b"0007payload"); - assert_no_temp(&dir.0); -} - /// Real file system with failures injected at chosen steps. #[derive(Default)] struct Faulty { @@ -333,19 +265,3 @@ fn sync_file_flushes_a_local_file() { let file = std::fs::OpenOptions::new().write(true).create_new(true).open(&path).unwrap(); sync_file(&file).unwrap(); } - -#[test] -fn streaming_encoder_unwind_closes_and_removes_temporary_file() { - let d = TempDir::new("stream-unwind"); - let p = d.0.join("doc.psb"); - std::fs::write(&p, ORIGINAL).unwrap(); - let result = std::panic::catch_unwind(|| { - let _ = atomic_write_stream(&p, |file| { - file.write_all(b"partial output")?; - panic!("injected encoder unwind"); - }); - }); - assert!(result.is_err()); - assert_eq!(std::fs::read(&p).unwrap(), ORIGINAL); - assert_no_temp(&d.0); -} diff --git a/crates/io/examples/scratch_psb_probe.rs b/crates/io/examples/scratch_psb_probe.rs deleted file mode 100644 index 99c210aae0..0000000000 --- a/crates/io/examples/scratch_psb_probe.rs +++ /dev/null @@ -1,36 +0,0 @@ -//! Read-only large-document import probe. Never exports or changes the input file. -use photocraft_raster::spill; -use std::time::Instant; - -fn main() -> Result<(), String> { - let mut args = std::env::args().skip(1); - let path = args.next().ok_or("usage: scratch_psb_probe ")?; - let dir = args.next().ok_or("missing scratch directory")?; - spill::configure(Some(std::path::Path::new(&dir)), 1 << 30)?; - let start = Instant::now(); - let result = photocraft_io::import_path_with(&path, std::path::Path::new(&path), &photocraft_raster::Interrupt::NONE).map_err(|e| e.to_string())?; - spill::settle(); - let d = &result.document; - println!( - "import_seconds={:.3} size={}x{} depth={:?} mode={:?} warnings={:?}", - start.elapsed().as_secs_f64(), - d.size.width, - d.size.height, - d.depth, - d.mode, - result.warnings - ); - for (_, _, layer) in d.walk() { - if let Some(surface) = layer.surface() { - // Access source pixels directly, without creating a GPU canvas or flattening. - let bounds = surface.tile_bounds(); - let x = bounds.x0.saturating_add(bounds.width() as i32 / 2); - let y = bounds.y0.saturating_add(bounds.height() as i32 / 2); - println!("layer={:?} tiles={} bounds={bounds:?} sample={:?}", layer.name, surface.tile_count(), surface.pixel(x, y)); - } - } - spill::check_integrity()?; - println!("scratch={}", serde_json::to_string(&spill::stats()).map_err(|e| e.to_string())?); - drop(result); - spill::configure(None, 0) -} diff --git a/crates/io/src/lib.rs b/crates/io/src/lib.rs index 0cb2b481f9..936bab32b9 100644 --- a/crates/io/src/lib.rs +++ b/crates/io/src/lib.rs @@ -41,8 +41,6 @@ pub mod camera_raw_xmp; mod channel_map; pub mod comps_map; pub mod effects_map; -#[cfg(not(target_arch = "wasm32"))] -mod exr_native; pub mod exr_parts; mod flat; mod gradient_bake; @@ -57,8 +55,6 @@ mod pdn; pub mod pixels; mod psd_export; mod psd_import; -#[cfg(not(target_arch = "wasm32"))] -mod psd_stream; pub mod psd_thumbnail; pub mod raw; pub mod slices_map; @@ -245,120 +241,6 @@ pub fn import(name: &str, bytes: &[u8]) -> Result { import_with_svg_group_depth(name, bytes, photocraft_doc::MAX_GROUP_DEPTH) } -/// Native seekable import. Layered PSD/PSB and flat scanline EXR decode bounded bands directly into tiles; -/// the bytes API remains available for web and for formats without a seekable decoder. -#[cfg(not(target_arch = "wasm32"))] -pub fn import_path_with(name: &str, path: &std::path::Path, ctl: &photocraft_raster::Interrupt<'_>) -> Result { - import_path_with_svg_group_depth(name, path, photocraft_doc::MAX_GROUP_DEPTH, ctl) -} - -/// Seekable import preserving the caller's SVG rasterization preference on fallback. -#[cfg(not(target_arch = "wasm32"))] -pub fn import_path_with_svg_group_depth( - name: &str, - path: &std::path::Path, - max_svg_group_depth: usize, - ctl: &photocraft_raster::Interrupt<'_>, -) -> Result { - use std::io::Read; - ctl.check().map_err(|_| IoError::Cancelled)?; - photocraft_compose::discard_unreadable_pixel_caches(); - photocraft_raster::spill::check_integrity().map_err(IoError::Unsupported)?; - let before = photocraft_raster::spill::read_error_generation(); - let mut source = std::fs::File::open(path).map_err(|e| IoError::Unsupported(e.to_string()))?; - let original = source.metadata().map_err(|e| IoError::Unsupported(e.to_string()))?; - let size = original.len(); - let mut magic = [0; 4]; - let n = source.read(&mut magic).map_err(|e| IoError::Unsupported(e.to_string()))?; - if n == 4 && is_psd(&magic) { - let budget = (photocraft_raster::memory::stats().1.transient_bytes / 8).min(128 << 20); - let _metadata = photocraft_raster::memory::try_reserve(budget.saturating_mul(3)).map_err(IoError::Unsupported)?; - let index = photocraft_psd::stream::Index::read(&mut source, budget, &|| ctl.cancelled()) - .map_err(|e| if ctl.cancelled() { IoError::Cancelled } else { e.into() })?; - if psd_stream::eligible(&index) { - if psd_import::group_depth(&index.metadata) > photocraft_doc::MAX_GROUP_DEPTH { - return Err(IoError::Unsupported("PSD layer nesting exceeds the document limit".into())); - } - let mut result = psd_stream::import(&|| std::fs::File::open(path), &index, *ctl)?; - let current = std::fs::metadata(path).map_err(|e| IoError::Unsupported(e.to_string()))?; - if current.len() != original.len() || current.modified().ok() != original.modified().ok() { - return Err(IoError::Unsupported("the source file changed while it was being opened; open it again".into())); - } - result.document.name = name.into(); - photocraft_raster::spill::check_since(before).map_err(IoError::Unsupported)?; - ctl.check().map_err(|_| IoError::Cancelled)?; - return Ok(result); - } - } - if n == 4 - && photocraft_codecs::detect(&magic) == Some(photocraft_codecs::Format::OpenExr) - && let Some(mut result) = exr_native::import(&mut source, ctl)? - { - let current = std::fs::metadata(path).map_err(|e| IoError::Unsupported(e.to_string()))?; - if current.len() != original.len() || current.modified().ok() != original.modified().ok() { - return Err(IoError::Unsupported("the source file changed while it was being opened; open it again".into())); - } - result.document.name = name.into(); - photocraft_raster::spill::check_since(before).map_err(IoError::Unsupported)?; - ctl.check().map_err(|_| IoError::Cancelled)?; - return Ok(result); - } - drop(source); - let _bytes = photocraft_raster::memory::try_reserve(size) - .map_err(|_| IoError::Unsupported("this format requires a whole-file decoder and exceeds the working-memory budget".into()))?; - let bytes = photocraft_format::read_file(path).map_err(|e| IoError::Unsupported(e.to_string()))?; - import_with_svg_group_depth_and_interrupt(name, &bytes, max_svg_group_depth, ctl) -} - -/// Read from handles already authorized by the caller. Independent opens prevent channel -/// readers sharing a seek cursor, and no ambient filesystem access occurs in this function. -#[cfg(not(target_arch = "wasm32"))] -pub fn import_seekable_with( - name: &str, - open: &dyn Fn() -> std::io::Result, - ctl: &photocraft_raster::Interrupt<'_>, -) -> Result { - use std::io::SeekFrom; - ctl.check().map_err(|_| IoError::Cancelled)?; - photocraft_compose::discard_unreadable_pixel_caches(); - photocraft_raster::spill::check_integrity().map_err(IoError::Unsupported)?; - let before = photocraft_raster::spill::read_error_generation(); - let mut source = open().map_err(|e| IoError::Unsupported(e.to_string()))?; - let size = source.seek(SeekFrom::End(0)).map_err(|e| IoError::Unsupported(e.to_string()))?; - source.seek(SeekFrom::Start(0)).map_err(|e| IoError::Unsupported(e.to_string()))?; - let mut magic = [0; 4]; - let n = source.read(&mut magic).map_err(|e| IoError::Unsupported(e.to_string()))?; - if n == 4 && is_psd(&magic) { - let budget = (photocraft_raster::memory::stats().1.transient_bytes / 8).min(128 << 20); - let _metadata = photocraft_raster::memory::try_reserve(budget.saturating_mul(3)).map_err(IoError::Unsupported)?; - let index = photocraft_psd::stream::Index::read(&mut source, budget, &|| ctl.cancelled()) - .map_err(|e| if ctl.cancelled() { IoError::Cancelled } else { e.into() })?; - if psd_stream::eligible(&index) { - if psd_import::group_depth(&index.metadata) > photocraft_doc::MAX_GROUP_DEPTH { - return Err(IoError::Unsupported("PSD layer nesting exceeds the document limit".into())); - } - let mut result = psd_stream::import(open, &index, *ctl)?; - result.document.name = name.into(); - photocraft_raster::spill::check_since(before).map_err(IoError::Unsupported)?; - ctl.check().map_err(|_| IoError::Cancelled)?; - return Ok(result); - } - } - if n == 4 - && photocraft_codecs::detect(&magic) == Some(photocraft_codecs::Format::OpenExr) - && let Some(mut result) = exr_native::import(&mut source, ctl)? - { - result.document.name = name.into(); - photocraft_raster::spill::check_since(before).map_err(IoError::Unsupported)?; - ctl.check().map_err(|_| IoError::Cancelled)?; - return Ok(result); - } - source.seek(SeekFrom::Start(0)).map_err(|e| IoError::Unsupported(e.to_string()))?; - let _working = photocraft_raster::memory::try_reserve_operation(size).map_err(IoError::Unsupported)?; - let bytes = photocraft_format::read::read_all(&mut source, size).map_err(|e| IoError::Unsupported(e.to_string()))?; - import_with(name, &bytes, ctl) -} - /// [`import`] for a background open: checks `ctl` between stages (and per layer for PSD/PSB) and /// reports progress. A cancelled import fails with [`IoError::Cancelled`]. pub fn import_with(name: &str, bytes: &[u8], ctl: &photocraft_raster::Interrupt) -> Result { @@ -477,17 +359,6 @@ pub fn export(doc: &Document, name_or_ext: &str, opts: &ExportOptions) -> Result /// export fails with [`IoError::Cancelled`]; otherwise the bytes are exactly [`export`]'s. pub fn export_with(doc: &Document, name_or_ext: &str, opts: &ExportOptions, ctl: &photocraft_raster::Interrupt) -> Result { ctl.check().map_err(|_| IoError::Cancelled)?; - // The byte API has to retain its complete output. Native large saves use the seekable - // path below; reject an inadmissible legacy working set before building channel arrays. - let tiles: u64 = doc - .walk() - .iter() - .filter_map(|(_, _, l)| l.surface()) - .map(|s| (s.tile_count() as u64).saturating_mul(256 * 256).saturating_mul(s.format().bytes_per_pixel() as u64)) - .sum(); - let estimate = doc.size.area().saturating_mul(32).saturating_add(tiles.saturating_mul(3)); - let _working = photocraft_raster::memory::try_reserve_operation(estimate) - .map_err(|_| IoError::Unsupported("export exceeds the working-memory budget; use native PSB streaming or export a smaller region".into()))?; photocraft_compose::discard_unreadable_pixel_caches(); photocraft_raster::spill::check_integrity().map_err(IoError::Unsupported)?; let scratch_generation = photocraft_raster::spill::read_error_generation(); @@ -498,42 +369,6 @@ pub fn export_with(doc: &Document, name_or_ext: &str, opts: &ExportOptions, ctl: Ok(r) } -/// Native atomic save. PSB writes metadata and bounded original-pixel bands directly to disk; -/// formats with only a bytes encoder use the checked working-set admission in [`export`]. -#[cfg(not(target_arch = "wasm32"))] -pub fn export_path_with(doc: &Document, path: &std::path::Path, opts: &ExportOptions, ctl: &photocraft_raster::Interrupt<'_>) -> Result, IoError> { - ctl.check().map_err(|_| IoError::Cancelled)?; - let ext = path.extension().and_then(|s| s.to_str()).unwrap_or("").to_ascii_lowercase(); - if ext == "psb" || ext == "psd" && (opts.force_psb || doc.size.width > 30_000 || doc.size.height > 30_000) { - photocraft_raster::spill::check_integrity().map_err(IoError::Unsupported)?; - let mut warnings = Vec::new(); - let result = photocraft_format::atomic::atomic_write_stream_with_commit( - path, - |out| { - warnings = psd_stream::export_into(doc, out, *ctl).map_err(|e| std::io::Error::other(e.to_string()))?; - Ok(()) - }, - || ctl.check().map_err(|e| std::io::Error::new(std::io::ErrorKind::Interrupted, e)), - ); - if result.is_err() && ctl.cancelled() { - return Err(IoError::Cancelled); - } - result.map_err(|e| IoError::Unsupported(e.to_string()))?; - return Ok(warnings); - } - let result = export(doc, &path.to_string_lossy(), opts)?; - ctl.check().map_err(|_| IoError::Cancelled)?; - photocraft_format::atomic_write(path, &result.bytes).map_err(|e| IoError::Unsupported(e.to_string()))?; - Ok(result.warnings) -} - -/// Write a native PSB to an already authorized seekable output. The caller owns atomic -/// publication and cleanup; this never opens an ambient path. -#[cfg(not(target_arch = "wasm32"))] -pub fn export_psb_into(doc: &Document, out: &mut W, ctl: &photocraft_raster::Interrupt<'_>) -> Result, IoError> { - psd_stream::export_into(doc, out, *ctl) -} - /// Run `f` with an interrupt whose progress `0..=1` fills `lo..=hi` of `ctl`'s (same cancellation). pub(crate) fn with_span(ctl: &photocraft_raster::Interrupt, lo: f32, hi: f32, f: impl FnOnce(&photocraft_raster::Interrupt) -> R) -> R { let cancelled = || ctl.cancelled(); diff --git a/crates/io/src/psd_export.rs b/crates/io/src/psd_export.rs index e2117b237e..9db3d4799f 100644 --- a/crates/io/src/psd_export.rs +++ b/crates/io/src/psd_export.rs @@ -81,8 +81,6 @@ struct Ex { patterns: Vec, /// The layer written as Photoshop's Background (see [`psd_background`]). background: Option, - deferred: bool, - sources: StreamSources, } /// The layer Photoshop should open as its Background: the bottom top-level layer when it is a @@ -105,14 +103,10 @@ fn psd_background(doc: &Document) -> Option { return None; } let n = s.channels(); - let opaque = canvas - .tiles() - .all(|tc| s.try_read_region(tc.rect().intersect(&canvas)).is_ok_and(|pixels| pixels.chunks_exact(n).all(|p| p.last().is_some_and(|a| *a >= 1.0)))); + let opaque = s.read_region(canvas).chunks_exact(n).all(|p| p.last().is_some_and(|a| *a >= 1.0)); opaque.then_some(l.id) } -pub(crate) type StreamSources = std::collections::HashMap, Option)>; - /// How deep embedded documents are exported inside each other before a smart object is written /// as pixels instead. const MAX_NESTING: u32 = 8; @@ -342,13 +336,10 @@ impl Ex { } fn empty_channels(&self) -> Vec { - if self.deferred { - return (-1..self.cc as i16).map(|id| ChannelData { id, compression: Some(Compression::Raw), data: Vec::new() }).collect(); - } (-1..self.cc as i16).map(|id| self.encode(id, &[], 0, 0)).collect() } - fn pixel_channels(&mut self, s: &Surface, name: &str) -> (PsdRect, Vec, Vec, Option) { + fn pixel_channels(&self, s: &Surface, name: &str) -> (PsdRect, Vec, Vec) { let mut warnings = Vec::new(); let s = if s.format() != self.fmt { warnings.push(format!("layer \"{name}\": pixels converted from {:?} to {:?}", s.format(), self.fmt)); @@ -358,11 +349,7 @@ impl Ex { }; let r = s.content_bounds(); if r.is_empty() { - return (PsdRect::default(), self.empty_channels(), warnings, None); - } - if self.deferred { - let channels = (-1..self.cc as i16).map(|id| ChannelData { id, compression: Some(Compression::Raw), data: Vec::new() }).collect(); - return (to_psd_rect(r), channels, warnings, Some(s)); + return (PsdRect::default(), self.empty_channels(), warnings); } // The interleaved gather is the sequential pass for full-canvas layers, so it runs // in row bands on all cores and the band results concatenate in order. @@ -386,15 +373,15 @@ impl Ex { let order: Vec<(i16, &Vec)> = std::iter::once((-1, &planes[self.cc])).chain(planes.iter().take(self.cc).enumerate().map(|(c, p)| (c as i16, p))).collect(); let ch = crate::pixels::par_map(order, |(id, plane)| self.encode(id, plane, w, h)); - (to_psd_rect(r), ch, warnings, None) + (to_psd_rect(r), ch, warnings) } - fn mask(&self, m: &LayerMask, vector: Option<(f32, f32)>) -> (MaskData, Option, Option) { + fn mask(&self, m: &LayerMask, vector: Option<(f32, f32)>) -> (MaskData, Option) { let s = if m.surface.format() != self.mask_fmt { m.surface.convert(self.mask_fmt) } else { m.surface.clone() }; let default = s.default_pixel().first().copied().unwrap_or(0.0); let r = s.content_bounds(); let (w, h) = (r.width() as usize, r.height() as usize); - let plane = if self.deferred || r.is_empty() { Vec::new() } else { deinterleave(&s.to_interleaved(r), 1, self.mask_fmt.sample, &[false]).remove(0) }; + let plane = if r.is_empty() { Vec::new() } else { deinterleave(&s.to_interleaved(r), 1, self.mask_fmt.sample, &[false]).remove(0) }; let mut flags = 0u8; if !m.linked { flags |= PsdMask::FLAG_RELATIVE; @@ -415,11 +402,7 @@ impl Ex { real: None, real_first: true, }; - if self.deferred { - (MaskData::Mask(pm), Some(ChannelData { id: -2, compression: Some(Compression::Raw), data: Vec::new() }), Some(s)) - } else { - (MaskData::Mask(pm), Some(self.encode(-2, &plane, w, h)), None) - } + (MaskData::Mask(pm), Some(self.encode(-2, &plane, w, h))) } /// Prefer numbers above imported layer IDs, wrapping safely at u32::MAX. @@ -441,18 +424,16 @@ impl Ex { fn record(&mut self, l: &Layer, extra: Vec, pixels: Option<&Surface>) -> LayerRecord { let id = self.layer_id(l); - let (rect, mut channels, warnings, source) = match pixels { + let (rect, mut channels, warnings) = match pixels { Some(s) => self.pixel_channels(s, &l.name), - None => (PsdRect::default(), self.empty_channels(), Vec::new(), None), + None => (PsdRect::default(), self.empty_channels(), Vec::new()), }; self.warnings.extend(warnings); let mut mask = MaskData::None; - let mut mask_source = None; // Vector mask density/feather live in the mask parameters, with or without a user mask. let vector = l.vector_mask.as_ref().map(|vm| (vm.density, vm.feather)); if let Some(m) = &l.mask { - let (md, ch, s) = self.mask(m, vector); - mask_source = s; + let (md, ch) = self.mask(m, vector); mask = md; channels.extend(ch); } else if let Some(parameters) = mask_parameters(None, vector) { @@ -467,9 +448,6 @@ impl Ex { real_first: true, }); } - if self.deferred { - self.sources.insert(id, (source, mask_source)); - } // A Background carries no transparency channel: that is how Photoshop tells it apart. if self.background == Some(l.id) { channels.retain(|c| c.id != -1); @@ -1000,7 +978,7 @@ impl Ex { } let doc = photocraft_format::load_from_bytes(bytes).map_err(|e| format!("its contents can't be read: {e}"))?; let (file, warnings) = - document_to_psd_nested(&doc, &PsdExportOptions { force_psb: true, ..Default::default() }, self.smart.depth + 1, &Interrupt::NONE, None); + document_to_psd_nested(&doc, &PsdExportOptions { force_psb: true, ..Default::default() }, self.smart.depth + 1, &Interrupt::NONE); let data = file.to_bytes().map_err(|e| format!("its contents can't be written: {e}"))?; let stem = file_name.rsplit_once('.').map_or(file_name, |(a, _)| a); self.warnings.extend(warnings.into_iter().map(|w| format!("smart object {stem}: {w}"))); @@ -1365,21 +1343,14 @@ pub fn document_to_psd(doc: &Document) -> PsdFile { /// anything that could not be represented. The merged composite is rendered /// with `photocraft_compose::flatten`. pub fn document_to_psd_with(doc: &Document, opts: &PsdExportOptions) -> (PsdFile, Vec) { - document_to_psd_nested(doc, opts, 0, &Interrupt::NONE, None) -} - -#[cfg(not(target_arch = "wasm32"))] -pub(crate) fn stream_metadata(doc: &Document, opts: &PsdExportOptions) -> (PsdFile, Vec, StreamSources) { - let mut sources = StreamSources::new(); - let (file, warnings) = document_to_psd_nested(doc, opts, 0, &Interrupt::NONE, Some(&mut sources)); - (file, warnings, sources) + document_to_psd_nested(doc, opts, 0, &Interrupt::NONE) } /// [`document_to_psd_with`] for a background save (#3389): reports progress through `ctl` after /// each layer record and each band of the merged composite. Once `ctl` is cancelled it stops /// early and returns an **incomplete** model: callers check `ctl` and discard it. pub(crate) fn document_to_psd_interruptible(doc: &Document, opts: &PsdExportOptions, ctl: &Interrupt) -> (PsdFile, Vec) { - document_to_psd_nested(doc, opts, 0, ctl, None) + document_to_psd_nested(doc, opts, 0, ctl) } /// How much of a PSD conversion the merged composite weighs, in layer records: rendering it @@ -1509,13 +1480,7 @@ pub(crate) fn psd_size_exceeds_limit(doc: &Document) -> bool { } /// [`document_to_psd_with`] for a document embedded `depth` smart objects deep. -fn document_to_psd_nested( - doc: &Document, - opts: &PsdExportOptions, - depth: u32, - ctl: &Interrupt, - deferred: Option<&mut StreamSources>, -) -> (PsdFile, Vec) { +fn document_to_psd_nested(doc: &Document, opts: &PsdExportOptions, depth: u32, ctl: &Interrupt) -> (PsdFile, Vec) { if doc.mode == ColorMode::Multichannel { return crate::multichannel_map::document_to_psd(doc, opts.force_psb); } @@ -1545,8 +1510,6 @@ fn document_to_psd_nested( smart: SmartOut::new(doc, depth), patterns: doc.patterns.clone(), background: psd_background(doc), - deferred: deferred.is_some(), - sources: Default::default(), }; if big && !opts.force_psb { ex.warnings.push("document exceeds 30000 px; written as PSB".into()); @@ -1621,9 +1584,7 @@ fn document_to_psd_nested( // isn't rendered at all. A document without layer records keeps the real image: its pixels // live nowhere else. let real_merged = opts.maximize_compatibility || ex.records.is_empty(); - let (mut planes, has_alpha) = if ex.deferred { - (Vec::new(), true) - } else if real_merged { + let (mut planes, has_alpha) = if real_merged { // Merged composite, rendered and encoded in bands (no full-size float composite). Matting // against white only changes pixels with alpha < 1; if some are slightly translucent but // all round to opaque (so no alpha channel is written), encode once more without the matte. @@ -1661,9 +1622,6 @@ fn document_to_psd_nested( None => None, }; for a in &extra { - if ex.deferred { - continue; - } let s = if a.surface.format() != ex.mask_fmt { a.surface.convert(ex.mask_fmt) } else { a.surface.clone() }; let bytes = s.to_interleaved(canvas); planes.extend(deinterleave(&bytes, 1, sample, &[false]).remove(0)); @@ -1671,13 +1629,9 @@ fn document_to_psd_nested( let channels = (cc + usize::from(has_alpha) + extra.len()) as u16; let header = Header::new(version, doc.size.width, doc.size.height, channels, psd_depth(sample), psd_mode(fmt.mode)); let mcomp = if sample == SampleType::F32 { Compression::Raw } else { opts.merged_compression }; - let image_data = if ex.deferred { - ImageData { compression: Compression::Raw, data: Vec::new() } - } else { - ImageData::encode(mcomp, &planes, &header) - .or_else(|_| ImageData::encode(Compression::Raw, &planes, &header)) - .unwrap_or(ImageData { compression: Compression::Raw, data: planes }) - }; + let image_data = ImageData::encode(mcomp, &planes, &header) + .or_else(|_| ImageData::encode(Compression::Raw, &planes, &header)) + .unwrap_or(ImageData { compression: Compression::Raw, data: planes }); ctl.progress(1.0); // Resources. @@ -1823,9 +1777,6 @@ fn document_to_psd_nested( layer_mask_trailing: Vec::new(), image_data, }; - if let Some(out) = deferred { - *out = ex.sources; - } (file, ex.warnings) } diff --git a/crates/io/src/psd_import.rs b/crates/io/src/psd_import.rs index 82814f3762..444c2d6e5b 100644 --- a/crates/io/src/psd_import.rs +++ b/crates/io/src/psd_import.rs @@ -66,7 +66,6 @@ pub(crate) struct Ctx<'a> { /// Layer records decoded so far, out of `total` (progress). pub done: usize, pub total: usize, - pub streamed: Option<&'a std::collections::HashMap<(usize, i16), Surface>>, } fn doc_mode(m: PsdMode) -> Option { @@ -172,9 +171,6 @@ impl Ctx<'_> { } fn record_surface(&mut self, rec: &LayerRecord, name: &str) -> Surface { - if let Some(streamed) = self.streamed { - return streamed.get(&(rec as *const LayerRecord as usize, i16::MIN)).cloned().unwrap_or_else(|| Surface::new(self.fmt)); - } let r = rec.rect; if r.is_empty() || r.size().is_err() { return Surface::new(self.fmt); @@ -235,11 +231,7 @@ impl Ctx<'_> { } }; let mut surface = Surface::with_default(self.mask_fmt, &[f32::from(default) / 255.0]); - if let Some(streamed) = self.streamed { - if let Some(decoded) = streamed.get(&(rec as *const LayerRecord as usize, id)) { - surface = decoded.clone(); - } - } else if !rect.is_empty() + if !rect.is_empty() && let Ok((w, h)) = rect.size() && let Some(plane) = self.channel_plane(rec, id, name) { @@ -665,14 +657,6 @@ pub fn psd_to_document(file: &PsdFile) -> (Document, Vec) { /// [`psd_to_document`] for a background open: checks `ctl` per layer record and reports /// progress. `None` when cancelled. pub fn psd_to_document_with(file: &PsdFile, ctl: &photocraft_raster::Interrupt) -> Option<(Document, Vec)> { - psd_to_document_decoded(file, ctl, None) -} - -pub(crate) fn psd_to_document_decoded<'a>( - file: &'a PsdFile, - ctl: &photocraft_raster::Interrupt<'a>, - streamed: Option<&'a std::collections::HashMap<(usize, i16), Surface>>, -) -> Option<(Document, Vec)> { let h = &file.header; let mut warnings = Vec::new(); let mode = doc_mode(h.color_mode).unwrap_or_else(|| { @@ -770,7 +754,6 @@ pub(crate) fn psd_to_document_decoded<'a>( ctl: *ctl, done: 0, total: file.layers().len(), - streamed, }; for b in file.global_blocks.iter().filter(|b| matches!(&b.key, b"FEid" | b"FXid")) { // Smart-filter caches can be large: a cancelled open stops between blocks. diff --git a/crates/io/src/psd_stream.rs b/crates/io/src/psd_stream.rs deleted file mode 100644 index c11a9e593b..0000000000 --- a/crates/io/src/psd_stream.rs +++ /dev/null @@ -1,395 +0,0 @@ -//! Native import without a whole PSB byte buffer or whole decoded channel buffers. -use crate::{ - ImportResult, IoError, - pixels::{interleave, max_sample, zero_sample}, -}; -use photocraft_color::{ColorMode, PixelFormat, SampleType}; -use photocraft_geom::Rect; -use photocraft_psd::{ - LayerRecord, - stream::{Index, Rows}, -}; -use photocraft_raster::{Interrupt, Surface}; -use std::{ - collections::HashMap, - io::{BufReader, Read, Seek, SeekFrom, Write}, -}; - -pub(crate) fn eligible(index: &Index) -> bool { - let file = &index.metadata; - matches!( - file.header.color_mode, - photocraft_psd::ColorMode::Rgb | photocraft_psd::ColorMode::Grayscale | photocraft_psd::ColorMode::Cmyk | photocraft_psd::ColorMode::Lab - ) && matches!(file.header.depth, 8 | 16 | 32) - && !file.layers().is_empty() - && usize::from(file.header.channels) <= usize::from(file.header.color_mode.color_channels().unwrap_or(0)) + usize::from(file.merged_has_alpha()) -} - -struct Decode<'a, R> { - open: &'a dyn Fn() -> std::io::Result, - index: &'a Index, - ctl: Interrupt<'a>, - done: u64, - total: u64, - scratch_expected: bool, -} - -fn tile_footprint(r: photocraft_psd::Rect, bytes_per_pixel: usize) -> u64 { - if r.is_empty() { - return 0; - } - let side = i64::from(photocraft_geom::TILE_SIZE); - let columns = ((i64::from(r.right) - 1).div_euclid(side) - i64::from(r.left).div_euclid(side) + 1) as u64; - let rows = ((i64::from(r.bottom) - 1).div_euclid(side) - i64::from(r.top).div_euclid(side) + 1) as u64; - columns.saturating_mul(rows).saturating_mul(side as u64 * side as u64).saturating_mul(bytes_per_pixel as u64) -} - -pub(super) fn ram_tile_capacity(scratch: bool, resident: u64, budget: u64, growth: u64) -> Result<(), IoError> { - if !scratch && resident.checked_add(growth).is_none_or(|required| required > budget) { - return Err(IoError::Unsupported("decoded image tiles exceed the available RAM budget; enable a scratch disk or close other documents".into())); - } - Ok(()) -} - -pub(super) fn require_scratch(expected: bool) -> Result<(), IoError> { - if expected && !photocraft_raster::spill::enabled() { - return Err(IoError::Unsupported( - photocraft_raster::spill::last_error().unwrap_or_else(|| "scratch storage was disabled while loading; retry the import".into()), - )); - } - Ok(()) -} - -impl Decode<'_, R> { - fn surface(&mut self, layer: usize, rec: &LayerRecord, ids: &[i16], fmt: PixelFormat, default: &[f32]) -> Result { - let r = if ids.len() == 1 { rec.channel_rect(*ids.first().ok_or_else(|| IoError::Unsupported("empty channel set".into()))?) } else { rec.rect }; - if r.is_empty() { - return Ok(Surface::with_default(fmt, default)); - } - let (width, height) = r.size()?; - let working = - (width as u64).saturating_mul(64).saturating_mul(fmt.bytes_per_pixel() as u64).saturating_mul(3).saturating_add(ids.len() as u64 * (3 << 20)); - let _working = photocraft_raster::memory::try_reserve(working).map_err(IoError::Unsupported)?; - let ranges = self.index.channels.get(layer).ok_or_else(|| IoError::Unsupported("missing indexed layer".into()))?; - let mut readers = Vec::new(); - for id in ids { - readers.push(if let Some(range) = ranges.iter().find(|c| c.id == *id) { - Some(Rows::new( - BufReader::with_capacity(1 << 20, (self.open)().map_err(|e| IoError::Unsupported(e.to_string()))?), - *range, - width, - height, - self.index.metadata.header.depth, - self.index.metadata.header.version, - )?) - } else { - None - }); - } - let mut out = Surface::with_default(fmt, default); - let fill: Vec> = ids.iter().map(|id| if *id == -1 { max_sample(fmt.sample) } else { zero_sample(fmt.sample) }).collect(); - let invert: Vec = ids.iter().map(|id| fmt.mode == ColorMode::Cmyk && *id >= 0).collect(); - let mut planes: Vec> = ids.iter().map(|_| Vec::new()).collect(); - let mut row = Vec::new(); - let mut y = 0usize; - while y < height { - self.ctl.check().map_err(|_| IoError::Cancelled)?; - let rows = (height - y).min(64); - for (reader, plane) in readers.iter_mut().zip(&mut planes) { - plane.clear(); - if let Some(reader) = reader { - for _ in 0..rows { - self.ctl.check().map_err(|_| IoError::Cancelled)?; - reader.read_row(&mut row)?; - plane.extend_from_slice(&row); - } - } - } - let refs: Vec> = readers.iter().zip(&planes).map(|(r, p)| r.as_ref().map(|_| p.as_slice())).collect(); - let mut bytes = interleave(&refs, &fill, width * rows, fmt.sample, &invert); - if fmt.mode == ColorMode::Lab && fmt.sample == SampleType::U16 { - crate::pixels::lab16_chroma(&mut bytes, ids.len(), true); - } - let top = i32::try_from(i64::from(r.top) + y as i64).map_err(|_| IoError::Unsupported("layer coordinates overflow".into()))?; - let bottom = top.checked_add(rows as i32).ok_or_else(|| IoError::Unsupported("layer coordinates overflow".into()))?; - let band = Rect::new(r.left, top, r.right, bottom); - let scratch = photocraft_raster::spill::enabled(); - require_scratch(self.scratch_expected)?; - if !scratch { - let missing = band.tiles().filter(|c| out.tile(*c).is_none()).count() as u64; - let growth = missing.saturating_mul((photocraft_geom::TILE_SIZE as u64).pow(2)).saturating_mul(fmt.bytes_per_pixel() as u64); - ram_tile_capacity(false, photocraft_raster::spill::stats().resident_bytes as u64, photocraft_raster::memory::stats().1.tile_bytes, growth)?; - } - out.write_interleaved(band, &bytes); - // A final band can trigger a scratch write failure too. - require_scratch(self.scratch_expected)?; - y += rows; - self.done = self.done.saturating_add(rows as u64 * ids.len() as u64); - self.ctl.progress(0.05 + 0.85 * self.done as f32 / self.total.max(1) as f32); - } - for reader in readers.into_iter().flatten() { - reader.finish()?; - } - out.prune(); - require_scratch(self.scratch_expected)?; - Ok(out) - } -} - -pub(crate) fn import(open: &dyn Fn() -> std::io::Result, index: &Index, ctl: Interrupt<'_>) -> Result { - let file = &index.metadata; - let mode = match file.header.color_mode { - photocraft_psd::ColorMode::Rgb => ColorMode::Rgb, - photocraft_psd::ColorMode::Grayscale => ColorMode::Grayscale, - photocraft_psd::ColorMode::Cmyk => ColorMode::Cmyk, - photocraft_psd::ColorMode::Lab => ColorMode::Lab, - _ => return Err(IoError::Unsupported("stream color mode".into())), - }; - let sample = crate::pixels::sample_for_depth(file.header.depth); - let fmt = PixelFormat::new(mode, sample, true); - let scratch_expected = photocraft_raster::spill::enabled(); - if !scratch_expected { - let budget = photocraft_raster::memory::refresh(); - let required = file.layers().iter().fold(0_u64, |sum, rec| { - let mut bytes = tile_footprint(rec.rect, fmt.bytes_per_pixel()); - for id in [-2, -3] { - if rec.channel(id).is_some() { - bytes = bytes.saturating_add(tile_footprint(rec.channel_rect(id), sample.bytes())); - } - } - sum.saturating_add(bytes) - }); - ram_tile_capacity(false, photocraft_raster::spill::stats().resident_bytes as u64, budget.tile_bytes, required)?; - } - let total = file.layers().iter().map(|r| r.channel_rect(0).size().map_or(0, |(_, h)| h as u64 * (mode.color_channels() as u64 + 1))).sum(); - let mut decoder = Decode { open, index, ctl, done: 0, total, scratch_expected }; - let mut surfaces = HashMap::new(); - for (i, rec) in file.layers().iter().enumerate() { - ctl.check().map_err(|_| IoError::Cancelled)?; - let key = rec as *const LayerRecord as usize; - let ids: Vec = (0..mode.color_channels() as i16).chain([-1]).collect(); - surfaces.insert((key, i16::MIN), decoder.surface(i, rec, &ids, fmt, &vec![0.0; fmt.channels()])?); - for id in [-2, -3] { - if rec.channel(id).is_some() { - let default = rec.layer_mask().map_or(0, |m| if id == -3 { m.real.map_or(m.default_color, |m| m.background) } else { m.default_color }); - let fmt = PixelFormat::new(ColorMode::Grayscale, sample, false); - surfaces.insert((key, id), decoder.surface(i, rec, &[id], fmt, &[f32::from(default) / 255.0])?); - } - } - } - let (document, warnings) = crate::psd_import::psd_to_document_decoded(file, &ctl, Some(&surfaces)).ok_or(IoError::Cancelled)?; - Ok(ImportResult { document, warnings, source_read_only: false, preview_only: false }) -} - -fn sink_error(e: std::io::Error) -> photocraft_psd::PsdError { - photocraft_psd::PsdError::Invalid(format!("stream output: {e}")) -} - -pub(crate) fn export_into(doc: &photocraft_doc::Document, out: &mut W, ctl: Interrupt<'_>) -> Result, IoError> { - if !matches!(doc.mode, ColorMode::Rgb | ColorMode::Grayscale | ColorMode::Cmyk | ColorMode::Lab) { - return Err(IoError::Unsupported("seekable PSB export requires RGB, Gray, CMYK or Lab".into())); - } - ctl.check().map_err(|_| IoError::Cancelled)?; - let before = photocraft_raster::spill::read_error_generation(); - let fmt = doc.pixel_format(); - let metadata: u64 = doc - .metadata - .psd_resources - .iter() - .map(|(_, _, bytes)| bytes.len() as u64) - .chain(doc.metadata.psd_global_blocks.iter().map(|(_, _, bytes)| bytes.len() as u64)) - .chain(doc.walk().iter().flat_map(|(_, _, l)| l.psd_blocks.iter().map(|(_, b)| b.len() as u64))) - .fold(0_u64, u64::saturating_add); - let embedded = doc - .walk() - .iter() - .filter_map(|(_, _, l)| match &l.content { - photocraft_doc::LayerContent::Smart(sm) => match &sm.source { - photocraft_doc::SmartSource::Embedded { bytes, .. } => Some(bytes.len() as u64), - _ => None, - }, - _ => None, - }) - .fold(0_u64, u64::saturating_add); - let working = u64::from(doc.size.width) - .saturating_mul(64) - .saturating_mul(128) - .saturating_add(64 << 20) - .saturating_add(metadata.saturating_mul(4)) - .saturating_add(embedded.saturating_mul(4)); - let _working = photocraft_raster::memory::try_reserve(working).map_err(IoError::Unsupported)?; - let opts = crate::psd_export::PsdExportOptions { force_psb: true, ..Default::default() }; - let (file, warnings, sources) = crate::psd_export::stream_metadata(doc, &opts); - let check = || -> photocraft_psd::Result<()> { - if ctl.cancelled() { - return Err(photocraft_psd::PsdError::Invalid("stream export cancelled".into())); - } - photocraft_raster::spill::check_since(before).map_err(photocraft_psd::PsdError::Invalid) - }; - let result = photocraft_psd::stream::write_layered( - &file, - out, - |i, id, out| { - check()?; - let rec = file.layers().get(i).ok_or_else(|| photocraft_psd::PsdError::Invalid("missing output record".into()))?; - let source = rec.layer_id().and_then(|id| sources.get(&id)).and_then(|(pixels, mask)| if id == -2 { mask.as_ref() } else { pixels.as_ref() }); - let r = rec.channel_rect(id); - if r.is_empty() { - return Ok(()); - } - let s = source.ok_or_else(|| photocraft_psd::PsdError::Invalid("missing output pixel source".into()))?; - let r = Rect::new(r.left, r.top, r.right, r.bottom); - let channels = s.format().channels(); - let channel = if id == -1 { channels.saturating_sub(1) } else { id.max(0) as usize }; - let mut invert = vec![s.format().mode == ColorMode::Cmyk; channels]; - if (id < 0 || s.format().alpha) - && let Some(a) = invert.last_mut() - { - *a = false; - } - let mut y = r.y0; - while y < r.y1 { - check()?; - let band = Rect::new(r.x0, y, r.x1, y.saturating_add(64).min(r.y1)); - let mut bytes = s.to_interleaved(band); - if s.format().mode == ColorMode::Lab && s.format().sample == SampleType::U16 { - crate::pixels::lab16_chroma(&mut bytes, channels, false); - } - let planes = crate::pixels::deinterleave(&bytes, channels, s.format().sample, &invert); - out.write_all(planes.get(channel).ok_or_else(|| photocraft_psd::PsdError::Invalid("invalid output channel".into()))?).map_err(sink_error)?; - y = band.y1; - } - Ok(()) - }, - |out| { - check()?; - let start = out.stream_position().map_err(sink_error)?; - let row_bytes = u64::from(doc.size.width) * fmt.sample.bytes() as u64; - let plane_bytes = - row_bytes.checked_mul(u64::from(doc.size.height)).ok_or_else(|| photocraft_psd::PsdError::Invalid("output size overflow".into()))?; - let cc = fmt.mode.color_channels(); - let white = photocraft_raster::from_rgba(&fmt, [1.0; 4]); - let space = photocraft_compose::cmyk_space(doc); - photocraft_compose::render_bands(doc, doc.bounds(), 64, |band| -> photocraft_psd::Result<()> { - check()?; - let mut planes: Vec> = (0..=cc).map(|_| Vec::with_capacity(band.px.len() * fmt.sample.bytes())).collect(); - photocraft_color::convert::with_cmyk_space(space.as_ref(), || { - let mut v = [0.0; 5]; - for p in &band.px { - photocraft_raster::from_rgba_into(&fmt, *p, &mut v); - for (c, plane) in planes.iter_mut().enumerate() { - let value = v.get(c).copied().unwrap_or(0.0); - let value = if c < cc { - crate::pixels::matte(value, v.get(cc).copied().unwrap_or(0.0), white.get(c).copied().unwrap_or(1.0)) - } else { - value - }; - let value = if fmt.mode == ColorMode::Cmyk && c < cc { 1.0 - value } else { value }; - let value = if fmt.mode == ColorMode::Lab && fmt.sample == SampleType::U16 && matches!(c, 1 | 2) { - value * crate::pixels::LAB16_CHROMA_MAX / 65535.0 - } else { - value - }; - crate::pixels::encode_be(value, fmt.sample, plane); - } - } - }); - for (c, bytes) in planes.iter().enumerate() { - let offset = start - .checked_add(c as u64 * plane_bytes) - .and_then(|v| v.checked_add(band.rect.y0 as u64 * row_bytes)) - .ok_or_else(|| photocraft_psd::PsdError::Invalid("output offset overflow".into()))?; - out.seek(SeekFrom::Start(offset)).map_err(sink_error)?; - out.write_all(bytes).map_err(sink_error)?; - } - ctl.progress(0.5 + 0.5 * band.rect.y1 as f32 / doc.size.height.max(1) as f32); - Ok(()) - })?; - for (i, channel) in doc.channels.iter().chain(doc.quick_mask.iter()).take(usize::from(file.header.channels).saturating_sub(cc + 1)).enumerate() { - let index = cc + 1 + i; - let mask_fmt = photocraft_color::PixelFormat::new(ColorMode::Grayscale, fmt.sample, false); - let converted = (channel.surface.format() != mask_fmt).then(|| channel.surface.convert(mask_fmt)); - let surface = converted.as_ref().unwrap_or(&channel.surface); - let mut y = 0; - while y < doc.size.height { - check()?; - let end = y.saturating_add(64).min(doc.size.height); - let r = Rect::from_xywh(0, y as i32, doc.size.width, end - y); - let bytes = surface.to_interleaved(r); - let plane = crate::pixels::deinterleave(&bytes, 1, surface.format().sample, &[false]); - out.seek(SeekFrom::Start(start + index as u64 * plane_bytes + u64::from(y) * row_bytes)).map_err(sink_error)?; - out.write_all(plane.first().ok_or_else(|| photocraft_psd::PsdError::Invalid("missing extra output plane".into()))?).map_err(sink_error)?; - y = end; - } - } - let end = start - .checked_add(u64::from(file.header.channels).saturating_mul(plane_bytes)) - .ok_or_else(|| photocraft_psd::PsdError::Invalid("output size overflow".into()))?; - out.seek(SeekFrom::Start(end)).map_err(sink_error)?; - check() - }, - ); - if ctl.cancelled() { - return Err(IoError::Cancelled); - } - result?; - photocraft_raster::spill::check_since(before).map_err(IoError::Unsupported)?; - Ok(warnings) -} - -#[cfg(test)] -mod memory_tests { - use super::*; - use photocraft_psd::{Compression, Version}; - use std::io::Cursor; - - #[test] - fn tile_footprint_counts_partial_negative_tiles_at_each_depth() { - let side = photocraft_geom::TILE_SIZE as u64; - for bpp in [1, 2, 4, 8, 16, 20] { - let rect = photocraft_psd::Rect::from_xywh(-1, -1, 2, 2); - assert_eq!(tile_footprint(rect, bpp), 4 * side * side * bpp as u64); - let aligned = photocraft_psd::Rect::from_xywh(-(side as i32), 0, side as u32, side as u32); - assert_eq!(tile_footprint(aligned, bpp), side * side * bpp as u64); - } - assert_eq!(tile_footprint(photocraft_psd::Rect::default(), 16), 0); - let hostile = photocraft_psd::Rect { top: i32::MIN, left: i32::MIN, bottom: i32::MAX, right: i32::MAX }; - assert_eq!(tile_footprint(hostile, 16), u64::MAX); - } - - #[test] - fn ram_only_admission_accounts_for_live_tiles_and_overflow() { - assert!(ram_tile_capacity(false, 60, 100, 40).is_ok()); - assert!(ram_tile_capacity(false, 60, 100, 41).is_err()); - assert!(ram_tile_capacity(false, u64::MAX, u64::MAX, 1).is_err()); - assert!(ram_tile_capacity(true, 60, 100, 1000).is_ok()); - } - - #[test] - fn oversized_ram_only_import_is_rejected_before_opening_channels() { - assert!(!photocraft_raster::spill::enabled()); - let file = photocraft_psd::testgen::layered(Version::Psb, photocraft_psd::ColorMode::Rgb, 32, Compression::Raw); - let bytes = file.to_bytes().unwrap(); - let mut index = Index::read(&mut Cursor::new(&bytes), 128 << 20, &|| false).unwrap(); - // Valid dimensions would need over a terabyte; the source remains a tiny fixture. - index.metadata.layers_mut()[0].rect = photocraft_psd::Rect::from_xywh(0, 0, 300_000, 300_000); - let opened = std::cell::Cell::new(0); - let result = import( - &|| { - opened.set(opened.get() + 1); - Ok(Cursor::new(&bytes)) - }, - &index, - Interrupt::NONE, - ); - assert!(matches!(result, Err(IoError::Unsupported(ref message)) if message.contains("enable a scratch disk"))); - assert_eq!(opened.get(), 0); - } - - #[test] - fn missing_scratch_is_not_silently_treated_as_ram_only() { - assert!(!photocraft_raster::spill::enabled()); - assert!(require_scratch(false).is_ok()); - assert!(require_scratch(true).is_err()); - } -} diff --git a/crates/io/tests/streaming_psd.rs b/crates/io/tests/streaming_psd.rs deleted file mode 100644 index 0a8038b3ac..0000000000 --- a/crates/io/tests/streaming_psd.rs +++ /dev/null @@ -1,136 +0,0 @@ -#![cfg(not(target_arch = "wasm32"))] -use photocraft_psd::{ColorMode, Compression, Version}; -use photocraft_raster::Interrupt; - -// Native imports and exports reserve the same process-wide working-memory allowance. -// Keep fidelity fixtures from rejecting each other solely because the test runner is parallel. -static WORKING_MEMORY_TESTS: std::sync::Mutex<()> = std::sync::Mutex::new(()); - -#[test] -fn seekable_import_matches_byte_import_pixels_masks_and_metadata() { - let _test = WORKING_MEMORY_TESTS.lock().unwrap(); - let path = std::env::temp_dir().join(format!("photocraft-stream-parity-{}.psb", std::process::id())); - for version in [Version::Psd, Version::Psb] { - for mode in [ColorMode::Rgb, ColorMode::Grayscale, ColorMode::Cmyk, ColorMode::Lab] { - for depth in [8, 16, 32] { - for compression in Compression::ALL { - let mut file = photocraft_psd::testgen::layered(version, mode, depth, compression); - // The streaming layer path need not consume the stored merged image. - file.header.channels = mode.color_channels().unwrap() + u16::from(file.merged_has_alpha()); - let bytes = file.to_bytes().unwrap(); - std::fs::write(&path, &bytes).unwrap(); - let a = photocraft_io::import("d", &bytes).unwrap(); - let b = photocraft_io::import_path_with("d", &path, &Interrupt::NONE).unwrap(); - let authorized = photocraft_io::import_seekable_with("d", &|| std::fs::File::open(&path), &Interrupt::NONE).unwrap(); - assert_eq!( - photocraft_compose::render(&authorized.document, authorized.document.bounds()).px, - photocraft_compose::render(&b.document, b.document.bounds()).px - ); - assert_eq!(a.warnings, b.warnings, "{version:?}/{mode:?}/{depth}/{compression:?}"); - let aa = photocraft_compose::render(&a.document, a.document.bounds()); - let bb = photocraft_compose::render(&b.document, b.document.bounds()); - assert_eq!(aa.px, bb.px, "{version:?}/{mode:?}/{depth}/{compression:?}"); - let options = photocraft_io::PsdExportOptions::default(); - let (a, _) = photocraft_io::document_to_psd_with(&a.document, &options); - let (b, _) = photocraft_io::document_to_psd_with(&b.document, &options); - assert_eq!(a, b, "layer, mask and unknown-block round trip: {version:?}/{mode:?}/{depth}/{compression:?}"); - } - } - } - } - std::fs::remove_file(path).unwrap(); -} - -#[test] -fn cancelled_and_truncated_imports_publish_no_document() { - let _test = WORKING_MEMORY_TESTS.lock().unwrap(); - let path = std::env::temp_dir().join(format!("photocraft-stream-cancel-{}.psb", std::process::id())); - let mut file = photocraft_psd::testgen::layered(Version::Psb, ColorMode::Rgb, 8, Compression::Zip); - file.header.channels = 3 + u16::from(file.merged_has_alpha()); - let bytes = file.to_bytes().unwrap(); - std::fs::write(&path, &bytes).unwrap(); - assert!(matches!(photocraft_io::import_path_with("d", &path, &Interrupt::cancel_only(&|| true)), Err(photocraft_io::IoError::Cancelled))); - std::fs::write(&path, &bytes[..bytes.len() / 2]).unwrap(); - assert!(photocraft_io::import_path_with("d", &path, &Interrupt::NONE).is_err()); - std::fs::remove_file(path).unwrap(); -} - -#[test] -fn seekable_export_preserves_merged_layers_masks_and_unknown_metadata() { - let _test = WORKING_MEMORY_TESTS.lock().unwrap(); - let path = std::env::temp_dir().join(format!("photocraft-export-parity-{}.psb", std::process::id())); - for mode in [ColorMode::Rgb, ColorMode::Grayscale, ColorMode::Cmyk, ColorMode::Lab] { - for depth in [8, 16, 32] { - let input = photocraft_psd::testgen::layered(Version::Psb, mode, depth, Compression::ZipPrediction); - let doc = photocraft_io::import("input", &input.to_bytes().unwrap()).unwrap().document; - let (reference, _) = photocraft_io::document_to_psd_with(&doc, &photocraft_io::PsdExportOptions { force_psb: true, ..Default::default() }); - photocraft_io::export_path_with(&doc, &path, &Default::default(), &Interrupt::NONE).unwrap(); - let output = photocraft_psd::PsdFile::from_bytes(&std::fs::read(&path).unwrap()).unwrap(); - assert_eq!(output.header, reference.header, "{mode:?}/{depth}"); - assert_eq!(output.resources, reference.resources); - let normalized_reference = photocraft_psd::PsdFile::from_bytes(&reference.to_bytes().unwrap()).unwrap(); - assert_eq!(output.global_blocks, normalized_reference.global_blocks); - assert_eq!(output.image_data.decode(&output.header).unwrap(), reference.image_data.decode(&reference.header).unwrap(), "merged {mode:?}/{depth}"); - let loaded = photocraft_io::import_path_with("output", &path, &Interrupt::NONE).unwrap(); - let (loaded, _) = photocraft_io::document_to_psd_with(&loaded.document, &photocraft_io::PsdExportOptions { force_psb: true, ..Default::default() }); - assert_eq!(loaded.to_bytes().unwrap(), reference.to_bytes().unwrap(), "layer/mask/metadata {mode:?}/{depth}"); - } - } - std::fs::remove_file(path).unwrap(); -} - -#[test] -fn cancelled_stream_save_keeps_destination_and_removes_temporary_output() { - let _test = WORKING_MEMORY_TESTS.lock().unwrap(); - let dir = std::env::temp_dir().join(format!("photocraft-export-cancel-{}", std::process::id())); - std::fs::create_dir_all(&dir).unwrap(); - let path = dir.join("kept.psb"); - std::fs::write(&path, b"precious original").unwrap(); - let input = photocraft_psd::testgen::layered(Version::Psb, ColorMode::Rgb, 8, Compression::Raw); - let doc = photocraft_io::import("input", &input.to_bytes().unwrap()).unwrap().document; - let calls = std::sync::atomic::AtomicUsize::new(0); - let cancel = || calls.fetch_add(1, std::sync::atomic::Ordering::Relaxed) > 5; - assert!(matches!( - photocraft_io::export_path_with(&doc, &path, &Default::default(), &Interrupt::cancel_only(&cancel)), - Err(photocraft_io::IoError::Cancelled) - )); - assert_eq!(std::fs::read(&path).unwrap(), b"precious original"); - assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 1); - std::fs::remove_dir_all(dir).unwrap(); -} - -#[cfg(feature = "corpus")] -#[test] -fn native_path_import_matches_byte_import_across_real_psd_corpus() { - let _test = WORKING_MEMORY_TESTS.lock().unwrap(); - fn collect(p: &std::path::Path, out: &mut Vec) { - for e in std::fs::read_dir(p).unwrap().flatten() { - let p = e.path(); - if p.is_dir() { - collect(&p, out); - } else if p.extension().is_some_and(|x| x.eq_ignore_ascii_case("psd") || x.eq_ignore_ascii_case("psb")) { - out.push(p); - } - } - } - let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../corpus/psd"); - assert!(root.is_dir(), "run cargo xtask corpus --all"); - let mut files = Vec::new(); - collect(&root, &mut files); - let mut compared = 0; - for path in files { - let bytes = std::fs::read(&path).unwrap(); - let Ok(a) = photocraft_io::import("d", &bytes) else { continue }; - let b = photocraft_io::import_path_with("d", &path, &Interrupt::NONE).unwrap_or_else(|e| panic!("{}: {e}", path.display())); - assert_eq!(a.warnings, b.warnings, "{}", path.display()); - assert_eq!( - photocraft_compose::render(&a.document, a.document.bounds()).px, - photocraft_compose::render(&b.document, b.document.bounds()).px, - "{}", - path.display() - ); - compared += 1; - } - assert!(compared >= 20, "{compared} corpus files compared"); - eprintln!("seekable corpus parity: {compared} files"); -} diff --git a/crates/io/tests/svg.rs b/crates/io/tests/svg.rs index fd56d35b86..d1baf418fa 100644 --- a/crates/io/tests/svg.rs +++ b/crates/io/tests/svg.rs @@ -93,23 +93,6 @@ fn svg_group_rasterization_threshold_is_configurable_and_clamped() { let LayerContent::Group(outer) = &unlimited.document.layers[0].content else { panic!("outer group should remain editable") }; assert!(outer.children.iter().any(|child| matches!(child.content, LayerContent::Group(_))), "zero disables the preference threshold"); - #[cfg(not(target_arch = "wasm32"))] - { - let path = std::env::temp_dir().join(format!("photocraft-svg-depth-{}.svg", std::process::id())); - std::fs::write(&path, source.as_bytes()).unwrap(); - for (depth, reference) in [(1, &shallow), (0, &unlimited)] { - let seekable = photocraft_io::import_path_with_svg_group_depth("nested.svg", &path, depth, &photocraft_raster::Interrupt::NONE).unwrap(); - assert_eq!(seekable.warnings, reference.warnings); - let LayerContent::Group(outer) = &seekable.document.layers[0].content else { panic!("outer group should remain editable") }; - assert!(outer.children.iter().any(|child| if depth == 1 { - matches!(child.content, LayerContent::Raster(_)) - } else { - matches!(child.content, LayerContent::Group(_)) - })); - } - std::fs::remove_file(path).unwrap(); - } - let clamped = import_with_svg_group_depth("nested.svg", source.as_bytes(), usize::MAX).unwrap(); assert!(matches!(clamped.document.layers[0].content, LayerContent::Group(_)), "out-of-range settings clamp to the document's hard group limit"); } diff --git a/crates/psd/src/layer.rs b/crates/psd/src/layer.rs index 42036a994b..5450ed7e67 100644 --- a/crates/psd/src/layer.rs +++ b/crates/psd/src/layer.rs @@ -544,7 +544,7 @@ impl LayerRecord { ch.decode(w, h, depth, version) } - pub(crate) fn read(r: &mut Reader<'_>, version: Version) -> Result<(Self, Vec)> { + fn read(r: &mut Reader<'_>, version: Version) -> Result<(Self, Vec)> { let rect = Rect::read(r)?; let nch = r.u16()?; if nch > MAX_LAYER_CHANNELS { @@ -586,7 +586,7 @@ impl LayerRecord { Ok((LayerRecord { rect, channels, blend_mode, opacity, clipping, flags, filler, mask, blending_ranges, name, blocks, extra_trailing }, lens)) } - pub(crate) fn write(&self, out: &mut Vec, version: Version) -> Result<()> { + fn write(&self, out: &mut Vec, version: Version) -> Result<()> { self.rect.write(out); let nch = u16::try_from(self.channels.len()).map_err(|_| PsdError::LimitExceeded("too many channels"))?; out.put_u16(nch); diff --git a/crates/psd/src/lib.rs b/crates/psd/src/lib.rs index 32efb0267c..00d73c4003 100644 --- a/crates/psd/src/lib.rs +++ b/crates/psd/src/lib.rs @@ -64,7 +64,6 @@ pub mod pixels; pub mod resources; pub mod shc; pub mod slices; -pub mod stream; pub mod tagged; #[cfg(any(test, feature = "testgen"))] // Test-data generator (tests and fuzz seeds only): failing loudly is the point. diff --git a/crates/psd/src/stream.rs b/crates/psd/src/stream.rs deleted file mode 100644 index 3023b5750a..0000000000 --- a/crates/psd/src/stream.rs +++ /dev/null @@ -1,499 +0,0 @@ -//! Seekable PSD/PSB metadata and row decoding. Pixel ranges stay in the source; no whole-file -//! or whole-channel buffer is created. `metadata` deliberately contains no encoded pixels. -use std::io::{Read, Seek, SeekFrom, Take, Write}; - -use crate::image_data::ImageData; -use crate::io::Reader; -use crate::{Compression, GlobalLayerMask, Header, LayerInfo, LayerInfoPlacement, LayerRecord, PlaneLayout, PsdError, PsdFile, Result, TaggedBlock, Version}; - -fn io(e: std::io::Error) -> PsdError { - PsdError::invalid(format!("stream I/O: {e}")) -} -fn invalid() -> PsdError { - PsdError::invalid("invalid stream range") -} -fn array(r: &mut impl Read) -> Result<[u8; N]> { - let mut b = [0; N]; - r.read_exact(&mut b).map_err(io)?; - Ok(b) -} -fn length(r: &mut impl Read, long: bool) -> Result { - Ok(if long { u64::from_be_bytes(array(r)?) } else { u64::from(u32::from_be_bytes(array(r)?)) }) -} - -#[derive(Clone, Copy, Debug)] -/// One checked encoded layer-channel range, excluding its compression marker. -pub struct ChannelRange { - /// Photoshop channel id (negative for transparency and masks). - pub id: i16, - /// Absolute byte offset in the source. - pub offset: u64, - /// Encoded byte count. - pub length: u64, - /// Channel codec. - pub compression: Compression, -} - -/// Metadata and independent source ranges; encoded pixels are never retained in this model. -pub struct Index { - /// Parsed metadata only. Do not serialize this model: pixel channels are in `channels`. - pub metadata: PsdFile, - /// Ranges per layer record, in stored order. - pub channels: Vec>, -} - -struct Scan<'a, R> { - reader: &'a mut R, - allowance: u64, - cancelled: &'a dyn Fn() -> bool, -} -impl Scan<'_, R> { - fn position(&mut self) -> Result { - self.reader.stream_position().map_err(io) - } - fn end(&mut self, length: u64, boundary: u64) -> Result { - self.position()?.checked_add(length).filter(|end| *end <= boundary).ok_or_else(invalid) - } - fn bytes(&mut self, n: u64, boundary: u64) -> Result> { - self.end(n, boundary)?; - self.allowance = self.allowance.checked_sub(n).ok_or(PsdError::LimitExceeded("stream metadata exceeds working-memory budget"))?; - let n = usize::try_from(n).map_err(|_| invalid())?; - let mut out = Vec::new(); - out.try_reserve_exact(n).map_err(|_| PsdError::LimitExceeded("stream metadata allocation"))?; - out.resize(n, 0); - for chunk in out.chunks_mut(64 << 10) { - if (self.cancelled)() { - return Err(PsdError::invalid("stream cancelled")); - } - self.reader.read_exact(chunk).map_err(io)?; - } - Ok(out) - } - fn layers(&mut self, end: u64, version: Version) -> Result<(LayerInfo, Vec>)> { - let start = self.position()?; - let count = i16::from_be_bytes(array(self.reader)?); - let n = usize::from(count.unsigned_abs()); - if (n as u64).saturating_mul(34) > end.saturating_sub(self.position()?) { - return Err(invalid()); - } - let mut layers = Vec::with_capacity(n); - let mut lengths = Vec::with_capacity(n); - for _ in 0..n { - let mut record = self.bytes(18, end)?; - let nch = u16::from_be_bytes(record.get(16..18).ok_or_else(invalid)?.try_into().map_err(|_| invalid())?); - if nch > crate::layer::MAX_LAYER_CHANNELS { - return Err(PsdError::LimitExceeded("too many layer channels")); - } - let fixed = u64::from(nch) * if version.is_psb() { 10 } else { 6 } + 16; - record.extend(self.bytes(fixed, end)?); - let extra = record.get(record.len().saturating_sub(4)..).ok_or_else(invalid)?; - let extra = u32::from_be_bytes(extra.try_into().map_err(|_| invalid())?); - record.extend(self.bytes(u64::from(extra), end)?); - let (rec, lens) = LayerRecord::read(&mut Reader::new(&record), version)?; - layers.push(rec); - lengths.push(lens); - } - let mut ranges = Vec::with_capacity(n); - for (rec, lens) in layers.iter_mut().zip(lengths) { - let mut channels = Vec::new(); - for (ch, len) in rec.channels.iter_mut().zip(lens) { - if len == 0 { - continue; - } - if len < 2 { - return Err(invalid()); - } - let channel_end = self.end(len, end)?; - let compression = Compression::from_u16(u16::from_be_bytes(array(self.reader)?)); - ch.compression = Some(compression); - channels.push(ChannelRange { id: ch.id, offset: self.position()?, length: len - 2, compression }); - self.reader.seek(SeekFrom::Start(channel_end)).map_err(io)?; - } - ranges.push(channels); - } - let body = self.position()?.saturating_sub(start); - let rest = end.saturating_sub(self.position()?); - let pad = self.bytes(rest, end)?; - let padding = if pad.len() as u64 == body % 2 && pad.iter().all(|b| *b == 0) { None } else { Some(pad) }; - Ok((LayerInfo { merged_alpha: count < 0, layers, padding }, ranges)) - } -} - -impl Index { - /// Index a seekable file with an aggregate metadata cap and cooperative cancellation. - pub fn read(reader: &mut R, metadata_budget: u64, cancelled: &dyn Fn() -> bool) -> Result { - let end = reader.seek(SeekFrom::End(0)).map_err(io)?; - reader.seek(SeekFrom::Start(0)).map_err(io)?; - let header = Header::read(&mut Reader::new(&array::<26>(reader)?))?; - let mut s = Scan { reader, allowance: metadata_budget, cancelled }; - let n = length(s.reader, false)?; - let color_mode_data = s.bytes(n, end)?; - let n = length(s.reader, false)?; - let resources = s.bytes(n, end)?; - let mut resource_bytes = (n as u32).to_be_bytes().to_vec(); - resource_bytes.extend(resources); - let resources = crate::resources::read_section(&mut Reader::new(&resource_bytes))?; - drop(resource_bytes); - let lm_len = length(s.reader, header.version.is_psb())?; - let lm_end = s.end(lm_len, end)?; - let mut layer_info = None; - let mut channels = Vec::new(); - let mut global_layer_mask = None; - let mut global_blocks = Vec::new(); - let mut layer_mask_trailing = Vec::new(); - let mut layer_info_placement = LayerInfoPlacement::Section; - if lm_len > 0 { - let n = length(s.reader, header.version.is_psb())?; - let li_end = s.end(n, lm_end)?; - if n > 0 { - let (info, c) = s.layers(li_end, header.version)?; - layer_info = Some(info); - channels = c; - } - if lm_end.saturating_sub(s.position()?) >= 4 { - let n = length(s.reader, false)?; - global_layer_mask = Some(GlobalLayerMask { data: s.bytes(n, lm_end)? }); - while lm_end.saturating_sub(s.position()?) >= 12 { - let start = s.position()?; - let signature = array::<4>(s.reader)?; - if signature != *b"8BIM" && signature != *b"8B64" { - s.reader.seek(SeekFrom::Start(start)).map_err(io)?; - break; - } - let key = array::<4>(s.reader)?; - let n = length(s.reader, crate::tagged::uses_long_length(header.version, &key))?; - let block_end = s.end(n, lm_end)?; - let is_layers = layer_info.is_none() && matches!(&key, b"Lr16" | b"Lr32" | b"Layr"); - let data = if is_layers { - let (info, c) = s.layers(block_end, header.version)?; - layer_info = Some(info); - channels = c; - Vec::new() - } else { - s.bytes(n, lm_end)? - }; - // Same padding detector as the byte parser, without reading the next block. - let at = s.position()?; - let rest = lm_end.saturating_sub(at); - let peek = s.bytes(rest.min(7), lm_end)?; - s.reader.seek(SeekFrom::Start(at)).map_err(io)?; - let pad = (0..=3usize) - .find(|k| *k as u64 == rest || peek.get(*k..).is_some_and(|b| b.starts_with(b"8BIM") || b.starts_with(b"8B64"))) - .unwrap_or(0); - let bytes = s.bytes(pad as u64, lm_end)?; - let padding = if bytes.len() as u64 == (4 - n % 4) % 4 && bytes.iter().all(|b| *b == 0) { None } else { Some(bytes) }; - if is_layers { - layer_info_placement = LayerInfoPlacement::GlobalBlock { index: global_blocks.len(), signature, key, padding }; - } else { - global_blocks.push(TaggedBlock { signature, key, data, padding }); - } - } - } - let left = lm_end.saturating_sub(s.position()?); - layer_mask_trailing = s.bytes(left, lm_end)?; - } - // The merged image is intentionally not read: layered imports below consume only layers. - let compression = Compression::from_u16(u16::from_be_bytes(array(s.reader)?)); - Ok(Self { - metadata: PsdFile { - header, - color_mode_data, - resources, - layer_info, - layer_info_placement, - global_layer_mask, - global_blocks, - layer_mask_trailing, - image_data: ImageData { compression, data: Vec::new() }, - }, - channels, - }) - } -} - -enum Input { - Raw(Take), - Rle(Take, Vec), - Zip(flate2::read::ZlibDecoder>), -} -/// A sequential channel decoder with bounded row buffers, including ZIP prediction. -pub struct Rows { - input: Input, - layout: PlaneLayout, - prediction: bool, - row: usize, - encoded: Vec, -} -impl Rows { - /// Open a checked range on an independent reader. Clone file handles with shared seek - /// positions must not be used for concurrent channels. - pub fn new(mut reader: R, range: ChannelRange, width: usize, height: usize, depth: u16, version: Version) -> Result { - if width > 300_000 || height > 300_000 || !matches!(depth, 1 | 8 | 16 | 32) { - return Err(PsdError::LimitExceeded("stream channel dimensions")); - } - reader.seek(SeekFrom::Start(range.offset)).map_err(io)?; - let mut source = reader.take(range.length); - let layout = PlaneLayout { planes: 1, width, height, depth, version }; - let input = match range.compression { - Compression::Raw => Input::Raw(source), - Compression::Rle => { - let mut counts = Vec::with_capacity(height); - let mut total = (height as u64).checked_mul(if version.is_psb() { 4 } else { 2 }).ok_or_else(invalid)?; - for _ in 0..height { - let n = - if version.is_psb() { u32::from_be_bytes(array(&mut source)?) as usize } else { usize::from(u16::from_be_bytes(array(&mut source)?)) }; - if n > layout.row_bytes().saturating_mul(2).saturating_add(1024) { - return Err(PsdError::LimitExceeded("encoded RLE row")); - } - total = total.checked_add(n as u64).ok_or_else(invalid)?; - counts.push(n); - } - if total > range.length { - return Err(invalid()); - } - Input::Rle(source, counts) - } - Compression::Zip | Compression::ZipPrediction => Input::Zip(flate2::read::ZlibDecoder::new(source)), - Compression::Unknown(n) => return Err(PsdError::Unsupported(format!("stream compression {n}"))), - }; - Ok(Self { input, layout, prediction: range.compression == Compression::ZipPrediction, row: 0, encoded: Vec::new() }) - } - /// Decode the next native big-endian sample row, reusing `out`. - pub fn read_row(&mut self, out: &mut Vec) -> Result<()> { - if self.row >= self.layout.height { - return Err(invalid()); - } - let bytes = self.layout.row_bytes(); - out.resize(bytes, 0); - match &mut self.input { - Input::Raw(r) => r.read_exact(out).map_err(io)?, - Input::Zip(r) => r.read_exact(out).map_err(io)?, - Input::Rle(r, counts) => { - out.clear(); - self.encoded.resize(*counts.get(self.row).ok_or_else(invalid)?, 0); - r.read_exact(&mut self.encoded).map_err(io)?; - crate::compression::packbits::decode_into(&self.encoded, bytes, out)?; - } - } - if self.prediction { - let row = PlaneLayout { height: 1, ..self.layout }; - crate::compression::unpredict(out, &row)?; - } - self.row += 1; - Ok(()) - } - /// Checks the zlib checksum and rejects extra decoded samples after the expected last row. - pub fn finish(mut self) -> Result<()> { - if self.row != self.layout.height { - return Err(invalid()); - } - let mut extra = [0; 1]; - let n = match &mut self.input { - Input::Raw(r) | Input::Rle(r, _) => r.read(&mut extra), - Input::Zip(r) => r.read(&mut extra), - } - .map_err(io)?; - if n != 0 { - return Err(PsdError::invalid("extra stream channel samples")); - } - Ok(()) - } -} - -fn patch(out: &mut W, at: u64, value: u64) -> Result<()> { - let end = out.stream_position().map_err(io)?; - out.seek(SeekFrom::Start(at)).map_err(io)?; - out.write_all(&value.to_be_bytes()).map_err(io)?; - out.seek(SeekFrom::Start(end)).map_err(io)?; - Ok(()) -} - -fn write_layers(out: &mut W, info: &LayerInfo, pixels: &mut impl FnMut(usize, i16, &mut W) -> Result<()>) -> Result<()> { - let start = out.stream_position().map_err(io)?; - let count = i16::try_from(info.layers.len()).map_err(|_| PsdError::LimitExceeded("too many streamed layers"))?; - out.write_all(&(if info.merged_alpha { -count } else { count }).to_be_bytes()).map_err(io)?; - let mut records = Vec::with_capacity(info.layers.len()); - for rec in &info.layers { - records.push(out.stream_position().map_err(io)?); - let mut encoded = Vec::new(); - rec.write(&mut encoded, Version::Psb)?; - out.write_all(&encoded).map_err(io)?; - } - for (i, rec) in info.layers.iter().enumerate() { - for (c, channel) in rec.channels.iter().enumerate() { - let begin = out.stream_position().map_err(io)?; - if let Some(compression) = channel.compression { - out.write_all(&compression.as_u16().to_be_bytes()).map_err(io)?; - pixels(i, channel.id, out)?; - } - let len = out.stream_position().map_err(io)?.saturating_sub(begin); - let at = records.get(i).copied().ok_or_else(invalid)?.checked_add(20 + c as u64 * 10).ok_or_else(invalid)?; - patch(out, at, len)?; - } - } - let length = out.stream_position().map_err(io)?.saturating_sub(start); - let pad = (4 - length % 4) % 4; - for _ in 0..pad { - out.write_all(&[0]).map_err(io)?; - } - Ok(()) -} - -/// Write a PSB directly to a seekable sink, patching channel and section lengths after each -/// bounded encoded stream. Callbacks supply encoded layer channels and the merged image; this -/// function never buffers their data. Publication/atomic replacement belongs to the caller. -pub fn write_layered( - file: &PsdFile, - out: &mut W, - mut pixels: impl FnMut(usize, i16, &mut W) -> Result<()>, - merged: impl FnOnce(&mut W) -> Result<()>, -) -> Result<()> { - if file.header.version != Version::Psb { - return Err(PsdError::Unsupported("seekable writer requires PSB".into())); - } - file.header.validate()?; - let mut prefix = Vec::new(); - file.header.write(&mut prefix); - prefix.extend(u32::try_from(file.color_mode_data.len()).map_err(|_| invalid())?.to_be_bytes()); - prefix.extend(&file.color_mode_data); - crate::resources::write_section(&mut prefix, &file.resources)?; - out.write_all(&prefix).map_err(io)?; - let lm_at = out.stream_position().map_err(io)?; - out.write_all(&0u64.to_be_bytes()).map_err(io)?; - let start = out.stream_position().map_err(io)?; - let li_at = start; - out.write_all(&0u64.to_be_bytes()).map_err(io)?; - if let (Some(info), LayerInfoPlacement::Section) = (&file.layer_info, &file.layer_info_placement) { - let begin = out.stream_position().map_err(io)?; - write_layers(out, info, &mut pixels)?; - let n = out.stream_position().map_err(io)?.saturating_sub(begin); - patch(out, li_at, n)?; - } - let mask = file.global_layer_mask.as_ref().map_or(&[][..], |m| m.data.as_slice()); - out.write_all(&u32::try_from(mask.len()).map_err(|_| invalid())?.to_be_bytes()).map_err(io)?; - out.write_all(mask).map_err(io)?; - for i in 0..=file.global_blocks.len() { - if let (Some(info), LayerInfoPlacement::GlobalBlock { index, signature, key, padding }) = (&file.layer_info, &file.layer_info_placement) - && i == (*index).min(file.global_blocks.len()) - { - out.write_all(signature).map_err(io)?; - out.write_all(key).map_err(io)?; - let at = out.stream_position().map_err(io)?; - out.write_all(&0u64.to_be_bytes()).map_err(io)?; - let begin = out.stream_position().map_err(io)?; - write_layers(out, info, &mut pixels)?; - let n = out.stream_position().map_err(io)?.saturating_sub(begin); - patch(out, at, n)?; - match padding { - Some(bytes) => out.write_all(bytes).map_err(io)?, - None => out.write_all(&[0u8; 3][..((4 - n % 4) % 4) as usize]).map_err(io)?, - } - } - if let Some(block) = file.global_blocks.get(i) { - let mut bytes = Vec::new(); - crate::tagged::write_blocks(&mut bytes, std::slice::from_ref(block), Version::Psb, 4)?; - out.write_all(&bytes).map_err(io)?; - } - } - out.write_all(&file.layer_mask_trailing).map_err(io)?; - let n = out.stream_position().map_err(io)?.saturating_sub(start); - patch(out, lm_at, n)?; - out.write_all(&file.image_data.compression.as_u16().to_be_bytes()).map_err(io)?; - merged(out) -} - -#[cfg(test)] -mod tests { - use super::*; - use std::io::Cursor; - - #[test] - fn indexed_channels_match_byte_decoder_at_every_depth_and_compression() { - for version in [Version::Psd, Version::Psb] { - for depth in [8, 16, 32] { - for compression in Compression::ALL { - let file = crate::testgen::layered(version, crate::ColorMode::Rgb, depth, compression); - let bytes = file.to_bytes().unwrap(); - let index = Index::read(&mut Cursor::new(&bytes), 32 << 20, &|| false).unwrap(); - assert_eq!(index.metadata.resources, file.resources); - assert_eq!(index.metadata.global_blocks, file.global_blocks); - for (rec, ranges) in index.metadata.layers().iter().zip(&index.channels) { - for range in ranges { - let (w, h) = rec.channel_rect(range.id).size().unwrap(); - let mut rows = Rows::new(Cursor::new(&bytes), *range, w, h, depth, version).unwrap(); - let mut decoded = Vec::new(); - let mut row = Vec::new(); - for _ in 0..h { - rows.read_row(&mut row).unwrap(); - decoded.extend_from_slice(&row); - } - rows.finish().unwrap(); - let original = file.layers().iter().find(|l| l.name == rec.name).unwrap(); - assert_eq!(decoded, original.decode_channel(range.id, depth, version).unwrap()); - } - } - } - } - } - } - - #[test] - fn bounded_index_and_cancel_do_not_allocate_declared_pixel_bytes() { - let file = crate::testgen::layered(Version::Psb, crate::ColorMode::Rgb, 8, Compression::Rle); - let bytes = file.to_bytes().unwrap(); - assert!(Index::read(&mut Cursor::new(&bytes), 2, &|| false).is_err()); - assert!(Index::read(&mut Cursor::new(&bytes), 32 << 20, &|| true).is_err()); - for n in [0, 25, 30, bytes.len() / 2] { - assert!(Index::read(&mut Cursor::new(&bytes[..n]), 32 << 20, &|| false).is_err()); - } - } - - #[test] - fn rows_reject_corrupt_checksum_and_truncated_data() { - let layout = PlaneLayout { planes: 1, width: 10, height: 4, depth: 8, version: Version::Psb }; - let encoded = crate::compression::encode_planes(Compression::Zip, &[1; 40], &layout).unwrap(); - for corrupt in [false, true] { - let mut bytes = encoded.clone(); - if corrupt { - let n = bytes.len(); - bytes[n - 1] ^= 0xff; - } else { - bytes.truncate(bytes.len() - 3); - } - let range = ChannelRange { id: 0, offset: 0, length: bytes.len() as u64, compression: Compression::Zip }; - let mut rows = Rows::new(Cursor::new(&bytes), range, 10, 4, 8, Version::Psb).unwrap(); - let mut row = Vec::new(); - let mut error = false; - for _ in 0..4 { - if rows.read_row(&mut row).is_err() { - error = true; - break; - } - } - assert!(error || rows.finish().is_err(), "truncated or corrupt zlib must fail"); - } - } - - #[test] - fn seekable_writer_patches_layer_channel_and_global_block_lengths() { - for depth in [8, 16, 32] { - for compression in Compression::ALL { - let file = crate::testgen::layered(Version::Psb, crate::ColorMode::Rgb, depth, compression); - let mut out = Cursor::new(Vec::new()); - write_layered( - &file, - &mut out, - |layer, id, out| out.write_all(&file.layers()[layer].channel(id).unwrap().data).map_err(io), - |out| out.write_all(&file.image_data.data).map_err(io), - ) - .unwrap(); - let read = PsdFile::from_bytes(&out.into_inner()).unwrap(); - assert_eq!(read.header, file.header); - assert_eq!(read.global_blocks, file.global_blocks); - assert_eq!(read.resources, file.resources); - assert_eq!(read.image_data, file.image_data); - assert_eq!(read.layers(), file.layers()); - } - } - } -} diff --git a/crates/ui-egui/src/control.rs b/crates/ui-egui/src/control.rs index 69dc5ddc34..64f9c31556 100644 --- a/crates/ui-egui/src/control.rs +++ b/crates/ui-egui/src/control.rs @@ -1219,11 +1219,6 @@ fn dispatch(app: &mut PhotocraftApp, ctx: &egui::Context, req: &ControlRequest) Err(e) => return err(e), } } - if matches!(path.rsplit('.').next().map(|s| s.to_ascii_lowercase()).as_deref(), Some("psd" | "psb" | "psdt" | "exr")) - && let Some(import) = app.services.automation_import_path.clone() - { - return run_waiting(app, wait, |app| crate::jobs_ui::open_authorized(app, path.to_string(), import)); - } let opened = match app.services.automation_read.as_mut() { Some(read) => read(path), None => Err("automation read authority is not configured".into()), @@ -2332,51 +2327,6 @@ mod tests { assert_eq!(written.borrow().len(), 3, "a bad tgaBits writes nothing"); } - #[test] - fn automation_exr_open_uses_authorized_seekable_import() { - use std::sync::{ - Arc, - atomic::{AtomicUsize, Ordering}, - }; - for source_read_only in [false, true] { - let reads = Arc::new(AtomicUsize::new(0)); - let imported = reads.clone(); - let services = crate::Services { - automation_import_path: Some(Arc::new(move |path, _| { - assert_eq!(path, "in/starmap.EXR"); - imported.fetch_add(1, Ordering::Relaxed); - Ok(photocraft_io::ImportResult { - document: photocraft_doc::Document::new( - "starmap.EXR", - photocraft_geom::Size::new(65536, 32768), - photocraft_color::ColorMode::Rgb, - photocraft_color::SampleType::F32, - ), - warnings: Vec::new(), - source_read_only, - preview_only: source_read_only, - }) - })), - automation_read: Some(Box::new(|_| Err("whole-file read must not run".into()))), - ..Default::default() - }; - let mut app = PhotocraftApp::new(photocraft_engine::Session::new(), services); - let response = call(&mut app, &egui::Context::default(), "app.open", json!({"path": "in/starmap.EXR", "wait": false})); - assert_eq!(response["ok"], true, "{response}"); - let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); - while !app.session.jobs().is_empty() && std::time::Instant::now() < deadline { - app.session.poll_jobs(); - std::thread::sleep(std::time::Duration::from_millis(1)); - } - assert!(app.session.jobs().is_empty(), "EXR import did not finish"); - assert_eq!(reads.load(Ordering::Relaxed), 1); - assert_eq!(app.session.active().unwrap().doc.size, photocraft_geom::Size::new(65536, 32768)); - let st = app.session.active().unwrap(); - assert_eq!(st.source_read_only, source_read_only); - assert_eq!(st.path.is_none(), source_read_only, "read-only imports must not keep a source save path"); - } - } - #[test] fn automation_open_and_save_use_the_roots_and_reply_with_warnings() { use photocraft_color::{ColorMode, SampleType}; diff --git a/crates/ui-egui/src/file_open.rs b/crates/ui-egui/src/file_open.rs index 1cec43e5df..d67e56dbf0 100644 --- a/crates/ui-egui/src/file_open.rs +++ b/crates/ui-egui/src/file_open.rs @@ -112,30 +112,8 @@ impl PhotocraftApp { crate::jobs_ui::start_open(self, &display_name(path), Some(path.to_string()), photocraft_engine::jobs::OpenSource::Path(path.to_string()))?; return Ok(Vec::new()); } - let seekable = ext == "exr" || matches!(ext.as_str(), "psd" | "psb" | "psdt") && std::fs::metadata(path).is_ok_and(|m| m.len() >= 64 << 20); - if !seekable { - let bytes = photocraft_format::read_file(std::path::Path::new(path)).map_err(|e| format!("{path}: {e}"))?; - return self.open_file(path, &bytes); - } - let name = self.open_name(&display_name(path)); - let imported = photocraft_io::import_path_with(&name, std::path::Path::new(path), &photocraft_raster::Interrupt::NONE).map_err(|e| e.to_string())?; - let warnings = with_missing_fonts_of(&imported.document, imported.warnings); - let question = self.session.color.open_question(&imported.document); - crate::open_prompt::arrive( - self, - crate::open_prompt::Held { - doc: imported.document, - read_only: imported.source_read_only, - name, - path: Some(path.to_string()), - warnings: warnings.clone(), - bytes: None, - slot: None, - question, - policy_applied: false, - }, - ); - Ok(warnings) + let bytes = photocraft_format::read_file(std::path::Path::new(path)).map_err(|e| format!("{path}: {e}"))?; + self.open_file(path, &bytes) } #[cfg(target_arch = "wasm32")] diff --git a/crates/ui-egui/src/jobs_ui.rs b/crates/ui-egui/src/jobs_ui.rs index 5fc586fe5b..29da41d481 100644 --- a/crates/ui-egui/src/jobs_ui.rs +++ b/crates/ui-egui/src/jobs_ui.rs @@ -249,51 +249,6 @@ pub(crate) fn finish_open_tab(app: &mut PhotocraftApp, tab: &OpenTab, v: &Value) r } -pub(crate) fn open_authorized(app: &mut PhotocraftApp, path: String, import: crate::AutomationImportPathFn) -> Result { - let name = app.open_name(&crate::file_open::display_name(&path)); - let worker_path = path.clone(); - let apply_name = name.clone(); - let apply_path = path.clone(); - match app - .session - .start_job( - OPEN_JOB, - json!({"path": path}), - "Opening", - false, - move |ctx| { - ctx.stage(0.0, 1.0, "Decoding", |ctl| import(&worker_path, ctl)) - .map_err(|e| if ctx.cancelled() { photocraft_engine::EngineError::Cancelled } else { photocraft_engine::EngineError::Other(e) }) - }, - move |session, imported| { - let source_read_only = imported.source_read_only; - let (mut doc, warnings) = (imported.document, imported.warnings); - doc.name = apply_name.clone(); - let document_path = (!source_read_only && !photocraft_engine::file_cmds::is_template(&apply_path)).then(|| apply_path.clone()); - let (index, color) = session.open_document(doc, document_path); - if let Some(st) = session.active_mut() { - st.source_read_only = source_read_only; - } - Ok(json!({"document": index, "name": apply_name, "warnings": warnings, "color": color, "path": apply_path, "automation": true})) - }, - ) - .map_err(|e| e.to_string())? - { - Started::Done(v) => { - finish_open(app, &name, Some(&path), &v, None)?; - Ok(v) - } - Started::Job(job) => { - app.jobs.opens.push(OpenTab { job, name, path: Some(path), slot: None, order: None }); - app.jobs.focus = Some(job); - app.jobs.last_started = Some(job); - app.ui.chrome.home = None; - app.ui.status_error = false; - Ok(json!({"job": job.0, "pending": true})) - } - } -} - /// Save the active document to `path` in the background: `save` encodes and writes it on a /// worker while the window keeps drawing and shows the progress dialog. The document is locked /// until the job ends, which [`tick`] records like a synchronous save. `Ok(None)`: started; diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index a62ab4c161..04027176b1 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -325,8 +325,6 @@ pub(crate) enum SaveKind { /// Automation: flat formats stay copies (#2579) and hooks never fire. Automation, } -/// Decode a document through handles beneath the desktop control session's authorized root. -pub type AutomationImportPathFn = std::sync::Arc) -> Result + Send + Sync>; /// Read bytes through the desktop control session's authorized read root. pub type AutomationReadFn = Box Result<(String, Vec), String>>; /// Write bytes through the desktop control session's authorized write root. @@ -417,7 +415,6 @@ pub struct Services { /// File access used only by control/MCP requests. Interactive dialogs keep /// using `file_dialog` and `write` with the user's authority. pub automation_read: Option, - pub automation_import_path: Option, pub automation_write: Option, pub automation_open_pcraft: Option, pub automation_command: Option, From 7e29a9999c581d55fc6adc377d7fb4e4c06305e8 Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Tue, 3 Nov 2026 09:23:15 +0100 Subject: [PATCH 3/3] Wire EXR band streaming into main's path open - import_file_for_open_with decodes single-part flat scanline EXR block by block into tiles; headers the band decoder refuses fall back to the whole-file decoder, so files main opened still open. Band-time failures and cancellation publish no document. - The RAM tile budget and scratch-disk checks live in exr_native (the PSD stream module they came from is dropped); over-budget opens fail with IoError::OutOfMemory. - The band decoder picks colour channels with exr_color_channels, like the byte decoder. - Tests move into the per-crate it binaries and use import_file; an engine open-job test covers the OpenSource::Path route. The ui-egui sync open test is dropped: main's synchronous open reads bytes, and the io test covers the oversized header. - docs/streaming-io.md, roadmap, scratch-disks and REL-212-1 describe EXR only. --- crates/codecs/src/exr_stream.rs | 18 ++++---- crates/codecs/tests/{ => it}/exr_stream.rs | 4 +- crates/codecs/tests/it/main.rs | 1 + crates/engine/src/jobs/tests.rs | 32 +++++++++++++ crates/io/src/exr_native.rs | 44 +++++++++++++----- crates/io/src/lib.rs | 23 +++++++++- crates/io/tests/it/main.rs | 1 + crates/io/tests/{ => it}/streaming_exr.rs | 24 +++++----- crates/ui-egui/tests/exr_open.rs | 44 ------------------ docs/roadmap.md | 2 +- docs/scorecard.md | 2 +- docs/scratch-disks.md | 4 +- docs/streaming-io.md | 53 +++++++++------------- scorecard/reliability.toml | 2 +- 14 files changed, 139 insertions(+), 115 deletions(-) rename crates/codecs/tests/{ => it}/exr_stream.rs (98%) rename crates/io/tests/{ => it}/streaming_exr.rs (82%) delete mode 100644 crates/ui-egui/tests/exr_open.rs diff --git a/crates/codecs/src/exr_stream.rs b/crates/codecs/src/exr_stream.rs index a1d5818a16..1ed8ba1276 100644 --- a/crates/codecs/src/exr_stream.rs +++ b/crates/codecs/src/exr_stream.rs @@ -141,15 +141,17 @@ impl Decoder { if list.is_empty() || list.len() > 64 || list.iter().any(|c| c.sampling.0 != 1 || c.sampling.1 != 1) { return Err(unsupported("EXR channel count or subsampling")); } + // The same channels the whole-file decoder shows, so both routes open the same image. let names: Vec = list.iter().map(|c| c.name.to_string()).collect(); - let find = |name| names.iter().position(|n| n.rsplit('.').next() == Some(name)); - let (selected, gray) = if let (Some(r), Some(g), Some(b)) = (find("R"), find("G"), find("B")) { - (vec![Some(r), Some(g), Some(b), find("A")], false) - } else if let Some(y) = find("Y").or_else(|| (list.len() == 1).then_some(0)) { - (vec![Some(y), find("A")], true) - } else { - return Err(unsupported("no R/G/B or Y channels")); - }; + let (layout, picked) = crate::image::exr_color_channels(names.iter().map(String::as_str)).ok_or_else(|| unsupported("no R/G/B or Y channels"))?; + let gray = matches!(layout, crate::ChannelLayout::Gray | crate::ChannelLayout::GrayA); + let mut selected: Vec> = picked.into_iter().map(Some).collect(); + if !matches!(layout, crate::ChannelLayout::GrayA | crate::ChannelLayout::Rgba) { + selected.push(None); + } + if selected.len() != if gray { 2 } else { 4 } { + return Err(unsupported("EXR channel selection")); + } // The document path currently tags Rec. 709 linear samples. Keep custom // primaries on the existing decoder until their profiles can be mapped. if !gray && header.shared_attributes.chromaticities.is_some() { diff --git a/crates/codecs/tests/exr_stream.rs b/crates/codecs/tests/it/exr_stream.rs similarity index 98% rename from crates/codecs/tests/exr_stream.rs rename to crates/codecs/tests/it/exr_stream.rs index acf03e8ae1..65be2387b9 100644 --- a/crates/codecs/tests/exr_stream.rs +++ b/crates/codecs/tests/it/exr_stream.rs @@ -135,8 +135,8 @@ fn mixed_samples_and_offset_data_window_keep_coordinates_and_values() { while let Some(band) = decoder.next_band(&|| false).unwrap() { assert_eq!(band.y, rows); rows += band.height; - for p in band.data.chunks_exact(16) { - let v: Vec = p.chunks_exact(4).map(|s| f32::from_le_bytes(s.try_into().unwrap())).collect(); + for p in band.data.as_chunks::<16>().0 { + let v: Vec = p.as_chunks::<4>().0.iter().map(|s| f32::from_le_bytes(*s)).collect(); assert_eq!(v, [-0.25, 3.5, 42.0, 1.0]); } } diff --git a/crates/codecs/tests/it/main.rs b/crates/codecs/tests/it/main.rs index 4293fb4a48..72e878c44e 100644 --- a/crates/codecs/tests/it/main.rs +++ b/crates/codecs/tests/it/main.rs @@ -10,6 +10,7 @@ mod detect_caps; mod dpx; mod exr_cryptomatte; mod exr_multipart; +mod exr_stream; mod fidelity; mod gif_frames; mod heif; diff --git a/crates/engine/src/jobs/tests.rs b/crates/engine/src/jobs/tests.rs index 0974a5c17a..271162e3c6 100644 --- a/crates/engine/src/jobs/tests.rs +++ b/crates/engine/src/jobs/tests.rs @@ -543,6 +543,38 @@ fn open_runs_as_a_job_and_cancel_adds_nothing() { assert!(matches!(wait_event(&mut s, id).outcome, JobOutcome::Failed(_))); } +/// A flat scanline EXR opened from a path goes through the band decoder in the open job and +/// arrives as the same 32-bit document the whole-file decoder makes; cancelling adds nothing. +#[cfg(any(unix, windows))] +#[test] +fn an_exr_path_open_decodes_in_bands_and_cancels_cleanly() { + use photocraft_codecs::{ChannelLayout, EncodeOptions, ExrCompression, Format, Image, SampleType}; + let values: Vec = (0..64 * 48 * 3).map(|i| [0.25, 3.0, -0.5][i % 3]).collect(); + let image = Image::from_f32(64, 48, ChannelLayout::Rgb, &values).unwrap().convert(ChannelLayout::Rgb, SampleType::F16); + let bytes = photocraft_codecs::encode(&image, Format::OpenExr, &EncodeOptions { exr_compression: ExrCompression::Zip16, ..Default::default() }).unwrap(); + let path = std::env::temp_dir().join(format!("photocraft-open-job-{}.exr", std::process::id())); + std::fs::write(&path, &bytes).unwrap(); + let reference = photocraft_io::import("band.exr", &bytes).unwrap().document; + let mut s = Session::new(); + let id = job(s.start_open("band.exr", OpenSource::Path(path.to_string_lossy().into_owned())).unwrap()); + let e = wait_event(&mut s, id); + assert!(matches!(e.outcome, JobOutcome::Done(_)), "{e:?}"); + let doc = &s.documents()[0].doc; + assert_eq!((doc.size, doc.mode, doc.depth), (reference.size, reference.mode, reference.depth)); + use photocraft_doc::LayerContent; + let (LayerContent::Raster(a), LayerContent::Raster(r)) = (&doc.layers[0].content, &reference.layers[0].content) else { panic!("raster background") }; + assert_eq!(a.to_interleaved(reference.bounds()), r.to_interleaved(reference.bounds())); + + let id = job(s.start_open("again.exr", OpenSource::Path(path.to_string_lossy().into_owned())).unwrap()); + s.cancel_job(id); + let outcome = wait_event(&mut s, id).outcome; + assert!(matches!(outcome, JobOutcome::Cancelled | JobOutcome::Done(_)), "{outcome:?}"); + if outcome == JobOutcome::Cancelled { + assert_eq!(s.documents().len(), 1, "a cancelled open adds no document"); + } + std::fs::remove_file(path).unwrap(); +} + /// Ask When Opening: a file whose profile doesn't match is decoded but not added; the shell takes /// it once the prompt is answered. Without Ask it opens as usual. #[test] diff --git a/crates/io/src/exr_native.rs b/crates/io/src/exr_native.rs index 0d7555bd7b..af76690056 100644 --- a/crates/io/src/exr_native.rs +++ b/crates/io/src/exr_native.rs @@ -6,13 +6,37 @@ use photocraft_geom::{Rect, Size, TILE_SIZE}; use photocraft_raster::{Interrupt, Surface}; use std::io::{Read, Seek}; +/// Tiles the decoded image would add must fit the RAM tile budget when no scratch disk can +/// take cold tiles. +fn ram_tile_capacity(resident: u64, budget: u64, growth: u64) -> Result<(), IoError> { + if resident.checked_add(growth).is_none_or(|required| required > budget) { + return Err(IoError::OutOfMemory("decoded image tiles exceed the available RAM budget; enable a scratch disk or close other documents".into())); + } + Ok(()) +} + +/// A scratch disk that was on when the import started must still be on: its tiles may +/// already have moved there. +fn require_scratch(expected: bool) -> Result<(), IoError> { + if expected && !photocraft_raster::spill::enabled() { + return Err(IoError::Read( + photocraft_raster::spill::last_error().unwrap_or_else(|| "scratch storage was disabled while loading; retry the import".into()), + )); + } + Ok(()) +} + +/// Decode a single-part flat scanline EXR from `source` one compression block at a time. +/// `Ok(None)`: the header is not one the band decoder takes (multi-part, deep, tiled, custom +/// chromaticities, over its block limits or malformed by its stricter checks); the caller +/// falls back to the whole-file decoder. Once pixels are being decoded any failure fails the +/// open, so no partial document is published. pub(crate) fn import(source: R, ctl: &Interrupt<'_>) -> Result, IoError> { - let _metadata = photocraft_raster::memory::try_reserve(32 << 20).map_err(IoError::Unsupported)?; + let _metadata = photocraft_raster::memory::try_reserve(32 << 20).map_err(IoError::OutOfMemory)?; let mut decoder = match photocraft_codecs::exr_stream::Decoder::new(source, &|| ctl.cancelled()) { Ok(decoder) => decoder, Err(_) if ctl.cancelled() => return Err(IoError::Cancelled), - Err(photocraft_codecs::CodecError::Unsupported { .. }) => return Ok(None), - Err(e) => return Err(e.into()), + Err(_) => return Ok(None), }; let info = decoder.info(); let mode = if info.gray { ColorMode::Grayscale } else { ColorMode::Rgb }; @@ -22,19 +46,18 @@ pub(crate) fn import(source: R, ctl: &Interrupt<'_>) -> Result(source: R, ctl: &Interrupt<'_>) -> Result(source: R, ctl: &Interrupt<'_>) -> Result Vec { let values: Vec = (0..31 * 35 * layout.channels()).map(|i| [-0.25, 4.0, 0.0, 0.0001, 0.5][i % 5]).collect(); @@ -13,6 +9,10 @@ fn fixture(layout: ChannelLayout, sample: SampleType, compression: ExrCompressio photocraft_codecs::encode(&image, Format::OpenExr, &EncodeOptions { exr_compression: compression, ..Default::default() }).unwrap() } +fn open(path: &std::path::Path, ctl: &Interrupt) -> Result { + photocraft_io::import_file("HDR", path, &photocraft_io::ImportOptions::default(), ctl) +} + // One test keeps these imports from competing for the global working allowance. #[test] fn native_scanline_import_preserves_hdr_document_and_cancels_without_publication() { @@ -23,9 +23,8 @@ fn native_scanline_import_preserves_hdr_document_and_cancels_without_publication let bytes = fixture(layout, sample, compression); std::fs::write(&path, &bytes).unwrap(); let reference = photocraft_io::import("HDR", &bytes).unwrap(); - let native = photocraft_io::import_path_with("HDR", &path, &Interrupt::NONE).unwrap(); - let authorized = photocraft_io::import_seekable_with("HDR", &|| Ok(Cursor::new(&bytes)), &Interrupt::NONE).unwrap(); - for actual in [native, authorized] { + let actual = open(&path, &Interrupt::NONE).unwrap(); + { let a = &actual.document; let r = &reference.document; assert_eq!(actual.warnings, reference.warnings); @@ -56,9 +55,9 @@ fn native_scanline_import_preserves_hdr_document_and_cancels_without_publication cancelled.store(true, Ordering::Relaxed); } }; - assert!(matches!(photocraft_io::import_path_with("HDR", &path, &Interrupt::new(&cancel, &progress)), Err(photocraft_io::IoError::Cancelled))); + assert!(matches!(open(&path, &Interrupt::new(&cancel, &progress)), Err(photocraft_io::IoError::Cancelled))); std::fs::write(&path, &bytes[..bytes.len() / 2]).unwrap(); - assert!(photocraft_io::import_path_with("HDR", &path, &Interrupt::NONE).is_err()); + assert!(open(&path, &Interrupt::NONE).is_err()); // A syntactically complete huge header/table must be refused before reading // its pixel blocks when scratch is unavailable. No giant fixture is needed. assert!(!photocraft_raster::spill::enabled()); @@ -87,8 +86,7 @@ fn native_scanline_import_preserves_hdr_document_and_cancels_without_publication huge.extend_from_slice(&offset.to_le_bytes()); } huge.extend_from_slice(&[0; 8]); - assert!( - matches!(photocraft_io::import_seekable_with("huge", &|| Ok(Cursor::new(&huge)), &Interrupt::NONE), Err(photocraft_io::IoError::Unsupported(ref message)) if message.contains("decoded image tiles")) - ); + std::fs::write(&path, &huge).unwrap(); + assert!(matches!(open(&path, &Interrupt::NONE), Err(photocraft_io::IoError::OutOfMemory(ref message)) if message.contains("decoded image tiles"))); std::fs::remove_file(path).unwrap(); } diff --git a/crates/ui-egui/tests/exr_open.rs b/crates/ui-egui/tests/exr_open.rs deleted file mode 100644 index ae3b737bec..0000000000 --- a/crates/ui-egui/tests/exr_open.rs +++ /dev/null @@ -1,44 +0,0 @@ -#![cfg(not(target_arch = "wasm32"))] -use photocraft_codecs::{ChannelLayout, EncodeOptions, Format, Image}; -use photocraft_ui_egui::PhotocraftApp; - -#[test] -fn synchronous_native_exr_open_uses_tile_admission_before_huge_pixel_decode() { - let image = Image::from_f32(1, 1, ChannelLayout::Rgb, &[0.5, 2.0, -0.25]).unwrap(); - let bytes = photocraft_codecs::encode(&image, Format::OpenExr, &EncodeOptions::default()).unwrap(); - let mut header_end = 8; - loop { - if bytes[header_end] == 0 { - header_end += 1; - break; - } - for _ in 0..2 { - while bytes[header_end] != 0 { - header_end += 1; - } - header_end += 1; - } - let length = i32::from_le_bytes(bytes[header_end..header_end + 4].try_into().unwrap()) as usize; - header_end += 4 + length; - } - let mut huge = bytes[..header_end].to_vec(); - let needle = b"dataWindow\0box2i\0\x10\0\0\0"; - let at = huge.windows(needle.len()).position(|v| v == needle).unwrap() + needle.len(); - huge[at + 8..at + 12].copy_from_slice(&65535_i32.to_le_bytes()); - huge[at + 12..at + 16].copy_from_slice(&32767_i32.to_le_bytes()); - let offset = (header_end + 2048 * 8) as u64; - for _ in 0..2048 { - huge.extend_from_slice(&offset.to_le_bytes()); - } - huge.extend_from_slice(&[0; 8]); - let path = std::env::temp_dir().join(format!("photocraft-exr-ui-{}.exr", std::process::id())); - std::fs::write(&path, huge).unwrap(); - let mut app = PhotocraftApp::new(photocraft_engine::Session::new(), Default::default()); - app.background_jobs = false; - photocraft_raster::spill::configure(None, 0).unwrap(); - photocraft_raster::memory::configure(photocraft_raster::memory::Policy { percent: None, absolute_bytes: 1 << 30 }); - let error = app.open_path(path.to_str().unwrap()).unwrap_err(); - assert!(error.contains("decoded image tiles"), "{error}"); - assert!(app.session.documents().is_empty()); - std::fs::remove_file(path).unwrap(); -} diff --git a/docs/roadmap.md b/docs/roadmap.md index 1710997e45..d2ca4a8435 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -125,4 +125,4 @@ Each milestone has a **definition of done (DoD)** and must leave `main` green on 2026-10-13: native scratch storage pages current and history tiles through compressed, CRC-checked files. Fresh RAM budgets default to 70%; engine-owned memory supervision stops with its last session. Scratch errors block edit/save commits and permit retry. Unix files are private and stale files swept at startup; Windows files use delete-on-close. See [scratch disks](scratch-disks.md). Streaming import and viewport rendering follow separately. -2026-10-13: seekable native PSD/PSB and supported scanline EXR imports decode bounded bands at the original resolution. Authorized opens retain read-only source protection. See [native streaming I/O](streaming-io.md). +2026-10-13: native path opens of single-part flat scanline EXR files decode one compression block at a time into tiles at the original resolution. See [EXR streaming](streaming-io.md). diff --git a/docs/scorecard.md b/docs/scorecard.md index 4733080bb1..db283ba4be 100644 --- a/docs/scorecard.md +++ b/docs/scorecard.md @@ -276,7 +276,7 @@ Never-crash attribute on 30 of 30 crate roots. | REL-2 | Every workspace crate carries the never-crash deny(clippy::unwrap_used, ...) attribute | done | | Counted automatically above (crate roots under crates/ and apps/); testkit gained it in #221 | | REL-3 | The app catches escaped panics around command dispatch and import/export | done | | a_panic_in_a_command_... runs the test-only `test.panic` spec through Session::execute and asserts Err plus an unchanged document (engine jobs_tests.rs); the worker guard is covered by a_panic_in_the_worker_...; crash_guard.rs guard_turns_a_panic_into_an_error covers import/export (apps/photocraft) | | REL-4 | Atomic save (temp, fsync, rename) for every document write | done | [#203](https://github.com/storytold/photocraft/issues/203) | photocraft_format::atomic_write (format/src/atomic.rs, #230): same-dir temp, sync_all, rename with Windows PermissionDenied retry, Unix dir fsync; every document write site routes through it and format/src/atomic/tests.rs failure-injects write, rename and read-only-dir errors | -| REL-212-1 | Fallible allocation for paste/erase/fill/transform/save/import, clear out-of-memory error | partial | [#212](https://github.com/storytold/photocraft/issues/212) | Raster region/tile buffers, paste/erase/fill/transform and warp output use fallible reservations; filter and lens working memory is admitted before edits, and current/history tiles spill to CRC-checked scratch. Native layered PSD/PSB and supported scanline EXR imports decode bounded bands; save/import boundaries preserve state; remaining encoder/decoder allocations still need broader audit. | +| REL-212-1 | Fallible allocation for paste/erase/fill/transform/save/import, clear out-of-memory error | partial | [#212](https://github.com/storytold/photocraft/issues/212) | Raster region/tile buffers, paste/erase/fill/transform and warp output use fallible reservations; filter and lens working memory is admitted before edits, and current/history tiles spill to CRC-checked scratch. Native path opens of flat scanline EXR decode bounded blocks into tiles; save/import boundaries preserve state; remaining encoder/decoder allocations still need broader audit. | | REL-212-2 | Export switches to PSB when the estimated file size exceeds 2 GB | done | [#212](https://github.com/storytold/photocraft/issues/212) | PSD export uses checked size estimates for raster, masks, channels, and retained metadata | | REL-212-3 | Tests: an allocation-limit injection makes each command Err with the document unchanged | partial | [#212](https://github.com/storytold/photocraft/issues/212) | Engine failpoint tests cover paste, erase, fill, transform, import and save transaction boundaries; codec-level allocator failure coverage remains open | diff --git a/docs/scratch-disks.md b/docs/scratch-disks.md index bf73af1948..bcc3103bce 100644 --- a/docs/scratch-disks.md +++ b/docs/scratch-disks.md @@ -57,5 +57,5 @@ read must never become saved pixels. Limitation: a tile that was already unreada read from outside the edited document (for example a clipboard copy) is not detected. Scratch storage alone does not make every format, filter or renderer streamable. See -[native PSD/EXR streaming](streaming-io.md) for the seekable import and output APIs. Viewport -rendering is a separate change built on these guards and memory reservations. +[EXR streaming](streaming-io.md) for the block-by-block EXR import. Viewport rendering is a +separate change built on these guards and memory reservations. diff --git a/docs/streaming-io.md b/docs/streaming-io.md index 3cc4d3a674..b7a84de6a2 100644 --- a/docs/streaming-io.md +++ b/docs/streaming-io.md @@ -1,36 +1,27 @@ -# Native PSD/EXR streaming +# EXR streaming -Native path opens inspect a seekable source before allocating decoded pixels. Supported layered -PSD/PSB files and flat scanline EXR files decode bounded bands directly into guarded raster -tiles. Their document dimensions and samples remain at the source resolution; scratch residency -can move cold tiles to disk. The byte decoder remains available for web and unsupported layouts, -subject to working-memory admission rather than a file-sized allocation without a budget. +Native path opens (`photocraft_io::import_file_for_open_with`, used by the background open job +and by automation file opens) decode single-part flat scanline EXR files one compression block +at a time straight into guarded raster tiles. The document keeps the source dimensions and +samples; HALF samples are stored as F32, with a warning. Tiles count against the RAM tile +budget, and scratch residency can move cold tiles to disk; without a scratch disk an image whose +tiles cannot fit the budget fails with a clear out-of-memory error before any pixels are +decoded. Layered PSD/PSB files use the existing streaming PSD reader on the same route. -PSD indexing retains bounded metadata and channel offsets without copying all channel data or -decoding the stored merged image. Independent seekable channel handles feed the existing document -mapping, preserving layers, masks, groups, metadata, colour models and depth. Unsupported nesting -and metadata limits produce errors. The native PSB writer uses the same metadata mapper with -deferred pixel sources and writes original-depth channel bands to a seekable output. +The block decoder (`photocraft_codecs::exr_stream`) treats the file as untrusted. Header metadata +is bounded (8 MiB, 4096 attributes); dimensions, channel count, chunk count, offsets, block +lengths and sample counts are checked against the file length and the header before anything is +allocated, and one decoded block is capped at 64 MiB. Duplicate, misaligned or truncated blocks +fail the open. It picks the same colour channels as the whole-file decoder. -EXR block decoding supports the flat scanline layouts accepted by the streaming decoder, including -the RGB HALF ZIP test image `65536 × 32768`. HALF samples are stored as F32, with a warning; this -does not reduce the document dimensions. Unsupported EXR layouts fall back to the normal codec -when its limits allow them. Chunk sizes, source bounds and transient buffers are checked before -allocation. Cancellation is checked between bands, and progress describes completed work. +Multi-part, deep, tiled, subsampled and custom-chromaticity files, and headers the block decoder +refuses, fall back to the whole-file decoder, under its own limits and working-memory admission. +Web and byte opens always use the whole-file decoder. -Control and CLI opens use the same importer. Authorized control opens create independent handles -under the held read-directory capability, including channel reopens. They do not gain ambient -filesystem access. Path opens check that file size and modification time remain unchanged. Read-only -and preview import flags survive the native and authorized routes, and templates keep an untitled -document without a source save path. +Cancellation is checked between reads and between scanlines, and progress reports completed +lines. A cancelled or failed open publishes no partial document. The open also fails if the +file's size or modification time changed while it was being read. -The seekable output API accepts a caller-owned writer; atomic path output encodes into a temporary -file beside the destination, syncs it using the shared platform helper, then renames it. Encoder -failure, cancellation or unwinding removes the temporary and preserves the old destination. Its -integration with the desktop background-save service is a separate change, using upstream's -`SaveCtl` and document-lock/saved-revision contract. - -Synthetic tests compare streamed import with the byte decoder across PSD/PSB versions, depths, -colour modes and compression, and verify masks, metadata, cancellation, EXR chunk handling, -source protection and capability bounds. Large-file runtime checks use the public fixtures at -[vistaero.es/photocraft](https://vistaero.es/photocraft/). +Tests compare the streamed document with the whole-file decoder across Gray/GrayA/RGB/RGBA, HALF +and FLOAT, and None/RLE/ZIP/ZIPS/PIZ compression, and cover cancellation, truncation, forged +offsets and lengths, hostile metadata and oversized headers. diff --git a/scorecard/reliability.toml b/scorecard/reliability.toml index 7bc708e548..2939a9152a 100644 --- a/scorecard/reliability.toml +++ b/scorecard/reliability.toml @@ -35,7 +35,7 @@ id = "REL-212-1" target = "Fallible allocation for paste/erase/fill/transform/save/import, clear out-of-memory error" status = "partial" issue = 212 -note = "Raster region/tile buffers, paste/erase/fill/transform and warp output use fallible reservations; filter and lens working memory is admitted before edits, and current/history tiles spill to CRC-checked scratch. Native layered PSD/PSB and supported scanline EXR imports decode bounded bands; save/import boundaries preserve state; remaining encoder/decoder allocations still need broader audit." +note = "Raster region/tile buffers, paste/erase/fill/transform and warp output use fallible reservations; filter and lens working memory is admitted before edits, and current/history tiles spill to CRC-checked scratch. Native path opens of flat scanline EXR decode bounded blocks into tiles; save/import boundaries preserve state; remaining encoder/decoder allocations still need broader audit." [[item]] id = "REL-212-2"