From dbcee2ba476d694b5e59ecdd5324f4dd709dfbac Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Tue, 3 Nov 2026 10:49:38 +0100 Subject: [PATCH] io: stage encoded PSD/PSB channels on the scratch disk for large saves Each colour channel and the merged image is parked in a scratch stage file as soon as it is encoded (documents of 1 GiB+ raw pixels, chunks of 64 KiB+), so a save no longer holds every encoded plane in memory before #8115's checked write. The written file is byte-identical; when staging is unavailable the data stays in memory. --- crates/io/src/lib.rs | 1 + crates/io/src/psd_export.rs | 17 ++++- crates/io/src/psd_export/document.rs | 8 +++ crates/io/src/psd_export/stage.rs | 103 +++++++++++++++++++++++++++ crates/io/src/psd_export/tests.rs | 53 ++++++++++++++ crates/io/src/tiff_layers.rs | 2 + 6 files changed, 182 insertions(+), 2 deletions(-) create mode 100644 crates/io/src/psd_export/stage.rs diff --git a/crates/io/src/lib.rs b/crates/io/src/lib.rs index 2d4371d4c1..f976867778 100644 --- a/crates/io/src/lib.rs +++ b/crates/io/src/lib.rs @@ -793,6 +793,7 @@ fn export_stages(doc: &Document, name_or_ext: &str, opts: &ExportOptions, ctl: & maximize_compatibility: opts.psd_maximize_compatibility, document_path: opts.document_path.clone(), read_linked_files: opts.read_linked_files, + stage_on_scratch_from: Some(psd_export::STAGE_DOCUMENTS_FROM), ..Default::default() }; // The preview is rendered first, while only the document is in memory: its full-size diff --git a/crates/io/src/psd_export.rs b/crates/io/src/psd_export.rs index 6086aaf644..6f87aa7588 100644 --- a/crates/io/src/psd_export.rs +++ b/crates/io/src/psd_export.rs @@ -20,6 +20,8 @@ mod estimate; mod merged; mod resources; mod smart; +mod stage; +pub(crate) use stage::STAGE_DOCUMENTS_FROM; pub(crate) use document::document_to_psd_interruptible; #[cfg(test)] @@ -74,6 +76,10 @@ pub struct PsdExportOptions { /// off, since a stored path is document data rather than a granted file: such a layer is /// then written as pixels, with a warning. pub read_linked_files: bool, + /// Park each encoded channel on the scratch disk, not in memory, when the document's raw + /// pixel bytes (layers and composite) reach this and spilling is on (`None`: never). The + /// save then holds the file model's structure, not a second copy of its pixels (#5333). + pub stage_on_scratch_from: Option, } impl Default for PsdExportOptions { @@ -86,6 +92,7 @@ impl Default for PsdExportOptions { maximize_compatibility: true, document_path: None, read_linked_files: true, + stage_on_scratch_from: None, } } } @@ -126,6 +133,8 @@ struct Ex { patterns: Vec, /// The layer written as Photoshop's Background (see [`psd_background`]). background: Option, + /// Where encoded channels go instead of memory ([`PsdExportOptions::stage_on_scratch_from`]). + stage: Option>, } /// The layer Photoshop should open as its Background: the bottom top-level layer when it is a @@ -186,9 +195,13 @@ impl Ex { // A ZIP stream for a zero-sized channel makes readers try to decompress an // invalid image extent (notably psd-tools for 32-bit adjustment layers). let compression = if w == 0 || h == 0 { Compression::Raw } else { self.compression() }; - ChannelData::encode(id, compression, plane, w, h, depth, self.version) + let c = ChannelData::encode(id, compression, plane, w, h, depth, self.version) .or_else(|_| ChannelData::encode(id, Compression::Raw, plane, w, h, depth, self.version)) - .unwrap_or(ChannelData::new(id, Some(Compression::Raw), plane.to_vec())) + .unwrap_or(ChannelData::new(id, Some(Compression::Raw), plane.to_vec())); + match &self.stage { + Some(stage) => stage.channel(c), + None => c, + } } fn empty_channels(&self) -> Vec { diff --git a/crates/io/src/psd_export/document.rs b/crates/io/src/psd_export/document.rs index f6c36c2d42..a87ed93010 100644 --- a/crates/io/src/psd_export/document.rs +++ b/crates/io/src/psd_export/document.rs @@ -88,6 +88,10 @@ fn export_supported_document( smart: SmartOut::new(doc, depth), patterns: doc.patterns.clone(), background: psd_background(doc), + stage: { + let raw = doc.size.area().saturating_mul(fmt.bytes_per_pixel() as u64).saturating_mul(doc.walk().len() as u64 + 1); + super::stage::Stage::for_document(raw, opts.stage_on_scratch_from) + }, }; add_format_warnings(doc, opts, fmt, big, duotone_inks.is_some(), &mut ex.warnings); index_layer_ids_and_text(doc, &mut ex); @@ -331,6 +335,10 @@ fn prepare_merged_planes<'a>( data: planes, stored: None, }); + let image_data = match &ex.stage { + Some(stage) => stage.image(image_data), + None => image_data, + }; ctl.progress(1.0); MergedExport { header, image_data, extra, has_alpha, quick_mask_id, real_merged } } diff --git a/crates/io/src/psd_export/stage.rs b/crates/io/src/psd_export/stage.rs new file mode 100644 index 0000000000..3e04d53247 --- /dev/null +++ b/crates/io/src/psd_export/stage.rs @@ -0,0 +1,103 @@ +//! Encoded channels parked on the scratch disk while a large PSD/PSB save is assembled (#5333). +//! +//! A save encodes the whole file model before it writes a byte (#8115: it can still be +//! cancelled, and the scratch reads behind it are checked, before the file is replaced). In +//! memory that model is a second, compressed copy of every layer. With +//! [`PsdExportOptions::stage_on_scratch_from`](super::PsdExportOptions::stage_on_scratch_from) +//! set and spilling on, each channel is appended to a [`StageFile`] in the scratch directory as +//! soon as it is encoded, and the model keeps only a [`Stored`] range: the writer then copies it +//! through in short reads (`photocraft_psd`'s `put_encoded`). A channel that cannot be staged +//! (the disk is full, say) stays in memory: staging only ever saves memory. + +use std::sync::Arc; + +use photocraft_psd::source::{ReadAt, Source, Stored}; +use photocraft_psd::{ChannelData, ImageData, PsdError}; +use photocraft_raster::stage_file::StageFile; + +/// Encoded data shorter than this stays in memory: a range costs more than it saves. +const MIN_STAGED_BYTES: usize = 64 << 10; +/// Raw pixel bytes (layers and composite, as sampled) from which a save made by +/// [`crate::export_with`] stages its channels: about a 24 MP RGBA/16 document with five layers. +pub(crate) const STAGE_DOCUMENTS_FROM: u64 = 1 << 30; + +#[cfg(test)] +thread_local! { + /// Where this thread's exports stage, instead of the process-wide scratch directory. + pub(crate) static DIR_FOR_TESTS: std::cell::RefCell> = const { std::cell::RefCell::new(None) }; +} + +struct Backing(StageFile); + +impl ReadAt for Backing { + fn len(&self) -> u64 { + self.0.len() + } + + fn read_at(&self, offset: u64, buf: &mut [u8]) -> photocraft_psd::Result<()> { + self.0.read_exact_at(buf, offset).map_err(|e| PsdError::Read(format!("staged export data, {} bytes at offset {offset}: {e}", buf.len()))) + } +} + +/// The stage of one export: a file on the scratch disk shared by every channel. +pub(crate) struct Stage { + file: Arc, + source: Source, +} + +impl Stage { + /// A stage for a document of `raw_bytes` pixel bytes, when `from` asks for one at that size + /// and a stage file can be made in the scratch directory. + pub(crate) fn for_document(raw_bytes: u64, from: Option) -> Option> { + if raw_bytes < from? { + return None; + } + #[cfg(test)] + if let Some(dir) = DIR_FOR_TESTS.with(|d| d.borrow().clone()) { + return StageFile::create(&dir).ok().map(Self::new); + } + StageFile::in_scratch().map(Self::new) + } + + fn new(file: StageFile) -> Arc { + let file = Arc::new(Backing(file)); + let source = Source::from_arc(file.clone()); + Arc::new(Self { file, source }) + } + + /// Moves `data` to the stage; `None` (the data kept by the caller) when it is short or the + /// write fails. + fn park(&self, data: &[u8]) -> Option { + if data.len() < MIN_STAGED_BYTES { + return None; + } + let offset = self.file.0.append(data).ok()?; + Some(Stored { source: self.source.clone(), offset, len: data.len() as u64 }) + } + + pub(crate) fn channel(&self, mut c: ChannelData) -> ChannelData { + if c.stored.is_none() + && let Some(stored) = self.park(&c.data) + { + c.data = Vec::new(); + c.stored = Some(stored); + } + c + } + + pub(crate) fn image(&self, mut image: ImageData) -> ImageData { + if image.stored.is_none() + && let Some(stored) = self.park(&image.data) + { + image.data = Vec::new(); + image.stored = Some(stored); + } + image + } + + /// Bytes on the scratch disk. + #[cfg(test)] + pub(crate) fn len(&self) -> u64 { + self.file.0.len() + } +} diff --git a/crates/io/src/psd_export/tests.rs b/crates/io/src/psd_export/tests.rs index c53f2c06f3..4fd78e8c73 100644 --- a/crates/io/src/psd_export/tests.rs +++ b/crates/io/src/psd_export/tests.rs @@ -619,3 +619,56 @@ fn a_linked_fifo_is_not_read() { let _ = std::fs::remove_dir_all(&dir); assert!(warnings.iter().any(|w| w.contains("smart object written as pixels") && w.contains("not a regular file")), "{warnings:?}"); } + +/// #5333: with staging on, every large encoded channel and the merged image are parked on the +/// scratch disk as they are encoded, and the file written from that model is the same, byte for +/// byte, as one assembled in memory: as PSD, as PSB, and as a PSD promoted to PSB on the way out. +#[test] +fn staged_channels_write_the_same_file_as_in_memory_ones() { + let dir = std::env::temp_dir().join(format!("pc-psd-stage-{}", std::process::id())); + stage::DIR_FOR_TESTS.with(|d| *d.borrow_mut() = Some(dir.clone())); + let doc = sized_document(true, SampleType::U8); + let ctl = Interrupt::NONE; + for (force_psb, limit) in [(false, PSD_MAX_FILE_SIZE), (true, PSD_MAX_FILE_SIZE), (false, SIZED_LIMIT)] { + let plain = PsdExportOptions { force_psb, ..Default::default() }; + let staged = PsdExportOptions { stage_on_scratch_from: Some(0), ..plain.clone() }; + let (want, _) = document_to_psd_sized(&doc, &plain, 0, &ctl, limit); + let (file, _) = document_to_psd_sized(&doc, &staged, 0, &ctl, limit); + let layer = &file.layers()[0]; + // The opaque alpha packs to a few KB and stays in memory; the noise colour channels go. + assert!(layer.channels.iter().filter(|c| c.id >= 0).all(|c| c.stored.is_some() && c.data.is_empty()), "every noise channel is staged"); + assert!(layer.channels.iter().filter(|c| c.id == -1).all(|c| c.stored.is_none()), "the short alpha stays in memory"); + assert!(file.image_data.stored.is_some() && file.image_data.data.is_empty(), "the merged image is staged"); + assert_eq!(file.write_as_psb, want.write_as_psb); + let bytes = want.to_bytes().unwrap(); + assert!(file.to_bytes().unwrap() == bytes, "psb {force_psb}, limit {limit}: staged bytes differ"); + let mut w = std::io::Cursor::new(Vec::new()); + file.write_to(&mut w).unwrap(); + assert!(w.into_inner() == bytes, "psb {force_psb}, limit {limit}: streamed staged bytes differ"); + } + // Under the threshold, or with no stage asked for, nothing is staged. + let (file, _) = document_to_psd_sized(&doc, &PsdExportOptions { stage_on_scratch_from: Some(u64::MAX), ..Default::default() }, 0, &ctl, PSD_MAX_FILE_SIZE); + assert!(file.layers()[0].channels.iter().all(|c| c.stored.is_none()) && file.image_data.stored.is_none()); + stage::DIR_FOR_TESTS.with(|d| *d.borrow_mut() = None); + #[cfg(unix)] + assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 0, "the stage file has no name on disk"); + let _ = std::fs::remove_dir(&dir); +} + +/// Short channels stay in memory, and a stage that cannot be written keeps the data in memory. +#[test] +fn staging_keeps_short_or_unwritable_data_in_memory() { + let dir = std::env::temp_dir().join(format!("pc-psd-stage-short-{}", std::process::id())); + stage::DIR_FOR_TESTS.with(|d| *d.borrow_mut() = Some(dir.clone())); + let stage = stage::Stage::for_document(1, Some(0)).unwrap(); + stage::DIR_FOR_TESTS.with(|d| *d.borrow_mut() = None); + let short = stage.channel(ChannelData::new(0, Some(Compression::Raw), vec![1; 100])); + assert!(short.stored.is_none() && short.data.len() == 100); + let long = stage.channel(ChannelData::new(0, Some(Compression::Raw), vec![2; 1 << 17])); + assert!(long.stored.is_some() && long.data.is_empty()); + assert_eq!(long.encoded().unwrap().as_ref(), &vec![2u8; 1 << 17][..]); + assert_eq!(stage.len(), 1 << 17); + assert!(stage::Stage::for_document(10, Some(11)).is_none(), "under the threshold"); + assert!(stage::Stage::for_document(10, None).is_none(), "not asked for"); + let _ = std::fs::remove_dir(&dir); +} diff --git a/crates/io/src/tiff_layers.rs b/crates/io/src/tiff_layers.rs index f001b6a265..f0bb148179 100644 --- a/crates/io/src/tiff_layers.rs +++ b/crates/io/src/tiff_layers.rs @@ -265,6 +265,8 @@ pub(crate) fn export_layered(doc: &Document, opts: &ExportOptions, ctl: &photocr maximize_compatibility: true, document_path: opts.document_path.clone(), read_linked_files: opts.read_linked_files, + // The layered TIFF reads the merged image's bytes directly: they stay in memory. + stage_on_scratch_from: None, }; // TIFF has no palette, bitmap or duotone layout: the PSD model of an Indexed Color, Bitmap // or Duotone document is made in its pixel format (RGB or Grayscale), which the PSD exporter