io: stage encoded PSD/PSB channels on the scratch disk for large saves

Each colour channel and the merged image is parked in a scratch stage file as
soon as it is encoded (documents of 1 GiB+ raw pixels, chunks of 64 KiB+), so a
save no longer holds every encoded plane in memory before #8115's checked write.
The written file is byte-identical; when staging is unavailable the data stays
in memory.
This commit is contained in:
FredApps committed 2026-11-03 10:49:38 +01:00
1 parent 4b27da939f
commit dbcee2ba47
6 files changed
+182 -2

No files matched your search

+1
View File
@@ -793,6 +793,7 @@ fn export_stages(doc: &Document, name_or_ext: &str, opts: &ExportOptions, ctl: &
maximize_compatibility: opts.psd_maximize_compatibility,
document_path: opts.document_path.clone(),
read_linked_files: opts.read_linked_files,
stage_on_scratch_from: Some(psd_export::STAGE_DOCUMENTS_FROM),
..Default::default()
};
// The preview is rendered first, while only the document is in memory: its full-size
+15 -2
View File
@@ -20,6 +20,8 @@ mod estimate;
mod merged;
mod resources;
mod smart;
mod stage;
pub(crate) use stage::STAGE_DOCUMENTS_FROM;
pub(crate) use document::document_to_psd_interruptible;
#[cfg(test)]
@@ -74,6 +76,10 @@ pub struct PsdExportOptions {
/// off, since a stored path is document data rather than a granted file: such a layer is
/// then written as pixels, with a warning.
pub read_linked_files: bool,
/// Park each encoded channel on the scratch disk, not in memory, when the document's raw
/// pixel bytes (layers and composite) reach this and spilling is on (`None`: never). The
/// save then holds the file model's structure, not a second copy of its pixels (#5333).
pub stage_on_scratch_from: Option<u64>,
}
impl Default for PsdExportOptions {
@@ -86,6 +92,7 @@ impl Default for PsdExportOptions {
maximize_compatibility: true,
document_path: None,
read_linked_files: true,
stage_on_scratch_from: None,
}
}
}
@@ -126,6 +133,8 @@ struct Ex {
patterns: Vec<photocraft_doc::Pattern>,
/// The layer written as Photoshop's Background (see [`psd_background`]).
background: Option<photocraft_doc::LayerId>,
/// Where encoded channels go instead of memory ([`PsdExportOptions::stage_on_scratch_from`]).
stage: Option<std::sync::Arc<stage::Stage>>,
}
/// The layer Photoshop should open as its Background: the bottom top-level layer when it is a
@@ -186,9 +195,13 @@ impl Ex {
// A ZIP stream for a zero-sized channel makes readers try to decompress an
// invalid image extent (notably psd-tools for 32-bit adjustment layers).
let compression = if w == 0 || h == 0 { Compression::Raw } else { self.compression() };
ChannelData::encode(id, compression, plane, w, h, depth, self.version)
let c = ChannelData::encode(id, compression, plane, w, h, depth, self.version)
.or_else(|_| ChannelData::encode(id, Compression::Raw, plane, w, h, depth, self.version))
.unwrap_or(ChannelData::new(id, Some(Compression::Raw), plane.to_vec()))
.unwrap_or(ChannelData::new(id, Some(Compression::Raw), plane.to_vec()));
match &self.stage {
Some(stage) => stage.channel(c),
None => c,
}
}
fn empty_channels(&self) -> Vec<ChannelData> {
+8
View File
@@ -88,6 +88,10 @@ fn export_supported_document(
smart: SmartOut::new(doc, depth),
patterns: doc.patterns.clone(),
background: psd_background(doc),
stage: {
let raw = doc.size.area().saturating_mul(fmt.bytes_per_pixel() as u64).saturating_mul(doc.walk().len() as u64 + 1);
super::stage::Stage::for_document(raw, opts.stage_on_scratch_from)
},
};
add_format_warnings(doc, opts, fmt, big, duotone_inks.is_some(), &mut ex.warnings);
index_layer_ids_and_text(doc, &mut ex);
@@ -331,6 +335,10 @@ fn prepare_merged_planes<'a>(
data: planes,
stored: None,
});
let image_data = match &ex.stage {
Some(stage) => stage.image(image_data),
None => image_data,
};
ctl.progress(1.0);
MergedExport { header, image_data, extra, has_alpha, quick_mask_id, real_merged }
}
+103
View File
@@ -0,0 +1,103 @@
//! Encoded channels parked on the scratch disk while a large PSD/PSB save is assembled (#5333).
//!
//! A save encodes the whole file model before it writes a byte (#8115: it can still be
//! cancelled, and the scratch reads behind it are checked, before the file is replaced). In
//! memory that model is a second, compressed copy of every layer. With
//! [`PsdExportOptions::stage_on_scratch_from`](super::PsdExportOptions::stage_on_scratch_from)
//! set and spilling on, each channel is appended to a [`StageFile`] in the scratch directory as
//! soon as it is encoded, and the model keeps only a [`Stored`] range: the writer then copies it
//! through in short reads (`photocraft_psd`'s `put_encoded`). A channel that cannot be staged
//! (the disk is full, say) stays in memory: staging only ever saves memory.
use std::sync::Arc;
use photocraft_psd::source::{ReadAt, Source, Stored};
use photocraft_psd::{ChannelData, ImageData, PsdError};
use photocraft_raster::stage_file::StageFile;
/// Encoded data shorter than this stays in memory: a range costs more than it saves.
const MIN_STAGED_BYTES: usize = 64 << 10;
/// Raw pixel bytes (layers and composite, as sampled) from which a save made by
/// [`crate::export_with`] stages its channels: about a 24 MP RGBA/16 document with five layers.
pub(crate) const STAGE_DOCUMENTS_FROM: u64 = 1 << 30;
#[cfg(test)]
thread_local! {
/// Where this thread's exports stage, instead of the process-wide scratch directory.
pub(crate) static DIR_FOR_TESTS: std::cell::RefCell<Option<std::path::PathBuf>> = const { std::cell::RefCell::new(None) };
}
struct Backing(StageFile);
impl ReadAt for Backing {
fn len(&self) -> u64 {
self.0.len()
}
fn read_at(&self, offset: u64, buf: &mut [u8]) -> photocraft_psd::Result<()> {
self.0.read_exact_at(buf, offset).map_err(|e| PsdError::Read(format!("staged export data, {} bytes at offset {offset}: {e}", buf.len())))
}
}
/// The stage of one export: a file on the scratch disk shared by every channel.
pub(crate) struct Stage {
file: Arc<Backing>,
source: Source,
}
impl Stage {
/// A stage for a document of `raw_bytes` pixel bytes, when `from` asks for one at that size
/// and a stage file can be made in the scratch directory.
pub(crate) fn for_document(raw_bytes: u64, from: Option<u64>) -> Option<Arc<Self>> {
if raw_bytes < from? {
return None;
}
#[cfg(test)]
if let Some(dir) = DIR_FOR_TESTS.with(|d| d.borrow().clone()) {
return StageFile::create(&dir).ok().map(Self::new);
}
StageFile::in_scratch().map(Self::new)
}
fn new(file: StageFile) -> Arc<Self> {
let file = Arc::new(Backing(file));
let source = Source::from_arc(file.clone());
Arc::new(Self { file, source })
}
/// Moves `data` to the stage; `None` (the data kept by the caller) when it is short or the
/// write fails.
fn park(&self, data: &[u8]) -> Option<Stored> {
if data.len() < MIN_STAGED_BYTES {
return None;
}
let offset = self.file.0.append(data).ok()?;
Some(Stored { source: self.source.clone(), offset, len: data.len() as u64 })
}
pub(crate) fn channel(&self, mut c: ChannelData) -> ChannelData {
if c.stored.is_none()
&& let Some(stored) = self.park(&c.data)
{
c.data = Vec::new();
c.stored = Some(stored);
}
c
}
pub(crate) fn image(&self, mut image: ImageData) -> ImageData {
if image.stored.is_none()
&& let Some(stored) = self.park(&image.data)
{
image.data = Vec::new();
image.stored = Some(stored);
}
image
}
/// Bytes on the scratch disk.
#[cfg(test)]
pub(crate) fn len(&self) -> u64 {
self.file.0.len()
}
}
+53
View File
@@ -619,3 +619,56 @@ fn a_linked_fifo_is_not_read() {
let _ = std::fs::remove_dir_all(&dir);
assert!(warnings.iter().any(|w| w.contains("smart object written as pixels") && w.contains("not a regular file")), "{warnings:?}");
}
/// #5333: with staging on, every large encoded channel and the merged image are parked on the
/// scratch disk as they are encoded, and the file written from that model is the same, byte for
/// byte, as one assembled in memory: as PSD, as PSB, and as a PSD promoted to PSB on the way out.
#[test]
fn staged_channels_write_the_same_file_as_in_memory_ones() {
let dir = std::env::temp_dir().join(format!("pc-psd-stage-{}", std::process::id()));
stage::DIR_FOR_TESTS.with(|d| *d.borrow_mut() = Some(dir.clone()));
let doc = sized_document(true, SampleType::U8);
let ctl = Interrupt::NONE;
for (force_psb, limit) in [(false, PSD_MAX_FILE_SIZE), (true, PSD_MAX_FILE_SIZE), (false, SIZED_LIMIT)] {
let plain = PsdExportOptions { force_psb, ..Default::default() };
let staged = PsdExportOptions { stage_on_scratch_from: Some(0), ..plain.clone() };
let (want, _) = document_to_psd_sized(&doc, &plain, 0, &ctl, limit);
let (file, _) = document_to_psd_sized(&doc, &staged, 0, &ctl, limit);
let layer = &file.layers()[0];
// The opaque alpha packs to a few KB and stays in memory; the noise colour channels go.
assert!(layer.channels.iter().filter(|c| c.id >= 0).all(|c| c.stored.is_some() && c.data.is_empty()), "every noise channel is staged");
assert!(layer.channels.iter().filter(|c| c.id == -1).all(|c| c.stored.is_none()), "the short alpha stays in memory");
assert!(file.image_data.stored.is_some() && file.image_data.data.is_empty(), "the merged image is staged");
assert_eq!(file.write_as_psb, want.write_as_psb);
let bytes = want.to_bytes().unwrap();
assert!(file.to_bytes().unwrap() == bytes, "psb {force_psb}, limit {limit}: staged bytes differ");
let mut w = std::io::Cursor::new(Vec::new());
file.write_to(&mut w).unwrap();
assert!(w.into_inner() == bytes, "psb {force_psb}, limit {limit}: streamed staged bytes differ");
}
// Under the threshold, or with no stage asked for, nothing is staged.
let (file, _) = document_to_psd_sized(&doc, &PsdExportOptions { stage_on_scratch_from: Some(u64::MAX), ..Default::default() }, 0, &ctl, PSD_MAX_FILE_SIZE);
assert!(file.layers()[0].channels.iter().all(|c| c.stored.is_none()) && file.image_data.stored.is_none());
stage::DIR_FOR_TESTS.with(|d| *d.borrow_mut() = None);
#[cfg(unix)]
assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 0, "the stage file has no name on disk");
let _ = std::fs::remove_dir(&dir);
}
/// Short channels stay in memory, and a stage that cannot be written keeps the data in memory.
#[test]
fn staging_keeps_short_or_unwritable_data_in_memory() {
let dir = std::env::temp_dir().join(format!("pc-psd-stage-short-{}", std::process::id()));
stage::DIR_FOR_TESTS.with(|d| *d.borrow_mut() = Some(dir.clone()));
let stage = stage::Stage::for_document(1, Some(0)).unwrap();
stage::DIR_FOR_TESTS.with(|d| *d.borrow_mut() = None);
let short = stage.channel(ChannelData::new(0, Some(Compression::Raw), vec![1; 100]));
assert!(short.stored.is_none() && short.data.len() == 100);
let long = stage.channel(ChannelData::new(0, Some(Compression::Raw), vec![2; 1 << 17]));
assert!(long.stored.is_some() && long.data.is_empty());
assert_eq!(long.encoded().unwrap().as_ref(), &vec![2u8; 1 << 17][..]);
assert_eq!(stage.len(), 1 << 17);
assert!(stage::Stage::for_document(10, Some(11)).is_none(), "under the threshold");
assert!(stage::Stage::for_document(10, None).is_none(), "not asked for");
let _ = std::fs::remove_dir(&dir);
}
+2
View File
@@ -265,6 +265,8 @@ pub(crate) fn export_layered(doc: &Document, opts: &ExportOptions, ctl: &photocr
maximize_compatibility: true,
document_path: opts.document_path.clone(),
read_linked_files: opts.read_linked_files,
// The layered TIFF reads the merged image's bytes directly: they stay in memory.
stage_on_scratch_from: None,
};
// TIFF has no palette, bitmap or duotone layout: the PSD model of an Indexed Color, Bitmap
// or Duotone document is made in its pixel format (RGB or Grayscale), which the PSD exporter