mirror of
https://github.com/storytold/photocraft.git
synced 2026-11-21 01:18:43 +00:00
io: stage encoded PSD/PSB channels on the scratch disk for large saves
Each colour channel and the merged image is parked in a scratch stage file as soon as it is encoded (documents of 1 GiB+ raw pixels, chunks of 64 KiB+), so a save no longer holds every encoded plane in memory before #8115's checked write. The written file is byte-identical; when staging is unavailable the data stays in memory.
This commit is contained in:
1 parent
4b27da939f
commit
dbcee2ba47
6 files changed
+182
-2
No files matched your search
@@ -793,6 +793,7 @@ fn export_stages(doc: &Document, name_or_ext: &str, opts: &ExportOptions, ctl: &
|
||||
maximize_compatibility: opts.psd_maximize_compatibility,
|
||||
document_path: opts.document_path.clone(),
|
||||
read_linked_files: opts.read_linked_files,
|
||||
stage_on_scratch_from: Some(psd_export::STAGE_DOCUMENTS_FROM),
|
||||
..Default::default()
|
||||
};
|
||||
// The preview is rendered first, while only the document is in memory: its full-size
|
||||
|
||||
@@ -20,6 +20,8 @@ mod estimate;
|
||||
mod merged;
|
||||
mod resources;
|
||||
mod smart;
|
||||
mod stage;
|
||||
pub(crate) use stage::STAGE_DOCUMENTS_FROM;
|
||||
|
||||
pub(crate) use document::document_to_psd_interruptible;
|
||||
#[cfg(test)]
|
||||
@@ -74,6 +76,10 @@ pub struct PsdExportOptions {
|
||||
/// off, since a stored path is document data rather than a granted file: such a layer is
|
||||
/// then written as pixels, with a warning.
|
||||
pub read_linked_files: bool,
|
||||
/// Park each encoded channel on the scratch disk, not in memory, when the document's raw
|
||||
/// pixel bytes (layers and composite) reach this and spilling is on (`None`: never). The
|
||||
/// save then holds the file model's structure, not a second copy of its pixels (#5333).
|
||||
pub stage_on_scratch_from: Option<u64>,
|
||||
}
|
||||
|
||||
impl Default for PsdExportOptions {
|
||||
@@ -86,6 +92,7 @@ impl Default for PsdExportOptions {
|
||||
maximize_compatibility: true,
|
||||
document_path: None,
|
||||
read_linked_files: true,
|
||||
stage_on_scratch_from: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -126,6 +133,8 @@ struct Ex {
|
||||
patterns: Vec<photocraft_doc::Pattern>,
|
||||
/// The layer written as Photoshop's Background (see [`psd_background`]).
|
||||
background: Option<photocraft_doc::LayerId>,
|
||||
/// Where encoded channels go instead of memory ([`PsdExportOptions::stage_on_scratch_from`]).
|
||||
stage: Option<std::sync::Arc<stage::Stage>>,
|
||||
}
|
||||
|
||||
/// The layer Photoshop should open as its Background: the bottom top-level layer when it is a
|
||||
@@ -186,9 +195,13 @@ impl Ex {
|
||||
// A ZIP stream for a zero-sized channel makes readers try to decompress an
|
||||
// invalid image extent (notably psd-tools for 32-bit adjustment layers).
|
||||
let compression = if w == 0 || h == 0 { Compression::Raw } else { self.compression() };
|
||||
ChannelData::encode(id, compression, plane, w, h, depth, self.version)
|
||||
let c = ChannelData::encode(id, compression, plane, w, h, depth, self.version)
|
||||
.or_else(|_| ChannelData::encode(id, Compression::Raw, plane, w, h, depth, self.version))
|
||||
.unwrap_or(ChannelData::new(id, Some(Compression::Raw), plane.to_vec()))
|
||||
.unwrap_or(ChannelData::new(id, Some(Compression::Raw), plane.to_vec()));
|
||||
match &self.stage {
|
||||
Some(stage) => stage.channel(c),
|
||||
None => c,
|
||||
}
|
||||
}
|
||||
|
||||
fn empty_channels(&self) -> Vec<ChannelData> {
|
||||
|
||||
@@ -88,6 +88,10 @@ fn export_supported_document(
|
||||
smart: SmartOut::new(doc, depth),
|
||||
patterns: doc.patterns.clone(),
|
||||
background: psd_background(doc),
|
||||
stage: {
|
||||
let raw = doc.size.area().saturating_mul(fmt.bytes_per_pixel() as u64).saturating_mul(doc.walk().len() as u64 + 1);
|
||||
super::stage::Stage::for_document(raw, opts.stage_on_scratch_from)
|
||||
},
|
||||
};
|
||||
add_format_warnings(doc, opts, fmt, big, duotone_inks.is_some(), &mut ex.warnings);
|
||||
index_layer_ids_and_text(doc, &mut ex);
|
||||
@@ -331,6 +335,10 @@ fn prepare_merged_planes<'a>(
|
||||
data: planes,
|
||||
stored: None,
|
||||
});
|
||||
let image_data = match &ex.stage {
|
||||
Some(stage) => stage.image(image_data),
|
||||
None => image_data,
|
||||
};
|
||||
ctl.progress(1.0);
|
||||
MergedExport { header, image_data, extra, has_alpha, quick_mask_id, real_merged }
|
||||
}
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
//! Encoded channels parked on the scratch disk while a large PSD/PSB save is assembled (#5333).
|
||||
//!
|
||||
//! A save encodes the whole file model before it writes a byte (#8115: it can still be
|
||||
//! cancelled, and the scratch reads behind it are checked, before the file is replaced). In
|
||||
//! memory that model is a second, compressed copy of every layer. With
|
||||
//! [`PsdExportOptions::stage_on_scratch_from`](super::PsdExportOptions::stage_on_scratch_from)
|
||||
//! set and spilling on, each channel is appended to a [`StageFile`] in the scratch directory as
|
||||
//! soon as it is encoded, and the model keeps only a [`Stored`] range: the writer then copies it
|
||||
//! through in short reads (`photocraft_psd`'s `put_encoded`). A channel that cannot be staged
|
||||
//! (the disk is full, say) stays in memory: staging only ever saves memory.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use photocraft_psd::source::{ReadAt, Source, Stored};
|
||||
use photocraft_psd::{ChannelData, ImageData, PsdError};
|
||||
use photocraft_raster::stage_file::StageFile;
|
||||
|
||||
/// Encoded data shorter than this stays in memory: a range costs more than it saves.
|
||||
const MIN_STAGED_BYTES: usize = 64 << 10;
|
||||
/// Raw pixel bytes (layers and composite, as sampled) from which a save made by
|
||||
/// [`crate::export_with`] stages its channels: about a 24 MP RGBA/16 document with five layers.
|
||||
pub(crate) const STAGE_DOCUMENTS_FROM: u64 = 1 << 30;
|
||||
|
||||
#[cfg(test)]
|
||||
thread_local! {
|
||||
/// Where this thread's exports stage, instead of the process-wide scratch directory.
|
||||
pub(crate) static DIR_FOR_TESTS: std::cell::RefCell<Option<std::path::PathBuf>> = const { std::cell::RefCell::new(None) };
|
||||
}
|
||||
|
||||
struct Backing(StageFile);
|
||||
|
||||
impl ReadAt for Backing {
|
||||
fn len(&self) -> u64 {
|
||||
self.0.len()
|
||||
}
|
||||
|
||||
fn read_at(&self, offset: u64, buf: &mut [u8]) -> photocraft_psd::Result<()> {
|
||||
self.0.read_exact_at(buf, offset).map_err(|e| PsdError::Read(format!("staged export data, {} bytes at offset {offset}: {e}", buf.len())))
|
||||
}
|
||||
}
|
||||
|
||||
/// The stage of one export: a file on the scratch disk shared by every channel.
|
||||
pub(crate) struct Stage {
|
||||
file: Arc<Backing>,
|
||||
source: Source,
|
||||
}
|
||||
|
||||
impl Stage {
|
||||
/// A stage for a document of `raw_bytes` pixel bytes, when `from` asks for one at that size
|
||||
/// and a stage file can be made in the scratch directory.
|
||||
pub(crate) fn for_document(raw_bytes: u64, from: Option<u64>) -> Option<Arc<Self>> {
|
||||
if raw_bytes < from? {
|
||||
return None;
|
||||
}
|
||||
#[cfg(test)]
|
||||
if let Some(dir) = DIR_FOR_TESTS.with(|d| d.borrow().clone()) {
|
||||
return StageFile::create(&dir).ok().map(Self::new);
|
||||
}
|
||||
StageFile::in_scratch().map(Self::new)
|
||||
}
|
||||
|
||||
fn new(file: StageFile) -> Arc<Self> {
|
||||
let file = Arc::new(Backing(file));
|
||||
let source = Source::from_arc(file.clone());
|
||||
Arc::new(Self { file, source })
|
||||
}
|
||||
|
||||
/// Moves `data` to the stage; `None` (the data kept by the caller) when it is short or the
|
||||
/// write fails.
|
||||
fn park(&self, data: &[u8]) -> Option<Stored> {
|
||||
if data.len() < MIN_STAGED_BYTES {
|
||||
return None;
|
||||
}
|
||||
let offset = self.file.0.append(data).ok()?;
|
||||
Some(Stored { source: self.source.clone(), offset, len: data.len() as u64 })
|
||||
}
|
||||
|
||||
pub(crate) fn channel(&self, mut c: ChannelData) -> ChannelData {
|
||||
if c.stored.is_none()
|
||||
&& let Some(stored) = self.park(&c.data)
|
||||
{
|
||||
c.data = Vec::new();
|
||||
c.stored = Some(stored);
|
||||
}
|
||||
c
|
||||
}
|
||||
|
||||
pub(crate) fn image(&self, mut image: ImageData) -> ImageData {
|
||||
if image.stored.is_none()
|
||||
&& let Some(stored) = self.park(&image.data)
|
||||
{
|
||||
image.data = Vec::new();
|
||||
image.stored = Some(stored);
|
||||
}
|
||||
image
|
||||
}
|
||||
|
||||
/// Bytes on the scratch disk.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn len(&self) -> u64 {
|
||||
self.file.0.len()
|
||||
}
|
||||
}
|
||||
@@ -619,3 +619,56 @@ fn a_linked_fifo_is_not_read() {
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
assert!(warnings.iter().any(|w| w.contains("smart object written as pixels") && w.contains("not a regular file")), "{warnings:?}");
|
||||
}
|
||||
|
||||
/// #5333: with staging on, every large encoded channel and the merged image are parked on the
|
||||
/// scratch disk as they are encoded, and the file written from that model is the same, byte for
|
||||
/// byte, as one assembled in memory: as PSD, as PSB, and as a PSD promoted to PSB on the way out.
|
||||
#[test]
|
||||
fn staged_channels_write_the_same_file_as_in_memory_ones() {
|
||||
let dir = std::env::temp_dir().join(format!("pc-psd-stage-{}", std::process::id()));
|
||||
stage::DIR_FOR_TESTS.with(|d| *d.borrow_mut() = Some(dir.clone()));
|
||||
let doc = sized_document(true, SampleType::U8);
|
||||
let ctl = Interrupt::NONE;
|
||||
for (force_psb, limit) in [(false, PSD_MAX_FILE_SIZE), (true, PSD_MAX_FILE_SIZE), (false, SIZED_LIMIT)] {
|
||||
let plain = PsdExportOptions { force_psb, ..Default::default() };
|
||||
let staged = PsdExportOptions { stage_on_scratch_from: Some(0), ..plain.clone() };
|
||||
let (want, _) = document_to_psd_sized(&doc, &plain, 0, &ctl, limit);
|
||||
let (file, _) = document_to_psd_sized(&doc, &staged, 0, &ctl, limit);
|
||||
let layer = &file.layers()[0];
|
||||
// The opaque alpha packs to a few KB and stays in memory; the noise colour channels go.
|
||||
assert!(layer.channels.iter().filter(|c| c.id >= 0).all(|c| c.stored.is_some() && c.data.is_empty()), "every noise channel is staged");
|
||||
assert!(layer.channels.iter().filter(|c| c.id == -1).all(|c| c.stored.is_none()), "the short alpha stays in memory");
|
||||
assert!(file.image_data.stored.is_some() && file.image_data.data.is_empty(), "the merged image is staged");
|
||||
assert_eq!(file.write_as_psb, want.write_as_psb);
|
||||
let bytes = want.to_bytes().unwrap();
|
||||
assert!(file.to_bytes().unwrap() == bytes, "psb {force_psb}, limit {limit}: staged bytes differ");
|
||||
let mut w = std::io::Cursor::new(Vec::new());
|
||||
file.write_to(&mut w).unwrap();
|
||||
assert!(w.into_inner() == bytes, "psb {force_psb}, limit {limit}: streamed staged bytes differ");
|
||||
}
|
||||
// Under the threshold, or with no stage asked for, nothing is staged.
|
||||
let (file, _) = document_to_psd_sized(&doc, &PsdExportOptions { stage_on_scratch_from: Some(u64::MAX), ..Default::default() }, 0, &ctl, PSD_MAX_FILE_SIZE);
|
||||
assert!(file.layers()[0].channels.iter().all(|c| c.stored.is_none()) && file.image_data.stored.is_none());
|
||||
stage::DIR_FOR_TESTS.with(|d| *d.borrow_mut() = None);
|
||||
#[cfg(unix)]
|
||||
assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 0, "the stage file has no name on disk");
|
||||
let _ = std::fs::remove_dir(&dir);
|
||||
}
|
||||
|
||||
/// Short channels stay in memory, and a stage that cannot be written keeps the data in memory.
|
||||
#[test]
|
||||
fn staging_keeps_short_or_unwritable_data_in_memory() {
|
||||
let dir = std::env::temp_dir().join(format!("pc-psd-stage-short-{}", std::process::id()));
|
||||
stage::DIR_FOR_TESTS.with(|d| *d.borrow_mut() = Some(dir.clone()));
|
||||
let stage = stage::Stage::for_document(1, Some(0)).unwrap();
|
||||
stage::DIR_FOR_TESTS.with(|d| *d.borrow_mut() = None);
|
||||
let short = stage.channel(ChannelData::new(0, Some(Compression::Raw), vec![1; 100]));
|
||||
assert!(short.stored.is_none() && short.data.len() == 100);
|
||||
let long = stage.channel(ChannelData::new(0, Some(Compression::Raw), vec![2; 1 << 17]));
|
||||
assert!(long.stored.is_some() && long.data.is_empty());
|
||||
assert_eq!(long.encoded().unwrap().as_ref(), &vec![2u8; 1 << 17][..]);
|
||||
assert_eq!(stage.len(), 1 << 17);
|
||||
assert!(stage::Stage::for_document(10, Some(11)).is_none(), "under the threshold");
|
||||
assert!(stage::Stage::for_document(10, None).is_none(), "not asked for");
|
||||
let _ = std::fs::remove_dir(&dir);
|
||||
}
|
||||
@@ -265,6 +265,8 @@ pub(crate) fn export_layered(doc: &Document, opts: &ExportOptions, ctl: &photocr
|
||||
maximize_compatibility: true,
|
||||
document_path: opts.document_path.clone(),
|
||||
read_linked_files: opts.read_linked_files,
|
||||
// The layered TIFF reads the merged image's bytes directly: they stay in memory.
|
||||
stage_on_scratch_from: None,
|
||||
};
|
||||
// TIFF has no palette, bitmap or duotone layout: the PSD model of an Indexed Color, Bitmap
|
||||
// or Duotone document is made in its pixel format (RGB or Grayscale), which the PSD exporter
|
||||
|
||||
Reference in new issue
Block a user