From 4b27da939f7d70f614e72fa03517c3bc8efaecf3 Mon Sep 17 00:00:00 2001 From: FredApps <153173854+FredApps@users.noreply.github.com> Date: Tue, 3 Nov 2026 10:49:38 +0100 Subject: [PATCH] raster: anonymous append-only stage file on the scratch disk Shares ScratchFile's private open (0600, unlinked at once, delete-on-close on Windows) so encoded save data can be parked off the heap and read back by offset. --- crates/raster/src/lib.rs | 1 + crates/raster/src/scratch_file.rs | 81 ++++++++++++--------- crates/raster/src/stage_file.rs | 112 ++++++++++++++++++++++++++++++ 3 files changed, 160 insertions(+), 34 deletions(-) create mode 100644 crates/raster/src/stage_file.rs diff --git a/crates/raster/src/lib.rs b/crates/raster/src/lib.rs index 994c02c57c..77121d0e7d 100644 --- a/crates/raster/src/lib.rs +++ b/crates/raster/src/lib.rs @@ -49,6 +49,7 @@ impl std::error::Error for AllocationError {} pub mod memory; mod scratch_file; pub mod spill; +pub mod stage_file; /// One shared tile per pixel value (and tile size), for [`Surface::share_uniform_tiles`]. #[derive(Debug, Default)] diff --git a/crates/raster/src/scratch_file.rs b/crates/raster/src/scratch_file.rs index bcb46c3d0f..fc131543b4 100644 --- a/crates/raster/src/scratch_file.rs +++ b/crates/raster/src/scratch_file.rs @@ -118,35 +118,9 @@ impl std::fmt::Debug for ScratchFile { impl ScratchFile { pub fn create(dir: &Path) -> std::io::Result { - std::fs::create_dir_all(dir)?; - static COUNTER: AtomicUsize = AtomicUsize::new(0); - // A reused process id must not collide with an abandoned file from an older launch. - let n = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_nanos() - .saturating_add(COUNTER.fetch_add(1, Ordering::Relaxed) as u128); - let path = dir.join(format!("photocraft-{}-{n}.scratch", std::process::id())); - let mut options = std::fs::OpenOptions::new(); - options.read(true).write(true).create_new(true); - #[cfg(unix)] - { - use std::os::unix::fs::OpenOptionsExt; - options.mode(0o600); - } + let (main, path, linked) = open_private(dir)?; #[cfg(windows)] - { - use std::os::windows::fs::OpenOptionsExt; - const FILE_FLAG_DELETE_ON_CLOSE: u32 = 0x0400_0000; - options.custom_flags(FILE_FLAG_DELETE_ON_CLOSE); - } - let main = options.open(&path)?; - #[cfg(unix)] - reap_abandoned(dir, &main); - // Unix: the open handle keeps the inode; nothing is left behind at any exit. A - // filesystem that refuses the unlink keeps the name until drop or the next reap. - #[cfg(not(windows))] - let linked = std::fs::remove_file(&path).is_err(); + let _ = linked; Ok(Self { main, #[cfg(windows)] @@ -231,6 +205,45 @@ impl ScratchFile { } } +/// Creates a file only this process can reach in `dir`, gone once its last handle closes: Unix +/// opens it private (0600) and unlinks it right away, Windows opens it delete-on-close. Files +/// of dead processes are reaped. Returns the file, its name, and whether the name is still +/// there (a filesystem refused the unlink: remove it when done). +pub(crate) fn open_private(dir: &Path) -> std::io::Result<(File, PathBuf, bool)> { + std::fs::create_dir_all(dir)?; + static COUNTER: AtomicUsize = AtomicUsize::new(0); + // A reused process id must not collide with an abandoned file from an older launch. + let n = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_nanos() + .saturating_add(COUNTER.fetch_add(1, Ordering::Relaxed) as u128); + let path = dir.join(format!("photocraft-{}-{n}.scratch", std::process::id())); + let mut options = std::fs::OpenOptions::new(); + options.read(true).write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + #[cfg(windows)] + { + use std::os::windows::fs::OpenOptionsExt; + const FILE_FLAG_DELETE_ON_CLOSE: u32 = 0x0400_0000; + options.custom_flags(FILE_FLAG_DELETE_ON_CLOSE); + } + let file = options.open(&path)?; + #[cfg(unix)] + reap_abandoned(dir, &file); + // Unix: the open handle keeps the inode; nothing is left behind at any exit. A + // filesystem that refuses the unlink keeps the name until drop or the next reap. + #[cfg(not(windows))] + let linked = std::fs::remove_file(&path).is_err(); + #[cfg(windows)] + let linked = false; + Ok((file, path, linked)) +} + /// Accept only the names we generate, never other temporary files or directories. #[cfg(any(unix, test))] fn scratch_pid(name: &str) -> Option { @@ -297,17 +310,17 @@ fn thread_slot() -> usize { } #[cfg(unix)] -fn write_all_at(file: &File, data: &[u8], offset: u64) -> std::io::Result<()> { +pub(crate) fn write_all_at(file: &File, data: &[u8], offset: u64) -> std::io::Result<()> { std::os::unix::fs::FileExt::write_all_at(file, data, offset) } #[cfg(unix)] -fn read_exact_at(file: &File, buf: &mut [u8], offset: u64) -> std::io::Result<()> { +pub(crate) fn read_exact_at(file: &File, buf: &mut [u8], offset: u64) -> std::io::Result<()> { std::os::unix::fs::FileExt::read_exact_at(file, buf, offset) } #[cfg(windows)] -fn write_all_at(file: &File, mut data: &[u8], mut offset: u64) -> std::io::Result<()> { +pub(crate) fn write_all_at(file: &File, mut data: &[u8], mut offset: u64) -> std::io::Result<()> { use std::os::windows::fs::FileExt; while !data.is_empty() { let n = file.seek_write(data, offset)?; @@ -321,7 +334,7 @@ fn write_all_at(file: &File, mut data: &[u8], mut offset: u64) -> std::io::Resul } #[cfg(windows)] -fn read_exact_at(file: &File, mut buf: &mut [u8], mut offset: u64) -> std::io::Result<()> { +pub(crate) fn read_exact_at(file: &File, mut buf: &mut [u8], mut offset: u64) -> std::io::Result<()> { use std::os::windows::fs::FileExt; while !buf.is_empty() { let n = file.seek_read(buf, offset)?; @@ -335,12 +348,12 @@ fn read_exact_at(file: &File, mut buf: &mut [u8], mut offset: u64) -> std::io::R } #[cfg(not(any(unix, windows)))] -fn write_all_at(_file: &File, _data: &[u8], _offset: u64) -> std::io::Result<()> { +pub(crate) fn write_all_at(_file: &File, _data: &[u8], _offset: u64) -> std::io::Result<()> { Err(std::io::ErrorKind::Unsupported.into()) } #[cfg(not(any(unix, windows)))] -fn read_exact_at(_file: &File, _buf: &mut [u8], _offset: u64) -> std::io::Result<()> { +pub(crate) fn read_exact_at(_file: &File, _buf: &mut [u8], _offset: u64) -> std::io::Result<()> { Err(std::io::ErrorKind::Unsupported.into()) } diff --git a/crates/raster/src/stage_file.rs b/crates/raster/src/stage_file.rs new file mode 100644 index 0000000000..e371fbe6be --- /dev/null +++ b/crates/raster/src/stage_file.rs @@ -0,0 +1,112 @@ +//! Encoded data an exporter parks on the scratch disk instead of in memory (#5333). +//! +//! A large PSD/PSB save encodes every channel before it writes a byte (so it can still be +//! cancelled, and its scratch reads are checked, before the file is replaced). Kept in memory, +//! those channels are a second copy of the document. A [`StageFile`] holds them on the scratch +//! disk instead: an append-only file made like the tile scratch file (private, unlinked on Unix, +//! delete-on-close on Windows), so nothing is left behind at any exit, and the space returns as +//! soon as the save drops it. + +use std::fs::File; +use std::path::{Path, PathBuf}; +use std::sync::{Mutex, PoisonError}; + +use crate::scratch_file::{open_private, read_exact_at, write_all_at}; + +pub struct StageFile { + file: File, + path: PathBuf, + /// The directory entry still exists (the unlink was refused): remove it on drop. + linked: bool, + /// Bytes reserved so far; appends take their offset from it. + end: Mutex, +} + +impl std::fmt::Debug for StageFile { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("StageFile").field("path", &self.path).field("len", &self.len()).finish() + } +} + +impl StageFile { + /// A new, empty stage file in `dir`. + pub fn create(dir: &Path) -> std::io::Result { + let (file, path, linked) = open_private(dir)?; + Ok(Self { file, path, linked, end: Mutex::new(0) }) + } + + /// A stage file in the configured scratch directory, when spilling is on (`None` otherwise, + /// or when the file cannot be made: the caller then keeps its data in memory). + pub fn in_scratch() -> Option { + if !crate::spill::enabled() { + return None; + } + Self::create(&crate::spill::stats().dir?).ok() + } + + /// Appends `data` and returns its offset. Parallel appends each get their own range. + pub fn append(&self, data: &[u8]) -> std::io::Result { + let offset = { + let mut end = self.end.lock().unwrap_or_else(PoisonError::into_inner); + let offset = *end; + *end = offset.checked_add(data.len() as u64).ok_or_else(|| std::io::Error::other("stage file too large"))?; + offset + }; + write_all_at(&self.file, data, offset)?; + Ok(offset) + } + + /// Fills `buf` from `offset`, which must lie within what was appended. + pub fn read_exact_at(&self, buf: &mut [u8], offset: u64) -> std::io::Result<()> { + if offset.checked_add(buf.len() as u64).is_none_or(|end| end > self.len()) { + return Err(std::io::ErrorKind::UnexpectedEof.into()); + } + read_exact_at(&self.file, buf, offset) + } + + /// Bytes appended (reserved) so far. + pub fn len(&self) -> u64 { + *self.end.lock().unwrap_or_else(PoisonError::into_inner) + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +impl Drop for StageFile { + fn drop(&mut self) { + if self.linked { + let _ = std::fs::remove_file(&self.path); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn appends_read_back_and_leave_nothing_behind() { + let dir = std::env::temp_dir().join(format!("pc-stage-{}", std::process::id())); + let stage = StageFile::create(&dir).unwrap(); + #[cfg(unix)] + assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 0, "the stage file is unlinked while open"); + let parts: Vec> = (0..8u8).map(|i| vec![i; 1000 + usize::from(i)]).collect(); + let offsets: Vec = std::thread::scope(|s| { + let handles: Vec<_> = parts.iter().map(|p| s.spawn(|| stage.append(p).unwrap())).collect(); + handles.into_iter().map(|h| h.join().unwrap()).collect() + }); + assert_eq!(stage.len(), parts.iter().map(|p| p.len() as u64).sum::()); + for (p, &at) in parts.iter().zip(&offsets) { + let mut buf = vec![0; p.len()]; + stage.read_exact_at(&mut buf, at).unwrap(); + assert_eq!(&buf, p); + } + let mut past = [0u8; 2]; + assert!(stage.read_exact_at(&mut past, stage.len() - 1).is_err(), "reads stop at what was appended"); + drop(stage); + assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 0); + let _ = std::fs::remove_dir(&dir); + } +}