raster: anonymous append-only stage file on the scratch disk

Shares ScratchFile's private open (0600, unlinked at once, delete-on-close on
Windows) so encoded save data can be parked off the heap and read back by offset.
This commit is contained in:
FredApps committed 2026-11-03 10:49:38 +01:00
1 parent 2bad559628
commit 4b27da939f
3 files changed
+160 -34

No files matched your search

+1
View File
@@ -49,6 +49,7 @@ impl std::error::Error for AllocationError {}
pub mod memory;
mod scratch_file;
pub mod spill;
pub mod stage_file;
/// One shared tile per pixel value (and tile size), for [`Surface::share_uniform_tiles`].
#[derive(Debug, Default)]
+47 -34
View File
@@ -118,35 +118,9 @@ impl std::fmt::Debug for ScratchFile {
impl ScratchFile {
pub fn create(dir: &Path) -> std::io::Result<Self> {
std::fs::create_dir_all(dir)?;
static COUNTER: AtomicUsize = AtomicUsize::new(0);
// A reused process id must not collide with an abandoned file from an older launch.
let n = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_nanos()
.saturating_add(COUNTER.fetch_add(1, Ordering::Relaxed) as u128);
let path = dir.join(format!("photocraft-{}-{n}.scratch", std::process::id()));
let mut options = std::fs::OpenOptions::new();
options.read(true).write(true).create_new(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
options.mode(0o600);
}
let (main, path, linked) = open_private(dir)?;
#[cfg(windows)]
{
use std::os::windows::fs::OpenOptionsExt;
const FILE_FLAG_DELETE_ON_CLOSE: u32 = 0x0400_0000;
options.custom_flags(FILE_FLAG_DELETE_ON_CLOSE);
}
let main = options.open(&path)?;
#[cfg(unix)]
reap_abandoned(dir, &main);
// Unix: the open handle keeps the inode; nothing is left behind at any exit. A
// filesystem that refuses the unlink keeps the name until drop or the next reap.
#[cfg(not(windows))]
let linked = std::fs::remove_file(&path).is_err();
let _ = linked;
Ok(Self {
main,
#[cfg(windows)]
@@ -231,6 +205,45 @@ impl ScratchFile {
}
}
/// Creates a file only this process can reach in `dir`, gone once its last handle closes: Unix
/// opens it private (0600) and unlinks it right away, Windows opens it delete-on-close. Files
/// of dead processes are reaped. Returns the file, its name, and whether the name is still
/// there (a filesystem refused the unlink: remove it when done).
pub(crate) fn open_private(dir: &Path) -> std::io::Result<(File, PathBuf, bool)> {
std::fs::create_dir_all(dir)?;
static COUNTER: AtomicUsize = AtomicUsize::new(0);
// A reused process id must not collide with an abandoned file from an older launch.
let n = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.unwrap_or_default()
.as_nanos()
.saturating_add(COUNTER.fetch_add(1, Ordering::Relaxed) as u128);
let path = dir.join(format!("photocraft-{}-{n}.scratch", std::process::id()));
let mut options = std::fs::OpenOptions::new();
options.read(true).write(true).create_new(true);
#[cfg(unix)]
{
use std::os::unix::fs::OpenOptionsExt;
options.mode(0o600);
}
#[cfg(windows)]
{
use std::os::windows::fs::OpenOptionsExt;
const FILE_FLAG_DELETE_ON_CLOSE: u32 = 0x0400_0000;
options.custom_flags(FILE_FLAG_DELETE_ON_CLOSE);
}
let file = options.open(&path)?;
#[cfg(unix)]
reap_abandoned(dir, &file);
// Unix: the open handle keeps the inode; nothing is left behind at any exit. A
// filesystem that refuses the unlink keeps the name until drop or the next reap.
#[cfg(not(windows))]
let linked = std::fs::remove_file(&path).is_err();
#[cfg(windows)]
let linked = false;
Ok((file, path, linked))
}
/// Accept only the names we generate, never other temporary files or directories.
#[cfg(any(unix, test))]
fn scratch_pid(name: &str) -> Option<u32> {
@@ -297,17 +310,17 @@ fn thread_slot() -> usize {
}
#[cfg(unix)]
fn write_all_at(file: &File, data: &[u8], offset: u64) -> std::io::Result<()> {
pub(crate) fn write_all_at(file: &File, data: &[u8], offset: u64) -> std::io::Result<()> {
std::os::unix::fs::FileExt::write_all_at(file, data, offset)
}
#[cfg(unix)]
fn read_exact_at(file: &File, buf: &mut [u8], offset: u64) -> std::io::Result<()> {
pub(crate) fn read_exact_at(file: &File, buf: &mut [u8], offset: u64) -> std::io::Result<()> {
std::os::unix::fs::FileExt::read_exact_at(file, buf, offset)
}
#[cfg(windows)]
fn write_all_at(file: &File, mut data: &[u8], mut offset: u64) -> std::io::Result<()> {
pub(crate) fn write_all_at(file: &File, mut data: &[u8], mut offset: u64) -> std::io::Result<()> {
use std::os::windows::fs::FileExt;
while !data.is_empty() {
let n = file.seek_write(data, offset)?;
@@ -321,7 +334,7 @@ fn write_all_at(file: &File, mut data: &[u8], mut offset: u64) -> std::io::Resul
}
#[cfg(windows)]
fn read_exact_at(file: &File, mut buf: &mut [u8], mut offset: u64) -> std::io::Result<()> {
pub(crate) fn read_exact_at(file: &File, mut buf: &mut [u8], mut offset: u64) -> std::io::Result<()> {
use std::os::windows::fs::FileExt;
while !buf.is_empty() {
let n = file.seek_read(buf, offset)?;
@@ -335,12 +348,12 @@ fn read_exact_at(file: &File, mut buf: &mut [u8], mut offset: u64) -> std::io::R
}
#[cfg(not(any(unix, windows)))]
fn write_all_at(_file: &File, _data: &[u8], _offset: u64) -> std::io::Result<()> {
pub(crate) fn write_all_at(_file: &File, _data: &[u8], _offset: u64) -> std::io::Result<()> {
Err(std::io::ErrorKind::Unsupported.into())
}
#[cfg(not(any(unix, windows)))]
fn read_exact_at(_file: &File, _buf: &mut [u8], _offset: u64) -> std::io::Result<()> {
pub(crate) fn read_exact_at(_file: &File, _buf: &mut [u8], _offset: u64) -> std::io::Result<()> {
Err(std::io::ErrorKind::Unsupported.into())
}
+112
View File
@@ -0,0 +1,112 @@
//! Encoded data an exporter parks on the scratch disk instead of in memory (#5333).
//!
//! A large PSD/PSB save encodes every channel before it writes a byte (so it can still be
//! cancelled, and its scratch reads are checked, before the file is replaced). Kept in memory,
//! those channels are a second copy of the document. A [`StageFile`] holds them on the scratch
//! disk instead: an append-only file made like the tile scratch file (private, unlinked on Unix,
//! delete-on-close on Windows), so nothing is left behind at any exit, and the space returns as
//! soon as the save drops it.
use std::fs::File;
use std::path::{Path, PathBuf};
use std::sync::{Mutex, PoisonError};
use crate::scratch_file::{open_private, read_exact_at, write_all_at};
pub struct StageFile {
file: File,
path: PathBuf,
/// The directory entry still exists (the unlink was refused): remove it on drop.
linked: bool,
/// Bytes reserved so far; appends take their offset from it.
end: Mutex<u64>,
}
impl std::fmt::Debug for StageFile {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("StageFile").field("path", &self.path).field("len", &self.len()).finish()
}
}
impl StageFile {
/// A new, empty stage file in `dir`.
pub fn create(dir: &Path) -> std::io::Result<Self> {
let (file, path, linked) = open_private(dir)?;
Ok(Self { file, path, linked, end: Mutex::new(0) })
}
/// A stage file in the configured scratch directory, when spilling is on (`None` otherwise,
/// or when the file cannot be made: the caller then keeps its data in memory).
pub fn in_scratch() -> Option<Self> {
if !crate::spill::enabled() {
return None;
}
Self::create(&crate::spill::stats().dir?).ok()
}
/// Appends `data` and returns its offset. Parallel appends each get their own range.
pub fn append(&self, data: &[u8]) -> std::io::Result<u64> {
let offset = {
let mut end = self.end.lock().unwrap_or_else(PoisonError::into_inner);
let offset = *end;
*end = offset.checked_add(data.len() as u64).ok_or_else(|| std::io::Error::other("stage file too large"))?;
offset
};
write_all_at(&self.file, data, offset)?;
Ok(offset)
}
/// Fills `buf` from `offset`, which must lie within what was appended.
pub fn read_exact_at(&self, buf: &mut [u8], offset: u64) -> std::io::Result<()> {
if offset.checked_add(buf.len() as u64).is_none_or(|end| end > self.len()) {
return Err(std::io::ErrorKind::UnexpectedEof.into());
}
read_exact_at(&self.file, buf, offset)
}
/// Bytes appended (reserved) so far.
pub fn len(&self) -> u64 {
*self.end.lock().unwrap_or_else(PoisonError::into_inner)
}
pub fn is_empty(&self) -> bool {
self.len() == 0
}
}
impl Drop for StageFile {
fn drop(&mut self) {
if self.linked {
let _ = std::fs::remove_file(&self.path);
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn appends_read_back_and_leave_nothing_behind() {
let dir = std::env::temp_dir().join(format!("pc-stage-{}", std::process::id()));
let stage = StageFile::create(&dir).unwrap();
#[cfg(unix)]
assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 0, "the stage file is unlinked while open");
let parts: Vec<Vec<u8>> = (0..8u8).map(|i| vec![i; 1000 + usize::from(i)]).collect();
let offsets: Vec<u64> = std::thread::scope(|s| {
let handles: Vec<_> = parts.iter().map(|p| s.spawn(|| stage.append(p).unwrap())).collect();
handles.into_iter().map(|h| h.join().unwrap()).collect()
});
assert_eq!(stage.len(), parts.iter().map(|p| p.len() as u64).sum::<u64>());
for (p, &at) in parts.iter().zip(&offsets) {
let mut buf = vec![0; p.len()];
stage.read_exact_at(&mut buf, at).unwrap();
assert_eq!(&buf, p);
}
let mut past = [0u8; 2];
assert!(stage.read_exact_at(&mut past, stage.len() - 1).is_err(), "reads stop at what was appended");
drop(stage);
assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 0);
let _ = std::fs::remove_dir(&dir);
}
}