mirror of
https://github.com/storytold/photocraft.git
synced 2026-11-15 22:52:35 +00:00
raster: anonymous append-only stage file on the scratch disk
Shares ScratchFile's private open (0600, unlinked at once, delete-on-close on Windows) so encoded save data can be parked off the heap and read back by offset.
This commit is contained in:
1 parent
2bad559628
commit
4b27da939f
3 files changed
+160
-34
No files matched your search
@@ -49,6 +49,7 @@ impl std::error::Error for AllocationError {}
|
||||
pub mod memory;
|
||||
mod scratch_file;
|
||||
pub mod spill;
|
||||
pub mod stage_file;
|
||||
|
||||
/// One shared tile per pixel value (and tile size), for [`Surface::share_uniform_tiles`].
|
||||
#[derive(Debug, Default)]
|
||||
|
||||
@@ -118,35 +118,9 @@ impl std::fmt::Debug for ScratchFile {
|
||||
|
||||
impl ScratchFile {
|
||||
pub fn create(dir: &Path) -> std::io::Result<Self> {
|
||||
std::fs::create_dir_all(dir)?;
|
||||
static COUNTER: AtomicUsize = AtomicUsize::new(0);
|
||||
// A reused process id must not collide with an abandoned file from an older launch.
|
||||
let n = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_nanos()
|
||||
.saturating_add(COUNTER.fetch_add(1, Ordering::Relaxed) as u128);
|
||||
let path = dir.join(format!("photocraft-{}-{n}.scratch", std::process::id()));
|
||||
let mut options = std::fs::OpenOptions::new();
|
||||
options.read(true).write(true).create_new(true);
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::OpenOptionsExt;
|
||||
options.mode(0o600);
|
||||
}
|
||||
let (main, path, linked) = open_private(dir)?;
|
||||
#[cfg(windows)]
|
||||
{
|
||||
use std::os::windows::fs::OpenOptionsExt;
|
||||
const FILE_FLAG_DELETE_ON_CLOSE: u32 = 0x0400_0000;
|
||||
options.custom_flags(FILE_FLAG_DELETE_ON_CLOSE);
|
||||
}
|
||||
let main = options.open(&path)?;
|
||||
#[cfg(unix)]
|
||||
reap_abandoned(dir, &main);
|
||||
// Unix: the open handle keeps the inode; nothing is left behind at any exit. A
|
||||
// filesystem that refuses the unlink keeps the name until drop or the next reap.
|
||||
#[cfg(not(windows))]
|
||||
let linked = std::fs::remove_file(&path).is_err();
|
||||
let _ = linked;
|
||||
Ok(Self {
|
||||
main,
|
||||
#[cfg(windows)]
|
||||
@@ -231,6 +205,45 @@ impl ScratchFile {
|
||||
}
|
||||
}
|
||||
|
||||
/// Creates a file only this process can reach in `dir`, gone once its last handle closes: Unix
|
||||
/// opens it private (0600) and unlinks it right away, Windows opens it delete-on-close. Files
|
||||
/// of dead processes are reaped. Returns the file, its name, and whether the name is still
|
||||
/// there (a filesystem refused the unlink: remove it when done).
|
||||
pub(crate) fn open_private(dir: &Path) -> std::io::Result<(File, PathBuf, bool)> {
|
||||
std::fs::create_dir_all(dir)?;
|
||||
static COUNTER: AtomicUsize = AtomicUsize::new(0);
|
||||
// A reused process id must not collide with an abandoned file from an older launch.
|
||||
let n = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_nanos()
|
||||
.saturating_add(COUNTER.fetch_add(1, Ordering::Relaxed) as u128);
|
||||
let path = dir.join(format!("photocraft-{}-{n}.scratch", std::process::id()));
|
||||
let mut options = std::fs::OpenOptions::new();
|
||||
options.read(true).write(true).create_new(true);
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::OpenOptionsExt;
|
||||
options.mode(0o600);
|
||||
}
|
||||
#[cfg(windows)]
|
||||
{
|
||||
use std::os::windows::fs::OpenOptionsExt;
|
||||
const FILE_FLAG_DELETE_ON_CLOSE: u32 = 0x0400_0000;
|
||||
options.custom_flags(FILE_FLAG_DELETE_ON_CLOSE);
|
||||
}
|
||||
let file = options.open(&path)?;
|
||||
#[cfg(unix)]
|
||||
reap_abandoned(dir, &file);
|
||||
// Unix: the open handle keeps the inode; nothing is left behind at any exit. A
|
||||
// filesystem that refuses the unlink keeps the name until drop or the next reap.
|
||||
#[cfg(not(windows))]
|
||||
let linked = std::fs::remove_file(&path).is_err();
|
||||
#[cfg(windows)]
|
||||
let linked = false;
|
||||
Ok((file, path, linked))
|
||||
}
|
||||
|
||||
/// Accept only the names we generate, never other temporary files or directories.
|
||||
#[cfg(any(unix, test))]
|
||||
fn scratch_pid(name: &str) -> Option<u32> {
|
||||
@@ -297,17 +310,17 @@ fn thread_slot() -> usize {
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn write_all_at(file: &File, data: &[u8], offset: u64) -> std::io::Result<()> {
|
||||
pub(crate) fn write_all_at(file: &File, data: &[u8], offset: u64) -> std::io::Result<()> {
|
||||
std::os::unix::fs::FileExt::write_all_at(file, data, offset)
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn read_exact_at(file: &File, buf: &mut [u8], offset: u64) -> std::io::Result<()> {
|
||||
pub(crate) fn read_exact_at(file: &File, buf: &mut [u8], offset: u64) -> std::io::Result<()> {
|
||||
std::os::unix::fs::FileExt::read_exact_at(file, buf, offset)
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
fn write_all_at(file: &File, mut data: &[u8], mut offset: u64) -> std::io::Result<()> {
|
||||
pub(crate) fn write_all_at(file: &File, mut data: &[u8], mut offset: u64) -> std::io::Result<()> {
|
||||
use std::os::windows::fs::FileExt;
|
||||
while !data.is_empty() {
|
||||
let n = file.seek_write(data, offset)?;
|
||||
@@ -321,7 +334,7 @@ fn write_all_at(file: &File, mut data: &[u8], mut offset: u64) -> std::io::Resul
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
fn read_exact_at(file: &File, mut buf: &mut [u8], mut offset: u64) -> std::io::Result<()> {
|
||||
pub(crate) fn read_exact_at(file: &File, mut buf: &mut [u8], mut offset: u64) -> std::io::Result<()> {
|
||||
use std::os::windows::fs::FileExt;
|
||||
while !buf.is_empty() {
|
||||
let n = file.seek_read(buf, offset)?;
|
||||
@@ -335,12 +348,12 @@ fn read_exact_at(file: &File, mut buf: &mut [u8], mut offset: u64) -> std::io::R
|
||||
}
|
||||
|
||||
#[cfg(not(any(unix, windows)))]
|
||||
fn write_all_at(_file: &File, _data: &[u8], _offset: u64) -> std::io::Result<()> {
|
||||
pub(crate) fn write_all_at(_file: &File, _data: &[u8], _offset: u64) -> std::io::Result<()> {
|
||||
Err(std::io::ErrorKind::Unsupported.into())
|
||||
}
|
||||
|
||||
#[cfg(not(any(unix, windows)))]
|
||||
fn read_exact_at(_file: &File, _buf: &mut [u8], _offset: u64) -> std::io::Result<()> {
|
||||
pub(crate) fn read_exact_at(_file: &File, _buf: &mut [u8], _offset: u64) -> std::io::Result<()> {
|
||||
Err(std::io::ErrorKind::Unsupported.into())
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
//! Encoded data an exporter parks on the scratch disk instead of in memory (#5333).
|
||||
//!
|
||||
//! A large PSD/PSB save encodes every channel before it writes a byte (so it can still be
|
||||
//! cancelled, and its scratch reads are checked, before the file is replaced). Kept in memory,
|
||||
//! those channels are a second copy of the document. A [`StageFile`] holds them on the scratch
|
||||
//! disk instead: an append-only file made like the tile scratch file (private, unlinked on Unix,
|
||||
//! delete-on-close on Windows), so nothing is left behind at any exit, and the space returns as
|
||||
//! soon as the save drops it.
|
||||
|
||||
use std::fs::File;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Mutex, PoisonError};
|
||||
|
||||
use crate::scratch_file::{open_private, read_exact_at, write_all_at};
|
||||
|
||||
pub struct StageFile {
|
||||
file: File,
|
||||
path: PathBuf,
|
||||
/// The directory entry still exists (the unlink was refused): remove it on drop.
|
||||
linked: bool,
|
||||
/// Bytes reserved so far; appends take their offset from it.
|
||||
end: Mutex<u64>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for StageFile {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("StageFile").field("path", &self.path).field("len", &self.len()).finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl StageFile {
|
||||
/// A new, empty stage file in `dir`.
|
||||
pub fn create(dir: &Path) -> std::io::Result<Self> {
|
||||
let (file, path, linked) = open_private(dir)?;
|
||||
Ok(Self { file, path, linked, end: Mutex::new(0) })
|
||||
}
|
||||
|
||||
/// A stage file in the configured scratch directory, when spilling is on (`None` otherwise,
|
||||
/// or when the file cannot be made: the caller then keeps its data in memory).
|
||||
pub fn in_scratch() -> Option<Self> {
|
||||
if !crate::spill::enabled() {
|
||||
return None;
|
||||
}
|
||||
Self::create(&crate::spill::stats().dir?).ok()
|
||||
}
|
||||
|
||||
/// Appends `data` and returns its offset. Parallel appends each get their own range.
|
||||
pub fn append(&self, data: &[u8]) -> std::io::Result<u64> {
|
||||
let offset = {
|
||||
let mut end = self.end.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
let offset = *end;
|
||||
*end = offset.checked_add(data.len() as u64).ok_or_else(|| std::io::Error::other("stage file too large"))?;
|
||||
offset
|
||||
};
|
||||
write_all_at(&self.file, data, offset)?;
|
||||
Ok(offset)
|
||||
}
|
||||
|
||||
/// Fills `buf` from `offset`, which must lie within what was appended.
|
||||
pub fn read_exact_at(&self, buf: &mut [u8], offset: u64) -> std::io::Result<()> {
|
||||
if offset.checked_add(buf.len() as u64).is_none_or(|end| end > self.len()) {
|
||||
return Err(std::io::ErrorKind::UnexpectedEof.into());
|
||||
}
|
||||
read_exact_at(&self.file, buf, offset)
|
||||
}
|
||||
|
||||
/// Bytes appended (reserved) so far.
|
||||
pub fn len(&self) -> u64 {
|
||||
*self.end.lock().unwrap_or_else(PoisonError::into_inner)
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.len() == 0
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for StageFile {
|
||||
fn drop(&mut self) {
|
||||
if self.linked {
|
||||
let _ = std::fs::remove_file(&self.path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn appends_read_back_and_leave_nothing_behind() {
|
||||
let dir = std::env::temp_dir().join(format!("pc-stage-{}", std::process::id()));
|
||||
let stage = StageFile::create(&dir).unwrap();
|
||||
#[cfg(unix)]
|
||||
assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 0, "the stage file is unlinked while open");
|
||||
let parts: Vec<Vec<u8>> = (0..8u8).map(|i| vec![i; 1000 + usize::from(i)]).collect();
|
||||
let offsets: Vec<u64> = std::thread::scope(|s| {
|
||||
let handles: Vec<_> = parts.iter().map(|p| s.spawn(|| stage.append(p).unwrap())).collect();
|
||||
handles.into_iter().map(|h| h.join().unwrap()).collect()
|
||||
});
|
||||
assert_eq!(stage.len(), parts.iter().map(|p| p.len() as u64).sum::<u64>());
|
||||
for (p, &at) in parts.iter().zip(&offsets) {
|
||||
let mut buf = vec![0; p.len()];
|
||||
stage.read_exact_at(&mut buf, at).unwrap();
|
||||
assert_eq!(&buf, p);
|
||||
}
|
||||
let mut past = [0u8; 2];
|
||||
assert!(stage.read_exact_at(&mut past, stage.len() - 1).is_err(), "reads stop at what was appended");
|
||||
drop(stage);
|
||||
assert_eq!(std::fs::read_dir(&dir).unwrap().count(), 0);
|
||||
let _ = std::fs::remove_dir(&dir);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user