mirror of
https://github.com/storytold/effectcraft.git
synced 2026-11-19 16:45:28 +00:00
Footage cache (#3271): optional background localizer for network footage
Settings > Disk > Localize Network Footage (off by default), with a folder and a size limit: OpenEXR files on network shares (UNC paths) are copied one at a time on a background thread, never on the playback path, under a temporary name and renamed when complete. A copy is read only while its source has the size and modification time it had when copied and the copy is complete; entries made of it keep naming the share's file and identity. The least recently used copies go beyond the limit; Settings > Disk shows what the localizer is doing.
This commit is contained in:
1 parent
d4a86e037e
commit
043344445f
11 files changed
+491
-4
No files matched your search
@@ -401,6 +401,14 @@ impl FootageSource for ConfinedFootage {
|
||||
fn set_aux_footage(&self, paths: &[String]) {
|
||||
self.inner.set_aux_footage(paths)
|
||||
}
|
||||
/// Forwards the localizer settings to the inner source.
|
||||
fn set_localizer(&self, folder: Option<(std::path::PathBuf, u64)>) {
|
||||
self.inner.set_localizer(folder)
|
||||
}
|
||||
/// The inner source's localizer state.
|
||||
fn localize_status(&self) -> Option<String> {
|
||||
self.inner.localize_status()
|
||||
}
|
||||
/// Forwards the disk cache to the inner source.
|
||||
fn set_disk_cache(&self, cache: Option<Arc<effectcraft_engine::render::disk_cache::DiskCache>>) {
|
||||
self.inner.set_disk_cache(cache)
|
||||
|
||||
@@ -298,6 +298,10 @@ pub fn pages() -> Vec<Page> {
|
||||
Section("Media Cache"),
|
||||
s("disk.mediaCacheFolder", "Database and Cache Folder", Kind::Path, true),
|
||||
s("disk.conformedMediaFolder", "Conformed Audio Folder", Kind::Path, true),
|
||||
Section("Network Footage"),
|
||||
s("disk.localizeFootage", "Localize Network Footage", B, true),
|
||||
s("disk.localizeMaxGb", "Maximum Local Copies Size", Kind::Int(1, 100_000, "GB"), true),
|
||||
s("disk.localizeFolder", "Local Copies Folder", Kind::Path, true),
|
||||
],
|
||||
},
|
||||
Page {
|
||||
|
||||
@@ -190,6 +190,12 @@ page!(Disk {
|
||||
disk_cache_folder: String = String::new(),
|
||||
media_cache_folder: String = String::new(),
|
||||
conformed_media_folder: String = String::new(),
|
||||
/// Copy footage on network shares to a local folder in the background (#3271).
|
||||
localize_footage: bool = false,
|
||||
/// Where the local copies go (empty: beside the media cache).
|
||||
localize_folder: String = String::new(),
|
||||
/// The most the local copies take (GB).
|
||||
localize_max_gb: u32 = 100,
|
||||
});
|
||||
|
||||
page!(Memory {
|
||||
|
||||
@@ -30,6 +30,7 @@ impl Session {
|
||||
self.apply_cache_budgets();
|
||||
self.configure_disk_cache();
|
||||
self.footage.set_conform_folder(self.conformed_audio_folder());
|
||||
self.footage.set_localizer(self.localize_settings());
|
||||
// Switches Affect Nested Comps changes what precomp layers render: drop cached pixels.
|
||||
let nested = self.prefs.general.switches_affect_nested_comps;
|
||||
if self.applied_nested_switches.replace(nested).is_some_and(|was| was != nested) {
|
||||
@@ -65,6 +66,22 @@ impl Session {
|
||||
self.cache_folder(&self.prefs.disk.media_cache_folder, "Media Cache")
|
||||
}
|
||||
|
||||
/// Settings ▸ Disk ▸ Localize Network Footage (#3271): the local copies' folder and size
|
||||
/// limit, `None` when off (or there is no folder for them).
|
||||
pub fn localize_settings(&self) -> Option<(std::path::PathBuf, u64)> {
|
||||
let d = &self.prefs.disk;
|
||||
if !d.localize_footage || cfg!(target_arch = "wasm32") {
|
||||
return None;
|
||||
}
|
||||
let folder = self.cache_folder(&d.localize_folder, "Local Footage Copies")?;
|
||||
Some((folder, u64::from(d.localize_max_gb.max(1)) << 30))
|
||||
}
|
||||
|
||||
/// What the footage localizer is doing, as a line for Settings ▸ Disk (`None`: off).
|
||||
pub fn localize_status(&self) -> Option<String> {
|
||||
self.footage.localize_status()
|
||||
}
|
||||
|
||||
/// Settings ▸ Disk ▸ Conformed Audio Folder (decoded footage audio, see
|
||||
/// `MediaPool::set_conform_folder`).
|
||||
pub fn conformed_audio_folder(&self) -> Option<std::path::PathBuf> {
|
||||
|
||||
@@ -939,3 +939,35 @@ fn the_plane_store_is_trimmed_by_the_accountant() {
|
||||
assert_eq!(store.cached(), ["/3271-acc-b.exr", "/3271-acc-c.exr"], "held for effects, and in use");
|
||||
drop(in_use);
|
||||
}
|
||||
|
||||
/// #3271: with the localizer on, a network OpenEXR is copied in the background and then read
|
||||
/// from its local copy, with the same pixels; what is read is still named by the share's path
|
||||
/// and identity (caches key it the same either way).
|
||||
#[test]
|
||||
fn a_localized_network_file_is_read_from_its_copy() {
|
||||
let _serial = crate::localize::TEST_SERIAL.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
let root = std::env::temp_dir().join(format!("ec-3271-localized-{}", std::process::id()));
|
||||
let (share, local) = (root.join("share"), root.join("local"));
|
||||
std::fs::create_dir_all(&share).unwrap();
|
||||
crate::localize::treat_as_network(&share);
|
||||
let file = share.join("plate_0001.exr");
|
||||
std::fs::write(&file, multipart()).unwrap();
|
||||
let f = crate::probe(&file).unwrap();
|
||||
let pool = crate::MediaPool::new();
|
||||
pool.set_localizer(Some((local.clone(), 1 << 30)));
|
||||
let from_share = pool.frame_at(&f, Tick::ZERO).unwrap();
|
||||
let until = std::time::Instant::now() + std::time::Duration::from_secs(10);
|
||||
while crate::localize::status().is_some_and(|s| s.files == 0) && std::time::Instant::now() < until {
|
||||
std::thread::sleep(std::time::Duration::from_millis(10));
|
||||
}
|
||||
assert!(pool.localize_status().is_some_and(|s| s.contains("1 files")), "{:?}", pool.localize_status());
|
||||
pool.forget(&f.path);
|
||||
let before = crate::opens_under(&local);
|
||||
let (img, read) = crate::reads::captured(|| pool.frame_at(&f, Tick::ZERO).unwrap());
|
||||
assert!(crate::opens_under(&local) > before, "read from the local copy");
|
||||
assert_eq!(bits(&img), bits(&from_share));
|
||||
let stamp = FileStamp::of_metadata(&std::fs::metadata(&file).unwrap());
|
||||
assert!(read.files().iter().any(|r| *r.path == *f.path && r.stamp == stamp), "named by the share's file: {read:?}");
|
||||
pool.set_localizer(None);
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
}
|
||||
@@ -37,6 +37,9 @@ pub(crate) enum ExrSource {
|
||||
Bytes(Arc<[u8]>),
|
||||
/// A file, opened for each read and read in place.
|
||||
File(std::path::PathBuf),
|
||||
/// A network file read from its verified local copy ([`crate::localize`]): it opens the
|
||||
/// copy, but its identity (and every entry made of it) is the original's.
|
||||
Localized { original: std::path::PathBuf, copy: std::path::PathBuf, stamp: FileStamp },
|
||||
}
|
||||
|
||||
/// An opened [`ExrSource`].
|
||||
@@ -51,7 +54,7 @@ impl ExrSource {
|
||||
pub(crate) fn stamp(&self) -> std::io::Result<FileStamp> {
|
||||
match self {
|
||||
ExrSource::Bytes(_) => Ok(FileStamp::Virtual),
|
||||
ExrSource::File(path) => std::fs::metadata(path).map(|m| FileStamp::of_metadata(&m)),
|
||||
ExrSource::File(path) | ExrSource::Localized { original: path, .. } => std::fs::metadata(path).map(|m| FileStamp::of_metadata(&m)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -70,6 +73,18 @@ impl ExrSource {
|
||||
}
|
||||
Ok(Opened { len: meta.len(), stamp: FileStamp::of_metadata(&meta), reader: Box::new(file) })
|
||||
}
|
||||
ExrSource::Localized { copy, stamp, .. } => {
|
||||
#[cfg(test)]
|
||||
crate::count_open(copy);
|
||||
let file = std::fs::File::open(copy)?;
|
||||
let meta = file.metadata()?;
|
||||
// (the copy must still be complete: its size is the original's)
|
||||
let complete = matches!(stamp, FileStamp::Native { len, .. } if *len == meta.len());
|
||||
if !meta.is_file() || !complete {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, format!("{}: not a complete copy", copy.display())));
|
||||
}
|
||||
Ok(Opened { len: meta.len(), stamp: *stamp, reader: Box::new(file) })
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,6 +25,8 @@ mod exr_planes;
|
||||
mod exr_read;
|
||||
mod gif_anim;
|
||||
mod layered;
|
||||
mod localize;
|
||||
pub use localize::{LocalizeSettings, LocalizeStatus};
|
||||
pub use layered::vector_doc;
|
||||
mod pool;
|
||||
mod probe;
|
||||
|
||||
@@ -0,0 +1,351 @@
|
||||
//! The background footage localizer (#3271, Settings ▸ Disk ▸ Localize Network Footage, off by
|
||||
//! default): footage on a network share is copied, file by file, to a local folder while the
|
||||
//! user works, so later reads come from the local disk.
|
||||
//!
|
||||
//! - **Never on the playback path.** A read asks for a copy ([`request`]) and goes on reading
|
||||
//! the share; one background thread copies, one file at a time.
|
||||
//! - **Used only when complete and verified.** A copy is made under a temporary name, renamed
|
||||
//! when complete, and its size and the source's identity (size and modification time) as they
|
||||
//! were when it was copied are recorded. It is read only while the source still has that
|
||||
//! identity and the copy that size ([`local_copy`]); a source that changed is read from the
|
||||
//! share (and copied again).
|
||||
//! - **Readers keep naming the source.** Entries made of a localized file list the share's path
|
||||
//! and identity (see `ExrSource::Localized`), so caches are keyed the same either way.
|
||||
//! - **Bounded.** The copies take at most the size limit; the least recently used go first.
|
||||
//! - **Visible.** [`status`] says how many files are local, their size, what is queued and what
|
||||
//! is being copied.
|
||||
|
||||
use std::collections::{HashMap, HashSet, VecDeque};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Condvar, Mutex, PoisonError};
|
||||
|
||||
use effectcraft_render::reads::FileStamp;
|
||||
|
||||
/// Where copies go and how much they may take.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct LocalizeSettings {
|
||||
pub folder: PathBuf,
|
||||
pub max_bytes: u64,
|
||||
}
|
||||
|
||||
/// What the localizer is doing (Settings ▸ Disk shows it).
|
||||
#[derive(Clone, Debug, Default, PartialEq)]
|
||||
pub struct LocalizeStatus {
|
||||
/// Complete local copies and their bytes.
|
||||
pub files: usize,
|
||||
pub bytes: u64,
|
||||
pub max_bytes: u64,
|
||||
/// Files waiting to be copied.
|
||||
pub queued: usize,
|
||||
/// The file being copied now.
|
||||
pub copying: Option<String>,
|
||||
/// Files that were too large for the limit, or failed to copy.
|
||||
pub skipped: usize,
|
||||
}
|
||||
|
||||
/// A complete local copy.
|
||||
struct Copy {
|
||||
local: PathBuf,
|
||||
/// The copy's size and the source's identity when it was copied.
|
||||
len: u64,
|
||||
source: FileStamp,
|
||||
last_use: u64,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct State {
|
||||
settings: Option<LocalizeSettings>,
|
||||
copies: HashMap<PathBuf, Copy>,
|
||||
queue: VecDeque<PathBuf>,
|
||||
queued: HashSet<PathBuf>,
|
||||
bytes: u64,
|
||||
clock: u64,
|
||||
copying: Option<PathBuf>,
|
||||
skipped: HashSet<PathBuf>,
|
||||
worker: bool,
|
||||
/// Bumped with each new setting: a copy started under another one is thrown away.
|
||||
epoch: u64,
|
||||
}
|
||||
|
||||
/// At most this many files wait to be copied (older requests are dropped first).
|
||||
const QUEUE_LIMIT: usize = 4096;
|
||||
|
||||
fn state() -> &'static (Mutex<State>, Condvar) {
|
||||
static STATE: std::sync::OnceLock<(Mutex<State>, Condvar)> = std::sync::OnceLock::new();
|
||||
STATE.get_or_init(Default::default)
|
||||
}
|
||||
|
||||
fn lock() -> std::sync::MutexGuard<'static, State> {
|
||||
state().0.lock().unwrap_or_else(PoisonError::into_inner)
|
||||
}
|
||||
|
||||
/// Turn the localizer on with `settings`, or off (`None`): the copies made so far stay on disk
|
||||
/// but are no longer read; a new folder starts empty.
|
||||
pub fn configure(settings: Option<LocalizeSettings>) {
|
||||
let mut st = lock();
|
||||
if st.settings == settings {
|
||||
return;
|
||||
}
|
||||
st.epoch = st.epoch.wrapping_add(1);
|
||||
if st.settings.as_ref().map(|s| &s.folder) != settings.as_ref().map(|s| &s.folder) {
|
||||
st.copies.clear();
|
||||
st.bytes = 0;
|
||||
}
|
||||
st.queue.clear();
|
||||
st.queued.clear();
|
||||
st.skipped.clear();
|
||||
st.settings = settings;
|
||||
if let Some(max) = st.settings.as_ref().map(|s| s.max_bytes) {
|
||||
evict_to(&mut st, max);
|
||||
}
|
||||
}
|
||||
|
||||
/// What the localizer is doing; `None` when it is off.
|
||||
pub fn status() -> Option<LocalizeStatus> {
|
||||
let st = lock();
|
||||
let s = st.settings.as_ref()?;
|
||||
Some(LocalizeStatus {
|
||||
files: st.copies.len(),
|
||||
bytes: st.bytes,
|
||||
max_bytes: s.max_bytes,
|
||||
queued: st.queue.len(),
|
||||
copying: st.copying.as_ref().map(|p| p.to_string_lossy().to_string()),
|
||||
skipped: st.skipped.len(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Paths under these roots count as network footage (tests: a local folder standing in for a
|
||||
/// share).
|
||||
#[cfg(test)]
|
||||
static NETWORK_ROOTS: Mutex<Vec<PathBuf>> = Mutex::new(Vec::new());
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn treat_as_network(root: &Path) {
|
||||
NETWORK_ROOTS.lock().unwrap_or_else(PoisonError::into_inner).push(root.to_path_buf());
|
||||
}
|
||||
|
||||
/// Is `path` on a network share: a UNC path (`\\server\share\…`, `//server/share/…`). Drive
|
||||
/// letters mapped to a share are not told apart from local disks.
|
||||
pub(crate) fn is_network(path: &Path) -> bool {
|
||||
#[cfg(test)]
|
||||
if NETWORK_ROOTS.lock().unwrap_or_else(PoisonError::into_inner).iter().any(|r| path.starts_with(r)) {
|
||||
return true;
|
||||
}
|
||||
let s = path.to_string_lossy();
|
||||
s.starts_with("\\\\") || s.starts_with("//")
|
||||
}
|
||||
|
||||
/// Ask for a local copy of the network file `original` (nothing when the localizer is off, the
|
||||
/// file is local, already copied, queued or skipped). Returns at once.
|
||||
pub(crate) fn request(original: &Path) {
|
||||
if !is_network(original) {
|
||||
return;
|
||||
}
|
||||
let mut st = lock();
|
||||
if st.settings.is_none() || st.copies.contains_key(original) || st.queued.contains(original) || st.skipped.contains(original) {
|
||||
return;
|
||||
}
|
||||
if st.queue.len() >= QUEUE_LIMIT
|
||||
&& let Some(old) = st.queue.pop_front()
|
||||
{
|
||||
st.queued.remove(&old);
|
||||
}
|
||||
st.queue.push_back(original.to_path_buf());
|
||||
st.queued.insert(original.to_path_buf());
|
||||
start_worker(&mut st);
|
||||
state().1.notify_all();
|
||||
}
|
||||
|
||||
/// The complete local copy of `original` when its source still has identity `now` (its size and
|
||||
/// modification time when it was copied) and the copy its size; `None` otherwise (read the
|
||||
/// source). A copy whose source changed is forgotten (and copied again when asked for).
|
||||
pub(crate) fn local_copy(original: &Path, now: &FileStamp) -> Option<PathBuf> {
|
||||
let mut st = lock();
|
||||
st.settings.as_ref()?;
|
||||
st.clock += 1;
|
||||
let clock = st.clock;
|
||||
let c = st.copies.get_mut(original)?;
|
||||
let ok = c.source == *now && std::fs::metadata(&c.local).is_ok_and(|m| m.is_file() && m.len() == c.len);
|
||||
if ok {
|
||||
c.last_use = clock;
|
||||
return Some(c.local.clone());
|
||||
}
|
||||
if let Some(gone) = st.copies.remove(original) {
|
||||
st.bytes = st.bytes.saturating_sub(gone.len);
|
||||
let _ = std::fs::remove_file(&gone.local);
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
fn start_worker(st: &mut State) {
|
||||
if st.worker {
|
||||
return;
|
||||
}
|
||||
st.worker = std::thread::Builder::new().name("ec-localize".into()).spawn(worker).is_ok();
|
||||
}
|
||||
|
||||
/// (no threads in the browser: nothing is copied there)
|
||||
#[cfg(target_arch = "wasm32")]
|
||||
fn start_worker(_st: &mut State) {}
|
||||
|
||||
/// Drop the least recently used copies until they take at most `max` bytes.
|
||||
fn evict_to(st: &mut State, max: u64) {
|
||||
while st.bytes > max {
|
||||
let Some(oldest) = st.copies.iter().min_by_key(|(_, c)| c.last_use).map(|(p, _)| p.clone()) else { break };
|
||||
if let Some(c) = st.copies.remove(&oldest) {
|
||||
st.bytes = st.bytes.saturating_sub(c.len);
|
||||
let _ = std::fs::remove_file(&c.local);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The copies' folder name of `original`: a hash of its path, then its file name.
|
||||
fn local_name(original: &Path) -> String {
|
||||
use std::hash::{Hash, Hasher};
|
||||
let mut h = std::collections::hash_map::DefaultHasher::new();
|
||||
original.hash(&mut h);
|
||||
let name = original.file_name().map(|n| n.to_string_lossy().to_string()).unwrap_or_default();
|
||||
format!("{:016x}_{name}", h.finish())
|
||||
}
|
||||
|
||||
/// The copying thread: one file at a time, until the queue is empty for a while.
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
fn worker() {
|
||||
let (m, cv) = state();
|
||||
loop {
|
||||
let (job, settings, epoch) = {
|
||||
let mut st = m.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
loop {
|
||||
if let (Some(p), Some(s)) = (st.queue.front().cloned(), st.settings.clone()) {
|
||||
st.queue.pop_front();
|
||||
st.queued.remove(&p);
|
||||
st.copying = Some(p.clone());
|
||||
break (p, s, st.epoch);
|
||||
}
|
||||
let (g, timeout) = match cv.wait_timeout(st, std::time::Duration::from_secs(30)) {
|
||||
Ok(r) => r,
|
||||
Err(p) => p.into_inner(),
|
||||
};
|
||||
st = g;
|
||||
if timeout.timed_out() && st.queue.is_empty() {
|
||||
st.worker = false;
|
||||
return;
|
||||
}
|
||||
}
|
||||
};
|
||||
let copied = copy_file(&job, &settings);
|
||||
let mut st = m.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
st.copying = None;
|
||||
match copied {
|
||||
Some((local, len, source)) if st.epoch == epoch => {
|
||||
if let Some(old) = st.copies.remove(&job) {
|
||||
st.bytes = st.bytes.saturating_sub(old.len);
|
||||
}
|
||||
st.clock += 1;
|
||||
let last_use = st.clock;
|
||||
st.copies.insert(job, Copy { local, len, source, last_use });
|
||||
st.bytes = st.bytes.saturating_add(len);
|
||||
evict_to(&mut st, settings.max_bytes);
|
||||
}
|
||||
Some((local, _, _)) => {
|
||||
let _ = std::fs::remove_file(local);
|
||||
}
|
||||
None => {
|
||||
st.skipped.insert(job);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Copy `original` into the folder of `settings`: under a temporary name, then renamed when
|
||||
/// complete; its modification time set to the source's. The copy, its size and the source's
|
||||
/// identity, or `None` (too large for the limit, unreadable, or changed while it was copied).
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
fn copy_file(original: &Path, settings: &LocalizeSettings) -> Option<(PathBuf, u64, FileStamp)> {
|
||||
let meta = std::fs::metadata(original).ok().filter(std::fs::Metadata::is_file)?;
|
||||
if meta.len() > settings.max_bytes {
|
||||
return None;
|
||||
}
|
||||
let before = FileStamp::of_metadata(&meta);
|
||||
std::fs::create_dir_all(&settings.folder).ok()?;
|
||||
let local = settings.folder.join(local_name(original));
|
||||
let tmp = settings.folder.join(format!("{}.part", local_name(original)));
|
||||
let copied = (|| -> std::io::Result<u64> {
|
||||
let mut from = std::fs::File::open(original)?;
|
||||
let mut to = std::fs::File::create(&tmp)?;
|
||||
let n = std::io::copy(&mut from, &mut to)?;
|
||||
to.sync_all()?;
|
||||
if let Ok(t) = meta.modified() {
|
||||
to.set_modified(t)?;
|
||||
}
|
||||
Ok(n)
|
||||
})();
|
||||
let after = std::fs::metadata(original).ok().map(|m| FileStamp::of_metadata(&m));
|
||||
match copied {
|
||||
Ok(n) if n == meta.len() && after.as_ref() == Some(&before) && std::fs::rename(&tmp, &local).is_ok() => Some((local, n, before)),
|
||||
_ => {
|
||||
let _ = std::fs::remove_file(&tmp);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The localizer state is process-wide: tests that use it take turns.
|
||||
#[cfg(test)]
|
||||
pub(crate) static TEST_SERIAL: Mutex<()> = Mutex::new(());
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn wait_for(f: impl Fn(&LocalizeStatus) -> bool) -> LocalizeStatus {
|
||||
let until = std::time::Instant::now() + std::time::Duration::from_secs(10);
|
||||
loop {
|
||||
let s = status().unwrap_or_default();
|
||||
if f(&s) || std::time::Instant::now() > until {
|
||||
return s;
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_millis(10));
|
||||
}
|
||||
}
|
||||
|
||||
/// A network file is copied in the background and read locally only while it matches its
|
||||
/// source; a source that changed is read from the share again; files over the limit are not
|
||||
/// copied; local files are never queued.
|
||||
#[test]
|
||||
fn network_files_are_copied_and_used_only_while_they_match() {
|
||||
let _serial = TEST_SERIAL.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
let root = std::env::temp_dir().join(format!("ec-3271-localize-{}", std::process::id()));
|
||||
let (share, local) = (root.join("share"), root.join("local"));
|
||||
std::fs::create_dir_all(&share).unwrap();
|
||||
treat_as_network(&share);
|
||||
let file = share.join("plate_0001.exr");
|
||||
std::fs::write(&file, vec![7u8; 4096]).unwrap();
|
||||
let stamp = |p: &Path| FileStamp::of_metadata(&std::fs::metadata(p).unwrap());
|
||||
configure(Some(LocalizeSettings { folder: local.clone(), max_bytes: 1 << 20 }));
|
||||
assert!(local_copy(&file, &stamp(&file)).is_none(), "not copied yet");
|
||||
request(&file);
|
||||
let s = wait_for(|s| s.files == 1);
|
||||
assert_eq!((s.files, s.bytes, s.queued), (1, 4096, 0), "{s:?}");
|
||||
let copy = local_copy(&file, &stamp(&file)).expect("a verified copy");
|
||||
assert!(copy.starts_with(&local));
|
||||
assert_eq!(std::fs::read(©).unwrap(), vec![7u8; 4096]);
|
||||
// The source changes: the copy is not read any more.
|
||||
std::fs::write(&file, vec![8u8; 5000]).unwrap();
|
||||
assert!(local_copy(&file, &stamp(&file)).is_none());
|
||||
// Over the limit: skipped.
|
||||
let big = share.join("big.exr");
|
||||
std::fs::write(&big, vec![1u8; 4096]).unwrap();
|
||||
configure(Some(LocalizeSettings { folder: local.clone(), max_bytes: 1000 }));
|
||||
request(&big);
|
||||
let s = wait_for(|s| s.skipped == 1);
|
||||
assert_eq!((s.skipped, s.files), (1, 0), "{s:?}");
|
||||
// A local file is never queued.
|
||||
request(&root.join("elsewhere.exr"));
|
||||
assert_eq!(status().unwrap().queued, 0);
|
||||
configure(None);
|
||||
assert!(status().is_none());
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
}
|
||||
}
|
||||
@@ -892,10 +892,23 @@ impl Inner {
|
||||
|
||||
/// Where the OpenEXR file at `path` is read from: registered bytes, else the file.
|
||||
fn exr_source(&self, path: &str) -> crate::exr_read::ExrSource {
|
||||
match lock(&self.files).get(path) {
|
||||
Some(b) => crate::exr_read::ExrSource::Bytes(b.clone()),
|
||||
None => crate::exr_read::ExrSource::File(std::path::PathBuf::from(path)),
|
||||
use crate::exr_read::ExrSource;
|
||||
if let Some(b) = lock(&self.files).get(path) {
|
||||
return ExrSource::Bytes(b.clone());
|
||||
}
|
||||
let file = std::path::PathBuf::from(path);
|
||||
// A network file: copied in the background when the localizer is on, and read from its
|
||||
// copy once that is complete and the file unchanged (#3271).
|
||||
if crate::localize::is_network(&file) && crate::localize::status().is_some() {
|
||||
crate::localize::request(&file);
|
||||
if let Ok(meta) = std::fs::metadata(&file) {
|
||||
let stamp = FileStamp::of_metadata(&meta);
|
||||
if let Some(copy) = crate::localize::local_copy(&file, &stamp) {
|
||||
return ExrSource::Localized { original: file, copy, stamp };
|
||||
}
|
||||
}
|
||||
}
|
||||
ExrSource::File(file)
|
||||
}
|
||||
|
||||
/// The bytes of the file at `path` and its identity as read ([`FileStamp::Missing`] when it
|
||||
@@ -1495,6 +1508,24 @@ impl FootageSource for MediaPool {
|
||||
wanted.learned.clear();
|
||||
}
|
||||
|
||||
fn set_localizer(&self, folder: Option<(std::path::PathBuf, u64)>) {
|
||||
crate::localize::configure(folder.map(|(folder, max_bytes)| crate::localize::LocalizeSettings { folder, max_bytes }));
|
||||
}
|
||||
|
||||
fn localize_status(&self) -> Option<String> {
|
||||
let s = crate::localize::status()?;
|
||||
let gb = |b: u64| b as f64 / f64::from(1u32 << 30);
|
||||
let copying = s.copying.as_deref().map(|p| format!(", copying {p}")).unwrap_or_default();
|
||||
Some(format!(
|
||||
"Local copies: {} files, {:.1} of {:.1} GB; {} queued, {} skipped{copying}",
|
||||
s.files,
|
||||
gb(s.bytes),
|
||||
gb(s.max_bytes),
|
||||
s.queued,
|
||||
s.skipped
|
||||
))
|
||||
}
|
||||
|
||||
fn set_disk_cache(&self, cache: Option<Arc<effectcraft_render::disk_cache::DiskCache>>) {
|
||||
crate::exr_planes::set_disk_cache(cache.as_ref());
|
||||
}
|
||||
|
||||
@@ -82,6 +82,14 @@ pub trait FootageSource: Send + Sync {
|
||||
/// decode costly footage keep what they decoded there too (#3271: OpenEXR channels).
|
||||
/// Sources without such a tier ignore it.
|
||||
fn set_disk_cache(&self, _cache: Option<Arc<crate::disk_cache::DiskCache>>) {}
|
||||
/// Settings ▸ Disk ▸ Localize Network Footage: copy footage on network shares to `folder`
|
||||
/// in the background, up to the byte limit, and read the copies once they are complete and
|
||||
/// verified (`None`: off). Sources that don't read files ignore it.
|
||||
fn set_localizer(&self, _folder: Option<(std::path::PathBuf, u64)>) {}
|
||||
/// What the localizer is doing, as a line to show (`None`: off or not supported).
|
||||
fn localize_status(&self) -> Option<String> {
|
||||
None
|
||||
}
|
||||
/// The parsed 3D model of a [`effectcraft_project::FootageKind::Model`] item (Advanced 3D).
|
||||
fn model(&self, _item: ItemId, _footage: &Footage) -> Option<Arc<effectcraft_model::Model>> {
|
||||
None
|
||||
@@ -222,6 +230,12 @@ impl FootageSource for CancellableFootage<'_> {
|
||||
fn set_disk_cache(&self, cache: Option<Arc<crate::disk_cache::DiskCache>>) {
|
||||
self.inner.set_disk_cache(cache)
|
||||
}
|
||||
fn set_localizer(&self, folder: Option<(std::path::PathBuf, u64)>) {
|
||||
self.inner.set_localizer(folder)
|
||||
}
|
||||
fn localize_status(&self) -> Option<String> {
|
||||
self.inner.localize_status()
|
||||
}
|
||||
fn model(&self, item: ItemId, footage: &Footage) -> Option<Arc<effectcraft_model::Model>> {
|
||||
self.inner.model(item, footage)
|
||||
}
|
||||
|
||||
@@ -238,6 +238,13 @@ fn page_ui(app: &mut EffectcraftApp, ui: &mut egui::Ui, page: &Page, cur: &Value
|
||||
if key == "disk.diskCacheMaxGb" {
|
||||
disk_cache_limit_ui(app, ui, t);
|
||||
}
|
||||
if key == "disk.localizeFootage"
|
||||
&& let Some(line) = app.session.localize_status()
|
||||
{
|
||||
// (the localizer's state: local copies, what is queued and being copied)
|
||||
let r = ui.label(RichText::new(line).color(t.text_dim));
|
||||
app.auto.add("settings.disk.localizeStatus", r.rect, "localize status");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user