diff --git a/crates/automation/src/confine.rs b/crates/automation/src/confine.rs index 0bde6d8ca..46e656d39 100644 --- a/crates/automation/src/confine.rs +++ b/crates/automation/src/confine.rs @@ -406,11 +406,11 @@ impl FootageSource for ConfinedFootage { self.inner.set_aux_footage(paths) } /// Forwards the localizer settings to the inner source. - fn set_localizer(&self, folder: Option<(std::path::PathBuf, u64)>) { - self.inner.set_localizer(folder) + fn set_localizer(&self, config: Option) { + self.inner.set_localizer(config) } /// The inner source's localizer state. - fn localize_status(&self) -> Option { + fn localize_status(&self) -> Option { self.inner.localize_status() } /// Forwards the disk cache to the inner source. diff --git a/crates/effects/src/lib.rs b/crates/effects/src/lib.rs index 2bc54ba16..5e5a97343 100644 --- a/crates/effects/src/lib.rs +++ b/crates/effects/src/lib.rs @@ -116,8 +116,8 @@ pub use time_fx::{PixelMotion, TimewarpPlan, TwStep, pmb_plan, posterized_time, pub use ocio::color_stabilizer_maps; // GPU effects, part B (3D Channel, Immersive Video, colour management, simulation render passes). pub use channel3d::{ - CRYPTO_LAYERS, CryptoAux, MATTE_ONLY_IN_ALPHA, aux_needs, aux_point, pixel_margin, crypto_aux_image, crypto_aux_image_with, crypto_layer_choices, crypto_layer_menu, - crypto_layer_name, crypto_layer_named, id_color, matte_only_px, reads_aux, selection_hashes, + CRYPTO_LAYERS, CryptoAux, MATTE_ONLY_IN_ALPHA, aux_needs, aux_point, crypto_aux_image, crypto_aux_image_with, crypto_layer_choices, crypto_layer_menu, + crypto_layer_name, crypto_layer_named, id_color, matte_only_px, pixel_margin, reads_aux, selection_hashes, }; pub use ocio::{ColorOp, Straight, Tf, color_program}; pub use sim::{Acc, Blob, Bubble, Post, Shape, Sprite, SpritePlan, Tint, bubble_list, bubble_sample_bounds, drizzle_drops, mercury_blobs, sprite_plan}; diff --git a/crates/engine/src/footage/footage_reload.rs b/crates/engine/src/footage/footage_reload.rs index e56e16a96..77989b16c 100644 --- a/crates/engine/src/footage/footage_reload.rs +++ b/crates/engine/src/footage/footage_reload.rs @@ -335,6 +335,8 @@ pub(crate) fn forget_media(s: &mut Session, paths: &BTreeSet) { for p in paths { s.footage.forget(p); } + // (thumbnails look at no file: a reload gives them new keys, #3271) + s.footage_reloads.fetch_add(1, std::sync::atomic::Ordering::Relaxed); s.layer_cache.clear(); s.events.push(crate::Event::PurgeCaches); } diff --git a/crates/engine/src/settings/prefs/dialog.rs b/crates/engine/src/settings/prefs/dialog.rs index a14f37955..00e7a057a 100644 --- a/crates/engine/src/settings/prefs/dialog.rs +++ b/crates/engine/src/settings/prefs/dialog.rs @@ -302,6 +302,7 @@ pub fn pages() -> Vec { s("disk.localizeFootage", "Localize Network Footage", B, true), s("disk.localizeMaxGb", "Maximum Local Copies Size", Kind::Int(1, 100_000, "GB"), true), s("disk.localizeFolder", "Local Copies Folder", Kind::Path, true), + s("disk.networkFootageLocations", "Network Footage Locations", Kind::Text, true), ], }, Page { diff --git a/crates/engine/src/settings/prefs/sections.rs b/crates/engine/src/settings/prefs/sections.rs index 1efec5aee..0d31702ee 100644 --- a/crates/engine/src/settings/prefs/sections.rs +++ b/crates/engine/src/settings/prefs/sections.rs @@ -195,7 +195,9 @@ page!(Disk { /// Where the local copies go (empty: beside the media cache). localize_folder: String = String::new(), /// The most the local copies take (GB). - localize_max_gb: u32 = 100, + localize_max_gb: u32 = 20, + /// Path prefixes counted as network footage besides UNC paths (`P:\`), `;`-separated. + network_footage_locations: String = String::new(), }); page!(Memory { diff --git a/crates/engine/src/settings/session_settings.rs b/crates/engine/src/settings/session_settings.rs index 65267d5a2..77010a8cf 100644 --- a/crates/engine/src/settings/session_settings.rs +++ b/crates/engine/src/settings/session_settings.rs @@ -68,17 +68,37 @@ impl Session { /// Settings ▸ Disk ▸ Localize Network Footage (#3271): the local copies' folder and size /// limit, `None` when off (or there is no folder for them). - pub fn localize_settings(&self) -> Option<(std::path::PathBuf, u64)> { + pub fn localize_settings(&self) -> Option { let d = &self.prefs.disk; if !d.localize_footage || cfg!(target_arch = "wasm32") { return None; } - let folder = self.cache_folder(&d.localize_folder, "Local Footage Copies")?; - Some((folder, u64::from(d.localize_max_gb.max(1)) << 30)) + // Without a folder of their own, the copies go beside the disk cache's folder (a disk + // the user chose for caches), else beside the platform cache folder. + let folder = if d.localize_folder.trim().is_empty() && !d.disk_cache_folder.trim().is_empty() { + let dc = std::path::PathBuf::from(d.disk_cache_folder.trim()); + dc.parent().map_or_else(|| dc.join("Local Footage Copies"), |p| p.join("Local Footage Copies")) + } else { + self.cache_folder(&d.localize_folder, "Local Footage Copies")? + }; + let space_of = folder.clone(); + Some(effectcraft_render::LocalizeConfig { + folder, + max_bytes: u64::from(d.localize_max_gb.max(1)) << 30, + network: network_locations(&d.network_footage_locations), + disk_space: Some(std::sync::Arc::new(move || { + // (the folder may not exist yet: its nearest existing parent's disk) + let mut at = space_of.as_path(); + while !at.exists() { + at = at.parent()?; + } + crate::sysinfo::disk_space(at) + })), + }) } - /// What the footage localizer is doing, as a line for Settings ▸ Disk (`None`: off). - pub fn localize_status(&self) -> Option { + /// What the footage localizer is doing, for Settings ▸ Disk (`None`: off). + pub fn localize_status(&self) -> Option { self.footage.localize_status() } @@ -99,7 +119,7 @@ impl Session { let accountant = effectcraft_render::memory::Accountant::global(); let layers: std::sync::Arc = self.layer_cache.clone(); accountant.register(&layers); - accountant.set_budget(b.layer.saturating_add(b.media).saturating_add(b.preview)); + accountant.set_budget(b.layer.saturating_add(b.media).saturating_add(b.preview).saturating_add(effectcraft_render::memory::PLANE_BUDGET)); } /// Take the latest reading of the system's memory and start the next one in the background @@ -403,3 +423,9 @@ impl Session { } } } + +/// Settings ▸ Disk ▸ Network Footage Locations: path prefixes separated by `;` or new lines +/// (`P:\; Q:\renders`), each counted as network footage (#3271). +pub fn network_locations(setting: &str) -> Vec { + setting.split([';', '\n']).map(str::trim).filter(|s| !s.is_empty()).map(std::path::PathBuf::from).collect() +} diff --git a/crates/engine/src/tests.rs b/crates/engine/src/tests.rs index 8576e4a9a..38ec0e30a 100644 --- a/crates/engine/src/tests.rs +++ b/crates/engine/src/tests.rs @@ -59,9 +59,9 @@ mod tail; mod text; mod text_edit; mod three_d; +mod thumbnails; mod timeline; mod timeline_interchange; pub(crate) mod track; mod vector_import; mod viewer; -mod thumbnails; diff --git a/crates/engine/src/tests/thumbnails.rs b/crates/engine/src/tests/thumbnails.rs index 34595d9c7..abd6ca906 100644 --- a/crates/engine/src/tests/thumbnails.rs +++ b/crates/engine/src/tests/thumbnails.rs @@ -28,3 +28,40 @@ fn thumbnail_keys_follow_the_content() { s.forget_decoded(&effectcraft_project::Footage::default()); assert_ne!(key(&s), before, "a reload gives a new key"); } + +/// #3271 review: File ▸ Reload Footage gives the footage's thumbnail a new key (the file may +/// have changed without the project knowing). +#[test] +fn reload_footage_gives_thumbnails_a_new_key() { + struct Probe; + impl crate::Importer for Probe { + fn probe(&self, path: &str) -> Result { + Ok(effectcraft_project::Footage { + path: path.into(), + kind: effectcraft_project::FootageKind::Still, + width: 1, + height: 1, + has_video: true, + ..Default::default() + }) + } + } + let dir = std::env::temp_dir().join(format!("ec-3271-thumb-reload-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let file = dir.join("a.png"); + std::fs::write(&file, b"aa").unwrap(); + let mut s = Session { importer: Some(std::sync::Arc::new(Probe)), ..Default::default() }; + let f = effectcraft_project::Footage { + path: file.to_string_lossy().to_string(), + kind: effectcraft_project::FootageKind::Still, + width: 1, + height: 1, + has_video: true, + ..Default::default() + }; + let id = s.edit("add", None, |p, _| Ok(p.add_item("a", effectcraft_color::Label::None, None, effectcraft_project::ItemKind::Footage(f)))).unwrap(); + let before = s.thumbnail_key(id, Tick::ZERO, 192).unwrap(); + s.execute_checked("file.reloadFootage", json!({"item": id.0})).unwrap(); + assert_ne!(s.thumbnail_key(id, Tick::ZERO, 192).unwrap(), before); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/crates/media/src/exr_channels.rs b/crates/media/src/exr_channels.rs index c3ed51a60..42c3e47eb 100644 --- a/crates/media/src/exr_channels.rs +++ b/crates/media/src/exr_channels.rs @@ -84,10 +84,7 @@ pub(crate) enum Rgb { /// pictures waiting for their effects, decodes in progress), so memory can go above it while /// they are. (Measured on such a sequence decoded one frame at a time: 1 GiB held peak memory /// about 1 GB lower than 2 GiB, at the same speed.) -#[cfg(not(target_arch = "wasm32"))] -pub const CACHE_BUDGET: usize = 1 << 30; -#[cfg(target_arch = "wasm32")] -pub const CACHE_BUDGET: usize = 512 << 20; +pub const CACHE_BUDGET: usize = effectcraft_render::memory::PLANE_BUDGET; /// The index of an EXR file's bytes and the planes of `ids` (all when `None`). `None` when it is /// not a readable EXR or the planes need too much memory. diff --git a/crates/media/src/exr_planes.rs b/crates/media/src/exr_planes.rs index 4c47f985c..f295366e2 100644 --- a/crates/media/src/exr_planes.rs +++ b/crates/media/src/exr_planes.rs @@ -163,6 +163,10 @@ struct Flight { /// The disk cache's drop generation before the file was opened for this request (#3191: a /// plane entry made of a file reloaded since is refused). since: u64, + /// The cancel flag and decoding hints of the request that asked for the decode: a decode + /// thread that claims it decodes under them (its own being called off doesn't call off the + /// asker's decode). + hints: filmcraft_media::cancel::Context, } /// [`Flight::phase`] values. @@ -197,6 +201,7 @@ impl Flight { handed_ready: Condvar::new(), base, since, + hints: filmcraft_media::cancel::Context::current(), }) } @@ -291,7 +296,7 @@ impl Owning<'_> { self.ended = true; let outcome = self.store.finish(self.path, self.level, self.flight, outcome); self.flight.end(outcome.clone()); - effectcraft_render::memory::Accountant::global().balance(); + effectcraft_render::memory::Accountant::global().note_growth(); outcome } } @@ -354,9 +359,11 @@ pub(crate) struct DecodeRecord { #[cfg(test)] static DECODES: Mutex> = Mutex::new(Vec::new()); -/// Paths whose queued decode a decode thread took over (tests). +/// Paths whose queued decode a decode thread took over (tests), and a signal for each. #[cfg(test)] pub(crate) static CLAIMS: Mutex> = Mutex::new(Vec::new()); +#[cfg(test)] +pub(crate) static CLAIM_SIGNAL: Condvar = Condvar::new(); /// The decodes of `path` so far (tests; paths are unique per test). #[cfg(test)] @@ -364,6 +371,21 @@ pub(crate) fn decodes_of(path: &str) -> Vec { DECODES.lock().unwrap_or_else(PoisonError::into_inner).iter().filter(|(p, _)| p == path).map(|(_, d)| d.clone()).collect() } +/// Run before a decode of a path is queued on the decode threads (`QUEUING`) and when the +/// queued job begins (`STARTING`), keyed by path (tests: holding a decode queued). +#[cfg(test)] +pub(crate) static QUEUING: Mutex> = Mutex::new(Vec::new()); +#[cfg(test)] +pub(crate) static STARTING: Mutex> = Mutex::new(Vec::new()); + +#[cfg(test)] +fn run_hook(hooks: &Mutex>, path: &str) { + let hook = hooks.lock().unwrap_or_else(PoisonError::into_inner).iter().find(|(p, _)| p == path).map(|(_, h)| h.clone()); + if let Some(h) = hook { + h(path); + } +} + /// Runs before a decode reads its chunks, with the file's path (tests). #[cfg(test)] type Hook = Arc; @@ -414,6 +436,7 @@ impl PlaneStore { /// from a decode in progress, or decoded now (only what the store doesn't hold yet). The /// file it was read from is noted ([`crate::reads`]). A request that keeps finding the /// decodes it waits for called off decodes a copy itself. + #[cfg_attr(not(test), allow(dead_code))] pub(crate) fn get(&self, path: &str, source: &ExrSource, index: &Arc, want: &Want, by: Use) -> Result, Refused> { self.get_opened(path, source, index, want, by, None, drop_generation()) } @@ -677,11 +700,6 @@ impl PlaneStore { self.lock().entries.iter().filter(|e| e.batch.is_some() || !e.failed.is_empty()).map(|e| e.path.clone()).collect() } - #[cfg(test)] - pub(crate) fn in_flight(&self, path: &str) -> bool { - self.lock().entries.iter().any(|e| e.path == path && e.flight.is_some()) - } - #[cfg(test)] pub(crate) fn is_held(&self, path: &str) -> bool { self.lock().held.iter().any(|(p, _)| p == path) @@ -722,9 +740,12 @@ impl Drop for Handing<'_> { /// request that queued it, and answer this request with it (or a copy when it doesn't cover it). fn decode_claimed(path: &str, source: &ExrSource, flight: &Arc, index: &Arc, want: &Want) -> Result, Refused> { #[cfg(test)] - CLAIMS.lock().unwrap_or_else(PoisonError::into_inner).push(path.to_string()); + { + CLAIMS.lock().unwrap_or_else(PoisonError::into_inner).push(path.to_string()); + CLAIM_SIGNAL.notify_all(); + } let mut handing = Handing { flight, outcome: None }; - let outcome = decode(path, source, flight, flight.base.clone(), None); + let outcome = flight.hints.clone().run(|| decode(path, source, flight, flight.base.clone(), None)); handing.outcome = Some(outcome.clone()); drop(handing); match outcome { @@ -747,7 +768,11 @@ fn run(path: &str, source: &ExrSource, flight: &Arc, base: Option, base: Option = wanted.iter().copied().filter(|id| !base.as_ref().is_some_and(|b| b.ids.binary_search(id).is_ok())).collect(); let names: Vec = missing.iter().filter_map(|i| index.channels.get(*i)).map(|c| c.name.clone()).collect(); - // The disk tier first (#3271): what an earlier session or a decode RAM let go of kept. - let disk = disk_cache().filter(|_| !missing.is_empty() && unique(&names)); - let since = flight.since; - let key = { - let refs: Vec<&str> = names.iter().map(String::as_str).collect(); - effectcraft_render::disk_cache::plane_key(path, &opened.stamp, &refs, 0, None) - }; let (w, h) = index.size(); - let kept = disk.as_ref().and_then(|d| d.get_planes(key, None, (w as u32, h as u32, names.len()))).map(|(p, _)| p).filter(|p| { - (p.width as usize, p.height as usize) == (w, h) - && p.planes.len() == names.len() - && p.planes.iter().zip(&names).all(|((n, v), want)| n == want && v.len() == w * h) - }); - let from_disk = kept.is_some(); - let (planes, stats) = match kept { - Some(p) => (Ok(p.planes.into_iter().map(|(_, v)| v).collect()), Default::default()), - None => { - let request = ReadRequest { max_bytes: cap_of(path), ..ReadRequest::full(missing.clone()) }; - read_planes(&mut *opened.reader, &index, &request) - } + // The disk tier first (#3271): what an earlier session or a decode RAM let go of kept, one + // entry per channel (any request is assembled from them, whatever was asked before). + // (a file naming two channels alike can't tell their entries apart: not kept) + let unique = index.channels.iter().map(|c| c.name.as_str()).collect::>().len() == index.channels.len(); + let disk = disk_cache_for(path).filter(|_| !missing.is_empty() && unique); + let since = flight.since; + let key = |name: &str| effectcraft_render::disk_cache::plane_key(path, &opened.stamp, &[name], 0, None); + let mut kept: Vec>> = names + .iter() + .map(|n| { + let (planes, _) = disk.as_ref()?.get_planes(key(n), None, (w as u32, h as u32, 1))?; + let (name, plane) = planes.planes.into_iter().next()?; + (name == *n && plane.len() == w.saturating_mul(h)).then_some(plane) + }) + .collect(); + let to_read: Vec = missing.iter().zip(&kept).filter(|(_, k)| k.is_none()).map(|(id, _)| *id).collect(); + let from_disk = to_read.is_empty(); + let (read, stats) = if to_read.is_empty() { + (Ok(vec![]), Default::default()) + } else { + let request = ReadRequest { max_bytes: cap_of(path), ..ReadRequest::full(to_read.clone()) }; + read_planes(&mut *opened.reader, &index, &request) }; #[cfg(test)] DECODES.lock().unwrap_or_else(PoisonError::into_inner).push((path.to_string(), DecodeRecord { channels: names.clone(), stats: stats.clone(), from_disk })); #[cfg(not(test))] - let _ = &stats; - let planes: Vec> = planes.map_err(|e| match e { + let _ = (&stats, from_disk); + let read: Vec> = read.map_err(|e| match e { ReadError::Cancelled => Refused::Cancelled, // (a read that failed just now: the next request reads again) ReadError::Io(why) => { @@ -847,6 +875,22 @@ fn decode(path: &str, source: &ExrSource, flight: &Arc, base: Option> = kept.iter_mut().map(|k| k.take().or_else(|| read.next()).unwrap_or_default()).collect(); // The union, in file order: the planes held and the ones just read. let mut new: Vec<(usize, Vec)> = missing.into_iter().zip(planes).collect(); if let Some(b) = &base { @@ -868,25 +912,10 @@ fn decode(path: &str, source: &ExrSource, flight: &Arc, base: Option = new.iter().map(|(id, _)| *id).collect(); aux.channels = new.into_iter().filter_map(|(id, p)| Some((index.channels.get(id)?.name.clone(), p))).collect(); let file = ReadFile::new(path, opened.stamp); - let batch = Arc::new(Batch { index, ids, all, aux: Arc::new(aux), file: file.clone() }); - // Kept on disk for the next time (encoded on the disk cache's writer thread; a file from - // memory, which can't be checked later, isn't). - if let Some(d) = disk.filter(|_| !from_disk) { - let mut read = effectcraft_render::reads::ReadFiles::default(); - read.add(file); - if let Some(entry) = effectcraft_render::disk_cache::EntryFiles::of(&read, None) { - d.put_planes(key, &batch.aux, names, &entry, since); - } - } + let batch = Arc::new(Batch { index, ids, all, aux: Arc::new(aux), file }); Ok(batch) } -/// Every name once (a plane entry is read back by name). -fn unique(names: &[String]) -> bool { - let set: std::collections::HashSet<&String> = names.iter().collect(); - set.len() == names.len() -} - /// The disk cache decoded planes are kept in ([`set_disk_cache`]). static DISK: Mutex>> = Mutex::new(None); @@ -900,6 +929,26 @@ fn disk_cache() -> Option> { DISK.lock().unwrap_or_else(PoisonError::into_inner).as_ref().and_then(std::sync::Weak::upgrade) } +/// The disk cache planes of `path` are kept in (tests: only for the paths of the test that set +/// it, [`use_disk_for`], so tests running beside it don't share it). +fn disk_cache_for(path: &str) -> Option> { + #[cfg(test)] + if !DISK_PATHS.lock().unwrap_or_else(PoisonError::into_inner).iter().any(|p| path.starts_with(p.as_str())) { + return None; + } + let _ = path; + disk_cache() +} + +#[cfg(test)] +static DISK_PATHS: Mutex> = Mutex::new(Vec::new()); + +/// Keep the planes of paths under `prefix` in the disk cache (tests). +#[cfg(test)] +pub(crate) fn use_disk_for(prefix: &str) { + DISK_PATHS.lock().unwrap_or_else(PoisonError::into_inner).push(prefix.to_string()); +} + /// The store all footage shares. static STORE: PlaneStore = PlaneStore::new(CACHE_BUDGET); diff --git a/crates/media/src/exr_planes/tests.rs b/crates/media/src/exr_planes/tests.rs index c820d43db..7dc72e6f7 100644 --- a/crates/media/src/exr_planes/tests.rs +++ b/crates/media/src/exr_planes/tests.rs @@ -822,9 +822,10 @@ fn a_new_frame_opens_its_file_once() { let _ = std::fs::remove_dir_all(&dir); } -/// #3271: decoded channels are kept in the disk cache: once RAM has let them go they are read -/// back from there, bit for bit (Cryptomatte IDs included), instead of decoding the file; Reload -/// Footage drops them. +/// #3271: decoded channels are kept in the disk cache, one entry per channel: once RAM has let +/// them go they are read back from there, bit for bit (Cryptomatte IDs included), whatever was +/// asked for before (the picture first, then the channels, then all of it after an eviction); +/// Reload Footage drops them. #[test] fn decoded_planes_are_kept_on_disk_and_read_back_exactly() { use effectcraft_render::disk_cache::{DiskCache, Kind}; @@ -833,30 +834,36 @@ fn decoded_planes_are_kept_on_disk_and_read_back_exactly() { let file = dir.join("crypto_0001.exr"); std::fs::write(&file, multipart()).unwrap(); let dc = DiskCache::open(dir.join("cache"), 1 << 30).unwrap(); + // (only this test's files use it) + use_disk_for(&dir.to_string_lossy()); set_disk_cache(Some(&dc)); let f = crate::probe(&file).unwrap(); let pool = crate::MediaPool::new(); + // The picture first, then every channel: 4, then the other 5 read from the file. + pool.frame_at(&f, Tick::ZERO).unwrap(); let first = pool.aux(ItemId(1), &f, Tick::ZERO).unwrap(); dc.flush(); - assert_eq!(dc.keys(Kind::Plane).len(), 1, "{:?}", dc.stats()); - forget(&f.path); + assert_eq!(dc.keys(Kind::Plane).len(), 9, "one entry per channel: {:?}", dc.stats()); + // RAM lets go: all of it again comes from disk, without reading the file. + pool.forget(&f.path); let again = pool.aux(ItemId(1), &f, Tick::ZERO).unwrap(); let bits = |a: &AuxChannels| a.channels.iter().map(|(n, p)| (n.clone(), p.iter().map(|v| v.to_bits()).collect::>())).collect::>(); assert_eq!(bits(&again), bits(&first), "bit for bit"); - // Reload Footage: the entry goes, the file is decoded again. + // Reload Footage: the entries go, the file is decoded again. dc.drop_files([f.path.clone()]); dc.flush(); - forget(&f.path); + pool.forget(&f.path); pool.aux(ItemId(1), &f, Tick::ZERO).unwrap(); set_disk_cache(None); let from_disk: Vec = decodes_of(&f.path).iter().map(|d| d.from_disk).collect(); - assert_eq!(from_disk, [false, true, false]); + assert_eq!(from_disk, [false, false, true, false]); let _ = std::fs::remove_dir_all(&dir); } /// #3271: a decode thread asking for planes whose decode is still queued on the decode threads /// (an aux request on a frame worker queued it, then the picture's decode asks) takes it over -/// instead of decoding a copy: one decode, and both requests get its planes. +/// instead of decoding a copy: one decode, and both requests get its planes. The queued job is +/// held back until the claim (a hook), so the order is the same every time. #[test] fn a_decode_thread_takes_over_a_queued_decode() { let bytes: Arc<[u8]> = single_part_zips().into(); @@ -866,43 +873,31 @@ fn a_decode_thread_takes_over_a_queued_decode() { let index = crate::exr_index::get(path, &source).unwrap(); let all = Want { all: true, ids: Default::default() }; let threads = crate::pool::decode_threads().unwrap(); - let n = threads.current_num_threads(); - // Every decode thread busy: n - 1 hold until released, one waits for the queued decode. - let (release, held) = std::sync::mpsc::channel::<()>(); - let held = Arc::new(Mutex::new(held)); - for _ in 1..n { - let held = held.clone(); - threads.spawn(move || { - let _ = held.lock().unwrap().recv_timeout(std::time::Duration::from_secs(20)); - }); - } let (tx, rx) = std::sync::mpsc::channel(); + // When the asker queues its decode: a decode thread asks for the same planes. { let (store, source, index, all, tx) = (store.clone(), source.clone(), index.clone(), all.clone(), tx.clone()); - threads.spawn(move || { - let until = std::time::Instant::now() + std::time::Duration::from_secs(10); - while !store.in_flight(path) && std::time::Instant::now() < until { - std::thread::sleep(std::time::Duration::from_millis(1)); - } - let _ = tx.send(store.get(path, &source, &index, &all, Use::Other).map(|b| b.ids.len())); + let hook: Hook = Arc::new(move |_: &str| { + let (store, source, index, all, tx) = (store.clone(), source.clone(), index.clone(), all.clone(), tx.clone()); + threads.spawn(move || { + let _ = tx.send(store.get(path, &source, &index, &all, Use::Other).map(|b| b.ids.len())); + }); }); + QUEUING.lock().unwrap().push((path.to_string(), hook)); } - // The other request, from a thread of its own: its decode is queued behind them. - let asker = { - let (store, source, index, all) = (store.clone(), source.clone(), index.clone(), all.clone()); - std::thread::spawn(move || store.get(path, &source, &index, &all, Use::Other).map(|b| b.ids.len())) - }; + // The queued job waits until that decode thread claimed it. + let hold: Hook = Arc::new(|p: &str| { + let claims = CLAIMS.lock().unwrap(); + let _ = CLAIM_SIGNAL.wait_timeout_while(claims, std::time::Duration::from_secs(10), |c| !c.iter().any(|c| c == p)); + }); + STARTING.lock().unwrap().push((path.to_string(), hold)); + let asked = store.get(path, &source, &index, &all, Use::Other).map(|b| b.ids.len()); let claimed = rx.recv_timeout(std::time::Duration::from_secs(20)).unwrap(); - let asked = asker.join().unwrap(); - for _ in 1..n { - let _ = release.send(()); - } - assert!(claimed.is_ok() && claimed == asked, "{claimed:?} {asked:?}"); - // (other tests share the decode threads: when one of theirs ran first, the queued decode - // began before the decode thread asked, and it decoded a copy, as before) - if CLAIMS.lock().unwrap_or_else(PoisonError::into_inner).iter().any(|p| p == path) { - assert_eq!(decodes_of(path).len(), 1, "decoded once"); - } + QUEUING.lock().unwrap().retain(|(p, _)| p != path); + STARTING.lock().unwrap().retain(|(p, _)| p != path); + assert!(CLAIMS.lock().unwrap().iter().any(|p| p == path), "claimed"); + assert!(asked.is_ok() && claimed == asked, "{claimed:?} {asked:?}"); + assert_eq!(decodes_of(path).len(), 1, "decoded once"); } /// #3271: the plane store in the common memory budget: trimmed least recently used first, @@ -958,13 +953,13 @@ fn a_localized_network_file_is_read_from_its_copy() { std::fs::write(&file, multipart()).unwrap(); let f = crate::probe(&file).unwrap(); let pool = crate::MediaPool::new(); - pool.set_localizer(Some((local.clone(), 1 << 30))); + pool.set_localizer(Some(effectcraft_render::LocalizeConfig { folder: local.clone(), max_bytes: 1 << 30, network: vec![], disk_space: None })); let from_share = pool.frame_at(&f, Tick::ZERO).unwrap(); let until = std::time::Instant::now() + std::time::Duration::from_secs(10); while crate::localize::status().is_some_and(|s| s.files == 0) && std::time::Instant::now() < until { std::thread::sleep(std::time::Duration::from_millis(10)); } - assert!(pool.localize_status().is_some_and(|s| s.contains("1 files")), "{:?}", pool.localize_status()); + assert!(pool.localize_status().is_some_and(|s| s.files == 1), "{:?}", pool.localize_status()); pool.forget(&f.path); let before = crate::opens_under(&local); let (img, read) = crate::reads::captured(|| pool.frame_at(&f, Tick::ZERO).unwrap()); @@ -978,13 +973,15 @@ fn a_localized_network_file_is_read_from_its_copy() { /// #3271: a viewer region of interest reads only the footage rows it shows of an OpenEXR layer /// (with one scan line per chunk, only those lines are decompressed), and the region renders -/// bit for bit what reading the whole frame renders, at full and reduced resolution. +/// bit for bit what reading the whole frame renders, at full and reduced resolution and at +/// several layer scales. In a collapsed precomp the whole frame is read (its layers are placed +/// through the precomp layer too). #[test] fn a_region_of_interest_reads_only_its_rows() { use effectcraft_project::{Comp, ItemKind, LayerSource, Project, Value, build}; use effectcraft_render::{RenderOpts, Renderer}; use effectcraft_time::FrameRate; - /// The pool, reading whole frames (the region's rows ignored). + /// A pool reading whole frames (the region's rows ignored). struct Whole<'a>(&'a crate::MediaPool); impl FootageSource for Whole<'_> { fn frame(&self, item: ItemId, f: &Footage, t: Tick) -> Option> { @@ -996,9 +993,12 @@ fn a_region_of_interest_reads_only_its_rows() { } let bytes: Arc<[u8]> = single_part_zips().into(); let path = "/3271-roi.exr"; - let pool = crate::MediaPool::new(); - pool.add_bytes(path, bytes.clone()); - let f = crate::probe_bytes(path, bytes).unwrap(); + let fresh = || { + let pool = crate::MediaPool::new(); + pool.add_bytes(path, bytes.clone()); + pool + }; + let f = crate::probe_bytes(path, bytes.clone()).unwrap(); let mut p = Project::default(); let comp = Comp::new(96, 96, FrameRate::FPS_30, Tick::from_seconds_f64(1.0)); let cid = p.add_item("Comp", effectcraft_color::Label::Sandstone, None, ItemKind::Comp(comp.clone().into())); @@ -1006,18 +1006,35 @@ fn a_region_of_interest_reads_only_its_rows() { let mut l = build::layer(&mut p, &comp, "plate", LayerSource::Footage { item: fid }, (64, 64), None); l.props.prop_mut("transform/position").unwrap().value = Value::Vec3([50.0, 46.0, 0.0]); l.props.prop_mut("transform/rotation").unwrap().value = Value::Scalar(12.0); - l.props.prop_mut("transform/scale").unwrap().value = Value::Vec3([90.0, 110.0, 100.0]); - p.comp_mut(cid).unwrap().layers = vec![l].into(); + p.comp_mut(cid).unwrap().layers = vec![l.clone()].into(); + // The same layer inside a collapsed precomp, moved by the precomp layer. + let inner = p.add_item("Inner", effectcraft_color::Label::Sandstone, None, ItemKind::Comp(comp.clone().into())); + p.comp_mut(inner).unwrap().layers = vec![l].into(); + let outer = p.add_item("Outer", effectcraft_color::Label::Sandstone, None, ItemKind::Comp(comp.clone().into())); + let mut pre = build::layer(&mut p, &comp, "Inner", LayerSource::Comp { item: inner }, (96, 96), None); + pre.switches.collapse = true; + pre.props.prop_mut("transform/position").unwrap().value = Value::Vec3([40.0, 60.0, 0.0]); + p.comp_mut(outer).unwrap().layers = vec![pre].into(); for (layer_scale, scale) in [(100.0, 1.0), (100.0, 0.5), (30.0, 0.25), (30.0, 1.0), (250.0, 0.5)] { - let comp = p.comp_mut(cid).unwrap(); - comp.layers[0].props.prop_mut("transform/scale").unwrap().value = Value::Vec3([layer_scale * 0.9, layer_scale * 1.1, 100.0]); + for c in [cid, inner] { + p.comp_mut(c).unwrap().layers[0].props.prop_mut("transform/scale").unwrap().value = Value::Vec3([layer_scale * 0.9, layer_scale * 1.1, 100.0]); + } let opts = RenderOpts { scale, roi: Some([38.0, 40.0, 24.0, 10.0]), ..Default::default() }; - let whole = Renderer::new(&p, &Whole(&pool), opts).comp_frame(cid, Tick::ZERO); + let whole = Renderer::new(&p, &Whole(&fresh()), opts).comp_frame(cid, Tick::ZERO); let before = decodes_of(path).len(); - let rows = Renderer::new(&p, &pool, opts).comp_frame(cid, Tick::ZERO); - assert_eq!(bits(&rows), bits(&whole), "at {scale}"); - let d = decodes_of(path); - let last = d.get(before..).and_then(|d| d.last()).expect("a read of the region's rows"); - assert!(last.stats.lines < SIZE.1, "at {scale}: {} lines", last.stats.lines); + let rows = Renderer::new(&p, &fresh(), opts).comp_frame(cid, Tick::ZERO); + assert_eq!(bits(&rows), bits(&whole), "{layer_scale}% at {scale}"); + // (a layer shrunk this far shows all of its rows in the region, grown by the warp's reach) + if layer_scale >= 100.0 { + let d = decodes_of(path); + let last = d.get(before..).and_then(|d| d.last()).expect("a read of the region's rows"); + assert!(last.stats.lines < SIZE.1, "{layer_scale}% at {scale}: {} lines", last.stats.lines); + } + // Collapsed: the whole frame. + let whole = Renderer::new(&p, &Whole(&fresh()), opts).comp_frame(outer, Tick::ZERO); + let before = decodes_of(path).len(); + let collapsed = Renderer::new(&p, &fresh(), opts).comp_frame(outer, Tick::ZERO); + assert_eq!(bits(&collapsed), bits(&whole), "collapsed, {layer_scale}% at {scale}"); + assert!(decodes_of(path).get(before..).unwrap_or_default().iter().all(|d| d.stats.lines == SIZE.1), "collapsed: the whole frame"); } } diff --git a/crates/media/src/exr_read.rs b/crates/media/src/exr_read.rs index 53a62cf85..0276de0ef 100644 --- a/crates/media/src/exr_read.rs +++ b/crates/media/src/exr_read.rs @@ -73,17 +73,21 @@ impl ExrSource { } Ok(Opened { len: meta.len(), stamp: FileStamp::of_metadata(&meta), reader: Box::new(file) }) } - ExrSource::Localized { copy, stamp, .. } => { - #[cfg(test)] - crate::count_open(copy); - let file = std::fs::File::open(copy)?; - let meta = file.metadata()?; - // (the copy must still be complete: its size is the original's) - let complete = matches!(stamp, FileStamp::Native { len, .. } if *len == meta.len()); - if !meta.is_file() || !complete { - return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, format!("{}: not a complete copy", copy.display()))); - } - Ok(Opened { len: meta.len(), stamp: *stamp, reader: Box::new(file) }) + ExrSource::Localized { original, copy, stamp } => { + let local = || -> std::io::Result { + #[cfg(test)] + crate::count_open(copy); + let file = std::fs::File::open(copy)?; + let meta = file.metadata()?; + // (the copy must still be complete: its size is the original's) + let complete = matches!(stamp, FileStamp::Native { len, .. } if *len == meta.len()); + if !meta.is_file() || !complete { + return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, format!("{}: not a complete copy", copy.display()))); + } + Ok(Opened { len: meta.len(), stamp: *stamp, reader: Box::new(file) }) + }; + // (a copy evicted or damaged meanwhile: the share) + local().or_else(|_| ExrSource::File(original.clone()).open()) } } } diff --git a/crates/media/src/lib.rs b/crates/media/src/lib.rs index 02b29bc27..d54dbd0d7 100644 --- a/crates/media/src/lib.rs +++ b/crates/media/src/lib.rs @@ -26,7 +26,6 @@ mod exr_read; mod gif_anim; mod layered; mod localize; -pub use localize::{LocalizeSettings, LocalizeStatus}; pub use layered::vector_doc; mod pool; mod probe; diff --git a/crates/media/src/localize.rs b/crates/media/src/localize.rs index 26442e98f..ac8817db9 100644 --- a/crates/media/src/localize.rs +++ b/crates/media/src/localize.rs @@ -2,46 +2,30 @@ //! default): footage on a network share is copied, file by file, to a local folder while the //! user works, so later reads come from the local disk. //! +//! - **Network footage**: UNC paths (`\\server\share\…`) and the locations the user lists +//! (Settings ▸ Disk ▸ Network Footage Locations, such as a drive letter mapped to a share, +//! `P:\`). No system call is made to tell a mapped drive apart. //! - **Never on the playback path.** A read asks for a copy ([`request`]) and goes on reading //! the share; one background thread copies, one file at a time. -//! - **Used only when complete and verified.** A copy is made under a temporary name, renamed -//! when complete, and its size and the source's identity (size and modification time) as they -//! were when it was copied are recorded. It is read only while the source still has that -//! identity and the copy that size ([`local_copy`]); a source that changed is read from the -//! share (and copied again). +//! - **Used only when complete and verified.** A copy is made under a temporary name (`.part`), +//! renamed when complete, and recorded beside it (`.src`: the source's path and identity when +//! it was copied). It is read only while the source still has that identity and the copy its +//! size ([`local_copy`]); a source that changed is read from the share (and copied again). //! - **Readers keep naming the source.** Entries made of a localized file list the share's path //! and identity (see `ExrSource::Localized`), so caches are keyed the same either way. -//! - **Bounded.** The copies take at most the size limit; the least recently used go first. -//! - **Visible.** [`status`] says how many files are local, their size, what is queued and what -//! is being copied. +//! - **Bounded, across sessions.** The copies take at most the size limit and leave the disk +//! cache's reserve free on their disk ([`effectcraft_render::disk_cache::disk_reserve`]); the +//! least recently used go first. The folder's copies are adopted when the localizer starts +//! (and stale temporaries deleted), so they count towards the limit after a restart. +//! - **Visible.** [`status`] says how many files are local, their size, what is queued, what is +//! being copied and which locations count as network. use std::collections::{HashMap, HashSet, VecDeque}; use std::path::{Path, PathBuf}; use std::sync::{Condvar, Mutex, PoisonError}; use effectcraft_render::reads::FileStamp; - -/// Where copies go and how much they may take. -#[derive(Clone, Debug, PartialEq)] -pub struct LocalizeSettings { - pub folder: PathBuf, - pub max_bytes: u64, -} - -/// What the localizer is doing (Settings ▸ Disk shows it). -#[derive(Clone, Debug, Default, PartialEq)] -pub struct LocalizeStatus { - /// Complete local copies and their bytes. - pub files: usize, - pub bytes: u64, - pub max_bytes: u64, - /// Files waiting to be copied. - pub queued: usize, - /// The file being copied now. - pub copying: Option, - /// Files that were too large for the limit, or failed to copy. - pub skipped: usize, -} +use effectcraft_render::{LocalizeConfig, LocalizeReport}; /// A complete local copy. struct Copy { @@ -54,7 +38,7 @@ struct Copy { #[derive(Default)] struct State { - settings: Option, + config: Option, copies: HashMap, queue: VecDeque, queued: HashSet, @@ -65,6 +49,10 @@ struct State { worker: bool, /// Bumped with each new setting: a copy started under another one is thrown away. epoch: u64, + /// The free space seen last on the copies' disk and when (read at most every 30 s). + space: Option<((u64, u64), std::time::Instant)>, + /// The limit is held below the setting by free disk space. + reserve_limited: bool, } /// At most this many files wait to be copied (older requests are dropped first). @@ -79,38 +67,123 @@ fn lock() -> std::sync::MutexGuard<'static, State> { state().0.lock().unwrap_or_else(PoisonError::into_inner) } -/// Turn the localizer on with `settings`, or off (`None`): the copies made so far stay on disk -/// but are no longer read; a new folder starts empty. -pub fn configure(settings: Option) { +/// The settings that decide what is copied where (the free-space reader aside). +fn same(a: Option<&LocalizeConfig>, b: Option<&LocalizeConfig>) -> bool { + match (a, b) { + (None, None) => true, + (Some(a), Some(b)) => a.folder == b.folder && a.max_bytes == b.max_bytes && a.network == b.network, + _ => false, + } +} + +/// Turn the localizer on with `config`, or off (`None`): the copies stay on disk but are no +/// longer read. A new folder's earlier copies are adopted (their records beside them), stale +/// temporaries deleted. +pub fn configure(config: Option) { let mut st = lock(); - if st.settings == settings { + if same(st.config.as_ref(), config.as_ref()) { + if let Some(c) = config { + st.config = Some(c); + } return; } st.epoch = st.epoch.wrapping_add(1); - if st.settings.as_ref().map(|s| &s.folder) != settings.as_ref().map(|s| &s.folder) { - st.copies.clear(); - st.bytes = 0; - } + let new_folder = st.config.as_ref().map(|c| &c.folder) != config.as_ref().map(|c| &c.folder); st.queue.clear(); st.queued.clear(); st.skipped.clear(); - st.settings = settings; - if let Some(max) = st.settings.as_ref().map(|s| s.max_bytes) { - evict_to(&mut st, max); + st.space = None; + st.config = config; + if new_folder { + st.copies.clear(); + st.bytes = 0; + if let Some(folder) = st.config.as_ref().map(|c| c.folder.clone()) { + adopt(&mut st, &folder); + } + } + let max = limit(&mut st); + evict_to(&mut st, max); +} + +/// The copies' record beside a copy. +fn record_path(local: &Path) -> PathBuf { + let mut p = local.as_os_str().to_owned(); + p.push(".src"); + PathBuf::from(p) +} + +/// A record's text: the source's path, size and modification time (ns, or `-`), the copy's size. +fn record(original: &Path, source: &FileStamp, len: u64) -> Option { + let FileStamp::Native { len: slen, modified } = source else { return None }; + let modified = modified.map_or("-".to_string(), |m| m.to_string()); + Some(format!("{}\n{slen}\n{modified}\n{len}\n", original.to_string_lossy())) +} + +fn parse_record(text: &str) -> Option<(PathBuf, FileStamp, u64)> { + let mut lines = text.lines(); + let original = PathBuf::from(lines.next()?); + let slen = lines.next()?.parse().ok()?; + let modified = match lines.next()? { + "-" => None, + m => Some(m.parse().ok()?), + }; + let len = lines.next()?.parse().ok()?; + Some((original, FileStamp::Native { len: slen, modified }, len)) +} + +/// Is `name` one of the localizer's own files (`<16 hex>_…`). +fn ours(name: &str) -> bool { + name.len() > 17 && name.as_bytes().get(16) == Some(&b'_') && name.get(..16).is_some_and(|h| h.bytes().all(|b| b.is_ascii_hexdigit())) +} + +/// Take over the copies `folder` holds from an earlier session; delete temporaries and copies +/// without a record (only the localizer's own files: other files in the folder are left alone). +fn adopt(st: &mut State, folder: &Path) { + let Ok(dir) = std::fs::read_dir(folder) else { return }; + let mut records = vec![]; + for e in dir.flatten() { + let name = e.file_name().to_string_lossy().to_string(); + if !ours(&name) { + continue; + } + if name.ends_with(".part") { + let _ = std::fs::remove_file(e.path()); + } else if name.ends_with(".src") { + records.push(e.path()); + } + } + for rec in records { + let local = rec.with_extension(""); + let parsed = std::fs::read_to_string(&rec).ok().and_then(|t| parse_record(&t)); + let fine = parsed.as_ref().is_some_and(|(_, _, len)| std::fs::metadata(&local).is_ok_and(|m| m.is_file() && m.len() == *len)); + match parsed { + Some((original, source, len)) if fine => { + st.clock += 1; + let last_use = st.clock; + st.bytes = st.bytes.saturating_add(len); + st.copies.insert(original, Copy { local, len, source, last_use }); + } + _ => { + let _ = std::fs::remove_file(&rec); + let _ = std::fs::remove_file(&local); + } + } } } /// What the localizer is doing; `None` when it is off. -pub fn status() -> Option { +pub fn status() -> Option { let st = lock(); - let s = st.settings.as_ref()?; - Some(LocalizeStatus { + let c = st.config.as_ref()?; + Some(LocalizeReport { files: st.copies.len(), bytes: st.bytes, - max_bytes: s.max_bytes, + max_bytes: c.max_bytes, queued: st.queue.len(), copying: st.copying.as_ref().map(|p| p.to_string_lossy().to_string()), skipped: st.skipped.len(), + network: c.network.iter().map(|p| p.to_string_lossy().to_string()).collect(), + reserve_limited: st.reserve_limited, }) } @@ -124,15 +197,29 @@ pub(crate) fn treat_as_network(root: &Path) { NETWORK_ROOTS.lock().unwrap_or_else(PoisonError::into_inner).push(root.to_path_buf()); } -/// Is `path` on a network share: a UNC path (`\\server\share\…`, `//server/share/…`). Drive -/// letters mapped to a share are not told apart from local disks. +/// `path` lies under `prefix` (separators and, on Windows, letter case don't matter). +fn under(path: &Path, prefix: &Path) -> bool { + let norm = |p: &Path| { + let s = p.to_string_lossy().replace('\\', "/"); + if cfg!(windows) { s.to_lowercase() } else { s } + }; + let (p, pre) = (norm(path), norm(prefix)); + let pre = pre.trim_end_matches('/'); + !pre.is_empty() && (p == pre || p.starts_with(&format!("{pre}/"))) +} + +/// Is `path` network footage: a UNC path (`\\server\share\…`, `//server/share/…`), or under one +/// of the locations the localizer is given (`P:\`). pub(crate) fn is_network(path: &Path) -> bool { #[cfg(test)] if NETWORK_ROOTS.lock().unwrap_or_else(PoisonError::into_inner).iter().any(|r| path.starts_with(r)) { return true; } let s = path.to_string_lossy(); - s.starts_with("\\\\") || s.starts_with("//") + if s.starts_with("\\\\") || s.starts_with("//") { + return true; + } + lock().config.as_ref().is_some_and(|c| c.network.iter().any(|n| under(path, n))) } /// Ask for a local copy of the network file `original` (nothing when the localizer is off, the @@ -142,7 +229,7 @@ pub(crate) fn request(original: &Path) { return; } let mut st = lock(); - if st.settings.is_none() || st.copies.contains_key(original) || st.queued.contains(original) || st.skipped.contains(original) { + if st.config.is_none() || st.copies.contains_key(original) || st.queued.contains(original) || st.skipped.contains(original) { return; } if st.queue.len() >= QUEUE_LIMIT @@ -161,7 +248,7 @@ pub(crate) fn request(original: &Path) { /// source). A copy whose source changed is forgotten (and copied again when asked for). pub(crate) fn local_copy(original: &Path, now: &FileStamp) -> Option { let mut st = lock(); - st.settings.as_ref()?; + st.config.as_ref()?; st.clock += 1; let clock = st.clock; let c = st.copies.get_mut(original)?; @@ -172,11 +259,16 @@ pub(crate) fn local_copy(original: &Path, now: &FileStamp) -> Option { } if let Some(gone) = st.copies.remove(original) { st.bytes = st.bytes.saturating_sub(gone.len); - let _ = std::fs::remove_file(&gone.local); + remove_copy(&gone.local); } None } +fn remove_copy(local: &Path) { + let _ = std::fs::remove_file(local); + let _ = std::fs::remove_file(record_path(local)); +} + #[cfg(not(target_arch = "wasm32"))] fn start_worker(st: &mut State) { if st.worker { @@ -189,13 +281,30 @@ fn start_worker(st: &mut State) { #[cfg(target_arch = "wasm32")] fn start_worker(_st: &mut State) {} +/// The most the copies may take now: the setting, or less when free disk space holds them +/// below it (the disk cache's reserve stays free on their disk; free space is read at most +/// every 30 s). +fn limit(st: &mut State) -> u64 { + let Some(c) = st.config.clone() else { return 0 }; + let fresh = st.space.is_some_and(|(_, at)| at.elapsed() < std::time::Duration::from_secs(30)); + if !fresh && let Some(read) = &c.disk_space { + st.space = read().map(|s| (s, std::time::Instant::now())); + } + let cap = match st.space { + Some(((total, available), _)) if total > 0 => st.bytes.saturating_add(available).saturating_sub(effectcraft_render::disk_cache::disk_reserve(total)), + _ => u64::MAX, + }; + st.reserve_limited = cap < c.max_bytes; + c.max_bytes.min(cap) +} + /// Drop the least recently used copies until they take at most `max` bytes. fn evict_to(st: &mut State, max: u64) { while st.bytes > max { let Some(oldest) = st.copies.iter().min_by_key(|(_, c)| c.last_use).map(|(p, _)| p.clone()) else { break }; if let Some(c) = st.copies.remove(&oldest) { st.bytes = st.bytes.saturating_sub(c.len); - let _ = std::fs::remove_file(&c.local); + remove_copy(&c.local); } } } @@ -214,14 +323,15 @@ fn local_name(original: &Path) -> String { fn worker() { let (m, cv) = state(); loop { - let (job, settings, epoch) = { + let (job, config, epoch, room) = { let mut st = m.lock().unwrap_or_else(PoisonError::into_inner); loop { - if let (Some(p), Some(s)) = (st.queue.front().cloned(), st.settings.clone()) { + if let (Some(p), Some(c)) = (st.queue.front().cloned(), st.config.clone()) { st.queue.pop_front(); st.queued.remove(&p); st.copying = Some(p.clone()); - break (p, s, st.epoch); + let room = limit(&mut st); + break (p, c, st.epoch, room); } let (g, timeout) = match cv.wait_timeout(st, std::time::Duration::from_secs(30)) { Ok(r) => r, @@ -234,7 +344,7 @@ fn worker() { } } }; - let copied = copy_file(&job, &settings); + let copied = copy_file(&job, &config.folder, room); let mut st = m.lock().unwrap_or_else(PoisonError::into_inner); st.copying = None; match copied { @@ -246,11 +356,10 @@ fn worker() { let last_use = st.clock; st.copies.insert(job, Copy { local, len, source, last_use }); st.bytes = st.bytes.saturating_add(len); - evict_to(&mut st, settings.max_bytes); - } - Some((local, _, _)) => { - let _ = std::fs::remove_file(local); + let max = limit(&mut st); + evict_to(&mut st, max); } + Some((local, _, _)) => remove_copy(&local), None => { st.skipped.insert(job); } @@ -258,19 +367,20 @@ fn worker() { } } -/// Copy `original` into the folder of `settings`: under a temporary name, then renamed when -/// complete; its modification time set to the source's. The copy, its size and the source's -/// identity, or `None` (too large for the limit, unreadable, or changed while it was copied). +/// Copy `original` into `folder`: under a temporary name, then renamed when complete, with its +/// record beside it; its modification time set to the source's. The copy, its size and the +/// source's identity, or `None` (larger than `room`, unreadable, or changed while it was +/// copied). #[cfg(not(target_arch = "wasm32"))] -fn copy_file(original: &Path, settings: &LocalizeSettings) -> Option<(PathBuf, u64, FileStamp)> { +fn copy_file(original: &Path, folder: &Path, room: u64) -> Option<(PathBuf, u64, FileStamp)> { let meta = std::fs::metadata(original).ok().filter(std::fs::Metadata::is_file)?; - if meta.len() > settings.max_bytes { + if meta.len() > room { return None; } let before = FileStamp::of_metadata(&meta); - std::fs::create_dir_all(&settings.folder).ok()?; - let local = settings.folder.join(local_name(original)); - let tmp = settings.folder.join(format!("{}.part", local_name(original))); + std::fs::create_dir_all(folder).ok()?; + let local = folder.join(local_name(original)); + let tmp = folder.join(format!("{}.part", local_name(original))); let copied = (|| -> std::io::Result { let mut from = std::fs::File::open(original)?; let mut to = std::fs::File::create(&tmp)?; @@ -282,8 +392,15 @@ fn copy_file(original: &Path, settings: &LocalizeSettings) -> Option<(PathBuf, u Ok(n) })(); let after = std::fs::metadata(original).ok().map(|m| FileStamp::of_metadata(&m)); - match copied { - Ok(n) if n == meta.len() && after.as_ref() == Some(&before) && std::fs::rename(&tmp, &local).is_ok() => Some((local, n, before)), + let text = record(original, &before, meta.len()); + match (copied, text) { + (Ok(n), Some(text)) if n == meta.len() && after.as_ref() == Some(&before) && std::fs::rename(&tmp, &local).is_ok() => { + if std::fs::write(record_path(&local), text).is_err() { + remove_copy(&local); + return None; + } + Some((local, n, before)) + } _ => { let _ = std::fs::remove_file(&tmp); None @@ -299,7 +416,11 @@ pub(crate) static TEST_SERIAL: Mutex<()> = Mutex::new(()); mod tests { use super::*; - fn wait_for(f: impl Fn(&LocalizeStatus) -> bool) -> LocalizeStatus { + fn config(folder: &Path, max_bytes: u64, network: Vec) -> Option { + Some(LocalizeConfig { folder: folder.to_path_buf(), max_bytes, network, disk_space: None }) + } + + fn wait_for(f: impl Fn(&LocalizeReport) -> bool) -> LocalizeReport { let until = std::time::Instant::now() + std::time::Duration::from_secs(10); loop { let s = status().unwrap_or_default(); @@ -312,18 +433,21 @@ mod tests { /// A network file is copied in the background and read locally only while it matches its /// source; a source that changed is read from the share again; files over the limit are not - /// copied; local files are never queued. + /// copied; local files are never queued; a listed location counts as network; the folder's + /// copies are adopted after a restart (stale temporaries deleted). #[test] fn network_files_are_copied_and_used_only_while_they_match() { let _serial = TEST_SERIAL.lock().unwrap_or_else(PoisonError::into_inner); let root = std::env::temp_dir().join(format!("ec-3271-localize-{}", std::process::id())); - let (share, local) = (root.join("share"), root.join("local")); + let (share, local, mapped) = (root.join("share"), root.join("local"), root.join("mapped")); std::fs::create_dir_all(&share).unwrap(); + std::fs::create_dir_all(&mapped).unwrap(); treat_as_network(&share); let file = share.join("plate_0001.exr"); std::fs::write(&file, vec![7u8; 4096]).unwrap(); let stamp = |p: &Path| FileStamp::of_metadata(&std::fs::metadata(p).unwrap()); - configure(Some(LocalizeSettings { folder: local.clone(), max_bytes: 1 << 20 })); + configure(config(&local, 1 << 20, vec![mapped.clone()])); + assert!(is_network(&mapped.join("x.exr")), "a listed location"); assert!(local_copy(&file, &stamp(&file)).is_none(), "not copied yet"); request(&file); let s = wait_for(|s| s.files == 1); @@ -331,13 +455,21 @@ mod tests { let copy = local_copy(&file, &stamp(&file)).expect("a verified copy"); assert!(copy.starts_with(&local)); assert_eq!(std::fs::read(©).unwrap(), vec![7u8; 4096]); + // A restart: the copy is adopted, a stale temporary deleted. + let stale = local.join("0123456789abcdef_old.exr.part"); + std::fs::write(&stale, b"x").unwrap(); + configure(None); + configure(config(&local, 1 << 20, vec![])); + assert_eq!(status().unwrap().files, 1, "adopted"); + assert!(!stale.exists(), "temporaries deleted"); + assert!(local_copy(&file, &stamp(&file)).is_some()); // The source changes: the copy is not read any more. std::fs::write(&file, vec![8u8; 5000]).unwrap(); assert!(local_copy(&file, &stamp(&file)).is_none()); // Over the limit: skipped. let big = share.join("big.exr"); std::fs::write(&big, vec![1u8; 4096]).unwrap(); - configure(Some(LocalizeSettings { folder: local.clone(), max_bytes: 1000 })); + configure(config(&local, 1000, vec![])); request(&big); let s = wait_for(|s| s.skipped == 1); assert_eq!((s.skipped, s.files), (1, 0), "{s:?}"); diff --git a/crates/media/src/pool.rs b/crates/media/src/pool.rs index 790303825..0a7a70676 100644 --- a/crates/media/src/pool.rs +++ b/crates/media/src/pool.rs @@ -530,7 +530,7 @@ impl Drop for Owner<'_> { drop(c); *lock(&self.flight.outcome) = Some(outcome); self.flight.ready.notify_all(); - effectcraft_render::memory::Accountant::global().balance(); + effectcraft_render::memory::Accountant::global().note_growth(); } } @@ -1097,6 +1097,12 @@ impl Inner { loc.key.nominal = (footage.width, footage.height); } } + // Some rows of an OpenEXR frame (a region of interest, #3271): the whole frame when the + // cache has it, else those rows read for this request alone (not kept: a frame cache + // entry of a few rows would hold a whole frame's memory). + if loc.key.rows.is_some() { + return self.frame_rows_of(&loc, footage); + } // What is decoded: a movie frame at its size, a still whole (its key has no size). let source = Loc { key: if movie { loc.key.clone() } else { Key { size: (0, 0), nominal: (0, 0), ..loc.key.clone() } }, media_t: loc.media_t, float: loc.float }; @@ -1173,6 +1179,32 @@ impl Inner { Ok(if as_asked { whole } else { self.keep(&loc.key, shape(whole), files, generation) }) } + /// [`Inner::frame_at_rows`] of a request for rows (`loc.key.rows`): the whole frame at that + /// size or the whole still from the frame cache, else the rows decoded now. + fn frame_rows_of(self: &Arc, loc: &Loc, footage: &Footage) -> Result> { + let whole = Key { rows: None, ..loc.key.clone() }; + let still = Key { size: (0, 0), nominal: (0, 0), ..whole.clone() }; + let mut c = lock(&self.cache); + if let Some((img, files)) = c.lru.get(&whole) { + drop(c); + self.hits.fetch_add(1, Ordering::Relaxed); + reads::note_all(&files); + return Ok(img); + } + if let Some((img, files)) = c.lru.get(&still).or_else(|| c.recent(&still)) { + drop(c); + self.hits.fetch_add(1, Ordering::Relaxed); + reads::note_all(&files); + return Ok(Self::sized_still(img, loc, footage)); + } + drop(c); + self.misses.fetch_add(1, Ordering::Relaxed); + let source = Loc { key: Key { size: (0, 0), nominal: (0, 0), ..loc.key.clone() }, media_t: None, float: loc.float }; + let (img, files) = self.run_decode(&source, footage); + reads::note_all(&files); + Ok(Self::sized_still(Arc::new(img?), loc, footage)) + } + /// Keep `img`, made of `files`, in the frame cache as `key` (unless a file was forgotten /// since `generation`, see [`CacheState::generation`]), and return it. fn keep(&self, key: &Key, img: Arc, files: Arc, generation: u64) -> Arc { @@ -1181,7 +1213,7 @@ impl Inner { c.lru.insert(key.clone(), img.clone(), files, self.budget()); } drop(c); - effectcraft_render::memory::Accountant::global().balance(); + effectcraft_render::memory::Accountant::global().note_growth(); img } @@ -1536,22 +1568,12 @@ impl FootageSource for MediaPool { wanted.learned.clear(); } - fn set_localizer(&self, folder: Option<(std::path::PathBuf, u64)>) { - crate::localize::configure(folder.map(|(folder, max_bytes)| crate::localize::LocalizeSettings { folder, max_bytes })); + fn set_localizer(&self, config: Option) { + crate::localize::configure(config); } - fn localize_status(&self) -> Option { - let s = crate::localize::status()?; - let gb = |b: u64| b as f64 / f64::from(1u32 << 30); - let copying = s.copying.as_deref().map(|p| format!(", copying {p}")).unwrap_or_default(); - Some(format!( - "Local copies: {} files, {:.1} of {:.1} GB; {} queued, {} skipped{copying}", - s.files, - gb(s.bytes), - gb(s.max_bytes), - s.queued, - s.skipped - )) + fn localize_status(&self) -> Option { + crate::localize::status() } fn set_disk_cache(&self, cache: Option>) { diff --git a/crates/render/src/accel.rs b/crates/render/src/accel.rs index e3aa1b3f9..06d204a1a 100644 --- a/crates/render/src/accel.rs +++ b/crates/render/src/accel.rs @@ -377,6 +377,8 @@ impl<'a> Renderer<'a> { pub fn adjustment_footprint(&self, ctx: &EvalCtx, layer: &Layer) -> Option> { // Cached, so an accelerator sees the same buffer (and uploads it once) every frame. let key = self.cache.and_then(|_| cache::footprint_key(ctx, layer, self.raster_scale(ctx, layer), self.opts.draft, self.mb_on(ctx, layer))); + // (a footage layer read in part is never kept under a whole layer's key, #3271) + let key = key.filter(|_| !self.reads_footage_rows(ctx, layer)); let key = self.mask_gate_key(ctx, layer, key); if let Some((b, _)) = self.cached(key) { return Some(b); diff --git a/crates/render/src/cache.rs b/crates/render/src/cache.rs index 50856b0b7..a9feff5bf 100644 --- a/crates/render/src/cache.rs +++ b/crates/render/src/cache.rs @@ -349,7 +349,7 @@ impl LayerCache { } drop(g); // (with the lock released: the accountant asks every cache for its bytes) - crate::memory::Accountant::global().balance(); + crate::memory::Accountant::global().note_growth(); (buf, files) } @@ -829,7 +829,19 @@ fn key_any(ctx: &EvalCtx, layer: &Layer, scale: f64, draft: bool, blur: bool, fo item.hash(&mut h); // A still is the same at every time (unless a proxy, which may be a movie, stands in). if f.kind != FootageKind::Still || it.proxy.is_some() { - ctx.source_time(layer).0.hash(&mut h); + // A sequence frame without frame blending or fields shows one file for its whole + // frame duration: keyed by its frame, so times within it (a faster comp, a + // stretched layer) share the entry (#3271). + let whole_frames = f.kind == FootageKind::Sequence + && it.proxy.is_none() + && f.fields == effectcraft_project::FieldOrder::Off + && crate::frames::frame_blend_mode(ctx, layer) == effectcraft_project::FrameBlend::Off; + if whole_frames { + 0x5e9_u16.hash(&mut h); + crate::frames::frame_position(ctx.source_time(layer), f.frame_rate).0.hash(&mut h); + } else { + ctx.source_time(layer).0.hash(&mut h); + } } // Frame blending mixes neighbouring source frames. ctx.comp.enable_frame_blending.hash(&mut h); @@ -974,7 +986,6 @@ pub fn derive(key: u64, i: u64) -> u64 { h.finish() } - /// The layer cache in the common memory budget (#3271): rendered layer buffers, costly to make /// again, are freed after footage frames and planes. impl crate::memory::MemoryHolder for LayerCache { diff --git a/crates/render/src/disk_cache.rs b/crates/render/src/disk_cache.rs index 99fc38a21..fbf03105d 100644 --- a/crates/render/src/disk_cache.rs +++ b/crates/render/src/disk_cache.rs @@ -321,15 +321,15 @@ enum Job { /// the pixels they made (a layer buffer shared with the RAM cache, a frame moved in) and go on; /// converting and compressing them is the writer's work. enum Payload { - /// Already encoded. - Bytes(Vec), /// A layer buffer and its source files. Layer(Arc, EntryFiles), /// An 8-bit or half-float frame (width, height, pixels) and its source files. Frame8(u32, u32, Vec, EntryFiles), FrameF16(u32, u32, Vec, EntryFiles), - /// The planes named (in order) of decoded footage channels, and their source file. - Planes(Arc, Vec, EntryFiles), + /// Decoded footage planes (width, height, named planes), and their source file. + Planes(u32, u32, Vec<(String, Vec)>, EntryFiles), + /// Bytes to compress with LZ4 (a segmentation). + Lz4(Vec), } #[cfg(test)] @@ -348,15 +348,15 @@ impl Payload { /// The entry's payload bytes: its source files, then its pixels, compressed. fn encode(self) -> Vec { #[cfg(test)] - if !matches!(self, Payload::Bytes(_)) { + if !matches!(self, Payload::Lz4(_)) { ENCODED_HERE.with(|n| n.set(n.get() + 1)); } match self { - Payload::Bytes(b) => b, Payload::Layer(buf, files) => with_files(&files, encode_layer(&buf)), Payload::Frame8(w, h, rgba, files) => with_files(&files, encode_frame(w, h, &rgba)), Payload::FrameF16(w, h, rgba, files) => with_files(&files, encode_frame_f16(w, h, &rgba)), - Payload::Planes(aux, names, files) => with_files(&files, encode_planes(&aux, &names)), + Payload::Planes(w, h, planes, files) => with_files(&files, encode_planes(w, h, &planes)), + Payload::Lz4(raw) => lz4_flex::compress_prepend_size(&raw), } } @@ -364,11 +364,19 @@ impl Payload { #[cfg_attr(target_arch = "wasm32", allow(dead_code))] fn held(&self) -> usize { match self { - Payload::Bytes(b) => b.len(), Payload::Layer(buf, _) => buf.img.data.len().saturating_mul(16), Payload::Frame8(_, _, p, _) => p.len(), Payload::FrameF16(_, _, p, _) => p.len().saturating_mul(2), - Payload::Planes(aux, names, _) => (aux.width as usize).saturating_mul(aux.height as usize).saturating_mul(4).saturating_mul(names.len()), + Payload::Planes(_, _, planes, _) => planes.iter().map(|(_, p)| p.len().saturating_mul(4)).fold(0, usize::saturating_add), + Payload::Lz4(raw) => raw.len(), + } + } + + /// The bytes only this payload holds: a layer buffer the RAM cache shares counts there. + fn unique(&self) -> usize { + match self { + Payload::Layer(buf, _) if Arc::strong_count(buf) > 1 => 0, + _ => self.held(), } } @@ -451,13 +459,14 @@ impl Queue { self.frames.is_empty() && self.jobs.is_empty() } - /// The bytes the queued writes hold. - fn held_bytes(&self) -> usize { + /// The bytes only the queued layer and plane writes hold (no frames: see the memory holder). + #[cfg_attr(target_arch = "wasm32", allow(dead_code))] + fn unique_job_bytes(&self) -> usize { let job = |j: &Job| match j { - Job::Write(_, _, _, p) => p.held(), + Job::Write(_, _, _, p) => p.unique(), Job::Drop(_) => 0, }; - self.frames.iter().chain(&self.jobs).map(job).fold(0, usize::saturating_add) + self.jobs.iter().map(job).fold(0, usize::saturating_add) } } @@ -894,11 +903,6 @@ impl DiskCache { } } - /// Queue a write; `false` when it was dropped (a full queue). - fn enqueue(&self, kind: Kind, key: u128, payload: Vec) -> bool { - self.enqueue_format(kind, key, FORMAT_RGBA8, Payload::Bytes(payload), None) - } - /// [`DiskCache::enqueue`] with the entry's header format byte. `made`: the source files of a /// frame or layer entry and the drop generation its reads started at /// ([`DiskCache::drop_generation`]); refused (`false`) when one of them was dropped since. @@ -913,7 +917,7 @@ impl DiskCache { if self.queue.0.lock().is_ok_and(|q| stale(&q)) { return false; } - if !matches!(payload, Payload::Bytes(_)) { + if !matches!(payload, Payload::Lz4(_)) { self.count_encode(); } self.write_format(kind, key, format, &payload.encode()); @@ -935,7 +939,7 @@ impl DiskCache { return false; } // (counted as it is accepted: the writer compresses it) - if !matches!(payload, Payload::Bytes(_)) { + if !matches!(payload, Payload::Lz4(_)) { self.count_encode(); } if kind == Kind::Frame { @@ -1196,7 +1200,16 @@ impl DiskCache { /// [`DiskCache::put_frame`] taking the pixels: they move to the writer thread, which converts /// and compresses them (#3271); the caller goes on at once. #[allow(clippy::too_many_arguments)] - pub fn put_frame_owned(&self, key: u128, width: u32, height: u32, rgba: Vec, files: &ReadFiles, virtual_id: Option>, since: u64) -> bool { + pub fn put_frame_owned( + &self, + key: u128, + width: u32, + height: u32, + rgba: Vec, + files: &ReadFiles, + virtual_id: Option>, + since: u64, + ) -> bool { if rgba.len() as u64 != width as u64 * height as u64 * 4 { return false; } @@ -1256,18 +1269,19 @@ impl DiskCache { self.get_checked(Kind::Plane, key, FORMAT_PLANES_FILES, virtual_id, |p| decode_planes(p, expect)) } - /// Queue decoded footage channels for writing: the planes of `aux` named `names` (in that - /// order, as [`plane_key`] names them), with the source file they were read from. Encoded on - /// the writer thread (a reference to the planes, no copy). Refused when there is no room, the - /// file can't be checked later, or it was dropped after `since`. - pub fn put_planes(&self, key: u128, aux: &Arc, names: Vec, files: &EntryFiles, since: u64) -> bool { + /// Queue decoded footage channels for writing: named `width` × `height` planes (in the order + /// [`plane_key`] names them), with the source file they were read from; encoded on the writer + /// thread. Refused when a plane is of another size, there is no room, or the source file was + /// dropped after `since`. + pub fn put_planes(&self, key: u128, width: u32, height: u32, planes: Vec<(String, Vec)>, files: &EntryFiles, since: u64) -> bool { if self.contains(Kind::Plane, key) { return true; } - if names.is_empty() || !self.accepts_writes() { + let n = (width as usize).checked_mul(height as usize); + if planes.is_empty() || planes.iter().any(|(_, p)| Some(p.len()) != n) || !self.accepts_writes() { return false; } - self.enqueue_format(Kind::Plane, key, FORMAT_PLANES_FILES, Payload::Planes(aux.clone(), names, files.clone()), Some((files, since))) + self.enqueue_format(Kind::Plane, key, FORMAT_PLANES_FILES, Payload::Planes(width, height, planes, files.clone()), Some((files, since))) } /// A segmentation (chain key, scale) written by [`DiskCache::put_seg`]. @@ -1283,7 +1297,8 @@ impl DiskCache { if self.contains(Kind::Matte, k) { return; } - let _ = self.enqueue(Kind::Matte, k, lz4_flex::compress_prepend_size(&seg.to_bytes())); + // (compressed on the writer thread) + let _ = self.enqueue_format(Kind::Matte, k, FORMAT_RGBA8, Payload::Lz4(seg.to_bytes()), None); } /// Make the frame at key `from` the frame at key `to` too (an edit that can't change it gave @@ -1472,37 +1487,21 @@ impl crate::memory::MemoryHolder for DiskCache { fn name(&self) -> &'static str { "disk write queue" } + /// The bytes only the queue holds (a layer buffer the RAM cache still holds counts there). fn bytes(&self) -> usize { - self.queue.0.lock().map(|q| q.held_bytes()).unwrap_or(0) + self.queue.0.lock().map(|q| q.unique_job_bytes()).unwrap_or(0) } fn priority(&self) -> u8 { 0 } + /// Drops queued layer and plane writes (newest first) that alone hold their bytes; frame + /// writes are never dropped (a fully cached comp would come back with gaps). fn trim_to(&self, bytes: usize) { let Ok(mut q) = self.queue.0.lock() else { return }; - while q.held_bytes() > bytes { - // Layer and plane writes first (newest first), then frames. - let job = match q.jobs.iter().rposition(|j| matches!(j, Job::Write(..))) { - Some(i) => { - let j = q.jobs.remove(i); - if let Some(Job::Write(_, _, _, p)) = &j { - q.job_bytes = q.job_bytes.saturating_sub(p.held()); - } - j - } - None => match q.frames.pop_back() { - Some(j) => { - if let Job::Write(_, key, _, p) = &j { - q.frame_bytes = q.frame_bytes.saturating_sub(p.held()); - q.pending_frames.remove(key); - } - Some(j) - } - None => None, - }, - }; - if job.is_none() { - break; + while q.unique_job_bytes() > bytes { + let Some(i) = q.jobs.iter().rposition(|j| matches!(j, Job::Write(_, _, _, p) if p.unique() > 0)) else { break }; + if let Some(Job::Write(_, _, _, p)) = q.jobs.remove(i) { + q.job_bytes = q.job_bytes.saturating_sub(p.held()); } self.dropped.fetch_add(1, Ordering::Relaxed); } @@ -1614,26 +1613,25 @@ fn decode_frame_f16(p: &[u8]) -> Option { /// bytes) and type (2: `f32`), then every plane's bytes shuffled into byte planes (plane by plane, /// the first bytes of its floats, then the second…), LZ4-compressed. Planes of `aux` named /// `names` that it doesn't hold, or holds at another size, are written as zeros. -fn encode_planes(aux: &effectcraft_raster::AuxChannels, names: &[String]) -> Vec { - let (w, h) = (aux.width, aux.height); +fn encode_planes(w: u32, h: u32, planes: &[(String, Vec)]) -> Vec { let n = (w as usize).saturating_mul(h as usize); let mut out = Vec::with_capacity(64); out.extend_from_slice(&w.to_le_bytes()); out.extend_from_slice(&h.to_le_bytes()); - out.extend_from_slice(&(names.len() as u32).to_le_bytes()); - for name in names { + out.extend_from_slice(&(planes.len() as u32).to_le_bytes()); + for (name, _) in planes { let bytes = name.as_bytes(); - out.extend_from_slice(&(bytes.len().min(u16::MAX as usize) as u16).to_le_bytes()); - out.extend_from_slice(bytes.get(..bytes.len().min(u16::MAX as usize)).unwrap_or_default()); + let len = bytes.len().min(u16::MAX as usize); + out.extend_from_slice(&(len as u16).to_le_bytes()); + out.extend_from_slice(bytes.get(..len).unwrap_or_default()); out.push(2); } - let mut shuffled = vec![0u8; n.saturating_mul(4).saturating_mul(names.len())]; - for (k, name) in names.iter().enumerate() { - let Some(plane) = aux.channels.iter().find(|(c, _)| c == name).map(|(_, p)| p).filter(|p| p.len() == n) else { continue }; - let base = k * n * 4; - for (i, v) in plane.iter().enumerate() { - let b = v.to_bits().to_le_bytes(); - for (byte, value) in b.iter().enumerate() { + let Some(total) = n.checked_mul(4).and_then(|b| b.checked_mul(planes.len())) else { return out }; + let mut shuffled = vec![0u8; total]; + for (k, (_, plane)) in planes.iter().enumerate() { + let Some(base) = k.checked_mul(n).and_then(|b| b.checked_mul(4)) else { continue }; + for (i, v) in plane.iter().take(n).enumerate() { + for (byte, value) in v.to_bits().to_le_bytes().iter().enumerate() { if let Some(o) = shuffled.get_mut(base + byte * n + i) { *o = *value; } diff --git a/crates/render/src/footage.rs b/crates/render/src/footage.rs index 367167e3f..3407d062c 100644 --- a/crates/render/src/footage.rs +++ b/crates/render/src/footage.rs @@ -90,12 +90,12 @@ pub trait FootageSource: Send + Sync { /// decode costly footage keep what they decoded there too (#3271: OpenEXR channels). /// Sources without such a tier ignore it. fn set_disk_cache(&self, _cache: Option>) {} - /// Settings ▸ Disk ▸ Localize Network Footage: copy footage on network shares to `folder` - /// in the background, up to the byte limit, and read the copies once they are complete and - /// verified (`None`: off). Sources that don't read files ignore it. - fn set_localizer(&self, _folder: Option<(std::path::PathBuf, u64)>) {} - /// What the localizer is doing, as a line to show (`None`: off or not supported). - fn localize_status(&self) -> Option { + /// Settings ▸ Disk ▸ Localize Network Footage: copy network footage in the background and + /// read the copies once they are complete and verified (`None`: off). Sources that don't read + /// files ignore it. + fn set_localizer(&self, _config: Option) {} + /// What the localizer is doing (`None`: off or not supported). + fn localize_status(&self) -> Option { None } /// The parsed 3D model of a [`effectcraft_project::FootageKind::Model`] item (Advanced 3D). @@ -238,10 +238,10 @@ impl FootageSource for CancellableFootage<'_> { fn set_disk_cache(&self, cache: Option>) { self.inner.set_disk_cache(cache) } - fn set_localizer(&self, folder: Option<(std::path::PathBuf, u64)>) { - self.inner.set_localizer(folder) + fn set_localizer(&self, config: Option) { + self.inner.set_localizer(config) } - fn localize_status(&self) -> Option { + fn localize_status(&self) -> Option { self.inner.localize_status() } fn model(&self, item: ItemId, footage: &Footage) -> Option> { @@ -304,3 +304,34 @@ impl FootageSource for NoFootage { f() } } + +/// Settings ▸ Disk ▸ Localize Network Footage (#3271): where network footage is copied, how much +/// the copies may take, which locations count as network besides UNC paths, and how to read the +/// copies' disk's free space (the disk cache's reserve stays free there). +#[derive(Clone)] +pub struct LocalizeConfig { + pub folder: std::path::PathBuf, + pub max_bytes: u64, + /// Path prefixes that count as network footage (a drive letter mapped to a share, `P:\`). + pub network: Vec, + /// The copies' disk: `(total, available)` bytes, `None` unknown. + pub disk_space: Option Option<(u64, u64)> + Send + Sync>>, +} + +/// What the footage localizer is doing (Settings ▸ Disk shows it). +#[derive(Clone, Debug, Default, PartialEq)] +pub struct LocalizeReport { + /// Complete local copies and their bytes, and the most they may take. + pub files: usize, + pub bytes: u64, + pub max_bytes: u64, + /// Files waiting to be copied, and those too large or that failed to copy. + pub queued: usize, + pub skipped: usize, + /// The file being copied now. + pub copying: Option, + /// The locations that count as network besides UNC paths. + pub network: Vec, + /// Free disk space holds the copies below the size limit. + pub reserve_limited: bool, +} diff --git a/crates/render/src/lib.rs b/crates/render/src/lib.rs index 532fc4e79..20c93022d 100644 --- a/crates/render/src/lib.rs +++ b/crates/render/src/lib.rs @@ -48,7 +48,7 @@ mod opts; mod pipeline; pub use accel::{Accelerator, Backend, FxStep, FxTarget}; -pub use footage::{CancellableFootage, FloatCurve, FootageSource, NoFootage, is_vector_footage, missing_item_problem}; +pub use footage::{CancellableFootage, FloatCurve, FootageSource, LocalizeConfig, LocalizeReport, NoFootage, is_vector_footage, missing_item_problem}; pub use frames::{ adaptive_samples, blend_frames, cineon_frame, content_bounds, extended_region, field_frame, frame_position, frame_size, interpret_pixels, kurbo_path, render_frame, roi_inside, diff --git a/crates/render/src/memory.rs b/crates/render/src/memory.rs index 74ec65ac8..be806b507 100644 --- a/crates/render/src/memory.rs +++ b/crates/render/src/memory.rs @@ -19,6 +19,19 @@ use std::sync::{Arc, Mutex, PoisonError, Weak}; /// The share of the budget the accountant frees down to once it is over (hysteresis). pub const TRIM_TO: f64 = 0.9; +/// The decoded footage planes' own budget (the plane store in `effectcraft-media`): part of the +/// common budget's sum. +#[cfg(not(target_arch = "wasm32"))] +pub const PLANE_BUDGET: usize = 1 << 30; +#[cfg(target_arch = "wasm32")] +pub const PLANE_BUDGET: usize = 512 << 20; + +/// How long a holder that refilled stays protected. +const THRASH_FOR: std::time::Duration = std::time::Duration::from_secs(10); + +/// Inserts balance at most this often ([`Accountant::note_growth`]). +const BALANCE_EVERY: std::time::Duration = std::time::Duration::from_millis(100); + /// A cache the accountant keeps within the common budget. pub trait MemoryHolder: Send + Sync { /// A name for logs and stats. @@ -39,9 +52,9 @@ pub trait MemoryHolder: Send + Sync { struct Registered { holder: Weak, - /// Its refills at the last balance, and whether it was thrashing then. + /// Its refills at the last balance, and until when it counts as thrashing (it refilled). refills: u64, - thrashing: bool, + thrash_until: Option, } /// The common budget of the big caches (see the module notes). @@ -50,6 +63,9 @@ pub struct Accountant { holders: Mutex>, /// Bytes freed by the accountant so far (stats, tests). freed: AtomicUsize, + /// One balance at a time (another one asked for meanwhile is skipped), and when the last + /// one ran. + balancing: Mutex>, } impl Default for Accountant { @@ -61,7 +77,7 @@ impl Default for Accountant { impl Accountant { /// An accountant with `budget` bytes (`usize::MAX`: no limit). pub const fn new(budget: usize) -> Accountant { - Accountant { budget: AtomicUsize::new(budget), holders: Mutex::new(Vec::new()), freed: AtomicUsize::new(0) } + Accountant { budget: AtomicUsize::new(budget), holders: Mutex::new(Vec::new()), freed: AtomicUsize::new(0), balancing: Mutex::new(None) } } /// The process's accountant. @@ -89,7 +105,19 @@ impl Accountant { if h.iter().any(|r| r.holder.ptr_eq(&weak)) { return; } - h.push(Registered { holder: weak, refills: holder.refills(), thrashing: false }); + h.push(Registered { holder: weak, refills: holder.refills(), thrash_until: None }); + } + + /// A holder grew (an insert): balance, at most every [`BALANCE_EVERY`] (the holders' locks + /// aren't taken on every insert). + pub fn note_growth(&self) { + if self.budget() == usize::MAX { + return; + } + let recent = self.balancing.try_lock().map(|g| g.is_some_and(|t| t.elapsed() < BALANCE_EVERY)).unwrap_or(true); + if !recent { + self.balance(); + } } /// The holders alive now, with their effective priority (raised while thrashing). @@ -100,9 +128,13 @@ impl Accountant { .filter_map(|r| { let holder = r.holder.upgrade()?; let refills = holder.refills(); - r.thrashing = refills > r.refills; + let now = web_time::Instant::now(); + if refills > r.refills { + r.thrash_until = Some(now + THRASH_FOR); + } r.refills = refills; - let p = holder.priority().saturating_add(u8::from(r.thrashing)); + let thrashing = r.thrash_until.is_some_and(|t| t > now); + let p = holder.priority().saturating_add(u8::from(thrashing)); Some((holder, p)) }) .collect() @@ -128,6 +160,9 @@ impl Accountant { if budget == usize::MAX { return; } + // (one at a time: a balance running meanwhile does the work) + let Ok(mut last) = self.balancing.try_lock() else { return }; + *last = Some(web_time::Instant::now()); let mut holders = self.alive(); let mut total: usize = holders.iter().map(|(h, _)| h.bytes()).fold(0, usize::saturating_add); if total <= budget { diff --git a/crates/render/src/pipeline.rs b/crates/render/src/pipeline.rs index fd634db61..d000ec2d0 100644 --- a/crates/render/src/pipeline.rs +++ b/crates/render/src/pipeline.rs @@ -947,6 +947,11 @@ impl<'a> Renderer<'a> { /// parameter, an expression). pub(crate) fn footage_rows(&self, ctx: &EvalCtx, layer: &Layer, f: &Footage) -> Option<(i64, i64)> { let roi = self.opts.roi.filter(|_| self.top)?; + // (a collapsed precomp's layers are placed through the outer layers too, and an + // adjustment layer's footprint is keyed whole: the whole frame there) + if self.outer.is_some() || self.collapse3d.is_some() || self.depth > 0 || layer.switches.adjustment { + return None; + } if layer.switches.three_d || f.fields != effectcraft_project::FieldOrder::Off || !matches!(f.kind, FootageKind::Still | FootageKind::Sequence) @@ -974,7 +979,8 @@ impl<'a> Renderer<'a> { // pixels are 1/scale footage rows), plus the region's rounding to output pixels: one // output pixel is `k / scale` footage rows, `k` the footage rows per comp pixel. let s = self.opts.scale.max(1e-3); - let k = [effectcraft_geom::vec3(1.0, 0.0, 0.0), effectcraft_geom::vec3(0.0, 1.0, 0.0)].iter().map(|v| inv.apply_vec(*v).length()).fold(1.0f64, f64::max); + let k = + [effectcraft_geom::vec3(1.0, 0.0, 0.0), effectcraft_geom::vec3(0.0, 1.0, 0.0)].iter().map(|v| inv.apply_vec(*v).length()).fold(1.0f64, f64::max); if !k.is_finite() { return None; } @@ -986,7 +992,16 @@ impl<'a> Renderer<'a> { /// [`Renderer::footage_frame`] of only the rows `rows` (see [`Renderer::footage_rows`]). #[allow(clippy::too_many_arguments)] - fn footage_frame_in(&self, ctx: &EvalCtx, layer: &Layer, item: ItemId, f: &Footage, t: Tick, size: Option<(u32, u32)>, rows: Option<(i64, i64)>) -> Option> { + fn footage_frame_in( + &self, + ctx: &EvalCtx, + layer: &Layer, + item: ItemId, + f: &Footage, + t: Tick, + size: Option<(u32, u32)>, + rows: Option<(i64, i64)>, + ) -> Option> { // Interpret Footage ▸ Separate Fields: each field is a frame at twice the rate. if f.fields != effectcraft_project::FieldOrder::Off && matches!(f.kind, FootageKind::Video | FootageKind::Sequence) { let field_rate = FrameRate::new(f.frame_rate.num * 2, f.frame_rate.den); @@ -1053,7 +1068,7 @@ impl<'a> Renderer<'a> { } /// Is footage layer `layer` read in part ([`Renderer::footage_rows`]). - fn reads_footage_rows(&self, ctx: &EvalCtx, layer: &Layer) -> bool { + pub(crate) fn reads_footage_rows(&self, ctx: &EvalCtx, layer: &Layer) -> bool { let LayerSource::Footage { item } = &layer.source else { return false }; let item = ctx.media_item(layer, *item); if self.opts.roi.is_none() || !self.top || self.proxy_for(item).is_some() { @@ -1187,6 +1202,8 @@ impl<'a> Renderer<'a> { /// so scrubbing reuses frames rendered for earlier output frames. pub fn layer_input(&self, ctx: &EvalCtx, layer: &Layer, effects: usize) -> Option> { let key = self.cache.and_then(|_| cache::input_key(ctx, layer, self.raster_scale(ctx, layer), self.opts.draft, self.mb_on(ctx, layer), effects)); + // (a footage layer read in part is never kept under a whole layer's key, #3271) + let key = key.filter(|_| !self.reads_footage_rows(ctx, layer)); let key = self.content_key(ctx, layer, key); if let Some((b, _)) = self.cached(key) { return Some(b); diff --git a/crates/ui-egui/src/i18n/ui.rs b/crates/ui-egui/src/i18n/ui.rs index 15bf774f9..c95a2bd37 100644 --- a/crates/ui-egui/src/i18n/ui.rs +++ b/crates/ui-egui/src/i18n/ui.rs @@ -884,6 +884,16 @@ const JAPANESE: &[(&str, &str)] = &[ ("Media Cache", "メディアキャッシュ"), ("Database and Cache Folder", "データベースとキャッシュのフォルダー"), ("Conformed Audio Folder", "適合オーディオフォルダー"), + ("Network Footage", "ネットワークフッテージ"), + ("Localize Network Footage", "ネットワークフッテージをローカルにコピー"), + ("Maximum Local Copies Size", "ローカルコピーの最大サイズ"), + ("Local Copies Folder", "ローカルコピーのフォルダー"), + ("Network Footage Locations", "ネットワークフッテージの場所"), + ("Local copies: {} files, {} of {} GB; {} queued, {} skipped", "ローカルコピー: {} ファイル、{} / {} GB、待機 {}、スキップ {}"), + ("Copying {}", "コピー中: {}"), + ("UNC paths (\\\\server\\share)", "UNC パス (\\\\server\\share)"), + ("Network footage: {}", "ネットワークフッテージ: {}"), + ("Limited by free disk space", "ディスクの空き容量により制限されています"), ("Memory & CPU", "メモリとCPU"), ("RAM Reserved for Other Applications", "他のアプリケーション用に確保するRAM"), ("Cache Budgets", "キャッシュの割り当て"), @@ -2015,6 +2025,16 @@ const SIMPLIFIED_CHINESE: &[(&str, &str)] = &[ ("Media Cache", "媒体缓存"), ("Database and Cache Folder", "数据库和缓存文件夹"), ("Conformed Audio Folder", "符合的音频文件夹"), + ("Network Footage", "网络素材"), + ("Localize Network Footage", "将网络素材复制到本地"), + ("Maximum Local Copies Size", "本地副本最大大小"), + ("Local Copies Folder", "本地副本文件夹"), + ("Network Footage Locations", "网络素材位置"), + ("Local copies: {} files, {} of {} GB; {} queued, {} skipped", "本地副本:{} 个文件,{} / {} GB;{} 个排队,{} 个跳过"), + ("Copying {}", "正在复制 {}"), + ("UNC paths (\\\\server\\share)", "UNC 路径 (\\\\server\\share)"), + ("Network footage: {}", "网络素材:{}"), + ("Limited by free disk space", "受磁盘可用空间限制"), ("Memory & CPU", "内存和 CPU"), ("RAM Reserved for Other Applications", "为其他应用程序保留的内存"), ("Cache Budgets", "缓存预算"), @@ -3126,6 +3146,16 @@ const SWEDISH: &[(&str, &str)] = &[ ("Media Cache", "Mediecache"), ("Database and Cache Folder", "Databas- och cachemapp"), ("Conformed Audio Folder", "Mapp för konformerat ljud"), + ("Network Footage", "Nätverksmaterial"), + ("Localize Network Footage", "Kopiera nätverksmaterial lokalt"), + ("Maximum Local Copies Size", "Största storlek för lokala kopior"), + ("Local Copies Folder", "Mapp för lokala kopior"), + ("Network Footage Locations", "Platser för nätverksmaterial"), + ("Local copies: {} files, {} of {} GB; {} queued, {} skipped", "Lokala kopior: {} filer, {} av {} GB; {} i kö, {} överhoppade"), + ("Copying {}", "Kopierar {}"), + ("UNC paths (\\\\server\\share)", "UNC-sökvägar (\\\\server\\share)"), + ("Network footage: {}", "Nätverksmaterial: {}"), + ("Limited by free disk space", "Begränsas av ledigt diskutrymme"), ("Memory & CPU", "Minne och processor"), ("RAM Reserved for Other Applications", "RAM reserverat för andra program"), ("Cache Budgets", "Cachebudgetar"), @@ -4343,6 +4373,16 @@ const TRADITIONAL_CHINESE: &[(&str, &str)] = &[ ("Media Cache", "媒體快取"), ("Database and Cache Folder", "資料庫和快取資料夾"), ("Conformed Audio Folder", "符合的音訊資料夾"), + ("Network Footage", "網路素材"), + ("Localize Network Footage", "將網路素材複製到本機"), + ("Maximum Local Copies Size", "本機副本最大大小"), + ("Local Copies Folder", "本機副本資料夾"), + ("Network Footage Locations", "網路素材位置"), + ("Local copies: {} files, {} of {} GB; {} queued, {} skipped", "本機副本:{} 個檔案,{} / {} GB;{} 個排隊,{} 個略過"), + ("Copying {}", "正在複製 {}"), + ("UNC paths (\\\\server\\share)", "UNC 路徑 (\\\\server\\share)"), + ("Network footage: {}", "網路素材:{}"), + ("Limited by free disk space", "受磁碟可用空間限制"), ("Memory & CPU", "記憶體和 CPU"), ("RAM Reserved for Other Applications", "為其他應用程式保留的記憶體"), ("Cache Budgets", "快取預算"), @@ -5829,6 +5869,16 @@ const PORTUGUESE: &[(&str, &str)] = &[ ("Brightness", "Brilho"), ("Cache Budgets", "Orçamentos de cache"), ("Conformed Audio Folder", "Pasta de áudio conformado"), + ("Network Footage", "Mídia de rede"), + ("Localize Network Footage", "Copiar mídia de rede localmente"), + ("Maximum Local Copies Size", "Tamanho máximo das cópias locais"), + ("Local Copies Folder", "Pasta das cópias locais"), + ("Network Footage Locations", "Locais de mídia de rede"), + ("Local copies: {} files, {} of {} GB; {} queued, {} skipped", "Cópias locais: {} arquivos, {} de {} GB; {} na fila, {} ignorados"), + ("Copying {}", "Copiando {}"), + ("UNC paths (\\\\server\\share)", "Caminhos UNC (\\\\server\\share)"), + ("Network footage: {}", "Mídia de rede: {}"), + ("Limited by free disk space", "Limitado pelo espaço livre em disco"), ("Create Layers at Composition Start Time", "Criar camadas no tempo inicial da composição"), ("Create Split Layers Above Original Layer", "Criar camadas divididas acima da camada original"), ("Custom", "Personalizado"), @@ -6805,6 +6855,16 @@ const POLISH: &[(&str, &str)] = &[ ("Media Cache", "Pamięć multimediów"), ("Database and Cache Folder", "Folder bazy danych i pamięci"), ("Conformed Audio Folder", "Folder dopasowanego audio"), + ("Network Footage", "Materiał sieciowy"), + ("Localize Network Footage", "Kopiuj materiał sieciowy lokalnie"), + ("Maximum Local Copies Size", "Maksymalny rozmiar kopii lokalnych"), + ("Local Copies Folder", "Folder kopii lokalnych"), + ("Network Footage Locations", "Lokalizacje materiału sieciowego"), + ("Local copies: {} files, {} of {} GB; {} queued, {} skipped", "Kopie lokalne: {} plików, {} z {} GB; {} w kolejce, {} pominiętych"), + ("Copying {}", "Kopiowanie {}"), + ("UNC paths (\\\\server\\share)", "Ścieżki UNC (\\\\server\\share)"), + ("Network footage: {}", "Materiał sieciowy: {}"), + ("Limited by free disk space", "Ograniczone wolnym miejscem na dysku"), ("Memory & CPU", "Pamięć i procesor"), ("RAM Reserved for Other Applications", "Pamięć RAM zarezerwowana dla innych aplikacji"), ("Cache Budgets", "Budżety pamięci podręcznej"), @@ -8707,6 +8767,16 @@ const GEORGIAN: &[(&str, &str)] = &[ ("Media Cache", "მედიაქეში"), ("Database and Cache Folder", "მონაცემთა ბაზისა და ქეშის საქაღალდე"), ("Conformed Audio Folder", "მისადაგებული აუდიოს საქაღალდე"), + ("Network Footage", "ქსელური მასალა"), + ("Localize Network Footage", "ქსელური მასალის ლოკალურად კოპირება"), + ("Maximum Local Copies Size", "ლოკალური ასლების მაქსიმალური ზომა"), + ("Local Copies Folder", "ლოკალური ასლების საქაღალდე"), + ("Network Footage Locations", "ქსელური მასალის მდებარეობები"), + ("Local copies: {} files, {} of {} GB; {} queued, {} skipped", "ლოკალური ასლები: {} ფაილი, {} / {} GB; რიგში {}, გამოტოვებული {}"), + ("Copying {}", "კოპირდება {}"), + ("UNC paths (\\\\server\\share)", "UNC ბილიკები (\\\\server\\share)"), + ("Network footage: {}", "ქსელური მასალა: {}"), + ("Limited by free disk space", "შეზღუდულია დისკის თავისუფალი ადგილით"), ("Memory & CPU", "მეხსიერება და CPU"), ("RAM Reserved for Other Applications", "სხვა აპლიკაციებისთვის დაჯავშნული RAM"), ("Cache Budgets", "ქეშის ლიმიტები"), @@ -9689,6 +9759,16 @@ const FRENCH: &[(&str, &str)] = &[ ("Media Cache", "Cache des médias"), ("Database and Cache Folder", "Dossier de la base de données et du cache"), ("Conformed Audio Folder", "Dossier de l’audio conformé"), + ("Network Footage", "Métrage réseau"), + ("Localize Network Footage", "Copier le métrage réseau en local"), + ("Maximum Local Copies Size", "Taille maximale des copies locales"), + ("Local Copies Folder", "Dossier des copies locales"), + ("Network Footage Locations", "Emplacements du métrage réseau"), + ("Local copies: {} files, {} of {} GB; {} queued, {} skipped", "Copies locales : {} fichiers, {} sur {} Go ; {} en attente, {} ignorés"), + ("Copying {}", "Copie de {}"), + ("UNC paths (\\\\server\\share)", "Chemins UNC (\\\\server\\share)"), + ("Network footage: {}", "Métrage réseau : {}"), + ("Limited by free disk space", "Limité par l’espace disque libre"), ("Memory & CPU", "Mémoire et CPU"), ("RAM Reserved for Other Applications", "Mémoire vive réservée aux autres applications"), ("Cache Budgets", "Budgets de cache"), diff --git a/crates/ui-egui/src/panels/project.rs b/crates/ui-egui/src/panels/project.rs index a2b48d04d..ba9da58ae 100644 --- a/crates/ui-egui/src/panels/project.rs +++ b/crates/ui-egui/src/panels/project.rs @@ -127,21 +127,30 @@ type PendingThumb = (u128, Arc)>>>>); fn thumbnail_key(app: &EffectcraftApp, ctx: &egui::Context, it: &Item, t: effectcraft_engine::time::Tick) -> u128 { let id = egui::Id::unique(("proj-thumb-key", it.id.0)); let rev = app.session.revision; - if let Some((r, at, k)) = ctx.data(|d| d.get_temp::<(u64, i64, u128)>(id)) - && r == rev - && at == t.0 - { - return k; + let now = ctx.input(|i| i.time); + if let Some((r, at, k, when)) = ctx.data(|d| d.get_temp::<(u64, i64, u128, f64)>(id)) { + if r == rev && at == t.0 { + return k; + } + // (the key hashes the comp's content: while edits come fast, at most every 0.25 s) + if now - when < THUMB_KEY_EVERY { + ctx.request_repaint_after(std::time::Duration::from_secs_f64(THUMB_KEY_EVERY)); + return k; + } } // (no key: the revision stands in, as before) let k = app.session.thumbnail_key(it.id, t, THUMB_SIDE).unwrap_or(u128::from(rev) | (1 << 127)); - ctx.data_mut(|d| d.insert_temp(id, (rev, t.0, k))); + ctx.data_mut(|d| d.insert_temp(id, (rev, t.0, k, now))); k } /// The longest side of a Project panel thumbnail, in pixels. const THUMB_SIDE: u32 = 192; +/// A thumbnail's content key is worked out again at most this often (seconds) while the +/// project keeps changing. +const THUMB_KEY_EVERY: f64 = 0.25; + /// Thumbnail of a comp (its current time) or footage (first frame), cached per item by what it /// shows ([`thumbnail_key`], #3271): an edit that doesn't change the picture doesn't render it /// again. Rendered on a background thread (inline on wasm32): until the new one lands the diff --git a/crates/ui-egui/src/panels/settings.rs b/crates/ui-egui/src/panels/settings.rs index 20732394b..2b4a3afa7 100644 --- a/crates/ui-egui/src/panels/settings.rs +++ b/crates/ui-egui/src/panels/settings.rs @@ -239,17 +239,44 @@ fn page_ui(app: &mut EffectcraftApp, ui: &mut egui::Ui, page: &Page, cur: &Value disk_cache_limit_ui(app, ui, t); } if key == "disk.localizeFootage" - && let Some(line) = app.session.localize_status() + && let Some(s) = app.session.localize_status() { - // (the localizer's state: local copies, what is queued and being copied) - let r = ui.label(RichText::new(line).color(t.text_dim)); - app.auto.add("settings.disk.localizeStatus", r.rect, "localize status"); + localize_status_ui(app, ui, t, &s); } } } } } +/// Settings ▸ Disk, under Localize Network Footage (#3271): the local copies, what is queued and +/// being copied, and which locations count as network. Automation id +/// `settings.disk.localizeStatus`, labelled `files=… bytes=… queued=… skipped=…`. +fn localize_status_ui(app: &mut EffectcraftApp, ui: &mut egui::Ui, t: &Tokens, s: &effectcraft_engine::render::LocalizeReport) { + const GB: f64 = (1u64 << 30) as f64; + let mut lines = vec![tr_args( + "Local copies: {} files, {} of {} GB; {} queued, {} skipped", + &[&s.files, &format!("{:.1}", s.bytes as f64 / GB), &format!("{:.1}", s.max_bytes as f64 / GB), &s.queued, &s.skipped], + )]; + if let Some(p) = &s.copying { + lines.push(tr_args("Copying {}", &[p])); + } + let mut network = vec![tr("UNC paths (\\\\server\\share)").to_string()]; + network.extend(s.network.iter().cloned()); + lines.push(tr_args("Network footage: {}", &[&network.join(", ")])); + if s.reserve_limited { + lines.push(tr("Limited by free disk space").to_string()); + } + let r = ui + .vertical(|ui| { + for line in &lines { + ui.label(RichText::new(line).color(t.text_dim)); + } + }) + .response; + let label = format!("files={} bytes={} queued={} skipped={}", s.files, s.bytes, s.queued, s.skipped); + app.auto.add("settings.disk.localizeStatus", r.rect, &label); +} + /// Settings ▸ Disk, under Maximum Disk Cache Size: the limit in effect, and why it is lower when /// free disk space holds the cache below the setting (it never fills the disk). Automation id /// `settings.disk.cacheLimit`, labelled `limitMb=… maxMb=… reserveMb=… spaceLimited=… writes=…` diff --git a/docs/preferences.md b/docs/preferences.md index b8bbb2bcf..85185c221 100644 --- a/docs/preferences.md +++ b/docs/preferences.md @@ -153,6 +153,15 @@ Commands (CLI, MCP, control channel): - `disk.mediaCacheFolder`: audio waveform summaries kept between sessions (`Peaks/`) - `disk.conformedMediaFolder`: decoded (conformed) footage audio, written once per file and sample rate and read back instead of decoding again +- `disk.localizeFootage` (off by default), `disk.localizeMaxGb` (20), `disk.localizeFolder`, + `disk.networkFootageLocations`: OpenEXR footage on network shares is copied, one file at a + time on a background thread, to a local folder (by default beside the disk cache's folder), + and read from the copy once it is complete and the source still has the size and modification + time it had when copied. Network footage is UNC paths (`\server\share\…`) and the locations + listed in `disk.networkFootageLocations` (`;`-separated path prefixes, such as a drive letter + mapped to a share: `P:\`). The copies take at most `disk.localizeMaxGb` and leave the disk + cache's reserve free on their disk; earlier sessions' copies are adopted. The Disk page shows + the local copies, what is queued and being copied, and the network locations - `memory.ramReservedGb`: the cache budgets together leave this much physical memory free - `memory.reduceCacheWhenLow`: cache budgets halve while the system is low on memory (checked every 30 s by the desktop app, on a background thread)