mirror of
https://github.com/storytold/effectcraft.git
synced 2026-11-20 00:55:43 +00:00
Footage cache (#3271): a file being copied by the localizer is not queued again
A read asking for a network file while the localizer copied it queued it again, and the file was copied over the network a second time. The file being copied is now remembered with the setting it is copied under; asked for again under the same setting it isn't queued, under a new one it is. A test holds a copy and asks again; it fails on the old code.
This commit is contained in:
1 parent
643c795bbf
commit
086c53093f
1 file changed
+71
-5
@@ -44,7 +44,9 @@ struct State {
|
||||
queued: HashSet<PathBuf>,
|
||||
bytes: u64,
|
||||
clock: u64,
|
||||
copying: Option<PathBuf>,
|
||||
/// The file being copied now and the setting ([`State::epoch`]) it is copied under: asked
|
||||
/// for again meanwhile it isn't queued a second time (unless the setting changed since).
|
||||
copying: Option<(PathBuf, u64)>,
|
||||
skipped: HashSet<PathBuf>,
|
||||
worker: bool,
|
||||
/// Bumped with each new setting: a copy started under another one is thrown away.
|
||||
@@ -180,7 +182,7 @@ pub fn status() -> Option<LocalizeReport> {
|
||||
bytes: st.bytes,
|
||||
max_bytes: c.max_bytes,
|
||||
queued: st.queue.len(),
|
||||
copying: st.copying.as_ref().map(|p| p.to_string_lossy().to_string()),
|
||||
copying: st.copying.as_ref().map(|(p, _)| p.to_string_lossy().to_string()),
|
||||
skipped: st.skipped.len(),
|
||||
network: c.network.iter().map(|p| p.to_string_lossy().to_string()).collect(),
|
||||
reserve_limited: st.reserve_limited,
|
||||
@@ -223,13 +225,15 @@ pub(crate) fn is_network(path: &Path) -> bool {
|
||||
}
|
||||
|
||||
/// Ask for a local copy of the network file `original` (nothing when the localizer is off, the
|
||||
/// file is local, already copied, queued or skipped). Returns at once.
|
||||
/// file is local, already copied, queued, being copied under this setting, or skipped).
|
||||
/// Returns at once.
|
||||
pub(crate) fn request(original: &Path) {
|
||||
if !is_network(original) {
|
||||
return;
|
||||
}
|
||||
let mut st = lock();
|
||||
if st.config.is_none() || st.copies.contains_key(original) || st.queued.contains(original) || st.skipped.contains(original) {
|
||||
let copying = st.copying.as_ref().is_some_and(|(p, epoch)| p == original && *epoch == st.epoch);
|
||||
if st.config.is_none() || copying || st.copies.contains_key(original) || st.queued.contains(original) || st.skipped.contains(original) {
|
||||
return;
|
||||
}
|
||||
if st.queue.len() >= QUEUE_LIMIT
|
||||
@@ -329,7 +333,7 @@ fn worker() {
|
||||
if let (Some(p), Some(c)) = (st.queue.front().cloned(), st.config.clone()) {
|
||||
st.queue.pop_front();
|
||||
st.queued.remove(&p);
|
||||
st.copying = Some(p.clone());
|
||||
st.copying = Some((p.clone(), st.epoch));
|
||||
let room = limit(&mut st);
|
||||
break (p, c, st.epoch, room);
|
||||
}
|
||||
@@ -373,6 +377,13 @@ fn worker() {
|
||||
/// copied).
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
fn copy_file(original: &Path, folder: &Path, room: u64) -> Option<(PathBuf, u64, FileStamp)> {
|
||||
#[cfg(test)]
|
||||
{
|
||||
let hook = BEFORE_COPY.lock().unwrap_or_else(PoisonError::into_inner).clone();
|
||||
if let Some(h) = hook {
|
||||
h(original);
|
||||
}
|
||||
}
|
||||
let meta = std::fs::metadata(original).ok().filter(std::fs::Metadata::is_file)?;
|
||||
if meta.len() > room {
|
||||
return None;
|
||||
@@ -408,6 +419,12 @@ fn copy_file(original: &Path, folder: &Path, room: u64) -> Option<(PathBuf, u64,
|
||||
}
|
||||
}
|
||||
|
||||
/// Runs before each copy, with the source's path (tests).
|
||||
#[cfg(test)]
|
||||
type CopyHook = std::sync::Arc<dyn Fn(&Path) + Send + Sync>;
|
||||
#[cfg(test)]
|
||||
static BEFORE_COPY: Mutex<Option<CopyHook>> = Mutex::new(None);
|
||||
|
||||
/// The localizer state is process-wide: tests that use it take turns.
|
||||
#[cfg(test)]
|
||||
pub(crate) static TEST_SERIAL: Mutex<()> = Mutex::new(());
|
||||
@@ -480,4 +497,53 @@ mod tests {
|
||||
assert!(status().is_none());
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
}
|
||||
|
||||
/// #3271 review: a file asked for again while it is being copied is not copied a second
|
||||
/// time, but asked for after the setting changed it is (the copy under the old setting is
|
||||
/// thrown away).
|
||||
#[test]
|
||||
fn a_file_being_copied_is_not_queued_again() {
|
||||
let _serial = TEST_SERIAL.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
let root = std::env::temp_dir().join(format!("ec-3271-localize-dup-{}", std::process::id()));
|
||||
let (share, local) = (root.join("share"), root.join("local"));
|
||||
std::fs::create_dir_all(&share).unwrap();
|
||||
treat_as_network(&share);
|
||||
let file = share.join("plate_0002.exr");
|
||||
std::fs::write(&file, vec![3u8; 2048]).unwrap();
|
||||
// Each copy waits until the test lets it go on.
|
||||
let copies = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let (go, wait) = std::sync::mpsc::channel::<()>();
|
||||
{
|
||||
let (copies, wait) = (copies.clone(), Mutex::new(wait));
|
||||
*BEFORE_COPY.lock().unwrap() = Some(std::sync::Arc::new(move |_: &Path| {
|
||||
copies.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
let _ = wait.lock().unwrap().recv_timeout(std::time::Duration::from_secs(10));
|
||||
}));
|
||||
}
|
||||
let copies_now = || copies.load(std::sync::atomic::Ordering::SeqCst);
|
||||
configure(config(&local, 1 << 20, vec![]));
|
||||
request(&file);
|
||||
wait_for(|s| s.copying.is_some());
|
||||
request(&file);
|
||||
assert_eq!(status().unwrap().queued, 0, "not queued while it is copied");
|
||||
go.send(()).unwrap();
|
||||
wait_for(|s| s.files == 1 && s.copying.is_none());
|
||||
assert_eq!(copies_now(), 1, "copied once");
|
||||
// A new setting while a copy runs: asked for again, it is copied again.
|
||||
let file2 = share.join("plate_0003.exr");
|
||||
std::fs::write(&file2, vec![4u8; 1024]).unwrap();
|
||||
request(&file2);
|
||||
wait_for(|s| s.copying.is_some());
|
||||
configure(config(&local, 2 << 20, vec![]));
|
||||
request(&file2);
|
||||
assert_eq!(status().unwrap().queued, 1, "queued again under the new setting");
|
||||
go.send(()).unwrap();
|
||||
go.send(()).unwrap();
|
||||
let s = wait_for(|s| s.files == 2 && s.queued == 0 && s.copying.is_none());
|
||||
assert_eq!(s.files, 2, "{s:?}");
|
||||
assert_eq!(copies_now(), 3);
|
||||
*BEFORE_COPY.lock().unwrap() = None;
|
||||
configure(None);
|
||||
let _ = std::fs::remove_dir_all(&root);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user