#3538: OCIO transform files and configs follow their files through #3493's LUT cache

Move the LUT cache that follows files by size and modification time into
effects::file_cache and use it for ocio::lut::load_file and
ocio_config::load_config. A config's entry depends on the .ocio file, the LUTs
its FileTransforms read and the search-path places it looked for missing ones.
check_lut_files and lut_generation now cover all three caches.

crates/effects/src/file_cache.rs

@@ -0,0 +1,415 @@
+//! Caches of what effects read from files named by a parameter (a LUT, an OCIO transform file, an
+//! OCIO config), keyed by the parameter text and checked against the files each result was read
+//! from (#3493, #3538).
+//!
+//! A cached result remembers every file its load read, with the file's size and modification time
+//! (its stamp) taken before reading, including the files a nested cached load read (a config's
+//! `FileTransform` LUTs) and files looked for but missing. Each lookup stamps those files again
+//! (one `metadata` call each) and loads again once one moved, so a file edited, created, deleted or
+//! repaired on disk is followed. Inline text reads no file and is its own content. An edit that keeps
+//! both the size and the modification time can't be told apart.
+//!
+//! Every file read is also followed by [`check_lut_files`], and [`lut_generation`] moves when one is
+//! found changed: the viewers and frame caches drop the frames made before.
+
+use std::cell::RefCell;
+use std::collections::{HashMap, HashSet};
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::{Mutex, MutexGuard, OnceLock, PoisonError};
+use std::time::SystemTime;
+
+/// A file's size and modification time when it was read (`None`: missing, or a file that could
+/// not be described). A file whose stamp moved is read again.
+pub(crate) type FileStamp = Option<(u64, Option<SystemTime>)>;
+
+/// The files a load read and their stamps.
+type Files = Vec<(String, FileStamp)>;
+
+/// At most this many files are followed by [`check_lut_files`]; the least recently read go first.
+const MAX_FOLLOWED_FILES: usize = 1024;
+
+/// One cached result and what it was read from.
+struct Entry<V> {
+    files: Files,
+    value: V,
+    used: u64,
+    /// The result's size estimate and the parameter text, counted against the cache's byte bound.
+    bytes: usize,
+}
+
+/// Results by parameter text, at most `max_entries` and `max_bytes` of them, the least recently
+/// used going first.
+pub(crate) struct SourceCache<V> {
+    map: HashMap<String, Entry<V>>,
+    clock: u64,
+    bytes: usize,
+    max_entries: usize,
+    max_bytes: usize,
+}
+
+impl<V> SourceCache<V> {
+    pub(crate) fn new(max_entries: usize, max_bytes: usize) -> Self {
+        SourceCache { map: HashMap::new(), clock: 0, bytes: 0, max_entries, max_bytes }
+    }
+
+    fn remove(&mut self, src: &str) {
+        if let Some(e) = self.map.remove(src) {
+            self.bytes = self.bytes.saturating_sub(e.bytes);
+        }
+    }
+
+    /// Forget the results read from `path`.
+    fn forget_file(&mut self, path: &str) {
+        let stale: Vec<String> = self.map.iter().filter(|(_, e)| e.files.iter().any(|(p, _)| p == path)).map(|(k, _)| k.clone()).collect();
+        for src in stale {
+            self.remove(&src);
+        }
+    }
+
+    /// Keep within the bounds, always the result for `keep`.
+    fn evict(&mut self, keep: &str) {
+        while self.map.len() > self.max_entries || (self.bytes > self.max_bytes && self.map.len() > 1) {
+            let Some(oldest) = self.map.iter().filter(|(k, _)| k.as_str() != keep).min_by_key(|(_, e)| e.used).map(|(k, _)| k.clone()) else { break };
+            self.remove(&oldest);
+        }
+    }
+
+    /// The files the result cached for `src` was read from.
+    pub(crate) fn files(&self, src: &str) -> Option<Vec<String>> {
+        self.map.get(src).map(|e| e.files.iter().map(|(p, _)| p.clone()).collect())
+    }
+
+    #[cfg(test)]
+    pub(crate) fn len(&self) -> usize {
+        self.map.len()
+    }
+
+    #[cfg(test)]
+    pub(crate) fn contains(&self, src: &str) -> bool {
+        self.map.contains_key(src)
+    }
+
+    #[cfg(test)]
+    pub(crate) fn bytes(&self) -> (usize, usize, usize) {
+        (self.bytes, self.map.values().map(|e| e.bytes).sum(), self.max_bytes)
+    }
+}
+
+/// Lock a cache; a panic elsewhere while it was held leaves a usable map.
+pub(crate) fn lock<V>(cache: &Mutex<SourceCache<V>>) -> MutexGuard<'_, SourceCache<V>> {
+    cache.lock().unwrap_or_else(PoisonError::into_inner)
+}
+
+/// A cache that registers with [`check_lut_files`] on first use.
+pub(crate) struct Registered<V: 'static> {
+    cache: OnceLock<Mutex<SourceCache<V>>>,
+    max_entries: usize,
+    max_bytes: usize,
+}
+
+impl<V: Clone + Send + 'static> Registered<V> {
+    pub(crate) const fn new(max_entries: usize, max_bytes: usize) -> Self {
+        Registered { cache: OnceLock::new(), max_entries, max_bytes }
+    }
+
+    pub(crate) fn get(&'static self) -> &'static Mutex<SourceCache<V>> {
+        self.cache.get_or_init(|| {
+            registry().push(self);
+            Mutex::new(SourceCache::new(self.max_entries, self.max_bytes))
+        })
+    }
+}
+
+/// A cache [`check_lut_files`] tells to forget the results read from a changed file.
+trait ForgetFile: Sync {
+    fn forget_file(&self, path: &str);
+    /// Add the files the cached results were read from to `out`.
+    fn held_files(&self, out: &mut HashSet<String>);
+}
+
+impl<V: Send> ForgetFile for Registered<V> {
+    fn forget_file(&self, path: &str) {
+        if let Some(c) = self.cache.get() {
+            lock(c).forget_file(path);
+        }
+    }
+
+    fn held_files(&self, out: &mut HashSet<String>) {
+        if let Some(c) = self.cache.get() {
+            out.extend(lock(c).map.values().flat_map(|e| e.files.iter().map(|(p, _)| p.clone())));
+        }
+    }
+}
+
+fn registry() -> MutexGuard<'static, Vec<&'static dyn ForgetFile>> {
+    static REGISTRY: OnceLock<Mutex<Vec<&'static dyn ForgetFile>>> = OnceLock::new();
+    REGISTRY.get_or_init(Mutex::default).lock().unwrap_or_else(PoisonError::into_inner)
+}
+
+thread_local! {
+    /// The files read by the loads running on this thread, innermost last.
+    static READING: RefCell<Vec<Files>> = const { RefCell::new(Vec::new()) };
+}
+
+/// Note that the running load read (or looked for) `path`, stamped `stamp`.
+pub(crate) fn note_file(path: &str, stamp: FileStamp) {
+    READING.with(|r| {
+        if let Ok(mut r) = r.try_borrow_mut()
+            && let Some(top) = r.last_mut()
+            && !top.iter().any(|(p, _)| p == path)
+        {
+            top.push((path.to_string(), stamp));
+        }
+    });
+}
+
+/// Pops the files of a load when it ends, even by a panic.
+struct Reading {
+    popped: bool,
+}
+
+impl Reading {
+    fn start() -> Reading {
+        READING.with(|r| {
+            if let Ok(mut r) = r.try_borrow_mut() {
+                r.push(Vec::new());
+            }
+        });
+        Reading { popped: false }
+    }
+
+    fn pop() -> Option<Files> {
+        READING.with(|r| r.try_borrow_mut().ok().and_then(|mut r| r.pop()))
+    }
+
+    fn files(mut self) -> Files {
+        self.popped = true;
+        Self::pop().unwrap_or_default()
+    }
+}
+
+impl Drop for Reading {
+    fn drop(&mut self) {
+        if !self.popped {
+            let _ = Self::pop();
+        }
+    }
+}
+
+/// The stamp of the file at `path` now: one `metadata` call.
+pub(crate) fn file_stamp(path: &str) -> FileStamp {
+    std::fs::metadata(path).ok().map(|md| (md.len(), md.modified().ok()))
+}
+
+/// Read the text file at `path` for the running load, stamped before reading (a file written
+/// after it was stamped has a newer stamp, and is read again next time).
+pub(crate) fn read(path: &str) -> std::io::Result<String> {
+    note_file(path, file_stamp(path));
+    std::fs::read_to_string(path)
+}
+
+/// Note that the running load looked for `path` and did not find it: it loads again once the file
+/// appears.
+pub(crate) fn note_missing(path: &str) {
+    note_file(path, None);
+}
+
+/// The result cached for `src`, or `load`'s (and its size estimate in bytes), cached. `load` reads
+/// files with [`read`] (and notes the ones it looked for with [`note_missing`]); no lock is held
+/// while it runs or while the files are stamped.
+pub(crate) fn cached<V: Clone>(cache: &Mutex<SourceCache<V>>, src: &str, load: impl FnOnce() -> (V, usize)) -> V {
+    let files = lock(cache).map.get(src).map(|e| e.files.clone());
+    let stale = files.is_some();
+    if let Some(files) = files
+        && files.iter().all(|(p, s)| file_stamp(p) == *s)
+    {
+        let mut c = lock(cache);
+        c.clock += 1;
+        let now = c.clock;
+        if let Some(e) = c.map.get_mut(src)
+            && e.files == files
+        {
+            e.used = now;
+            let value = e.value.clone();
+            drop(c);
+            // (an enclosing load depends on these files too)
+            for (p, s) in &files {
+                note_file(p, *s);
+            }
+            return value;
+        }
+    }
+    let reading = Reading::start();
+    let (value, size) = load();
+    let files = reading.files();
+    follow(&files, stale);
+    for (p, s) in &files {
+        note_file(p, *s);
+    }
+    let bytes = size.saturating_add(src.len());
+    let mut c = lock(cache);
+    c.clock += 1;
+    let used = c.clock;
+    c.remove(src);
+    c.map.insert(src.to_string(), Entry { files, value: value.clone(), used, bytes });
+    c.bytes = c.bytes.saturating_add(bytes);
+    c.evict(src);
+    value
+}
+
+/// How often a file a cached load read was found changed on disk ([`lut_generation`]).
+static GENERATION: AtomicU64 = AtomicU64::new(0);
+
+/// The files followed by [`check_lut_files`]: the stamp each was last read with, and when.
+#[derive(Default)]
+struct Followed {
+    files: HashMap<String, (FileStamp, u64)>,
+    clock: u64,
+}
+
+fn followed() -> MutexGuard<'static, Followed> {
+    static FOLLOWED: OnceLock<Mutex<Followed>> = OnceLock::new();
+    FOLLOWED.get_or_init(Mutex::default).lock().unwrap_or_else(PoisonError::into_inner)
+}
+
+impl Followed {
+    /// Note that `files` were read with these stamps; true when one has a different stamp than
+    /// its last read's: a file changed on disk.
+    fn read(&mut self, files: &Files) -> bool {
+        let mut moved = false;
+        for (path, stamp) in files {
+            self.clock += 1;
+            let now = self.clock;
+            if self.files.insert(path.clone(), (*stamp, now)).is_some_and(|(old, _)| old != *stamp) {
+                moved = true;
+            }
+        }
+        moved
+    }
+
+    /// Keep at most [`MAX_FOLLOWED_FILES`], letting go of the least recently read first but never
+    /// of a file in `held` (a cached result was read from it) or in `files` (just read).
+    fn trim(&mut self, held: &HashSet<String>, files: &Files) {
+        while self.files.len() > MAX_FOLLOWED_FILES {
+            let Some(oldest) = self
+                .files
+                .iter()
+                .filter(|(p, _)| !held.contains(p.as_str()) && !files.iter().any(|(f, _)| f == *p))
+                .min_by_key(|(_, (_, used))| *used)
+                .map(|(p, _)| p.clone())
+            else {
+                break;
+            };
+            self.files.remove(&oldest);
+        }
+    }
+}
+
+/// The files the registered caches' results were read from.
+fn held_files() -> HashSet<String> {
+    let caches: Vec<&'static dyn ForgetFile> = registry().clone();
+    let mut out = HashSet::new();
+    for c in caches {
+        c.held_files(&mut out);
+    }
+    out
+}
+
+/// Note that `files` were read with these stamps. A file read with a different stamp than last
+/// time, or a load replacing a result whose files moved (`stale`), means a file changed on disk,
+/// and frames made before may show the old contents.
+fn follow(files: &Files, stale: bool) {
+    let (moved, over) = {
+        let mut f = followed();
+        (f.read(files), f.files.len() > MAX_FOLLOWED_FILES)
+    };
+    if over {
+        // (gathered without the followed files' lock: no two of these locks are ever nested)
+        let held = held_files();
+        followed().trim(&held, files);
+    }
+    if moved || stale {
+        GENERATION.fetch_add(1, Ordering::AcqRel);
+    }
+}
+
+/// Changes each time a file a cached load read is found changed on disk (by a load itself or
+/// [`check_lut_files`]): frames made before then may show the old contents.
+pub fn lut_generation() -> u64 {
+    GENERATION.load(Ordering::Acquire)
+}
+
+/// Look at every file read (one `metadata` call each) and forget the results read from the ones
+/// that changed on disk since, so the next load reads them again. Returns [`lut_generation`].
+pub fn check_lut_files() -> u64 {
+    let files: Vec<(String, FileStamp)> = followed().files.iter().map(|(p, (s, _))| (p.clone(), *s)).collect();
+    // (stamped without the lock: a file on a slow share must not hold up renders)
+    let changed: Vec<(String, FileStamp, FileStamp)> =
+        files.into_iter().filter_map(|(path, old)| Some(file_stamp(&path)).filter(|now| *now != old).map(|now| (path, old, now))).collect();
+    let caches: Vec<&'static dyn ForgetFile> = registry().clone();
+    for (path, old, now) in changed {
+        {
+            let mut f = followed();
+            // (a render that read the file meanwhile has noted its new stamp already)
+            match f.files.get_mut(&path) {
+                Some((s, _)) if *s == old => *s = now,
+                _ => continue,
+            }
+        }
+        for c in &caches {
+            c.forget_file(&path);
+        }
+        GENERATION.fetch_add(1, Ordering::AcqRel);
+    }
+    lut_generation()
+}
+
+#[cfg(test)]
+pub(crate) fn followed_stamp(path: &str) -> Option<FileStamp> {
+    followed().files.get(path).map(|(s, _)| *s)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    /// Past [`MAX_FOLLOWED_FILES`], the file just read is still followed (a list of its own: the
+    /// shared one's files belong to the other tests too).
+    #[test]
+    fn the_file_just_read_stays_followed_past_the_limit() {
+        let mut f = Followed::default();
+        let held: HashSet<String> = ["/followed/0.cube".to_string()].into();
+        for i in 0..MAX_FOLLOWED_FILES + 8 {
+            let files = vec![(format!("/followed/{i}.cube"), Some((1, None)))];
+            assert!(!f.read(&files));
+            f.trim(&held, &files);
+            assert!(f.files.contains_key(&files[0].0), "{i}: the file just read");
+            assert!(f.files.len() <= MAX_FOLLOWED_FILES);
+        }
+        assert!(f.files.contains_key("/followed/0.cube"), "a file a cached result was read from");
+        assert!(!f.files.contains_key("/followed/1.cube"), "the least recently read other");
+        assert!(f.read(&vec![("/followed/1030.cube".into(), Some((2, None)))]), "a moved stamp");
+    }
+
+    /// A nested cached load's files count for the enclosing load.
+    #[test]
+    fn an_enclosing_load_depends_on_a_nested_loads_files() {
+        static INNER: Registered<u32> = Registered::new(8, 1 << 20);
+        static OUTER: Registered<u32> = Registered::new(8, 1 << 20);
+        let v = cached(OUTER.get(), "outer", || {
+            note_missing("/nested/outer-own.cube");
+            let inner = cached(INNER.get(), "inner", || {
+                note_missing("/nested/inner.cube");
+                (7, 0)
+            });
+            (inner, 0)
+        });
+        assert_eq!(v, 7);
+        let files = lock(OUTER.get()).files("outer").unwrap();
+        assert_eq!(files, vec!["/nested/outer-own.cube".to_string(), "/nested/inner.cube".to_string()]);
+        // A hit on the inner result still reports its files to a new enclosing load.
+        let w = cached(OUTER.get(), "outer 2", || (cached(INNER.get(), "inner", || (0, 0)), 0));
+        assert_eq!(w, 7);
+        assert_eq!(lock(OUTER.get()).files("outer 2").unwrap(), vec!["/nested/inner.cube".to_string()]);
+    }
+}

crates/effects/src/lib.rs

@@ -39,6 +39,7 @@ mod keying2;
 pub mod keylight;
 mod matte;
 
+mod file_cache;
 pub mod migrate;
 mod misc;
 pub mod mocha_shape;
@@ -123,13 +124,14 @@ pub use channel3d::{
     crypto_layer_name, crypto_layer_named, extractor_aux_channel, extractor_own_channel, id_color, is_extractor_copy, matte_only_px, reads_aux,
     selection_hashes,
 };
+pub use file_cache::{check_lut_files, lut_generation};
 pub use ocio::{ColorOp, Straight, Tf, color_program};
 pub use sim::{Acc, Blob, Bubble, Post, Shape, Sprite, SpritePlan, Tint, bubble_list, bubble_sample_bounds, drizzle_drops, mercury_blobs, sprite_plan};
 pub use sim2::{
     CausticsSetup, FoamDisc, FoamEnv, FoamPlan, Piece, PiecePlan, PieceTex, PixelPlan, WavePlan, Waves, caustics_setup, foam_full_plan, piece_plan, pixel_plan,
     wave_world_plan,
 };
-pub use utility::{Lut, check_lut_files, load_lut, lut_file, lut_generation, lut_params};
+pub use utility::{Lut, load_lut, lut_file, lut_files, lut_params};
 pub use vr::{equi_dir as vr_equi_dir, rotation as vr_rotation};
 
 /// Effect categories in Effects & Presets order.

crates/effects/src/ocio.rs

@@ -24,8 +24,7 @@
 //! * Configuration ▸ Custom reads a `.ocio` file instead ([`crate::ocio_config`]); what it can't
 //!   apply is reported by [`warning`].
 
-use std::collections::HashMap;
-use std::sync::{Arc, Mutex, OnceLock};
+use std::sync::Arc;
 
 use effectcraft_color::ColorSpace;
 use effectcraft_color::space::{invert, mul, mul_vec, rgb_to_xyz};
@@ -51,6 +50,7 @@ pub use effects::parse_looks;
 pub use effects::{color_stabilizer_maps, warning};
 #[cfg(test)]
 use effects::{tonemap, tonemap_inv};
+pub(crate) use lut::cached_files;
 pub use lut::{FileXform, load_file};
 #[cfg(test)]
 pub use lut::{parse_file, parse_spi1d, parse_spi3d, parse_spimtx};

crates/effects/src/ocio/lut.rs

@@ -1,5 +1,6 @@
 use super::cdl::{Cdl, parse_cdl_xml};
-use super::{Arc, HashMap, Lut, Mutex, OnceLock, parse_cube};
+use super::{Arc, Lut, parse_cube};
+use crate::file_cache::{self, Registered};
 
 // ---------------------------------------------------------------- LUT files
 
@@ -264,33 +265,38 @@ pub fn parse_file(text: &str, ext: &str) -> Option<FileXform> {
     parse_3dl(text).map(|lut| FileXform::Lut { lut, shaper: None })
 }
 
-type FileCache = Mutex<HashMap<String, Option<Arc<FileXform>>>>;
+/// [`load_file`]'s cache, by parameter text (bounded like [`crate::load_lut`]'s).
+static FILES: Registered<Option<Arc<FileXform>>> = Registered::new(64, 512 << 20);
 
-/// Load (and cache) a transform from a file path or inline content.
+/// The files the transform cached for `src` was read from.
+pub(crate) fn cached_files(src: &str) -> Option<Vec<String>> {
+    file_cache::lock(FILES.get()).files(src)
+}
+
+/// Load (and cache) a transform from a file path or inline content. A file is stamped (one
+/// `metadata` call) on every load and read again once its size or modification time moved (see
+/// [`crate::file_cache`]); inline content is its own content.
 pub fn load_file(src: &str) -> Option<Arc<FileXform>> {
     let src_t = src.trim();
     if src_t.is_empty() {
         return None;
     }
-    static CACHE: OnceLock<FileCache> = OnceLock::new();
-    let cache = CACHE.get_or_init(|| Mutex::new(HashMap::new()));
-    if let Ok(m) = cache.lock()
-        && let Some(v) = m.get(src)
-    {
-        return v.clone();
-    }
-    let inline = src.contains('\n');
-    let parsed = if inline {
-        parse_file(src, "")
-    } else {
-        let ext = std::path::Path::new(src_t).extension().and_then(std::ffi::OsStr::to_str).unwrap_or("");
-        std::fs::read_to_string(src_t).ok().and_then(|t| parse_file(&t, ext))
-    }
-    .map(Arc::new);
-    if let Ok(mut m) = cache.lock() {
-        m.insert(src.to_string(), parsed.clone());
-    }
-    parsed
+    file_cache::cached(FILES.get(), src, || {
+        let parsed = if src.contains('\n') {
+            parse_file(src, "")
+        } else {
+            let ext = std::path::Path::new(src_t).extension().and_then(std::ffi::OsStr::to_str).unwrap_or("");
+            file_cache::read(src_t).ok().and_then(|t| parse_file(&t, ext))
+        }
+        .map(Arc::new);
+        let bytes = match parsed.as_deref() {
+            Some(FileXform::Lut { lut, shaper }) => crate::utility::lut_bytes(lut)
+                .saturating_add(shaper.as_ref().map_or(0, |s| s.iter().map(Vec::len).sum::<usize>().saturating_mul(std::mem::size_of::<(f32, f32)>()))),
+            Some(FileXform::Cdl(c)) => c.len().saturating_mul(std::mem::size_of::<(String, Cdl)>()),
+            Some(FileXform::Matrix { .. }) | None => 0,
+        };
+        (parsed, bytes)
+    })
 }
 
 impl FileXform {

crates/effects/src/ocio_config.rs

@@ -42,9 +42,10 @@
 //! resolving named transforms [`MAX_NAMED_DEPTH`] levels down or past [`MAX_OPS`] steps
 //! (a config that refers to itself in a loop), so a hostile config can't exhaust the stack.
 
-use std::collections::HashMap;
 use std::path::{Path, PathBuf};
-use std::sync::{Arc, Mutex, OnceLock};
+use std::sync::Arc;
+
+use crate::file_cache::{self, Registered};
 
 use super::ocio::{ACES_CG, ACES2065_1, Cdl, D65, FileXform, P3, REC709, REC2020, Space, Tf, load_file, matrix, sp};
 
@@ -882,13 +883,21 @@ fn resolve(src: &str, dir: &Path, search: &[String]) -> Option<PathBuf> {
     if src.trim().is_empty() {
         return None;
     }
+    // (a candidate not there is noted: the config is read again once it appears)
+    let found = |c: PathBuf| {
+        let there = c.exists();
+        if !there {
+            file_cache::note_missing(&c.to_string_lossy());
+        }
+        there.then_some(c)
+    };
     let p = Path::new(src);
     if p.is_absolute() {
-        return p.exists().then(|| p.to_path_buf());
+        return found(p.to_path_buf());
     }
     let mut dirs: Vec<PathBuf> = search.iter().map(|s| dir.join(s)).collect();
     dirs.push(dir.to_path_buf());
-    dirs.into_iter().map(|d| d.join(src)).find(|c| c.exists())
+    dirs.into_iter().map(|d| d.join(src)).find_map(found)
 }
 
 // ---------------------------------------------------------------- the config
@@ -1338,33 +1347,34 @@ impl<'c> Build<'c> {
     }
 }
 
-type ConfigCache = Mutex<HashMap<String, Result<Arc<Config>, String>>>;
+/// [`load_config`]'s cache, by parameter text.
+static CONFIGS: Registered<Result<Arc<Config>, String>> = Registered::new(16, 64 << 20);
+
+/// The files the config cached for `src` was read from: the `.ocio` file, the files its
+/// `FileTransform`s read and the places it looked for missing ones.
+pub(crate) fn cached_files(src: &str) -> Option<Vec<String>> {
+    file_cache::lock(CONFIGS.get()).files(src)
+}
 
 /// Load (and cache) a config from a `.ocio` path or the config text itself; why not, when it
-/// can't be read.
+/// can't be read. The config file and the files its `FileTransform`s read are stamped (one
+/// `metadata` call each) on every load, and the config is read again once one of them moved or a
+/// missing one appeared (see [`crate::file_cache`]).
 pub fn load_config(src: &str) -> Result<Arc<Config>, String> {
     let t = src.trim();
     if t.is_empty() {
         return Err("no config file chosen".into());
     }
-    static CACHE: OnceLock<ConfigCache> = OnceLock::new();
-    let cache = CACHE.get_or_init(|| Mutex::new(HashMap::new()));
-    if let Ok(m) = cache.lock()
-        && let Some(v) = m.get(src)
-    {
-        return v.clone();
-    }
-    let parsed = if src.contains('\n') {
-        Config::parse(src, Path::new("."))
-    } else {
+    file_cache::cached(CONFIGS.get(), src, || {
+        if src.contains('\n') {
+            return (Config::parse(src, Path::new(".")).map(Arc::new), 0);
+        }
         let p = Path::new(t);
-        std::fs::read_to_string(p).map_err(|e| format!("{t}: {e}")).and_then(|text| Config::parse(&text, p.parent().unwrap_or(Path::new("."))))
-    }
-    .map(Arc::new);
-    if let Ok(mut m) = cache.lock() {
-        m.insert(src.to_string(), parsed.clone());
-    }
-    parsed
+        match file_cache::read(t) {
+            Ok(text) => (Config::parse(&text, p.parent().unwrap_or(Path::new("."))).map(Arc::new), text.len()),
+            Err(e) => (Err(format!("{t}: {e}")), 0),
+        }
+    })
 }
 
 #[cfg(test)]

crates/effects/src/ocio_config/tests.rs

@@ -452,3 +452,26 @@ fn an_ocio_config_and_its_luts_are_read_again_once_they_change() {
     assert!(load_config(&src).is_err(), "a deleted config");
     std::fs::remove_dir_all(&dir).unwrap();
 }
+
+/// A LUT a cached config read that changes is noticed without a render loading the config (the
+/// interface's watcher), and the config's files name it (frame and layer cache keys hold them).
+#[test]
+fn check_lut_files_notices_a_configs_lut() {
+    let dir = std::env::temp_dir().join(format!("ec-fx-3538-check-{}", std::process::id()));
+    let _ = std::fs::remove_dir_all(&dir);
+    std::fs::create_dir_all(&dir).unwrap();
+    let lut = dir.join("watched.cube");
+    std::fs::write(&lut, "LUT_1D_SIZE 2\n0 0 0\n1 1 1\n").unwrap();
+    let config = dir.join("config.ocio");
+    std::fs::write(&config, "ocio_profile_version: 2\ncolorspaces:\n  - !<ColorSpace>\n    name: lin\n  - !<ColorSpace>\n    name: graded\n    to_reference: !<FileTransform> {src: watched.cube}\n").unwrap();
+    let src = config.to_string_lossy().into_owned();
+    assert!(load_config(&src).is_ok());
+    let files = crate::lut_files(&src);
+    assert!(files.contains(&lut.to_string_lossy().into_owned()), "{files:?}");
+    let before = crate::check_lut_files();
+    assert!(cached_files(&src).is_some(), "an unchanged config stays");
+    std::fs::write(&lut, "LUT_1D_SIZE 3\n0 0 0\n0.5 0.5 0.5\n1 1 1\n").unwrap();
+    assert!(crate::check_lut_files() > before);
+    assert!(cached_files(&src).is_none(), "a config whose LUT changed is read again");
+    std::fs::remove_dir_all(&dir).unwrap();
+}

crates/effects/src/utility.rs

@@ -1,10 +1,9 @@
 //! Utility effects: bounds growth, HDR inspection/compression, Cineon log conversion and colour
 //! lookup tables (.cube).
 
-use std::collections::HashMap;
-use std::sync::atomic::{AtomicU64, Ordering};
-use std::sync::{Arc, Mutex, OnceLock, PoisonError};
+use std::sync::Arc;
 
+use crate::file_cache::{self, Registered};
 use effectcraft_color::luminance;
 use effectcraft_keyframe::Value;
 use effectcraft_project::ParamUi;
@@ -331,84 +330,22 @@ impl Lut {
     }
 }
 
-/// A LUT file's size and modification time when it was read (`None`: inline text, or a file
-/// that could not be described). A file whose stamp moved is read again (#3493).
-type FileStamp = Option<(u64, Option<std::time::SystemTime>)>;
-
 /// At most this many parsed LUTs (files and inline texts) are kept, and at most
 /// [`MAX_CACHED_BYTES`] of them; the least recently used go first.
 const MAX_CACHED_LUTS: usize = 64;
 const MAX_CACHED_BYTES: usize = 512 << 20;
 
-/// At most this many LUT files are followed by [`check_lut_files`] beyond the ones whose LUT is
-/// kept: the frames made with a LUT that left the cache are still dropped when its file changes.
-const MAX_FOLLOWED_FILES: usize = 1024;
+/// [`load_lut`]'s cache, by parameter text.
+static LUTS: Registered<Option<Arc<Lut>>> = Registered::new(MAX_CACHED_LUTS, MAX_CACHED_BYTES);
 
-/// A parsed LUT (or the failure to parse one) and what it was read from.
-struct CachedLut {
-    stamp: FileStamp,
-    lut: Option<Arc<Lut>>,
-    used: u64,
-    /// The parsed tables and the parameter text, counted against [`MAX_CACHED_BYTES`].
-    bytes: usize,
-}
-
-/// [`load_lut`]'s cache, by parameter text, and the stamp each LUT file was last read with.
-#[derive(Default)]
-struct LutCache {
-    map: HashMap<String, CachedLut>,
-    files: HashMap<String, FileStamp>,
-    clock: u64,
-    bytes: usize,
-}
-
-impl LutCache {
-    /// Forget the LUT kept for `src`.
-    fn remove(&mut self, src: &str) {
-        if let Some(e) = self.map.remove(src) {
-            self.bytes = self.bytes.saturating_sub(e.bytes);
-        }
-    }
-
-    /// Note that the file at `path` was read with `stamp`: a different stamp than the last read's
-    /// means it changed on disk, and frames made before may show the old LUT.
-    fn read_file(&mut self, path: &str, stamp: FileStamp) {
-        if self.files.insert(path.to_string(), stamp).is_some_and(|old| old != stamp) {
-            GENERATION.fetch_add(1, Ordering::AcqRel);
-        }
-        if self.files.len() > MAX_FOLLOWED_FILES {
-            let map = &self.map;
-            self.files.retain(|p, _| p.as_str() == path || map.keys().any(|src| lut_file(src) == Some(p.as_str())));
-        }
-    }
-
-    /// Keep at most [`MAX_CACHED_LUTS`] LUTs and [`MAX_CACHED_BYTES`], always the one for `keep`.
-    fn evict(&mut self, keep: &str) {
-        while self.map.len() > MAX_CACHED_LUTS || (self.bytes > MAX_CACHED_BYTES && self.map.len() > 1) {
-            let Some(oldest) = self.map.iter().filter(|(k, _)| k.as_str() != keep).min_by_key(|(_, e)| e.used).map(|(k, _)| k.clone()) else { break };
-            self.remove(&oldest);
-        }
-    }
-}
-
-/// How often a LUT file was found changed on disk ([`lut_generation`]).
-static GENERATION: AtomicU64 = AtomicU64::new(0);
-
-fn lut_cache() -> std::sync::MutexGuard<'static, LutCache> {
-    static CACHE: OnceLock<Mutex<LutCache>> = OnceLock::new();
-    CACHE.get_or_init(Mutex::default).lock().unwrap_or_else(PoisonError::into_inner)
-}
-
-/// The stamp of the file at `path` now: one `metadata` call.
-fn file_stamp(path: &str) -> FileStamp {
-    std::fs::metadata(path).ok().map(|md| (md.len(), md.modified().ok()))
-}
-
-/// The parameters (paths in the effect's group) of `effect` that [`load_lut`] reads.
+/// The parameters (paths in the effect's group) of `effect` that name a file an effect reads
+/// through a cache that follows it ([`load_lut`], the OCIO file transforms and configs).
 pub fn lut_params(effect: &str) -> &'static [&'static str] {
     match effect {
         "ec.utility.applylut" => &["lut"],
         "ec.color.lumetri" => &["basicCorrection/inputLutFile", "creative/lookFile"],
+        "ec.color.ociofile" => &["file"],
+        "ec.color.ociocolorspace" | "ec.color.ociodisplay" => &["configFile"],
         _ => &[],
     }
 }
@@ -421,75 +358,37 @@ pub fn lut_file(src: &str) -> Option<&str> {
     (!path.is_empty() && !path.contains('\n')).then_some(path)
 }
 
-/// Changes each time a LUT file [`load_lut`] read is found changed on disk (by [`load_lut`]
-/// itself or [`check_lut_files`]): frames made before then may show the old LUT.
-pub fn lut_generation() -> u64 {
-    GENERATION.load(Ordering::Acquire)
-}
-
-/// Look at every LUT file read (one `metadata` call each) and forget the LUTs of the ones that
-/// changed on disk since, so the next [`load_lut`] reads them again. Returns [`lut_generation`].
-pub fn check_lut_files() -> u64 {
-    let files: Vec<(String, FileStamp)> = lut_cache().files.iter().map(|(p, s)| (p.clone(), *s)).collect();
-    // (stamped without the lock: a file on a slow share must not hold up renders)
-    let changed: Vec<(String, FileStamp, FileStamp)> =
-        files.into_iter().filter_map(|(path, old)| Some(file_stamp(&path)).filter(|now| *now != old).map(|now| (path, old, now))).collect();
-    if !changed.is_empty() {
-        let mut c = lut_cache();
-        for (path, old, now) in changed {
-            // (a render that read the file meanwhile has noted its new stamp already)
-            if c.files.get(&path) == Some(&old) {
-                let stale: Vec<String> = c.map.keys().filter(|src| lut_file(src) == Some(path.as_str())).cloned().collect();
-                for src in stale {
-                    c.remove(&src);
-                }
-                c.read_file(&path, now);
-            }
-        }
-    }
-    lut_generation()
+/// The files a [`lut_params`] parameter set to `src` reads: the ones its cached result was read
+/// from (an OCIO config's also the LUTs it names, and the places it looked for missing ones), else
+/// the file it names.
+pub fn lut_files(src: &str) -> Vec<String> {
+    crate::ocio_config::cached_files(src)
+        .or_else(|| crate::ocio::cached_files(src))
+        .or_else(|| file_cache::lock(LUTS.get()).files(src))
+        .unwrap_or_else(|| lut_file(src).map(str::to_string).into_iter().collect())
 }
 
 /// Load (and cache) a LUT from inline `.cube` text or a file path. A file is stamped (one
-/// `metadata` call) on every load and read again once its size or modification time moved, so a
-/// file edited, created, deleted or repaired on disk is followed; inline text is its own content.
-/// An edit that keeps both the size and the modification time can't be told apart.
+/// `metadata` call) on every load and read again once its size or modification time moved (see
+/// [`crate::file_cache`]); inline text is its own content.
 pub fn load_lut(src: &str) -> Option<Arc<Lut>> {
     if src.trim().is_empty() {
         return None;
     }
-    let file = lut_file(src);
-    let stamp = file.and_then(file_stamp);
-    {
-        let mut c = lut_cache();
-        c.clock += 1;
-        let now = c.clock;
-        if let Some(e) = c.map.get_mut(src)
-            && e.stamp == stamp
-        {
-            e.used = now;
-            return e.lut.clone();
+    file_cache::cached(LUTS.get(), src, || {
+        let lut = match lut_file(src) {
+            Some(path) => file_cache::read(path).ok().and_then(|t| parse_cube(&t)),
+            None => parse_cube(src),
         }
-    }
-    // (read and parsed without the lock; a file written after it was stamped has a newer stamp,
-    // and is read again next time)
-    let lut = match file {
-        Some(path) => std::fs::read_to_string(path).ok().and_then(|t| parse_cube(&t)),
-        None => parse_cube(src),
-    }
-    .map(Arc::new);
-    let tables = lut.as_ref().map_or(0, |l| l.d1.len().saturating_add(l.d3.len()).saturating_mul(std::mem::size_of::<[f32; 3]>()));
-    let bytes = tables.saturating_add(src.len());
-    let mut c = lut_cache();
-    if let Some(path) = file {
-        c.read_file(path, stamp);
-    }
-    c.remove(src);
-    let used = c.clock;
-    c.map.insert(src.to_string(), CachedLut { stamp, lut: lut.clone(), used, bytes });
-    c.bytes = c.bytes.saturating_add(bytes);
-    c.evict(src);
-    lut
+        .map(Arc::new);
+        let bytes = lut.as_ref().map_or(0, |l| lut_bytes(l));
+        (lut, bytes)
+    })
+}
+
+/// The size of a LUT's tables.
+pub(crate) fn lut_bytes(l: &Lut) -> usize {
+    l.d1.len().saturating_add(l.d3.len()).saturating_mul(std::mem::size_of::<[f32; 3]>())
 }
 
 fn apply_lut(ctx: &EffectCtx, mut b: Buf) -> Buf {
@@ -685,11 +584,11 @@ mod tests {
         std::fs::write(&path, identity_cube(2)).unwrap();
         let src = path.to_string_lossy().into_owned();
         assert!(load_lut(&src).is_some());
-        let before = check_lut_files();
-        assert!(lut_cache().map.contains_key(&src), "an unchanged file stays");
+        let before = crate::check_lut_files();
+        assert!(file_cache::lock(LUTS.get()).contains(&src), "an unchanged file stays");
         std::fs::write(&path, identity_cube(3)).unwrap();
-        assert!(check_lut_files() > before);
-        assert!(!lut_cache().map.contains_key(&src), "a changed file is read again");
+        assert!(crate::check_lut_files() > before);
+        assert!(!file_cache::lock(LUTS.get()).contains(&src), "a changed file is read again");
         assert_eq!(load_lut(&src).map(|l| l.n3), Some(3));
         std::fs::remove_dir_all(&dir).unwrap();
     }
@@ -703,12 +602,12 @@ mod tests {
         for i in 0..MAX_CACHED_LUTS + 8 {
             let text = format!("TITLE \"bounded {i}\"\n{}", identity_cube(2));
             assert!(load_lut(&text).is_some());
-            assert!(lut_cache().map.len() <= MAX_CACHED_LUTS);
+            assert!(file_cache::lock(LUTS.get()).len() <= MAX_CACHED_LUTS);
         }
         assert!(!Arc::ptr_eq(&first, &load_lut(&oldest).unwrap()), "the oldest was let go");
-        let c = lut_cache();
-        assert_eq!(c.bytes, c.map.values().map(|e| e.bytes).sum::<usize>(), "the bytes kept add up");
-        assert!(c.bytes <= MAX_CACHED_BYTES);
+        let (bytes, sum, max) = file_cache::lock(LUTS.get()).bytes();
+        assert_eq!(bytes, sum, "the bytes kept add up");
+        assert!(bytes <= max);
     }
 
     /// A LUT file whose LUT left the cache is still followed: its frames are dropped when it
@@ -725,26 +624,13 @@ mod tests {
         for i in 0..MAX_CACHED_LUTS + 8 {
             assert!(load_lut(&format!("TITLE \"evict {i}\"\n{}", identity_cube(2))).is_some());
         }
-        assert!(!lut_cache().map.contains_key(&src), "its LUT left the cache");
+        assert!(!file_cache::lock(LUTS.get()).contains(&src), "its LUT left the cache");
         std::fs::write(&path, identity_cube(3)).unwrap();
-        check_lut_files();
-        assert_eq!(lut_cache().files.get(&src), Some(&file_stamp(&src)), "the change was noticed");
+        crate::check_lut_files();
+        assert_eq!(file_cache::followed_stamp(&src), Some(file_cache::file_stamp(&src)), "the change was noticed");
         std::fs::remove_dir_all(&dir).unwrap();
     }
 
-    /// Past [`MAX_FOLLOWED_FILES`], the file just read is still followed (a cache of its own:
-    /// the shared one's followed files belong to the other tests too).
-    #[test]
-    fn the_file_just_read_stays_followed_past_the_limit() {
-        let mut c = LutCache::default();
-        for i in 0..MAX_FOLLOWED_FILES + 8 {
-            let path = format!("/followed/{i}.cube");
-            c.read_file(&path, Some((1, None)));
-            assert!(c.files.contains_key(&path), "{i}: the file just read");
-        }
-        assert!(c.files.len() <= MAX_FOLLOWED_FILES, "the others no longer cached are let go");
-    }
-
     #[test]
     fn one_d_lut_with_domain() {
         let lut = parse_cube("LUT_1D_SIZE 2\nDOMAIN_MIN 0 0 0\nDOMAIN_MAX 2 2 2\n1 1 1\n0 0 0\n").unwrap();