media: add a pooled FileReader ByteSource

FileReader reads a file in place and keeps at most MAX_OPEN_FILES
descriptors open in a shared LRU pool, reopening by path when a file
was closed. A project with hundreds of clips can stay under the process
file-descriptor limit. A file truncated since it was opened is refused.

Part of #778 and effectcraft#781.
This commit is contained in:
FredApps committed 2026-11-16 14:52:55 +01:00
1 parent e836b6fd89
commit 16bf5f8578
2 files changed
+212

No files matched your search

+206
View File
@@ -0,0 +1,206 @@
//! Pooled file [`ByteSource`]s: many clips can be open at once without holding a descriptor each.
//!
//! Past [`MAX_OPEN_FILES`] the least recently read file is closed and reopened by path on demand,
//! so a project with hundreds of sources does not hit the process file-descriptor limit (often 256
//! on macOS, 1024 on Linux).
#![cfg(not(target_arch = "wasm32"))]
use std::io;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use crate::source::ByteSource;
fn eof() -> io::Error {
io::Error::new(io::ErrorKind::UnexpectedEof, "read past end of byte source")
}
/// Files [`FileReader`]s keep open at most, together. Past this many, the least recently read file
/// is closed and its reader reopens it on demand.
pub const MAX_OPEN_FILES: usize = 128;
/// The open files of a set of [`FileReader`]s, least recently used first.
#[derive(Debug)]
pub(crate) struct FilePool {
cap: usize,
state: std::sync::Mutex<PoolState>,
}
#[derive(Debug, Default)]
struct PoolState {
next_id: u64,
open: std::collections::VecDeque<(u64, Arc<std::fs::File>)>,
}
impl FilePool {
pub(crate) fn new(cap: usize) -> Self {
Self { cap: cap.max(1), state: Default::default() }
}
fn state(&self) -> std::sync::MutexGuard<'_, PoolState> {
self.state.lock().unwrap_or_else(|e| e.into_inner())
}
fn next_id(&self) -> u64 {
let mut s = self.state();
s.next_id = s.next_id.wrapping_add(1);
s.next_id
}
/// The open file of reader `id`, now the most recently used.
fn get(&self, id: u64) -> Option<Arc<std::fs::File>> {
let mut s = self.state();
let at = s.open.iter().position(|(i, _)| *i == id)?;
let entry = s.open.remove(at)?;
let file = entry.1.clone();
s.open.push_back(entry);
Some(file)
}
/// Keep `file` open for reader `id`, closing the least recently used files past the cap. A
/// file another thread is reading from stays open until that read ends.
fn insert(&self, id: u64, file: std::fs::File) -> Arc<std::fs::File> {
let mut s = self.state();
if let Some((_, open)) = s.open.iter().find(|(i, _)| *i == id) {
return open.clone(); // another thread reopened it first
}
let file = Arc::new(file);
s.open.push_back((id, file.clone()));
while s.open.len() > self.cap {
s.open.pop_front();
}
file
}
fn remove(&self, id: u64) {
self.state().open.retain(|(i, _)| *i != id);
}
#[cfg(test)]
fn open_count(&self) -> usize {
self.state().open.len()
}
}
static FILE_POOL: std::sync::LazyLock<Arc<FilePool>> = std::sync::LazyLock::new(|| Arc::new(FilePool::new(MAX_OPEN_FILES)));
/// A file on disk, read in place: only the requested ranges are read, so opening a clip costs its
/// index rather than its size. The file is kept open in a shared pool of at most
/// [`MAX_OPEN_FILES`] and reopened by path when the pool closed it.
#[derive(Debug)]
pub struct FileReader {
path: PathBuf,
len: u64,
id: u64,
pool: Arc<FilePool>,
}
impl FileReader {
/// Open `path` in the process-wide pool.
pub fn open(path: &Path) -> io::Result<Self> {
Self::open_in(path, &FILE_POOL)
}
pub(crate) fn open_in(path: &Path, pool: &Arc<FilePool>) -> io::Result<Self> {
let file = std::fs::File::open(path)?;
let m = file.metadata()?;
if !m.is_file() {
return Err(io::Error::new(io::ErrorKind::NotFound, format!("{} is not a file", path.display())));
}
let id = pool.next_id();
pool.insert(id, file);
Ok(Self { path: path.to_path_buf(), len: m.len(), id, pool: pool.clone() })
}
/// The open file, reopened when the pool closed it. A file now shorter than when it was opened
/// (replaced or truncated) is refused rather than read as the old one; a longer one (still
/// being written) is read up to the opened length.
fn file(&self) -> io::Result<Arc<std::fs::File>> {
if let Some(file) = self.pool.get(self.id) {
return Ok(file);
}
let file = std::fs::File::open(&self.path)?;
let m = file.metadata()?;
if !m.is_file() || m.len() < self.len {
return Err(io::Error::other(format!("{} changed since it was opened", self.path.display())));
}
Ok(self.pool.insert(self.id, file))
}
}
impl Drop for FileReader {
fn drop(&mut self) {
self.pool.remove(self.id);
}
}
impl ByteSource for FileReader {
fn len(&self) -> u64 {
self.len
}
/// Positional reads: threads decoding different parts of the file do not share a cursor.
fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> {
if offset.checked_add(buf.len() as u64).is_none_or(|e| e > self.len) {
return Err(eof());
}
let file = self.file()?;
file.read_at(offset, buf)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn file_reader_reads_ranges_in_place() {
let v: Vec<u8> = (0..=255).collect();
let dir = craft_media_testkit::private_dir("file-reader");
let path = dir.join("bytes.bin");
std::fs::write(&path, &v).unwrap();
let r = FileReader::open(&path).unwrap();
assert_eq!(ByteSource::len(&r), 256);
let mut b = [0u8; 3];
r.read_at(4, &mut b).unwrap();
assert_eq!(&b, &v[4..7]);
// reads are positional: an earlier offset after a later one
r.read_at(0, &mut b).unwrap();
assert_eq!(&b, &v[0..3]);
assert_eq!(r.read_at(254, &mut [0u8; 3]).unwrap_err().kind(), io::ErrorKind::UnexpectedEof);
assert!(FileReader::open(&dir).is_err(), "a directory is not a media file");
assert_eq!(FileReader::open(&dir.join("missing")).unwrap_err().kind(), io::ErrorKind::NotFound);
drop(r);
std::fs::remove_dir_all(&dir).ok();
}
/// More readers than the pool keeps open stay readable; past the cap the least recently read
/// file is closed and reopened on demand, and a file truncated meanwhile is refused.
#[test]
fn more_file_readers_than_the_pool_holds_stay_readable() {
let dir = craft_media_testkit::private_dir("file-pool");
let pool = Arc::new(FilePool::new(2));
let paths: Vec<_> = (0..5).map(|i| dir.join(format!("{i}.bin"))).collect();
for (i, p) in paths.iter().enumerate() {
std::fs::write(p, format!("file{i}-0123456789")).unwrap();
}
let readers: Vec<_> = paths.iter().map(|p| FileReader::open_in(p, &pool).unwrap()).collect();
assert_eq!(pool.open_count(), 2);
for _ in 0..2 {
for (i, r) in readers.iter().enumerate() {
let mut b = [0u8; 5];
r.read_at(0, &mut b).unwrap();
assert_eq!(b, *format!("file{i}").as_bytes());
assert!(pool.open_count() <= 2);
}
}
// reader 0 was closed by the later reads; its file is now shorter than when opened
std::fs::write(&paths[0], b"short").unwrap();
let mut b = [0u8; 5];
assert!(readers[0].read_at(0, &mut b).is_err());
drop(readers);
assert_eq!(pool.open_count(), 0, "dropped readers close their files");
std::fs::remove_dir_all(&dir).ok();
}
}
+6
View File
@@ -14,6 +14,8 @@
//! program streams).
//! - [`Container`] and [`CodecId`] (no feature): the identifiers the codec crates report a file's
//! container and codec with.
//! - `FileReader` (native): a pooled file `ByteSource` so many clips can be open without holding a
//! descriptor each.
//!
//! `bitstream` and `isobmff` are on by default; `matroska`, `ogg`, `avi`, `mxf` and `mpegts` are opt-in. A crate that needs
//! only bit I/O uses `default-features = false, features = ["bitstream"]`.
@@ -22,6 +24,8 @@
pub mod avi;
#[cfg(feature = "bitstream")]
pub mod bitstream;
#[cfg(not(target_arch = "wasm32"))]
mod file;
mod id;
#[cfg(feature = "isobmff")]
pub mod isobmff;
@@ -34,6 +38,8 @@ pub mod mxf;
#[cfg(feature = "ogg")]
pub mod ogg;
pub mod source;
#[cfg(not(target_arch = "wasm32"))]
pub use file::{FileReader, MAX_OPEN_FILES};
pub use id::{CodecId, CodecParams, Container, PcmFormat, xiph_lace};
pub use source::{ByteSource, ReadSeekSource};