mirror of
https://github.com/storytold/craft-libs.git
synced 2026-11-18 00:05:08 +00:00
media: add a pooled FileReader ByteSource
FileReader reads a file in place and keeps at most MAX_OPEN_FILES descriptors open in a shared LRU pool, reopening by path when a file was closed. A project with hundreds of clips can stay under the process file-descriptor limit. A file truncated since it was opened is refused. Part of #778 and effectcraft#781.
This commit is contained in:
1 parent
e836b6fd89
commit
16bf5f8578
2 files changed
+212
No files matched your search
@@ -0,0 +1,206 @@
|
||||
//! Pooled file [`ByteSource`]s: many clips can be open at once without holding a descriptor each.
|
||||
//!
|
||||
//! Past [`MAX_OPEN_FILES`] the least recently read file is closed and reopened by path on demand,
|
||||
//! so a project with hundreds of sources does not hit the process file-descriptor limit (often 256
|
||||
//! on macOS, 1024 on Linux).
|
||||
|
||||
#![cfg(not(target_arch = "wasm32"))]
|
||||
|
||||
use std::io;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::source::ByteSource;
|
||||
|
||||
fn eof() -> io::Error {
|
||||
io::Error::new(io::ErrorKind::UnexpectedEof, "read past end of byte source")
|
||||
}
|
||||
|
||||
/// Files [`FileReader`]s keep open at most, together. Past this many, the least recently read file
|
||||
/// is closed and its reader reopens it on demand.
|
||||
pub const MAX_OPEN_FILES: usize = 128;
|
||||
|
||||
/// The open files of a set of [`FileReader`]s, least recently used first.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct FilePool {
|
||||
cap: usize,
|
||||
state: std::sync::Mutex<PoolState>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct PoolState {
|
||||
next_id: u64,
|
||||
open: std::collections::VecDeque<(u64, Arc<std::fs::File>)>,
|
||||
}
|
||||
|
||||
impl FilePool {
|
||||
pub(crate) fn new(cap: usize) -> Self {
|
||||
Self { cap: cap.max(1), state: Default::default() }
|
||||
}
|
||||
|
||||
fn state(&self) -> std::sync::MutexGuard<'_, PoolState> {
|
||||
self.state.lock().unwrap_or_else(|e| e.into_inner())
|
||||
}
|
||||
|
||||
fn next_id(&self) -> u64 {
|
||||
let mut s = self.state();
|
||||
s.next_id = s.next_id.wrapping_add(1);
|
||||
s.next_id
|
||||
}
|
||||
|
||||
/// The open file of reader `id`, now the most recently used.
|
||||
fn get(&self, id: u64) -> Option<Arc<std::fs::File>> {
|
||||
let mut s = self.state();
|
||||
let at = s.open.iter().position(|(i, _)| *i == id)?;
|
||||
let entry = s.open.remove(at)?;
|
||||
let file = entry.1.clone();
|
||||
s.open.push_back(entry);
|
||||
Some(file)
|
||||
}
|
||||
|
||||
/// Keep `file` open for reader `id`, closing the least recently used files past the cap. A
|
||||
/// file another thread is reading from stays open until that read ends.
|
||||
fn insert(&self, id: u64, file: std::fs::File) -> Arc<std::fs::File> {
|
||||
let mut s = self.state();
|
||||
if let Some((_, open)) = s.open.iter().find(|(i, _)| *i == id) {
|
||||
return open.clone(); // another thread reopened it first
|
||||
}
|
||||
let file = Arc::new(file);
|
||||
s.open.push_back((id, file.clone()));
|
||||
while s.open.len() > self.cap {
|
||||
s.open.pop_front();
|
||||
}
|
||||
file
|
||||
}
|
||||
|
||||
fn remove(&self, id: u64) {
|
||||
self.state().open.retain(|(i, _)| *i != id);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn open_count(&self) -> usize {
|
||||
self.state().open.len()
|
||||
}
|
||||
}
|
||||
|
||||
static FILE_POOL: std::sync::LazyLock<Arc<FilePool>> = std::sync::LazyLock::new(|| Arc::new(FilePool::new(MAX_OPEN_FILES)));
|
||||
|
||||
/// A file on disk, read in place: only the requested ranges are read, so opening a clip costs its
|
||||
/// index rather than its size. The file is kept open in a shared pool of at most
|
||||
/// [`MAX_OPEN_FILES`] and reopened by path when the pool closed it.
|
||||
#[derive(Debug)]
|
||||
pub struct FileReader {
|
||||
path: PathBuf,
|
||||
len: u64,
|
||||
id: u64,
|
||||
pool: Arc<FilePool>,
|
||||
}
|
||||
|
||||
impl FileReader {
|
||||
/// Open `path` in the process-wide pool.
|
||||
pub fn open(path: &Path) -> io::Result<Self> {
|
||||
Self::open_in(path, &FILE_POOL)
|
||||
}
|
||||
|
||||
pub(crate) fn open_in(path: &Path, pool: &Arc<FilePool>) -> io::Result<Self> {
|
||||
let file = std::fs::File::open(path)?;
|
||||
let m = file.metadata()?;
|
||||
if !m.is_file() {
|
||||
return Err(io::Error::new(io::ErrorKind::NotFound, format!("{} is not a file", path.display())));
|
||||
}
|
||||
let id = pool.next_id();
|
||||
pool.insert(id, file);
|
||||
Ok(Self { path: path.to_path_buf(), len: m.len(), id, pool: pool.clone() })
|
||||
}
|
||||
|
||||
/// The open file, reopened when the pool closed it. A file now shorter than when it was opened
|
||||
/// (replaced or truncated) is refused rather than read as the old one; a longer one (still
|
||||
/// being written) is read up to the opened length.
|
||||
fn file(&self) -> io::Result<Arc<std::fs::File>> {
|
||||
if let Some(file) = self.pool.get(self.id) {
|
||||
return Ok(file);
|
||||
}
|
||||
let file = std::fs::File::open(&self.path)?;
|
||||
let m = file.metadata()?;
|
||||
if !m.is_file() || m.len() < self.len {
|
||||
return Err(io::Error::other(format!("{} changed since it was opened", self.path.display())));
|
||||
}
|
||||
Ok(self.pool.insert(self.id, file))
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for FileReader {
|
||||
fn drop(&mut self) {
|
||||
self.pool.remove(self.id);
|
||||
}
|
||||
}
|
||||
|
||||
impl ByteSource for FileReader {
|
||||
fn len(&self) -> u64 {
|
||||
self.len
|
||||
}
|
||||
|
||||
/// Positional reads: threads decoding different parts of the file do not share a cursor.
|
||||
fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> {
|
||||
if offset.checked_add(buf.len() as u64).is_none_or(|e| e > self.len) {
|
||||
return Err(eof());
|
||||
}
|
||||
let file = self.file()?;
|
||||
file.read_at(offset, buf)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn file_reader_reads_ranges_in_place() {
|
||||
let v: Vec<u8> = (0..=255).collect();
|
||||
let dir = craft_media_testkit::private_dir("file-reader");
|
||||
let path = dir.join("bytes.bin");
|
||||
std::fs::write(&path, &v).unwrap();
|
||||
let r = FileReader::open(&path).unwrap();
|
||||
assert_eq!(ByteSource::len(&r), 256);
|
||||
let mut b = [0u8; 3];
|
||||
r.read_at(4, &mut b).unwrap();
|
||||
assert_eq!(&b, &v[4..7]);
|
||||
// reads are positional: an earlier offset after a later one
|
||||
r.read_at(0, &mut b).unwrap();
|
||||
assert_eq!(&b, &v[0..3]);
|
||||
assert_eq!(r.read_at(254, &mut [0u8; 3]).unwrap_err().kind(), io::ErrorKind::UnexpectedEof);
|
||||
assert!(FileReader::open(&dir).is_err(), "a directory is not a media file");
|
||||
assert_eq!(FileReader::open(&dir.join("missing")).unwrap_err().kind(), io::ErrorKind::NotFound);
|
||||
drop(r);
|
||||
std::fs::remove_dir_all(&dir).ok();
|
||||
}
|
||||
|
||||
/// More readers than the pool keeps open stay readable; past the cap the least recently read
|
||||
/// file is closed and reopened on demand, and a file truncated meanwhile is refused.
|
||||
#[test]
|
||||
fn more_file_readers_than_the_pool_holds_stay_readable() {
|
||||
let dir = craft_media_testkit::private_dir("file-pool");
|
||||
let pool = Arc::new(FilePool::new(2));
|
||||
let paths: Vec<_> = (0..5).map(|i| dir.join(format!("{i}.bin"))).collect();
|
||||
for (i, p) in paths.iter().enumerate() {
|
||||
std::fs::write(p, format!("file{i}-0123456789")).unwrap();
|
||||
}
|
||||
let readers: Vec<_> = paths.iter().map(|p| FileReader::open_in(p, &pool).unwrap()).collect();
|
||||
assert_eq!(pool.open_count(), 2);
|
||||
for _ in 0..2 {
|
||||
for (i, r) in readers.iter().enumerate() {
|
||||
let mut b = [0u8; 5];
|
||||
r.read_at(0, &mut b).unwrap();
|
||||
assert_eq!(b, *format!("file{i}").as_bytes());
|
||||
assert!(pool.open_count() <= 2);
|
||||
}
|
||||
}
|
||||
// reader 0 was closed by the later reads; its file is now shorter than when opened
|
||||
std::fs::write(&paths[0], b"short").unwrap();
|
||||
let mut b = [0u8; 5];
|
||||
assert!(readers[0].read_at(0, &mut b).is_err());
|
||||
drop(readers);
|
||||
assert_eq!(pool.open_count(), 0, "dropped readers close their files");
|
||||
std::fs::remove_dir_all(&dir).ok();
|
||||
}
|
||||
}
|
||||
@@ -14,6 +14,8 @@
|
||||
//! program streams).
|
||||
//! - [`Container`] and [`CodecId`] (no feature): the identifiers the codec crates report a file's
|
||||
//! container and codec with.
|
||||
//! - `FileReader` (native): a pooled file `ByteSource` so many clips can be open without holding a
|
||||
//! descriptor each.
|
||||
//!
|
||||
//! `bitstream` and `isobmff` are on by default; `matroska`, `ogg`, `avi`, `mxf` and `mpegts` are opt-in. A crate that needs
|
||||
//! only bit I/O uses `default-features = false, features = ["bitstream"]`.
|
||||
@@ -22,6 +24,8 @@
|
||||
pub mod avi;
|
||||
#[cfg(feature = "bitstream")]
|
||||
pub mod bitstream;
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
mod file;
|
||||
mod id;
|
||||
#[cfg(feature = "isobmff")]
|
||||
pub mod isobmff;
|
||||
@@ -34,6 +38,8 @@ pub mod mxf;
|
||||
#[cfg(feature = "ogg")]
|
||||
pub mod ogg;
|
||||
pub mod source;
|
||||
#[cfg(not(target_arch = "wasm32"))]
|
||||
pub use file::{FileReader, MAX_OPEN_FILES};
|
||||
|
||||
pub use id::{CodecId, CodecParams, Container, PcmFormat, xiph_lace};
|
||||
pub use source::{ByteSource, ReadSeekSource};
|
||||
Reference in new issue
Block a user