Don't crash on MP4 files that claim more data than they hold

An M4A whose handler atom claimed a 64-bit size of 2^60 bytes ended the
process on import: symphonia reads `hdlr` and the `data`, `name` and
`mean` atoms of iTunes metadata into buffers of their declared size, and
an allocation that fails aborts. Its sample tables reserve room for
whatever entry count they claim (up to 68 GB from a 19 KB file), and
every sample is read into a buffer of the size `stsz` gives it. macOS
hands out most of that lazily; on Linux, Windows and the 32-bit Windows
build such a request fails and aborts.

When the probe finds an MP4 file, the atoms symphonia descends into are
now walked first, the way symphonia reads them: a header that starts
inside its container is read whole, and the walk ends where symphonia
stops, after an atom that runs past its container. A file is refused as
malformed when one of those atoms, a sample table or a sample claims
more bytes than the file holds. Atoms symphonia skips, and an `mdat` cut
short, are left alone, so truncated recordings still read.

crates/audio-io/src/lib.rs

@@ -16,6 +16,7 @@ mod aiff;
 mod buffer;
 mod caf;
 mod flac;
+mod mp4;
 mod pcm;
 pub mod peaks;
 mod symph;

crates/audio-io/src/mp4.rs

@@ -0,0 +1,106 @@
+//! Size checks on MP4 atoms before symphonia reads them: it allocates whatever an atom, a sample
+//! table or a sample claims, and a failed allocation aborts the process.
+
+use crate::pcm::{be_u32, be_u64};
+use crate::{AudioError, Result};
+
+#[derive(Clone, Copy)]
+enum Level {
+    Top,
+    Moov,
+    Trak,
+    Mdia,
+    Minf,
+    Stbl,
+    Udta,
+    Meta,
+    Ilst,
+    Tag,
+}
+
+/// Refuse the MP4 file whose top-level atoms start at `at` if an atom symphonia reads whole, a
+/// sample table or a sample claims more bytes than the file holds.
+pub(crate) fn check(b: &[u8], at: usize) -> Result<()> {
+    atoms(b, at, b.len(), Level::Top).map(|_| ())
+}
+
+fn malformed(what: &str) -> AudioError {
+    AudioError::Malformed(format!("MP4 {what} is larger than the file"))
+}
+
+/// Walk the atoms in `pos..end` that symphonia descends into, as it reads them. False where
+/// symphonia stops reading: at a header it refuses, or after an atom that runs past `end`.
+fn atoms(b: &[u8], mut pos: usize, end: usize, level: Level) -> Result<bool> {
+    // A header that starts inside the container is read whole, even past its end.
+    while pos < end {
+        let (Some(size), Some(kind)) = (be_u32(b, pos), b.get(pos + 4..pos + 8)) else { return Ok(false) };
+        let head = if size == 1 { 16 } else { 8 };
+        // Size 0: no data, and the next atom starts at the container's end.
+        let len = match size {
+            0 => head,
+            1 => be_u64(b, pos + 8).unwrap_or(0),
+            n => u64::from(n),
+        };
+        if len < head {
+            return Ok(false);
+        }
+        let body = pos + head as usize;
+        let data_end = usize::try_from(len).ok().and_then(|n| pos.checked_add(n));
+        let next = if size == 0 { Some(end) } else { data_end };
+        let inner = data_end.map_or(b.len(), |n| n.min(b.len()));
+        let more = match (level, kind) {
+            (Level::Top, b"moov") => atoms(b, body, inner, Level::Moov)?,
+            (Level::Top | Level::Udta, b"meta") => atoms(b, body + 4, inner, Level::Meta)?,
+            (Level::Moov, b"trak") => atoms(b, body, inner, Level::Trak)?,
+            (Level::Moov, b"udta") => atoms(b, body, inner, Level::Udta)?,
+            (Level::Trak, b"mdia") => atoms(b, body, inner, Level::Mdia)?,
+            (Level::Mdia, b"minf") => atoms(b, body, inner, Level::Minf)?,
+            (Level::Minf, b"stbl") => atoms(b, body, inner, Level::Stbl)?,
+            (Level::Meta, b"ilst") => atoms(b, body, inner, Level::Ilst)?,
+            (Level::Ilst, _) => atoms(b, body, inner, Level::Tag)?,
+            (Level::Stbl, b"stts" | b"co64") => table(b, body + 4, 8, inner)?,
+            (Level::Stbl, b"stsc") => table(b, body + 4, 12, inner)?,
+            (Level::Stbl, b"stco") => table(b, body + 4, 4, inner)?,
+            (Level::Stbl, b"stsz") => sample_sizes(b, body, inner)?,
+            (Level::Mdia, b"hdlr") | (Level::Tag, b"data" | b"name" | b"mean") if len > (b.len() - pos) as u64 => {
+                return Err(malformed(&format!("`{}` atom", String::from_utf8_lossy(kind))));
+            }
+            _ => true,
+        };
+        match next {
+            Some(n) if more && n <= end => pos = n,
+            _ => return Ok(false),
+        }
+    }
+    Ok(true)
+}
+
+/// A sample table's entries, counted at `count`, must fit in the file: symphonia reserves room for
+/// all of them first. False if they run past the table's atom (`end`), where symphonia stops.
+fn table(b: &[u8], count: usize, entry: u64, end: usize) -> Result<bool> {
+    let Some(n) = be_u32(b, count) else { return Ok(false) };
+    let (first, bytes) = (count + 4, u64::from(n) * entry);
+    if bytes > b.len().saturating_sub(first) as u64 {
+        return Err(malformed("sample table"));
+    }
+    Ok(first as u64 + bytes <= end as u64)
+}
+
+/// Every sample's size must fit in the file: symphonia reads each one into a buffer of that size.
+/// Sizes past the atom are never used, as symphonia stops before reading a packet.
+fn sample_sizes(b: &[u8], body: usize, end: usize) -> Result<bool> {
+    let (Some(size), Some(count)) = (be_u32(b, body + 4), be_u32(b, body + 8)) else { return Ok(false) };
+    let file = b.len() as u64;
+    if u64::from(size) > file {
+        return Err(malformed("sample"));
+    }
+    if size != 0 {
+        return Ok(true);
+    }
+    let fits = table(b, body + 8, 4, end)?;
+    let (sizes, _) = b.get(body + 12..end).unwrap_or_default().as_chunks::<4>();
+    if sizes.iter().take(count as usize).any(|s| u64::from(u32::from_be_bytes(*s)) > file) {
+        return Err(malformed("sample"));
+    }
+    Ok(fits)
+}

crates/audio-io/src/symph.rs

@@ -41,8 +41,10 @@ fn open(bytes: &[u8]) -> Result<Box<dyn FormatReader>> {
         match symphonia::default::get_probe().next(&mut mss).map_err(map_err)? {
             Instantiate::Format(reader) => {
                 let at = usize::try_from(mss.pos()).unwrap_or(usize::MAX);
-                if bytes.get(at..at + 4) == Some(b"caff") {
-                    crate::caf::check(bytes, at)?;
+                match bytes.get(at..at + 4) {
+                    Some(b"caff") => crate::caf::check(bytes, at)?,
+                    Some(b"ftyp") => crate::mp4::check(bytes, at.saturating_sub(4))?,
+                    _ => {}
                 }
                 return reader(mss, &fmt_opts).map_err(map_err);
             }

crates/audio-io/tests/malformed.rs

@@ -210,6 +210,67 @@ fn well_formed_caf_still_reads() {
     assert_eq!(probe(&aac, None).unwrap().frames, 3072);
 }
 
+fn atom(kind: &[u8; 4], body: &[u8]) -> Vec<u8> {
+    [&(8 + body.len() as u32).to_be_bytes()[..], kind, body].concat()
+}
+
+/// An MP4 of `frames` frames of 16-bit mono PCM; `swap` may replace any version-0 atom.
+fn pcm_mp4(frames: u32, swap: impl Fn(&[u8; 4], Vec<u8>) -> Vec<u8>) -> Vec<u8> {
+    let full = |kind: &[u8; 4], fields: &[u32]| {
+        let body: Vec<u8> = [0].iter().chain(fields).flat_map(|v: &u32| v.to_be_bytes()).collect();
+        swap(kind, atom(kind, &body))
+    };
+    let entry = atom(b"sowt", &[&[0u8; 6][..], &[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 16, 0, 0, 0, 0], &(48_000u32 << 16).to_be_bytes()].concat());
+    let moov = |offset: u32| {
+        let stsd = atom(b"stsd", &[&[0, 0, 0, 0, 0, 0, 0, 1][..], &entry].concat());
+        let stbl =
+            [stsd, full(b"stts", &[1, frames, 1]), full(b"stsc", &[1, 1, frames, 1]), full(b"stsz", &[2, frames]), full(b"stco", &[1, offset])];
+        let minf = [full(b"smhd", &[0]), atom(b"stbl", &stbl.concat())].concat();
+        let mdia = [full(b"mdhd", &[0, 0, 48_000, frames, 0]), full(b"hdlr", &[0, u32::from_be_bytes(*b"soun"), 0, 0, 0, 0]), atom(b"minf", &minf)];
+        let trak = [full(b"tkhd", &[0; 20]), atom(b"mdia", &mdia.concat())].concat();
+        atom(b"moov", &[full(b"mvhd", &[[0, 0, 48_000, frames, 1 << 16].as_slice(), &[0; 18], &[2]].concat()), atom(b"trak", &trak)].concat())
+    };
+    let ftyp = atom(b"ftyp", b"M4A \0\0\0\0M4A isom");
+    let offset = (ftyp.len() + moov(0).len() + 8) as u32;
+    let pcm: Vec<u8> = (0..frames as i16).flat_map(|i| (i * 40).to_le_bytes()).collect();
+    [ftyp, moov(offset), atom(b"mdat", &pcm)].concat()
+}
+
+#[test]
+fn mp4_sizes_beyond_the_file_are_errors_not_aborts() {
+    let (_, buf) = decode(&pcm_mp4(480, |_, a| a), None).unwrap();
+    assert_eq!(buf.frames(), 480);
+    let huge = |kind: &[u8; 4], a: Vec<u8>| [&1u32.to_be_bytes()[..], kind, &(1u64 << 60).to_be_bytes(), &a[8..]].concat();
+    // The first field after version and flags: stts's entry count, stsz's sample size.
+    let first_field_max = |mut a: Vec<u8>| {
+        a[12..16].copy_from_slice(&u32::MAX.to_be_bytes());
+        a
+    };
+    let sizes: Vec<u8> = (0..480u32).flat_map(|i| if i == 479 { u32::MAX } else { 2 }.to_be_bytes()).collect();
+    let per_sample = atom(b"stsz", &[&[0; 8][..], &480u32.to_be_bytes(), &sizes].concat());
+    // A header that starts inside its container is read whole, past the container's end.
+    let mdia = atom(b"mdia", &1u32.to_be_bytes());
+    let straddling = [&b"hdlr"[..], &(1u64 << 60).to_be_bytes(), &[0; 24]].concat();
+    let files = [
+        pcm_mp4(480, |k, a| if k == b"hdlr" { huge(k, a) } else { a }),
+        pcm_mp4(480, |k, a| if k == b"stts" { first_field_max(a) } else { a }),
+        pcm_mp4(480, |k, a| if k == b"stsz" { first_field_max(a) } else { a }),
+        pcm_mp4(480, |k, a| if k == b"stsz" { per_sample.clone() } else { a }),
+        [atom(b"ftyp", b"M4A \0\0\0\0M4A isom"), atom(b"moov", &atom(b"trak", &mdia)), straddling].concat(),
+    ];
+    for f in files {
+        assert!(matches!(decode(&f, None), Err(AudioError::Malformed(m)) if m.contains("larger than the file")));
+    }
+}
+
+#[test]
+fn atoms_running_past_their_parent_are_walked_once() {
+    // Each moov holds a trak that runs to the end of the file; symphonia stops after the first.
+    let unit = atom(b"moov", &[&u32::MAX.to_be_bytes()[..], b"trak"].concat());
+    let f = [atom(b"ftyp", b"M4A \0\0\0\0M4A isom"), unit.repeat(1 << 18)].concat();
+    assert!(decode(&f, None).is_err());
+}
+
 #[test]
 fn many_tiny_chunks_terminate() {
     let mut chunks: Vec<(&[u8; 4], Vec<u8>)> = vec![(b"fmt ", fmt(1, 1, 8_000, 16))];