diff --git a/crates/audio/src/vmd.rs b/crates/audio/src/vmd.rs index 6c2a0b829..137396671 100644 --- a/crates/audio/src/vmd.rs +++ b/crates/audio/src/vmd.rs @@ -19,7 +19,7 @@ //! A [`Decoder`] is made from the 816-byte file header. [`Decoder::decode`] takes a sound //! frame's flags (from its record in the file's table of contents; `craft_media::vmd` reads it) //! and its data, and returns interleaved `i16` samples: one block of sound or silence, or, for a -//! frame with a silence mask, several. 8-bit samples are widened to 16 bits. Blocks do not depend +//! frame with a silence mask (the first sound frame of a file), several. 8-bit samples are widened to 16 bits. Blocks do not depend //! on each other, so decoding can start at any frame. //! //! Not decoded: IMA ADPCM sound (audio flag 0x10) and the external codecs of later Coktel Vision @@ -72,7 +72,7 @@ impl Decoder { if usize::from(u16_at(0)) != HEADER_LEN - 2 { return Err(Error::Invalid("not a VMD header")); } - if u16_at(16) & 0x1000 == 0 { + if u16_at(16) & 0x1000 == 0 || u16_at(804) == 0 { return Err(Error::Invalid("the file has no sound")); } let flags = u16_at(810); @@ -119,20 +119,33 @@ impl Decoder { /// frame data. Returns interleaved samples, [`Decoder::block_samples`] for each channel and /// block. /// - /// - flags 1: one block of sound; + /// - flags 1: one block of sound (every whole block, if the frame holds more); /// - flags 2: a 32-bit mask, then blocks of sound: the sound a player loads ahead at the /// start of a file. From bit 0 up, a set bit is a block of silence and a clear bit the next /// block of data, while there is one. ffmpeg plays the silent blocks first instead; /// the order here is the description's; /// - flags 3: one block of silence. /// - /// An empty frame gives no samples. A block cut short is an error. + /// An empty frame gives no samples. A frame's first block cut short is an error; a part of a + /// block after the last whole one is left out. + /// + /// The sound of a file is its sound frames decoded in file order and played end to end from + /// the first picture on: see `craft_media::vmd` for how that lines up with the pictures. pub fn decode(&self, flags: u8, data: &[u8]) -> Result, Error> { let mut out = Vec::new(); match flags { 3 => out.resize(self.block_samples, 0), _ if data.is_empty() => {} - 1 => self.block(data, &mut out)?, + 1 => { + // one block; a frame that holds more whole blocks plays them all, as in ffmpeg + self.block(data, &mut out)?; + let size = self.block_bytes(); + if size > 0 { + for more in data[size..].chunks_exact(size) { + self.block(more, &mut out)?; + } + } + } 2 => { let (mask, mut data) = data.split_first_chunk::<4>().ok_or(Error::Invalid("the silence mask is cut short"))?; let mask = u32::from_le_bytes(*mask); diff --git a/crates/audio/tests/vmd.rs b/crates/audio/tests/vmd.rs index d7004a888..c1746d4df 100644 --- a/crates/audio/tests/vmd.rs +++ b/crates/audio/tests/vmd.rs @@ -4,6 +4,7 @@ use std::process::Command; +use craft_audio::AudioError; use craft_audio::vmd::{Decoder, Error}; use craft_media::vmd::{FrameKind, open}; use craft_media_testkit::hostile::Rng; @@ -153,6 +154,11 @@ fn eight_bit_and_silence() { assert_eq!(d.decode(3, &[]).unwrap(), [0; 4]); assert_eq!(d.decode(1, &[]).unwrap(), [0i16; 0]); assert_eq!(decoder(4, 0x200).decode(3, &[]).unwrap(), [0; 8]); + // a flags 1 frame of more than one block plays every whole block + assert_eq!( + d.decode(1, &[0x80, 0x81, 0x82, 0x83, 0x90, 0x91, 0x92, 0x93, 0xA0]).unwrap(), + [0, 0x100, 0x200, 0x300, 0x1000, 0x1100, 0x1200, 0x1300] + ); assert_eq!(decoder(-4, 0x8000).decode(3, &[]).unwrap(), [0; 4]); } @@ -177,6 +183,9 @@ fn bad_input_is_an_error() { let d = decoder(-20, 0); assert_eq!(d.decode(1, &[0; 20]), Err(Error::Invalid("a block of sound is cut short"))); assert_eq!(d.decode(4, &[0; 21]), Err(Error::Unsupported("unknown sound frame flags"))); + // in the file API's error: unsupported stays unsupported, the rest is malformed input + assert!(matches!(AudioError::from(Error::Unsupported("x")), AudioError::Unsupported(_))); + assert!(matches!(AudioError::from(Error::Invalid("x")), AudioError::Malformed(_))); let good = build(&spec(20, 0), &[]); let with = |patch: &dyn Fn(&mut Vec)| { @@ -188,6 +197,7 @@ fn bad_input_is_an_error() { assert_eq!(with(&|h| h.truncate(815)), Some(Error::Invalid("the header is shorter than 816 bytes"))); assert_eq!(with(&|h| h[0] = 0), Some(Error::Invalid("not a VMD header"))); assert_eq!(with(&|h| h[17] = 0), Some(Error::Invalid("the file has no sound"))); + assert_eq!(with(&|h| h[804..806].fill(0)), Some(Error::Invalid("the file has no sound"))); assert_eq!(with(&|h| h[810] = 0x10), Some(Error::Unsupported("IMA ADPCM sound"))); // a frame length of nothing decodes to nothing let mut empty = good.clone(); diff --git a/crates/media/README.md b/crates/media/README.md index 0d2e10b7c..e827a3d09 100644 --- a/crates/media/README.md +++ b/crates/media/README.md @@ -829,12 +829,18 @@ for (i, frame) in file.frames.iter().enumerate() { type 0 in its place, and the picture stays as it was. - `frame_rate()`: a file with sound plays one block of sound a picture (sample rate over block length); a file without plays 10 pictures a second. +- Timing: the video frame of block `n` is shown at picture period `n`. Sound is one continuous + stream from the first picture on: decode the sound frames in file order and play the samples + end to end. The first sound frame holds many blocks (28 in most Phantasmagoria files), loaded + ahead by the player, so a later sound frame is heard that much later than the block it is + stored in, and the sound outlasts the pictures. - `video_params()` and `audio_params()` return `CodecParams` with `CodecId::VmdVideo` and `CodecId::VmdAudio` and the 816-byte header as `extradata`, which is what `craft_video::vmd::Decoder::new` and `craft_audio::vmd::Decoder::new` take. `header_bytes()` is the same header. -- `picture_rect(frame)` places a video frame's update in the picture (frame coordinates are - screen coordinates when the picture does not sit at the screen origin). +- `picture_rect(frame)` places a video frame's update in the picture. Frame coordinates are + screen coordinates; the first frame as large as the picture gives the picture's position + (`origin()`), as ffmpeg was seen to do. Gaps: only the 816-byte header is read. The later Coktel Vision revisions with a longer header or one without a palette are refused (`Error::NotVmd`). Files with 16-bit, 24-bit or Indeo 3 diff --git a/crates/media/src/id.rs b/crates/media/src/id.rs index 0eb239ad5..7ba597d82 100644 --- a/crates/media/src/id.rs +++ b/crates/media/src/id.rs @@ -371,13 +371,14 @@ mod tests { #[test] fn audio_and_video_are_disjoint() { - for c in [CodecId::Aac, CodecId::Opus, CodecId::Pcm, CodecId::Eac3, CodecId::Speex] { + for c in [CodecId::Aac, CodecId::Opus, CodecId::Pcm, CodecId::Eac3, CodecId::Speex, CodecId::VmdAudio] { assert!(c.is_audio() && !c.is_video(), "{c:?}"); } - for c in [CodecId::H264, CodecId::Av1, CodecId::ProRes, CodecId::Theora, CodecId::Dnxhd, CodecId::Apv] { + for c in [CodecId::H264, CodecId::Av1, CodecId::ProRes, CodecId::Theora, CodecId::Dnxhd, CodecId::Apv, CodecId::VmdVideo] { assert!(c.is_video() && !c.is_audio(), "{c:?}"); } assert!(!CodecId::Unknown.is_audio() && !CodecId::Unknown.is_video()); + assert_eq!((CodecId::VmdVideo.name(), CodecId::VmdAudio.name(), Container::Vmd.name()), ("VMD video", "VMD audio", "VMD")); } #[test] diff --git a/crates/media/src/vmd/mod.rs b/crates/media/src/vmd/mod.rs index 574d871bc..054a43052 100644 --- a/crates/media/src/vmd/mod.rs +++ b/crates/media/src/vmd/mod.rs @@ -8,6 +8,13 @@ //! 16-byte record a frame. A block is one picture period and holds the same number of frames in //! every block: usually a sound frame and a video frame. //! +//! Pictures and sound are timed differently. The video frame of block `n` is shown at picture +//! period `n` ([`Frame::block`], [`VmdFile::frame_rate`]). Sound is one continuous stream that +//! starts with the first picture: decode the sound frames in file order and play the samples end +//! to end. The first sound frame holds many blocks of sound, which the player loads ahead, so +//! each later sound frame is heard that much later than the block it is stored in, and the sound +//! runs on after the last picture. +//! //! [`open`] reads the header and the table of contents; [`VmdFile::read_frame`] reads one frame's //! data on demand. The video decoder is `craft_video::vmd` and the sound decoder //! `craft_audio::vmd`; both take the header ([`VmdFile::header_bytes`]) as their configuration. @@ -109,7 +116,8 @@ pub struct AudioHeader { pub bits: u8, /// 1 or 2. pub channels: u8, - /// Number of sound buffers: how many blocks a frame with a silence mask covers. + /// Number of sound buffers. In the game files seen it is one more than the blocks of sound + /// (silent ones included) in the file's first sound frame. pub buffers: u16, /// The audio flags as stored (0x8000 old-style stereo, 0x200 new-style stereo, 0x10 IMA /// ADPCM). @@ -146,7 +154,7 @@ pub struct Header { pub width: u16, /// Picture height in pixels. pub height: u16, - /// Flags (0x1000: the file has sound). + /// Flags (0x1000: the file has sound, if its sample rate is not 0). pub flags: u16, /// Frames in every block. pub frames_per_block: u16, @@ -187,7 +195,8 @@ impl Header { return Err(Error::NotVmd); } let flags = u16_at(16); - let audio = (flags & 0x1000 != 0).then(|| { + // a sample rate of 0 is no sound either: ffmpeg goes by the rate alone + let audio = (flags & 0x1000 != 0 && u16_at(804) != 0).then(|| { let length = u16_at(806) as i16; let audio_flags = u16_at(810); AudioHeader { @@ -254,7 +263,8 @@ pub enum FrameKind { pub struct Frame { /// What the frame holds. pub kind: FrameKind, - /// The block the frame belongs to: its picture period. + /// The block the frame is stored in. A block is one picture period, so this is when a video + /// frame is shown. It is not when a sound frame is heard: see the module documentation. pub block: u32, /// Offset of the frame data. pub offset: u64, @@ -270,6 +280,7 @@ pub struct VmdFile { /// The frames, block after block. pub frames: Vec, raw_header: Vec, + origin: (u16, u16), } /// Open a VMD file: read the header and the table of contents. @@ -313,7 +324,17 @@ pub fn open(src: &S) -> Result { offset += u64::from(size); } } - Ok(VmdFile { header, frames, raw_header: raw.to_vec() }) + // the picture's place on the screen: see `picture_rect` + let origin = frames + .iter() + .find_map(|f| match f.kind { + FrameKind::Video { left, top, right, bottom, .. } => (right.checked_sub(left).and_then(|w| w.checked_add(1)) == Some(header.width) + && bottom.checked_sub(top).and_then(|h| h.checked_add(1)) == Some(header.height)) + .then_some((left, top)), + _ => None, + }) + .unwrap_or((0, 0)); + Ok(VmdFile { header, frames, raw_header: raw.to_vec(), origin }) } impl VmdFile { @@ -345,29 +366,28 @@ impl VmdFile { } } + /// The screen position `(left, top)` that video frame coordinates are measured from: see + /// [`VmdFile::picture_rect`]. + pub fn origin(&self) -> (u16, u16) { + self.origin + } + /// Where a video frame's update lies in the picture: `(left, top, right, bottom)` with the /// last column and row included, or `None` for a frame that is not video or lies outside the /// picture. /// - /// Frame coordinates are screen coordinates in files whose picture does not sit at the - /// screen origin, and the description does not say how to tell. This follows what ffmpeg - /// does with such files: coordinates that still fit the picture once the header's origin is - /// taken off are screen coordinates, any others are picture coordinates. + /// Frame coordinates are screen coordinates, and a picture need not sit at the screen's + /// corner. The description does not say where the corner comes from. This does what ffmpeg + /// was seen to do: the first video frame that is exactly as large as the picture gives the + /// picture's position ([`VmdFile::origin`]), which is taken off every frame; a file without + /// such a frame has its picture at the corner. The header's own position fields are not + /// used. pub fn picture_rect(&self, frame: &Frame) -> Option<(u16, u16, u16, u16)> { let FrameKind::Video { left, top, right, bottom, .. } = frame.kind else { return None }; let h = &self.header; - if right < left || bottom < top { - return None; - } - let on_screen = left - .checked_sub(h.left) - .zip(top.checked_sub(h.top)) - .zip(right.checked_sub(h.left).zip(bottom.checked_sub(h.top))) - .filter(|&(_, (r, b))| r < h.width && b < h.height); - match on_screen { - Some(((l, t), (r, b))) => Some((l, t, r, b)), - None => (right < h.width && bottom < h.height).then_some((left, top, right, bottom)), - } + let (left, right) = (left.checked_sub(self.origin.0)?, right.checked_sub(self.origin.0)?); + let (top, bottom) = (top.checked_sub(self.origin.1)?, bottom.checked_sub(self.origin.1)?); + (left <= right && top <= bottom && right < h.width && bottom < h.height).then_some((left, top, right, bottom)) } /// What a video decoder needs, in the container-neutral form the codec crates take: diff --git a/crates/media/src/vmd/tests.rs b/crates/media/src/vmd/tests.rs index 58c32f233..eb4b52bd3 100644 --- a/crates/media/src/vmd/tests.rs +++ b/crates/media/src/vmd/tests.rs @@ -72,6 +72,14 @@ fn sound_formats() { assert_eq!(open_with(500, 0x8000), (8, 2, 250, CodecId::VmdAudio)); assert_eq!(open_with(i16::MIN, 0), (16, 1, 32768, CodecId::VmdAudio)); assert_eq!(open_with(-500, 0x10).3, CodecId::Unknown); + + // sound needs the header flag and a sample rate + let mut d = sample(true); + d[804..806].fill(0); + assert_eq!(open(&d).unwrap().header.audio, None); + let mut d = sample(true); + d[17] &= !0x10; + assert_eq!(open(&d).unwrap().header.audio, None); } #[test] @@ -88,18 +96,27 @@ fn other_picture_formats_are_not_vmd_video() { assert_eq!(params(|d| d[24..28].copy_from_slice(b"iv32")).0, VideoFormat::Indeo3); } +/// The cases ffmpeg was checked on: the first frame as large as the picture places it. #[test] -fn picture_rect_takes_off_the_screen_origin() { - let mut spec = Spec::new(16, 8); - (spec.left, spec.top) = (4, 2); - let frame = |rect| F::video(rect, false, vec![]); - let d = build( - &spec, - &[vec![frame([4, 2, 19, 9])], vec![frame([2, 0, 5, 0])], vec![frame([6, 2, 9, 2])], vec![frame([0, 0, 16, 7])], vec![frame([5, 0, 4, 0])]], +fn picture_rect_takes_off_the_picture_position() { + let rects = |header_origin: (u16, u16), frames: &[[u16; 4]]| { + let mut spec = Spec::new(20, 10); + (spec.left, spec.top) = header_origin; + let blocks: Vec<_> = frames.iter().map(|&rect| vec![F::video(rect, false, vec![])]).collect(); + let f = open(&build(&spec, &blocks)).unwrap(); + (f.origin(), f.frames.iter().map(|fr| f.picture_rect(fr)).collect::>()) + }; + // a picture at the screen's corner, whatever the header says + assert_eq!(rects((10, 5), &[[0, 0, 19, 9], [12, 6, 18, 9]]), ((0, 0), vec![Some((0, 0, 19, 9)), Some((12, 6, 18, 9))])); + // a picture at (3, 2): frames left of or above it, or past its far edges, are outside + assert_eq!( + rects((0, 0), &[[3, 2, 22, 11], [3, 2, 5, 5], [0, 0, 5, 5], [3, 2, 23, 11], [3, 2, 22, 12]]), + ((3, 2), vec![Some((0, 0, 19, 9)), Some((0, 0, 2, 3)), None, None, None]) ); - let f = open(&d).unwrap(); - let rects: Vec<_> = f.frames.iter().map(|fr| f.picture_rect(fr)).collect(); - assert_eq!(rects, [Some((0, 0, 15, 7)), Some((2, 0, 5, 0)), Some((2, 0, 5, 0)), None, None]); + // the placing frame need not be the first + assert_eq!(rects((0, 0), &[[4, 3, 5, 4], [4, 3, 23, 12]]), ((4, 3), vec![Some((0, 0, 1, 1)), Some((0, 0, 19, 9))])); + // no frame as large as the picture: the corner; inverted and oversized frames are outside + assert_eq!(rects((7, 7), &[[2, 0, 5, 0], [5, 0, 4, 0], [0, 1, 0, 0], [0, 0, 20, 9]]), ((0, 0), vec![Some((2, 0, 5, 0)), None, None, None])); } #[test] @@ -128,7 +145,7 @@ fn table_of_contents_must_be_in_the_file() { let mut many = d.clone(); many[6..8].copy_from_slice(&[0xFF; 2]); many[18..20].copy_from_slice(&[0xFF; 2]); - assert!(matches!(open(&many), Err(Error::Invalid(_)))); + assert!(matches!(open(&many), Err(Error::Invalid(e)) if e.contains("frames in the table"))); } #[test] diff --git a/crates/video/src/vmd/mod.rs b/crates/video/src/vmd/mod.rs index 98ba08f2a..a25b3a603 100644 --- a/crates/video/src/vmd/mod.rs +++ b/crates/video/src/vmd/mod.rs @@ -37,8 +37,8 @@ //! files ([`Error::Unsupported`]). //! //! Input is treated as hostile: the picture is checked against the decode memory budget -//! ([`crate::budget`]), LZ output is bounded by the header's unpack buffer size, and every copy -//! is bounds-checked. No `unsafe`, no panics. +//! ([`crate::budget`]), LZ output is bounded by the header's unpack buffer size and by a few +//! bytes a pixel of the picture, and every copy is bounds-checked. No `unsafe`, no panics. #![warn(missing_docs)] @@ -205,7 +205,10 @@ impl Decoder { return self.render(rect, method, body); } let mut unpacked = std::mem::take(&mut self.unpacked); - let result = lz_unpack(body, self.unpack_limit, &mut unpacked).and_then(|()| self.render(rect, method & 0x7F, &unpacked)); + // no rendering method reads more than a few bytes a pixel: unpacking stops there, so that + // a frame's cost goes by the picture's size and not by what the file says + let useful = self.indices.len().saturating_mul(4).saturating_add(1024); + let result = lz_unpack(body, self.unpack_limit, useful, &mut unpacked).and_then(|()| self.render(rect, method & 0x7F, &unpacked)); self.unpacked = unpacked; result } @@ -313,8 +316,9 @@ fn unpack_rle(src: &mut Reader, run: &mut [u8]) -> Result<()> { /// Unpack an LZ-packed frame body into `out`: the unpacked length, an optional marker that /// selects long chains, then tag bytes whose bits (lowest first) each announce a literal byte /// or a chain copied from a 4096-byte window of earlier output. Packed data that ends before -/// the length is reached leaves `out` short; the rendering then fails if it needs more. -fn lz_unpack(src: &[u8], limit: usize, out: &mut Vec) -> Result<()> { +/// the length is reached leaves `out` short; the rendering then fails if it needs more. A length +/// above `limit` is an error; unpacking stops after `useful` bytes. +fn lz_unpack(src: &[u8], limit: usize, useful: usize, out: &mut Vec) -> Result<()> { out.clear(); let mut src = Reader(src); let len = src.take(4).ok_or(Error::Invalid("the frame data ends early"))?; @@ -322,6 +326,7 @@ fn lz_unpack(src: &[u8], limit: usize, out: &mut Vec) -> Result<()> { if left > limit { return Err(Error::Invalid("an LZ-packed frame is larger than the unpack buffer")); } + left = left.min(useful); let mut window = Window { bytes: [0x20; 4096], at: 0xFEE }; let mut long_chains = false; if src.0.starts_with(&[0x34, 0x12, 0x78, 0x56]) { diff --git a/crates/video/tests/vmd/main.rs b/crates/video/tests/vmd/main.rs index 9dd460065..b44aa3753 100644 --- a/crates/video/tests/vmd/main.rs +++ b/crates/video/tests/vmd/main.rs @@ -256,6 +256,34 @@ fn lz_chains() { raw.extend_from_slice(chunk); } assert_eq!(unpack(&raw).unwrap(), (0u8..32).collect::>()); + // with 8 bytes or fewer left, a tag of 0xFF is eight literal bits, of which only the bytes + // left are read: 13 bytes are 8 raw and 5 literals + let mut d = flat(); + let tail = [0x82, 13, 0, 0, 0, 0xFF, 1, 2, 3, 4, 5, 6, 7, 8, 0xFF, 9, 10, 11, 12, 13]; + d.decode(Rect { right: 12, bottom: 0, ..FULL }, false, &tail).unwrap(); + assert_eq!(d.indices()[..14], [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 0x40]); +} + +/// A frame that says it unpacks to the whole unpack buffer costs no more than its picture: the +/// same 8 packed bytes repeated would otherwise be unpacked to megabytes for every frame. +#[test] +fn lz_unpacking_stops_at_what_a_picture_can_use() { + let mut spec = Spec::new(16, 2); + spec.unpack = u32::MAX; + let mut d = Decoder::new(&build(&spec, &[])).unwrap(); + // 4 GiB declared; long chains of 273 bytes from the window's fill, far more than 32 pixels + let mut data = vec![0x82, 0xFF, 0xFF, 0xFF, 0xFF, 0x34, 0x12, 0x78, 0x56]; + for _ in 0..40_000 { + data.push(0); + data.extend([0x00, 0x0F, 0xFF].repeat(8)); + } + let start = std::time::Instant::now(); + for _ in 0..200 { + d.decode(FULL, false, &data).unwrap(); + } + assert_eq!(d.indices(), [0x20; 32]); + // 87 MB of output for each frame without the stop: seconds, not milliseconds + assert!(start.elapsed() < std::time::Duration::from_secs(2), "{:?}", start.elapsed()); } #[test] @@ -276,6 +304,9 @@ fn palette_changes() { chunk[..2].copy_from_slice(&[4, 2]); d.decode(FULL, true, &chunk).unwrap(); assert_eq!((d.palette()[0], d.palette()[255]), ([255; 3], [255; 3])); + // components are 6 bits: higher bits are dropped + d.decode(FULL, true, &palette_chunk(&[0x41; 768])).unwrap(); + assert_eq!(d.palette()[7], [4; 3]); assert_eq!(d.decode(FULL, true, &[0; 769]), Err(Error::Invalid("the palette is cut short"))); }