diff --git a/Dockerfile b/Dockerfile index dd52eb70d6..ee90b49cae 100644 --- a/Dockerfile +++ b/Dockerfile @@ -64,6 +64,7 @@ COPY Cargo.toml . COPY sqlx-data.json . COPY crates/ ./crates COPY db/ ./db +COPY test_data/ ./test_data # Run all of the tests RUN SQLX_OFFLINE=true \ diff --git a/crates/lib/media/Cargo.toml b/crates/lib/media/Cargo.toml index fc36b15be5..e6c8a945c2 100644 --- a/crates/lib/media/Cargo.toml +++ b/crates/lib/media/Cargo.toml @@ -19,7 +19,9 @@ errors = { path = "../errors" } # External log = "0.4.14" -symphonia = { version = "0.5.1", features = ["aac", "alac", "mp3", "isomp4"] } # Powerful audio decoder and demuxer + +# Powerful audio decoder and demuxer +symphonia = { version = "0.5.1", features = ["aac", "alac", "flac", "isomp4", "mkv", "mp3", "ogg", "vorbis", "wav"] } [dev-dependencies] # None yet diff --git a/crates/lib/media/src/decode_basic_audio_info.rs b/crates/lib/media/src/decode_basic_audio_info.rs index 7549d8147b..06ccd1f048 100644 --- a/crates/lib/media/src/decode_basic_audio_info.rs +++ b/crates/lib/media/src/decode_basic_audio_info.rs @@ -1,7 +1,7 @@ use errors::AnyhowResult; use std::io::Cursor; use symphonia::core::audio::SampleBuffer; -use symphonia::core::formats::{FormatOptions, FormatReader}; +use symphonia::core::formats::{FormatOptions, FormatReader, Track}; use symphonia::core::io::{MediaSourceStream, ReadOnlySource}; use symphonia::core::meta::MetadataOptions; use symphonia::core::probe::Hint; @@ -16,6 +16,8 @@ pub struct BasicAudioInfo { pub required_full_decode: bool, } +/// Decode audio info from an audio or video file containing audio streams. +/// This handles multiple formats and codecs. pub fn decode_basic_audio_info( audio_bytes: &[u8], maybe_mimetype: Option<&str>, @@ -46,13 +48,13 @@ pub fn decode_basic_audio_info( let mut format = probed.format; - let default_track = match format.default_track() { + let audio_track = match find_audio_track(&format) { None => return Ok(NO_AUDIO_INFO.clone()), Some(default_track) => default_track, }; - let mut maybe_track_duration = default_track.codec_params.time_base - .zip(default_track.codec_params.n_frames) + let mut maybe_track_duration = audio_track.codec_params.time_base + .zip(audio_track.codec_params.n_frames) .map(|(time_base, n_frames)| { // NB: This yields the duration of the track time_base.calc_time(n_frames) @@ -63,10 +65,7 @@ pub fn decode_basic_audio_info( duration_millis + frac_millis }); - let codec_registry = symphonia::default::get_codecs(); - - let maybe_codec_name = codec_registry.get_codec(default_track.codec_params.codec) - .map(|codec_descriptor| codec_descriptor.short_name.to_string()); + let maybe_codec_name = get_codec_short_name(&audio_track); let mut required_full_decode = false; @@ -85,9 +84,39 @@ pub fn decode_basic_audio_info( }) } +fn find_audio_track(format: &Box) -> Option<&Track> { + // For audio files, the default track is sufficient. + // Video files include multiple streams, and they will need to be searched. + + for track in format.tracks() { + match track.codec_params.codec { + symphonia::core::codecs::CODEC_TYPE_NULL => continue, // NB: Eg. unsupported, video, etc. + _ => return Some(track), + } + } + + format.default_track() +} + +fn get_codec_short_name(track: &Track) -> Option { + let codec_registry = symphonia::default::get_codecs(); + + if let Some(codec_descriptor) = codec_registry.get_codec(track.codec_params.codec) { + return Some(codec_descriptor.short_name.to_string()); + } + + match track.codec_params.codec { + // NB: Opus (which is newer than Vorbis and for streaming) support has not landed yet, but the + // Symphonia library knows which tracks are Opus-encoded. + // https://github.com/pdeljanov/Symphonia/issues/8 + symphonia::core::codecs::CODEC_TYPE_OPUS => Some("opus".to_string()), + _ => None, + } +} + fn read_duration(format: &mut Box) -> AnyhowResult> { // NB: Code adapted from symphonia repo example code. - let track = match format.default_track() { + let audio_track = match find_audio_track(format) { None => return Ok(None), Some(track) => track, }; @@ -96,20 +125,20 @@ fn read_duration(format: &mut Box) -> AnyhowResult // Create a decoder for the track. let mut decoder = symphonia::default::get_codecs() - .make(&track.codec_params, &decoder_opts)?; + .make(&audio_track.codec_params, &decoder_opts)?; - let channel_count = match track.codec_params.channels { + let channel_count = match audio_track.codec_params.channels { None => return Ok(None), Some(channels) => channels.count(), }; - let sample_rate = match track.codec_params.sample_rate { + let sample_rate = match audio_track.codec_params.sample_rate { None => return Ok(None), Some(sample_rate) => sample_rate, }; // Store the track identifier, we'll use it to filter packets. - let track_id = track.id; + let track_id = audio_track.id; let mut sample_count = 0; let mut sample_buf = None; @@ -175,83 +204,185 @@ mod tests { use errors::AnyhowResult; use crate::decode_basic_audio_info::decode_basic_audio_info; - fn test_file(root_file_path: &str) -> PathBuf { + fn test_file(path_from_repo_root: &str) -> PathBuf { // https://doc.rust-lang.org/cargo/reference/environment-variables.html let mut path = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - path.push(format!("../../../{}", root_file_path)); + path.push(format!("../../../{}", path_from_repo_root)); path } - #[test] - fn decode_flac_info() -> AnyhowResult<()> { - let path = test_file("test_data/audio/flac/zelda_ocarina_small_item.flac"); - let bytes = std::fs::read(path)?; - let info = decode_basic_audio_info(&bytes, None, None)?; - assert_eq!(info.codec_name, Some("flac".to_string())); - assert_eq!(info.duration_millis, Some(5120)); - assert_eq!(info.required_full_decode, false); - Ok(()) + mod audio { + use super::*; + + #[test] + fn aac() -> AnyhowResult<()> { + // NB: ffprobe -i output: + // [aac @ 0x55d4e03a4ec0] Estimating duration from bitrate, this may be inaccurate + // Input #0, aac, from 'audio/aac/golden_sun_elemental_stars_cyanne.aac': + // Duration: 00:00:40.05, bitrate: 128 kb/s + // Stream #0:0: Audio: aac (LC), 16000 Hz, stereo, fltp, 128 kb/s + // It might be faster to use ffmpeg's estimation method. + let path = test_file("test_data/audio/aac/golden_sun_elemental_stars_cyanne.aac"); + let bytes = std::fs::read(path)?; + let info = decode_basic_audio_info(&bytes, None, None)?; + assert_eq!(info.codec_name, Some("aac".to_string())); + assert_eq!(info.duration_millis, Some(40128)); + assert_eq!(info.required_full_decode, true); + Ok(()) + } + + #[test] + fn flac() -> AnyhowResult<()> { + let path = test_file("test_data/audio/flac/zelda_ocarina_small_item.flac"); + let bytes = std::fs::read(path)?; + let info = decode_basic_audio_info(&bytes, None, None)?; + assert_eq!(info.codec_name, Some("flac".to_string())); + assert_eq!(info.duration_millis, Some(5120)); + assert_eq!(info.required_full_decode, false); + Ok(()) + } + + #[test] + fn flac_wrong_extension() -> AnyhowResult<()> { + let path = test_file("test_data/audio/flac/zelda_ocarina_small_item.flac"); + let bytes = std::fs::read(path)?; + let incorrect_mimetype = Some("audio/wav"); + let incorrect_extension = Some("wav"); + let info = decode_basic_audio_info(&bytes, incorrect_mimetype, incorrect_extension)?; + assert_eq!(info.codec_name, Some("flac".to_string())); + assert_eq!(info.duration_millis, Some(5120)); + assert_eq!(info.required_full_decode, false); + Ok(()) + } + + // NB: Requires symphonia 'aac' and 'isomp4' feature flags + #[test] + fn m4a() -> AnyhowResult<()> { + let path = test_file("test_data/audio/m4a/super_mario_bros_lost_life.m4a"); + let bytes = std::fs::read(path)?; + let info = decode_basic_audio_info(&bytes, None, None)?; + assert_eq!(info.codec_name, Some("alac".to_string())); + assert_eq!(info.duration_millis, Some(5493)); + assert_eq!(info.required_full_decode, false); + Ok(()) + } + + // NB: Requires symphonia 'mp3' feature flag + #[test] + fn mp3() -> AnyhowResult<()> { + let path = test_file("test_data/audio/mp3/super_mario_rpg_beware_the_forests_mushrooms.mp3"); + let bytes = std::fs::read(path)?; + let info = decode_basic_audio_info(&bytes, None, None)?; + assert_eq!(info.codec_name, Some("mp3".to_string())); + assert_eq!(info.duration_millis, Some(15023)); + assert_eq!(info.required_full_decode, false); + Ok(()) + } + + #[test] + fn ogg() -> AnyhowResult<()> { + // According to ffprobe (which ever so slightly disagrees with our calculation), + // Duration: 00:00:04.90, start: 0.000000, bitrate: 83 kb/s + // length : 4.94 + let path = test_file("test_data/audio/ogg/banjo-kazooie_jiggy_appearance.ogg"); + let bytes = std::fs::read(path)?; + let info = decode_basic_audio_info(&bytes, None, None)?; + assert_eq!(info.codec_name, Some("vorbis".to_string())); + assert_eq!(info.duration_millis, Some(4903)); // NB: This disagrees with ffprobe, but it's pretty close. + assert_eq!(info.required_full_decode, true); + Ok(()) + } + + #[test] + fn wav_pcm_s16le_16khz() -> AnyhowResult<()> { + let path = test_file("test_data/audio/wav/sm64_mario_its_me.wav"); + let bytes = std::fs::read(path)?; + let info = decode_basic_audio_info(&bytes, None, None)?; + assert_eq!(info.codec_name, Some("pcm_s16le".to_string())); + assert_eq!(info.duration_millis, Some(1891)); + assert_eq!(info.required_full_decode, false); + Ok(()) + } + + #[test] + fn wav_pcm_s16le_44khz() -> AnyhowResult<()> { + let path = test_file("test_data/audio/wav/smrpg_correct.wav"); + let bytes = std::fs::read(path)?; + let info = decode_basic_audio_info(&bytes, None, None)?; + assert_eq!(info.codec_name, Some("pcm_s16le".to_string())); + assert_eq!(info.duration_millis, Some(847)); + assert_eq!(info.required_full_decode, false); + Ok(()) + } + + #[test] + fn wav_pcm_f32() -> AnyhowResult<()> { + let path = test_file("test_data/audio/wav/smrpg_battlestart_f32.wav"); + let bytes = std::fs::read(path)?; + let info = decode_basic_audio_info(&bytes, None, None)?; + assert_eq!(info.codec_name, Some("pcm_f32le".to_string())); + assert_eq!(info.duration_millis, Some(708)); + assert_eq!(info.required_full_decode, false); + Ok(()) + } + + #[test] + fn wav_unsigned() -> AnyhowResult<()> { + let path = test_file("test_data/audio/wav/smrpg_item_mushroom_8u.wav"); + let bytes = std::fs::read(path)?; + let info = decode_basic_audio_info(&bytes, None, None)?; + assert_eq!(info.codec_name, Some("pcm_u8".to_string())); + assert_eq!(info.duration_millis, Some(1741)); + assert_eq!(info.required_full_decode, false); + Ok(()) + } } - #[test] - fn decode_flac_info_with_wrong_extension() -> AnyhowResult<()> { - let path = test_file("test_data/audio/flac/zelda_ocarina_small_item.flac"); - let bytes = std::fs::read(path)?; - let incorrect_mimetype = Some("audio/wav"); - let incorrect_extension = Some("wav"); - let info = decode_basic_audio_info(&bytes, incorrect_mimetype, incorrect_extension)?; - assert_eq!(info.codec_name, Some("flac".to_string())); - assert_eq!(info.duration_millis, Some(5120)); - assert_eq!(info.required_full_decode, false); - Ok(()) - } + mod video { + use super::*; - // NB: Requires symphonia 'aac' and 'isomp4' feature flags - #[test] - fn decode_m4a_alac_info() -> AnyhowResult<()> { - let path = test_file("test_data/audio/m4a/super_mario_bros_lost_life.m4a"); - let bytes = std::fs::read(path)?; - let info = decode_basic_audio_info(&bytes, None, None)?; - assert_eq!(info.codec_name, Some("alac".to_string())); - assert_eq!(info.duration_millis, Some(5493)); - assert_eq!(info.required_full_decode, false); - Ok(()) - } + #[test] + fn mkv_h264_video_opus_audio() -> AnyhowResult<()> { + let path = test_file("test_data/video/mkv/fake_you.mkv"); + let bytes = std::fs::read(path)?; + let info = decode_basic_audio_info(&bytes, None, None)?; + assert_eq!(info.codec_name, Some("opus".to_string())); + assert_eq!(info.duration_millis, Some(15007)); + assert_eq!(info.required_full_decode, false); + Ok(()) + } - // NB: Requires symphonia 'mp3' feature flag - #[test] - fn decode_mp3_info() -> AnyhowResult<()> { - let path = test_file("test_data/audio/mp3/super_mario_rpg_beware_the_forests_mushrooms.mp3"); - let bytes = std::fs::read(path)?; - let info = decode_basic_audio_info(&bytes, None, None)?; - assert_eq!(info.codec_name, Some("mp3".to_string())); - assert_eq!(info.duration_millis, Some(15023)); - assert_eq!(info.required_full_decode, false); - Ok(()) - } + #[test] + fn mov_h264_video_aac_audio() -> AnyhowResult<()> { + let path = test_file("test_data/video/mov/majoras_mask_intro.mov"); + let bytes = std::fs::read(path)?; + let info = decode_basic_audio_info(&bytes, None, None)?; + assert_eq!(info.codec_name, Some("aac".to_string())); + assert_eq!(info.duration_millis, Some(30128)); + assert_eq!(info.required_full_decode, false); + Ok(()) + } - #[test] - fn decode_ogg_info() -> AnyhowResult<()> { - // According to ffprobe (which ever so slightly disagrees with our calculation), - // Duration: 00:00:04.90, start: 0.000000, bitrate: 83 kb/s - // length : 4.94 - let path = test_file("test_data/audio/ogg/banjo-kazooie_jiggy_appearance.ogg"); - let bytes = std::fs::read(path)?; - let info = decode_basic_audio_info(&bytes, None, None)?; - assert_eq!(info.codec_name, Some("vorbis".to_string())); - assert_eq!(info.duration_millis, Some(4903)); // NB: This disagrees with ffprobe, but it's pretty close. - assert_eq!(info.required_full_decode, true); - Ok(()) - } + #[test] + fn mp4_h264_video_aac_audio() -> AnyhowResult<()> { + let path = test_file("test_data/video/mp4/golden_sun_garoh.mp4"); + let bytes = std::fs::read(path)?; + let info = decode_basic_audio_info(&bytes, None, None)?; + assert_eq!(info.codec_name, Some("aac".to_string())); + assert_eq!(info.duration_millis, Some(15295)); + assert_eq!(info.required_full_decode, false); + Ok(()) + } - #[test] - fn decode_wav_info_pcm_s16le() -> AnyhowResult<()> { - let path = test_file("test_data/audio/wav/sm64_mario_its_me.wav"); - let bytes = std::fs::read(path)?; - let info = decode_basic_audio_info(&bytes, None, None)?; - assert_eq!(info.codec_name, Some("pcm_s16le".to_string())); - assert_eq!(info.duration_millis, Some(1891)); - assert_eq!(info.required_full_decode, false); - Ok(()) + #[test] + fn webm_vp9_video_opus_audio() -> AnyhowResult<()> { + let path = test_file("test_data/video/webm/laser_pong.webm"); + let bytes = std::fs::read(path)?; + let info = decode_basic_audio_info(&bytes, None, None)?; + assert_eq!(info.codec_name, Some("opus".to_string())); + assert_eq!(info.duration_millis, Some(10016)); + assert_eq!(info.required_full_decode, false); + Ok(()) + } } } diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/upload_audio.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/upload_audio.rs index 955d7be1ad..e713269cab 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/upload_audio.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/upload_audio.rs @@ -169,31 +169,58 @@ pub async fn upload_audio_handler( let mut maybe_duration_millis = None; let mut maybe_codec_name = None; + let mut media_upload_type = None; if let Some(mimetype) = maybe_mimetype.as_deref() { - match mimetype { - "video/x-msvideo" => { - // .avi - }, - "audio/x-wav" | "audio/ogg" => { - // .wav, .ogg - let basic_info = decode_basic_audio_info(bytes.as_ref(), Some(mimetype), None) - .map_err(|e| { - warn!("file decoding error: {:?}", e); - UploadAudioError::BadInput("could not decode file".to_string()) - })?; + // NB: .aiff (audio/aiff) isn't supported by Symphonia: + // It contains uncompressed PCM-encoded audio similar to wav. + // See: https://github.com/pdeljanov/Symphonia/issues/75 + // NB: The following formats are not supported by Symphonia and + // do not have any open issues filed. They may simply be too old: + // - .wma (audio/x-ms-wma) + // - .avi (video/x-msvideo) + media_upload_type = match mimetype { + // Audio + "audio/aac" /* .aac */ => Some(MediaUploadType::Audio), + "audio/m4a" /* .m4a */ => Some(MediaUploadType::Audio), + "audio/mpeg" /* .mp3 */ => Some(MediaUploadType::Audio), + "audio/ogg" /* .ogg */ => Some(MediaUploadType::Audio), + "audio/x-flac" /* .flac */ => Some(MediaUploadType::Audio), + "audio/x-wav" /* .wav */ => Some(MediaUploadType::Audio), + // Video + "video/mp4" /* .mp4 */ => Some(MediaUploadType::Video), + "video/quicktime" /* .mov */ => Some(MediaUploadType::Video), + "video/webm" /* .webm */ => Some(MediaUploadType::Video), + _ => None, + }; - maybe_duration_millis = basic_info.duration_millis; - maybe_codec_name = basic_info.codec_name; - } - _ => {} + if media_upload_type.is_some() { + let basic_info = decode_basic_audio_info( + bytes.as_ref(), + Some(mimetype), + None + ).map_err(|e| { + warn!("file decoding error: {:?}", e); + UploadAudioError::BadInput("could not decode file".to_string()) + })?; + + maybe_duration_millis = basic_info.duration_millis; + maybe_codec_name = basic_info.codec_name; } } + let media_upload_type = match media_upload_type { + Some(m) => m, + None => { + warn!("Invalid mimetype: {:?}", maybe_mimetype); + return Err(UploadAudioError::BadInput(format!("Bad mimetype: {:?}", maybe_mimetype))) + }, + }; + let record_id = insert_media_upload(Args { token: &token, uuid_idempotency_token: &uuid_idempotency_token, - media_type: MediaUploadType::Audio, + media_type: media_upload_type, maybe_original_filename: upload_media_request.file_name.as_deref(), original_file_size_bytes: file_size_bytes as u64, maybe_original_duration_millis: maybe_duration_millis, diff --git a/test_data/audio/aac/golden_sun_elemental_stars_cyanne.aac b/test_data/audio/aac/golden_sun_elemental_stars_cyanne.aac new file mode 100644 index 0000000000..f8ea4e26fc Binary files /dev/null and b/test_data/audio/aac/golden_sun_elemental_stars_cyanne.aac differ diff --git a/test_data/audio/aiff/jet_force_gemini_anubis.aiff b/test_data/audio/aiff/jet_force_gemini_anubis.aiff new file mode 100644 index 0000000000..3b0d909ca1 Binary files /dev/null and b/test_data/audio/aiff/jet_force_gemini_anubis.aiff differ diff --git a/test_data/audio/wav/smrpg_battlestart_f32.wav b/test_data/audio/wav/smrpg_battlestart_f32.wav new file mode 100644 index 0000000000..3c38b85ed7 Binary files /dev/null and b/test_data/audio/wav/smrpg_battlestart_f32.wav differ diff --git a/test_data/audio/wav/smrpg_correct.wav b/test_data/audio/wav/smrpg_correct.wav new file mode 100644 index 0000000000..ccb8c51776 Binary files /dev/null and b/test_data/audio/wav/smrpg_correct.wav differ diff --git a/test_data/audio/wav/smrpg_item_mushroom_8u.wav b/test_data/audio/wav/smrpg_item_mushroom_8u.wav new file mode 100644 index 0000000000..398a936b76 Binary files /dev/null and b/test_data/audio/wav/smrpg_item_mushroom_8u.wav differ diff --git a/test_data/video/mkv/fake_you.mkv b/test_data/video/mkv/fake_you.mkv new file mode 100644 index 0000000000..c59840b9a8 Binary files /dev/null and b/test_data/video/mkv/fake_you.mkv differ diff --git a/test_data/video/mov/majoras_mask_intro.mov b/test_data/video/mov/majoras_mask_intro.mov new file mode 100644 index 0000000000..b0b1a8ae59 Binary files /dev/null and b/test_data/video/mov/majoras_mask_intro.mov differ diff --git a/test_data/video/mp4/golden_sun_garoh.mp4 b/test_data/video/mp4/golden_sun_garoh.mp4 new file mode 100644 index 0000000000..c2047dcfb6 Binary files /dev/null and b/test_data/video/mp4/golden_sun_garoh.mp4 differ diff --git a/test_data/video/webm/laser_pong.webm b/test_data/video/webm/laser_pong.webm new file mode 100644 index 0000000000..157938c6a3 Binary files /dev/null and b/test_data/video/webm/laser_pong.webm differ