From 11ddeb05b50982b80f475ceec172e539243db8c4 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 10 Jul 2026 16:58:34 -0400 Subject: [PATCH] handle audio results (#1772) --- .../hydrated/hydrated_webhook_contents.rs | 17 +++ .../extract_audio_payload.rs | 103 ++++++++++++++ .../hydrated/success_case_extractors/mod.rs | 9 +- .../seed_audio_1p0_audio_payload_1.json | 18 +++ .../handle_successful_fal_webhook.rs | 9 ++ .../webhooks/fal/process_success/mod.rs | 1 + .../process_success/process_audio_payload.rs | 132 ++++++++++++++++++ 7 files changed, 286 insertions(+), 3 deletions(-) create mode 100644 crates/api_clients/fal_client/src/webhook_payload/hydrated/success_case_extractors/extract_audio_payload.rs create mode 100644 crates/api_clients/fal_client/test_data/webhooks/success/seed_audio_1p0_audio_payload_1.json create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/webhooks/fal/process_success/process_audio_payload.rs diff --git a/crates/api_clients/fal_client/src/webhook_payload/hydrated/hydrated_webhook_contents.rs b/crates/api_clients/fal_client/src/webhook_payload/hydrated/hydrated_webhook_contents.rs index 5458106d7f..8487e68371 100644 --- a/crates/api_clients/fal_client/src/webhook_payload/hydrated/hydrated_webhook_contents.rs +++ b/crates/api_clients/fal_client/src/webhook_payload/hydrated/hydrated_webhook_contents.rs @@ -43,6 +43,9 @@ pub struct ExtractedContents { /// Parsed from `payload.video`. pub video: Option, + /// Parsed from `payload.audio` (e.g. Seed Audio 1.0 speech results). + pub audio: Option, + /// Parsed from `payload.model_glb`. pub model_glb: Option, @@ -108,6 +111,20 @@ pub struct VideoData { pub file_size: Option, } +/// Data under `payload.audio` (e.g. Seed Audio 1.0): +#[derive(Debug, Deserialize)] +pub struct AudioData { + pub url: Option, + pub content_type: Option, + pub file_name: Option, + pub file_size: Option, + pub bitrate: Option, + pub channels: Option, + /// Duration in seconds. + pub duration: Option, + pub sample_rate: Option, +} + /// Data under `payload.model_glb` (there may be other sibling keys too). /// Also used for `payload.model_glb_pbr`, which has the same shape. #[derive(Debug, Deserialize)] diff --git a/crates/api_clients/fal_client/src/webhook_payload/hydrated/success_case_extractors/extract_audio_payload.rs b/crates/api_clients/fal_client/src/webhook_payload/hydrated/success_case_extractors/extract_audio_payload.rs new file mode 100644 index 0000000000..b965a9a2f1 --- /dev/null +++ b/crates/api_clients/fal_client/src/webhook_payload/hydrated/success_case_extractors/extract_audio_payload.rs @@ -0,0 +1,103 @@ +use serde_json::{Map, Value}; + +use crate::webhook_payload::hydrated::hydrated_webhook_contents::AudioData; + +/// Extract and deserialize the `audio` key from a webhook success payload +/// (e.g. Seed Audio 1.0 speech results). +pub(crate) fn extract_audio(obj: &Map) -> Option { + let value = obj.get("audio")?; + serde_json::from_value(value.clone()).ok() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::webhook_payload::hydrate_webhook_contents::hydrate_webhook_contents; + use crate::webhook_payload::hydrated::hydrated_webhook_contents::HydratedWebhookContents; + use crate::webhook_payload::raw::raw_webhook_payload::RawWebhookPayload; + + fn load_test_webhook(filename: &str) -> RawWebhookPayload { + let path = format!("test_data/webhooks/{}", filename); + let json = std::fs::read_to_string(&path) + .unwrap_or_else(|e| panic!("Failed to read {}: {}", path, e)); + serde_json::from_str(&json) + .unwrap_or_else(|e| panic!("Failed to parse {}: {}", path, e)) + } + + #[test] + fn audio_from_seed_audio_1p0_test_file() { + let webhook = load_test_webhook("success/seed_audio_1p0_audio_payload_1.json"); + let result = hydrate_webhook_contents(&webhook); + + let HydratedWebhookContents::Success(data) = result else { + panic!("Expected Success, got {:?}", result); + }; + + let contents = data.extracted_contents + .expect("extracted_contents should be Some"); + + let audio = contents.audio.expect("audio should be Some"); + assert_eq!(audio.url.as_deref(), Some("https://v3b.fal.media/files/b/0aa1bc36/vOSCcxsbibkZIiXR9BIZn_speech.mp3")); + assert_eq!(audio.content_type.as_deref(), Some("audio/mpeg")); + assert_eq!(audio.file_name.as_deref(), Some("speech.mp3")); + assert_eq!(audio.file_size, Some(53612)); + assert_eq!(audio.duration, Some(6.63995)); + assert_eq!(audio.sample_rate, Some(24000)); + assert!(audio.bitrate.is_none()); + assert!(audio.channels.is_none()); + + // No other content types in this payload. + assert!(contents.image.is_none()); + assert!(contents.images.is_none()); + assert!(contents.video.is_none()); + assert!(contents.model_glb.is_none()); + assert!(contents.thumbnail.is_none()); + } + + #[test] + fn synthetic_audio_payload() { + let obj: serde_json::Map = serde_json::from_str(r#"{ + "audio": { + "url": "https://cdn.example.com/speech.mp3", + "content_type": "audio/mpeg", + "file_name": "speech.mp3", + "file_size": 12345, + "bitrate": 128000, + "channels": 2, + "duration": 3.5, + "sample_rate": 44100 + } + }"#).unwrap(); + + let audio = extract_audio(&obj).expect("should extract audio"); + assert_eq!(audio.url.as_deref(), Some("https://cdn.example.com/speech.mp3")); + assert_eq!(audio.content_type.as_deref(), Some("audio/mpeg")); + assert_eq!(audio.file_name.as_deref(), Some("speech.mp3")); + assert_eq!(audio.file_size, Some(12345)); + assert_eq!(audio.bitrate, Some(128000)); + assert_eq!(audio.channels, Some(2)); + assert_eq!(audio.duration, Some(3.5)); + assert_eq!(audio.sample_rate, Some(44100)); + } + + #[test] + fn audio_url_only() { + let obj: serde_json::Map = serde_json::from_str(r#"{ + "audio": {"url": "https://cdn.example.com/speech.wav"} + }"#).unwrap(); + + let audio = extract_audio(&obj).expect("should extract audio"); + assert_eq!(audio.url.as_deref(), Some("https://cdn.example.com/speech.wav")); + assert!(audio.content_type.is_none()); + assert!(audio.duration.is_none()); + } + + #[test] + fn missing_audio_key_returns_none() { + let obj: serde_json::Map = serde_json::from_str(r#"{ + "video": {"url": "https://example.com/video.mp4"} + }"#).unwrap(); + + assert!(extract_audio(&obj).is_none()); + } +} diff --git a/crates/api_clients/fal_client/src/webhook_payload/hydrated/success_case_extractors/mod.rs b/crates/api_clients/fal_client/src/webhook_payload/hydrated/success_case_extractors/mod.rs index a9a63cedc1..06578003bb 100644 --- a/crates/api_clients/fal_client/src/webhook_payload/hydrated/success_case_extractors/mod.rs +++ b/crates/api_clients/fal_client/src/webhook_payload/hydrated/success_case_extractors/mod.rs @@ -1,3 +1,4 @@ +pub mod extract_audio_payload; pub mod extract_image_payload; pub mod extract_images_payload; pub mod extract_video_payload; @@ -18,8 +19,8 @@ use crate::webhook_payload::hydrated::hydrated_webhook_contents::ExtractedConten /// Try to extract known content keys from a success payload. /// /// Checks the payload (as a JSON object) for any of: `image`, `images`, -/// `video`, `model_glb`, `model_glb_pbr`, `model_urls`, `model_mesh`, -/// `model_obj`, `result_files`, `thumbnail`, `rendered_image`, +/// `video`, `audio`, `model_glb`, `model_glb_pbr`, `model_urls`, +/// `model_mesh`, `model_obj`, `result_files`, `thumbnail`, `rendered_image`, /// `preprocessed_image`. If at least one is found, returns /// `Some(ExtractedContents)` with copies of the relevant values. Multiple /// keys can be populated simultaneously. Returns `None` if the payload is @@ -30,6 +31,7 @@ pub fn extract_contents_from_payload(payload: &Value) -> Option Option Option = Some("artcraft_"); + +/// Process an `audio` payload (e.g. Seed Audio 1.0 speech results): download +/// the file, upload it to the public bucket, and insert an audio media file +/// record. +pub async fn process_audio_payload( + audio_data: &AudioData, + job: &FalJobDetails, + server_state: &ServerState, +) -> Result { + let audio_url = audio_data.url + .as_deref() + .ok_or_else(|| { + warn!("No `url` in audio payload"); + CommonWebError::server_error_with_message("no `url` in audio payload") + })?; + + // Download with a retry if the first attempt returns suspiciously few bytes. + let mut file_bytes = http_download_url_to_bytes(audio_url) + .await + .map_err(|err| { + warn!("Failed to download audio from {}: {:?}", audio_url, err); + CommonWebError::from_error(err) + })?; + + if file_bytes.len() <= 10 { + warn!( + "Downloaded only {} bytes from {} — retrying once", + file_bytes.len(), + audio_url, + ); + file_bytes = http_download_url_to_bytes(audio_url) + .await + .map_err(|err| { + warn!("Failed to download audio on retry from {}: {:?}", audio_url, err); + CommonWebError::from_error(err) + })?; + } + + // Resolve mime type: magic bytes first, fal content_type as fallback. + let metadata = resolve_file_metadata(&file_bytes, audio_data.content_type.as_deref()) + .ok_or_else(|| { + warn!( + "Could not determine file type for audio (bytes: {}, fal content_type: {:?})", + file_bytes.len(), + audio_data.content_type, + ); + CommonWebError::server_error_with_message( + &format!("Could not determine file type for audio (bytes: {}, fal content_type: {:?})", + file_bytes.len(), audio_data.content_type)) + })?; + + let mime_type = metadata.mime_type.as_str(); + + info!("Mime type of audio: {}, source: {:?}", mime_type, metadata.source); + + let media_file_type = MediaFileType::try_from_mime_type(mime_type) + .ok_or_else(|| { + warn!("Unsupported media file type: {}", mime_type); + CommonWebError::server_error_with_message( + &format!("Unsupported media file type: {}", mime_type)) + })?; + + let extension_with_period = metadata.file_extension.extension_with_period(); + + let file_size_bytes = file_bytes.len(); + let file_hash = sha256_hash_bytes(&file_bytes) + .map_err(|err| { + warn!("Failed to hash audio bytes: {:?}", err); + CommonWebError::from_anyhow_error(err) + })?; + + // Fal reports the duration in (fractional) seconds. + let maybe_duration_millis = audio_data.duration + .map(|seconds| (seconds * 1000.0) as u64); + + let public_upload_path = MediaFileBucketPath::generate_new(PREFIX, Some(&extension_with_period)); + + info!("Uploading media to bucket path: {}", public_upload_path.get_full_object_path_str()); + + server_state.public_bucket_client.upload_file_with_content_type_process( + public_upload_path.get_full_object_path_str(), + file_bytes.as_ref(), + &mime_type) + .await + .map_err(|err| { + warn!("Failed to upload audio to bucket: {:?}", err); + CommonWebError::from_anyhow_error(err) + })?; + + let media_token = MediaFileInsertBuilder::new() + .checksum_sha2(&file_hash) + .creator_ip_address(&job.creator_ip_address) + .file_size_bytes(file_size_bytes as u64) + .maybe_creator_anonymous_visitor(job.maybe_creator_anonymous_visitor_token.as_ref()) + .maybe_creator_user(job.maybe_creator_user_token.as_ref()) + .maybe_duration_millis(maybe_duration_millis) + .maybe_generation_provider(Some(GenerationProvider::Artcraft)) + .maybe_prompt_token(job.maybe_prompt_token.as_ref()) + .maybe_platform_type(job.maybe_platform_type) + .media_file_class(MediaFileClass::Audio) + .media_file_origin_category(MediaFileOriginCategory::Inference) + .media_file_type(media_file_type) + .mime_type(mime_type) + .public_bucket_directory_hash(&public_upload_path) + .insert_pool(&server_state.mysql_pool) + .await + .map_err(|err| { + warn!("Failed to insert audio media file record: {:?}", err); + CommonWebError::from_error(err) + })?; + + info!("Audio media file uploaded with token: {}", media_token); + + Ok(media_token) +}