From 755dcc1d7b8732aa85f2a94e54892b78c7e3bc3d Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 28 Aug 2025 10:53:45 -0400 Subject: [PATCH] seedance pro handler --- ...e_seedance_1_pro_image_to_video_webhook.rs | 14 + ...enerate_seedance_1_0_pro_image_to_video.rs | 54 ++++ .../src/generate/video/mod.rs | 1 + .../public/enums/src/common/model_type.rs | 11 +- ...seedance_1_0_pro_image_to_video_handler.rs | 247 ++++++++++++++++++ ...erate_veo_3_fast_image_to_video_handler.rs | 2 +- .../endpoints/generate/video/mod.rs | 1 + .../http_server/routes/add_generate_routes.rs | 5 + 8 files changed, 333 insertions(+), 2 deletions(-) create mode 100644 crates/schema/public/api/artcraft_api_defs/src/generate/video/generate_seedance_1_0_pro_image_to_video.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/generate_seedance_1_0_pro_image_to_video_handler.rs diff --git a/crates/api_clients/fal_client/src/requests/webhook/video/enqueue_seedance_1_pro_image_to_video_webhook.rs b/crates/api_clients/fal_client/src/requests/webhook/video/enqueue_seedance_1_pro_image_to_video_webhook.rs index 173c0f7b9a..b1863b8602 100644 --- a/crates/api_clients/fal_client/src/requests/webhook/video/enqueue_seedance_1_pro_image_to_video_webhook.rs +++ b/crates/api_clients/fal_client/src/requests/webhook/video/enqueue_seedance_1_pro_image_to_video_webhook.rs @@ -18,8 +18,15 @@ pub struct Seedance1ProArgs<'a, U: IntoUrl, V: IntoUrl> { #[derive(Copy, Clone, Debug)] pub enum Seedance1ProDuration { + ThreeSeconds, + FourSeconds, FiveSeconds, + SixSeconds, + SevenSeconds, + EightSeconds, + NineSeconds, TenSeconds, + ElevenSeconds, TwelveSeconds, } @@ -34,8 +41,15 @@ pub async fn enqueue_seedance_1_pro_image_to_video_webhook ) -> Result { let duration = match args.duration { + Seedance1ProDuration::ThreeSeconds => Some("3".to_string()), + Seedance1ProDuration::FourSeconds => Some("4".to_string()), Seedance1ProDuration::FiveSeconds => Some("5".to_string()), + Seedance1ProDuration::SixSeconds => Some("6".to_string()), + Seedance1ProDuration::SevenSeconds => Some("7".to_string()), + Seedance1ProDuration::EightSeconds => Some("8".to_string()), + Seedance1ProDuration::NineSeconds => Some("9".to_string()), Seedance1ProDuration::TenSeconds => Some("10".to_string()), + Seedance1ProDuration::ElevenSeconds => Some("11".to_string()), Seedance1ProDuration::TwelveSeconds => Some("12".to_string()), }; diff --git a/crates/schema/public/api/artcraft_api_defs/src/generate/video/generate_seedance_1_0_pro_image_to_video.rs b/crates/schema/public/api/artcraft_api_defs/src/generate/video/generate_seedance_1_0_pro_image_to_video.rs new file mode 100644 index 0000000000..f14680f28b --- /dev/null +++ b/crates/schema/public/api/artcraft_api_defs/src/generate/video/generate_seedance_1_0_pro_image_to_video.rs @@ -0,0 +1,54 @@ +use serde::{Deserialize, Serialize}; +use tokens::tokens::generic_inference_jobs::InferenceJobToken; +use tokens::tokens::media_files::MediaFileToken; +use utoipa::ToSchema; + +pub const GENERATE_SEEDANCE_1_0_PRO_IMAGE_TO_VIDEO_URL_PATH: &str = "/v1/generate/video/seedance_1.0_pro_image_to_video"; + +#[derive(Serialize, Deserialize, ToSchema)] +pub struct GenerateSeedance10ProImageToVideoRequest { + /// Idempotency token to prevent duplicate requests. + pub uuid_idempotency_token: String, + + /// Source of the image file to convert to video. + /// It must be an image. + pub media_file_token: Option, + + /// Optional text prompt. + pub prompt: Option, + + /// Optional: aspect ratio of the generated video. + pub resolution: Option, + + /// Optional: aspect ratio of the generated video. + pub duration: Option, +} + +#[derive(Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum GenerateSeedance10ProResolution { + FourEightyP, + SevenTwentyP, + TenEightyP, +} + +#[derive(Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum GenerateSeedance10ProDuration { + ThreeSeconds, + FourSeconds, + FiveSeconds, + SixSeconds, + SevenSeconds, + EightSeconds, + NineSeconds, + TenSeconds, + ElevenSeconds, + TwelveSeconds, +} + +#[derive(Serialize, Deserialize, ToSchema)] +pub struct GenerateSeedance10ProImageToVideoResponse { + pub success: bool, + pub inference_job_token: InferenceJobToken, +} diff --git a/crates/schema/public/api/artcraft_api_defs/src/generate/video/mod.rs b/crates/schema/public/api/artcraft_api_defs/src/generate/video/mod.rs index 193249cb12..1b61842a11 100644 --- a/crates/schema/public/api/artcraft_api_defs/src/generate/video/mod.rs +++ b/crates/schema/public/api/artcraft_api_defs/src/generate/video/mod.rs @@ -2,6 +2,7 @@ pub mod generate_kling_1_6_pro_image_to_video; pub mod generate_kling_2_1_master_image_to_video; pub mod generate_kling_2_1_pro_image_to_video; pub mod generate_seedance_1_0_lite_image_to_video; +pub mod generate_seedance_1_0_pro_image_to_video; pub mod generate_veo_2_image_to_video; pub mod generate_veo_3_fast_image_to_video; pub mod generate_veo_3_image_to_video; \ No newline at end of file diff --git a/crates/schema/public/enums/src/common/model_type.rs b/crates/schema/public/enums/src/common/model_type.rs index 5894615308..bbed5c66e0 100644 --- a/crates/schema/public/enums/src/common/model_type.rs +++ b/crates/schema/public/enums/src/common/model_type.rs @@ -65,6 +65,8 @@ pub enum ModelType { Kling21Master, #[serde(rename = "seedance_1p0_lite")] Seedance10Lite, + #[serde(rename = "seedance_1p0_pro")] + Seedance10Pro, #[serde(rename = "veo_2")] Veo2, #[serde(rename = "veo_3")] @@ -113,6 +115,7 @@ impl ModelType { Self::Kling21Pro => "kling_2p1_pro", Self::Kling21Master => "kling_2p1_master", Self::Seedance10Lite => "seedance_1p0_lite", + Self::Seedance10Pro => "seedance_1p0_pro", Self::Veo2 => "veo_2", Self::Veo3 => "veo_3", Self::Veo3Fast => "veo_3_fast", @@ -150,6 +153,7 @@ impl ModelType { "kling_2p1_pro" => Ok(Self::Kling21Pro), "kling_2p1_master" => Ok(Self::Kling21Master), "seedance_1p0_lite" => Ok(Self::Seedance10Lite), + "seedance_1p0_pro" => Ok(Self::Seedance10Pro), "veo_2" => Ok(Self::Veo2), "veo_3" => Ok(Self::Veo3), "veo_3_fast" => Ok(Self::Veo3Fast), @@ -191,6 +195,7 @@ impl ModelType { Self::Kling21Pro, Self::Kling21Master, Self::Seedance10Lite, + Self::Seedance10Pro, Self::Veo2, Self::Veo3, Self::Veo3Fast, @@ -236,6 +241,7 @@ mod tests { assert_serialization(ModelType::Kling21Pro, "kling_2p1_pro"); assert_serialization(ModelType::Kling21Master, "kling_2p1_master"); assert_serialization(ModelType::Seedance10Lite, "seedance_1p0_lite"); + assert_serialization(ModelType::Seedance10Pro, "seedance_1p0_pro"); assert_serialization(ModelType::Veo2, "veo_2"); assert_serialization(ModelType::Veo3, "veo_3"); assert_serialization(ModelType::Veo3Fast, "veo_3_fast"); @@ -271,6 +277,7 @@ mod tests { assert_eq!(ModelType::Kling21Pro.to_str(), "kling_2p1_pro"); assert_eq!(ModelType::Kling21Master.to_str(), "kling_2p1_master"); assert_eq!(ModelType::Seedance10Lite.to_str(), "seedance_1p0_lite"); + assert_eq!(ModelType::Seedance10Pro.to_str(), "seedance_1p0_pro"); assert_eq!(ModelType::Veo2.to_str(), "veo_2"); assert_eq!(ModelType::Veo3.to_str(), "veo_3"); assert_eq!(ModelType::Veo3Fast.to_str(), "veo_3_fast"); @@ -306,6 +313,7 @@ mod tests { assert_eq!(ModelType::from_str("kling_2p1_pro").unwrap(), ModelType::Kling21Pro); assert_eq!(ModelType::from_str("kling_2p1_master").unwrap(), ModelType::Kling21Master); assert_eq!(ModelType::from_str("seedance_1p0_lite").unwrap(), ModelType::Seedance10Lite); + assert_eq!(ModelType::from_str("seedance_1p0_pro").unwrap(), ModelType::Seedance10Pro); assert_eq!(ModelType::from_str("veo_2").unwrap(), ModelType::Veo2); assert_eq!(ModelType::from_str("veo_3").unwrap(), ModelType::Veo3); assert_eq!(ModelType::from_str("veo_3_fast").unwrap(), ModelType::Veo3Fast); @@ -317,7 +325,7 @@ mod tests { #[test] fn all_variants() { let mut variants = ModelType::all_variants(); - assert_eq!(variants.len(), 27); + assert_eq!(variants.len(), 28); // Image models assert_eq!(variants.pop_first(), Some(ModelType::Flux1Dev)); assert_eq!(variants.pop_first(), Some(ModelType::Flux1Schnell)); @@ -342,6 +350,7 @@ mod tests { assert_eq!(variants.pop_first(), Some(ModelType::Kling21Pro)); assert_eq!(variants.pop_first(), Some(ModelType::Kling21Master)); assert_eq!(variants.pop_first(), Some(ModelType::Seedance10Lite)); + assert_eq!(variants.pop_first(), Some(ModelType::Seedance10Pro)); assert_eq!(variants.pop_first(), Some(ModelType::Veo2)); assert_eq!(variants.pop_first(), Some(ModelType::Veo3)); assert_eq!(variants.pop_first(), Some(ModelType::Veo3Fast)); diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/generate_seedance_1_0_pro_image_to_video_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/generate_seedance_1_0_pro_image_to_video_handler.rs new file mode 100644 index 0000000000..d71887bf1e --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/generate_seedance_1_0_pro_image_to_video_handler.rs @@ -0,0 +1,247 @@ +use std::sync::Arc; + +use crate::http_server::common_responses::common_web_error::CommonWebError; +use crate::http_server::common_responses::media::media_links_builder::MediaLinksBuilder; +use crate::http_server::endpoints::media_files::helpers::get_media_domain::get_media_domain; +use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format; +use crate::state::server_state::ServerState; +use crate::util::lookup::fetch_all_required_media_files::fetch_all_required_media_files; +use crate::util::traits::into_media_links_trait::IntoMediaLinks; +use actix_web::web::Json; +use actix_web::{web, HttpRequest}; +use artcraft_api_defs::generate::video::generate_seedance_1_0_pro_image_to_video::GenerateSeedance10ProDuration; +use artcraft_api_defs::generate::video::generate_seedance_1_0_pro_image_to_video::GenerateSeedance10ProImageToVideoRequest; +use artcraft_api_defs::generate::video::generate_seedance_1_0_pro_image_to_video::GenerateSeedance10ProImageToVideoResponse; +use artcraft_api_defs::generate::video::generate_seedance_1_0_pro_image_to_video::GenerateSeedance10ProResolution; +use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; +use enums::by_table::prompts::prompt_type::PromptType; +use enums::common::generation_provider::GenerationProvider; +use enums::common::model_type::ModelType; +use enums::common::visibility::Visibility; +use fal_client::requests::webhook::video::enqueue_seedance_1_pro_image_to_video_webhook::enqueue_seedance_1_pro_image_to_video_webhook; +use fal_client::requests::webhook::video::enqueue_seedance_1_pro_image_to_video_webhook::Seedance1ProDuration; +use fal_client::requests::webhook::video::enqueue_seedance_1_pro_image_to_video_webhook::{Seedance1ProArgs, Seedance1ProResolution}; +use http_server_common::request::get_request_ip::get_request_ip; +use log::{error, info, warn}; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::insert_generic_inference_job_for_fal_queue; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::FalCategory; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::InsertGenericInferenceForFalArgs; +use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token; +use mysql_queries::queries::media_files::get::get_media_file::{get_media_file, get_media_file_with_connection}; +use mysql_queries::queries::prompts::insert_prompt::{insert_prompt, InsertPromptArgs}; +use sqlx::Acquire; +use utoipa::ToSchema; + +/// Seedance 1.0 Pro Image to Video +#[utoipa::path( + post, + tag = "Generate Videos", + path = "/v1/generate/video/seedance_1.0_pro_image_to_video", + responses( + (status = 200, description = "Success", body = GenerateSeedance10ProImageToVideoResponse), + ), + params( + ("request" = GenerateSeedance10ProImageToVideoRequest, description = "Payload for Request"), + ) +)] +pub async fn generate_seedance_1_0_pro_image_to_video_handler( + http_request: HttpRequest, + request: Json, + server_state: web::Data> +) -> Result, CommonWebError> { + let mut mysql_connection = server_state.mysql_pool + .acquire() + .await?; + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session_from_connection(&http_request, &mut mysql_connection) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + CommonWebError::ServerError + })?; + + let maybe_avt_token = server_state + .avt_cookie_manager + .get_avt_token_from_request(&http_request); + + // TODO: Limit usage for new accounts. Billing, free credits metering, etc. + + //let user_session = match maybe_user_session { + // Some(session) => session, + // None => { + // warn!("not logged in"); + // return Err(RemoveImageBackgroundError::NotAuthorized); + // } + //}; + + let start_frame_media_file_token = match &request.media_file_token { + Some(token) => token, + None => { + warn!("No media file token provided"); + return Err(CommonWebError::BadInputWithSimpleMessage("No media file token provided".to_string())); + } + }; + + let mut tokens = vec![ + start_frame_media_file_token.clone(), + ]; + + let media_files = fetch_all_required_media_files( + &mut mysql_connection, + &tokens, + ).await?; + + for media_file in media_files.iter() { + if !media_file.media_type.is_jpg_or_png_or_legacy_image() { + return Err(CommonWebError::BadInputWithSimpleMessage("Media file must be a JPG or PNG image".to_string())); + } + } + + if let Err(reason) = validate_idempotency_token_format(&request.uuid_idempotency_token) { + return Err(CommonWebError::BadInputWithSimpleMessage(reason)); + } + + insert_idempotency_token(&request.uuid_idempotency_token, &mut *mysql_connection) + .await + .map_err(|err| { + error!("Error inserting idempotency token: {:?}", err); + CommonWebError::BadInputWithSimpleMessage("invalid idempotency token".to_string()) + })?; + + let media_domain = get_media_domain(&http_request); + + let start_frame_url = media_files.iter() + .find(|file| &file.token == start_frame_media_file_token) + .map(|file| file.to_media_links(media_domain, server_state.server_environment)) + .map(|file| file.cdn_url) + .ok_or_else(|| { + warn!("Start frame media file not found after fetch"); + CommonWebError::NotFound + })?; + + info!("Fal webhook URL: {}", server_state.fal.webhook_url); + + let prompt = request.prompt + .as_deref() + .map(|prompt| prompt.trim()) + .unwrap_or_else(|| ""); + + let resolution = match &request.resolution { + Some(GenerateSeedance10ProResolution::FourEightyP) => Seedance1ProResolution::FourEightyP, + Some(GenerateSeedance10ProResolution::SevenTwentyP) => Seedance1ProResolution::SevenTwentyP, + Some(GenerateSeedance10ProResolution::TenEightyP) => Seedance1ProResolution::TenEightyP, + None => Seedance1ProResolution::SevenTwentyP, + }; + + let duration = match &request.duration { + Some(GenerateSeedance10ProDuration::ThreeSeconds) => Seedance1ProDuration::ThreeSeconds, + Some(GenerateSeedance10ProDuration::FourSeconds) => Seedance1ProDuration::FourSeconds, + Some(GenerateSeedance10ProDuration::FiveSeconds) => Seedance1ProDuration::FiveSeconds, + Some(GenerateSeedance10ProDuration::SixSeconds) => Seedance1ProDuration::SixSeconds, + Some(GenerateSeedance10ProDuration::SevenSeconds) => Seedance1ProDuration::SevenSeconds, + Some(GenerateSeedance10ProDuration::EightSeconds) => Seedance1ProDuration::EightSeconds, + Some(GenerateSeedance10ProDuration::NineSeconds) => Seedance1ProDuration::NineSeconds, + Some(GenerateSeedance10ProDuration::TenSeconds) => Seedance1ProDuration::TenSeconds, + Some(GenerateSeedance10ProDuration::ElevenSeconds) => Seedance1ProDuration::ElevenSeconds, + Some(GenerateSeedance10ProDuration::TwelveSeconds) => Seedance1ProDuration::TwelveSeconds, + None => Seedance1ProDuration::FiveSeconds, + }; + + let args = Seedance1ProArgs { + image_url: start_frame_url, + webhook_url: &server_state.fal.webhook_url, + api_key: &server_state.fal.api_key, + duration, + resolution, + prompt, + camera_fixed: false, // TODO: Parameterize + seed: None, // TODO: Parameterize + }; + + let fal_result = enqueue_seedance_1_pro_image_to_video_webhook(args) + .await + .map_err(|err| { + warn!("Error calling enqueue_seedance_1_pro_image_to_video_webhook: {:?}", err); + CommonWebError::ServerError + })?; + + let external_job_id = fal_result.request_id + .ok_or_else(|| { + warn!("Fal request_id is None"); + CommonWebError::ServerError + })?; + + info!("Fal request_id: {}", external_job_id); + + let ip_address = get_request_ip(&http_request); + + let mut transaction = mysql_connection + .begin() + .await + .map_err(|err| { + error!("Error starting MySQL transaction: {:?}", err); + CommonWebError::ServerError + })?; + + // NB: Don't fail the job if the query fails. + let prompt_result = insert_prompt(InsertPromptArgs { + maybe_apriori_prompt_token: None, + prompt_type: PromptType::ArtcraftApp, + maybe_creator_user_token: maybe_user_session + .as_ref() + .map(|s| &s.user_token), + maybe_model_type: Some(ModelType::Seedance10Pro), + maybe_generation_provider: Some(GenerationProvider::Artcraft), + maybe_positive_prompt: Some(prompt), + maybe_negative_prompt: None, + maybe_other_args: None, + creator_ip_address: &ip_address, + mysql_executor: &mut *transaction, + phantom: Default::default(), + }).await; + + let prompt_token = match prompt_result { + Ok(token) => Some(token), + Err(err) => { + warn!("Error inserting prompt: {:?}", err); + None // Don't fail the job if the prompt insertion fails. + } + }; + + let db_result = insert_generic_inference_job_for_fal_queue(InsertGenericInferenceForFalArgs { + uuid_idempotency_token: &request.uuid_idempotency_token, + maybe_external_third_party_id: &external_job_id, + fal_category: FalCategory::VideoGeneration, + maybe_inference_args: None, + maybe_prompt_token: prompt_token.as_ref(), + maybe_creator_user_token: maybe_user_session.as_ref().map(|s| &s.user_token), + maybe_avt_token: maybe_avt_token.as_ref(), + creator_ip_address: &ip_address, + creator_set_visibility: Visibility::Public, + mysql_executor: &mut *transaction, + phantom: Default::default(), + }).await; + + let job_token = match db_result { + Ok(token) => token, + Err(err) => { + warn!("Error inserting generic inference job for FAL queue: {:?}", err); + return Err(CommonWebError::ServerError); + } + }; + + let _r = transaction + .commit() + .await + .map_err(|err| { + error!("Error committing MySQL transaction: {:?}", err); + CommonWebError::ServerError + })?; + + Ok(Json(GenerateSeedance10ProImageToVideoResponse { + success: true, + inference_job_token: job_token, + })) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/generate_veo_3_fast_image_to_video_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/generate_veo_3_fast_image_to_video_handler.rs index 4966677849..4854c2b9f4 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/generate_veo_3_fast_image_to_video_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/generate_veo_3_fast_image_to_video_handler.rs @@ -28,7 +28,7 @@ use sqlx::Acquire; use tokens::tokens::prompts::PromptToken; use utoipa::ToSchema; -/// Veo 3 Image to Video +/// Veo 3 Fast Image to Video #[utoipa::path( post, tag = "Generate Videos", diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/mod.rs index 09e29abafb..ad5d429cef 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/mod.rs @@ -2,6 +2,7 @@ pub mod generate_kling_1_6_pro_video_handler; pub mod generate_kling_2_1_master_video_handler; pub mod generate_kling_2_1_pro_video_handler; pub mod generate_seedance_1_0_lite_image_to_video_handler; +pub mod generate_seedance_1_0_pro_image_to_video_handler; pub mod generate_veo_2_image_to_video_handler; pub mod generate_veo_3_fast_image_to_video_handler; pub mod generate_veo_3_image_to_video_handler; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/routes/add_generate_routes.rs b/crates/service/web/storyteller_web/src/http_server/routes/add_generate_routes.rs index a5e614559d..37fe29b11f 100644 --- a/crates/service/web/storyteller_web/src/http_server/routes/add_generate_routes.rs +++ b/crates/service/web/storyteller_web/src/http_server/routes/add_generate_routes.rs @@ -22,6 +22,7 @@ use actix_http::body::MessageBody; use actix_service::ServiceFactory; use actix_web::dev::{ServiceRequest, ServiceResponse}; use actix_web::{web, App, Error, HttpResponse}; +use crate::http_server::endpoints::generate::video::generate_seedance_1_0_pro_image_to_video_handler::generate_seedance_1_0_pro_image_to_video_handler; pub fn add_generate_routes (app: App) -> App where @@ -103,6 +104,10 @@ where .route(web::post().to(generate_seedance_1_0_lite_image_to_video_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service(web::resource("/seedance_1.0_pro_image_to_video") + .route(web::post().to(generate_seedance_1_0_pro_image_to_video_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) .service(web::resource("/veo_2_image_to_video") .route(web::post().to(generate_veo_2_image_to_video_handler)) .route(web::head().to(|| HttpResponse::Ok()))