From b54953bc0404c9505591625484ed68bc6a99e371 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 26 Aug 2025 19:02:25 -0400 Subject: [PATCH] seedance end frame --- ...eue_kling_16_pro_image_to_video_webhook.rs | 21 +++-- ...eue_kling_21_pro_image_to_video_webhook.rs | 8 +- ..._seedance_1_lite_image_to_video_webhook.rs | 21 +++-- .../generic/handle_video_artcraft.rs | 1 + crates/lib/test_data/src/web/image_urls.rs | 2 +- ...nerate_seedance_1_0_lite_image_to_video.rs | 6 +- ...eedance_1_0_lite_image_to_video_handler.rs | 84 +++++++++++-------- .../seedance/v1/lite/image_to_video/mod.rs | 38 +++++---- 8 files changed, 104 insertions(+), 77 deletions(-) diff --git a/crates/api_clients/fal_client/src/requests/webhook/video/enqueue_kling_16_pro_image_to_video_webhook.rs b/crates/api_clients/fal_client/src/requests/webhook/video/enqueue_kling_16_pro_image_to_video_webhook.rs index 3baaa5babd..c97ff638e8 100644 --- a/crates/api_clients/fal_client/src/requests/webhook/video/enqueue_kling_16_pro_image_to_video_webhook.rs +++ b/crates/api_clients/fal_client/src/requests/webhook/video/enqueue_kling_16_pro_image_to_video_webhook.rs @@ -68,35 +68,34 @@ pub async fn enqueue_kling_16_pro_image_to_video_webhook AnyhowResult<()> { - let image = test_file_path("test_data/image/juno.jpg")?; - + async fn test() -> AnyhowResult<()> { // XXX: Don't commit secrets! let secret = read_to_string("/Users/bt/Artcraft/credentials/fal_api_key.txt")?; let api_key = FalApiKey::from_str(&secret); - let args = Kling16Args { - image_path: image, - prompt: "a corgi looks out over the water", + let args = Kling16ProArgs { + image_url: TALL_MOCHI_WITH_GLASSES_IMAGE_URL, + end_frame_image_url: Some(JUNO_AT_LAKE_IMAGE_URL.to_string()), + prompt: "shiba in glasses runs to the lake and stands by the shore", api_key: &api_key, duration: Kling16Duration::Default, + aspect_ratio: Kling16ProAspectRatio::WideSixteenNine, + webhook_url: "https://example.com/webhook", }; - let result = enqueue_kling_16_image_to_video_webhook(args).await?; + let result = enqueue_kling_16_pro_image_to_video_webhook(args).await?; Ok(()) } } -*/ \ No newline at end of file diff --git a/crates/api_clients/fal_client/src/requests/webhook/video/enqueue_kling_21_pro_image_to_video_webhook.rs b/crates/api_clients/fal_client/src/requests/webhook/video/enqueue_kling_21_pro_image_to_video_webhook.rs index f52a882212..2c7bc991de 100644 --- a/crates/api_clients/fal_client/src/requests/webhook/video/enqueue_kling_21_pro_image_to_video_webhook.rs +++ b/crates/api_clients/fal_client/src/requests/webhook/video/enqueue_kling_21_pro_image_to_video_webhook.rs @@ -74,20 +74,18 @@ mod tests { use crate::requests::webhook::video::enqueue_kling_21_pro_image_to_video_webhook::{enqueue_kling_21_pro_image_to_video_webhook, Kling21ProArgs, Kling21ProAspectRatio, Kling21ProDuration}; use errors::AnyhowResult; use std::fs::read_to_string; - use test_data::web::image_urls::{JUNO_AT_LAKE_IMAGE_URL, WIDE_FALL_MOUNTAINS_IMAGE_URL}; + use test_data::web::image_urls::{JUNO_AT_LAKE_IMAGE_URL, SUPER_WIDE_FALL_MOUNTAINS_IMAGE_URL}; #[tokio::test] #[ignore] - async fn test_kling21_pro_video() -> AnyhowResult<()> { - let image_url = "https://cdn-2.fakeyou.com/media/3/4/h/f/s/34hfsmt8e38rvne6mwa4pwbxr6292sgy/image_34hfsmt8e38rvne6mwa4pwbxr6292sgy.png"; - + async fn test() -> AnyhowResult<()> { // XXX: Don't commit secrets! let secret = read_to_string("/Users/bt/Artcraft/credentials/fal_api_key.txt")?; let api_key = FalApiKey::from_str(&secret); let args = Kling21ProArgs { - image_url: WIDE_FALL_MOUNTAINS_IMAGE_URL, + image_url: SUPER_WIDE_FALL_MOUNTAINS_IMAGE_URL, end_frame_image_url: Some(JUNO_AT_LAKE_IMAGE_URL.to_string()), prompt: "a shot of the mountains, the camera pulls back to show a corgi waiting to jump into the lake", api_key: &api_key, diff --git a/crates/api_clients/fal_client/src/requests/webhook/video/enqueue_seedance_1_lite_image_to_video_webhook.rs b/crates/api_clients/fal_client/src/requests/webhook/video/enqueue_seedance_1_lite_image_to_video_webhook.rs index c13c277dcb..996259cba6 100644 --- a/crates/api_clients/fal_client/src/requests/webhook/video/enqueue_seedance_1_lite_image_to_video_webhook.rs +++ b/crates/api_clients/fal_client/src/requests/webhook/video/enqueue_seedance_1_lite_image_to_video_webhook.rs @@ -6,8 +6,9 @@ use fal::endpoints::fal_ai::bytedance::seedance::v1::lite::image_to_video::Image use fal::webhook::WebhookResponse; use reqwest::IntoUrl; -pub struct Seedance1LiteArgs<'a, U: IntoUrl, V: IntoUrl> { +pub struct Seedance1LiteArgs<'a, U: IntoUrl, T: IntoUrl, V: IntoUrl> { pub image_url: U, + pub end_frame_image_url: Option, pub webhook_url: V, pub prompt: &'a str, pub api_key: &'a FalApiKey, @@ -29,8 +30,8 @@ pub enum Seedance1LiteResolution { SevenTwentyP, // 720p } -pub async fn enqueue_seedance_1_lite_image_to_video_webhook( - args: Seedance1LiteArgs<'_, U, V> +pub async fn enqueue_seedance_1_lite_image_to_video_webhook( + args: Seedance1LiteArgs<'_, U, T, V> ) -> Result { let duration = match args.duration { Seedance1LiteDuration::FiveSeconds => Some("5".to_string()), @@ -44,8 +45,12 @@ pub async fn enqueue_seedance_1_lite_image_to_video_webhook AnyhowResult<()> { - let image_url = "https://cdn-2.fakeyou.com/media/3/4/h/f/s/34hfsmt8e38rvne6mwa4pwbxr6292sgy/image_34hfsmt8e38rvne6mwa4pwbxr6292sgy.png"; - + async fn test() -> AnyhowResult<()> { // XXX: Don't commit secrets! let secret = read_to_string("/Users/bt/Artcraft/credentials/fal_api_key.txt")?; let api_key = FalApiKey::from_str(&secret); let args = Seedance1LiteArgs { - image_url: image_url, - prompt: "a shot of the mountains as the sun sets and reveals the moon and stars", + image_url: TALL_MOCHI_WITH_GLASSES_IMAGE_URL, + end_frame_image_url: Some(JUNO_AT_LAKE_IMAGE_URL.to_string()), + prompt: "shiba in glasses runs to the lake and stands by the shore", api_key: &api_key, camera_fixed: false, duration: Seedance1LiteDuration::FiveSeconds, diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/generic/handle_video_artcraft.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/generic/handle_video_artcraft.rs index 97582d217e..1dfd4bbbec 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/generic/handle_video_artcraft.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_to_video/generic/handle_video_artcraft.rs @@ -156,6 +156,7 @@ pub async fn handle_video_artcraft( let request = GenerateSeedance10LiteImageToVideoRequest { uuid_idempotency_token, media_file_token: request.image_media_token, + end_frame_image_media_token: request.end_frame_image_media_token, prompt: request.prompt, resolution: None, duration: None, diff --git a/crates/lib/test_data/src/web/image_urls.rs b/crates/lib/test_data/src/web/image_urls.rs index 370632946c..e7b83a2812 100644 --- a/crates/lib/test_data/src/web/image_urls.rs +++ b/crates/lib/test_data/src/web/image_urls.rs @@ -22,7 +22,7 @@ pub const MOUNTAIN_TREE_IMAGE_URL :&str = "https://cdn-2.fakeyou.com/media/3/4/h pub const WHITE_HOUSE_SUNSET_IMAGE_URL : &str = "https://cdn-2.fakeyou.com/media/q/t/z/w/k/qtzwka8dd0g2nfyk94k1qh3zzfwqtfhn/asset_qtzwka8dd0g2nfyk94k1qh3zzfwqtfhn.jpg"; // media token: m_z61yg5w7dnnvhb1f9sqt8jf02y61bv -pub const WIDE_FALL_MOUNTAINS_IMAGE_URL : &str = "https://cdn-2.fakeyou.com/media/w/6/m/j/p/w6mjpcqkky7f2f3xkyvkcv3qeedkmejs/asset_w6mjpcqkky7f2f3xkyvkcv3qeedkmejs.jpg"; +pub const SUPER_WIDE_FALL_MOUNTAINS_IMAGE_URL: &str = "https://cdn-2.fakeyou.com/media/w/6/m/j/p/w6mjpcqkky7f2f3xkyvkcv3qeedkmejs/asset_w6mjpcqkky7f2f3xkyvkcv3qeedkmejs.jpg"; // media token: m_m1bz02z1kkzanxy6rb4vk1kvq9de9g pub const JUNO_AT_LAKE_IMAGE_URL : &str = "https://cdn-2.fakeyou.com/media/s/v/c/p/p/svcpp441gyb6mjtt8413kf5zxw1ehy4t/upload_svcpp441gyb6mjtt8413kf5zxw1ehy4t.jpg"; diff --git a/crates/schema/public/api/artcraft_api_defs/src/generate/video/generate_seedance_1_0_lite_image_to_video.rs b/crates/schema/public/api/artcraft_api_defs/src/generate/video/generate_seedance_1_0_lite_image_to_video.rs index a0464bf8cf..9d4755b73a 100644 --- a/crates/schema/public/api/artcraft_api_defs/src/generate/video/generate_seedance_1_0_lite_image_to_video.rs +++ b/crates/schema/public/api/artcraft_api_defs/src/generate/video/generate_seedance_1_0_lite_image_to_video.rs @@ -13,7 +13,11 @@ pub struct GenerateSeedance10LiteImageToVideoRequest { /// Source of the image file to convert to video. /// It must be an image. pub media_file_token: Option, - + + /// Optional. + /// Image media file; the image to remove the background from. + pub end_frame_image_media_token: Option, + /// Optional text prompt. pub prompt: Option, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/generate_seedance_1_0_lite_image_to_video_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/generate_seedance_1_0_lite_image_to_video_handler.rs index 680c685c13..77bd95ef34 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/generate_seedance_1_0_lite_image_to_video_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/generate_seedance_1_0_lite_image_to_video_handler.rs @@ -5,6 +5,8 @@ use crate::http_server::common_responses::media::media_links_builder::MediaLinks use crate::http_server::endpoints::media_files::helpers::get_media_domain::get_media_domain; use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format; use crate::state::server_state::ServerState; +use crate::util::lookup::fetch_all_required_media_files::fetch_all_required_media_files; +use crate::util::traits::into_media_links_trait::IntoMediaLinks; use actix_web::web::Json; use actix_web::{web, HttpRequest}; use artcraft_api_defs::generate::video::generate_seedance_1_0_lite_image_to_video::GenerateSeedance10LiteDuration; @@ -74,14 +76,35 @@ pub async fn generate_seedance_1_0_lite_image_to_video_handler( // } //}; - let media_file_token = match &request.media_file_token { + let start_frame_media_file_token = match &request.media_file_token { Some(token) => token, None => { warn!("No media file token provided"); return Err(CommonWebError::BadInputWithSimpleMessage("No media file token provided".to_string())); } }; - + + let mut tokens = vec![ + start_frame_media_file_token.clone(), + ]; + + let maybe_end_frame_image_media_token = request.end_frame_image_media_token.as_ref(); + + if let Some(end_frame_token) = maybe_end_frame_image_media_token { + tokens.push(end_frame_token.clone()); + } + + let media_files = fetch_all_required_media_files( + &mut mysql_connection, + &tokens, + ).await?; + + for media_file in media_files.iter() { + if !media_file.media_type.is_jpg_or_png_or_legacy_image() { + return Err(CommonWebError::BadInputWithSimpleMessage("Media file must be a JPG or PNG image".to_string())); + } + } + if let Err(reason) = validate_idempotency_token_format(&request.uuid_idempotency_token) { return Err(CommonWebError::BadInputWithSimpleMessage(reason)); } @@ -92,42 +115,30 @@ pub async fn generate_seedance_1_0_lite_image_to_video_handler( error!("Error inserting idempotency token: {:?}", err); CommonWebError::BadInputWithSimpleMessage("invalid idempotency token".to_string()) })?; - const IS_MOD : bool = false; - - let media_file_lookup_result = get_media_file_with_connection( - media_file_token, - IS_MOD, - &mut mysql_connection, - ).await; - let media_file = match media_file_lookup_result { - Ok(Some(media_file)) => media_file, - Ok(None) => { - warn!("MediaFile not found: {:?}", media_file_token); - return Err(CommonWebError::NotFound); - }, - Err(err) => { - warn!("Error looking up media_file: {:?}", err); - return Err(CommonWebError::ServerError); - } + let media_domain = get_media_domain(&http_request); + + let start_frame_url = media_files.iter() + .find(|file| &file.token == start_frame_media_file_token) + .map(|file| file.to_media_links(media_domain, server_state.server_environment)) + .map(|file| file.cdn_url) + .ok_or_else(|| { + warn!("Start frame media file not found after fetch"); + CommonWebError::NotFound + })?; + + let maybe_end_frame_url = match maybe_end_frame_image_media_token { + None => None, + Some(end_frame_token) => Some(media_files.iter() + .find(|file| &file.token == end_frame_token) + .map(|file| file.to_media_links(media_domain, server_state.server_environment)) + .map(|file| file.cdn_url) + .ok_or_else(|| { + warn!("End frame media file not found after fetch"); + CommonWebError::NotFound + })?), }; - if !media_file.media_type.is_jpg_or_png_or_legacy_image() { - return Err(CommonWebError::BadInputWithSimpleMessage("Media file must be a JPG or PNG image".to_string())); - } - - let media_domain = get_media_domain(&http_request); - - let bucket_path = MediaFileBucketPath::from_object_hash( - &media_file.public_bucket_directory_hash, - media_file.maybe_public_bucket_prefix.as_deref(), - media_file.maybe_public_bucket_extension.as_deref()); - - let media_links = MediaLinksBuilder::from_media_path_and_env( - media_domain, - server_state.server_environment, - &bucket_path); - info!("Fal webhook URL: {}", server_state.fal.webhook_url); let prompt = request.prompt @@ -148,7 +159,8 @@ pub async fn generate_seedance_1_0_lite_image_to_video_handler( }; let args = Seedance1LiteArgs { - image_url: media_links.cdn_url, + image_url: start_frame_url, + end_frame_image_url: maybe_end_frame_url, webhook_url: &server_state.fal.webhook_url, api_key: &server_state.fal.api_key, duration, diff --git a/crates/vendor/fal-rs/src/endpoints/fal_ai/bytedance/seedance/v1/lite/image_to_video/mod.rs b/crates/vendor/fal-rs/src/endpoints/fal_ai/bytedance/seedance/v1/lite/image_to_video/mod.rs index 017f2d7f39..06fc738325 100644 --- a/crates/vendor/fal-rs/src/endpoints/fal_ai/bytedance/seedance/v1/lite/image_to_video/mod.rs +++ b/crates/vendor/fal-rs/src/endpoints/fal_ai/bytedance/seedance/v1/lite/image_to_video/mod.rs @@ -5,6 +5,29 @@ use serde::{Deserialize, Serialize}; #[allow(unused_imports)] use std::collections::HashMap; +#[derive(Debug, Serialize, Deserialize, Default)] +pub struct ImageToVideoRequest { + pub image_url: String, + + #[serde(skip_serializing_if = "Option::is_none")] + pub end_image_url: Option, + + /// The resolution of the generated video frame + #[serde(skip_serializing_if = "Option::is_none")] + pub resolution: Option, + + /// The duration of the generated video in seconds + #[serde(skip_serializing_if = "Option::is_none")] + pub duration: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub camera_fixed: Option, + + pub prompt: String, + + pub seed: i32, +} + #[derive(Debug, Serialize, Deserialize, Default)] pub struct File { /// The mime type of the file. @@ -39,21 +62,6 @@ pub struct I2VOutput { pub video: File, } -#[derive(Debug, Serialize, Deserialize, Default)] -pub struct ImageToVideoRequest { - /// The resolution of the generated video frame - #[serde(skip_serializing_if = "Option::is_none")] - pub resolution: Option, - /// The duration of the generated video in seconds - #[serde(skip_serializing_if = "Option::is_none")] - pub duration: Option, - pub image_url: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub camera_fixed: Option, - pub prompt: String, - pub seed: i32, -} - #[derive(Debug, Serialize, Deserialize, Default)] pub struct ValidationError { pub loc: Vec,