diff --git a/crates/api_clients/artcraft/artcraft_api_defs/src/generate/video/mod.rs b/crates/api_clients/artcraft/artcraft_api_defs/src/generate/video/mod.rs index 1b61842a1..dcbdb40fb 100644 --- a/crates/api_clients/artcraft/artcraft_api_defs/src/generate/video/mod.rs +++ b/crates/api_clients/artcraft/artcraft_api_defs/src/generate/video/mod.rs @@ -5,4 +5,5 @@ pub mod generate_seedance_1_0_lite_image_to_video; pub mod generate_seedance_1_0_pro_image_to_video; pub mod generate_veo_2_image_to_video; pub mod generate_veo_3_fast_image_to_video; -pub mod generate_veo_3_image_to_video; \ No newline at end of file +pub mod generate_veo_3_image_to_video; +pub mod multi_function; \ No newline at end of file diff --git a/crates/api_clients/artcraft/artcraft_api_defs/src/generate/video/multi_function/kling_2_5_turbo_multi_function_video_gen.rs b/crates/api_clients/artcraft/artcraft_api_defs/src/generate/video/multi_function/kling_2_5_turbo_multi_function_video_gen.rs new file mode 100644 index 000000000..0caf69524 --- /dev/null +++ b/crates/api_clients/artcraft/artcraft_api_defs/src/generate/video/multi_function/kling_2_5_turbo_multi_function_video_gen.rs @@ -0,0 +1,64 @@ +use serde::{Deserialize, Serialize}; +use tokens::tokens::generic_inference_jobs::InferenceJobToken; +use tokens::tokens::media_files::MediaFileToken; +use utoipa::ToSchema; + +pub const KLING_2P5_TURBO_PRO_MULTI_FUNCTION_VIDEO_VIDEO_PATH: &str = "/v1/generate/video/multi_function/kling_2p5_turbo_pro"; + +/// Both text-to-video and image-to-video in one request. +#[derive(Serialize, Deserialize, ToSchema)] +pub struct Kling2p5TurboProMultiFunctionVideoGenRequest { + /// Idempotency token to prevent duplicate requests. + pub uuid_idempotency_token: String, + + /// "Required". + /// Required downstream, but we'll coerce null to empty string. + /// Text prompt. + pub prompt: Option, + + /// Optional. + /// Only for text-to-video. + pub negative_prompt: Option, + + /// Optional. + /// Only for image-to-video + /// Source of the starting frame + /// If present, we're doing image-to-video + /// If absent, we're doing text-to-video + pub start_frame_image_media_token: Option, + + /// Optional. + /// Only for image-to-video + /// Source of the ending frame (if desired) + pub end_frame_image_media_token: Option, + + /// Optional + /// Duration of the video + pub duration: Option, + + /// Optional. + /// Only for text-to-video + pub aspect_ratio: Option, +} + +#[derive(Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum Kling2p5TurboProMultiFunctionVideoGenDuration { + FiveSeconds, + TenSeconds, +} + +#[derive(Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum Kling2p5TurboProMultiFunctionVideoGenAspectRatio { + Square, + SixteenByNine, + NineBySixteen, +} + + +#[derive(Serialize, Deserialize, ToSchema)] +pub struct Kling2p5TurboProMultiFunctionVideoGenResponse { + pub success: bool, + pub inference_job_token: InferenceJobToken, +} diff --git a/crates/api_clients/artcraft/artcraft_api_defs/src/generate/video/multi_function/kling_2_6_multi_function_video_gen.rs b/crates/api_clients/artcraft/artcraft_api_defs/src/generate/video/multi_function/kling_2_6_multi_function_video_gen.rs new file mode 100644 index 000000000..164fa6de1 --- /dev/null +++ b/crates/api_clients/artcraft/artcraft_api_defs/src/generate/video/multi_function/kling_2_6_multi_function_video_gen.rs @@ -0,0 +1,68 @@ +use serde::{Deserialize, Serialize}; +use tokens::tokens::generic_inference_jobs::InferenceJobToken; +use tokens::tokens::media_files::MediaFileToken; +use utoipa::ToSchema; + +pub const KLING_2P6_PRO_MULTI_FUNCTION_VIDEO_VIDEO_PATH: &str = "/v1/generate/video/multi_function/kling_2p6_pro"; + +/// Both text-to-video and image-to-video in one request. +#[derive(Serialize, Deserialize, ToSchema)] +pub struct Kling2p6ProMultiFunctionVideoGenRequest { + /// Idempotency token to prevent duplicate requests. + pub uuid_idempotency_token: String, + + /// "Required". + /// Required downstream, but we'll coerce null to empty string. + /// Text prompt. + pub prompt: Option, + + /// Optional. + /// Only for text-to-video. + pub negative_prompt: Option, + + /// Optional. + /// Only for image-to-video + /// Source of the starting frame + /// If present, we're doing image-to-video + /// If absent, we're doing text-to-video + pub start_frame_image_media_token: Option, + + // /// Optional. + // /// Only for image-to-video + // /// Source of the ending frame (if desired) + // pub end_frame_image_media_token: Option, + + /// Optional + /// Duration of the video + pub duration: Option, + + /// Optional + /// Whether to generate audio. + pub generate_audio: Option, + + /// Optional. + /// Only for text-to-video + pub aspect_ratio: Option, +} + +#[derive(Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum Kling2p6ProMultiFunctionVideoGenDuration { + FiveSeconds, + TenSeconds, +} + +#[derive(Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum Kling2p6ProMultiFunctionVideoGenAspectRatio { + Square, + SixteenByNine, + NineBySixteen, +} + + +#[derive(Serialize, Deserialize, ToSchema)] +pub struct Kling2p6ProMultiFunctionVideoGenResponse { + pub success: bool, + pub inference_job_token: InferenceJobToken, +} diff --git a/crates/api_clients/artcraft/artcraft_api_defs/src/generate/video/multi_function/mod.rs b/crates/api_clients/artcraft/artcraft_api_defs/src/generate/video/multi_function/mod.rs new file mode 100644 index 000000000..faf3e5d7c --- /dev/null +++ b/crates/api_clients/artcraft/artcraft_api_defs/src/generate/video/multi_function/mod.rs @@ -0,0 +1,6 @@ +pub mod kling_2_5_turbo_multi_function_video_gen; +pub mod kling_2_6_multi_function_video_gen; +pub mod sora_2_multi_function_video_gen; +pub mod sora_2_pro_multi_function_video_gen; +pub mod veo_3p1_fast_multi_function_video_gen; +pub mod veo_3p1_multi_function_video_gen; \ No newline at end of file diff --git a/crates/api_clients/artcraft/artcraft_api_defs/src/generate/video/multi_function/sora_2_multi_function_video_gen.rs b/crates/api_clients/artcraft/artcraft_api_defs/src/generate/video/multi_function/sora_2_multi_function_video_gen.rs new file mode 100644 index 000000000..85fbfe373 --- /dev/null +++ b/crates/api_clients/artcraft/artcraft_api_defs/src/generate/video/multi_function/sora_2_multi_function_video_gen.rs @@ -0,0 +1,66 @@ +use serde::{Deserialize, Serialize}; +use tokens::tokens::generic_inference_jobs::InferenceJobToken; +use tokens::tokens::media_files::MediaFileToken; +use utoipa::ToSchema; + +pub const SORA_2_MULTI_FUNCTION_VIDEO_VIDEO_PATH: &str = "/v1/generate/video/multi_function/sora_2"; + +/// Both text-to-video and image-to-video in one request. +#[derive(Serialize, Deserialize, ToSchema)] +pub struct Sora2MultiFunctionVideoGenRequest { + /// Idempotency token to prevent duplicate requests. + pub uuid_idempotency_token: String, + + /// "Required". + /// Required downstream, but we'll coerce null to empty string. + /// Text prompt. + pub prompt: Option, + + /// Optional. + /// Only for image-to-video + /// Source of the starting frame + /// If present, we're doing image-to-video + /// If absent, we're doing text-to-video + pub image_media_token: Option, + + /// Optional. + pub resolution: Option, + + /// Optional + /// Duration of the video + /// (this is uniform against all modes) + pub duration: Option, + + /// Optional. + pub aspect_ratio: Option, +} + +#[derive(Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum Sora2MultiFunctionVideoGenResolution { + Auto, + SevenTwentyP, +} + +#[derive(Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum Sora2MultiFunctionVideoGenDuration { + FourSeconds, + EightSeconds, + TwelveSeconds, +} + +#[derive(Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum Sora2MultiFunctionVideoGenAspectRatio { + Auto, + NineBySixteen, + SixteenByNine, +} + + +#[derive(Serialize, Deserialize, ToSchema)] +pub struct Sora2MultiFunctionVideoGenResponse { + pub success: bool, + pub inference_job_token: InferenceJobToken, +} diff --git a/crates/api_clients/artcraft/artcraft_api_defs/src/generate/video/multi_function/sora_2_pro_multi_function_video_gen.rs b/crates/api_clients/artcraft/artcraft_api_defs/src/generate/video/multi_function/sora_2_pro_multi_function_video_gen.rs new file mode 100644 index 000000000..44e69b7bd --- /dev/null +++ b/crates/api_clients/artcraft/artcraft_api_defs/src/generate/video/multi_function/sora_2_pro_multi_function_video_gen.rs @@ -0,0 +1,67 @@ +use serde::{Deserialize, Serialize}; +use tokens::tokens::generic_inference_jobs::InferenceJobToken; +use tokens::tokens::media_files::MediaFileToken; +use utoipa::ToSchema; + +pub const SORA_2_PRO_MULTI_FUNCTION_VIDEO_VIDEO_PATH: &str = "/v1/generate/video/multi_function/sora_2_pro"; + +/// Both text-to-video and image-to-video in one request. +#[derive(Serialize, Deserialize, ToSchema)] +pub struct Sora2ProMultiFunctionVideoGenRequest { + /// Idempotency token to prevent duplicate requests. + pub uuid_idempotency_token: String, + + /// "Required". + /// Required downstream, but we'll coerce null to empty string. + /// Text prompt. + pub prompt: Option, + + /// Optional. + /// Only for image-to-video + /// Source of the starting frame + /// If present, we're doing image-to-video + /// If absent, we're doing text-to-video + pub image_media_token: Option, + + /// Optional. + pub resolution: Option, + + /// Optional + /// Duration of the video + /// (this is uniform against all modes) + pub duration: Option, + + /// Optional. + pub aspect_ratio: Option, +} + +#[derive(Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum Sora2ProMultiFunctionVideoGenResolution { + Auto, + SevenTwentyP, + TenEightyP, +} + +#[derive(Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum Sora2ProMultiFunctionVideoGenDuration { + FourSeconds, + EightSeconds, + TwelveSeconds, +} + +#[derive(Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum Sora2ProMultiFunctionVideoGenAspectRatio { + Auto, + NineBySixteen, + SixteenByNine, +} + + +#[derive(Serialize, Deserialize, ToSchema)] +pub struct Sora2ProMultiFunctionVideoGenResponse { + pub success: bool, + pub inference_job_token: InferenceJobToken, +} diff --git a/crates/api_clients/artcraft/artcraft_api_defs/src/generate/video/multi_function/veo_3p1_fast_multi_function_video_gen.rs b/crates/api_clients/artcraft/artcraft_api_defs/src/generate/video/multi_function/veo_3p1_fast_multi_function_video_gen.rs new file mode 100644 index 000000000..f3ac22b74 --- /dev/null +++ b/crates/api_clients/artcraft/artcraft_api_defs/src/generate/video/multi_function/veo_3p1_fast_multi_function_video_gen.rs @@ -0,0 +1,90 @@ +use serde::{Deserialize, Serialize}; +use tokens::tokens::generic_inference_jobs::InferenceJobToken; +use tokens::tokens::media_files::MediaFileToken; +use utoipa::ToSchema; + +pub const VEO_3P1_FAST_MULTI_FUNCTION_VIDEO_VIDEO_PATH: &str = "/v1/generate/video/multi_function/veo_3p1_fast"; + +/// Both text-to-video and image-to-video in one request. +#[derive(Serialize, Deserialize, ToSchema)] +pub struct Veo3p1FastMultiFunctionVideoGenRequest { + /// Idempotency token to prevent duplicate requests. + pub uuid_idempotency_token: String, + + /// "Required". + /// Required downstream, but we'll coerce null to empty string. + /// Text prompt. + pub prompt: Option, + + /// Optional. + /// Only for text-to-video. + pub negative_prompt: Option, + + /// Optional. + /// Only for image-to-video + /// Source of the starting frame + /// If present, we're doing image-to-video + /// If absent, we're doing text-to-video + pub start_frame_image_media_token: Option, + + /// Optional. + /// Only for image-to-video + /// Source of the ending frame (if desired) + pub end_frame_image_media_token: Option, + + /// Optional + /// Duration of the video + /// (this is uniform against all modes) + pub duration: Option, + + /// Optional. + pub aspect_ratio: Option, + + /// Optional. + pub resolution: Option, + + /// Optional + /// Whether to generate audio. + pub generate_audio: Option, + + /// Optional + /// Only for text-to-video + pub enhance_prompt: Option, + + /// Optional + /// Only for text-to-video + pub seed: Option, + + /// Optional + /// Only for text-to-video + pub auto_fix: Option, +} + +#[derive(Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum Veo3p1FastMultiFunctionVideoGenDuration { + FourSeconds, + SixSeconds, + EightSeconds, +} + +#[derive(Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum Veo3p1FastMultiFunctionVideoGenAspectRatio { + Auto, + SixteenByNine, + NineBySixteen, +} + +#[derive(Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum Veo3p1FastMultiFunctionVideoGenResolution { + SevenTwentyP, + TenEightyP, +} + +#[derive(Serialize, Deserialize, ToSchema)] +pub struct Veo3p1FastMultiFunctionVideoGenResponse { + pub success: bool, + pub inference_job_token: InferenceJobToken, +} diff --git a/crates/api_clients/artcraft/artcraft_api_defs/src/generate/video/multi_function/veo_3p1_multi_function_video_gen.rs b/crates/api_clients/artcraft/artcraft_api_defs/src/generate/video/multi_function/veo_3p1_multi_function_video_gen.rs new file mode 100644 index 000000000..dfe622d6f --- /dev/null +++ b/crates/api_clients/artcraft/artcraft_api_defs/src/generate/video/multi_function/veo_3p1_multi_function_video_gen.rs @@ -0,0 +1,90 @@ +use serde::{Deserialize, Serialize}; +use tokens::tokens::generic_inference_jobs::InferenceJobToken; +use tokens::tokens::media_files::MediaFileToken; +use utoipa::ToSchema; + +pub const VEO_3P1_MULTI_FUNCTION_VIDEO_VIDEO_PATH: &str = "/v1/generate/video/multi_function/veo_3p1"; + +/// Both text-to-video and image-to-video in one request. +#[derive(Serialize, Deserialize, ToSchema)] +pub struct Veo3p1MultiFunctionVideoGenRequest { + /// Idempotency token to prevent duplicate requests. + pub uuid_idempotency_token: String, + + /// "Required". + /// Required downstream, but we'll coerce null to empty string. + /// Text prompt. + pub prompt: Option, + + /// Optional. + /// Only for text-to-video. + pub negative_prompt: Option, + + /// Optional. + /// Only for image-to-video + /// Source of the starting frame + /// If present, we're doing image-to-video + /// If absent, we're doing text-to-video + pub start_frame_image_media_token: Option, + + /// Optional. + /// Only for image-to-video + /// Source of the ending frame (if desired) + pub end_frame_image_media_token: Option, + + /// Optional + /// Duration of the video + /// (this is uniform against all modes) + pub duration: Option, + + /// Optional. + pub aspect_ratio: Option, + + /// Optional. + pub resolution: Option, + + /// Optional + /// Whether to generate audio. + pub generate_audio: Option, + + /// Optional + /// Only for text-to-video + pub enhance_prompt: Option, + + /// Optional + /// Only for text-to-video + pub seed: Option, + + /// Optional + /// Only for text-to-video + pub auto_fix: Option, +} + +#[derive(Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum Veo3p1MultiFunctionVideoGenDuration { + FourSeconds, + SixSeconds, + EightSeconds, +} + +#[derive(Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum Veo3p1MultiFunctionVideoGenAspectRatio { + Auto, + SixteenByNine, + NineBySixteen, +} + +#[derive(Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum Veo3p1MultiFunctionVideoGenResolution { + SevenTwentyP, + TenEightyP, +} + +#[derive(Serialize, Deserialize, ToSchema)] +pub struct Veo3p1MultiFunctionVideoGenResponse { + pub success: bool, + pub inference_job_token: InferenceJobToken, +} diff --git a/crates/api_clients/artcraft/storyteller_client/src/endpoints/generate/video/mod.rs b/crates/api_clients/artcraft/storyteller_client/src/endpoints/generate/video/mod.rs index ab29e4f1e..28a28bdbb 100644 --- a/crates/api_clients/artcraft/storyteller_client/src/endpoints/generate/video/mod.rs +++ b/crates/api_clients/artcraft/storyteller_client/src/endpoints/generate/video/mod.rs @@ -5,3 +5,4 @@ pub mod generate_seedance_1_0_lite_image_to_video; pub mod generate_veo_2_image_to_video; pub mod generate_veo_3_fast_image_to_video; pub mod generate_veo_3_image_to_video; +pub mod multi_function; diff --git a/crates/api_clients/artcraft/storyteller_client/src/endpoints/generate/video/multi_function/kling_2p5_turbo_pro_multi_function_video_gen.rs b/crates/api_clients/artcraft/storyteller_client/src/endpoints/generate/video/multi_function/kling_2p5_turbo_pro_multi_function_video_gen.rs new file mode 100644 index 000000000..82536540c --- /dev/null +++ b/crates/api_clients/artcraft/storyteller_client/src/endpoints/generate/video/multi_function/kling_2p5_turbo_pro_multi_function_video_gen.rs @@ -0,0 +1,18 @@ +use crate::credentials::storyteller_credential_set::StorytellerCredentialSet; +use crate::error::storyteller_error::StorytellerError; +use crate::utils::api_host::ApiHost; +use crate::utils::basic_json_post_request::basic_json_post_request; +use artcraft_api_defs::generate::video::multi_function::kling_2_5_turbo_multi_function_video_gen::{Kling2p5TurboProMultiFunctionVideoGenRequest, Kling2p5TurboProMultiFunctionVideoGenResponse, KLING_2P5_TURBO_PRO_MULTI_FUNCTION_VIDEO_VIDEO_PATH}; + +pub async fn kling_2p5_turbo_pro_multi_function_image_gen( + api_host: &ApiHost, + maybe_creds: Option<&StorytellerCredentialSet>, + request: Kling2p5TurboProMultiFunctionVideoGenRequest, +) -> Result { + Ok(basic_json_post_request( + api_host, + KLING_2P5_TURBO_PRO_MULTI_FUNCTION_VIDEO_VIDEO_PATH, + maybe_creds, + request, + ).await?) +} diff --git a/crates/api_clients/artcraft/storyteller_client/src/endpoints/generate/video/multi_function/kling_2p6_pro_multi_function_video_gen.rs b/crates/api_clients/artcraft/storyteller_client/src/endpoints/generate/video/multi_function/kling_2p6_pro_multi_function_video_gen.rs new file mode 100644 index 000000000..520efd06c --- /dev/null +++ b/crates/api_clients/artcraft/storyteller_client/src/endpoints/generate/video/multi_function/kling_2p6_pro_multi_function_video_gen.rs @@ -0,0 +1,18 @@ +use crate::credentials::storyteller_credential_set::StorytellerCredentialSet; +use crate::error::storyteller_error::StorytellerError; +use crate::utils::api_host::ApiHost; +use crate::utils::basic_json_post_request::basic_json_post_request; +use artcraft_api_defs::generate::video::multi_function::kling_2_6_multi_function_video_gen::{Kling2p6ProMultiFunctionVideoGenRequest, Kling2p6ProMultiFunctionVideoGenResponse, KLING_2P6_PRO_MULTI_FUNCTION_VIDEO_VIDEO_PATH}; + +pub async fn kling_2p6_pro_multi_function_image_gen( + api_host: &ApiHost, + maybe_creds: Option<&StorytellerCredentialSet>, + request: Kling2p6ProMultiFunctionVideoGenRequest, +) -> Result { + Ok(basic_json_post_request( + api_host, + KLING_2P6_PRO_MULTI_FUNCTION_VIDEO_VIDEO_PATH, + maybe_creds, + request, + ).await?) +} diff --git a/crates/api_clients/artcraft/storyteller_client/src/endpoints/generate/video/multi_function/mod.rs b/crates/api_clients/artcraft/storyteller_client/src/endpoints/generate/video/multi_function/mod.rs new file mode 100644 index 000000000..6c8d7eea9 --- /dev/null +++ b/crates/api_clients/artcraft/storyteller_client/src/endpoints/generate/video/multi_function/mod.rs @@ -0,0 +1,6 @@ +pub mod kling_2p5_turbo_pro_multi_function_video_gen; +pub mod kling_2p6_pro_multi_function_video_gen; +pub mod sora_2_multi_function_video_gen; +pub mod sora_2_pro_multi_function_video_gen; +pub mod veo_3p1_fast_multi_function_video_gen; +pub mod veo_3p1_multi_function_video_gen; \ No newline at end of file diff --git a/crates/api_clients/artcraft/storyteller_client/src/endpoints/generate/video/multi_function/sora_2_multi_function_video_gen.rs b/crates/api_clients/artcraft/storyteller_client/src/endpoints/generate/video/multi_function/sora_2_multi_function_video_gen.rs new file mode 100644 index 000000000..607c8f1e6 --- /dev/null +++ b/crates/api_clients/artcraft/storyteller_client/src/endpoints/generate/video/multi_function/sora_2_multi_function_video_gen.rs @@ -0,0 +1,18 @@ +use crate::credentials::storyteller_credential_set::StorytellerCredentialSet; +use crate::error::storyteller_error::StorytellerError; +use crate::utils::api_host::ApiHost; +use crate::utils::basic_json_post_request::basic_json_post_request; +use artcraft_api_defs::generate::video::multi_function::sora_2_multi_function_video_gen::{Sora2MultiFunctionVideoGenRequest, Sora2MultiFunctionVideoGenResponse, SORA_2_MULTI_FUNCTION_VIDEO_VIDEO_PATH}; + +pub async fn sora_2_multi_function_image_gen( + api_host: &ApiHost, + maybe_creds: Option<&StorytellerCredentialSet>, + request: Sora2MultiFunctionVideoGenRequest, +) -> Result { + Ok(basic_json_post_request( + api_host, + SORA_2_MULTI_FUNCTION_VIDEO_VIDEO_PATH, + maybe_creds, + request, + ).await?) +} diff --git a/crates/api_clients/artcraft/storyteller_client/src/endpoints/generate/video/multi_function/sora_2_pro_multi_function_video_gen.rs b/crates/api_clients/artcraft/storyteller_client/src/endpoints/generate/video/multi_function/sora_2_pro_multi_function_video_gen.rs new file mode 100644 index 000000000..1394898dc --- /dev/null +++ b/crates/api_clients/artcraft/storyteller_client/src/endpoints/generate/video/multi_function/sora_2_pro_multi_function_video_gen.rs @@ -0,0 +1,18 @@ +use crate::credentials::storyteller_credential_set::StorytellerCredentialSet; +use crate::error::storyteller_error::StorytellerError; +use crate::utils::api_host::ApiHost; +use crate::utils::basic_json_post_request::basic_json_post_request; +use artcraft_api_defs::generate::video::multi_function::sora_2_pro_multi_function_video_gen::{Sora2ProMultiFunctionVideoGenRequest, Sora2ProMultiFunctionVideoGenResponse, SORA_2_PRO_MULTI_FUNCTION_VIDEO_VIDEO_PATH}; + +pub async fn sora_2_fast_multi_function_image_gen( + api_host: &ApiHost, + maybe_creds: Option<&StorytellerCredentialSet>, + request: Sora2ProMultiFunctionVideoGenRequest, +) -> Result { + Ok(basic_json_post_request( + api_host, + SORA_2_PRO_MULTI_FUNCTION_VIDEO_VIDEO_PATH, + maybe_creds, + request, + ).await?) +} diff --git a/crates/api_clients/artcraft/storyteller_client/src/endpoints/generate/video/multi_function/veo_3p1_fast_multi_function_video_gen.rs b/crates/api_clients/artcraft/storyteller_client/src/endpoints/generate/video/multi_function/veo_3p1_fast_multi_function_video_gen.rs new file mode 100644 index 000000000..d261aa77d --- /dev/null +++ b/crates/api_clients/artcraft/storyteller_client/src/endpoints/generate/video/multi_function/veo_3p1_fast_multi_function_video_gen.rs @@ -0,0 +1,18 @@ +use crate::credentials::storyteller_credential_set::StorytellerCredentialSet; +use crate::error::storyteller_error::StorytellerError; +use crate::utils::api_host::ApiHost; +use crate::utils::basic_json_post_request::basic_json_post_request; +use artcraft_api_defs::generate::video::multi_function::veo_3p1_fast_multi_function_video_gen::{Veo3p1FastMultiFunctionVideoGenRequest, Veo3p1FastMultiFunctionVideoGenResponse, VEO_3P1_FAST_MULTI_FUNCTION_VIDEO_VIDEO_PATH}; + +pub async fn veo_3p1_fast_multi_function_image_gen( + api_host: &ApiHost, + maybe_creds: Option<&StorytellerCredentialSet>, + request: Veo3p1FastMultiFunctionVideoGenRequest, +) -> Result { + Ok(basic_json_post_request( + api_host, + VEO_3P1_FAST_MULTI_FUNCTION_VIDEO_VIDEO_PATH, + maybe_creds, + request, + ).await?) +} diff --git a/crates/api_clients/artcraft/storyteller_client/src/endpoints/generate/video/multi_function/veo_3p1_multi_function_video_gen.rs b/crates/api_clients/artcraft/storyteller_client/src/endpoints/generate/video/multi_function/veo_3p1_multi_function_video_gen.rs new file mode 100644 index 000000000..062ab20c1 --- /dev/null +++ b/crates/api_clients/artcraft/storyteller_client/src/endpoints/generate/video/multi_function/veo_3p1_multi_function_video_gen.rs @@ -0,0 +1,18 @@ +use crate::credentials::storyteller_credential_set::StorytellerCredentialSet; +use crate::error::storyteller_error::StorytellerError; +use crate::utils::api_host::ApiHost; +use crate::utils::basic_json_post_request::basic_json_post_request; +use artcraft_api_defs::generate::video::multi_function::veo_3p1_multi_function_video_gen::{Veo3p1MultiFunctionVideoGenRequest, Veo3p1MultiFunctionVideoGenResponse, VEO_3P1_MULTI_FUNCTION_VIDEO_VIDEO_PATH}; + +pub async fn veo_3p1_multi_function_image_gen( + api_host: &ApiHost, + maybe_creds: Option<&StorytellerCredentialSet>, + request: Veo3p1MultiFunctionVideoGenRequest, +) -> Result { + Ok(basic_json_post_request( + api_host, + VEO_3P1_MULTI_FUNCTION_VIDEO_VIDEO_PATH, + maybe_creds, + request, + ).await?) +} diff --git a/crates/api_clients/fal_client/src/requests/webhook/video/image/enqueue_veo_3p1_fast_first_last_frame_image_to_video_webhook.rs b/crates/api_clients/fal_client/src/requests/webhook/video/image/enqueue_veo_3p1_fast_first_last_frame_image_to_video_webhook.rs index 015bf5e07..68f195dce 100644 --- a/crates/api_clients/fal_client/src/requests/webhook/video/image/enqueue_veo_3p1_fast_first_last_frame_image_to_video_webhook.rs +++ b/crates/api_clients/fal_client/src/requests/webhook/video/image/enqueue_veo_3p1_fast_first_last_frame_image_to_video_webhook.rs @@ -36,7 +36,6 @@ pub enum EnqueueVeo3p1FastFirstLastFrameImageToVideoDurationSeconds { #[derive(Copy, Clone, Debug)] pub enum EnqueueVeo3p1FastFirstLastFrameImageToVideoAspectRatio { Auto, - Square, SixteenByNine, NineBySixteen, } @@ -62,7 +61,6 @@ pub async fn enqueue_veo_3p1_fast_first_last_frame_image_to_video_webhook "auto", - EnqueueVeo3p1FastFirstLastFrameImageToVideoAspectRatio::Square => "1:1", EnqueueVeo3p1FastFirstLastFrameImageToVideoAspectRatio::SixteenByNine => "16:9", EnqueueVeo3p1FastFirstLastFrameImageToVideoAspectRatio::NineBySixteen => "9:16", }) diff --git a/crates/api_clients/fal_client/src/requests/webhook/video/image/enqueue_veo_3p1_fast_image_to_video_webhook.rs b/crates/api_clients/fal_client/src/requests/webhook/video/image/enqueue_veo_3p1_fast_image_to_video_webhook.rs index 3d135953c..6a9785047 100644 --- a/crates/api_clients/fal_client/src/requests/webhook/video/image/enqueue_veo_3p1_fast_image_to_video_webhook.rs +++ b/crates/api_clients/fal_client/src/requests/webhook/video/image/enqueue_veo_3p1_fast_image_to_video_webhook.rs @@ -33,7 +33,6 @@ pub enum EnqueueVeo3p1FastImageToVideoDurationSeconds { #[derive(Copy, Clone, Debug)] pub enum EnqueueVeo3p1FastImageToVideoAspectRatio { Auto, - Square, SixteenByNine, NineBySixteen, } @@ -59,7 +58,6 @@ pub async fn enqueue_veo_3p1_fast_image_to_video_webhook( let aspect_ratio = args.aspect_ratio .map(|aspect_ratio| match aspect_ratio { EnqueueVeo3p1FastImageToVideoAspectRatio::Auto => "auto", - EnqueueVeo3p1FastImageToVideoAspectRatio::Square => "1:1", EnqueueVeo3p1FastImageToVideoAspectRatio::SixteenByNine => "16:9", EnqueueVeo3p1FastImageToVideoAspectRatio::NineBySixteen => "9:16", }) diff --git a/crates/api_clients/fal_client/src/requests/webhook/video/image/enqueue_veo_3p1_first_last_frame_image_to_video_webhook.rs b/crates/api_clients/fal_client/src/requests/webhook/video/image/enqueue_veo_3p1_first_last_frame_image_to_video_webhook.rs index ee188ad6f..948219c1f 100644 --- a/crates/api_clients/fal_client/src/requests/webhook/video/image/enqueue_veo_3p1_first_last_frame_image_to_video_webhook.rs +++ b/crates/api_clients/fal_client/src/requests/webhook/video/image/enqueue_veo_3p1_first_last_frame_image_to_video_webhook.rs @@ -36,7 +36,6 @@ pub enum EnqueueVeo3p1FirstLastFrameImageToVideoDurationSeconds { #[derive(Copy, Clone, Debug)] pub enum EnqueueVeo3p1FirstLastFrameImageToVideoAspectRatio { Auto, - Square, SixteenByNine, NineBySixteen, } @@ -62,7 +61,6 @@ pub async fn enqueue_veo_3p1_first_last_frame_image_to_video_webhook let aspect_ratio = args.aspect_ratio .map(|aspect_ratio| match aspect_ratio { EnqueueVeo3p1FirstLastFrameImageToVideoAspectRatio::Auto => "auto", - EnqueueVeo3p1FirstLastFrameImageToVideoAspectRatio::Square => "1:1", EnqueueVeo3p1FirstLastFrameImageToVideoAspectRatio::SixteenByNine => "16:9", EnqueueVeo3p1FirstLastFrameImageToVideoAspectRatio::NineBySixteen => "9:16", }) diff --git a/crates/api_clients/fal_client/src/requests/webhook/video/image/enqueue_veo_3p1_image_to_video_webhook.rs b/crates/api_clients/fal_client/src/requests/webhook/video/image/enqueue_veo_3p1_image_to_video_webhook.rs index cd49031d6..1ff41c3e2 100644 --- a/crates/api_clients/fal_client/src/requests/webhook/video/image/enqueue_veo_3p1_image_to_video_webhook.rs +++ b/crates/api_clients/fal_client/src/requests/webhook/video/image/enqueue_veo_3p1_image_to_video_webhook.rs @@ -33,7 +33,6 @@ pub enum EnqueueVeo3p1ImageToVideoDurationSeconds { #[derive(Copy, Clone, Debug)] pub enum EnqueueVeo3p1ImageToVideoAspectRatio { Auto, - Square, SixteenByNine, NineBySixteen, } @@ -59,7 +58,6 @@ pub async fn enqueue_veo_3p1_image_to_video_webhook( let aspect_ratio = args.aspect_ratio .map(|aspect_ratio| match aspect_ratio { EnqueueVeo3p1ImageToVideoAspectRatio::Auto => "auto", - EnqueueVeo3p1ImageToVideoAspectRatio::Square => "1:1", EnqueueVeo3p1ImageToVideoAspectRatio::SixteenByNine => "16:9", EnqueueVeo3p1ImageToVideoAspectRatio::NineBySixteen => "9:16", }) diff --git a/crates/api_clients/fal_client/src/requests/webhook/video/text/enqueue_veo_3p1_fast_text_to_video_webhook.rs b/crates/api_clients/fal_client/src/requests/webhook/video/text/enqueue_veo_3p1_fast_text_to_video_webhook.rs index 5a069bdca..74d31345c 100644 --- a/crates/api_clients/fal_client/src/requests/webhook/video/text/enqueue_veo_3p1_fast_text_to_video_webhook.rs +++ b/crates/api_clients/fal_client/src/requests/webhook/video/text/enqueue_veo_3p1_fast_text_to_video_webhook.rs @@ -34,7 +34,6 @@ pub enum EnqueueVeo3p1FastTextToVideoDurationSeconds { #[derive(Copy, Clone, Debug)] pub enum EnqueueVeo3p1FastTextToVideoAspectRatio { Auto, - Square, SixteenByNine, NineBySixteen, } @@ -60,7 +59,6 @@ pub async fn enqueue_veo_3p1_fast_text_to_video_webhook( let aspect_ratio = args.aspect_ratio .map(|aspect_ratio| match aspect_ratio { EnqueueVeo3p1FastTextToVideoAspectRatio::Auto => "auto", - EnqueueVeo3p1FastTextToVideoAspectRatio::Square => "1:1", EnqueueVeo3p1FastTextToVideoAspectRatio::SixteenByNine => "16:9", EnqueueVeo3p1FastTextToVideoAspectRatio::NineBySixteen => "9:16", }) diff --git a/crates/api_clients/fal_client/src/requests/webhook/video/text/enqueue_veo_3p1_text_to_video_webhook.rs b/crates/api_clients/fal_client/src/requests/webhook/video/text/enqueue_veo_3p1_text_to_video_webhook.rs index 676d6e832..d8a839da0 100644 --- a/crates/api_clients/fal_client/src/requests/webhook/video/text/enqueue_veo_3p1_text_to_video_webhook.rs +++ b/crates/api_clients/fal_client/src/requests/webhook/video/text/enqueue_veo_3p1_text_to_video_webhook.rs @@ -34,7 +34,6 @@ pub enum EnqueueVeo3p1TextToVideoDurationSeconds { #[derive(Copy, Clone, Debug)] pub enum EnqueueVeo3p1TextToVideoAspectRatio { Auto, - Square, SixteenByNine, NineBySixteen, } @@ -60,7 +59,6 @@ pub async fn enqueue_veo_3p1_text_to_video_webhook( let aspect_ratio = args.aspect_ratio .map(|aspect_ratio| match aspect_ratio { EnqueueVeo3p1TextToVideoAspectRatio::Auto => "auto", - EnqueueVeo3p1TextToVideoAspectRatio::Square => "1:1", EnqueueVeo3p1TextToVideoAspectRatio::SixteenByNine => "16:9", EnqueueVeo3p1TextToVideoAspectRatio::NineBySixteen => "9:16", }) diff --git a/crates/schema/public/enums/src/common/model_type.rs b/crates/schema/public/enums/src/common/model_type.rs index 90809dd27..648f6ec0c 100644 --- a/crates/schema/public/enums/src/common/model_type.rs +++ b/crates/schema/public/enums/src/common/model_type.rs @@ -85,18 +85,28 @@ pub enum ModelType { Kling21Pro, #[serde(rename = "kling_2p1_master")] Kling21Master, + #[serde(rename = "kling_2p5_turbo_pro")] + Kling2p5TurboPro, + #[serde(rename = "kling_2p6_pro")] + Kling2p6Pro, #[serde(rename = "seedance_1p0_lite")] Seedance10Lite, #[serde(rename = "seedance_1p0_pro")] Seedance10Pro, #[serde(rename = "sora_2")] Sora2, + #[serde(rename = "sora_2_pro")] + Sora2Pro, #[serde(rename = "veo_2")] Veo2, #[serde(rename = "veo_3")] Veo3, #[serde(rename = "veo_3_fast")] Veo3Fast, + #[serde(rename = "veo_3p1")] + Veo3p1, + #[serde(rename = "veo_3p1_fast")] + Veo3p1Fast, // 3D Object generation models #[serde(rename = "hunyuan_3d_2p0")] @@ -147,12 +157,17 @@ impl ModelType { Self::Kling16Pro => "kling_1p6_pro", Self::Kling21Pro => "kling_2p1_pro", Self::Kling21Master => "kling_2p1_master", + Self::Kling2p5TurboPro => "kling_2p5_turbo_pro", + Self::Kling2p6Pro => "kling_2p6_pro", Self::Seedance10Lite => "seedance_1p0_lite", Self::Seedance10Pro => "seedance_1p0_pro", Self::Sora2 => "sora_2", + Self::Sora2Pro => "sora_2_pro", Self::Veo2 => "veo_2", Self::Veo3 => "veo_3", Self::Veo3Fast => "veo_3_fast", + Self::Veo3p1 => "veo_3p1", + Self::Veo3p1Fast => "veo_3p1_fast", // 3D Object generation models Self::Hunyuan3d2_0 => "hunyuan_3d_2p0", @@ -195,12 +210,17 @@ impl ModelType { "kling_1p6_pro" => Ok(Self::Kling16Pro), "kling_2p1_pro" => Ok(Self::Kling21Pro), "kling_2p1_master" => Ok(Self::Kling21Master), + "kling_2p5_turbo_pro" => Ok(Self::Kling2p5TurboPro), + "kling_2p6_pro" => Ok(Self::Kling2p6Pro), "seedance_1p0_lite" => Ok(Self::Seedance10Lite), "seedance_1p0_pro" => Ok(Self::Seedance10Pro), "sora_2" => Ok(Self::Sora2), + "sora_2_pro" => Ok(Self::Sora2Pro), "veo_2" => Ok(Self::Veo2), "veo_3" => Ok(Self::Veo3), "veo_3_fast" => Ok(Self::Veo3Fast), + "veo_3p1" => Ok(Self::Veo3p1), + "veo_3p1_fast" => Ok(Self::Veo3p1Fast), // 3D Object generation models "hunyuan_3d_2p0" => Ok(Self::Hunyuan3d2_0), @@ -247,12 +267,17 @@ impl ModelType { Self::Kling16Pro, Self::Kling21Pro, Self::Kling21Master, + Self::Kling2p5TurboPro, + Self::Kling2p6Pro, Self::Seedance10Lite, Self::Seedance10Pro, Self::Sora2, + Self::Sora2Pro, Self::Veo2, Self::Veo3, Self::Veo3Fast, + Self::Veo3p1, + Self::Veo3p1Fast, // 3D Object generation models Self::Hunyuan3d2_0, @@ -303,12 +328,17 @@ mod tests { assert_serialization(ModelType::Kling16Pro, "kling_1p6_pro"); assert_serialization(ModelType::Kling21Pro, "kling_2p1_pro"); assert_serialization(ModelType::Kling21Master, "kling_2p1_master"); + assert_serialization(ModelType::Kling2p5TurboPro, "kling_2p5_turbo_pro"); + assert_serialization(ModelType::Kling2p6Pro, "kling_2p6_pro"); assert_serialization(ModelType::Seedance10Lite, "seedance_1p0_lite"); assert_serialization(ModelType::Seedance10Pro, "seedance_1p0_pro"); assert_serialization(ModelType::Sora2, "sora_2"); + assert_serialization(ModelType::Sora2Pro, "sora_2_pro"); assert_serialization(ModelType::Veo2, "veo_2"); assert_serialization(ModelType::Veo3, "veo_3"); assert_serialization(ModelType::Veo3Fast, "veo_3_fast"); + assert_serialization(ModelType::Veo3p1, "veo_3p1"); + assert_serialization(ModelType::Veo3p1Fast, "veo_3p1_fast"); // 3D Object generation models assert_serialization(ModelType::Hunyuan3d2_0, "hunyuan_3d_2p0"); assert_serialization(ModelType::Hunyuan3d2_1, "hunyuan_3d_2p1"); @@ -349,12 +379,17 @@ mod tests { assert_eq!(ModelType::Kling16Pro.to_str(), "kling_1p6_pro"); assert_eq!(ModelType::Kling21Pro.to_str(), "kling_2p1_pro"); assert_eq!(ModelType::Kling21Master.to_str(), "kling_2p1_master"); + assert_eq!(ModelType::Kling2p5TurboPro.to_str(), "kling_2p5_turbo_pro"); + assert_eq!(ModelType::Kling2p6Pro.to_str(), "kling_2p6_pro"); assert_eq!(ModelType::Seedance10Lite.to_str(), "seedance_1p0_lite"); assert_eq!(ModelType::Seedance10Pro.to_str(), "seedance_1p0_pro"); assert_eq!(ModelType::Sora2.to_str(), "sora_2"); + assert_eq!(ModelType::Sora2Pro.to_str(), "sora_2_pro"); assert_eq!(ModelType::Veo2.to_str(), "veo_2"); assert_eq!(ModelType::Veo3.to_str(), "veo_3"); assert_eq!(ModelType::Veo3Fast.to_str(), "veo_3_fast"); + assert_eq!(ModelType::Veo3p1.to_str(), "veo_3p1"); + assert_eq!(ModelType::Veo3p1Fast.to_str(), "veo_3p1_fast"); // 3D Object generation models assert_eq!(ModelType::Hunyuan3d2_0.to_str(), "hunyuan_3d_2p0"); @@ -395,12 +430,18 @@ mod tests { assert_eq!(ModelType::from_str("kling_1p6_pro").unwrap(), ModelType::Kling16Pro); assert_eq!(ModelType::from_str("kling_2p1_pro").unwrap(), ModelType::Kling21Pro); assert_eq!(ModelType::from_str("kling_2p1_master").unwrap(), ModelType::Kling21Master); + assert_eq!(ModelType::from_str("kling_2p5_turbo_pro").unwrap(), ModelType::Kling2p5TurboPro); + assert_eq!(ModelType::from_str("kling_2p6_pro").unwrap(), ModelType::Kling2p6Pro); assert_eq!(ModelType::from_str("seedance_1p0_lite").unwrap(), ModelType::Seedance10Lite); assert_eq!(ModelType::from_str("seedance_1p0_pro").unwrap(), ModelType::Seedance10Pro); assert_eq!(ModelType::from_str("sora_2").unwrap(), ModelType::Sora2); + assert_eq!(ModelType::from_str("sora_2_pro").unwrap(), ModelType::Sora2Pro); assert_eq!(ModelType::from_str("veo_2").unwrap(), ModelType::Veo2); assert_eq!(ModelType::from_str("veo_3").unwrap(), ModelType::Veo3); assert_eq!(ModelType::from_str("veo_3_fast").unwrap(), ModelType::Veo3Fast); + assert_eq!(ModelType::from_str("veo_3p1").unwrap(), ModelType::Veo3p1); + assert_eq!(ModelType::from_str("veo_3p1_fast").unwrap(), ModelType::Veo3p1Fast); + // 3D Object generation models assert_eq!(ModelType::from_str("hunyuan_3d_2p0").unwrap(), ModelType::Hunyuan3d2_0); assert_eq!(ModelType::from_str("hunyuan_3d_2p1").unwrap(), ModelType::Hunyuan3d2_1); @@ -409,7 +450,7 @@ mod tests { #[test] fn all_variants() { let mut variants = ModelType::all_variants(); - assert_eq!(variants.len(), 38); + assert_eq!(variants.len(), 43); // Image models assert_eq!(variants.pop_first(), Some(ModelType::Flux1Dev)); assert_eq!(variants.pop_first(), Some(ModelType::Flux1Schnell)); @@ -442,12 +483,17 @@ mod tests { assert_eq!(variants.pop_first(), Some(ModelType::Kling16Pro)); assert_eq!(variants.pop_first(), Some(ModelType::Kling21Pro)); assert_eq!(variants.pop_first(), Some(ModelType::Kling21Master)); + assert_eq!(variants.pop_first(), Some(ModelType::Kling2p5TurboPro)); + assert_eq!(variants.pop_first(), Some(ModelType::Kling2p6Pro)); assert_eq!(variants.pop_first(), Some(ModelType::Seedance10Lite)); assert_eq!(variants.pop_first(), Some(ModelType::Seedance10Pro)); assert_eq!(variants.pop_first(), Some(ModelType::Sora2)); + assert_eq!(variants.pop_first(), Some(ModelType::Sora2Pro)); assert_eq!(variants.pop_first(), Some(ModelType::Veo2)); assert_eq!(variants.pop_first(), Some(ModelType::Veo3)); assert_eq!(variants.pop_first(), Some(ModelType::Veo3Fast)); + assert_eq!(variants.pop_first(), Some(ModelType::Veo3p1)); + assert_eq!(variants.pop_first(), Some(ModelType::Veo3p1Fast)); // 3D Object generation models assert_eq!(variants.pop_first(), Some(ModelType::Hunyuan3d2_0)); assert_eq!(variants.pop_first(), Some(ModelType::Hunyuan3d2_1)); diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/mod.rs index ad5d429ce..5c5501eba 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/mod.rs @@ -5,4 +5,5 @@ pub mod generate_seedance_1_0_lite_image_to_video_handler; pub mod generate_seedance_1_0_pro_image_to_video_handler; pub mod generate_veo_2_image_to_video_handler; pub mod generate_veo_3_fast_image_to_video_handler; -pub mod generate_veo_3_image_to_video_handler; \ No newline at end of file +pub mod generate_veo_3_image_to_video_handler; +pub mod multi_function; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/kling_2p5_turbo_pro_multi_function_video_gen_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/kling_2p5_turbo_pro_multi_function_video_gen_handler.rs new file mode 100644 index 000000000..086f1d5e2 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/kling_2p5_turbo_pro_multi_function_video_gen_handler.rs @@ -0,0 +1,317 @@ +use std::collections::{HashMap, HashSet}; +use std::iter::FromIterator; +use std::sync::Arc; + +use crate::http_server::common_responses::common_web_error::CommonWebError; +use crate::http_server::common_responses::media::media_links_builder::MediaLinksBuilder; +use crate::http_server::endpoints::generate::common::payments_error_test::payments_error_test; +use crate::http_server::endpoints::media_files::helpers::get_media_domain::get_media_domain; +use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format; +use crate::state::server_state::ServerState; +use crate::util::lookup::lookup_image_urls_as_map::lookup_image_urls_as_map; +use actix_web::web::Json; +use actix_web::{web, HttpRequest}; +use artcraft_api_defs::generate::video::multi_function::kling_2_5_turbo_multi_function_video_gen::{Kling2p5TurboProMultiFunctionVideoGenAspectRatio, Kling2p5TurboProMultiFunctionVideoGenDuration, Kling2p5TurboProMultiFunctionVideoGenRequest, Kling2p5TurboProMultiFunctionVideoGenResponse}; +use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; +use enums::by_table::prompt_context_items::prompt_context_semantic_type::PromptContextSemanticType; +use enums::by_table::prompts::prompt_type::PromptType; +use enums::common::generation_provider::GenerationProvider; +use enums::common::model_type::ModelType; +use enums::common::visibility::Visibility; +use fal_client::creds::open_ai_api_key::OpenAiApiKey; +use fal_client::requests::webhook::image::edit::enqueue_gpt_image_1p5_edit_image_webhook::{enqueue_gpt_image_1p5_image_edit_webhook, EnqueueGptImage1p5EditImageArgs, EnqueueGptImage1p5EditImageBackground, EnqueueGptImage1p5EditImageInputFidelity, EnqueueGptImage1p5EditImageNumImages, EnqueueGptImage1p5EditImageQuality, EnqueueGptImage1p5EditImageSize}; +use fal_client::requests::webhook::image::text::enqueue_gpt_image_1p5_text_to_image_webhook::{enqueue_gpt_image_1p5_text_to_image_webhook, EnqueueGptImage1p5TextToImageArgs, EnqueueGptImage1p5TextToImageBackground, EnqueueGptImage1p5TextToImageNumImages, EnqueueGptImage1p5TextToImageQuality, EnqueueGptImage1p5TextToImageSize}; +use fal_client::requests::webhook::video::image::enqueue_kling_v2p5_turbo_pro_image_to_video_webhook::{enqueue_kling_v2p5_turbo_pro_image_to_video_webhook, EnqueueKlingV2p5TurboProImageToVideoArgs, EnqueueKlingV2p5TurboProImageToVideoDurationSeconds}; +use fal_client::requests::webhook::video::text::enqueue_kling_v2p5_turbo_pro_text_to_video_webhook::{enqueue_kling_v2p5_turbo_pro_text_to_video_webhook, EnqueueKlingV2p5TurboProTextToVideoArgs, EnqueueKlingV2p5TurboProTextToVideoAspectRatio, EnqueueKlingV2p5TurboProTextToVideoDurationSeconds}; +use http_server_common::request::get_request_ip::get_request_ip; +use log::{error, info, warn}; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::insert_generic_inference_job_for_fal_queue; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::FalCategory; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::InsertGenericInferenceForFalArgs; +use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token; +use mysql_queries::queries::media_files::get::batch_get_media_files_by_tokens::{batch_get_media_files_by_tokens, batch_get_media_files_by_tokens_with_connection}; +use mysql_queries::queries::prompt_context_items::insert_batch_prompt_context_items::{insert_batch_prompt_context_items, InsertBatchArgs, PromptContextItem}; +use mysql_queries::queries::prompts::insert_prompt::{insert_prompt, InsertPromptArgs}; +use server_environment::ServerEnvironment; +use sqlx::pool::PoolConnection; +use sqlx::{Acquire, MySql}; +use tokens::tokens::media_files::MediaFileToken; +use utoipa::ToSchema; + +/// Kling 2.5 Turbo Pro Multi-Function (text and image to video) +#[utoipa::path( + post, + tag = "Generate Video (Multi-Function)", + path = "/v1/generate/video/multi_function/kling_2p5_turbo_pro", + responses( + (status = 200, description = "Success", body = Kling2p5TurboProMultiFunctionVideoGenResponse), + ), + params( + ("request" = Kling2p5TurboProMultiFunctionVideoGenRequest, description = "Payload for Request"), + ) +)] +pub async fn kling_2p5_turbo_pro_multi_function_video_gen_handler( + http_request: HttpRequest, + request: Json, + server_state: web::Data> +) -> Result, CommonWebError> { + + payments_error_test(&request.prompt.as_deref().unwrap_or(""))?; + + let mut mysql_connection = server_state.mysql_pool + .acquire() + .await?; + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session_from_connection(&http_request, &mut mysql_connection) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + CommonWebError::ServerError + })?; + + let maybe_avt_token = server_state + .avt_cookie_manager + .get_avt_token_from_request(&http_request); + + // TODO: Limit usage for new accounts. Billing, free credits metering, etc. + + //let user_session = match maybe_user_session { + // Some(session) => session, + // None => { + // warn!("not logged in"); + // return Err(CommonWebError::NotAuthorized); + // } + //}; + + if let Err(reason) = validate_idempotency_token_format(&request.uuid_idempotency_token) { + return Err(CommonWebError::BadInputWithSimpleMessage(reason)); + } + + let mut query_media_tokens = None; + + if let Some(start_frame_token) = request.start_frame_image_media_token.as_ref() { + let mut tokens = Vec::new(); + tokens.push(start_frame_token.to_owned()); + + if let Some(end_frame_token) = request.end_frame_image_media_token.as_ref() { + // NB: Masks are only relevant when this is an image editing exercise. + tokens.push(end_frame_token.to_owned()); + } + + query_media_tokens = Some(tokens); + } + + let image_urls_by_token = match query_media_tokens { + None => HashMap::new(), + Some(media_tokens) => { + info!("Looking up image media tokens: {:?}", media_tokens); + lookup_image_urls_as_map( + &http_request, + &mut mysql_connection, + server_state.server_environment, + &media_tokens, + ).await? + } + }; + + let maybe_start_frame_image_url = match request.start_frame_image_media_token.as_ref() { + None => None, + Some(token) => match image_urls_by_token.get(token) { + Some(url) => Some(url.to_string()), + None => { + return Err(CommonWebError::BadInputWithSimpleMessage("Media for start frame not found.".to_string())); + } + } + }; + + let maybe_end_frame_image_url = match request.end_frame_image_media_token.as_ref() { + None => None, + Some(token) => match image_urls_by_token.get(token) { + Some(url) => Some(url.to_string()), + None => { + return Err(CommonWebError::BadInputWithSimpleMessage("Media for end frame not found.".to_string())); + } + } + }; + + insert_idempotency_token(&request.uuid_idempotency_token, &mut *mysql_connection) + .await + .map_err(|err| { + error!("Error inserting idempotency token: {:?}", err); + CommonWebError::BadInputWithSimpleMessage("repeated idempotency token".to_string()) + })?; + + info!("Fal webhook URL: {}", server_state.fal.webhook_url); + + let fal_result; + + if let Some(start_frame_url) = maybe_start_frame_image_url { + info!("image-to-video case"); + + let duration = match request.duration { + Some(Kling2p5TurboProMultiFunctionVideoGenDuration::FiveSeconds) => EnqueueKlingV2p5TurboProImageToVideoDurationSeconds::Five, + Some(Kling2p5TurboProMultiFunctionVideoGenDuration::TenSeconds) => EnqueueKlingV2p5TurboProImageToVideoDurationSeconds::Ten, + None => EnqueueKlingV2p5TurboProImageToVideoDurationSeconds::Five, + }; + + let args = EnqueueKlingV2p5TurboProImageToVideoArgs { + prompt: request.prompt.as_deref().unwrap_or("").to_string(), + image_url: start_frame_url, + tail_image_url: maybe_end_frame_image_url, + negative_prompt: request.negative_prompt.clone(), + duration: Some(duration), + webhook_url: &server_state.fal.webhook_url, + api_key: &server_state.fal.api_key, + }; + + fal_result = enqueue_kling_v2p5_turbo_pro_image_to_video_webhook(args) + .await + .map_err(|err| { + warn!("Error calling enqueue_kling_v2p5_turbo_pro_image_to_video_webhook: {:?}", err); + CommonWebError::ServerError + })?; + + } else { + info!("text-to-video case"); + + let duration = match request.duration { + Some(Kling2p5TurboProMultiFunctionVideoGenDuration::FiveSeconds) => EnqueueKlingV2p5TurboProTextToVideoDurationSeconds::Five, + Some(Kling2p5TurboProMultiFunctionVideoGenDuration::TenSeconds) => EnqueueKlingV2p5TurboProTextToVideoDurationSeconds::Ten, + None => EnqueueKlingV2p5TurboProTextToVideoDurationSeconds::Five, + }; + + let aspect_ratio = match request.aspect_ratio { + Some(Kling2p5TurboProMultiFunctionVideoGenAspectRatio::Square) => EnqueueKlingV2p5TurboProTextToVideoAspectRatio::Square, + Some(Kling2p5TurboProMultiFunctionVideoGenAspectRatio::SixteenByNine) => EnqueueKlingV2p5TurboProTextToVideoAspectRatio::SixteenByNine, + Some(Kling2p5TurboProMultiFunctionVideoGenAspectRatio::NineBySixteen) => EnqueueKlingV2p5TurboProTextToVideoAspectRatio::NineBySixteen, + None => EnqueueKlingV2p5TurboProTextToVideoAspectRatio::Square, + }; + + let args = EnqueueKlingV2p5TurboProTextToVideoArgs { + prompt: request.prompt.as_deref().unwrap_or("").to_string(), + negative_prompt: request.negative_prompt.clone(), + duration: Some(duration), + aspect_ratio: Some(aspect_ratio), + webhook_url: &server_state.fal.webhook_url, + api_key: &server_state.fal.api_key, + }; + + fal_result = enqueue_kling_v2p5_turbo_pro_text_to_video_webhook(args) + .await + .map_err(|err| { + warn!("Error calling enqueue_kling_v2p5_turbo_pro_text_to_video_webhook: {:?}", err); + CommonWebError::ServerError + })?; + } + + let external_job_id = fal_result.request_id + .ok_or_else(|| { + warn!("Fal request_id is None"); + CommonWebError::ServerError + })?; + + info!("Fal request_id: {}", external_job_id); + + let ip_address = get_request_ip(&http_request); + + let mut transaction = mysql_connection + .begin() + .await + .map_err(|err| { + error!("Error starting MySQL transaction: {:?}", err); + CommonWebError::ServerError + })?; + + // NB: Don't fail the job if the query fails. + let prompt_result = insert_prompt(InsertPromptArgs { + maybe_apriori_prompt_token: None, + prompt_type: PromptType::ArtcraftApp, + maybe_creator_user_token: maybe_user_session + .as_ref() + .map(|s| &s.user_token), + maybe_model_type: Some(ModelType::Kling2p5TurboPro), + maybe_generation_provider: Some(GenerationProvider::Artcraft), + maybe_positive_prompt: request.prompt.as_deref(), + maybe_negative_prompt: None, + maybe_other_args: None, + creator_ip_address: &ip_address, + mysql_executor: &mut *transaction, + phantom: Default::default(), + }).await; + + let prompt_token = match prompt_result { + Ok(token) => Some(token), + Err(err) => { + warn!("Error inserting prompt: {:?}", err); + None // Don't fail the job if the prompt insertion fails. + } + }; + + if let Some(token) = prompt_token.as_ref() { + let mut context_items = Vec::with_capacity(2); + + if let Some(media_token) = &request.start_frame_image_media_token { + context_items.push(PromptContextItem { + media_token: media_token.clone(), + context_semantic_type: PromptContextSemanticType::VidStartFrame, + }); + } + + if let Some(media_token) = &request.end_frame_image_media_token { + context_items.push(PromptContextItem { + media_token: media_token.clone(), + context_semantic_type: PromptContextSemanticType::VidEndFrame, + }); + } + + if !context_items.is_empty() { + let result = insert_batch_prompt_context_items(InsertBatchArgs { + prompt_token: token.clone(), + items: context_items, + transaction: &mut transaction, + }).await; + + if let Err(err) = result { + // NB: Fail open. + warn!("Error inserting batch prompt context items: {:?}", err); + } + } + } + + let db_result = insert_generic_inference_job_for_fal_queue(InsertGenericInferenceForFalArgs { + uuid_idempotency_token: &request.uuid_idempotency_token, + maybe_external_third_party_id: &external_job_id, + fal_category: FalCategory::VideoGeneration, + maybe_inference_args: None, + maybe_prompt_token: prompt_token.as_ref(), + maybe_creator_user_token: maybe_user_session.as_ref().map(|s| &s.user_token), + maybe_avt_token: maybe_avt_token.as_ref(), + creator_ip_address: &ip_address, + creator_set_visibility: Visibility::Public, + mysql_executor: &mut *transaction, + phantom: Default::default(), + }).await; + + let job_token = match db_result { + Ok(token) => token, + Err(err) => { + warn!("Error inserting generic inference job for FAL queue: {:?}", err); + return Err(CommonWebError::ServerError); + } + }; + + let _r = transaction + .commit() + .await + .map_err(|err| { + error!("Error committing MySQL transaction: {:?}", err); + CommonWebError::ServerError + })?; + + Ok(Json(Kling2p5TurboProMultiFunctionVideoGenResponse { + success: true, + inference_job_token: job_token, + })) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/kling_2p6_pro_multi_function_video_gen_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/kling_2p6_pro_multi_function_video_gen_handler.rs new file mode 100644 index 000000000..a0396a214 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/kling_2p6_pro_multi_function_video_gen_handler.rs @@ -0,0 +1,316 @@ +use std::collections::{HashMap, HashSet}; +use std::iter::FromIterator; +use std::sync::Arc; + +use crate::http_server::common_responses::common_web_error::CommonWebError; +use crate::http_server::common_responses::media::media_links_builder::MediaLinksBuilder; +use crate::http_server::endpoints::generate::common::payments_error_test::payments_error_test; +use crate::http_server::endpoints::media_files::helpers::get_media_domain::get_media_domain; +use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format; +use crate::state::server_state::ServerState; +use crate::util::lookup::lookup_image_urls_as_map::lookup_image_urls_as_map; +use actix_web::web::Json; +use actix_web::{web, HttpRequest}; +use artcraft_api_defs::generate::video::multi_function::kling_2_6_multi_function_video_gen::{Kling2p6ProMultiFunctionVideoGenAspectRatio, Kling2p6ProMultiFunctionVideoGenDuration, Kling2p6ProMultiFunctionVideoGenRequest, Kling2p6ProMultiFunctionVideoGenResponse}; +use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; +use enums::by_table::prompt_context_items::prompt_context_semantic_type::PromptContextSemanticType; +use enums::by_table::prompts::prompt_type::PromptType; +use enums::common::generation_provider::GenerationProvider; +use enums::common::model_type::ModelType; +use enums::common::visibility::Visibility; +use fal_client::creds::open_ai_api_key::OpenAiApiKey; +use fal_client::requests::webhook::video::image::enqueue_kling_v2p6_pro_image_to_video_webhook::{enqueue_kling_v2p6_pro_image_to_video_webhook, EnqueueKlingV2p6ProImageToVideoArgs, EnqueueKlingV2p6ProImageToVideoDurationSeconds}; +use fal_client::requests::webhook::video::text::enqueue_kling_v2p6_pro_text_to_video_webhook::{enqueue_kling_v2p6_pro_text_to_video_webhook, EnqueueKlingV2p6ProTextToVideoArgs, EnqueueKlingV2p6ProTextToVideoAspectRatio, EnqueueKlingV2p6ProTextToVideoDurationSeconds}; +use http_server_common::request::get_request_ip::get_request_ip; +use log::{error, info, warn}; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::insert_generic_inference_job_for_fal_queue; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::FalCategory; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::InsertGenericInferenceForFalArgs; +use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token; +use mysql_queries::queries::media_files::get::batch_get_media_files_by_tokens::{batch_get_media_files_by_tokens, batch_get_media_files_by_tokens_with_connection}; +use mysql_queries::queries::prompt_context_items::insert_batch_prompt_context_items::{insert_batch_prompt_context_items, InsertBatchArgs, PromptContextItem}; +use mysql_queries::queries::prompts::insert_prompt::{insert_prompt, InsertPromptArgs}; +use server_environment::ServerEnvironment; +use sqlx::pool::PoolConnection; +use sqlx::{Acquire, MySql}; +use tokens::tokens::media_files::MediaFileToken; +use utoipa::ToSchema; + +/// Kling 2.6 Pro Multi-Function (text and image to video) +#[utoipa::path( + post, + tag = "Generate Video (Multi-Function)", + path = "/v1/generate/video/multi_function/kling_2p6_pro", + responses( + (status = 200, description = "Success", body = Kling2p6ProMultiFunctionVideoGenResponse), + ), + params( + ("request" = Kling2p6ProMultiFunctionVideoGenRequest, description = "Payload for Request"), + ) +)] +pub async fn kling_2p6_pro_multi_function_video_gen_handler( + http_request: HttpRequest, + request: Json, + server_state: web::Data> +) -> Result, CommonWebError> { + + payments_error_test(&request.prompt.as_deref().unwrap_or(""))?; + + let mut mysql_connection = server_state.mysql_pool + .acquire() + .await?; + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session_from_connection(&http_request, &mut mysql_connection) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + CommonWebError::ServerError + })?; + + let maybe_avt_token = server_state + .avt_cookie_manager + .get_avt_token_from_request(&http_request); + + // TODO: Limit usage for new accounts. Billing, free credits metering, etc. + + //let user_session = match maybe_user_session { + // Some(session) => session, + // None => { + // warn!("not logged in"); + // return Err(CommonWebError::NotAuthorized); + // } + //}; + + if let Err(reason) = validate_idempotency_token_format(&request.uuid_idempotency_token) { + return Err(CommonWebError::BadInputWithSimpleMessage(reason)); + } + + let mut query_media_tokens = None; + + if let Some(start_frame_token) = request.start_frame_image_media_token.as_ref() { + let mut tokens = Vec::new(); + tokens.push(start_frame_token.to_owned()); + + // if let Some(end_frame_token) = request.end_frame_image_media_token.as_ref() { + // // NB: Masks are only relevant when this is an image editing exercise. + // tokens.push(end_frame_token.to_owned()); + // } + + query_media_tokens = Some(tokens); + } + + let image_urls_by_token = match query_media_tokens { + None => HashMap::new(), + Some(media_tokens) => { + info!("Looking up image media tokens: {:?}", media_tokens); + lookup_image_urls_as_map( + &http_request, + &mut mysql_connection, + server_state.server_environment, + &media_tokens, + ).await? + } + }; + + let maybe_start_frame_image_url = match request.start_frame_image_media_token.as_ref() { + None => None, + Some(token) => match image_urls_by_token.get(token) { + Some(url) => Some(url.to_string()), + None => { + return Err(CommonWebError::BadInputWithSimpleMessage("Media for start frame not found.".to_string())); + } + } + }; + + // let maybe_end_frame_image_url = match request.end_frame_image_media_token.as_ref() { + // None => None, + // Some(token) => match image_urls_by_token.get(token) { + // Some(url) => Some(url.to_string()), + // None => { + // return Err(CommonWebError::BadInputWithSimpleMessage("Media for end frame not found.".to_string())); + // } + // } + // }; + + insert_idempotency_token(&request.uuid_idempotency_token, &mut *mysql_connection) + .await + .map_err(|err| { + error!("Error inserting idempotency token: {:?}", err); + CommonWebError::BadInputWithSimpleMessage("repeated idempotency token".to_string()) + })?; + + info!("Fal webhook URL: {}", server_state.fal.webhook_url); + + let fal_result; + + if let Some(start_frame_url) = maybe_start_frame_image_url { + info!("image-to-video case"); + + let duration = match request.duration { + Some(Kling2p6ProMultiFunctionVideoGenDuration::FiveSeconds) => EnqueueKlingV2p6ProImageToVideoDurationSeconds::Five, + Some(Kling2p6ProMultiFunctionVideoGenDuration::TenSeconds) => EnqueueKlingV2p6ProImageToVideoDurationSeconds::Ten, + None => EnqueueKlingV2p6ProImageToVideoDurationSeconds::Five, + }; + + let args = EnqueueKlingV2p6ProImageToVideoArgs { + prompt: request.prompt.as_deref().unwrap_or("").to_string(), + image_url: start_frame_url, + generate_audio: request.generate_audio, + negative_prompt: request.negative_prompt.clone(), + duration: Some(duration), + webhook_url: &server_state.fal.webhook_url, + api_key: &server_state.fal.api_key, + }; + + fal_result = enqueue_kling_v2p6_pro_image_to_video_webhook(args) + .await + .map_err(|err| { + warn!("Error calling enqueue_kling_v2p6_pro_image_to_video_webhook: {:?}", err); + CommonWebError::ServerError + })?; + + } else { + info!("text-to-video case"); + + let duration = match request.duration { + Some(Kling2p6ProMultiFunctionVideoGenDuration::FiveSeconds) => EnqueueKlingV2p6ProTextToVideoDurationSeconds::Five, + Some(Kling2p6ProMultiFunctionVideoGenDuration::TenSeconds) => EnqueueKlingV2p6ProTextToVideoDurationSeconds::Ten, + None => EnqueueKlingV2p6ProTextToVideoDurationSeconds::Five, + }; + + let aspect_ratio = match request.aspect_ratio { + Some(Kling2p6ProMultiFunctionVideoGenAspectRatio::Square) => EnqueueKlingV2p6ProTextToVideoAspectRatio::Square, + Some(Kling2p6ProMultiFunctionVideoGenAspectRatio::SixteenByNine) => EnqueueKlingV2p6ProTextToVideoAspectRatio::SixteenByNine, + Some(Kling2p6ProMultiFunctionVideoGenAspectRatio::NineBySixteen) => EnqueueKlingV2p6ProTextToVideoAspectRatio::NineBySixteen, + None => EnqueueKlingV2p6ProTextToVideoAspectRatio::Square, + }; + + let args = EnqueueKlingV2p6ProTextToVideoArgs { + prompt: request.prompt.as_deref().unwrap_or("").to_string(), + negative_prompt: request.negative_prompt.clone(), + generate_audio: request.generate_audio, + duration: Some(duration), + aspect_ratio: Some(aspect_ratio), + webhook_url: &server_state.fal.webhook_url, + api_key: &server_state.fal.api_key, + }; + + fal_result = enqueue_kling_v2p6_pro_text_to_video_webhook(args) + .await + .map_err(|err| { + warn!("Error calling enqueue_kling_v2p6_pro_text_to_video_webhook: {:?}", err); + CommonWebError::ServerError + })?; + } + + let external_job_id = fal_result.request_id + .ok_or_else(|| { + warn!("Fal request_id is None"); + CommonWebError::ServerError + })?; + + info!("Fal request_id: {}", external_job_id); + + let ip_address = get_request_ip(&http_request); + + let mut transaction = mysql_connection + .begin() + .await + .map_err(|err| { + error!("Error starting MySQL transaction: {:?}", err); + CommonWebError::ServerError + })?; + + // NB: Don't fail the job if the query fails. + let prompt_result = insert_prompt(InsertPromptArgs { + maybe_apriori_prompt_token: None, + prompt_type: PromptType::ArtcraftApp, + maybe_creator_user_token: maybe_user_session + .as_ref() + .map(|s| &s.user_token), + maybe_model_type: Some(ModelType::Kling2p6Pro), + maybe_generation_provider: Some(GenerationProvider::Artcraft), + maybe_positive_prompt: request.prompt.as_deref(), + maybe_negative_prompt: None, + maybe_other_args: None, + creator_ip_address: &ip_address, + mysql_executor: &mut *transaction, + phantom: Default::default(), + }).await; + + let prompt_token = match prompt_result { + Ok(token) => Some(token), + Err(err) => { + warn!("Error inserting prompt: {:?}", err); + None // Don't fail the job if the prompt insertion fails. + } + }; + + if let Some(token) = prompt_token.as_ref() { + let mut context_items = Vec::with_capacity(2); + + if let Some(media_token) = &request.start_frame_image_media_token { + context_items.push(PromptContextItem { + media_token: media_token.clone(), + context_semantic_type: PromptContextSemanticType::VidStartFrame, + }); + } + + // if let Some(media_token) = &request.end_frame_image_media_token { + // context_items.push(PromptContextItem { + // media_token: media_token.clone(), + // context_semantic_type: PromptContextSemanticType::VidEndFrame, + // }); + // } + + if !context_items.is_empty() { + let result = insert_batch_prompt_context_items(InsertBatchArgs { + prompt_token: token.clone(), + items: context_items, + transaction: &mut transaction, + }).await; + + if let Err(err) = result { + // NB: Fail open. + warn!("Error inserting batch prompt context items: {:?}", err); + } + } + } + + let db_result = insert_generic_inference_job_for_fal_queue(InsertGenericInferenceForFalArgs { + uuid_idempotency_token: &request.uuid_idempotency_token, + maybe_external_third_party_id: &external_job_id, + fal_category: FalCategory::VideoGeneration, + maybe_inference_args: None, + maybe_prompt_token: prompt_token.as_ref(), + maybe_creator_user_token: maybe_user_session.as_ref().map(|s| &s.user_token), + maybe_avt_token: maybe_avt_token.as_ref(), + creator_ip_address: &ip_address, + creator_set_visibility: Visibility::Public, + mysql_executor: &mut *transaction, + phantom: Default::default(), + }).await; + + let job_token = match db_result { + Ok(token) => token, + Err(err) => { + warn!("Error inserting generic inference job for FAL queue: {:?}", err); + return Err(CommonWebError::ServerError); + } + }; + + let _r = transaction + .commit() + .await + .map_err(|err| { + error!("Error committing MySQL transaction: {:?}", err); + CommonWebError::ServerError + })?; + + Ok(Json(Kling2p6ProMultiFunctionVideoGenResponse { + success: true, + inference_job_token: job_token, + })) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/mod.rs new file mode 100644 index 000000000..d32080165 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/mod.rs @@ -0,0 +1,6 @@ +pub mod kling_2p5_turbo_pro_multi_function_video_gen_handler; +pub mod kling_2p6_pro_multi_function_video_gen_handler; +pub mod sora_2_multi_function_video_gen_handler; +pub mod sora_2_pro_multi_function_video_gen_handler; +pub mod veo_3p1_fast_multi_function_video_gen_handler; +pub mod veo_3p1_multi_function_video_gen_handler; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/sora_2_multi_function_video_gen_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/sora_2_multi_function_video_gen_handler.rs new file mode 100644 index 000000000..8f46b4e72 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/sora_2_multi_function_video_gen_handler.rs @@ -0,0 +1,336 @@ +use std::collections::{HashMap, HashSet}; +use std::iter::FromIterator; +use std::sync::Arc; + +use crate::http_server::common_responses::common_web_error::CommonWebError; +use crate::http_server::common_responses::media::media_links_builder::MediaLinksBuilder; +use crate::http_server::endpoints::generate::common::payments_error_test::payments_error_test; +use crate::http_server::endpoints::media_files::helpers::get_media_domain::get_media_domain; +use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format; +use crate::state::server_state::ServerState; +use crate::util::lookup::lookup_image_urls_as_map::lookup_image_urls_as_map; +use actix_web::web::Json; +use actix_web::{web, HttpRequest}; +use artcraft_api_defs::generate::video::multi_function::sora_2_multi_function_video_gen::{Sora2MultiFunctionVideoGenAspectRatio, Sora2MultiFunctionVideoGenDuration, Sora2MultiFunctionVideoGenRequest, Sora2MultiFunctionVideoGenResolution, Sora2MultiFunctionVideoGenResponse}; +use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; +use enums::by_table::prompt_context_items::prompt_context_semantic_type::PromptContextSemanticType; +use enums::by_table::prompts::prompt_type::PromptType; +use enums::common::generation_provider::GenerationProvider; +use enums::common::model_type::ModelType; +use enums::common::visibility::Visibility; +use fal_client::creds::open_ai_api_key::OpenAiApiKey; +use fal_client::requests::webhook::video::image::enqueue_sora_2_image_to_video_webhook::{enqueue_sora_2_image_to_video_webhook, EnqueueSora2ImageToVideoArgs, EnqueueSora2ImageToVideoAspectRatio, EnqueueSora2ImageToVideoDurationSeconds, EnqueueSora2ImageToVideoResolution}; +use fal_client::requests::webhook::video::text::enqueue_sora_2_text_to_video_webhook::{enqueue_sora_2_text_to_video_webhook, EnqueueSora2TextToVideoArgs, EnqueueSora2TextToVideoAspectRatio, EnqueueSora2TextToVideoDurationSeconds, EnqueueSora2TextToVideoResolution}; +use http_server_common::request::get_request_ip::get_request_ip; +use log::{error, info, warn}; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::insert_generic_inference_job_for_fal_queue; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::FalCategory; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::InsertGenericInferenceForFalArgs; +use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token; +use mysql_queries::queries::media_files::get::batch_get_media_files_by_tokens::{batch_get_media_files_by_tokens, batch_get_media_files_by_tokens_with_connection}; +use mysql_queries::queries::prompt_context_items::insert_batch_prompt_context_items::{insert_batch_prompt_context_items, InsertBatchArgs, PromptContextItem}; +use mysql_queries::queries::prompts::insert_prompt::{insert_prompt, InsertPromptArgs}; +use server_environment::ServerEnvironment; +use sqlx::pool::PoolConnection; +use sqlx::{Acquire, MySql}; +use tokens::tokens::media_files::MediaFileToken; +use utoipa::ToSchema; + +/// Sora 2 Multi-Function (text and image to video) +#[utoipa::path( + post, + tag = "Generate Video (Multi-Function)", + path = "/v1/generate/video/multi_function/sora_2", + responses( + (status = 200, description = "Success", body = Sora2MultiFunctionVideoGenResponse), + ), + params( + ("request" = Sora2MultiFunctionVideoGenRequest, description = "Payload for Request"), + ) +)] +pub async fn sora_2_multi_function_video_gen_handler( + http_request: HttpRequest, + request: Json, + server_state: web::Data> +) -> Result, CommonWebError> { + + payments_error_test(&request.prompt.as_deref().unwrap_or(""))?; + + let mut mysql_connection = server_state.mysql_pool + .acquire() + .await?; + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session_from_connection(&http_request, &mut mysql_connection) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + CommonWebError::ServerError + })?; + + let maybe_avt_token = server_state + .avt_cookie_manager + .get_avt_token_from_request(&http_request); + + // TODO: Limit usage for new accounts. Billing, free credits metering, etc. + + //let user_session = match maybe_user_session { + // Some(session) => session, + // None => { + // warn!("not logged in"); + // return Err(CommonWebError::NotAuthorized); + // } + //}; + + if let Err(reason) = validate_idempotency_token_format(&request.uuid_idempotency_token) { + return Err(CommonWebError::BadInputWithSimpleMessage(reason)); + } + + let mut query_media_tokens = None; + + if let Some(start_frame_token) = request.image_media_token.as_ref() { + let mut tokens = Vec::new(); + tokens.push(start_frame_token.to_owned()); + + // if let Some(end_frame_token) = request.end_frame_image_media_token.as_ref() { + // // NB: Masks are only relevant when this is an image editing exercise. + // tokens.push(end_frame_token.to_owned()); + // } + + query_media_tokens = Some(tokens); + } + + let image_urls_by_token = match query_media_tokens { + None => HashMap::new(), + Some(media_tokens) => { + info!("Looking up image media tokens: {:?}", media_tokens); + lookup_image_urls_as_map( + &http_request, + &mut mysql_connection, + server_state.server_environment, + &media_tokens, + ).await? + } + }; + + let maybe_image_url = match request.image_media_token.as_ref() { + None => None, + Some(token) => match image_urls_by_token.get(token) { + Some(url) => Some(url.to_string()), + None => { + return Err(CommonWebError::BadInputWithSimpleMessage("Media for start frame not found.".to_string())); + } + } + }; + + // let maybe_end_frame_image_url = match request.end_frame_image_media_token.as_ref() { + // None => None, + // Some(token) => match image_urls_by_token.get(token) { + // Some(url) => Some(url.to_string()), + // None => { + // return Err(CommonWebError::BadInputWithSimpleMessage("Media for end frame not found.".to_string())); + // } + // } + // }; + + insert_idempotency_token(&request.uuid_idempotency_token, &mut *mysql_connection) + .await + .map_err(|err| { + error!("Error inserting idempotency token: {:?}", err); + CommonWebError::BadInputWithSimpleMessage("repeated idempotency token".to_string()) + })?; + + info!("Fal webhook URL: {}", server_state.fal.webhook_url); + + let fal_result; + + if let Some(image_url) = maybe_image_url { + info!("image-to-video case"); + + let duration = match request.duration { + Some(Sora2MultiFunctionVideoGenDuration::FourSeconds) => EnqueueSora2ImageToVideoDurationSeconds::Four, + Some(Sora2MultiFunctionVideoGenDuration::EightSeconds) => EnqueueSora2ImageToVideoDurationSeconds::Eight, + Some(Sora2MultiFunctionVideoGenDuration::TwelveSeconds) => EnqueueSora2ImageToVideoDurationSeconds::Twelve, + None => EnqueueSora2ImageToVideoDurationSeconds::Eight, + }; + + let aspect_ratio = match request.aspect_ratio { + Some(Sora2MultiFunctionVideoGenAspectRatio::Auto) => EnqueueSora2ImageToVideoAspectRatio::Auto, + Some(Sora2MultiFunctionVideoGenAspectRatio::SixteenByNine) => EnqueueSora2ImageToVideoAspectRatio::SixteenByNine, + Some(Sora2MultiFunctionVideoGenAspectRatio::NineBySixteen) => EnqueueSora2ImageToVideoAspectRatio::NineBySixteen, + None => EnqueueSora2ImageToVideoAspectRatio::SixteenByNine, + }; + + let resolution = match request.resolution { + Some(Sora2MultiFunctionVideoGenResolution::Auto) => EnqueueSora2ImageToVideoResolution::Auto, + Some(Sora2MultiFunctionVideoGenResolution::SevenTwentyP) => EnqueueSora2ImageToVideoResolution::SevenTwentyP, + None => EnqueueSora2ImageToVideoResolution::SevenTwentyP, + }; + + let args = EnqueueSora2ImageToVideoArgs { + image_url, + prompt: request.prompt.as_deref().unwrap_or("").to_string(), + duration: Some(duration), + aspect_ratio: Some(aspect_ratio), + resolution: Some(resolution), + webhook_url: &server_state.fal.webhook_url, + api_key: &server_state.fal.api_key, + }; + + fal_result = enqueue_sora_2_image_to_video_webhook(args) + .await + .map_err(|err| { + warn!("Error calling enqueue_sora_2_image_to_video_webhook: {:?}", err); + CommonWebError::ServerError + })?; + + } else { + info!("text-to-video case"); + + let duration = match request.duration { + Some(Sora2MultiFunctionVideoGenDuration::FourSeconds) => EnqueueSora2TextToVideoDurationSeconds::Four, + Some(Sora2MultiFunctionVideoGenDuration::EightSeconds) => EnqueueSora2TextToVideoDurationSeconds::Eight, + Some(Sora2MultiFunctionVideoGenDuration::TwelveSeconds) => EnqueueSora2TextToVideoDurationSeconds::Twelve, + None => EnqueueSora2TextToVideoDurationSeconds::Eight, + }; + + let aspect_ratio = match request.aspect_ratio { + Some(Sora2MultiFunctionVideoGenAspectRatio::Auto) => EnqueueSora2TextToVideoAspectRatio::Auto, + Some(Sora2MultiFunctionVideoGenAspectRatio::SixteenByNine) => EnqueueSora2TextToVideoAspectRatio::SixteenByNine, + Some(Sora2MultiFunctionVideoGenAspectRatio::NineBySixteen) => EnqueueSora2TextToVideoAspectRatio::NineBySixteen, + None => EnqueueSora2TextToVideoAspectRatio::SixteenByNine, + }; + + let resolution = match request.resolution { + Some(Sora2MultiFunctionVideoGenResolution::Auto) => EnqueueSora2TextToVideoResolution::Auto, + Some(Sora2MultiFunctionVideoGenResolution::SevenTwentyP) => EnqueueSora2TextToVideoResolution::SevenTwentyP, + None => EnqueueSora2TextToVideoResolution::SevenTwentyP, + }; + + let args = EnqueueSora2TextToVideoArgs { + prompt: request.prompt.as_deref().unwrap_or("").to_string(), + duration: Some(duration), + aspect_ratio: Some(aspect_ratio), + resolution: Some(resolution), + webhook_url: &server_state.fal.webhook_url, + api_key: &server_state.fal.api_key, + }; + + fal_result = enqueue_sora_2_text_to_video_webhook(args) + .await + .map_err(|err| { + warn!("Error calling enqueue_sora_2_text_to_video_webhook: {:?}", err); + CommonWebError::ServerError + })?; + } + + let external_job_id = fal_result.request_id + .ok_or_else(|| { + warn!("Fal request_id is None"); + CommonWebError::ServerError + })?; + + info!("Fal request_id: {}", external_job_id); + + let ip_address = get_request_ip(&http_request); + + let mut transaction = mysql_connection + .begin() + .await + .map_err(|err| { + error!("Error starting MySQL transaction: {:?}", err); + CommonWebError::ServerError + })?; + + // NB: Don't fail the job if the query fails. + let prompt_result = insert_prompt(InsertPromptArgs { + maybe_apriori_prompt_token: None, + prompt_type: PromptType::ArtcraftApp, + maybe_creator_user_token: maybe_user_session + .as_ref() + .map(|s| &s.user_token), + maybe_model_type: Some(ModelType::Sora2), + maybe_generation_provider: Some(GenerationProvider::Artcraft), + maybe_positive_prompt: request.prompt.as_deref(), + maybe_negative_prompt: None, + maybe_other_args: None, + creator_ip_address: &ip_address, + mysql_executor: &mut *transaction, + phantom: Default::default(), + }).await; + + let prompt_token = match prompt_result { + Ok(token) => Some(token), + Err(err) => { + warn!("Error inserting prompt: {:?}", err); + None // Don't fail the job if the prompt insertion fails. + } + }; + + if let Some(token) = prompt_token.as_ref() { + let mut context_items = Vec::with_capacity(2); + + if let Some(media_token) = &request.image_media_token { + context_items.push(PromptContextItem { + media_token: media_token.clone(), + context_semantic_type: PromptContextSemanticType::VidStartFrame, + }); + } + + // if let Some(media_token) = &request.end_frame_image_media_token { + // context_items.push(PromptContextItem { + // media_token: media_token.clone(), + // context_semantic_type: PromptContextSemanticType::VidEndFrame, + // }); + // } + + if !context_items.is_empty() { + let result = insert_batch_prompt_context_items(InsertBatchArgs { + prompt_token: token.clone(), + items: context_items, + transaction: &mut transaction, + }).await; + + if let Err(err) = result { + // NB: Fail open. + warn!("Error inserting batch prompt context items: {:?}", err); + } + } + } + + let db_result = insert_generic_inference_job_for_fal_queue(InsertGenericInferenceForFalArgs { + uuid_idempotency_token: &request.uuid_idempotency_token, + maybe_external_third_party_id: &external_job_id, + fal_category: FalCategory::VideoGeneration, + maybe_inference_args: None, + maybe_prompt_token: prompt_token.as_ref(), + maybe_creator_user_token: maybe_user_session.as_ref().map(|s| &s.user_token), + maybe_avt_token: maybe_avt_token.as_ref(), + creator_ip_address: &ip_address, + creator_set_visibility: Visibility::Public, + mysql_executor: &mut *transaction, + phantom: Default::default(), + }).await; + + let job_token = match db_result { + Ok(token) => token, + Err(err) => { + warn!("Error inserting generic inference job for FAL queue: {:?}", err); + return Err(CommonWebError::ServerError); + } + }; + + let _r = transaction + .commit() + .await + .map_err(|err| { + error!("Error committing MySQL transaction: {:?}", err); + CommonWebError::ServerError + })?; + + Ok(Json(Sora2MultiFunctionVideoGenResponse { + success: true, + inference_job_token: job_token, + })) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/sora_2_pro_multi_function_video_gen_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/sora_2_pro_multi_function_video_gen_handler.rs new file mode 100644 index 000000000..28ad08b2f --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/sora_2_pro_multi_function_video_gen_handler.rs @@ -0,0 +1,338 @@ +use std::collections::{HashMap, HashSet}; +use std::iter::FromIterator; +use std::sync::Arc; + +use crate::http_server::common_responses::common_web_error::CommonWebError; +use crate::http_server::common_responses::media::media_links_builder::MediaLinksBuilder; +use crate::http_server::endpoints::generate::common::payments_error_test::payments_error_test; +use crate::http_server::endpoints::media_files::helpers::get_media_domain::get_media_domain; +use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format; +use crate::state::server_state::ServerState; +use crate::util::lookup::lookup_image_urls_as_map::lookup_image_urls_as_map; +use actix_web::web::Json; +use actix_web::{web, HttpRequest}; +use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; +use enums::by_table::prompt_context_items::prompt_context_semantic_type::PromptContextSemanticType; +use enums::by_table::prompts::prompt_type::PromptType; +use enums::common::generation_provider::GenerationProvider; +use enums::common::model_type::ModelType; +use enums::common::visibility::Visibility; +use fal_client::creds::open_ai_api_key::OpenAiApiKey; +use http_server_common::request::get_request_ip::get_request_ip; +use log::{error, info, warn}; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::insert_generic_inference_job_for_fal_queue; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::FalCategory; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::InsertGenericInferenceForFalArgs; +use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token; +use mysql_queries::queries::media_files::get::batch_get_media_files_by_tokens::{batch_get_media_files_by_tokens, batch_get_media_files_by_tokens_with_connection}; +use mysql_queries::queries::prompt_context_items::insert_batch_prompt_context_items::{insert_batch_prompt_context_items, InsertBatchArgs, PromptContextItem}; +use mysql_queries::queries::prompts::insert_prompt::{insert_prompt, InsertPromptArgs}; +use server_environment::ServerEnvironment; +use sqlx::pool::PoolConnection; +use sqlx::{Acquire, MySql}; +use tokens::tokens::media_files::MediaFileToken; +use utoipa::ToSchema; +use artcraft_api_defs::generate::video::multi_function::sora_2_pro_multi_function_video_gen::{Sora2ProMultiFunctionVideoGenAspectRatio, Sora2ProMultiFunctionVideoGenDuration, Sora2ProMultiFunctionVideoGenRequest, Sora2ProMultiFunctionVideoGenResolution, Sora2ProMultiFunctionVideoGenResponse}; +use fal_client::requests::webhook::video::image::enqueue_sora_2_pro_image_to_video_webhook::{enqueue_sora_2_pro_image_to_video_webhook, EnqueueSora2ProImageToVideoArgs, EnqueueSora2ProImageToVideoAspectRatio, EnqueueSora2ProImageToVideoDurationSeconds, EnqueueSora2ProImageToVideoResolution}; +use fal_client::requests::webhook::video::text::enqueue_sora_2_pro_text_to_video_webhook::{enqueue_sora_2_pro_text_to_video_webhook, EnqueueSora2ProTextToVideoArgs, EnqueueSora2ProTextToVideoAspectRatio, EnqueueSora2ProTextToVideoDurationSeconds, EnqueueSora2ProTextToVideoResolution}; + +/// Sora 2 Pro Multi-Function (text and image to video) +#[utoipa::path( + post, + tag = "Generate Video (Multi-Function)", + path = "/v1/generate/video/multi_function/sora_2_pro", + responses( + (status = 200, description = "Success", body = Sora2ProMultiFunctionVideoGenResponse), + ), + params( + ("request" = Sora2ProMultiFunctionVideoGenRequest, description = "Payload for Request"), + ) +)] +pub async fn sora_2_pro_multi_function_video_gen_handler( + http_request: HttpRequest, + request: Json, + server_state: web::Data> +) -> Result, CommonWebError> { + + payments_error_test(&request.prompt.as_deref().unwrap_or(""))?; + + let mut mysql_connection = server_state.mysql_pool + .acquire() + .await?; + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session_from_connection(&http_request, &mut mysql_connection) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + CommonWebError::ServerError + })?; + + let maybe_avt_token = server_state + .avt_cookie_manager + .get_avt_token_from_request(&http_request); + + // TODO: Limit usage for new accounts. Billing, free credits metering, etc. + + //let user_session = match maybe_user_session { + // Some(session) => session, + // None => { + // warn!("not logged in"); + // return Err(CommonWebError::NotAuthorized); + // } + //}; + + if let Err(reason) = validate_idempotency_token_format(&request.uuid_idempotency_token) { + return Err(CommonWebError::BadInputWithSimpleMessage(reason)); + } + + let mut query_media_tokens = None; + + if let Some(start_frame_token) = request.image_media_token.as_ref() { + let mut tokens = Vec::new(); + tokens.push(start_frame_token.to_owned()); + + // if let Some(end_frame_token) = request.end_frame_image_media_token.as_ref() { + // // NB: Masks are only relevant when this is an image editing exercise. + // tokens.push(end_frame_token.to_owned()); + // } + + query_media_tokens = Some(tokens); + } + + let image_urls_by_token = match query_media_tokens { + None => HashMap::new(), + Some(media_tokens) => { + info!("Looking up image media tokens: {:?}", media_tokens); + lookup_image_urls_as_map( + &http_request, + &mut mysql_connection, + server_state.server_environment, + &media_tokens, + ).await? + } + }; + + let maybe_image_url = match request.image_media_token.as_ref() { + None => None, + Some(token) => match image_urls_by_token.get(token) { + Some(url) => Some(url.to_string()), + None => { + return Err(CommonWebError::BadInputWithSimpleMessage("Media for start frame not found.".to_string())); + } + } + }; + + // let maybe_end_frame_image_url = match request.end_frame_image_media_token.as_ref() { + // None => None, + // Some(token) => match image_urls_by_token.get(token) { + // Some(url) => Some(url.to_string()), + // None => { + // return Err(CommonWebError::BadInputWithSimpleMessage("Media for end frame not found.".to_string())); + // } + // } + // }; + + insert_idempotency_token(&request.uuid_idempotency_token, &mut *mysql_connection) + .await + .map_err(|err| { + error!("Error inserting idempotency token: {:?}", err); + CommonWebError::BadInputWithSimpleMessage("repeated idempotency token".to_string()) + })?; + + info!("Fal webhook URL: {}", server_state.fal.webhook_url); + + let fal_result; + + if let Some(image_url) = maybe_image_url { + info!("image-to-video case"); + + let duration = match request.duration { + Some(Sora2ProMultiFunctionVideoGenDuration::FourSeconds) => EnqueueSora2ProImageToVideoDurationSeconds::Four, + Some(Sora2ProMultiFunctionVideoGenDuration::EightSeconds) => EnqueueSora2ProImageToVideoDurationSeconds::Eight, + Some(Sora2ProMultiFunctionVideoGenDuration::TwelveSeconds) => EnqueueSora2ProImageToVideoDurationSeconds::Twelve, + None => EnqueueSora2ProImageToVideoDurationSeconds::Eight, + }; + + let aspect_ratio = match request.aspect_ratio { + Some(Sora2ProMultiFunctionVideoGenAspectRatio::Auto) => EnqueueSora2ProImageToVideoAspectRatio::Auto, + Some(Sora2ProMultiFunctionVideoGenAspectRatio::SixteenByNine) => EnqueueSora2ProImageToVideoAspectRatio::SixteenByNine, + Some(Sora2ProMultiFunctionVideoGenAspectRatio::NineBySixteen) => EnqueueSora2ProImageToVideoAspectRatio::NineBySixteen, + None => EnqueueSora2ProImageToVideoAspectRatio::SixteenByNine, + }; + + let resolution = match request.resolution { + Some(Sora2ProMultiFunctionVideoGenResolution::Auto) => EnqueueSora2ProImageToVideoResolution::Auto, + Some(Sora2ProMultiFunctionVideoGenResolution::SevenTwentyP) => EnqueueSora2ProImageToVideoResolution::SevenTwentyP, + Some(Sora2ProMultiFunctionVideoGenResolution::TenEightyP) => EnqueueSora2ProImageToVideoResolution::TenEightyP, + None => EnqueueSora2ProImageToVideoResolution::TenEightyP, + }; + + let args = EnqueueSora2ProImageToVideoArgs { + image_url, + prompt: request.prompt.as_deref().unwrap_or("").to_string(), + duration: Some(duration), + aspect_ratio: Some(aspect_ratio), + resolution: Some(resolution), + webhook_url: &server_state.fal.webhook_url, + api_key: &server_state.fal.api_key, + }; + + fal_result = enqueue_sora_2_pro_image_to_video_webhook(args) + .await + .map_err(|err| { + warn!("Error calling enqueue_sora_2_pro_image_to_video_webhook: {:?}", err); + CommonWebError::ServerError + })?; + + } else { + info!("text-to-video case"); + + let duration = match request.duration { + Some(Sora2ProMultiFunctionVideoGenDuration::FourSeconds) => EnqueueSora2ProTextToVideoDurationSeconds::Four, + Some(Sora2ProMultiFunctionVideoGenDuration::EightSeconds) => EnqueueSora2ProTextToVideoDurationSeconds::Eight, + Some(Sora2ProMultiFunctionVideoGenDuration::TwelveSeconds) => EnqueueSora2ProTextToVideoDurationSeconds::Twelve, + None => EnqueueSora2ProTextToVideoDurationSeconds::Eight, + }; + + let aspect_ratio = match request.aspect_ratio { + Some(Sora2ProMultiFunctionVideoGenAspectRatio::Auto) => EnqueueSora2ProTextToVideoAspectRatio::Auto, + Some(Sora2ProMultiFunctionVideoGenAspectRatio::SixteenByNine) => EnqueueSora2ProTextToVideoAspectRatio::SixteenByNine, + Some(Sora2ProMultiFunctionVideoGenAspectRatio::NineBySixteen) => EnqueueSora2ProTextToVideoAspectRatio::NineBySixteen, + None => EnqueueSora2ProTextToVideoAspectRatio::SixteenByNine, + }; + + let resolution = match request.resolution { + Some(Sora2ProMultiFunctionVideoGenResolution::Auto) => EnqueueSora2ProTextToVideoResolution::Auto, + Some(Sora2ProMultiFunctionVideoGenResolution::SevenTwentyP) => EnqueueSora2ProTextToVideoResolution::SevenTwentyP, + Some(Sora2ProMultiFunctionVideoGenResolution::TenEightyP) => EnqueueSora2ProTextToVideoResolution::TenEightyP, + None => EnqueueSora2ProTextToVideoResolution::TenEightyP, + }; + + let args = EnqueueSora2ProTextToVideoArgs { + prompt: request.prompt.as_deref().unwrap_or("").to_string(), + duration: Some(duration), + aspect_ratio: Some(aspect_ratio), + resolution: Some(resolution), + webhook_url: &server_state.fal.webhook_url, + api_key: &server_state.fal.api_key, + }; + + fal_result = enqueue_sora_2_pro_text_to_video_webhook(args) + .await + .map_err(|err| { + warn!("Error calling enqueue_sora_2_pro_text_to_video_webhook: {:?}", err); + CommonWebError::ServerError + })?; + } + + let external_job_id = fal_result.request_id + .ok_or_else(|| { + warn!("Fal request_id is None"); + CommonWebError::ServerError + })?; + + info!("Fal request_id: {}", external_job_id); + + let ip_address = get_request_ip(&http_request); + + let mut transaction = mysql_connection + .begin() + .await + .map_err(|err| { + error!("Error starting MySQL transaction: {:?}", err); + CommonWebError::ServerError + })?; + + // NB: Don't fail the job if the query fails. + let prompt_result = insert_prompt(InsertPromptArgs { + maybe_apriori_prompt_token: None, + prompt_type: PromptType::ArtcraftApp, + maybe_creator_user_token: maybe_user_session + .as_ref() + .map(|s| &s.user_token), + maybe_model_type: Some(ModelType::Sora2Pro), + maybe_generation_provider: Some(GenerationProvider::Artcraft), + maybe_positive_prompt: request.prompt.as_deref(), + maybe_negative_prompt: None, + maybe_other_args: None, + creator_ip_address: &ip_address, + mysql_executor: &mut *transaction, + phantom: Default::default(), + }).await; + + let prompt_token = match prompt_result { + Ok(token) => Some(token), + Err(err) => { + warn!("Error inserting prompt: {:?}", err); + None // Don't fail the job if the prompt insertion fails. + } + }; + + if let Some(token) = prompt_token.as_ref() { + let mut context_items = Vec::with_capacity(2); + + if let Some(media_token) = &request.image_media_token { + context_items.push(PromptContextItem { + media_token: media_token.clone(), + context_semantic_type: PromptContextSemanticType::VidStartFrame, + }); + } + + // if let Some(media_token) = &request.end_frame_image_media_token { + // context_items.push(PromptContextItem { + // media_token: media_token.clone(), + // context_semantic_type: PromptContextSemanticType::VidEndFrame, + // }); + // } + + if !context_items.is_empty() { + let result = insert_batch_prompt_context_items(InsertBatchArgs { + prompt_token: token.clone(), + items: context_items, + transaction: &mut transaction, + }).await; + + if let Err(err) = result { + // NB: Fail open. + warn!("Error inserting batch prompt context items: {:?}", err); + } + } + } + + let db_result = insert_generic_inference_job_for_fal_queue(InsertGenericInferenceForFalArgs { + uuid_idempotency_token: &request.uuid_idempotency_token, + maybe_external_third_party_id: &external_job_id, + fal_category: FalCategory::VideoGeneration, + maybe_inference_args: None, + maybe_prompt_token: prompt_token.as_ref(), + maybe_creator_user_token: maybe_user_session.as_ref().map(|s| &s.user_token), + maybe_avt_token: maybe_avt_token.as_ref(), + creator_ip_address: &ip_address, + creator_set_visibility: Visibility::Public, + mysql_executor: &mut *transaction, + phantom: Default::default(), + }).await; + + let job_token = match db_result { + Ok(token) => token, + Err(err) => { + warn!("Error inserting generic inference job for FAL queue: {:?}", err); + return Err(CommonWebError::ServerError); + } + }; + + let _r = transaction + .commit() + .await + .map_err(|err| { + error!("Error committing MySQL transaction: {:?}", err); + CommonWebError::ServerError + })?; + + Ok(Json(Sora2ProMultiFunctionVideoGenResponse { + success: true, + inference_job_token: job_token, + })) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/veo_3p1_fast_multi_function_video_gen_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/veo_3p1_fast_multi_function_video_gen_handler.rs new file mode 100644 index 000000000..a141c7484 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/veo_3p1_fast_multi_function_video_gen_handler.rs @@ -0,0 +1,390 @@ +use std::collections::{HashMap, HashSet}; +use std::iter::FromIterator; +use std::sync::Arc; + +use crate::http_server::common_responses::common_web_error::CommonWebError; +use crate::http_server::common_responses::media::media_links_builder::MediaLinksBuilder; +use crate::http_server::endpoints::generate::common::payments_error_test::payments_error_test; +use crate::http_server::endpoints::media_files::helpers::get_media_domain::get_media_domain; +use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format; +use crate::state::server_state::ServerState; +use crate::util::lookup::lookup_image_urls_as_map::lookup_image_urls_as_map; +use actix_web::web::Json; +use actix_web::{web, HttpRequest}; +use artcraft_api_defs::generate::video::multi_function::veo_3p1_fast_multi_function_video_gen::{Veo3p1FastMultiFunctionVideoGenAspectRatio, Veo3p1FastMultiFunctionVideoGenDuration, Veo3p1FastMultiFunctionVideoGenRequest, Veo3p1FastMultiFunctionVideoGenResolution, Veo3p1FastMultiFunctionVideoGenResponse}; +use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; +use enums::by_table::prompt_context_items::prompt_context_semantic_type::PromptContextSemanticType; +use enums::by_table::prompts::prompt_type::PromptType; +use enums::common::generation_provider::GenerationProvider; +use enums::common::model_type::ModelType; +use enums::common::visibility::Visibility; +use fal_client::creds::open_ai_api_key::OpenAiApiKey; +use fal_client::requests::webhook::video::image::enqueue_veo_3p1_fast_first_last_frame_image_to_video_webhook::{enqueue_veo_3p1_fast_first_last_frame_image_to_video_webhook, EnqueueVeo3p1FastFirstLastFrameImageToVideoArgs, EnqueueVeo3p1FastFirstLastFrameImageToVideoAspectRatio, EnqueueVeo3p1FastFirstLastFrameImageToVideoDurationSeconds, EnqueueVeo3p1FastFirstLastFrameImageToVideoResolution}; +use fal_client::requests::webhook::video::image::enqueue_veo_3p1_fast_image_to_video_webhook::{enqueue_veo_3p1_fast_image_to_video_webhook, EnqueueVeo3p1FastImageToVideoArgs, EnqueueVeo3p1FastImageToVideoAspectRatio, EnqueueVeo3p1FastImageToVideoDurationSeconds, EnqueueVeo3p1FastImageToVideoResolution}; +use fal_client::requests::webhook::video::text::enqueue_veo_3p1_fast_text_to_video_webhook::{enqueue_veo_3p1_fast_text_to_video_webhook, EnqueueVeo3p1FastTextToVideoArgs, EnqueueVeo3p1FastTextToVideoAspectRatio, EnqueueVeo3p1FastTextToVideoDurationSeconds, EnqueueVeo3p1FastTextToVideoResolution}; +use http_server_common::request::get_request_ip::get_request_ip; +use log::{error, info, warn}; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::insert_generic_inference_job_for_fal_queue; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::FalCategory; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::InsertGenericInferenceForFalArgs; +use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token; +use mysql_queries::queries::media_files::get::batch_get_media_files_by_tokens::{batch_get_media_files_by_tokens, batch_get_media_files_by_tokens_with_connection}; +use mysql_queries::queries::prompt_context_items::insert_batch_prompt_context_items::{insert_batch_prompt_context_items, InsertBatchArgs, PromptContextItem}; +use mysql_queries::queries::prompts::insert_prompt::{insert_prompt, InsertPromptArgs}; +use server_environment::ServerEnvironment; +use sqlx::pool::PoolConnection; +use sqlx::{Acquire, MySql}; +use tokens::tokens::media_files::MediaFileToken; +use utoipa::ToSchema; + +/// Veo 3.1 Fast Multi-Function (text and image to video) +#[utoipa::path( + post, + tag = "Generate Video (Multi-Function)", + path = "/v1/generate/video/multi_function/veo_3p1_fast", + responses( + (status = 200, description = "Success", body = Veo3p1FastMultiFunctionVideoGenResponse), + ), + params( + ("request" = Veo3p1FastMultiFunctionVideoGenRequest, description = "Payload for Request"), + ) +)] +pub async fn veo_3p1_fast_multi_function_video_gen_handler( + http_request: HttpRequest, + request: Json, + server_state: web::Data> +) -> Result, CommonWebError> { + + payments_error_test(&request.prompt.as_deref().unwrap_or(""))?; + + let mut mysql_connection = server_state.mysql_pool + .acquire() + .await?; + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session_from_connection(&http_request, &mut mysql_connection) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + CommonWebError::ServerError + })?; + + let maybe_avt_token = server_state + .avt_cookie_manager + .get_avt_token_from_request(&http_request); + + // TODO: Limit usage for new accounts. Billing, free credits metering, etc. + + //let user_session = match maybe_user_session { + // Some(session) => session, + // None => { + // warn!("not logged in"); + // return Err(CommonWebError::NotAuthorized); + // } + //}; + + if let Err(reason) = validate_idempotency_token_format(&request.uuid_idempotency_token) { + return Err(CommonWebError::BadInputWithSimpleMessage(reason)); + } + + let mut query_media_tokens = None; + + if let Some(start_frame_token) = request.start_frame_image_media_token.as_ref() { + let mut tokens = Vec::new(); + tokens.push(start_frame_token.to_owned()); + + if let Some(end_frame_token) = request.end_frame_image_media_token.as_ref() { + // NB: Masks are only relevant when this is an image editing exercise. + tokens.push(end_frame_token.to_owned()); + } + + query_media_tokens = Some(tokens); + } + + let image_urls_by_token = match query_media_tokens { + None => HashMap::new(), + Some(media_tokens) => { + info!("Looking up image media tokens: {:?}", media_tokens); + lookup_image_urls_as_map( + &http_request, + &mut mysql_connection, + server_state.server_environment, + &media_tokens, + ).await? + } + }; + + let maybe_start_frame_image_url = match request.start_frame_image_media_token.as_ref() { + None => None, + Some(token) => match image_urls_by_token.get(token) { + Some(url) => Some(url.to_string()), + None => { + return Err(CommonWebError::BadInputWithSimpleMessage("Media for start frame not found.".to_string())); + } + } + }; + + let maybe_end_frame_image_url = match request.end_frame_image_media_token.as_ref() { + None => None, + Some(token) => match image_urls_by_token.get(token) { + Some(url) => Some(url.to_string()), + None => { + return Err(CommonWebError::BadInputWithSimpleMessage("Media for end frame not found.".to_string())); + } + } + }; + + insert_idempotency_token(&request.uuid_idempotency_token, &mut *mysql_connection) + .await + .map_err(|err| { + error!("Error inserting idempotency token: {:?}", err); + CommonWebError::BadInputWithSimpleMessage("repeated idempotency token".to_string()) + })?; + + info!("Fal webhook URL: {}", server_state.fal.webhook_url); + + let fal_result; + + let generate_audio = request.generate_audio.unwrap_or(true); + + if let Some(start_frame_url) = maybe_start_frame_image_url { + if let Some(end_frame_url) = maybe_end_frame_image_url { + + info!("image-to-video case (start and end frame)"); + + let duration = match request.duration { + Some(Veo3p1FastMultiFunctionVideoGenDuration::FourSeconds) => EnqueueVeo3p1FastFirstLastFrameImageToVideoDurationSeconds::Four, + Some(Veo3p1FastMultiFunctionVideoGenDuration::SixSeconds) => EnqueueVeo3p1FastFirstLastFrameImageToVideoDurationSeconds::Six, + Some(Veo3p1FastMultiFunctionVideoGenDuration::EightSeconds) => EnqueueVeo3p1FastFirstLastFrameImageToVideoDurationSeconds::Eight, + None => EnqueueVeo3p1FastFirstLastFrameImageToVideoDurationSeconds::Six, + }; + + let aspect_ratio = match request.aspect_ratio { + Some(Veo3p1FastMultiFunctionVideoGenAspectRatio::Auto) => EnqueueVeo3p1FastFirstLastFrameImageToVideoAspectRatio::Auto, + Some(Veo3p1FastMultiFunctionVideoGenAspectRatio::SixteenByNine) => EnqueueVeo3p1FastFirstLastFrameImageToVideoAspectRatio::SixteenByNine, + Some(Veo3p1FastMultiFunctionVideoGenAspectRatio::NineBySixteen) => EnqueueVeo3p1FastFirstLastFrameImageToVideoAspectRatio::NineBySixteen, + None => EnqueueVeo3p1FastFirstLastFrameImageToVideoAspectRatio::SixteenByNine, + }; + + let resolution = match request.resolution { + Some(Veo3p1FastMultiFunctionVideoGenResolution::SevenTwentyP) => EnqueueVeo3p1FastFirstLastFrameImageToVideoResolution::SevenTwentyP, + Some(Veo3p1FastMultiFunctionVideoGenResolution::TenEightyP) => EnqueueVeo3p1FastFirstLastFrameImageToVideoResolution::TenEightyP, + None => EnqueueVeo3p1FastFirstLastFrameImageToVideoResolution::TenEightyP, + }; + + let args = EnqueueVeo3p1FastFirstLastFrameImageToVideoArgs { + last_frame_url: start_frame_url, + first_frame_url: end_frame_url, + prompt: request.prompt.as_deref().unwrap_or("").to_string(), + duration: Some(duration), + aspect_ratio: Some(aspect_ratio), + resolution: Some(resolution), + generate_audio: Some(generate_audio), + webhook_url: &server_state.fal.webhook_url, + api_key: &server_state.fal.api_key, + }; + + fal_result = enqueue_veo_3p1_fast_first_last_frame_image_to_video_webhook(args) + .await + .map_err(|err| { + warn!("Error calling veo_3p1_fast_first_last_frame_image_to_video_webhook: {:?}", err); + CommonWebError::ServerError + })?; + + } else { + info!("image-to-video case (start frame only)"); + + let duration = match request.duration { + Some(Veo3p1FastMultiFunctionVideoGenDuration::FourSeconds) => EnqueueVeo3p1FastImageToVideoDurationSeconds::Four, + Some(Veo3p1FastMultiFunctionVideoGenDuration::SixSeconds) => EnqueueVeo3p1FastImageToVideoDurationSeconds::Six, + Some(Veo3p1FastMultiFunctionVideoGenDuration::EightSeconds) => EnqueueVeo3p1FastImageToVideoDurationSeconds::Eight, + None => EnqueueVeo3p1FastImageToVideoDurationSeconds::Six, + }; + + let aspect_ratio = match request.aspect_ratio { + Some(Veo3p1FastMultiFunctionVideoGenAspectRatio::Auto) => EnqueueVeo3p1FastImageToVideoAspectRatio::Auto, + Some(Veo3p1FastMultiFunctionVideoGenAspectRatio::SixteenByNine) => EnqueueVeo3p1FastImageToVideoAspectRatio::SixteenByNine, + Some(Veo3p1FastMultiFunctionVideoGenAspectRatio::NineBySixteen) => EnqueueVeo3p1FastImageToVideoAspectRatio::NineBySixteen, + None => EnqueueVeo3p1FastImageToVideoAspectRatio::SixteenByNine, + }; + + let resolution = match request.resolution { + Some(Veo3p1FastMultiFunctionVideoGenResolution::SevenTwentyP) => EnqueueVeo3p1FastImageToVideoResolution::SevenTwentyP, + Some(Veo3p1FastMultiFunctionVideoGenResolution::TenEightyP) => EnqueueVeo3p1FastImageToVideoResolution::TenEightyP, + None => EnqueueVeo3p1FastImageToVideoResolution::TenEightyP, + }; + + let args = EnqueueVeo3p1FastImageToVideoArgs { + prompt: request.prompt.as_deref().unwrap_or("").to_string(), + image_url: start_frame_url, + duration: Some(duration), + aspect_ratio: Some(aspect_ratio), + resolution: Some(resolution), + generate_audio: Some(generate_audio), + webhook_url: &server_state.fal.webhook_url, + api_key: &server_state.fal.api_key, + }; + + fal_result = enqueue_veo_3p1_fast_image_to_video_webhook(args) + .await + .map_err(|err| { + warn!("Error calling enqueue_veo_3p1_fast_image_to_video_webhook: {:?}", err); + CommonWebError::ServerError + })?; + } + + } else { + info!("text-to-video case"); + + let duration = match request.duration { + Some(Veo3p1FastMultiFunctionVideoGenDuration::FourSeconds) => EnqueueVeo3p1FastTextToVideoDurationSeconds::Four, + Some(Veo3p1FastMultiFunctionVideoGenDuration::SixSeconds) => EnqueueVeo3p1FastTextToVideoDurationSeconds::Six, + Some(Veo3p1FastMultiFunctionVideoGenDuration::EightSeconds) => EnqueueVeo3p1FastTextToVideoDurationSeconds::Eight, + None => EnqueueVeo3p1FastTextToVideoDurationSeconds::Six, + }; + + let aspect_ratio = match request.aspect_ratio { + Some(Veo3p1FastMultiFunctionVideoGenAspectRatio::Auto) => EnqueueVeo3p1FastTextToVideoAspectRatio::Auto, + Some(Veo3p1FastMultiFunctionVideoGenAspectRatio::SixteenByNine) => EnqueueVeo3p1FastTextToVideoAspectRatio::SixteenByNine, + Some(Veo3p1FastMultiFunctionVideoGenAspectRatio::NineBySixteen) => EnqueueVeo3p1FastTextToVideoAspectRatio::NineBySixteen, + None => EnqueueVeo3p1FastTextToVideoAspectRatio::SixteenByNine, + }; + + let resolution = match request.resolution { + Some(Veo3p1FastMultiFunctionVideoGenResolution::SevenTwentyP) => EnqueueVeo3p1FastTextToVideoResolution::SevenTwentyP, + Some(Veo3p1FastMultiFunctionVideoGenResolution::TenEightyP) => EnqueueVeo3p1FastTextToVideoResolution::TenEightyP, + None => EnqueueVeo3p1FastTextToVideoResolution::TenEightyP, + }; + + let args = EnqueueVeo3p1FastTextToVideoArgs { + prompt: request.prompt.as_deref().unwrap_or("").to_string(), + negative_prompt: request.negative_prompt.clone(), + duration: Some(duration), + aspect_ratio: Some(aspect_ratio), + resolution: Some(resolution), + generate_audio: Some(generate_audio), + enhance_prompt: Some(true), + auto_fix: Some(true), + seed: None, + webhook_url: &server_state.fal.webhook_url, + api_key: &server_state.fal.api_key, + }; + + fal_result = enqueue_veo_3p1_fast_text_to_video_webhook(args) + .await + .map_err(|err| { + warn!("Error calling enqueue_veo_3p1_fast_text_to_video_webhook: {:?}", err); + CommonWebError::ServerError + })?; + } + + let external_job_id = fal_result.request_id + .ok_or_else(|| { + warn!("Fal request_id is None"); + CommonWebError::ServerError + })?; + + info!("Fal request_id: {}", external_job_id); + + let ip_address = get_request_ip(&http_request); + + let mut transaction = mysql_connection + .begin() + .await + .map_err(|err| { + error!("Error starting MySQL transaction: {:?}", err); + CommonWebError::ServerError + })?; + + // NB: Don't fail the job if the query fails. + let prompt_result = insert_prompt(InsertPromptArgs { + maybe_apriori_prompt_token: None, + prompt_type: PromptType::ArtcraftApp, + maybe_creator_user_token: maybe_user_session + .as_ref() + .map(|s| &s.user_token), + maybe_model_type: Some(ModelType::Veo3p1Fast), + maybe_generation_provider: Some(GenerationProvider::Artcraft), + maybe_positive_prompt: request.prompt.as_deref(), + maybe_negative_prompt: None, + maybe_other_args: None, + creator_ip_address: &ip_address, + mysql_executor: &mut *transaction, + phantom: Default::default(), + }).await; + + let prompt_token = match prompt_result { + Ok(token) => Some(token), + Err(err) => { + warn!("Error inserting prompt: {:?}", err); + None // Don't fail the job if the prompt insertion fails. + } + }; + + if let Some(token) = prompt_token.as_ref() { + let mut context_items = Vec::with_capacity(2); + + if let Some(media_token) = &request.start_frame_image_media_token { + context_items.push(PromptContextItem { + media_token: media_token.clone(), + context_semantic_type: PromptContextSemanticType::VidStartFrame, + }); + } + + if let Some(media_token) = &request.end_frame_image_media_token { + context_items.push(PromptContextItem { + media_token: media_token.clone(), + context_semantic_type: PromptContextSemanticType::VidEndFrame, + }); + } + + if !context_items.is_empty() { + let result = insert_batch_prompt_context_items(InsertBatchArgs { + prompt_token: token.clone(), + items: context_items, + transaction: &mut transaction, + }).await; + + if let Err(err) = result { + // NB: Fail open. + warn!("Error inserting batch prompt context items: {:?}", err); + } + } + } + + let db_result = insert_generic_inference_job_for_fal_queue(InsertGenericInferenceForFalArgs { + uuid_idempotency_token: &request.uuid_idempotency_token, + maybe_external_third_party_id: &external_job_id, + fal_category: FalCategory::VideoGeneration, + maybe_inference_args: None, + maybe_prompt_token: prompt_token.as_ref(), + maybe_creator_user_token: maybe_user_session.as_ref().map(|s| &s.user_token), + maybe_avt_token: maybe_avt_token.as_ref(), + creator_ip_address: &ip_address, + creator_set_visibility: Visibility::Public, + mysql_executor: &mut *transaction, + phantom: Default::default(), + }).await; + + let job_token = match db_result { + Ok(token) => token, + Err(err) => { + warn!("Error inserting generic inference job for FAL queue: {:?}", err); + return Err(CommonWebError::ServerError); + } + }; + + let _r = transaction + .commit() + .await + .map_err(|err| { + error!("Error committing MySQL transaction: {:?}", err); + CommonWebError::ServerError + })?; + + Ok(Json(Veo3p1FastMultiFunctionVideoGenResponse { + success: true, + inference_job_token: job_token, + })) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/veo_3p1_multi_function_video_gen_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/veo_3p1_multi_function_video_gen_handler.rs new file mode 100644 index 000000000..a5a6df56a --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/video/multi_function/veo_3p1_multi_function_video_gen_handler.rs @@ -0,0 +1,390 @@ +use std::collections::{HashMap, HashSet}; +use std::iter::FromIterator; +use std::sync::Arc; + +use crate::http_server::common_responses::common_web_error::CommonWebError; +use crate::http_server::common_responses::media::media_links_builder::MediaLinksBuilder; +use crate::http_server::endpoints::generate::common::payments_error_test::payments_error_test; +use crate::http_server::endpoints::media_files::helpers::get_media_domain::get_media_domain; +use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format; +use crate::state::server_state::ServerState; +use crate::util::lookup::lookup_image_urls_as_map::lookup_image_urls_as_map; +use actix_web::web::Json; +use actix_web::{web, HttpRequest}; +use artcraft_api_defs::generate::video::multi_function::veo_3p1_multi_function_video_gen::{Veo3p1MultiFunctionVideoGenAspectRatio, Veo3p1MultiFunctionVideoGenDuration, Veo3p1MultiFunctionVideoGenRequest, Veo3p1MultiFunctionVideoGenResolution, Veo3p1MultiFunctionVideoGenResponse}; +use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; +use enums::by_table::prompt_context_items::prompt_context_semantic_type::PromptContextSemanticType; +use enums::by_table::prompts::prompt_type::PromptType; +use enums::common::generation_provider::GenerationProvider; +use enums::common::model_type::ModelType; +use enums::common::visibility::Visibility; +use fal_client::creds::open_ai_api_key::OpenAiApiKey; +use fal_client::requests::webhook::video::image::enqueue_veo_3p1_first_last_frame_image_to_video_webhook::{enqueue_veo_3p1_first_last_frame_image_to_video_webhook, EnqueueVeo3p1FirstLastFrameImageToVideoArgs, EnqueueVeo3p1FirstLastFrameImageToVideoAspectRatio, EnqueueVeo3p1FirstLastFrameImageToVideoDurationSeconds, EnqueueVeo3p1FirstLastFrameImageToVideoResolution}; +use fal_client::requests::webhook::video::image::enqueue_veo_3p1_image_to_video_webhook::{enqueue_veo_3p1_image_to_video_webhook, EnqueueVeo3p1ImageToVideoArgs, EnqueueVeo3p1ImageToVideoAspectRatio, EnqueueVeo3p1ImageToVideoDurationSeconds, EnqueueVeo3p1ImageToVideoResolution}; +use fal_client::requests::webhook::video::text::enqueue_veo_3p1_text_to_video_webhook::{enqueue_veo_3p1_text_to_video_webhook, EnqueueVeo3p1TextToVideoArgs, EnqueueVeo3p1TextToVideoAspectRatio, EnqueueVeo3p1TextToVideoDurationSeconds, EnqueueVeo3p1TextToVideoResolution}; +use http_server_common::request::get_request_ip::get_request_ip; +use log::{error, info, warn}; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::insert_generic_inference_job_for_fal_queue; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::FalCategory; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::InsertGenericInferenceForFalArgs; +use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token; +use mysql_queries::queries::media_files::get::batch_get_media_files_by_tokens::{batch_get_media_files_by_tokens, batch_get_media_files_by_tokens_with_connection}; +use mysql_queries::queries::prompt_context_items::insert_batch_prompt_context_items::{insert_batch_prompt_context_items, InsertBatchArgs, PromptContextItem}; +use mysql_queries::queries::prompts::insert_prompt::{insert_prompt, InsertPromptArgs}; +use server_environment::ServerEnvironment; +use sqlx::pool::PoolConnection; +use sqlx::{Acquire, MySql}; +use tokens::tokens::media_files::MediaFileToken; +use utoipa::ToSchema; + +/// Veo 3.1 Multi-Function (text and image to video) +#[utoipa::path( + post, + tag = "Generate Video (Multi-Function)", + path = "/v1/generate/video/multi_function/veo_3p1", + responses( + (status = 200, description = "Success", body = Veo3p1MultiFunctionVideoGenResponse), + ), + params( + ("request" = Veo3p1MultiFunctionVideoGenRequest, description = "Payload for Request"), + ) +)] +pub async fn veo_3p1_multi_function_video_gen_handler( + http_request: HttpRequest, + request: Json, + server_state: web::Data> +) -> Result, CommonWebError> { + + payments_error_test(&request.prompt.as_deref().unwrap_or(""))?; + + let mut mysql_connection = server_state.mysql_pool + .acquire() + .await?; + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session_from_connection(&http_request, &mut mysql_connection) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + CommonWebError::ServerError + })?; + + let maybe_avt_token = server_state + .avt_cookie_manager + .get_avt_token_from_request(&http_request); + + // TODO: Limit usage for new accounts. Billing, free credits metering, etc. + + //let user_session = match maybe_user_session { + // Some(session) => session, + // None => { + // warn!("not logged in"); + // return Err(CommonWebError::NotAuthorized); + // } + //}; + + if let Err(reason) = validate_idempotency_token_format(&request.uuid_idempotency_token) { + return Err(CommonWebError::BadInputWithSimpleMessage(reason)); + } + + let mut query_media_tokens = None; + + if let Some(start_frame_token) = request.start_frame_image_media_token.as_ref() { + let mut tokens = Vec::new(); + tokens.push(start_frame_token.to_owned()); + + if let Some(end_frame_token) = request.end_frame_image_media_token.as_ref() { + // NB: Masks are only relevant when this is an image editing exercise. + tokens.push(end_frame_token.to_owned()); + } + + query_media_tokens = Some(tokens); + } + + let image_urls_by_token = match query_media_tokens { + None => HashMap::new(), + Some(media_tokens) => { + info!("Looking up image media tokens: {:?}", media_tokens); + lookup_image_urls_as_map( + &http_request, + &mut mysql_connection, + server_state.server_environment, + &media_tokens, + ).await? + } + }; + + let maybe_start_frame_image_url = match request.start_frame_image_media_token.as_ref() { + None => None, + Some(token) => match image_urls_by_token.get(token) { + Some(url) => Some(url.to_string()), + None => { + return Err(CommonWebError::BadInputWithSimpleMessage("Media for start frame not found.".to_string())); + } + } + }; + + let maybe_end_frame_image_url = match request.end_frame_image_media_token.as_ref() { + None => None, + Some(token) => match image_urls_by_token.get(token) { + Some(url) => Some(url.to_string()), + None => { + return Err(CommonWebError::BadInputWithSimpleMessage("Media for end frame not found.".to_string())); + } + } + }; + + insert_idempotency_token(&request.uuid_idempotency_token, &mut *mysql_connection) + .await + .map_err(|err| { + error!("Error inserting idempotency token: {:?}", err); + CommonWebError::BadInputWithSimpleMessage("repeated idempotency token".to_string()) + })?; + + info!("Fal webhook URL: {}", server_state.fal.webhook_url); + + let fal_result; + + let generate_audio = request.generate_audio.unwrap_or(true); + + if let Some(start_frame_url) = maybe_start_frame_image_url { + if let Some(end_frame_url) = maybe_end_frame_image_url { + + info!("image-to-video case (start and end frame)"); + + let duration = match request.duration { + Some(Veo3p1MultiFunctionVideoGenDuration::FourSeconds) => EnqueueVeo3p1FirstLastFrameImageToVideoDurationSeconds::Four, + Some(Veo3p1MultiFunctionVideoGenDuration::SixSeconds) => EnqueueVeo3p1FirstLastFrameImageToVideoDurationSeconds::Six, + Some(Veo3p1MultiFunctionVideoGenDuration::EightSeconds) => EnqueueVeo3p1FirstLastFrameImageToVideoDurationSeconds::Eight, + None => EnqueueVeo3p1FirstLastFrameImageToVideoDurationSeconds::Six, + }; + + let aspect_ratio = match request.aspect_ratio { + Some(Veo3p1MultiFunctionVideoGenAspectRatio::Auto) => EnqueueVeo3p1FirstLastFrameImageToVideoAspectRatio::Auto, + Some(Veo3p1MultiFunctionVideoGenAspectRatio::SixteenByNine) => EnqueueVeo3p1FirstLastFrameImageToVideoAspectRatio::SixteenByNine, + Some(Veo3p1MultiFunctionVideoGenAspectRatio::NineBySixteen) => EnqueueVeo3p1FirstLastFrameImageToVideoAspectRatio::NineBySixteen, + None => EnqueueVeo3p1FirstLastFrameImageToVideoAspectRatio::SixteenByNine, + }; + + let resolution = match request.resolution { + Some(Veo3p1MultiFunctionVideoGenResolution::SevenTwentyP) => EnqueueVeo3p1FirstLastFrameImageToVideoResolution::SevenTwentyP, + Some(Veo3p1MultiFunctionVideoGenResolution::TenEightyP) => EnqueueVeo3p1FirstLastFrameImageToVideoResolution::TenEightyP, + None => EnqueueVeo3p1FirstLastFrameImageToVideoResolution::TenEightyP, + }; + + let args = EnqueueVeo3p1FirstLastFrameImageToVideoArgs { + last_frame_url: start_frame_url, + first_frame_url: end_frame_url, + prompt: request.prompt.as_deref().unwrap_or("").to_string(), + duration: Some(duration), + aspect_ratio: Some(aspect_ratio), + resolution: Some(resolution), + generate_audio: Some(generate_audio), + webhook_url: &server_state.fal.webhook_url, + api_key: &server_state.fal.api_key, + }; + + fal_result = enqueue_veo_3p1_first_last_frame_image_to_video_webhook(args) + .await + .map_err(|err| { + warn!("Error calling veo_3p1_first_last_frame_image_to_video_webhook: {:?}", err); + CommonWebError::ServerError + })?; + + } else { + info!("image-to-video case (start frame only)"); + + let duration = match request.duration { + Some(Veo3p1MultiFunctionVideoGenDuration::FourSeconds) => EnqueueVeo3p1ImageToVideoDurationSeconds::Four, + Some(Veo3p1MultiFunctionVideoGenDuration::SixSeconds) => EnqueueVeo3p1ImageToVideoDurationSeconds::Six, + Some(Veo3p1MultiFunctionVideoGenDuration::EightSeconds) => EnqueueVeo3p1ImageToVideoDurationSeconds::Eight, + None => EnqueueVeo3p1ImageToVideoDurationSeconds::Six, + }; + + let aspect_ratio = match request.aspect_ratio { + Some(Veo3p1MultiFunctionVideoGenAspectRatio::Auto) => EnqueueVeo3p1ImageToVideoAspectRatio::Auto, + Some(Veo3p1MultiFunctionVideoGenAspectRatio::SixteenByNine) => EnqueueVeo3p1ImageToVideoAspectRatio::SixteenByNine, + Some(Veo3p1MultiFunctionVideoGenAspectRatio::NineBySixteen) => EnqueueVeo3p1ImageToVideoAspectRatio::NineBySixteen, + None => EnqueueVeo3p1ImageToVideoAspectRatio::SixteenByNine, + }; + + let resolution = match request.resolution { + Some(Veo3p1MultiFunctionVideoGenResolution::SevenTwentyP) => EnqueueVeo3p1ImageToVideoResolution::SevenTwentyP, + Some(Veo3p1MultiFunctionVideoGenResolution::TenEightyP) => EnqueueVeo3p1ImageToVideoResolution::TenEightyP, + None => EnqueueVeo3p1ImageToVideoResolution::TenEightyP, + }; + + let args = EnqueueVeo3p1ImageToVideoArgs { + prompt: request.prompt.as_deref().unwrap_or("").to_string(), + image_url: start_frame_url, + duration: Some(duration), + aspect_ratio: Some(aspect_ratio), + resolution: Some(resolution), + generate_audio: Some(generate_audio), + webhook_url: &server_state.fal.webhook_url, + api_key: &server_state.fal.api_key, + }; + + fal_result = enqueue_veo_3p1_image_to_video_webhook(args) + .await + .map_err(|err| { + warn!("Error calling enqueue_veo_3p1_image_to_video_webhook: {:?}", err); + CommonWebError::ServerError + })?; + } + + } else { + info!("text-to-video case"); + + let duration = match request.duration { + Some(Veo3p1MultiFunctionVideoGenDuration::FourSeconds) => EnqueueVeo3p1TextToVideoDurationSeconds::Four, + Some(Veo3p1MultiFunctionVideoGenDuration::SixSeconds) => EnqueueVeo3p1TextToVideoDurationSeconds::Six, + Some(Veo3p1MultiFunctionVideoGenDuration::EightSeconds) => EnqueueVeo3p1TextToVideoDurationSeconds::Eight, + None => EnqueueVeo3p1TextToVideoDurationSeconds::Six, + }; + + let aspect_ratio = match request.aspect_ratio { + Some(Veo3p1MultiFunctionVideoGenAspectRatio::Auto) => EnqueueVeo3p1TextToVideoAspectRatio::Auto, + Some(Veo3p1MultiFunctionVideoGenAspectRatio::SixteenByNine) => EnqueueVeo3p1TextToVideoAspectRatio::SixteenByNine, + Some(Veo3p1MultiFunctionVideoGenAspectRatio::NineBySixteen) => EnqueueVeo3p1TextToVideoAspectRatio::NineBySixteen, + None => EnqueueVeo3p1TextToVideoAspectRatio::SixteenByNine, + }; + + let resolution = match request.resolution { + Some(Veo3p1MultiFunctionVideoGenResolution::SevenTwentyP) => EnqueueVeo3p1TextToVideoResolution::SevenTwentyP, + Some(Veo3p1MultiFunctionVideoGenResolution::TenEightyP) => EnqueueVeo3p1TextToVideoResolution::TenEightyP, + None => EnqueueVeo3p1TextToVideoResolution::TenEightyP, + }; + + let args = EnqueueVeo3p1TextToVideoArgs { + prompt: request.prompt.as_deref().unwrap_or("").to_string(), + negative_prompt: request.negative_prompt.clone(), + duration: Some(duration), + aspect_ratio: Some(aspect_ratio), + resolution: Some(resolution), + generate_audio: Some(generate_audio), + enhance_prompt: Some(true), + auto_fix: Some(true), + seed: None, + webhook_url: &server_state.fal.webhook_url, + api_key: &server_state.fal.api_key, + }; + + fal_result = enqueue_veo_3p1_text_to_video_webhook(args) + .await + .map_err(|err| { + warn!("Error calling enqueue_veo_3p1_text_to_video_webhook: {:?}", err); + CommonWebError::ServerError + })?; + } + + let external_job_id = fal_result.request_id + .ok_or_else(|| { + warn!("Fal request_id is None"); + CommonWebError::ServerError + })?; + + info!("Fal request_id: {}", external_job_id); + + let ip_address = get_request_ip(&http_request); + + let mut transaction = mysql_connection + .begin() + .await + .map_err(|err| { + error!("Error starting MySQL transaction: {:?}", err); + CommonWebError::ServerError + })?; + + // NB: Don't fail the job if the query fails. + let prompt_result = insert_prompt(InsertPromptArgs { + maybe_apriori_prompt_token: None, + prompt_type: PromptType::ArtcraftApp, + maybe_creator_user_token: maybe_user_session + .as_ref() + .map(|s| &s.user_token), + maybe_model_type: Some(ModelType::Veo3p1), + maybe_generation_provider: Some(GenerationProvider::Artcraft), + maybe_positive_prompt: request.prompt.as_deref(), + maybe_negative_prompt: None, + maybe_other_args: None, + creator_ip_address: &ip_address, + mysql_executor: &mut *transaction, + phantom: Default::default(), + }).await; + + let prompt_token = match prompt_result { + Ok(token) => Some(token), + Err(err) => { + warn!("Error inserting prompt: {:?}", err); + None // Don't fail the job if the prompt insertion fails. + } + }; + + if let Some(token) = prompt_token.as_ref() { + let mut context_items = Vec::with_capacity(2); + + if let Some(media_token) = &request.start_frame_image_media_token { + context_items.push(PromptContextItem { + media_token: media_token.clone(), + context_semantic_type: PromptContextSemanticType::VidStartFrame, + }); + } + + if let Some(media_token) = &request.end_frame_image_media_token { + context_items.push(PromptContextItem { + media_token: media_token.clone(), + context_semantic_type: PromptContextSemanticType::VidEndFrame, + }); + } + + if !context_items.is_empty() { + let result = insert_batch_prompt_context_items(InsertBatchArgs { + prompt_token: token.clone(), + items: context_items, + transaction: &mut transaction, + }).await; + + if let Err(err) = result { + // NB: Fail open. + warn!("Error inserting batch prompt context items: {:?}", err); + } + } + } + + let db_result = insert_generic_inference_job_for_fal_queue(InsertGenericInferenceForFalArgs { + uuid_idempotency_token: &request.uuid_idempotency_token, + maybe_external_third_party_id: &external_job_id, + fal_category: FalCategory::VideoGeneration, + maybe_inference_args: None, + maybe_prompt_token: prompt_token.as_ref(), + maybe_creator_user_token: maybe_user_session.as_ref().map(|s| &s.user_token), + maybe_avt_token: maybe_avt_token.as_ref(), + creator_ip_address: &ip_address, + creator_set_visibility: Visibility::Public, + mysql_executor: &mut *transaction, + phantom: Default::default(), + }).await; + + let job_token = match db_result { + Ok(token) => token, + Err(err) => { + warn!("Error inserting generic inference job for FAL queue: {:?}", err); + return Err(CommonWebError::ServerError); + } + }; + + let _r = transaction + .commit() + .await + .map_err(|err| { + error!("Error committing MySQL transaction: {:?}", err); + CommonWebError::ServerError + })?; + + Ok(Json(Veo3p1MultiFunctionVideoGenResponse { + success: true, + inference_job_token: job_token, + })) +} diff --git a/crates/service/web/storyteller_web/src/http_server/routes/application_routes/generate_routes.rs b/crates/service/web/storyteller_web/src/http_server/routes/application_routes/generate_routes.rs index 76f3f78cf..df9fa2749 100644 --- a/crates/service/web/storyteller_web/src/http_server/routes/application_routes/generate_routes.rs +++ b/crates/service/web/storyteller_web/src/http_server/routes/application_routes/generate_routes.rs @@ -25,6 +25,12 @@ use crate::http_server::endpoints::generate::video::generate_seedance_1_0_pro_im use crate::http_server::endpoints::generate::video::generate_veo_2_image_to_video_handler::generate_veo_2_image_to_video_handler; use crate::http_server::endpoints::generate::video::generate_veo_3_fast_image_to_video_handler::generate_veo_3_fast_image_to_video_handler; use crate::http_server::endpoints::generate::video::generate_veo_3_image_to_video_handler::generate_veo_3_image_to_video_handler; +use crate::http_server::endpoints::generate::video::multi_function::kling_2p5_turbo_pro_multi_function_video_gen_handler::kling_2p5_turbo_pro_multi_function_video_gen_handler; +use crate::http_server::endpoints::generate::video::multi_function::kling_2p6_pro_multi_function_video_gen_handler::kling_2p6_pro_multi_function_video_gen_handler; +use crate::http_server::endpoints::generate::video::multi_function::sora_2_multi_function_video_gen_handler::sora_2_multi_function_video_gen_handler; +use crate::http_server::endpoints::generate::video::multi_function::sora_2_pro_multi_function_video_gen_handler::sora_2_pro_multi_function_video_gen_handler; +use crate::http_server::endpoints::generate::video::multi_function::veo_3p1_fast_multi_function_video_gen_handler::veo_3p1_fast_multi_function_video_gen_handler; +use crate::http_server::endpoints::generate::video::multi_function::veo_3p1_multi_function_video_gen_handler::veo_3p1_multi_function_video_gen_handler; use actix_http::body::MessageBody; use actix_service::ServiceFactory; use actix_web::dev::{ServiceRequest, ServiceResponse}; @@ -119,6 +125,32 @@ where ) ) .service(web::scope("/video") + .service(web::scope("/multi_function") + .service(web::resource("/kling_2p5_turbo_pro") + .route(web::post().to(kling_2p5_turbo_pro_multi_function_video_gen_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service(web::resource("/kling_2p6_pro") + .route(web::post().to(kling_2p6_pro_multi_function_video_gen_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service(web::resource("/sora_2") + .route(web::post().to(sora_2_multi_function_video_gen_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service(web::resource("/sora_2_pro") + .route(web::post().to(sora_2_pro_multi_function_video_gen_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service(web::resource("/veo_3p1") + .route(web::post().to(veo_3p1_multi_function_video_gen_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + .service(web::resource("/veo_3p1_fast") + .route(web::post().to(veo_3p1_fast_multi_function_video_gen_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + ) .service(web::resource("/kling_1.6_pro_image_to_video") .route(web::post().to(generate_kling_1_6_pro_video_handler)) .route(web::head().to(|| HttpResponse::Ok()))