diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/video/enqueue_image_to_video_command.rs b/crates/desktop/artcraft/src/core/commands/enqueue/video/enqueue_image_to_video_command.rs index cb80436360..b94a4ede32 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/video/enqueue_image_to_video_command.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/video/enqueue_image_to_video_command.rs @@ -7,6 +7,7 @@ use crate::core::commands::response::failure_response_wrapper::{CommandErrorResp use crate::core::commands::response::shorthand::Response; use crate::core::commands::response::success_response_wrapper::SerializeMarker; use crate::core::events::sendable_event_trait::SendableEvent; +use crate::core::model::video_models::VideoModel; use crate::core::state::data_dir::app_data_root::AppDataRoot; use crate::core::state::data_dir::trait_data_subdir::DataSubdir; use crate::core::utils::download_media_file_to_temp_dir::download_media_file_to_temp_dir; @@ -66,26 +67,9 @@ pub struct EnqueueImageToVideoRequest { pub image_media_token: Option, /// The model to use. - pub model: Option, + pub model: Option, } -#[derive(Deserialize, Debug, Copy, Clone)] -#[serde(rename_all = "snake_case")] -pub enum EnqueueImageToVideoModel { - #[deprecated(note = "Use `kling_1_6_pro` instead")] - #[serde(rename = "kling1_6")] - Kling16, - #[serde(rename = "kling_1_6_pro")] - Kling16Pro, - #[serde(rename = "kling_2_1_pro")] - Kling21Pro, - #[serde(rename = "kling_2_1_master")] - Kling21Master, - #[serde(rename = "seedance_1_0_lite")] - Seedance10Lite, - #[serde(rename = "veo_2")] - Veo2, -} #[derive(Serialize)] pub struct EnqueueImageToVideoSuccessResponse { diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/video/handle_video_artcraft.rs b/crates/desktop/artcraft/src/core/commands/enqueue/video/handle_video_artcraft.rs index bc64caa1e8..873760ec5a 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/video/handle_video_artcraft.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/video/handle_video_artcraft.rs @@ -1,11 +1,12 @@ use crate::core::commands::enqueue::image::enqueue_text_to_image_command::{EnqueueTextToImageModel, EnqueueTextToImageRequest}; use crate::core::commands::enqueue::object::enqueue_image_to_3d_object_command::{EnqueueImageTo3dObjectModel, EnqueueImageTo3dObjectRequest}; -use crate::core::commands::enqueue::video::enqueue_image_to_video_command::{EnqueueImageToVideoModel, EnqueueImageToVideoRequest}; +use crate::core::commands::enqueue::video::enqueue_image_to_video_command::EnqueueImageToVideoRequest; use crate::core::commands::enqueue::video::internal_video_error::InternalVideoError; use crate::core::events::basic_sendable_event_trait::BasicSendableEvent; use crate::core::events::generation_events::common::{GenerationAction, GenerationModel, GenerationServiceProvider}; use crate::core::events::generation_events::generation_enqueue_failure_event::GenerationEnqueueFailureEvent; use crate::core::events::generation_events::generation_enqueue_success_event::GenerationEnqueueSuccessEvent; +use crate::core::model::video_models::VideoModel; use crate::core::state::data_dir::app_data_root::AppDataRoot; use crate::core::utils::download_media_file_to_temp_dir::download_media_file_to_temp_dir; use crate::core::utils::save_base64_image_to_temp_dir::save_base64_image_to_temp_dir; @@ -17,26 +18,26 @@ use crate::services::storyteller::state::storyteller_credential_manager::Storyte use anyhow::anyhow; use artcraft_api_defs::generate::object::generate_hunyuan_2_0_image_to_3d::GenerateHunyuan20ImageTo3dRequest; use artcraft_api_defs::generate::video::generate_kling_1_6_pro_image_to_video::{GenerateKling16ProAspectRatio, GenerateKling16ProImageToVideoRequest}; +use artcraft_api_defs::generate::video::generate_kling_2_1_master_image_to_video::{GenerateKling21MasterAspectRatio, GenerateKling21MasterImageToVideoRequest}; +use artcraft_api_defs::generate::video::generate_kling_2_1_pro_image_to_video::{GenerateKling21ProAspectRatio, GenerateKling21ProImageToVideoRequest}; +use artcraft_api_defs::generate::video::generate_seedance_1_0_lite_image_to_video::GenerateSeedance10LiteImageToVideoRequest; +use artcraft_api_defs::generate::video::generate_veo_2_image_to_video::{GenerateVeo2AspectRatio, GenerateVeo2ImageToVideoRequest}; use fal_client::creds::fal_api_key::FalApiKey; use fal_client::requests::queue::enqueue_hunyuan2_image_to_3d::{enqueue_hunyuan2_image_to_3d, Hunyuan2Args}; use fal_client::requests::queue::image_gen::enqueue_flux_pro_11_ultra_text_to_image::{enqueue_flux_pro_11_ultra_text_to_image, FluxPro11UltraTextToImageArgs}; use fal_client::requests::queue::image_gen::enqueue_recraft3_text_to_image::{enqueue_recraft3_text_to_image, Recraft3TextToImageArgs}; use fal_client::requests::queue::video_gen::enqueue_kling_16_pro_image_to_video::Kling16ProAspectRatio; +use fal_client::requests::webhook::video::enqueue_veo_2_image_to_video_webhook::Veo2AspectRatio; use idempotency::uuid::generate_random_uuid; use log::{error, info, warn}; use storyteller_client::generate::object::generate_hunyuan_3d_2_0_image_to_3d::generate_hunyuan3d_2_0_image_to_3d; use storyteller_client::generate::video::generate_kling_16_pro_image_to_video::generate_kling_16_pro_image_to_video; -use storyteller_client::utils::api_host::ApiHost; -use tauri::{AppHandle, State}; -use artcraft_api_defs::generate::video::generate_kling_2_1_master_image_to_video::{GenerateKling21MasterAspectRatio, GenerateKling21MasterImageToVideoRequest}; -use artcraft_api_defs::generate::video::generate_kling_2_1_pro_image_to_video::{GenerateKling21ProAspectRatio, GenerateKling21ProImageToVideoRequest}; -use artcraft_api_defs::generate::video::generate_seedance_1_0_lite_image_to_video::GenerateSeedance10LiteImageToVideoRequest; -use artcraft_api_defs::generate::video::generate_veo_2_image_to_video::{GenerateVeo2AspectRatio, GenerateVeo2ImageToVideoRequest}; -use fal_client::requests::webhook::video::enqueue_veo_2_image_to_video_webhook::Veo2AspectRatio; use storyteller_client::generate::video::generate_kling_21_master_image_to_video::generate_kling_21_master_image_to_video; use storyteller_client::generate::video::generate_kling_21_pro_image_to_video::generate_kling_21_pro_image_to_video; use storyteller_client::generate::video::generate_seedance_1_0_lite_image_to_video::generate_seedance_1_0_lite_image_to_video; use storyteller_client::generate::video::generate_veo_2_image_to_video::generate_veo_2_image_to_video; +use storyteller_client::utils::api_host::ApiHost; +use tauri::{AppHandle, State}; pub async fn handle_video_artcraft( request: EnqueueImageToVideoRequest, @@ -78,10 +79,7 @@ pub async fn handle_video_artcraft( None => { return Err(InternalVideoError::NoModelSpecified); } - Some( - EnqueueImageToVideoModel::Kling16 | - EnqueueImageToVideoModel::Kling16Pro - ) => { + Some(VideoModel::Kling16Pro) => { info!("enqueue Kling 1.6 Pro with Artcraft API"); selected_model = Some(GenerationModel::Kling1_6); let request = GenerateKling16ProImageToVideoRequest { @@ -107,7 +105,7 @@ pub async fn handle_video_artcraft( } } } - Some(EnqueueImageToVideoModel::Kling21Master) => { + Some(VideoModel::Kling21Master) => { info!("enqueue Kling 2.1 Master with Artcraft API"); selected_model = Some(GenerationModel::Kling21Master); let request = GenerateKling21MasterImageToVideoRequest { @@ -133,7 +131,7 @@ pub async fn handle_video_artcraft( } } } - Some(EnqueueImageToVideoModel::Kling21Pro) => { + Some(VideoModel::Kling21Pro) => { info!("enqueue Kling 2.1 Pro with Artcraft API"); selected_model = Some(GenerationModel::Kling21Pro); let request = GenerateKling21ProImageToVideoRequest { @@ -159,7 +157,7 @@ pub async fn handle_video_artcraft( } } } - Some(EnqueueImageToVideoModel::Seedance10Lite) => { + Some(VideoModel::Seedance10Lite) => { info!("enqueue Seedance 1.0 Lite with Artcraft API"); selected_model = Some(GenerationModel::Seedance10Lite); let request = GenerateSeedance10LiteImageToVideoRequest { @@ -185,7 +183,7 @@ pub async fn handle_video_artcraft( } } } - Some(EnqueueImageToVideoModel::Veo2) => { + Some(VideoModel::Veo2) => { info!("enqueue Veo 2 with Artcraft API"); selected_model = Some(GenerationModel::Veo2); let request = GenerateVeo2ImageToVideoRequest { diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/video/handle_video_fal.rs b/crates/desktop/artcraft/src/core/commands/enqueue/video/handle_video_fal.rs index 4ce4aa1c4c..f7ecd0c4af 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/video/handle_video_fal.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/video/handle_video_fal.rs @@ -1,10 +1,11 @@ use crate::core::commands::enqueue::image::enqueue_text_to_image_command::{EnqueueTextToImageModel, EnqueueTextToImageRequest}; -use crate::core::commands::enqueue::video::enqueue_image_to_video_command::{EnqueueImageToVideoModel, EnqueueImageToVideoRequest}; +use crate::core::commands::enqueue::video::enqueue_image_to_video_command::EnqueueImageToVideoRequest; use crate::core::commands::enqueue::video::internal_video_error::InternalVideoError; use crate::core::events::basic_sendable_event_trait::BasicSendableEvent; use crate::core::events::generation_events::common::{GenerationAction, GenerationModel, GenerationServiceProvider}; use crate::core::events::generation_events::generation_enqueue_failure_event::GenerationEnqueueFailureEvent; use crate::core::events::generation_events::generation_enqueue_success_event::GenerationEnqueueSuccessEvent; +use crate::core::model::video_models::VideoModel; use crate::core::state::data_dir::app_data_root::AppDataRoot; use crate::core::utils::download_media_file_to_temp_dir::download_media_file_to_temp_dir; use crate::core::utils::save_base64_image_to_temp_dir::save_base64_image_to_temp_dir; @@ -68,10 +69,7 @@ pub async fn handle_video_fal( None => { return Err(InternalVideoError::NoModelSpecified); } - Some( - EnqueueImageToVideoModel::Kling16 | - EnqueueImageToVideoModel::Kling16Pro - ) => { + Some(VideoModel::Kling16Pro) => { info!("enqueue Kling 1.6 image to video with Kling API"); selected_model = Some(GenerationModel::Kling1_6); enqueue_kling_16_pro_image_to_video(Kling16ProArgs { diff --git a/crates/desktop/artcraft/src/core/mod.rs b/crates/desktop/artcraft/src/core/mod.rs index 208ed1126b..e4b36574bf 100644 --- a/crates/desktop/artcraft/src/core/mod.rs +++ b/crates/desktop/artcraft/src/core/mod.rs @@ -1,6 +1,7 @@ pub mod artcraft_error; pub mod commands; pub mod events; +pub mod model; pub mod state; pub mod threads; pub mod utils; diff --git a/crates/desktop/artcraft/src/core/model/image_models.rs b/crates/desktop/artcraft/src/core/model/image_models.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crates/desktop/artcraft/src/core/model/mod.rs b/crates/desktop/artcraft/src/core/model/mod.rs new file mode 100644 index 0000000000..7ee84bd8bc --- /dev/null +++ b/crates/desktop/artcraft/src/core/model/mod.rs @@ -0,0 +1,2 @@ +pub mod image_models; +pub mod video_models; \ No newline at end of file diff --git a/crates/desktop/artcraft/src/core/model/video_models.rs b/crates/desktop/artcraft/src/core/model/video_models.rs new file mode 100644 index 0000000000..7e2aadb784 --- /dev/null +++ b/crates/desktop/artcraft/src/core/model/video_models.rs @@ -0,0 +1,22 @@ +use serde_derive::Deserialize; + +/// This is used in the Tauri command bridge. +/// Don't change the serializations without coordinating with the frontend. +#[derive(Deserialize, Debug, Copy, Clone)] +#[serde(rename_all = "snake_case")] +pub enum VideoModel { + #[serde(rename = "kling_1.6_pro")] + Kling16Pro, + + #[serde(rename = "kling_2.1_pro")] + Kling21Pro, + + #[serde(rename = "kling_2.1_master")] + Kling21Master, + + #[serde(rename = "seedance_1.0_lite")] + Seedance10Lite, + + #[serde(rename = "veo_2")] + Veo2, +} diff --git a/crates/service/web/storyteller_web/src/docs/api_doc.rs b/crates/service/web/storyteller_web/src/docs/api_doc.rs index 01184a7ca8..ae7270a2d8 100644 --- a/crates/service/web/storyteller_web/src/docs/api_doc.rs +++ b/crates/service/web/storyteller_web/src/docs/api_doc.rs @@ -527,8 +527,8 @@ use crate::http_server::web_utils::response_success_helpers::*; GenerateFluxPro11UltraTextToImageResponse, GenerateHunyuan21ImageTo3dRequest, GenerateHunyuan21ImageTo3dResponse, - GenerateHunyuan2ImageTo3dRequest, - GenerateHunyuan2ImageTo3dResponse, + GenerateHunyuan20ImageTo3dRequest, + GenerateHunyuan20ImageTo3dResponse, GenerateKling16ProAspectRatio, GenerateKling16ProDuration, GenerateKling16ProImageToVideoRequest, diff --git a/frontend/apps/editor3d/app/src/pages/PageEnigma/PageEditor.tsx b/frontend/apps/editor3d/app/src/pages/PageEnigma/PageEditor.tsx index 24532cb96b..46fec9ce4f 100644 --- a/frontend/apps/editor3d/app/src/pages/PageEnigma/PageEditor.tsx +++ b/frontend/apps/editor3d/app/src/pages/PageEnigma/PageEditor.tsx @@ -527,7 +527,7 @@ export const PageEditor = () => { )} {tabStore.activeTabId == "IMAGE" && (
- @@ -535,7 +535,7 @@ export const PageEditor = () => { )} {tabStore.activeTabId == "VIDEO" && (
- diff --git a/frontend/apps/editor3d/app/src/pages/PageVideo/ImageToVideo.tsx b/frontend/apps/editor3d/app/src/pages/PageVideo/ImageToVideo.tsx index d338db554a..70b46e5c81 100644 --- a/frontend/apps/editor3d/app/src/pages/PageVideo/ImageToVideo.tsx +++ b/frontend/apps/editor3d/app/src/pages/PageVideo/ImageToVideo.tsx @@ -21,6 +21,10 @@ const ImageToVideo = ({ imageMediaId, imageUrl }: ImageToVideoProps) => { selectedModels[ModelCategory.ImageToVideo] || videoGenerationModels[0]?.label; + console.log("Selected model:", selectedModel) + console.log("Image Media Id:", imageMediaId) + console.log("Image URL:", imageUrl) + const jobContext: JobContextType = { jobTokens: [], addJobToken: () => {}, diff --git a/frontend/libs/components/model-selector/src/lib/model-lists.tsx b/frontend/libs/components/model-selector/src/lib/model-lists.tsx index 9d40ac57ec..a65e97a4f3 100644 --- a/frontend/libs/components/model-selector/src/lib/model-lists.tsx +++ b/frontend/libs/components/model-selector/src/lib/model-lists.tsx @@ -7,13 +7,36 @@ export type ModelList = Omit[]; export const allModels = { video: [ { - label: "Kling 2.0", + id: "kling_1.6_pro", + label: "Kling 1.6 Pro", + icon: , + description: "Good quality model", + badges: [{ label: "2 min.", icon: }], + }, + { + id: "kling_2.1_pro", + label: "Kling 2.1 Pro", icon: , description: "High quality model", badges: [{ label: "2 min.", icon: }], }, { - label: "Runway", + id: "kling_2.1_master", + label: "Kling 2.1 Master", + icon: , + description: "Master quality model ($$)", + badges: [{ label: "2 min.", icon: }], + }, + { + id: "seedance_1.0_lite", + label: "Seedance 1.0 Lite", + icon: , + description: "Fast and high-quality model", + badges: [{ label: "2 min.", icon: }], + }, + { + id: "veo_2", + label: "Google Veo 2", icon: , description: "Fast and high-quality model", badges: [{ label: "2 min.", icon: }], @@ -21,24 +44,28 @@ export const allModels = { ], image: [ { + id: "todo", label: "GPT Image 1 (GPT-4o)", icon: , description: "Slow, ultra instructive model", badges: [{ label: "45 sec.", icon: }], }, { + id: "todo", label: "Flux Pro Ultra", icon: , description: "High quality model", badges: [{ label: "15 sec.", icon: }], }, { + id: "todo", label: "Recraft 3", icon: , description: "Fast and high-quality model", badges: [{ label: "15 sec.", icon: }], }, { + id: "todo", label: "Flux.1 Kontext", icon: , description: "New model with advanced context", diff --git a/frontend/libs/components/promptbox/src/lib/PromptBoxVideo.tsx b/frontend/libs/components/promptbox/src/lib/PromptBoxVideo.tsx index 9f1f680a39..6fd173dadb 100644 --- a/frontend/libs/components/promptbox/src/lib/PromptBoxVideo.tsx +++ b/frontend/libs/components/promptbox/src/lib/PromptBoxVideo.tsx @@ -183,6 +183,8 @@ export const PromptBoxVideo = ({ const handleEnqueue = async () => { setIsEnqueueing(true); + console.log("Handle Enqueue"); + setTimeout(() => { // TODO(bt,2025-05-08): This is a hack so we don't accidentally wind up with a permanently disabled prompt box if // the backend hangs on a given request.