From 3cac35aba4001ef464dd4bf59b11b97bcce07a75 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 18 May 2025 10:56:29 -0400 Subject: [PATCH] add hunyuan3d enqueue api --- .../src/core/state/data_dir/temporary_dir.rs | 2 +- .../utils/download_media_file_to_temp_dir.rs | 29 +++ crates/desktop/artcraft/src/core/utils/mod.rs | 2 + .../utils/save_base64_image_to_temp_dir.rs | 25 ++ crates/desktop/artcraft/src/lib.rs | 2 + .../fal_background_removal_command.rs | 60 ++--- .../fal_hunyuan_image_to_3d_command.rs | 220 ++++++++++++++++++ .../fal_kling_image_to_video_command.rs | 63 ++--- .../artcraft/src/services/fal/commands/mod.rs | 1 + .../requests/enqueue_hunyuan2_image_to_3d.rs | 69 ++++++ .../fal_client/src/requests/mod.rs | 1 + 11 files changed, 379 insertions(+), 95 deletions(-) create mode 100644 crates/desktop/artcraft/src/core/utils/download_media_file_to_temp_dir.rs create mode 100644 crates/desktop/artcraft/src/core/utils/save_base64_image_to_temp_dir.rs create mode 100644 crates/desktop/artcraft/src/services/fal/commands/fal_hunyuan_image_to_3d_command.rs create mode 100644 crates/lib/api_clients/fal_client/src/requests/enqueue_hunyuan2_image_to_3d.rs diff --git a/crates/desktop/artcraft/src/core/state/data_dir/temporary_dir.rs b/crates/desktop/artcraft/src/core/state/data_dir/temporary_dir.rs index d7b801e3fb..4456562e35 100644 --- a/crates/desktop/artcraft/src/core/state/data_dir/temporary_dir.rs +++ b/crates/desktop/artcraft/src/core/state/data_dir/temporary_dir.rs @@ -36,7 +36,7 @@ impl TemporaryDir { Ok(tempfile) } - pub fn new_named_temp_file_with_extension(&self, extension: &str) -> anyhow::Result { + pub fn new_named_temp_file_with_extension(&self, extension: &str) -> Result { let extension = if extension.starts_with(".") { extension.to_string() } else { diff --git a/crates/desktop/artcraft/src/core/utils/download_media_file_to_temp_dir.rs b/crates/desktop/artcraft/src/core/utils/download_media_file_to_temp_dir.rs new file mode 100644 index 0000000000..611f88affa --- /dev/null +++ b/crates/desktop/artcraft/src/core/utils/download_media_file_to_temp_dir.rs @@ -0,0 +1,29 @@ +use crate::core::state::data_dir::app_data_root::AppDataRoot; +use crate::core::utils::get_url_file_extension::get_url_file_extension; +use crate::core::utils::simple_http_download_to_tempfile::simple_http_download_to_tempfile; +use errors::AnyhowResult; +use storyteller_client::media_files::get_media_file::get_media_file; +use storyteller_client::utils::api_host::ApiHost; +use tempfile::NamedTempFile; +use tokens::tokens::media_files::MediaFileToken; + +// TODO: Better more concrete error handling + +pub async fn download_media_file_to_temp_dir(app_data_root: &AppDataRoot, token: &MediaFileToken) -> AnyhowResult { + let response = get_media_file(&ApiHost::Storyteller, token).await?; + + let media_file_url = &response.media_file.media_links.cdn_url; + + let extension_with_dot = get_url_file_extension(media_file_url) + .map(|ext| format!(".{}", ext)) + .unwrap_or_else(|| ".png".to_string()); // TODO: Better default extension passed by caller as heuristic. + + //let filename = format!("{}{}", response.media_file.token.as_str(), extension_with_dot); + //let filename = app_data_root.downloads_dir().path().join(&filename); + + let mut file = app_data_root.temp_dir().new_named_temp_file_with_extension(&extension_with_dot)?; + + simple_http_download_to_tempfile(&media_file_url, &mut file).await?; + + Ok(file) // NB: Must return TempFile to not drop / delete it +} diff --git a/crates/desktop/artcraft/src/core/utils/mod.rs b/crates/desktop/artcraft/src/core/utils/mod.rs index 6c77dfed2c..c78fe103da 100644 --- a/crates/desktop/artcraft/src/core/utils/mod.rs +++ b/crates/desktop/artcraft/src/core/utils/mod.rs @@ -1,9 +1,11 @@ pub mod best_window_size_heuristic; pub mod clear_all_webview_cookies; +pub mod download_media_file_to_temp_dir; pub mod get_url_file_extension; pub mod image; pub mod large_file_transfer; pub mod object_store; +pub mod save_base64_image_to_temp_dir; pub mod simple_http_download; pub mod simple_http_download_to_tempfile; pub mod webview_unsafe; \ No newline at end of file diff --git a/crates/desktop/artcraft/src/core/utils/save_base64_image_to_temp_dir.rs b/crates/desktop/artcraft/src/core/utils/save_base64_image_to_temp_dir.rs new file mode 100644 index 0000000000..4dcb759f2f --- /dev/null +++ b/crates/desktop/artcraft/src/core/utils/save_base64_image_to_temp_dir.rs @@ -0,0 +1,25 @@ +use crate::core::state::data_dir::app_data_root::AppDataRoot; +use base64::prelude::BASE64_STANDARD; +use base64::Engine; +use errors::AnyhowResult; +use mimetypes::mimetype_info::mimetype_info::MimetypeInfo; +use std::io::Write; +use tempfile::NamedTempFile; + +// TODO: Retain error types (io/decode) + +pub async fn save_base64_image_to_temp_dir(app_data_root: &AppDataRoot, base64_image: String) -> AnyhowResult { + let bytes = BASE64_STANDARD.decode(base64_image)?; + + let extension = MimetypeInfo::get_for_bytes(&bytes) + .map(|info| info.file_extension()) + .flatten() + .map(|ext| ext.extension_with_period().to_string()) + .unwrap_or_else(|| ".png".to_string()); + + let mut file = app_data_root.temp_dir().new_named_temp_file_with_extension(&extension)?; + + file.write_all(bytes.as_ref())?; + + Ok(file) // NB: Must return TempFile to not drop / delete it +} diff --git a/crates/desktop/artcraft/src/lib.rs b/crates/desktop/artcraft/src/lib.rs index 4c40176377..fccba3bd01 100644 --- a/crates/desktop/artcraft/src/lib.rs +++ b/crates/desktop/artcraft/src/lib.rs @@ -15,6 +15,7 @@ use crate::core::threads::discord_presence_thread::discord_presence_thread; use crate::core::threads::main_window_thread::main_window_thread::main_window_thread; use crate::core::utils::webview_unsafe::webview_unsafe_for_app; use crate::services::fal::commands::fal_background_removal_command::fal_background_removal_command; +use crate::services::fal::commands::fal_hunyuan_image_to_3d_command::fal_hunyuan_image_to_3d_command; use crate::services::fal::commands::fal_kling_image_to_video_command::fal_kling_image_to_video_command; use crate::services::fal::commands::get_fal_api_key_command::get_fal_api_key_command; use crate::services::fal::commands::set_fal_api_key_command::set_fal_api_key_command; @@ -159,6 +160,7 @@ pub fn run() { .invoke_handler(tauri::generate_handler![ check_sora_session_command, fal_background_removal_command, + fal_hunyuan_image_to_3d_command, fal_kling_image_to_video_command, flip_image, get_app_preferences_command, diff --git a/crates/desktop/artcraft/src/services/fal/commands/fal_background_removal_command.rs b/crates/desktop/artcraft/src/services/fal/commands/fal_background_removal_command.rs index b6f17b9cb8..e9838840d2 100644 --- a/crates/desktop/artcraft/src/services/fal/commands/fal_background_removal_command.rs +++ b/crates/desktop/artcraft/src/services/fal/commands/fal_background_removal_command.rs @@ -1,18 +1,20 @@ use crate::core::commands::command_response_wrapper::{CommandErrorResponseWrapper, CommandErrorStatus, CommandResult, SerializeMarker}; use crate::core::events::sendable_event_trait::SendableEvent; +use crate::core::state::data_dir::app_data_root::AppDataRoot; +use crate::core::state::data_dir::trait_data_subdir::DataSubdir; +use crate::core::utils::download_media_file_to_temp_dir::download_media_file_to_temp_dir; +use crate::core::utils::get_url_file_extension::get_url_file_extension; +use crate::core::utils::save_base64_image_to_temp_dir::save_base64_image_to_temp_dir; +use crate::core::utils::simple_http_download::simple_http_download; +use crate::core::utils::simple_http_download_to_tempfile::simple_http_download_to_tempfile; +use crate::services::fal::state::fal_credential_manager::FalCredentialManager; use crate::services::sora::events::sora_image_enqueue_failure_event::SoraImageEnqueueFailureEvent; use crate::services::sora::events::sora_image_enqueue_success_event::SoraImageEnqueueSuccessEvent; -use crate::services::fal::state::fal_credential_manager::FalCredentialManager; use crate::services::sora::state::read_sora_credentials_from_disk::read_sora_credentials_from_disk; use crate::services::sora::state::sora_credential_holder::SoraCredentialHolder; use crate::services::sora::state::sora_credential_manager::SoraCredentialManager; use crate::services::sora::state::sora_task_queue::SoraTaskQueue; -use crate::core::state::data_dir::app_data_root::AppDataRoot; -use crate::core::state::data_dir::trait_data_subdir::DataSubdir; use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager; -use crate::core::utils::get_url_file_extension::get_url_file_extension; -use crate::core::utils::simple_http_download::simple_http_download; -use crate::core::utils::simple_http_download_to_tempfile::simple_http_download_to_tempfile; use anyhow::anyhow; use base64::prelude::BASE64_STANDARD; use base64::{DecodeError, Engine}; @@ -73,7 +75,7 @@ impl SerializeMarker for FalBackgroundRemovalSuccessResponse {} #[derive(Serialize, Debug)] #[serde(rename_all = "snake_case")] -pub enum SoraImageRemixErrorType { +pub enum BackgroundRemovalErrorType { /// Generic server error ServerError, /// No Fal API key available @@ -87,7 +89,7 @@ pub async fn fal_background_removal_command( app_data_root: State<'_, AppDataRoot>, fal_creds_manager: State<'_, FalCredentialManager>, storyteller_creds_manager: State<'_, StorytellerCredentialManager>, -) -> CommandResult { +) -> CommandResult { info!("fal_background_removal_command called; image media token: {:?}", request.image_media_token); @@ -100,7 +102,7 @@ pub async fn fal_background_removal_command( return Err(CommandErrorResponseWrapper { status: CommandErrorStatus::Unauthorized, error_message: Some("You need to set a FAL api key".to_string()), - error_type: Some(SoraImageRemixErrorType::NeedsFalApiKey), + error_type: Some(BackgroundRemovalErrorType::NeedsFalApiKey), error_details: None, }); } @@ -123,7 +125,7 @@ pub async fn fal_background_removal_command( //} let mut status = CommandErrorStatus::ServerError; - let mut error_type = SoraImageRemixErrorType::ServerError; + let mut error_type = BackgroundRemovalErrorType::ServerError; let mut error_message = "A server error occurred. Please try again. If it continues, please tell our staff about the problem."; //match (err) { @@ -205,10 +207,10 @@ pub async fn remove_background( let mut temp_download; if let Some(media_token) = request.image_media_token { - temp_download = download_media_file(&app_data_root, &media_token).await?; + temp_download = download_media_file_to_temp_dir(&app_data_root, &media_token).await?; } else if let Some(base64_bytes) = request.base64_image { - temp_download = persist_base64(&app_data_root, base64_bytes).await?; + temp_download = save_base64_image_to_temp_dir(&app_data_root, base64_bytes).await?; } else { return Err(InnerError::AnyhowError(anyhow!("No image media token or base64 image provided"))); } @@ -253,37 +255,3 @@ pub async fn remove_background( Ok((response, base64_bytes)) } -async fn download_media_file(app_data_root: &AppDataRoot, token: &MediaFileToken) -> Result { - let response = get_media_file(&ApiHost::Storyteller, token).await?; - - let media_file_url = &response.media_file.media_links.cdn_url; - - let extension_with_dot = get_url_file_extension(media_file_url) - .map(|ext| format!(".{}", ext)) - .unwrap_or_else(|| ".png".to_string()); - - //let filename = format!("{}{}", response.media_file.token.as_str(), extension_with_dot); - //let filename = app_data_root.downloads_dir().path().join(&filename); - - let mut file = app_data_root.temp_dir().new_named_temp_file_with_extension(&extension_with_dot)?; - - simple_http_download_to_tempfile(&media_file_url, &mut file).await?; - - Ok(file) // NB: Must return TempFile to not drop / delete it -} - -async fn persist_base64(app_data_root: &AppDataRoot, base64_image: String) -> Result { - let bytes = BASE64_STANDARD.decode(base64_image)?; - - let extension = MimetypeInfo::get_for_bytes(&bytes) - .map(|info| info.file_extension()) - .flatten() - .map(|ext| ext.extension_with_period().to_string()) - .unwrap_or_else(|| ".png".to_string()); - - let mut file = app_data_root.temp_dir().new_named_temp_file_with_extension(&extension)?; - - file.write_all(bytes.as_ref())?; - - Ok(file) // NB: Must return TempFile to not drop / delete it -} diff --git a/crates/desktop/artcraft/src/services/fal/commands/fal_hunyuan_image_to_3d_command.rs b/crates/desktop/artcraft/src/services/fal/commands/fal_hunyuan_image_to_3d_command.rs new file mode 100644 index 0000000000..8eca4d5989 --- /dev/null +++ b/crates/desktop/artcraft/src/services/fal/commands/fal_hunyuan_image_to_3d_command.rs @@ -0,0 +1,220 @@ +use crate::core::commands::command_response_wrapper::{CommandErrorResponseWrapper, CommandErrorStatus, CommandResult, SerializeMarker}; +use crate::core::events::sendable_event_trait::SendableEvent; +use crate::core::state::data_dir::app_data_root::AppDataRoot; +use crate::core::state::data_dir::trait_data_subdir::DataSubdir; +use crate::core::utils::download_media_file_to_temp_dir::download_media_file_to_temp_dir; +use crate::core::utils::get_url_file_extension::get_url_file_extension; +use crate::core::utils::save_base64_image_to_temp_dir::save_base64_image_to_temp_dir; +use crate::core::utils::simple_http_download::simple_http_download; +use crate::core::utils::simple_http_download_to_tempfile::simple_http_download_to_tempfile; +use crate::services::fal::state::fal_credential_manager::FalCredentialManager; +use crate::services::fal::state::fal_task_queue::FalTaskQueue; +use crate::services::sora::events::sora_image_enqueue_failure_event::SoraImageEnqueueFailureEvent; +use crate::services::sora::events::sora_image_enqueue_success_event::SoraImageEnqueueSuccessEvent; +use crate::services::sora::state::read_sora_credentials_from_disk::read_sora_credentials_from_disk; +use crate::services::sora::state::sora_credential_holder::SoraCredentialHolder; +use crate::services::sora::state::sora_credential_manager::SoraCredentialManager; +use crate::services::sora::state::sora_task_queue::SoraTaskQueue; +use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager; +use anyhow::anyhow; +use base64::prelude::BASE64_STANDARD; +use base64::{DecodeError, Engine}; +use errors::{AnyhowError, AnyhowResult}; +use fal_client::fal_error_plus::FalErrorPlus; +use fal_client::requests::enqueue_hunyuan2_image_to_3d::{enqueue_hunyuan2_image_to_3d, Hunyuan2Args}; +use fal_client::requests::enqueue_kling_16_image_to_video::{enqueue_kling_16_image_to_video, Kling16Args, Kling16Duration}; +use fal_client::requests::remove_background_rembg::remove_background_rembg; +use filesys::file_read_bytes::file_read_bytes; +use image::codecs::png::{CompressionType, FilterType, PngEncoder}; +use image::{DynamicImage, EncodableLayout, ImageReader}; +use log::{debug, error, info, warn}; +use mimetypes::mimetype_info::mimetype_info::MimetypeInfo; +use openai_sora_client::credentials::SoraCredentials; +use openai_sora_client::creds::credential_migration::CredentialMigrationRef; +use openai_sora_client::recipes::image_remix_with_session_auto_renew::{image_remix_with_session_auto_renew, ImageRemixAutoRenewRequest}; +use openai_sora_client::recipes::image_upload_from_file_with_session_auto_renew::{image_upload_from_file_with_session_auto_renew, ImageUploadFromFileAutoRenewRequest}; +use openai_sora_client::recipes::maybe_upgrade_or_renew_session::maybe_upgrade_or_renew_session; +use openai_sora_client::requests::image_gen::common::{ImageSize, NumImages}; +use openai_sora_client::requests::image_gen::sora_image_gen_remix::{sora_image_gen_remix, SoraImageGenRemixRequest}; +use openai_sora_client::requests::upload::upload_media_from_bytes::sora_media_upload_from_bytes; +use openai_sora_client::requests::upload::upload_media_from_file::sora_media_upload_from_file; +use openai_sora_client::sora_error::SoraError; +use reqwest::Url; +use serde_derive::{Deserialize, Serialize}; +use std::fs::read_to_string; +use std::io::{Cursor, Read, Write}; +use std::ops::Add; +use std::path::PathBuf; +use std::time::Duration; +use storyteller_client::api_error::ApiError; +use storyteller_client::api_error::ApiError::InternalServerError; +use storyteller_client::media_files::get_media_file::{get_media_file, GetMediaFileSuccessResponse}; +use storyteller_client::media_files::upload_image_media_file_from_file::upload_image_media_file_from_file; +use storyteller_client::utils::api_host::ApiHost; +use tauri::{AppHandle, Emitter, Manager, State}; +use tempfile::NamedTempFile; +use tokens::tokens::media_files::MediaFileToken; + +#[derive(Deserialize)] +pub struct FalHunyuanImageTo3dRequest { + /// Image media file; the image to remove the background from. + pub image_media_token: Option, + + /// Base64-encoded image + pub base64_image: Option, +} + +#[derive(Serialize, Debug)] +#[serde(rename_all = "snake_case")] +pub enum HunyuanErrorType { + /// Generic server error + ServerError, + /// No Fal API key available + NeedsFalApiKey, +} + + +#[tauri::command] +pub async fn fal_hunyuan_image_to_3d_command( + request: FalHunyuanImageTo3dRequest, + app_data_root: State<'_, AppDataRoot>, + fal_creds_manager: State<'_, FalCredentialManager>, + storyteller_creds_manager: State<'_, StorytellerCredentialManager>, + fal_task_queue: State<'_, FalTaskQueue>, +) -> CommandResult<(), HunyuanErrorType, ()> { + + info!("fal_hunyuan_image_to_3d_command called; image media token: {:?}", request.image_media_token); + + let has_credentials = fal_creds_manager + .has_apparent_api_token() + .unwrap_or(true); // NB: Failures would be lock issues + + if !has_credentials { + warn!("No API key found"); + return Err(CommandErrorResponseWrapper { + status: CommandErrorStatus::Unauthorized, + error_message: Some("You need to set a FAL api key".to_string()), + error_type: Some(HunyuanErrorType::NeedsFalApiKey), + error_details: None, + }); + } + + let result = image_to_3d( + request, + &app_data_root, + &fal_creds_manager, + &storyteller_creds_manager, + &fal_task_queue, + ).await; + + if let Err(err) = result { + error!("error: {:?}", err); + + //let event = SoraImageEnqueueFailureEvent {}; + //let result = event.send(&app); + //if let Err(err) = result { + // error!("Failed to emit event: {:?}", err); + //} + + let mut status = CommandErrorStatus::ServerError; + let mut error_type = HunyuanErrorType::ServerError; + let mut error_message = "A server error occurred. Please try again. If it continues, please tell our staff about the problem."; + + //match (err) { + // _ => {}, + //} + + return Err(CommandErrorResponseWrapper { + status, + error_message: Some(error_message.to_string()), + error_type: Some(error_type), + error_details: None, + }) + } + + //let event = SoraImageEnqueueSuccessEvent {}; + //let result = event.send(&app); + //if let Err(err) = result { + // error!("Failed to emit event: {:?}", err); + //} + + Ok(().into()) +} + +#[derive(Debug)] +enum InnerError { + FalError(FalErrorPlus), + AnyhowError(AnyhowError), + StorytellerApiError(ApiError), + DecodeError(DecodeError), + IoError(std::io::Error), +} + +impl From for InnerError { + fn from(value: AnyhowError) -> Self { + Self::AnyhowError(value) + } +} + +impl From for InnerError { + fn from(value: FalErrorPlus) -> Self { + Self::FalError(value) + } +} + +impl From for InnerError { + fn from(value: ApiError) -> Self { + Self::StorytellerApiError(value) + } +} + +impl From for InnerError { + fn from(value: DecodeError) -> Self { + Self::DecodeError(value) + } +} + +impl From for InnerError { + fn from(value: std::io::Error) -> Self { + Self::IoError(value) + } +} + +pub async fn image_to_3d( + request: FalHunyuanImageTo3dRequest, + app_data_root: &AppDataRoot, + fal_creds_manager: &FalCredentialManager, + storyteller_creds_manager: &StorytellerCredentialManager, + fal_task_queue: &FalTaskQueue, +) -> Result<(), InnerError> { + + let api_key = fal_creds_manager.get_key_required()?; + let creds = storyteller_creds_manager.get_credentials_required()?; + + let mut temp_download; + + if let Some(media_token) = request.image_media_token { + temp_download = download_media_file_to_temp_dir(&app_data_root, &media_token).await?; + + } else if let Some(base64_bytes) = request.base64_image { + temp_download = save_base64_image_to_temp_dir(&app_data_root, base64_bytes).await?; + } else { + return Err(InnerError::AnyhowError(anyhow!("No image media token or base64 image provided"))); + } + + info!("Calling FAL image to 3d ..."); + + let filename = temp_download.path().to_path_buf(); + + let enqueued = enqueue_hunyuan2_image_to_3d(Hunyuan2Args { + image_path: filename, + api_key: &api_key, + }).await?; + + if let Err(err) = fal_task_queue.insert(&enqueued) { + error!("Failed to enqueue task: {:?}", err); + return Err(InnerError::AnyhowError(anyhow!("Failed to enqueue task: {:?}", err))); + } + + Ok(()) +} diff --git a/crates/desktop/artcraft/src/services/fal/commands/fal_kling_image_to_video_command.rs b/crates/desktop/artcraft/src/services/fal/commands/fal_kling_image_to_video_command.rs index dc17bee2d8..fb74daffdb 100644 --- a/crates/desktop/artcraft/src/services/fal/commands/fal_kling_image_to_video_command.rs +++ b/crates/desktop/artcraft/src/services/fal/commands/fal_kling_image_to_video_command.rs @@ -1,19 +1,21 @@ use crate::core::commands::command_response_wrapper::{CommandErrorResponseWrapper, CommandErrorStatus, CommandResult, SerializeMarker}; use crate::core::events::sendable_event_trait::SendableEvent; -use crate::services::sora::events::sora_image_enqueue_failure_event::SoraImageEnqueueFailureEvent; -use crate::services::sora::events::sora_image_enqueue_success_event::SoraImageEnqueueSuccessEvent; +use crate::core::state::data_dir::app_data_root::AppDataRoot; +use crate::core::state::data_dir::trait_data_subdir::DataSubdir; +use crate::core::utils::download_media_file_to_temp_dir::download_media_file_to_temp_dir; +use crate::core::utils::get_url_file_extension::get_url_file_extension; +use crate::core::utils::save_base64_image_to_temp_dir::save_base64_image_to_temp_dir; +use crate::core::utils::simple_http_download::simple_http_download; +use crate::core::utils::simple_http_download_to_tempfile::simple_http_download_to_tempfile; use crate::services::fal::state::fal_credential_manager::FalCredentialManager; use crate::services::fal::state::fal_task_queue::FalTaskQueue; +use crate::services::sora::events::sora_image_enqueue_failure_event::SoraImageEnqueueFailureEvent; +use crate::services::sora::events::sora_image_enqueue_success_event::SoraImageEnqueueSuccessEvent; use crate::services::sora::state::read_sora_credentials_from_disk::read_sora_credentials_from_disk; use crate::services::sora::state::sora_credential_holder::SoraCredentialHolder; use crate::services::sora::state::sora_credential_manager::SoraCredentialManager; use crate::services::sora::state::sora_task_queue::SoraTaskQueue; -use crate::core::state::data_dir::app_data_root::AppDataRoot; -use crate::core::state::data_dir::trait_data_subdir::DataSubdir; use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager; -use crate::core::utils::get_url_file_extension::get_url_file_extension; -use crate::core::utils::simple_http_download::simple_http_download; -use crate::core::utils::simple_http_download_to_tempfile::simple_http_download_to_tempfile; use anyhow::anyhow; use base64::prelude::BASE64_STANDARD; use base64::{DecodeError, Engine}; @@ -63,7 +65,7 @@ pub struct FalKlingImageToVideoRequest { #[derive(Serialize, Debug)] #[serde(rename_all = "snake_case")] -pub enum SoraImageRemixErrorType { +pub enum KlingImageToVideoErrorType { /// Generic server error ServerError, /// No Fal API key available @@ -78,7 +80,7 @@ pub async fn fal_kling_image_to_video_command( fal_creds_manager: State<'_, FalCredentialManager>, storyteller_creds_manager: State<'_, StorytellerCredentialManager>, fal_task_queue: State<'_, FalTaskQueue>, -) -> CommandResult<(), SoraImageRemixErrorType, ()> { +) -> CommandResult<(), KlingImageToVideoErrorType, ()> { info!("fal_kling_image_to_video_command called; image media token: {:?}", request.image_media_token); @@ -91,7 +93,7 @@ pub async fn fal_kling_image_to_video_command( return Err(CommandErrorResponseWrapper { status: CommandErrorStatus::Unauthorized, error_message: Some("You need to set a FAL api key".to_string()), - error_type: Some(SoraImageRemixErrorType::NeedsFalApiKey), + error_type: Some(KlingImageToVideoErrorType::NeedsFalApiKey), error_details: None, }); } @@ -114,7 +116,7 @@ pub async fn fal_kling_image_to_video_command( //} let mut status = CommandErrorStatus::ServerError; - let mut error_type = SoraImageRemixErrorType::ServerError; + let mut error_type = KlingImageToVideoErrorType::ServerError; let mut error_message = "A server error occurred. Please try again. If it continues, please tell our staff about the problem."; //match (err) { @@ -191,10 +193,10 @@ pub async fn image_to_video( let mut temp_download; if let Some(media_token) = request.image_media_token { - temp_download = download_media_file(&app_data_root, &media_token).await?; + temp_download = download_media_file_to_temp_dir(&app_data_root, &media_token).await?; } else if let Some(base64_bytes) = request.base64_image { - temp_download = persist_base64(&app_data_root, base64_bytes).await?; + temp_download = save_base64_image_to_temp_dir(&app_data_root, base64_bytes).await?; } else { return Err(InnerError::AnyhowError(anyhow!("No image media token or base64 image provided"))); } @@ -216,38 +218,3 @@ pub async fn image_to_video( Ok(()) } - -async fn download_media_file(app_data_root: &AppDataRoot, token: &MediaFileToken) -> Result { - let response = get_media_file(&ApiHost::Storyteller, token).await?; - - let media_file_url = &response.media_file.media_links.cdn_url; - - let extension_with_dot = get_url_file_extension(media_file_url) - .map(|ext| format!(".{}", ext)) - .unwrap_or_else(|| ".png".to_string()); - - //let filename = format!("{}{}", response.media_file.token.as_str(), extension_with_dot); - //let filename = app_data_root.downloads_dir().path().join(&filename); - - let mut file = app_data_root.temp_dir().new_named_temp_file_with_extension(&extension_with_dot)?; - - simple_http_download_to_tempfile(&media_file_url, &mut file).await?; - - Ok(file) // NB: Must return TempFile to not drop / delete it -} - -async fn persist_base64(app_data_root: &AppDataRoot, base64_image: String) -> Result { - let bytes = BASE64_STANDARD.decode(base64_image)?; - - let extension = MimetypeInfo::get_for_bytes(&bytes) - .map(|info| info.file_extension()) - .flatten() - .map(|ext| ext.extension_with_period().to_string()) - .unwrap_or_else(|| ".png".to_string()); - - let mut file = app_data_root.temp_dir().new_named_temp_file_with_extension(&extension)?; - - file.write_all(bytes.as_ref())?; - - Ok(file) // NB: Must return TempFile to not drop / delete it -} diff --git a/crates/desktop/artcraft/src/services/fal/commands/mod.rs b/crates/desktop/artcraft/src/services/fal/commands/mod.rs index d547c4ca54..49f725d47b 100644 --- a/crates/desktop/artcraft/src/services/fal/commands/mod.rs +++ b/crates/desktop/artcraft/src/services/fal/commands/mod.rs @@ -1,4 +1,5 @@ pub mod fal_background_removal_command; +pub mod fal_hunyuan_image_to_3d_command; pub mod fal_kling_image_to_video_command; pub mod get_fal_api_key_command; pub mod set_fal_api_key_command; \ No newline at end of file diff --git a/crates/lib/api_clients/fal_client/src/requests/enqueue_hunyuan2_image_to_3d.rs b/crates/lib/api_clients/fal_client/src/requests/enqueue_hunyuan2_image_to_3d.rs new file mode 100644 index 0000000000..546f8862f4 --- /dev/null +++ b/crates/lib/api_clients/fal_client/src/requests/enqueue_hunyuan2_image_to_3d.rs @@ -0,0 +1,69 @@ +use crate::creds::fal_api_key::FalApiKey; +use crate::fal_error_plus::FalErrorPlus; +use crate::model::enqueued_request::EnqueuedRequest; +use fal::endpoints::fal_ai::hunyuan3d::v2::v2; +use fal::endpoints::fal_ai::hunyuan3d::v2::Hunyuan3DInput; +use fal_client::file_to_base64_url::file_to_base64_url; +use futures::StreamExt; +use std::io::Write; +use std::path::Path; + +pub struct Hunyuan2Args<'a, P: AsRef> { + pub image_path: P, + pub api_key: &'a FalApiKey, +} + +pub async fn enqueue_hunyuan2_image_to_3d>(args: Hunyuan2Args<'_, P>) -> Result { + let image_url = file_to_base64_url(args.image_path)?; + + /* + TODO: Handle error messages - + FalError(FalError(Other("{\"detail\": \"Invalid Key Authorization header format. Expected ':'.\"}"))) + FalError(FalError(Other("{\"detail\": \"No user found for Key ID and Secret\"}"))) + */ + + let request = Hunyuan3DInput { + input_image_url: image_url, + textured_mesh: Some(true), + guidance_scale: None, + num_inference_steps: None, + octree_resolution: None, + seed: None, + }; + + let result = v2(request) + .with_api_key(&args.api_key.0) + .queue() + .await?; + + Ok(EnqueuedRequest::from_queue_response(&result)?) +} + +#[cfg(test)] +mod tests { + use crate::creds::fal_api_key::FalApiKey; + use crate::requests::enqueue_hunyuan2_image_to_3d::{enqueue_hunyuan2_image_to_3d, Hunyuan2Args}; + use errors::AnyhowResult; + use std::fs::read_to_string; + use testing::test_file_path::test_file_path; + + #[tokio::test] + #[ignore] + async fn test() -> AnyhowResult<()> { + let image = test_file_path("test_data/image/juno.jpg")?; + + // XXX: Don't commit secrets! + let secret = read_to_string("/Users/bt/Artcraft/credentials/fal_api_key.txt")?; + + let api_key = FalApiKey::from_str(&secret); + + let args = Hunyuan2Args { + image_path: image, + api_key: &api_key, + }; + + let result = enqueue_hunyuan2_image_to_3d(args).await?; + + Ok(()) + } +} diff --git a/crates/lib/api_clients/fal_client/src/requests/mod.rs b/crates/lib/api_clients/fal_client/src/requests/mod.rs index 5490aa7586..3544169e94 100644 --- a/crates/lib/api_clients/fal_client/src/requests/mod.rs +++ b/crates/lib/api_clients/fal_client/src/requests/mod.rs @@ -1,2 +1,3 @@ +pub mod enqueue_hunyuan2_image_to_3d; pub mod enqueue_kling_16_image_to_video; pub mod remove_background_rembg; \ No newline at end of file