diff --git a/Cargo.lock b/Cargo.lock index c71cf60273..e5f9ec03f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8806,7 +8806,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" dependencies = [ "phf_macros 0.11.2", - "phf_shared 0.11.2", + "phf_shared 0.11.3", ] [[package]] @@ -8855,7 +8855,7 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" dependencies = [ - "phf_shared 0.11.2", + "phf_shared 0.11.3", "rand 0.8.5", ] @@ -8880,7 +8880,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3444646e286606587e49f3bcf1679b8cef1dc2c5ecc29ddacaffc305180d464b" dependencies = [ "phf_generator 0.11.2", - "phf_shared 0.11.2", + "phf_shared 0.11.3", "proc-macro2", "quote", "syn 2.0.87", @@ -8892,7 +8892,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c00cf8b9eafe68dde5e9eaa2cef8ee84a9336a47d566ec55ca16589633b65af7" dependencies = [ - "siphasher", + "siphasher 0.3.11", ] [[package]] @@ -8901,16 +8901,16 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096" dependencies = [ - "siphasher", + "siphasher 0.3.11", ] [[package]] name = "phf_shared" -version = "0.11.2" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" dependencies = [ - "siphasher", + "siphasher 1.0.1", ] [[package]] @@ -11282,6 +11282,12 @@ version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" +[[package]] +name = "siphasher" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" + [[package]] name = "slab" version = "0.4.9" @@ -11840,6 +11846,7 @@ dependencies = [ "errors", "idempotency", "log", + "mimetypes", "reqwest 0.12.15", "serde", "serde_derive", @@ -13899,12 +13906,9 @@ dependencies = [ [[package]] name = "unicase" -version = "2.7.0" +version = "2.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d2d4dafb69621809a81864c9c1b864479e1235c0dd4e199924b9742439ed89" -dependencies = [ - "version_check", -] +checksum = "75b844d17643ee918803943289730bec8aac480150456169e647ed0b576ba539" [[package]] name = "unicode-bidi" @@ -15321,7 +15325,7 @@ dependencies = [ "openssl-sys", "parking_lot 0.12.2", "phf 0.11.2", - "phf_shared 0.11.2", + "phf_shared 0.11.3", "proc-macro2", "prost", "pyo3", diff --git a/crates/desktop/artcraft/src/core/state/data_dir/temporary_dir.rs b/crates/desktop/artcraft/src/core/state/data_dir/temporary_dir.rs index d7b801e3fb..4456562e35 100644 --- a/crates/desktop/artcraft/src/core/state/data_dir/temporary_dir.rs +++ b/crates/desktop/artcraft/src/core/state/data_dir/temporary_dir.rs @@ -36,7 +36,7 @@ impl TemporaryDir { Ok(tempfile) } - pub fn new_named_temp_file_with_extension(&self, extension: &str) -> anyhow::Result { + pub fn new_named_temp_file_with_extension(&self, extension: &str) -> Result { let extension = if extension.starts_with(".") { extension.to_string() } else { diff --git a/crates/desktop/artcraft/src/core/utils/download_media_file_to_temp_dir.rs b/crates/desktop/artcraft/src/core/utils/download_media_file_to_temp_dir.rs new file mode 100644 index 0000000000..611f88affa --- /dev/null +++ b/crates/desktop/artcraft/src/core/utils/download_media_file_to_temp_dir.rs @@ -0,0 +1,29 @@ +use crate::core::state::data_dir::app_data_root::AppDataRoot; +use crate::core::utils::get_url_file_extension::get_url_file_extension; +use crate::core::utils::simple_http_download_to_tempfile::simple_http_download_to_tempfile; +use errors::AnyhowResult; +use storyteller_client::media_files::get_media_file::get_media_file; +use storyteller_client::utils::api_host::ApiHost; +use tempfile::NamedTempFile; +use tokens::tokens::media_files::MediaFileToken; + +// TODO: Better more concrete error handling + +pub async fn download_media_file_to_temp_dir(app_data_root: &AppDataRoot, token: &MediaFileToken) -> AnyhowResult { + let response = get_media_file(&ApiHost::Storyteller, token).await?; + + let media_file_url = &response.media_file.media_links.cdn_url; + + let extension_with_dot = get_url_file_extension(media_file_url) + .map(|ext| format!(".{}", ext)) + .unwrap_or_else(|| ".png".to_string()); // TODO: Better default extension passed by caller as heuristic. + + //let filename = format!("{}{}", response.media_file.token.as_str(), extension_with_dot); + //let filename = app_data_root.downloads_dir().path().join(&filename); + + let mut file = app_data_root.temp_dir().new_named_temp_file_with_extension(&extension_with_dot)?; + + simple_http_download_to_tempfile(&media_file_url, &mut file).await?; + + Ok(file) // NB: Must return TempFile to not drop / delete it +} diff --git a/crates/desktop/artcraft/src/core/utils/download_url_to_temp_dir.rs b/crates/desktop/artcraft/src/core/utils/download_url_to_temp_dir.rs new file mode 100644 index 0000000000..60122a08d5 --- /dev/null +++ b/crates/desktop/artcraft/src/core/utils/download_url_to_temp_dir.rs @@ -0,0 +1,20 @@ +use crate::core::state::data_dir::app_data_root::AppDataRoot; +use errors::AnyhowResult; +use std::io::Write; +use tempfile::NamedTempFile; +use url::Url; + +pub async fn download_url_to_temp_dir(url: &str, app_data_root: &AppDataRoot) -> AnyhowResult { + let url = Url::parse(&url)?; + + let response = reqwest::get(url.clone()).await?; + let response_bytes = response.bytes().await?; + + let ext = url.path().split(".").last().unwrap_or("png"); + + let mut temp_file = app_data_root.temp_dir().new_named_temp_file_with_extension(ext)?; + + temp_file.write_all(&response_bytes)?; + + Ok(temp_file) +} diff --git a/crates/desktop/artcraft/src/core/utils/mod.rs b/crates/desktop/artcraft/src/core/utils/mod.rs index 6c77dfed2c..d9003fc855 100644 --- a/crates/desktop/artcraft/src/core/utils/mod.rs +++ b/crates/desktop/artcraft/src/core/utils/mod.rs @@ -1,9 +1,12 @@ pub mod best_window_size_heuristic; pub mod clear_all_webview_cookies; +pub mod download_media_file_to_temp_dir; +pub mod download_url_to_temp_dir; pub mod get_url_file_extension; pub mod image; pub mod large_file_transfer; pub mod object_store; +pub mod save_base64_image_to_temp_dir; pub mod simple_http_download; pub mod simple_http_download_to_tempfile; pub mod webview_unsafe; \ No newline at end of file diff --git a/crates/desktop/artcraft/src/core/utils/save_base64_image_to_temp_dir.rs b/crates/desktop/artcraft/src/core/utils/save_base64_image_to_temp_dir.rs new file mode 100644 index 0000000000..4dcb759f2f --- /dev/null +++ b/crates/desktop/artcraft/src/core/utils/save_base64_image_to_temp_dir.rs @@ -0,0 +1,25 @@ +use crate::core::state::data_dir::app_data_root::AppDataRoot; +use base64::prelude::BASE64_STANDARD; +use base64::Engine; +use errors::AnyhowResult; +use mimetypes::mimetype_info::mimetype_info::MimetypeInfo; +use std::io::Write; +use tempfile::NamedTempFile; + +// TODO: Retain error types (io/decode) + +pub async fn save_base64_image_to_temp_dir(app_data_root: &AppDataRoot, base64_image: String) -> AnyhowResult { + let bytes = BASE64_STANDARD.decode(base64_image)?; + + let extension = MimetypeInfo::get_for_bytes(&bytes) + .map(|info| info.file_extension()) + .flatten() + .map(|ext| ext.extension_with_period().to_string()) + .unwrap_or_else(|| ".png".to_string()); + + let mut file = app_data_root.temp_dir().new_named_temp_file_with_extension(&extension)?; + + file.write_all(bytes.as_ref())?; + + Ok(file) // NB: Must return TempFile to not drop / delete it +} diff --git a/crates/desktop/artcraft/src/lib.rs b/crates/desktop/artcraft/src/lib.rs index 4c40176377..fccba3bd01 100644 --- a/crates/desktop/artcraft/src/lib.rs +++ b/crates/desktop/artcraft/src/lib.rs @@ -15,6 +15,7 @@ use crate::core::threads::discord_presence_thread::discord_presence_thread; use crate::core::threads::main_window_thread::main_window_thread::main_window_thread; use crate::core::utils::webview_unsafe::webview_unsafe_for_app; use crate::services::fal::commands::fal_background_removal_command::fal_background_removal_command; +use crate::services::fal::commands::fal_hunyuan_image_to_3d_command::fal_hunyuan_image_to_3d_command; use crate::services::fal::commands::fal_kling_image_to_video_command::fal_kling_image_to_video_command; use crate::services::fal::commands::get_fal_api_key_command::get_fal_api_key_command; use crate::services::fal::commands::set_fal_api_key_command::set_fal_api_key_command; @@ -159,6 +160,7 @@ pub fn run() { .invoke_handler(tauri::generate_handler![ check_sora_session_command, fal_background_removal_command, + fal_hunyuan_image_to_3d_command, fal_kling_image_to_video_command, flip_image, get_app_preferences_command, diff --git a/crates/desktop/artcraft/src/services/fal/commands/fal_background_removal_command.rs b/crates/desktop/artcraft/src/services/fal/commands/fal_background_removal_command.rs index b6f17b9cb8..265e45a0a2 100644 --- a/crates/desktop/artcraft/src/services/fal/commands/fal_background_removal_command.rs +++ b/crates/desktop/artcraft/src/services/fal/commands/fal_background_removal_command.rs @@ -1,18 +1,20 @@ use crate::core::commands::command_response_wrapper::{CommandErrorResponseWrapper, CommandErrorStatus, CommandResult, SerializeMarker}; use crate::core::events::sendable_event_trait::SendableEvent; +use crate::core::state::data_dir::app_data_root::AppDataRoot; +use crate::core::state::data_dir::trait_data_subdir::DataSubdir; +use crate::core::utils::download_media_file_to_temp_dir::download_media_file_to_temp_dir; +use crate::core::utils::get_url_file_extension::get_url_file_extension; +use crate::core::utils::save_base64_image_to_temp_dir::save_base64_image_to_temp_dir; +use crate::core::utils::simple_http_download::simple_http_download; +use crate::core::utils::simple_http_download_to_tempfile::simple_http_download_to_tempfile; +use crate::services::fal::state::fal_credential_manager::FalCredentialManager; use crate::services::sora::events::sora_image_enqueue_failure_event::SoraImageEnqueueFailureEvent; use crate::services::sora::events::sora_image_enqueue_success_event::SoraImageEnqueueSuccessEvent; -use crate::services::fal::state::fal_credential_manager::FalCredentialManager; use crate::services::sora::state::read_sora_credentials_from_disk::read_sora_credentials_from_disk; use crate::services::sora::state::sora_credential_holder::SoraCredentialHolder; use crate::services::sora::state::sora_credential_manager::SoraCredentialManager; use crate::services::sora::state::sora_task_queue::SoraTaskQueue; -use crate::core::state::data_dir::app_data_root::AppDataRoot; -use crate::core::state::data_dir::trait_data_subdir::DataSubdir; use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager; -use crate::core::utils::get_url_file_extension::get_url_file_extension; -use crate::core::utils::simple_http_download::simple_http_download; -use crate::core::utils::simple_http_download_to_tempfile::simple_http_download_to_tempfile; use anyhow::anyhow; use base64::prelude::BASE64_STANDARD; use base64::{DecodeError, Engine}; @@ -41,8 +43,8 @@ use std::io::{Cursor, Read, Write}; use std::ops::Add; use std::path::PathBuf; use std::time::Duration; -use storyteller_client::api_error::ApiError; -use storyteller_client::api_error::ApiError::InternalServerError; +use storyteller_client::error::api_error::ApiError; +use storyteller_client::error::api_error::ApiError::InternalServerError; use storyteller_client::media_files::get_media_file::{get_media_file, GetMediaFileSuccessResponse}; use storyteller_client::media_files::upload_image_media_file_from_file::upload_image_media_file_from_file; use storyteller_client::utils::api_host::ApiHost; @@ -73,7 +75,7 @@ impl SerializeMarker for FalBackgroundRemovalSuccessResponse {} #[derive(Serialize, Debug)] #[serde(rename_all = "snake_case")] -pub enum SoraImageRemixErrorType { +pub enum BackgroundRemovalErrorType { /// Generic server error ServerError, /// No Fal API key available @@ -87,7 +89,7 @@ pub async fn fal_background_removal_command( app_data_root: State<'_, AppDataRoot>, fal_creds_manager: State<'_, FalCredentialManager>, storyteller_creds_manager: State<'_, StorytellerCredentialManager>, -) -> CommandResult { +) -> CommandResult { info!("fal_background_removal_command called; image media token: {:?}", request.image_media_token); @@ -100,7 +102,7 @@ pub async fn fal_background_removal_command( return Err(CommandErrorResponseWrapper { status: CommandErrorStatus::Unauthorized, error_message: Some("You need to set a FAL api key".to_string()), - error_type: Some(SoraImageRemixErrorType::NeedsFalApiKey), + error_type: Some(BackgroundRemovalErrorType::NeedsFalApiKey), error_details: None, }); } @@ -123,7 +125,7 @@ pub async fn fal_background_removal_command( //} let mut status = CommandErrorStatus::ServerError; - let mut error_type = SoraImageRemixErrorType::ServerError; + let mut error_type = BackgroundRemovalErrorType::ServerError; let mut error_message = "A server error occurred. Please try again. If it continues, please tell our staff about the problem."; //match (err) { @@ -205,10 +207,10 @@ pub async fn remove_background( let mut temp_download; if let Some(media_token) = request.image_media_token { - temp_download = download_media_file(&app_data_root, &media_token).await?; + temp_download = download_media_file_to_temp_dir(&app_data_root, &media_token).await?; } else if let Some(base64_bytes) = request.base64_image { - temp_download = persist_base64(&app_data_root, base64_bytes).await?; + temp_download = save_base64_image_to_temp_dir(&app_data_root, base64_bytes).await?; } else { return Err(InnerError::AnyhowError(anyhow!("No image media token or base64 image provided"))); } @@ -253,37 +255,3 @@ pub async fn remove_background( Ok((response, base64_bytes)) } -async fn download_media_file(app_data_root: &AppDataRoot, token: &MediaFileToken) -> Result { - let response = get_media_file(&ApiHost::Storyteller, token).await?; - - let media_file_url = &response.media_file.media_links.cdn_url; - - let extension_with_dot = get_url_file_extension(media_file_url) - .map(|ext| format!(".{}", ext)) - .unwrap_or_else(|| ".png".to_string()); - - //let filename = format!("{}{}", response.media_file.token.as_str(), extension_with_dot); - //let filename = app_data_root.downloads_dir().path().join(&filename); - - let mut file = app_data_root.temp_dir().new_named_temp_file_with_extension(&extension_with_dot)?; - - simple_http_download_to_tempfile(&media_file_url, &mut file).await?; - - Ok(file) // NB: Must return TempFile to not drop / delete it -} - -async fn persist_base64(app_data_root: &AppDataRoot, base64_image: String) -> Result { - let bytes = BASE64_STANDARD.decode(base64_image)?; - - let extension = MimetypeInfo::get_for_bytes(&bytes) - .map(|info| info.file_extension()) - .flatten() - .map(|ext| ext.extension_with_period().to_string()) - .unwrap_or_else(|| ".png".to_string()); - - let mut file = app_data_root.temp_dir().new_named_temp_file_with_extension(&extension)?; - - file.write_all(bytes.as_ref())?; - - Ok(file) // NB: Must return TempFile to not drop / delete it -} diff --git a/crates/desktop/artcraft/src/services/fal/commands/fal_hunyuan_image_to_3d_command.rs b/crates/desktop/artcraft/src/services/fal/commands/fal_hunyuan_image_to_3d_command.rs new file mode 100644 index 0000000000..c3418f664e --- /dev/null +++ b/crates/desktop/artcraft/src/services/fal/commands/fal_hunyuan_image_to_3d_command.rs @@ -0,0 +1,220 @@ +use crate::core::commands::command_response_wrapper::{CommandErrorResponseWrapper, CommandErrorStatus, CommandResult, SerializeMarker}; +use crate::core::events::sendable_event_trait::SendableEvent; +use crate::core::state::data_dir::app_data_root::AppDataRoot; +use crate::core::state::data_dir::trait_data_subdir::DataSubdir; +use crate::core::utils::download_media_file_to_temp_dir::download_media_file_to_temp_dir; +use crate::core::utils::get_url_file_extension::get_url_file_extension; +use crate::core::utils::save_base64_image_to_temp_dir::save_base64_image_to_temp_dir; +use crate::core::utils::simple_http_download::simple_http_download; +use crate::core::utils::simple_http_download_to_tempfile::simple_http_download_to_tempfile; +use crate::services::fal::state::fal_credential_manager::FalCredentialManager; +use crate::services::fal::state::fal_task_queue::FalTaskQueue; +use crate::services::sora::events::sora_image_enqueue_failure_event::SoraImageEnqueueFailureEvent; +use crate::services::sora::events::sora_image_enqueue_success_event::SoraImageEnqueueSuccessEvent; +use crate::services::sora::state::read_sora_credentials_from_disk::read_sora_credentials_from_disk; +use crate::services::sora::state::sora_credential_holder::SoraCredentialHolder; +use crate::services::sora::state::sora_credential_manager::SoraCredentialManager; +use crate::services::sora::state::sora_task_queue::SoraTaskQueue; +use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager; +use anyhow::anyhow; +use base64::prelude::BASE64_STANDARD; +use base64::{DecodeError, Engine}; +use errors::{AnyhowError, AnyhowResult}; +use fal_client::fal_error_plus::FalErrorPlus; +use fal_client::requests::enqueue_hunyuan2_image_to_3d::{enqueue_hunyuan2_image_to_3d, Hunyuan2Args}; +use fal_client::requests::enqueue_kling_16_image_to_video::{enqueue_kling_16_image_to_video, Kling16Args, Kling16Duration}; +use fal_client::requests::remove_background_rembg::remove_background_rembg; +use filesys::file_read_bytes::file_read_bytes; +use image::codecs::png::{CompressionType, FilterType, PngEncoder}; +use image::{DynamicImage, EncodableLayout, ImageReader}; +use log::{debug, error, info, warn}; +use mimetypes::mimetype_info::mimetype_info::MimetypeInfo; +use openai_sora_client::credentials::SoraCredentials; +use openai_sora_client::creds::credential_migration::CredentialMigrationRef; +use openai_sora_client::recipes::image_remix_with_session_auto_renew::{image_remix_with_session_auto_renew, ImageRemixAutoRenewRequest}; +use openai_sora_client::recipes::image_upload_from_file_with_session_auto_renew::{image_upload_from_file_with_session_auto_renew, ImageUploadFromFileAutoRenewRequest}; +use openai_sora_client::recipes::maybe_upgrade_or_renew_session::maybe_upgrade_or_renew_session; +use openai_sora_client::requests::image_gen::common::{ImageSize, NumImages}; +use openai_sora_client::requests::image_gen::sora_image_gen_remix::{sora_image_gen_remix, SoraImageGenRemixRequest}; +use openai_sora_client::requests::upload::upload_media_from_bytes::sora_media_upload_from_bytes; +use openai_sora_client::requests::upload::upload_media_from_file::sora_media_upload_from_file; +use openai_sora_client::sora_error::SoraError; +use reqwest::Url; +use serde_derive::{Deserialize, Serialize}; +use std::fs::read_to_string; +use std::io::{Cursor, Read, Write}; +use std::ops::Add; +use std::path::PathBuf; +use std::time::Duration; +use storyteller_client::error::api_error::ApiError; +use storyteller_client::error::api_error::ApiError::InternalServerError; +use storyteller_client::media_files::get_media_file::{get_media_file, GetMediaFileSuccessResponse}; +use storyteller_client::media_files::upload_image_media_file_from_file::upload_image_media_file_from_file; +use storyteller_client::utils::api_host::ApiHost; +use tauri::{AppHandle, Emitter, Manager, State}; +use tempfile::NamedTempFile; +use tokens::tokens::media_files::MediaFileToken; + +#[derive(Deserialize)] +pub struct FalHunyuanImageTo3dRequest { + /// Image media file; the image to remove the background from. + pub image_media_token: Option, + + /// Base64-encoded image + pub base64_image: Option, +} + +#[derive(Serialize, Debug)] +#[serde(rename_all = "snake_case")] +pub enum HunyuanErrorType { + /// Generic server error + ServerError, + /// No Fal API key available + NeedsFalApiKey, +} + + +#[tauri::command] +pub async fn fal_hunyuan_image_to_3d_command( + request: FalHunyuanImageTo3dRequest, + app_data_root: State<'_, AppDataRoot>, + fal_creds_manager: State<'_, FalCredentialManager>, + storyteller_creds_manager: State<'_, StorytellerCredentialManager>, + fal_task_queue: State<'_, FalTaskQueue>, +) -> CommandResult<(), HunyuanErrorType, ()> { + + info!("fal_hunyuan_image_to_3d_command called; image media token: {:?}", request.image_media_token); + + let has_credentials = fal_creds_manager + .has_apparent_api_token() + .unwrap_or(true); // NB: Failures would be lock issues + + if !has_credentials { + warn!("No API key found"); + return Err(CommandErrorResponseWrapper { + status: CommandErrorStatus::Unauthorized, + error_message: Some("You need to set a FAL api key".to_string()), + error_type: Some(HunyuanErrorType::NeedsFalApiKey), + error_details: None, + }); + } + + let result = image_to_3d( + request, + &app_data_root, + &fal_creds_manager, + &storyteller_creds_manager, + &fal_task_queue, + ).await; + + if let Err(err) = result { + error!("error: {:?}", err); + + //let event = SoraImageEnqueueFailureEvent {}; + //let result = event.send(&app); + //if let Err(err) = result { + // error!("Failed to emit event: {:?}", err); + //} + + let mut status = CommandErrorStatus::ServerError; + let mut error_type = HunyuanErrorType::ServerError; + let mut error_message = "A server error occurred. Please try again. If it continues, please tell our staff about the problem."; + + //match (err) { + // _ => {}, + //} + + return Err(CommandErrorResponseWrapper { + status, + error_message: Some(error_message.to_string()), + error_type: Some(error_type), + error_details: None, + }) + } + + //let event = SoraImageEnqueueSuccessEvent {}; + //let result = event.send(&app); + //if let Err(err) = result { + // error!("Failed to emit event: {:?}", err); + //} + + Ok(().into()) +} + +#[derive(Debug)] +enum InnerError { + FalError(FalErrorPlus), + AnyhowError(AnyhowError), + StorytellerApiError(ApiError), + DecodeError(DecodeError), + IoError(std::io::Error), +} + +impl From for InnerError { + fn from(value: AnyhowError) -> Self { + Self::AnyhowError(value) + } +} + +impl From for InnerError { + fn from(value: FalErrorPlus) -> Self { + Self::FalError(value) + } +} + +impl From for InnerError { + fn from(value: ApiError) -> Self { + Self::StorytellerApiError(value) + } +} + +impl From for InnerError { + fn from(value: DecodeError) -> Self { + Self::DecodeError(value) + } +} + +impl From for InnerError { + fn from(value: std::io::Error) -> Self { + Self::IoError(value) + } +} + +pub async fn image_to_3d( + request: FalHunyuanImageTo3dRequest, + app_data_root: &AppDataRoot, + fal_creds_manager: &FalCredentialManager, + storyteller_creds_manager: &StorytellerCredentialManager, + fal_task_queue: &FalTaskQueue, +) -> Result<(), InnerError> { + + let api_key = fal_creds_manager.get_key_required()?; + let creds = storyteller_creds_manager.get_credentials_required()?; + + let mut temp_download; + + if let Some(media_token) = request.image_media_token { + temp_download = download_media_file_to_temp_dir(&app_data_root, &media_token).await?; + + } else if let Some(base64_bytes) = request.base64_image { + temp_download = save_base64_image_to_temp_dir(&app_data_root, base64_bytes).await?; + } else { + return Err(InnerError::AnyhowError(anyhow!("No image media token or base64 image provided"))); + } + + info!("Calling FAL image to 3d ..."); + + let filename = temp_download.path().to_path_buf(); + + let enqueued = enqueue_hunyuan2_image_to_3d(Hunyuan2Args { + image_path: filename, + api_key: &api_key, + }).await?; + + if let Err(err) = fal_task_queue.insert(&enqueued) { + error!("Failed to enqueue task: {:?}", err); + return Err(InnerError::AnyhowError(anyhow!("Failed to enqueue task: {:?}", err))); + } + + Ok(()) +} diff --git a/crates/desktop/artcraft/src/services/fal/commands/fal_kling_image_to_video_command.rs b/crates/desktop/artcraft/src/services/fal/commands/fal_kling_image_to_video_command.rs index dc17bee2d8..3ebf5f4406 100644 --- a/crates/desktop/artcraft/src/services/fal/commands/fal_kling_image_to_video_command.rs +++ b/crates/desktop/artcraft/src/services/fal/commands/fal_kling_image_to_video_command.rs @@ -1,19 +1,21 @@ use crate::core::commands::command_response_wrapper::{CommandErrorResponseWrapper, CommandErrorStatus, CommandResult, SerializeMarker}; use crate::core::events::sendable_event_trait::SendableEvent; -use crate::services::sora::events::sora_image_enqueue_failure_event::SoraImageEnqueueFailureEvent; -use crate::services::sora::events::sora_image_enqueue_success_event::SoraImageEnqueueSuccessEvent; +use crate::core::state::data_dir::app_data_root::AppDataRoot; +use crate::core::state::data_dir::trait_data_subdir::DataSubdir; +use crate::core::utils::download_media_file_to_temp_dir::download_media_file_to_temp_dir; +use crate::core::utils::get_url_file_extension::get_url_file_extension; +use crate::core::utils::save_base64_image_to_temp_dir::save_base64_image_to_temp_dir; +use crate::core::utils::simple_http_download::simple_http_download; +use crate::core::utils::simple_http_download_to_tempfile::simple_http_download_to_tempfile; use crate::services::fal::state::fal_credential_manager::FalCredentialManager; use crate::services::fal::state::fal_task_queue::FalTaskQueue; +use crate::services::sora::events::sora_image_enqueue_failure_event::SoraImageEnqueueFailureEvent; +use crate::services::sora::events::sora_image_enqueue_success_event::SoraImageEnqueueSuccessEvent; use crate::services::sora::state::read_sora_credentials_from_disk::read_sora_credentials_from_disk; use crate::services::sora::state::sora_credential_holder::SoraCredentialHolder; use crate::services::sora::state::sora_credential_manager::SoraCredentialManager; use crate::services::sora::state::sora_task_queue::SoraTaskQueue; -use crate::core::state::data_dir::app_data_root::AppDataRoot; -use crate::core::state::data_dir::trait_data_subdir::DataSubdir; use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager; -use crate::core::utils::get_url_file_extension::get_url_file_extension; -use crate::core::utils::simple_http_download::simple_http_download; -use crate::core::utils::simple_http_download_to_tempfile::simple_http_download_to_tempfile; use anyhow::anyhow; use base64::prelude::BASE64_STANDARD; use base64::{DecodeError, Engine}; @@ -43,8 +45,8 @@ use std::io::{Cursor, Read, Write}; use std::ops::Add; use std::path::PathBuf; use std::time::Duration; -use storyteller_client::api_error::ApiError; -use storyteller_client::api_error::ApiError::InternalServerError; +use storyteller_client::error::api_error::ApiError; +use storyteller_client::error::api_error::ApiError::InternalServerError; use storyteller_client::media_files::get_media_file::{get_media_file, GetMediaFileSuccessResponse}; use storyteller_client::media_files::upload_image_media_file_from_file::upload_image_media_file_from_file; use storyteller_client::utils::api_host::ApiHost; @@ -63,7 +65,7 @@ pub struct FalKlingImageToVideoRequest { #[derive(Serialize, Debug)] #[serde(rename_all = "snake_case")] -pub enum SoraImageRemixErrorType { +pub enum KlingImageToVideoErrorType { /// Generic server error ServerError, /// No Fal API key available @@ -78,7 +80,7 @@ pub async fn fal_kling_image_to_video_command( fal_creds_manager: State<'_, FalCredentialManager>, storyteller_creds_manager: State<'_, StorytellerCredentialManager>, fal_task_queue: State<'_, FalTaskQueue>, -) -> CommandResult<(), SoraImageRemixErrorType, ()> { +) -> CommandResult<(), KlingImageToVideoErrorType, ()> { info!("fal_kling_image_to_video_command called; image media token: {:?}", request.image_media_token); @@ -91,7 +93,7 @@ pub async fn fal_kling_image_to_video_command( return Err(CommandErrorResponseWrapper { status: CommandErrorStatus::Unauthorized, error_message: Some("You need to set a FAL api key".to_string()), - error_type: Some(SoraImageRemixErrorType::NeedsFalApiKey), + error_type: Some(KlingImageToVideoErrorType::NeedsFalApiKey), error_details: None, }); } @@ -114,7 +116,7 @@ pub async fn fal_kling_image_to_video_command( //} let mut status = CommandErrorStatus::ServerError; - let mut error_type = SoraImageRemixErrorType::ServerError; + let mut error_type = KlingImageToVideoErrorType::ServerError; let mut error_message = "A server error occurred. Please try again. If it continues, please tell our staff about the problem."; //match (err) { @@ -191,10 +193,10 @@ pub async fn image_to_video( let mut temp_download; if let Some(media_token) = request.image_media_token { - temp_download = download_media_file(&app_data_root, &media_token).await?; + temp_download = download_media_file_to_temp_dir(&app_data_root, &media_token).await?; } else if let Some(base64_bytes) = request.base64_image { - temp_download = persist_base64(&app_data_root, base64_bytes).await?; + temp_download = save_base64_image_to_temp_dir(&app_data_root, base64_bytes).await?; } else { return Err(InnerError::AnyhowError(anyhow!("No image media token or base64 image provided"))); } @@ -216,38 +218,3 @@ pub async fn image_to_video( Ok(()) } - -async fn download_media_file(app_data_root: &AppDataRoot, token: &MediaFileToken) -> Result { - let response = get_media_file(&ApiHost::Storyteller, token).await?; - - let media_file_url = &response.media_file.media_links.cdn_url; - - let extension_with_dot = get_url_file_extension(media_file_url) - .map(|ext| format!(".{}", ext)) - .unwrap_or_else(|| ".png".to_string()); - - //let filename = format!("{}{}", response.media_file.token.as_str(), extension_with_dot); - //let filename = app_data_root.downloads_dir().path().join(&filename); - - let mut file = app_data_root.temp_dir().new_named_temp_file_with_extension(&extension_with_dot)?; - - simple_http_download_to_tempfile(&media_file_url, &mut file).await?; - - Ok(file) // NB: Must return TempFile to not drop / delete it -} - -async fn persist_base64(app_data_root: &AppDataRoot, base64_image: String) -> Result { - let bytes = BASE64_STANDARD.decode(base64_image)?; - - let extension = MimetypeInfo::get_for_bytes(&bytes) - .map(|info| info.file_extension()) - .flatten() - .map(|ext| ext.extension_with_period().to_string()) - .unwrap_or_else(|| ".png".to_string()); - - let mut file = app_data_root.temp_dir().new_named_temp_file_with_extension(&extension)?; - - file.write_all(bytes.as_ref())?; - - Ok(file) // NB: Must return TempFile to not drop / delete it -} diff --git a/crates/desktop/artcraft/src/services/fal/commands/mod.rs b/crates/desktop/artcraft/src/services/fal/commands/mod.rs index d547c4ca54..49f725d47b 100644 --- a/crates/desktop/artcraft/src/services/fal/commands/mod.rs +++ b/crates/desktop/artcraft/src/services/fal/commands/mod.rs @@ -1,4 +1,5 @@ pub mod fal_background_removal_command; +pub mod fal_hunyuan_image_to_3d_command; pub mod fal_kling_image_to_video_command; pub mod get_fal_api_key_command; pub mod set_fal_api_key_command; \ No newline at end of file diff --git a/crates/desktop/artcraft/src/services/fal/threads/fal_task_polling_thread.rs b/crates/desktop/artcraft/src/services/fal/threads/fal_task_polling_thread.rs index 7a95e691ca..bf8f1f4883 100644 --- a/crates/desktop/artcraft/src/services/fal/threads/fal_task_polling_thread.rs +++ b/crates/desktop/artcraft/src/services/fal/threads/fal_task_polling_thread.rs @@ -1,32 +1,38 @@ use crate::core::events::sendable_event_trait::SendableEvent; -use crate::services::sora::events::sora_image_generation_complete_event::SoraImageGenerationCompleteEvent; -use crate::services::sora::events::sora_image_generation_failed_event::SoraImageGenerationFailedEvent; use crate::core::state::data_dir::app_data_root::AppDataRoot; use crate::core::state::data_dir::trait_data_subdir::DataSubdir; +use crate::core::utils::download_url_to_temp_dir::download_url_to_temp_dir; use crate::services::fal::state::fal_credential_manager::FalCredentialManager; use crate::services::fal::state::fal_task_queue::FalTaskQueue; +use crate::services::sora::events::sora_image_generation_complete_event::SoraImageGenerationCompleteEvent; +use crate::services::sora::events::sora_image_generation_failed_event::SoraImageGenerationFailedEvent; use crate::services::sora::state::sora_credential_manager::SoraCredentialManager; use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager; use errors::AnyhowResult; use fal_client::export::queue::Status; use fal_client::fal_error_plus::FalErrorPlus; +use fal_client::model::fal_request_id::FalRequestId; use fal_client::utils::queue_status_checker::QueueStatusChecker; use log::{error, info, warn}; +use mimetypes::mimetype_for_file::get_mimetype_for_file; +use mimetypes::mimetype_info::file_extension::FileExtension; +use mimetypes::mimetype_info::mimetype_info::MimetypeInfo; use openai_sora_client::requests::image_gen::image_gen_status::{Generation, TaskId, TaskStatus}; use serde_derive::Serialize; use std::fs::File; use std::io::Write; use std::path::PathBuf; +use storyteller_client::error::api_error::ApiError; use storyteller_client::media_files::upload_image_media_file_from_file::upload_image_media_file_from_file; +use storyteller_client::media_files::upload_new_engine_asset_from_file::upload_new_engine_asset_from_file; +use storyteller_client::media_files::upload_video_media_file_from_file::{upload_video_media_file_from_file, UploadVideoMediaFileSuccessResponse}; +use storyteller_client::recipes::upload_media_file_from_file::upload_media_file_from_file; use storyteller_client::utils::api_host::ApiHost; use tauri::{AppHandle, Emitter}; use tempdir::TempDir; use tempfile::NamedTempFile; use tokens::tokens::media_files::MediaFileToken; use url::Url; -use fal_client::model::fal_request_id::FalRequestId; -use storyteller_client::api_error::ApiError; -use storyteller_client::media_files::upload_video_media_file_from_file::{upload_video_media_file_from_file, UploadVideoMediaFileSuccessResponse}; pub async fn fal_task_polling_thread( app_handle: AppHandle, @@ -125,20 +131,21 @@ async fn polling_loop( } } }; - - - let download_file = download_generation(&url, &app_data_root).await?; - let result = upload_video_media_file_from_file( - &ApiHost::Storyteller, - Some(&storyteller_creds), + // TODO: Clean up + + let download_file = download_url_to_temp_dir(&url, &app_data_root).await?; + + let result = upload_media_file_from_file( + &ApiHost::Storyteller, + Some(&storyteller_creds), download_file ).await; - + match result { Ok(success) => { info!("Uploaded to API backend: {:?}", success.media_file_token); - + //let event = SoraImageGenerationCompleteEvent { // media_file_token: success.media_file_token, //}; @@ -147,12 +154,12 @@ async fn polling_loop( succeeded_tasks.push(task_id); } Err(err) => { - warn!("Failed to upload video media file: {:?}", err); + warn!("Failed to upload media file: {:?}", err); continue; } } - } + } // // TODO: The cursoring logic likely needs to improve. @@ -225,17 +232,3 @@ async fn polling_loop( } } -async fn download_generation(url: &str, app_data_root: &AppDataRoot) -> AnyhowResult { - let url = Url::parse(&url)?; - - let response = reqwest::get(url.clone()).await?; - let response_bytes = response.bytes().await?; - - let ext = url.path().split(".").last().unwrap_or("png"); - - let mut temp_file = app_data_root.temp_dir().new_named_temp_file_with_extension(ext)?; - - temp_file.write_all(&response_bytes)?; - - Ok(temp_file) -} diff --git a/crates/desktop/artcraft/src/services/sora/commands/sora_image_remix_command.rs b/crates/desktop/artcraft/src/services/sora/commands/sora_image_remix_command.rs index 1e23b3d9dc..6ccb34bd89 100644 --- a/crates/desktop/artcraft/src/services/sora/commands/sora_image_remix_command.rs +++ b/crates/desktop/artcraft/src/services/sora/commands/sora_image_remix_command.rs @@ -1,15 +1,15 @@ use crate::core::commands::command_response_wrapper::{CommandErrorResponseWrapper, CommandErrorStatus, CommandResult}; use crate::core::events::sendable_event_trait::SendableEvent; +use crate::core::state::data_dir::app_data_root::AppDataRoot; +use crate::core::state::data_dir::trait_data_subdir::DataSubdir; +use crate::core::utils::get_url_file_extension::get_url_file_extension; +use crate::core::utils::simple_http_download::simple_http_download; use crate::services::sora::events::sora_image_enqueue_failure_event::SoraImageEnqueueFailureEvent; use crate::services::sora::events::sora_image_enqueue_success_event::SoraImageEnqueueSuccessEvent; use crate::services::sora::state::read_sora_credentials_from_disk::read_sora_credentials_from_disk; use crate::services::sora::state::sora_credential_holder::SoraCredentialHolder; use crate::services::sora::state::sora_credential_manager::SoraCredentialManager; use crate::services::sora::state::sora_task_queue::SoraTaskQueue; -use crate::core::state::data_dir::app_data_root::AppDataRoot; -use crate::core::state::data_dir::trait_data_subdir::DataSubdir; -use crate::core::utils::get_url_file_extension::get_url_file_extension; -use crate::core::utils::simple_http_download::simple_http_download; use base64::prelude::BASE64_STANDARD; use base64::Engine; use errors::{AnyhowError, AnyhowResult}; @@ -31,8 +31,8 @@ use std::fs::read_to_string; use std::io::Cursor; use std::ops::Add; use std::time::Duration; -use storyteller_client::api_error::ApiError; -use storyteller_client::api_error::ApiError::InternalServerError; +use storyteller_client::error::api_error::ApiError; +use storyteller_client::error::api_error::ApiError::InternalServerError; use storyteller_client::media_files::get_media_file::get_media_file; use storyteller_client::utils::api_host::ApiHost; use tauri::{AppHandle, Emitter, Manager, State}; diff --git a/crates/lib/api_clients/fal_client/src/model/fal_endpoint.rs b/crates/lib/api_clients/fal_client/src/model/fal_endpoint.rs index 3661d1e155..75d866a243 100644 --- a/crates/lib/api_clients/fal_client/src/model/fal_endpoint.rs +++ b/crates/lib/api_clients/fal_client/src/model/fal_endpoint.rs @@ -1,22 +1,28 @@ use fal::queue::Queue; use serde::de::DeserializeOwned; +const HUNYUAN_3D_2_BASE: &str = "fal-ai/hunyuan3d/v2"; + const KLING16_IMAGE_TO_VIDEO: &str = "fal-ai/kling-video/v1.6/pro/image-to-video"; + #[derive(Clone, Debug)] pub enum FalEndpoint { Kling16ImageToVideo, + Hunyuan3d2Base, Other(String), } impl FalEndpoint { pub fn from_queue_response(queue: &Queue) -> Self { match queue.endpoint.as_str() { + HUNYUAN_3D_2_BASE => FalEndpoint::Hunyuan3d2Base, KLING16_IMAGE_TO_VIDEO => FalEndpoint::Kling16ImageToVideo, _ => FalEndpoint::Other(queue.endpoint.clone()), } } pub fn url(&self) -> &str { match self { + Self::Hunyuan3d2Base => HUNYUAN_3D_2_BASE, Self::Kling16ImageToVideo => KLING16_IMAGE_TO_VIDEO, Self::Other(url) => url, } diff --git a/crates/lib/api_clients/fal_client/src/requests/enqueue_hunyuan2_image_to_3d.rs b/crates/lib/api_clients/fal_client/src/requests/enqueue_hunyuan2_image_to_3d.rs new file mode 100644 index 0000000000..546f8862f4 --- /dev/null +++ b/crates/lib/api_clients/fal_client/src/requests/enqueue_hunyuan2_image_to_3d.rs @@ -0,0 +1,69 @@ +use crate::creds::fal_api_key::FalApiKey; +use crate::fal_error_plus::FalErrorPlus; +use crate::model::enqueued_request::EnqueuedRequest; +use fal::endpoints::fal_ai::hunyuan3d::v2::v2; +use fal::endpoints::fal_ai::hunyuan3d::v2::Hunyuan3DInput; +use fal_client::file_to_base64_url::file_to_base64_url; +use futures::StreamExt; +use std::io::Write; +use std::path::Path; + +pub struct Hunyuan2Args<'a, P: AsRef> { + pub image_path: P, + pub api_key: &'a FalApiKey, +} + +pub async fn enqueue_hunyuan2_image_to_3d>(args: Hunyuan2Args<'_, P>) -> Result { + let image_url = file_to_base64_url(args.image_path)?; + + /* + TODO: Handle error messages - + FalError(FalError(Other("{\"detail\": \"Invalid Key Authorization header format. Expected ':'.\"}"))) + FalError(FalError(Other("{\"detail\": \"No user found for Key ID and Secret\"}"))) + */ + + let request = Hunyuan3DInput { + input_image_url: image_url, + textured_mesh: Some(true), + guidance_scale: None, + num_inference_steps: None, + octree_resolution: None, + seed: None, + }; + + let result = v2(request) + .with_api_key(&args.api_key.0) + .queue() + .await?; + + Ok(EnqueuedRequest::from_queue_response(&result)?) +} + +#[cfg(test)] +mod tests { + use crate::creds::fal_api_key::FalApiKey; + use crate::requests::enqueue_hunyuan2_image_to_3d::{enqueue_hunyuan2_image_to_3d, Hunyuan2Args}; + use errors::AnyhowResult; + use std::fs::read_to_string; + use testing::test_file_path::test_file_path; + + #[tokio::test] + #[ignore] + async fn test() -> AnyhowResult<()> { + let image = test_file_path("test_data/image/juno.jpg")?; + + // XXX: Don't commit secrets! + let secret = read_to_string("/Users/bt/Artcraft/credentials/fal_api_key.txt")?; + + let api_key = FalApiKey::from_str(&secret); + + let args = Hunyuan2Args { + image_path: image, + api_key: &api_key, + }; + + let result = enqueue_hunyuan2_image_to_3d(args).await?; + + Ok(()) + } +} diff --git a/crates/lib/api_clients/fal_client/src/requests/mod.rs b/crates/lib/api_clients/fal_client/src/requests/mod.rs index 5490aa7586..3544169e94 100644 --- a/crates/lib/api_clients/fal_client/src/requests/mod.rs +++ b/crates/lib/api_clients/fal_client/src/requests/mod.rs @@ -1,2 +1,3 @@ +pub mod enqueue_hunyuan2_image_to_3d; pub mod enqueue_kling_16_image_to_video; pub mod remove_background_rembg; \ No newline at end of file diff --git a/crates/lib/api_clients/fal_client/src/utils/queue_status_checker.rs b/crates/lib/api_clients/fal_client/src/utils/queue_status_checker.rs index 8944cf16ad..e93e300cf3 100644 --- a/crates/lib/api_clients/fal_client/src/utils/queue_status_checker.rs +++ b/crates/lib/api_clients/fal_client/src/utils/queue_status_checker.rs @@ -1,15 +1,13 @@ -use fal::endpoints::fal_ai::kling_video::v1_6::pro::effects::I2VOutput; -use fal::FalError; -use fal::prelude::fal_ai::imageutils::sam::RemoveBackgroundOutput; -use fal::prelude::QueueResponse; -use fal::queue::{Queue, QueueStatus}; -use reqwest::Client; -use serde::de::DeserializeOwned; -use url::Url; use crate::creds::fal_api_key::FalApiKey; use crate::fal_error_plus::FalErrorPlus; use crate::model::enqueued_request::EnqueuedRequest; use crate::model::fal_endpoint::FalEndpoint; +use fal::endpoints::fal_ai::hunyuan3d::v2::ObjectOutput; +use fal::endpoints::fal_ai::kling_video::v1_6::pro::effects::I2VOutput; +use fal::prelude::QueueResponse; +use fal::queue::{Queue, QueueStatus}; +use reqwest::Client; +use serde::de::DeserializeOwned; pub struct QueueStatusChecker { api_key: FalApiKey, @@ -22,6 +20,11 @@ impl QueueStatusChecker { pub async fn check_status(&self, request: &EnqueuedRequest) -> Result { match &request.fal_endpoint { + FalEndpoint::Hunyuan3d2Base => { + let queue = self.make_queue::(&request); + let status = queue.status(false).await?; + Ok(status) + } FalEndpoint::Kling16ImageToVideo => { let queue = self.make_queue::(&request); let status = queue.status(false).await?; @@ -35,6 +38,11 @@ impl QueueStatusChecker { pub async fn get_download_url(&self, request: &EnqueuedRequest) -> Result { match &request.fal_endpoint { + FalEndpoint::Hunyuan3d2Base => { + let queue = self.make_queue::(&request); + let status = queue.response().await?; + Ok(status.model_mesh.url) + } FalEndpoint::Kling16ImageToVideo => { let queue = self.make_queue::(&request); let status = queue.response().await?; diff --git a/crates/lib/api_clients/storyteller_client/Cargo.toml b/crates/lib/api_clients/storyteller_client/Cargo.toml index 55dd190279..03d6332dac 100644 --- a/crates/lib/api_clients/storyteller_client/Cargo.toml +++ b/crates/lib/api_clients/storyteller_client/Cargo.toml @@ -22,6 +22,7 @@ enums_public = { workspace = true } errors = { workspace = true } idempotency = { workspace = true } tokens = { workspace = true } +mimetypes = { workspace = true } # External anyhow = { workspace = true } diff --git a/crates/lib/api_clients/storyteller_client/src/api_error.rs b/crates/lib/api_clients/storyteller_client/src/error/api_error.rs similarity index 100% rename from crates/lib/api_clients/storyteller_client/src/api_error.rs rename to crates/lib/api_clients/storyteller_client/src/error/api_error.rs diff --git a/crates/lib/api_clients/storyteller_client/src/error/client_error.rs b/crates/lib/api_clients/storyteller_client/src/error/client_error.rs new file mode 100644 index 0000000000..1bf04115c1 --- /dev/null +++ b/crates/lib/api_clients/storyteller_client/src/error/client_error.rs @@ -0,0 +1,35 @@ +use std::error::Error; +use std::fmt::{Display, Formatter}; +use std::io; +use std::path::PathBuf; + +/// An error arising from the client, either before or after the request is sent. +#[derive(Debug)] +pub enum ClientError { + /// Could not determine the filetype for the file. + FileTypeNotKnown(PathBuf), + + /// Could not handle the file type. + FileTypeNotHandled(PathBuf), + + /// An error doing file I/O (on our side) + IoError(io::Error), +} + +impl Error for ClientError {} + +impl Display for ClientError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Self::FileTypeNotKnown(path) => write!(f, "Could not determine the filetype for the file: {:?}", path), + Self::FileTypeNotHandled(path) => write!(f, "Could not handle the file type for the file: {:?}", path), + Self::IoError(err) => write!(f, "IO error: {}", err), + } + } +} + +impl From for ClientError { + fn from(error: io::Error) -> Self { + ClientError::IoError(error) + } +} diff --git a/crates/lib/api_clients/storyteller_client/src/error/mod.rs b/crates/lib/api_clients/storyteller_client/src/error/mod.rs new file mode 100644 index 0000000000..4b238e7786 --- /dev/null +++ b/crates/lib/api_clients/storyteller_client/src/error/mod.rs @@ -0,0 +1,3 @@ +pub mod api_error; +pub mod client_error; +pub mod storyteller_error; \ No newline at end of file diff --git a/crates/lib/api_clients/storyteller_client/src/error/storyteller_error.rs b/crates/lib/api_clients/storyteller_client/src/error/storyteller_error.rs new file mode 100644 index 0000000000..1c0fdec7e8 --- /dev/null +++ b/crates/lib/api_clients/storyteller_client/src/error/storyteller_error.rs @@ -0,0 +1,32 @@ +use crate::error::api_error::ApiError; +use crate::error::client_error::ClientError; +use std::error::Error; + +#[derive(Debug)] +pub enum StorytellerError { + Api(ApiError), + Client(ClientError), +} + +impl Error for StorytellerError {} + +impl std::fmt::Display for StorytellerError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + StorytellerError::Api(e) => write!(f, "API error: {:?}", e), + StorytellerError::Client(e) => write!(f, "Client error: {:?}", e), + } + } +} + +impl From for StorytellerError { + fn from(error: ApiError) -> Self { + StorytellerError::Api(error) + } +} + +impl From for StorytellerError { + fn from(error: ClientError) -> Self { + StorytellerError::Client(error) + } +} diff --git a/crates/lib/api_clients/storyteller_client/src/lib.rs b/crates/lib/api_clients/storyteller_client/src/lib.rs index d0b6be4a4e..e1d6792184 100644 --- a/crates/lib/api_clients/storyteller_client/src/lib.rs +++ b/crates/lib/api_clients/storyteller_client/src/lib.rs @@ -1,7 +1,8 @@ //! An API client for Storyteller / FakeYou -pub mod api_error; pub mod credentials; +pub mod error; pub mod media_files; +pub mod recipes; pub mod shared_response_types; pub mod utils; diff --git a/crates/lib/api_clients/storyteller_client/src/media_files/get_media_file.rs b/crates/lib/api_clients/storyteller_client/src/media_files/get_media_file.rs index f83104fca4..78d897cf86 100644 --- a/crates/lib/api_clients/storyteller_client/src/media_files/get_media_file.rs +++ b/crates/lib/api_clients/storyteller_client/src/media_files/get_media_file.rs @@ -1,4 +1,4 @@ -use crate::api_error::ApiError; +use crate::error::api_error::ApiError; use crate::shared_response_types::media_file_cover_image_details::MediaFileCoverImageDetails; use crate::shared_response_types::media_links::MediaLinks; use crate::shared_response_types::simple_entity_stats::SimpleEntityStats; diff --git a/crates/lib/api_clients/storyteller_client/src/media_files/mod.rs b/crates/lib/api_clients/storyteller_client/src/media_files/mod.rs index b10beebb3a..f37ac72cb9 100644 --- a/crates/lib/api_clients/storyteller_client/src/media_files/mod.rs +++ b/crates/lib/api_clients/storyteller_client/src/media_files/mod.rs @@ -1,3 +1,4 @@ pub mod get_media_file; pub mod upload_image_media_file_from_file; +pub mod upload_new_engine_asset_from_file; pub mod upload_video_media_file_from_file; diff --git a/crates/lib/api_clients/storyteller_client/src/media_files/upload_image_media_file_from_file.rs b/crates/lib/api_clients/storyteller_client/src/media_files/upload_image_media_file_from_file.rs index eb9faa09de..6b9f1083f7 100644 --- a/crates/lib/api_clients/storyteller_client/src/media_files/upload_image_media_file_from_file.rs +++ b/crates/lib/api_clients/storyteller_client/src/media_files/upload_image_media_file_from_file.rs @@ -1,5 +1,5 @@ -use crate::api_error::ApiError; use crate::credentials::storyteller_credential_set::StorytellerCredentialSet; +use crate::error::api_error::ApiError; use crate::shared_response_types::media_file_cover_image_details::MediaFileCoverImageDetails; use crate::shared_response_types::media_links::MediaLinks; use crate::shared_response_types::simple_entity_stats::SimpleEntityStats; diff --git a/crates/lib/api_clients/storyteller_client/src/media_files/upload_new_engine_asset_from_file.rs b/crates/lib/api_clients/storyteller_client/src/media_files/upload_new_engine_asset_from_file.rs new file mode 100644 index 0000000000..054f283b55 --- /dev/null +++ b/crates/lib/api_clients/storyteller_client/src/media_files/upload_new_engine_asset_from_file.rs @@ -0,0 +1,93 @@ +use crate::credentials::storyteller_credential_set::StorytellerCredentialSet; +use crate::error::api_error::ApiError; +use crate::shared_response_types::media_file_cover_image_details::MediaFileCoverImageDetails; +use crate::shared_response_types::media_links::MediaLinks; +use crate::shared_response_types::simple_entity_stats::SimpleEntityStats; +use crate::shared_response_types::user_details_light::UserDetailsLight; +use crate::utils::api_host::ApiHost; +use crate::utils::constants::{APPLICATION_JSON, USER_AGENT}; +use crate::utils::filter_bad_response::filter_bad_response; +use crate::utils::http_get_anonymous::http_get_anonymous; +use chrono::{DateTime, Utc}; +use enums::by_table::media_files::media_file_animation_type::MediaFileAnimationType; +use enums::by_table::media_files::media_file_class::MediaFileClass; +use enums::by_table::media_files::media_file_engine_category::MediaFileEngineCategory; +use enums::by_table::media_files::media_file_type::MediaFileType; +use enums::by_table::model_weights::weights_category::WeightsCategory; +use enums::common::visibility::Visibility; +use enums::no_table::style_transfer::style_transfer_name::StyleTransferName; +use enums_public::by_table::model_weights::public_weights_types::PublicWeightsType; +use idempotency::uuid::generate_random_uuid; +use log::debug; +use reqwest::multipart::Form; +use reqwest::Client; +use serde_derive::{Deserialize, Serialize}; +use std::fs::File; +use std::io::Read; +use std::path::Path; +use tokens::tokens::batch_generations::BatchGenerationToken; +use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::model_weights::ModelWeightToken; +use tokens::tokens::prompts::PromptToken; +use uuid::uuid; + +// TODO: The naming of these methods is ridiculous. It needs to be cleaned up. + +/// Upload an image media file from a file. +pub async fn upload_new_engine_asset_from_file>( + api_host: &ApiHost, + maybe_creds: Option<&StorytellerCredentialSet>, + path: P, +) -> Result { + + let url = get_route(api_host); + + debug!("Requesting {:?}", &url); + + //let mut file = File::open(path)?; + + let client = Client::builder() + .gzip(true) + .build()?; + + let form = Form::new() + .text("uuid_idempotency_token", generate_random_uuid()) + .text("engine_category", "object") // TODO: Which type should we use? + .file("file", path) + .await?; + + let mut request_builder = client.post(url) + .header("User-Agent", USER_AGENT) + .header("Accept", APPLICATION_JSON); + + if let Some(creds) = maybe_creds { + if let Some(header) = &creds.maybe_as_cookie_header() { + request_builder = request_builder.header("Cookie", header); + } + } + + let response = request_builder + .multipart(form) + .send() + .await?; + + let response = filter_bad_response(response).await?; + let response_body = &response.text().await?; + + let media_file = serde_json::from_str(&response_body)?; + + Ok(media_file) +} + +fn get_route(api_host: &ApiHost) -> String { + let api_hostname = api_host.to_api_hostname(); + format!("https://{}/v1/media_files/upload/new_engine_asset", api_hostname) +} + +// TODO(bt,2025-04-22): Share API definitions between client and server in common crate. + +#[derive(Deserialize, Debug)] +pub struct UploadImageMediaFileSuccessResponse { + pub success: bool, + pub media_file_token: MediaFileToken, +} diff --git a/crates/lib/api_clients/storyteller_client/src/media_files/upload_video_media_file_from_file.rs b/crates/lib/api_clients/storyteller_client/src/media_files/upload_video_media_file_from_file.rs index 05cb9ae2ae..aecc6805df 100644 --- a/crates/lib/api_clients/storyteller_client/src/media_files/upload_video_media_file_from_file.rs +++ b/crates/lib/api_clients/storyteller_client/src/media_files/upload_video_media_file_from_file.rs @@ -1,5 +1,5 @@ -use crate::api_error::ApiError; use crate::credentials::storyteller_credential_set::StorytellerCredentialSet; +use crate::error::api_error::ApiError; use crate::shared_response_types::media_file_cover_image_details::MediaFileCoverImageDetails; use crate::shared_response_types::media_links::MediaLinks; use crate::shared_response_types::simple_entity_stats::SimpleEntityStats; diff --git a/crates/lib/api_clients/storyteller_client/src/recipes/mod.rs b/crates/lib/api_clients/storyteller_client/src/recipes/mod.rs new file mode 100644 index 0000000000..1232fbe14e --- /dev/null +++ b/crates/lib/api_clients/storyteller_client/src/recipes/mod.rs @@ -0,0 +1 @@ +pub mod upload_media_file_from_file; \ No newline at end of file diff --git a/crates/lib/api_clients/storyteller_client/src/recipes/upload_media_file_from_file.rs b/crates/lib/api_clients/storyteller_client/src/recipes/upload_media_file_from_file.rs new file mode 100644 index 0000000000..714da66e23 --- /dev/null +++ b/crates/lib/api_clients/storyteller_client/src/recipes/upload_media_file_from_file.rs @@ -0,0 +1,63 @@ +use crate::credentials::storyteller_credential_set::StorytellerCredentialSet; +use crate::error::client_error::ClientError; +use crate::error::storyteller_error::StorytellerError; +use crate::media_files::upload_image_media_file_from_file::upload_image_media_file_from_file; +use crate::media_files::upload_new_engine_asset_from_file::upload_new_engine_asset_from_file; +use crate::media_files::upload_video_media_file_from_file::upload_video_media_file_from_file; +use crate::utils::api_host::ApiHost; +use mimetypes::mimetype_info::file_extension::FileExtension; +use mimetypes::mimetype_info::mimetype_info::MimetypeInfo; +use serde_derive::Deserialize; +use std::path::Path; +use tokens::tokens::media_files::MediaFileToken; + +#[derive(Deserialize, Debug)] +pub struct UploadMediaFileSuccessResponse { + pub success: bool, + pub media_file_token: MediaFileToken, +} + +/// Upload an image media file from a file. +pub async fn upload_media_file_from_file>( + api_host: &ApiHost, + maybe_creds: Option<&StorytellerCredentialSet>, + path: P, +) -> Result { + + let maybe_type = MimetypeInfo::get_for_path(path.as_ref()) + .map_err(|err| ClientError::from(err))? + .map(|mime| mime.file_extension()) + .flatten(); + + match maybe_type { + Some(FileExtension::Glb) => { + match upload_new_engine_asset_from_file(api_host, maybe_creds, path).await { + Ok(result) => Ok(UploadMediaFileSuccessResponse { + success: result.success, + media_file_token: result.media_file_token, + }), + Err(err) => Err(StorytellerError::Api(err)), + } + } + Some(FileExtension::Mp4) => { + match upload_video_media_file_from_file(api_host, maybe_creds, path).await { + Ok(result) => Ok(UploadMediaFileSuccessResponse { + success: result.success, + media_file_token: result.media_file_token, + }), + Err(err) => Err(StorytellerError::Api(err)), + } + } + Some(FileExtension::Png) | Some(FileExtension::Jpg) | Some(FileExtension::Gif) | Some(FileExtension::Webp) => { + match upload_image_media_file_from_file(api_host, maybe_creds, path).await { + Ok(result) => Ok(UploadMediaFileSuccessResponse { + success: result.success, + media_file_token: result.media_file_token, + }), + Err(err) => Err(StorytellerError::Api(err)), + } + } + Some(_) => Err(StorytellerError::Client(ClientError::FileTypeNotHandled(path.as_ref().to_path_buf()))), + None => Err(StorytellerError::Client(ClientError::FileTypeNotKnown(path.as_ref().to_path_buf()))), + } +} diff --git a/crates/lib/api_clients/storyteller_client/src/utils/filter_bad_response.rs b/crates/lib/api_clients/storyteller_client/src/utils/filter_bad_response.rs index 3bd27d1b17..aecf663d3c 100644 --- a/crates/lib/api_clients/storyteller_client/src/utils/filter_bad_response.rs +++ b/crates/lib/api_clients/storyteller_client/src/utils/filter_bad_response.rs @@ -1,7 +1,6 @@ -use anyhow::anyhow; -use crate::api_error::ApiError; +use crate::error::api_error::ApiError; use crate::utils::status_codes::*; -use reqwest::StatusCode; +use anyhow::anyhow; /// Pass request errors to this method for standard error handling behavior. /// This function needs to take temporary ownership, but will return it to the caller. diff --git a/crates/lib/api_clients/storyteller_client/src/utils/http_get_anonymous.rs b/crates/lib/api_clients/storyteller_client/src/utils/http_get_anonymous.rs index ece6bcb577..f4c66d20c7 100644 --- a/crates/lib/api_clients/storyteller_client/src/utils/http_get_anonymous.rs +++ b/crates/lib/api_clients/storyteller_client/src/utils/http_get_anonymous.rs @@ -1,5 +1,5 @@ +use crate::error::api_error::ApiError; use reqwest::Client; -use crate::api_error::ApiError; const USER_AGENT: &str = "storyteller-client/1.0"; pub async fn http_get_anonymous(url: String) -> Result { diff --git a/crates/lib/files/mimetypes/src/custom/custom_infer.rs b/crates/lib/files/mimetypes/src/custom/custom_infer.rs new file mode 100644 index 0000000000..9a27dd7af9 --- /dev/null +++ b/crates/lib/files/mimetypes/src/custom/custom_infer.rs @@ -0,0 +1,32 @@ +use infer::Infer; +use std::sync::LazyLock; + +pub (crate) static CUSTOM_INFER: LazyLock = LazyLock::new(|| { + let mut infer = Infer::new(); + add_custom_types(&mut infer); + infer +}); + +fn add_custom_types(infer: &mut Infer) { + infer.add("model/gltf-binary", "glb", gltf_matcher); +} + +fn gltf_matcher(buf: &[u8]) -> bool { + // Magic is 'glTF' (103 108 84 70) + // https://github.com/KhronosGroup/glTF/blob/main/extensions/1.0/Khronos/KHR_binary_glTF/README.md + buf.len() >= 4 && buf[0] == 103 && buf[1] == 108 && buf[2] == 84 && buf[3] == 70 +} + +#[cfg(test)] +mod tests { + use crate::custom::custom_infer::CUSTOM_INFER; + use testing::test_file_path::test_file_path; + + #[test] + fn test_gltf() -> anyhow::Result<()> { + let path = test_file_path("test_data/3d/hanashi.glb")?; + let maybe_type = CUSTOM_INFER.get_from_path(path)?.unwrap(); + assert_eq!(maybe_type.mime_type(), "model/gltf-binary"); + Ok(()) + } +} diff --git a/crates/lib/files/mimetypes/src/custom/mod.rs b/crates/lib/files/mimetypes/src/custom/mod.rs new file mode 100644 index 0000000000..69519270dd --- /dev/null +++ b/crates/lib/files/mimetypes/src/custom/mod.rs @@ -0,0 +1 @@ +pub (crate) mod custom_infer; \ No newline at end of file diff --git a/crates/lib/files/mimetypes/src/lib.rs b/crates/lib/files/mimetypes/src/lib.rs index 4d04fd6188..286b1abf88 100644 --- a/crates/lib/files/mimetypes/src/lib.rs +++ b/crates/lib/files/mimetypes/src/lib.rs @@ -19,6 +19,7 @@ #![allow(dead_code)] #![allow(non_snake_case)] +pub (crate) mod custom; pub mod mimetype_for_bytes; pub mod mimetype_for_file; pub mod mimetype_info; diff --git a/crates/lib/files/mimetypes/src/mimetype_info/file_extension.rs b/crates/lib/files/mimetypes/src/mimetype_info/file_extension.rs index 4b88194ef5..aa00627ac2 100644 --- a/crates/lib/files/mimetypes/src/mimetype_info/file_extension.rs +++ b/crates/lib/files/mimetypes/src/mimetype_info/file_extension.rs @@ -8,6 +8,7 @@ pub enum FileExtension { Aac, Flac, Gif, + Glb, Jpg, Mov, Mp3, @@ -26,6 +27,7 @@ impl FileExtension { "audio/aac" => Some(FileExtension::Aac), "audio/flac" => Some(FileExtension::Flac), "image/gif" => Some(FileExtension::Gif), + "model/gltf-binary" => Some(FileExtension::Glb), "image/jpeg" => Some(FileExtension::Jpg), "video/quicktime" => Some(FileExtension::Mov), "audio/mpeg" => Some(FileExtension::Mp3), @@ -53,6 +55,7 @@ impl FileExtension { FileExtension::Aac => (".aac", "aac"), FileExtension::Flac => (".flac", "flac"), FileExtension::Gif => (".gif", "gif"), + FileExtension::Glb => (".glb", "glb"), FileExtension::Jpg => (".jpg", "jpg"), FileExtension::Mov => (".mov", "mov"), FileExtension::Mp3 => (".mp3", "mp3"), diff --git a/crates/lib/files/mimetypes/src/mimetype_info/mimetype_info.rs b/crates/lib/files/mimetypes/src/mimetype_info/mimetype_info.rs index c8336bc908..46314838e3 100644 --- a/crates/lib/files/mimetypes/src/mimetype_info/mimetype_info.rs +++ b/crates/lib/files/mimetypes/src/mimetype_info/mimetype_info.rs @@ -1,3 +1,4 @@ +use crate::custom::custom_infer::CUSTOM_INFER; use crate::mimetype_info::file_extension::FileExtension; use infer::Type; use std::io; @@ -14,12 +15,12 @@ pub struct MimetypeInfo { impl MimetypeInfo { pub fn get_for_bytes(bytes: &[u8]) -> Option { - infer::get(bytes) + CUSTOM_INFER.get(bytes) .map(|t| Self::for_type(t)) } pub fn get_for_path>(path: P) -> io::Result> { - infer::get_from_path(path) + CUSTOM_INFER.get_from_path(path) .map(|maybe_type| maybe_type.map(|t| Self::for_type(t))) } @@ -62,8 +63,8 @@ mod tests { } mod for_files { - use testing::test_file_path::test_file_path; use super::*; + use testing::test_file_path::test_file_path; #[test] fn mp3() -> anyhow::Result<()> { @@ -76,5 +77,17 @@ mod tests { assert_eq!(ext.extension_without_period(), "mp3"); Ok(()) } + + #[test] + fn glb() -> anyhow::Result<()> { + let path = test_file_path("test_data/3d/hanashi.glb")?; + let t = MimetypeInfo::get_for_path(path)?.expect("should detect type"); + assert_eq!(t.mime_type(), "model/gltf-binary"); + let ext = t.file_extension.expect("should have extension"); + assert_eq!(ext, FileExtension::Glb); + assert_eq!(ext.extension_with_period(), ".glb"); + assert_eq!(ext.extension_without_period(), "glb"); + Ok(()) + } } } diff --git a/frontend/libs/components/lightbox-modal/src/lib/lightbox-modal.tsx b/frontend/libs/components/lightbox-modal/src/lib/lightbox-modal.tsx index 52f31035c9..4aa1bbb0bb 100644 --- a/frontend/libs/components/lightbox-modal/src/lib/lightbox-modal.tsx +++ b/frontend/libs/components/lightbox-modal/src/lib/lightbox-modal.tsx @@ -4,7 +4,7 @@ import { Transition, TransitionChild } from "@headlessui/react"; import { createPortal } from "react-dom"; import dayjs from "dayjs"; import { faDownToLine } from "@fortawesome/pro-solid-svg-icons"; -import { FalKlingImageToVideo } from "@storyteller/tauri-api" +import { FalKlingImageToVideo, FalHunyuanImageTo3d } from "@storyteller/tauri-api" interface LightboxModalProps { isOpen: boolean; @@ -130,7 +130,22 @@ export function LightboxModal({ //onCloseGallery(); // close the gallery }} > - Generate Video + Video + + + {onAddToSceneClicked && downloadUrl && ( diff --git a/frontend/libs/tauri-api/src/index.ts b/frontend/libs/tauri-api/src/index.ts index 0c69746b34..ddc1247fdf 100644 --- a/frontend/libs/tauri-api/src/index.ts +++ b/frontend/libs/tauri-api/src/index.ts @@ -1,11 +1,12 @@ export * from "./lib/common/CommandStatus.js"; +export * from "./lib/fal/FalBackgroundRemoval.js"; +export * from "./lib/fal/FalHunyuanImageTo3d.js"; +export * from "./lib/fal/FalKlingImageToVideo.js"; +export * from "./lib/fal/GetFalApiKey.js"; +export * from "./lib/fal/SetFalApiKey.js"; export * from "./lib/settings/GetAppPreferences.js"; export * from "./lib/settings/UpdateAppPreference.js"; export * from "./lib/sora/CheckSoraSession.js"; export * from "./lib/sora/LogoutSoraSession.js"; export * from "./lib/sora/SoraImageRemix.js"; export * from "./lib/sora/useSoraLoginListener.js"; -export * from "./lib/fal/FalBackgroundRemoval.js"; -export * from "./lib/fal/FalKlingImageToVideo.js"; -export * from "./lib/fal/GetFalApiKey.js"; -export * from "./lib/fal/SetFalApiKey.js"; diff --git a/frontend/libs/tauri-api/src/lib/fal/FalHunyuanImageTo3d.ts b/frontend/libs/tauri-api/src/lib/fal/FalHunyuanImageTo3d.ts new file mode 100644 index 0000000000..b57ab4a9da --- /dev/null +++ b/frontend/libs/tauri-api/src/lib/fal/FalHunyuanImageTo3d.ts @@ -0,0 +1,39 @@ +import { invoke } from "@tauri-apps/api/core"; +import { CommandResult } from "../common/CommandStatus"; + +export enum FalHunyuanImageTo3dErrorType { + ServerError = "server_error", + NeedsFalApiKey = "needs_fal_api_key", +} + +export interface FalHunyuanImageTo3dRequest { + image_media_token?: string; + base64_image?: string; +} + +export interface FalHunyuanImageTo3dError extends CommandResult { + error_type: FalHunyuanImageTo3dErrorType; + error_message?: string; +} + +export interface FalHunyuanImageTo3dPayload { + media_token: string; + cdn_url: string; + base64_bytes: string; +} + +export interface FalHunyuanImageTo3dSuccess extends CommandResult { + payload: FalHunyuanImageTo3dPayload; +} + +export type FalHunyuanImageTo3dResult = FalHunyuanImageTo3dSuccess | FalHunyuanImageTo3dError; + +export const FalHunyuanImageTo3d = async (request: FalHunyuanImageTo3dRequest) : Promise => { + let result = await invoke("fal_hunyuan_image_to_3d_command", { + request: { + image_media_token: request.image_media_token, + base64_image: request.base64_image, + } + }); + return (result as FalHunyuanImageTo3dResult); +} diff --git a/test_data/3d/hanashi.glb b/test_data/3d/hanashi.glb new file mode 100644 index 0000000000..50bad29628 Binary files /dev/null and b/test_data/3d/hanashi.glb differ