From dcbd67c803f9b9c32bb869fc4358911d256070de Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sat, 5 Jul 2025 21:06:02 -0400 Subject: [PATCH] tauri endpoint that handles gpt-image-1 (contextual editing) --- .../image/edit/gpt_image_1_edit_image.rs | 19 ++ .../src/generate/image/edit/mod.rs | 1 + .../src/generate/image/mod.rs | 1 + .../image/enqueue_text_to_image_command.rs | 2 +- .../enqueue_contextual_edit_image_command.rs | 217 ++++++++++++++++++ .../commands/enqueue/image_edit/errors.rs | 101 ++++++++ .../gpt_image_1/handle_gpt_image_1.rs | 64 ++++++ .../handle_gpt_image_1_artcraft.rs | 117 ++++++++++ .../gpt_image_1/handle_gpt_image_1_sora.rs | 172 ++++++++++++++ .../enqueue/image_edit/gpt_image_1/mod.rs | 3 + .../core/commands/enqueue/image_edit/mod.rs | 4 + .../enqueue/image_edit/success_event.rs | 16 ++ .../artcraft/src/core/commands/enqueue/mod.rs | 1 + .../src/core/commands/response/shorthand.rs | 9 + .../model/contextual_image_edit_models.rs | 12 + crates/desktop/artcraft/src/core/model/mod.rs | 1 + crates/desktop/artcraft/src/lib.rs | 4 +- 17 files changed, 742 insertions(+), 2 deletions(-) create mode 100644 crates/api_clients/storyteller_client/src/generate/image/edit/gpt_image_1_edit_image.rs create mode 100644 crates/api_clients/storyteller_client/src/generate/image/edit/mod.rs create mode 100644 crates/desktop/artcraft/src/core/commands/enqueue/image_edit/enqueue_contextual_edit_image_command.rs create mode 100644 crates/desktop/artcraft/src/core/commands/enqueue/image_edit/errors.rs create mode 100644 crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/handle_gpt_image_1.rs create mode 100644 crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/handle_gpt_image_1_artcraft.rs create mode 100644 crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/handle_gpt_image_1_sora.rs create mode 100644 crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/mod.rs create mode 100644 crates/desktop/artcraft/src/core/commands/enqueue/image_edit/mod.rs create mode 100644 crates/desktop/artcraft/src/core/commands/enqueue/image_edit/success_event.rs create mode 100644 crates/desktop/artcraft/src/core/model/contextual_image_edit_models.rs diff --git a/crates/api_clients/storyteller_client/src/generate/image/edit/gpt_image_1_edit_image.rs b/crates/api_clients/storyteller_client/src/generate/image/edit/gpt_image_1_edit_image.rs new file mode 100644 index 0000000000..658be83ee7 --- /dev/null +++ b/crates/api_clients/storyteller_client/src/generate/image/edit/gpt_image_1_edit_image.rs @@ -0,0 +1,19 @@ +use crate::credentials::storyteller_credential_set::StorytellerCredentialSet; +use crate::error::storyteller_error::StorytellerError; +use crate::utils::api_host::ApiHost; +use crate::utils::basic_json_post_request::basic_json_post_request; +use artcraft_api_defs::generate::image::edit::gpt_image_1_edit_image::{GptImage1EditImageRequest, GptImage1EditImageResponse, GPT_IMAGE_1_EDIT_IMAGE_PATH}; + + +pub async fn gpt_image_1_edit_image( + api_host: &ApiHost, + maybe_creds: Option<&StorytellerCredentialSet>, + request: GptImage1EditImageRequest, +) -> Result { + Ok(basic_json_post_request( + api_host, + GPT_IMAGE_1_EDIT_IMAGE_PATH, + maybe_creds, + request, + ).await?) +} diff --git a/crates/api_clients/storyteller_client/src/generate/image/edit/mod.rs b/crates/api_clients/storyteller_client/src/generate/image/edit/mod.rs new file mode 100644 index 0000000000..e839328df0 --- /dev/null +++ b/crates/api_clients/storyteller_client/src/generate/image/edit/mod.rs @@ -0,0 +1 @@ +pub mod gpt_image_1_edit_image; \ No newline at end of file diff --git a/crates/api_clients/storyteller_client/src/generate/image/mod.rs b/crates/api_clients/storyteller_client/src/generate/image/mod.rs index 8a6a0561c6..3ad14a3291 100644 --- a/crates/api_clients/storyteller_client/src/generate/image/mod.rs +++ b/crates/api_clients/storyteller_client/src/generate/image/mod.rs @@ -1,3 +1,4 @@ +pub mod edit; pub mod generate_flux_1_dev_text_to_image; pub mod generate_flux_1_schnell_text_to_image; pub mod generate_flux_pro_11_text_to_image; diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image/enqueue_text_to_image_command.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image/enqueue_text_to_image_command.rs index 84af5d10f7..a9c03482e0 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image/enqueue_text_to_image_command.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image/enqueue_text_to_image_command.rs @@ -21,7 +21,7 @@ use crate::services::sora::state::sora_task_queue::SoraTaskQueue; use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager; use fal_client::requests::queue::image_gen::enqueue_flux_pro_11_ultra_text_to_image::{enqueue_flux_pro_11_ultra_text_to_image, FluxPro11UltraTextToImageArgs}; use log::{error, info, warn}; -use serde_derive::{Deserialize, Serialize}; +use serde::{Deserialize, Serialize}; use tauri::{AppHandle, State}; #[derive(Deserialize)] diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/enqueue_contextual_edit_image_command.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/enqueue_contextual_edit_image_command.rs new file mode 100644 index 0000000000..92ec97e080 --- /dev/null +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/enqueue_contextual_edit_image_command.rs @@ -0,0 +1,217 @@ +use crate::core::commands::enqueue::image::enqueue_text_to_image_command::{EnqueueTextToImageRequest, EnqueueTextToImageSuccessResponse}; +use crate::core::commands::enqueue::image::handle_image_artcraft::handle_image_artcraft; +use crate::core::commands::enqueue::image::handle_image_fal::handle_image_fal; +use crate::core::commands::enqueue::image::handle_image_sora::handle_image_sora; +use crate::core::commands::enqueue::image::internal_image_error::InternalImageError; +use crate::core::commands::enqueue::image_edit::errors::InternalContextualEditImageError; +use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1::handle_gpt_image_1; +use crate::core::commands::enqueue::image_edit::success_event::ContextualEditImageSuccessEvent; +use crate::core::commands::response::failure_response_wrapper::{CommandErrorResponseWrapper, CommandErrorStatus}; +use crate::core::commands::response::shorthand::{Response, ResponseOrErrorType}; +use crate::core::commands::response::success_response_wrapper::SerializeMarker; +use crate::core::events::basic_sendable_event_trait::BasicSendableEvent; +use crate::core::events::generation_events::common::{GenerationAction, GenerationServiceProvider}; +use crate::core::events::generation_events::generation_enqueue_failure_event::GenerationEnqueueFailureEvent; +use crate::core::events::generation_events::generation_enqueue_success_event::GenerationEnqueueSuccessEvent; +use crate::core::events::sendable_event_trait::SendableEvent; +use crate::core::model::contextual_image_edit_models::ContextualImageEditModel; +use crate::core::model::image_models::ImageModel; +use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs; +use crate::core::state::data_dir::app_data_root::AppDataRoot; +use crate::core::state::data_dir::trait_data_subdir::DataSubdir; +use crate::core::state::provider_priority::{Provider, ProviderPriorityStore}; +use crate::core::utils::get_url_file_extension::get_url_file_extension; +use crate::core::utils::simple_http_download::simple_http_download; +use crate::services::fal::state::fal_credential_manager::FalCredentialManager; +use crate::services::fal::state::fal_task_queue::FalTaskQueue; +use crate::services::sora::state::sora_credential_manager::SoraCredentialManager; +use crate::services::sora::state::sora_task_queue::SoraTaskQueue; +use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager; +use errors::AnyhowError; +use log::{error, info, warn}; +use openai_sora_client::recipes::image_remix_with_session_auto_renew::{image_remix_with_session_auto_renew, ImageRemixAutoRenewRequest}; +use openai_sora_client::recipes::image_upload_from_file_with_session_auto_renew::{image_upload_from_file_with_session_auto_renew, ImageUploadFromFileAutoRenewRequest}; +use openai_sora_client::recipes::maybe_upgrade_or_renew_session::maybe_upgrade_or_renew_session; +use openai_sora_client::requests::image_gen::common::{ImageSize, NumImages}; +use openai_sora_client::sora_error::SoraError; +use serde_derive::{Deserialize, Serialize}; +use std::time::Duration; +use storyteller_client::error::storyteller_error::StorytellerError; +use storyteller_client::media_files::get_media_file::get_media_file; +use storyteller_client::utils::api_host::ApiHost; +use tauri::{AppHandle, Manager, State}; +use tokens::tokens::media_files::MediaFileToken; + + +#[derive(Deserialize, Debug)] +pub struct EnqueueContextualEditImageCommand { + /// The model to use. + pub model: Option, + + /// Images to use for the image edit. + /// The first image is typically a 2D canvas or 3D stage, but doesn't have to be. + /// There must be at least one image. + pub image_media_tokens: Vec, + + /// The user's image generation prompt. + pub prompt: String, + + /// Turn off the system prompt. + pub disable_system_prompt: Option, + + /// Number of images to generate. + pub image_count: Option, + + /// Aspect ratio. + pub aspect_ratio: Option, + + /// Image quality. + pub image_quality: Option, +} + +#[derive(Deserialize, Debug, Copy, Clone)] +#[serde(rename_all = "snake_case")] +pub enum EditImageSize { + Auto, + Square, + Wide, + Tall, +} + +#[derive(Deserialize, Debug, Copy, Clone)] +#[serde(rename_all = "snake_case")] +pub enum EditImageQuality { + Auto, + High, + Medium, + Low, +} + +#[derive(Serialize, Debug)] +#[serde(rename_all = "snake_case")] +pub enum EnqueueContextualEditImageErrorType { + /// Caller didn't specify a model + ModelNotSpecified, + + /// No model available for image generation + NoProviderAvailable, + + /// Generic bad request error + BadRequest, + + /// Generic server error + ServerError, + + /// The user is sending too many requests + TooManyConcurrentTasks, + /// User is not logged into Sora! + SoraLoginRequired, + /// The user needs to create a Sora account + SoraUsernameNotYetCreated, + /// The Sora service is having problems. Try again soon. + SoraIsHavingProblems, +} + +#[derive(Serialize)] +pub struct EnqueueContextualEditImageSuccessResponse { +} + +impl SerializeMarker for EnqueueContextualEditImageSuccessResponse {} + +#[tauri::command] +pub async fn enqueue_contextual_edit_image_command( + app: AppHandle, + request: EnqueueContextualEditImageCommand, + app_data_root: State<'_, AppDataRoot>, + app_env_configs: State<'_, AppEnvConfigs>, + provider_priority_store: State<'_, ProviderPriorityStore>, + storyteller_creds_manager: State<'_, StorytellerCredentialManager>, + fal_creds_manager: State<'_, FalCredentialManager>, + fal_task_queue: State<'_, FalTaskQueue>, + sora_creds_manager: State<'_, SoraCredentialManager>, + sora_task_queue: State<'_, SoraTaskQueue>, +) -> ResponseOrErrorType { + + info!("enqueue_contextual_edit_image_command called; image media tokens : {:?}, full request: {:?}", + &request.image_media_tokens, &request); + + let result = handle_request( + &request, + &app, + &app_data_root, + &app_env_configs, + &provider_priority_store, + &storyteller_creds_manager, + &fal_creds_manager, + &fal_task_queue, + &sora_creds_manager, + &sora_task_queue, + ).await; + + match result { + Err(err) => { + error!("Error enqueuing contextual edit image: {:?}", err); + + // TODO: Derive from err. Make service provider optional. + let event = GenerationEnqueueFailureEvent { + action: GenerationAction::GenerateImage, + service: GenerationServiceProvider::Artcraft, // FIXME: This is wrong. + model: None, + reason: None, + }; + + if let Err(err) = event.send(&app) { + error!("Failed to emit event: {:?}", err); // Fail open. + } + + Err(err.to_tauri_response()) + } + Ok(event) => { + let event = GenerationEnqueueSuccessEvent { + action: GenerationAction::GenerateImage, + service: event.service_provider, + model: Some(event.tauri_event_model()), + }; + + if let Err(err) = event.send(&app) { + error!("Failed to emit event: {:?}", err); // Fail open. + } + + Ok(EnqueueContextualEditImageSuccessResponse {}.into()) + } + } +} + +pub async fn handle_request( + request: &EnqueueContextualEditImageCommand, + app: &AppHandle, + app_data_root: &AppDataRoot, + app_env_configs: &AppEnvConfigs, + provider_priority_store: &ProviderPriorityStore, + storyteller_creds_manager: &StorytellerCredentialManager, + fal_creds_manager: &FalCredentialManager, + fal_task_queue: &FalTaskQueue, + sora_creds_manager: &SoraCredentialManager, + sora_task_queue: &SoraTaskQueue, +) -> Result { + match request.model { + None => { + Err(InternalContextualEditImageError::NoModelSpecified) + } + Some(ContextualImageEditModel::GptImage1) => { + handle_gpt_image_1( + request, + app, + app_data_root, + app_env_configs, + provider_priority_store, + storyteller_creds_manager, + fal_creds_manager, + fal_task_queue, + sora_creds_manager, + sora_task_queue, + ).await + } + // TODO(bt,2025-07-05): Flux Kontext, etc. + } +} diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/errors.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/errors.rs new file mode 100644 index 0000000000..8e46f12817 --- /dev/null +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/errors.rs @@ -0,0 +1,101 @@ +use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::EnqueueContextualEditImageErrorType; +use crate::core::commands::response::failure_response_wrapper::{CommandErrorResponseWrapper, CommandErrorStatus}; +use errors::AnyhowError; +use openai_sora_client::sora_error::SoraError; +use serde::Serialize; +use storyteller_client::error::storyteller_error::StorytellerError; + +#[derive(Debug)] +pub enum InternalContextualEditImageError { + /// Request did not specify a model. + NoModelSpecified, + + /// No provider is configured for the model type. + NoProviderAvailable, + + /// No storyteller credentials are set. + NeedsStorytellerCredentials, + + /// An invalid number of images was requested. + InvalidNumberOfRequestedImages { + min: u32, + max: u32, + requested: u32, + }, + + /// An invalid number of input images was requested. + InvalidNumberOfInputImages { + min: u32, + max: u32, + requested: u32, + }, + + /// No sora credentials are set. + NeedsSoraCredentials, + + SoraError(SoraError), + AnyhowError(AnyhowError), + StorytellerError(StorytellerError), +} + +impl From for InternalContextualEditImageError { + fn from(value: AnyhowError) -> Self { + Self::AnyhowError(value) + } +} + +impl From for InternalContextualEditImageError { + fn from(value: SoraError) -> Self { + Self::SoraError(value) + } +} + +impl From for InternalContextualEditImageError { + fn from(value: StorytellerError) -> Self { + Self::StorytellerError(value) + } +} + +impl InternalContextualEditImageError { + pub fn to_tauri_response(&self) -> CommandErrorResponseWrapper { + let mut status = CommandErrorStatus::ServerError; + let mut error_type = EnqueueContextualEditImageErrorType::ServerError; + let mut error_message = "A server error occurred. Please try again. If it continues, please tell our staff about the problem.".to_string(); + + match self { + InternalContextualEditImageError::NoModelSpecified => { + status = CommandErrorStatus::BadRequest; + error_type = EnqueueContextualEditImageErrorType::ModelNotSpecified; + error_message = "No model specified for image generation".to_string(); + } + InternalContextualEditImageError::NoProviderAvailable => { + status = CommandErrorStatus::ServerError; + error_type = EnqueueContextualEditImageErrorType::NoProviderAvailable; + error_message = "No configured provider available for image generation".to_string(); + } + InternalContextualEditImageError::InvalidNumberOfRequestedImages { min, max, requested } => { + status = CommandErrorStatus::BadRequest; + error_type = EnqueueContextualEditImageErrorType::BadRequest; + error_message = format!("Invalid number of images requested ({}). Must be between {} and {}", requested, min, max); + + } + InternalContextualEditImageError::InvalidNumberOfInputImages { min, max, requested } => { + status = CommandErrorStatus::BadRequest; + error_type = EnqueueContextualEditImageErrorType::BadRequest; + error_message = format!("Invalid number of input images ({}). Must be between {} and {}", requested, min, max); + } + InternalContextualEditImageError::SoraError(_) => {} + InternalContextualEditImageError::AnyhowError(_) => {} + InternalContextualEditImageError::StorytellerError(_) => {} + InternalContextualEditImageError::NeedsStorytellerCredentials => {} + InternalContextualEditImageError::NeedsSoraCredentials => {} + } + + CommandErrorResponseWrapper { + status, + error_message: Some(error_message.to_string()), + error_type: Some(error_type), + error_details: None, + } + } +} \ No newline at end of file diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/handle_gpt_image_1.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/handle_gpt_image_1.rs new file mode 100644 index 0000000000..575487dbc4 --- /dev/null +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/handle_gpt_image_1.rs @@ -0,0 +1,64 @@ +use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::EnqueueContextualEditImageCommand; +use crate::core::commands::enqueue::image_edit::errors::InternalContextualEditImageError; +use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_artcraft::handle_gpt_image_1_artcraft; +use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_sora::handle_gpt_image_1_sora; +use crate::core::commands::enqueue::image_edit::success_event::ContextualEditImageSuccessEvent; +use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs; +use crate::core::state::data_dir::app_data_root::AppDataRoot; +use crate::core::state::provider_priority::{Provider, ProviderPriorityStore}; +use crate::services::fal::state::fal_credential_manager::FalCredentialManager; +use crate::services::fal::state::fal_task_queue::FalTaskQueue; +use crate::services::sora::state::sora_credential_manager::SoraCredentialManager; +use crate::services::sora::state::sora_task_queue::SoraTaskQueue; +use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager; +use tauri::AppHandle; + +pub async fn handle_gpt_image_1( + request: &EnqueueContextualEditImageCommand, + app: &AppHandle, + app_data_root: &AppDataRoot, + app_env_configs: &AppEnvConfigs, + provider_priority_store: &ProviderPriorityStore, + storyteller_creds_manager: &StorytellerCredentialManager, + fal_creds_manager: &FalCredentialManager, + fal_task_queue: &FalTaskQueue, + sora_creds_manager: &SoraCredentialManager, + sora_task_queue: &SoraTaskQueue, +) -> Result { + + let priority = provider_priority_store.get_priority()?; + + // TODO: Check if providers are available before proceeding. + + for provider in priority.iter() { + match provider { + Provider::Fal => { + // Fallthrough + // If in the future we support OpenAI API keys, it's worth considering whether we + // send those requests ourselves, or if we use FAL as an intermediary. FAL makes + // the API nicer to deal with, but the user needs an additional key. + } + Provider::Artcraft => { + return handle_gpt_image_1_artcraft( + request, + app, + app_data_root, + app_env_configs, + storyteller_creds_manager + ).await; + } + Provider::Sora => { + return handle_gpt_image_1_sora( + request, + app, + app_data_root, + app_env_configs, + sora_creds_manager, + sora_task_queue, + ).await; + } + } + } + + Err(InternalContextualEditImageError::NoProviderAvailable) +} diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/handle_gpt_image_1_artcraft.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/handle_gpt_image_1_artcraft.rs new file mode 100644 index 0000000000..3cb4fde2a4 --- /dev/null +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/handle_gpt_image_1_artcraft.rs @@ -0,0 +1,117 @@ +use crate::core::commands::enqueue::image::enqueue_text_to_image_command::EnqueueTextToImageRequest; +use crate::core::commands::enqueue::image::internal_image_error::InternalImageError; +use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::{EditImageQuality, EditImageSize, EnqueueContextualEditImageCommand}; +use crate::core::commands::enqueue::image_edit::errors::InternalContextualEditImageError; +use crate::core::commands::enqueue::image_edit::success_event::ContextualEditImageSuccessEvent; +use crate::core::events::basic_sendable_event_trait::BasicSendableEvent; +use crate::core::events::generation_events::common::{GenerationAction, GenerationServiceProvider}; +use crate::core::events::generation_events::generation_enqueue_failure_event::GenerationEnqueueFailureEvent; +use crate::core::model::contextual_image_edit_models::ContextualImageEditModel; +use crate::core::model::image_models::ImageModel; +use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs; +use crate::core::state::data_dir::app_data_root::AppDataRoot; +use crate::core::state::provider_priority::ProviderPriorityStore; +use crate::services::fal::state::fal_credential_manager::FalCredentialManager; +use crate::services::fal::state::fal_task_queue::FalTaskQueue; +use crate::services::sora::state::sora_credential_manager::SoraCredentialManager; +use crate::services::sora::state::sora_task_queue::SoraTaskQueue; +use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager; +use anyhow::anyhow; +use artcraft_api_defs::generate::image::edit::gpt_image_1_edit_image::{GptImage1EditImageImageQuality, GptImage1EditImageImageSize, GptImage1EditImageNumImages, GptImage1EditImageRequest}; +use artcraft_api_defs::generate::image::generate_flux_1_dev_text_to_image::GenerateFlux1DevTextToImageRequest; +use artcraft_api_defs::generate::image::generate_flux_1_schnell_text_to_image::GenerateFlux1SchnellTextToImageRequest; +use artcraft_api_defs::generate::image::generate_flux_pro_11_text_to_image::GenerateFluxPro11TextToImageRequest; +use artcraft_api_defs::generate::image::generate_flux_pro_11_ultra_text_to_image::GenerateFluxPro11UltraTextToImageRequest; +use fal_client::requests::queue::image_gen::enqueue_flux_pro_11_ultra_text_to_image::{enqueue_flux_pro_11_ultra_text_to_image, FluxPro11UltraTextToImageArgs}; +use fal_client::requests::webhook::image::enqueue_gpt_image_1_edit_image_webhook::GptEditImageQuality; +use idempotency::uuid::generate_random_uuid; +use log::{error, info}; +use storyteller_client::generate::image::edit::gpt_image_1_edit_image::gpt_image_1_edit_image; +use storyteller_client::generate::image::generate_flux_1_dev_text_to_image::generate_flux_1_dev_text_to_image; +use storyteller_client::generate::image::generate_flux_1_schnell_text_to_image::generate_flux_1_schnell_text_to_image; +use storyteller_client::generate::image::generate_flux_pro_11_text_to_image::generate_flux_pro_11_text_to_image; +use storyteller_client::generate::image::generate_flux_pro_11_ultra_text_to_image::generate_flux_pro_11_ultra_text_to_image; +use tauri::AppHandle; + +pub async fn handle_gpt_image_1_artcraft( + request: &EnqueueContextualEditImageCommand, + app: &AppHandle, + app_data_root: &AppDataRoot, + app_env_configs: &AppEnvConfigs, + storyteller_creds_manager: &StorytellerCredentialManager, +) -> Result { + + let creds = match storyteller_creds_manager.get_credentials()? { + Some(creds) => creds, + None => { + return Err(InternalContextualEditImageError::NeedsStorytellerCredentials); + }, + }; + + info!("Calling Artcraft gpt-image-1 ..."); + + let uuid_idempotency_token = generate_random_uuid(); + + let image_quality = request.image_quality + .map(|quality| match quality { + EditImageQuality::Auto => GptImage1EditImageImageQuality::Auto, + EditImageQuality::High => GptImage1EditImageImageQuality::High, + EditImageQuality::Medium => GptImage1EditImageImageQuality::Medium, + EditImageQuality::Low => GptImage1EditImageImageQuality::Low, + }); + + let image_size = request.aspect_ratio + .map(|size| match size { + EditImageSize::Auto => GptImage1EditImageImageSize::Square, + EditImageSize::Square => GptImage1EditImageImageSize::Square, + EditImageSize::Tall => GptImage1EditImageImageSize::Vertical, + EditImageSize::Wide => GptImage1EditImageImageSize::Horizontal, + }); + + let num_images = match request.image_count { + None => None, + Some(1) => Some(GptImage1EditImageNumImages::One), + Some(2) => Some(GptImage1EditImageNumImages::Two), + Some(3) => Some(GptImage1EditImageNumImages::Three), + Some(4) => Some(GptImage1EditImageNumImages::Four), + Some(other) => { + return Err(InternalContextualEditImageError::InvalidNumberOfRequestedImages { + min: 1, + max: 4, + requested: other, + }); + }, + }; + + let request = GptImage1EditImageRequest { + uuid_idempotency_token, + prompt: Some(request.prompt.clone()), + image_media_tokens: Some(request.image_media_tokens.clone()), + image_size, + num_images, + image_quality, + }; + + let result = gpt_image_1_edit_image( + &app_env_configs.storyteller_host, + Some(&creds), + request, + ).await; + + match result { + Ok(enqueued) => { + // TODO(bt,2025-07-05): Enqueue job token? + info!("Successfully enqueued Artcraft gpt-image-1. Job token: {}", + enqueued.inference_job_token); + } + Err(err) => { + error!("Failed to use Artcraft gpt-image-1: {:?}", err); + return Err(InternalContextualEditImageError::StorytellerError(err)); + } + } + + Ok(ContextualEditImageSuccessEvent { + service_provider: GenerationServiceProvider::Artcraft, + model: ContextualImageEditModel::GptImage1, + }) +} diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/handle_gpt_image_1_sora.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/handle_gpt_image_1_sora.rs new file mode 100644 index 0000000000..263291d099 --- /dev/null +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/handle_gpt_image_1_sora.rs @@ -0,0 +1,172 @@ +use crate::core::commands::enqueue::image::enqueue_text_to_image_command::EnqueueTextToImageRequest; +use crate::core::commands::enqueue::image::internal_image_error::InternalImageError; +use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::{EditImageSize, EnqueueContextualEditImageCommand, EnqueueContextualEditImageErrorType}; +use crate::core::commands::enqueue::image_edit::errors::InternalContextualEditImageError; +use crate::core::commands::enqueue::image_edit::success_event::ContextualEditImageSuccessEvent; +use crate::core::commands::response::failure_response_wrapper::{CommandErrorResponseWrapper, CommandErrorStatus}; +use crate::core::events::basic_sendable_event_trait::BasicSendableEvent; +use crate::core::events::generation_events::common::{GenerationAction, GenerationServiceProvider}; +use crate::core::events::generation_events::generation_enqueue_failure_event::GenerationEnqueueFailureEvent; +use crate::core::events::generation_events::generation_enqueue_success_event::GenerationEnqueueSuccessEvent; +use crate::core::model::contextual_image_edit_models::ContextualImageEditModel; +use crate::core::model::image_models::ImageModel; +use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs; +use crate::core::state::data_dir::app_data_root::AppDataRoot; +use crate::core::state::data_dir::trait_data_subdir::DataSubdir; +use crate::core::state::provider_priority::ProviderPriorityStore; +use crate::core::utils::get_url_file_extension::get_url_file_extension; +use crate::core::utils::simple_http_download::simple_http_download; +use crate::services::fal::state::fal_credential_manager::FalCredentialManager; +use crate::services::fal::state::fal_task_queue::FalTaskQueue; +use crate::services::sora::state::sora_credential_manager::SoraCredentialManager; +use crate::services::sora::state::sora_task_queue::SoraTaskQueue; +use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager; +use anyhow::anyhow; +use artcraft_api_defs::generate::image::generate_flux_1_dev_text_to_image::GenerateFlux1DevTextToImageRequest; +use artcraft_api_defs::generate::image::generate_flux_1_schnell_text_to_image::GenerateFlux1SchnellTextToImageRequest; +use artcraft_api_defs::generate::image::generate_flux_pro_11_text_to_image::GenerateFluxPro11TextToImageRequest; +use artcraft_api_defs::generate::image::generate_flux_pro_11_ultra_text_to_image::GenerateFluxPro11UltraTextToImageRequest; +use errors::AnyhowResult; +use fal_client::requests::queue::image_gen::enqueue_flux_pro_11_ultra_text_to_image::{enqueue_flux_pro_11_ultra_text_to_image, FluxPro11UltraTextToImageArgs}; +use idempotency::uuid::generate_random_uuid; +use log::{error, info, warn}; +use openai_sora_client::recipes::image_remix_with_session_auto_renew::{image_remix_with_session_auto_renew, ImageRemixAutoRenewRequest}; +use openai_sora_client::recipes::image_upload_from_file_with_session_auto_renew::{image_upload_from_file_with_session_auto_renew, ImageUploadFromFileAutoRenewRequest}; +use openai_sora_client::recipes::maybe_upgrade_or_renew_session::maybe_upgrade_or_renew_session; +use openai_sora_client::requests::image_gen::common::{ImageSize, NumImages}; +use openai_sora_client::sora_error::SoraError; +use std::time::Duration; +use storyteller_client::generate::image::generate_flux_1_dev_text_to_image::generate_flux_1_dev_text_to_image; +use storyteller_client::generate::image::generate_flux_1_schnell_text_to_image::generate_flux_1_schnell_text_to_image; +use storyteller_client::generate::image::generate_flux_pro_11_text_to_image::generate_flux_pro_11_text_to_image; +use storyteller_client::generate::image::generate_flux_pro_11_ultra_text_to_image::generate_flux_pro_11_ultra_text_to_image; +use storyteller_client::media_files::get_media_file::get_media_file; +use storyteller_client::utils::api_host::ApiHost; +use tauri::AppHandle; + +const SORA_IMAGE_UPLOAD_TIMEOUT: Duration = Duration::from_millis(1000 * 30); // 30 seconds + +const SORA_IMAGE_REMIX_TIMEOUT: Duration = Duration::from_millis(1000 * 30); // 30 seconds + +const DEFAULT_ASPECT_RATIO : EditImageSize = EditImageSize::Square; + +pub async fn handle_gpt_image_1_sora( + request: &EnqueueContextualEditImageCommand, + app: &AppHandle, + app_data_root: &AppDataRoot, + app_env_configs: &AppEnvConfigs, + sora_creds_manager: &SoraCredentialManager, + sora_task_queue: &SoraTaskQueue, + +) -> Result { + + let sora_creds = match sora_creds_manager.get_credentials() { + Ok(Some(creds)) => creds, + Ok(None) => { + warn!("No Sora credentials found."); + return Err(InternalContextualEditImageError::NeedsSoraCredentials); + } + Err(err) => { + error!("Failed to get Sora credentials: {:?}", err); + return Err(InternalContextualEditImageError::AnyhowError(err)); + } + }; + + // TODO(bt,2025-07-08): Handle many images. + // TODO(bt,2025-07-08): Handle authenticated sessions (for private images). + // TODO(bt,2025-07-08): Handle HTTP batch get requests. + let first_image_token = request.image_media_tokens.first() + .ok_or_else(|| InternalContextualEditImageError::InvalidNumberOfInputImages { + min: 1, + max: 10, // TODO Confirm + requested: request.image_media_tokens.len() as u32, + })?; + + let response = get_media_file(&ApiHost::Storyteller, first_image_token).await?; + + let media_file_url = &response.media_file.media_links.cdn_url; + let extension_with_dot = get_url_file_extension(media_file_url) + .map(|ext| format!(".{}", ext)) + .unwrap_or_else(|| ".png".to_string()); + + let filename = format!("{}{}", response.media_file.token.as_str(), extension_with_dot); + let filename = app_data_root.downloads_dir().path().join(&filename); + + simple_http_download(&media_file_url, &filename).await?; + + let files_to_upload = vec![filename]; + + let mut creds = sora_creds_manager.get_credentials_required()?; + + let credential_updated = maybe_upgrade_or_renew_session(&mut creds) + .await + .map_err(|err| { + error!("Failed to upgrade or renew session: {:?}", err); + err + })?; + + if credential_updated { + info!("Storing updated credentials"); + sora_creds_manager.set_credentials(&creds)?; + } + + let mut sora_media_tokens = Vec::with_capacity(files_to_upload.len()); + + for (i, file_path) in files_to_upload.iter().enumerate() { + info!("Uploading image {} of {}...", (i+1), files_to_upload.len()); + + let (response, maybe_new_credentials) = + image_upload_from_file_with_session_auto_renew(ImageUploadFromFileAutoRenewRequest { + file_path, + credentials: &creds, + request_timeout: Some(SORA_IMAGE_UPLOAD_TIMEOUT), // TODO: Centralize and make configurable. + }).await?; + + if let Some(new_creds) = maybe_new_credentials { + info!("Storing updated credentials."); + sora_creds_manager.set_credentials(&new_creds)?; + creds = new_creds; + } + + sora_media_tokens.push(response.id); + } + + let aspect_ratio = request.aspect_ratio.unwrap_or(DEFAULT_ASPECT_RATIO); + + let aspect_ratio = match aspect_ratio { + EditImageSize::Auto => ImageSize::Square, + EditImageSize::Square => ImageSize::Square, + EditImageSize::Wide => ImageSize::Wide, + EditImageSize::Tall => ImageSize::Tall, + }; + + info!("Calling Sora image generation..."); + + // TODO(bt,2025-04-21): Download media tokens. + // Note: This is incredibly inefficient. We should keep a local cache. + // Also, if they've already been uploaded to OpenAI, we shouldn't continue to re-upload. + + let (response, maybe_new_creds) = + image_remix_with_session_auto_renew(ImageRemixAutoRenewRequest { + prompt: request.prompt.to_string(), + num_images: NumImages::One, + image_size: aspect_ratio, + sora_media_tokens: sora_media_tokens.clone(), + credentials: &creds, + request_timeout: Some(SORA_IMAGE_REMIX_TIMEOUT), + }).await?; + + if let Some(new_creds) = maybe_new_creds { + info!("Storing updated credentials."); + sora_creds_manager.set_credentials(&new_creds)?; + } + + info!("New Sora Task ID: {:?} ", response.task_id); + + sora_task_queue.insert(&response.task_id)?; + + Ok(ContextualEditImageSuccessEvent { + service_provider: GenerationServiceProvider::Sora, + model: ContextualImageEditModel::GptImage1, + }) +} diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/mod.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/mod.rs new file mode 100644 index 0000000000..73b5e0a985 --- /dev/null +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/mod.rs @@ -0,0 +1,3 @@ +pub mod handle_gpt_image_1; +pub mod handle_gpt_image_1_artcraft; +pub mod handle_gpt_image_1_sora; \ No newline at end of file diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/mod.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/mod.rs new file mode 100644 index 0000000000..7c3f2af36f --- /dev/null +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/mod.rs @@ -0,0 +1,4 @@ +pub mod enqueue_contextual_edit_image_command; +pub mod errors; +pub mod gpt_image_1; +pub mod success_event; \ No newline at end of file diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/success_event.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/success_event.rs new file mode 100644 index 0000000000..70cd4a5bc4 --- /dev/null +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/success_event.rs @@ -0,0 +1,16 @@ +use crate::core::events::generation_events::common::{GenerationModel, GenerationServiceProvider}; +use crate::core::model::contextual_image_edit_models::ContextualImageEditModel; +use crate::core::model::image_models::ImageModel; + +pub struct ContextualEditImageSuccessEvent { + pub service_provider: GenerationServiceProvider, + pub model: ContextualImageEditModel, +} + +impl ContextualEditImageSuccessEvent { + pub fn tauri_event_model(&self) -> GenerationModel { + match self.model { + ContextualImageEditModel::GptImage1 => GenerationModel::GptImage1, + } + } +} diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/mod.rs b/crates/desktop/artcraft/src/core/commands/enqueue/mod.rs index c70c64e67d..25feeea8e6 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/mod.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/mod.rs @@ -1,3 +1,4 @@ pub mod image; +pub mod image_edit; pub mod object; pub mod video; \ No newline at end of file diff --git a/crates/desktop/artcraft/src/core/commands/response/shorthand.rs b/crates/desktop/artcraft/src/core/commands/response/shorthand.rs index fbb333f4ad..be12a6cfea 100644 --- a/crates/desktop/artcraft/src/core/commands/response/shorthand.rs +++ b/crates/desktop/artcraft/src/core/commands/response/shorthand.rs @@ -22,9 +22,18 @@ use crate::core::commands::response::success_response_wrapper::CommandSuccessRes // - sora_image_remix_command /// Easy to use infallible response. +/// This always contains a success payload and never results in an error. pub type InfallibleResponse = CommandSuccessResponseWrapper; /// Easy to use Result type. +/// +/// There are easier ways to use Response : +/// - `SimpleResponse` just returns strings on success or error (but provides clues to the frontend) +/// - `SuccessOrErrorMessage` returns an empty success payload or an error message +/// - `ResponseOrErrorMessage` returns a success payload or an error message +/// - `ResponseOrErrorType` returns a success payload or an error type +/// - `ResponseOrError` returns a success payload or an error payload +/// pub type Response = Result, CommandErrorResponseWrapper>; diff --git a/crates/desktop/artcraft/src/core/model/contextual_image_edit_models.rs b/crates/desktop/artcraft/src/core/model/contextual_image_edit_models.rs new file mode 100644 index 0000000000..5224376a32 --- /dev/null +++ b/crates/desktop/artcraft/src/core/model/contextual_image_edit_models.rs @@ -0,0 +1,12 @@ +use serde_derive::Deserialize; + +/// This is used in the Tauri command bridge. +/// Don't change the serializations without coordinating with the frontend. +#[derive(Deserialize, Debug, Copy, Clone)] +#[serde(rename_all = "snake_case")] +pub enum ContextualImageEditModel { + #[serde(rename = "gpt_image_1")] + GptImage1, + + // TODO: Flux Kontext models. +} diff --git a/crates/desktop/artcraft/src/core/model/mod.rs b/crates/desktop/artcraft/src/core/model/mod.rs index 7ee84bd8bc..8cecc07236 100644 --- a/crates/desktop/artcraft/src/core/model/mod.rs +++ b/crates/desktop/artcraft/src/core/model/mod.rs @@ -1,2 +1,3 @@ +pub mod contextual_image_edit_models; pub mod image_models; pub mod video_models; \ No newline at end of file diff --git a/crates/desktop/artcraft/src/lib.rs b/crates/desktop/artcraft/src/lib.rs index 06b9ad6397..04036fcaf9 100644 --- a/crates/desktop/artcraft/src/lib.rs +++ b/crates/desktop/artcraft/src/lib.rs @@ -6,6 +6,7 @@ use tauri::Manager; use crate::core::commands::app_preferences::get_app_preferences_command::get_app_preferences_command; use crate::core::commands::app_preferences::update_app_preference_command::update_app_preferences_command; use crate::core::commands::enqueue::image::enqueue_text_to_image_command::enqueue_text_to_image_command; +use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::enqueue_contextual_edit_image_command; use crate::core::commands::enqueue::object::enqueue_image_to_3d_object_command::enqueue_image_to_3d_object_command; use crate::core::commands::enqueue::video::enqueue_image_to_video_command::enqueue_image_to_video_command; use crate::core::commands::flip_image::flip_image; @@ -189,6 +190,7 @@ pub fn run() { .manage(storyteller_creds_manager_3) .invoke_handler(tauri::generate_handler![ check_sora_session_command, + enqueue_contextual_edit_image_command, enqueue_image_to_3d_object_command, enqueue_image_to_video_command, enqueue_text_to_image_command, @@ -200,10 +202,10 @@ pub fn run() { get_app_preferences_command, get_fal_api_key_command, get_provider_order_command, - set_provider_order_command, open_sora_login_command, platform_info_command, set_fal_api_key_command, + set_provider_order_command, sora_image_generation_command, sora_image_remix_command, sora_logout_command,