mirror of
https://github.com/storytold/artcraft.git
synced 2026-11-21 00:55:08 +00:00
tauri endpoint that handles gpt-image-1 (contextual editing)
This commit is contained in:
1 parent
4b901a9ea2
commit
dcbd67c803
17 files changed
+742
-2
No files matched your search
+19
@@ -0,0 +1,19 @@
|
||||
use crate::credentials::storyteller_credential_set::StorytellerCredentialSet;
|
||||
use crate::error::storyteller_error::StorytellerError;
|
||||
use crate::utils::api_host::ApiHost;
|
||||
use crate::utils::basic_json_post_request::basic_json_post_request;
|
||||
use artcraft_api_defs::generate::image::edit::gpt_image_1_edit_image::{GptImage1EditImageRequest, GptImage1EditImageResponse, GPT_IMAGE_1_EDIT_IMAGE_PATH};
|
||||
|
||||
|
||||
pub async fn gpt_image_1_edit_image(
|
||||
api_host: &ApiHost,
|
||||
maybe_creds: Option<&StorytellerCredentialSet>,
|
||||
request: GptImage1EditImageRequest,
|
||||
) -> Result<GptImage1EditImageResponse, StorytellerError> {
|
||||
Ok(basic_json_post_request(
|
||||
api_host,
|
||||
GPT_IMAGE_1_EDIT_IMAGE_PATH,
|
||||
maybe_creds,
|
||||
request,
|
||||
).await?)
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
pub mod gpt_image_1_edit_image;
|
||||
@@ -1,3 +1,4 @@
|
||||
pub mod edit;
|
||||
pub mod generate_flux_1_dev_text_to_image;
|
||||
pub mod generate_flux_1_schnell_text_to_image;
|
||||
pub mod generate_flux_pro_11_text_to_image;
|
||||
|
||||
+1
-1
@@ -21,7 +21,7 @@ use crate::services::sora::state::sora_task_queue::SoraTaskQueue;
|
||||
use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager;
|
||||
use fal_client::requests::queue::image_gen::enqueue_flux_pro_11_ultra_text_to_image::{enqueue_flux_pro_11_ultra_text_to_image, FluxPro11UltraTextToImageArgs};
|
||||
use log::{error, info, warn};
|
||||
use serde_derive::{Deserialize, Serialize};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tauri::{AppHandle, State};
|
||||
|
||||
#[derive(Deserialize)]
|
||||
|
||||
+217
@@ -0,0 +1,217 @@
|
||||
use crate::core::commands::enqueue::image::enqueue_text_to_image_command::{EnqueueTextToImageRequest, EnqueueTextToImageSuccessResponse};
|
||||
use crate::core::commands::enqueue::image::handle_image_artcraft::handle_image_artcraft;
|
||||
use crate::core::commands::enqueue::image::handle_image_fal::handle_image_fal;
|
||||
use crate::core::commands::enqueue::image::handle_image_sora::handle_image_sora;
|
||||
use crate::core::commands::enqueue::image::internal_image_error::InternalImageError;
|
||||
use crate::core::commands::enqueue::image_edit::errors::InternalContextualEditImageError;
|
||||
use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1::handle_gpt_image_1;
|
||||
use crate::core::commands::enqueue::image_edit::success_event::ContextualEditImageSuccessEvent;
|
||||
use crate::core::commands::response::failure_response_wrapper::{CommandErrorResponseWrapper, CommandErrorStatus};
|
||||
use crate::core::commands::response::shorthand::{Response, ResponseOrErrorType};
|
||||
use crate::core::commands::response::success_response_wrapper::SerializeMarker;
|
||||
use crate::core::events::basic_sendable_event_trait::BasicSendableEvent;
|
||||
use crate::core::events::generation_events::common::{GenerationAction, GenerationServiceProvider};
|
||||
use crate::core::events::generation_events::generation_enqueue_failure_event::GenerationEnqueueFailureEvent;
|
||||
use crate::core::events::generation_events::generation_enqueue_success_event::GenerationEnqueueSuccessEvent;
|
||||
use crate::core::events::sendable_event_trait::SendableEvent;
|
||||
use crate::core::model::contextual_image_edit_models::ContextualImageEditModel;
|
||||
use crate::core::model::image_models::ImageModel;
|
||||
use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs;
|
||||
use crate::core::state::data_dir::app_data_root::AppDataRoot;
|
||||
use crate::core::state::data_dir::trait_data_subdir::DataSubdir;
|
||||
use crate::core::state::provider_priority::{Provider, ProviderPriorityStore};
|
||||
use crate::core::utils::get_url_file_extension::get_url_file_extension;
|
||||
use crate::core::utils::simple_http_download::simple_http_download;
|
||||
use crate::services::fal::state::fal_credential_manager::FalCredentialManager;
|
||||
use crate::services::fal::state::fal_task_queue::FalTaskQueue;
|
||||
use crate::services::sora::state::sora_credential_manager::SoraCredentialManager;
|
||||
use crate::services::sora::state::sora_task_queue::SoraTaskQueue;
|
||||
use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager;
|
||||
use errors::AnyhowError;
|
||||
use log::{error, info, warn};
|
||||
use openai_sora_client::recipes::image_remix_with_session_auto_renew::{image_remix_with_session_auto_renew, ImageRemixAutoRenewRequest};
|
||||
use openai_sora_client::recipes::image_upload_from_file_with_session_auto_renew::{image_upload_from_file_with_session_auto_renew, ImageUploadFromFileAutoRenewRequest};
|
||||
use openai_sora_client::recipes::maybe_upgrade_or_renew_session::maybe_upgrade_or_renew_session;
|
||||
use openai_sora_client::requests::image_gen::common::{ImageSize, NumImages};
|
||||
use openai_sora_client::sora_error::SoraError;
|
||||
use serde_derive::{Deserialize, Serialize};
|
||||
use std::time::Duration;
|
||||
use storyteller_client::error::storyteller_error::StorytellerError;
|
||||
use storyteller_client::media_files::get_media_file::get_media_file;
|
||||
use storyteller_client::utils::api_host::ApiHost;
|
||||
use tauri::{AppHandle, Manager, State};
|
||||
use tokens::tokens::media_files::MediaFileToken;
|
||||
|
||||
|
||||
#[derive(Deserialize, Debug)]
|
||||
pub struct EnqueueContextualEditImageCommand {
|
||||
/// The model to use.
|
||||
pub model: Option<ContextualImageEditModel>,
|
||||
|
||||
/// Images to use for the image edit.
|
||||
/// The first image is typically a 2D canvas or 3D stage, but doesn't have to be.
|
||||
/// There must be at least one image.
|
||||
pub image_media_tokens: Vec<MediaFileToken>,
|
||||
|
||||
/// The user's image generation prompt.
|
||||
pub prompt: String,
|
||||
|
||||
/// Turn off the system prompt.
|
||||
pub disable_system_prompt: Option<bool>,
|
||||
|
||||
/// Number of images to generate.
|
||||
pub image_count: Option<u32>,
|
||||
|
||||
/// Aspect ratio.
|
||||
pub aspect_ratio: Option<EditImageSize>,
|
||||
|
||||
/// Image quality.
|
||||
pub image_quality: Option<EditImageQuality>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Copy, Clone)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum EditImageSize {
|
||||
Auto,
|
||||
Square,
|
||||
Wide,
|
||||
Tall,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Copy, Clone)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum EditImageQuality {
|
||||
Auto,
|
||||
High,
|
||||
Medium,
|
||||
Low,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Debug)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum EnqueueContextualEditImageErrorType {
|
||||
/// Caller didn't specify a model
|
||||
ModelNotSpecified,
|
||||
|
||||
/// No model available for image generation
|
||||
NoProviderAvailable,
|
||||
|
||||
/// Generic bad request error
|
||||
BadRequest,
|
||||
|
||||
/// Generic server error
|
||||
ServerError,
|
||||
|
||||
/// The user is sending too many requests
|
||||
TooManyConcurrentTasks,
|
||||
/// User is not logged into Sora!
|
||||
SoraLoginRequired,
|
||||
/// The user needs to create a Sora account
|
||||
SoraUsernameNotYetCreated,
|
||||
/// The Sora service is having problems. Try again soon.
|
||||
SoraIsHavingProblems,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct EnqueueContextualEditImageSuccessResponse {
|
||||
}
|
||||
|
||||
impl SerializeMarker for EnqueueContextualEditImageSuccessResponse {}
|
||||
|
||||
#[tauri::command]
|
||||
pub async fn enqueue_contextual_edit_image_command(
|
||||
app: AppHandle,
|
||||
request: EnqueueContextualEditImageCommand,
|
||||
app_data_root: State<'_, AppDataRoot>,
|
||||
app_env_configs: State<'_, AppEnvConfigs>,
|
||||
provider_priority_store: State<'_, ProviderPriorityStore>,
|
||||
storyteller_creds_manager: State<'_, StorytellerCredentialManager>,
|
||||
fal_creds_manager: State<'_, FalCredentialManager>,
|
||||
fal_task_queue: State<'_, FalTaskQueue>,
|
||||
sora_creds_manager: State<'_, SoraCredentialManager>,
|
||||
sora_task_queue: State<'_, SoraTaskQueue>,
|
||||
) -> ResponseOrErrorType<EnqueueContextualEditImageSuccessResponse, EnqueueContextualEditImageErrorType> {
|
||||
|
||||
info!("enqueue_contextual_edit_image_command called; image media tokens : {:?}, full request: {:?}",
|
||||
&request.image_media_tokens, &request);
|
||||
|
||||
let result = handle_request(
|
||||
&request,
|
||||
&app,
|
||||
&app_data_root,
|
||||
&app_env_configs,
|
||||
&provider_priority_store,
|
||||
&storyteller_creds_manager,
|
||||
&fal_creds_manager,
|
||||
&fal_task_queue,
|
||||
&sora_creds_manager,
|
||||
&sora_task_queue,
|
||||
).await;
|
||||
|
||||
match result {
|
||||
Err(err) => {
|
||||
error!("Error enqueuing contextual edit image: {:?}", err);
|
||||
|
||||
// TODO: Derive from err. Make service provider optional.
|
||||
let event = GenerationEnqueueFailureEvent {
|
||||
action: GenerationAction::GenerateImage,
|
||||
service: GenerationServiceProvider::Artcraft, // FIXME: This is wrong.
|
||||
model: None,
|
||||
reason: None,
|
||||
};
|
||||
|
||||
if let Err(err) = event.send(&app) {
|
||||
error!("Failed to emit event: {:?}", err); // Fail open.
|
||||
}
|
||||
|
||||
Err(err.to_tauri_response())
|
||||
}
|
||||
Ok(event) => {
|
||||
let event = GenerationEnqueueSuccessEvent {
|
||||
action: GenerationAction::GenerateImage,
|
||||
service: event.service_provider,
|
||||
model: Some(event.tauri_event_model()),
|
||||
};
|
||||
|
||||
if let Err(err) = event.send(&app) {
|
||||
error!("Failed to emit event: {:?}", err); // Fail open.
|
||||
}
|
||||
|
||||
Ok(EnqueueContextualEditImageSuccessResponse {}.into())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn handle_request(
|
||||
request: &EnqueueContextualEditImageCommand,
|
||||
app: &AppHandle,
|
||||
app_data_root: &AppDataRoot,
|
||||
app_env_configs: &AppEnvConfigs,
|
||||
provider_priority_store: &ProviderPriorityStore,
|
||||
storyteller_creds_manager: &StorytellerCredentialManager,
|
||||
fal_creds_manager: &FalCredentialManager,
|
||||
fal_task_queue: &FalTaskQueue,
|
||||
sora_creds_manager: &SoraCredentialManager,
|
||||
sora_task_queue: &SoraTaskQueue,
|
||||
) -> Result<ContextualEditImageSuccessEvent, InternalContextualEditImageError> {
|
||||
match request.model {
|
||||
None => {
|
||||
Err(InternalContextualEditImageError::NoModelSpecified)
|
||||
}
|
||||
Some(ContextualImageEditModel::GptImage1) => {
|
||||
handle_gpt_image_1(
|
||||
request,
|
||||
app,
|
||||
app_data_root,
|
||||
app_env_configs,
|
||||
provider_priority_store,
|
||||
storyteller_creds_manager,
|
||||
fal_creds_manager,
|
||||
fal_task_queue,
|
||||
sora_creds_manager,
|
||||
sora_task_queue,
|
||||
).await
|
||||
}
|
||||
// TODO(bt,2025-07-05): Flux Kontext, etc.
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::EnqueueContextualEditImageErrorType;
|
||||
use crate::core::commands::response::failure_response_wrapper::{CommandErrorResponseWrapper, CommandErrorStatus};
|
||||
use errors::AnyhowError;
|
||||
use openai_sora_client::sora_error::SoraError;
|
||||
use serde::Serialize;
|
||||
use storyteller_client::error::storyteller_error::StorytellerError;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum InternalContextualEditImageError {
|
||||
/// Request did not specify a model.
|
||||
NoModelSpecified,
|
||||
|
||||
/// No provider is configured for the model type.
|
||||
NoProviderAvailable,
|
||||
|
||||
/// No storyteller credentials are set.
|
||||
NeedsStorytellerCredentials,
|
||||
|
||||
/// An invalid number of images was requested.
|
||||
InvalidNumberOfRequestedImages {
|
||||
min: u32,
|
||||
max: u32,
|
||||
requested: u32,
|
||||
},
|
||||
|
||||
/// An invalid number of input images was requested.
|
||||
InvalidNumberOfInputImages {
|
||||
min: u32,
|
||||
max: u32,
|
||||
requested: u32,
|
||||
},
|
||||
|
||||
/// No sora credentials are set.
|
||||
NeedsSoraCredentials,
|
||||
|
||||
SoraError(SoraError),
|
||||
AnyhowError(AnyhowError),
|
||||
StorytellerError(StorytellerError),
|
||||
}
|
||||
|
||||
impl From<AnyhowError> for InternalContextualEditImageError {
|
||||
fn from(value: AnyhowError) -> Self {
|
||||
Self::AnyhowError(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<SoraError> for InternalContextualEditImageError {
|
||||
fn from(value: SoraError) -> Self {
|
||||
Self::SoraError(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<StorytellerError> for InternalContextualEditImageError {
|
||||
fn from(value: StorytellerError) -> Self {
|
||||
Self::StorytellerError(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl InternalContextualEditImageError {
|
||||
pub fn to_tauri_response<E: Serialize>(&self) -> CommandErrorResponseWrapper<EnqueueContextualEditImageErrorType, E> {
|
||||
let mut status = CommandErrorStatus::ServerError;
|
||||
let mut error_type = EnqueueContextualEditImageErrorType::ServerError;
|
||||
let mut error_message = "A server error occurred. Please try again. If it continues, please tell our staff about the problem.".to_string();
|
||||
|
||||
match self {
|
||||
InternalContextualEditImageError::NoModelSpecified => {
|
||||
status = CommandErrorStatus::BadRequest;
|
||||
error_type = EnqueueContextualEditImageErrorType::ModelNotSpecified;
|
||||
error_message = "No model specified for image generation".to_string();
|
||||
}
|
||||
InternalContextualEditImageError::NoProviderAvailable => {
|
||||
status = CommandErrorStatus::ServerError;
|
||||
error_type = EnqueueContextualEditImageErrorType::NoProviderAvailable;
|
||||
error_message = "No configured provider available for image generation".to_string();
|
||||
}
|
||||
InternalContextualEditImageError::InvalidNumberOfRequestedImages { min, max, requested } => {
|
||||
status = CommandErrorStatus::BadRequest;
|
||||
error_type = EnqueueContextualEditImageErrorType::BadRequest;
|
||||
error_message = format!("Invalid number of images requested ({}). Must be between {} and {}", requested, min, max);
|
||||
|
||||
}
|
||||
InternalContextualEditImageError::InvalidNumberOfInputImages { min, max, requested } => {
|
||||
status = CommandErrorStatus::BadRequest;
|
||||
error_type = EnqueueContextualEditImageErrorType::BadRequest;
|
||||
error_message = format!("Invalid number of input images ({}). Must be between {} and {}", requested, min, max);
|
||||
}
|
||||
InternalContextualEditImageError::SoraError(_) => {}
|
||||
InternalContextualEditImageError::AnyhowError(_) => {}
|
||||
InternalContextualEditImageError::StorytellerError(_) => {}
|
||||
InternalContextualEditImageError::NeedsStorytellerCredentials => {}
|
||||
InternalContextualEditImageError::NeedsSoraCredentials => {}
|
||||
}
|
||||
|
||||
CommandErrorResponseWrapper {
|
||||
status,
|
||||
error_message: Some(error_message.to_string()),
|
||||
error_type: Some(error_type),
|
||||
error_details: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
+64
@@ -0,0 +1,64 @@
|
||||
use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::EnqueueContextualEditImageCommand;
|
||||
use crate::core::commands::enqueue::image_edit::errors::InternalContextualEditImageError;
|
||||
use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_artcraft::handle_gpt_image_1_artcraft;
|
||||
use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1_sora::handle_gpt_image_1_sora;
|
||||
use crate::core::commands::enqueue::image_edit::success_event::ContextualEditImageSuccessEvent;
|
||||
use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs;
|
||||
use crate::core::state::data_dir::app_data_root::AppDataRoot;
|
||||
use crate::core::state::provider_priority::{Provider, ProviderPriorityStore};
|
||||
use crate::services::fal::state::fal_credential_manager::FalCredentialManager;
|
||||
use crate::services::fal::state::fal_task_queue::FalTaskQueue;
|
||||
use crate::services::sora::state::sora_credential_manager::SoraCredentialManager;
|
||||
use crate::services::sora::state::sora_task_queue::SoraTaskQueue;
|
||||
use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager;
|
||||
use tauri::AppHandle;
|
||||
|
||||
pub async fn handle_gpt_image_1(
|
||||
request: &EnqueueContextualEditImageCommand,
|
||||
app: &AppHandle,
|
||||
app_data_root: &AppDataRoot,
|
||||
app_env_configs: &AppEnvConfigs,
|
||||
provider_priority_store: &ProviderPriorityStore,
|
||||
storyteller_creds_manager: &StorytellerCredentialManager,
|
||||
fal_creds_manager: &FalCredentialManager,
|
||||
fal_task_queue: &FalTaskQueue,
|
||||
sora_creds_manager: &SoraCredentialManager,
|
||||
sora_task_queue: &SoraTaskQueue,
|
||||
) -> Result<ContextualEditImageSuccessEvent, InternalContextualEditImageError> {
|
||||
|
||||
let priority = provider_priority_store.get_priority()?;
|
||||
|
||||
// TODO: Check if providers are available before proceeding.
|
||||
|
||||
for provider in priority.iter() {
|
||||
match provider {
|
||||
Provider::Fal => {
|
||||
// Fallthrough
|
||||
// If in the future we support OpenAI API keys, it's worth considering whether we
|
||||
// send those requests ourselves, or if we use FAL as an intermediary. FAL makes
|
||||
// the API nicer to deal with, but the user needs an additional key.
|
||||
}
|
||||
Provider::Artcraft => {
|
||||
return handle_gpt_image_1_artcraft(
|
||||
request,
|
||||
app,
|
||||
app_data_root,
|
||||
app_env_configs,
|
||||
storyteller_creds_manager
|
||||
).await;
|
||||
}
|
||||
Provider::Sora => {
|
||||
return handle_gpt_image_1_sora(
|
||||
request,
|
||||
app,
|
||||
app_data_root,
|
||||
app_env_configs,
|
||||
sora_creds_manager,
|
||||
sora_task_queue,
|
||||
).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Err(InternalContextualEditImageError::NoProviderAvailable)
|
||||
}
|
||||
+117
@@ -0,0 +1,117 @@
|
||||
use crate::core::commands::enqueue::image::enqueue_text_to_image_command::EnqueueTextToImageRequest;
|
||||
use crate::core::commands::enqueue::image::internal_image_error::InternalImageError;
|
||||
use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::{EditImageQuality, EditImageSize, EnqueueContextualEditImageCommand};
|
||||
use crate::core::commands::enqueue::image_edit::errors::InternalContextualEditImageError;
|
||||
use crate::core::commands::enqueue::image_edit::success_event::ContextualEditImageSuccessEvent;
|
||||
use crate::core::events::basic_sendable_event_trait::BasicSendableEvent;
|
||||
use crate::core::events::generation_events::common::{GenerationAction, GenerationServiceProvider};
|
||||
use crate::core::events::generation_events::generation_enqueue_failure_event::GenerationEnqueueFailureEvent;
|
||||
use crate::core::model::contextual_image_edit_models::ContextualImageEditModel;
|
||||
use crate::core::model::image_models::ImageModel;
|
||||
use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs;
|
||||
use crate::core::state::data_dir::app_data_root::AppDataRoot;
|
||||
use crate::core::state::provider_priority::ProviderPriorityStore;
|
||||
use crate::services::fal::state::fal_credential_manager::FalCredentialManager;
|
||||
use crate::services::fal::state::fal_task_queue::FalTaskQueue;
|
||||
use crate::services::sora::state::sora_credential_manager::SoraCredentialManager;
|
||||
use crate::services::sora::state::sora_task_queue::SoraTaskQueue;
|
||||
use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager;
|
||||
use anyhow::anyhow;
|
||||
use artcraft_api_defs::generate::image::edit::gpt_image_1_edit_image::{GptImage1EditImageImageQuality, GptImage1EditImageImageSize, GptImage1EditImageNumImages, GptImage1EditImageRequest};
|
||||
use artcraft_api_defs::generate::image::generate_flux_1_dev_text_to_image::GenerateFlux1DevTextToImageRequest;
|
||||
use artcraft_api_defs::generate::image::generate_flux_1_schnell_text_to_image::GenerateFlux1SchnellTextToImageRequest;
|
||||
use artcraft_api_defs::generate::image::generate_flux_pro_11_text_to_image::GenerateFluxPro11TextToImageRequest;
|
||||
use artcraft_api_defs::generate::image::generate_flux_pro_11_ultra_text_to_image::GenerateFluxPro11UltraTextToImageRequest;
|
||||
use fal_client::requests::queue::image_gen::enqueue_flux_pro_11_ultra_text_to_image::{enqueue_flux_pro_11_ultra_text_to_image, FluxPro11UltraTextToImageArgs};
|
||||
use fal_client::requests::webhook::image::enqueue_gpt_image_1_edit_image_webhook::GptEditImageQuality;
|
||||
use idempotency::uuid::generate_random_uuid;
|
||||
use log::{error, info};
|
||||
use storyteller_client::generate::image::edit::gpt_image_1_edit_image::gpt_image_1_edit_image;
|
||||
use storyteller_client::generate::image::generate_flux_1_dev_text_to_image::generate_flux_1_dev_text_to_image;
|
||||
use storyteller_client::generate::image::generate_flux_1_schnell_text_to_image::generate_flux_1_schnell_text_to_image;
|
||||
use storyteller_client::generate::image::generate_flux_pro_11_text_to_image::generate_flux_pro_11_text_to_image;
|
||||
use storyteller_client::generate::image::generate_flux_pro_11_ultra_text_to_image::generate_flux_pro_11_ultra_text_to_image;
|
||||
use tauri::AppHandle;
|
||||
|
||||
pub async fn handle_gpt_image_1_artcraft(
|
||||
request: &EnqueueContextualEditImageCommand,
|
||||
app: &AppHandle,
|
||||
app_data_root: &AppDataRoot,
|
||||
app_env_configs: &AppEnvConfigs,
|
||||
storyteller_creds_manager: &StorytellerCredentialManager,
|
||||
) -> Result<ContextualEditImageSuccessEvent, InternalContextualEditImageError> {
|
||||
|
||||
let creds = match storyteller_creds_manager.get_credentials()? {
|
||||
Some(creds) => creds,
|
||||
None => {
|
||||
return Err(InternalContextualEditImageError::NeedsStorytellerCredentials);
|
||||
},
|
||||
};
|
||||
|
||||
info!("Calling Artcraft gpt-image-1 ...");
|
||||
|
||||
let uuid_idempotency_token = generate_random_uuid();
|
||||
|
||||
let image_quality = request.image_quality
|
||||
.map(|quality| match quality {
|
||||
EditImageQuality::Auto => GptImage1EditImageImageQuality::Auto,
|
||||
EditImageQuality::High => GptImage1EditImageImageQuality::High,
|
||||
EditImageQuality::Medium => GptImage1EditImageImageQuality::Medium,
|
||||
EditImageQuality::Low => GptImage1EditImageImageQuality::Low,
|
||||
});
|
||||
|
||||
let image_size = request.aspect_ratio
|
||||
.map(|size| match size {
|
||||
EditImageSize::Auto => GptImage1EditImageImageSize::Square,
|
||||
EditImageSize::Square => GptImage1EditImageImageSize::Square,
|
||||
EditImageSize::Tall => GptImage1EditImageImageSize::Vertical,
|
||||
EditImageSize::Wide => GptImage1EditImageImageSize::Horizontal,
|
||||
});
|
||||
|
||||
let num_images = match request.image_count {
|
||||
None => None,
|
||||
Some(1) => Some(GptImage1EditImageNumImages::One),
|
||||
Some(2) => Some(GptImage1EditImageNumImages::Two),
|
||||
Some(3) => Some(GptImage1EditImageNumImages::Three),
|
||||
Some(4) => Some(GptImage1EditImageNumImages::Four),
|
||||
Some(other) => {
|
||||
return Err(InternalContextualEditImageError::InvalidNumberOfRequestedImages {
|
||||
min: 1,
|
||||
max: 4,
|
||||
requested: other,
|
||||
});
|
||||
},
|
||||
};
|
||||
|
||||
let request = GptImage1EditImageRequest {
|
||||
uuid_idempotency_token,
|
||||
prompt: Some(request.prompt.clone()),
|
||||
image_media_tokens: Some(request.image_media_tokens.clone()),
|
||||
image_size,
|
||||
num_images,
|
||||
image_quality,
|
||||
};
|
||||
|
||||
let result = gpt_image_1_edit_image(
|
||||
&app_env_configs.storyteller_host,
|
||||
Some(&creds),
|
||||
request,
|
||||
).await;
|
||||
|
||||
match result {
|
||||
Ok(enqueued) => {
|
||||
// TODO(bt,2025-07-05): Enqueue job token?
|
||||
info!("Successfully enqueued Artcraft gpt-image-1. Job token: {}",
|
||||
enqueued.inference_job_token);
|
||||
}
|
||||
Err(err) => {
|
||||
error!("Failed to use Artcraft gpt-image-1: {:?}", err);
|
||||
return Err(InternalContextualEditImageError::StorytellerError(err));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(ContextualEditImageSuccessEvent {
|
||||
service_provider: GenerationServiceProvider::Artcraft,
|
||||
model: ContextualImageEditModel::GptImage1,
|
||||
})
|
||||
}
|
||||
+172
@@ -0,0 +1,172 @@
|
||||
use crate::core::commands::enqueue::image::enqueue_text_to_image_command::EnqueueTextToImageRequest;
|
||||
use crate::core::commands::enqueue::image::internal_image_error::InternalImageError;
|
||||
use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::{EditImageSize, EnqueueContextualEditImageCommand, EnqueueContextualEditImageErrorType};
|
||||
use crate::core::commands::enqueue::image_edit::errors::InternalContextualEditImageError;
|
||||
use crate::core::commands::enqueue::image_edit::success_event::ContextualEditImageSuccessEvent;
|
||||
use crate::core::commands::response::failure_response_wrapper::{CommandErrorResponseWrapper, CommandErrorStatus};
|
||||
use crate::core::events::basic_sendable_event_trait::BasicSendableEvent;
|
||||
use crate::core::events::generation_events::common::{GenerationAction, GenerationServiceProvider};
|
||||
use crate::core::events::generation_events::generation_enqueue_failure_event::GenerationEnqueueFailureEvent;
|
||||
use crate::core::events::generation_events::generation_enqueue_success_event::GenerationEnqueueSuccessEvent;
|
||||
use crate::core::model::contextual_image_edit_models::ContextualImageEditModel;
|
||||
use crate::core::model::image_models::ImageModel;
|
||||
use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs;
|
||||
use crate::core::state::data_dir::app_data_root::AppDataRoot;
|
||||
use crate::core::state::data_dir::trait_data_subdir::DataSubdir;
|
||||
use crate::core::state::provider_priority::ProviderPriorityStore;
|
||||
use crate::core::utils::get_url_file_extension::get_url_file_extension;
|
||||
use crate::core::utils::simple_http_download::simple_http_download;
|
||||
use crate::services::fal::state::fal_credential_manager::FalCredentialManager;
|
||||
use crate::services::fal::state::fal_task_queue::FalTaskQueue;
|
||||
use crate::services::sora::state::sora_credential_manager::SoraCredentialManager;
|
||||
use crate::services::sora::state::sora_task_queue::SoraTaskQueue;
|
||||
use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager;
|
||||
use anyhow::anyhow;
|
||||
use artcraft_api_defs::generate::image::generate_flux_1_dev_text_to_image::GenerateFlux1DevTextToImageRequest;
|
||||
use artcraft_api_defs::generate::image::generate_flux_1_schnell_text_to_image::GenerateFlux1SchnellTextToImageRequest;
|
||||
use artcraft_api_defs::generate::image::generate_flux_pro_11_text_to_image::GenerateFluxPro11TextToImageRequest;
|
||||
use artcraft_api_defs::generate::image::generate_flux_pro_11_ultra_text_to_image::GenerateFluxPro11UltraTextToImageRequest;
|
||||
use errors::AnyhowResult;
|
||||
use fal_client::requests::queue::image_gen::enqueue_flux_pro_11_ultra_text_to_image::{enqueue_flux_pro_11_ultra_text_to_image, FluxPro11UltraTextToImageArgs};
|
||||
use idempotency::uuid::generate_random_uuid;
|
||||
use log::{error, info, warn};
|
||||
use openai_sora_client::recipes::image_remix_with_session_auto_renew::{image_remix_with_session_auto_renew, ImageRemixAutoRenewRequest};
|
||||
use openai_sora_client::recipes::image_upload_from_file_with_session_auto_renew::{image_upload_from_file_with_session_auto_renew, ImageUploadFromFileAutoRenewRequest};
|
||||
use openai_sora_client::recipes::maybe_upgrade_or_renew_session::maybe_upgrade_or_renew_session;
|
||||
use openai_sora_client::requests::image_gen::common::{ImageSize, NumImages};
|
||||
use openai_sora_client::sora_error::SoraError;
|
||||
use std::time::Duration;
|
||||
use storyteller_client::generate::image::generate_flux_1_dev_text_to_image::generate_flux_1_dev_text_to_image;
|
||||
use storyteller_client::generate::image::generate_flux_1_schnell_text_to_image::generate_flux_1_schnell_text_to_image;
|
||||
use storyteller_client::generate::image::generate_flux_pro_11_text_to_image::generate_flux_pro_11_text_to_image;
|
||||
use storyteller_client::generate::image::generate_flux_pro_11_ultra_text_to_image::generate_flux_pro_11_ultra_text_to_image;
|
||||
use storyteller_client::media_files::get_media_file::get_media_file;
|
||||
use storyteller_client::utils::api_host::ApiHost;
|
||||
use tauri::AppHandle;
|
||||
|
||||
const SORA_IMAGE_UPLOAD_TIMEOUT: Duration = Duration::from_millis(1000 * 30); // 30 seconds
|
||||
|
||||
const SORA_IMAGE_REMIX_TIMEOUT: Duration = Duration::from_millis(1000 * 30); // 30 seconds
|
||||
|
||||
const DEFAULT_ASPECT_RATIO : EditImageSize = EditImageSize::Square;
|
||||
|
||||
pub async fn handle_gpt_image_1_sora(
|
||||
request: &EnqueueContextualEditImageCommand,
|
||||
app: &AppHandle,
|
||||
app_data_root: &AppDataRoot,
|
||||
app_env_configs: &AppEnvConfigs,
|
||||
sora_creds_manager: &SoraCredentialManager,
|
||||
sora_task_queue: &SoraTaskQueue,
|
||||
|
||||
) -> Result<ContextualEditImageSuccessEvent, InternalContextualEditImageError> {
|
||||
|
||||
let sora_creds = match sora_creds_manager.get_credentials() {
|
||||
Ok(Some(creds)) => creds,
|
||||
Ok(None) => {
|
||||
warn!("No Sora credentials found.");
|
||||
return Err(InternalContextualEditImageError::NeedsSoraCredentials);
|
||||
}
|
||||
Err(err) => {
|
||||
error!("Failed to get Sora credentials: {:?}", err);
|
||||
return Err(InternalContextualEditImageError::AnyhowError(err));
|
||||
}
|
||||
};
|
||||
|
||||
// TODO(bt,2025-07-08): Handle many images.
|
||||
// TODO(bt,2025-07-08): Handle authenticated sessions (for private images).
|
||||
// TODO(bt,2025-07-08): Handle HTTP batch get requests.
|
||||
let first_image_token = request.image_media_tokens.first()
|
||||
.ok_or_else(|| InternalContextualEditImageError::InvalidNumberOfInputImages {
|
||||
min: 1,
|
||||
max: 10, // TODO Confirm
|
||||
requested: request.image_media_tokens.len() as u32,
|
||||
})?;
|
||||
|
||||
let response = get_media_file(&ApiHost::Storyteller, first_image_token).await?;
|
||||
|
||||
let media_file_url = &response.media_file.media_links.cdn_url;
|
||||
let extension_with_dot = get_url_file_extension(media_file_url)
|
||||
.map(|ext| format!(".{}", ext))
|
||||
.unwrap_or_else(|| ".png".to_string());
|
||||
|
||||
let filename = format!("{}{}", response.media_file.token.as_str(), extension_with_dot);
|
||||
let filename = app_data_root.downloads_dir().path().join(&filename);
|
||||
|
||||
simple_http_download(&media_file_url, &filename).await?;
|
||||
|
||||
let files_to_upload = vec![filename];
|
||||
|
||||
let mut creds = sora_creds_manager.get_credentials_required()?;
|
||||
|
||||
let credential_updated = maybe_upgrade_or_renew_session(&mut creds)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
error!("Failed to upgrade or renew session: {:?}", err);
|
||||
err
|
||||
})?;
|
||||
|
||||
if credential_updated {
|
||||
info!("Storing updated credentials");
|
||||
sora_creds_manager.set_credentials(&creds)?;
|
||||
}
|
||||
|
||||
let mut sora_media_tokens = Vec::with_capacity(files_to_upload.len());
|
||||
|
||||
for (i, file_path) in files_to_upload.iter().enumerate() {
|
||||
info!("Uploading image {} of {}...", (i+1), files_to_upload.len());
|
||||
|
||||
let (response, maybe_new_credentials) =
|
||||
image_upload_from_file_with_session_auto_renew(ImageUploadFromFileAutoRenewRequest {
|
||||
file_path,
|
||||
credentials: &creds,
|
||||
request_timeout: Some(SORA_IMAGE_UPLOAD_TIMEOUT), // TODO: Centralize and make configurable.
|
||||
}).await?;
|
||||
|
||||
if let Some(new_creds) = maybe_new_credentials {
|
||||
info!("Storing updated credentials.");
|
||||
sora_creds_manager.set_credentials(&new_creds)?;
|
||||
creds = new_creds;
|
||||
}
|
||||
|
||||
sora_media_tokens.push(response.id);
|
||||
}
|
||||
|
||||
let aspect_ratio = request.aspect_ratio.unwrap_or(DEFAULT_ASPECT_RATIO);
|
||||
|
||||
let aspect_ratio = match aspect_ratio {
|
||||
EditImageSize::Auto => ImageSize::Square,
|
||||
EditImageSize::Square => ImageSize::Square,
|
||||
EditImageSize::Wide => ImageSize::Wide,
|
||||
EditImageSize::Tall => ImageSize::Tall,
|
||||
};
|
||||
|
||||
info!("Calling Sora image generation...");
|
||||
|
||||
// TODO(bt,2025-04-21): Download media tokens.
|
||||
// Note: This is incredibly inefficient. We should keep a local cache.
|
||||
// Also, if they've already been uploaded to OpenAI, we shouldn't continue to re-upload.
|
||||
|
||||
let (response, maybe_new_creds) =
|
||||
image_remix_with_session_auto_renew(ImageRemixAutoRenewRequest {
|
||||
prompt: request.prompt.to_string(),
|
||||
num_images: NumImages::One,
|
||||
image_size: aspect_ratio,
|
||||
sora_media_tokens: sora_media_tokens.clone(),
|
||||
credentials: &creds,
|
||||
request_timeout: Some(SORA_IMAGE_REMIX_TIMEOUT),
|
||||
}).await?;
|
||||
|
||||
if let Some(new_creds) = maybe_new_creds {
|
||||
info!("Storing updated credentials.");
|
||||
sora_creds_manager.set_credentials(&new_creds)?;
|
||||
}
|
||||
|
||||
info!("New Sora Task ID: {:?} ", response.task_id);
|
||||
|
||||
sora_task_queue.insert(&response.task_id)?;
|
||||
|
||||
Ok(ContextualEditImageSuccessEvent {
|
||||
service_provider: GenerationServiceProvider::Sora,
|
||||
model: ContextualImageEditModel::GptImage1,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
pub mod handle_gpt_image_1;
|
||||
pub mod handle_gpt_image_1_artcraft;
|
||||
pub mod handle_gpt_image_1_sora;
|
||||
@@ -0,0 +1,4 @@
|
||||
pub mod enqueue_contextual_edit_image_command;
|
||||
pub mod errors;
|
||||
pub mod gpt_image_1;
|
||||
pub mod success_event;
|
||||
@@ -0,0 +1,16 @@
|
||||
use crate::core::events::generation_events::common::{GenerationModel, GenerationServiceProvider};
|
||||
use crate::core::model::contextual_image_edit_models::ContextualImageEditModel;
|
||||
use crate::core::model::image_models::ImageModel;
|
||||
|
||||
pub struct ContextualEditImageSuccessEvent {
|
||||
pub service_provider: GenerationServiceProvider,
|
||||
pub model: ContextualImageEditModel,
|
||||
}
|
||||
|
||||
impl ContextualEditImageSuccessEvent {
|
||||
pub fn tauri_event_model(&self) -> GenerationModel {
|
||||
match self.model {
|
||||
ContextualImageEditModel::GptImage1 => GenerationModel::GptImage1,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
pub mod image;
|
||||
pub mod image_edit;
|
||||
pub mod object;
|
||||
pub mod video;
|
||||
@@ -22,9 +22,18 @@ use crate::core::commands::response::success_response_wrapper::CommandSuccessRes
|
||||
// - sora_image_remix_command
|
||||
|
||||
/// Easy to use infallible response.
|
||||
/// This always contains a success payload and never results in an error.
|
||||
pub type InfallibleResponse<SuccessPayload> = CommandSuccessResponseWrapper<SuccessPayload>;
|
||||
|
||||
/// Easy to use Result<T, E> type.
|
||||
///
|
||||
/// There are easier ways to use Response<S, Et, Ep> :
|
||||
/// - `SimpleResponse` just returns strings on success or error (but provides clues to the frontend)
|
||||
/// - `SuccessOrErrorMessage` returns an empty success payload or an error message
|
||||
/// - `ResponseOrErrorMessage<SuccessPayload>` returns a success payload or an error message
|
||||
/// - `ResponseOrErrorType<SuccessPayload, ErrType>` returns a success payload or an error type
|
||||
/// - `ResponseOrError<SuccessPayload, ErrPayload>` returns a success payload or an error payload
|
||||
///
|
||||
pub type Response<SuccessPayload, ErrType, ErrPayload> =
|
||||
Result<CommandSuccessResponseWrapper<SuccessPayload>, CommandErrorResponseWrapper<ErrType, ErrPayload>>;
|
||||
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
use serde_derive::Deserialize;
|
||||
|
||||
/// This is used in the Tauri command bridge.
|
||||
/// Don't change the serializations without coordinating with the frontend.
|
||||
#[derive(Deserialize, Debug, Copy, Clone)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ContextualImageEditModel {
|
||||
#[serde(rename = "gpt_image_1")]
|
||||
GptImage1,
|
||||
|
||||
// TODO: Flux Kontext models.
|
||||
}
|
||||
@@ -1,2 +1,3 @@
|
||||
pub mod contextual_image_edit_models;
|
||||
pub mod image_models;
|
||||
pub mod video_models;
|
||||
@@ -6,6 +6,7 @@ use tauri::Manager;
|
||||
use crate::core::commands::app_preferences::get_app_preferences_command::get_app_preferences_command;
|
||||
use crate::core::commands::app_preferences::update_app_preference_command::update_app_preferences_command;
|
||||
use crate::core::commands::enqueue::image::enqueue_text_to_image_command::enqueue_text_to_image_command;
|
||||
use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::enqueue_contextual_edit_image_command;
|
||||
use crate::core::commands::enqueue::object::enqueue_image_to_3d_object_command::enqueue_image_to_3d_object_command;
|
||||
use crate::core::commands::enqueue::video::enqueue_image_to_video_command::enqueue_image_to_video_command;
|
||||
use crate::core::commands::flip_image::flip_image;
|
||||
@@ -189,6 +190,7 @@ pub fn run() {
|
||||
.manage(storyteller_creds_manager_3)
|
||||
.invoke_handler(tauri::generate_handler![
|
||||
check_sora_session_command,
|
||||
enqueue_contextual_edit_image_command,
|
||||
enqueue_image_to_3d_object_command,
|
||||
enqueue_image_to_video_command,
|
||||
enqueue_text_to_image_command,
|
||||
@@ -200,10 +202,10 @@ pub fn run() {
|
||||
get_app_preferences_command,
|
||||
get_fal_api_key_command,
|
||||
get_provider_order_command,
|
||||
set_provider_order_command,
|
||||
open_sora_login_command,
|
||||
platform_info_command,
|
||||
set_fal_api_key_command,
|
||||
set_provider_order_command,
|
||||
sora_image_generation_command,
|
||||
sora_image_remix_command,
|
||||
sora_logout_command,
|
||||
|
||||
Reference in new issue
Block a user