diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/enqueue_contextual_edit_image_command.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/enqueue_contextual_edit_image_command.rs index 6661f98316..0b08837f4d 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/enqueue_contextual_edit_image_command.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/enqueue_contextual_edit_image_command.rs @@ -51,7 +51,12 @@ pub struct EnqueueContextualEditImageCommand { /// Images to use for the image edit. /// The first image is typically a 2D canvas or 3D stage, but doesn't have to be. /// There must be at least one image. - pub image_media_tokens: Vec, + pub image_media_tokens: Option>, + + /// If set, this becomes the first image in the image media tokens (pushing back + /// each of the `image_media_tokens` by one). + /// This is useful if we want to do prompt engineering. + pub scene_image_media_token: Option, /// The user's image generation prompt. pub prompt: String, diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/handle_gpt_image_1.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/handle_gpt_image_1.rs index dbae5fcad8..dcfebab0e2 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/handle_gpt_image_1.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/handle_gpt_image_1.rs @@ -14,6 +14,8 @@ use crate::services::storyteller::state::storyteller_credential_manager::Storyte use log::info; use tauri::AppHandle; +pub(super) const MAX_IMAGES: usize = 10; + pub async fn handle_gpt_image_1( request: &EnqueueContextualEditImageCommand, app: &AppHandle, diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/handle_gpt_image_1_artcraft.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/handle_gpt_image_1_artcraft.rs index 3cb4fde2a4..d5f7492165 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/handle_gpt_image_1_artcraft.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/handle_gpt_image_1_artcraft.rs @@ -2,6 +2,7 @@ use crate::core::commands::enqueue::image::enqueue_text_to_image_command::Enqueu use crate::core::commands::enqueue::image::internal_image_error::InternalImageError; use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::{EditImageQuality, EditImageSize, EnqueueContextualEditImageCommand}; use crate::core::commands::enqueue::image_edit::errors::InternalContextualEditImageError; +use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1::MAX_IMAGES; use crate::core::commands::enqueue::image_edit::success_event::ContextualEditImageSuccessEvent; use crate::core::events::basic_sendable_event_trait::BasicSendableEvent; use crate::core::events::generation_events::common::{GenerationAction, GenerationServiceProvider}; @@ -82,11 +83,29 @@ pub async fn handle_gpt_image_1_artcraft( }); }, }; + + let mut media_tokens = Vec::with_capacity(10); + + if let Some(scene_image_media_token) = request.scene_image_media_token.clone() { + media_tokens.push(scene_image_media_token); + } + + if let Some(image_media_tokens) = request.image_media_tokens.as_ref() { + media_tokens.extend_from_slice(image_media_tokens); + } + + if media_tokens.len() > MAX_IMAGES { + return Err(InternalContextualEditImageError::InvalidNumberOfInputImages { + min: 1, + max: MAX_IMAGES as u32, + requested: media_tokens.len() as u32, + }); + } let request = GptImage1EditImageRequest { uuid_idempotency_token, prompt: Some(request.prompt.clone()), - image_media_tokens: Some(request.image_media_tokens.clone()), + image_media_tokens: Some(media_tokens), image_size, num_images, image_quality, diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/handle_gpt_image_1_sora.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/handle_gpt_image_1_sora.rs index 78e5856a69..c7b66bece0 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/handle_gpt_image_1_sora.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/gpt_image_1/handle_gpt_image_1_sora.rs @@ -2,6 +2,7 @@ use crate::core::commands::enqueue::image::enqueue_text_to_image_command::Enqueu use crate::core::commands::enqueue::image::internal_image_error::InternalImageError; use crate::core::commands::enqueue::image_edit::enqueue_contextual_edit_image_command::{EditImageSize, EnqueueContextualEditImageCommand, EnqueueContextualEditImageErrorType}; use crate::core::commands::enqueue::image_edit::errors::InternalContextualEditImageError; +use crate::core::commands::enqueue::image_edit::gpt_image_1::handle_gpt_image_1::MAX_IMAGES; use crate::core::commands::enqueue::image_edit::success_event::ContextualEditImageSuccessEvent; use crate::core::commands::response::failure_response_wrapper::{CommandErrorResponseWrapper, CommandErrorStatus}; use crate::core::events::basic_sendable_event_trait::BasicSendableEvent; @@ -67,34 +68,50 @@ pub async fn handle_gpt_image_1_sora( } }; - // TODO(bt,2025-07-08): Handle many images. - // TODO(bt,2025-07-08): Handle authenticated sessions (for private images). - // TODO(bt,2025-07-08): Handle HTTP batch get requests. - let first_image_token = request.image_media_tokens.first() - .ok_or_else(|| InternalContextualEditImageError::InvalidNumberOfInputImages { - min: 1, - max: 10, // TODO Confirm - requested: request.image_media_tokens.len() as u32, - })?; + let mut media_tokens = Vec::with_capacity(10); + + if let Some(scene_image_media_token) = request.scene_image_media_token.clone() { + media_tokens.push(scene_image_media_token); + } + + if let Some(image_media_tokens) = request.image_media_tokens.as_ref() { + media_tokens.extend_from_slice(image_media_tokens); + } + + if media_tokens.len() > MAX_IMAGES { + return Err(InternalContextualEditImageError::InvalidNumberOfInputImages { + min: 1, + max: MAX_IMAGES as u32, + requested: media_tokens.len() as u32, + }); + } info!("Calling get media file API: {:?}", app_env_configs.storyteller_host); + + let mut files_to_upload_to_sora = Vec::with_capacity(10); + + // TODO(bt,2025-07-07): This is inefficient. Cache and parallelize this. + for media_token in media_tokens.iter() { + info!("Using media token: {:?}", media_token); - let response = get_media_file( - &app_env_configs.storyteller_host, - first_image_token - ).await?; + let response = get_media_file( + &app_env_configs.storyteller_host, + media_token + ).await?; - let media_file_url = &response.media_file.media_links.cdn_url; - let extension_with_dot = get_url_file_extension(media_file_url) - .map(|ext| format!(".{}", ext)) - .unwrap_or_else(|| ".png".to_string()); + let media_file_url = &response.media_file.media_links.cdn_url; + let extension_with_dot = get_url_file_extension(media_file_url) + .map(|ext| format!(".{}", ext)) + .unwrap_or_else(|| ".png".to_string()); - let filename = format!("{}{}", response.media_file.token.as_str(), extension_with_dot); - let filename = app_data_root.downloads_dir().path().join(&filename); + let filename = format!("{}{}", response.media_file.token.as_str(), extension_with_dot); + let filename = app_data_root.downloads_dir().path().join(&filename); - simple_http_download(&media_file_url, &filename).await?; + simple_http_download(&media_file_url, &filename).await?; + + files_to_upload_to_sora.push(filename); + } - let files_to_upload = vec![filename]; let mut creds = sora_creds_manager.get_credentials_required()?; @@ -110,10 +127,10 @@ pub async fn handle_gpt_image_1_sora( sora_creds_manager.set_credentials(&creds)?; } - let mut sora_media_tokens = Vec::with_capacity(files_to_upload.len()); + let mut sora_media_tokens = Vec::with_capacity(files_to_upload_to_sora.len()); - for (i, file_path) in files_to_upload.iter().enumerate() { - info!("Uploading image {} of {}...", (i+1), files_to_upload.len()); + for (i, file_path) in files_to_upload_to_sora.iter().enumerate() { + info!("Uploading image {} of {}...", (i+1), files_to_upload_to_sora.len()); let (response, maybe_new_credentials) = image_upload_from_file_with_session_auto_renew(ImageUploadFromFileAutoRenewRequest {