From c8228c5073dfb3ec4b83cae1aee63d924374a780 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 31 Dec 2025 07:03:34 -0500 Subject: [PATCH] more model controls (#948) still need to roll this out to everything, but this lands standardized aspect ratio for several models --- .../generate/image/multi_function/mod.rs | 1 + ...o_banana_multi_function_image_gen_image.rs | 18 ++++ .../aspect_ratio/common_aspect_ratio.rs | 4 + .../aspect_ratio_to_artcraft_gpt_image_1.rs | 7 ++ .../aspect_ratio_to_artcraft_gpt_image_1p5.rs | 28 ++++++ .../aspect_ratio_to_artcraft_nano_banana.rs | 26 +++++ ...spect_ratio_to_artcraft_nano_banana_pro.rs | 3 + .../convert/aspect_ratio_to_grok_image.rs | 28 ++++++ ...aspect_ratio_to_sora_native_gpt_image_1.rs | 28 ++++++ .../api_adapters/aspect_ratio/convert/mod.rs | 7 +- .../handle_artcraft_gpt_image_1p5_edit.rs | 31 ++++-- .../handle_artcraft_nano_banana_edit.rs | 44 +++++++-- ...le_artcraft_gpt_image_1p5_text_to_image.rs | 33 +++++-- ...ndle_artcraft_nano_banana_text_to_image.rs | 53 +++++++--- .../enqueue/text_to_image/grok/handle_grok.rs | 98 ++++--------------- .../handle_gpt_image_1_sora_text_to_image.rs | 34 +++++-- .../app/src/pages/PageEdit/PageEdit.tsx | 19 +++- .../promptbox/src/lib/PromptBox3D.tsx | 19 +++- .../promptbox/src/lib/PromptBoxEdit.tsx | 16 ++- .../promptbox/src/lib/PromptBoxImage.tsx | 4 +- .../src/lib/common/AspectRatioPicker.tsx | 49 ++++++---- .../classes/properties/CommonAspectRatio.ts | 5 + .../model-list/src/lib/lists/ImageModels.ts | 38 +++++-- .../src/lib/enqueue/EnqueueEditImage.ts | 11 ++- 24 files changed, 444 insertions(+), 160 deletions(-) create mode 100644 crates/api_clients/artcraft/storyteller_client/src/endpoints/generate/image/multi_function/nano_banana_multi_function_image_gen_image.rs create mode 100644 crates/desktop/artcraft/src/core/api_adapters/aspect_ratio/convert/aspect_ratio_to_artcraft_gpt_image_1.rs create mode 100644 crates/desktop/artcraft/src/core/api_adapters/aspect_ratio/convert/aspect_ratio_to_artcraft_gpt_image_1p5.rs create mode 100644 crates/desktop/artcraft/src/core/api_adapters/aspect_ratio/convert/aspect_ratio_to_artcraft_nano_banana.rs create mode 100644 crates/desktop/artcraft/src/core/api_adapters/aspect_ratio/convert/aspect_ratio_to_grok_image.rs create mode 100644 crates/desktop/artcraft/src/core/api_adapters/aspect_ratio/convert/aspect_ratio_to_sora_native_gpt_image_1.rs diff --git a/crates/api_clients/artcraft/storyteller_client/src/endpoints/generate/image/multi_function/mod.rs b/crates/api_clients/artcraft/storyteller_client/src/endpoints/generate/image/multi_function/mod.rs index 292729e202..190a87b48c 100644 --- a/crates/api_clients/artcraft/storyteller_client/src/endpoints/generate/image/multi_function/mod.rs +++ b/crates/api_clients/artcraft/storyteller_client/src/endpoints/generate/image/multi_function/mod.rs @@ -1,4 +1,5 @@ pub mod bytedance_seedream_v4_multi_function_image_gen_image; pub mod bytedance_seedream_v4p5_multi_function_image_gen_image; pub mod gpt_image_1p5_multi_function_image_gen_image; +pub mod nano_banana_multi_function_image_gen_image; pub mod nano_banana_pro_multi_function_image_gen_image; diff --git a/crates/api_clients/artcraft/storyteller_client/src/endpoints/generate/image/multi_function/nano_banana_multi_function_image_gen_image.rs b/crates/api_clients/artcraft/storyteller_client/src/endpoints/generate/image/multi_function/nano_banana_multi_function_image_gen_image.rs new file mode 100644 index 0000000000..c63118e435 --- /dev/null +++ b/crates/api_clients/artcraft/storyteller_client/src/endpoints/generate/image/multi_function/nano_banana_multi_function_image_gen_image.rs @@ -0,0 +1,18 @@ +use crate::credentials::storyteller_credential_set::StorytellerCredentialSet; +use crate::error::storyteller_error::StorytellerError; +use crate::utils::api_host::ApiHost; +use crate::utils::basic_json_post_request::basic_json_post_request; +use artcraft_api_defs::generate::image::multi_function::nano_banana_multi_function_image_gen::{NanoBananaMultiFunctionImageGenRequest, NanoBananaMultiFunctionImageGenResponse, NANO_BANANA_MULTI_FUNCTION_IMAGE_GEN_PATH}; + +pub async fn nano_banana_multi_function_image_gen( + api_host: &ApiHost, + maybe_creds: Option<&StorytellerCredentialSet>, + request: NanoBananaMultiFunctionImageGenRequest, +) -> Result { + Ok(basic_json_post_request( + api_host, + NANO_BANANA_MULTI_FUNCTION_IMAGE_GEN_PATH, + maybe_creds, + request, + ).await?) +} diff --git a/crates/desktop/artcraft/src/core/api_adapters/aspect_ratio/common_aspect_ratio.rs b/crates/desktop/artcraft/src/core/api_adapters/aspect_ratio/common_aspect_ratio.rs index c61910bd1f..12efcfd902 100644 --- a/crates/desktop/artcraft/src/core/api_adapters/aspect_ratio/common_aspect_ratio.rs +++ b/crates/desktop/artcraft/src/core/api_adapters/aspect_ratio/common_aspect_ratio.rs @@ -22,4 +22,8 @@ pub enum CommonAspectRatio { TallTwoByThree, TallNineBySixteen, TallNineByTwentyOne, + // Imprecise semantic values that we probably remap to other meanings + // on a model-by-model basis. + Wide, + Tall, } diff --git a/crates/desktop/artcraft/src/core/api_adapters/aspect_ratio/convert/aspect_ratio_to_artcraft_gpt_image_1.rs b/crates/desktop/artcraft/src/core/api_adapters/aspect_ratio/convert/aspect_ratio_to_artcraft_gpt_image_1.rs new file mode 100644 index 0000000000..1c1d4954bc --- /dev/null +++ b/crates/desktop/artcraft/src/core/api_adapters/aspect_ratio/convert/aspect_ratio_to_artcraft_gpt_image_1.rs @@ -0,0 +1,7 @@ +use crate::core::api_adapters::aspect_ratio::common_aspect_ratio::CommonAspectRatio; + +pub fn aspect_ratio_to_artcraft_gpt_image_1(aspect_ratio: CommonAspectRatio) -> Option { + // TODO: We still make calls to two distinct endpoints for image-to-image and text-to-image. + // These need to be consolidated to benefit from this approach. + None +} diff --git a/crates/desktop/artcraft/src/core/api_adapters/aspect_ratio/convert/aspect_ratio_to_artcraft_gpt_image_1p5.rs b/crates/desktop/artcraft/src/core/api_adapters/aspect_ratio/convert/aspect_ratio_to_artcraft_gpt_image_1p5.rs new file mode 100644 index 0000000000..c01ff29e1e --- /dev/null +++ b/crates/desktop/artcraft/src/core/api_adapters/aspect_ratio/convert/aspect_ratio_to_artcraft_gpt_image_1p5.rs @@ -0,0 +1,28 @@ +use crate::core::api_adapters::aspect_ratio::common_aspect_ratio::CommonAspectRatio; +use artcraft_api_defs::generate::image::multi_function::gpt_image_1p5_multi_function_image_gen::GptImage1p5MultiFunctionImageGenSize; + +pub fn aspect_ratio_to_artcraft_gpt_image_1p5(aspect_ratio: CommonAspectRatio) -> GptImage1p5MultiFunctionImageGenSize { + match aspect_ratio { + // Exact + CommonAspectRatio::Square => GptImage1p5MultiFunctionImageGenSize::Square, + CommonAspectRatio::Wide => GptImage1p5MultiFunctionImageGenSize::Wide, + CommonAspectRatio::Tall => GptImage1p5MultiFunctionImageGenSize::Tall, + + // Non-matching + CommonAspectRatio::Auto => GptImage1p5MultiFunctionImageGenSize::Square, + + // Mismatch - wide + CommonAspectRatio::WideThreeByTwo + | CommonAspectRatio::WideFiveByFour + | CommonAspectRatio::WideFourByThree + | CommonAspectRatio::WideSixteenByNine + | CommonAspectRatio::WideTwentyOneByNine => GptImage1p5MultiFunctionImageGenSize::Wide, + + // Mismatch - tall + CommonAspectRatio::TallTwoByThree + | CommonAspectRatio::TallFourByFive + | CommonAspectRatio::TallThreeByFour + | CommonAspectRatio::TallNineBySixteen + | CommonAspectRatio::TallNineByTwentyOne => GptImage1p5MultiFunctionImageGenSize::Tall, + } +} diff --git a/crates/desktop/artcraft/src/core/api_adapters/aspect_ratio/convert/aspect_ratio_to_artcraft_nano_banana.rs b/crates/desktop/artcraft/src/core/api_adapters/aspect_ratio/convert/aspect_ratio_to_artcraft_nano_banana.rs new file mode 100644 index 0000000000..3488097ec9 --- /dev/null +++ b/crates/desktop/artcraft/src/core/api_adapters/aspect_ratio/convert/aspect_ratio_to_artcraft_nano_banana.rs @@ -0,0 +1,26 @@ +use crate::core::api_adapters::aspect_ratio::common_aspect_ratio::CommonAspectRatio; +use artcraft_api_defs::generate::image::multi_function::nano_banana_multi_function_image_gen::NanoBananaMultiFunctionImageGenAspectRatio; + +pub fn aspect_ratio_to_artcraft_nano_banana(aspect_ratio: CommonAspectRatio) -> NanoBananaMultiFunctionImageGenAspectRatio { + match aspect_ratio { + // Exact + CommonAspectRatio::Square => NanoBananaMultiFunctionImageGenAspectRatio::OneByOne, + CommonAspectRatio::WideFiveByFour => NanoBananaMultiFunctionImageGenAspectRatio::FiveByFour, + CommonAspectRatio::WideFourByThree => NanoBananaMultiFunctionImageGenAspectRatio::FourByThree, + CommonAspectRatio::WideThreeByTwo => NanoBananaMultiFunctionImageGenAspectRatio::ThreeByTwo, + CommonAspectRatio::WideSixteenByNine => NanoBananaMultiFunctionImageGenAspectRatio::SixteenByNine, + CommonAspectRatio::WideTwentyOneByNine => NanoBananaMultiFunctionImageGenAspectRatio::TwentyOneByNine, + CommonAspectRatio::TallFourByFive => NanoBananaMultiFunctionImageGenAspectRatio::FourByFive, + CommonAspectRatio::TallThreeByFour => NanoBananaMultiFunctionImageGenAspectRatio::ThreeByFour, + CommonAspectRatio::TallTwoByThree => NanoBananaMultiFunctionImageGenAspectRatio::TwoByThree, + CommonAspectRatio::TallNineBySixteen => NanoBananaMultiFunctionImageGenAspectRatio::NineBySixteen, + // Special + // NB: this is only for image-to-image editing, not text-to-image. + CommonAspectRatio::Auto => NanoBananaMultiFunctionImageGenAspectRatio::Auto, + // NB: "9:21" value does not exist. Interpolated. + CommonAspectRatio::TallNineByTwentyOne => NanoBananaMultiFunctionImageGenAspectRatio::NineBySixteen, + // Semantic values + CommonAspectRatio::Wide => NanoBananaMultiFunctionImageGenAspectRatio::SixteenByNine, + CommonAspectRatio::Tall => NanoBananaMultiFunctionImageGenAspectRatio::NineBySixteen, + } +} diff --git a/crates/desktop/artcraft/src/core/api_adapters/aspect_ratio/convert/aspect_ratio_to_artcraft_nano_banana_pro.rs b/crates/desktop/artcraft/src/core/api_adapters/aspect_ratio/convert/aspect_ratio_to_artcraft_nano_banana_pro.rs index a784aadabc..49cbbfd73c 100644 --- a/crates/desktop/artcraft/src/core/api_adapters/aspect_ratio/convert/aspect_ratio_to_artcraft_nano_banana_pro.rs +++ b/crates/desktop/artcraft/src/core/api_adapters/aspect_ratio/convert/aspect_ratio_to_artcraft_nano_banana_pro.rs @@ -19,5 +19,8 @@ pub fn aspect_ratio_to_artcraft_nano_banana_pro(aspect_ratio: CommonAspectRatio) CommonAspectRatio::Auto => NanoBananaProMultiFunctionImageGenAspectRatio::Auto, // NB: "9:21" value does not exist. Interpolated. CommonAspectRatio::TallNineByTwentyOne => NanoBananaProMultiFunctionImageGenAspectRatio::NineBySixteen, + // Semantic values + CommonAspectRatio::Wide => NanoBananaProMultiFunctionImageGenAspectRatio::SixteenByNine, + CommonAspectRatio::Tall => NanoBananaProMultiFunctionImageGenAspectRatio::NineBySixteen, } } diff --git a/crates/desktop/artcraft/src/core/api_adapters/aspect_ratio/convert/aspect_ratio_to_grok_image.rs b/crates/desktop/artcraft/src/core/api_adapters/aspect_ratio/convert/aspect_ratio_to_grok_image.rs new file mode 100644 index 0000000000..f2f7305198 --- /dev/null +++ b/crates/desktop/artcraft/src/core/api_adapters/aspect_ratio/convert/aspect_ratio_to_grok_image.rs @@ -0,0 +1,28 @@ +use crate::core::api_adapters::aspect_ratio::common_aspect_ratio::CommonAspectRatio; +use grok_client::requests::image_websocket::messages::websocket_client_message::ClientMessageAspectRatio; + +pub fn aspect_ratio_to_grok_image(aspect_ratio: CommonAspectRatio) -> ClientMessageAspectRatio { + match aspect_ratio { + // Exact + CommonAspectRatio::Square => ClientMessageAspectRatio::Square, + CommonAspectRatio::WideThreeByTwo => ClientMessageAspectRatio::WideThreeByTwo, + CommonAspectRatio::TallTwoByThree => ClientMessageAspectRatio::TallTwoByThree, + + // Non-matching + CommonAspectRatio::Auto => ClientMessageAspectRatio::Square, + + // Mismatch - wide + CommonAspectRatio::Wide + | CommonAspectRatio::WideFiveByFour + | CommonAspectRatio::WideFourByThree + | CommonAspectRatio::WideSixteenByNine + | CommonAspectRatio::WideTwentyOneByNine => ClientMessageAspectRatio::WideThreeByTwo, + + // Mismatch - tall + CommonAspectRatio::Tall + | CommonAspectRatio::TallFourByFive + | CommonAspectRatio::TallThreeByFour + | CommonAspectRatio::TallNineBySixteen + | CommonAspectRatio::TallNineByTwentyOne => ClientMessageAspectRatio::TallTwoByThree, + } +} diff --git a/crates/desktop/artcraft/src/core/api_adapters/aspect_ratio/convert/aspect_ratio_to_sora_native_gpt_image_1.rs b/crates/desktop/artcraft/src/core/api_adapters/aspect_ratio/convert/aspect_ratio_to_sora_native_gpt_image_1.rs new file mode 100644 index 0000000000..5efda6e287 --- /dev/null +++ b/crates/desktop/artcraft/src/core/api_adapters/aspect_ratio/convert/aspect_ratio_to_sora_native_gpt_image_1.rs @@ -0,0 +1,28 @@ +use crate::core::api_adapters::aspect_ratio::common_aspect_ratio::CommonAspectRatio; +use openai_sora_client::requests::image_gen::common::ImageSize; + +pub fn aspect_ratio_to_sora_native_gpt_image_1(aspect_ratio: CommonAspectRatio) -> ImageSize { + match aspect_ratio { + // Exact + CommonAspectRatio::Square => ImageSize::Square, + CommonAspectRatio::Wide => ImageSize::Wide, + CommonAspectRatio::Tall => ImageSize::Tall, + + // Non-matching + CommonAspectRatio::Auto => ImageSize::Square, + + // Mismatch - wide + CommonAspectRatio::WideThreeByTwo + | CommonAspectRatio::WideFiveByFour + | CommonAspectRatio::WideFourByThree + | CommonAspectRatio::WideSixteenByNine + | CommonAspectRatio::WideTwentyOneByNine => ImageSize::Wide, + + // Mismatch - tall + CommonAspectRatio::TallTwoByThree + | CommonAspectRatio::TallFourByFive + | CommonAspectRatio::TallThreeByFour + | CommonAspectRatio::TallNineBySixteen + | CommonAspectRatio::TallNineByTwentyOne => ImageSize::Tall, + } +} diff --git a/crates/desktop/artcraft/src/core/api_adapters/aspect_ratio/convert/mod.rs b/crates/desktop/artcraft/src/core/api_adapters/aspect_ratio/convert/mod.rs index 7cd722714f..37a0c0790a 100644 --- a/crates/desktop/artcraft/src/core/api_adapters/aspect_ratio/convert/mod.rs +++ b/crates/desktop/artcraft/src/core/api_adapters/aspect_ratio/convert/mod.rs @@ -1 +1,6 @@ -pub mod aspect_ratio_to_artcraft_nano_banana_pro; \ No newline at end of file +pub mod aspect_ratio_to_artcraft_gpt_image_1; +pub mod aspect_ratio_to_artcraft_gpt_image_1p5; +pub mod aspect_ratio_to_artcraft_nano_banana; +pub mod aspect_ratio_to_artcraft_nano_banana_pro; +pub mod aspect_ratio_to_grok_image; +pub mod aspect_ratio_to_sora_native_gpt_image_1; \ No newline at end of file diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/artcraft/handle_artcraft_gpt_image_1p5_edit.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/artcraft/handle_artcraft_gpt_image_1p5_edit.rs index c9f3becce2..ce9c15352f 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/artcraft/handle_artcraft_gpt_image_1p5_edit.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/artcraft/handle_artcraft_gpt_image_1p5_edit.rs @@ -1,3 +1,6 @@ +use crate::core::api_adapters::aspect_ratio::convert::aspect_ratio_to_artcraft_gpt_image_1::aspect_ratio_to_artcraft_gpt_image_1; +use crate::core::api_adapters::aspect_ratio::convert::aspect_ratio_to_artcraft_gpt_image_1p5::aspect_ratio_to_artcraft_gpt_image_1p5; +use crate::core::api_adapters::aspect_ratio::convert::aspect_ratio_to_artcraft_nano_banana_pro::aspect_ratio_to_artcraft_nano_banana_pro; use crate::core::commands::enqueue::generate_error::{BadInputReason, GenerateError}; use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::{EditImageQuality, EditImageSize, EnqueueEditImageCommand}; use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess; @@ -57,13 +60,7 @@ pub async fn handle_artcraft_gpt_image_1p5_edit( EditImageQuality::Low => GptImage1p5MultiFunctionImageGenQuality::Low, }); - let image_size = request.aspect_ratio - .map(|size| match size { - EditImageSize::Auto => GptImage1p5MultiFunctionImageGenSize::Square, - EditImageSize::Square => GptImage1p5MultiFunctionImageGenSize::Square, - EditImageSize::Tall => GptImage1p5MultiFunctionImageGenSize::Tall, - EditImageSize::Wide => GptImage1p5MultiFunctionImageGenSize::Wide, - }); + let image_size = get_aspect_ratio(request); let num_images = match request.image_count { None => None, @@ -139,3 +136,23 @@ pub async fn handle_artcraft_gpt_image_1p5_edit( task_type: TaskType::ImageGeneration, }) } + +fn get_aspect_ratio(request: &EnqueueEditImageCommand) -> Option { + if let Some(common_aspect_ratio) = request.common_aspect_ratio { + // Handle modern aspect ratio + let aspect = aspect_ratio_to_artcraft_gpt_image_1p5(common_aspect_ratio); + return Some(aspect); + } + + if let Some(aspect_ratio) = request.aspect_ratio { + // Handle deprecated aspect ratio + return match aspect_ratio { + EditImageSize::Auto => Some(GptImage1p5MultiFunctionImageGenSize::Square), + EditImageSize::Square => Some(GptImage1p5MultiFunctionImageGenSize::Square), + EditImageSize::Wide => Some(GptImage1p5MultiFunctionImageGenSize::Wide), + EditImageSize::Tall => Some(GptImage1p5MultiFunctionImageGenSize::Tall), + } + } + + None +} diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/artcraft/handle_artcraft_nano_banana_edit.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/artcraft/handle_artcraft_nano_banana_edit.rs index 045a20375a..683db6276c 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/artcraft/handle_artcraft_nano_banana_edit.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/artcraft/handle_artcraft_nano_banana_edit.rs @@ -1,3 +1,5 @@ +use crate::core::api_adapters::aspect_ratio::convert::aspect_ratio_to_artcraft_nano_banana::aspect_ratio_to_artcraft_nano_banana; +use crate::core::api_adapters::aspect_ratio::convert::aspect_ratio_to_artcraft_nano_banana_pro::aspect_ratio_to_artcraft_nano_banana_pro; use crate::core::commands::enqueue::generate_error::{BadInputReason, GenerateError}; use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::{EditImageQuality, EditImageSize, EnqueueEditImageCommand}; use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess; @@ -18,6 +20,8 @@ use artcraft_api_defs::generate::image::generate_flux_1_dev_text_to_image::Gener use artcraft_api_defs::generate::image::generate_flux_1_schnell_text_to_image::GenerateFlux1SchnellTextToImageRequest; use artcraft_api_defs::generate::image::generate_flux_pro_11_text_to_image::GenerateFluxPro11TextToImageRequest; use artcraft_api_defs::generate::image::generate_flux_pro_11_ultra_text_to_image::GenerateFluxPro11UltraTextToImageRequest; +use artcraft_api_defs::generate::image::multi_function::nano_banana_multi_function_image_gen::{NanoBananaMultiFunctionImageGenAspectRatio, NanoBananaMultiFunctionImageGenNumImages, NanoBananaMultiFunctionImageGenRequest}; +use artcraft_api_defs::generate::image::multi_function::nano_banana_pro_multi_function_image_gen::{NanoBananaProMultiFunctionImageGenAspectRatio, NanoBananaProMultiFunctionImageGenRequest}; use enums::common::generation_provider::GenerationProvider; use enums::tauri::tasks::task_type::TaskType; use idempotency::uuid::generate_random_uuid; @@ -25,6 +29,8 @@ use log::{error, info}; use storyteller_client::endpoints::generate::image::edit::flux_pro_kontext_max_edit_image::flux_pro_kontext_max_edit_image; use storyteller_client::endpoints::generate::image::edit::gemini_25_flash_edit_image::gemini_25_flash_edit_image; use storyteller_client::endpoints::generate::image::edit::gpt_image_1_edit_image::gpt_image_1_edit_image; +use storyteller_client::endpoints::generate::image::multi_function::nano_banana_multi_function_image_gen_image::nano_banana_multi_function_image_gen; +use storyteller_client::endpoints::generate::image::multi_function::nano_banana_pro_multi_function_image_gen_image::nano_banana_pro_multi_function_image_gen; use tauri::AppHandle; pub(super) const MAX_IMAGES: usize = 10; @@ -47,13 +53,15 @@ pub async fn handle_artcraft_nano_banana_edit( info!("Calling Artcraft Nano Banana (edit) ..."); let uuid_idempotency_token = generate_random_uuid(); - + + let aspect_ratio = get_aspect_ratio(request); + let num_images = match request.image_count { None => None, - Some(1) => Some(Gemini25FlashEditImageNumImages::One), - Some(2) => Some(Gemini25FlashEditImageNumImages::Two), - Some(3) => Some(Gemini25FlashEditImageNumImages::Three), - Some(4) => Some(Gemini25FlashEditImageNumImages::Four), + Some(1) => Some(NanoBananaMultiFunctionImageGenNumImages::One), + Some(2) => Some(NanoBananaMultiFunctionImageGenNumImages::Two), + Some(3) => Some(NanoBananaMultiFunctionImageGenNumImages::Three), + Some(4) => Some(NanoBananaMultiFunctionImageGenNumImages::Four), Some(other) => { return Err(GenerateError::BadInput(BadInputReason::InvalidNumberOfRequestedImages { min: 1, @@ -81,15 +89,15 @@ pub async fn handle_artcraft_nano_banana_edit( })); } - let request = Gemini25FlashEditImageRequest { + let request = NanoBananaMultiFunctionImageGenRequest { uuid_idempotency_token, prompt: Some(request.prompt.clone()), image_media_tokens: Some(media_tokens), num_images, - image_quality: None, + aspect_ratio, }; - let result = gemini_25_flash_edit_image( + let result = nano_banana_multi_function_image_gen( &app_env_configs.storyteller_host, Some(&creds), request, @@ -115,3 +123,23 @@ pub async fn handle_artcraft_nano_banana_edit( task_type: TaskType::ImageGeneration, }) } + +fn get_aspect_ratio(request: &EnqueueEditImageCommand) -> Option { + if let Some(common_aspect_ratio) = request.common_aspect_ratio { + // Handle modern aspect ratio + let aspect = aspect_ratio_to_artcraft_nano_banana(common_aspect_ratio); + return Some(aspect); + } + + if let Some(aspect_ratio) = request.aspect_ratio { + // Handle deprecated aspect ratio + return match aspect_ratio { + EditImageSize::Auto => Some(NanoBananaMultiFunctionImageGenAspectRatio::Auto), + EditImageSize::Square => Some(NanoBananaMultiFunctionImageGenAspectRatio::OneByOne), + EditImageSize::Wide => Some(NanoBananaMultiFunctionImageGenAspectRatio::SixteenByNine), + EditImageSize::Tall => Some(NanoBananaMultiFunctionImageGenAspectRatio::NineBySixteen), + } + } + + None +} diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/artcraft/handle_artcraft_gpt_image_1p5_text_to_image.rs b/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/artcraft/handle_artcraft_gpt_image_1p5_text_to_image.rs index ba96501496..91d508b77a 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/artcraft/handle_artcraft_gpt_image_1p5_text_to_image.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/artcraft/handle_artcraft_gpt_image_1p5_text_to_image.rs @@ -1,5 +1,6 @@ +use crate::core::api_adapters::aspect_ratio::convert::aspect_ratio_to_artcraft_gpt_image_1p5::aspect_ratio_to_artcraft_gpt_image_1p5; use crate::core::commands::enqueue::generate_error::GenerateError; -use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::{EditImageQuality, EditImageSize}; +use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::{EditImageQuality, EditImageSize, EnqueueEditImageCommand}; use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess; use crate::core::commands::enqueue::text_to_image::enqueue_text_to_image_command::{EnqueueTextToImageRequest, TextToImageSize}; use crate::core::events::basic_sendable_event_trait::BasicSendableEvent; @@ -37,13 +38,7 @@ pub async fn handle_artcraft_gpt_image_1p5_text_to_image( let uuid_idempotency_token = generate_random_uuid(); - let image_size = request.aspect_ratio - .map(|size| match size { - TextToImageSize::Auto => GptImage1p5MultiFunctionImageGenSize::Square, - TextToImageSize::Square => GptImage1p5MultiFunctionImageGenSize::Square, - TextToImageSize::Tall => GptImage1p5MultiFunctionImageGenSize::Tall, - TextToImageSize::Wide => GptImage1p5MultiFunctionImageGenSize::Wide, - }); + let aspect_ratio = get_aspect_ratio(request); let num_images = match request.number_images { None => None, @@ -65,7 +60,7 @@ pub async fn handle_artcraft_gpt_image_1p5_text_to_image( let request = GptImage1p5MultiFunctionImageGenRequest { uuid_idempotency_token, prompt: request.prompt.clone(), - image_size, + image_size: aspect_ratio, num_images, // Not provided for text-to-image image_media_tokens: None, @@ -102,3 +97,23 @@ pub async fn handle_artcraft_gpt_image_1p5_text_to_image( task_type: TaskType::ImageGeneration, }) } + +fn get_aspect_ratio(request: &EnqueueTextToImageRequest) -> Option { + if let Some(common_aspect_ratio) = request.common_aspect_ratio { + // Handle modern aspect ratio + let aspect = aspect_ratio_to_artcraft_gpt_image_1p5(common_aspect_ratio); + return Some(aspect); + } + + if let Some(aspect_ratio) = request.aspect_ratio { + // Handle deprecated aspect ratio + return match aspect_ratio { + TextToImageSize::Auto => Some(GptImage1p5MultiFunctionImageGenSize::Square), + TextToImageSize::Square => Some(GptImage1p5MultiFunctionImageGenSize::Square), + TextToImageSize::Wide => Some(GptImage1p5MultiFunctionImageGenSize::Wide), + TextToImageSize::Tall => Some(GptImage1p5MultiFunctionImageGenSize::Tall), + } + } + + None +} diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/artcraft/handle_artcraft_nano_banana_text_to_image.rs b/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/artcraft/handle_artcraft_nano_banana_text_to_image.rs index 8541f684d4..c1a15d2826 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/artcraft/handle_artcraft_nano_banana_text_to_image.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/artcraft/handle_artcraft_nano_banana_text_to_image.rs @@ -1,3 +1,4 @@ +use crate::core::api_adapters::aspect_ratio::convert::aspect_ratio_to_artcraft_nano_banana::aspect_ratio_to_artcraft_nano_banana; use crate::core::commands::enqueue::generate_error::{GenerateError, MissingCredentialsReason, ProviderFailureReason}; use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::{EditImageQuality, EditImageSize}; use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess; @@ -13,17 +14,15 @@ use crate::services::sora::state::sora_task_queue::SoraTaskQueue; use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager; use artcraft_api_defs::generate::image::edit::gemini_25_flash_edit_image::{Gemini25FlashEditImageNumImages, Gemini25FlashEditImageRequest}; use artcraft_api_defs::generate::image::generate_gpt_image_1_text_to_image::{GenerateGptImage1TextToImageImageQuality, GenerateGptImage1TextToImageImageSize, GenerateGptImage1TextToImageNumImages, GenerateGptImage1TextToImageRequest}; +use artcraft_api_defs::generate::image::multi_function::nano_banana_multi_function_image_gen::{NanoBananaMultiFunctionImageGenAspectRatio, NanoBananaMultiFunctionImageGenNumImages, NanoBananaMultiFunctionImageGenRequest}; +use artcraft_api_defs::generate::image::multi_function::nano_banana_pro_multi_function_image_gen::NanoBananaProMultiFunctionImageGenAspectRatio; use enums::common::generation_provider::GenerationProvider; use enums::tauri::tasks::task_type::TaskType; use idempotency::uuid::generate_random_uuid; use log::{error, info}; -use openai_sora_client::recipes::maybe_upgrade_or_renew_session::maybe_upgrade_or_renew_session; -use openai_sora_client::recipes::simple_image_gen_with_session_auto_renew::{simple_image_gen_with_session_auto_renew, SimpleImageGenAutoRenewRequest}; -use openai_sora_client::requests::image_gen::common::{ImageSize, NumImages}; use std::time::Duration; -use storyteller_client::endpoints::generate::image::edit::gemini_25_flash_edit_image::gemini_25_flash_edit_image; -use storyteller_client::endpoints::generate::image::edit::gpt_image_1_edit_image::gpt_image_1_edit_image; -use storyteller_client::endpoints::generate::image::generate_gpt_image_1_text_to_image::generate_gpt_image_1_text_to_image; +use storyteller_client::endpoints::generate::image::multi_function::nano_banana_multi_function_image_gen_image::nano_banana_multi_function_image_gen; +use storyteller_client::endpoints::generate::image::multi_function::nano_banana_pro_multi_function_image_gen_image::nano_banana_pro_multi_function_image_gen; use tauri::AppHandle; pub async fn handle_artcraft_nano_banana_text_to_image( @@ -43,24 +42,26 @@ pub async fn handle_artcraft_nano_banana_text_to_image( let uuid_idempotency_token = generate_random_uuid(); - let num_images = match request.number_images{ + let num_images = match request.number_images { None => None, - Some(1) => Some(Gemini25FlashEditImageNumImages::One), - Some(2) => Some(Gemini25FlashEditImageNumImages::Two), - Some(3) => Some(Gemini25FlashEditImageNumImages::Three), - Some(4) => Some(Gemini25FlashEditImageNumImages::Four), - _ => Some(Gemini25FlashEditImageNumImages::One), // Default to one image if invalid number + Some(1) => Some(NanoBananaMultiFunctionImageGenNumImages::One), + Some(2) => Some(NanoBananaMultiFunctionImageGenNumImages::Two), + Some(3) => Some(NanoBananaMultiFunctionImageGenNumImages::Three), + Some(4) => Some(NanoBananaMultiFunctionImageGenNumImages::Four), + _ => Some(NanoBananaMultiFunctionImageGenNumImages::One), // Default to one image if invalid number }; - let request = Gemini25FlashEditImageRequest { + let aspect_ratio = get_aspect_ratio(request); + + let request = NanoBananaMultiFunctionImageGenRequest { uuid_idempotency_token, prompt: request.prompt.clone(), image_media_tokens: request.image_media_tokens.clone(), num_images, - image_quality: None, + aspect_ratio, }; - let result = gemini_25_flash_edit_image( + let result = nano_banana_multi_function_image_gen( &app_env_configs.storyteller_host, Some(&creds), request, @@ -81,8 +82,28 @@ pub async fn handle_artcraft_nano_banana_text_to_image( Ok(TaskEnqueueSuccess { provider: GenerationProvider::Artcraft, - model: Some(GenerationModel::GptImage1), + model: Some(GenerationModel::NanoBanana), provider_job_id: Some(job_id.to_string()), task_type: TaskType::ImageGeneration, }) } + +fn get_aspect_ratio(request: &EnqueueTextToImageRequest) -> Option { + if let Some(common_aspect_ratio) = request.common_aspect_ratio { + // Handle modern aspect ratio + let aspect = aspect_ratio_to_artcraft_nano_banana(common_aspect_ratio); + return Some(aspect); + } + + if let Some(aspect_ratio) = request.aspect_ratio { + // Handle deprecated aspect ratio + return match aspect_ratio { + TextToImageSize::Auto => Some(NanoBananaMultiFunctionImageGenAspectRatio::Auto), + TextToImageSize::Square => Some(NanoBananaMultiFunctionImageGenAspectRatio::OneByOne), + TextToImageSize::Wide => Some(NanoBananaMultiFunctionImageGenAspectRatio::SixteenByNine), + TextToImageSize::Tall => Some(NanoBananaMultiFunctionImageGenAspectRatio::NineBySixteen), + } + } + + None +} diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/grok/handle_grok.rs b/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/grok/handle_grok.rs index 249f78aeec..dee51f43cd 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/grok/handle_grok.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/grok/handle_grok.rs @@ -1,3 +1,4 @@ +use crate::core::api_adapters::aspect_ratio::convert::aspect_ratio_to_grok_image::aspect_ratio_to_grok_image; use crate::core::commands::enqueue::generate_error::GenerateError; use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess; use crate::core::commands::enqueue::text_to_image::enqueue_text_to_image_command::{EnqueueTextToImageRequest, TextToImageSize}; @@ -54,18 +55,12 @@ pub async fn handle_grok( .map(|prompt| prompt.trim().to_string()) .unwrap_or_else(|| "".to_string()); - let aspect_ratio = match request.aspect_ratio { - None => ClientMessageAspectRatio::Square, - Some(TextToImageSize::Auto) => ClientMessageAspectRatio::Square, - Some(TextToImageSize::Square) => ClientMessageAspectRatio::Square, - Some(TextToImageSize::Wide) => ClientMessageAspectRatio::WideThreeByTwo, - Some(TextToImageSize::Tall) => ClientMessageAspectRatio::TallTwoByThree, - }; + let aspect_ratio = get_aspect_ratio(request); grok_image_prompt_queue.enqueue(PromptItem { task_id: job_id.clone(), prompt, - aspect_ratio, + aspect_ratio: aspect_ratio.unwrap_or(ClientMessageAspectRatio::Square), })?; Ok(TaskEnqueueSuccess { @@ -76,73 +71,22 @@ pub async fn handle_grok( }) } -// TODO: -// fn handle_midjourney_errors( -// app: &AppHandle, -// maybe_errors: Option> -// ) -> Result { -// if let Some(errors) = maybe_errors { -// if !errors.is_empty() { -// let messages: Vec = errors.iter() -// .map(|e| format!("{:?}", e)) -// .collect(); -// -// let combined_message = messages.join("; "); -// -// let event = FlashUserInputErrorEvent { -// message: format!("Midjourney Error: {}", combined_message), -// }; -// -// if let Err(err) = event.send(&app) { -// error!("Failed to send FlashUserInputErrorEvent: {:?}", err); // Fail open -// } -// } -// } -// -// Err(GenerateError::ProviderFailure(ProviderFailureReason::MidjourneyJobEnqueueFailed)) -// } +fn get_aspect_ratio(request: &EnqueueTextToImageRequest) -> Option { + if let Some(common_aspect_ratio) = request.common_aspect_ratio { + // Handle modern aspect ratio + let aspect = aspect_ratio_to_grok_image(common_aspect_ratio); + return Some(aspect); + } -//async fn get_websocket(app: &AppHandle, creds_manager: &GrokCredentialManager, websocket_manager: &GrokWebsocketManager) -> Result { -// let maybe_websocket = websocket_manager.grab_websocket()?; -// -// let websocket = match maybe_websocket { -// Some(websocket) => websocket, -// None => { -// let cookies = match creds_manager.maybe_copy_cookie_store() { -// Ok(Some(cookies)) => cookies.to_cookie_string(), -// Ok(None) => { -// error!("Grok credentials not found."); -// ShowProviderLoginModalEvent::send_for_provider(GenerationProvider::Grok, &app); -// return Err(GenerateError::needs_grok_credentials()); -// } -// Err(err) => { -// error!("Error reading Midjourney credentials: {:?}", err); -// ShowProviderLoginModalEvent::send_for_provider(GenerationProvider::Grok, &app); -// return Err(GenerateError::needs_grok_credentials()); -// }, -// }; -// -// info!("Creating Grok websocket..."); -// -// let create_websocket_result = create_listen_websocket(CreateListenWebsocketArgs { -// cookies: &cookies, -// }).await; -// -// let websocket = match create_websocket_result { -// Ok(websocket) => GrokWebsocket::new(websocket), -// Err(err) => { -// error!("Error creating websocket: {:?}", err); -// ShowProviderLoginModalEvent::send_for_provider(GenerationProvider::Grok, &app); -// return Err(GenerateError::needs_grok_credentials()); -// } -// }; -// -// info!("Setting managed Grok websocket..."); -// websocket_manager.set_websocket(websocket.clone())?; -// -// websocket -// } -// }; -// -// Ok(websocket) -//} + if let Some(aspect_ratio) = request.aspect_ratio { + // Handle deprecated aspect ratio + return match aspect_ratio { + TextToImageSize::Auto => Some(ClientMessageAspectRatio::Square), + TextToImageSize::Square => Some(ClientMessageAspectRatio::Square), + TextToImageSize::Wide => Some(ClientMessageAspectRatio::WideThreeByTwo), + TextToImageSize::Tall => Some(ClientMessageAspectRatio::TallTwoByThree), + } + } + + None +} diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/sora/handle_gpt_image_1_sora_text_to_image.rs b/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/sora/handle_gpt_image_1_sora_text_to_image.rs index 42b16a584c..82a1abedbf 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/sora/handle_gpt_image_1_sora_text_to_image.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/sora/handle_gpt_image_1_sora_text_to_image.rs @@ -1,3 +1,5 @@ +use crate::core::api_adapters::aspect_ratio::convert::aspect_ratio_to_grok_image::aspect_ratio_to_grok_image; +use crate::core::api_adapters::aspect_ratio::convert::aspect_ratio_to_sora_native_gpt_image_1::aspect_ratio_to_sora_native_gpt_image_1; use crate::core::commands::enqueue::generate_error::GenerateError; use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess; use crate::core::commands::enqueue::text_to_image::enqueue_text_to_image_command::{EnqueueTextToImageRequest, TextToImageSize}; @@ -10,6 +12,7 @@ use crate::services::sora::state::sora_task_queue::SoraTaskQueue; use enums::common::generation_provider::GenerationProvider; use enums::tauri::tasks::task_type::TaskType; use errors::AnyhowResult; +use grok_client::requests::image_websocket::messages::websocket_client_message::ClientMessageAspectRatio; use log::{error, info}; use openai_sora_client::recipes::maybe_upgrade_or_renew_session::maybe_upgrade_or_renew_session; use openai_sora_client::recipes::simple_image_gen_with_session_auto_renew::{simple_image_gen_with_session_auto_renew, SimpleImageGenAutoRenewRequest}; @@ -54,13 +57,8 @@ pub async fn handle_gpt_image_1_sora_text_to_image( let prompt = request.prompt.as_deref().unwrap_or(""); - let size = match request.aspect_ratio { - None => ImageSize::Square, - Some(TextToImageSize::Auto) => ImageSize::Square, - Some(TextToImageSize::Square) => ImageSize::Square, - Some(TextToImageSize::Tall) => ImageSize::Tall, - Some(TextToImageSize::Wide) => ImageSize::Wide, - }; + let aspect_ratio = get_aspect_ratio(request) + .unwrap_or(ImageSize::Square); let num_images = match request.number_images { None => NumImages::One, @@ -75,7 +73,7 @@ pub async fn handle_gpt_image_1_sora_text_to_image( simple_image_gen_with_session_auto_renew(SimpleImageGenAutoRenewRequest { prompt: prompt.to_string(), num_images, - image_size: size, + image_size: aspect_ratio, credentials: &creds, request_timeout: Some(SORA_SIMPLE_IMAGE_GEN_TIMEOUT), }).await; @@ -115,3 +113,23 @@ pub async fn handle_gpt_image_1_sora_text_to_image( provider_job_id: Some(response.task_id.to_string()), }) } + +fn get_aspect_ratio(request: &EnqueueTextToImageRequest) -> Option { + if let Some(common_aspect_ratio) = request.common_aspect_ratio { + // Handle modern aspect ratio + let aspect = aspect_ratio_to_sora_native_gpt_image_1(common_aspect_ratio); + return Some(aspect); + } + + if let Some(aspect_ratio) = request.aspect_ratio { + // Handle deprecated aspect ratio + return match aspect_ratio { + TextToImageSize::Auto => Some(ImageSize::Square), + TextToImageSize::Square => Some(ImageSize::Square), + TextToImageSize::Wide => Some(ImageSize::Wide), + TextToImageSize::Tall => Some(ImageSize::Tall), + } + } + + None +} diff --git a/frontend/apps/editor3d/app/src/pages/PageEdit/PageEdit.tsx b/frontend/apps/editor3d/app/src/pages/PageEdit/PageEdit.tsx index 57f8aa22f6..f0e36d3966 100644 --- a/frontend/apps/editor3d/app/src/pages/PageEdit/PageEdit.tsx +++ b/frontend/apps/editor3d/app/src/pages/PageEdit/PageEdit.tsx @@ -29,7 +29,7 @@ import { //ProviderSelector, //PROVIDER_LOOKUP_BY_PAGE, } from "@storyteller/ui-model-selector"; -import { ImageModel } from "@storyteller/model-list"; +import { CommonAspectRatio, ImageModel } from "@storyteller/model-list"; import { HistoryStack, ImageBundle } from "./HistoryStack"; import { TutorialModalButton } from "@storyteller/ui-tutorial-modal"; import { GenerationProvider } from "@storyteller/api-enums"; @@ -63,6 +63,9 @@ const PageEdit = () => { >([]); const [generationCount, setGenerationCount] = useState(1); + // New aspect ratio state will begin to phase out old aspect ratio + const [commonAspectRatio, setCommonAspectRatio] = useState(undefined); + // Use the Zustand store const store = useEditStore(); const baseImageUrl = store.baseImageInfo?.url; @@ -382,6 +385,7 @@ const PageEdit = () => { let result; if (selectedImageModel?.editingIsInpainting) { + // CASE 1 - INPAINTING (Only a few models do this!) const arrayBuffer = await getMaskArrayBuffer(); let request : EnqueueImageInpaintRequest = { model: selectedImageModel, @@ -395,8 +399,12 @@ const PageEdit = () => { if (!!options?.selectedProvider) { request.provider = options.selectedProvider; } + //if (selectedImageModel?.supportsNewAspectRatio()) { + // request.common_aspect_ratio = commonAspectRatio; + //} result = await EnqueueImageInpaint(request); } else if (selectedImageModel?.isNanoBananaModel()) { + // CASE 2 - NANO BANANA const compositeFile = await getCompositeCanvasFile(); if (!compositeFile) { console.error("Failed to create composite canvas"); @@ -427,8 +435,12 @@ const PageEdit = () => { if (!!options?.selectedProvider) { request.provider = options.selectedProvider; } + if (selectedImageModel?.supportsNewAspectRatio()) { + request.common_aspect_ratio = commonAspectRatio; + } result = await EnqueueEditImage(request); } else { + // CASE 3 - DEFAULT const imgs = options?.images || []; let request : EnqueueEditImageRequest = { model: selectedImageModel, @@ -449,6 +461,9 @@ const PageEdit = () => { if (!!options?.selectedProvider) { request.provider = options.selectedProvider; } + if (selectedImageModel?.supportsNewAspectRatio()) { + request.common_aspect_ratio = commonAspectRatio; + } result = await EnqueueEditImage(request); } @@ -631,6 +646,8 @@ const PageEdit = () => { { store.setActiveTool(mode as ActiveEditTool); }} diff --git a/frontend/libs/components/promptbox/src/lib/PromptBox3D.tsx b/frontend/libs/components/promptbox/src/lib/PromptBox3D.tsx index 9257caeafd..8722323cc1 100644 --- a/frontend/libs/components/promptbox/src/lib/PromptBox3D.tsx +++ b/frontend/libs/components/promptbox/src/lib/PromptBox3D.tsx @@ -46,9 +46,10 @@ import { } from "@storyteller/tauri-api"; import { usePrompt3DStore } from "./promptStore"; import { gtagEvent } from "@storyteller/google-analytics"; -import { ImageModel } from "@storyteller/model-list"; +import { CommonAspectRatio, ImageModel } from "@storyteller/model-list"; import { GenerationProvider } from "@storyteller/api-enums"; import { ImagePromptRow } from "./ImagePromptRow"; +import { AspectRatioPicker } from "./common/AspectRatioPicker"; interface PromptBox3DProps { cameras: Signal; @@ -156,6 +157,9 @@ export const PromptBox3D = ({ (selectedImageModel?.canUseImagePrompt && referenceImages.length > 0) || false; + // New aspect ratio state will begin to phase out old aspect ratio + const [commonAspectRatio, setCommonAspectRatio] = useState(undefined); + useEffect(() => { if (textareaRef.current) { textareaRef.current.style.height = "auto"; @@ -384,6 +388,10 @@ export const PromptBox3D = ({ if (!!selectedProvider) { request.provider = selectedProvider; } + + if (selectedImageModel?.supportsNewAspectRatio()) { + request.common_aspect_ratio = commonAspectRatio; + } const generateResponse = await EnqueueEditImage(request); @@ -594,7 +602,14 @@ export const PromptBox3D = ({
- {selectedImageModel?.canChangeAspectRatio && ( + {selectedImageModel?.supportsNewAspectRatio() && ( + + )} + {selectedImageModel?.canChangeAspectRatio && !selectedImageModel?.supportsNewAspectRatio() && ( void; @@ -48,6 +49,8 @@ export interface PromptBoxEditProps { selectedImageModel?: ImageModel; selectedProvider?: GenerationProvider; generationCount?: number; + commonAspectRatio?: CommonAspectRatio; + setCommonAspectRatio: (aspectRatio: CommonAspectRatio) => void; onGenerationCountChange?: (count: number) => void; supportsMaskedInpainting?: boolean; isEnqueueing?: boolean; @@ -67,6 +70,8 @@ export const PromptBoxEdit = ({ selectedImageModel, selectedProvider, generationCount: generationCountProp, + commonAspectRatio, + setCommonAspectRatio, onGenerationCountChange, supportsMaskedInpainting = false, uploadImage, @@ -443,7 +448,14 @@ export const PromptBoxEdit = ({
- {selectedImageModel?.canChangeAspectRatio && ( + {selectedImageModel?.supportsNewAspectRatio() && ( + + )} + {selectedImageModel?.canChangeAspectRatio && !selectedImageModel?.supportsNewAspectRatio() && ( case CommonAspectRatio.Square: return faSquare; + case CommonAspectRatio.Wide: case CommonAspectRatio.WideFiveByFour: case CommonAspectRatio.WideFourByThree: case CommonAspectRatio.WideThreeByTwo: @@ -101,6 +102,7 @@ const getAspectRatioIcon = (aspectRatio: CommonAspectRatio) : IconDefinition => return faRectangleWide; // Tall + case CommonAspectRatio.Tall: case CommonAspectRatio.TallFourByFive: case CommonAspectRatio.TallThreeByFour: case CommonAspectRatio.TallTwoByThree: @@ -123,27 +125,34 @@ const getAspectRatioTextLabel = (aspectRatio: CommonAspectRatio) : string => { // Wide case CommonAspectRatio.WideFiveByFour: - return "5:4"; + return "5:4 (Wide)"; case CommonAspectRatio.WideFourByThree: - return "4:3"; + return "4:3 (Wide)"; case CommonAspectRatio.WideThreeByTwo: - return "3:2"; + return "3:2 (Wide)"; case CommonAspectRatio.WideSixteenByNine: - return "16:9"; + return "16:9 (Wide)"; case CommonAspectRatio.WideTwentyOneByNine: - return "21:9"; + return "21:9 (Wide)"; // Tall case CommonAspectRatio.TallFourByFive: - return "4:5"; + return "4:5 (Tall)"; case CommonAspectRatio.TallThreeByFour: - return "3:4"; + return "3:4 (Tall)"; case CommonAspectRatio.TallTwoByThree: - return "2:3"; + return "2:3 (Tall)"; case CommonAspectRatio.TallNineBySixteen: - return "9:16"; + return "9:16 (Tall)"; case CommonAspectRatio.TallNineByTwentyOne: - return "9:21"; + return "9:21 (Tall)"; + + + // Semantic cases + case CommonAspectRatio.Wide: + return "Wide"; + case CommonAspectRatio.Tall: + return "Tall"; default: return "Square"; // Fail open-ish @@ -155,16 +164,16 @@ const popOverLabelToAspectRatio = (label: string, model: ImageModel): CommonAspe switch (label) { case "Auto": return CommonAspectRatio.Auto; case "Square": return CommonAspectRatio.Square; - case "5:4": return CommonAspectRatio.WideFiveByFour; - case "4:3": return CommonAspectRatio.WideFourByThree; - case "3:2": return CommonAspectRatio.WideThreeByTwo; - case "16:9": return CommonAspectRatio.WideSixteenByNine; - case "21:9": return CommonAspectRatio.WideTwentyOneByNine; - case "4:5": return CommonAspectRatio.TallFourByFive; - case "3:4": return CommonAspectRatio.TallThreeByFour; - case "2:3": return CommonAspectRatio.TallTwoByThree; - case "9:16": return CommonAspectRatio.TallNineBySixteen; - case "9:21": return CommonAspectRatio.TallNineByTwentyOne; + case "5:4 (Wide)": return CommonAspectRatio.WideFiveByFour; + case "4:3 (Wide)": return CommonAspectRatio.WideFourByThree; + case "3:2 (Wide)": return CommonAspectRatio.WideThreeByTwo; + case "16:9 (Wide)": return CommonAspectRatio.WideSixteenByNine; + case "21:9 (Wide)": return CommonAspectRatio.WideTwentyOneByNine; + case "4:5 (Tall)": return CommonAspectRatio.TallFourByFive; + case "3:4 (Tall)": return CommonAspectRatio.TallThreeByFour; + case "2:3 (Tall)": return CommonAspectRatio.TallTwoByThree; + case "9:16 (Tall)": return CommonAspectRatio.TallNineBySixteen; + case "9:21 (Tall)": return CommonAspectRatio.TallNineByTwentyOne; } // If we can't find it, return the model's default aspect ratio or Square as fallback return model.defaultAspectRatio || CommonAspectRatio.Square; diff --git a/frontend/libs/model-list/src/lib/classes/properties/CommonAspectRatio.ts b/frontend/libs/model-list/src/lib/classes/properties/CommonAspectRatio.ts index 901a0d4c05..e20af0dfa7 100644 --- a/frontend/libs/model-list/src/lib/classes/properties/CommonAspectRatio.ts +++ b/frontend/libs/model-list/src/lib/classes/properties/CommonAspectRatio.ts @@ -24,4 +24,9 @@ export enum CommonAspectRatio { TallTwoByThree = "tall_two_by_three", TallNineBySixteen = "tall_nine_by_sixteen", TallNineByTwentyOne = "tall_nine_by_twenty_one", + + // Imprecise semantic values (that probably are mapped to other meanings) + // Not appropriate for all models to use these values. + Wide = "wide", + Tall = "tall", } diff --git a/frontend/libs/model-list/src/lib/lists/ImageModels.ts b/frontend/libs/model-list/src/lib/lists/ImageModels.ts index fe8b75a7fb..b3cce7aaa1 100644 --- a/frontend/libs/model-list/src/lib/lists/ImageModels.ts +++ b/frontend/libs/model-list/src/lib/lists/ImageModels.ts @@ -25,6 +25,12 @@ export const IMAGE_MODELS: ImageModel[] = [ maxImagePromptCount: 6, progressBarTime: 10000, canChangeAspectRatio: true, + defaultAspectRatio: CommonAspectRatio.Square, + aspectRatios: [ + CommonAspectRatio.Square, + CommonAspectRatio.WideThreeByTwo, + CommonAspectRatio.TallTwoByThree, + ] }), new ImageModel({ id: "midjourney", @@ -110,24 +116,38 @@ export const IMAGE_MODELS: ImageModel[] = [ canChangeAspectRatio: true, }), new ImageModel({ - id: "gemini_25_flash", - tauriId: "gemini_25_flash", - fullName: "Gemini 2.5 Flash", + id: "nano_banana", + tauriId: "nano_banana", + fullName: "Nano Banana", category: "image", creator: ModelCreator.Google, selectorName: "Nano Banana", - selectorDescription: "Incredibly instructive editing", + selectorDescription: "Fast instructive editing", selectorBadges: ["25 sec."], maxGenerationCount: 4, defaultGenerationCount: 1, canEditImages: true, canUseImagePrompt: true, maxImagePromptCount: 6, - canTextToImage: false, + canTextToImage: true, tags: [ModelTag.InstructiveEdit], progressBarTime: 25000, // canChangeResolution: true, canChangeAspectRatio: true, + defaultAspectRatio: CommonAspectRatio.Square, + aspectRatios: [ + CommonAspectRatio.Auto, + CommonAspectRatio.Square, + CommonAspectRatio.WideFiveByFour, + CommonAspectRatio.WideFourByThree, + CommonAspectRatio.WideThreeByTwo, + CommonAspectRatio.WideSixteenByNine, + CommonAspectRatio.WideTwentyOneByNine, + CommonAspectRatio.TallFourByFive, + CommonAspectRatio.TallThreeByFour, + CommonAspectRatio.TallTwoByThree, + CommonAspectRatio.TallNineBySixteen, + ] }), new ImageModel({ id: "nano_banana_pro", @@ -136,7 +156,7 @@ export const IMAGE_MODELS: ImageModel[] = [ category: "image", creator: ModelCreator.Google, selectorName: "Nano Banana Pro", - selectorDescription: "Extremely powerful", + selectorDescription: "Powerful instructive editing", selectorBadges: ["30 sec."], maxGenerationCount: 4, defaultGenerationCount: 1, @@ -202,6 +222,12 @@ export const IMAGE_MODELS: ImageModel[] = [ maxImagePromptCount: 6, progressBarTime: 60000, canChangeAspectRatio: true, + defaultAspectRatio: CommonAspectRatio.Square, + aspectRatios: [ + CommonAspectRatio.Square, + CommonAspectRatio.Wide, + CommonAspectRatio.Tall, + ] }), new ImageModel({ id: "seedream_4", diff --git a/frontend/libs/tauri-api/src/lib/enqueue/EnqueueEditImage.ts b/frontend/libs/tauri-api/src/lib/enqueue/EnqueueEditImage.ts index 536c5a1c71..ec1af86ec2 100644 --- a/frontend/libs/tauri-api/src/lib/enqueue/EnqueueEditImage.ts +++ b/frontend/libs/tauri-api/src/lib/enqueue/EnqueueEditImage.ts @@ -1,6 +1,6 @@ import { invoke } from "@tauri-apps/api/core"; import { CommandResult } from "../common/CommandStatus"; -import { ImageModel } from "@storyteller/model-list"; +import { CommonAspectRatio, ImageModel } from "@storyteller/model-list"; import { GenerationProvider } from "@storyteller/api-enums"; export interface EnqueueEditImageRequest { @@ -29,8 +29,12 @@ export interface EnqueueEditImageRequest { /// Number of images to generate. image_count?: number; + // TODO: This is deprecated and will be phased out. aspect_ratio?: EnqueueEditImageSize; + // This is the new aspect ratio. + common_aspect_ratio?: CommonAspectRatio; + image_quality?: EnqueueEditImageQuality; /// Image resolution. Support: Nano Banana Pro @@ -55,6 +59,7 @@ interface RawEnqueueEditImageRequest { disable_system_prompt?: boolean; image_count?: number; aspect_ratio?: EnqueueEditImageSize; + common_aspect_ratio?: CommonAspectRatio; image_quality?: EnqueueEditImageQuality; frontend_caller?: string; frontend_subscriber_id?: string; @@ -147,6 +152,10 @@ export const EnqueueEditImage = async (request: EnqueueEditImageRequest) : Promi mutableRequest.aspect_ratio = request.aspect_ratio; } + if (!!request.common_aspect_ratio) { + mutableRequest.common_aspect_ratio = request.common_aspect_ratio; + } + if (!!request.image_count) { mutableRequest.image_count = request.image_count; }