diff --git a/crates/api_clients/artcraft/artcraft_api_defs/src/generate/image/mod.rs b/crates/api_clients/artcraft/artcraft_api_defs/src/generate/image/mod.rs index c236019482..5cd3af7174 100644 --- a/crates/api_clients/artcraft/artcraft_api_defs/src/generate/image/mod.rs +++ b/crates/api_clients/artcraft/artcraft_api_defs/src/generate/image/mod.rs @@ -6,4 +6,5 @@ pub mod generate_flux_pro_11_ultra_text_to_image; pub mod generate_gpt_image_1_text_to_image; pub mod inpaint; pub mod multi_function; -pub mod remove_image_background; \ No newline at end of file +pub mod remove_image_background; +pub mod text; \ No newline at end of file diff --git a/crates/api_clients/artcraft/artcraft_api_defs/src/generate/image/multi_function/mod.rs b/crates/api_clients/artcraft/artcraft_api_defs/src/generate/image/multi_function/mod.rs index b343838335..43e7b65fb6 100644 --- a/crates/api_clients/artcraft/artcraft_api_defs/src/generate/image/multi_function/mod.rs +++ b/crates/api_clients/artcraft/artcraft_api_defs/src/generate/image/multi_function/mod.rs @@ -1,4 +1,5 @@ pub mod bytedance_seedream_v4_multi_function_image_gen; pub mod bytedance_seedream_v4p5_multi_function_image_gen; pub mod gpt_image_1p5_multi_function_image_gen; +pub mod nano_banana_multi_function_image_gen; pub mod nano_banana_pro_multi_function_image_gen; \ No newline at end of file diff --git a/crates/api_clients/artcraft/artcraft_api_defs/src/generate/image/multi_function/nano_banana_multi_function_image_gen.rs b/crates/api_clients/artcraft/artcraft_api_defs/src/generate/image/multi_function/nano_banana_multi_function_image_gen.rs new file mode 100644 index 0000000000..bdd69110f7 --- /dev/null +++ b/crates/api_clients/artcraft/artcraft_api_defs/src/generate/image/multi_function/nano_banana_multi_function_image_gen.rs @@ -0,0 +1,62 @@ +use serde_derive::{Deserialize, Serialize}; +use tokens::tokens::generic_inference_jobs::InferenceJobToken; +use tokens::tokens::media_files::MediaFileToken; +use utoipa::ToSchema; + +pub const NANO_BANANA_MULTI_FUNCTION_IMAGE_GEN_PATH: &str = "/v1/generate/image/multi_function/nano_banana"; + +#[derive(Serialize, Deserialize, ToSchema, Debug)] +pub struct NanoBananaMultiFunctionImageGenRequest { + /// Idempotency token to prevent duplicate requests. + pub uuid_idempotency_token: String, + + /// Text prompt to generate the image from. + pub prompt: Option, + + /// Image media tokens to include in the editing context. + /// If present, we're doing image editing (image-to-image / image-editing) + /// If absent, we're doing image generation (text-to-image) + pub image_media_tokens: Option>, + + /// Number of images to generate. Default is one. + pub num_images: Option, + + /// Aspect ratio of the images to generate. Default is "1:1" + pub aspect_ratio: Option, +} + +#[derive(Copy, Clone, Debug, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum NanoBananaMultiFunctionImageGenNumImages { + One, // Default + Two, + Three, + Four, +} + +#[derive(Copy, Clone, Debug, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum NanoBananaMultiFunctionImageGenAspectRatio { + // Auto (only for image editing) + Auto, // Default for image editing. + // Square + OneByOne, // Default for text-to-image + // Wide + FiveByFour, + FourByThree, + ThreeByTwo, + SixteenByNine, + TwentyOneByNine, + // Tall + FourByFive, + ThreeByFour, + TwoByThree, + NineBySixteen, // NB: No NineByTwentyOne ? +} + + +#[derive(Serialize, Deserialize, ToSchema)] +pub struct NanoBananaMultiFunctionImageGenResponse { + pub success: bool, + pub inference_job_token: InferenceJobToken, +} diff --git a/crates/api_clients/artcraft/artcraft_api_defs/src/generate/image/multi_function/nano_banana_pro_multi_function_image_gen.rs b/crates/api_clients/artcraft/artcraft_api_defs/src/generate/image/multi_function/nano_banana_pro_multi_function_image_gen.rs index 31a8e52b67..ce60ef5632 100644 --- a/crates/api_clients/artcraft/artcraft_api_defs/src/generate/image/multi_function/nano_banana_pro_multi_function_image_gen.rs +++ b/crates/api_clients/artcraft/artcraft_api_defs/src/generate/image/multi_function/nano_banana_pro_multi_function_image_gen.rs @@ -5,7 +5,7 @@ use utoipa::ToSchema; pub const NANO_BANANA_PRO_MULTI_FUNCTION_IMAGE_GEN_PATH: &str = "/v1/generate/image/multi_function/nano_banana_pro"; -#[derive(Serialize, Deserialize, ToSchema)] +#[derive(Serialize, Deserialize, ToSchema, Debug)] pub struct NanoBananaProMultiFunctionImageGenRequest { /// Idempotency token to prevent duplicate requests. pub uuid_idempotency_token: String, @@ -48,8 +48,10 @@ pub enum NanoBananaProMultiFunctionImageGenImageResolution { #[derive(Copy, Clone, Debug, Serialize, Deserialize, ToSchema)] #[serde(rename_all = "snake_case")] pub enum NanoBananaProMultiFunctionImageGenAspectRatio { + // Auto (only for image editing) + Auto, // Default for image editing. // Square - OneByOne, // Default + OneByOne, // Default for text-to-image // Wide FiveByFour, FourByThree, diff --git a/crates/api_clients/artcraft/artcraft_api_defs/src/generate/image/text/gemini_25_flash_text_to_image.rs b/crates/api_clients/artcraft/artcraft_api_defs/src/generate/image/text/gemini_25_flash_text_to_image.rs new file mode 100644 index 0000000000..a343ef3081 --- /dev/null +++ b/crates/api_clients/artcraft/artcraft_api_defs/src/generate/image/text/gemini_25_flash_text_to_image.rs @@ -0,0 +1,50 @@ +use serde_derive::{Deserialize, Serialize}; +use tokens::tokens::generic_inference_jobs::InferenceJobToken; +use tokens::tokens::media_files::MediaFileToken; +use utoipa::ToSchema; + +pub const GEMINI_25_FLASH_EDIT_IMAGE_PATH: &str = "/v1/generate/image/gemini_25_flash"; + +#[derive(Serialize, Deserialize, ToSchema)] +pub struct Gemini25FlashTextToImageImageRequest { + /// Idempotency token to prevent duplicate requests. + pub uuid_idempotency_token: String, + + /// Text prompt to generate the image from. + pub prompt: Option, + + // TODO: OneOf for image_urls or image_media_tokens + /// Image media tokens to include in the editing context. + pub image_media_tokens: Option>, + + /// Number of images to generate. Default is one. + pub num_images: Option, + + /// Quality of the image to generate. Default is High. + pub image_quality: Option, +} + +#[derive(Copy, Clone, Debug, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum Gemini25FlashTextToImageImageNumImages { + One, // Default + Two, + Three, + Four, +} + +#[derive(Copy, Clone, Debug, Serialize, Deserialize, ToSchema)] +#[serde(rename_all = "snake_case")] +pub enum Gemini25FlashTextToImageImageImageQuality { + Auto, + Low, + Medium, + High, // Default +} + + +#[derive(Serialize, Deserialize, ToSchema)] +pub struct Gemini25FlashTextToImageImageResponse { + pub success: bool, + pub inference_job_token: InferenceJobToken, +} diff --git a/crates/api_clients/artcraft/artcraft_api_defs/src/generate/image/text/mod.rs b/crates/api_clients/artcraft/artcraft_api_defs/src/generate/image/text/mod.rs new file mode 100644 index 0000000000..61f401bc12 --- /dev/null +++ b/crates/api_clients/artcraft/artcraft_api_defs/src/generate/image/text/mod.rs @@ -0,0 +1 @@ +pub mod gemini_25_flash_text_to_image; \ No newline at end of file diff --git a/crates/api_clients/fal_client/src/requests/webhook/image/edit/enqueue_gemini_25_flash_edit_webhook.rs b/crates/api_clients/fal_client/src/requests/webhook/image/edit/enqueue_gemini_25_flash_edit_webhook.rs index f31e685162..54e14d6ca6 100644 --- a/crates/api_clients/fal_client/src/requests/webhook/image/edit/enqueue_gemini_25_flash_edit_webhook.rs +++ b/crates/api_clients/fal_client/src/requests/webhook/image/edit/enqueue_gemini_25_flash_edit_webhook.rs @@ -11,6 +11,9 @@ pub struct Gemini25FlashEditArgs<'a, R: IntoUrl> { pub prompt: &'a str, pub image_urls: Vec, pub num_images: Gemini25FlashEditNumImages, + + // Optional + pub aspect_ratio: Option, // Fulfillment pub webhook_url: R, @@ -25,6 +28,26 @@ pub enum Gemini25FlashEditNumImages { Four, } +/// auto, 21:9, 16:9, 3:2, 4:3, 5:4, 1:1, 4:5, 3:4, 2:3, 9:16 +/// Default is "auto" +#[derive(Copy, Clone, Debug)] +pub enum Gemini25FlashEditAspectRatio { + // Automatic (default) + Auto, + // Square + OneByOne, + // Wide + FiveByFour, + FourByThree, + ThreeByTwo, + SixteenByNine, + TwentyOneByNine, + // Tall + FourByFive, + ThreeByFour, + TwoByThree, + NineBySixteen, // NB: No NineByTwentyOne ? +} pub async fn enqueue_gemini_25_flash_edit_webhook( args: Gemini25FlashEditArgs<'_, R> @@ -37,9 +60,30 @@ pub async fn enqueue_gemini_25_flash_edit_webhook( Gemini25FlashEditNumImages::Four => 4, }; + let aspect_ratio = args.aspect_ratio + .map(|aspect_ratio| match aspect_ratio { + // Auto + Gemini25FlashEditAspectRatio::Auto => "auto", + // Square + Gemini25FlashEditAspectRatio::OneByOne => "1:1", + // Wide + Gemini25FlashEditAspectRatio::FiveByFour => "5:4", + Gemini25FlashEditAspectRatio::FourByThree => "4:3", + Gemini25FlashEditAspectRatio::ThreeByTwo => "3:2", + Gemini25FlashEditAspectRatio::SixteenByNine => "16:9", + Gemini25FlashEditAspectRatio::TwentyOneByNine => "21:9", + // Tall + Gemini25FlashEditAspectRatio::FourByFive => "4:5", + Gemini25FlashEditAspectRatio::ThreeByFour => "3:4", + Gemini25FlashEditAspectRatio::TwoByThree => "2:3", + Gemini25FlashEditAspectRatio::NineBySixteen => "9:16", + }) + .map(|aspect_ratio| aspect_ratio.to_string()); + let request = Gemini25FlashImageEditInput { prompt: args.prompt.to_string(), image_urls: args.image_urls, + aspect_ratio, // Constants num_images: Some(num_images), output_format: Some("png".to_string()), @@ -75,6 +119,7 @@ mod tests { TREX_SKELETON_IMAGE_URL.to_string(), ], num_images: Gemini25FlashEditNumImages::Two, + aspect_ratio: None, prompt: "add the ghost to the image of the t-rex skeleton, make it look spooky but friendly", api_key: &api_key, webhook_url: "https://example.com/webhook", diff --git a/crates/api_clients/fal_client/src/requests/webhook/image/edit/enqueue_nano_banana_pro_edit_image_webhook.rs b/crates/api_clients/fal_client/src/requests/webhook/image/edit/enqueue_nano_banana_pro_edit_image_webhook.rs index 2ee07fadc3..1f7a64c28b 100644 --- a/crates/api_clients/fal_client/src/requests/webhook/image/edit/enqueue_nano_banana_pro_edit_image_webhook.rs +++ b/crates/api_clients/fal_client/src/requests/webhook/image/edit/enqueue_nano_banana_pro_edit_image_webhook.rs @@ -30,15 +30,17 @@ pub enum EnqueueNanoBananaProEditImageNumImages { #[derive(Copy, Clone, Debug)] pub enum EnqueueNanoBananaProEditImageResolution { - OneK, + OneK, // Default value "1K" TwoK, FourK, } -/// 21:9, 16:9, 3:2, 4:3, 5:4, 1:1, 4:5, 3:4, 2:3, 9:16 -/// Default is "1:1" +/// auto, 21:9, 16:9, 3:2, 4:3, 5:4, 1:1, 4:5, 3:4, 2:3, 9:16 +/// Default is "auto" #[derive(Copy, Clone, Debug)] pub enum EnqueueNanoBananaProEditImageAspectRatio { + // Automatic (default) + Auto, // Square OneByOne, // Wide @@ -75,6 +77,8 @@ pub async fn enqueue_nano_banana_pro_image_edit_webhook( let aspect_ratio = args.aspect_ratio .map(|aspect_ratio| match aspect_ratio { + // Auto + EnqueueNanoBananaProEditImageAspectRatio::Auto => "auto", // Square EnqueueNanoBananaProEditImageAspectRatio::OneByOne => "1:1", // Wide diff --git a/crates/api_clients/fal_client/src/requests/webhook/image/text/enqueue_gemini_25_flash_text_to_image_webhook.rs b/crates/api_clients/fal_client/src/requests/webhook/image/text/enqueue_gemini_25_flash_text_to_image_webhook.rs new file mode 100644 index 0000000000..6ae117ec6c --- /dev/null +++ b/crates/api_clients/fal_client/src/requests/webhook/image/text/enqueue_gemini_25_flash_text_to_image_webhook.rs @@ -0,0 +1,121 @@ +use crate::creds::fal_api_key::FalApiKey; +use crate::error::classify_fal_error::classify_fal_error; +use crate::error::fal_error_plus::FalErrorPlus; +use fal::prelude::fal_ai::gemini_25_flash_image::text_to_image::gemini_25_flash_text_to_image; +use fal::prelude::fal_ai::gemini_25_flash_image::text_to_image::Gemini25FlashTextToImageInput; +use fal::webhook::WebhookResponse; +use reqwest::IntoUrl; + +pub struct Gemini25FlashTextToImageArgs<'a, R: IntoUrl> { + // Request required + pub prompt: &'a str, + pub num_images: Gemini25FlashTextToImageNumImages, + + // Optional + pub aspect_ratio: Option, + + // Fulfillment + pub webhook_url: R, + pub api_key: &'a FalApiKey, +} + +#[derive(Copy, Clone, Debug)] +pub enum Gemini25FlashTextToImageNumImages { + One, + Two, + Three, + Four, +} + +/// 21:9, 16:9, 3:2, 4:3, 5:4, 1:1, 4:5, 3:4, 2:3, 9:16 +/// Default is "1:1" +#[derive(Copy, Clone, Debug)] +pub enum Gemini25FlashTextToImageAspectRatio { + // Square + OneByOne, + // Wide + FiveByFour, + FourByThree, + ThreeByTwo, + SixteenByNine, + TwentyOneByNine, + // Tall + FourByFive, + ThreeByFour, + TwoByThree, + NineBySixteen, // NB: No NineByTwentyOne ? +} + +pub async fn enqueue_gemini_25_flash_text_to_image_webhook( + args: Gemini25FlashTextToImageArgs<'_, R> +) -> Result { + + let num_images = match args.num_images { + Gemini25FlashTextToImageNumImages::One => 1, + Gemini25FlashTextToImageNumImages::Two => 2, + Gemini25FlashTextToImageNumImages::Three => 3, + Gemini25FlashTextToImageNumImages::Four => 4, + }; + + let aspect_ratio = args.aspect_ratio + .map(|aspect_ratio| match aspect_ratio { + // Square + Gemini25FlashTextToImageAspectRatio::OneByOne => "1:1", + // Wide + Gemini25FlashTextToImageAspectRatio::FiveByFour => "5:4", + Gemini25FlashTextToImageAspectRatio::FourByThree => "4:3", + Gemini25FlashTextToImageAspectRatio::ThreeByTwo => "3:2", + Gemini25FlashTextToImageAspectRatio::SixteenByNine => "16:9", + Gemini25FlashTextToImageAspectRatio::TwentyOneByNine => "21:9", + // Tall + Gemini25FlashTextToImageAspectRatio::FourByFive => "4:5", + Gemini25FlashTextToImageAspectRatio::ThreeByFour => "3:4", + Gemini25FlashTextToImageAspectRatio::TwoByThree => "2:3", + Gemini25FlashTextToImageAspectRatio::NineBySixteen => "9:16", + }) + .map(|aspect_ratio| aspect_ratio.to_string()); + + let request = Gemini25FlashTextToImageInput { + prompt: args.prompt.to_string(), + aspect_ratio, + // Constants + num_images: Some(num_images), + output_format: Some("png".to_string()), + }; + + let result = gemini_25_flash_text_to_image(request) + .with_api_key(&args.api_key.0) + .queue_webhook(args.webhook_url) + .await; + + result.map_err(|err| classify_fal_error(err)) +} + +#[cfg(test)] +mod tests { + use crate::creds::fal_api_key::FalApiKey; + use crate::requests::webhook::image::text::enqueue_gemini_25_flash_text_to_image_webhook::{enqueue_gemini_25_flash_text_to_image_webhook, Gemini25FlashTextToImageArgs, Gemini25FlashTextToImageNumImages}; + use errors::AnyhowResult; + use std::fs::read_to_string; + + #[tokio::test] + #[ignore] + async fn test() -> AnyhowResult<()> { + // XXX: Don't commit secrets! + let secret = read_to_string("/Users/bt/Artcraft/credentials/fal_api_key.txt")?; + + let api_key = FalApiKey::from_str(&secret); + + let args = Gemini25FlashTextToImageArgs { + prompt: "a warrior on the field of battle, final fantasy style, lots of hungry raptors rushing the heroes", + num_images: Gemini25FlashTextToImageNumImages::Two, + aspect_ratio: None, + api_key: &api_key, + webhook_url: "https://example.com/webhook", + }; + + let result = enqueue_gemini_25_flash_text_to_image_webhook(args).await?; + + Ok(()) + } +} diff --git a/crates/api_clients/fal_client/src/requests/webhook/image/text/mod.rs b/crates/api_clients/fal_client/src/requests/webhook/image/text/mod.rs index b35e5a0ecb..5fa70ecc96 100644 --- a/crates/api_clients/fal_client/src/requests/webhook/image/text/mod.rs +++ b/crates/api_clients/fal_client/src/requests/webhook/image/text/mod.rs @@ -1,4 +1,5 @@ pub mod enqueue_bytedance_seedream_v4_text_to_image_webhook; pub mod enqueue_bytedance_seedream_v4p5_text_to_image_webhook; -pub mod enqueue_nano_banana_pro_text_to_image_webhook; -pub mod enqueue_gpt_image_1p5_text_to_image_webhook; \ No newline at end of file +pub mod enqueue_gemini_25_flash_text_to_image_webhook; +pub mod enqueue_gpt_image_1p5_text_to_image_webhook; +pub mod enqueue_nano_banana_pro_text_to_image_webhook; \ No newline at end of file diff --git a/crates/desktop/artcraft/src/core/api_adapters/aspect_ratio/common_aspect_ratio.rs b/crates/desktop/artcraft/src/core/api_adapters/aspect_ratio/common_aspect_ratio.rs new file mode 100644 index 0000000000..c61910bd1f --- /dev/null +++ b/crates/desktop/artcraft/src/core/api_adapters/aspect_ratio/common_aspect_ratio.rs @@ -0,0 +1,25 @@ +use serde_derive::{Deserialize, Serialize}; + +/// This is a comprehensive list of common aspect ratios. +/// Not every model will support every aspect ratio. +/// In the case a model doesn't support the aspect ratio, pick the nearest option. +#[derive(Copy, Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum CommonAspectRatio { + // Auto (eg. for image editing to use the source; used by Nano Banana Pro edit image, but not text-to-image) + Auto, + // Square + Square, + // Wide + WideFiveByFour, + WideFourByThree, + WideThreeByTwo, + WideSixteenByNine, + WideTwentyOneByNine, + // Tall + TallFourByFive, + TallThreeByFour, + TallTwoByThree, + TallNineBySixteen, + TallNineByTwentyOne, +} diff --git a/crates/desktop/artcraft/src/core/api_adapters/aspect_ratio/convert/aspect_ratio_to_artcraft_nano_banana_pro.rs b/crates/desktop/artcraft/src/core/api_adapters/aspect_ratio/convert/aspect_ratio_to_artcraft_nano_banana_pro.rs new file mode 100644 index 0000000000..a784aadabc --- /dev/null +++ b/crates/desktop/artcraft/src/core/api_adapters/aspect_ratio/convert/aspect_ratio_to_artcraft_nano_banana_pro.rs @@ -0,0 +1,23 @@ +use crate::core::api_adapters::aspect_ratio::common_aspect_ratio::CommonAspectRatio; +use artcraft_api_defs::generate::image::multi_function::nano_banana_pro_multi_function_image_gen::NanoBananaProMultiFunctionImageGenAspectRatio; + +pub fn aspect_ratio_to_artcraft_nano_banana_pro(aspect_ratio: CommonAspectRatio) -> NanoBananaProMultiFunctionImageGenAspectRatio { + match aspect_ratio { + // Exact + CommonAspectRatio::Square => NanoBananaProMultiFunctionImageGenAspectRatio::OneByOne, + CommonAspectRatio::WideFiveByFour => NanoBananaProMultiFunctionImageGenAspectRatio::FiveByFour, + CommonAspectRatio::WideFourByThree => NanoBananaProMultiFunctionImageGenAspectRatio::FourByThree, + CommonAspectRatio::WideThreeByTwo => NanoBananaProMultiFunctionImageGenAspectRatio::ThreeByTwo, + CommonAspectRatio::WideSixteenByNine => NanoBananaProMultiFunctionImageGenAspectRatio::SixteenByNine, + CommonAspectRatio::WideTwentyOneByNine => NanoBananaProMultiFunctionImageGenAspectRatio::TwentyOneByNine, + CommonAspectRatio::TallFourByFive => NanoBananaProMultiFunctionImageGenAspectRatio::FourByFive, + CommonAspectRatio::TallThreeByFour => NanoBananaProMultiFunctionImageGenAspectRatio::ThreeByFour, + CommonAspectRatio::TallTwoByThree => NanoBananaProMultiFunctionImageGenAspectRatio::TwoByThree, + CommonAspectRatio::TallNineBySixteen => NanoBananaProMultiFunctionImageGenAspectRatio::NineBySixteen, + // Special + // NB: this is only for image-to-image editing, not text-to-image. + CommonAspectRatio::Auto => NanoBananaProMultiFunctionImageGenAspectRatio::Auto, + // NB: "9:21" value does not exist. Interpolated. + CommonAspectRatio::TallNineByTwentyOne => NanoBananaProMultiFunctionImageGenAspectRatio::NineBySixteen, + } +} diff --git a/crates/desktop/artcraft/src/core/api_adapters/aspect_ratio/convert/mod.rs b/crates/desktop/artcraft/src/core/api_adapters/aspect_ratio/convert/mod.rs new file mode 100644 index 0000000000..7cd722714f --- /dev/null +++ b/crates/desktop/artcraft/src/core/api_adapters/aspect_ratio/convert/mod.rs @@ -0,0 +1 @@ +pub mod aspect_ratio_to_artcraft_nano_banana_pro; \ No newline at end of file diff --git a/crates/desktop/artcraft/src/core/api_adapters/aspect_ratio/mod.rs b/crates/desktop/artcraft/src/core/api_adapters/aspect_ratio/mod.rs new file mode 100644 index 0000000000..9dbd41f347 --- /dev/null +++ b/crates/desktop/artcraft/src/core/api_adapters/aspect_ratio/mod.rs @@ -0,0 +1,2 @@ +pub mod common_aspect_ratio; +pub mod convert; diff --git a/crates/desktop/artcraft/src/core/api_adapters/mod.rs b/crates/desktop/artcraft/src/core/api_adapters/mod.rs new file mode 100644 index 0000000000..02401adae4 --- /dev/null +++ b/crates/desktop/artcraft/src/core/api_adapters/mod.rs @@ -0,0 +1 @@ +pub mod aspect_ratio; \ No newline at end of file diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/common/mod.rs b/crates/desktop/artcraft/src/core/commands/enqueue/common/mod.rs index 2ad065a846..d81449e79c 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/common/mod.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/common/mod.rs @@ -1,3 +1,3 @@ -pub mod notify_frontend_of_errors; mod maybe_notify_frontend_of_billing_errors; -mod maybe_notify_frontend_of_grok_errors; \ No newline at end of file +mod maybe_notify_frontend_of_grok_errors; +pub mod notify_frontend_of_errors; \ No newline at end of file diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/artcraft/handle_artcraft_nano_banana_pro_edit.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/artcraft/handle_artcraft_nano_banana_pro_edit.rs index 3af27c4cfb..0bec15e383 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/artcraft/handle_artcraft_nano_banana_pro_edit.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/artcraft/handle_artcraft_nano_banana_pro_edit.rs @@ -1,6 +1,9 @@ +use crate::core::api_adapters::aspect_ratio::convert::aspect_ratio_to_artcraft_nano_banana_pro::aspect_ratio_to_artcraft_nano_banana_pro; use crate::core::commands::enqueue::generate_error::{BadInputReason, GenerateError}; +use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::ImageEditModel::NanoBanana; use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::{EditImageQuality, EditImageResolution, EditImageSize, EnqueueEditImageCommand}; use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess; +use crate::core::commands::enqueue::text_to_image::enqueue_text_to_image_command::{EnqueueTextToImageRequest, TextToImageSize}; use crate::core::events::basic_sendable_event_trait::BasicSendableEvent; use crate::core::events::generation_events::common::{GenerationAction, GenerationModel, GenerationServiceProvider}; use crate::core::events::generation_events::generation_enqueue_failure_event::GenerationEnqueueFailureEvent; @@ -18,6 +21,7 @@ use artcraft_api_defs::generate::image::generate_flux_1_dev_text_to_image::Gener use artcraft_api_defs::generate::image::generate_flux_1_schnell_text_to_image::GenerateFlux1SchnellTextToImageRequest; use artcraft_api_defs::generate::image::generate_flux_pro_11_text_to_image::GenerateFluxPro11TextToImageRequest; use artcraft_api_defs::generate::image::generate_flux_pro_11_ultra_text_to_image::GenerateFluxPro11UltraTextToImageRequest; +use artcraft_api_defs::generate::image::multi_function::nano_banana_pro_multi_function_image_gen::{NanoBananaProMultiFunctionImageGenAspectRatio, NanoBananaProMultiFunctionImageGenImageResolution, NanoBananaProMultiFunctionImageGenNumImages, NanoBananaProMultiFunctionImageGenRequest}; use enums::common::generation_provider::GenerationProvider; use enums::tauri::tasks::task_type::TaskType; use idempotency::uuid::generate_random_uuid; @@ -25,10 +29,8 @@ use log::{error, info}; use storyteller_client::endpoints::generate::image::edit::flux_pro_kontext_max_edit_image::flux_pro_kontext_max_edit_image; use storyteller_client::endpoints::generate::image::edit::gemini_25_flash_edit_image::gemini_25_flash_edit_image; use storyteller_client::endpoints::generate::image::edit::gpt_image_1_edit_image::gpt_image_1_edit_image; -use tauri::AppHandle; -use artcraft_api_defs::generate::image::multi_function::nano_banana_pro_multi_function_image_gen::{NanoBananaProMultiFunctionImageGenAspectRatio, NanoBananaProMultiFunctionImageGenImageResolution, NanoBananaProMultiFunctionImageGenNumImages, NanoBananaProMultiFunctionImageGenRequest}; use storyteller_client::endpoints::generate::image::multi_function::nano_banana_pro_multi_function_image_gen_image::nano_banana_pro_multi_function_image_gen; -use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::ImageEditModel::NanoBanana; +use tauri::AppHandle; pub(super) const MAX_IMAGES: usize = 10; @@ -66,14 +68,7 @@ pub async fn handle_artcraft_nano_banana_pro_edit( }, }; - let aspect_ratio = request.aspect_ratio - .map(|aspect_ratio| match aspect_ratio { - EditImageSize::Auto => None, - EditImageSize::Square => Some(NanoBananaProMultiFunctionImageGenAspectRatio::OneByOne), - EditImageSize::Wide => Some(NanoBananaProMultiFunctionImageGenAspectRatio::SixteenByNine), - EditImageSize::Tall => Some(NanoBananaProMultiFunctionImageGenAspectRatio::NineBySixteen), - }) - .flatten(); + let aspect_ratio = get_aspect_ratio(request); let resolution = request.image_resolution .map(|image_resolution| match image_resolution { @@ -135,3 +130,23 @@ pub async fn handle_artcraft_nano_banana_pro_edit( task_type: TaskType::ImageGeneration, }) } + +fn get_aspect_ratio(request: &EnqueueEditImageCommand) -> Option { + if let Some(common_aspect_ratio) = request.common_aspect_ratio { + // Handle modern aspect ratio + let aspect = aspect_ratio_to_artcraft_nano_banana_pro(common_aspect_ratio); + return Some(aspect); + } + + if let Some(aspect_ratio) = request.aspect_ratio { + // Handle deprecated aspect ratio + return match aspect_ratio { + EditImageSize::Auto => Some(NanoBananaProMultiFunctionImageGenAspectRatio::Auto), + EditImageSize::Square => Some(NanoBananaProMultiFunctionImageGenAspectRatio::OneByOne), + EditImageSize::Wide => Some(NanoBananaProMultiFunctionImageGenAspectRatio::SixteenByNine), + EditImageSize::Tall => Some(NanoBananaProMultiFunctionImageGenAspectRatio::NineBySixteen), + } + } + + None +} diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/enqueue_edit_image_command.rs b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/enqueue_edit_image_command.rs index 7e6fd0d75a..d740ec276b 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/enqueue_edit_image_command.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/image_edit/enqueue_edit_image_command.rs @@ -1,3 +1,4 @@ +use crate::core::api_adapters::aspect_ratio::common_aspect_ratio::CommonAspectRatio; use crate::core::commands::enqueue::common::notify_frontend_of_errors::notify_frontend_of_errors; use crate::core::commands::enqueue::generate_error::{BadInputReason, GenerateError}; use crate::core::commands::enqueue::image_edit::artcraft::handle_image_edit_artcraft::handle_image_edit_artcraft; @@ -108,8 +109,13 @@ pub struct EnqueueEditImageCommand { pub image_count: Option, /// Aspect ratio. + #[deprecated(note="use common_aspect_ratio")] pub aspect_ratio: Option, + /// New field for aspect ratio. + /// Not all models support each of these aspect ratios, but we can choose or interpolate sensibly. + pub common_aspect_ratio: Option, + /// Image quality. pub image_quality: Option, diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/artcraft/handle_artcraft_nano_banana_pro_text_to_image.rs b/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/artcraft/handle_artcraft_nano_banana_pro_text_to_image.rs index 0e6d6d6324..09a1096b7a 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/artcraft/handle_artcraft_nano_banana_pro_text_to_image.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/artcraft/handle_artcraft_nano_banana_pro_text_to_image.rs @@ -1,3 +1,5 @@ +use crate::core::api_adapters::aspect_ratio::common_aspect_ratio::CommonAspectRatio; +use crate::core::api_adapters::aspect_ratio::convert::aspect_ratio_to_artcraft_nano_banana_pro::aspect_ratio_to_artcraft_nano_banana_pro; use crate::core::commands::enqueue::generate_error::{BadInputReason, GenerateError, MissingCredentialsReason, ProviderFailureReason}; use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::{EditImageQuality, EditImageResolution, EditImageSize}; use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess; @@ -58,14 +60,7 @@ pub async fn handle_artcraft_nano_banana_pro_text_to_image( }, }; - let aspect_ratio = request.aspect_ratio - .map(|aspect_ratio| match aspect_ratio { - TextToImageSize::Auto => None, - TextToImageSize::Square => Some(NanoBananaProMultiFunctionImageGenAspectRatio::OneByOne), - TextToImageSize::Wide => Some(NanoBananaProMultiFunctionImageGenAspectRatio::SixteenByNine), - TextToImageSize::Tall => Some(NanoBananaProMultiFunctionImageGenAspectRatio::NineBySixteen), - }) - .flatten(); + let aspect_ratio = get_aspect_ratio(request); let resolution = request.image_resolution .map(|image_resolution| match image_resolution { @@ -82,6 +77,8 @@ pub async fn handle_artcraft_nano_banana_pro_text_to_image( resolution, aspect_ratio, }; + + info!("Calling Artcraft with request {:?}", request); let result = nano_banana_pro_multi_function_image_gen( &app_env_configs.storyteller_host, @@ -109,3 +106,23 @@ pub async fn handle_artcraft_nano_banana_pro_text_to_image( task_type: TaskType::ImageGeneration, }) } + +fn get_aspect_ratio(request: &EnqueueTextToImageRequest) -> Option { + if let Some(common_aspect_ratio) = request.common_aspect_ratio { + // Handle modern aspect ratio + let aspect = aspect_ratio_to_artcraft_nano_banana_pro(common_aspect_ratio); + return Some(aspect); + } + + if let Some(aspect_ratio) = request.aspect_ratio { + // Handle deprecated aspect ratio + return match aspect_ratio { + TextToImageSize::Auto => Some(NanoBananaProMultiFunctionImageGenAspectRatio::Auto), + TextToImageSize::Square => Some(NanoBananaProMultiFunctionImageGenAspectRatio::OneByOne), + TextToImageSize::Wide => Some(NanoBananaProMultiFunctionImageGenAspectRatio::SixteenByNine), + TextToImageSize::Tall => Some(NanoBananaProMultiFunctionImageGenAspectRatio::NineBySixteen), + } + } + + None +} diff --git a/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/enqueue_text_to_image_command.rs b/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/enqueue_text_to_image_command.rs index 947d607fbc..48a578c3fc 100644 --- a/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/enqueue_text_to_image_command.rs +++ b/crates/desktop/artcraft/src/core/commands/enqueue/text_to_image/enqueue_text_to_image_command.rs @@ -34,6 +34,7 @@ use serde::{Deserialize, Serialize}; use sqlite_tasks::queries::create_task::{create_task, CreateTaskArgs}; use tauri::{AppHandle, State}; use tokens::tokens::media_files::MediaFileToken; +use crate::core::api_adapters::aspect_ratio::common_aspect_ratio::CommonAspectRatio; /// This is used in the Tauri command bridge. /// Don't change the serializations without coordinating with the frontend. @@ -73,7 +74,7 @@ pub enum TextToImageModel { Midjourney, } -#[derive(Deserialize)] +#[derive(Deserialize, Debug)] pub struct EnqueueTextToImageRequest { /// The provider to use (defaults to Artcraft/Storyteller). /// Not all (provider, model) combinations are valid. @@ -86,7 +87,12 @@ pub struct EnqueueTextToImageRequest { pub prompt: Option, /// Aspect ratio. + #[deprecated(note="use common_aspect_ratio")] pub aspect_ratio: Option, + + /// New field for aspect ratio. + /// Not all models support each of these aspect ratios, but we can choose or interpolate sensibly. + pub common_aspect_ratio: Option, /// Aspect ratio. pub image_resolution: Option, @@ -185,6 +191,8 @@ pub async fn enqueue_text_to_image_command( info!("enqueue_text_to_image called"); + info!("request: {:?}", request); + let result = handle_request( request, &app, diff --git a/crates/desktop/artcraft/src/core/mod.rs b/crates/desktop/artcraft/src/core/mod.rs index e2998e8f92..810a551526 100644 --- a/crates/desktop/artcraft/src/core/mod.rs +++ b/crates/desktop/artcraft/src/core/mod.rs @@ -1,3 +1,4 @@ +pub mod api_adapters; pub mod artcraft_error; pub mod commands; pub mod events; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/edit/gemini_25_flash_edit_image_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/edit/gemini_25_flash_edit_image_handler.rs index 83f7d766f7..db545a8ceb 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/edit/gemini_25_flash_edit_image_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/edit/gemini_25_flash_edit_image_handler.rs @@ -44,6 +44,7 @@ use utoipa::ToSchema; ("request" = Gemini25FlashEditImageRequest, description = "Payload for Request"), ) )] +#[deprecated(note="use `nano_banana_multi_function_image_gen_handler` instead")] pub async fn gemini_25_flash_edit_image_handler( http_request: HttpRequest, request: Json, @@ -148,13 +149,14 @@ pub async fn gemini_25_flash_edit_image_handler( Some(Gemini25FlashEditImageNumImages::Four) => Gemini25FlashEditNumImages::Four, None => Gemini25FlashEditNumImages::One, // Default to One }; - + let args = Gemini25FlashEditArgs { image_urls, prompt: request.prompt.as_deref().unwrap_or(""), webhook_url: &server_state.fal.webhook_url, api_key: &server_state.fal.api_key, num_images, + aspect_ratio: None, // TODO }; let fal_result = enqueue_gemini_25_flash_edit_webhook(args) diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/multi_function/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/multi_function/mod.rs index d120272219..237eafc4c3 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/multi_function/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/multi_function/mod.rs @@ -1,4 +1,5 @@ pub mod bytedance_seedream_v4_multi_function_image_gen_handler; pub mod bytedance_seedream_v4p5_multi_function_image_gen_handler; pub mod gpt_image_1p5_multi_function_image_gen_handler; +pub mod nano_banana_multi_function_image_gen_handler; pub mod nano_banana_pro_multi_function_image_gen_handler; diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/multi_function/nano_banana_multi_function_image_gen_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/multi_function/nano_banana_multi_function_image_gen_handler.rs new file mode 100644 index 0000000000..d41b555325 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/multi_function/nano_banana_multi_function_image_gen_handler.rs @@ -0,0 +1,298 @@ +use std::collections::HashSet; +use std::iter::FromIterator; +use std::sync::Arc; + +use crate::http_server::common_responses::common_web_error::CommonWebError; +use crate::http_server::common_responses::media::media_links_builder::MediaLinksBuilder; +use crate::http_server::endpoints::generate::common::payments_error_test::payments_error_test; +use crate::http_server::endpoints::media_files::helpers::get_media_domain::get_media_domain; +use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format; +use crate::state::server_state::ServerState; +use crate::util::lookup::lookup_image_urls_as_optional_list::lookup_image_urls_as_optional_list; +use actix_web::web::Json; +use actix_web::{web, HttpRequest}; +use artcraft_api_defs::generate::image::edit::gemini_25_flash_edit_image::{Gemini25FlashEditImageNumImages, Gemini25FlashEditImageRequest}; +use artcraft_api_defs::generate::image::multi_function::nano_banana_multi_function_image_gen::{NanoBananaMultiFunctionImageGenAspectRatio, NanoBananaMultiFunctionImageGenNumImages, NanoBananaMultiFunctionImageGenRequest, NanoBananaMultiFunctionImageGenResponse}; +use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; +use enums::by_table::prompt_context_items::prompt_context_semantic_type::PromptContextSemanticType; +use enums::by_table::prompts::prompt_type::PromptType; +use enums::common::generation_provider::GenerationProvider; +use enums::common::model_type::ModelType; +use enums::common::visibility::Visibility; +use fal_client::creds::open_ai_api_key::OpenAiApiKey; +use fal_client::requests::webhook::image::edit::enqueue_gemini_25_flash_edit_webhook::{enqueue_gemini_25_flash_edit_webhook, Gemini25FlashEditArgs, Gemini25FlashEditAspectRatio, Gemini25FlashEditNumImages}; +use fal_client::requests::webhook::image::edit::enqueue_nano_banana_pro_edit_image_webhook::{enqueue_nano_banana_pro_image_edit_webhook, EnqueueNanoBananaProEditImageArgs, EnqueueNanoBananaProEditImageAspectRatio, EnqueueNanoBananaProEditImageNumImages, EnqueueNanoBananaProEditImageResolution}; +use fal_client::requests::webhook::image::text::enqueue_gemini_25_flash_text_to_image_webhook::{enqueue_gemini_25_flash_text_to_image_webhook, Gemini25FlashTextToImageArgs, Gemini25FlashTextToImageAspectRatio, Gemini25FlashTextToImageNumImages}; +use fal_client::requests::webhook::image::text::enqueue_nano_banana_pro_text_to_image_webhook::{enqueue_nano_banana_pro_text_to_image_webhook, EnqueueNanoBananaProTextToImageArgs, EnqueueNanoBananaProTextToImageNumImages, EnqueueNanoBananaProTextToImageResolution}; +use http_server_common::request::get_request_ip::get_request_ip; +use log::{error, info, warn}; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::insert_generic_inference_job_for_fal_queue; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::FalCategory; +use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::InsertGenericInferenceForFalArgs; +use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token; +use mysql_queries::queries::media_files::get::batch_get_media_files_by_tokens::{batch_get_media_files_by_tokens, batch_get_media_files_by_tokens_with_connection}; +use mysql_queries::queries::prompt_context_items::insert_batch_prompt_context_items::{insert_batch_prompt_context_items, InsertBatchArgs, PromptContextItem}; +use mysql_queries::queries::prompts::insert_prompt::{insert_prompt, InsertPromptArgs}; +use server_environment::ServerEnvironment; +use sqlx::pool::PoolConnection; +use sqlx::{Acquire, MySql}; +use tokens::tokens::media_files::MediaFileToken; +use utoipa::ToSchema; + +/// Nano Banana Multi-Function (generate + edit) +#[utoipa::path( + post, + tag = "Generate Images (Multi-Function)", + path = "/v1/generate/image/multi_function/nano_banana", + responses( + (status = 200, description = "Success", body = NanoBananaMultiFunctionImageGenResponse), + ), + params( + ("request" = NanoBananaMultiFunctionImageGenRequest, description = "Payload for Request"), + ) +)] +pub async fn nano_banana_multi_function_image_gen_handler( + http_request: HttpRequest, + request: Json, + server_state: web::Data> +) -> Result, CommonWebError> { + + payments_error_test(&request.prompt.as_deref().unwrap_or(""))?; + + let mut mysql_connection = server_state.mysql_pool + .acquire() + .await?; + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session_from_connection(&http_request, &mut mysql_connection) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + CommonWebError::ServerError + })?; + + let maybe_avt_token = server_state + .avt_cookie_manager + .get_avt_token_from_request(&http_request); + + // TODO: Limit usage for new accounts. Billing, free credits metering, etc. + + //let user_session = match maybe_user_session { + // Some(session) => session, + // None => { + // warn!("not logged in"); + // return Err(CommonWebError::NotAuthorized); + // } + //}; + + if let Err(reason) = validate_idempotency_token_format(&request.uuid_idempotency_token) { + return Err(CommonWebError::BadInputWithSimpleMessage(reason)); + } + + let image_urls = match request.image_media_tokens.as_ref() { + Some(media_tokens) => { + info!("Looking up image media tokens: {:?}", media_tokens); + lookup_image_urls_as_optional_list( + &http_request, + &mut mysql_connection, + server_state.server_environment, + media_tokens, + ).await? + }, + None => None, + }; + + insert_idempotency_token(&request.uuid_idempotency_token, &mut *mysql_connection) + .await + .map_err(|err| { + error!("Error inserting idempotency token: {:?}", err); + CommonWebError::BadInputWithSimpleMessage("repeated idempotency token".to_string()) + })?; + + info!("Fal webhook URL: {}", server_state.fal.webhook_url); + + let fal_result; + + if let Some(input_image_urls) = image_urls.as_deref() { + info!("nano banana edit image"); + + let num_images = match request.num_images { + Some(NanoBananaMultiFunctionImageGenNumImages::One) => Gemini25FlashEditNumImages::One, + Some(NanoBananaMultiFunctionImageGenNumImages::Two) => Gemini25FlashEditNumImages::Two, + Some(NanoBananaMultiFunctionImageGenNumImages::Three) => Gemini25FlashEditNumImages::Three, + Some(NanoBananaMultiFunctionImageGenNumImages::Four) => Gemini25FlashEditNumImages::Four, + None => Gemini25FlashEditNumImages::One, // Default to One + }; + + let aspect_ratio = match request.aspect_ratio { + Some(NanoBananaMultiFunctionImageGenAspectRatio::Auto) => Gemini25FlashEditAspectRatio::Auto, + Some(NanoBananaMultiFunctionImageGenAspectRatio::OneByOne) => Gemini25FlashEditAspectRatio::OneByOne, + Some(NanoBananaMultiFunctionImageGenAspectRatio::FiveByFour) => Gemini25FlashEditAspectRatio::FiveByFour, + Some(NanoBananaMultiFunctionImageGenAspectRatio::FourByThree) => Gemini25FlashEditAspectRatio::FourByThree, + Some(NanoBananaMultiFunctionImageGenAspectRatio::ThreeByTwo) => Gemini25FlashEditAspectRatio::ThreeByTwo, + Some(NanoBananaMultiFunctionImageGenAspectRatio::SixteenByNine) => Gemini25FlashEditAspectRatio::SixteenByNine, + Some(NanoBananaMultiFunctionImageGenAspectRatio::TwentyOneByNine) => Gemini25FlashEditAspectRatio::TwentyOneByNine, + Some(NanoBananaMultiFunctionImageGenAspectRatio::FourByFive) => Gemini25FlashEditAspectRatio::FourByFive, + Some(NanoBananaMultiFunctionImageGenAspectRatio::ThreeByFour) => Gemini25FlashEditAspectRatio::ThreeByFour, + Some(NanoBananaMultiFunctionImageGenAspectRatio::TwoByThree) => Gemini25FlashEditAspectRatio::TwoByThree, + Some(NanoBananaMultiFunctionImageGenAspectRatio::NineBySixteen) => Gemini25FlashEditAspectRatio::NineBySixteen, + None => Gemini25FlashEditAspectRatio::OneByOne, + }; + + let args = Gemini25FlashEditArgs { + prompt: request.prompt.as_deref().unwrap_or(""), + image_urls: input_image_urls.to_owned(), + num_images, + aspect_ratio: Some(aspect_ratio), + webhook_url: &server_state.fal.webhook_url, + api_key: &server_state.fal.api_key, + }; + + fal_result = enqueue_gemini_25_flash_edit_webhook(args) + .await + .map_err(|err| { + warn!("Error calling enqueue_gemini_25_flash_edit_webhook : {:?}", err); + CommonWebError::ServerError + })?; + + } else { + info!("nano banana text-to-image"); + + let num_images = match request.num_images { + Some(NanoBananaMultiFunctionImageGenNumImages::One) => Gemini25FlashTextToImageNumImages::One, + Some(NanoBananaMultiFunctionImageGenNumImages::Two) => Gemini25FlashTextToImageNumImages::Two, + Some(NanoBananaMultiFunctionImageGenNumImages::Three) => Gemini25FlashTextToImageNumImages::Three, + Some(NanoBananaMultiFunctionImageGenNumImages::Four) => Gemini25FlashTextToImageNumImages::Four, + None => Gemini25FlashTextToImageNumImages::One, // Default to One + }; + + let aspect_ratio = match request.aspect_ratio { + Some(NanoBananaMultiFunctionImageGenAspectRatio::Auto) => Gemini25FlashTextToImageAspectRatio::OneByOne, // NB: "auto" is only for edit image, not text-to-image ! + Some(NanoBananaMultiFunctionImageGenAspectRatio::OneByOne) => Gemini25FlashTextToImageAspectRatio::OneByOne, + Some(NanoBananaMultiFunctionImageGenAspectRatio::FiveByFour) => Gemini25FlashTextToImageAspectRatio::FiveByFour, + Some(NanoBananaMultiFunctionImageGenAspectRatio::FourByThree) => Gemini25FlashTextToImageAspectRatio::FourByThree, + Some(NanoBananaMultiFunctionImageGenAspectRatio::ThreeByTwo) => Gemini25FlashTextToImageAspectRatio::ThreeByTwo, + Some(NanoBananaMultiFunctionImageGenAspectRatio::SixteenByNine) => Gemini25FlashTextToImageAspectRatio::SixteenByNine, + Some(NanoBananaMultiFunctionImageGenAspectRatio::TwentyOneByNine) => Gemini25FlashTextToImageAspectRatio::TwentyOneByNine, + Some(NanoBananaMultiFunctionImageGenAspectRatio::FourByFive) => Gemini25FlashTextToImageAspectRatio::FourByFive, + Some(NanoBananaMultiFunctionImageGenAspectRatio::ThreeByFour) => Gemini25FlashTextToImageAspectRatio::ThreeByFour, + Some(NanoBananaMultiFunctionImageGenAspectRatio::TwoByThree) => Gemini25FlashTextToImageAspectRatio::TwoByThree, + Some(NanoBananaMultiFunctionImageGenAspectRatio::NineBySixteen) => Gemini25FlashTextToImageAspectRatio::NineBySixteen, + None => Gemini25FlashTextToImageAspectRatio::OneByOne, + }; + + let args = Gemini25FlashTextToImageArgs { + prompt: request.prompt.as_deref().unwrap_or(""), + num_images, + aspect_ratio: Some(aspect_ratio), + webhook_url: &server_state.fal.webhook_url, + api_key: &server_state.fal.api_key, + }; + + fal_result = enqueue_gemini_25_flash_text_to_image_webhook(args) + .await + .map_err(|err| { + warn!("Error calling enqueue_gemini_25_flash_text_to_image_webhook: {:?}", err); + CommonWebError::ServerError + })?; + } + + let external_job_id = fal_result.request_id + .ok_or_else(|| { + warn!("Fal request_id is None"); + CommonWebError::ServerError + })?; + + info!("Fal request_id: {}", external_job_id); + + let ip_address = get_request_ip(&http_request); + + let mut transaction = mysql_connection + .begin() + .await + .map_err(|err| { + error!("Error starting MySQL transaction: {:?}", err); + CommonWebError::ServerError + })?; + + // NB: Don't fail the job if the query fails. + let prompt_result = insert_prompt(InsertPromptArgs { + maybe_apriori_prompt_token: None, + prompt_type: PromptType::ArtcraftApp, + maybe_creator_user_token: maybe_user_session + .as_ref() + .map(|s| &s.user_token), + maybe_model_type: Some(ModelType::NanoBanana), + maybe_generation_provider: Some(GenerationProvider::Artcraft), + maybe_positive_prompt: request.prompt.as_deref(), + maybe_negative_prompt: None, + maybe_other_args: None, + creator_ip_address: &ip_address, + mysql_executor: &mut *transaction, + phantom: Default::default(), + }).await; + + let prompt_token = match prompt_result { + Ok(token) => Some(token), + Err(err) => { + warn!("Error inserting prompt: {:?}", err); + None // Don't fail the job if the prompt insertion fails. + } + }; + + if let Some(media_tokens) = &request.image_media_tokens { + if let Some(token) = prompt_token.as_ref() { + let result = insert_batch_prompt_context_items(InsertBatchArgs { + prompt_token: token.clone(), + items: media_tokens.iter().map(|token| { + PromptContextItem { + media_token: token.clone(), + context_semantic_type: PromptContextSemanticType::Imgref, + } + }).collect(), + transaction: &mut transaction, + }).await; + + if let Err(err) = result { + // NB: Fail open. + warn!("Error inserting batch prompt context items: {:?}", err); + } + } + } + + let db_result = insert_generic_inference_job_for_fal_queue(InsertGenericInferenceForFalArgs { + uuid_idempotency_token: &request.uuid_idempotency_token, + maybe_external_third_party_id: &external_job_id, + fal_category: FalCategory::ImageGeneration, + maybe_inference_args: None, + maybe_prompt_token: prompt_token.as_ref(), + maybe_creator_user_token: maybe_user_session.as_ref().map(|s| &s.user_token), + maybe_avt_token: maybe_avt_token.as_ref(), + creator_ip_address: &ip_address, + creator_set_visibility: Visibility::Public, + mysql_executor: &mut *transaction, + phantom: Default::default(), + }).await; + + let job_token = match db_result { + Ok(token) => token, + Err(err) => { + warn!("Error inserting generic inference job for FAL queue: {:?}", err); + return Err(CommonWebError::ServerError); + } + }; + + let _r = transaction + .commit() + .await + .map_err(|err| { + error!("Error committing MySQL transaction: {:?}", err); + CommonWebError::ServerError + })?; + + Ok(Json(NanoBananaMultiFunctionImageGenResponse { + success: true, + inference_job_token: job_token, + })) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/multi_function/nano_banana_pro_multi_function_image_gen_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/multi_function/nano_banana_pro_multi_function_image_gen_handler.rs index 99b952438f..c344ffc956 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/multi_function/nano_banana_pro_multi_function_image_gen_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/generate/image/multi_function/nano_banana_pro_multi_function_image_gen_handler.rs @@ -130,6 +130,7 @@ pub async fn nano_banana_pro_multi_function_image_gen_handler( }; let aspect_ratio = match request.aspect_ratio { + Some(NanoBananaProMultiFunctionImageGenAspectRatio::Auto) => EnqueueNanoBananaProEditImageAspectRatio::Auto, Some(NanoBananaProMultiFunctionImageGenAspectRatio::OneByOne) => EnqueueNanoBananaProEditImageAspectRatio::OneByOne, Some(NanoBananaProMultiFunctionImageGenAspectRatio::FiveByFour) => EnqueueNanoBananaProEditImageAspectRatio::FiveByFour, Some(NanoBananaProMultiFunctionImageGenAspectRatio::FourByThree) => EnqueueNanoBananaProEditImageAspectRatio::FourByThree, @@ -179,6 +180,7 @@ pub async fn nano_banana_pro_multi_function_image_gen_handler( }; let aspect_ratio = match request.aspect_ratio { + Some(NanoBananaProMultiFunctionImageGenAspectRatio::Auto) => EnqueueNanoBananaProTextToImageAspectRatio::OneByOne, // NB: "auto" is only for edit image, not text-to-image ! Some(NanoBananaProMultiFunctionImageGenAspectRatio::OneByOne) => EnqueueNanoBananaProTextToImageAspectRatio::OneByOne, Some(NanoBananaProMultiFunctionImageGenAspectRatio::FiveByFour) => EnqueueNanoBananaProTextToImageAspectRatio::FiveByFour, Some(NanoBananaProMultiFunctionImageGenAspectRatio::FourByThree) => EnqueueNanoBananaProTextToImageAspectRatio::FourByThree, diff --git a/crates/service/web/storyteller_web/src/http_server/routes/application_routes/generate_routes.rs b/crates/service/web/storyteller_web/src/http_server/routes/application_routes/generate_routes.rs index df9fa27495..b8635771fd 100644 --- a/crates/service/web/storyteller_web/src/http_server/routes/application_routes/generate_routes.rs +++ b/crates/service/web/storyteller_web/src/http_server/routes/application_routes/generate_routes.rs @@ -13,6 +13,7 @@ use crate::http_server::endpoints::generate::image::inpaint::flux_pro_1_inpaint_ use crate::http_server::endpoints::generate::image::multi_function::bytedance_seedream_v4_multi_function_image_gen_handler::bytedance_seedream_v4_multi_function_image_gen_handler; use crate::http_server::endpoints::generate::image::multi_function::bytedance_seedream_v4p5_multi_function_image_gen_handler::bytedance_seedream_v4p5_multi_function_image_gen_handler; use crate::http_server::endpoints::generate::image::multi_function::gpt_image_1p5_multi_function_image_gen_handler::gpt_image_1p5_multi_function_image_gen_handler; +use crate::http_server::endpoints::generate::image::multi_function::nano_banana_multi_function_image_gen_handler::nano_banana_multi_function_image_gen_handler; use crate::http_server::endpoints::generate::image::multi_function::nano_banana_pro_multi_function_image_gen_handler::nano_banana_pro_multi_function_image_gen_handler; use crate::http_server::endpoints::generate::image::remove_image_background_handler::remove_image_background_handler; use crate::http_server::endpoints::generate::object::generate_hunyuan_2_0_image_to_3d_handler::generate_hunyuan_2_0_image_to_3d_handler; @@ -62,6 +63,10 @@ where .route(web::post().to(gpt_image_1p5_multi_function_image_gen_handler)) .route(web::head().to(|| HttpResponse::Ok())) ) + .service(web::resource("/nano_banana") + .route(web::post().to(nano_banana_multi_function_image_gen_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) .service(web::resource("/nano_banana_pro") .route(web::post().to(nano_banana_pro_multi_function_image_gen_handler)) .route(web::head().to(|| HttpResponse::Ok())) diff --git a/crates/vendor/fal-rs/src/endpoints/fal_ai/gemini_25_flash_image/edit.rs b/crates/vendor/fal-rs/src/endpoints/fal_ai/gemini_25_flash_image/edit.rs index 0a07eaa3ee..25c36977b3 100644 --- a/crates/vendor/fal-rs/src/endpoints/fal_ai/gemini_25_flash_image/edit.rs +++ b/crates/vendor/fal-rs/src/endpoints/fal_ai/gemini_25_flash_image/edit.rs @@ -14,6 +14,12 @@ pub struct Gemini25FlashImageEditInput { #[serde(skip_serializing_if = "Option::is_none")] pub output_format: Option, + /// The aspect ratio of the generated image. + /// Default value: "auto" + /// Possible enum values: auto, 21:9, 16:9, 3:2, 4:3, 5:4, 1:1, 4:5, 3:4, 2:3, 9:16 + #[serde(skip_serializing_if = "Option::is_none")] + pub aspect_ratio: Option, + // /// Resolution, eg. "720p" // #[serde(skip_serializing_if = "Option::is_none")] // pub resolution: Option, diff --git a/crates/vendor/fal-rs/src/endpoints/fal_ai/gemini_25_flash_image/mod.rs b/crates/vendor/fal-rs/src/endpoints/fal_ai/gemini_25_flash_image/mod.rs index 0a4a4d0e41..e929873bc2 100644 --- a/crates/vendor/fal-rs/src/endpoints/fal_ai/gemini_25_flash_image/mod.rs +++ b/crates/vendor/fal-rs/src/endpoints/fal_ai/gemini_25_flash_image/mod.rs @@ -1 +1,2 @@ -pub mod edit; \ No newline at end of file +pub mod edit; +pub mod text_to_image; \ No newline at end of file diff --git a/crates/vendor/fal-rs/src/endpoints/fal_ai/gemini_25_flash_image/text_to_image.rs b/crates/vendor/fal-rs/src/endpoints/fal_ai/gemini_25_flash_image/text_to_image.rs new file mode 100644 index 0000000000..282e6296d8 --- /dev/null +++ b/crates/vendor/fal-rs/src/endpoints/fal_ai/gemini_25_flash_image/text_to_image.rs @@ -0,0 +1,37 @@ +use crate::prelude::{Deserialize, FalRequest, Serialize}; + +#[derive(Debug, Serialize, Deserialize, Default)] +pub struct Gemini25FlashTextToImageInput { + pub prompt: String, + + /// 1 - 4 + #[serde(skip_serializing_if = "Option::is_none")] + pub num_images: Option, + + /// "jpeg" or "png" + #[serde(skip_serializing_if = "Option::is_none")] + pub output_format: Option, + + /// The aspect ratio of the generated image. + /// Default value: "1:1" + /// Possible enum values: 21:9, 16:9, 3:2, 4:3, 5:4, 1:1, 4:5, 3:4, 2:3, 9:16 + #[serde(skip_serializing_if = "Option::is_none")] + pub aspect_ratio: Option, +} + +#[derive(Debug, Serialize, Deserialize, Default)] +pub struct File { + /// The URL where the file can be downloaded from. + pub url: String, +} + +#[derive(Debug, Serialize, Deserialize)] +pub struct Gemini25FlashTextToImageOutput { + pub images: Vec, +} + +pub fn gemini_25_flash_text_to_image( + params: Gemini25FlashTextToImageInput, +) -> FalRequest { + FalRequest::new("fal-ai/gemini-25-flash-image", params) +} diff --git a/frontend/libs/components/promptbox/src/lib/PromptBoxImage.tsx b/frontend/libs/components/promptbox/src/lib/PromptBoxImage.tsx index d2780b2af4..2d2ac615a5 100644 --- a/frontend/libs/components/promptbox/src/lib/PromptBoxImage.tsx +++ b/frontend/libs/components/promptbox/src/lib/PromptBoxImage.tsx @@ -27,12 +27,13 @@ import { } from "@fortawesome/pro-regular-svg-icons"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { IconProp } from "@fortawesome/fontawesome-svg-core"; -import { ImageModel, getCapabilitiesForModel } from "@storyteller/model-list"; +import { CommonAspectRatio, ImageModel, getCapabilitiesForModel } from "@storyteller/model-list"; import { usePromptImageStore, RefImage } from "./promptStore"; import { gtagEvent } from "@storyteller/google-analytics"; import { twMerge } from "tailwind-merge"; import { ImagePromptRow } from "./ImagePromptRow"; import { GenerationProvider } from "@storyteller/api-enums"; +import { AspectRatioPicker } from "./common/AspectRatioPicker"; interface PromptBoxImageProps { useJobContext: () => JobContextType; @@ -109,6 +110,9 @@ export const PromptBoxImage = ({ referenceImages.length > 0 || uploadingImages.length > 0; + // New aspect ratio state will begin to phase out old aspect ratio + const [commonAspectRatio, setCommonAspectRatio] = useState(undefined); + useEffect(() => { onImageRowVisibilityChange?.(isImageRowVisible); }, [isImageRowVisible, onImageRowVisibilityChange]); @@ -289,7 +293,7 @@ export const PromptBoxImage = ({ }, 10000); try { - const aspectRatio = getCurrentAspectRatio(); + const aspectRatio = getCurrentLegacyAspectRatio(); const resolution = getCurrentResolution(); const request: EnqueueTextToImageRequest = { @@ -306,6 +310,10 @@ export const PromptBoxImage = ({ request.provider = selectedProvider; } + if (selectedModel?.supportsNewAspectRatio()) { + request.common_aspect_ratio = commonAspectRatio; + } + if ( selectedModel?.canUseImagePrompt && !!referenceImages && @@ -335,7 +343,7 @@ export const PromptBoxImage = ({ return iconElement.props.icon; }; - const getCurrentAspectRatio = (): EnqueueTextToImageSize => { + const getCurrentLegacyAspectRatio = (): EnqueueTextToImageSize => { const selected = aspectRatioList.find((item) => item.selected); switch (selected?.label.toLowerCase()) { case "wide": @@ -469,7 +477,14 @@ export const PromptBoxImage = ({
- {selectedModel?.canChangeAspectRatio && ( + {selectedModel?.supportsNewAspectRatio() && ( + + )} + {selectedModel?.canChangeAspectRatio && !selectedModel?.supportsNewAspectRatio() && ( void; + //model: ImageModel | VideoModel +} + +/** + * Stateless component. + * + * Picker for "common aspect ratios", the new data structure Tauri accepts for + * all image and video models (Note: this is not fully rolled out yet. Some/most + * models may still use the old format.) + * + * @param model - currently selected model + * @param currentAspectRatio - currently selected aspect ratio + * @param handleCommonAspectRatioSelect - callback when an aspect ratio is selected + */ +export const AspectRatioPicker = ({ + model, + currentAspectRatio, + handleCommonAspectRatioSelect, +}: AspectRatioPickerProps) => { + + const useAspectRatio = currentAspectRatio ?? model.defaultAspectRatio ?? undefined; + + console.log("aspect - currentAspectRatio:", currentAspectRatio); + console.log("aspect - useAspectRatio:", useAspectRatio); + + const getCurrentResolutionIcon = () : IconDefinition => { + if (!useAspectRatio) { + return faSquare; + } + return getAspectRatioIcon(useAspectRatio); + } + + const handleSelectAdapter = (item: PopoverItem) => { + const ratio = popOverLabelToAspectRatio(item.label, model); + handleCommonAspectRatioSelect(ratio); + } + + let aspectRatioList : PopoverItem[] = []; + + model.aspectRatios?.forEach((ratio: CommonAspectRatio) => { + aspectRatioList.push({ + label: getAspectRatioTextLabel(ratio), + selected: useAspectRatio === ratio, + description: `foo ${ratio}`, + icon: , + }); + }); + + return ( + <> + + + } + /> + + + ) +} + + +const getAspectRatioIcon = (aspectRatio: CommonAspectRatio) : IconDefinition => { + switch (aspectRatio) { + case CommonAspectRatio.Auto: + return faMagicWandSparkles; + case CommonAspectRatio.Square: + return faSquare; + + case CommonAspectRatio.WideFiveByFour: + case CommonAspectRatio.WideFourByThree: + case CommonAspectRatio.WideThreeByTwo: + return faRectangle; + case CommonAspectRatio.WideSixteenByNine: + case CommonAspectRatio.WideTwentyOneByNine: + return faRectangleWide; + + // Tall + case CommonAspectRatio.TallFourByFive: + case CommonAspectRatio.TallThreeByFour: + case CommonAspectRatio.TallTwoByThree: + return faRectangleVertical; + case CommonAspectRatio.TallNineBySixteen: + case CommonAspectRatio.TallNineByTwentyOne: + return faRectangleVertical; // TODO: New FontAwesome 7 has "Tall". + + default: + return faSquare; // Fail open-ish + } +} + +const getAspectRatioTextLabel = (aspectRatio: CommonAspectRatio) : string => { + switch (aspectRatio) { + case CommonAspectRatio.Auto: + return "Auto"; + case CommonAspectRatio.Square: + return "Square"; + + // Wide + case CommonAspectRatio.WideFiveByFour: + return "5:4"; + case CommonAspectRatio.WideFourByThree: + return "4:3"; + case CommonAspectRatio.WideThreeByTwo: + return "3:2"; + case CommonAspectRatio.WideSixteenByNine: + return "16:9"; + case CommonAspectRatio.WideTwentyOneByNine: + return "21:9"; + + // Tall + case CommonAspectRatio.TallFourByFive: + return "4:5"; + case CommonAspectRatio.TallThreeByFour: + return "3:4"; + case CommonAspectRatio.TallTwoByThree: + return "2:3"; + case CommonAspectRatio.TallNineBySixteen: + return "9:16"; + case CommonAspectRatio.TallNineByTwentyOne: + return "9:21"; + + default: + return "Square"; // Fail open-ish + } +} + +// Note: We only need this to deal with turning PopOverItems back into typesafe aspect ratios +const popOverLabelToAspectRatio = (label: string, model: ImageModel): CommonAspectRatio => { + switch (label) { + case "Auto": return CommonAspectRatio.Auto; + case "Square": return CommonAspectRatio.Square; + case "5:4": return CommonAspectRatio.WideFiveByFour; + case "4:3": return CommonAspectRatio.WideFourByThree; + case "3:2": return CommonAspectRatio.WideThreeByTwo; + case "16:9": return CommonAspectRatio.WideSixteenByNine; + case "21:9": return CommonAspectRatio.WideTwentyOneByNine; + case "4:5": return CommonAspectRatio.TallFourByFive; + case "3:4": return CommonAspectRatio.TallThreeByFour; + case "2:3": return CommonAspectRatio.TallTwoByThree; + case "9:16": return CommonAspectRatio.TallNineBySixteen; + case "9:21": return CommonAspectRatio.TallNineByTwentyOne; + } + // If we can't find it, return the model's default aspect ratio or Square as fallback + return model.defaultAspectRatio || CommonAspectRatio.Square; +} diff --git a/frontend/libs/model-list/src/index.ts b/frontend/libs/model-list/src/index.ts index 18ff980b8d..c0279ad633 100644 --- a/frontend/libs/model-list/src/index.ts +++ b/frontend/libs/model-list/src/index.ts @@ -1,6 +1,7 @@ export * from "./lib/classes/ImageModel.js"; export * from "./lib/classes/Model.js"; export * from "./lib/classes/VideoModel.js"; +export * from "./lib/classes/properties/CommonAspectRatio.js"; export * from "./lib/classes/metadata/ModelCreator.js"; export * from "./lib/classes/metadata/ModelCreatorIcons.js"; export * from "./lib/classes/metadata/ModelMapping.js"; diff --git a/frontend/libs/model-list/src/lib/classes/ImageModel.ts b/frontend/libs/model-list/src/lib/classes/ImageModel.ts index a21f0880c7..52e89a839b 100644 --- a/frontend/libs/model-list/src/lib/classes/ImageModel.ts +++ b/frontend/libs/model-list/src/lib/classes/ImageModel.ts @@ -3,6 +3,7 @@ import { Model, ModelKind } from "./Model.js"; import { ModelCategory } from "../legacy/ModelConfig.js"; import { ModelTag } from "./metadata/ModelTag.js"; import { GenerationProvider } from "@storyteller/api-enums"; +import { CommonAspectRatio } from "./properties/CommonAspectRatio.js"; export class ImageModel extends Model { // Typescript type discriminator property @@ -43,6 +44,14 @@ export class ImageModel extends Model { // Whether the model supports changing aspect ratio readonly canChangeAspectRatio: boolean; + // Aspect ratios supported by this model + // (This is a new field that will take time to roll out and replace the old aspect ratios) + readonly aspectRatios: CommonAspectRatio[]; + + // Aspect ratio to use as default when nothing is selected. + // Otherwise, use `aspectRatios[0]` as default. + readonly defaultAspectRatio?: CommonAspectRatio; + constructor(args: { id: string; tauriId: string; @@ -65,6 +74,8 @@ export class ImageModel extends Model { tags?: ModelTag[]; progressBarTime?: number; providers?: GenerationProvider[]; + aspectRatios?: CommonAspectRatio[]; + defaultAspectRatio?: CommonAspectRatio; }) { if (args.maxGenerationCount < 1) { throw new Error("maxGenerationCount must be at least 1"); @@ -88,6 +99,8 @@ export class ImageModel extends Model { this.canTextToImage = args.canTextToImage === false ? false : true; // Default to true ! this.canChangeResolution = args.canChangeResolution ?? false; this.canChangeAspectRatio = args.canChangeAspectRatio ?? false; + this.aspectRatios = args.aspectRatios ?? []; + this.defaultAspectRatio = args.defaultAspectRatio ?? (args.aspectRatios && args.aspectRatios.length > 0 ? args.aspectRatios[0] : undefined); } // If the model is a "Nano Banana"-type model, we may want to enable certain features. @@ -104,4 +117,12 @@ export class ImageModel extends Model { return false; } } + + // Rolling out new aspect ratio controls to models. + // If we have them set on the model, we can use the new UI controls. + supportsNewAspectRatio(): boolean { + return this.canChangeAspectRatio + && this.aspectRatios + && this.aspectRatios.length > 0; + } } diff --git a/frontend/libs/model-list/src/lib/classes/properties/CommonAspectRatio.ts b/frontend/libs/model-list/src/lib/classes/properties/CommonAspectRatio.ts new file mode 100644 index 0000000000..901a0d4c05 --- /dev/null +++ b/frontend/libs/model-list/src/lib/classes/properties/CommonAspectRatio.ts @@ -0,0 +1,27 @@ + +/// Common aspect ratio Tauri API +/// Not all models have each of these aspect ratios, +/// but Tauri will interpolate if a mistake is sent. +export enum CommonAspectRatio { + // "Auto" + // The idea here is that in image editing (not text-to-image), the model will use the source image. + // This is the semantics used by Nano Banana Pro image editing mode (but not text-to-image). + Auto = "auto", + + // Square + Square = "square", + + // Wide + WideFiveByFour = "wide_five_by_four", + WideFourByThree = "wide_four_by_three", + WideThreeByTwo = "wide_three_by_two", + WideSixteenByNine = "wide_sixteen_by_nine", + WideTwentyOneByNine = "wide_twenty_one_by_nine", + + // Tall + TallFourByFive = "tall_four_by_five", + TallThreeByFour = "tall_three_by_four", + TallTwoByThree = "tall_two_by_three", + TallNineBySixteen = "tall_nine_by_sixteen", + TallNineByTwentyOne = "tall_nine_by_twenty_one", +} diff --git a/frontend/libs/model-list/src/lib/lists/ImageModels.ts b/frontend/libs/model-list/src/lib/lists/ImageModels.ts index c09831605c..fe8b75a7fb 100644 --- a/frontend/libs/model-list/src/lib/lists/ImageModels.ts +++ b/frontend/libs/model-list/src/lib/lists/ImageModels.ts @@ -2,6 +2,7 @@ import { GenerationProvider } from "@storyteller/api-enums"; import { ImageModel } from "../classes/ImageModel.js"; import { ModelCreator } from "../classes/metadata/ModelCreator.js"; import { ModelTag } from "../classes/metadata/ModelTag.js"; +import { CommonAspectRatio } from "../classes/properties/CommonAspectRatio.js"; // TODO: Some of the model configs, such as generation counts, are authoritatively controlled in `legacy/Models.ts` @@ -147,6 +148,20 @@ export const IMAGE_MODELS: ImageModel[] = [ progressBarTime: 25000, canChangeResolution: true, canChangeAspectRatio: true, + defaultAspectRatio: CommonAspectRatio.Square, + aspectRatios: [ + CommonAspectRatio.Auto, + CommonAspectRatio.Square, + CommonAspectRatio.WideFiveByFour, + CommonAspectRatio.WideFourByThree, + CommonAspectRatio.WideThreeByTwo, + CommonAspectRatio.WideSixteenByNine, + CommonAspectRatio.WideTwentyOneByNine, + CommonAspectRatio.TallFourByFive, + CommonAspectRatio.TallThreeByFour, + CommonAspectRatio.TallTwoByThree, + CommonAspectRatio.TallNineBySixteen, + ] }), new ImageModel({ id: "gpt_image_1", diff --git a/frontend/libs/tauri-api/src/lib/enqueue/EnqueueTextToImage.ts b/frontend/libs/tauri-api/src/lib/enqueue/EnqueueTextToImage.ts index 99a13a1077..17b1d20d03 100644 --- a/frontend/libs/tauri-api/src/lib/enqueue/EnqueueTextToImage.ts +++ b/frontend/libs/tauri-api/src/lib/enqueue/EnqueueTextToImage.ts @@ -1,6 +1,6 @@ import { invoke } from "@tauri-apps/api/core"; import { CommandResult } from "../common/CommandStatus"; -import { Model } from "@storyteller/model-list"; +import { CommonAspectRatio, Model } from "@storyteller/model-list"; import { GenerationProvider } from "@storyteller/api-enums"; export enum EnqueueTextToImageErrorType { @@ -24,9 +24,13 @@ export interface EnqueueTextToImageRequest { // The text prompt. prompt?: string; + // TODO: This is deprecated and will be phased out. // The desired output aspect ratio. aspect_ratio?: EnqueueTextToImageSize; + // This is the new aspect ratio. + common_aspect_ratio?: CommonAspectRatio; + // Image resolution. Support: Nano Banana Pro image_resolution?: EnqueueTextToImageResolution; @@ -54,6 +58,7 @@ interface EnqueueTextToImageRawRequest { provider?: GenerationProvider; prompt?: string; aspect_ratio?: EnqueueTextToImageSize; + common_aspect_ratio?: CommonAspectRatio; image_resolution?: EnqueueTextToImageResolution; number_images?: number; image_media_tokens?: string[]; @@ -126,6 +131,10 @@ export const EnqueueTextToImage = async (request: EnqueueTextToImageRequest) : P mutableRequest.aspect_ratio = request.aspect_ratio; } + if (!!request.common_aspect_ratio) { + mutableRequest.common_aspect_ratio = request.common_aspect_ratio; + } + if (!!request.image_resolution) { mutableRequest.image_resolution = request.image_resolution; }