nano banana frontend options, unified nano banana endpoint, new aspect ratio controls (#947)

This commit is contained in:
Brandon Thomas authored and GitHub committed 2025-12-31 03:21:25 -05:00
1 parent eea31d3f94
commit cbf91709dc
36 files changed
+1035 -37

No files matched your search

@@ -6,4 +6,5 @@ pub mod generate_flux_pro_11_ultra_text_to_image;
pub mod generate_gpt_image_1_text_to_image;
pub mod inpaint;
pub mod multi_function;
pub mod remove_image_background;
pub mod remove_image_background;
pub mod text;
@@ -1,4 +1,5 @@
pub mod bytedance_seedream_v4_multi_function_image_gen;
pub mod bytedance_seedream_v4p5_multi_function_image_gen;
pub mod gpt_image_1p5_multi_function_image_gen;
pub mod nano_banana_multi_function_image_gen;
pub mod nano_banana_pro_multi_function_image_gen;
@@ -0,0 +1,62 @@
use serde_derive::{Deserialize, Serialize};
use tokens::tokens::generic_inference_jobs::InferenceJobToken;
use tokens::tokens::media_files::MediaFileToken;
use utoipa::ToSchema;
pub const NANO_BANANA_MULTI_FUNCTION_IMAGE_GEN_PATH: &str = "/v1/generate/image/multi_function/nano_banana";
#[derive(Serialize, Deserialize, ToSchema, Debug)]
pub struct NanoBananaMultiFunctionImageGenRequest {
/// Idempotency token to prevent duplicate requests.
pub uuid_idempotency_token: String,
/// Text prompt to generate the image from.
pub prompt: Option<String>,
/// Image media tokens to include in the editing context.
/// If present, we're doing image editing (image-to-image / image-editing)
/// If absent, we're doing image generation (text-to-image)
pub image_media_tokens: Option<Vec<MediaFileToken>>,
/// Number of images to generate. Default is one.
pub num_images: Option<NanoBananaMultiFunctionImageGenNumImages>,
/// Aspect ratio of the images to generate. Default is "1:1"
pub aspect_ratio: Option<NanoBananaMultiFunctionImageGenAspectRatio>,
}
#[derive(Copy, Clone, Debug, Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "snake_case")]
pub enum NanoBananaMultiFunctionImageGenNumImages {
One, // Default
Two,
Three,
Four,
}
#[derive(Copy, Clone, Debug, Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "snake_case")]
pub enum NanoBananaMultiFunctionImageGenAspectRatio {
// Auto (only for image editing)
Auto, // Default for image editing.
// Square
OneByOne, // Default for text-to-image
// Wide
FiveByFour,
FourByThree,
ThreeByTwo,
SixteenByNine,
TwentyOneByNine,
// Tall
FourByFive,
ThreeByFour,
TwoByThree,
NineBySixteen, // NB: No NineByTwentyOne ?
}
#[derive(Serialize, Deserialize, ToSchema)]
pub struct NanoBananaMultiFunctionImageGenResponse {
pub success: bool,
pub inference_job_token: InferenceJobToken,
}
@@ -5,7 +5,7 @@ use utoipa::ToSchema;
pub const NANO_BANANA_PRO_MULTI_FUNCTION_IMAGE_GEN_PATH: &str = "/v1/generate/image/multi_function/nano_banana_pro";
#[derive(Serialize, Deserialize, ToSchema)]
#[derive(Serialize, Deserialize, ToSchema, Debug)]
pub struct NanoBananaProMultiFunctionImageGenRequest {
/// Idempotency token to prevent duplicate requests.
pub uuid_idempotency_token: String,
@@ -48,8 +48,10 @@ pub enum NanoBananaProMultiFunctionImageGenImageResolution {
#[derive(Copy, Clone, Debug, Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "snake_case")]
pub enum NanoBananaProMultiFunctionImageGenAspectRatio {
// Auto (only for image editing)
Auto, // Default for image editing.
// Square
OneByOne, // Default
OneByOne, // Default for text-to-image
// Wide
FiveByFour,
FourByThree,
@@ -0,0 +1,50 @@
use serde_derive::{Deserialize, Serialize};
use tokens::tokens::generic_inference_jobs::InferenceJobToken;
use tokens::tokens::media_files::MediaFileToken;
use utoipa::ToSchema;
pub const GEMINI_25_FLASH_EDIT_IMAGE_PATH: &str = "/v1/generate/image/gemini_25_flash";
#[derive(Serialize, Deserialize, ToSchema)]
pub struct Gemini25FlashTextToImageImageRequest {
/// Idempotency token to prevent duplicate requests.
pub uuid_idempotency_token: String,
/// Text prompt to generate the image from.
pub prompt: Option<String>,
// TODO: OneOf for image_urls or image_media_tokens
/// Image media tokens to include in the editing context.
pub image_media_tokens: Option<Vec<MediaFileToken>>,
/// Number of images to generate. Default is one.
pub num_images: Option<Gemini25FlashTextToImageImageNumImages>,
/// Quality of the image to generate. Default is High.
pub image_quality: Option<Gemini25FlashTextToImageImageImageQuality>,
}
#[derive(Copy, Clone, Debug, Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "snake_case")]
pub enum Gemini25FlashTextToImageImageNumImages {
One, // Default
Two,
Three,
Four,
}
#[derive(Copy, Clone, Debug, Serialize, Deserialize, ToSchema)]
#[serde(rename_all = "snake_case")]
pub enum Gemini25FlashTextToImageImageImageQuality {
Auto,
Low,
Medium,
High, // Default
}
#[derive(Serialize, Deserialize, ToSchema)]
pub struct Gemini25FlashTextToImageImageResponse {
pub success: bool,
pub inference_job_token: InferenceJobToken,
}
@@ -0,0 +1 @@
pub mod gemini_25_flash_text_to_image;
@@ -11,6 +11,9 @@ pub struct Gemini25FlashEditArgs<'a, R: IntoUrl> {
pub prompt: &'a str,
pub image_urls: Vec<String>,
pub num_images: Gemini25FlashEditNumImages,
// Optional
pub aspect_ratio: Option<Gemini25FlashEditAspectRatio>,
// Fulfillment
pub webhook_url: R,
@@ -25,6 +28,26 @@ pub enum Gemini25FlashEditNumImages {
Four,
}
/// auto, 21:9, 16:9, 3:2, 4:3, 5:4, 1:1, 4:5, 3:4, 2:3, 9:16
/// Default is "auto"
#[derive(Copy, Clone, Debug)]
pub enum Gemini25FlashEditAspectRatio {
// Automatic (default)
Auto,
// Square
OneByOne,
// Wide
FiveByFour,
FourByThree,
ThreeByTwo,
SixteenByNine,
TwentyOneByNine,
// Tall
FourByFive,
ThreeByFour,
TwoByThree,
NineBySixteen, // NB: No NineByTwentyOne ?
}
pub async fn enqueue_gemini_25_flash_edit_webhook<R: IntoUrl>(
args: Gemini25FlashEditArgs<'_, R>
@@ -37,9 +60,30 @@ pub async fn enqueue_gemini_25_flash_edit_webhook<R: IntoUrl>(
Gemini25FlashEditNumImages::Four => 4,
};
let aspect_ratio = args.aspect_ratio
.map(|aspect_ratio| match aspect_ratio {
// Auto
Gemini25FlashEditAspectRatio::Auto => "auto",
// Square
Gemini25FlashEditAspectRatio::OneByOne => "1:1",
// Wide
Gemini25FlashEditAspectRatio::FiveByFour => "5:4",
Gemini25FlashEditAspectRatio::FourByThree => "4:3",
Gemini25FlashEditAspectRatio::ThreeByTwo => "3:2",
Gemini25FlashEditAspectRatio::SixteenByNine => "16:9",
Gemini25FlashEditAspectRatio::TwentyOneByNine => "21:9",
// Tall
Gemini25FlashEditAspectRatio::FourByFive => "4:5",
Gemini25FlashEditAspectRatio::ThreeByFour => "3:4",
Gemini25FlashEditAspectRatio::TwoByThree => "2:3",
Gemini25FlashEditAspectRatio::NineBySixteen => "9:16",
})
.map(|aspect_ratio| aspect_ratio.to_string());
let request = Gemini25FlashImageEditInput {
prompt: args.prompt.to_string(),
image_urls: args.image_urls,
aspect_ratio,
// Constants
num_images: Some(num_images),
output_format: Some("png".to_string()),
@@ -75,6 +119,7 @@ mod tests {
TREX_SKELETON_IMAGE_URL.to_string(),
],
num_images: Gemini25FlashEditNumImages::Two,
aspect_ratio: None,
prompt: "add the ghost to the image of the t-rex skeleton, make it look spooky but friendly",
api_key: &api_key,
webhook_url: "https://example.com/webhook",
@@ -30,15 +30,17 @@ pub enum EnqueueNanoBananaProEditImageNumImages {
#[derive(Copy, Clone, Debug)]
pub enum EnqueueNanoBananaProEditImageResolution {
OneK,
OneK, // Default value "1K"
TwoK,
FourK,
}
/// 21:9, 16:9, 3:2, 4:3, 5:4, 1:1, 4:5, 3:4, 2:3, 9:16
/// Default is "1:1"
/// auto, 21:9, 16:9, 3:2, 4:3, 5:4, 1:1, 4:5, 3:4, 2:3, 9:16
/// Default is "auto"
#[derive(Copy, Clone, Debug)]
pub enum EnqueueNanoBananaProEditImageAspectRatio {
// Automatic (default)
Auto,
// Square
OneByOne,
// Wide
@@ -75,6 +77,8 @@ pub async fn enqueue_nano_banana_pro_image_edit_webhook<R: IntoUrl>(
let aspect_ratio = args.aspect_ratio
.map(|aspect_ratio| match aspect_ratio {
// Auto
EnqueueNanoBananaProEditImageAspectRatio::Auto => "auto",
// Square
EnqueueNanoBananaProEditImageAspectRatio::OneByOne => "1:1",
// Wide
@@ -0,0 +1,121 @@
use crate::creds::fal_api_key::FalApiKey;
use crate::error::classify_fal_error::classify_fal_error;
use crate::error::fal_error_plus::FalErrorPlus;
use fal::prelude::fal_ai::gemini_25_flash_image::text_to_image::gemini_25_flash_text_to_image;
use fal::prelude::fal_ai::gemini_25_flash_image::text_to_image::Gemini25FlashTextToImageInput;
use fal::webhook::WebhookResponse;
use reqwest::IntoUrl;
pub struct Gemini25FlashTextToImageArgs<'a, R: IntoUrl> {
// Request required
pub prompt: &'a str,
pub num_images: Gemini25FlashTextToImageNumImages,
// Optional
pub aspect_ratio: Option<Gemini25FlashTextToImageAspectRatio>,
// Fulfillment
pub webhook_url: R,
pub api_key: &'a FalApiKey,
}
#[derive(Copy, Clone, Debug)]
pub enum Gemini25FlashTextToImageNumImages {
One,
Two,
Three,
Four,
}
/// 21:9, 16:9, 3:2, 4:3, 5:4, 1:1, 4:5, 3:4, 2:3, 9:16
/// Default is "1:1"
#[derive(Copy, Clone, Debug)]
pub enum Gemini25FlashTextToImageAspectRatio {
// Square
OneByOne,
// Wide
FiveByFour,
FourByThree,
ThreeByTwo,
SixteenByNine,
TwentyOneByNine,
// Tall
FourByFive,
ThreeByFour,
TwoByThree,
NineBySixteen, // NB: No NineByTwentyOne ?
}
pub async fn enqueue_gemini_25_flash_text_to_image_webhook<R: IntoUrl>(
args: Gemini25FlashTextToImageArgs<'_, R>
) -> Result<WebhookResponse, FalErrorPlus> {
let num_images = match args.num_images {
Gemini25FlashTextToImageNumImages::One => 1,
Gemini25FlashTextToImageNumImages::Two => 2,
Gemini25FlashTextToImageNumImages::Three => 3,
Gemini25FlashTextToImageNumImages::Four => 4,
};
let aspect_ratio = args.aspect_ratio
.map(|aspect_ratio| match aspect_ratio {
// Square
Gemini25FlashTextToImageAspectRatio::OneByOne => "1:1",
// Wide
Gemini25FlashTextToImageAspectRatio::FiveByFour => "5:4",
Gemini25FlashTextToImageAspectRatio::FourByThree => "4:3",
Gemini25FlashTextToImageAspectRatio::ThreeByTwo => "3:2",
Gemini25FlashTextToImageAspectRatio::SixteenByNine => "16:9",
Gemini25FlashTextToImageAspectRatio::TwentyOneByNine => "21:9",
// Tall
Gemini25FlashTextToImageAspectRatio::FourByFive => "4:5",
Gemini25FlashTextToImageAspectRatio::ThreeByFour => "3:4",
Gemini25FlashTextToImageAspectRatio::TwoByThree => "2:3",
Gemini25FlashTextToImageAspectRatio::NineBySixteen => "9:16",
})
.map(|aspect_ratio| aspect_ratio.to_string());
let request = Gemini25FlashTextToImageInput {
prompt: args.prompt.to_string(),
aspect_ratio,
// Constants
num_images: Some(num_images),
output_format: Some("png".to_string()),
};
let result = gemini_25_flash_text_to_image(request)
.with_api_key(&args.api_key.0)
.queue_webhook(args.webhook_url)
.await;
result.map_err(|err| classify_fal_error(err))
}
#[cfg(test)]
mod tests {
use crate::creds::fal_api_key::FalApiKey;
use crate::requests::webhook::image::text::enqueue_gemini_25_flash_text_to_image_webhook::{enqueue_gemini_25_flash_text_to_image_webhook, Gemini25FlashTextToImageArgs, Gemini25FlashTextToImageNumImages};
use errors::AnyhowResult;
use std::fs::read_to_string;
#[tokio::test]
#[ignore]
async fn test() -> AnyhowResult<()> {
// XXX: Don't commit secrets!
let secret = read_to_string("/Users/bt/Artcraft/credentials/fal_api_key.txt")?;
let api_key = FalApiKey::from_str(&secret);
let args = Gemini25FlashTextToImageArgs {
prompt: "a warrior on the field of battle, final fantasy style, lots of hungry raptors rushing the heroes",
num_images: Gemini25FlashTextToImageNumImages::Two,
aspect_ratio: None,
api_key: &api_key,
webhook_url: "https://example.com/webhook",
};
let result = enqueue_gemini_25_flash_text_to_image_webhook(args).await?;
Ok(())
}
}
@@ -1,4 +1,5 @@
pub mod enqueue_bytedance_seedream_v4_text_to_image_webhook;
pub mod enqueue_bytedance_seedream_v4p5_text_to_image_webhook;
pub mod enqueue_nano_banana_pro_text_to_image_webhook;
pub mod enqueue_gpt_image_1p5_text_to_image_webhook;
pub mod enqueue_gemini_25_flash_text_to_image_webhook;
pub mod enqueue_gpt_image_1p5_text_to_image_webhook;
pub mod enqueue_nano_banana_pro_text_to_image_webhook;
@@ -0,0 +1,25 @@
use serde_derive::{Deserialize, Serialize};
/// This is a comprehensive list of common aspect ratios.
/// Not every model will support every aspect ratio.
/// In the case a model doesn't support the aspect ratio, pick the nearest option.
#[derive(Copy, Clone, Debug, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum CommonAspectRatio {
// Auto (eg. for image editing to use the source; used by Nano Banana Pro edit image, but not text-to-image)
Auto,
// Square
Square,
// Wide
WideFiveByFour,
WideFourByThree,
WideThreeByTwo,
WideSixteenByNine,
WideTwentyOneByNine,
// Tall
TallFourByFive,
TallThreeByFour,
TallTwoByThree,
TallNineBySixteen,
TallNineByTwentyOne,
}
@@ -0,0 +1,23 @@
use crate::core::api_adapters::aspect_ratio::common_aspect_ratio::CommonAspectRatio;
use artcraft_api_defs::generate::image::multi_function::nano_banana_pro_multi_function_image_gen::NanoBananaProMultiFunctionImageGenAspectRatio;
pub fn aspect_ratio_to_artcraft_nano_banana_pro(aspect_ratio: CommonAspectRatio) -> NanoBananaProMultiFunctionImageGenAspectRatio {
match aspect_ratio {
// Exact
CommonAspectRatio::Square => NanoBananaProMultiFunctionImageGenAspectRatio::OneByOne,
CommonAspectRatio::WideFiveByFour => NanoBananaProMultiFunctionImageGenAspectRatio::FiveByFour,
CommonAspectRatio::WideFourByThree => NanoBananaProMultiFunctionImageGenAspectRatio::FourByThree,
CommonAspectRatio::WideThreeByTwo => NanoBananaProMultiFunctionImageGenAspectRatio::ThreeByTwo,
CommonAspectRatio::WideSixteenByNine => NanoBananaProMultiFunctionImageGenAspectRatio::SixteenByNine,
CommonAspectRatio::WideTwentyOneByNine => NanoBananaProMultiFunctionImageGenAspectRatio::TwentyOneByNine,
CommonAspectRatio::TallFourByFive => NanoBananaProMultiFunctionImageGenAspectRatio::FourByFive,
CommonAspectRatio::TallThreeByFour => NanoBananaProMultiFunctionImageGenAspectRatio::ThreeByFour,
CommonAspectRatio::TallTwoByThree => NanoBananaProMultiFunctionImageGenAspectRatio::TwoByThree,
CommonAspectRatio::TallNineBySixteen => NanoBananaProMultiFunctionImageGenAspectRatio::NineBySixteen,
// Special
// NB: this is only for image-to-image editing, not text-to-image.
CommonAspectRatio::Auto => NanoBananaProMultiFunctionImageGenAspectRatio::Auto,
// NB: "9:21" value does not exist. Interpolated.
CommonAspectRatio::TallNineByTwentyOne => NanoBananaProMultiFunctionImageGenAspectRatio::NineBySixteen,
}
}
@@ -0,0 +1 @@
pub mod aspect_ratio_to_artcraft_nano_banana_pro;
@@ -0,0 +1,2 @@
pub mod common_aspect_ratio;
pub mod convert;
@@ -0,0 +1 @@
pub mod aspect_ratio;
@@ -1,3 +1,3 @@
pub mod notify_frontend_of_errors;
mod maybe_notify_frontend_of_billing_errors;
mod maybe_notify_frontend_of_grok_errors;
mod maybe_notify_frontend_of_grok_errors;
pub mod notify_frontend_of_errors;
@@ -1,6 +1,9 @@
use crate::core::api_adapters::aspect_ratio::convert::aspect_ratio_to_artcraft_nano_banana_pro::aspect_ratio_to_artcraft_nano_banana_pro;
use crate::core::commands::enqueue::generate_error::{BadInputReason, GenerateError};
use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::ImageEditModel::NanoBanana;
use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::{EditImageQuality, EditImageResolution, EditImageSize, EnqueueEditImageCommand};
use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess;
use crate::core::commands::enqueue::text_to_image::enqueue_text_to_image_command::{EnqueueTextToImageRequest, TextToImageSize};
use crate::core::events::basic_sendable_event_trait::BasicSendableEvent;
use crate::core::events::generation_events::common::{GenerationAction, GenerationModel, GenerationServiceProvider};
use crate::core::events::generation_events::generation_enqueue_failure_event::GenerationEnqueueFailureEvent;
@@ -18,6 +21,7 @@ use artcraft_api_defs::generate::image::generate_flux_1_dev_text_to_image::Gener
use artcraft_api_defs::generate::image::generate_flux_1_schnell_text_to_image::GenerateFlux1SchnellTextToImageRequest;
use artcraft_api_defs::generate::image::generate_flux_pro_11_text_to_image::GenerateFluxPro11TextToImageRequest;
use artcraft_api_defs::generate::image::generate_flux_pro_11_ultra_text_to_image::GenerateFluxPro11UltraTextToImageRequest;
use artcraft_api_defs::generate::image::multi_function::nano_banana_pro_multi_function_image_gen::{NanoBananaProMultiFunctionImageGenAspectRatio, NanoBananaProMultiFunctionImageGenImageResolution, NanoBananaProMultiFunctionImageGenNumImages, NanoBananaProMultiFunctionImageGenRequest};
use enums::common::generation_provider::GenerationProvider;
use enums::tauri::tasks::task_type::TaskType;
use idempotency::uuid::generate_random_uuid;
@@ -25,10 +29,8 @@ use log::{error, info};
use storyteller_client::endpoints::generate::image::edit::flux_pro_kontext_max_edit_image::flux_pro_kontext_max_edit_image;
use storyteller_client::endpoints::generate::image::edit::gemini_25_flash_edit_image::gemini_25_flash_edit_image;
use storyteller_client::endpoints::generate::image::edit::gpt_image_1_edit_image::gpt_image_1_edit_image;
use tauri::AppHandle;
use artcraft_api_defs::generate::image::multi_function::nano_banana_pro_multi_function_image_gen::{NanoBananaProMultiFunctionImageGenAspectRatio, NanoBananaProMultiFunctionImageGenImageResolution, NanoBananaProMultiFunctionImageGenNumImages, NanoBananaProMultiFunctionImageGenRequest};
use storyteller_client::endpoints::generate::image::multi_function::nano_banana_pro_multi_function_image_gen_image::nano_banana_pro_multi_function_image_gen;
use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::ImageEditModel::NanoBanana;
use tauri::AppHandle;
pub(super) const MAX_IMAGES: usize = 10;
@@ -66,14 +68,7 @@ pub async fn handle_artcraft_nano_banana_pro_edit(
},
};
let aspect_ratio = request.aspect_ratio
.map(|aspect_ratio| match aspect_ratio {
EditImageSize::Auto => None,
EditImageSize::Square => Some(NanoBananaProMultiFunctionImageGenAspectRatio::OneByOne),
EditImageSize::Wide => Some(NanoBananaProMultiFunctionImageGenAspectRatio::SixteenByNine),
EditImageSize::Tall => Some(NanoBananaProMultiFunctionImageGenAspectRatio::NineBySixteen),
})
.flatten();
let aspect_ratio = get_aspect_ratio(request);
let resolution = request.image_resolution
.map(|image_resolution| match image_resolution {
@@ -135,3 +130,23 @@ pub async fn handle_artcraft_nano_banana_pro_edit(
task_type: TaskType::ImageGeneration,
})
}
fn get_aspect_ratio(request: &EnqueueEditImageCommand) -> Option<NanoBananaProMultiFunctionImageGenAspectRatio> {
if let Some(common_aspect_ratio) = request.common_aspect_ratio {
// Handle modern aspect ratio
let aspect = aspect_ratio_to_artcraft_nano_banana_pro(common_aspect_ratio);
return Some(aspect);
}
if let Some(aspect_ratio) = request.aspect_ratio {
// Handle deprecated aspect ratio
return match aspect_ratio {
EditImageSize::Auto => Some(NanoBananaProMultiFunctionImageGenAspectRatio::Auto),
EditImageSize::Square => Some(NanoBananaProMultiFunctionImageGenAspectRatio::OneByOne),
EditImageSize::Wide => Some(NanoBananaProMultiFunctionImageGenAspectRatio::SixteenByNine),
EditImageSize::Tall => Some(NanoBananaProMultiFunctionImageGenAspectRatio::NineBySixteen),
}
}
None
}
@@ -1,3 +1,4 @@
use crate::core::api_adapters::aspect_ratio::common_aspect_ratio::CommonAspectRatio;
use crate::core::commands::enqueue::common::notify_frontend_of_errors::notify_frontend_of_errors;
use crate::core::commands::enqueue::generate_error::{BadInputReason, GenerateError};
use crate::core::commands::enqueue::image_edit::artcraft::handle_image_edit_artcraft::handle_image_edit_artcraft;
@@ -108,8 +109,13 @@ pub struct EnqueueEditImageCommand {
pub image_count: Option<u32>,
/// Aspect ratio.
#[deprecated(note="use common_aspect_ratio")]
pub aspect_ratio: Option<EditImageSize>,
/// New field for aspect ratio.
/// Not all models support each of these aspect ratios, but we can choose or interpolate sensibly.
pub common_aspect_ratio: Option<CommonAspectRatio>,
/// Image quality.
pub image_quality: Option<EditImageQuality>,
@@ -1,3 +1,5 @@
use crate::core::api_adapters::aspect_ratio::common_aspect_ratio::CommonAspectRatio;
use crate::core::api_adapters::aspect_ratio::convert::aspect_ratio_to_artcraft_nano_banana_pro::aspect_ratio_to_artcraft_nano_banana_pro;
use crate::core::commands::enqueue::generate_error::{BadInputReason, GenerateError, MissingCredentialsReason, ProviderFailureReason};
use crate::core::commands::enqueue::image_edit::enqueue_edit_image_command::{EditImageQuality, EditImageResolution, EditImageSize};
use crate::core::commands::enqueue::task_enqueue_success::TaskEnqueueSuccess;
@@ -58,14 +60,7 @@ pub async fn handle_artcraft_nano_banana_pro_text_to_image(
},
};
let aspect_ratio = request.aspect_ratio
.map(|aspect_ratio| match aspect_ratio {
TextToImageSize::Auto => None,
TextToImageSize::Square => Some(NanoBananaProMultiFunctionImageGenAspectRatio::OneByOne),
TextToImageSize::Wide => Some(NanoBananaProMultiFunctionImageGenAspectRatio::SixteenByNine),
TextToImageSize::Tall => Some(NanoBananaProMultiFunctionImageGenAspectRatio::NineBySixteen),
})
.flatten();
let aspect_ratio = get_aspect_ratio(request);
let resolution = request.image_resolution
.map(|image_resolution| match image_resolution {
@@ -82,6 +77,8 @@ pub async fn handle_artcraft_nano_banana_pro_text_to_image(
resolution,
aspect_ratio,
};
info!("Calling Artcraft with request {:?}", request);
let result = nano_banana_pro_multi_function_image_gen(
&app_env_configs.storyteller_host,
@@ -109,3 +106,23 @@ pub async fn handle_artcraft_nano_banana_pro_text_to_image(
task_type: TaskType::ImageGeneration,
})
}
fn get_aspect_ratio(request: &EnqueueTextToImageRequest) -> Option<NanoBananaProMultiFunctionImageGenAspectRatio> {
if let Some(common_aspect_ratio) = request.common_aspect_ratio {
// Handle modern aspect ratio
let aspect = aspect_ratio_to_artcraft_nano_banana_pro(common_aspect_ratio);
return Some(aspect);
}
if let Some(aspect_ratio) = request.aspect_ratio {
// Handle deprecated aspect ratio
return match aspect_ratio {
TextToImageSize::Auto => Some(NanoBananaProMultiFunctionImageGenAspectRatio::Auto),
TextToImageSize::Square => Some(NanoBananaProMultiFunctionImageGenAspectRatio::OneByOne),
TextToImageSize::Wide => Some(NanoBananaProMultiFunctionImageGenAspectRatio::SixteenByNine),
TextToImageSize::Tall => Some(NanoBananaProMultiFunctionImageGenAspectRatio::NineBySixteen),
}
}
None
}
@@ -34,6 +34,7 @@ use serde::{Deserialize, Serialize};
use sqlite_tasks::queries::create_task::{create_task, CreateTaskArgs};
use tauri::{AppHandle, State};
use tokens::tokens::media_files::MediaFileToken;
use crate::core::api_adapters::aspect_ratio::common_aspect_ratio::CommonAspectRatio;
/// This is used in the Tauri command bridge.
/// Don't change the serializations without coordinating with the frontend.
@@ -73,7 +74,7 @@ pub enum TextToImageModel {
Midjourney,
}
#[derive(Deserialize)]
#[derive(Deserialize, Debug)]
pub struct EnqueueTextToImageRequest {
/// The provider to use (defaults to Artcraft/Storyteller).
/// Not all (provider, model) combinations are valid.
@@ -86,7 +87,12 @@ pub struct EnqueueTextToImageRequest {
pub prompt: Option<String>,
/// Aspect ratio.
#[deprecated(note="use common_aspect_ratio")]
pub aspect_ratio: Option<TextToImageSize>,
/// New field for aspect ratio.
/// Not all models support each of these aspect ratios, but we can choose or interpolate sensibly.
pub common_aspect_ratio: Option<CommonAspectRatio>,
/// Aspect ratio.
pub image_resolution: Option<TextToImageResolution>,
@@ -185,6 +191,8 @@ pub async fn enqueue_text_to_image_command(
info!("enqueue_text_to_image called");
info!("request: {:?}", request);
let result = handle_request(
request,
&app,
+1
View File
@@ -1,3 +1,4 @@
pub mod api_adapters;
pub mod artcraft_error;
pub mod commands;
pub mod events;
@@ -44,6 +44,7 @@ use utoipa::ToSchema;
("request" = Gemini25FlashEditImageRequest, description = "Payload for Request"),
)
)]
#[deprecated(note="use `nano_banana_multi_function_image_gen_handler` instead")]
pub async fn gemini_25_flash_edit_image_handler(
http_request: HttpRequest,
request: Json<Gemini25FlashEditImageRequest>,
@@ -148,13 +149,14 @@ pub async fn gemini_25_flash_edit_image_handler(
Some(Gemini25FlashEditImageNumImages::Four) => Gemini25FlashEditNumImages::Four,
None => Gemini25FlashEditNumImages::One, // Default to One
};
let args = Gemini25FlashEditArgs {
image_urls,
prompt: request.prompt.as_deref().unwrap_or(""),
webhook_url: &server_state.fal.webhook_url,
api_key: &server_state.fal.api_key,
num_images,
aspect_ratio: None, // TODO
};
let fal_result = enqueue_gemini_25_flash_edit_webhook(args)
@@ -1,4 +1,5 @@
pub mod bytedance_seedream_v4_multi_function_image_gen_handler;
pub mod bytedance_seedream_v4p5_multi_function_image_gen_handler;
pub mod gpt_image_1p5_multi_function_image_gen_handler;
pub mod nano_banana_multi_function_image_gen_handler;
pub mod nano_banana_pro_multi_function_image_gen_handler;
@@ -0,0 +1,298 @@
use std::collections::HashSet;
use std::iter::FromIterator;
use std::sync::Arc;
use crate::http_server::common_responses::common_web_error::CommonWebError;
use crate::http_server::common_responses::media::media_links_builder::MediaLinksBuilder;
use crate::http_server::endpoints::generate::common::payments_error_test::payments_error_test;
use crate::http_server::endpoints::media_files::helpers::get_media_domain::get_media_domain;
use crate::http_server::validations::validate_idempotency_token_format::validate_idempotency_token_format;
use crate::state::server_state::ServerState;
use crate::util::lookup::lookup_image_urls_as_optional_list::lookup_image_urls_as_optional_list;
use actix_web::web::Json;
use actix_web::{web, HttpRequest};
use artcraft_api_defs::generate::image::edit::gemini_25_flash_edit_image::{Gemini25FlashEditImageNumImages, Gemini25FlashEditImageRequest};
use artcraft_api_defs::generate::image::multi_function::nano_banana_multi_function_image_gen::{NanoBananaMultiFunctionImageGenAspectRatio, NanoBananaMultiFunctionImageGenNumImages, NanoBananaMultiFunctionImageGenRequest, NanoBananaMultiFunctionImageGenResponse};
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use enums::by_table::prompt_context_items::prompt_context_semantic_type::PromptContextSemanticType;
use enums::by_table::prompts::prompt_type::PromptType;
use enums::common::generation_provider::GenerationProvider;
use enums::common::model_type::ModelType;
use enums::common::visibility::Visibility;
use fal_client::creds::open_ai_api_key::OpenAiApiKey;
use fal_client::requests::webhook::image::edit::enqueue_gemini_25_flash_edit_webhook::{enqueue_gemini_25_flash_edit_webhook, Gemini25FlashEditArgs, Gemini25FlashEditAspectRatio, Gemini25FlashEditNumImages};
use fal_client::requests::webhook::image::edit::enqueue_nano_banana_pro_edit_image_webhook::{enqueue_nano_banana_pro_image_edit_webhook, EnqueueNanoBananaProEditImageArgs, EnqueueNanoBananaProEditImageAspectRatio, EnqueueNanoBananaProEditImageNumImages, EnqueueNanoBananaProEditImageResolution};
use fal_client::requests::webhook::image::text::enqueue_gemini_25_flash_text_to_image_webhook::{enqueue_gemini_25_flash_text_to_image_webhook, Gemini25FlashTextToImageArgs, Gemini25FlashTextToImageAspectRatio, Gemini25FlashTextToImageNumImages};
use fal_client::requests::webhook::image::text::enqueue_nano_banana_pro_text_to_image_webhook::{enqueue_nano_banana_pro_text_to_image_webhook, EnqueueNanoBananaProTextToImageArgs, EnqueueNanoBananaProTextToImageNumImages, EnqueueNanoBananaProTextToImageResolution};
use http_server_common::request::get_request_ip::get_request_ip;
use log::{error, info, warn};
use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::insert_generic_inference_job_for_fal_queue;
use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::FalCategory;
use mysql_queries::queries::generic_inference::fal::insert_generic_inference_job_for_fal_queue::InsertGenericInferenceForFalArgs;
use mysql_queries::queries::idepotency_tokens::insert_idempotency_token::insert_idempotency_token;
use mysql_queries::queries::media_files::get::batch_get_media_files_by_tokens::{batch_get_media_files_by_tokens, batch_get_media_files_by_tokens_with_connection};
use mysql_queries::queries::prompt_context_items::insert_batch_prompt_context_items::{insert_batch_prompt_context_items, InsertBatchArgs, PromptContextItem};
use mysql_queries::queries::prompts::insert_prompt::{insert_prompt, InsertPromptArgs};
use server_environment::ServerEnvironment;
use sqlx::pool::PoolConnection;
use sqlx::{Acquire, MySql};
use tokens::tokens::media_files::MediaFileToken;
use utoipa::ToSchema;
/// Nano Banana Multi-Function (generate + edit)
#[utoipa::path(
post,
tag = "Generate Images (Multi-Function)",
path = "/v1/generate/image/multi_function/nano_banana",
responses(
(status = 200, description = "Success", body = NanoBananaMultiFunctionImageGenResponse),
),
params(
("request" = NanoBananaMultiFunctionImageGenRequest, description = "Payload for Request"),
)
)]
pub async fn nano_banana_multi_function_image_gen_handler(
http_request: HttpRequest,
request: Json<NanoBananaMultiFunctionImageGenRequest>,
server_state: web::Data<Arc<ServerState>>
) -> Result<Json<NanoBananaMultiFunctionImageGenResponse>, CommonWebError> {
payments_error_test(&request.prompt.as_deref().unwrap_or(""))?;
let mut mysql_connection = server_state.mysql_pool
.acquire()
.await?;
let maybe_user_session = server_state
.session_checker
.maybe_get_user_session_from_connection(&http_request, &mut mysql_connection)
.await
.map_err(|e| {
warn!("Session checker error: {:?}", e);
CommonWebError::ServerError
})?;
let maybe_avt_token = server_state
.avt_cookie_manager
.get_avt_token_from_request(&http_request);
// TODO: Limit usage for new accounts. Billing, free credits metering, etc.
//let user_session = match maybe_user_session {
// Some(session) => session,
// None => {
// warn!("not logged in");
// return Err(CommonWebError::NotAuthorized);
// }
//};
if let Err(reason) = validate_idempotency_token_format(&request.uuid_idempotency_token) {
return Err(CommonWebError::BadInputWithSimpleMessage(reason));
}
let image_urls = match request.image_media_tokens.as_ref() {
Some(media_tokens) => {
info!("Looking up image media tokens: {:?}", media_tokens);
lookup_image_urls_as_optional_list(
&http_request,
&mut mysql_connection,
server_state.server_environment,
media_tokens,
).await?
},
None => None,
};
insert_idempotency_token(&request.uuid_idempotency_token, &mut *mysql_connection)
.await
.map_err(|err| {
error!("Error inserting idempotency token: {:?}", err);
CommonWebError::BadInputWithSimpleMessage("repeated idempotency token".to_string())
})?;
info!("Fal webhook URL: {}", server_state.fal.webhook_url);
let fal_result;
if let Some(input_image_urls) = image_urls.as_deref() {
info!("nano banana edit image");
let num_images = match request.num_images {
Some(NanoBananaMultiFunctionImageGenNumImages::One) => Gemini25FlashEditNumImages::One,
Some(NanoBananaMultiFunctionImageGenNumImages::Two) => Gemini25FlashEditNumImages::Two,
Some(NanoBananaMultiFunctionImageGenNumImages::Three) => Gemini25FlashEditNumImages::Three,
Some(NanoBananaMultiFunctionImageGenNumImages::Four) => Gemini25FlashEditNumImages::Four,
None => Gemini25FlashEditNumImages::One, // Default to One
};
let aspect_ratio = match request.aspect_ratio {
Some(NanoBananaMultiFunctionImageGenAspectRatio::Auto) => Gemini25FlashEditAspectRatio::Auto,
Some(NanoBananaMultiFunctionImageGenAspectRatio::OneByOne) => Gemini25FlashEditAspectRatio::OneByOne,
Some(NanoBananaMultiFunctionImageGenAspectRatio::FiveByFour) => Gemini25FlashEditAspectRatio::FiveByFour,
Some(NanoBananaMultiFunctionImageGenAspectRatio::FourByThree) => Gemini25FlashEditAspectRatio::FourByThree,
Some(NanoBananaMultiFunctionImageGenAspectRatio::ThreeByTwo) => Gemini25FlashEditAspectRatio::ThreeByTwo,
Some(NanoBananaMultiFunctionImageGenAspectRatio::SixteenByNine) => Gemini25FlashEditAspectRatio::SixteenByNine,
Some(NanoBananaMultiFunctionImageGenAspectRatio::TwentyOneByNine) => Gemini25FlashEditAspectRatio::TwentyOneByNine,
Some(NanoBananaMultiFunctionImageGenAspectRatio::FourByFive) => Gemini25FlashEditAspectRatio::FourByFive,
Some(NanoBananaMultiFunctionImageGenAspectRatio::ThreeByFour) => Gemini25FlashEditAspectRatio::ThreeByFour,
Some(NanoBananaMultiFunctionImageGenAspectRatio::TwoByThree) => Gemini25FlashEditAspectRatio::TwoByThree,
Some(NanoBananaMultiFunctionImageGenAspectRatio::NineBySixteen) => Gemini25FlashEditAspectRatio::NineBySixteen,
None => Gemini25FlashEditAspectRatio::OneByOne,
};
let args = Gemini25FlashEditArgs {
prompt: request.prompt.as_deref().unwrap_or(""),
image_urls: input_image_urls.to_owned(),
num_images,
aspect_ratio: Some(aspect_ratio),
webhook_url: &server_state.fal.webhook_url,
api_key: &server_state.fal.api_key,
};
fal_result = enqueue_gemini_25_flash_edit_webhook(args)
.await
.map_err(|err| {
warn!("Error calling enqueue_gemini_25_flash_edit_webhook : {:?}", err);
CommonWebError::ServerError
})?;
} else {
info!("nano banana text-to-image");
let num_images = match request.num_images {
Some(NanoBananaMultiFunctionImageGenNumImages::One) => Gemini25FlashTextToImageNumImages::One,
Some(NanoBananaMultiFunctionImageGenNumImages::Two) => Gemini25FlashTextToImageNumImages::Two,
Some(NanoBananaMultiFunctionImageGenNumImages::Three) => Gemini25FlashTextToImageNumImages::Three,
Some(NanoBananaMultiFunctionImageGenNumImages::Four) => Gemini25FlashTextToImageNumImages::Four,
None => Gemini25FlashTextToImageNumImages::One, // Default to One
};
let aspect_ratio = match request.aspect_ratio {
Some(NanoBananaMultiFunctionImageGenAspectRatio::Auto) => Gemini25FlashTextToImageAspectRatio::OneByOne, // NB: "auto" is only for edit image, not text-to-image !
Some(NanoBananaMultiFunctionImageGenAspectRatio::OneByOne) => Gemini25FlashTextToImageAspectRatio::OneByOne,
Some(NanoBananaMultiFunctionImageGenAspectRatio::FiveByFour) => Gemini25FlashTextToImageAspectRatio::FiveByFour,
Some(NanoBananaMultiFunctionImageGenAspectRatio::FourByThree) => Gemini25FlashTextToImageAspectRatio::FourByThree,
Some(NanoBananaMultiFunctionImageGenAspectRatio::ThreeByTwo) => Gemini25FlashTextToImageAspectRatio::ThreeByTwo,
Some(NanoBananaMultiFunctionImageGenAspectRatio::SixteenByNine) => Gemini25FlashTextToImageAspectRatio::SixteenByNine,
Some(NanoBananaMultiFunctionImageGenAspectRatio::TwentyOneByNine) => Gemini25FlashTextToImageAspectRatio::TwentyOneByNine,
Some(NanoBananaMultiFunctionImageGenAspectRatio::FourByFive) => Gemini25FlashTextToImageAspectRatio::FourByFive,
Some(NanoBananaMultiFunctionImageGenAspectRatio::ThreeByFour) => Gemini25FlashTextToImageAspectRatio::ThreeByFour,
Some(NanoBananaMultiFunctionImageGenAspectRatio::TwoByThree) => Gemini25FlashTextToImageAspectRatio::TwoByThree,
Some(NanoBananaMultiFunctionImageGenAspectRatio::NineBySixteen) => Gemini25FlashTextToImageAspectRatio::NineBySixteen,
None => Gemini25FlashTextToImageAspectRatio::OneByOne,
};
let args = Gemini25FlashTextToImageArgs {
prompt: request.prompt.as_deref().unwrap_or(""),
num_images,
aspect_ratio: Some(aspect_ratio),
webhook_url: &server_state.fal.webhook_url,
api_key: &server_state.fal.api_key,
};
fal_result = enqueue_gemini_25_flash_text_to_image_webhook(args)
.await
.map_err(|err| {
warn!("Error calling enqueue_gemini_25_flash_text_to_image_webhook: {:?}", err);
CommonWebError::ServerError
})?;
}
let external_job_id = fal_result.request_id
.ok_or_else(|| {
warn!("Fal request_id is None");
CommonWebError::ServerError
})?;
info!("Fal request_id: {}", external_job_id);
let ip_address = get_request_ip(&http_request);
let mut transaction = mysql_connection
.begin()
.await
.map_err(|err| {
error!("Error starting MySQL transaction: {:?}", err);
CommonWebError::ServerError
})?;
// NB: Don't fail the job if the query fails.
let prompt_result = insert_prompt(InsertPromptArgs {
maybe_apriori_prompt_token: None,
prompt_type: PromptType::ArtcraftApp,
maybe_creator_user_token: maybe_user_session
.as_ref()
.map(|s| &s.user_token),
maybe_model_type: Some(ModelType::NanoBanana),
maybe_generation_provider: Some(GenerationProvider::Artcraft),
maybe_positive_prompt: request.prompt.as_deref(),
maybe_negative_prompt: None,
maybe_other_args: None,
creator_ip_address: &ip_address,
mysql_executor: &mut *transaction,
phantom: Default::default(),
}).await;
let prompt_token = match prompt_result {
Ok(token) => Some(token),
Err(err) => {
warn!("Error inserting prompt: {:?}", err);
None // Don't fail the job if the prompt insertion fails.
}
};
if let Some(media_tokens) = &request.image_media_tokens {
if let Some(token) = prompt_token.as_ref() {
let result = insert_batch_prompt_context_items(InsertBatchArgs {
prompt_token: token.clone(),
items: media_tokens.iter().map(|token| {
PromptContextItem {
media_token: token.clone(),
context_semantic_type: PromptContextSemanticType::Imgref,
}
}).collect(),
transaction: &mut transaction,
}).await;
if let Err(err) = result {
// NB: Fail open.
warn!("Error inserting batch prompt context items: {:?}", err);
}
}
}
let db_result = insert_generic_inference_job_for_fal_queue(InsertGenericInferenceForFalArgs {
uuid_idempotency_token: &request.uuid_idempotency_token,
maybe_external_third_party_id: &external_job_id,
fal_category: FalCategory::ImageGeneration,
maybe_inference_args: None,
maybe_prompt_token: prompt_token.as_ref(),
maybe_creator_user_token: maybe_user_session.as_ref().map(|s| &s.user_token),
maybe_avt_token: maybe_avt_token.as_ref(),
creator_ip_address: &ip_address,
creator_set_visibility: Visibility::Public,
mysql_executor: &mut *transaction,
phantom: Default::default(),
}).await;
let job_token = match db_result {
Ok(token) => token,
Err(err) => {
warn!("Error inserting generic inference job for FAL queue: {:?}", err);
return Err(CommonWebError::ServerError);
}
};
let _r = transaction
.commit()
.await
.map_err(|err| {
error!("Error committing MySQL transaction: {:?}", err);
CommonWebError::ServerError
})?;
Ok(Json(NanoBananaMultiFunctionImageGenResponse {
success: true,
inference_job_token: job_token,
}))
}
@@ -130,6 +130,7 @@ pub async fn nano_banana_pro_multi_function_image_gen_handler(
};
let aspect_ratio = match request.aspect_ratio {
Some(NanoBananaProMultiFunctionImageGenAspectRatio::Auto) => EnqueueNanoBananaProEditImageAspectRatio::Auto,
Some(NanoBananaProMultiFunctionImageGenAspectRatio::OneByOne) => EnqueueNanoBananaProEditImageAspectRatio::OneByOne,
Some(NanoBananaProMultiFunctionImageGenAspectRatio::FiveByFour) => EnqueueNanoBananaProEditImageAspectRatio::FiveByFour,
Some(NanoBananaProMultiFunctionImageGenAspectRatio::FourByThree) => EnqueueNanoBananaProEditImageAspectRatio::FourByThree,
@@ -179,6 +180,7 @@ pub async fn nano_banana_pro_multi_function_image_gen_handler(
};
let aspect_ratio = match request.aspect_ratio {
Some(NanoBananaProMultiFunctionImageGenAspectRatio::Auto) => EnqueueNanoBananaProTextToImageAspectRatio::OneByOne, // NB: "auto" is only for edit image, not text-to-image !
Some(NanoBananaProMultiFunctionImageGenAspectRatio::OneByOne) => EnqueueNanoBananaProTextToImageAspectRatio::OneByOne,
Some(NanoBananaProMultiFunctionImageGenAspectRatio::FiveByFour) => EnqueueNanoBananaProTextToImageAspectRatio::FiveByFour,
Some(NanoBananaProMultiFunctionImageGenAspectRatio::FourByThree) => EnqueueNanoBananaProTextToImageAspectRatio::FourByThree,
@@ -13,6 +13,7 @@ use crate::http_server::endpoints::generate::image::inpaint::flux_pro_1_inpaint_
use crate::http_server::endpoints::generate::image::multi_function::bytedance_seedream_v4_multi_function_image_gen_handler::bytedance_seedream_v4_multi_function_image_gen_handler;
use crate::http_server::endpoints::generate::image::multi_function::bytedance_seedream_v4p5_multi_function_image_gen_handler::bytedance_seedream_v4p5_multi_function_image_gen_handler;
use crate::http_server::endpoints::generate::image::multi_function::gpt_image_1p5_multi_function_image_gen_handler::gpt_image_1p5_multi_function_image_gen_handler;
use crate::http_server::endpoints::generate::image::multi_function::nano_banana_multi_function_image_gen_handler::nano_banana_multi_function_image_gen_handler;
use crate::http_server::endpoints::generate::image::multi_function::nano_banana_pro_multi_function_image_gen_handler::nano_banana_pro_multi_function_image_gen_handler;
use crate::http_server::endpoints::generate::image::remove_image_background_handler::remove_image_background_handler;
use crate::http_server::endpoints::generate::object::generate_hunyuan_2_0_image_to_3d_handler::generate_hunyuan_2_0_image_to_3d_handler;
@@ -62,6 +63,10 @@ where
.route(web::post().to(gpt_image_1p5_multi_function_image_gen_handler))
.route(web::head().to(|| HttpResponse::Ok()))
)
.service(web::resource("/nano_banana")
.route(web::post().to(nano_banana_multi_function_image_gen_handler))
.route(web::head().to(|| HttpResponse::Ok()))
)
.service(web::resource("/nano_banana_pro")
.route(web::post().to(nano_banana_pro_multi_function_image_gen_handler))
.route(web::head().to(|| HttpResponse::Ok()))
@@ -14,6 +14,12 @@ pub struct Gemini25FlashImageEditInput {
#[serde(skip_serializing_if = "Option::is_none")]
pub output_format: Option<String>,
/// The aspect ratio of the generated image.
/// Default value: "auto"
/// Possible enum values: auto, 21:9, 16:9, 3:2, 4:3, 5:4, 1:1, 4:5, 3:4, 2:3, 9:16
#[serde(skip_serializing_if = "Option::is_none")]
pub aspect_ratio: Option<String>,
// /// Resolution, eg. "720p"
// #[serde(skip_serializing_if = "Option::is_none")]
// pub resolution: Option<String>,
@@ -1 +1,2 @@
pub mod edit;
pub mod edit;
pub mod text_to_image;
@@ -0,0 +1,37 @@
use crate::prelude::{Deserialize, FalRequest, Serialize};
#[derive(Debug, Serialize, Deserialize, Default)]
pub struct Gemini25FlashTextToImageInput {
pub prompt: String,
/// 1 - 4
#[serde(skip_serializing_if = "Option::is_none")]
pub num_images: Option<u8>,
/// "jpeg" or "png"
#[serde(skip_serializing_if = "Option::is_none")]
pub output_format: Option<String>,
/// The aspect ratio of the generated image.
/// Default value: "1:1"
/// Possible enum values: 21:9, 16:9, 3:2, 4:3, 5:4, 1:1, 4:5, 3:4, 2:3, 9:16
#[serde(skip_serializing_if = "Option::is_none")]
pub aspect_ratio: Option<String>,
}
#[derive(Debug, Serialize, Deserialize, Default)]
pub struct File {
/// The URL where the file can be downloaded from.
pub url: String,
}
#[derive(Debug, Serialize, Deserialize)]
pub struct Gemini25FlashTextToImageOutput {
pub images: Vec<File>,
}
pub fn gemini_25_flash_text_to_image(
params: Gemini25FlashTextToImageInput,
) -> FalRequest<Gemini25FlashTextToImageInput, Gemini25FlashTextToImageOutput> {
FalRequest::new("fal-ai/gemini-25-flash-image", params)
}
@@ -27,12 +27,13 @@ import {
} from "@fortawesome/pro-regular-svg-icons";
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
import { IconProp } from "@fortawesome/fontawesome-svg-core";
import { ImageModel, getCapabilitiesForModel } from "@storyteller/model-list";
import { CommonAspectRatio, ImageModel, getCapabilitiesForModel } from "@storyteller/model-list";
import { usePromptImageStore, RefImage } from "./promptStore";
import { gtagEvent } from "@storyteller/google-analytics";
import { twMerge } from "tailwind-merge";
import { ImagePromptRow } from "./ImagePromptRow";
import { GenerationProvider } from "@storyteller/api-enums";
import { AspectRatioPicker } from "./common/AspectRatioPicker";
interface PromptBoxImageProps {
useJobContext: () => JobContextType;
@@ -109,6 +110,9 @@ export const PromptBoxImage = ({
referenceImages.length > 0 ||
uploadingImages.length > 0;
// New aspect ratio state will begin to phase out old aspect ratio
const [commonAspectRatio, setCommonAspectRatio] = useState<CommonAspectRatio | undefined>(undefined);
useEffect(() => {
onImageRowVisibilityChange?.(isImageRowVisible);
}, [isImageRowVisible, onImageRowVisibilityChange]);
@@ -289,7 +293,7 @@ export const PromptBoxImage = ({
}, 10000);
try {
const aspectRatio = getCurrentAspectRatio();
const aspectRatio = getCurrentLegacyAspectRatio();
const resolution = getCurrentResolution();
const request: EnqueueTextToImageRequest = {
@@ -306,6 +310,10 @@ export const PromptBoxImage = ({
request.provider = selectedProvider;
}
if (selectedModel?.supportsNewAspectRatio()) {
request.common_aspect_ratio = commonAspectRatio;
}
if (
selectedModel?.canUseImagePrompt &&
!!referenceImages &&
@@ -335,7 +343,7 @@ export const PromptBoxImage = ({
return iconElement.props.icon;
};
const getCurrentAspectRatio = (): EnqueueTextToImageSize => {
const getCurrentLegacyAspectRatio = (): EnqueueTextToImageSize => {
const selected = aspectRatioList.find((item) => item.selected);
switch (selected?.label.toLowerCase()) {
case "wide":
@@ -469,7 +477,14 @@ export const PromptBoxImage = ({
</div>
<div className="mt-2 flex items-center justify-between gap-2">
<div className="flex items-center gap-2">
{selectedModel?.canChangeAspectRatio && (
{selectedModel?.supportsNewAspectRatio() && (
<AspectRatioPicker
model={selectedModel}
currentAspectRatio={commonAspectRatio}
handleCommonAspectRatioSelect={setCommonAspectRatio}
/>
)}
{selectedModel?.canChangeAspectRatio && !selectedModel?.supportsNewAspectRatio() && (
<Tooltip
content="Aspect Ratio"
position="top"
@@ -0,0 +1,171 @@
import { faMagicWandSparkles, faRectangle, faRectangleVertical, faRectangleWide, faSquare, IconDefinition } from "@fortawesome/pro-regular-svg-icons"
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"
import { PopoverItem, PopoverMenu } from "libs/components/popover/src/lib/popover"
import Tooltip from "libs/components/tooltip/src/lib/tooltip"
import { ImageModel } from "libs/model-list/src/lib/classes/ImageModel"
import { CommonAspectRatio } from "libs/model-list/src/lib/classes/properties/CommonAspectRatio"
interface AspectRatioPickerProps {
model: ImageModel,
currentAspectRatio?: CommonAspectRatio,
handleCommonAspectRatioSelect: (selected: CommonAspectRatio) => void;
//model: ImageModel | VideoModel
}
/**
* Stateless component.
*
* Picker for "common aspect ratios", the new data structure Tauri accepts for
* all image and video models (Note: this is not fully rolled out yet. Some/most
* models may still use the old format.)
*
* @param model - currently selected model
* @param currentAspectRatio - currently selected aspect ratio
* @param handleCommonAspectRatioSelect - callback when an aspect ratio is selected
*/
export const AspectRatioPicker = ({
model,
currentAspectRatio,
handleCommonAspectRatioSelect,
}: AspectRatioPickerProps) => {
const useAspectRatio = currentAspectRatio ?? model.defaultAspectRatio ?? undefined;
console.log("aspect - currentAspectRatio:", currentAspectRatio);
console.log("aspect - useAspectRatio:", useAspectRatio);
const getCurrentResolutionIcon = () : IconDefinition => {
if (!useAspectRatio) {
return faSquare;
}
return getAspectRatioIcon(useAspectRatio);
}
const handleSelectAdapter = (item: PopoverItem) => {
const ratio = popOverLabelToAspectRatio(item.label, model);
handleCommonAspectRatioSelect(ratio);
}
let aspectRatioList : PopoverItem[] = [];
model.aspectRatios?.forEach((ratio: CommonAspectRatio) => {
aspectRatioList.push({
label: getAspectRatioTextLabel(ratio),
selected: useAspectRatio === ratio,
description: `foo ${ratio}`,
icon: <FontAwesomeIcon icon={getAspectRatioIcon(ratio)} className="h-4 w-4" />,
});
});
return (
<>
<Tooltip
content="Aspect Ratio"
position="top"
className="z-50"
closeOnClick={true}
>
<PopoverMenu
items={aspectRatioList}
onSelect={handleSelectAdapter}
mode="toggle"
panelTitle="Aspect Ratio"
showIconsInList
triggerIcon={
<FontAwesomeIcon
icon={getCurrentResolutionIcon()}
className="h-4 w-4"
/>
}
/>
</Tooltip>
</>
)
}
const getAspectRatioIcon = (aspectRatio: CommonAspectRatio) : IconDefinition => {
switch (aspectRatio) {
case CommonAspectRatio.Auto:
return faMagicWandSparkles;
case CommonAspectRatio.Square:
return faSquare;
case CommonAspectRatio.WideFiveByFour:
case CommonAspectRatio.WideFourByThree:
case CommonAspectRatio.WideThreeByTwo:
return faRectangle;
case CommonAspectRatio.WideSixteenByNine:
case CommonAspectRatio.WideTwentyOneByNine:
return faRectangleWide;
// Tall
case CommonAspectRatio.TallFourByFive:
case CommonAspectRatio.TallThreeByFour:
case CommonAspectRatio.TallTwoByThree:
return faRectangleVertical;
case CommonAspectRatio.TallNineBySixteen:
case CommonAspectRatio.TallNineByTwentyOne:
return faRectangleVertical; // TODO: New FontAwesome 7 has "Tall".
default:
return faSquare; // Fail open-ish
}
}
const getAspectRatioTextLabel = (aspectRatio: CommonAspectRatio) : string => {
switch (aspectRatio) {
case CommonAspectRatio.Auto:
return "Auto";
case CommonAspectRatio.Square:
return "Square";
// Wide
case CommonAspectRatio.WideFiveByFour:
return "5:4";
case CommonAspectRatio.WideFourByThree:
return "4:3";
case CommonAspectRatio.WideThreeByTwo:
return "3:2";
case CommonAspectRatio.WideSixteenByNine:
return "16:9";
case CommonAspectRatio.WideTwentyOneByNine:
return "21:9";
// Tall
case CommonAspectRatio.TallFourByFive:
return "4:5";
case CommonAspectRatio.TallThreeByFour:
return "3:4";
case CommonAspectRatio.TallTwoByThree:
return "2:3";
case CommonAspectRatio.TallNineBySixteen:
return "9:16";
case CommonAspectRatio.TallNineByTwentyOne:
return "9:21";
default:
return "Square"; // Fail open-ish
}
}
// Note: We only need this to deal with turning PopOverItems back into typesafe aspect ratios
const popOverLabelToAspectRatio = (label: string, model: ImageModel): CommonAspectRatio => {
switch (label) {
case "Auto": return CommonAspectRatio.Auto;
case "Square": return CommonAspectRatio.Square;
case "5:4": return CommonAspectRatio.WideFiveByFour;
case "4:3": return CommonAspectRatio.WideFourByThree;
case "3:2": return CommonAspectRatio.WideThreeByTwo;
case "16:9": return CommonAspectRatio.WideSixteenByNine;
case "21:9": return CommonAspectRatio.WideTwentyOneByNine;
case "4:5": return CommonAspectRatio.TallFourByFive;
case "3:4": return CommonAspectRatio.TallThreeByFour;
case "2:3": return CommonAspectRatio.TallTwoByThree;
case "9:16": return CommonAspectRatio.TallNineBySixteen;
case "9:21": return CommonAspectRatio.TallNineByTwentyOne;
}
// If we can't find it, return the model's default aspect ratio or Square as fallback
return model.defaultAspectRatio || CommonAspectRatio.Square;
}
+1
View File
@@ -1,6 +1,7 @@
export * from "./lib/classes/ImageModel.js";
export * from "./lib/classes/Model.js";
export * from "./lib/classes/VideoModel.js";
export * from "./lib/classes/properties/CommonAspectRatio.js";
export * from "./lib/classes/metadata/ModelCreator.js";
export * from "./lib/classes/metadata/ModelCreatorIcons.js";
export * from "./lib/classes/metadata/ModelMapping.js";
@@ -3,6 +3,7 @@ import { Model, ModelKind } from "./Model.js";
import { ModelCategory } from "../legacy/ModelConfig.js";
import { ModelTag } from "./metadata/ModelTag.js";
import { GenerationProvider } from "@storyteller/api-enums";
import { CommonAspectRatio } from "./properties/CommonAspectRatio.js";
export class ImageModel extends Model {
// Typescript type discriminator property
@@ -43,6 +44,14 @@ export class ImageModel extends Model {
// Whether the model supports changing aspect ratio
readonly canChangeAspectRatio: boolean;
// Aspect ratios supported by this model
// (This is a new field that will take time to roll out and replace the old aspect ratios)
readonly aspectRatios: CommonAspectRatio[];
// Aspect ratio to use as default when nothing is selected.
// Otherwise, use `aspectRatios[0]` as default.
readonly defaultAspectRatio?: CommonAspectRatio;
constructor(args: {
id: string;
tauriId: string;
@@ -65,6 +74,8 @@ export class ImageModel extends Model {
tags?: ModelTag[];
progressBarTime?: number;
providers?: GenerationProvider[];
aspectRatios?: CommonAspectRatio[];
defaultAspectRatio?: CommonAspectRatio;
}) {
if (args.maxGenerationCount < 1) {
throw new Error("maxGenerationCount must be at least 1");
@@ -88,6 +99,8 @@ export class ImageModel extends Model {
this.canTextToImage = args.canTextToImage === false ? false : true; // Default to true !
this.canChangeResolution = args.canChangeResolution ?? false;
this.canChangeAspectRatio = args.canChangeAspectRatio ?? false;
this.aspectRatios = args.aspectRatios ?? [];
this.defaultAspectRatio = args.defaultAspectRatio ?? (args.aspectRatios && args.aspectRatios.length > 0 ? args.aspectRatios[0] : undefined);
}
// If the model is a "Nano Banana"-type model, we may want to enable certain features.
@@ -104,4 +117,12 @@ export class ImageModel extends Model {
return false;
}
}
// Rolling out new aspect ratio controls to models.
// If we have them set on the model, we can use the new UI controls.
supportsNewAspectRatio(): boolean {
return this.canChangeAspectRatio
&& this.aspectRatios
&& this.aspectRatios.length > 0;
}
}
@@ -0,0 +1,27 @@
/// Common aspect ratio Tauri API
/// Not all models have each of these aspect ratios,
/// but Tauri will interpolate if a mistake is sent.
export enum CommonAspectRatio {
// "Auto"
// The idea here is that in image editing (not text-to-image), the model will use the source image.
// This is the semantics used by Nano Banana Pro image editing mode (but not text-to-image).
Auto = "auto",
// Square
Square = "square",
// Wide
WideFiveByFour = "wide_five_by_four",
WideFourByThree = "wide_four_by_three",
WideThreeByTwo = "wide_three_by_two",
WideSixteenByNine = "wide_sixteen_by_nine",
WideTwentyOneByNine = "wide_twenty_one_by_nine",
// Tall
TallFourByFive = "tall_four_by_five",
TallThreeByFour = "tall_three_by_four",
TallTwoByThree = "tall_two_by_three",
TallNineBySixteen = "tall_nine_by_sixteen",
TallNineByTwentyOne = "tall_nine_by_twenty_one",
}
@@ -2,6 +2,7 @@ import { GenerationProvider } from "@storyteller/api-enums";
import { ImageModel } from "../classes/ImageModel.js";
import { ModelCreator } from "../classes/metadata/ModelCreator.js";
import { ModelTag } from "../classes/metadata/ModelTag.js";
import { CommonAspectRatio } from "../classes/properties/CommonAspectRatio.js";
// TODO: Some of the model configs, such as generation counts, are authoritatively controlled in `legacy/Models.ts`
@@ -147,6 +148,20 @@ export const IMAGE_MODELS: ImageModel[] = [
progressBarTime: 25000,
canChangeResolution: true,
canChangeAspectRatio: true,
defaultAspectRatio: CommonAspectRatio.Square,
aspectRatios: [
CommonAspectRatio.Auto,
CommonAspectRatio.Square,
CommonAspectRatio.WideFiveByFour,
CommonAspectRatio.WideFourByThree,
CommonAspectRatio.WideThreeByTwo,
CommonAspectRatio.WideSixteenByNine,
CommonAspectRatio.WideTwentyOneByNine,
CommonAspectRatio.TallFourByFive,
CommonAspectRatio.TallThreeByFour,
CommonAspectRatio.TallTwoByThree,
CommonAspectRatio.TallNineBySixteen,
]
}),
new ImageModel({
id: "gpt_image_1",
@@ -1,6 +1,6 @@
import { invoke } from "@tauri-apps/api/core";
import { CommandResult } from "../common/CommandStatus";
import { Model } from "@storyteller/model-list";
import { CommonAspectRatio, Model } from "@storyteller/model-list";
import { GenerationProvider } from "@storyteller/api-enums";
export enum EnqueueTextToImageErrorType {
@@ -24,9 +24,13 @@ export interface EnqueueTextToImageRequest {
// The text prompt.
prompt?: string;
// TODO: This is deprecated and will be phased out.
// The desired output aspect ratio.
aspect_ratio?: EnqueueTextToImageSize;
// This is the new aspect ratio.
common_aspect_ratio?: CommonAspectRatio;
// Image resolution. Support: Nano Banana Pro
image_resolution?: EnqueueTextToImageResolution;
@@ -54,6 +58,7 @@ interface EnqueueTextToImageRawRequest {
provider?: GenerationProvider;
prompt?: string;
aspect_ratio?: EnqueueTextToImageSize;
common_aspect_ratio?: CommonAspectRatio;
image_resolution?: EnqueueTextToImageResolution;
number_images?: number;
image_media_tokens?: string[];
@@ -126,6 +131,10 @@ export const EnqueueTextToImage = async (request: EnqueueTextToImageRequest) : P
mutableRequest.aspect_ratio = request.aspect_ratio;
}
if (!!request.common_aspect_ratio) {
mutableRequest.common_aspect_ratio = request.common_aspect_ratio;
}
if (!!request.image_resolution) {
mutableRequest.image_resolution = request.image_resolution;
}