diff --git a/crates/desktop/artcraft/src/commands/app_preferences/update_app_preference_command.rs b/crates/desktop/artcraft/src/commands/app_preferences/update_app_preference_command.rs index b2d2f340af..1b16b82972 100644 --- a/crates/desktop/artcraft/src/commands/app_preferences/update_app_preference_command.rs +++ b/crates/desktop/artcraft/src/commands/app_preferences/update_app_preference_command.rs @@ -29,9 +29,10 @@ pub enum ValueType { pub enum PreferenceName { PreferredDownloadDirectory, PlaySounds, + EnqueueSuccessSound, + EnqueueFailureSound, GenerationSuccessSound, GenerationFailureSound, - GenerationEnqueueSound, } #[derive(Serialize)] @@ -85,6 +86,16 @@ async fn update_prefs( return Err(anyhow!("Invalid value: {:?}", request.value)), } } + PreferenceName::EnqueueSuccessSound => { + prefs.enqueue_success_sound = request.value + .map(|val| string_value(&val)) + .transpose()?; + } + PreferenceName::EnqueueFailureSound => { + prefs.enqueue_failure_sound = request.value + .map(|val| string_value(&val)) + .transpose()?; + } PreferenceName::GenerationSuccessSound => { prefs.generation_success_sound = request.value .map(|val| string_value(&val)) @@ -95,11 +106,6 @@ async fn update_prefs( .map(|val| string_value(&val)) .transpose()?; } - PreferenceName::GenerationEnqueueSound => { - prefs.generation_enqueue_sound = request.value - .map(|val| string_value(&val)) - .transpose()?; - } } app_prefs.set_clone(&prefs)?; diff --git a/crates/desktop/artcraft/src/commands/command_response_wrapper.rs b/crates/desktop/artcraft/src/commands/command_response_wrapper.rs index aefe2a330b..610f129efb 100644 --- a/crates/desktop/artcraft/src/commands/command_response_wrapper.rs +++ b/crates/desktop/artcraft/src/commands/command_response_wrapper.rs @@ -3,7 +3,11 @@ use std::error::Error; use std::str::FromStr; /// Easy to use Result type. -pub type CommandResult = Result, CommandErrorResponseWrapper>; +pub type CommandResult = + Result, CommandErrorResponseWrapper>; + +/// No inner payloads for this type. Just strings as messages. +pub type SimpleCommandResult = CommandResult<(), (), ()>; /// Statuses for successful commands. #[derive(Serialize, Clone, Debug)] @@ -18,6 +22,7 @@ pub enum CommandSuccessStatus { pub enum CommandErrorStatus { BadRequest, // 400 Unauthorized, // 401 + TooManyRequests, // 429 ServerError, // 500 } @@ -34,15 +39,22 @@ pub struct CommandSuccessResponseWrapper { } #[derive(Serialize, Debug)] -pub struct CommandErrorResponseWrapper { +pub struct CommandErrorResponseWrapper { /// Both "success" and "error" types have a `status` field for the frontend. pub status: CommandErrorStatus, #[serde(skip_serializing_if = "Option::is_none")] pub error_message: Option, + /// Represents a "type" of error, preferably an enum without payloads. + /// Preferably this serializes to a string. #[serde(skip_serializing_if = "Option::is_none")] - pub error_details: Option, + pub error_type: Option, + + /// Represents a full error details object or payload. + /// This could be a structure, message, or whatever. + #[serde(skip_serializing_if = "Option::is_none")] + pub error_details: Option, } @@ -96,11 +108,12 @@ impl From<&str> for CommandSuccessResponseWrapper { } } -impl From<&str> for CommandErrorResponseWrapper { +impl From<&str> for CommandErrorResponseWrapper<(), ()> { fn from(value: &str) -> Self { CommandErrorResponseWrapper { status: CommandErrorStatus::BadRequest, // NB: Default to 500 error. error_message: Some(value.to_string()), + error_type: None, error_details: None, } } diff --git a/crates/desktop/artcraft/src/commands/sora/check_sora_session_command.rs b/crates/desktop/artcraft/src/commands/sora/check_sora_session_command.rs index 8c00c91aef..35aea32e98 100644 --- a/crates/desktop/artcraft/src/commands/sora/check_sora_session_command.rs +++ b/crates/desktop/artcraft/src/commands/sora/check_sora_session_command.rs @@ -1,3 +1,5 @@ +use std::ops::Add; +use chrono::{DateTime, TimeDelta, Utc}; use crate::commands::app_preferences::get_app_preferences_command::AppPreferencesPayload; use crate::state::app_preferences::app_preferences_manager::AppPreferencesManager; use crate::state::data_dir::app_data_root::AppDataRoot; @@ -12,6 +14,8 @@ use openai_sora_client::requests::list_media::list_media::list_media; use serde_derive::Serialize; use tauri::{AppHandle, State}; +const CACHE_PERIOD : TimeDelta = TimeDelta::milliseconds(1000 * 60 * 5); // 5 minutes + #[derive(Serialize)] #[serde(rename_all = "snake_case")] pub struct CheckSoraSessionCommand { @@ -63,10 +67,35 @@ async fn do_check( }); } }; + + let mut needs_upgrade_or_check = false; - let upgraded = maybe_upgrade_or_renew_session(&mut creds).await?; + let stats = sora_creds_manager.get_credential_stats()?; + + match stats.last_consecutive_credential_success { + None => { + info!("Session never checked, needs upgrade or status checking."); + needs_upgrade_or_check= true; + } + Some(last_success) => { + let now = chrono::Utc::now(); + let expires = last_success.add(CACHE_PERIOD); + if now.gt(&expires) { + info!("Session success expired, needs upgrade or status checking."); + needs_upgrade_or_check = true; + } + } + } - list_media(&creds).await?; + let mut upgraded = false; + + if needs_upgrade_or_check { + info!("Attempting to upgrade session..."); + upgraded = maybe_upgrade_or_renew_session(&mut creds).await?; + info!("Attempting to check session..."); + list_media(&creds).await?; + sora_creds_manager.record_credential_success()?; + } if upgraded { info!("Saving upgraded session to manager and disk..."); diff --git a/crates/desktop/artcraft/src/commands/sora/sora_image_generation_command.rs b/crates/desktop/artcraft/src/commands/sora/sora_image_generation_command.rs index 9f42a82d41..1eeea144bc 100644 --- a/crates/desktop/artcraft/src/commands/sora/sora_image_generation_command.rs +++ b/crates/desktop/artcraft/src/commands/sora/sora_image_generation_command.rs @@ -14,8 +14,13 @@ use openai_sora_client::requests::image_gen::sora_image_gen_remix::{sora_image_g use openai_sora_client::requests::upload::upload_media_from_bytes::sora_media_upload_from_bytes; use std::fs::read_to_string; use std::io::Cursor; +use std::time::Duration; use tauri::{AppHandle, Manager, State}; +const SORA_IMAGE_UPLOAD_TIMEOUT: Duration = Duration::from_millis(1000 * 30); // 30 seconds + +const SORA_IMAGE_REMIX_TIMEOUT: Duration = Duration::from_millis(1000 * 30); // 30 seconds + #[tauri::command] pub async fn sora_image_generation_command( prompt: &str, @@ -55,8 +60,13 @@ pub async fn generate_image( let image_bytes = BASE64_STANDARD.decode(image)?; let filename = "image.png".to_string(); + - let response = sora_media_upload_from_bytes(image_bytes, filename, CredentialMigrationRef::New(&creds)) + let response = sora_media_upload_from_bytes( + image_bytes, + filename, + CredentialMigrationRef::New(&creds), + Some(SORA_IMAGE_UPLOAD_TIMEOUT)) .await .map_err(|err| { error!("Failed to upload image to Sora: {:?}", err); @@ -72,6 +82,7 @@ pub async fn generate_image( image_size: ImageSize::Square, sora_media_tokens: sora_media_tokens.clone(), credentials: CredentialMigrationRef::New(&creds), + request_timeout: Some(SORA_IMAGE_REMIX_TIMEOUT), }).await .map_err(|err| { error!("Failed to call Sora image generation: {:?}", err); diff --git a/crates/desktop/artcraft/src/commands/sora/sora_image_remix_command.rs b/crates/desktop/artcraft/src/commands/sora/sora_image_remix_command.rs index 26b84e2667..b7ef12c517 100644 --- a/crates/desktop/artcraft/src/commands/sora/sora_image_remix_command.rs +++ b/crates/desktop/artcraft/src/commands/sora/sora_image_remix_command.rs @@ -1,4 +1,4 @@ -use crate::commands::command_response_wrapper::CommandResult; +use crate::commands::command_response_wrapper::{CommandErrorResponseWrapper, CommandErrorStatus, CommandResult}; use crate::events::sendable_event_trait::SendableEvent; use crate::events::sora::sora_image_enqueue_failure_event::SoraImageEnqueueFailureEvent; use crate::events::sora::sora_image_enqueue_success_event::SoraImageEnqueueSuccessEvent; @@ -12,7 +12,7 @@ use crate::utils::get_url_file_extension::get_url_file_extension; use crate::utils::simple_http_download::simple_http_download; use base64::prelude::BASE64_STANDARD; use base64::Engine; -use errors::AnyhowResult; +use errors::{AnyhowError, AnyhowResult}; use image::codecs::png::{CompressionType, FilterType, PngEncoder}; use image::{DynamicImage, ImageReader}; use log::{debug, error, info}; @@ -25,14 +25,22 @@ use openai_sora_client::requests::image_gen::common::{ImageSize, NumImages}; use openai_sora_client::requests::image_gen::sora_image_gen_remix::{sora_image_gen_remix, SoraImageGenRemixRequest}; use openai_sora_client::requests::upload::upload_media_from_bytes::sora_media_upload_from_bytes; use openai_sora_client::requests::upload::upload_media_from_file::sora_media_upload_from_file; +use openai_sora_client::sora_error::SoraError; use serde_derive::{Deserialize, Serialize}; use std::fs::read_to_string; use std::io::Cursor; +use std::time::Duration; +use storyteller_client::api_error::ApiError; +use storyteller_client::api_error::ApiError::InternalServerError; use storyteller_client::media_files::get_media_file::get_media_file; use storyteller_client::utils::api_host::ApiHost; use tauri::{AppHandle, Emitter, Manager, State}; use tokens::tokens::media_files::MediaFileToken; +const SORA_IMAGE_UPLOAD_TIMEOUT: Duration = Duration::from_millis(1000 * 30); // 30 seconds + +const SORA_IMAGE_REMIX_TIMEOUT: Duration = Duration::from_millis(1000 * 30); // 30 seconds + #[derive(Deserialize)] pub struct SoraImageRemixCommand { /// Image media file; the engine or canvas snapshot (screenshot). @@ -50,6 +58,20 @@ pub struct SoraImageRemixCommand { pub maybe_number_of_samples: Option, } +#[derive(Serialize, Debug)] +#[serde(rename_all = "snake_case")] +pub enum SoraImageRemixErrorType { + /// Generic server error + ServerError, + /// The user is sending too many requests + TooManyConcurrentTasks, + /// The user needs to create a Sora account + SoraUsernameNotYetCreated, + /// The Sora service is having problems. Try again soon. + SoraIsHavingProblems, +} + + #[tauri::command] pub async fn sora_image_remix_command( app: AppHandle, @@ -57,7 +79,7 @@ pub async fn sora_image_remix_command( app_data_root: State<'_, AppDataRoot>, sora_creds_manager: State<'_, SoraCredentialManager>, sora_task_queue: State<'_, SoraTaskQueue>, -) -> CommandResult<(), String> { +) -> CommandResult<(), SoraImageRemixErrorType, ()> { info!("image_generation_command called; scene media token: {:?}, additional images: {:?}", request.snapshot_media_token, request.maybe_additional_images); @@ -66,27 +88,83 @@ pub async fn sora_image_remix_command( let result = generate_image(request, &app_data_root, &sora_creds_manager, &sora_task_queue).await; - if let Err(err) = result { - error!("error: {:?}", err); + match result { + Err(err) => { + error!("error: {:?}", err); - let event = SoraImageEnqueueFailureEvent {}; - let result = event.send(&app); + let event = SoraImageEnqueueFailureEvent {}; + let result = event.send(&app); - if let Err(err) = result { - error!("Failed to emit event: {:?}", err); + if let Err(err) = result { + error!("Failed to emit event: {:?}", err); + } + + let mut status = CommandErrorStatus::ServerError; + let mut error_type = SoraImageRemixErrorType::ServerError; + let mut error_message = "A server error occurred. Please try again. If it continues, please tell our staff about the problem."; + + match (err) { + InnerError::SoraError(SoraError::TooManyConcurrentTasks) => { + error_type = SoraImageRemixErrorType::TooManyConcurrentTasks; + status = CommandErrorStatus::ServerError; + error_message = "You already have work in progress. Please wait for it to finish."; + }, + InnerError::SoraError(SoraError::SoraUsernameNotYetCreated) => { + error_type = SoraImageRemixErrorType::SoraUsernameNotYetCreated; + status = CommandErrorStatus::BadRequest; + error_message = "You need to create a username on Sora.com to continue."; + }, + InnerError::SoraError(SoraError::BadGateway(_) | SoraError::CloudFlareTimeout(_)) => { + error_type = SoraImageRemixErrorType::SoraIsHavingProblems; + status = CommandErrorStatus::ServerError; + error_message = "Sora is having problems. Please wait a moment and then retry."; + } + _ => {}, + } + + Err(CommandErrorResponseWrapper { + status, + error_message: Some(error_message.to_string()), + error_type: Some(error_type), + error_details: None, + }) + } + Ok(_) => { + let event = SoraImageEnqueueSuccessEvent {}; + let result = event.send(&app); + + if let Err(err) = result { + error!("Failed to emit event: {:?}", err); + } + + Ok(().into()) } - - return Err("there was an error".into()); } +} - let event = SoraImageEnqueueSuccessEvent {}; - let result = event.send(&app); +#[derive(Debug)] +enum InnerError { + SoraError(SoraError), + AnyhowError(AnyhowError), + StorytellerApiError(ApiError), +} - if let Err(err) = result { - error!("Failed to emit event: {:?}", err); +impl From for InnerError { + fn from(value: AnyhowError) -> Self { + Self::AnyhowError(value) } +} - Ok(().into()) +impl From for InnerError { + fn from(value: SoraError) -> Self { + Self::SoraError(value) + } +} + +impl From for InnerError { + fn from(value: ApiError) -> Self { + Self::StorytellerApiError(value) + } } pub async fn generate_image( @@ -94,7 +172,7 @@ pub async fn generate_image( app_data_root: &AppDataRoot, sora_creds_manager: &SoraCredentialManager, sora_task_queue: &SoraTaskQueue, -) -> AnyhowResult<()> { +) -> Result<(), InnerError> { let response = get_media_file(&ApiHost::Storyteller, &request.snapshot_media_token).await?; @@ -133,6 +211,7 @@ pub async fn generate_image( image_upload_from_file_with_session_auto_renew(ImageUploadFromFileAutoRenewRequest { file_path, credentials: &creds, + request_timeout: Some(SORA_IMAGE_UPLOAD_TIMEOUT), }).await?; if let Some(new_creds) = maybe_new_credentials { @@ -157,6 +236,7 @@ pub async fn generate_image( image_size: ImageSize::Square, sora_media_tokens: sora_media_tokens.clone(), credentials: &creds, + request_timeout: Some(SORA_IMAGE_REMIX_TIMEOUT), }).await?; if let Some(new_creds) = maybe_new_creds { diff --git a/crates/desktop/artcraft/src/commands/sora/sora_logout_command.rs b/crates/desktop/artcraft/src/commands/sora/sora_logout_command.rs index 1c87366fd1..531fee02f6 100644 --- a/crates/desktop/artcraft/src/commands/sora/sora_logout_command.rs +++ b/crates/desktop/artcraft/src/commands/sora/sora_logout_command.rs @@ -1,5 +1,5 @@ use crate::commands::app_preferences::get_app_preferences_command::AppPreferencesPayload; -use crate::commands::command_response_wrapper::{CommandResult, CommandSuccessResponseWrapper}; +use crate::commands::command_response_wrapper::{CommandResult, CommandSuccessResponseWrapper, SimpleCommandResult}; use crate::state::app_preferences::app_preferences_manager::AppPreferencesManager; use crate::state::data_dir::app_data_root::AppDataRoot; use crate::state::sora::sora_credential_manager::SoraCredentialManager; @@ -17,7 +17,7 @@ use tauri::{AppHandle, State}; #[tauri::command] pub async fn sora_logout_command( sora_creds_manager: State<'_, SoraCredentialManager>, -) -> CommandResult<(), String> { +) -> SimpleCommandResult { info!("sora_logout_command called"); if let Err(err) = sora_creds_manager.purge_credentials_from_disk() { @@ -30,6 +30,10 @@ pub async fn sora_logout_command( return Err("error clearing credentials from memory".into()); } + if let Err(err) = sora_creds_manager.reset_credential_stats() { + error!("Error resetting credential stats: {:?}", err); + return Err("error resetting credential stats".into()); + } + Ok(().into()) } - diff --git a/crates/desktop/artcraft/src/state/sora/mod.rs b/crates/desktop/artcraft/src/state/sora/mod.rs index f2b2834e3c..5a0e3cc740 100644 --- a/crates/desktop/artcraft/src/state/sora/mod.rs +++ b/crates/desktop/artcraft/src/state/sora/mod.rs @@ -2,4 +2,5 @@ pub mod read_sora_credentials_from_disk; pub mod read_sora_credentials_from_disk_with_upgrade; pub mod sora_credential_holder; pub mod sora_credential_manager; +pub mod sora_credential_stats; pub mod sora_task_queue; \ No newline at end of file diff --git a/crates/desktop/artcraft/src/state/sora/sora_credential_manager.rs b/crates/desktop/artcraft/src/state/sora/sora_credential_manager.rs index e69f277753..30ff2651ee 100644 --- a/crates/desktop/artcraft/src/state/sora/sora_credential_manager.rs +++ b/crates/desktop/artcraft/src/state/sora/sora_credential_manager.rs @@ -1,23 +1,29 @@ -use std::{fs, io}; use crate::state::data_dir::app_data_root::AppDataRoot; use crate::state::sora::read_sora_credentials_from_disk::read_sora_credentials_from_disk; use crate::state::sora::sora_credential_holder::SoraCredentialHolder; +use crate::state::sora::sora_credential_stats::SoraCredentialStats; use anyhow::anyhow; use errors::AnyhowResult; use log::{error, info, warn}; +use memory_store::clone_cell::CloneCell; +use openai_sora_client::creds::sora_cookies::SoraCookies; use openai_sora_client::creds::sora_credential_set::SoraCredentialSet; +use openai_sora_client::creds::sora_jwt_bearer_token::SoraJwtBearerToken; use openai_sora_client::creds::sora_sentinel::SoraSentinel; use openai_sora_client::requests::sentinel_refresh::generate::token::generate_token; use std::fs::OpenOptions; use std::io::Write; use std::path::Path; -use openai_sora_client::creds::sora_cookies::SoraCookies; -use openai_sora_client::creds::sora_jwt_bearer_token::SoraJwtBearerToken; +use std::{fs, io}; #[derive(Clone)] pub struct SoraCredentialManager { holder: SoraCredentialHolder, app_data_root: AppDataRoot, + + /// This is meant to help debug credential issues, keep credentials cached, fresh, etc. + // TODO: Put this into the same structure as the credentials and also persist to disk. + stats: CloneCell, } impl SoraCredentialManager { @@ -35,9 +41,36 @@ impl SoraCredentialManager { Self { holder, app_data_root: app_data_root.clone(), + stats: CloneCell::with_owned(SoraCredentialStats::new()), } } + pub fn record_credential_success(&self) -> AnyhowResult<()> { + let mut stats = self.stats.get_clone()?; + stats.record_credential_success(); + self.stats.set_owned(stats)?; + Ok(()) + } + + pub fn record_credential_failure(&self) -> AnyhowResult<()> { + let mut stats = self.stats.get_clone()?; + stats.record_credential_failure(); + self.stats.set_owned(stats)?; + Ok(()) + } + + pub fn reset_credential_stats(&self) -> AnyhowResult<()> { + self.stats.set_owned(SoraCredentialStats::new()) + } + + pub fn get_credential_stats(&self) -> AnyhowResult { + self.stats.get_clone() + } + + pub fn set_credential_stats(&self, stats: SoraCredentialStats) -> AnyhowResult<()> { + self.stats.set_owned(stats) + } + pub fn set_credentials(&self, creds: &SoraCredentialSet) -> AnyhowResult<()> { self.holder.set_credentials(creds) } diff --git a/crates/desktop/artcraft/src/state/sora/sora_credential_stats.rs b/crates/desktop/artcraft/src/state/sora/sora_credential_stats.rs new file mode 100644 index 0000000000..dad71f43fd --- /dev/null +++ b/crates/desktop/artcraft/src/state/sora/sora_credential_stats.rs @@ -0,0 +1,47 @@ + +/// This is meant to track 401s and other credential failures, not 500s. +#[derive(Clone)] +pub struct SoraCredentialStats { + pub last_credential_failure: Option>, + pub last_credential_success: Option>, + pub credential_success_count: u32, + pub credential_failure_count: u32, + /// This resets on each failure + pub last_consecutive_credential_success: Option>, + /// This resets on each failure + pub consecutive_credential_success_count: u32, + /// This resets on each success + pub consecutive_credential_failure_count: u32, +} + +impl SoraCredentialStats { + pub fn new() -> Self { + Self { + last_credential_failure: None, + last_credential_success: None, + last_consecutive_credential_success: None, + credential_success_count: 0, + credential_failure_count: 0, + consecutive_credential_success_count: 0, + consecutive_credential_failure_count: 0, + } + } + + pub fn record_credential_success(&mut self) { + let now = chrono::Utc::now(); + self.last_credential_success = Some(now); + self.last_consecutive_credential_success = Some(now); + self.credential_success_count += 1; + self.consecutive_credential_success_count += 1; + self.consecutive_credential_failure_count = 0; + } + + pub fn record_credential_failure(&mut self) { + let now = chrono::Utc::now(); + self.last_credential_failure = Some(now); + self.last_consecutive_credential_success = None; + self.credential_failure_count += 1; + self.consecutive_credential_failure_count += 1; + self.consecutive_credential_success_count = 0; + } +} diff --git a/crates/desktop/artcraft/tauri.conf.json b/crates/desktop/artcraft/tauri.conf.json index 700306c606..7c0adacb61 100644 --- a/crates/desktop/artcraft/tauri.conf.json +++ b/crates/desktop/artcraft/tauri.conf.json @@ -21,11 +21,43 @@ ], "security": { "csp": { - "default-src": "*", - "connect-src": "*", + "default-src": [ + "'self'", + "'unsafe-eval'", + "'unsafe-inline'", + "blob:", + "data:", + "https://*.fakeyou.com", + "https://*.storyteller.ai", + "ipc://localhost", + "tauri://localhost" + ], + "connect-src": [ + "'self'", + "'unsafe-eval'", + "'unsafe-inline'", + "blob:", + "data:", + "https://*.fakeyou.com", + "https://*.storyteller.ai", + "https://unpkg.com/detect-gpu@5.0.70/", + "ipc://localhost", + "tauri://localhost" + ], "font-src": "*", "img-src": "* data: blob: 'unsafe-inline'", - "style-src": "*" + "style-src": [ + "'self'", + "'unsafe-eval'", + "'unsafe-inline'", + "blob:", + "data:", + "https://*.fakeyou.com", + "https://*.storyteller.ai", + "https://fonts.googleapis.com", + "ipc://localhost", + "tauri://localhost" + ] } } }, diff --git a/crates/lib/api_clients/openai_sora_client/src/recipes/image_remix_with_session_auto_renew.rs b/crates/lib/api_clients/openai_sora_client/src/recipes/image_remix_with_session_auto_renew.rs index db28f678ce..eb17d90d4c 100644 --- a/crates/lib/api_clients/openai_sora_client/src/recipes/image_remix_with_session_auto_renew.rs +++ b/crates/lib/api_clients/openai_sora_client/src/recipes/image_remix_with_session_auto_renew.rs @@ -1,3 +1,4 @@ +use std::time::Duration; use anyhow::anyhow; use log::{info, warn}; use errors::AnyhowResult; @@ -7,7 +8,7 @@ use crate::creds::sora_jwt_bearer_token::SoraJwtBearerToken; use crate::creds::sora_sentinel::SoraSentinel; use crate::requests::bearer::generate::generate_bearer_with_cookie; use crate::requests::image_gen::common::{ImageSize, NumImages, SoraImageGenResponse}; -use crate::requests::image_gen::image_gen_http_request; +use crate::requests::image_gen::{image_gen_http_request, SoraImageGenError}; use crate::requests::image_gen::sora_image_gen_remix::{sora_image_gen_remix, SoraImageGenRemixRequest}; use crate::requests::sentinel_refresh::generate::token::generate_token; use crate::sora_error::SoraError; @@ -18,6 +19,8 @@ pub struct ImageRemixAutoRenewRequest<'a> { pub image_size: ImageSize, pub sora_media_tokens: Vec, pub credentials: &'a SoraCredentialSet, + /// This function can try several different requests. This is the timeout for *individual* requests. + pub request_timeout: Option, } /// Image request with retry and session auto-renewal. @@ -29,6 +32,7 @@ pub async fn image_remix_with_session_auto_renew(request: ImageRemixAutoRenewReq image_size: request.image_size, sora_media_tokens: request.sora_media_tokens.clone(), // NB: Clone because used again credentials: CredentialMigrationRef::New(request.credentials), + request_timeout: request.request_timeout, }).await; let err = match result { @@ -43,29 +47,32 @@ pub async fn image_remix_with_session_auto_renew(request: ImageRemixAutoRenewReq match err { // We'll fail these requests... - image_gen_http_request::SoraError::TooManyConcurrentTasks(_) => { + SoraImageGenError::TooManyConcurrentTasks(_) => { return Err(SoraError::TooManyConcurrentTasks); } - image_gen_http_request::SoraError::GenericError(err) => { + SoraImageGenError::GenericError(err) => { return Err(SoraError::OtherBadStatus(anyhow!("image remix failed with GenericError: {:?}", err))) } - image_gen_http_request::SoraError::NetworkError(err) => { + SoraImageGenError::NetworkError(err) => { // TODO: The underlying type should be a reqwest::Error. return Err(SoraError::OtherBadStatus(anyhow!("network error: {:?}", err))) } + SoraImageGenError::UsernameRequired(_err) => { + return Err(SoraError::SoraUsernameNotYetCreated) + } // We'll retry these requests... - image_gen_http_request::SoraError::SentinelBlock(err) => { + SoraImageGenError::SentinelBlock(err) => { warn!("Image generation failed due to sentinel block error: {:?}", err); refresh_sentinel = true; } - image_gen_http_request::SoraError::TokenExpired(err) => { + SoraImageGenError::TokenExpired(err) => { warn!("Image generation failed due to token expired error: {:?}", err); refresh_sentinel = true; // TODO: Not sure what this error is, actually. refresh_jwt = true; } - image_gen_http_request::SoraError::InvalidJwt(err) => { + SoraImageGenError::InvalidJwt(err) => { warn!("Image generation failed due to invalid jwt error: {:?}", err); refresh_jwt = true; } @@ -123,12 +130,13 @@ pub async fn image_remix_with_session_auto_renew(request: ImageRemixAutoRenewReq image_size: request.image_size, sora_media_tokens: request.sora_media_tokens, credentials: CredentialMigrationRef::New(&new_creds), + request_timeout: request.request_timeout, }).await; match result { Ok(response) => Ok((response, Some(new_creds))), Err(err) => match err { - image_gen_http_request::SoraError::TooManyConcurrentTasks(err) => { + image_gen_http_request::SoraImageGenError::TooManyConcurrentTasks(err) => { Err(SoraError::TooManyConcurrentTasks) } _ => { diff --git a/crates/lib/api_clients/openai_sora_client/src/recipes/image_upload_from_bytes_with_session_auto_renew.rs b/crates/lib/api_clients/openai_sora_client/src/recipes/image_upload_from_bytes_with_session_auto_renew.rs index 2f8fba3feb..1b06065637 100644 --- a/crates/lib/api_clients/openai_sora_client/src/recipes/image_upload_from_bytes_with_session_auto_renew.rs +++ b/crates/lib/api_clients/openai_sora_client/src/recipes/image_upload_from_bytes_with_session_auto_renew.rs @@ -14,12 +14,16 @@ use crate::sora_error::SoraError; use anyhow::anyhow; use errors::AnyhowResult; use log::{error, info, warn}; +use std::time::Duration; pub struct ImageUploadFromBytesAutoRenewRequest<'a> { pub file_bytes: Vec, pub filename: String, pub mime_type: &'a str, pub credentials: &'a SoraCredentialSet, + + /// This function can try to upload several times. This is the timeout for *individual* requests. + pub request_timeout: Option, } /// Image upload with retry and session auto-renewal. @@ -32,6 +36,7 @@ pub async fn image_upload_from_bytes_with_session_auto_renew( request.file_bytes.clone(), // FIXME(bt): This is horrible, but the client needs to take ownership. :( request.filename.clone(), // FIXME: Same CredentialMigrationRef::New(request.credentials), + request.request_timeout, ).await; let err = match result { @@ -51,6 +56,7 @@ pub async fn image_upload_from_bytes_with_session_auto_renew( request.file_bytes, request.filename, CredentialMigrationRef::New(&new_creds), + request.request_timeout, ).await?; Ok((result, Some(new_creds))) diff --git a/crates/lib/api_clients/openai_sora_client/src/recipes/image_upload_from_file_with_session_auto_renew.rs b/crates/lib/api_clients/openai_sora_client/src/recipes/image_upload_from_file_with_session_auto_renew.rs index f4cf28934f..695400f7db 100644 --- a/crates/lib/api_clients/openai_sora_client/src/recipes/image_upload_from_file_with_session_auto_renew.rs +++ b/crates/lib/api_clients/openai_sora_client/src/recipes/image_upload_from_file_with_session_auto_renew.rs @@ -6,10 +6,14 @@ use crate::requests::upload::upload_media_http_request::{upload_media_http_reque use crate::sora_error::SoraError; use log::{info, warn}; use std::path::Path; +use std::time::Duration; pub struct ImageUploadFromFileAutoRenewRequest<'a, P: AsRef> { pub file_path: P, pub credentials: &'a SoraCredentialSet, + + /// This function can try to upload several times. This is the timeout for *individual* requests. + pub request_timeout: Option, } /// Image upload with retry and session auto-renewal. @@ -21,6 +25,8 @@ pub async fn image_upload_from_file_with_session_auto_renew>( let result = sora_media_upload_from_file( request.file_path.as_ref().clone(), // FIXME(bt): This is horrible, but the client needs to take ownership. :( CredentialMigrationRef::New(request.credentials), + /// This function can try to upload several times. This is the timeout for *individual* requests. + request.request_timeout, ).await; let err = match result { @@ -37,6 +43,7 @@ pub async fn image_upload_from_file_with_session_auto_renew>( let result = sora_media_upload_from_file( request.file_path, CredentialMigrationRef::New(&new_creds), + request.request_timeout, ).await?; Ok((result, Some(new_creds))) diff --git a/crates/lib/api_clients/openai_sora_client/src/recipes/list_sora_task_status_with_session_auto_renew.rs b/crates/lib/api_clients/openai_sora_client/src/recipes/list_sora_task_status_with_session_auto_renew.rs index b25a8219c3..b7abbc5655 100644 --- a/crates/lib/api_clients/openai_sora_client/src/recipes/list_sora_task_status_with_session_auto_renew.rs +++ b/crates/lib/api_clients/openai_sora_client/src/recipes/list_sora_task_status_with_session_auto_renew.rs @@ -39,11 +39,10 @@ pub async fn list_sora_task_status_with_session_auto_renew( new_creds = Some(updated_creds); } - let legacy_creds = new_creds.as_ref() - .unwrap_or_else(|| &args.credentials) - .to_legacy_credentials()?; + let upgraded_creds = new_creds.as_ref() + .unwrap_or_else(|| &args.credentials); - let result = get_image_gen_status(&request, &legacy_creds).await; + let result = get_image_gen_status(&request, &upgraded_creds).await; let err = match result { Ok(response) => return Ok((response, new_creds)), @@ -56,7 +55,7 @@ pub async fn list_sora_task_status_with_session_auto_renew( info!("Retrying image upload with new credentials..."); - let result = get_image_gen_status(&request, &legacy_creds).await?; + let result = get_image_gen_status(&request, &upgraded_creds).await?; Ok((result, Some(new_creds))) } diff --git a/crates/lib/api_clients/openai_sora_client/src/recipes/wait_for_image_gen_status.rs b/crates/lib/api_clients/openai_sora_client/src/recipes/wait_for_image_gen_status.rs index 50d5bf0b74..faa2051dc0 100644 --- a/crates/lib/api_clients/openai_sora_client/src/recipes/wait_for_image_gen_status.rs +++ b/crates/lib/api_clients/openai_sora_client/src/recipes/wait_for_image_gen_status.rs @@ -2,6 +2,7 @@ use crate::credentials::SoraCredentials; use crate::requests::image_gen::image_gen_status::TaskStatus; use crate::requests::image_gen::image_gen_status::{get_image_gen_status, StatusRequest, TaskResponse, VideoGenStatusResponse}; use errors::AnyhowResult; +use crate::creds::sora_credential_set::SoraCredentialSet; pub async fn wait_for_image_gen_status(task_id: &String, credentials: &SoraCredentials, retry_limit: Option) -> AnyhowResult { let status_request = StatusRequest { @@ -10,11 +11,13 @@ pub async fn wait_for_image_gen_status(task_id: &String, credentials: &SoraCrede // before: Some(task_id.clone()), }; let retry_limit = retry_limit.unwrap_or(10); + + let new_creds = SoraCredentialSet::from_legacy_credentials(&credentials)?; for _ in 0..retry_limit { tokio::time::sleep(std::time::Duration::from_secs(10)).await; - let status_response = get_image_gen_status(&status_request, credentials).await?; + let status_response = get_image_gen_status(&status_request, &new_creds).await?; match find_task_response_by_id(&status_response, task_id.clone()) { Some(task_response) => match TaskStatus::from_str(&task_response.status) { diff --git a/crates/lib/api_clients/openai_sora_client/src/requests/image_gen/image_gen_http_request.rs b/crates/lib/api_clients/openai_sora_client/src/requests/image_gen/image_gen_http_request.rs index a69cdcec63..b09094f42f 100644 --- a/crates/lib/api_clients/openai_sora_client/src/requests/image_gen/image_gen_http_request.rs +++ b/crates/lib/api_clients/openai_sora_client/src/requests/image_gen/image_gen_http_request.rs @@ -1,7 +1,9 @@ use crate::creds::credential_migration::CredentialMigrationRef; -use serde_derive::{Deserialize, Serialize}; -use thiserror::Error; use crate::requests::image_gen::image_gen_status::TaskId; +use log::warn; +use serde_derive::{Deserialize, Serialize}; +use std::time::Duration; +use thiserror::Error; const SORA_IMAGE_GEN_URL: &str = "https://sora.com/backend/video_gen"; @@ -9,7 +11,7 @@ const SORA_IMAGE_GEN_URL: &str = "https://sora.com/backend/video_gen"; const USER_AGENT : &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36"; #[derive(Error, Debug)] -pub enum SoraError { +pub enum SoraImageGenError { #[error("Sora token expired: {0}")] TokenExpired(String), @@ -27,6 +29,9 @@ pub enum SoraError { #[error("Network error: {0}")] NetworkError(String), + + #[error("Sora username required: {0}")] + UsernameRequired(String), } #[derive(Serialize, Debug)] @@ -127,6 +132,7 @@ pub enum SoraErrorCode { InvalidJwt, TokenExpired, TooManyConcurrentTasks, + UsernameRequired, Unknown(String), } @@ -147,7 +153,11 @@ impl std::fmt::Display for RawSoraErrorInner { } /// Don't expose the internal request implementation as there are only a few "correct" ways to call the API. -pub (crate) async fn image_gen_http_request(sora_request: RawSoraImageGenRequest, credentials: CredentialMigrationRef<'_>) -> Result { +pub (crate) async fn image_gen_http_request( + sora_request: RawSoraImageGenRequest, + credentials: CredentialMigrationRef<'_>, + request_timeout: Option, +) -> Result { let client = reqwest::Client::new(); let mut cookie; @@ -161,78 +171,88 @@ pub (crate) async fn image_gen_http_request(sora_request: RawSoraImageGenRequest // TODO(bt,2025-04-23): We're using a Sora payload error in place of application state error. Surface this differently. sentinel = creds.sentinel.as_ref() .map(|sentinel| sentinel.to_string()) - .ok_or(SoraError::SentinelBlock("Sentinel is required for image generation.".to_string()))?; + .ok_or(SoraImageGenError::SentinelBlock("Sentinel is required for image generation.".to_string()))?; } CredentialMigrationRef::New(creds) => { cookie = creds.cookies.to_string(); // TODO(bt,2025-04-23): We're using a Sora payload error in place of application state error. Surface this differently. authorization_header = creds.jwt_bearer_token.as_ref() - .ok_or(SoraError::InvalidJwt("JWT bearer is required for image generation".to_string()))? + .ok_or(SoraImageGenError::InvalidJwt("JWT bearer is required for image generation".to_string()))? .to_authorization_header_value(); // TODO(bt,2025-04-23): We're using a Sora payload error in place of application state error. Surface this differently. sentinel = creds.sora_sentinel.as_ref() .map(|sentinel| sentinel.get_sentinel().to_string()) - .ok_or(SoraError::SentinelBlock("Sentinel is required for image generation.".to_string()))?; + .ok_or(SoraImageGenError::SentinelBlock("Sentinel is required for image generation.".to_string()))?; } } - let http_request = client.post(SORA_IMAGE_GEN_URL) + let mut http_request = client.post(SORA_IMAGE_GEN_URL) .header("User-Agent", USER_AGENT) .header("Cookie", &cookie) .header("Authorization", &authorization_header) .header("Content-Type", "application/json") .header("OpenAI-Sentinel-Token", &sentinel); + + + if let Some(timeout) = request_timeout { + http_request = http_request.timeout(timeout); + } let http_request = http_request.json(&sora_request).build() - .map_err(|e| SoraError::NetworkError(e.to_string()))?; + .map_err(|e| SoraImageGenError::NetworkError(e.to_string()))?; let response = client.execute(http_request) .await - .map_err(|e| SoraError::NetworkError(e.to_string()))?; + .map_err(|e| SoraImageGenError::NetworkError(e.to_string()))?; let status = response.status(); let response_body = &response.text().await - .map_err(|e| SoraError::NetworkError(e.to_string()))?; + .map_err(|e| SoraImageGenError::NetworkError(e.to_string()))?; if status != reqwest::StatusCode::OK { + warn!("Sora image generation failure. Raw response: {:?}", response_body); + let error_response: RawSoraErrorResponse = serde_json::from_str(response_body) - .map_err(|e| SoraError::GenericError(format!("Failed to parse error response: {}", e)))?; + .map_err(|e| SoraImageGenError::GenericError(format!("Failed to parse error response: {}", e)))?; // Check for specific error codes if let Some(code) = &error_response.error.code { match code { SoraErrorCode::TokenExpired => { - return Err(SoraError::TokenExpired(error_response.error.message)); + return Err(SoraImageGenError::TokenExpired(error_response.error.message)); } SoraErrorCode::SentinelBlock => { - return Err(SoraError::SentinelBlock(error_response.error.message)); + return Err(SoraImageGenError::SentinelBlock(error_response.error.message)); } SoraErrorCode::TooManyConcurrentTasks => { - return Err(SoraError::TooManyConcurrentTasks(error_response.error.message)); + return Err(SoraImageGenError::TooManyConcurrentTasks(error_response.error.message)); } SoraErrorCode::Unknown(_) => { - return Err(SoraError::GenericError(error_response.error.message)); + return Err(SoraImageGenError::GenericError(error_response.error.message)); } SoraErrorCode::InvalidJwt => { - return Err(SoraError::InvalidJwt(error_response.error.message)); + return Err(SoraImageGenError::InvalidJwt(error_response.error.message)); + } + SoraErrorCode::UsernameRequired => { + return Err(SoraImageGenError::UsernameRequired(error_response.error.message)); } } } - return Err(SoraError::GenericError(error_response.error.message)); + return Err(SoraImageGenError::GenericError(error_response.error.message)); } let response = serde_json::from_str(response_body) - .map_err(|e| SoraError::GenericError(format!("Failed to parse success response: {}", e)))?; + .map_err(|e| SoraImageGenError::GenericError(format!("Failed to parse success response: {}", e)))?; Ok(response) } #[cfg(test)] mod tests { - use errors::AnyhowResult; use crate::requests::image_gen::image_gen_http_request::RawSoraResponse; + use errors::AnyhowResult; #[test] fn deserialize_task_id() -> AnyhowResult<()> { diff --git a/crates/lib/api_clients/openai_sora_client/src/requests/image_gen/image_gen_status.rs b/crates/lib/api_clients/openai_sora_client/src/requests/image_gen/image_gen_status.rs index 1411f69670..4862926110 100644 --- a/crates/lib/api_clients/openai_sora_client/src/requests/image_gen/image_gen_status.rs +++ b/crates/lib/api_clients/openai_sora_client/src/requests/image_gen/image_gen_status.rs @@ -6,7 +6,9 @@ use reqwest::Url; use serde_derive::{Deserialize, Serialize}; use std::fmt::{Debug, Display}; use std::{fs::File, io::Write, path::Path}; -use log::debug; +use log::{debug, warn}; +use crate::creds::sora_credential_set::SoraCredentialSet; +use crate::creds::sora_jwt_bearer_token::SoraJwtBearerToken; const SORA_STATUS_URL: &str = "https://sora.com/backend/video_gen"; @@ -171,7 +173,7 @@ pub struct InpaintItem { pub r#type: String, pub preset_id: Option, pub generation_id: Option, - pub upload_media_id: String, + pub upload_media_id: Option, pub frame_index: i32, pub source_start_frame: i32, } @@ -199,7 +201,16 @@ impl StatusRequest { } /// Gets the status of image generation tasks from Sora API -pub async fn get_image_gen_status(status_request: &StatusRequest, credentials: &SoraCredentials) -> Result { +pub async fn get_image_gen_status(status_request: &StatusRequest, credentials: &SoraCredentialSet) -> Result { + + let bearer_header = match credentials.jwt_bearer_token.as_ref() { + Some(bearer) => bearer.to_authorization_header_value(), + None => { + warn!("No JWT bearer for getting image gen status!"); + return Err(SoraError::NoBearerTokenAvailable) + }, + }; + let client = reqwest::Client::new(); let mut url = reqwest::Url::parse(SORA_STATUS_URL) @@ -216,8 +227,8 @@ pub async fn get_image_gen_status(status_request: &StatusRequest, credentials: & let http_request = client.get(url) .header("User-Agent", USER_AGENT) - .header("Cookie", &credentials.cookie) - .header("Authorization", credentials.authorization_header_value()) + .header("Cookie", credentials.cookies.as_str()) + .header("Authorization", bearer_header) .header("Content-Type", "application/json"); let http_request = http_request.build()?; @@ -235,9 +246,15 @@ pub async fn get_image_gen_status(status_request: &StatusRequest, credentials: & return Err(SoraError::AnyhowError(anyhow::anyhow!("The status request failed; status = {:?}, message = {:?}", status, response_body))); } - let response: VideoGenStatusResponse = serde_json::from_str(response_body)?; + let maybe_response = serde_json::from_str::(response_body); - Ok(response) + match maybe_response { + Ok(response) => Ok(response), + Err(err) => { + warn!("Failed to parse status response: {:?}", err); + Err(SoraError::JsonErrorWithBody(err, response_body.to_string())) + } + } } pub async fn save_generations_to_dir(generations: &[Generation], dir: &str) -> AnyhowResult<()> { @@ -275,6 +292,7 @@ mod tests { use errors::AnyhowResult; use std::fs::read_to_string; use testing::test_file_path::test_file_path; + use crate::creds::sora_credential_set::SoraCredentialSet; #[ignore] #[tokio::test] @@ -297,10 +315,12 @@ mod tests { let bearer = bearer.trim().to_string(); let creds = SoraCredentials { bearer_token: bearer, cookie, sentinel: Some(sentinel) }; + let new_creds = SoraCredentialSet::from_legacy_credentials(&creds)?; + // Get the task status for a specific task // let response = get_image_gen_status(&StatusRequest::new(None, Some("task_01jr9yvpfyetx9r7qvvx38scna".to_string())), &creds).await?; - let response = get_image_gen_status(&StatusRequest::new(Some(50), None), &creds).await?; + let response = get_image_gen_status(&StatusRequest::new(Some(50), None), &new_creds).await?; println!("task_id: {}", response.task_responses[0].id); let task_id = response.task_responses[0].id.clone(); diff --git a/crates/lib/api_clients/openai_sora_client/src/requests/image_gen/mod.rs b/crates/lib/api_clients/openai_sora_client/src/requests/image_gen/mod.rs index f0247756fb..17eab50891 100644 --- a/crates/lib/api_clients/openai_sora_client/src/requests/image_gen/mod.rs +++ b/crates/lib/api_clients/openai_sora_client/src/requests/image_gen/mod.rs @@ -1,5 +1,5 @@ pub (crate) mod image_gen_http_request; -pub use image_gen_http_request::SoraError; +pub use image_gen_http_request::SoraImageGenError; pub mod common; pub mod sora_image_gen_remix; pub mod sora_image_gen_simple; diff --git a/crates/lib/api_clients/openai_sora_client/src/requests/image_gen/sora_image_gen_remix.rs b/crates/lib/api_clients/openai_sora_client/src/requests/image_gen/sora_image_gen_remix.rs index 8dcb0c2f41..819aa49403 100644 --- a/crates/lib/api_clients/openai_sora_client/src/requests/image_gen/sora_image_gen_remix.rs +++ b/crates/lib/api_clients/openai_sora_client/src/requests/image_gen/sora_image_gen_remix.rs @@ -1,7 +1,8 @@ -use log::info; use crate::creds::credential_migration::CredentialMigrationRef; use crate::requests::image_gen::common::{ImageSize, NumImages, SoraImageGenResponse}; -use crate::requests::image_gen::image_gen_http_request::{image_gen_http_request, InpaintItem, InpaintItemType, OperationType, RawSoraImageGenRequest, SoraError, VideoGenType}; +use crate::requests::image_gen::image_gen_http_request::{image_gen_http_request, InpaintItem, InpaintItemType, OperationType, RawSoraImageGenRequest, SoraImageGenError, VideoGenType}; +use log::info; +use std::time::Duration; pub struct SoraImageGenRemixRequest<'a> { pub prompt: String, @@ -9,11 +10,12 @@ pub struct SoraImageGenRemixRequest<'a> { pub image_size: ImageSize, pub sora_media_tokens: Vec, pub credentials: CredentialMigrationRef<'a>, + pub request_timeout: Option, } /// The "remix" commands let you supply additional images as context. /// Sora "media tokens" of previously uploaded images must be supplied. -pub async fn sora_image_gen_remix(request: SoraImageGenRemixRequest<'_>) -> Result { +pub async fn sora_image_gen_remix(request: SoraImageGenRemixRequest<'_>) -> Result { let args = RawSoraImageGenRequest { r#type: VideoGenType::ImageGen, operation: OperationType::Remix, @@ -39,7 +41,7 @@ pub async fn sora_image_gen_remix(request: SoraImageGenRemixRequest<'_>) -> Resu // TODO: Replace this with debug level logging in the future. info!("Sending remix request: {:?}", args); - let result = image_gen_http_request(args, request.credentials).await?; + let result = image_gen_http_request(args, request.credentials, request.request_timeout).await?; Ok(SoraImageGenResponse { task_id: result.id, @@ -80,6 +82,7 @@ mod tests { image_size: ImageSize::Square, sora_media_tokens: vec!["media_01jqyhrz4detyvtzwp2p4j63ad".to_string()], credentials: CredentialMigrationRef::Legacy(&creds), + request_timeout: None, }).await?; println!("task_id: {}", response.task_id); diff --git a/crates/lib/api_clients/openai_sora_client/src/requests/image_gen/sora_image_gen_simple.rs b/crates/lib/api_clients/openai_sora_client/src/requests/image_gen/sora_image_gen_simple.rs index 2fc3d34af6..93e247f230 100644 --- a/crates/lib/api_clients/openai_sora_client/src/requests/image_gen/sora_image_gen_simple.rs +++ b/crates/lib/api_clients/openai_sora_client/src/requests/image_gen/sora_image_gen_simple.rs @@ -1,14 +1,15 @@ -use crate::credentials::SoraCredentials; +use crate::creds::credential_migration::CredentialMigrationRef; use crate::requests::image_gen::common::{ImageSize, NumImages, SoraImageGenResponse}; use crate::requests::image_gen::image_gen_http_request::{image_gen_http_request, OperationType, RawSoraImageGenRequest, VideoGenType}; use errors::AnyhowResult; -use crate::creds::credential_migration::CredentialMigrationRef; +use std::time::Duration; pub struct SoraImageGenSimpleRequest<'a> { pub prompt: String, pub num_images: NumImages, pub image_size: ImageSize, pub credentials: CredentialMigrationRef<'a>, + pub request_timeout: Option, } pub async fn sora_image_gen_simple(args: SoraImageGenSimpleRequest<'_>) -> AnyhowResult { @@ -24,7 +25,7 @@ pub async fn sora_image_gen_simple(args: SoraImageGenSimpleRequest<'_>) -> Anyho }; // TODO: Error handling. - let result = image_gen_http_request(sora_request, args.credentials).await?; + let result = image_gen_http_request(sora_request, args.credentials, args.request_timeout).await?; Ok(SoraImageGenResponse { task_id: result.id, @@ -34,12 +35,12 @@ pub async fn sora_image_gen_simple(args: SoraImageGenSimpleRequest<'_>) -> Anyho #[cfg(test)] mod tests { use crate::credentials::SoraCredentials; + use crate::creds::credential_migration::CredentialMigrationRef; use crate::requests::image_gen::common::{ImageSize, NumImages}; use crate::requests::image_gen::sora_image_gen_simple::{sora_image_gen_simple, SoraImageGenSimpleRequest}; use errors::AnyhowResult; use std::fs::read_to_string; use testing::test_file_path::test_file_path; - use crate::creds::credential_migration::CredentialMigrationRef; #[ignore] // You can manually run "ignore" tests in the IDE, but they won't run in CI. #[tokio::test] @@ -64,6 +65,7 @@ mod tests { num_images: NumImages::One, image_size: ImageSize::Square, credentials: CredentialMigrationRef::Legacy(&creds), + request_timeout: None, }).await?; println!("task_id: {}", response.task_id); diff --git a/crates/lib/api_clients/openai_sora_client/src/requests/upload/upload_media_from_bytes.rs b/crates/lib/api_clients/openai_sora_client/src/requests/upload/upload_media_from_bytes.rs index 1902b9ada9..686bca9261 100644 --- a/crates/lib/api_clients/openai_sora_client/src/requests/upload/upload_media_from_bytes.rs +++ b/crates/lib/api_clients/openai_sora_client/src/requests/upload/upload_media_from_bytes.rs @@ -2,10 +2,16 @@ use crate::creds::credential_migration::CredentialMigrationRef; use crate::requests::upload::upload_media_http_request::{upload_media_http_request, SoraMediaUploadResponse}; use crate::sora_error::SoraError; use std::path::PathBuf; +use std::time::Duration; /// Upload bytes. /// The underlying reqwest lib needs to own the bytes, so we can't pass as a reference. -pub async fn sora_media_upload_from_bytes(bytes: Vec, file_name: String, creds: CredentialMigrationRef<'_>) -> Result { +pub async fn sora_media_upload_from_bytes( + bytes: Vec, + file_name: String, + creds: CredentialMigrationRef<'_>, + maybe_timeout: Option, +) -> Result { let file_path = PathBuf::from(&file_name); // TODO: Read file magic bytes first, then fall back to this. @@ -20,5 +26,5 @@ pub async fn sora_media_upload_from_bytes(bytes: Vec, file_name: String, cre _ => "application/octet-stream", }; - Ok(upload_media_http_request(bytes, file_name, mime_type, creds).await?) + Ok(upload_media_http_request(bytes, file_name, mime_type, creds, maybe_timeout).await?) } diff --git a/crates/lib/api_clients/openai_sora_client/src/requests/upload/upload_media_from_file.rs b/crates/lib/api_clients/openai_sora_client/src/requests/upload/upload_media_from_file.rs index dc023285c1..e7fb48cf0b 100644 --- a/crates/lib/api_clients/openai_sora_client/src/requests/upload/upload_media_from_file.rs +++ b/crates/lib/api_clients/openai_sora_client/src/requests/upload/upload_media_from_file.rs @@ -2,6 +2,7 @@ use crate::creds::credential_migration::CredentialMigrationRef; use crate::requests::upload::upload_media_http_request::{upload_media_http_request, SoraMediaUploadResponse}; use crate::sora_error::SoraError; use std::path::Path; +use std::time::Duration; use tokio::fs::File; use tokio::io::AsyncReadExt; @@ -9,7 +10,11 @@ use tokio::io::AsyncReadExt; /// There's a better way to implement this. const INITIAL_BUFFER_SIZE : usize = 1024*1024; -pub async fn sora_media_upload_from_file>(file_path: P, creds: CredentialMigrationRef<'_>) -> Result { +pub async fn sora_media_upload_from_file>( + file_path: P, + creds: CredentialMigrationRef<'_>, + maybe_timeout: Option, +) -> Result { let mut file = File::open(&file_path).await?; let mut buffer = Vec::with_capacity(INITIAL_BUFFER_SIZE); file.read_to_end(&mut buffer).await?; @@ -31,7 +36,7 @@ pub async fn sora_media_upload_from_file>(file_path: P, creds: Cr _ => "application/octet-stream", }; - Ok(upload_media_http_request(buffer, filename, mime_type, creds).await?) + Ok(upload_media_http_request(buffer, filename, mime_type, creds, maybe_timeout).await?) } #[cfg(test)] @@ -63,6 +68,7 @@ mod tests { let response = sora_media_upload_from_file( image_path, CredentialMigrationRef::Legacy(&creds), + None, ).await?; println!("media: {:?}", response); diff --git a/crates/lib/api_clients/openai_sora_client/src/requests/upload/upload_media_http_request.rs b/crates/lib/api_clients/openai_sora_client/src/requests/upload/upload_media_http_request.rs index 722fce788a..3187c28810 100644 --- a/crates/lib/api_clients/openai_sora_client/src/requests/upload/upload_media_http_request.rs +++ b/crates/lib/api_clients/openai_sora_client/src/requests/upload/upload_media_http_request.rs @@ -1,3 +1,4 @@ +use std::time::Duration; use crate::credentials::{SoraCredentials, USER_AGENT}; use crate::creds::credential_migration::CredentialMigrationRef; use crate::sora_error::SoraError; @@ -69,6 +70,7 @@ pub (crate) async fn upload_media_http_request( filename: String, mime_type: &str, credentials: CredentialMigrationRef<'_>, + maybe_timeout: Option, ) -> Result { // Create multipart form @@ -97,11 +99,15 @@ pub (crate) async fn upload_media_http_request( // Make API request let client = Client::new(); - let request_builder = client.post(SORA_UPLOAD_MEDIA_URL) + let mut request_builder = client.post(SORA_UPLOAD_MEDIA_URL) .multipart(form) .header("User-Agent", USER_AGENT) .header("Cookie", &cookie) .header("Authorization", &auth_header); + + if let Some(timeout) = maybe_timeout { + request_builder = request_builder.timeout(timeout); + } let response = request_builder.send().await?; diff --git a/crates/lib/api_clients/openai_sora_client/src/sora_error.rs b/crates/lib/api_clients/openai_sora_client/src/sora_error.rs index 824cb10d2a..897df7ff75 100644 --- a/crates/lib/api_clients/openai_sora_client/src/sora_error.rs +++ b/crates/lib/api_clients/openai_sora_client/src/sora_error.rs @@ -29,6 +29,10 @@ pub enum SoraError { /// Failed to generate a bearer token despite the `200 OK` response. /// This has been generated by sending old/expired session cookies or empty cookies. FailedToGenerateBearer, + + /// Sora requires users to set up an account username before they can generate images. + /// This requires the user to set up that username. + SoraUsernameNotYetCreated, /// Something is wrong with the JWT bearer token. /// This error originates on our end as we try to parse the JWT. @@ -58,6 +62,10 @@ pub enum SoraError { /// serde_json::Error, likely from JSON deserialization schema mismatch. JsonError(serde_json::Error), + + /// serde_json::Error, likely from JSON deserialization schema mismatch. + /// Includes the original body. + JsonErrorWithBody(serde_json::Error, String), } impl Error for SoraError {} @@ -71,6 +79,9 @@ impl Display for SoraError { Self::FailedToGenerateBearer => { write!(f, "Failed to generate bearer. Probably invalid session.") } + Self::SoraUsernameNotYetCreated => { + write!(f, "Sora username not created. Please set up a username.") + } Self::LocalJwtClaimsParseError(message) => { write!(f, "Local JWT claims parse error: {}", message) } @@ -101,6 +112,9 @@ impl Display for SoraError { Self::JsonError(err) => { write!(f, "serde_json error: {}", err) } + Self::JsonErrorWithBody(err, message) => { + write!(f, "serde_json error: {:?} body: {:?}", err, message) + } } } } diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/image_studio/prompt/enqueue_studio_image_generation_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/image_studio/prompt/enqueue_studio_image_generation_handler.rs index cb09355b0f..d115a6188a 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/image_studio/prompt/enqueue_studio_image_generation_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/image_studio/prompt/enqueue_studio_image_generation_handler.rs @@ -15,7 +15,7 @@ use log::warn; use log::{debug, error}; use mysql_queries::queries::media_files::get::get_media_file::get_media_file; use openai_sora_client::credentials::SoraCredentials; -use openai_sora_client::requests::image_gen::SoraError; +use openai_sora_client::requests::image_gen::SoraImageGenError; use openai_sora_client::requests::sentinel_refresh::generate::token::generate_token; use serde::Deserialize; use serde::Serialize; @@ -326,7 +326,7 @@ pub async fn enqueue_studio_image_generation_handler(http_request: HttpRequest, debug!("Sora image gen response: {:?}", response); - if let Err(SoraError::SentinelBlock(msg)) = &response { + if let Err(SoraImageGenError::SentinelBlock(msg)) = &response { error!("Sora sentinel block, attempting refresh: {}", msg); match generate_token().await { @@ -382,16 +382,16 @@ pub async fn enqueue_studio_image_generation_handler(http_request: HttpRequest, let response = response.map_err(|err| { match err { - SoraError::TokenExpired(msg) => { + SoraImageGenError::TokenExpired(msg) => { error!("Sora token expired, needs refresh: {}", msg); // TODO: Implement token refresh logic here EnqueueImageGenRequestError::ServerError }, - SoraError::SentinelBlock(msg) => { + SoraImageGenError::SentinelBlock(msg) => { error!("Sora sentinel block still occurring after refresh: {}", msg); EnqueueImageGenRequestError::ServerError }, - SoraError::TooManyConcurrentTasks(msg) => { + SoraImageGenError::TooManyConcurrentTasks(msg) => { error!("Sora too many concurrent tasks: {}", msg); EnqueueImageGenRequestError::TooManyConcurrentTasks }, diff --git a/frontend/apps/editor2d/src/components/PromptBox.tsx b/frontend/apps/editor2d/src/components/PromptBox.tsx index c8d0cff643..10a1f8f74b 100644 --- a/frontend/apps/editor2d/src/components/PromptBox.tsx +++ b/frontend/apps/editor2d/src/components/PromptBox.tsx @@ -320,7 +320,7 @@ export const PromptBox = () => { if (!prompt.trim()) return; try { console.log( - "Enqueuing with prompt:", + "(3) Enqueuing with prompt:", prompt, "and reference images:", referenceImages, diff --git a/frontend/apps/editor3d/app/public/resources/sound/metroidprime_UI_12.wav b/frontend/apps/editor3d/app/public/resources/sound/metroidprime_UI_12.wav new file mode 100644 index 0000000000..2323f269fa Binary files /dev/null and b/frontend/apps/editor3d/app/public/resources/sound/metroidprime_UI_12.wav differ diff --git a/frontend/apps/editor3d/app/public/resources/sound/metroidprime_UI_13.wav b/frontend/apps/editor3d/app/public/resources/sound/metroidprime_UI_13.wav new file mode 100644 index 0000000000..fdd1442c7e Binary files /dev/null and b/frontend/apps/editor3d/app/public/resources/sound/metroidprime_UI_13.wav differ diff --git a/frontend/apps/editor3d/app/public/resources/sound/metroidprime_UI_14.wav b/frontend/apps/editor3d/app/public/resources/sound/metroidprime_UI_14.wav new file mode 100644 index 0000000000..c43944278a Binary files /dev/null and b/frontend/apps/editor3d/app/public/resources/sound/metroidprime_UI_14.wav differ diff --git a/frontend/apps/editor3d/app/public/resources/sound/metroidprime_UI_15.wav b/frontend/apps/editor3d/app/public/resources/sound/metroidprime_UI_15.wav new file mode 100644 index 0000000000..a51077fe51 Binary files /dev/null and b/frontend/apps/editor3d/app/public/resources/sound/metroidprime_UI_15.wav differ diff --git a/frontend/apps/editor3d/app/public/resources/sound/metroidprime_UI_18.wav b/frontend/apps/editor3d/app/public/resources/sound/metroidprime_UI_18.wav new file mode 100644 index 0000000000..f6eb93512c Binary files /dev/null and b/frontend/apps/editor3d/app/public/resources/sound/metroidprime_UI_18.wav differ diff --git a/frontend/apps/editor3d/app/public/resources/sound/metroidprime_UI_51.wav b/frontend/apps/editor3d/app/public/resources/sound/metroidprime_UI_51.wav new file mode 100644 index 0000000000..649699fd36 Binary files /dev/null and b/frontend/apps/editor3d/app/public/resources/sound/metroidprime_UI_51.wav differ diff --git a/frontend/apps/editor3d/app/public/resources/sound/metroidprime_UI_52.wav b/frontend/apps/editor3d/app/public/resources/sound/metroidprime_UI_52.wav new file mode 100644 index 0000000000..467db39108 Binary files /dev/null and b/frontend/apps/editor3d/app/public/resources/sound/metroidprime_UI_52.wav differ diff --git a/frontend/apps/editor3d/app/public/resources/sound/smrpg_character_extrapower.wav b/frontend/apps/editor3d/app/public/resources/sound/smrpg_character_extrapower.wav new file mode 100644 index 0000000000..48eb4e459a Binary files /dev/null and b/frontend/apps/editor3d/app/public/resources/sound/smrpg_character_extrapower.wav differ diff --git a/frontend/apps/editor3d/app/public/resources/sound/smrpg_click.wav b/frontend/apps/editor3d/app/public/resources/sound/smrpg_click.wav new file mode 100644 index 0000000000..f954d9c709 Binary files /dev/null and b/frontend/apps/editor3d/app/public/resources/sound/smrpg_click.wav differ diff --git a/frontend/apps/editor3d/app/public/resources/sound/smrpg_enemy_staticE.wav b/frontend/apps/editor3d/app/public/resources/sound/smrpg_enemy_staticE.wav new file mode 100644 index 0000000000..f6507d6413 Binary files /dev/null and b/frontend/apps/editor3d/app/public/resources/sound/smrpg_enemy_staticE.wav differ diff --git a/frontend/apps/editor3d/app/public/resources/sound/smrpg_ghost.wav b/frontend/apps/editor3d/app/public/resources/sound/smrpg_ghost.wav new file mode 100644 index 0000000000..ebb3f365d5 Binary files /dev/null and b/frontend/apps/editor3d/app/public/resources/sound/smrpg_ghost.wav differ diff --git a/frontend/apps/editor3d/app/public/resources/sound/smrpg_mario_giantshellkick.wav b/frontend/apps/editor3d/app/public/resources/sound/smrpg_mario_giantshellkick.wav new file mode 100644 index 0000000000..0a0a24ec0c Binary files /dev/null and b/frontend/apps/editor3d/app/public/resources/sound/smrpg_mario_giantshellkick.wav differ diff --git a/frontend/apps/editor3d/app/public/resources/sound/smrpg_wrong.wav b/frontend/apps/editor3d/app/public/resources/sound/smrpg_wrong.wav new file mode 100644 index 0000000000..f7a026c942 Binary files /dev/null and b/frontend/apps/editor3d/app/public/resources/sound/smrpg_wrong.wav differ diff --git a/frontend/apps/editor3d/app/src/flags/DevConsoleFlags.ts b/frontend/apps/editor3d/app/src/flags/DevConsoleFlags.ts new file mode 100644 index 0000000000..2f02578243 --- /dev/null +++ b/frontend/apps/editor3d/app/src/flags/DevConsoleFlags.ts @@ -0,0 +1,36 @@ + +// This is used to do things such as disable job polling in the dev console. + +export const SetDevFlag = (flagName: string, value: any) => { + if (!!!(window as any)._dev_flags) { + (window as any)._dev_flags = {}; + } + (window as any)._dev_flags[flagName] = value; +} + +export const GetDevFlag = (flagName: string) => { + if (!!(window as any)._dev_flags && flagName in (window as any)._dev_flags) { + return (window as any)._dev_flags[flagName]; + } + return null; +} + +export const DisableJobPolling = () => { + SetDevFlag("disable_job_polling", true); +} + +export const EnableJobPolling = () => { + SetDevFlag("disable_job_polling", false); +} + +export const IsJobPollingDisabled = () : boolean => { + return GetDevFlag("disable_job_polling") === true; +} + +(window as any).setDevFlag = SetDevFlag; +(window as any).getDevFlag = GetDevFlag; + +(window as any).disableJobPolling = DisableJobPolling; +(window as any).enableJobPolling = EnableJobPolling; + + diff --git a/frontend/apps/editor3d/app/src/hooks/useActiveJobs/utilities.ts b/frontend/apps/editor3d/app/src/hooks/useActiveJobs/utilities.ts index 4da3db7378..f68cf8f755 100644 --- a/frontend/apps/editor3d/app/src/hooks/useActiveJobs/utilities.ts +++ b/frontend/apps/editor3d/app/src/hooks/useActiveJobs/utilities.ts @@ -1,9 +1,15 @@ import { ToastTypes } from "~/enums"; import { addToast, setJobs } from "~/signals"; import { JobsApi } from "~/Classes/ApiManager/JobsApi"; +import { IsJobPollingDisabled } from "~/flags/DevConsoleFlags"; + export async function PollRecentJobs() { const jobsApi = new JobsApi(); + if (IsJobPollingDisabled()) { + return; + } + const response = await jobsApi.ListRecentJobs(); if (response.success && response.data) { setJobs(response.data); diff --git a/frontend/apps/editor3d/app/src/pages/PageEnigma/Editor/editor.ts b/frontend/apps/editor3d/app/src/pages/PageEnigma/Editor/editor.ts index bad23acca6..a9ebcc1e31 100644 --- a/frontend/apps/editor3d/app/src/pages/PageEnigma/Editor/editor.ts +++ b/frontend/apps/editor3d/app/src/pages/PageEnigma/Editor/editor.ts @@ -860,7 +860,6 @@ class Editor { } const uuid = crypto.randomUUID(); const file = new File([ab], `${uuid}.png`, { type: mimeString }); - return { base64Snapshot, file }; } diff --git a/frontend/apps/editor3d/app/src/pages/PageEnigma/InstallSounds.ts b/frontend/apps/editor3d/app/src/pages/PageEnigma/InstallSounds.ts index 9de089b70b..0d3a795415 100644 --- a/frontend/apps/editor3d/app/src/pages/PageEnigma/InstallSounds.ts +++ b/frontend/apps/editor3d/app/src/pages/PageEnigma/InstallSounds.ts @@ -7,23 +7,37 @@ export const InstallSounds = () => { return; } - // Good for simple immediate enqueue - r.setSoundOnce("done", new SoundEffect("resources/sound/oot_dialogue_done.wav", { defaultVolume: 0.2 })); // DEFAULT + // Good for simple menu choices + r.setSoundOnce("click", new SoundEffect("resources/sound/smrpg_click.wav", { defaultVolume: 0.2 })); + r.setSoundOnce("scifi_menu_beep_1", new SoundEffect("resources/sound/metroidprime_UI_15.wav", { defaultVolume: 0.3 })); + r.setSoundOnce("scifi_menu_beep_2", new SoundEffect("resources/sound/metroidprime_UI_14.wav", { defaultVolume: 0.3 })); + r.setSoundOnce("scifi_menu_select", new SoundEffect("resources/sound/metroidprime_UI_18.wav", { defaultVolume: 0.3 })); + + // Good for simple immediate enqueue success + r.setSoundOnce("done", new SoundEffect("resources/sound/oot_dialogue_done.wav", { defaultVolume: 0.4 })); // DEFAULT // Good for simple immediate failure - r.setSoundOnce("error_chirp", new SoundEffect("resources/sound/goldensun_135.wav", { defaultVolume: 0.2 })); + r.setSoundOnce("error_chirp", new SoundEffect("resources/sound/goldensun_135.wav", { defaultVolume: 0.4 })); r.setSoundOnce("spike_throw", new SoundEffect("resources/sound/smrpg_enemy_spikethrow.wav", { defaultVolume: 0.1 })); // DEFAULT + r.setSoundOnce("giant_shell_kick", new SoundEffect("resources/sound/smrpg_mario_giantshellkick.wav", { defaultVolume: 0.2 })); + r.setSoundOnce("wrong", new SoundEffect("resources/sound/smrpg_wrong.wav", { defaultVolume: 0.4 })); // Good for simple async success r.setSoundOnce("special_flower", new SoundEffect("resources/sound/smrpg_specialflower.wav", { defaultVolume: 0.2 })); // DEFAULT + r.setSoundOnce("extra_power", new SoundEffect("resources/sound/smrpg_character_extrapower.wav", { defaultVolume: 0.2 })); // DEFAULT // Good for async errors r.setSoundOnce("crumble", new SoundEffect("resources/sound/smrpg_drybones_crumble.wav", { defaultVolume: 0.1 })); // DEFAULT + r.setSoundOnce("ghost", new SoundEffect("resources/sound/smrpg_ghost.wav", { defaultVolume: 0.2 })); r.setSoundOnce("special_alert", new SoundEffect("resources/sound/goldensun_214.wav", { defaultVolume: 0.2 })); + r.setSoundOnce("scifi_alert", new SoundEffect("resources/sound/metroidprime_UI_52.wav", { defaultVolume: 0.2 })); + r.setSoundOnce("scifi_shrill_alert", new SoundEffect("resources/sound/metroidprime_UI_51.wav", { defaultVolume: 0.2 })); // Good for menus r.setSoundOnce("next", new SoundEffect("resources/sound/oot_dialogue_next.wav", { defaultVolume: 0.2 })); r.setSoundOnce("select", new SoundEffect("resources/sound/goldensun_111.wav", { defaultVolume: 0.4 })); + r.setSoundOnce("scifi_menu_open", new SoundEffect("resources/sound/metroidprime_UI_12.wav", { defaultVolume: 0.3 })); + r.setSoundOnce("scifi_menu_close", new SoundEffect("resources/sound/metroidprime_UI_13.wav", { defaultVolume: 0.3 })); // Good for special reward / celebration r.setSoundOnce("correct", new SoundEffect("resources/sound/smrpg_correct.wav", { defaultVolume: 0.1 })); diff --git a/frontend/apps/editor3d/app/src/pages/PageEnigma/PageEditor.tsx b/frontend/apps/editor3d/app/src/pages/PageEnigma/PageEditor.tsx index 8bbc1c765a..872052e3e2 100644 --- a/frontend/apps/editor3d/app/src/pages/PageEnigma/PageEditor.tsx +++ b/frontend/apps/editor3d/app/src/pages/PageEnigma/PageEditor.tsx @@ -344,13 +344,14 @@ export const PageEditor = () => { handleCameraFocalLengthChange={handleCameraFocalLengthChange} onAspectRatioSelect={onAspectRatioSelect} setEnginePrompt={(prompt) => { + console.log("setEnginePrompt", prompt); if (!editorEngine) { + console.log("editorEngine is not available"); return; } - editorEngine!.positive_prompt = prompt; }} - snapshotCurrentFrame={editorEngine?.snapShotOfCurrentFrame} + snapshotCurrentFrame={editorEngine?.snapShotOfCurrentFrame.bind(editorEngine)} /> { useSignals(); @@ -90,11 +89,12 @@ export const PageEnigma = ({ sceneToken }: { sceneToken?: string }) => { }); }); + useImageGenerationSuccessEvent(); + useImageGenerationFailureEvent(); + useEffect(() => { console.log("installing event listeners"); InstallSounds(); - InstallImageGenerationSuccess(); - InstallImageGenerationFailure(); }); if (validGpu === "unknown") { diff --git a/frontend/apps/editor3d/app/src/pages/PageEnigma/TauriEvents/SoraImageGenerationFailure.ts b/frontend/apps/editor3d/app/src/pages/PageEnigma/TauriEvents/SoraImageGenerationFailure.ts deleted file mode 100644 index 8a3163cf13..0000000000 --- a/frontend/apps/editor3d/app/src/pages/PageEnigma/TauriEvents/SoraImageGenerationFailure.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { SoundRegistry, SoundEffect } from "@storyteller/soundboard"; -import { listen } from '@tauri-apps/api/event'; -import { AppPreferencesPayload, CustomDirectory, GetAppPreferences, SystemDirectory } from "@storyteller/tauri-api"; -import { toast } from "@storyteller/ui-toaster"; - -type SoraImageGenerationFailed = { - prompt: string, -}; - -export const InstallImageGenerationFailure = () => { - listen('sora-image-generation-failed', async (event) => { - const prefs = await GetAppPreferences(); - const soundName = prefs.preferences?.generation_failure_sound; - if (soundName !== undefined) { - const registry = SoundRegistry.getInstance(); - registry.playSound(soundName); - } - toast.error("Image generation failed!"); - }); -} - -export const ImageGenerationFailure = () => { -} - diff --git a/frontend/apps/editor3d/app/src/pages/PageEnigma/TauriEvents/SoraImageGenerationSuccess.ts b/frontend/apps/editor3d/app/src/pages/PageEnigma/TauriEvents/SoraImageGenerationSuccess.ts deleted file mode 100644 index da4e91cb58..0000000000 --- a/frontend/apps/editor3d/app/src/pages/PageEnigma/TauriEvents/SoraImageGenerationSuccess.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { SoundRegistry, SoundEffect } from "@storyteller/soundboard"; -import { listen } from '@tauri-apps/api/event'; -import { AppPreferencesPayload, CustomDirectory, GetAppPreferences, SystemDirectory } from "@storyteller/tauri-api"; -import { toast } from "@storyteller/ui-toaster"; - -type ImageGenerationSuccess = { - media_file_token: string, -}; - -export const InstallImageGenerationSuccess = () => { - listen('sora-image-generation-complete', async (event) => { - const prefs = await GetAppPreferences(); - const soundName = prefs.preferences?.generation_success_sound; - if (soundName !== undefined) { - const registry = SoundRegistry.getInstance(); - registry.playSound(soundName); - } - toast.success("Image generated!"); - }); -} - -export const ImageGenerationSuccess = () => { -} diff --git a/frontend/apps/editor3d/app/src/pages/PageEnigma/TauriEvents/useImageGenerationFailureEvent.ts b/frontend/apps/editor3d/app/src/pages/PageEnigma/TauriEvents/useImageGenerationFailureEvent.ts new file mode 100644 index 0000000000..87a5079084 --- /dev/null +++ b/frontend/apps/editor3d/app/src/pages/PageEnigma/TauriEvents/useImageGenerationFailureEvent.ts @@ -0,0 +1,41 @@ +import { useEffect } from "react"; +import { listen, UnlistenFn } from '@tauri-apps/api/event'; +import { SoundRegistry } from "@storyteller/soundboard"; +import { GetAppPreferences } from "@storyteller/tauri-api"; +import { toast } from "@storyteller/ui-toaster"; + +type SoraImageGenerationFailed = { + prompt: string, +}; + +export const useImageGenerationFailureEvent = () => { + useEffect(() => { + let isUnmounted = false; + let unlisten: Promise; + + const setup = async () => { + unlisten = listen('sora-image-generation-failed', async (event) => { + console.log("Image generation failure event received:", event); + const prefs = await GetAppPreferences(); + const soundName = prefs.preferences?.generation_failure_sound; + if (soundName !== undefined) { + const registry = SoundRegistry.getInstance(); + registry.playSound(soundName); + } + toast.error("Image generation failed!"); + }); + + if (isUnmounted) { + unlisten.then(f => f()); // Unsubscribe if unmounted early. + } + }; + + setup(); + + return () => { + isUnmounted = true; + unlisten.then(f => f()); + }; + + }, []); +} diff --git a/frontend/apps/editor3d/app/src/pages/PageEnigma/TauriEvents/useImageGenerationSuccessEvent.ts b/frontend/apps/editor3d/app/src/pages/PageEnigma/TauriEvents/useImageGenerationSuccessEvent.ts new file mode 100644 index 0000000000..f9f758aae2 --- /dev/null +++ b/frontend/apps/editor3d/app/src/pages/PageEnigma/TauriEvents/useImageGenerationSuccessEvent.ts @@ -0,0 +1,41 @@ +import { useEffect } from "react"; +import { listen, UnlistenFn } from '@tauri-apps/api/event'; +import { SoundRegistry } from "@storyteller/soundboard"; +import { GetAppPreferences } from "@storyteller/tauri-api"; +import { toast } from "@storyteller/ui-toaster"; + +type ImageGenerationSuccess = { + media_file_token: string, +}; + +export const useImageGenerationSuccessEvent = () => { + useEffect(() => { + let isUnmounted = false; + let unlisten: Promise; + + const setup = async () => { + unlisten = listen('sora-image-generation-complete', async (event) => { + console.log("Image generation success event received:", event); + const prefs = await GetAppPreferences(); + const soundName = prefs.preferences?.generation_success_sound; + if (soundName !== undefined) { + const registry = SoundRegistry.getInstance(); + registry.playSound(soundName); + } + toast.success("Image generated!"); + }); + + if (isUnmounted) { + unlisten.then(f => f()); // Unsubscribe if unmounted early. + } + }; + + setup(); + + return () => { + isUnmounted = true; + unlisten.then(f => f()); + }; + + }, []); +} diff --git a/frontend/apps/editor3d/app/src/pages/PageEnigma/comps/GenerateModals/Activity.tsx b/frontend/apps/editor3d/app/src/pages/PageEnigma/comps/GenerateModals/Activity.tsx index 5bb2ea6375..e76c5e879e 100644 --- a/frontend/apps/editor3d/app/src/pages/PageEnigma/comps/GenerateModals/Activity.tsx +++ b/frontend/apps/editor3d/app/src/pages/PageEnigma/comps/GenerateModals/Activity.tsx @@ -9,6 +9,7 @@ import { useEffect, useState, useRef, useCallback } from "react"; import { JobsApi } from "~/Classes/ApiManager/JobsApi"; import { toast } from "@storyteller/ui-toaster"; import { ActiveJob } from "~/pages/PageEnigma/models"; +import { IsJobPollingDisabled } from "~/flags/DevConsoleFlags"; // TODO ensure we de-dupe all this extra code. interface CompletedItem { token: string; @@ -85,6 +86,9 @@ export function Activity() { let mounted = true; async function fetchJobs() { + if (IsJobPollingDisabled()) { + return; + } try { const jobsApi = new JobsApi(); const response = await jobsApi.ListRecentJobs(); diff --git a/frontend/apps/editor3d/app/src/pages/PageEnigma/comps/PromptBox/PromptBox.tsx b/frontend/apps/editor3d/app/src/pages/PageEnigma/comps/PromptBox/PromptBox.tsx deleted file mode 100644 index 3f6665ccad..0000000000 --- a/frontend/apps/editor3d/app/src/pages/PageEnigma/comps/PromptBox/PromptBox.tsx +++ /dev/null @@ -1,936 +0,0 @@ -import { useState, useRef, useEffect, useContext } from "react"; -import { useSignals } from "@preact/signals-react/runtime"; -import { invoke } from "@tauri-apps/api/core"; -import { EngineContext } from "~/pages/PageEnigma/contexts/EngineContext"; -import { IsDesktopApp } from "@storyteller/tauri-utils"; -import { uploadImage } from "~/components/reusable/UploadModalMedia/uploadImage"; -import { downloadFileFromUrl } from "@storyteller/api"; -import { - faPlus, - faCamera, - faSave, - faSparkles, - faUpload, - faImages, - faDownload, - faSpinnerThird, - faTimes, - faMessageCheck, - faMessageXmark, -} from "@fortawesome/pro-solid-svg-icons"; -import { - faRectangleWide, - faRectangleVertical, - faSquare, - faRectangle, - faTableCellsLarge, -} from "@fortawesome/pro-regular-svg-icons"; - -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; - -import { PopoverItem, PopoverMenu } from "~/components/reusable/Popover"; -import { Button, ToggleButton, Tooltip } from "~/components"; -import { CameraAspectRatio } from "~/pages/PageEnigma/enums"; - -import { - cameraAspectRatio, - disableHotkeyInput, - DomLevels, - enableHotkeyInput, - gridVisibility, - setGridVisibility, -} from "~/pages/PageEnigma/signals"; - -import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames"; -import Queue from "~/pages/PageEnigma/Queue/Queue"; -import { toEngineActions } from "~/pages/PageEnigma/Queue/toEngineActions"; -import { PromptsApi } from "@storyteller/api"; -import { SoundRegistry } from "@storyteller/soundboard"; -import { GetAppPreferences, SoraImageRemix } from "@storyteller/tauri-api"; -import { CommandSuccessStatus } from "@storyteller/tauri-api"; - -import { - cameras, - selectedCameraId, - addCamera, - deleteCamera, - updateCamera, - focalLengthDragging, -} from "~/pages/PageEnigma/signals/camera"; -import { toast } from "@storyteller/ui-toaster"; -import { isPromptBoxFocused } from "~/pages/PageEnigma/signals/promptBox"; - -interface ReferenceImage { - id: string; - url: string; - file: File; - mediaToken: string; -} - -import { EngineApi } from "~/Classes/ApiManager/EngineApi"; -import { UploaderStates } from "~/enums/UploaderStates"; -import { CameraSettingsModal } from "../CameraSettingsModal"; -import { twMerge } from "tailwind-merge"; -import { GalleryModal, GalleryItem } from "@storyteller/ui-gallery-modal"; -import { Modal } from "@storyteller/ui-modal"; - -export const PromptBox = () => { - useSignals(); - const editorEngine = useContext(EngineContext); - const fileInputRef = useRef(null); - const textareaRef = useRef(null); - const [content, setContent] = useState(null); - const [isModalOpen, setIsModalOpen] = useState(false); - - const [prompt, setPrompt] = useState(""); - const [isEnqueueing, setisEnqueueing] = useState(false); - const [useSystemPrompt, setUseSystemPrompt] = useState(true); - const [referenceImages, setReferenceImages] = useState([]); - const [uploadingImages, setUploadingImages] = useState< - { id: string; file: File }[] - >([]); - const [aspectRatioList, setAspectRatioList] = useState([ - { - label: "3:2", - selected: true, - icon: , - }, - { - label: "2:3", - selected: false, - icon: , - }, - { - label: "1:1", - selected: false, - icon: , - }, - ]); - const [isCameraSettingsOpen, setIsCameraSettingsOpen] = useState(false); - const [isGalleryModalOpen, setIsGalleryModalOpen] = useState(false); - const [selectedGalleryImages, setSelectedGalleryImages] = useState( - [], - ); - const [activeGalleryTab, setActiveGalleryTab] = useState("my-media"); - - useEffect(() => { - if (textareaRef.current) { - textareaRef.current.style.height = "auto"; - textareaRef.current.style.height = `${textareaRef.current.scrollHeight}px`; - } - }, [prompt]); - - // Update aspect ratio list based on the current cameraAspectRatio signal - useEffect(() => { - setAspectRatioList((prev) => - prev.map((item) => ({ - ...item, - selected: - (item.label === "16:9" && - cameraAspectRatio.value === CameraAspectRatio.HORIZONTAL_16_9) || - (item.label === "3:2" && - cameraAspectRatio.value === CameraAspectRatio.HORIZONTAL_3_2) || - (item.label === "2:3" && - cameraAspectRatio.value === CameraAspectRatio.VERTICAL_2_3) || - (item.label === "1:1" && - cameraAspectRatio.value === CameraAspectRatio.SQUARE_1_1), - })), - ); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [cameraAspectRatio.value]); - - const handleCameraSelect = (selectedItem: PopoverItem) => { - const selectedCamera = cameras.value.find( - (cam) => cam.label === selectedItem.label, - ); - if (selectedCamera && editorEngine) { - selectedCameraId.value = selectedCamera.id; - - // Show focal length display temporarily - // TODO: Rename dragging to visible - BFlat - focalLengthDragging.value = { - isDragging: true, - focalLength: selectedCamera.focalLength, - }; - setTimeout(() => { - focalLengthDragging.value = { - isDragging: false, - focalLength: selectedCamera.focalLength, - }; - }, 1500); - - // Update the main camera to match the selected camera's properties - if (editorEngine.camera) { - // First update position and lookAt - editorEngine.camera.position.set( - selectedCamera.position.x, - selectedCamera.position.y, - selectedCamera.position.z, - ); - editorEngine.camera.lookAt( - selectedCamera.lookAt.x, - selectedCamera.lookAt.y, - selectedCamera.lookAt.z, - ); - - // Update FOV - editorEngine.camera.fov = editorEngine.focalLengthToFov( - selectedCamera.focalLength, - ); - editorEngine.camera.updateProjectionMatrix(); - - // Reset and update camera controls - if (editorEngine.cameraViewControls) { - editorEngine.cameraViewControls.reset(); - editorEngine.cameraViewControls.update(0); - } - - // Force a render to update the view - editorEngine.renderScene(); - - // Queue an update to ensure the engine processes the camera change - // Queue.publish({ - // queueName: QueueNames.TO_ENGINE, - // action: toEngineActions.CAMERA_CHANGED, - // data: selectedCamera, - // }); - } - - // Force update camera properties in the state - updateCamera(selectedCamera.id, { - focalLength: selectedCamera.focalLength, - position: selectedCamera.position, - rotation: selectedCamera.rotation, - lookAt: selectedCamera.lookAt, - }); - } - }; - - const handleAddCamera = () => { - // Check if we've reached the maximum number of cameras - if (cameras.value.length >= 6) { - console.warn("Maximum number of cameras (6) reached"); - return; - } - - const newIndex = cameras.value.length + 1; - const newId = `cam${newIndex}`; - - // This is for generating random orbital position for the new camera using spherical coordinates - const radius = Math.random() * 5 + 7; // Distance from center: 7 to 12 units - const theta = Math.random() * Math.PI * 2; // Azimuthal angle: 0 to 2π - const phi = Math.PI / 3 + (Math.random() * Math.PI) / 6; // Polar angle: π/3 to π/2 (60° to 90° from horizontal) - - // Convert spherical coordinates to Cartesian coordinates - const randomX = radius * Math.sin(phi) * Math.cos(theta); - const randomY = Math.abs(radius * Math.cos(phi)) + 2; // Ensure Y is positive and at least 2 units up - const randomZ = radius * Math.sin(phi) * Math.sin(theta); - - addCamera({ - id: newId, - label: `Camera ${newIndex}`, - focalLength: 24, - position: { - x: randomX, - y: randomY, - z: randomZ, - }, - rotation: { x: 0, y: 0, z: 0 }, - lookAt: { x: 0, y: 0, z: 0 }, - }); - - // Switch to the newly created camera - selectedCameraId.value = newId; - - // Update the engine camera to match the new camera's properties - if (editorEngine && editorEngine.camera) { - editorEngine.camera.position.set(randomX, randomY, randomZ); - editorEngine.camera.lookAt(0, 0, 0); - editorEngine.camera.fov = editorEngine.focalLengthToFov(24); - editorEngine.camera.updateProjectionMatrix(); - - // Reset and update camera controls - if (editorEngine.cameraViewControls) { - editorEngine.cameraViewControls.reset(); - editorEngine.cameraViewControls.update(0); - } - - // Force a render to update the view - editorEngine.renderScene(); - - // Queue.publish({ - // queueName: QueueNames.TO_ENGINE, - // action: toEngineActions.CAMERA_CHANGED, - // data: { - // id: newId, - // label: `Camera ${newIndex}`, - // focalLength: 24, - // position: { x: randomX, y: randomY, z: randomZ }, - // rotation: { x: 0, y: 0, z: 0 }, - // lookAt: { x: 0, y: 0, z: 0 }, - // }, - // }); - } - }; - - const handleCameraNameChange = (id: string, newName: string) => { - updateCamera(id, { label: newName }); - }; - - const handleCameraFocalLengthChange = (id: string, value: number) => { - const camera = cameras.value.find((cam) => cam.id === id); - if (camera) { - updateCamera(id, { focalLength: value }); - } - }; - - const handleAspectRatioSelect = (selectedItem: PopoverItem) => { - setAspectRatioList((prev) => - prev.map((item) => ({ - ...item, - selected: item.label === selectedItem.label, - })), - ); - - // Map the selected label to the corresponding CameraAspectRatio enum value - let newRatio: CameraAspectRatio; - switch (selectedItem.label) { - case "16:9": - newRatio = CameraAspectRatio.HORIZONTAL_16_9; - break; - case "3:2": - newRatio = CameraAspectRatio.HORIZONTAL_3_2; - break; - case "2:3": - newRatio = CameraAspectRatio.VERTICAL_2_3; - break; - case "1:1": - newRatio = CameraAspectRatio.SQUARE_1_1; - break; - default: - newRatio = CameraAspectRatio.HORIZONTAL_16_9; - } - - // Publish the change to the engine - Queue.publish({ - queueName: QueueNames.TO_ENGINE, - action: toEngineActions.CHANGE_CAMERA_ASPECT_RATIO, - data: newRatio, - }); - }; - - const handleFileUpload = (event: React.ChangeEvent) => { - const files = event.target.files; - if (files) { - Array.from(files).forEach((file) => { - const uploadId = Math.random().toString(36).substring(7); - setUploadingImages((prev) => [...prev, { id: uploadId, file }]); - - const reader = new FileReader(); - reader.onloadend = () => { - uploadImage({ - title: `reference-image-${Math.random().toString(36).substring(2, 15)}`, - assetFile: file, - progressCallback: (newState) => { - console.debug("Upload progress:", newState.data); - if (newState.status === UploaderStates.success && newState.data) { - const referenceImage: ReferenceImage = { - id: Math.random().toString(36).substring(7), - url: reader.result as string, - file, - mediaToken: newState.data || "", - }; - setReferenceImages((prev) => [...prev, referenceImage]); - setUploadingImages((prev) => - prev.filter((img) => img.id !== uploadId), - ); - console.log("Reference image added:", referenceImage); - } else if ( - newState.status === UploaderStates.assetError || - newState.status === UploaderStates.imageCreateError - ) { - setUploadingImages((prev) => - prev.filter((img) => img.id !== uploadId), - ); - console.error("Upload failed"); - } - }, - }); - }; - - reader.readAsDataURL(file); - }); - } - }; - - const handleRemoveReference = (id: string) => { - setReferenceImages((prev) => prev.filter((img) => img.id !== id)); - if (fileInputRef.current) { - fileInputRef.current.value = ""; - } - }; - - const handleUploadClick = () => { - fileInputRef.current?.click(); - }; - - const handleGallerySelect = () => { - setIsGalleryModalOpen(true); - }; - - const handleGalleryClose = () => { - setIsGalleryModalOpen(false); - setSelectedGalleryImages([]); - }; - - const handleImageSelect = (id: string) => { - setSelectedGalleryImages((prev) => { - if (prev.includes(id)) { - return prev.filter((imageId) => imageId !== id); - } - if (prev.length >= 4) { - return prev; - } - return [...prev, id]; - }); - }; - - const handleGalleryImages = (selectedItems: GalleryItem[]) => { - selectedItems.forEach((item) => { - if (!item.fullImage) return; - const referenceImage: ReferenceImage = { - id: Math.random().toString(36).substring(7), - url: item.fullImage, - file: new File([], "gallery-image"), - mediaToken: item.id, - }; - setReferenceImages((prev) => [...prev, referenceImage]); - }); - setIsGalleryModalOpen(false); - setSelectedGalleryImages([]); - }; - - const handleAction = (action: string) => { - switch (action) { - case "upload": - handleUploadClick(); - break; - case "gallery": - handleGallerySelect(); - break; - default: - console.log("Unknown action:", action); - } - }; - - const handlePaste = (e: React.ClipboardEvent) => { - e.preventDefault(); - const pastedText = e.clipboardData.getData("text").trim(); - setPrompt(pastedText); - }; - - const handleChange = (e: React.ChangeEvent) => { - setPrompt(e.target.value); - }; - - const handleError = (errorMessage: string) => { - setContent(errorMessage); - setIsModalOpen(true); - }; - - const handleEnqueue = async () => { - const isDesktop = IsDesktopApp(); - console.log("Is this a desktop app?", isDesktop); - - const r = SoundRegistry.getInstance(); - r.playSound("test"); - - if (isDesktop) { - await handleTauriEnqueue(); - } else { - await handleWebEnqueue(); - } - }; - - const handleWebEnqueue = async () => { - if (!prompt.trim()) return; - - setisEnqueueing(true); - - if (editorEngine) { - editorEngine.positive_prompt = prompt; - - //const engineApi = new EngineApi(); - const promptsApi = new PromptsApi(); - - const snapshot = editorEngine.snapShotOfCurrentFrame(false); - - if (snapshot) { - const snapshotResult = await promptsApi.uploadSceneSnapshot({ - screenshot: snapshot.file, - sceneMediaToken: "", - }); - - console.log("useSystemPrompt", useSystemPrompt); - - const response = await promptsApi.enqueueImageGeneration({ - disableSystemPrompt: !useSystemPrompt, - prompt: prompt, - snapshotMediaToken: snapshotResult.data || "", - additionalImages: referenceImages.map((image) => image.mediaToken), - }); - - console.log("response", response); - - if (response.errorMessage) { - handleError(response.errorMessage); - setisEnqueueing(false); - return; - } - } - - try { - // Here we would pass both the prompt and reference images to the generation - console.log( - "Enqueuing with prompt:", - prompt, - "and reference images:", - referenceImages, - ); - await new Promise((resolve) => setTimeout(resolve, 2000)); - } finally { - setisEnqueueing(false); - toast.success("Image added to queue"); - } - } - setisEnqueueing(false); - }; - - const handleTauriEnqueue = async () => { - if (!prompt.trim()) return; - - setisEnqueueing(true); - - setTimeout(() => { - // TODO(bt,2025-05-08): This is a hack so we don't accidentally wind up with a permanently disabled prompt box if - // the backend hangs on a given request. (Sometimes Sora doesn't respond.) Ideally what we would do instead is only - // force the prompt box to be enabled again as long as another request isn't running, ie. if we just fired off two - // requests in succession, we wouldn't want to turn it off as the first one gets submitted. - console.debug('Turn off blocking of prompt box...'); - setisEnqueueing(false); - }, 10000); - - if (editorEngine) { - editorEngine.positive_prompt = prompt; - - const promptsApi = new PromptsApi(); - - const snapshot = editorEngine.snapShotOfCurrentFrame(false); - - if (snapshot) { - const snapshotResult = await promptsApi.uploadSceneSnapshot({ - screenshot: snapshot.file, - //sceneMediaToken: "", - }); - - console.log("snapshotResult", snapshotResult) - - const generateResponse = await SoraImageRemix({ - snapshot_media_token: snapshotResult.data!, - disable_system_prompt: !useSystemPrompt, - prompt: prompt, - maybe_additional_images: referenceImages.map( - (image) => image.mediaToken, - ), - maybe_number_of_samples: 1, - }); - - console.log("generateResponse", generateResponse) - - if (generateResponse.status === CommandSuccessStatus.Success) { - const prefs = await GetAppPreferences(); - const soundName = prefs.preferences?.enqueue_success_sound; - if (soundName !== undefined) { - const registry = SoundRegistry.getInstance(); - registry.playSound(soundName); - } - toast.success("Image added to queue"); - setisEnqueueing(false); - return; - - } else { - const prefs = await GetAppPreferences(); - const soundName = prefs.preferences?.enqueue_failure_sound; - if (soundName !== undefined) { - const registry = SoundRegistry.getInstance(); - registry.playSound(soundName); - } - toast.error("Failed to enqueue image"); - setisEnqueueing(false); - return; - } - - // const generateResponse = await invoke("sora_image_remix_command", { - // request: { - // snapshot_media_token: snapshotResult.data!, - // disable_system_prompt: !useSystemPrompt, - // prompt: prompt, - // maybe_additional_images: referenceImages.map( - // (image) => image.mediaToken, - // ), - // maybe_number_of_samples: 1, - // }, - // }); - - //if (generateResponse.errorMessage) { - // handleError(response.errorMessage); - // setisEnqueueing(false); - // return; - //} - } - - try { - // Here we would pass both the prompt and reference images to the generation - console.log( - "Enqueuing with prompt:", - prompt, - "and reference images:", - referenceImages, - ); - await new Promise((resolve) => setTimeout(resolve, 2000)); - } finally { - setisEnqueueing(false); - toast.success("Image added to queue"); - } - } - setisEnqueueing(false); - }; - - const handleKeyDown = (e: React.KeyboardEvent) => { - if (e.key === "Enter" && !e.shiftKey) { - e.preventDefault(); - handleWebEnqueue(); - } - }; - - // Get the current aspect ratio icon based on the cameraAspectRatio signal - const getCurrentAspectRatioIcon = () => { - switch (cameraAspectRatio.value) { - case CameraAspectRatio.HORIZONTAL_3_2: - return faRectangle; - case CameraAspectRatio.VERTICAL_2_3: - return faRectangleVertical; - case CameraAspectRatio.VERTICAL_9_16: - return faRectangleVertical; - case CameraAspectRatio.SQUARE_1_1: - return faSquare; - default: - return faRectangleWide; - } - }; - - const handleSaveFrame = async () => { - if (editorEngine) { - const snapshot = editorEngine.snapShotOfCurrentFrame(false); - if (snapshot) { - const engineApi = new EngineApi(); - await engineApi.uploadSceneSnapshot({ - screenshot: snapshot.file || "", - sceneMediaToken: "", - }); - } - } - }; - - const handleDownloadFrame = () => { - if (editorEngine) { - editorEngine.snapShotOfCurrentFrame(true); - } - }; - - return ( - <> - { - setIsModalOpen(false); - setContent(""); - }} - > - {content} - -
- {(referenceImages.length > 0 || uploadingImages.length > 0) && ( -
- {referenceImages.map((image) => ( -
- Reference - -
- ))} - {uploadingImages.map(({ id, file }) => { - const previewUrl = URL.createObjectURL(file); - return ( -
-
- Uploading preview -
-
- -
-
- ); - })} -
- )} -
- -
- , - action: "upload", - }, - { - label: "Choose from library", - selected: false, - icon: , - action: "gallery", - }, - ]} - onPanelAction={handleAction} - showIconsInList - buttonClassName="bg-transparent hover:bg-transparent py-1.5 px-0 pr-1 m-0 hover:opacity-50 transition-opacity duration-100 ring-0 border-none focus:ring-0 outline-none" - triggerIcon={ - - } - /> - -