From c4f401923b6ae58183c09b598577ff2f75bc4783 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 11 May 2025 01:02:00 -0400 Subject: [PATCH] various sora / generate improvements (#600) --- .../src/commands/command_response_wrapper.rs | 21 +- .../commands/sora/sora_image_remix_command.rs | 107 +- .../src/commands/sora/sora_logout_command.rs | 5 +- ...ora_task_status_with_session_auto_renew.rs | 9 +- .../src/recipes/wait_for_image_gen_status.rs | 5 +- .../image_gen/image_gen_http_request.rs | 3 + .../requests/image_gen/image_gen_status.rs | 36 +- .../openai_sora_client/src/sora_error.rs | 7 + .../editor2d/src/components/PromptBox.tsx | 2 +- .../editor3d/app/src/flags/DevConsoleFlags.ts | 36 + .../app/src/hooks/useActiveJobs/utilities.ts | 6 + .../comps/GenerateModals/Activity.tsx | 4 + .../PageEnigma/comps/PromptBox/PromptBox.tsx | 936 ------------------ .../PageEnigma/comps/PromptBox/index.tsx | 1 - .../promptbox/src/lib/PromptBox2D.tsx | 2 +- .../promptbox/src/lib/PromptBox3D.tsx | 109 +- .../tauri-api/src/lib/sora/SoraImageRemix.ts | 16 +- 17 files changed, 295 insertions(+), 1010 deletions(-) create mode 100644 frontend/apps/editor3d/app/src/flags/DevConsoleFlags.ts delete mode 100644 frontend/apps/editor3d/app/src/pages/PageEnigma/comps/PromptBox/PromptBox.tsx delete mode 100644 frontend/apps/editor3d/app/src/pages/PageEnigma/comps/PromptBox/index.tsx diff --git a/crates/desktop/artcraft/src/commands/command_response_wrapper.rs b/crates/desktop/artcraft/src/commands/command_response_wrapper.rs index aefe2a330b..610f129efb 100644 --- a/crates/desktop/artcraft/src/commands/command_response_wrapper.rs +++ b/crates/desktop/artcraft/src/commands/command_response_wrapper.rs @@ -3,7 +3,11 @@ use std::error::Error; use std::str::FromStr; /// Easy to use Result type. -pub type CommandResult = Result, CommandErrorResponseWrapper>; +pub type CommandResult = + Result, CommandErrorResponseWrapper>; + +/// No inner payloads for this type. Just strings as messages. +pub type SimpleCommandResult = CommandResult<(), (), ()>; /// Statuses for successful commands. #[derive(Serialize, Clone, Debug)] @@ -18,6 +22,7 @@ pub enum CommandSuccessStatus { pub enum CommandErrorStatus { BadRequest, // 400 Unauthorized, // 401 + TooManyRequests, // 429 ServerError, // 500 } @@ -34,15 +39,22 @@ pub struct CommandSuccessResponseWrapper { } #[derive(Serialize, Debug)] -pub struct CommandErrorResponseWrapper { +pub struct CommandErrorResponseWrapper { /// Both "success" and "error" types have a `status` field for the frontend. pub status: CommandErrorStatus, #[serde(skip_serializing_if = "Option::is_none")] pub error_message: Option, + /// Represents a "type" of error, preferably an enum without payloads. + /// Preferably this serializes to a string. #[serde(skip_serializing_if = "Option::is_none")] - pub error_details: Option, + pub error_type: Option, + + /// Represents a full error details object or payload. + /// This could be a structure, message, or whatever. + #[serde(skip_serializing_if = "Option::is_none")] + pub error_details: Option, } @@ -96,11 +108,12 @@ impl From<&str> for CommandSuccessResponseWrapper { } } -impl From<&str> for CommandErrorResponseWrapper { +impl From<&str> for CommandErrorResponseWrapper<(), ()> { fn from(value: &str) -> Self { CommandErrorResponseWrapper { status: CommandErrorStatus::BadRequest, // NB: Default to 500 error. error_message: Some(value.to_string()), + error_type: None, error_details: None, } } diff --git a/crates/desktop/artcraft/src/commands/sora/sora_image_remix_command.rs b/crates/desktop/artcraft/src/commands/sora/sora_image_remix_command.rs index 26b84e2667..8120a4e42e 100644 --- a/crates/desktop/artcraft/src/commands/sora/sora_image_remix_command.rs +++ b/crates/desktop/artcraft/src/commands/sora/sora_image_remix_command.rs @@ -1,4 +1,4 @@ -use crate::commands::command_response_wrapper::CommandResult; +use crate::commands::command_response_wrapper::{CommandErrorResponseWrapper, CommandErrorStatus, CommandResult}; use crate::events::sendable_event_trait::SendableEvent; use crate::events::sora::sora_image_enqueue_failure_event::SoraImageEnqueueFailureEvent; use crate::events::sora::sora_image_enqueue_success_event::SoraImageEnqueueSuccessEvent; @@ -12,7 +12,7 @@ use crate::utils::get_url_file_extension::get_url_file_extension; use crate::utils::simple_http_download::simple_http_download; use base64::prelude::BASE64_STANDARD; use base64::Engine; -use errors::AnyhowResult; +use errors::{AnyhowError, AnyhowResult}; use image::codecs::png::{CompressionType, FilterType, PngEncoder}; use image::{DynamicImage, ImageReader}; use log::{debug, error, info}; @@ -31,6 +31,9 @@ use std::io::Cursor; use storyteller_client::media_files::get_media_file::get_media_file; use storyteller_client::utils::api_host::ApiHost; use tauri::{AppHandle, Emitter, Manager, State}; +use openai_sora_client::sora_error::SoraError; +use storyteller_client::api_error::ApiError; +use storyteller_client::api_error::ApiError::InternalServerError; use tokens::tokens::media_files::MediaFileToken; #[derive(Deserialize)] @@ -50,6 +53,20 @@ pub struct SoraImageRemixCommand { pub maybe_number_of_samples: Option, } +#[derive(Serialize, Debug)] +#[serde(rename_all = "snake_case")] +pub enum SoraImageRemixErrorType { + /// Generic server error + ServerError, + /// The user is sending too many requests + TooManyConcurrentTasks, + /// The user needs to create a Sora account + SoraUsernameNotYetCreated, + /// The Sora service is having problems. Try again soon. + SoraIsHavingProblems, +} + + #[tauri::command] pub async fn sora_image_remix_command( app: AppHandle, @@ -57,7 +74,7 @@ pub async fn sora_image_remix_command( app_data_root: State<'_, AppDataRoot>, sora_creds_manager: State<'_, SoraCredentialManager>, sora_task_queue: State<'_, SoraTaskQueue>, -) -> CommandResult<(), String> { +) -> CommandResult<(), SoraImageRemixErrorType, ()> { info!("image_generation_command called; scene media token: {:?}, additional images: {:?}", request.snapshot_media_token, request.maybe_additional_images); @@ -66,27 +83,83 @@ pub async fn sora_image_remix_command( let result = generate_image(request, &app_data_root, &sora_creds_manager, &sora_task_queue).await; - if let Err(err) = result { - error!("error: {:?}", err); + match result { + Err(err) => { + error!("error: {:?}", err); - let event = SoraImageEnqueueFailureEvent {}; - let result = event.send(&app); + let event = SoraImageEnqueueFailureEvent {}; + let result = event.send(&app); - if let Err(err) = result { - error!("Failed to emit event: {:?}", err); + if let Err(err) = result { + error!("Failed to emit event: {:?}", err); + } + + let mut status = CommandErrorStatus::ServerError; + let mut error_type = SoraImageRemixErrorType::ServerError; + let mut error_message = "A server error occurred."; + + match (err) { + InnerError::SoraError(SoraError::TooManyConcurrentTasks) => { + error_type = SoraImageRemixErrorType::TooManyConcurrentTasks; + status = CommandErrorStatus::ServerError; + error_message = "You already have work in progress. Please wait."; + }, + InnerError::SoraError(SoraError::SoraUsernameNotYetCreated) => { + error_type = SoraImageRemixErrorType::SoraUsernameNotYetCreated; + status = CommandErrorStatus::BadRequest; + error_message = "You need to create a username on Sora.com."; + }, + InnerError::SoraError(SoraError::BadGateway(_) | SoraError::CloudFlareTimeout(_)) => { + error_type = SoraImageRemixErrorType::SoraIsHavingProblems; + status = CommandErrorStatus::ServerError; + error_message = "Sora is having problems. Please wait a moment before retrying."; + } + _ => {}, + } + + Err(CommandErrorResponseWrapper { + status, + error_message: Some(error_message.to_string()), + error_type: Some(error_type), + error_details: None, + }) + } + Ok(_) => { + let event = SoraImageEnqueueSuccessEvent {}; + let result = event.send(&app); + + if let Err(err) = result { + error!("Failed to emit event: {:?}", err); + } + + Ok(().into()) } - - return Err("there was an error".into()); } +} - let event = SoraImageEnqueueSuccessEvent {}; - let result = event.send(&app); +#[derive(Debug)] +enum InnerError { + SoraError(SoraError), + AnyhowError(AnyhowError), + StorytellerApiError(ApiError), +} - if let Err(err) = result { - error!("Failed to emit event: {:?}", err); +impl From for InnerError { + fn from(value: AnyhowError) -> Self { + Self::AnyhowError(value) } +} - Ok(().into()) +impl From for InnerError { + fn from(value: SoraError) -> Self { + Self::SoraError(value) + } +} + +impl From for InnerError { + fn from(value: ApiError) -> Self { + Self::StorytellerApiError(value) + } } pub async fn generate_image( @@ -94,7 +167,7 @@ pub async fn generate_image( app_data_root: &AppDataRoot, sora_creds_manager: &SoraCredentialManager, sora_task_queue: &SoraTaskQueue, -) -> AnyhowResult<()> { +) -> Result<(), InnerError> { let response = get_media_file(&ApiHost::Storyteller, &request.snapshot_media_token).await?; diff --git a/crates/desktop/artcraft/src/commands/sora/sora_logout_command.rs b/crates/desktop/artcraft/src/commands/sora/sora_logout_command.rs index 1c87366fd1..99a726794b 100644 --- a/crates/desktop/artcraft/src/commands/sora/sora_logout_command.rs +++ b/crates/desktop/artcraft/src/commands/sora/sora_logout_command.rs @@ -1,5 +1,5 @@ use crate::commands::app_preferences::get_app_preferences_command::AppPreferencesPayload; -use crate::commands::command_response_wrapper::{CommandResult, CommandSuccessResponseWrapper}; +use crate::commands::command_response_wrapper::{CommandResult, CommandSuccessResponseWrapper, SimpleCommandResult}; use crate::state::app_preferences::app_preferences_manager::AppPreferencesManager; use crate::state::data_dir::app_data_root::AppDataRoot; use crate::state::sora::sora_credential_manager::SoraCredentialManager; @@ -17,7 +17,7 @@ use tauri::{AppHandle, State}; #[tauri::command] pub async fn sora_logout_command( sora_creds_manager: State<'_, SoraCredentialManager>, -) -> CommandResult<(), String> { +) -> SimpleCommandResult { info!("sora_logout_command called"); if let Err(err) = sora_creds_manager.purge_credentials_from_disk() { @@ -32,4 +32,3 @@ pub async fn sora_logout_command( Ok(().into()) } - diff --git a/crates/lib/api_clients/openai_sora_client/src/recipes/list_sora_task_status_with_session_auto_renew.rs b/crates/lib/api_clients/openai_sora_client/src/recipes/list_sora_task_status_with_session_auto_renew.rs index b25a8219c3..b7abbc5655 100644 --- a/crates/lib/api_clients/openai_sora_client/src/recipes/list_sora_task_status_with_session_auto_renew.rs +++ b/crates/lib/api_clients/openai_sora_client/src/recipes/list_sora_task_status_with_session_auto_renew.rs @@ -39,11 +39,10 @@ pub async fn list_sora_task_status_with_session_auto_renew( new_creds = Some(updated_creds); } - let legacy_creds = new_creds.as_ref() - .unwrap_or_else(|| &args.credentials) - .to_legacy_credentials()?; + let upgraded_creds = new_creds.as_ref() + .unwrap_or_else(|| &args.credentials); - let result = get_image_gen_status(&request, &legacy_creds).await; + let result = get_image_gen_status(&request, &upgraded_creds).await; let err = match result { Ok(response) => return Ok((response, new_creds)), @@ -56,7 +55,7 @@ pub async fn list_sora_task_status_with_session_auto_renew( info!("Retrying image upload with new credentials..."); - let result = get_image_gen_status(&request, &legacy_creds).await?; + let result = get_image_gen_status(&request, &upgraded_creds).await?; Ok((result, Some(new_creds))) } diff --git a/crates/lib/api_clients/openai_sora_client/src/recipes/wait_for_image_gen_status.rs b/crates/lib/api_clients/openai_sora_client/src/recipes/wait_for_image_gen_status.rs index 50d5bf0b74..faa2051dc0 100644 --- a/crates/lib/api_clients/openai_sora_client/src/recipes/wait_for_image_gen_status.rs +++ b/crates/lib/api_clients/openai_sora_client/src/recipes/wait_for_image_gen_status.rs @@ -2,6 +2,7 @@ use crate::credentials::SoraCredentials; use crate::requests::image_gen::image_gen_status::TaskStatus; use crate::requests::image_gen::image_gen_status::{get_image_gen_status, StatusRequest, TaskResponse, VideoGenStatusResponse}; use errors::AnyhowResult; +use crate::creds::sora_credential_set::SoraCredentialSet; pub async fn wait_for_image_gen_status(task_id: &String, credentials: &SoraCredentials, retry_limit: Option) -> AnyhowResult { let status_request = StatusRequest { @@ -10,11 +11,13 @@ pub async fn wait_for_image_gen_status(task_id: &String, credentials: &SoraCrede // before: Some(task_id.clone()), }; let retry_limit = retry_limit.unwrap_or(10); + + let new_creds = SoraCredentialSet::from_legacy_credentials(&credentials)?; for _ in 0..retry_limit { tokio::time::sleep(std::time::Duration::from_secs(10)).await; - let status_response = get_image_gen_status(&status_request, credentials).await?; + let status_response = get_image_gen_status(&status_request, &new_creds).await?; match find_task_response_by_id(&status_response, task_id.clone()) { Some(task_response) => match TaskStatus::from_str(&task_response.status) { diff --git a/crates/lib/api_clients/openai_sora_client/src/requests/image_gen/image_gen_http_request.rs b/crates/lib/api_clients/openai_sora_client/src/requests/image_gen/image_gen_http_request.rs index 53907f11f1..c446840548 100644 --- a/crates/lib/api_clients/openai_sora_client/src/requests/image_gen/image_gen_http_request.rs +++ b/crates/lib/api_clients/openai_sora_client/src/requests/image_gen/image_gen_http_request.rs @@ -1,3 +1,4 @@ +use log::warn; use crate::creds::credential_migration::CredentialMigrationRef; use serde_derive::{Deserialize, Serialize}; use thiserror::Error; @@ -200,6 +201,8 @@ pub (crate) async fn image_gen_http_request(sora_request: RawSoraImageGenRequest .map_err(|e| SoraImageGenError::NetworkError(e.to_string()))?; if status != reqwest::StatusCode::OK { + warn!("Sora image generation failure. Raw response: {:?}", response_body); + let error_response: RawSoraErrorResponse = serde_json::from_str(response_body) .map_err(|e| SoraImageGenError::GenericError(format!("Failed to parse error response: {}", e)))?; diff --git a/crates/lib/api_clients/openai_sora_client/src/requests/image_gen/image_gen_status.rs b/crates/lib/api_clients/openai_sora_client/src/requests/image_gen/image_gen_status.rs index 1411f69670..4862926110 100644 --- a/crates/lib/api_clients/openai_sora_client/src/requests/image_gen/image_gen_status.rs +++ b/crates/lib/api_clients/openai_sora_client/src/requests/image_gen/image_gen_status.rs @@ -6,7 +6,9 @@ use reqwest::Url; use serde_derive::{Deserialize, Serialize}; use std::fmt::{Debug, Display}; use std::{fs::File, io::Write, path::Path}; -use log::debug; +use log::{debug, warn}; +use crate::creds::sora_credential_set::SoraCredentialSet; +use crate::creds::sora_jwt_bearer_token::SoraJwtBearerToken; const SORA_STATUS_URL: &str = "https://sora.com/backend/video_gen"; @@ -171,7 +173,7 @@ pub struct InpaintItem { pub r#type: String, pub preset_id: Option, pub generation_id: Option, - pub upload_media_id: String, + pub upload_media_id: Option, pub frame_index: i32, pub source_start_frame: i32, } @@ -199,7 +201,16 @@ impl StatusRequest { } /// Gets the status of image generation tasks from Sora API -pub async fn get_image_gen_status(status_request: &StatusRequest, credentials: &SoraCredentials) -> Result { +pub async fn get_image_gen_status(status_request: &StatusRequest, credentials: &SoraCredentialSet) -> Result { + + let bearer_header = match credentials.jwt_bearer_token.as_ref() { + Some(bearer) => bearer.to_authorization_header_value(), + None => { + warn!("No JWT bearer for getting image gen status!"); + return Err(SoraError::NoBearerTokenAvailable) + }, + }; + let client = reqwest::Client::new(); let mut url = reqwest::Url::parse(SORA_STATUS_URL) @@ -216,8 +227,8 @@ pub async fn get_image_gen_status(status_request: &StatusRequest, credentials: & let http_request = client.get(url) .header("User-Agent", USER_AGENT) - .header("Cookie", &credentials.cookie) - .header("Authorization", credentials.authorization_header_value()) + .header("Cookie", credentials.cookies.as_str()) + .header("Authorization", bearer_header) .header("Content-Type", "application/json"); let http_request = http_request.build()?; @@ -235,9 +246,15 @@ pub async fn get_image_gen_status(status_request: &StatusRequest, credentials: & return Err(SoraError::AnyhowError(anyhow::anyhow!("The status request failed; status = {:?}, message = {:?}", status, response_body))); } - let response: VideoGenStatusResponse = serde_json::from_str(response_body)?; + let maybe_response = serde_json::from_str::(response_body); - Ok(response) + match maybe_response { + Ok(response) => Ok(response), + Err(err) => { + warn!("Failed to parse status response: {:?}", err); + Err(SoraError::JsonErrorWithBody(err, response_body.to_string())) + } + } } pub async fn save_generations_to_dir(generations: &[Generation], dir: &str) -> AnyhowResult<()> { @@ -275,6 +292,7 @@ mod tests { use errors::AnyhowResult; use std::fs::read_to_string; use testing::test_file_path::test_file_path; + use crate::creds::sora_credential_set::SoraCredentialSet; #[ignore] #[tokio::test] @@ -297,10 +315,12 @@ mod tests { let bearer = bearer.trim().to_string(); let creds = SoraCredentials { bearer_token: bearer, cookie, sentinel: Some(sentinel) }; + let new_creds = SoraCredentialSet::from_legacy_credentials(&creds)?; + // Get the task status for a specific task // let response = get_image_gen_status(&StatusRequest::new(None, Some("task_01jr9yvpfyetx9r7qvvx38scna".to_string())), &creds).await?; - let response = get_image_gen_status(&StatusRequest::new(Some(50), None), &creds).await?; + let response = get_image_gen_status(&StatusRequest::new(Some(50), None), &new_creds).await?; println!("task_id: {}", response.task_responses[0].id); let task_id = response.task_responses[0].id.clone(); diff --git a/crates/lib/api_clients/openai_sora_client/src/sora_error.rs b/crates/lib/api_clients/openai_sora_client/src/sora_error.rs index d8edda998c..897df7ff75 100644 --- a/crates/lib/api_clients/openai_sora_client/src/sora_error.rs +++ b/crates/lib/api_clients/openai_sora_client/src/sora_error.rs @@ -62,6 +62,10 @@ pub enum SoraError { /// serde_json::Error, likely from JSON deserialization schema mismatch. JsonError(serde_json::Error), + + /// serde_json::Error, likely from JSON deserialization schema mismatch. + /// Includes the original body. + JsonErrorWithBody(serde_json::Error, String), } impl Error for SoraError {} @@ -108,6 +112,9 @@ impl Display for SoraError { Self::JsonError(err) => { write!(f, "serde_json error: {}", err) } + Self::JsonErrorWithBody(err, message) => { + write!(f, "serde_json error: {:?} body: {:?}", err, message) + } } } } diff --git a/frontend/apps/editor2d/src/components/PromptBox.tsx b/frontend/apps/editor2d/src/components/PromptBox.tsx index c8d0cff643..10a1f8f74b 100644 --- a/frontend/apps/editor2d/src/components/PromptBox.tsx +++ b/frontend/apps/editor2d/src/components/PromptBox.tsx @@ -320,7 +320,7 @@ export const PromptBox = () => { if (!prompt.trim()) return; try { console.log( - "Enqueuing with prompt:", + "(3) Enqueuing with prompt:", prompt, "and reference images:", referenceImages, diff --git a/frontend/apps/editor3d/app/src/flags/DevConsoleFlags.ts b/frontend/apps/editor3d/app/src/flags/DevConsoleFlags.ts new file mode 100644 index 0000000000..2f02578243 --- /dev/null +++ b/frontend/apps/editor3d/app/src/flags/DevConsoleFlags.ts @@ -0,0 +1,36 @@ + +// This is used to do things such as disable job polling in the dev console. + +export const SetDevFlag = (flagName: string, value: any) => { + if (!!!(window as any)._dev_flags) { + (window as any)._dev_flags = {}; + } + (window as any)._dev_flags[flagName] = value; +} + +export const GetDevFlag = (flagName: string) => { + if (!!(window as any)._dev_flags && flagName in (window as any)._dev_flags) { + return (window as any)._dev_flags[flagName]; + } + return null; +} + +export const DisableJobPolling = () => { + SetDevFlag("disable_job_polling", true); +} + +export const EnableJobPolling = () => { + SetDevFlag("disable_job_polling", false); +} + +export const IsJobPollingDisabled = () : boolean => { + return GetDevFlag("disable_job_polling") === true; +} + +(window as any).setDevFlag = SetDevFlag; +(window as any).getDevFlag = GetDevFlag; + +(window as any).disableJobPolling = DisableJobPolling; +(window as any).enableJobPolling = EnableJobPolling; + + diff --git a/frontend/apps/editor3d/app/src/hooks/useActiveJobs/utilities.ts b/frontend/apps/editor3d/app/src/hooks/useActiveJobs/utilities.ts index 4da3db7378..f68cf8f755 100644 --- a/frontend/apps/editor3d/app/src/hooks/useActiveJobs/utilities.ts +++ b/frontend/apps/editor3d/app/src/hooks/useActiveJobs/utilities.ts @@ -1,9 +1,15 @@ import { ToastTypes } from "~/enums"; import { addToast, setJobs } from "~/signals"; import { JobsApi } from "~/Classes/ApiManager/JobsApi"; +import { IsJobPollingDisabled } from "~/flags/DevConsoleFlags"; + export async function PollRecentJobs() { const jobsApi = new JobsApi(); + if (IsJobPollingDisabled()) { + return; + } + const response = await jobsApi.ListRecentJobs(); if (response.success && response.data) { setJobs(response.data); diff --git a/frontend/apps/editor3d/app/src/pages/PageEnigma/comps/GenerateModals/Activity.tsx b/frontend/apps/editor3d/app/src/pages/PageEnigma/comps/GenerateModals/Activity.tsx index 5bb2ea6375..e76c5e879e 100644 --- a/frontend/apps/editor3d/app/src/pages/PageEnigma/comps/GenerateModals/Activity.tsx +++ b/frontend/apps/editor3d/app/src/pages/PageEnigma/comps/GenerateModals/Activity.tsx @@ -9,6 +9,7 @@ import { useEffect, useState, useRef, useCallback } from "react"; import { JobsApi } from "~/Classes/ApiManager/JobsApi"; import { toast } from "@storyteller/ui-toaster"; import { ActiveJob } from "~/pages/PageEnigma/models"; +import { IsJobPollingDisabled } from "~/flags/DevConsoleFlags"; // TODO ensure we de-dupe all this extra code. interface CompletedItem { token: string; @@ -85,6 +86,9 @@ export function Activity() { let mounted = true; async function fetchJobs() { + if (IsJobPollingDisabled()) { + return; + } try { const jobsApi = new JobsApi(); const response = await jobsApi.ListRecentJobs(); diff --git a/frontend/apps/editor3d/app/src/pages/PageEnigma/comps/PromptBox/PromptBox.tsx b/frontend/apps/editor3d/app/src/pages/PageEnigma/comps/PromptBox/PromptBox.tsx deleted file mode 100644 index 3f6665ccad..0000000000 --- a/frontend/apps/editor3d/app/src/pages/PageEnigma/comps/PromptBox/PromptBox.tsx +++ /dev/null @@ -1,936 +0,0 @@ -import { useState, useRef, useEffect, useContext } from "react"; -import { useSignals } from "@preact/signals-react/runtime"; -import { invoke } from "@tauri-apps/api/core"; -import { EngineContext } from "~/pages/PageEnigma/contexts/EngineContext"; -import { IsDesktopApp } from "@storyteller/tauri-utils"; -import { uploadImage } from "~/components/reusable/UploadModalMedia/uploadImage"; -import { downloadFileFromUrl } from "@storyteller/api"; -import { - faPlus, - faCamera, - faSave, - faSparkles, - faUpload, - faImages, - faDownload, - faSpinnerThird, - faTimes, - faMessageCheck, - faMessageXmark, -} from "@fortawesome/pro-solid-svg-icons"; -import { - faRectangleWide, - faRectangleVertical, - faSquare, - faRectangle, - faTableCellsLarge, -} from "@fortawesome/pro-regular-svg-icons"; - -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; - -import { PopoverItem, PopoverMenu } from "~/components/reusable/Popover"; -import { Button, ToggleButton, Tooltip } from "~/components"; -import { CameraAspectRatio } from "~/pages/PageEnigma/enums"; - -import { - cameraAspectRatio, - disableHotkeyInput, - DomLevels, - enableHotkeyInput, - gridVisibility, - setGridVisibility, -} from "~/pages/PageEnigma/signals"; - -import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames"; -import Queue from "~/pages/PageEnigma/Queue/Queue"; -import { toEngineActions } from "~/pages/PageEnigma/Queue/toEngineActions"; -import { PromptsApi } from "@storyteller/api"; -import { SoundRegistry } from "@storyteller/soundboard"; -import { GetAppPreferences, SoraImageRemix } from "@storyteller/tauri-api"; -import { CommandSuccessStatus } from "@storyteller/tauri-api"; - -import { - cameras, - selectedCameraId, - addCamera, - deleteCamera, - updateCamera, - focalLengthDragging, -} from "~/pages/PageEnigma/signals/camera"; -import { toast } from "@storyteller/ui-toaster"; -import { isPromptBoxFocused } from "~/pages/PageEnigma/signals/promptBox"; - -interface ReferenceImage { - id: string; - url: string; - file: File; - mediaToken: string; -} - -import { EngineApi } from "~/Classes/ApiManager/EngineApi"; -import { UploaderStates } from "~/enums/UploaderStates"; -import { CameraSettingsModal } from "../CameraSettingsModal"; -import { twMerge } from "tailwind-merge"; -import { GalleryModal, GalleryItem } from "@storyteller/ui-gallery-modal"; -import { Modal } from "@storyteller/ui-modal"; - -export const PromptBox = () => { - useSignals(); - const editorEngine = useContext(EngineContext); - const fileInputRef = useRef(null); - const textareaRef = useRef(null); - const [content, setContent] = useState(null); - const [isModalOpen, setIsModalOpen] = useState(false); - - const [prompt, setPrompt] = useState(""); - const [isEnqueueing, setisEnqueueing] = useState(false); - const [useSystemPrompt, setUseSystemPrompt] = useState(true); - const [referenceImages, setReferenceImages] = useState([]); - const [uploadingImages, setUploadingImages] = useState< - { id: string; file: File }[] - >([]); - const [aspectRatioList, setAspectRatioList] = useState([ - { - label: "3:2", - selected: true, - icon: , - }, - { - label: "2:3", - selected: false, - icon: , - }, - { - label: "1:1", - selected: false, - icon: , - }, - ]); - const [isCameraSettingsOpen, setIsCameraSettingsOpen] = useState(false); - const [isGalleryModalOpen, setIsGalleryModalOpen] = useState(false); - const [selectedGalleryImages, setSelectedGalleryImages] = useState( - [], - ); - const [activeGalleryTab, setActiveGalleryTab] = useState("my-media"); - - useEffect(() => { - if (textareaRef.current) { - textareaRef.current.style.height = "auto"; - textareaRef.current.style.height = `${textareaRef.current.scrollHeight}px`; - } - }, [prompt]); - - // Update aspect ratio list based on the current cameraAspectRatio signal - useEffect(() => { - setAspectRatioList((prev) => - prev.map((item) => ({ - ...item, - selected: - (item.label === "16:9" && - cameraAspectRatio.value === CameraAspectRatio.HORIZONTAL_16_9) || - (item.label === "3:2" && - cameraAspectRatio.value === CameraAspectRatio.HORIZONTAL_3_2) || - (item.label === "2:3" && - cameraAspectRatio.value === CameraAspectRatio.VERTICAL_2_3) || - (item.label === "1:1" && - cameraAspectRatio.value === CameraAspectRatio.SQUARE_1_1), - })), - ); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [cameraAspectRatio.value]); - - const handleCameraSelect = (selectedItem: PopoverItem) => { - const selectedCamera = cameras.value.find( - (cam) => cam.label === selectedItem.label, - ); - if (selectedCamera && editorEngine) { - selectedCameraId.value = selectedCamera.id; - - // Show focal length display temporarily - // TODO: Rename dragging to visible - BFlat - focalLengthDragging.value = { - isDragging: true, - focalLength: selectedCamera.focalLength, - }; - setTimeout(() => { - focalLengthDragging.value = { - isDragging: false, - focalLength: selectedCamera.focalLength, - }; - }, 1500); - - // Update the main camera to match the selected camera's properties - if (editorEngine.camera) { - // First update position and lookAt - editorEngine.camera.position.set( - selectedCamera.position.x, - selectedCamera.position.y, - selectedCamera.position.z, - ); - editorEngine.camera.lookAt( - selectedCamera.lookAt.x, - selectedCamera.lookAt.y, - selectedCamera.lookAt.z, - ); - - // Update FOV - editorEngine.camera.fov = editorEngine.focalLengthToFov( - selectedCamera.focalLength, - ); - editorEngine.camera.updateProjectionMatrix(); - - // Reset and update camera controls - if (editorEngine.cameraViewControls) { - editorEngine.cameraViewControls.reset(); - editorEngine.cameraViewControls.update(0); - } - - // Force a render to update the view - editorEngine.renderScene(); - - // Queue an update to ensure the engine processes the camera change - // Queue.publish({ - // queueName: QueueNames.TO_ENGINE, - // action: toEngineActions.CAMERA_CHANGED, - // data: selectedCamera, - // }); - } - - // Force update camera properties in the state - updateCamera(selectedCamera.id, { - focalLength: selectedCamera.focalLength, - position: selectedCamera.position, - rotation: selectedCamera.rotation, - lookAt: selectedCamera.lookAt, - }); - } - }; - - const handleAddCamera = () => { - // Check if we've reached the maximum number of cameras - if (cameras.value.length >= 6) { - console.warn("Maximum number of cameras (6) reached"); - return; - } - - const newIndex = cameras.value.length + 1; - const newId = `cam${newIndex}`; - - // This is for generating random orbital position for the new camera using spherical coordinates - const radius = Math.random() * 5 + 7; // Distance from center: 7 to 12 units - const theta = Math.random() * Math.PI * 2; // Azimuthal angle: 0 to 2π - const phi = Math.PI / 3 + (Math.random() * Math.PI) / 6; // Polar angle: π/3 to π/2 (60° to 90° from horizontal) - - // Convert spherical coordinates to Cartesian coordinates - const randomX = radius * Math.sin(phi) * Math.cos(theta); - const randomY = Math.abs(radius * Math.cos(phi)) + 2; // Ensure Y is positive and at least 2 units up - const randomZ = radius * Math.sin(phi) * Math.sin(theta); - - addCamera({ - id: newId, - label: `Camera ${newIndex}`, - focalLength: 24, - position: { - x: randomX, - y: randomY, - z: randomZ, - }, - rotation: { x: 0, y: 0, z: 0 }, - lookAt: { x: 0, y: 0, z: 0 }, - }); - - // Switch to the newly created camera - selectedCameraId.value = newId; - - // Update the engine camera to match the new camera's properties - if (editorEngine && editorEngine.camera) { - editorEngine.camera.position.set(randomX, randomY, randomZ); - editorEngine.camera.lookAt(0, 0, 0); - editorEngine.camera.fov = editorEngine.focalLengthToFov(24); - editorEngine.camera.updateProjectionMatrix(); - - // Reset and update camera controls - if (editorEngine.cameraViewControls) { - editorEngine.cameraViewControls.reset(); - editorEngine.cameraViewControls.update(0); - } - - // Force a render to update the view - editorEngine.renderScene(); - - // Queue.publish({ - // queueName: QueueNames.TO_ENGINE, - // action: toEngineActions.CAMERA_CHANGED, - // data: { - // id: newId, - // label: `Camera ${newIndex}`, - // focalLength: 24, - // position: { x: randomX, y: randomY, z: randomZ }, - // rotation: { x: 0, y: 0, z: 0 }, - // lookAt: { x: 0, y: 0, z: 0 }, - // }, - // }); - } - }; - - const handleCameraNameChange = (id: string, newName: string) => { - updateCamera(id, { label: newName }); - }; - - const handleCameraFocalLengthChange = (id: string, value: number) => { - const camera = cameras.value.find((cam) => cam.id === id); - if (camera) { - updateCamera(id, { focalLength: value }); - } - }; - - const handleAspectRatioSelect = (selectedItem: PopoverItem) => { - setAspectRatioList((prev) => - prev.map((item) => ({ - ...item, - selected: item.label === selectedItem.label, - })), - ); - - // Map the selected label to the corresponding CameraAspectRatio enum value - let newRatio: CameraAspectRatio; - switch (selectedItem.label) { - case "16:9": - newRatio = CameraAspectRatio.HORIZONTAL_16_9; - break; - case "3:2": - newRatio = CameraAspectRatio.HORIZONTAL_3_2; - break; - case "2:3": - newRatio = CameraAspectRatio.VERTICAL_2_3; - break; - case "1:1": - newRatio = CameraAspectRatio.SQUARE_1_1; - break; - default: - newRatio = CameraAspectRatio.HORIZONTAL_16_9; - } - - // Publish the change to the engine - Queue.publish({ - queueName: QueueNames.TO_ENGINE, - action: toEngineActions.CHANGE_CAMERA_ASPECT_RATIO, - data: newRatio, - }); - }; - - const handleFileUpload = (event: React.ChangeEvent) => { - const files = event.target.files; - if (files) { - Array.from(files).forEach((file) => { - const uploadId = Math.random().toString(36).substring(7); - setUploadingImages((prev) => [...prev, { id: uploadId, file }]); - - const reader = new FileReader(); - reader.onloadend = () => { - uploadImage({ - title: `reference-image-${Math.random().toString(36).substring(2, 15)}`, - assetFile: file, - progressCallback: (newState) => { - console.debug("Upload progress:", newState.data); - if (newState.status === UploaderStates.success && newState.data) { - const referenceImage: ReferenceImage = { - id: Math.random().toString(36).substring(7), - url: reader.result as string, - file, - mediaToken: newState.data || "", - }; - setReferenceImages((prev) => [...prev, referenceImage]); - setUploadingImages((prev) => - prev.filter((img) => img.id !== uploadId), - ); - console.log("Reference image added:", referenceImage); - } else if ( - newState.status === UploaderStates.assetError || - newState.status === UploaderStates.imageCreateError - ) { - setUploadingImages((prev) => - prev.filter((img) => img.id !== uploadId), - ); - console.error("Upload failed"); - } - }, - }); - }; - - reader.readAsDataURL(file); - }); - } - }; - - const handleRemoveReference = (id: string) => { - setReferenceImages((prev) => prev.filter((img) => img.id !== id)); - if (fileInputRef.current) { - fileInputRef.current.value = ""; - } - }; - - const handleUploadClick = () => { - fileInputRef.current?.click(); - }; - - const handleGallerySelect = () => { - setIsGalleryModalOpen(true); - }; - - const handleGalleryClose = () => { - setIsGalleryModalOpen(false); - setSelectedGalleryImages([]); - }; - - const handleImageSelect = (id: string) => { - setSelectedGalleryImages((prev) => { - if (prev.includes(id)) { - return prev.filter((imageId) => imageId !== id); - } - if (prev.length >= 4) { - return prev; - } - return [...prev, id]; - }); - }; - - const handleGalleryImages = (selectedItems: GalleryItem[]) => { - selectedItems.forEach((item) => { - if (!item.fullImage) return; - const referenceImage: ReferenceImage = { - id: Math.random().toString(36).substring(7), - url: item.fullImage, - file: new File([], "gallery-image"), - mediaToken: item.id, - }; - setReferenceImages((prev) => [...prev, referenceImage]); - }); - setIsGalleryModalOpen(false); - setSelectedGalleryImages([]); - }; - - const handleAction = (action: string) => { - switch (action) { - case "upload": - handleUploadClick(); - break; - case "gallery": - handleGallerySelect(); - break; - default: - console.log("Unknown action:", action); - } - }; - - const handlePaste = (e: React.ClipboardEvent) => { - e.preventDefault(); - const pastedText = e.clipboardData.getData("text").trim(); - setPrompt(pastedText); - }; - - const handleChange = (e: React.ChangeEvent) => { - setPrompt(e.target.value); - }; - - const handleError = (errorMessage: string) => { - setContent(errorMessage); - setIsModalOpen(true); - }; - - const handleEnqueue = async () => { - const isDesktop = IsDesktopApp(); - console.log("Is this a desktop app?", isDesktop); - - const r = SoundRegistry.getInstance(); - r.playSound("test"); - - if (isDesktop) { - await handleTauriEnqueue(); - } else { - await handleWebEnqueue(); - } - }; - - const handleWebEnqueue = async () => { - if (!prompt.trim()) return; - - setisEnqueueing(true); - - if (editorEngine) { - editorEngine.positive_prompt = prompt; - - //const engineApi = new EngineApi(); - const promptsApi = new PromptsApi(); - - const snapshot = editorEngine.snapShotOfCurrentFrame(false); - - if (snapshot) { - const snapshotResult = await promptsApi.uploadSceneSnapshot({ - screenshot: snapshot.file, - sceneMediaToken: "", - }); - - console.log("useSystemPrompt", useSystemPrompt); - - const response = await promptsApi.enqueueImageGeneration({ - disableSystemPrompt: !useSystemPrompt, - prompt: prompt, - snapshotMediaToken: snapshotResult.data || "", - additionalImages: referenceImages.map((image) => image.mediaToken), - }); - - console.log("response", response); - - if (response.errorMessage) { - handleError(response.errorMessage); - setisEnqueueing(false); - return; - } - } - - try { - // Here we would pass both the prompt and reference images to the generation - console.log( - "Enqueuing with prompt:", - prompt, - "and reference images:", - referenceImages, - ); - await new Promise((resolve) => setTimeout(resolve, 2000)); - } finally { - setisEnqueueing(false); - toast.success("Image added to queue"); - } - } - setisEnqueueing(false); - }; - - const handleTauriEnqueue = async () => { - if (!prompt.trim()) return; - - setisEnqueueing(true); - - setTimeout(() => { - // TODO(bt,2025-05-08): This is a hack so we don't accidentally wind up with a permanently disabled prompt box if - // the backend hangs on a given request. (Sometimes Sora doesn't respond.) Ideally what we would do instead is only - // force the prompt box to be enabled again as long as another request isn't running, ie. if we just fired off two - // requests in succession, we wouldn't want to turn it off as the first one gets submitted. - console.debug('Turn off blocking of prompt box...'); - setisEnqueueing(false); - }, 10000); - - if (editorEngine) { - editorEngine.positive_prompt = prompt; - - const promptsApi = new PromptsApi(); - - const snapshot = editorEngine.snapShotOfCurrentFrame(false); - - if (snapshot) { - const snapshotResult = await promptsApi.uploadSceneSnapshot({ - screenshot: snapshot.file, - //sceneMediaToken: "", - }); - - console.log("snapshotResult", snapshotResult) - - const generateResponse = await SoraImageRemix({ - snapshot_media_token: snapshotResult.data!, - disable_system_prompt: !useSystemPrompt, - prompt: prompt, - maybe_additional_images: referenceImages.map( - (image) => image.mediaToken, - ), - maybe_number_of_samples: 1, - }); - - console.log("generateResponse", generateResponse) - - if (generateResponse.status === CommandSuccessStatus.Success) { - const prefs = await GetAppPreferences(); - const soundName = prefs.preferences?.enqueue_success_sound; - if (soundName !== undefined) { - const registry = SoundRegistry.getInstance(); - registry.playSound(soundName); - } - toast.success("Image added to queue"); - setisEnqueueing(false); - return; - - } else { - const prefs = await GetAppPreferences(); - const soundName = prefs.preferences?.enqueue_failure_sound; - if (soundName !== undefined) { - const registry = SoundRegistry.getInstance(); - registry.playSound(soundName); - } - toast.error("Failed to enqueue image"); - setisEnqueueing(false); - return; - } - - // const generateResponse = await invoke("sora_image_remix_command", { - // request: { - // snapshot_media_token: snapshotResult.data!, - // disable_system_prompt: !useSystemPrompt, - // prompt: prompt, - // maybe_additional_images: referenceImages.map( - // (image) => image.mediaToken, - // ), - // maybe_number_of_samples: 1, - // }, - // }); - - //if (generateResponse.errorMessage) { - // handleError(response.errorMessage); - // setisEnqueueing(false); - // return; - //} - } - - try { - // Here we would pass both the prompt and reference images to the generation - console.log( - "Enqueuing with prompt:", - prompt, - "and reference images:", - referenceImages, - ); - await new Promise((resolve) => setTimeout(resolve, 2000)); - } finally { - setisEnqueueing(false); - toast.success("Image added to queue"); - } - } - setisEnqueueing(false); - }; - - const handleKeyDown = (e: React.KeyboardEvent) => { - if (e.key === "Enter" && !e.shiftKey) { - e.preventDefault(); - handleWebEnqueue(); - } - }; - - // Get the current aspect ratio icon based on the cameraAspectRatio signal - const getCurrentAspectRatioIcon = () => { - switch (cameraAspectRatio.value) { - case CameraAspectRatio.HORIZONTAL_3_2: - return faRectangle; - case CameraAspectRatio.VERTICAL_2_3: - return faRectangleVertical; - case CameraAspectRatio.VERTICAL_9_16: - return faRectangleVertical; - case CameraAspectRatio.SQUARE_1_1: - return faSquare; - default: - return faRectangleWide; - } - }; - - const handleSaveFrame = async () => { - if (editorEngine) { - const snapshot = editorEngine.snapShotOfCurrentFrame(false); - if (snapshot) { - const engineApi = new EngineApi(); - await engineApi.uploadSceneSnapshot({ - screenshot: snapshot.file || "", - sceneMediaToken: "", - }); - } - } - }; - - const handleDownloadFrame = () => { - if (editorEngine) { - editorEngine.snapShotOfCurrentFrame(true); - } - }; - - return ( - <> - { - setIsModalOpen(false); - setContent(""); - }} - > - {content} - -
- {(referenceImages.length > 0 || uploadingImages.length > 0) && ( -
- {referenceImages.map((image) => ( -
- Reference - -
- ))} - {uploadingImages.map(({ id, file }) => { - const previewUrl = URL.createObjectURL(file); - return ( -
-
- Uploading preview -
-
- -
-
- ); - })} -
- )} -
- -
- , - action: "upload", - }, - { - label: "Choose from library", - selected: false, - icon: , - action: "gallery", - }, - ]} - onPanelAction={handleAction} - showIconsInList - buttonClassName="bg-transparent hover:bg-transparent py-1.5 px-0 pr-1 m-0 hover:opacity-50 transition-opacity duration-100 ring-0 border-none focus:ring-0 outline-none" - triggerIcon={ - - } - /> - -