mirror of
https://github.com/storytold/artcraft.git
synced 2026-11-05 17:58:55 +00:00
Merge branch 'main' into feature/3D-unmountable
This commit is contained in:
commit
746695b77e
59 files changed
+730
-1143
No files matched your search
+12
-6
@@ -29,9 +29,10 @@ pub enum ValueType {
|
||||
pub enum PreferenceName {
|
||||
PreferredDownloadDirectory,
|
||||
PlaySounds,
|
||||
EnqueueSuccessSound,
|
||||
EnqueueFailureSound,
|
||||
GenerationSuccessSound,
|
||||
GenerationFailureSound,
|
||||
GenerationEnqueueSound,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
@@ -85,6 +86,16 @@ async fn update_prefs(
|
||||
return Err(anyhow!("Invalid value: {:?}", request.value)),
|
||||
}
|
||||
}
|
||||
PreferenceName::EnqueueSuccessSound => {
|
||||
prefs.enqueue_success_sound = request.value
|
||||
.map(|val| string_value(&val))
|
||||
.transpose()?;
|
||||
}
|
||||
PreferenceName::EnqueueFailureSound => {
|
||||
prefs.enqueue_failure_sound = request.value
|
||||
.map(|val| string_value(&val))
|
||||
.transpose()?;
|
||||
}
|
||||
PreferenceName::GenerationSuccessSound => {
|
||||
prefs.generation_success_sound = request.value
|
||||
.map(|val| string_value(&val))
|
||||
@@ -95,11 +106,6 @@ async fn update_prefs(
|
||||
.map(|val| string_value(&val))
|
||||
.transpose()?;
|
||||
}
|
||||
PreferenceName::GenerationEnqueueSound => {
|
||||
prefs.generation_enqueue_sound = request.value
|
||||
.map(|val| string_value(&val))
|
||||
.transpose()?;
|
||||
}
|
||||
}
|
||||
|
||||
app_prefs.set_clone(&prefs)?;
|
||||
|
||||
@@ -3,7 +3,11 @@ use std::error::Error;
|
||||
use std::str::FromStr;
|
||||
|
||||
/// Easy to use Result<T, E> type.
|
||||
pub type CommandResult<T, E> = Result<CommandSuccessResponseWrapper<T>, CommandErrorResponseWrapper<E>>;
|
||||
pub type CommandResult<SuccessPayload, ErrType, ErrPayload> =
|
||||
Result<CommandSuccessResponseWrapper<SuccessPayload>, CommandErrorResponseWrapper<ErrType, ErrPayload>>;
|
||||
|
||||
/// No inner payloads for this type. Just strings as messages.
|
||||
pub type SimpleCommandResult = CommandResult<(), (), ()>;
|
||||
|
||||
/// Statuses for successful commands.
|
||||
#[derive(Serialize, Clone, Debug)]
|
||||
@@ -18,6 +22,7 @@ pub enum CommandSuccessStatus {
|
||||
pub enum CommandErrorStatus {
|
||||
BadRequest, // 400
|
||||
Unauthorized, // 401
|
||||
TooManyRequests, // 429
|
||||
ServerError, // 500
|
||||
}
|
||||
|
||||
@@ -34,15 +39,22 @@ pub struct CommandSuccessResponseWrapper<T: Serialize> {
|
||||
}
|
||||
|
||||
#[derive(Serialize, Debug)]
|
||||
pub struct CommandErrorResponseWrapper<T: Serialize> {
|
||||
pub struct CommandErrorResponseWrapper<ErrType: Serialize, ErrPayload: Serialize> {
|
||||
/// Both "success" and "error" types have a `status` field for the frontend.
|
||||
pub status: CommandErrorStatus,
|
||||
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub error_message: Option<String>,
|
||||
|
||||
/// Represents a "type" of error, preferably an enum without payloads.
|
||||
/// Preferably this serializes to a string.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub error_details: Option<T>,
|
||||
pub error_type: Option<ErrType>,
|
||||
|
||||
/// Represents a full error details object or payload.
|
||||
/// This could be a structure, message, or whatever.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub error_details: Option<ErrPayload>,
|
||||
}
|
||||
|
||||
|
||||
@@ -96,11 +108,12 @@ impl From<&str> for CommandSuccessResponseWrapper<String> {
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&str> for CommandErrorResponseWrapper<String> {
|
||||
impl From<&str> for CommandErrorResponseWrapper<(), ()> {
|
||||
fn from(value: &str) -> Self {
|
||||
CommandErrorResponseWrapper {
|
||||
status: CommandErrorStatus::BadRequest, // NB: Default to 500 error.
|
||||
error_message: Some(value.to_string()),
|
||||
error_type: None,
|
||||
error_details: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
use std::ops::Add;
|
||||
use chrono::{DateTime, TimeDelta, Utc};
|
||||
use crate::commands::app_preferences::get_app_preferences_command::AppPreferencesPayload;
|
||||
use crate::state::app_preferences::app_preferences_manager::AppPreferencesManager;
|
||||
use crate::state::data_dir::app_data_root::AppDataRoot;
|
||||
@@ -12,6 +14,8 @@ use openai_sora_client::requests::list_media::list_media::list_media;
|
||||
use serde_derive::Serialize;
|
||||
use tauri::{AppHandle, State};
|
||||
|
||||
const CACHE_PERIOD : TimeDelta = TimeDelta::milliseconds(1000 * 60 * 5); // 5 minutes
|
||||
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub struct CheckSoraSessionCommand {
|
||||
@@ -63,10 +67,35 @@ async fn do_check(
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
let mut needs_upgrade_or_check = false;
|
||||
|
||||
let upgraded = maybe_upgrade_or_renew_session(&mut creds).await?;
|
||||
let stats = sora_creds_manager.get_credential_stats()?;
|
||||
|
||||
match stats.last_consecutive_credential_success {
|
||||
None => {
|
||||
info!("Session never checked, needs upgrade or status checking.");
|
||||
needs_upgrade_or_check= true;
|
||||
}
|
||||
Some(last_success) => {
|
||||
let now = chrono::Utc::now();
|
||||
let expires = last_success.add(CACHE_PERIOD);
|
||||
if now.gt(&expires) {
|
||||
info!("Session success expired, needs upgrade or status checking.");
|
||||
needs_upgrade_or_check = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
list_media(&creds).await?;
|
||||
let mut upgraded = false;
|
||||
|
||||
if needs_upgrade_or_check {
|
||||
info!("Attempting to upgrade session...");
|
||||
upgraded = maybe_upgrade_or_renew_session(&mut creds).await?;
|
||||
info!("Attempting to check session...");
|
||||
list_media(&creds).await?;
|
||||
sora_creds_manager.record_credential_success()?;
|
||||
}
|
||||
|
||||
if upgraded {
|
||||
info!("Saving upgraded session to manager and disk...");
|
||||
|
||||
@@ -14,8 +14,13 @@ use openai_sora_client::requests::image_gen::sora_image_gen_remix::{sora_image_g
|
||||
use openai_sora_client::requests::upload::upload_media_from_bytes::sora_media_upload_from_bytes;
|
||||
use std::fs::read_to_string;
|
||||
use std::io::Cursor;
|
||||
use std::time::Duration;
|
||||
use tauri::{AppHandle, Manager, State};
|
||||
|
||||
const SORA_IMAGE_UPLOAD_TIMEOUT: Duration = Duration::from_millis(1000 * 30); // 30 seconds
|
||||
|
||||
const SORA_IMAGE_REMIX_TIMEOUT: Duration = Duration::from_millis(1000 * 30); // 30 seconds
|
||||
|
||||
#[tauri::command]
|
||||
pub async fn sora_image_generation_command(
|
||||
prompt: &str,
|
||||
@@ -55,8 +60,13 @@ pub async fn generate_image(
|
||||
let image_bytes = BASE64_STANDARD.decode(image)?;
|
||||
|
||||
let filename = "image.png".to_string();
|
||||
|
||||
|
||||
let response = sora_media_upload_from_bytes(image_bytes, filename, CredentialMigrationRef::New(&creds))
|
||||
let response = sora_media_upload_from_bytes(
|
||||
image_bytes,
|
||||
filename,
|
||||
CredentialMigrationRef::New(&creds),
|
||||
Some(SORA_IMAGE_UPLOAD_TIMEOUT))
|
||||
.await
|
||||
.map_err(|err| {
|
||||
error!("Failed to upload image to Sora: {:?}", err);
|
||||
@@ -72,6 +82,7 @@ pub async fn generate_image(
|
||||
image_size: ImageSize::Square,
|
||||
sora_media_tokens: sora_media_tokens.clone(),
|
||||
credentials: CredentialMigrationRef::New(&creds),
|
||||
request_timeout: Some(SORA_IMAGE_REMIX_TIMEOUT),
|
||||
}).await
|
||||
.map_err(|err| {
|
||||
error!("Failed to call Sora image generation: {:?}", err);
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use crate::commands::command_response_wrapper::CommandResult;
|
||||
use crate::commands::command_response_wrapper::{CommandErrorResponseWrapper, CommandErrorStatus, CommandResult};
|
||||
use crate::events::sendable_event_trait::SendableEvent;
|
||||
use crate::events::sora::sora_image_enqueue_failure_event::SoraImageEnqueueFailureEvent;
|
||||
use crate::events::sora::sora_image_enqueue_success_event::SoraImageEnqueueSuccessEvent;
|
||||
@@ -12,7 +12,7 @@ use crate::utils::get_url_file_extension::get_url_file_extension;
|
||||
use crate::utils::simple_http_download::simple_http_download;
|
||||
use base64::prelude::BASE64_STANDARD;
|
||||
use base64::Engine;
|
||||
use errors::AnyhowResult;
|
||||
use errors::{AnyhowError, AnyhowResult};
|
||||
use image::codecs::png::{CompressionType, FilterType, PngEncoder};
|
||||
use image::{DynamicImage, ImageReader};
|
||||
use log::{debug, error, info};
|
||||
@@ -25,14 +25,22 @@ use openai_sora_client::requests::image_gen::common::{ImageSize, NumImages};
|
||||
use openai_sora_client::requests::image_gen::sora_image_gen_remix::{sora_image_gen_remix, SoraImageGenRemixRequest};
|
||||
use openai_sora_client::requests::upload::upload_media_from_bytes::sora_media_upload_from_bytes;
|
||||
use openai_sora_client::requests::upload::upload_media_from_file::sora_media_upload_from_file;
|
||||
use openai_sora_client::sora_error::SoraError;
|
||||
use serde_derive::{Deserialize, Serialize};
|
||||
use std::fs::read_to_string;
|
||||
use std::io::Cursor;
|
||||
use std::time::Duration;
|
||||
use storyteller_client::api_error::ApiError;
|
||||
use storyteller_client::api_error::ApiError::InternalServerError;
|
||||
use storyteller_client::media_files::get_media_file::get_media_file;
|
||||
use storyteller_client::utils::api_host::ApiHost;
|
||||
use tauri::{AppHandle, Emitter, Manager, State};
|
||||
use tokens::tokens::media_files::MediaFileToken;
|
||||
|
||||
const SORA_IMAGE_UPLOAD_TIMEOUT: Duration = Duration::from_millis(1000 * 30); // 30 seconds
|
||||
|
||||
const SORA_IMAGE_REMIX_TIMEOUT: Duration = Duration::from_millis(1000 * 30); // 30 seconds
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct SoraImageRemixCommand {
|
||||
/// Image media file; the engine or canvas snapshot (screenshot).
|
||||
@@ -50,6 +58,20 @@ pub struct SoraImageRemixCommand {
|
||||
pub maybe_number_of_samples: Option<u32>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Debug)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum SoraImageRemixErrorType {
|
||||
/// Generic server error
|
||||
ServerError,
|
||||
/// The user is sending too many requests
|
||||
TooManyConcurrentTasks,
|
||||
/// The user needs to create a Sora account
|
||||
SoraUsernameNotYetCreated,
|
||||
/// The Sora service is having problems. Try again soon.
|
||||
SoraIsHavingProblems,
|
||||
}
|
||||
|
||||
|
||||
#[tauri::command]
|
||||
pub async fn sora_image_remix_command(
|
||||
app: AppHandle,
|
||||
@@ -57,7 +79,7 @@ pub async fn sora_image_remix_command(
|
||||
app_data_root: State<'_, AppDataRoot>,
|
||||
sora_creds_manager: State<'_, SoraCredentialManager>,
|
||||
sora_task_queue: State<'_, SoraTaskQueue>,
|
||||
) -> CommandResult<(), String> {
|
||||
) -> CommandResult<(), SoraImageRemixErrorType, ()> {
|
||||
|
||||
info!("image_generation_command called; scene media token: {:?}, additional images: {:?}",
|
||||
request.snapshot_media_token, request.maybe_additional_images);
|
||||
@@ -66,27 +88,83 @@ pub async fn sora_image_remix_command(
|
||||
|
||||
let result = generate_image(request, &app_data_root, &sora_creds_manager, &sora_task_queue).await;
|
||||
|
||||
if let Err(err) = result {
|
||||
error!("error: {:?}", err);
|
||||
match result {
|
||||
Err(err) => {
|
||||
error!("error: {:?}", err);
|
||||
|
||||
let event = SoraImageEnqueueFailureEvent {};
|
||||
let result = event.send(&app);
|
||||
let event = SoraImageEnqueueFailureEvent {};
|
||||
let result = event.send(&app);
|
||||
|
||||
if let Err(err) = result {
|
||||
error!("Failed to emit event: {:?}", err);
|
||||
if let Err(err) = result {
|
||||
error!("Failed to emit event: {:?}", err);
|
||||
}
|
||||
|
||||
let mut status = CommandErrorStatus::ServerError;
|
||||
let mut error_type = SoraImageRemixErrorType::ServerError;
|
||||
let mut error_message = "A server error occurred. Please try again. If it continues, please tell our staff about the problem.";
|
||||
|
||||
match (err) {
|
||||
InnerError::SoraError(SoraError::TooManyConcurrentTasks) => {
|
||||
error_type = SoraImageRemixErrorType::TooManyConcurrentTasks;
|
||||
status = CommandErrorStatus::ServerError;
|
||||
error_message = "You already have work in progress. Please wait for it to finish.";
|
||||
},
|
||||
InnerError::SoraError(SoraError::SoraUsernameNotYetCreated) => {
|
||||
error_type = SoraImageRemixErrorType::SoraUsernameNotYetCreated;
|
||||
status = CommandErrorStatus::BadRequest;
|
||||
error_message = "You need to create a username on Sora.com to continue.";
|
||||
},
|
||||
InnerError::SoraError(SoraError::BadGateway(_) | SoraError::CloudFlareTimeout(_)) => {
|
||||
error_type = SoraImageRemixErrorType::SoraIsHavingProblems;
|
||||
status = CommandErrorStatus::ServerError;
|
||||
error_message = "Sora is having problems. Please wait a moment and then retry.";
|
||||
}
|
||||
_ => {},
|
||||
}
|
||||
|
||||
Err(CommandErrorResponseWrapper {
|
||||
status,
|
||||
error_message: Some(error_message.to_string()),
|
||||
error_type: Some(error_type),
|
||||
error_details: None,
|
||||
})
|
||||
}
|
||||
Ok(_) => {
|
||||
let event = SoraImageEnqueueSuccessEvent {};
|
||||
let result = event.send(&app);
|
||||
|
||||
if let Err(err) = result {
|
||||
error!("Failed to emit event: {:?}", err);
|
||||
}
|
||||
|
||||
Ok(().into())
|
||||
}
|
||||
|
||||
return Err("there was an error".into());
|
||||
}
|
||||
}
|
||||
|
||||
let event = SoraImageEnqueueSuccessEvent {};
|
||||
let result = event.send(&app);
|
||||
#[derive(Debug)]
|
||||
enum InnerError {
|
||||
SoraError(SoraError),
|
||||
AnyhowError(AnyhowError),
|
||||
StorytellerApiError(ApiError),
|
||||
}
|
||||
|
||||
if let Err(err) = result {
|
||||
error!("Failed to emit event: {:?}", err);
|
||||
impl From<AnyhowError> for InnerError {
|
||||
fn from(value: AnyhowError) -> Self {
|
||||
Self::AnyhowError(value)
|
||||
}
|
||||
}
|
||||
|
||||
Ok(().into())
|
||||
impl From<SoraError> for InnerError {
|
||||
fn from(value: SoraError) -> Self {
|
||||
Self::SoraError(value)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<ApiError> for InnerError {
|
||||
fn from(value: ApiError) -> Self {
|
||||
Self::StorytellerApiError(value)
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn generate_image(
|
||||
@@ -94,7 +172,7 @@ pub async fn generate_image(
|
||||
app_data_root: &AppDataRoot,
|
||||
sora_creds_manager: &SoraCredentialManager,
|
||||
sora_task_queue: &SoraTaskQueue,
|
||||
) -> AnyhowResult<()> {
|
||||
) -> Result<(), InnerError> {
|
||||
|
||||
let response = get_media_file(&ApiHost::Storyteller, &request.snapshot_media_token).await?;
|
||||
|
||||
@@ -133,6 +211,7 @@ pub async fn generate_image(
|
||||
image_upload_from_file_with_session_auto_renew(ImageUploadFromFileAutoRenewRequest {
|
||||
file_path,
|
||||
credentials: &creds,
|
||||
request_timeout: Some(SORA_IMAGE_UPLOAD_TIMEOUT),
|
||||
}).await?;
|
||||
|
||||
if let Some(new_creds) = maybe_new_credentials {
|
||||
@@ -157,6 +236,7 @@ pub async fn generate_image(
|
||||
image_size: ImageSize::Square,
|
||||
sora_media_tokens: sora_media_tokens.clone(),
|
||||
credentials: &creds,
|
||||
request_timeout: Some(SORA_IMAGE_REMIX_TIMEOUT),
|
||||
}).await?;
|
||||
|
||||
if let Some(new_creds) = maybe_new_creds {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use crate::commands::app_preferences::get_app_preferences_command::AppPreferencesPayload;
|
||||
use crate::commands::command_response_wrapper::{CommandResult, CommandSuccessResponseWrapper};
|
||||
use crate::commands::command_response_wrapper::{CommandResult, CommandSuccessResponseWrapper, SimpleCommandResult};
|
||||
use crate::state::app_preferences::app_preferences_manager::AppPreferencesManager;
|
||||
use crate::state::data_dir::app_data_root::AppDataRoot;
|
||||
use crate::state::sora::sora_credential_manager::SoraCredentialManager;
|
||||
@@ -17,7 +17,7 @@ use tauri::{AppHandle, State};
|
||||
#[tauri::command]
|
||||
pub async fn sora_logout_command(
|
||||
sora_creds_manager: State<'_, SoraCredentialManager>,
|
||||
) -> CommandResult<(), String> {
|
||||
) -> SimpleCommandResult {
|
||||
info!("sora_logout_command called");
|
||||
|
||||
if let Err(err) = sora_creds_manager.purge_credentials_from_disk() {
|
||||
@@ -30,6 +30,10 @@ pub async fn sora_logout_command(
|
||||
return Err("error clearing credentials from memory".into());
|
||||
}
|
||||
|
||||
if let Err(err) = sora_creds_manager.reset_credential_stats() {
|
||||
error!("Error resetting credential stats: {:?}", err);
|
||||
return Err("error resetting credential stats".into());
|
||||
}
|
||||
|
||||
Ok(().into())
|
||||
}
|
||||
|
||||
@@ -2,4 +2,5 @@ pub mod read_sora_credentials_from_disk;
|
||||
pub mod read_sora_credentials_from_disk_with_upgrade;
|
||||
pub mod sora_credential_holder;
|
||||
pub mod sora_credential_manager;
|
||||
pub mod sora_credential_stats;
|
||||
pub mod sora_task_queue;
|
||||
@@ -1,23 +1,29 @@
|
||||
use std::{fs, io};
|
||||
use crate::state::data_dir::app_data_root::AppDataRoot;
|
||||
use crate::state::sora::read_sora_credentials_from_disk::read_sora_credentials_from_disk;
|
||||
use crate::state::sora::sora_credential_holder::SoraCredentialHolder;
|
||||
use crate::state::sora::sora_credential_stats::SoraCredentialStats;
|
||||
use anyhow::anyhow;
|
||||
use errors::AnyhowResult;
|
||||
use log::{error, info, warn};
|
||||
use memory_store::clone_cell::CloneCell;
|
||||
use openai_sora_client::creds::sora_cookies::SoraCookies;
|
||||
use openai_sora_client::creds::sora_credential_set::SoraCredentialSet;
|
||||
use openai_sora_client::creds::sora_jwt_bearer_token::SoraJwtBearerToken;
|
||||
use openai_sora_client::creds::sora_sentinel::SoraSentinel;
|
||||
use openai_sora_client::requests::sentinel_refresh::generate::token::generate_token;
|
||||
use std::fs::OpenOptions;
|
||||
use std::io::Write;
|
||||
use std::path::Path;
|
||||
use openai_sora_client::creds::sora_cookies::SoraCookies;
|
||||
use openai_sora_client::creds::sora_jwt_bearer_token::SoraJwtBearerToken;
|
||||
use std::{fs, io};
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct SoraCredentialManager {
|
||||
holder: SoraCredentialHolder,
|
||||
app_data_root: AppDataRoot,
|
||||
|
||||
/// This is meant to help debug credential issues, keep credentials cached, fresh, etc.
|
||||
// TODO: Put this into the same structure as the credentials and also persist to disk.
|
||||
stats: CloneCell<SoraCredentialStats>,
|
||||
}
|
||||
|
||||
impl SoraCredentialManager {
|
||||
@@ -35,9 +41,36 @@ impl SoraCredentialManager {
|
||||
Self {
|
||||
holder,
|
||||
app_data_root: app_data_root.clone(),
|
||||
stats: CloneCell::with_owned(SoraCredentialStats::new()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn record_credential_success(&self) -> AnyhowResult<()> {
|
||||
let mut stats = self.stats.get_clone()?;
|
||||
stats.record_credential_success();
|
||||
self.stats.set_owned(stats)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn record_credential_failure(&self) -> AnyhowResult<()> {
|
||||
let mut stats = self.stats.get_clone()?;
|
||||
stats.record_credential_failure();
|
||||
self.stats.set_owned(stats)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn reset_credential_stats(&self) -> AnyhowResult<()> {
|
||||
self.stats.set_owned(SoraCredentialStats::new())
|
||||
}
|
||||
|
||||
pub fn get_credential_stats(&self) -> AnyhowResult<SoraCredentialStats> {
|
||||
self.stats.get_clone()
|
||||
}
|
||||
|
||||
pub fn set_credential_stats(&self, stats: SoraCredentialStats) -> AnyhowResult<()> {
|
||||
self.stats.set_owned(stats)
|
||||
}
|
||||
|
||||
pub fn set_credentials(&self, creds: &SoraCredentialSet) -> AnyhowResult<()> {
|
||||
self.holder.set_credentials(creds)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
|
||||
/// This is meant to track 401s and other credential failures, not 500s.
|
||||
#[derive(Clone)]
|
||||
pub struct SoraCredentialStats {
|
||||
pub last_credential_failure: Option<chrono::DateTime<chrono::Utc>>,
|
||||
pub last_credential_success: Option<chrono::DateTime<chrono::Utc>>,
|
||||
pub credential_success_count: u32,
|
||||
pub credential_failure_count: u32,
|
||||
/// This resets on each failure
|
||||
pub last_consecutive_credential_success: Option<chrono::DateTime<chrono::Utc>>,
|
||||
/// This resets on each failure
|
||||
pub consecutive_credential_success_count: u32,
|
||||
/// This resets on each success
|
||||
pub consecutive_credential_failure_count: u32,
|
||||
}
|
||||
|
||||
impl SoraCredentialStats {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
last_credential_failure: None,
|
||||
last_credential_success: None,
|
||||
last_consecutive_credential_success: None,
|
||||
credential_success_count: 0,
|
||||
credential_failure_count: 0,
|
||||
consecutive_credential_success_count: 0,
|
||||
consecutive_credential_failure_count: 0,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn record_credential_success(&mut self) {
|
||||
let now = chrono::Utc::now();
|
||||
self.last_credential_success = Some(now);
|
||||
self.last_consecutive_credential_success = Some(now);
|
||||
self.credential_success_count += 1;
|
||||
self.consecutive_credential_success_count += 1;
|
||||
self.consecutive_credential_failure_count = 0;
|
||||
}
|
||||
|
||||
pub fn record_credential_failure(&mut self) {
|
||||
let now = chrono::Utc::now();
|
||||
self.last_credential_failure = Some(now);
|
||||
self.last_consecutive_credential_success = None;
|
||||
self.credential_failure_count += 1;
|
||||
self.consecutive_credential_failure_count += 1;
|
||||
self.consecutive_credential_success_count = 0;
|
||||
}
|
||||
}
|
||||
@@ -21,11 +21,43 @@
|
||||
],
|
||||
"security": {
|
||||
"csp": {
|
||||
"default-src": "*",
|
||||
"connect-src": "*",
|
||||
"default-src": [
|
||||
"'self'",
|
||||
"'unsafe-eval'",
|
||||
"'unsafe-inline'",
|
||||
"blob:",
|
||||
"data:",
|
||||
"https://*.fakeyou.com",
|
||||
"https://*.storyteller.ai",
|
||||
"ipc://localhost",
|
||||
"tauri://localhost"
|
||||
],
|
||||
"connect-src": [
|
||||
"'self'",
|
||||
"'unsafe-eval'",
|
||||
"'unsafe-inline'",
|
||||
"blob:",
|
||||
"data:",
|
||||
"https://*.fakeyou.com",
|
||||
"https://*.storyteller.ai",
|
||||
"https://unpkg.com/detect-gpu@5.0.70/",
|
||||
"ipc://localhost",
|
||||
"tauri://localhost"
|
||||
],
|
||||
"font-src": "*",
|
||||
"img-src": "* data: blob: 'unsafe-inline'",
|
||||
"style-src": "*"
|
||||
"style-src": [
|
||||
"'self'",
|
||||
"'unsafe-eval'",
|
||||
"'unsafe-inline'",
|
||||
"blob:",
|
||||
"data:",
|
||||
"https://*.fakeyou.com",
|
||||
"https://*.storyteller.ai",
|
||||
"https://fonts.googleapis.com",
|
||||
"ipc://localhost",
|
||||
"tauri://localhost"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
+16
-8
@@ -1,3 +1,4 @@
|
||||
use std::time::Duration;
|
||||
use anyhow::anyhow;
|
||||
use log::{info, warn};
|
||||
use errors::AnyhowResult;
|
||||
@@ -7,7 +8,7 @@ use crate::creds::sora_jwt_bearer_token::SoraJwtBearerToken;
|
||||
use crate::creds::sora_sentinel::SoraSentinel;
|
||||
use crate::requests::bearer::generate::generate_bearer_with_cookie;
|
||||
use crate::requests::image_gen::common::{ImageSize, NumImages, SoraImageGenResponse};
|
||||
use crate::requests::image_gen::image_gen_http_request;
|
||||
use crate::requests::image_gen::{image_gen_http_request, SoraImageGenError};
|
||||
use crate::requests::image_gen::sora_image_gen_remix::{sora_image_gen_remix, SoraImageGenRemixRequest};
|
||||
use crate::requests::sentinel_refresh::generate::token::generate_token;
|
||||
use crate::sora_error::SoraError;
|
||||
@@ -18,6 +19,8 @@ pub struct ImageRemixAutoRenewRequest<'a> {
|
||||
pub image_size: ImageSize,
|
||||
pub sora_media_tokens: Vec<String>,
|
||||
pub credentials: &'a SoraCredentialSet,
|
||||
/// This function can try several different requests. This is the timeout for *individual* requests.
|
||||
pub request_timeout: Option<Duration>,
|
||||
}
|
||||
|
||||
/// Image request with retry and session auto-renewal.
|
||||
@@ -29,6 +32,7 @@ pub async fn image_remix_with_session_auto_renew(request: ImageRemixAutoRenewReq
|
||||
image_size: request.image_size,
|
||||
sora_media_tokens: request.sora_media_tokens.clone(), // NB: Clone because used again
|
||||
credentials: CredentialMigrationRef::New(request.credentials),
|
||||
request_timeout: request.request_timeout,
|
||||
}).await;
|
||||
|
||||
let err = match result {
|
||||
@@ -43,29 +47,32 @@ pub async fn image_remix_with_session_auto_renew(request: ImageRemixAutoRenewReq
|
||||
|
||||
match err {
|
||||
// We'll fail these requests...
|
||||
image_gen_http_request::SoraError::TooManyConcurrentTasks(_) => {
|
||||
SoraImageGenError::TooManyConcurrentTasks(_) => {
|
||||
return Err(SoraError::TooManyConcurrentTasks);
|
||||
}
|
||||
image_gen_http_request::SoraError::GenericError(err) => {
|
||||
SoraImageGenError::GenericError(err) => {
|
||||
return Err(SoraError::OtherBadStatus(anyhow!("image remix failed with GenericError: {:?}", err)))
|
||||
}
|
||||
image_gen_http_request::SoraError::NetworkError(err) => {
|
||||
SoraImageGenError::NetworkError(err) => {
|
||||
// TODO: The underlying type should be a reqwest::Error.
|
||||
return Err(SoraError::OtherBadStatus(anyhow!("network error: {:?}", err)))
|
||||
}
|
||||
SoraImageGenError::UsernameRequired(_err) => {
|
||||
return Err(SoraError::SoraUsernameNotYetCreated)
|
||||
}
|
||||
|
||||
// We'll retry these requests...
|
||||
|
||||
image_gen_http_request::SoraError::SentinelBlock(err) => {
|
||||
SoraImageGenError::SentinelBlock(err) => {
|
||||
warn!("Image generation failed due to sentinel block error: {:?}", err);
|
||||
refresh_sentinel = true;
|
||||
}
|
||||
image_gen_http_request::SoraError::TokenExpired(err) => {
|
||||
SoraImageGenError::TokenExpired(err) => {
|
||||
warn!("Image generation failed due to token expired error: {:?}", err);
|
||||
refresh_sentinel = true; // TODO: Not sure what this error is, actually.
|
||||
refresh_jwt = true;
|
||||
}
|
||||
image_gen_http_request::SoraError::InvalidJwt(err) => {
|
||||
SoraImageGenError::InvalidJwt(err) => {
|
||||
warn!("Image generation failed due to invalid jwt error: {:?}", err);
|
||||
refresh_jwt = true;
|
||||
}
|
||||
@@ -123,12 +130,13 @@ pub async fn image_remix_with_session_auto_renew(request: ImageRemixAutoRenewReq
|
||||
image_size: request.image_size,
|
||||
sora_media_tokens: request.sora_media_tokens,
|
||||
credentials: CredentialMigrationRef::New(&new_creds),
|
||||
request_timeout: request.request_timeout,
|
||||
}).await;
|
||||
|
||||
match result {
|
||||
Ok(response) => Ok((response, Some(new_creds))),
|
||||
Err(err) => match err {
|
||||
image_gen_http_request::SoraError::TooManyConcurrentTasks(err) => {
|
||||
image_gen_http_request::SoraImageGenError::TooManyConcurrentTasks(err) => {
|
||||
Err(SoraError::TooManyConcurrentTasks)
|
||||
}
|
||||
_ => {
|
||||
|
||||
+6
@@ -14,12 +14,16 @@ use crate::sora_error::SoraError;
|
||||
use anyhow::anyhow;
|
||||
use errors::AnyhowResult;
|
||||
use log::{error, info, warn};
|
||||
use std::time::Duration;
|
||||
|
||||
pub struct ImageUploadFromBytesAutoRenewRequest<'a> {
|
||||
pub file_bytes: Vec<u8>,
|
||||
pub filename: String,
|
||||
pub mime_type: &'a str,
|
||||
pub credentials: &'a SoraCredentialSet,
|
||||
|
||||
/// This function can try to upload several times. This is the timeout for *individual* requests.
|
||||
pub request_timeout: Option<Duration>,
|
||||
}
|
||||
|
||||
/// Image upload with retry and session auto-renewal.
|
||||
@@ -32,6 +36,7 @@ pub async fn image_upload_from_bytes_with_session_auto_renew(
|
||||
request.file_bytes.clone(), // FIXME(bt): This is horrible, but the client needs to take ownership. :(
|
||||
request.filename.clone(), // FIXME: Same
|
||||
CredentialMigrationRef::New(request.credentials),
|
||||
request.request_timeout,
|
||||
).await;
|
||||
|
||||
let err = match result {
|
||||
@@ -51,6 +56,7 @@ pub async fn image_upload_from_bytes_with_session_auto_renew(
|
||||
request.file_bytes,
|
||||
request.filename,
|
||||
CredentialMigrationRef::New(&new_creds),
|
||||
request.request_timeout,
|
||||
).await?;
|
||||
|
||||
Ok((result, Some(new_creds)))
|
||||
|
||||
+7
@@ -6,10 +6,14 @@ use crate::requests::upload::upload_media_http_request::{upload_media_http_reque
|
||||
use crate::sora_error::SoraError;
|
||||
use log::{info, warn};
|
||||
use std::path::Path;
|
||||
use std::time::Duration;
|
||||
|
||||
pub struct ImageUploadFromFileAutoRenewRequest<'a, P: AsRef<Path>> {
|
||||
pub file_path: P,
|
||||
pub credentials: &'a SoraCredentialSet,
|
||||
|
||||
/// This function can try to upload several times. This is the timeout for *individual* requests.
|
||||
pub request_timeout: Option<Duration>,
|
||||
}
|
||||
|
||||
/// Image upload with retry and session auto-renewal.
|
||||
@@ -21,6 +25,8 @@ pub async fn image_upload_from_file_with_session_auto_renew<P: AsRef<Path>>(
|
||||
let result = sora_media_upload_from_file(
|
||||
request.file_path.as_ref().clone(), // FIXME(bt): This is horrible, but the client needs to take ownership. :(
|
||||
CredentialMigrationRef::New(request.credentials),
|
||||
/// This function can try to upload several times. This is the timeout for *individual* requests.
|
||||
request.request_timeout,
|
||||
).await;
|
||||
|
||||
let err = match result {
|
||||
@@ -37,6 +43,7 @@ pub async fn image_upload_from_file_with_session_auto_renew<P: AsRef<Path>>(
|
||||
let result = sora_media_upload_from_file(
|
||||
request.file_path,
|
||||
CredentialMigrationRef::New(&new_creds),
|
||||
request.request_timeout,
|
||||
).await?;
|
||||
|
||||
Ok((result, Some(new_creds)))
|
||||
|
||||
+4
-5
@@ -39,11 +39,10 @@ pub async fn list_sora_task_status_with_session_auto_renew(
|
||||
new_creds = Some(updated_creds);
|
||||
}
|
||||
|
||||
let legacy_creds = new_creds.as_ref()
|
||||
.unwrap_or_else(|| &args.credentials)
|
||||
.to_legacy_credentials()?;
|
||||
let upgraded_creds = new_creds.as_ref()
|
||||
.unwrap_or_else(|| &args.credentials);
|
||||
|
||||
let result = get_image_gen_status(&request, &legacy_creds).await;
|
||||
let result = get_image_gen_status(&request, &upgraded_creds).await;
|
||||
|
||||
let err = match result {
|
||||
Ok(response) => return Ok((response, new_creds)),
|
||||
@@ -56,7 +55,7 @@ pub async fn list_sora_task_status_with_session_auto_renew(
|
||||
|
||||
info!("Retrying image upload with new credentials...");
|
||||
|
||||
let result = get_image_gen_status(&request, &legacy_creds).await?;
|
||||
let result = get_image_gen_status(&request, &upgraded_creds).await?;
|
||||
|
||||
Ok((result, Some(new_creds)))
|
||||
}
|
||||
@@ -2,6 +2,7 @@ use crate::credentials::SoraCredentials;
|
||||
use crate::requests::image_gen::image_gen_status::TaskStatus;
|
||||
use crate::requests::image_gen::image_gen_status::{get_image_gen_status, StatusRequest, TaskResponse, VideoGenStatusResponse};
|
||||
use errors::AnyhowResult;
|
||||
use crate::creds::sora_credential_set::SoraCredentialSet;
|
||||
|
||||
pub async fn wait_for_image_gen_status(task_id: &String, credentials: &SoraCredentials, retry_limit: Option<u32>) -> AnyhowResult<TaskResponse> {
|
||||
let status_request = StatusRequest {
|
||||
@@ -10,11 +11,13 @@ pub async fn wait_for_image_gen_status(task_id: &String, credentials: &SoraCrede
|
||||
// before: Some(task_id.clone()),
|
||||
};
|
||||
let retry_limit = retry_limit.unwrap_or(10);
|
||||
|
||||
let new_creds = SoraCredentialSet::from_legacy_credentials(&credentials)?;
|
||||
|
||||
for _ in 0..retry_limit {
|
||||
tokio::time::sleep(std::time::Duration::from_secs(10)).await;
|
||||
|
||||
let status_response = get_image_gen_status(&status_request, credentials).await?;
|
||||
let status_response = get_image_gen_status(&status_request, &new_creds).await?;
|
||||
|
||||
match find_task_response_by_id(&status_response, task_id.clone()) {
|
||||
Some(task_response) => match TaskStatus::from_str(&task_response.status) {
|
||||
|
||||
+40
-20
@@ -1,7 +1,9 @@
|
||||
use crate::creds::credential_migration::CredentialMigrationRef;
|
||||
use serde_derive::{Deserialize, Serialize};
|
||||
use thiserror::Error;
|
||||
use crate::requests::image_gen::image_gen_status::TaskId;
|
||||
use log::warn;
|
||||
use serde_derive::{Deserialize, Serialize};
|
||||
use std::time::Duration;
|
||||
use thiserror::Error;
|
||||
|
||||
const SORA_IMAGE_GEN_URL: &str = "https://sora.com/backend/video_gen";
|
||||
|
||||
@@ -9,7 +11,7 @@ const SORA_IMAGE_GEN_URL: &str = "https://sora.com/backend/video_gen";
|
||||
const USER_AGENT : &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36";
|
||||
|
||||
#[derive(Error, Debug)]
|
||||
pub enum SoraError {
|
||||
pub enum SoraImageGenError {
|
||||
#[error("Sora token expired: {0}")]
|
||||
TokenExpired(String),
|
||||
|
||||
@@ -27,6 +29,9 @@ pub enum SoraError {
|
||||
|
||||
#[error("Network error: {0}")]
|
||||
NetworkError(String),
|
||||
|
||||
#[error("Sora username required: {0}")]
|
||||
UsernameRequired(String),
|
||||
}
|
||||
|
||||
#[derive(Serialize, Debug)]
|
||||
@@ -127,6 +132,7 @@ pub enum SoraErrorCode {
|
||||
InvalidJwt,
|
||||
TokenExpired,
|
||||
TooManyConcurrentTasks,
|
||||
UsernameRequired,
|
||||
Unknown(String),
|
||||
}
|
||||
|
||||
@@ -147,7 +153,11 @@ impl std::fmt::Display for RawSoraErrorInner {
|
||||
}
|
||||
|
||||
/// Don't expose the internal request implementation as there are only a few "correct" ways to call the API.
|
||||
pub (crate) async fn image_gen_http_request(sora_request: RawSoraImageGenRequest, credentials: CredentialMigrationRef<'_>) -> Result<RawSoraResponse, SoraError> {
|
||||
pub (crate) async fn image_gen_http_request(
|
||||
sora_request: RawSoraImageGenRequest,
|
||||
credentials: CredentialMigrationRef<'_>,
|
||||
request_timeout: Option<Duration>,
|
||||
) -> Result<RawSoraResponse, SoraImageGenError> {
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
let mut cookie;
|
||||
@@ -161,78 +171,88 @@ pub (crate) async fn image_gen_http_request(sora_request: RawSoraImageGenRequest
|
||||
// TODO(bt,2025-04-23): We're using a Sora payload error in place of application state error. Surface this differently.
|
||||
sentinel = creds.sentinel.as_ref()
|
||||
.map(|sentinel| sentinel.to_string())
|
||||
.ok_or(SoraError::SentinelBlock("Sentinel is required for image generation.".to_string()))?;
|
||||
.ok_or(SoraImageGenError::SentinelBlock("Sentinel is required for image generation.".to_string()))?;
|
||||
}
|
||||
CredentialMigrationRef::New(creds) => {
|
||||
cookie = creds.cookies.to_string();
|
||||
// TODO(bt,2025-04-23): We're using a Sora payload error in place of application state error. Surface this differently.
|
||||
authorization_header = creds.jwt_bearer_token.as_ref()
|
||||
.ok_or(SoraError::InvalidJwt("JWT bearer is required for image generation".to_string()))?
|
||||
.ok_or(SoraImageGenError::InvalidJwt("JWT bearer is required for image generation".to_string()))?
|
||||
.to_authorization_header_value();
|
||||
// TODO(bt,2025-04-23): We're using a Sora payload error in place of application state error. Surface this differently.
|
||||
sentinel = creds.sora_sentinel.as_ref()
|
||||
.map(|sentinel| sentinel.get_sentinel().to_string())
|
||||
.ok_or(SoraError::SentinelBlock("Sentinel is required for image generation.".to_string()))?;
|
||||
.ok_or(SoraImageGenError::SentinelBlock("Sentinel is required for image generation.".to_string()))?;
|
||||
}
|
||||
}
|
||||
|
||||
let http_request = client.post(SORA_IMAGE_GEN_URL)
|
||||
let mut http_request = client.post(SORA_IMAGE_GEN_URL)
|
||||
.header("User-Agent", USER_AGENT)
|
||||
.header("Cookie", &cookie)
|
||||
.header("Authorization", &authorization_header)
|
||||
.header("Content-Type", "application/json")
|
||||
.header("OpenAI-Sentinel-Token", &sentinel);
|
||||
|
||||
|
||||
if let Some(timeout) = request_timeout {
|
||||
http_request = http_request.timeout(timeout);
|
||||
}
|
||||
|
||||
let http_request = http_request.json(&sora_request).build()
|
||||
.map_err(|e| SoraError::NetworkError(e.to_string()))?;
|
||||
.map_err(|e| SoraImageGenError::NetworkError(e.to_string()))?;
|
||||
|
||||
let response = client.execute(http_request)
|
||||
.await
|
||||
.map_err(|e| SoraError::NetworkError(e.to_string()))?;
|
||||
.map_err(|e| SoraImageGenError::NetworkError(e.to_string()))?;
|
||||
|
||||
let status = response.status();
|
||||
|
||||
let response_body = &response.text().await
|
||||
.map_err(|e| SoraError::NetworkError(e.to_string()))?;
|
||||
.map_err(|e| SoraImageGenError::NetworkError(e.to_string()))?;
|
||||
|
||||
if status != reqwest::StatusCode::OK {
|
||||
warn!("Sora image generation failure. Raw response: {:?}", response_body);
|
||||
|
||||
let error_response: RawSoraErrorResponse = serde_json::from_str(response_body)
|
||||
.map_err(|e| SoraError::GenericError(format!("Failed to parse error response: {}", e)))?;
|
||||
.map_err(|e| SoraImageGenError::GenericError(format!("Failed to parse error response: {}", e)))?;
|
||||
|
||||
// Check for specific error codes
|
||||
if let Some(code) = &error_response.error.code {
|
||||
match code {
|
||||
SoraErrorCode::TokenExpired => {
|
||||
return Err(SoraError::TokenExpired(error_response.error.message));
|
||||
return Err(SoraImageGenError::TokenExpired(error_response.error.message));
|
||||
}
|
||||
SoraErrorCode::SentinelBlock => {
|
||||
return Err(SoraError::SentinelBlock(error_response.error.message));
|
||||
return Err(SoraImageGenError::SentinelBlock(error_response.error.message));
|
||||
}
|
||||
SoraErrorCode::TooManyConcurrentTasks => {
|
||||
return Err(SoraError::TooManyConcurrentTasks(error_response.error.message));
|
||||
return Err(SoraImageGenError::TooManyConcurrentTasks(error_response.error.message));
|
||||
}
|
||||
SoraErrorCode::Unknown(_) => {
|
||||
return Err(SoraError::GenericError(error_response.error.message));
|
||||
return Err(SoraImageGenError::GenericError(error_response.error.message));
|
||||
}
|
||||
SoraErrorCode::InvalidJwt => {
|
||||
return Err(SoraError::InvalidJwt(error_response.error.message));
|
||||
return Err(SoraImageGenError::InvalidJwt(error_response.error.message));
|
||||
}
|
||||
SoraErrorCode::UsernameRequired => {
|
||||
return Err(SoraImageGenError::UsernameRequired(error_response.error.message));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return Err(SoraError::GenericError(error_response.error.message));
|
||||
return Err(SoraImageGenError::GenericError(error_response.error.message));
|
||||
}
|
||||
|
||||
let response = serde_json::from_str(response_body)
|
||||
.map_err(|e| SoraError::GenericError(format!("Failed to parse success response: {}", e)))?;
|
||||
.map_err(|e| SoraImageGenError::GenericError(format!("Failed to parse success response: {}", e)))?;
|
||||
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use errors::AnyhowResult;
|
||||
use crate::requests::image_gen::image_gen_http_request::RawSoraResponse;
|
||||
use errors::AnyhowResult;
|
||||
|
||||
#[test]
|
||||
fn deserialize_task_id() -> AnyhowResult<()> {
|
||||
|
||||
+28
-8
@@ -6,7 +6,9 @@ use reqwest::Url;
|
||||
use serde_derive::{Deserialize, Serialize};
|
||||
use std::fmt::{Debug, Display};
|
||||
use std::{fs::File, io::Write, path::Path};
|
||||
use log::debug;
|
||||
use log::{debug, warn};
|
||||
use crate::creds::sora_credential_set::SoraCredentialSet;
|
||||
use crate::creds::sora_jwt_bearer_token::SoraJwtBearerToken;
|
||||
|
||||
const SORA_STATUS_URL: &str = "https://sora.com/backend/video_gen";
|
||||
|
||||
@@ -171,7 +173,7 @@ pub struct InpaintItem {
|
||||
pub r#type: String,
|
||||
pub preset_id: Option<String>,
|
||||
pub generation_id: Option<String>,
|
||||
pub upload_media_id: String,
|
||||
pub upload_media_id: Option<String>,
|
||||
pub frame_index: i32,
|
||||
pub source_start_frame: i32,
|
||||
}
|
||||
@@ -199,7 +201,16 @@ impl StatusRequest {
|
||||
}
|
||||
|
||||
/// Gets the status of image generation tasks from Sora API
|
||||
pub async fn get_image_gen_status(status_request: &StatusRequest, credentials: &SoraCredentials) -> Result<VideoGenStatusResponse, SoraError> {
|
||||
pub async fn get_image_gen_status(status_request: &StatusRequest, credentials: &SoraCredentialSet) -> Result<VideoGenStatusResponse, SoraError> {
|
||||
|
||||
let bearer_header = match credentials.jwt_bearer_token.as_ref() {
|
||||
Some(bearer) => bearer.to_authorization_header_value(),
|
||||
None => {
|
||||
warn!("No JWT bearer for getting image gen status!");
|
||||
return Err(SoraError::NoBearerTokenAvailable)
|
||||
},
|
||||
};
|
||||
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
let mut url = reqwest::Url::parse(SORA_STATUS_URL)
|
||||
@@ -216,8 +227,8 @@ pub async fn get_image_gen_status(status_request: &StatusRequest, credentials: &
|
||||
|
||||
let http_request = client.get(url)
|
||||
.header("User-Agent", USER_AGENT)
|
||||
.header("Cookie", &credentials.cookie)
|
||||
.header("Authorization", credentials.authorization_header_value())
|
||||
.header("Cookie", credentials.cookies.as_str())
|
||||
.header("Authorization", bearer_header)
|
||||
.header("Content-Type", "application/json");
|
||||
|
||||
let http_request = http_request.build()?;
|
||||
@@ -235,9 +246,15 @@ pub async fn get_image_gen_status(status_request: &StatusRequest, credentials: &
|
||||
return Err(SoraError::AnyhowError(anyhow::anyhow!("The status request failed; status = {:?}, message = {:?}", status, response_body)));
|
||||
}
|
||||
|
||||
let response: VideoGenStatusResponse = serde_json::from_str(response_body)?;
|
||||
let maybe_response = serde_json::from_str::<VideoGenStatusResponse>(response_body);
|
||||
|
||||
Ok(response)
|
||||
match maybe_response {
|
||||
Ok(response) => Ok(response),
|
||||
Err(err) => {
|
||||
warn!("Failed to parse status response: {:?}", err);
|
||||
Err(SoraError::JsonErrorWithBody(err, response_body.to_string()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn save_generations_to_dir(generations: &[Generation], dir: &str) -> AnyhowResult<()> {
|
||||
@@ -275,6 +292,7 @@ mod tests {
|
||||
use errors::AnyhowResult;
|
||||
use std::fs::read_to_string;
|
||||
use testing::test_file_path::test_file_path;
|
||||
use crate::creds::sora_credential_set::SoraCredentialSet;
|
||||
|
||||
#[ignore]
|
||||
#[tokio::test]
|
||||
@@ -297,10 +315,12 @@ mod tests {
|
||||
let bearer = bearer.trim().to_string();
|
||||
|
||||
let creds = SoraCredentials { bearer_token: bearer, cookie, sentinel: Some(sentinel) };
|
||||
let new_creds = SoraCredentialSet::from_legacy_credentials(&creds)?;
|
||||
|
||||
// Get the task status for a specific task
|
||||
// let response = get_image_gen_status(&StatusRequest::new(None, Some("task_01jr9yvpfyetx9r7qvvx38scna".to_string())), &creds).await?;
|
||||
|
||||
let response = get_image_gen_status(&StatusRequest::new(Some(50), None), &creds).await?;
|
||||
let response = get_image_gen_status(&StatusRequest::new(Some(50), None), &new_creds).await?;
|
||||
println!("task_id: {}", response.task_responses[0].id);
|
||||
|
||||
let task_id = response.task_responses[0].id.clone();
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
pub (crate) mod image_gen_http_request;
|
||||
pub use image_gen_http_request::SoraError;
|
||||
pub use image_gen_http_request::SoraImageGenError;
|
||||
pub mod common;
|
||||
pub mod sora_image_gen_remix;
|
||||
pub mod sora_image_gen_simple;
|
||||
|
||||
+7
-4
@@ -1,7 +1,8 @@
|
||||
use log::info;
|
||||
use crate::creds::credential_migration::CredentialMigrationRef;
|
||||
use crate::requests::image_gen::common::{ImageSize, NumImages, SoraImageGenResponse};
|
||||
use crate::requests::image_gen::image_gen_http_request::{image_gen_http_request, InpaintItem, InpaintItemType, OperationType, RawSoraImageGenRequest, SoraError, VideoGenType};
|
||||
use crate::requests::image_gen::image_gen_http_request::{image_gen_http_request, InpaintItem, InpaintItemType, OperationType, RawSoraImageGenRequest, SoraImageGenError, VideoGenType};
|
||||
use log::info;
|
||||
use std::time::Duration;
|
||||
|
||||
pub struct SoraImageGenRemixRequest<'a> {
|
||||
pub prompt: String,
|
||||
@@ -9,11 +10,12 @@ pub struct SoraImageGenRemixRequest<'a> {
|
||||
pub image_size: ImageSize,
|
||||
pub sora_media_tokens: Vec<String>,
|
||||
pub credentials: CredentialMigrationRef<'a>,
|
||||
pub request_timeout: Option<Duration>,
|
||||
}
|
||||
|
||||
/// The "remix" commands let you supply additional images as context.
|
||||
/// Sora "media tokens" of previously uploaded images must be supplied.
|
||||
pub async fn sora_image_gen_remix(request: SoraImageGenRemixRequest<'_>) -> Result<SoraImageGenResponse, SoraError> {
|
||||
pub async fn sora_image_gen_remix(request: SoraImageGenRemixRequest<'_>) -> Result<SoraImageGenResponse, SoraImageGenError> {
|
||||
let args = RawSoraImageGenRequest {
|
||||
r#type: VideoGenType::ImageGen,
|
||||
operation: OperationType::Remix,
|
||||
@@ -39,7 +41,7 @@ pub async fn sora_image_gen_remix(request: SoraImageGenRemixRequest<'_>) -> Resu
|
||||
// TODO: Replace this with debug level logging in the future.
|
||||
info!("Sending remix request: {:?}", args);
|
||||
|
||||
let result = image_gen_http_request(args, request.credentials).await?;
|
||||
let result = image_gen_http_request(args, request.credentials, request.request_timeout).await?;
|
||||
|
||||
Ok(SoraImageGenResponse {
|
||||
task_id: result.id,
|
||||
@@ -80,6 +82,7 @@ mod tests {
|
||||
image_size: ImageSize::Square,
|
||||
sora_media_tokens: vec!["media_01jqyhrz4detyvtzwp2p4j63ad".to_string()],
|
||||
credentials: CredentialMigrationRef::Legacy(&creds),
|
||||
request_timeout: None,
|
||||
}).await?;
|
||||
|
||||
println!("task_id: {}", response.task_id);
|
||||
|
||||
+6
-4
@@ -1,14 +1,15 @@
|
||||
use crate::credentials::SoraCredentials;
|
||||
use crate::creds::credential_migration::CredentialMigrationRef;
|
||||
use crate::requests::image_gen::common::{ImageSize, NumImages, SoraImageGenResponse};
|
||||
use crate::requests::image_gen::image_gen_http_request::{image_gen_http_request, OperationType, RawSoraImageGenRequest, VideoGenType};
|
||||
use errors::AnyhowResult;
|
||||
use crate::creds::credential_migration::CredentialMigrationRef;
|
||||
use std::time::Duration;
|
||||
|
||||
pub struct SoraImageGenSimpleRequest<'a> {
|
||||
pub prompt: String,
|
||||
pub num_images: NumImages,
|
||||
pub image_size: ImageSize,
|
||||
pub credentials: CredentialMigrationRef<'a>,
|
||||
pub request_timeout: Option<Duration>,
|
||||
}
|
||||
|
||||
pub async fn sora_image_gen_simple(args: SoraImageGenSimpleRequest<'_>) -> AnyhowResult<SoraImageGenResponse> {
|
||||
@@ -24,7 +25,7 @@ pub async fn sora_image_gen_simple(args: SoraImageGenSimpleRequest<'_>) -> Anyho
|
||||
};
|
||||
|
||||
// TODO: Error handling.
|
||||
let result = image_gen_http_request(sora_request, args.credentials).await?;
|
||||
let result = image_gen_http_request(sora_request, args.credentials, args.request_timeout).await?;
|
||||
|
||||
Ok(SoraImageGenResponse {
|
||||
task_id: result.id,
|
||||
@@ -34,12 +35,12 @@ pub async fn sora_image_gen_simple(args: SoraImageGenSimpleRequest<'_>) -> Anyho
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::credentials::SoraCredentials;
|
||||
use crate::creds::credential_migration::CredentialMigrationRef;
|
||||
use crate::requests::image_gen::common::{ImageSize, NumImages};
|
||||
use crate::requests::image_gen::sora_image_gen_simple::{sora_image_gen_simple, SoraImageGenSimpleRequest};
|
||||
use errors::AnyhowResult;
|
||||
use std::fs::read_to_string;
|
||||
use testing::test_file_path::test_file_path;
|
||||
use crate::creds::credential_migration::CredentialMigrationRef;
|
||||
|
||||
#[ignore] // You can manually run "ignore" tests in the IDE, but they won't run in CI.
|
||||
#[tokio::test]
|
||||
@@ -64,6 +65,7 @@ mod tests {
|
||||
num_images: NumImages::One,
|
||||
image_size: ImageSize::Square,
|
||||
credentials: CredentialMigrationRef::Legacy(&creds),
|
||||
request_timeout: None,
|
||||
}).await?;
|
||||
|
||||
println!("task_id: {}", response.task_id);
|
||||
|
||||
+8
-2
@@ -2,10 +2,16 @@ use crate::creds::credential_migration::CredentialMigrationRef;
|
||||
use crate::requests::upload::upload_media_http_request::{upload_media_http_request, SoraMediaUploadResponse};
|
||||
use crate::sora_error::SoraError;
|
||||
use std::path::PathBuf;
|
||||
use std::time::Duration;
|
||||
|
||||
/// Upload bytes.
|
||||
/// The underlying reqwest lib needs to own the bytes, so we can't pass as a reference.
|
||||
pub async fn sora_media_upload_from_bytes(bytes: Vec<u8>, file_name: String, creds: CredentialMigrationRef<'_>) -> Result<SoraMediaUploadResponse, SoraError> {
|
||||
pub async fn sora_media_upload_from_bytes(
|
||||
bytes: Vec<u8>,
|
||||
file_name: String,
|
||||
creds: CredentialMigrationRef<'_>,
|
||||
maybe_timeout: Option<Duration>,
|
||||
) -> Result<SoraMediaUploadResponse, SoraError> {
|
||||
let file_path = PathBuf::from(&file_name);
|
||||
|
||||
// TODO: Read file magic bytes first, then fall back to this.
|
||||
@@ -20,5 +26,5 @@ pub async fn sora_media_upload_from_bytes(bytes: Vec<u8>, file_name: String, cre
|
||||
_ => "application/octet-stream",
|
||||
};
|
||||
|
||||
Ok(upload_media_http_request(bytes, file_name, mime_type, creds).await?)
|
||||
Ok(upload_media_http_request(bytes, file_name, mime_type, creds, maybe_timeout).await?)
|
||||
}
|
||||
+8
-2
@@ -2,6 +2,7 @@ use crate::creds::credential_migration::CredentialMigrationRef;
|
||||
use crate::requests::upload::upload_media_http_request::{upload_media_http_request, SoraMediaUploadResponse};
|
||||
use crate::sora_error::SoraError;
|
||||
use std::path::Path;
|
||||
use std::time::Duration;
|
||||
use tokio::fs::File;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
@@ -9,7 +10,11 @@ use tokio::io::AsyncReadExt;
|
||||
/// There's a better way to implement this.
|
||||
const INITIAL_BUFFER_SIZE : usize = 1024*1024;
|
||||
|
||||
pub async fn sora_media_upload_from_file<P: AsRef<Path>>(file_path: P, creds: CredentialMigrationRef<'_>) -> Result<SoraMediaUploadResponse, SoraError> {
|
||||
pub async fn sora_media_upload_from_file<P: AsRef<Path>>(
|
||||
file_path: P,
|
||||
creds: CredentialMigrationRef<'_>,
|
||||
maybe_timeout: Option<Duration>,
|
||||
) -> Result<SoraMediaUploadResponse, SoraError> {
|
||||
let mut file = File::open(&file_path).await?;
|
||||
let mut buffer = Vec::with_capacity(INITIAL_BUFFER_SIZE);
|
||||
file.read_to_end(&mut buffer).await?;
|
||||
@@ -31,7 +36,7 @@ pub async fn sora_media_upload_from_file<P: AsRef<Path>>(file_path: P, creds: Cr
|
||||
_ => "application/octet-stream",
|
||||
};
|
||||
|
||||
Ok(upload_media_http_request(buffer, filename, mime_type, creds).await?)
|
||||
Ok(upload_media_http_request(buffer, filename, mime_type, creds, maybe_timeout).await?)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -63,6 +68,7 @@ mod tests {
|
||||
let response = sora_media_upload_from_file(
|
||||
image_path,
|
||||
CredentialMigrationRef::Legacy(&creds),
|
||||
None,
|
||||
).await?;
|
||||
|
||||
println!("media: {:?}", response);
|
||||
|
||||
+7
-1
@@ -1,3 +1,4 @@
|
||||
use std::time::Duration;
|
||||
use crate::credentials::{SoraCredentials, USER_AGENT};
|
||||
use crate::creds::credential_migration::CredentialMigrationRef;
|
||||
use crate::sora_error::SoraError;
|
||||
@@ -69,6 +70,7 @@ pub (crate) async fn upload_media_http_request(
|
||||
filename: String,
|
||||
mime_type: &str,
|
||||
credentials: CredentialMigrationRef<'_>,
|
||||
maybe_timeout: Option<Duration>,
|
||||
) -> Result<SoraMediaUploadResponse, SoraError> {
|
||||
|
||||
// Create multipart form
|
||||
@@ -97,11 +99,15 @@ pub (crate) async fn upload_media_http_request(
|
||||
|
||||
// Make API request
|
||||
let client = Client::new();
|
||||
let request_builder = client.post(SORA_UPLOAD_MEDIA_URL)
|
||||
let mut request_builder = client.post(SORA_UPLOAD_MEDIA_URL)
|
||||
.multipart(form)
|
||||
.header("User-Agent", USER_AGENT)
|
||||
.header("Cookie", &cookie)
|
||||
.header("Authorization", &auth_header);
|
||||
|
||||
if let Some(timeout) = maybe_timeout {
|
||||
request_builder = request_builder.timeout(timeout);
|
||||
}
|
||||
|
||||
let response = request_builder.send().await?;
|
||||
|
||||
|
||||
@@ -29,6 +29,10 @@ pub enum SoraError {
|
||||
/// Failed to generate a bearer token despite the `200 OK` response.
|
||||
/// This has been generated by sending old/expired session cookies or empty cookies.
|
||||
FailedToGenerateBearer,
|
||||
|
||||
/// Sora requires users to set up an account username before they can generate images.
|
||||
/// This requires the user to set up that username.
|
||||
SoraUsernameNotYetCreated,
|
||||
|
||||
/// Something is wrong with the JWT bearer token.
|
||||
/// This error originates on our end as we try to parse the JWT.
|
||||
@@ -58,6 +62,10 @@ pub enum SoraError {
|
||||
|
||||
/// serde_json::Error, likely from JSON deserialization schema mismatch.
|
||||
JsonError(serde_json::Error),
|
||||
|
||||
/// serde_json::Error, likely from JSON deserialization schema mismatch.
|
||||
/// Includes the original body.
|
||||
JsonErrorWithBody(serde_json::Error, String),
|
||||
}
|
||||
|
||||
impl Error for SoraError {}
|
||||
@@ -71,6 +79,9 @@ impl Display for SoraError {
|
||||
Self::FailedToGenerateBearer => {
|
||||
write!(f, "Failed to generate bearer. Probably invalid session.")
|
||||
}
|
||||
Self::SoraUsernameNotYetCreated => {
|
||||
write!(f, "Sora username not created. Please set up a username.")
|
||||
}
|
||||
Self::LocalJwtClaimsParseError(message) => {
|
||||
write!(f, "Local JWT claims parse error: {}", message)
|
||||
}
|
||||
@@ -101,6 +112,9 @@ impl Display for SoraError {
|
||||
Self::JsonError(err) => {
|
||||
write!(f, "serde_json error: {}", err)
|
||||
}
|
||||
Self::JsonErrorWithBody(err, message) => {
|
||||
write!(f, "serde_json error: {:?} body: {:?}", err, message)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+5
-5
@@ -15,7 +15,7 @@ use log::warn;
|
||||
use log::{debug, error};
|
||||
use mysql_queries::queries::media_files::get::get_media_file::get_media_file;
|
||||
use openai_sora_client::credentials::SoraCredentials;
|
||||
use openai_sora_client::requests::image_gen::SoraError;
|
||||
use openai_sora_client::requests::image_gen::SoraImageGenError;
|
||||
use openai_sora_client::requests::sentinel_refresh::generate::token::generate_token;
|
||||
use serde::Deserialize;
|
||||
use serde::Serialize;
|
||||
@@ -326,7 +326,7 @@ pub async fn enqueue_studio_image_generation_handler(http_request: HttpRequest,
|
||||
|
||||
debug!("Sora image gen response: {:?}", response);
|
||||
|
||||
if let Err(SoraError::SentinelBlock(msg)) = &response {
|
||||
if let Err(SoraImageGenError::SentinelBlock(msg)) = &response {
|
||||
error!("Sora sentinel block, attempting refresh: {}", msg);
|
||||
|
||||
match generate_token().await {
|
||||
@@ -382,16 +382,16 @@ pub async fn enqueue_studio_image_generation_handler(http_request: HttpRequest,
|
||||
|
||||
let response = response.map_err(|err| {
|
||||
match err {
|
||||
SoraError::TokenExpired(msg) => {
|
||||
SoraImageGenError::TokenExpired(msg) => {
|
||||
error!("Sora token expired, needs refresh: {}", msg);
|
||||
// TODO: Implement token refresh logic here
|
||||
EnqueueImageGenRequestError::ServerError
|
||||
},
|
||||
SoraError::SentinelBlock(msg) => {
|
||||
SoraImageGenError::SentinelBlock(msg) => {
|
||||
error!("Sora sentinel block still occurring after refresh: {}", msg);
|
||||
EnqueueImageGenRequestError::ServerError
|
||||
},
|
||||
SoraError::TooManyConcurrentTasks(msg) => {
|
||||
SoraImageGenError::TooManyConcurrentTasks(msg) => {
|
||||
error!("Sora too many concurrent tasks: {}", msg);
|
||||
EnqueueImageGenRequestError::TooManyConcurrentTasks
|
||||
},
|
||||
|
||||
@@ -320,7 +320,7 @@ export const PromptBox = () => {
|
||||
if (!prompt.trim()) return;
|
||||
try {
|
||||
console.log(
|
||||
"Enqueuing with prompt:",
|
||||
"(3) Enqueuing with prompt:",
|
||||
prompt,
|
||||
"and reference images:",
|
||||
referenceImages,
|
||||
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,36 @@
|
||||
|
||||
// This is used to do things such as disable job polling in the dev console.
|
||||
|
||||
export const SetDevFlag = (flagName: string, value: any) => {
|
||||
if (!!!(window as any)._dev_flags) {
|
||||
(window as any)._dev_flags = {};
|
||||
}
|
||||
(window as any)._dev_flags[flagName] = value;
|
||||
}
|
||||
|
||||
export const GetDevFlag = (flagName: string) => {
|
||||
if (!!(window as any)._dev_flags && flagName in (window as any)._dev_flags) {
|
||||
return (window as any)._dev_flags[flagName];
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export const DisableJobPolling = () => {
|
||||
SetDevFlag("disable_job_polling", true);
|
||||
}
|
||||
|
||||
export const EnableJobPolling = () => {
|
||||
SetDevFlag("disable_job_polling", false);
|
||||
}
|
||||
|
||||
export const IsJobPollingDisabled = () : boolean => {
|
||||
return GetDevFlag("disable_job_polling") === true;
|
||||
}
|
||||
|
||||
(window as any).setDevFlag = SetDevFlag;
|
||||
(window as any).getDevFlag = GetDevFlag;
|
||||
|
||||
(window as any).disableJobPolling = DisableJobPolling;
|
||||
(window as any).enableJobPolling = EnableJobPolling;
|
||||
|
||||
|
||||
@@ -1,9 +1,15 @@
|
||||
import { ToastTypes } from "~/enums";
|
||||
import { addToast, setJobs } from "~/signals";
|
||||
import { JobsApi } from "~/Classes/ApiManager/JobsApi";
|
||||
import { IsJobPollingDisabled } from "~/flags/DevConsoleFlags";
|
||||
|
||||
export async function PollRecentJobs() {
|
||||
const jobsApi = new JobsApi();
|
||||
|
||||
if (IsJobPollingDisabled()) {
|
||||
return;
|
||||
}
|
||||
|
||||
const response = await jobsApi.ListRecentJobs();
|
||||
if (response.success && response.data) {
|
||||
setJobs(response.data);
|
||||
|
||||
@@ -860,7 +860,6 @@ class Editor {
|
||||
}
|
||||
const uuid = crypto.randomUUID();
|
||||
const file = new File([ab], `${uuid}.png`, { type: mimeString });
|
||||
|
||||
return { base64Snapshot, file };
|
||||
}
|
||||
|
||||
|
||||
@@ -7,23 +7,37 @@ export const InstallSounds = () => {
|
||||
return;
|
||||
}
|
||||
|
||||
// Good for simple immediate enqueue
|
||||
r.setSoundOnce("done", new SoundEffect("resources/sound/oot_dialogue_done.wav", { defaultVolume: 0.2 })); // DEFAULT
|
||||
// Good for simple menu choices
|
||||
r.setSoundOnce("click", new SoundEffect("resources/sound/smrpg_click.wav", { defaultVolume: 0.2 }));
|
||||
r.setSoundOnce("scifi_menu_beep_1", new SoundEffect("resources/sound/metroidprime_UI_15.wav", { defaultVolume: 0.3 }));
|
||||
r.setSoundOnce("scifi_menu_beep_2", new SoundEffect("resources/sound/metroidprime_UI_14.wav", { defaultVolume: 0.3 }));
|
||||
r.setSoundOnce("scifi_menu_select", new SoundEffect("resources/sound/metroidprime_UI_18.wav", { defaultVolume: 0.3 }));
|
||||
|
||||
// Good for simple immediate enqueue success
|
||||
r.setSoundOnce("done", new SoundEffect("resources/sound/oot_dialogue_done.wav", { defaultVolume: 0.4 })); // DEFAULT
|
||||
|
||||
// Good for simple immediate failure
|
||||
r.setSoundOnce("error_chirp", new SoundEffect("resources/sound/goldensun_135.wav", { defaultVolume: 0.2 }));
|
||||
r.setSoundOnce("error_chirp", new SoundEffect("resources/sound/goldensun_135.wav", { defaultVolume: 0.4 }));
|
||||
r.setSoundOnce("spike_throw", new SoundEffect("resources/sound/smrpg_enemy_spikethrow.wav", { defaultVolume: 0.1 })); // DEFAULT
|
||||
r.setSoundOnce("giant_shell_kick", new SoundEffect("resources/sound/smrpg_mario_giantshellkick.wav", { defaultVolume: 0.2 }));
|
||||
r.setSoundOnce("wrong", new SoundEffect("resources/sound/smrpg_wrong.wav", { defaultVolume: 0.4 }));
|
||||
|
||||
// Good for simple async success
|
||||
r.setSoundOnce("special_flower", new SoundEffect("resources/sound/smrpg_specialflower.wav", { defaultVolume: 0.2 })); // DEFAULT
|
||||
r.setSoundOnce("extra_power", new SoundEffect("resources/sound/smrpg_character_extrapower.wav", { defaultVolume: 0.2 })); // DEFAULT
|
||||
|
||||
// Good for async errors
|
||||
r.setSoundOnce("crumble", new SoundEffect("resources/sound/smrpg_drybones_crumble.wav", { defaultVolume: 0.1 })); // DEFAULT
|
||||
r.setSoundOnce("ghost", new SoundEffect("resources/sound/smrpg_ghost.wav", { defaultVolume: 0.2 }));
|
||||
r.setSoundOnce("special_alert", new SoundEffect("resources/sound/goldensun_214.wav", { defaultVolume: 0.2 }));
|
||||
r.setSoundOnce("scifi_alert", new SoundEffect("resources/sound/metroidprime_UI_52.wav", { defaultVolume: 0.2 }));
|
||||
r.setSoundOnce("scifi_shrill_alert", new SoundEffect("resources/sound/metroidprime_UI_51.wav", { defaultVolume: 0.2 }));
|
||||
|
||||
// Good for menus
|
||||
r.setSoundOnce("next", new SoundEffect("resources/sound/oot_dialogue_next.wav", { defaultVolume: 0.2 }));
|
||||
r.setSoundOnce("select", new SoundEffect("resources/sound/goldensun_111.wav", { defaultVolume: 0.4 }));
|
||||
r.setSoundOnce("scifi_menu_open", new SoundEffect("resources/sound/metroidprime_UI_12.wav", { defaultVolume: 0.3 }));
|
||||
r.setSoundOnce("scifi_menu_close", new SoundEffect("resources/sound/metroidprime_UI_13.wav", { defaultVolume: 0.3 }));
|
||||
|
||||
// Good for special reward / celebration
|
||||
r.setSoundOnce("correct", new SoundEffect("resources/sound/smrpg_correct.wav", { defaultVolume: 0.1 }));
|
||||
|
||||
@@ -344,13 +344,14 @@ export const PageEditor = () => {
|
||||
handleCameraFocalLengthChange={handleCameraFocalLengthChange}
|
||||
onAspectRatioSelect={onAspectRatioSelect}
|
||||
setEnginePrompt={(prompt) => {
|
||||
console.log("setEnginePrompt", prompt);
|
||||
if (!editorEngine) {
|
||||
console.log("editorEngine is not available");
|
||||
return;
|
||||
}
|
||||
|
||||
editorEngine!.positive_prompt = prompt;
|
||||
}}
|
||||
snapshotCurrentFrame={editorEngine?.snapShotOfCurrentFrame}
|
||||
snapshotCurrentFrame={editorEngine?.snapShotOfCurrentFrame.bind(editorEngine)}
|
||||
/>
|
||||
|
||||
<LoadingDots
|
||||
|
||||
@@ -8,7 +8,6 @@ import { ErrorDialog, LoadingDots } from "~/components";
|
||||
import { GenerateModals } from "~/pages/PageEnigma/comps/GenerateModals/GenerateModals";
|
||||
import { Toaster } from "@storyteller/ui-toaster";
|
||||
import { EditorLoadingBar } from "./comps/EditorLoadingBar";
|
||||
// import { Wizard } from "~/pages/PageEnigma/Wizard/Wizard";
|
||||
import { useSignals } from "@preact/signals-react/runtime";
|
||||
import { useEffect, useState } from "react";
|
||||
import * as gpu from "detect-gpu";
|
||||
@@ -23,8 +22,8 @@ import {
|
||||
showPrecisionSelector,
|
||||
} from "./signals/precisionSelectorMenu";
|
||||
import { InstallSounds } from "~/pages/PageEnigma/InstallSounds";
|
||||
import { InstallImageGenerationFailure } from "~/pages/PageEnigma/TauriEvents/SoraImageGenerationFailure";
|
||||
import { InstallImageGenerationSuccess } from "~/pages/PageEnigma/TauriEvents/SoraImageGenerationSuccess";
|
||||
import { useImageGenerationFailureEvent } from "./TauriEvents/useImageGenerationFailureEvent";
|
||||
import { useImageGenerationSuccessEvent } from "./TauriEvents/useImageGenerationSuccessEvent";
|
||||
|
||||
export const PageEnigma = ({ sceneToken }: { sceneToken?: string }) => {
|
||||
useSignals();
|
||||
@@ -90,11 +89,12 @@ export const PageEnigma = ({ sceneToken }: { sceneToken?: string }) => {
|
||||
});
|
||||
});
|
||||
|
||||
useImageGenerationSuccessEvent();
|
||||
useImageGenerationFailureEvent();
|
||||
|
||||
useEffect(() => {
|
||||
console.log("installing event listeners");
|
||||
InstallSounds();
|
||||
InstallImageGenerationSuccess();
|
||||
InstallImageGenerationFailure();
|
||||
});
|
||||
|
||||
if (validGpu === "unknown") {
|
||||
|
||||
-24
@@ -1,24 +0,0 @@
|
||||
import { SoundRegistry, SoundEffect } from "@storyteller/soundboard";
|
||||
import { listen } from '@tauri-apps/api/event';
|
||||
import { AppPreferencesPayload, CustomDirectory, GetAppPreferences, SystemDirectory } from "@storyteller/tauri-api";
|
||||
import { toast } from "@storyteller/ui-toaster";
|
||||
|
||||
type SoraImageGenerationFailed = {
|
||||
prompt: string,
|
||||
};
|
||||
|
||||
export const InstallImageGenerationFailure = () => {
|
||||
listen<SoraImageGenerationFailed>('sora-image-generation-failed', async (event) => {
|
||||
const prefs = await GetAppPreferences();
|
||||
const soundName = prefs.preferences?.generation_failure_sound;
|
||||
if (soundName !== undefined) {
|
||||
const registry = SoundRegistry.getInstance();
|
||||
registry.playSound(soundName);
|
||||
}
|
||||
toast.error("Image generation failed!");
|
||||
});
|
||||
}
|
||||
|
||||
export const ImageGenerationFailure = () => {
|
||||
}
|
||||
|
||||
-23
@@ -1,23 +0,0 @@
|
||||
import { SoundRegistry, SoundEffect } from "@storyteller/soundboard";
|
||||
import { listen } from '@tauri-apps/api/event';
|
||||
import { AppPreferencesPayload, CustomDirectory, GetAppPreferences, SystemDirectory } from "@storyteller/tauri-api";
|
||||
import { toast } from "@storyteller/ui-toaster";
|
||||
|
||||
type ImageGenerationSuccess = {
|
||||
media_file_token: string,
|
||||
};
|
||||
|
||||
export const InstallImageGenerationSuccess = () => {
|
||||
listen<ImageGenerationSuccess>('sora-image-generation-complete', async (event) => {
|
||||
const prefs = await GetAppPreferences();
|
||||
const soundName = prefs.preferences?.generation_success_sound;
|
||||
if (soundName !== undefined) {
|
||||
const registry = SoundRegistry.getInstance();
|
||||
registry.playSound(soundName);
|
||||
}
|
||||
toast.success("Image generated!");
|
||||
});
|
||||
}
|
||||
|
||||
export const ImageGenerationSuccess = () => {
|
||||
}
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
import { useEffect } from "react";
|
||||
import { listen, UnlistenFn } from '@tauri-apps/api/event';
|
||||
import { SoundRegistry } from "@storyteller/soundboard";
|
||||
import { GetAppPreferences } from "@storyteller/tauri-api";
|
||||
import { toast } from "@storyteller/ui-toaster";
|
||||
|
||||
type SoraImageGenerationFailed = {
|
||||
prompt: string,
|
||||
};
|
||||
|
||||
export const useImageGenerationFailureEvent = () => {
|
||||
useEffect(() => {
|
||||
let isUnmounted = false;
|
||||
let unlisten: Promise<UnlistenFn>;
|
||||
|
||||
const setup = async () => {
|
||||
unlisten = listen<SoraImageGenerationFailed>('sora-image-generation-failed', async (event) => {
|
||||
console.log("Image generation failure event received:", event);
|
||||
const prefs = await GetAppPreferences();
|
||||
const soundName = prefs.preferences?.generation_failure_sound;
|
||||
if (soundName !== undefined) {
|
||||
const registry = SoundRegistry.getInstance();
|
||||
registry.playSound(soundName);
|
||||
}
|
||||
toast.error("Image generation failed!");
|
||||
});
|
||||
|
||||
if (isUnmounted) {
|
||||
unlisten.then(f => f()); // Unsubscribe if unmounted early.
|
||||
}
|
||||
};
|
||||
|
||||
setup();
|
||||
|
||||
return () => {
|
||||
isUnmounted = true;
|
||||
unlisten.then(f => f());
|
||||
};
|
||||
|
||||
}, []);
|
||||
}
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
import { useEffect } from "react";
|
||||
import { listen, UnlistenFn } from '@tauri-apps/api/event';
|
||||
import { SoundRegistry } from "@storyteller/soundboard";
|
||||
import { GetAppPreferences } from "@storyteller/tauri-api";
|
||||
import { toast } from "@storyteller/ui-toaster";
|
||||
|
||||
type ImageGenerationSuccess = {
|
||||
media_file_token: string,
|
||||
};
|
||||
|
||||
export const useImageGenerationSuccessEvent = () => {
|
||||
useEffect(() => {
|
||||
let isUnmounted = false;
|
||||
let unlisten: Promise<UnlistenFn>;
|
||||
|
||||
const setup = async () => {
|
||||
unlisten = listen<ImageGenerationSuccess>('sora-image-generation-complete', async (event) => {
|
||||
console.log("Image generation success event received:", event);
|
||||
const prefs = await GetAppPreferences();
|
||||
const soundName = prefs.preferences?.generation_success_sound;
|
||||
if (soundName !== undefined) {
|
||||
const registry = SoundRegistry.getInstance();
|
||||
registry.playSound(soundName);
|
||||
}
|
||||
toast.success("Image generated!");
|
||||
});
|
||||
|
||||
if (isUnmounted) {
|
||||
unlisten.then(f => f()); // Unsubscribe if unmounted early.
|
||||
}
|
||||
};
|
||||
|
||||
setup();
|
||||
|
||||
return () => {
|
||||
isUnmounted = true;
|
||||
unlisten.then(f => f());
|
||||
};
|
||||
|
||||
}, []);
|
||||
}
|
||||
@@ -9,6 +9,7 @@ import { useEffect, useState, useRef, useCallback } from "react";
|
||||
import { JobsApi } from "~/Classes/ApiManager/JobsApi";
|
||||
import { toast } from "@storyteller/ui-toaster";
|
||||
import { ActiveJob } from "~/pages/PageEnigma/models";
|
||||
import { IsJobPollingDisabled } from "~/flags/DevConsoleFlags";
|
||||
// TODO ensure we de-dupe all this extra code.
|
||||
interface CompletedItem {
|
||||
token: string;
|
||||
@@ -85,6 +86,9 @@ export function Activity() {
|
||||
let mounted = true;
|
||||
|
||||
async function fetchJobs() {
|
||||
if (IsJobPollingDisabled()) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const jobsApi = new JobsApi();
|
||||
const response = await jobsApi.ListRecentJobs();
|
||||
|
||||
@@ -1,936 +0,0 @@
|
||||
import { useState, useRef, useEffect, useContext } from "react";
|
||||
import { useSignals } from "@preact/signals-react/runtime";
|
||||
import { invoke } from "@tauri-apps/api/core";
|
||||
import { EngineContext } from "~/pages/PageEnigma/contexts/EngineContext";
|
||||
import { IsDesktopApp } from "@storyteller/tauri-utils";
|
||||
import { uploadImage } from "~/components/reusable/UploadModalMedia/uploadImage";
|
||||
import { downloadFileFromUrl } from "@storyteller/api";
|
||||
import {
|
||||
faPlus,
|
||||
faCamera,
|
||||
faSave,
|
||||
faSparkles,
|
||||
faUpload,
|
||||
faImages,
|
||||
faDownload,
|
||||
faSpinnerThird,
|
||||
faTimes,
|
||||
faMessageCheck,
|
||||
faMessageXmark,
|
||||
} from "@fortawesome/pro-solid-svg-icons";
|
||||
import {
|
||||
faRectangleWide,
|
||||
faRectangleVertical,
|
||||
faSquare,
|
||||
faRectangle,
|
||||
faTableCellsLarge,
|
||||
} from "@fortawesome/pro-regular-svg-icons";
|
||||
|
||||
import { FontAwesomeIcon } from "@fortawesome/react-fontawesome";
|
||||
|
||||
import { PopoverItem, PopoverMenu } from "~/components/reusable/Popover";
|
||||
import { Button, ToggleButton, Tooltip } from "~/components";
|
||||
import { CameraAspectRatio } from "~/pages/PageEnigma/enums";
|
||||
|
||||
import {
|
||||
cameraAspectRatio,
|
||||
disableHotkeyInput,
|
||||
DomLevels,
|
||||
enableHotkeyInput,
|
||||
gridVisibility,
|
||||
setGridVisibility,
|
||||
} from "~/pages/PageEnigma/signals";
|
||||
|
||||
import { QueueNames } from "~/pages/PageEnigma/Queue/QueueNames";
|
||||
import Queue from "~/pages/PageEnigma/Queue/Queue";
|
||||
import { toEngineActions } from "~/pages/PageEnigma/Queue/toEngineActions";
|
||||
import { PromptsApi } from "@storyteller/api";
|
||||
import { SoundRegistry } from "@storyteller/soundboard";
|
||||
import { GetAppPreferences, SoraImageRemix } from "@storyteller/tauri-api";
|
||||
import { CommandSuccessStatus } from "@storyteller/tauri-api";
|
||||
|
||||
import {
|
||||
cameras,
|
||||
selectedCameraId,
|
||||
addCamera,
|
||||
deleteCamera,
|
||||
updateCamera,
|
||||
focalLengthDragging,
|
||||
} from "~/pages/PageEnigma/signals/camera";
|
||||
import { toast } from "@storyteller/ui-toaster";
|
||||
import { isPromptBoxFocused } from "~/pages/PageEnigma/signals/promptBox";
|
||||
|
||||
interface ReferenceImage {
|
||||
id: string;
|
||||
url: string;
|
||||
file: File;
|
||||
mediaToken: string;
|
||||
}
|
||||
|
||||
import { EngineApi } from "~/Classes/ApiManager/EngineApi";
|
||||
import { UploaderStates } from "~/enums/UploaderStates";
|
||||
import { CameraSettingsModal } from "../CameraSettingsModal";
|
||||
import { twMerge } from "tailwind-merge";
|
||||
import { GalleryModal, GalleryItem } from "@storyteller/ui-gallery-modal";
|
||||
import { Modal } from "@storyteller/ui-modal";
|
||||
|
||||
export const PromptBox = () => {
|
||||
useSignals();
|
||||
const editorEngine = useContext(EngineContext);
|
||||
const fileInputRef = useRef<HTMLInputElement>(null);
|
||||
const textareaRef = useRef<HTMLTextAreaElement>(null);
|
||||
const [content, setContent] = useState<React.ReactNode>(null);
|
||||
const [isModalOpen, setIsModalOpen] = useState(false);
|
||||
|
||||
const [prompt, setPrompt] = useState("");
|
||||
const [isEnqueueing, setisEnqueueing] = useState(false);
|
||||
const [useSystemPrompt, setUseSystemPrompt] = useState(true);
|
||||
const [referenceImages, setReferenceImages] = useState<ReferenceImage[]>([]);
|
||||
const [uploadingImages, setUploadingImages] = useState<
|
||||
{ id: string; file: File }[]
|
||||
>([]);
|
||||
const [aspectRatioList, setAspectRatioList] = useState<PopoverItem[]>([
|
||||
{
|
||||
label: "3:2",
|
||||
selected: true,
|
||||
icon: <FontAwesomeIcon icon={faRectangle} className="h-4 w-4" />,
|
||||
},
|
||||
{
|
||||
label: "2:3",
|
||||
selected: false,
|
||||
icon: <FontAwesomeIcon icon={faRectangleVertical} className="h-4 w-4" />,
|
||||
},
|
||||
{
|
||||
label: "1:1",
|
||||
selected: false,
|
||||
icon: <FontAwesomeIcon icon={faSquare} className="h-4 w-4" />,
|
||||
},
|
||||
]);
|
||||
const [isCameraSettingsOpen, setIsCameraSettingsOpen] = useState(false);
|
||||
const [isGalleryModalOpen, setIsGalleryModalOpen] = useState(false);
|
||||
const [selectedGalleryImages, setSelectedGalleryImages] = useState<string[]>(
|
||||
[],
|
||||
);
|
||||
const [activeGalleryTab, setActiveGalleryTab] = useState("my-media");
|
||||
|
||||
useEffect(() => {
|
||||
if (textareaRef.current) {
|
||||
textareaRef.current.style.height = "auto";
|
||||
textareaRef.current.style.height = `${textareaRef.current.scrollHeight}px`;
|
||||
}
|
||||
}, [prompt]);
|
||||
|
||||
// Update aspect ratio list based on the current cameraAspectRatio signal
|
||||
useEffect(() => {
|
||||
setAspectRatioList((prev) =>
|
||||
prev.map((item) => ({
|
||||
...item,
|
||||
selected:
|
||||
(item.label === "16:9" &&
|
||||
cameraAspectRatio.value === CameraAspectRatio.HORIZONTAL_16_9) ||
|
||||
(item.label === "3:2" &&
|
||||
cameraAspectRatio.value === CameraAspectRatio.HORIZONTAL_3_2) ||
|
||||
(item.label === "2:3" &&
|
||||
cameraAspectRatio.value === CameraAspectRatio.VERTICAL_2_3) ||
|
||||
(item.label === "1:1" &&
|
||||
cameraAspectRatio.value === CameraAspectRatio.SQUARE_1_1),
|
||||
})),
|
||||
);
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [cameraAspectRatio.value]);
|
||||
|
||||
const handleCameraSelect = (selectedItem: PopoverItem) => {
|
||||
const selectedCamera = cameras.value.find(
|
||||
(cam) => cam.label === selectedItem.label,
|
||||
);
|
||||
if (selectedCamera && editorEngine) {
|
||||
selectedCameraId.value = selectedCamera.id;
|
||||
|
||||
// Show focal length display temporarily
|
||||
// TODO: Rename dragging to visible - BFlat
|
||||
focalLengthDragging.value = {
|
||||
isDragging: true,
|
||||
focalLength: selectedCamera.focalLength,
|
||||
};
|
||||
setTimeout(() => {
|
||||
focalLengthDragging.value = {
|
||||
isDragging: false,
|
||||
focalLength: selectedCamera.focalLength,
|
||||
};
|
||||
}, 1500);
|
||||
|
||||
// Update the main camera to match the selected camera's properties
|
||||
if (editorEngine.camera) {
|
||||
// First update position and lookAt
|
||||
editorEngine.camera.position.set(
|
||||
selectedCamera.position.x,
|
||||
selectedCamera.position.y,
|
||||
selectedCamera.position.z,
|
||||
);
|
||||
editorEngine.camera.lookAt(
|
||||
selectedCamera.lookAt.x,
|
||||
selectedCamera.lookAt.y,
|
||||
selectedCamera.lookAt.z,
|
||||
);
|
||||
|
||||
// Update FOV
|
||||
editorEngine.camera.fov = editorEngine.focalLengthToFov(
|
||||
selectedCamera.focalLength,
|
||||
);
|
||||
editorEngine.camera.updateProjectionMatrix();
|
||||
|
||||
// Reset and update camera controls
|
||||
if (editorEngine.cameraViewControls) {
|
||||
editorEngine.cameraViewControls.reset();
|
||||
editorEngine.cameraViewControls.update(0);
|
||||
}
|
||||
|
||||
// Force a render to update the view
|
||||
editorEngine.renderScene();
|
||||
|
||||
// Queue an update to ensure the engine processes the camera change
|
||||
// Queue.publish({
|
||||
// queueName: QueueNames.TO_ENGINE,
|
||||
// action: toEngineActions.CAMERA_CHANGED,
|
||||
// data: selectedCamera,
|
||||
// });
|
||||
}
|
||||
|
||||
// Force update camera properties in the state
|
||||
updateCamera(selectedCamera.id, {
|
||||
focalLength: selectedCamera.focalLength,
|
||||
position: selectedCamera.position,
|
||||
rotation: selectedCamera.rotation,
|
||||
lookAt: selectedCamera.lookAt,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
const handleAddCamera = () => {
|
||||
// Check if we've reached the maximum number of cameras
|
||||
if (cameras.value.length >= 6) {
|
||||
console.warn("Maximum number of cameras (6) reached");
|
||||
return;
|
||||
}
|
||||
|
||||
const newIndex = cameras.value.length + 1;
|
||||
const newId = `cam${newIndex}`;
|
||||
|
||||
// This is for generating random orbital position for the new camera using spherical coordinates
|
||||
const radius = Math.random() * 5 + 7; // Distance from center: 7 to 12 units
|
||||
const theta = Math.random() * Math.PI * 2; // Azimuthal angle: 0 to 2π
|
||||
const phi = Math.PI / 3 + (Math.random() * Math.PI) / 6; // Polar angle: π/3 to π/2 (60° to 90° from horizontal)
|
||||
|
||||
// Convert spherical coordinates to Cartesian coordinates
|
||||
const randomX = radius * Math.sin(phi) * Math.cos(theta);
|
||||
const randomY = Math.abs(radius * Math.cos(phi)) + 2; // Ensure Y is positive and at least 2 units up
|
||||
const randomZ = radius * Math.sin(phi) * Math.sin(theta);
|
||||
|
||||
addCamera({
|
||||
id: newId,
|
||||
label: `Camera ${newIndex}`,
|
||||
focalLength: 24,
|
||||
position: {
|
||||
x: randomX,
|
||||
y: randomY,
|
||||
z: randomZ,
|
||||
},
|
||||
rotation: { x: 0, y: 0, z: 0 },
|
||||
lookAt: { x: 0, y: 0, z: 0 },
|
||||
});
|
||||
|
||||
// Switch to the newly created camera
|
||||
selectedCameraId.value = newId;
|
||||
|
||||
// Update the engine camera to match the new camera's properties
|
||||
if (editorEngine && editorEngine.camera) {
|
||||
editorEngine.camera.position.set(randomX, randomY, randomZ);
|
||||
editorEngine.camera.lookAt(0, 0, 0);
|
||||
editorEngine.camera.fov = editorEngine.focalLengthToFov(24);
|
||||
editorEngine.camera.updateProjectionMatrix();
|
||||
|
||||
// Reset and update camera controls
|
||||
if (editorEngine.cameraViewControls) {
|
||||
editorEngine.cameraViewControls.reset();
|
||||
editorEngine.cameraViewControls.update(0);
|
||||
}
|
||||
|
||||
// Force a render to update the view
|
||||
editorEngine.renderScene();
|
||||
|
||||
// Queue.publish({
|
||||
// queueName: QueueNames.TO_ENGINE,
|
||||
// action: toEngineActions.CAMERA_CHANGED,
|
||||
// data: {
|
||||
// id: newId,
|
||||
// label: `Camera ${newIndex}`,
|
||||
// focalLength: 24,
|
||||
// position: { x: randomX, y: randomY, z: randomZ },
|
||||
// rotation: { x: 0, y: 0, z: 0 },
|
||||
// lookAt: { x: 0, y: 0, z: 0 },
|
||||
// },
|
||||
// });
|
||||
}
|
||||
};
|
||||
|
||||
const handleCameraNameChange = (id: string, newName: string) => {
|
||||
updateCamera(id, { label: newName });
|
||||
};
|
||||
|
||||
const handleCameraFocalLengthChange = (id: string, value: number) => {
|
||||
const camera = cameras.value.find((cam) => cam.id === id);
|
||||
if (camera) {
|
||||
updateCamera(id, { focalLength: value });
|
||||
}
|
||||
};
|
||||
|
||||
const handleAspectRatioSelect = (selectedItem: PopoverItem) => {
|
||||
setAspectRatioList((prev) =>
|
||||
prev.map((item) => ({
|
||||
...item,
|
||||
selected: item.label === selectedItem.label,
|
||||
})),
|
||||
);
|
||||
|
||||
// Map the selected label to the corresponding CameraAspectRatio enum value
|
||||
let newRatio: CameraAspectRatio;
|
||||
switch (selectedItem.label) {
|
||||
case "16:9":
|
||||
newRatio = CameraAspectRatio.HORIZONTAL_16_9;
|
||||
break;
|
||||
case "3:2":
|
||||
newRatio = CameraAspectRatio.HORIZONTAL_3_2;
|
||||
break;
|
||||
case "2:3":
|
||||
newRatio = CameraAspectRatio.VERTICAL_2_3;
|
||||
break;
|
||||
case "1:1":
|
||||
newRatio = CameraAspectRatio.SQUARE_1_1;
|
||||
break;
|
||||
default:
|
||||
newRatio = CameraAspectRatio.HORIZONTAL_16_9;
|
||||
}
|
||||
|
||||
// Publish the change to the engine
|
||||
Queue.publish({
|
||||
queueName: QueueNames.TO_ENGINE,
|
||||
action: toEngineActions.CHANGE_CAMERA_ASPECT_RATIO,
|
||||
data: newRatio,
|
||||
});
|
||||
};
|
||||
|
||||
const handleFileUpload = (event: React.ChangeEvent<HTMLInputElement>) => {
|
||||
const files = event.target.files;
|
||||
if (files) {
|
||||
Array.from(files).forEach((file) => {
|
||||
const uploadId = Math.random().toString(36).substring(7);
|
||||
setUploadingImages((prev) => [...prev, { id: uploadId, file }]);
|
||||
|
||||
const reader = new FileReader();
|
||||
reader.onloadend = () => {
|
||||
uploadImage({
|
||||
title: `reference-image-${Math.random().toString(36).substring(2, 15)}`,
|
||||
assetFile: file,
|
||||
progressCallback: (newState) => {
|
||||
console.debug("Upload progress:", newState.data);
|
||||
if (newState.status === UploaderStates.success && newState.data) {
|
||||
const referenceImage: ReferenceImage = {
|
||||
id: Math.random().toString(36).substring(7),
|
||||
url: reader.result as string,
|
||||
file,
|
||||
mediaToken: newState.data || "",
|
||||
};
|
||||
setReferenceImages((prev) => [...prev, referenceImage]);
|
||||
setUploadingImages((prev) =>
|
||||
prev.filter((img) => img.id !== uploadId),
|
||||
);
|
||||
console.log("Reference image added:", referenceImage);
|
||||
} else if (
|
||||
newState.status === UploaderStates.assetError ||
|
||||
newState.status === UploaderStates.imageCreateError
|
||||
) {
|
||||
setUploadingImages((prev) =>
|
||||
prev.filter((img) => img.id !== uploadId),
|
||||
);
|
||||
console.error("Upload failed");
|
||||
}
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
reader.readAsDataURL(file);
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
const handleRemoveReference = (id: string) => {
|
||||
setReferenceImages((prev) => prev.filter((img) => img.id !== id));
|
||||
if (fileInputRef.current) {
|
||||
fileInputRef.current.value = "";
|
||||
}
|
||||
};
|
||||
|
||||
const handleUploadClick = () => {
|
||||
fileInputRef.current?.click();
|
||||
};
|
||||
|
||||
const handleGallerySelect = () => {
|
||||
setIsGalleryModalOpen(true);
|
||||
};
|
||||
|
||||
const handleGalleryClose = () => {
|
||||
setIsGalleryModalOpen(false);
|
||||
setSelectedGalleryImages([]);
|
||||
};
|
||||
|
||||
const handleImageSelect = (id: string) => {
|
||||
setSelectedGalleryImages((prev) => {
|
||||
if (prev.includes(id)) {
|
||||
return prev.filter((imageId) => imageId !== id);
|
||||
}
|
||||
if (prev.length >= 4) {
|
||||
return prev;
|
||||
}
|
||||
return [...prev, id];
|
||||
});
|
||||
};
|
||||
|
||||
const handleGalleryImages = (selectedItems: GalleryItem[]) => {
|
||||
selectedItems.forEach((item) => {
|
||||
if (!item.fullImage) return;
|
||||
const referenceImage: ReferenceImage = {
|
||||
id: Math.random().toString(36).substring(7),
|
||||
url: item.fullImage,
|
||||
file: new File([], "gallery-image"),
|
||||
mediaToken: item.id,
|
||||
};
|
||||
setReferenceImages((prev) => [...prev, referenceImage]);
|
||||
});
|
||||
setIsGalleryModalOpen(false);
|
||||
setSelectedGalleryImages([]);
|
||||
};
|
||||
|
||||
const handleAction = (action: string) => {
|
||||
switch (action) {
|
||||
case "upload":
|
||||
handleUploadClick();
|
||||
break;
|
||||
case "gallery":
|
||||
handleGallerySelect();
|
||||
break;
|
||||
default:
|
||||
console.log("Unknown action:", action);
|
||||
}
|
||||
};
|
||||
|
||||
const handlePaste = (e: React.ClipboardEvent<HTMLTextAreaElement>) => {
|
||||
e.preventDefault();
|
||||
const pastedText = e.clipboardData.getData("text").trim();
|
||||
setPrompt(pastedText);
|
||||
};
|
||||
|
||||
const handleChange = (e: React.ChangeEvent<HTMLTextAreaElement>) => {
|
||||
setPrompt(e.target.value);
|
||||
};
|
||||
|
||||
const handleError = (errorMessage: string) => {
|
||||
setContent(errorMessage);
|
||||
setIsModalOpen(true);
|
||||
};
|
||||
|
||||
const handleEnqueue = async () => {
|
||||
const isDesktop = IsDesktopApp();
|
||||
console.log("Is this a desktop app?", isDesktop);
|
||||
|
||||
const r = SoundRegistry.getInstance();
|
||||
r.playSound("test");
|
||||
|
||||
if (isDesktop) {
|
||||
await handleTauriEnqueue();
|
||||
} else {
|
||||
await handleWebEnqueue();
|
||||
}
|
||||
};
|
||||
|
||||
const handleWebEnqueue = async () => {
|
||||
if (!prompt.trim()) return;
|
||||
|
||||
setisEnqueueing(true);
|
||||
|
||||
if (editorEngine) {
|
||||
editorEngine.positive_prompt = prompt;
|
||||
|
||||
//const engineApi = new EngineApi();
|
||||
const promptsApi = new PromptsApi();
|
||||
|
||||
const snapshot = editorEngine.snapShotOfCurrentFrame(false);
|
||||
|
||||
if (snapshot) {
|
||||
const snapshotResult = await promptsApi.uploadSceneSnapshot({
|
||||
screenshot: snapshot.file,
|
||||
sceneMediaToken: "",
|
||||
});
|
||||
|
||||
console.log("useSystemPrompt", useSystemPrompt);
|
||||
|
||||
const response = await promptsApi.enqueueImageGeneration({
|
||||
disableSystemPrompt: !useSystemPrompt,
|
||||
prompt: prompt,
|
||||
snapshotMediaToken: snapshotResult.data || "",
|
||||
additionalImages: referenceImages.map((image) => image.mediaToken),
|
||||
});
|
||||
|
||||
console.log("response", response);
|
||||
|
||||
if (response.errorMessage) {
|
||||
handleError(response.errorMessage);
|
||||
setisEnqueueing(false);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
// Here we would pass both the prompt and reference images to the generation
|
||||
console.log(
|
||||
"Enqueuing with prompt:",
|
||||
prompt,
|
||||
"and reference images:",
|
||||
referenceImages,
|
||||
);
|
||||
await new Promise((resolve) => setTimeout(resolve, 2000));
|
||||
} finally {
|
||||
setisEnqueueing(false);
|
||||
toast.success("Image added to queue");
|
||||
}
|
||||
}
|
||||
setisEnqueueing(false);
|
||||
};
|
||||
|
||||
const handleTauriEnqueue = async () => {
|
||||
if (!prompt.trim()) return;
|
||||
|
||||
setisEnqueueing(true);
|
||||
|
||||
setTimeout(() => {
|
||||
// TODO(bt,2025-05-08): This is a hack so we don't accidentally wind up with a permanently disabled prompt box if
|
||||
// the backend hangs on a given request. (Sometimes Sora doesn't respond.) Ideally what we would do instead is only
|
||||
// force the prompt box to be enabled again as long as another request isn't running, ie. if we just fired off two
|
||||
// requests in succession, we wouldn't want to turn it off as the first one gets submitted.
|
||||
console.debug('Turn off blocking of prompt box...');
|
||||
setisEnqueueing(false);
|
||||
}, 10000);
|
||||
|
||||
if (editorEngine) {
|
||||
editorEngine.positive_prompt = prompt;
|
||||
|
||||
const promptsApi = new PromptsApi();
|
||||
|
||||
const snapshot = editorEngine.snapShotOfCurrentFrame(false);
|
||||
|
||||
if (snapshot) {
|
||||
const snapshotResult = await promptsApi.uploadSceneSnapshot({
|
||||
screenshot: snapshot.file,
|
||||
//sceneMediaToken: "",
|
||||
});
|
||||
|
||||
console.log("snapshotResult", snapshotResult)
|
||||
|
||||
const generateResponse = await SoraImageRemix({
|
||||
snapshot_media_token: snapshotResult.data!,
|
||||
disable_system_prompt: !useSystemPrompt,
|
||||
prompt: prompt,
|
||||
maybe_additional_images: referenceImages.map(
|
||||
(image) => image.mediaToken,
|
||||
),
|
||||
maybe_number_of_samples: 1,
|
||||
});
|
||||
|
||||
console.log("generateResponse", generateResponse)
|
||||
|
||||
if (generateResponse.status === CommandSuccessStatus.Success) {
|
||||
const prefs = await GetAppPreferences();
|
||||
const soundName = prefs.preferences?.enqueue_success_sound;
|
||||
if (soundName !== undefined) {
|
||||
const registry = SoundRegistry.getInstance();
|
||||
registry.playSound(soundName);
|
||||
}
|
||||
toast.success("Image added to queue");
|
||||
setisEnqueueing(false);
|
||||
return;
|
||||
|
||||
} else {
|
||||
const prefs = await GetAppPreferences();
|
||||
const soundName = prefs.preferences?.enqueue_failure_sound;
|
||||
if (soundName !== undefined) {
|
||||
const registry = SoundRegistry.getInstance();
|
||||
registry.playSound(soundName);
|
||||
}
|
||||
toast.error("Failed to enqueue image");
|
||||
setisEnqueueing(false);
|
||||
return;
|
||||
}
|
||||
|
||||
// const generateResponse = await invoke("sora_image_remix_command", {
|
||||
// request: {
|
||||
// snapshot_media_token: snapshotResult.data!,
|
||||
// disable_system_prompt: !useSystemPrompt,
|
||||
// prompt: prompt,
|
||||
// maybe_additional_images: referenceImages.map(
|
||||
// (image) => image.mediaToken,
|
||||
// ),
|
||||
// maybe_number_of_samples: 1,
|
||||
// },
|
||||
// });
|
||||
|
||||
//if (generateResponse.errorMessage) {
|
||||
// handleError(response.errorMessage);
|
||||
// setisEnqueueing(false);
|
||||
// return;
|
||||
//}
|
||||
}
|
||||
|
||||
try {
|
||||
// Here we would pass both the prompt and reference images to the generation
|
||||
console.log(
|
||||
"Enqueuing with prompt:",
|
||||
prompt,
|
||||
"and reference images:",
|
||||
referenceImages,
|
||||
);
|
||||
await new Promise((resolve) => setTimeout(resolve, 2000));
|
||||
} finally {
|
||||
setisEnqueueing(false);
|
||||
toast.success("Image added to queue");
|
||||
}
|
||||
}
|
||||
setisEnqueueing(false);
|
||||
};
|
||||
|
||||
const handleKeyDown = (e: React.KeyboardEvent<HTMLTextAreaElement>) => {
|
||||
if (e.key === "Enter" && !e.shiftKey) {
|
||||
e.preventDefault();
|
||||
handleWebEnqueue();
|
||||
}
|
||||
};
|
||||
|
||||
// Get the current aspect ratio icon based on the cameraAspectRatio signal
|
||||
const getCurrentAspectRatioIcon = () => {
|
||||
switch (cameraAspectRatio.value) {
|
||||
case CameraAspectRatio.HORIZONTAL_3_2:
|
||||
return faRectangle;
|
||||
case CameraAspectRatio.VERTICAL_2_3:
|
||||
return faRectangleVertical;
|
||||
case CameraAspectRatio.VERTICAL_9_16:
|
||||
return faRectangleVertical;
|
||||
case CameraAspectRatio.SQUARE_1_1:
|
||||
return faSquare;
|
||||
default:
|
||||
return faRectangleWide;
|
||||
}
|
||||
};
|
||||
|
||||
const handleSaveFrame = async () => {
|
||||
if (editorEngine) {
|
||||
const snapshot = editorEngine.snapShotOfCurrentFrame(false);
|
||||
if (snapshot) {
|
||||
const engineApi = new EngineApi();
|
||||
await engineApi.uploadSceneSnapshot({
|
||||
screenshot: snapshot.file || "",
|
||||
sceneMediaToken: "",
|
||||
});
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const handleDownloadFrame = () => {
|
||||
if (editorEngine) {
|
||||
editorEngine.snapShotOfCurrentFrame(true);
|
||||
}
|
||||
};
|
||||
|
||||
return (
|
||||
<>
|
||||
<Modal
|
||||
isOpen={isModalOpen}
|
||||
onClose={() => {
|
||||
setIsModalOpen(false);
|
||||
setContent("");
|
||||
}}
|
||||
>
|
||||
{content}
|
||||
</Modal>
|
||||
<div className="absolute bottom-4 left-1/2 flex -translate-x-1/2 flex-col gap-3">
|
||||
{(referenceImages.length > 0 || uploadingImages.length > 0) && (
|
||||
<div className="flex w-full gap-2">
|
||||
{referenceImages.map((image) => (
|
||||
<div
|
||||
key={image.id}
|
||||
className="glass relative aspect-square w-20 rounded-lg"
|
||||
>
|
||||
<img
|
||||
src={image.url}
|
||||
alt="Reference"
|
||||
className="h-full w-full rounded-lg object-cover"
|
||||
/>
|
||||
<button
|
||||
onClick={() => handleRemoveReference(image.id)}
|
||||
className="absolute right-[2px] top-[2px] flex h-5 w-5 items-center justify-center rounded-full bg-black/40 text-white backdrop-blur-md transition-colors hover:bg-black"
|
||||
>
|
||||
<FontAwesomeIcon icon={faTimes} className="h-2.5 w-2.5" />
|
||||
</button>
|
||||
</div>
|
||||
))}
|
||||
{uploadingImages.map(({ id, file }) => {
|
||||
const previewUrl = URL.createObjectURL(file);
|
||||
return (
|
||||
<div
|
||||
key={id}
|
||||
className="glass relative aspect-square w-20 overflow-hidden rounded-lg"
|
||||
>
|
||||
<div className="absolute inset-0">
|
||||
<img
|
||||
src={previewUrl}
|
||||
alt="Uploading preview"
|
||||
className="h-full w-full object-cover blur-sm"
|
||||
/>
|
||||
</div>
|
||||
<div className="absolute inset-0 flex items-center justify-center bg-black/20">
|
||||
<FontAwesomeIcon
|
||||
icon={faSpinnerThird}
|
||||
className="h-6 w-6 animate-spin text-white"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
<div
|
||||
className={twMerge(
|
||||
"glass w-[730px] rounded-xl p-4",
|
||||
isPromptBoxFocused.value ? "!border !border-primary" : "",
|
||||
)}
|
||||
>
|
||||
<input
|
||||
type="file"
|
||||
ref={fileInputRef}
|
||||
className="hidden"
|
||||
accept="image/*"
|
||||
onChange={handleFileUpload}
|
||||
multiple
|
||||
/>
|
||||
<div className="flex justify-center gap-2">
|
||||
<PopoverMenu
|
||||
mode="button"
|
||||
panelTitle="Add Image"
|
||||
items={[
|
||||
{
|
||||
label: "Upload from device",
|
||||
selected: false,
|
||||
icon: <FontAwesomeIcon icon={faUpload} className="h-4 w-4" />,
|
||||
action: "upload",
|
||||
},
|
||||
{
|
||||
label: "Choose from library",
|
||||
selected: false,
|
||||
icon: <FontAwesomeIcon icon={faImages} className="h-4 w-4" />,
|
||||
action: "gallery",
|
||||
},
|
||||
]}
|
||||
onPanelAction={handleAction}
|
||||
showIconsInList
|
||||
buttonClassName="bg-transparent hover:bg-transparent py-1.5 px-0 pr-1 m-0 hover:opacity-50 transition-opacity duration-100 ring-0 border-none focus:ring-0 outline-none"
|
||||
triggerIcon={
|
||||
<FontAwesomeIcon icon={faPlus} className="text-xl" />
|
||||
}
|
||||
/>
|
||||
|
||||
<textarea
|
||||
ref={textareaRef}
|
||||
rows={1}
|
||||
placeholder="Describe your image..."
|
||||
className="text-md mb-2 max-h-[5.5em] flex-1 resize-none overflow-y-auto rounded bg-transparent pb-2 pr-2 pt-1 text-white placeholder-white placeholder:text-white/60 focus:outline-none"
|
||||
value={prompt}
|
||||
onChange={handleChange}
|
||||
onPaste={handlePaste}
|
||||
onKeyDown={handleKeyDown}
|
||||
onFocus={() => {
|
||||
disableHotkeyInput(DomLevels.INPUT);
|
||||
isPromptBoxFocused.value = true;
|
||||
}}
|
||||
onBlur={() => {
|
||||
enableHotkeyInput(DomLevels.INPUT);
|
||||
isPromptBoxFocused.value = false;
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
<div className="mt-2 flex items-center justify-between gap-2">
|
||||
<div className="flex items-center gap-2">
|
||||
<Tooltip
|
||||
content="Aspect ratio"
|
||||
position="top"
|
||||
className="z-50"
|
||||
closeOnClick={true}
|
||||
>
|
||||
<PopoverMenu
|
||||
items={aspectRatioList}
|
||||
onSelect={handleAspectRatioSelect}
|
||||
mode="toggle"
|
||||
panelTitle="Aspect Ratio"
|
||||
showIconsInList
|
||||
triggerIcon={
|
||||
<FontAwesomeIcon
|
||||
icon={getCurrentAspectRatioIcon()}
|
||||
className="h-4 w-4"
|
||||
/>
|
||||
}
|
||||
/>
|
||||
</Tooltip>
|
||||
<Tooltip
|
||||
content="Camera"
|
||||
position="top"
|
||||
className="z-50"
|
||||
delay={300}
|
||||
closeOnClick={true}
|
||||
>
|
||||
<PopoverMenu
|
||||
items={cameras.value.map((cam) => ({
|
||||
id: cam.id,
|
||||
label: cam.label,
|
||||
selected: cam.id === selectedCameraId.value,
|
||||
icon: (
|
||||
<FontAwesomeIcon icon={faCamera} className="h-4 w-4" />
|
||||
),
|
||||
focalLength: cam.focalLength,
|
||||
position: cam.position,
|
||||
rotation: cam.rotation,
|
||||
lookAt: cam.lookAt,
|
||||
}))}
|
||||
onSelect={handleCameraSelect}
|
||||
onAdd={handleAddCamera}
|
||||
triggerIcon={
|
||||
<FontAwesomeIcon icon={faCamera} className="h-4 w-4" />
|
||||
}
|
||||
showAddButton
|
||||
disableAddButton={cameras.value.length >= 6}
|
||||
showIconsInList
|
||||
mode="toggle"
|
||||
panelTitle="Camera"
|
||||
panelActionLabel="Settings"
|
||||
onPanelAction={() => setIsCameraSettingsOpen(true)}
|
||||
buttonClassName="max-w-[130px] h-9"
|
||||
/>
|
||||
</Tooltip>
|
||||
<Tooltip
|
||||
content={
|
||||
useSystemPrompt
|
||||
? "Use system prompt: ON"
|
||||
: "Use system prompt: OFF"
|
||||
}
|
||||
position="top"
|
||||
className="z-50"
|
||||
delay={200}
|
||||
>
|
||||
<ToggleButton
|
||||
isActive={useSystemPrompt}
|
||||
icon={faMessageXmark}
|
||||
activeIcon={faMessageCheck}
|
||||
onClick={() => setUseSystemPrompt(!useSystemPrompt)}
|
||||
/>
|
||||
</Tooltip>
|
||||
<Tooltip
|
||||
content={gridVisibility.value ? "Grid: ON" : "Grid: OFF"}
|
||||
position="top"
|
||||
className="z-50"
|
||||
delay={200}
|
||||
>
|
||||
<ToggleButton
|
||||
isActive={gridVisibility.value}
|
||||
icon={faTableCellsLarge}
|
||||
activeIcon={faTableCellsLarge}
|
||||
onClick={() => setGridVisibility(!gridVisibility.value)}
|
||||
/>
|
||||
</Tooltip>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<Tooltip
|
||||
content="Download frame"
|
||||
position="top"
|
||||
className="z-50"
|
||||
closeOnClick={true}
|
||||
delay={200}
|
||||
>
|
||||
<Button
|
||||
className="flex h-9 items-center border-none bg-[#5F5F68]/60 px-3 text-sm text-white backdrop-blur-lg hover:bg-[#5F5F68]/90"
|
||||
variant="secondary"
|
||||
icon={faDownload}
|
||||
onClick={handleDownloadFrame}
|
||||
/>
|
||||
</Tooltip>
|
||||
|
||||
<Button
|
||||
className="flex items-center border-none bg-[#5F5F68]/60 px-3 text-sm text-white backdrop-blur-lg hover:bg-[#5F5F68]/90"
|
||||
variant="secondary"
|
||||
icon={faSave}
|
||||
onClick={handleSaveFrame}
|
||||
>
|
||||
Save frame
|
||||
</Button>
|
||||
<Button
|
||||
className="flex items-center border-none bg-brand-primary px-3 text-sm text-white disabled:cursor-not-allowed disabled:opacity-50"
|
||||
icon={!isEnqueueing ? faSparkles : undefined}
|
||||
onClick={handleEnqueue}
|
||||
disabled={isEnqueueing || !prompt.trim()}
|
||||
>
|
||||
{isEnqueueing ? (
|
||||
<FontAwesomeIcon
|
||||
icon={faSpinnerThird}
|
||||
className="animate-spin text-lg"
|
||||
/>
|
||||
) : (
|
||||
"Generate"
|
||||
)}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<CameraSettingsModal
|
||||
isOpen={isCameraSettingsOpen}
|
||||
onClose={() => setIsCameraSettingsOpen(false)}
|
||||
cameras={cameras.value.map((cam) => ({
|
||||
id: cam.id,
|
||||
label: cam.label,
|
||||
selected: cam.id === selectedCameraId.value,
|
||||
icon: <FontAwesomeIcon icon={faCamera} className="h-4 w-4" />,
|
||||
focalLength: cam.focalLength,
|
||||
position: cam.position,
|
||||
rotation: cam.rotation,
|
||||
lookAt: cam.lookAt,
|
||||
}))}
|
||||
onCameraNameChange={handleCameraNameChange}
|
||||
onCameraFocalLengthChange={handleCameraFocalLengthChange}
|
||||
onAddCamera={handleAddCamera}
|
||||
selectedCameraId={selectedCameraId.value}
|
||||
handleCameraSelect={handleCameraSelect}
|
||||
onDeleteCamera={deleteCamera}
|
||||
/>
|
||||
</div>
|
||||
<GalleryModal
|
||||
isOpen={isGalleryModalOpen}
|
||||
onClose={handleGalleryClose}
|
||||
mode="select"
|
||||
selectedItemIds={selectedGalleryImages}
|
||||
onSelectItem={handleImageSelect}
|
||||
maxSelections={4}
|
||||
onUseSelected={handleGalleryImages}
|
||||
tabs={[
|
||||
{ id: "my-media", label: "My generations" },
|
||||
{ id: "uploads", label: "My uploads" },
|
||||
]}
|
||||
activeTab={activeGalleryTab}
|
||||
onTabChange={setActiveGalleryTab}
|
||||
onDownloadClicked={downloadFileFromUrl}
|
||||
/>
|
||||
</>
|
||||
);
|
||||
};
|
||||
@@ -1 +0,0 @@
|
||||
export * from "./PromptBox";
|
||||
@@ -49,9 +49,6 @@
|
||||
{
|
||||
"path": "../../libs/components/toaster"
|
||||
},
|
||||
{
|
||||
"path": "../../libs/components/demo-modal"
|
||||
},
|
||||
{
|
||||
"path": "../../libs/components/login-modal"
|
||||
},
|
||||
|
||||
@@ -357,7 +357,7 @@ export const PromptBox2D = ({
|
||||
if (!prompt.trim()) return;
|
||||
try {
|
||||
console.log(
|
||||
"Enqueuing with prompt:",
|
||||
"(4) Enqueuing with prompt:",
|
||||
prompt,
|
||||
"and reference images:",
|
||||
referenceImages
|
||||
|
||||
@@ -42,19 +42,20 @@ import { PromptsApi } from "@storyteller/api";
|
||||
import { SoundRegistry } from "@storyteller/soundboard";
|
||||
import { toast } from "@storyteller/ui-toaster";
|
||||
|
||||
interface ReferenceImage {
|
||||
id: string;
|
||||
url: string;
|
||||
file: File;
|
||||
mediaToken: string;
|
||||
}
|
||||
|
||||
import { EngineApi } from "@storyteller/api";
|
||||
import { CameraSettingsModal } from "@storyteller/ui-camera-settings-modal";
|
||||
import { twMerge } from "tailwind-merge";
|
||||
import { GalleryModal, GalleryItem } from "@storyteller/ui-gallery-modal";
|
||||
import { Modal } from "@storyteller/ui-modal";
|
||||
import { Signal } from "@preact/signals-react";
|
||||
import { CommandSuccessStatus, GetAppPreferences, SoraImageRemix, SoraImageRemixErrorType } from "@storyteller/tauri-api";
|
||||
|
||||
interface ReferenceImage {
|
||||
id: string;
|
||||
url: string;
|
||||
file: File;
|
||||
mediaToken: string;
|
||||
}
|
||||
|
||||
interface PromptBox3DProps {
|
||||
cameras: Signal<Camera[]>;
|
||||
@@ -317,14 +318,12 @@ export const PromptBox3D = ({
|
||||
|
||||
const handleEnqueue = async () => {
|
||||
const isDesktop = IsDesktopApp();
|
||||
console.log("Is this a desktop app?", isDesktop);
|
||||
|
||||
const r = SoundRegistry.getInstance();
|
||||
r.playSound("test");
|
||||
|
||||
console.log("Is Desktop?", isDesktop);
|
||||
if (isDesktop) {
|
||||
console.log("Desktop app - tauri enqueue");
|
||||
await handleTauriEnqueue();
|
||||
} else {
|
||||
console.log("Web app - web enqueue");
|
||||
await handleWebEnqueue();
|
||||
}
|
||||
};
|
||||
@@ -348,7 +347,7 @@ export const PromptBox3D = ({
|
||||
sceneMediaToken: "",
|
||||
});
|
||||
|
||||
console.log("useSystemPrompt", useSystemPrompt);
|
||||
console.log("(web 3d) useSystemPrompt", useSystemPrompt);
|
||||
|
||||
const response = await promptsApi.enqueueImageGeneration({
|
||||
disableSystemPrompt: !useSystemPrompt,
|
||||
@@ -369,7 +368,7 @@ export const PromptBox3D = ({
|
||||
try {
|
||||
// Here we would pass both the prompt and reference images to the generation
|
||||
console.log(
|
||||
"Enqueuing with prompt:",
|
||||
"(3D.2) Enqueuing with prompt:",
|
||||
prompt,
|
||||
"and reference images:",
|
||||
referenceImages
|
||||
@@ -388,6 +387,15 @@ export const PromptBox3D = ({
|
||||
|
||||
setisEnqueueing(true);
|
||||
|
||||
setTimeout(() => {
|
||||
// TODO(bt,2025-05-08): This is a hack so we don't accidentally wind up with a permanently disabled prompt box if
|
||||
// the backend hangs on a given request. (Sometimes Sora doesn't respond.) Ideally what we would do instead is only
|
||||
// force the prompt box to be enabled again as long as another request isn't running, ie. if we just fired off two
|
||||
// requests in succession, we wouldn't want to turn it off as the first one gets submitted.
|
||||
console.debug('Turn off blocking of prompt box...');
|
||||
setisEnqueueing(false);
|
||||
}, 10000);
|
||||
|
||||
setPrompt(prompt);
|
||||
|
||||
const promptsApi = new PromptsApi();
|
||||
@@ -401,31 +409,68 @@ export const PromptBox3D = ({
|
||||
//sceneMediaToken: "",
|
||||
});
|
||||
|
||||
const generateResponse = await invoke("sora_image_remix_command", {
|
||||
request: {
|
||||
snapshot_media_token: snapshotResult.data!,
|
||||
disable_system_prompt: !useSystemPrompt,
|
||||
prompt: prompt,
|
||||
maybe_additional_images: referenceImages.map(
|
||||
(image) => image.mediaToken
|
||||
),
|
||||
maybe_number_of_samples: 1,
|
||||
},
|
||||
console.log("snapshotResult", snapshotResult);
|
||||
|
||||
const generateResponse = await SoraImageRemix({
|
||||
snapshot_media_token: snapshotResult.data!,
|
||||
disable_system_prompt: !useSystemPrompt,
|
||||
prompt: prompt,
|
||||
maybe_additional_images: referenceImages.map(
|
||||
(image) => image.mediaToken,
|
||||
),
|
||||
maybe_number_of_samples: 1,
|
||||
});
|
||||
|
||||
console.log("response", generateResponse);
|
||||
console.log("generateResponse", generateResponse);
|
||||
|
||||
if (generateResponse.status === CommandSuccessStatus.Success) {
|
||||
const prefs = await GetAppPreferences();
|
||||
const soundName = prefs.preferences?.enqueue_success_sound;
|
||||
if (soundName !== undefined) {
|
||||
const registry = SoundRegistry.getInstance();
|
||||
registry.playSound(soundName);
|
||||
}
|
||||
toast.success("Image added to queue");
|
||||
setisEnqueueing(false);
|
||||
return;
|
||||
|
||||
} else {
|
||||
const prefs = await GetAppPreferences();
|
||||
const soundName = prefs.preferences?.enqueue_failure_sound;
|
||||
if (soundName !== undefined) {
|
||||
const registry = SoundRegistry.getInstance();
|
||||
registry.playSound(soundName);
|
||||
}
|
||||
|
||||
let errorMessage = "Failed to enqueue image";
|
||||
|
||||
if ("error_type" in generateResponse) {
|
||||
switch (generateResponse.error_type) {
|
||||
case SoraImageRemixErrorType.ServerError:
|
||||
errorMessage = "Server error. Failed to enqueue image.";
|
||||
break;
|
||||
case SoraImageRemixErrorType.TooManyConcurrentTasks:
|
||||
errorMessage = "You have too many Sora image generation tasks running. Please wait a moment.";
|
||||
break;
|
||||
case SoraImageRemixErrorType.SoraUsernameNotYetCreated:
|
||||
errorMessage = "Your Sora username is not yet created. Please visit the Sora.com website to create it first.";
|
||||
break;
|
||||
case SoraImageRemixErrorType.SoraIsHavingProblems:
|
||||
errorMessage = "Sora is having problems. Please try again later.";
|
||||
break;
|
||||
}
|
||||
}
|
||||
toast.error(errorMessage);
|
||||
setisEnqueueing(false);
|
||||
return;
|
||||
}
|
||||
|
||||
//if (generateResponse.errorMessage) {
|
||||
// handleError(response.errorMessage);
|
||||
// setisEnqueueing(false);
|
||||
// return;
|
||||
//}
|
||||
}
|
||||
|
||||
try {
|
||||
// Here we would pass both the prompt and reference images to the generation
|
||||
console.log(
|
||||
"Enqueuing with prompt:",
|
||||
"(3D.1) Enqueuing with prompt:",
|
||||
prompt,
|
||||
"and reference images:",
|
||||
referenceImages
|
||||
@@ -443,7 +488,7 @@ export const PromptBox3D = ({
|
||||
const handleKeyDown = (e: React.KeyboardEvent<HTMLTextAreaElement>) => {
|
||||
if (e.key === "Enter" && !e.shiftKey) {
|
||||
e.preventDefault();
|
||||
handleWebEnqueue();
|
||||
handleEnqueue();
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -2,6 +2,9 @@
|
||||
"files": [],
|
||||
"include": [],
|
||||
"references": [
|
||||
{
|
||||
"path": "../../tauri-api"
|
||||
},
|
||||
{
|
||||
"path": "../camera-settings-modal"
|
||||
},
|
||||
|
||||
@@ -43,6 +43,9 @@
|
||||
"src/**/*.tsx"
|
||||
],
|
||||
"references": [
|
||||
{
|
||||
"path": "../../tauri-api/tsconfig.lib.json"
|
||||
},
|
||||
{
|
||||
"path": "../camera-settings-modal/tsconfig.lib.json"
|
||||
},
|
||||
|
||||
@@ -16,16 +16,27 @@ const SOUND_OPTIONS = [
|
||||
{ value: "none", label: "None (Silent)" }, // NB: MUST BE FIRST
|
||||
|
||||
// Other options
|
||||
{ value: "correct", label: "Correct" },
|
||||
{ value: "crumble", label: "Crumble" },
|
||||
{ value: "done", label: "Done" },
|
||||
{ value: "error_chirp", label: "Error Chirp" },
|
||||
{ value: "flower", label: "Flower" },
|
||||
{ value: "next", label: "Next" },
|
||||
{ value: "select", label: "Select" },
|
||||
{ value: "special_alert", label: "Special Alert" },
|
||||
{ value: "special_flower", label: "Special Flower" },
|
||||
{ value: "spike_throw", label: "Spike Throw" },
|
||||
{ label: "Click", value: "click" },
|
||||
{ label: "Correct", value: "correct" },
|
||||
{ label: "Crumble", value: "crumble" },
|
||||
{ label: "Dialog Done", value: "done" },
|
||||
{ label: "Dialog Next", value: "next" },
|
||||
{ label: "Error Chirp", value: "error_chirp" },
|
||||
{ label: "Flower", value: "flower" },
|
||||
{ label: "Ghost", value: "ghost" },
|
||||
{ label: "Sci-Fi Alert", value: "scifi_alert" },
|
||||
{ label: "Sci-Fi Menu Beep 1", value: "scifi_menu_beep_1" },
|
||||
{ label: "Sci-Fi Menu Beep 2", value: "scifi_menu_beep_2" },
|
||||
{ label: "Sci-Fi Menu Close", value: "scifi_menu_close" },
|
||||
{ label: "Sci-Fi Menu Open", value: "scifi_menu_open" },
|
||||
{ label: "Sci-Fi Menu Select", value: "scifi_menu_select" },
|
||||
{ label: "Sci-Fi Shrill Alert", value: "scifi_shrill_alert" },
|
||||
{ label: "Select", value: "select" },
|
||||
{ label: "Shell Kick", value: "giant_shell_kick" },
|
||||
{ label: "Special Alert", value: "special_alert" },
|
||||
{ label: "Special Flower", value: "special_flower" },
|
||||
{ label: "Spike Throw", value: "spike_throw" },
|
||||
{ label: "Wrong", value: "wrong" },
|
||||
];
|
||||
|
||||
interface AudioSettingsPaneProps {}
|
||||
|
||||
@@ -19,10 +19,24 @@ export interface SoraImageRemixRequest {
|
||||
maybe_number_of_samples: number,
|
||||
}
|
||||
|
||||
export interface SoraImageRemixResult extends CommandResult {
|
||||
export interface SoraImageRemixSuccess extends CommandResult {
|
||||
// TODO: Status fields (they're not on the backend yet)
|
||||
}
|
||||
|
||||
export enum SoraImageRemixErrorType {
|
||||
ServerError = "server_error",
|
||||
TooManyConcurrentTasks = "too_many_concurrent_tasks",
|
||||
SoraUsernameNotYetCreated = "sora_username_not_yet_created",
|
||||
SoraIsHavingProblems = "sora_is_having_problems",
|
||||
}
|
||||
|
||||
export interface SoraImageRemixError extends CommandResult {
|
||||
error_type: SoraImageRemixErrorType;
|
||||
error_message?: string;
|
||||
}
|
||||
|
||||
export type SoraImageRemixResult = SoraImageRemixSuccess | SoraImageRemixError;
|
||||
|
||||
// Returns the Success and Error variants directly.
|
||||
// Throws on Network/Tauri errors.
|
||||
export const SoraImageRemix = async (request: SoraImageRemixRequest) : Promise<SoraImageRemixResult> => {
|
||||
|
||||
Reference in new issue
Block a user