diff --git a/crates/api_clients/openai_sora_client/src/recipes/list_sora_task_status_with_session_auto_renew.rs b/crates/api_clients/openai_sora_client/src/recipes/list_sora_task_status_with_session_auto_renew.rs index 1f2d044274..29c6f0e4c7 100644 --- a/crates/api_clients/openai_sora_client/src/recipes/list_sora_task_status_with_session_auto_renew.rs +++ b/crates/api_clients/openai_sora_client/src/recipes/list_sora_task_status_with_session_auto_renew.rs @@ -11,16 +11,16 @@ use anyhow::Error; use log::{info, warn}; use std::path::Path; -pub struct StatusRequestArgs { +pub struct StatusRequestArgs<'a> { pub limit: Option, pub before: Option, - pub credentials: SoraCredentialSet, + pub credentials: &'a SoraCredentialSet, } /// Check Sora task statuses with session auto-renewal. /// If a new sora credential is returned, replace the old one with the new one. pub async fn list_sora_task_status_with_session_auto_renew( - args: StatusRequestArgs, + args: StatusRequestArgs<'_>, ) -> Result<(VideoGenStatusResponse, Option), SoraError> { let request = StatusRequest { diff --git a/crates/api_clients/openai_sora_client/src/requests/common/task_id.rs b/crates/api_clients/openai_sora_client/src/requests/common/task_id.rs index e1d54106d0..0352754ddc 100644 --- a/crates/api_clients/openai_sora_client/src/requests/common/task_id.rs +++ b/crates/api_clients/openai_sora_client/src/requests/common/task_id.rs @@ -5,6 +5,20 @@ use serde_derive::{Deserialize, Serialize}; #[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, Serialize, Deserialize)] pub struct TaskId(pub String); +impl TaskId { + pub fn new(id: String) -> Self { + Self(id) + } + + pub fn as_str(&self) -> &str { + &self.0 + } + + pub fn to_string(&self) -> String { + self.0.to_string() + } +} + impl Display for TaskId { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { std::fmt::Display::fmt(&self.0, f) diff --git a/crates/desktop/artcraft/src/core/utils/mod.rs b/crates/desktop/artcraft/src/core/utils/mod.rs index dde08b91aa..1fc9359db5 100644 --- a/crates/desktop/artcraft/src/core/utils/mod.rs +++ b/crates/desktop/artcraft/src/core/utils/mod.rs @@ -9,4 +9,5 @@ pub mod object_store; pub mod save_base64_image_to_temp_dir; pub mod simple_http_download; pub mod simple_http_download_to_tempfile; +pub mod task_database_pending_statuses; pub mod window; \ No newline at end of file diff --git a/crates/desktop/artcraft/src/core/utils/task_database_pending_statuses.rs b/crates/desktop/artcraft/src/core/utils/task_database_pending_statuses.rs new file mode 100644 index 0000000000..f74e5e9703 --- /dev/null +++ b/crates/desktop/artcraft/src/core/utils/task_database_pending_statuses.rs @@ -0,0 +1,12 @@ +use enums::tauri::tasks::task_status::TaskStatus; +use once_cell::sync::Lazy; +use std::collections::HashSet; + +/// Reused predicate for querying pending SQLite tasks +pub static TASK_DATABASE_PENDING_STATUSES: Lazy> = Lazy::new(|| { + let mut statuses = HashSet::new(); + statuses.insert(TaskStatus::Pending); + statuses.insert(TaskStatus::Started); + statuses.insert(TaskStatus::AttemptFailed); + statuses +}); diff --git a/crates/desktop/artcraft/src/services/midjourney/threads/midjourney_long_polling_thread.rs b/crates/desktop/artcraft/src/services/midjourney/threads/midjourney_long_polling_thread.rs index ab8669c3d4..e04aa361e5 100644 --- a/crates/desktop/artcraft/src/services/midjourney/threads/midjourney_long_polling_thread.rs +++ b/crates/desktop/artcraft/src/services/midjourney/threads/midjourney_long_polling_thread.rs @@ -5,6 +5,7 @@ use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs; use crate::core::state::data_dir::app_data_root::AppDataRoot; use crate::core::state::data_dir::trait_data_subdir::DataSubdir; use crate::core::state::task_database::TaskDatabase; +use crate::core::utils::task_database_pending_statuses::TASK_DATABASE_PENDING_STATUSES; use crate::services::midjourney::state::midjourney_credential_manager::MidjourneyCredentialManager; use crate::services::midjourney::threads::events::maybe_handle_text_to_image_complete_event::maybe_handle_text_to_image_complete_event; use crate::services::midjourney::utils::download_midjourney_image::download_midjourney_image; @@ -31,22 +32,14 @@ use std::fs::File; use std::io::Write; use std::path::PathBuf; use storyteller_client::credentials::storyteller_credential_set::StorytellerCredentialSet; -use storyteller_client::error::api_error::ApiError; -use storyteller_client::error::storyteller_error::StorytellerError; use storyteller_client::endpoints::media_files::upload_image_media_file_from_file::{upload_image_media_file_from_file, UploadImageFromFileArgs}; use storyteller_client::endpoints::prompts::create_prompt::create_prompt; +use storyteller_client::error::api_error::ApiError; +use storyteller_client::error::storyteller_error::StorytellerError; use tauri::AppHandle; use tokens::tokens::batch_generations::BatchGenerationToken; use url::Url; -static PENDING_STATUSES : Lazy> = Lazy::new(|| { - let mut statuses = HashSet::new(); - statuses.insert(TaskStatus::Pending); - statuses.insert(TaskStatus::Started); - statuses.insert(TaskStatus::AttemptFailed); - statuses -}); - /// This thread is responsible for picking up tasks that fell through the cracks of /// the faster websocket thread. pub async fn midjourney_long_polling_thread( @@ -125,10 +118,10 @@ async fn polling_loop( let local_tasks = list_tasks_by_provider_and_status(ListTasksByProviderAndStatusArgs { db: task_database.get_connection(), provider: GenerationProvider::Midjourney, - task_statuses: &PENDING_STATUSES, + task_statuses: &TASK_DATABASE_PENDING_STATUSES, }).await?; - check_midjourney_tasks( + poll_midjourney_tasks( app_handle, app_env_configs, app_data_root, @@ -143,7 +136,7 @@ async fn polling_loop( } } -async fn check_midjourney_tasks( +async fn poll_midjourney_tasks( app_handle: &AppHandle, app_env_configs: &AppEnvConfigs, app_data_root: &AppDataRoot, diff --git a/crates/desktop/artcraft/src/services/sora/threads/sora_task_polling_thread.rs b/crates/desktop/artcraft/src/services/sora/threads/sora_task_polling_thread.rs index 0450900912..d0a19b54c0 100644 --- a/crates/desktop/artcraft/src/services/sora/threads/sora_task_polling_thread.rs +++ b/crates/desktop/artcraft/src/services/sora/threads/sora_task_polling_thread.rs @@ -6,19 +6,27 @@ use crate::core::events::sendable_event_trait::SendableEvent; use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs; use crate::core::state::data_dir::app_data_root::AppDataRoot; use crate::core::state::data_dir::trait_data_subdir::DataSubdir; +use crate::core::state::task_database::TaskDatabase; +use crate::core::utils::task_database_pending_statuses::TASK_DATABASE_PENDING_STATUSES; use crate::services::sora::state::sora_credential_manager::SoraCredentialManager; use crate::services::sora::state::sora_task_queue::SoraTaskQueue; use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager; use artcraft_api_defs::prompts::create_prompt::CreatePromptRequest; use enums::common::generation_provider::GenerationProvider; use enums::common::model_type::ModelType; +use enums::tauri::tasks::task_status; use errors::AnyhowResult; use idempotency::uuid::generate_random_uuid; use log::{error, info}; +use once_cell::sync::Lazy; +use openai_sora_client::creds::sora_credential_set::SoraCredentialSet; use openai_sora_client::recipes::list_sora_task_status_with_session_auto_renew::{list_sora_task_status_with_session_auto_renew, StatusRequestArgs}; use openai_sora_client::requests::common::task_id::TaskId; use openai_sora_client::requests::job_status::sora_job_status::{Generation, TaskStatus}; use reqwest::Url; +use sqlite_tasks::queries::list_tasks_by_provider_and_status::{list_tasks_by_provider_and_status, ListTasksByProviderAndStatusArgs, Task, TaskList}; +use sqlite_tasks::queries::update_task_status::{update_task_status, UpdateTaskArgs}; +use std::collections::{HashMap, HashSet}; use std::fs::File; use std::io::Write; use std::path::PathBuf; @@ -31,6 +39,7 @@ pub async fn sora_task_polling_thread( app_handle: AppHandle, app_env_configs: AppEnvConfigs, app_data_root: AppDataRoot, + task_database: TaskDatabase, sora_creds_manager: SoraCredentialManager, storyteller_creds_manager: StorytellerCredentialManager, sora_task_queue: SoraTaskQueue, @@ -39,6 +48,7 @@ pub async fn sora_task_polling_thread( let res = polling_loop( &app_handle, &app_env_configs, + &task_database, &sora_creds_manager, &storyteller_creds_manager, &sora_task_queue, @@ -54,6 +64,7 @@ pub async fn sora_task_polling_thread( async fn polling_loop( app_handle: &AppHandle, app_env_configs: &AppEnvConfigs, + task_database: &TaskDatabase, sora_creds_manager: &SoraCredentialManager, storyteller_creds_manager: &StorytellerCredentialManager, sora_task_queue: &SoraTaskQueue, @@ -70,127 +81,198 @@ async fn polling_loop( let creds = sora_creds_manager.get_credentials_required()?; - let (response, maybe_new_creds) = list_sora_task_status_with_session_auto_renew(StatusRequestArgs { - limit: None, - // TODO: How can we use the task id to poll better? Our existing code doesn't seem to illuminate this. - before: None, - credentials: creds, + let local_tasks = list_tasks_by_provider_and_status(ListTasksByProviderAndStatusArgs { + db: task_database.get_connection(), + provider: GenerationProvider::Sora, + task_statuses: &TASK_DATABASE_PENDING_STATUSES, }).await?; - if let Some(new_creds) = maybe_new_creds { - info!("Saving new credentials."); - sora_creds_manager.set_credentials(&new_creds)?; - } + poll_sora_tasks( + &app_handle, + &app_env_configs, + &task_database, + &sora_creds_manager, + &creds, + &storyteller_creds_manager, + &sora_task_queue, + &app_data_root, + local_tasks, + ).await?; - let mut succeeded_tasks = Vec::new(); - let mut failed_tasks = Vec::new(); + tokio::time::sleep(std::time::Duration::from_millis(2_000)).await; + } - // TODO: The cursoring logic likely needs to improve. - for task in response.task_responses { - let status = TaskStatus::from_str(&task.status); +async fn poll_sora_tasks( + app_handle: &AppHandle, + app_env_configs: &AppEnvConfigs, + task_database: &TaskDatabase, + sora_creds_manager: &SoraCredentialManager, + sora_creds: &SoraCredentialSet, + storyteller_creds_manager: &StorytellerCredentialManager, + sora_task_queue: &SoraTaskQueue, + app_data_root: &AppDataRoot, + local_tasks: TaskList, +) -> AnyhowResult<()> { - match status { - TaskStatus::Succeeded => { - succeeded_tasks.push(task); - } - TaskStatus::Failed => { - failed_tasks.push(task); - } - TaskStatus::Queued => {} - TaskStatus::Running => {} - TaskStatus::Unknown(_) => {} - } - } - - for task in failed_tasks.iter() { - if sora_task_queue.contains_key(&task.id)? { - let event = GenerationFailedEvent { - action: GenerationAction::GenerateImage, - service: GenerationServiceProvider::Sora, - model: None, - reason: None, - // prompt: task.prompt - // .as_deref() - // .map(|s| s.to_string()) - // .unwrap_or_else(||"".to_string()), - }; - if let Err(err) = event.send(&app_handle) { - error!("Failed to send GenerationFailedEvent: {:?}", err); // Fail open + let local_tasks = local_tasks.tasks; + + if local_tasks.is_empty() { + return Ok(()) + } + + // Map of Sora Task ID to Local Task. + let local_tasks_by_sora_task_id = local_tasks.iter() + .filter_map(|task| { + if let Some(provider_job_id) = &task.provider_job_id { + Some((provider_job_id.clone(), task.clone())) + } else { + None } + }) + .collect::>(); + + let (sora_response, maybe_new_creds) = list_sora_task_status_with_session_auto_renew(StatusRequestArgs { + limit: None, + // TODO: How can we use the task id to poll better? Our existing code doesn't seem to illuminate this. + before: None, + credentials: sora_creds, + }).await?; + + if let Some(new_creds) = maybe_new_creds { + info!("Saving new credentials."); + sora_creds_manager.set_credentials(&new_creds)?; + } + + let sora_items = sora_response.task_responses; + + let mut succeeded_tasks_by_id= HashMap::new(); + let mut failed_tasks_by_id = HashMap::new(); + + let storyteller_creds = storyteller_creds_manager.get_credentials_required()?; + + for task in sora_items.iter() { + let status = TaskStatus::from_str(&task.status); + + match status { + TaskStatus::Succeeded => { + succeeded_tasks_by_id.insert(task.id.clone(), task.clone()); } + TaskStatus::Failed => { + failed_tasks_by_id.insert(task.id.clone(), task.clone()); + } + TaskStatus::Queued => {} + TaskStatus::Running => {} + TaskStatus::Unknown(_) => {} } + } - let failed_task_ids: Vec<&TaskId> = failed_tasks - .iter() - .map(|task| &task.id) - .collect(); + // Clear dead tasks. - sora_task_queue.remove_list(&failed_task_ids)?; - - let creds = storyteller_creds_manager.get_credentials_required()?; - - for task in succeeded_tasks.iter() { - if !sora_task_queue.contains_key(&task.id)? { - continue; - } - - info!("Task succeeded: {:?}", task.id); - - let request = CreatePromptRequest { - uuid_idempotency_token: generate_random_uuid(), - positive_prompt: task.prompt.clone(), - negative_prompt: None, - model_type: Some(ModelType::GptImage1), - generation_provider: Some(GenerationProvider::Sora), + for (task_id, task) in failed_tasks_by_id.iter() { + // Emit events for failed tasks. + if sora_task_queue.contains_key(task_id)? { + let event = GenerationFailedEvent { + action: GenerationAction::GenerateImage, + service: GenerationServiceProvider::Sora, + model: None, + reason: None, }; - let prompt_response = create_prompt( - &app_env_configs.storyteller_host, - Some(&creds), - request - ).await?; - - info!("Created prompt: {:?}", &prompt_response.prompt_token); - - for (i, generation) in task.generations.iter().enumerate() { - info!("Downloading generated file..."); - let download_path = download_generation(generation, &app_data_root).await?; - - info!("Uploading to backend..."); - - let result = upload_image_media_file_from_file(UploadImageFromFileArgs { - api_host: &app_env_configs.storyteller_host, - maybe_creds: Some(&creds), - path: download_path, - is_intermediate_system_file: false, - maybe_prompt_token: Some(&prompt_response.prompt_token), - maybe_batch_token: None, // TODO: This should be added soon. - }).await?; - - info!("Uploaded to API backend: {:?}", result.media_file_token); - - let event = GenerationCompleteEvent { - //media_file_token: result.media_file_token, - action: Some(GenerationAction::GenerateImage), - service: GenerationServiceProvider::Sora, - model: None, - }; - - if let Err(err) = event.send(&app_handle) { - error!("Failed to send GenerationCompleteEvent: {:?}", err); // Fail open - } - } + event.send_infallible(&app_handle); } - let succeeded_task_ids : Vec<&TaskId> = succeeded_tasks - .iter() - .map(|task| &task.id) - .collect(); + // Clear from SQLite task database. + if let Some(local_task) = local_tasks_by_sora_task_id.get(task_id.as_str()) { + info!("Marking local task as failed: {:?}", local_task.id); - sora_task_queue.remove_list(&succeeded_task_ids)?; - - tokio::time::sleep(std::time::Duration::from_millis(6_000)).await; + let _updated = update_task_status(UpdateTaskArgs { + db: task_database.get_connection(), + task_id: &local_task.id, + status: task_status::TaskStatus::CompleteFailure, + }).await?; + } } + + // Clear from in memory DB + // TODO: Remove the in-memory queue in favor of SQLite only. + let failed_task_ids: Vec<&TaskId> = failed_tasks_by_id.keys().collect(); + sora_task_queue.remove_list(&failed_task_ids)?; + + + // Process succeeded tasks. + + for (task_id, task) in succeeded_tasks_by_id.iter() { + if !sora_task_queue.contains_key(task_id)? { + // TODO: Remove once the in-memory queue is gone. + continue; + } + + info!("Task succeeded: {:?}", task.id); + + let request = CreatePromptRequest { + uuid_idempotency_token: generate_random_uuid(), + positive_prompt: task.prompt.clone(), + negative_prompt: None, + model_type: Some(ModelType::GptImage1), + generation_provider: Some(GenerationProvider::Sora), + }; + + let prompt_response = create_prompt( + &app_env_configs.storyteller_host, + Some(&storyteller_creds), + request + ).await?; + + info!("Created prompt: {:?}", &prompt_response.prompt_token); + + for (i, generation) in task.generations.iter().enumerate() { + info!("Downloading generated file..."); + let download_path = download_generation(generation, &app_data_root).await?; + + info!("Uploading to backend..."); + + let result = upload_image_media_file_from_file(UploadImageFromFileArgs { + api_host: &app_env_configs.storyteller_host, + maybe_creds: Some(&storyteller_creds), + path: download_path, + is_intermediate_system_file: false, + maybe_prompt_token: Some(&prompt_response.prompt_token), + maybe_batch_token: None, // TODO: This should be added soon. + }).await?; + + info!("Uploaded to API backend: {:?}", result.media_file_token); + + + // Clear from SQLite task database. + if let Some(local_task) = local_tasks_by_sora_task_id.get(task_id.as_str()) { + info!("Marking local task as failed: {:?}", local_task.id); + + let updated = update_task_status(UpdateTaskArgs { + db: task_database.get_connection(), + task_id: &local_task.id, + status: task_status::TaskStatus::CompleteSuccess, + }).await?; + + // if !updated { + // return Ok(()); // If anything breaks with queries, don't spam events. + // } + } + + let event = GenerationCompleteEvent { + //media_file_token: result.media_file_token, + action: Some(GenerationAction::GenerateImage), + service: GenerationServiceProvider::Sora, + model: None, + }; + + event.send_infallible(&app_handle); + } + + } + + tokio::time::sleep(std::time::Duration::from_millis(3_000)).await; + + Ok(()) } async fn download_generation(generation: &Generation, app_data_root: &AppDataRoot) -> AnyhowResult {