wip refactor of sora task polling thread

This commit is contained in:
Brandon Thomas committed 2025-10-02 02:52:34 -04:00
1 parent 79650fd950
commit 19b13e6744
6 files changed
+223 -121

No files matched your search

@@ -11,16 +11,16 @@ use anyhow::Error;
use log::{info, warn};
use std::path::Path;
pub struct StatusRequestArgs {
pub struct StatusRequestArgs<'a> {
pub limit: Option<u32>,
pub before: Option<TaskId>,
pub credentials: SoraCredentialSet,
pub credentials: &'a SoraCredentialSet,
}
/// Check Sora task statuses with session auto-renewal.
/// If a new sora credential is returned, replace the old one with the new one.
pub async fn list_sora_task_status_with_session_auto_renew(
args: StatusRequestArgs,
args: StatusRequestArgs<'_>,
) -> Result<(VideoGenStatusResponse, Option<SoraCredentialSet>), SoraError> {
let request = StatusRequest {
@@ -5,6 +5,20 @@ use serde_derive::{Deserialize, Serialize};
#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, Serialize, Deserialize)]
pub struct TaskId(pub String);
impl TaskId {
pub fn new(id: String) -> Self {
Self(id)
}
pub fn as_str(&self) -> &str {
&self.0
}
pub fn to_string(&self) -> String {
self.0.to_string()
}
}
impl Display for TaskId {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
std::fmt::Display::fmt(&self.0, f)
@@ -9,4 +9,5 @@ pub mod object_store;
pub mod save_base64_image_to_temp_dir;
pub mod simple_http_download;
pub mod simple_http_download_to_tempfile;
pub mod task_database_pending_statuses;
pub mod window;
@@ -0,0 +1,12 @@
use enums::tauri::tasks::task_status::TaskStatus;
use once_cell::sync::Lazy;
use std::collections::HashSet;
/// Reused predicate for querying pending SQLite tasks
pub static TASK_DATABASE_PENDING_STATUSES: Lazy<HashSet<TaskStatus>> = Lazy::new(|| {
let mut statuses = HashSet::new();
statuses.insert(TaskStatus::Pending);
statuses.insert(TaskStatus::Started);
statuses.insert(TaskStatus::AttemptFailed);
statuses
});
@@ -5,6 +5,7 @@ use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs;
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use crate::core::state::data_dir::trait_data_subdir::DataSubdir;
use crate::core::state::task_database::TaskDatabase;
use crate::core::utils::task_database_pending_statuses::TASK_DATABASE_PENDING_STATUSES;
use crate::services::midjourney::state::midjourney_credential_manager::MidjourneyCredentialManager;
use crate::services::midjourney::threads::events::maybe_handle_text_to_image_complete_event::maybe_handle_text_to_image_complete_event;
use crate::services::midjourney::utils::download_midjourney_image::download_midjourney_image;
@@ -31,22 +32,14 @@ use std::fs::File;
use std::io::Write;
use std::path::PathBuf;
use storyteller_client::credentials::storyteller_credential_set::StorytellerCredentialSet;
use storyteller_client::error::api_error::ApiError;
use storyteller_client::error::storyteller_error::StorytellerError;
use storyteller_client::endpoints::media_files::upload_image_media_file_from_file::{upload_image_media_file_from_file, UploadImageFromFileArgs};
use storyteller_client::endpoints::prompts::create_prompt::create_prompt;
use storyteller_client::error::api_error::ApiError;
use storyteller_client::error::storyteller_error::StorytellerError;
use tauri::AppHandle;
use tokens::tokens::batch_generations::BatchGenerationToken;
use url::Url;
static PENDING_STATUSES : Lazy<HashSet<TaskStatus>> = Lazy::new(|| {
let mut statuses = HashSet::new();
statuses.insert(TaskStatus::Pending);
statuses.insert(TaskStatus::Started);
statuses.insert(TaskStatus::AttemptFailed);
statuses
});
/// This thread is responsible for picking up tasks that fell through the cracks of
/// the faster websocket thread.
pub async fn midjourney_long_polling_thread(
@@ -125,10 +118,10 @@ async fn polling_loop(
let local_tasks = list_tasks_by_provider_and_status(ListTasksByProviderAndStatusArgs {
db: task_database.get_connection(),
provider: GenerationProvider::Midjourney,
task_statuses: &PENDING_STATUSES,
task_statuses: &TASK_DATABASE_PENDING_STATUSES,
}).await?;
check_midjourney_tasks(
poll_midjourney_tasks(
app_handle,
app_env_configs,
app_data_root,
@@ -143,7 +136,7 @@ async fn polling_loop(
}
}
async fn check_midjourney_tasks(
async fn poll_midjourney_tasks(
app_handle: &AppHandle,
app_env_configs: &AppEnvConfigs,
app_data_root: &AppDataRoot,
@@ -6,19 +6,27 @@ use crate::core::events::sendable_event_trait::SendableEvent;
use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs;
use crate::core::state::data_dir::app_data_root::AppDataRoot;
use crate::core::state::data_dir::trait_data_subdir::DataSubdir;
use crate::core::state::task_database::TaskDatabase;
use crate::core::utils::task_database_pending_statuses::TASK_DATABASE_PENDING_STATUSES;
use crate::services::sora::state::sora_credential_manager::SoraCredentialManager;
use crate::services::sora::state::sora_task_queue::SoraTaskQueue;
use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager;
use artcraft_api_defs::prompts::create_prompt::CreatePromptRequest;
use enums::common::generation_provider::GenerationProvider;
use enums::common::model_type::ModelType;
use enums::tauri::tasks::task_status;
use errors::AnyhowResult;
use idempotency::uuid::generate_random_uuid;
use log::{error, info};
use once_cell::sync::Lazy;
use openai_sora_client::creds::sora_credential_set::SoraCredentialSet;
use openai_sora_client::recipes::list_sora_task_status_with_session_auto_renew::{list_sora_task_status_with_session_auto_renew, StatusRequestArgs};
use openai_sora_client::requests::common::task_id::TaskId;
use openai_sora_client::requests::job_status::sora_job_status::{Generation, TaskStatus};
use reqwest::Url;
use sqlite_tasks::queries::list_tasks_by_provider_and_status::{list_tasks_by_provider_and_status, ListTasksByProviderAndStatusArgs, Task, TaskList};
use sqlite_tasks::queries::update_task_status::{update_task_status, UpdateTaskArgs};
use std::collections::{HashMap, HashSet};
use std::fs::File;
use std::io::Write;
use std::path::PathBuf;
@@ -31,6 +39,7 @@ pub async fn sora_task_polling_thread(
app_handle: AppHandle,
app_env_configs: AppEnvConfigs,
app_data_root: AppDataRoot,
task_database: TaskDatabase,
sora_creds_manager: SoraCredentialManager,
storyteller_creds_manager: StorytellerCredentialManager,
sora_task_queue: SoraTaskQueue,
@@ -39,6 +48,7 @@ pub async fn sora_task_polling_thread(
let res = polling_loop(
&app_handle,
&app_env_configs,
&task_database,
&sora_creds_manager,
&storyteller_creds_manager,
&sora_task_queue,
@@ -54,6 +64,7 @@ pub async fn sora_task_polling_thread(
async fn polling_loop(
app_handle: &AppHandle,
app_env_configs: &AppEnvConfigs,
task_database: &TaskDatabase,
sora_creds_manager: &SoraCredentialManager,
storyteller_creds_manager: &StorytellerCredentialManager,
sora_task_queue: &SoraTaskQueue,
@@ -70,127 +81,198 @@ async fn polling_loop(
let creds = sora_creds_manager.get_credentials_required()?;
let (response, maybe_new_creds) = list_sora_task_status_with_session_auto_renew(StatusRequestArgs {
limit: None,
// TODO: How can we use the task id to poll better? Our existing code doesn't seem to illuminate this.
before: None,
credentials: creds,
let local_tasks = list_tasks_by_provider_and_status(ListTasksByProviderAndStatusArgs {
db: task_database.get_connection(),
provider: GenerationProvider::Sora,
task_statuses: &TASK_DATABASE_PENDING_STATUSES,
}).await?;
if let Some(new_creds) = maybe_new_creds {
info!("Saving new credentials.");
sora_creds_manager.set_credentials(&new_creds)?;
}
poll_sora_tasks(
&app_handle,
&app_env_configs,
&task_database,
&sora_creds_manager,
&creds,
&storyteller_creds_manager,
&sora_task_queue,
&app_data_root,
local_tasks,
).await?;
let mut succeeded_tasks = Vec::new();
let mut failed_tasks = Vec::new();
tokio::time::sleep(std::time::Duration::from_millis(2_000)).await;
}
// TODO: The cursoring logic likely needs to improve.
for task in response.task_responses {
let status = TaskStatus::from_str(&task.status);
async fn poll_sora_tasks(
app_handle: &AppHandle,
app_env_configs: &AppEnvConfigs,
task_database: &TaskDatabase,
sora_creds_manager: &SoraCredentialManager,
sora_creds: &SoraCredentialSet,
storyteller_creds_manager: &StorytellerCredentialManager,
sora_task_queue: &SoraTaskQueue,
app_data_root: &AppDataRoot,
local_tasks: TaskList,
) -> AnyhowResult<()> {
match status {
TaskStatus::Succeeded => {
succeeded_tasks.push(task);
}
TaskStatus::Failed => {
failed_tasks.push(task);
}
TaskStatus::Queued => {}
TaskStatus::Running => {}
TaskStatus::Unknown(_) => {}
}
}
for task in failed_tasks.iter() {
if sora_task_queue.contains_key(&task.id)? {
let event = GenerationFailedEvent {
action: GenerationAction::GenerateImage,
service: GenerationServiceProvider::Sora,
model: None,
reason: None,
// prompt: task.prompt
// .as_deref()
// .map(|s| s.to_string())
// .unwrap_or_else(||"".to_string()),
};
if let Err(err) = event.send(&app_handle) {
error!("Failed to send GenerationFailedEvent: {:?}", err); // Fail open
let local_tasks = local_tasks.tasks;
if local_tasks.is_empty() {
return Ok(())
}
// Map of Sora Task ID to Local Task.
let local_tasks_by_sora_task_id = local_tasks.iter()
.filter_map(|task| {
if let Some(provider_job_id) = &task.provider_job_id {
Some((provider_job_id.clone(), task.clone()))
} else {
None
}
})
.collect::<HashMap<String, Task>>();
let (sora_response, maybe_new_creds) = list_sora_task_status_with_session_auto_renew(StatusRequestArgs {
limit: None,
// TODO: How can we use the task id to poll better? Our existing code doesn't seem to illuminate this.
before: None,
credentials: sora_creds,
}).await?;
if let Some(new_creds) = maybe_new_creds {
info!("Saving new credentials.");
sora_creds_manager.set_credentials(&new_creds)?;
}
let sora_items = sora_response.task_responses;
let mut succeeded_tasks_by_id= HashMap::new();
let mut failed_tasks_by_id = HashMap::new();
let storyteller_creds = storyteller_creds_manager.get_credentials_required()?;
for task in sora_items.iter() {
let status = TaskStatus::from_str(&task.status);
match status {
TaskStatus::Succeeded => {
succeeded_tasks_by_id.insert(task.id.clone(), task.clone());
}
TaskStatus::Failed => {
failed_tasks_by_id.insert(task.id.clone(), task.clone());
}
TaskStatus::Queued => {}
TaskStatus::Running => {}
TaskStatus::Unknown(_) => {}
}
}
let failed_task_ids: Vec<&TaskId> = failed_tasks
.iter()
.map(|task| &task.id)
.collect();
// Clear dead tasks.
sora_task_queue.remove_list(&failed_task_ids)?;
let creds = storyteller_creds_manager.get_credentials_required()?;
for task in succeeded_tasks.iter() {
if !sora_task_queue.contains_key(&task.id)? {
continue;
}
info!("Task succeeded: {:?}", task.id);
let request = CreatePromptRequest {
uuid_idempotency_token: generate_random_uuid(),
positive_prompt: task.prompt.clone(),
negative_prompt: None,
model_type: Some(ModelType::GptImage1),
generation_provider: Some(GenerationProvider::Sora),
for (task_id, task) in failed_tasks_by_id.iter() {
// Emit events for failed tasks.
if sora_task_queue.contains_key(task_id)? {
let event = GenerationFailedEvent {
action: GenerationAction::GenerateImage,
service: GenerationServiceProvider::Sora,
model: None,
reason: None,
};
let prompt_response = create_prompt(
&app_env_configs.storyteller_host,
Some(&creds),
request
).await?;
info!("Created prompt: {:?}", &prompt_response.prompt_token);
for (i, generation) in task.generations.iter().enumerate() {
info!("Downloading generated file...");
let download_path = download_generation(generation, &app_data_root).await?;
info!("Uploading to backend...");
let result = upload_image_media_file_from_file(UploadImageFromFileArgs {
api_host: &app_env_configs.storyteller_host,
maybe_creds: Some(&creds),
path: download_path,
is_intermediate_system_file: false,
maybe_prompt_token: Some(&prompt_response.prompt_token),
maybe_batch_token: None, // TODO: This should be added soon.
}).await?;
info!("Uploaded to API backend: {:?}", result.media_file_token);
let event = GenerationCompleteEvent {
//media_file_token: result.media_file_token,
action: Some(GenerationAction::GenerateImage),
service: GenerationServiceProvider::Sora,
model: None,
};
if let Err(err) = event.send(&app_handle) {
error!("Failed to send GenerationCompleteEvent: {:?}", err); // Fail open
}
}
event.send_infallible(&app_handle);
}
let succeeded_task_ids : Vec<&TaskId> = succeeded_tasks
.iter()
.map(|task| &task.id)
.collect();
// Clear from SQLite task database.
if let Some(local_task) = local_tasks_by_sora_task_id.get(task_id.as_str()) {
info!("Marking local task as failed: {:?}", local_task.id);
sora_task_queue.remove_list(&succeeded_task_ids)?;
tokio::time::sleep(std::time::Duration::from_millis(6_000)).await;
let _updated = update_task_status(UpdateTaskArgs {
db: task_database.get_connection(),
task_id: &local_task.id,
status: task_status::TaskStatus::CompleteFailure,
}).await?;
}
}
// Clear from in memory DB
// TODO: Remove the in-memory queue in favor of SQLite only.
let failed_task_ids: Vec<&TaskId> = failed_tasks_by_id.keys().collect();
sora_task_queue.remove_list(&failed_task_ids)?;
// Process succeeded tasks.
for (task_id, task) in succeeded_tasks_by_id.iter() {
if !sora_task_queue.contains_key(task_id)? {
// TODO: Remove once the in-memory queue is gone.
continue;
}
info!("Task succeeded: {:?}", task.id);
let request = CreatePromptRequest {
uuid_idempotency_token: generate_random_uuid(),
positive_prompt: task.prompt.clone(),
negative_prompt: None,
model_type: Some(ModelType::GptImage1),
generation_provider: Some(GenerationProvider::Sora),
};
let prompt_response = create_prompt(
&app_env_configs.storyteller_host,
Some(&storyteller_creds),
request
).await?;
info!("Created prompt: {:?}", &prompt_response.prompt_token);
for (i, generation) in task.generations.iter().enumerate() {
info!("Downloading generated file...");
let download_path = download_generation(generation, &app_data_root).await?;
info!("Uploading to backend...");
let result = upload_image_media_file_from_file(UploadImageFromFileArgs {
api_host: &app_env_configs.storyteller_host,
maybe_creds: Some(&storyteller_creds),
path: download_path,
is_intermediate_system_file: false,
maybe_prompt_token: Some(&prompt_response.prompt_token),
maybe_batch_token: None, // TODO: This should be added soon.
}).await?;
info!("Uploaded to API backend: {:?}", result.media_file_token);
// Clear from SQLite task database.
if let Some(local_task) = local_tasks_by_sora_task_id.get(task_id.as_str()) {
info!("Marking local task as failed: {:?}", local_task.id);
let updated = update_task_status(UpdateTaskArgs {
db: task_database.get_connection(),
task_id: &local_task.id,
status: task_status::TaskStatus::CompleteSuccess,
}).await?;
// if !updated {
// return Ok(()); // If anything breaks with queries, don't spam events.
// }
}
let event = GenerationCompleteEvent {
//media_file_token: result.media_file_token,
action: Some(GenerationAction::GenerateImage),
service: GenerationServiceProvider::Sora,
model: None,
};
event.send_infallible(&app_handle);
}
}
tokio::time::sleep(std::time::Duration::from_millis(3_000)).await;
Ok(())
}
async fn download_generation(generation: &Generation, app_data_root: &AppDataRoot) -> AnyhowResult<PathBuf> {