mirror of
https://github.com/storytold/artcraft.git
synced 2026-11-16 06:42:14 +00:00
wip refactor of sora task polling thread
This commit is contained in:
1 parent
79650fd950
commit
19b13e6744
6 files changed
+223
-121
No files matched your search
+3
-3
@@ -11,16 +11,16 @@ use anyhow::Error;
|
||||
use log::{info, warn};
|
||||
use std::path::Path;
|
||||
|
||||
pub struct StatusRequestArgs {
|
||||
pub struct StatusRequestArgs<'a> {
|
||||
pub limit: Option<u32>,
|
||||
pub before: Option<TaskId>,
|
||||
pub credentials: SoraCredentialSet,
|
||||
pub credentials: &'a SoraCredentialSet,
|
||||
}
|
||||
|
||||
/// Check Sora task statuses with session auto-renewal.
|
||||
/// If a new sora credential is returned, replace the old one with the new one.
|
||||
pub async fn list_sora_task_status_with_session_auto_renew(
|
||||
args: StatusRequestArgs,
|
||||
args: StatusRequestArgs<'_>,
|
||||
) -> Result<(VideoGenStatusResponse, Option<SoraCredentialSet>), SoraError> {
|
||||
|
||||
let request = StatusRequest {
|
||||
|
||||
@@ -5,6 +5,20 @@ use serde_derive::{Deserialize, Serialize};
|
||||
#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, Serialize, Deserialize)]
|
||||
pub struct TaskId(pub String);
|
||||
|
||||
impl TaskId {
|
||||
pub fn new(id: String) -> Self {
|
||||
Self(id)
|
||||
}
|
||||
|
||||
pub fn as_str(&self) -> &str {
|
||||
&self.0
|
||||
}
|
||||
|
||||
pub fn to_string(&self) -> String {
|
||||
self.0.to_string()
|
||||
}
|
||||
}
|
||||
|
||||
impl Display for TaskId {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
std::fmt::Display::fmt(&self.0, f)
|
||||
|
||||
@@ -9,4 +9,5 @@ pub mod object_store;
|
||||
pub mod save_base64_image_to_temp_dir;
|
||||
pub mod simple_http_download;
|
||||
pub mod simple_http_download_to_tempfile;
|
||||
pub mod task_database_pending_statuses;
|
||||
pub mod window;
|
||||
@@ -0,0 +1,12 @@
|
||||
use enums::tauri::tasks::task_status::TaskStatus;
|
||||
use once_cell::sync::Lazy;
|
||||
use std::collections::HashSet;
|
||||
|
||||
/// Reused predicate for querying pending SQLite tasks
|
||||
pub static TASK_DATABASE_PENDING_STATUSES: Lazy<HashSet<TaskStatus>> = Lazy::new(|| {
|
||||
let mut statuses = HashSet::new();
|
||||
statuses.insert(TaskStatus::Pending);
|
||||
statuses.insert(TaskStatus::Started);
|
||||
statuses.insert(TaskStatus::AttemptFailed);
|
||||
statuses
|
||||
});
|
||||
+6
-13
@@ -5,6 +5,7 @@ use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs;
|
||||
use crate::core::state::data_dir::app_data_root::AppDataRoot;
|
||||
use crate::core::state::data_dir::trait_data_subdir::DataSubdir;
|
||||
use crate::core::state::task_database::TaskDatabase;
|
||||
use crate::core::utils::task_database_pending_statuses::TASK_DATABASE_PENDING_STATUSES;
|
||||
use crate::services::midjourney::state::midjourney_credential_manager::MidjourneyCredentialManager;
|
||||
use crate::services::midjourney::threads::events::maybe_handle_text_to_image_complete_event::maybe_handle_text_to_image_complete_event;
|
||||
use crate::services::midjourney::utils::download_midjourney_image::download_midjourney_image;
|
||||
@@ -31,22 +32,14 @@ use std::fs::File;
|
||||
use std::io::Write;
|
||||
use std::path::PathBuf;
|
||||
use storyteller_client::credentials::storyteller_credential_set::StorytellerCredentialSet;
|
||||
use storyteller_client::error::api_error::ApiError;
|
||||
use storyteller_client::error::storyteller_error::StorytellerError;
|
||||
use storyteller_client::endpoints::media_files::upload_image_media_file_from_file::{upload_image_media_file_from_file, UploadImageFromFileArgs};
|
||||
use storyteller_client::endpoints::prompts::create_prompt::create_prompt;
|
||||
use storyteller_client::error::api_error::ApiError;
|
||||
use storyteller_client::error::storyteller_error::StorytellerError;
|
||||
use tauri::AppHandle;
|
||||
use tokens::tokens::batch_generations::BatchGenerationToken;
|
||||
use url::Url;
|
||||
|
||||
static PENDING_STATUSES : Lazy<HashSet<TaskStatus>> = Lazy::new(|| {
|
||||
let mut statuses = HashSet::new();
|
||||
statuses.insert(TaskStatus::Pending);
|
||||
statuses.insert(TaskStatus::Started);
|
||||
statuses.insert(TaskStatus::AttemptFailed);
|
||||
statuses
|
||||
});
|
||||
|
||||
/// This thread is responsible for picking up tasks that fell through the cracks of
|
||||
/// the faster websocket thread.
|
||||
pub async fn midjourney_long_polling_thread(
|
||||
@@ -125,10 +118,10 @@ async fn polling_loop(
|
||||
let local_tasks = list_tasks_by_provider_and_status(ListTasksByProviderAndStatusArgs {
|
||||
db: task_database.get_connection(),
|
||||
provider: GenerationProvider::Midjourney,
|
||||
task_statuses: &PENDING_STATUSES,
|
||||
task_statuses: &TASK_DATABASE_PENDING_STATUSES,
|
||||
}).await?;
|
||||
|
||||
check_midjourney_tasks(
|
||||
poll_midjourney_tasks(
|
||||
app_handle,
|
||||
app_env_configs,
|
||||
app_data_root,
|
||||
@@ -143,7 +136,7 @@ async fn polling_loop(
|
||||
}
|
||||
}
|
||||
|
||||
async fn check_midjourney_tasks(
|
||||
async fn poll_midjourney_tasks(
|
||||
app_handle: &AppHandle,
|
||||
app_env_configs: &AppEnvConfigs,
|
||||
app_data_root: &AppDataRoot,
|
||||
|
||||
@@ -6,19 +6,27 @@ use crate::core::events::sendable_event_trait::SendableEvent;
|
||||
use crate::core::state::app_env_configs::app_env_configs::AppEnvConfigs;
|
||||
use crate::core::state::data_dir::app_data_root::AppDataRoot;
|
||||
use crate::core::state::data_dir::trait_data_subdir::DataSubdir;
|
||||
use crate::core::state::task_database::TaskDatabase;
|
||||
use crate::core::utils::task_database_pending_statuses::TASK_DATABASE_PENDING_STATUSES;
|
||||
use crate::services::sora::state::sora_credential_manager::SoraCredentialManager;
|
||||
use crate::services::sora::state::sora_task_queue::SoraTaskQueue;
|
||||
use crate::services::storyteller::state::storyteller_credential_manager::StorytellerCredentialManager;
|
||||
use artcraft_api_defs::prompts::create_prompt::CreatePromptRequest;
|
||||
use enums::common::generation_provider::GenerationProvider;
|
||||
use enums::common::model_type::ModelType;
|
||||
use enums::tauri::tasks::task_status;
|
||||
use errors::AnyhowResult;
|
||||
use idempotency::uuid::generate_random_uuid;
|
||||
use log::{error, info};
|
||||
use once_cell::sync::Lazy;
|
||||
use openai_sora_client::creds::sora_credential_set::SoraCredentialSet;
|
||||
use openai_sora_client::recipes::list_sora_task_status_with_session_auto_renew::{list_sora_task_status_with_session_auto_renew, StatusRequestArgs};
|
||||
use openai_sora_client::requests::common::task_id::TaskId;
|
||||
use openai_sora_client::requests::job_status::sora_job_status::{Generation, TaskStatus};
|
||||
use reqwest::Url;
|
||||
use sqlite_tasks::queries::list_tasks_by_provider_and_status::{list_tasks_by_provider_and_status, ListTasksByProviderAndStatusArgs, Task, TaskList};
|
||||
use sqlite_tasks::queries::update_task_status::{update_task_status, UpdateTaskArgs};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::fs::File;
|
||||
use std::io::Write;
|
||||
use std::path::PathBuf;
|
||||
@@ -31,6 +39,7 @@ pub async fn sora_task_polling_thread(
|
||||
app_handle: AppHandle,
|
||||
app_env_configs: AppEnvConfigs,
|
||||
app_data_root: AppDataRoot,
|
||||
task_database: TaskDatabase,
|
||||
sora_creds_manager: SoraCredentialManager,
|
||||
storyteller_creds_manager: StorytellerCredentialManager,
|
||||
sora_task_queue: SoraTaskQueue,
|
||||
@@ -39,6 +48,7 @@ pub async fn sora_task_polling_thread(
|
||||
let res = polling_loop(
|
||||
&app_handle,
|
||||
&app_env_configs,
|
||||
&task_database,
|
||||
&sora_creds_manager,
|
||||
&storyteller_creds_manager,
|
||||
&sora_task_queue,
|
||||
@@ -54,6 +64,7 @@ pub async fn sora_task_polling_thread(
|
||||
async fn polling_loop(
|
||||
app_handle: &AppHandle,
|
||||
app_env_configs: &AppEnvConfigs,
|
||||
task_database: &TaskDatabase,
|
||||
sora_creds_manager: &SoraCredentialManager,
|
||||
storyteller_creds_manager: &StorytellerCredentialManager,
|
||||
sora_task_queue: &SoraTaskQueue,
|
||||
@@ -70,127 +81,198 @@ async fn polling_loop(
|
||||
|
||||
let creds = sora_creds_manager.get_credentials_required()?;
|
||||
|
||||
let (response, maybe_new_creds) = list_sora_task_status_with_session_auto_renew(StatusRequestArgs {
|
||||
limit: None,
|
||||
// TODO: How can we use the task id to poll better? Our existing code doesn't seem to illuminate this.
|
||||
before: None,
|
||||
credentials: creds,
|
||||
let local_tasks = list_tasks_by_provider_and_status(ListTasksByProviderAndStatusArgs {
|
||||
db: task_database.get_connection(),
|
||||
provider: GenerationProvider::Sora,
|
||||
task_statuses: &TASK_DATABASE_PENDING_STATUSES,
|
||||
}).await?;
|
||||
|
||||
if let Some(new_creds) = maybe_new_creds {
|
||||
info!("Saving new credentials.");
|
||||
sora_creds_manager.set_credentials(&new_creds)?;
|
||||
}
|
||||
poll_sora_tasks(
|
||||
&app_handle,
|
||||
&app_env_configs,
|
||||
&task_database,
|
||||
&sora_creds_manager,
|
||||
&creds,
|
||||
&storyteller_creds_manager,
|
||||
&sora_task_queue,
|
||||
&app_data_root,
|
||||
local_tasks,
|
||||
).await?;
|
||||
|
||||
let mut succeeded_tasks = Vec::new();
|
||||
let mut failed_tasks = Vec::new();
|
||||
tokio::time::sleep(std::time::Duration::from_millis(2_000)).await;
|
||||
}
|
||||
|
||||
// TODO: The cursoring logic likely needs to improve.
|
||||
for task in response.task_responses {
|
||||
let status = TaskStatus::from_str(&task.status);
|
||||
async fn poll_sora_tasks(
|
||||
app_handle: &AppHandle,
|
||||
app_env_configs: &AppEnvConfigs,
|
||||
task_database: &TaskDatabase,
|
||||
sora_creds_manager: &SoraCredentialManager,
|
||||
sora_creds: &SoraCredentialSet,
|
||||
storyteller_creds_manager: &StorytellerCredentialManager,
|
||||
sora_task_queue: &SoraTaskQueue,
|
||||
app_data_root: &AppDataRoot,
|
||||
local_tasks: TaskList,
|
||||
) -> AnyhowResult<()> {
|
||||
|
||||
match status {
|
||||
TaskStatus::Succeeded => {
|
||||
succeeded_tasks.push(task);
|
||||
}
|
||||
TaskStatus::Failed => {
|
||||
failed_tasks.push(task);
|
||||
}
|
||||
TaskStatus::Queued => {}
|
||||
TaskStatus::Running => {}
|
||||
TaskStatus::Unknown(_) => {}
|
||||
}
|
||||
}
|
||||
|
||||
for task in failed_tasks.iter() {
|
||||
if sora_task_queue.contains_key(&task.id)? {
|
||||
let event = GenerationFailedEvent {
|
||||
action: GenerationAction::GenerateImage,
|
||||
service: GenerationServiceProvider::Sora,
|
||||
model: None,
|
||||
reason: None,
|
||||
// prompt: task.prompt
|
||||
// .as_deref()
|
||||
// .map(|s| s.to_string())
|
||||
// .unwrap_or_else(||"".to_string()),
|
||||
};
|
||||
if let Err(err) = event.send(&app_handle) {
|
||||
error!("Failed to send GenerationFailedEvent: {:?}", err); // Fail open
|
||||
let local_tasks = local_tasks.tasks;
|
||||
|
||||
if local_tasks.is_empty() {
|
||||
return Ok(())
|
||||
}
|
||||
|
||||
// Map of Sora Task ID to Local Task.
|
||||
let local_tasks_by_sora_task_id = local_tasks.iter()
|
||||
.filter_map(|task| {
|
||||
if let Some(provider_job_id) = &task.provider_job_id {
|
||||
Some((provider_job_id.clone(), task.clone()))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect::<HashMap<String, Task>>();
|
||||
|
||||
let (sora_response, maybe_new_creds) = list_sora_task_status_with_session_auto_renew(StatusRequestArgs {
|
||||
limit: None,
|
||||
// TODO: How can we use the task id to poll better? Our existing code doesn't seem to illuminate this.
|
||||
before: None,
|
||||
credentials: sora_creds,
|
||||
}).await?;
|
||||
|
||||
if let Some(new_creds) = maybe_new_creds {
|
||||
info!("Saving new credentials.");
|
||||
sora_creds_manager.set_credentials(&new_creds)?;
|
||||
}
|
||||
|
||||
let sora_items = sora_response.task_responses;
|
||||
|
||||
let mut succeeded_tasks_by_id= HashMap::new();
|
||||
let mut failed_tasks_by_id = HashMap::new();
|
||||
|
||||
let storyteller_creds = storyteller_creds_manager.get_credentials_required()?;
|
||||
|
||||
for task in sora_items.iter() {
|
||||
let status = TaskStatus::from_str(&task.status);
|
||||
|
||||
match status {
|
||||
TaskStatus::Succeeded => {
|
||||
succeeded_tasks_by_id.insert(task.id.clone(), task.clone());
|
||||
}
|
||||
TaskStatus::Failed => {
|
||||
failed_tasks_by_id.insert(task.id.clone(), task.clone());
|
||||
}
|
||||
TaskStatus::Queued => {}
|
||||
TaskStatus::Running => {}
|
||||
TaskStatus::Unknown(_) => {}
|
||||
}
|
||||
}
|
||||
|
||||
let failed_task_ids: Vec<&TaskId> = failed_tasks
|
||||
.iter()
|
||||
.map(|task| &task.id)
|
||||
.collect();
|
||||
// Clear dead tasks.
|
||||
|
||||
sora_task_queue.remove_list(&failed_task_ids)?;
|
||||
|
||||
let creds = storyteller_creds_manager.get_credentials_required()?;
|
||||
|
||||
for task in succeeded_tasks.iter() {
|
||||
if !sora_task_queue.contains_key(&task.id)? {
|
||||
continue;
|
||||
}
|
||||
|
||||
info!("Task succeeded: {:?}", task.id);
|
||||
|
||||
let request = CreatePromptRequest {
|
||||
uuid_idempotency_token: generate_random_uuid(),
|
||||
positive_prompt: task.prompt.clone(),
|
||||
negative_prompt: None,
|
||||
model_type: Some(ModelType::GptImage1),
|
||||
generation_provider: Some(GenerationProvider::Sora),
|
||||
for (task_id, task) in failed_tasks_by_id.iter() {
|
||||
// Emit events for failed tasks.
|
||||
if sora_task_queue.contains_key(task_id)? {
|
||||
let event = GenerationFailedEvent {
|
||||
action: GenerationAction::GenerateImage,
|
||||
service: GenerationServiceProvider::Sora,
|
||||
model: None,
|
||||
reason: None,
|
||||
};
|
||||
|
||||
let prompt_response = create_prompt(
|
||||
&app_env_configs.storyteller_host,
|
||||
Some(&creds),
|
||||
request
|
||||
).await?;
|
||||
|
||||
info!("Created prompt: {:?}", &prompt_response.prompt_token);
|
||||
|
||||
for (i, generation) in task.generations.iter().enumerate() {
|
||||
info!("Downloading generated file...");
|
||||
let download_path = download_generation(generation, &app_data_root).await?;
|
||||
|
||||
info!("Uploading to backend...");
|
||||
|
||||
let result = upload_image_media_file_from_file(UploadImageFromFileArgs {
|
||||
api_host: &app_env_configs.storyteller_host,
|
||||
maybe_creds: Some(&creds),
|
||||
path: download_path,
|
||||
is_intermediate_system_file: false,
|
||||
maybe_prompt_token: Some(&prompt_response.prompt_token),
|
||||
maybe_batch_token: None, // TODO: This should be added soon.
|
||||
}).await?;
|
||||
|
||||
info!("Uploaded to API backend: {:?}", result.media_file_token);
|
||||
|
||||
let event = GenerationCompleteEvent {
|
||||
//media_file_token: result.media_file_token,
|
||||
action: Some(GenerationAction::GenerateImage),
|
||||
service: GenerationServiceProvider::Sora,
|
||||
model: None,
|
||||
};
|
||||
|
||||
if let Err(err) = event.send(&app_handle) {
|
||||
error!("Failed to send GenerationCompleteEvent: {:?}", err); // Fail open
|
||||
}
|
||||
}
|
||||
event.send_infallible(&app_handle);
|
||||
}
|
||||
|
||||
let succeeded_task_ids : Vec<&TaskId> = succeeded_tasks
|
||||
.iter()
|
||||
.map(|task| &task.id)
|
||||
.collect();
|
||||
// Clear from SQLite task database.
|
||||
if let Some(local_task) = local_tasks_by_sora_task_id.get(task_id.as_str()) {
|
||||
info!("Marking local task as failed: {:?}", local_task.id);
|
||||
|
||||
sora_task_queue.remove_list(&succeeded_task_ids)?;
|
||||
|
||||
tokio::time::sleep(std::time::Duration::from_millis(6_000)).await;
|
||||
let _updated = update_task_status(UpdateTaskArgs {
|
||||
db: task_database.get_connection(),
|
||||
task_id: &local_task.id,
|
||||
status: task_status::TaskStatus::CompleteFailure,
|
||||
}).await?;
|
||||
}
|
||||
}
|
||||
|
||||
// Clear from in memory DB
|
||||
// TODO: Remove the in-memory queue in favor of SQLite only.
|
||||
let failed_task_ids: Vec<&TaskId> = failed_tasks_by_id.keys().collect();
|
||||
sora_task_queue.remove_list(&failed_task_ids)?;
|
||||
|
||||
|
||||
// Process succeeded tasks.
|
||||
|
||||
for (task_id, task) in succeeded_tasks_by_id.iter() {
|
||||
if !sora_task_queue.contains_key(task_id)? {
|
||||
// TODO: Remove once the in-memory queue is gone.
|
||||
continue;
|
||||
}
|
||||
|
||||
info!("Task succeeded: {:?}", task.id);
|
||||
|
||||
let request = CreatePromptRequest {
|
||||
uuid_idempotency_token: generate_random_uuid(),
|
||||
positive_prompt: task.prompt.clone(),
|
||||
negative_prompt: None,
|
||||
model_type: Some(ModelType::GptImage1),
|
||||
generation_provider: Some(GenerationProvider::Sora),
|
||||
};
|
||||
|
||||
let prompt_response = create_prompt(
|
||||
&app_env_configs.storyteller_host,
|
||||
Some(&storyteller_creds),
|
||||
request
|
||||
).await?;
|
||||
|
||||
info!("Created prompt: {:?}", &prompt_response.prompt_token);
|
||||
|
||||
for (i, generation) in task.generations.iter().enumerate() {
|
||||
info!("Downloading generated file...");
|
||||
let download_path = download_generation(generation, &app_data_root).await?;
|
||||
|
||||
info!("Uploading to backend...");
|
||||
|
||||
let result = upload_image_media_file_from_file(UploadImageFromFileArgs {
|
||||
api_host: &app_env_configs.storyteller_host,
|
||||
maybe_creds: Some(&storyteller_creds),
|
||||
path: download_path,
|
||||
is_intermediate_system_file: false,
|
||||
maybe_prompt_token: Some(&prompt_response.prompt_token),
|
||||
maybe_batch_token: None, // TODO: This should be added soon.
|
||||
}).await?;
|
||||
|
||||
info!("Uploaded to API backend: {:?}", result.media_file_token);
|
||||
|
||||
|
||||
// Clear from SQLite task database.
|
||||
if let Some(local_task) = local_tasks_by_sora_task_id.get(task_id.as_str()) {
|
||||
info!("Marking local task as failed: {:?}", local_task.id);
|
||||
|
||||
let updated = update_task_status(UpdateTaskArgs {
|
||||
db: task_database.get_connection(),
|
||||
task_id: &local_task.id,
|
||||
status: task_status::TaskStatus::CompleteSuccess,
|
||||
}).await?;
|
||||
|
||||
// if !updated {
|
||||
// return Ok(()); // If anything breaks with queries, don't spam events.
|
||||
// }
|
||||
}
|
||||
|
||||
let event = GenerationCompleteEvent {
|
||||
//media_file_token: result.media_file_token,
|
||||
action: Some(GenerationAction::GenerateImage),
|
||||
service: GenerationServiceProvider::Sora,
|
||||
model: None,
|
||||
};
|
||||
|
||||
event.send_infallible(&app_handle);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
tokio::time::sleep(std::time::Duration::from_millis(3_000)).await;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn download_generation(generation: &Generation, app_data_root: &AppDataRoot) -> AnyhowResult<PathBuf> {
|
||||
|
||||
Reference in new issue
Block a user