From 24de2f3275bfa6b751693e82ed5425b22f72fe0f Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Tue, 10 Mar 2026 08:06:08 -0400 Subject: [PATCH] poll and download results --- .../worldlabs/list_pending_worldlabs_jobs.rs | 98 +++++++++++++ .../generic_inference/worldlabs/mod.rs | 1 + .../job/worldlabs_job/src/main_loop.rs | 76 ++++++++++- .../src/process_job/process_failed_job.rs | 114 +++++++++++++++- .../src/process_job/process_successful_job.rs | 129 +++++++++++++++++- 5 files changed, 409 insertions(+), 9 deletions(-) create mode 100644 crates/schema/database/mysql_queries/src/queries/generic_inference/worldlabs/list_pending_worldlabs_jobs.rs diff --git a/crates/schema/database/mysql_queries/src/queries/generic_inference/worldlabs/list_pending_worldlabs_jobs.rs b/crates/schema/database/mysql_queries/src/queries/generic_inference/worldlabs/list_pending_worldlabs_jobs.rs new file mode 100644 index 0000000000..e3e9899e40 --- /dev/null +++ b/crates/schema/database/mysql_queries/src/queries/generic_inference/worldlabs/list_pending_worldlabs_jobs.rs @@ -0,0 +1,98 @@ +use anyhow::anyhow; +use log::warn; +use sqlx::MySqlPool; + +use enums::by_table::generic_inference_jobs::inference_job_external_third_party::InferenceJobExternalThirdParty; +use enums::common::visibility::Visibility; +use errors::AnyhowResult; +use tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken; +use tokens::tokens::generic_inference_jobs::InferenceJobToken; +use tokens::tokens::prompts::PromptToken; +use tokens::tokens::users::UserToken; +use tokens::tokens::wallet_ledger_entries::WalletLedgerEntryToken; + +/// A WorldLabs job that is waiting for a result from the external API. +#[derive(Debug)] +pub struct PendingWorldlabsJob { + pub job_token: InferenceJobToken, + + /// The external operation_id from World Labs. + pub operation_id: String, + + pub maybe_creator_user_token: Option, + pub maybe_creator_anonymous_visitor_token: Option, + pub creator_ip_address: String, + pub creator_set_visibility: Visibility, + + pub maybe_prompt_token: Option, + + pub maybe_wallet_ledger_entry_token: Option, +} + +#[derive(Debug, Default)] +struct RawRecord { + job_token: InferenceJobToken, + operation_id: Option, + maybe_creator_user_token: Option, + maybe_creator_anonymous_visitor_token: Option, + creator_ip_address: String, + creator_set_visibility: Visibility, + maybe_prompt_token: Option, + maybe_wallet_ledger_entry_token: Option, +} + +/// Returns all non-terminal WorldLabs jobs that have an associated operation_id. +pub async fn list_pending_worldlabs_jobs(pool: &MySqlPool) -> AnyhowResult> { + let records = sqlx::query_as!( + RawRecord, + r#" +SELECT + jobs.token as `job_token: tokens::tokens::generic_inference_jobs::InferenceJobToken`, + jobs.maybe_external_third_party_id as `operation_id`, + jobs.maybe_creator_user_token as `maybe_creator_user_token: tokens::tokens::users::UserToken`, + jobs.maybe_creator_anonymous_visitor_token as `maybe_creator_anonymous_visitor_token: tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken`, + jobs.creator_ip_address, + jobs.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`, + jobs.maybe_prompt_token as `maybe_prompt_token: tokens::tokens::prompts::PromptToken`, + jobs.maybe_wallet_ledger_entry_token as `maybe_wallet_ledger_entry_token: tokens::tokens::wallet_ledger_entries::WalletLedgerEntryToken` + +FROM generic_inference_jobs as jobs + +WHERE jobs.maybe_external_third_party = ? + AND jobs.status NOT IN ('complete_success', 'complete_failure') + AND jobs.maybe_external_third_party_id IS NOT NULL + +LIMIT 25000 + "#, + InferenceJobExternalThirdParty::Worldlabs.to_str(), + ) + .fetch_all(pool) + .await + .map_err(|err| anyhow!("error querying pending worldlabs jobs: {:?}", err))?; + + let jobs = records + .into_iter() + .filter_map(|record| { + let operation_id = match record.operation_id { + Some(id) => id, + None => { + warn!("PendingWorldlabsJob has no operation_id, skipping"); + return None; + } + }; + + Some(PendingWorldlabsJob { + job_token: record.job_token, + operation_id, + maybe_creator_user_token: record.maybe_creator_user_token, + maybe_creator_anonymous_visitor_token: record.maybe_creator_anonymous_visitor_token, + creator_ip_address: record.creator_ip_address, + creator_set_visibility: record.creator_set_visibility, + maybe_prompt_token: record.maybe_prompt_token, + maybe_wallet_ledger_entry_token: record.maybe_wallet_ledger_entry_token, + }) + }) + .collect(); + + Ok(jobs) +} diff --git a/crates/schema/database/mysql_queries/src/queries/generic_inference/worldlabs/mod.rs b/crates/schema/database/mysql_queries/src/queries/generic_inference/worldlabs/mod.rs index 259d0e2f25..f013474c15 100644 --- a/crates/schema/database/mysql_queries/src/queries/generic_inference/worldlabs/mod.rs +++ b/crates/schema/database/mysql_queries/src/queries/generic_inference/worldlabs/mod.rs @@ -1 +1,2 @@ pub mod insert_generic_inference_job_for_worldlabs_queue_with_apriori_job_token; +pub mod list_pending_worldlabs_jobs; diff --git a/crates/service/job/worldlabs_job/src/main_loop.rs b/crates/service/job/worldlabs_job/src/main_loop.rs index b9e3e04856..c3d7dea107 100644 --- a/crates/service/job/worldlabs_job/src/main_loop.rs +++ b/crates/service/job/worldlabs_job/src/main_loop.rs @@ -1,7 +1,12 @@ use std::time::Duration; use log::{error, info, warn}; +use mysql_queries::queries::generic_inference::worldlabs::list_pending_worldlabs_jobs::list_pending_worldlabs_jobs; +use world_labs_api::api::api_types::operation_id::OperationId; +use world_labs_api::api::requests::get_operation::get_operation::{get_operation, GetOperationArgs}; +use crate::process_job::process_failed_job::process_failed_job; +use crate::process_job::process_successful_job::process_successful_job; use crate::job_dependencies::JobDependencies; pub async fn main_loop(job_dependencies: JobDependencies) { @@ -19,8 +24,73 @@ pub async fn main_loop(job_dependencies: JobDependencies) { warn!("WorldLabs job runner main loop is shut down."); } -async fn run_poll_iteration(_deps: &JobDependencies) -> anyhow::Result<()> { - // TODO: Query pending WorldLabs jobs from DB, poll their status, and process results. - info!("WorldLabs poll iteration (not yet implemented)."); +async fn run_poll_iteration(deps: &JobDependencies) -> anyhow::Result<()> { + // 1. Query all non-terminal WorldLabs jobs from DB. + let pending_jobs = list_pending_worldlabs_jobs(&deps.mysql_pool).await?; + + if pending_jobs.is_empty() { + info!("No pending WorldLabs jobs."); + return Ok(()); + } + + info!("Found {} pending WorldLabs job(s).", pending_jobs.len()); + + // 2. For each pending job, poll its operation status from the World Labs API. + for job in &pending_jobs { + let operation_id = OperationId(job.operation_id.clone()); + + let operation = match get_operation(GetOperationArgs { + creds: &deps.worldlabs_creds, + operation_id: &operation_id, + request_timeout: Some(Duration::from_secs(30)), + }).await { + Ok(op) => op, + Err(err) => { + warn!( + "Error polling WorldLabs operation {} for job {}: {:?}", + job.operation_id, job.job_token.as_str(), err + ); + continue; + } + }; + + // 3. Check if the operation has an error. + if let Some(ref op_error) = operation.error { + let reason = op_error.message.as_deref().unwrap_or("unknown error"); + info!( + "Operation {} failed: {}. Processing job {} as failed.", + job.operation_id, reason, job.job_token.as_str() + ); + process_failed_job(deps, job, reason).await; + continue; + } + + // 4. Check if the operation is done. + if !operation.done { + // Still in progress — check again next poll. + info!( + "Operation {} for job {} is still in progress.", + job.operation_id, job.job_token.as_str() + ); + continue; + } + + // 5. Operation is done — process the result. + info!( + "Operation {} completed, processing job {}.", + job.operation_id, job.job_token.as_str() + ); + + if let Err(err) = process_successful_job(deps, job, &operation).await { + warn!( + "Error processing completed operation {} for job {}: {:?}", + job.operation_id, job.job_token.as_str(), err + ); + let _ = deps.job_stats.increment_failure_count(); + } else { + let _ = deps.job_stats.increment_success_count(); + } + } + Ok(()) } diff --git a/crates/service/job/worldlabs_job/src/process_job/process_failed_job.rs b/crates/service/job/worldlabs_job/src/process_job/process_failed_job.rs index d884ce640c..76bd4976e8 100644 --- a/crates/service/job/worldlabs_job/src/process_job/process_failed_job.rs +++ b/crates/service/job/worldlabs_job/src/process_job/process_failed_job.rs @@ -1,8 +1,116 @@ +use log::{error, info, warn}; + +use enums::by_table::generic_inference_jobs::frontend_failure_category::FrontendFailureCategory; +use mysql_queries::queries::generic_inference::job::mark_job_failed_by_token::{mark_job_failed_by_token, MarkJobFailedByTokenArgs}; +use mysql_queries::queries::generic_inference::worldlabs::list_pending_worldlabs_jobs::PendingWorldlabsJob; +use mysql_queries::queries::wallets::refund::try_to_refund_ledger_entry::{try_to_refund_ledger_entry, WalletRefundOutcome}; + use crate::job_dependencies::JobDependencies; pub async fn process_failed_job( - _deps: &JobDependencies, + deps: &JobDependencies, + job: &PendingWorldlabsJob, + reason: &str, ) { - // TODO: Implement WorldLabs failed job handling (refund + mark failed). - todo!("WorldLabs process_failed_job not yet implemented") + // --- Step 1: Attempt the refund before touching the job status. --- + // + // We do this first so that a crash or error between the refund and the status update + // can never result in a failed job with no refund. If the refund itself fails we bail + // out early: the job stays pending and will be retried on the next poll cycle. + + match &job.maybe_wallet_ledger_entry_token { + None => { + warn!( + "Job {} has no wallet ledger entry token; skipping refund.", + job.job_token.as_str() + ); + } + Some(ledger_token) => { + let mut transaction = match deps.mysql_pool.begin().await { + Ok(tx) => tx, + Err(err) => { + error!( + "Failed to begin refund transaction for job {} (ledger {}): {:?}. \ + Job will NOT be marked failed yet and will be retried next poll.", + job.job_token.as_str(), ledger_token.as_str(), err + ); + return; + } + }; + + match try_to_refund_ledger_entry(ledger_token, &mut transaction).await { + Ok(WalletRefundOutcome::Refunded(summary)) => { + info!( + "Refunded {} credits for failed job {} (ledger {} → refund ledger {}).", + summary.refund_amount, + job.job_token.as_str(), + ledger_token.as_str(), + summary.refund_ledger_entry_token.as_str(), + ); + if let Err(err) = transaction.commit().await { + error!( + "Failed to commit refund transaction for job {} (ledger {}): {:?}. \ + Job will NOT be marked failed yet and will be retried next poll.", + job.job_token.as_str(), ledger_token.as_str(), err + ); + return; + } + } + Ok(WalletRefundOutcome::AlreadyRefunded) => { + info!( + "Ledger entry {} for job {} was already refunded; proceeding to mark job failed.", + ledger_token.as_str(), + job.job_token.as_str(), + ); + let _ = transaction.rollback().await; + } + Err(err) => { + error!( + "Failed to refund ledger entry {} for job {}: {:?}. \ + Job will NOT be marked failed yet and will be retried next poll.", + ledger_token.as_str(), + job.job_token.as_str(), + err, + ); + let _ = transaction.rollback().await; + return; + } + } + } + } + + // --- Step 2: Mark the job record as failed. --- + + let reason_lower = reason.to_lowercase(); + + let platform_rules_violation = reason_lower.contains("violates") || + reason_lower.contains("platform rules") || + reason_lower.contains("please modify"); + + let frontend_failure_category = if platform_rules_violation { + Some(FrontendFailureCategory::ModelRulesViolation) + } else { + None + }; + + warn!( + "Operation for job {} failed: {}. Marking job failed.", + job.job_token.as_str(), reason + ); + + let mark_failed_result = mark_job_failed_by_token(MarkJobFailedByTokenArgs { + pool: &deps.mysql_pool, + job_token: &job.job_token, + maybe_public_failure_reason: Some(reason), + internal_debugging_failure_reason: reason, + maybe_frontend_failure_category: frontend_failure_category, + }).await; + + if let Err(err) = mark_failed_result { + error!( + "Error marking job {} as failed: {:?}", + job.job_token.as_str(), + err + ); + } } diff --git a/crates/service/job/worldlabs_job/src/process_job/process_successful_job.rs b/crates/service/job/worldlabs_job/src/process_job/process_successful_job.rs index a0397cc27f..5c39adf270 100644 --- a/crates/service/job/worldlabs_job/src/process_job/process_successful_job.rs +++ b/crates/service/job/worldlabs_job/src/process_job/process_successful_job.rs @@ -1,10 +1,133 @@ +use anyhow::anyhow; +use log::{error, info}; + +use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath; +use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; +use enums::by_table::media_files::media_file_class::MediaFileClass; +use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory; +use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory; +use enums::by_table::media_files::media_file_type::MediaFileType; use errors::AnyhowResult; +use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes; +use mysql_queries::queries::generic_inference::worldlabs::list_pending_worldlabs_jobs::PendingWorldlabsJob; +use mysql_queries::queries::media_files::create::insert_builder::media_file_insert_builder::MediaFileInsertBuilder; +use mysql_queries::queries::generic_inference::web::mark_generic_inference_job_successfully_done_by_token::mark_generic_inference_job_successfully_done_by_token; +use world_labs_api::api::requests::get_operation::get_operation::GetOperationResponse; use crate::job_dependencies::JobDependencies; +const PREFIX: &str = "artcraft_"; +const SUFFIX: &str = ".spz"; + /// Download the completed splat, upload to bucket, create media file record, and mark job done. pub async fn process_successful_job( - _deps: &JobDependencies, + deps: &JobDependencies, + job: &PendingWorldlabsJob, + operation: &GetOperationResponse, ) -> AnyhowResult<()> { - // TODO: Implement WorldLabs splat download and processing. - todo!("WorldLabs process_successful_job not yet implemented") + // Get the full-res splat URL from the completed world. + let world = operation.world.as_ref() + .ok_or_else(|| anyhow!("Completed operation {} has no world object", operation.operation_id))?; + + let splat_url = world.assets.as_ref() + .and_then(|a| a.splats.as_ref()) + .and_then(|s| s.spz_url_full_res.as_ref()) + .ok_or_else(|| anyhow!( + "Completed operation {} has no spz_url_full_res", + operation.operation_id + ))?; + + info!( + "Downloading splat for operation {} from: {}", + operation.operation_id, splat_url + ); + + // Download the splat bytes. + let splat_bytes: Vec = reqwest::get(splat_url) + .await + .map_err(|err| anyhow!("reqwest error downloading splat: {:?}", err))? + .bytes() + .await + .map_err(|err| anyhow!("error reading splat bytes: {:?}", err))? + .to_vec(); + + info!( + "Downloaded {} bytes for operation {}", + splat_bytes.len(), + operation.operation_id + ); + + // Hash the splat. + let checksum = sha256_hash_bytes(&splat_bytes) + .map_err(|err| anyhow!("error hashing splat: {:?}", err))?; + + // Build the bucket path. + let bucket_path = MediaFileBucketPath::generate_new(Some(PREFIX), Some(SUFFIX)); + + let object_path = bucket_path.get_full_object_path_str(); + + info!( + "Uploading splat to public bucket at path: {}", + object_path + ); + + // Upload to public bucket. + deps + .public_bucket_client + .upload_file_with_content_type_process(object_path, &splat_bytes, "application/gzip") + .await + .map_err(|err| anyhow!("error uploading splat to bucket: {:?}", err))?; + + info!( + "Uploaded splat for operation {}. Creating media file record.", + operation.operation_id + ); + + // Insert media file record. + let media_file_token = MediaFileInsertBuilder::new() + .maybe_creator_user(job.maybe_creator_user_token.as_ref()) + .maybe_creator_anonymous_visitor(job.maybe_creator_anonymous_visitor_token.as_ref()) + .creator_ip_address(&job.creator_ip_address) + .creator_set_visibility(job.creator_set_visibility) + .media_file_class(MediaFileClass::Dimensional) + .media_file_type(MediaFileType::Spz) + .media_file_origin_category(MediaFileOriginCategory::Inference) + .media_file_origin_product_category(MediaFileOriginProductCategory::ImageGeneration) + .mime_type("application/gzip") + .file_size_bytes(splat_bytes.len() as u64) + .checksum_sha2(&checksum) + .maybe_prompt_token(job.maybe_prompt_token.as_ref()) + .public_bucket_directory_hash(&bucket_path) + .insert_pool(&deps.mysql_pool) + .await + .map_err(|err| anyhow!("error inserting media file record: {:?}", err))?; + + info!( + "Created media file {} for operation {}. Marking job {} complete.", + media_file_token.as_str(), + operation.operation_id, + job.job_token.as_str() + ); + + // Mark inference job as successfully completed. + mark_generic_inference_job_successfully_done_by_token( + &deps.mysql_pool, + &job.job_token, + Some(InferenceResultType::MediaFile), + Some(media_file_token.as_str()), + None, + None, + ) + .await + .map_err(|err| { + error!( + "Error marking job {} done: {:?}", + job.job_token.as_str(), + err + ); + anyhow!("error marking job done: {:?}", err) + })?; + + info!("Job {} completed successfully.", job.job_token.as_str()); + + Ok(()) }