poll and download results

This commit is contained in:
Brandon Thomas committed 2026-03-10 08:06:08 -04:00
1 parent a07127fee0
commit 24de2f3275
5 files changed
+409 -9

No files matched your search

@@ -0,0 +1,98 @@
use anyhow::anyhow;
use log::warn;
use sqlx::MySqlPool;
use enums::by_table::generic_inference_jobs::inference_job_external_third_party::InferenceJobExternalThirdParty;
use enums::common::visibility::Visibility;
use errors::AnyhowResult;
use tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken;
use tokens::tokens::generic_inference_jobs::InferenceJobToken;
use tokens::tokens::prompts::PromptToken;
use tokens::tokens::users::UserToken;
use tokens::tokens::wallet_ledger_entries::WalletLedgerEntryToken;
/// A WorldLabs job that is waiting for a result from the external API.
#[derive(Debug)]
pub struct PendingWorldlabsJob {
pub job_token: InferenceJobToken,
/// The external operation_id from World Labs.
pub operation_id: String,
pub maybe_creator_user_token: Option<UserToken>,
pub maybe_creator_anonymous_visitor_token: Option<AnonymousVisitorTrackingToken>,
pub creator_ip_address: String,
pub creator_set_visibility: Visibility,
pub maybe_prompt_token: Option<PromptToken>,
pub maybe_wallet_ledger_entry_token: Option<WalletLedgerEntryToken>,
}
#[derive(Debug, Default)]
struct RawRecord {
job_token: InferenceJobToken,
operation_id: Option<String>,
maybe_creator_user_token: Option<UserToken>,
maybe_creator_anonymous_visitor_token: Option<AnonymousVisitorTrackingToken>,
creator_ip_address: String,
creator_set_visibility: Visibility,
maybe_prompt_token: Option<PromptToken>,
maybe_wallet_ledger_entry_token: Option<WalletLedgerEntryToken>,
}
/// Returns all non-terminal WorldLabs jobs that have an associated operation_id.
pub async fn list_pending_worldlabs_jobs(pool: &MySqlPool) -> AnyhowResult<Vec<PendingWorldlabsJob>> {
let records = sqlx::query_as!(
RawRecord,
r#"
SELECT
jobs.token as `job_token: tokens::tokens::generic_inference_jobs::InferenceJobToken`,
jobs.maybe_external_third_party_id as `operation_id`,
jobs.maybe_creator_user_token as `maybe_creator_user_token: tokens::tokens::users::UserToken`,
jobs.maybe_creator_anonymous_visitor_token as `maybe_creator_anonymous_visitor_token: tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken`,
jobs.creator_ip_address,
jobs.creator_set_visibility as `creator_set_visibility: enums::common::visibility::Visibility`,
jobs.maybe_prompt_token as `maybe_prompt_token: tokens::tokens::prompts::PromptToken`,
jobs.maybe_wallet_ledger_entry_token as `maybe_wallet_ledger_entry_token: tokens::tokens::wallet_ledger_entries::WalletLedgerEntryToken`
FROM generic_inference_jobs as jobs
WHERE jobs.maybe_external_third_party = ?
AND jobs.status NOT IN ('complete_success', 'complete_failure')
AND jobs.maybe_external_third_party_id IS NOT NULL
LIMIT 25000
"#,
InferenceJobExternalThirdParty::Worldlabs.to_str(),
)
.fetch_all(pool)
.await
.map_err(|err| anyhow!("error querying pending worldlabs jobs: {:?}", err))?;
let jobs = records
.into_iter()
.filter_map(|record| {
let operation_id = match record.operation_id {
Some(id) => id,
None => {
warn!("PendingWorldlabsJob has no operation_id, skipping");
return None;
}
};
Some(PendingWorldlabsJob {
job_token: record.job_token,
operation_id,
maybe_creator_user_token: record.maybe_creator_user_token,
maybe_creator_anonymous_visitor_token: record.maybe_creator_anonymous_visitor_token,
creator_ip_address: record.creator_ip_address,
creator_set_visibility: record.creator_set_visibility,
maybe_prompt_token: record.maybe_prompt_token,
maybe_wallet_ledger_entry_token: record.maybe_wallet_ledger_entry_token,
})
})
.collect();
Ok(jobs)
}
@@ -1 +1,2 @@
pub mod insert_generic_inference_job_for_worldlabs_queue_with_apriori_job_token;
pub mod list_pending_worldlabs_jobs;
@@ -1,7 +1,12 @@
use std::time::Duration;
use log::{error, info, warn};
use mysql_queries::queries::generic_inference::worldlabs::list_pending_worldlabs_jobs::list_pending_worldlabs_jobs;
use world_labs_api::api::api_types::operation_id::OperationId;
use world_labs_api::api::requests::get_operation::get_operation::{get_operation, GetOperationArgs};
use crate::process_job::process_failed_job::process_failed_job;
use crate::process_job::process_successful_job::process_successful_job;
use crate::job_dependencies::JobDependencies;
pub async fn main_loop(job_dependencies: JobDependencies) {
@@ -19,8 +24,73 @@ pub async fn main_loop(job_dependencies: JobDependencies) {
warn!("WorldLabs job runner main loop is shut down.");
}
async fn run_poll_iteration(_deps: &JobDependencies) -> anyhow::Result<()> {
// TODO: Query pending WorldLabs jobs from DB, poll their status, and process results.
info!("WorldLabs poll iteration (not yet implemented).");
async fn run_poll_iteration(deps: &JobDependencies) -> anyhow::Result<()> {
// 1. Query all non-terminal WorldLabs jobs from DB.
let pending_jobs = list_pending_worldlabs_jobs(&deps.mysql_pool).await?;
if pending_jobs.is_empty() {
info!("No pending WorldLabs jobs.");
return Ok(());
}
info!("Found {} pending WorldLabs job(s).", pending_jobs.len());
// 2. For each pending job, poll its operation status from the World Labs API.
for job in &pending_jobs {
let operation_id = OperationId(job.operation_id.clone());
let operation = match get_operation(GetOperationArgs {
creds: &deps.worldlabs_creds,
operation_id: &operation_id,
request_timeout: Some(Duration::from_secs(30)),
}).await {
Ok(op) => op,
Err(err) => {
warn!(
"Error polling WorldLabs operation {} for job {}: {:?}",
job.operation_id, job.job_token.as_str(), err
);
continue;
}
};
// 3. Check if the operation has an error.
if let Some(ref op_error) = operation.error {
let reason = op_error.message.as_deref().unwrap_or("unknown error");
info!(
"Operation {} failed: {}. Processing job {} as failed.",
job.operation_id, reason, job.job_token.as_str()
);
process_failed_job(deps, job, reason).await;
continue;
}
// 4. Check if the operation is done.
if !operation.done {
// Still in progress — check again next poll.
info!(
"Operation {} for job {} is still in progress.",
job.operation_id, job.job_token.as_str()
);
continue;
}
// 5. Operation is done — process the result.
info!(
"Operation {} completed, processing job {}.",
job.operation_id, job.job_token.as_str()
);
if let Err(err) = process_successful_job(deps, job, &operation).await {
warn!(
"Error processing completed operation {} for job {}: {:?}",
job.operation_id, job.job_token.as_str(), err
);
let _ = deps.job_stats.increment_failure_count();
} else {
let _ = deps.job_stats.increment_success_count();
}
}
Ok(())
}
@@ -1,8 +1,116 @@
use log::{error, info, warn};
use enums::by_table::generic_inference_jobs::frontend_failure_category::FrontendFailureCategory;
use mysql_queries::queries::generic_inference::job::mark_job_failed_by_token::{mark_job_failed_by_token, MarkJobFailedByTokenArgs};
use mysql_queries::queries::generic_inference::worldlabs::list_pending_worldlabs_jobs::PendingWorldlabsJob;
use mysql_queries::queries::wallets::refund::try_to_refund_ledger_entry::{try_to_refund_ledger_entry, WalletRefundOutcome};
use crate::job_dependencies::JobDependencies;
pub async fn process_failed_job(
_deps: &JobDependencies,
deps: &JobDependencies,
job: &PendingWorldlabsJob,
reason: &str,
) {
// TODO: Implement WorldLabs failed job handling (refund + mark failed).
todo!("WorldLabs process_failed_job not yet implemented")
// --- Step 1: Attempt the refund before touching the job status. ---
//
// We do this first so that a crash or error between the refund and the status update
// can never result in a failed job with no refund. If the refund itself fails we bail
// out early: the job stays pending and will be retried on the next poll cycle.
match &job.maybe_wallet_ledger_entry_token {
None => {
warn!(
"Job {} has no wallet ledger entry token; skipping refund.",
job.job_token.as_str()
);
}
Some(ledger_token) => {
let mut transaction = match deps.mysql_pool.begin().await {
Ok(tx) => tx,
Err(err) => {
error!(
"Failed to begin refund transaction for job {} (ledger {}): {:?}. \
Job will NOT be marked failed yet and will be retried next poll.",
job.job_token.as_str(), ledger_token.as_str(), err
);
return;
}
};
match try_to_refund_ledger_entry(ledger_token, &mut transaction).await {
Ok(WalletRefundOutcome::Refunded(summary)) => {
info!(
"Refunded {} credits for failed job {} (ledger {} → refund ledger {}).",
summary.refund_amount,
job.job_token.as_str(),
ledger_token.as_str(),
summary.refund_ledger_entry_token.as_str(),
);
if let Err(err) = transaction.commit().await {
error!(
"Failed to commit refund transaction for job {} (ledger {}): {:?}. \
Job will NOT be marked failed yet and will be retried next poll.",
job.job_token.as_str(), ledger_token.as_str(), err
);
return;
}
}
Ok(WalletRefundOutcome::AlreadyRefunded) => {
info!(
"Ledger entry {} for job {} was already refunded; proceeding to mark job failed.",
ledger_token.as_str(),
job.job_token.as_str(),
);
let _ = transaction.rollback().await;
}
Err(err) => {
error!(
"Failed to refund ledger entry {} for job {}: {:?}. \
Job will NOT be marked failed yet and will be retried next poll.",
ledger_token.as_str(),
job.job_token.as_str(),
err,
);
let _ = transaction.rollback().await;
return;
}
}
}
}
// --- Step 2: Mark the job record as failed. ---
let reason_lower = reason.to_lowercase();
let platform_rules_violation = reason_lower.contains("violates") ||
reason_lower.contains("platform rules") ||
reason_lower.contains("please modify");
let frontend_failure_category = if platform_rules_violation {
Some(FrontendFailureCategory::ModelRulesViolation)
} else {
None
};
warn!(
"Operation for job {} failed: {}. Marking job failed.",
job.job_token.as_str(), reason
);
let mark_failed_result = mark_job_failed_by_token(MarkJobFailedByTokenArgs {
pool: &deps.mysql_pool,
job_token: &job.job_token,
maybe_public_failure_reason: Some(reason),
internal_debugging_failure_reason: reason,
maybe_frontend_failure_category: frontend_failure_category,
}).await;
if let Err(err) = mark_failed_result {
error!(
"Error marking job {} as failed: {:?}",
job.job_token.as_str(),
err
);
}
}
@@ -1,10 +1,133 @@
use anyhow::anyhow;
use log::{error, info};
use bucket_paths::legacy::typified_paths::public::media_files::bucket_file_path::MediaFileBucketPath;
use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
use enums::by_table::media_files::media_file_class::MediaFileClass;
use enums::by_table::media_files::media_file_origin_category::MediaFileOriginCategory;
use enums::by_table::media_files::media_file_origin_product_category::MediaFileOriginProductCategory;
use enums::by_table::media_files::media_file_type::MediaFileType;
use errors::AnyhowResult;
use hashing::sha256::sha256_hash_bytes::sha256_hash_bytes;
use mysql_queries::queries::generic_inference::worldlabs::list_pending_worldlabs_jobs::PendingWorldlabsJob;
use mysql_queries::queries::media_files::create::insert_builder::media_file_insert_builder::MediaFileInsertBuilder;
use mysql_queries::queries::generic_inference::web::mark_generic_inference_job_successfully_done_by_token::mark_generic_inference_job_successfully_done_by_token;
use world_labs_api::api::requests::get_operation::get_operation::GetOperationResponse;
use crate::job_dependencies::JobDependencies;
const PREFIX: &str = "artcraft_";
const SUFFIX: &str = ".spz";
/// Download the completed splat, upload to bucket, create media file record, and mark job done.
pub async fn process_successful_job(
_deps: &JobDependencies,
deps: &JobDependencies,
job: &PendingWorldlabsJob,
operation: &GetOperationResponse,
) -> AnyhowResult<()> {
// TODO: Implement WorldLabs splat download and processing.
todo!("WorldLabs process_successful_job not yet implemented")
// Get the full-res splat URL from the completed world.
let world = operation.world.as_ref()
.ok_or_else(|| anyhow!("Completed operation {} has no world object", operation.operation_id))?;
let splat_url = world.assets.as_ref()
.and_then(|a| a.splats.as_ref())
.and_then(|s| s.spz_url_full_res.as_ref())
.ok_or_else(|| anyhow!(
"Completed operation {} has no spz_url_full_res",
operation.operation_id
))?;
info!(
"Downloading splat for operation {} from: {}",
operation.operation_id, splat_url
);
// Download the splat bytes.
let splat_bytes: Vec<u8> = reqwest::get(splat_url)
.await
.map_err(|err| anyhow!("reqwest error downloading splat: {:?}", err))?
.bytes()
.await
.map_err(|err| anyhow!("error reading splat bytes: {:?}", err))?
.to_vec();
info!(
"Downloaded {} bytes for operation {}",
splat_bytes.len(),
operation.operation_id
);
// Hash the splat.
let checksum = sha256_hash_bytes(&splat_bytes)
.map_err(|err| anyhow!("error hashing splat: {:?}", err))?;
// Build the bucket path.
let bucket_path = MediaFileBucketPath::generate_new(Some(PREFIX), Some(SUFFIX));
let object_path = bucket_path.get_full_object_path_str();
info!(
"Uploading splat to public bucket at path: {}",
object_path
);
// Upload to public bucket.
deps
.public_bucket_client
.upload_file_with_content_type_process(object_path, &splat_bytes, "application/gzip")
.await
.map_err(|err| anyhow!("error uploading splat to bucket: {:?}", err))?;
info!(
"Uploaded splat for operation {}. Creating media file record.",
operation.operation_id
);
// Insert media file record.
let media_file_token = MediaFileInsertBuilder::new()
.maybe_creator_user(job.maybe_creator_user_token.as_ref())
.maybe_creator_anonymous_visitor(job.maybe_creator_anonymous_visitor_token.as_ref())
.creator_ip_address(&job.creator_ip_address)
.creator_set_visibility(job.creator_set_visibility)
.media_file_class(MediaFileClass::Dimensional)
.media_file_type(MediaFileType::Spz)
.media_file_origin_category(MediaFileOriginCategory::Inference)
.media_file_origin_product_category(MediaFileOriginProductCategory::ImageGeneration)
.mime_type("application/gzip")
.file_size_bytes(splat_bytes.len() as u64)
.checksum_sha2(&checksum)
.maybe_prompt_token(job.maybe_prompt_token.as_ref())
.public_bucket_directory_hash(&bucket_path)
.insert_pool(&deps.mysql_pool)
.await
.map_err(|err| anyhow!("error inserting media file record: {:?}", err))?;
info!(
"Created media file {} for operation {}. Marking job {} complete.",
media_file_token.as_str(),
operation.operation_id,
job.job_token.as_str()
);
// Mark inference job as successfully completed.
mark_generic_inference_job_successfully_done_by_token(
&deps.mysql_pool,
&job.job_token,
Some(InferenceResultType::MediaFile),
Some(media_file_token.as_str()),
None,
None,
)
.await
.map_err(|err| {
error!(
"Error marking job {} done: {:?}",
job.job_token.as_str(),
err
);
anyhow!("error marking job done: {:?}", err)
})?;
info!("Job {} completed successfully.", job.job_token.as_str());
Ok(())
}