diff --git a/Cargo.lock b/Cargo.lock index 1c2f68a8d1..6fe0d05ec7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10172,6 +10172,7 @@ dependencies = [ "anyhow", "bootstrap", "bucket_paths", + "chrono", "cloud_storage", "concurrency", "easyenv", diff --git a/crates/service/job/seedance2_pro_job/Cargo.toml b/crates/service/job/seedance2_pro_job/Cargo.toml index 7f22c4e4b4..81cf23d945 100644 --- a/crates/service/job/seedance2_pro_job/Cargo.toml +++ b/crates/service/job/seedance2_pro_job/Cargo.toml @@ -41,6 +41,7 @@ tokens.workspace = true # External anyhow.workspace = true +chrono.workspace = true log.workspace = true sqlx = { version = "0.7.4", features = [ "mysql", "runtime-tokio-rustls", "chrono" ] } diff --git a/crates/service/job/seedance2_pro_job/src/job_dependencies.rs b/crates/service/job/seedance2_pro_job/src/job_dependencies.rs index b2c17edbcb..4554b3c3fc 100644 --- a/crates/service/job/seedance2_pro_job/src/job_dependencies.rs +++ b/crates/service/job/seedance2_pro_job/src/job_dependencies.rs @@ -1,3 +1,4 @@ +use chrono::Duration; use cloud_storage::bucket_client::BucketClient; use concurrency::relaxed_atomic_bool::RelaxedAtomicBool; use jobs_common::job_stats::JobStats; @@ -28,6 +29,11 @@ pub struct JobDependencies { /// when the order list is very long. pub maybe_pages_per_batch: Option, + /// If set, stop paginating backwards through orders once we encounter + /// an order older than this duration. This prevents endlessly scanning + /// ancient orders that will never match a pending job. + pub maybe_max_job_age: Option, + /// Set to `true` from another thread to trigger graceful shutdown. pub application_shutdown: RelaxedAtomicBool, diff --git a/crates/service/job/seedance2_pro_job/src/jobs/video_polling_job/video_polling_main_loop.rs b/crates/service/job/seedance2_pro_job/src/jobs/video_polling_job/video_polling_main_loop.rs index 0fe606023c..f40ee15168 100644 --- a/crates/service/job/seedance2_pro_job/src/jobs/video_polling_job/video_polling_main_loop.rs +++ b/crates/service/job/seedance2_pro_job/src/jobs/video_polling_job/video_polling_main_loop.rs @@ -1,6 +1,7 @@ use std::collections::HashMap; use std::time::{Duration, Instant}; +use chrono::Utc; use log::{error, info, warn}; use pager::notification::notification_details_builder::NotificationDetailsBuilder; use pager::notification::notification_urgency::NotificationUrgency; @@ -117,10 +118,32 @@ async fn run_poll_iteration(deps: &JobDependencies) -> anyhow::Result<()> { info!("Done polling page {}. Got {} orders on this page.", total_page_number, page_count); total_orders_seen += page_count; + + // Check if the last (oldest) order in this page exceeds the max age threshold. + // Orders are returned newest-first, so the last order is the oldest. + let exceeded_max_age = if let Some(ref max_age) = deps.maybe_max_job_age { + response.orders.last() + .and_then(|order| order.created_at_utc) + .map(|created_at| { + let age = Utc::now() - created_at; + let too_old = age > *max_age; + if too_old { + info!( + "Last order on page {} is {} days old (threshold: {} days). Stopping pagination.", + total_page_number, age.num_days(), max_age.num_days() + ); + } + too_old + }) + .unwrap_or(false) + } else { + false + }; + batch_orders.extend(response.orders); cursor = response.next_cursor; - let reached_end = cursor.is_none(); + let reached_end = cursor.is_none() || exceeded_max_age; // Determine if we should process the current batch now. let should_process_batch = reached_end diff --git a/crates/service/job/seedance2_pro_job/src/main.rs b/crates/service/job/seedance2_pro_job/src/main.rs index 457603930f..13d5af636a 100644 --- a/crates/service/job/seedance2_pro_job/src/main.rs +++ b/crates/service/job/seedance2_pro_job/src/main.rs @@ -113,6 +113,13 @@ async fn main() -> AnyhowResult<()> { info!("Batch page count: {}", count); } + let maybe_max_job_age = easyenv::try_get_env_num_optional::("MAX_JOB_AGE_THRESHOLD_DAYS")? + .and_then(chrono::Duration::try_days); + + if let Some(ref duration) = maybe_max_job_age { + info!("Max job age threshold: {} days", duration.num_days()); + } + let (pager, pager_worker) = build_pager(server_environment, &container_environment.hostname); info!("Spawning pager worker."); @@ -144,6 +151,7 @@ async fn main() -> AnyhowResult<()> { job_stats, poll_interval_millis, maybe_pages_per_batch, + maybe_max_job_age, application_shutdown: application_shutdown.clone(), pager, };