polling age threshold

This commit is contained in:
Brandon Thomas committed 2026-04-12 06:07:11 -04:00
1 parent 0fa0fbe97e
commit 225403c3d6
5 files changed
+40 -1

No files matched your search

Generated
+1
View File
@@ -10172,6 +10172,7 @@ dependencies = [
"anyhow",
"bootstrap",
"bucket_paths",
"chrono",
"cloud_storage",
"concurrency",
"easyenv",
@@ -41,6 +41,7 @@ tokens.workspace = true
# External
anyhow.workspace = true
chrono.workspace = true
log.workspace = true
sqlx = { version = "0.7.4", features = [ "mysql", "runtime-tokio-rustls", "chrono" ] }
@@ -1,3 +1,4 @@
use chrono::Duration;
use cloud_storage::bucket_client::BucketClient;
use concurrency::relaxed_atomic_bool::RelaxedAtomicBool;
use jobs_common::job_stats::JobStats;
@@ -28,6 +29,11 @@ pub struct JobDependencies {
/// when the order list is very long.
pub maybe_pages_per_batch: Option<u32>,
/// If set, stop paginating backwards through orders once we encounter
/// an order older than this duration. This prevents endlessly scanning
/// ancient orders that will never match a pending job.
pub maybe_max_job_age: Option<Duration>,
/// Set to `true` from another thread to trigger graceful shutdown.
pub application_shutdown: RelaxedAtomicBool,
@@ -1,6 +1,7 @@
use std::collections::HashMap;
use std::time::{Duration, Instant};
use chrono::Utc;
use log::{error, info, warn};
use pager::notification::notification_details_builder::NotificationDetailsBuilder;
use pager::notification::notification_urgency::NotificationUrgency;
@@ -117,10 +118,32 @@ async fn run_poll_iteration(deps: &JobDependencies) -> anyhow::Result<()> {
info!("Done polling page {}. Got {} orders on this page.", total_page_number, page_count);
total_orders_seen += page_count;
// Check if the last (oldest) order in this page exceeds the max age threshold.
// Orders are returned newest-first, so the last order is the oldest.
let exceeded_max_age = if let Some(ref max_age) = deps.maybe_max_job_age {
response.orders.last()
.and_then(|order| order.created_at_utc)
.map(|created_at| {
let age = Utc::now() - created_at;
let too_old = age > *max_age;
if too_old {
info!(
"Last order on page {} is {} days old (threshold: {} days). Stopping pagination.",
total_page_number, age.num_days(), max_age.num_days()
);
}
too_old
})
.unwrap_or(false)
} else {
false
};
batch_orders.extend(response.orders);
cursor = response.next_cursor;
let reached_end = cursor.is_none();
let reached_end = cursor.is_none() || exceeded_max_age;
// Determine if we should process the current batch now.
let should_process_batch = reached_end
@@ -113,6 +113,13 @@ async fn main() -> AnyhowResult<()> {
info!("Batch page count: {}", count);
}
let maybe_max_job_age = easyenv::try_get_env_num_optional::<i64>("MAX_JOB_AGE_THRESHOLD_DAYS")?
.and_then(chrono::Duration::try_days);
if let Some(ref duration) = maybe_max_job_age {
info!("Max job age threshold: {} days", duration.num_days());
}
let (pager, pager_worker) = build_pager(server_environment, &container_environment.hostname);
info!("Spawning pager worker.");
@@ -144,6 +151,7 @@ async fn main() -> AnyhowResult<()> {
job_stats,
poll_interval_millis,
maybe_pages_per_batch,
maybe_max_job_age,
application_shutdown: application_shutdown.clone(),
pager,
};