mirror of
https://github.com/storytold/artcraft.git
synced 2026-11-18 23:55:16 +00:00
polling age threshold
This commit is contained in:
1 parent
0fa0fbe97e
commit
225403c3d6
5 files changed
+40
-1
No files matched your search
Generated
+1
@@ -10172,6 +10172,7 @@ dependencies = [
|
||||
"anyhow",
|
||||
"bootstrap",
|
||||
"bucket_paths",
|
||||
"chrono",
|
||||
"cloud_storage",
|
||||
"concurrency",
|
||||
"easyenv",
|
||||
|
||||
@@ -41,6 +41,7 @@ tokens.workspace = true
|
||||
|
||||
# External
|
||||
anyhow.workspace = true
|
||||
chrono.workspace = true
|
||||
log.workspace = true
|
||||
|
||||
sqlx = { version = "0.7.4", features = [ "mysql", "runtime-tokio-rustls", "chrono" ] }
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
use chrono::Duration;
|
||||
use cloud_storage::bucket_client::BucketClient;
|
||||
use concurrency::relaxed_atomic_bool::RelaxedAtomicBool;
|
||||
use jobs_common::job_stats::JobStats;
|
||||
@@ -28,6 +29,11 @@ pub struct JobDependencies {
|
||||
/// when the order list is very long.
|
||||
pub maybe_pages_per_batch: Option<u32>,
|
||||
|
||||
/// If set, stop paginating backwards through orders once we encounter
|
||||
/// an order older than this duration. This prevents endlessly scanning
|
||||
/// ancient orders that will never match a pending job.
|
||||
pub maybe_max_job_age: Option<Duration>,
|
||||
|
||||
/// Set to `true` from another thread to trigger graceful shutdown.
|
||||
pub application_shutdown: RelaxedAtomicBool,
|
||||
|
||||
|
||||
+24
-1
@@ -1,6 +1,7 @@
|
||||
use std::collections::HashMap;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use chrono::Utc;
|
||||
use log::{error, info, warn};
|
||||
use pager::notification::notification_details_builder::NotificationDetailsBuilder;
|
||||
use pager::notification::notification_urgency::NotificationUrgency;
|
||||
@@ -117,10 +118,32 @@ async fn run_poll_iteration(deps: &JobDependencies) -> anyhow::Result<()> {
|
||||
info!("Done polling page {}. Got {} orders on this page.", total_page_number, page_count);
|
||||
|
||||
total_orders_seen += page_count;
|
||||
|
||||
// Check if the last (oldest) order in this page exceeds the max age threshold.
|
||||
// Orders are returned newest-first, so the last order is the oldest.
|
||||
let exceeded_max_age = if let Some(ref max_age) = deps.maybe_max_job_age {
|
||||
response.orders.last()
|
||||
.and_then(|order| order.created_at_utc)
|
||||
.map(|created_at| {
|
||||
let age = Utc::now() - created_at;
|
||||
let too_old = age > *max_age;
|
||||
if too_old {
|
||||
info!(
|
||||
"Last order on page {} is {} days old (threshold: {} days). Stopping pagination.",
|
||||
total_page_number, age.num_days(), max_age.num_days()
|
||||
);
|
||||
}
|
||||
too_old
|
||||
})
|
||||
.unwrap_or(false)
|
||||
} else {
|
||||
false
|
||||
};
|
||||
|
||||
batch_orders.extend(response.orders);
|
||||
|
||||
cursor = response.next_cursor;
|
||||
let reached_end = cursor.is_none();
|
||||
let reached_end = cursor.is_none() || exceeded_max_age;
|
||||
|
||||
// Determine if we should process the current batch now.
|
||||
let should_process_batch = reached_end
|
||||
|
||||
@@ -113,6 +113,13 @@ async fn main() -> AnyhowResult<()> {
|
||||
info!("Batch page count: {}", count);
|
||||
}
|
||||
|
||||
let maybe_max_job_age = easyenv::try_get_env_num_optional::<i64>("MAX_JOB_AGE_THRESHOLD_DAYS")?
|
||||
.and_then(chrono::Duration::try_days);
|
||||
|
||||
if let Some(ref duration) = maybe_max_job_age {
|
||||
info!("Max job age threshold: {} days", duration.num_days());
|
||||
}
|
||||
|
||||
let (pager, pager_worker) = build_pager(server_environment, &container_environment.hostname);
|
||||
|
||||
info!("Spawning pager worker.");
|
||||
@@ -144,6 +151,7 @@ async fn main() -> AnyhowResult<()> {
|
||||
job_stats,
|
||||
poll_interval_millis,
|
||||
maybe_pages_per_batch,
|
||||
maybe_max_job_age,
|
||||
application_shutdown: application_shutdown.clone(),
|
||||
pager,
|
||||
};
|
||||
|
||||
Reference in new issue
Block a user