pause on no updates

This commit is contained in:
Brandon Thomas committed 2024-02-05 04:37:04 -05:00
1 parent c0c86c90b8
commit 0719a929f8
4 files changed
+24 -4

No files matched your search

@@ -125,9 +125,9 @@ async fn list_model_weights<'e, 'c, E>(
r#"
SELECT
w.id,
w.token
w.token,
w.creator_set_visibility
w.creator_set_visibility,
w.weights_type,
w.weights_category,
@@ -174,7 +174,7 @@ LEFT OUTER JOIN entity_stats
LEFT OUTER JOIN media_files as cover_image
ON cover_image.token = w.maybe_cover_image_media_file_token
WHERE token IN (
WHERE w.token IN (
"#);
// NB: Syntax will be wrong if list has zero length
@@ -21,4 +21,7 @@ pub struct SleepConfigs {
// How long to wait between errors.
pub between_error_wait_millis: u64,
// How long to wait between periods of NO-OPs / no updates.
pub between_no_updates_wait_millis: u64,
}
@@ -56,6 +56,7 @@ async fn main() -> AnyhowResult<()> {
between_job_batch_wait_millis: easyenv::get_env_num("BETWEEN_JOB_BATCH_WAIT_MILLIS", 500)?,
between_query_wait_millis: easyenv::get_env_num("BETWEEN_QUERY_WAIT_MILLIS", 100)?,
between_error_wait_millis: easyenv::get_env_num("BETWEEN_ERROR_WAIT_MILLIS", 10_000)?,
between_no_updates_wait_millis: easyenv::get_env_num("BETWEEN_NO_UPDATES_WAIT_MILLIS", 20_000)?,
},
};
@@ -46,11 +46,20 @@ pub async fn with_database_main_loop(updated_at_cursor: &mut DateTime<Utc>, job_
let mut maybe_tokens =
list_model_weight_tokens_updated_since(&mut *mysql_connection, &updated_at_cursor).await?;
if maybe_tokens.is_empty() {
info!("No records updated since {:?}", &updated_at_cursor);
std::thread::sleep(Duration::from_millis(job_state.sleep_config.between_no_updates_wait_millis));
continue;
}
info!("Found {} updated records", maybe_tokens.len());
let mut last_observed_updated_at = updated_at_cursor.clone();
while !maybe_tokens.is_empty() {
// NB: This list might be very large if we query from (1) the epoch, or (2) there was a large series of updates
let drained_tokens = maybe_tokens.drain(0..50)
let last = 50.min(maybe_tokens.len());
let drained_tokens = maybe_tokens.drain(0..last)
.into_iter()
.map(|record| record.token)
.collect::<Vec<_>>();
@@ -75,12 +84,19 @@ pub async fn with_database_main_loop(updated_at_cursor: &mut DateTime<Utc>, job_
*updated_at_cursor = last_successful_update_at.max(*updated_at_cursor);
last_observed_updated_at = updated_at;
std::thread::sleep(Duration::from_millis(job_state.sleep_config.between_es_writes_wait_millis));
}
std::thread::sleep(Duration::from_millis(job_state.sleep_config.between_job_batch_wait_millis));
}
// NB: The last cursor math put us a second behind the current clock. Here we'll advance to the last record.
// NB: This technically could miss records if we're updating within the same second batch we read from, but that
// seems both unlikely and not worth solving at our present scale.
*updated_at_cursor = last_observed_updated_at.max(*updated_at_cursor);
info!("Up to date at cursor = {:?}", &updated_at_cursor);
std::thread::sleep(Duration::from_millis(job_state.sleep_config.between_query_wait_millis));