diff --git a/crates/lib/mysql_queries/src/queries/model_weights/batch_get/batch_get_model_weights_for_elastic_search_backfill.rs b/crates/lib/mysql_queries/src/queries/model_weights/batch_get/batch_get_model_weights_for_elastic_search_backfill.rs index d5aa4f8c86..24a95fbbf7 100644 --- a/crates/lib/mysql_queries/src/queries/model_weights/batch_get/batch_get_model_weights_for_elastic_search_backfill.rs +++ b/crates/lib/mysql_queries/src/queries/model_weights/batch_get/batch_get_model_weights_for_elastic_search_backfill.rs @@ -125,9 +125,9 @@ async fn list_model_weights<'e, 'c, E>( r#" SELECT w.id, - w.token + w.token, - w.creator_set_visibility + w.creator_set_visibility, w.weights_type, w.weights_category, @@ -174,7 +174,7 @@ LEFT OUTER JOIN entity_stats LEFT OUTER JOIN media_files as cover_image ON cover_image.token = w.maybe_cover_image_media_file_token -WHERE token IN ( +WHERE w.token IN ( "#); // NB: Syntax will be wrong if list has zero length diff --git a/crates/service/job/es-update-job/src/job_state.rs b/crates/service/job/es-update-job/src/job_state.rs index ce80365dce..0d235382bc 100644 --- a/crates/service/job/es-update-job/src/job_state.rs +++ b/crates/service/job/es-update-job/src/job_state.rs @@ -21,4 +21,7 @@ pub struct SleepConfigs { // How long to wait between errors. pub between_error_wait_millis: u64, + + // How long to wait between periods of NO-OPs / no updates. + pub between_no_updates_wait_millis: u64, } diff --git a/crates/service/job/es-update-job/src/main.rs b/crates/service/job/es-update-job/src/main.rs index de96ec6c52..8fb37d554e 100644 --- a/crates/service/job/es-update-job/src/main.rs +++ b/crates/service/job/es-update-job/src/main.rs @@ -56,6 +56,7 @@ async fn main() -> AnyhowResult<()> { between_job_batch_wait_millis: easyenv::get_env_num("BETWEEN_JOB_BATCH_WAIT_MILLIS", 500)?, between_query_wait_millis: easyenv::get_env_num("BETWEEN_QUERY_WAIT_MILLIS", 100)?, between_error_wait_millis: easyenv::get_env_num("BETWEEN_ERROR_WAIT_MILLIS", 10_000)?, + between_no_updates_wait_millis: easyenv::get_env_num("BETWEEN_NO_UPDATES_WAIT_MILLIS", 20_000)?, }, }; diff --git a/crates/service/job/es-update-job/src/main_loop.rs b/crates/service/job/es-update-job/src/main_loop.rs index 5b9465e44b..127f6f8ebc 100644 --- a/crates/service/job/es-update-job/src/main_loop.rs +++ b/crates/service/job/es-update-job/src/main_loop.rs @@ -46,11 +46,20 @@ pub async fn with_database_main_loop(updated_at_cursor: &mut DateTime, job_ let mut maybe_tokens = list_model_weight_tokens_updated_since(&mut *mysql_connection, &updated_at_cursor).await?; + if maybe_tokens.is_empty() { + info!("No records updated since {:?}", &updated_at_cursor); + std::thread::sleep(Duration::from_millis(job_state.sleep_config.between_no_updates_wait_millis)); + continue; + } + info!("Found {} updated records", maybe_tokens.len()); + let mut last_observed_updated_at = updated_at_cursor.clone(); + while !maybe_tokens.is_empty() { // NB: This list might be very large if we query from (1) the epoch, or (2) there was a large series of updates - let drained_tokens = maybe_tokens.drain(0..50) + let last = 50.min(maybe_tokens.len()); + let drained_tokens = maybe_tokens.drain(0..last) .into_iter() .map(|record| record.token) .collect::>(); @@ -75,12 +84,19 @@ pub async fn with_database_main_loop(updated_at_cursor: &mut DateTime, job_ *updated_at_cursor = last_successful_update_at.max(*updated_at_cursor); + last_observed_updated_at = updated_at; + std::thread::sleep(Duration::from_millis(job_state.sleep_config.between_es_writes_wait_millis)); } std::thread::sleep(Duration::from_millis(job_state.sleep_config.between_job_batch_wait_millis)); } + // NB: The last cursor math put us a second behind the current clock. Here we'll advance to the last record. + // NB: This technically could miss records if we're updating within the same second batch we read from, but that + // seems both unlikely and not worth solving at our present scale. + *updated_at_cursor = last_observed_updated_at.max(*updated_at_cursor); + info!("Up to date at cursor = {:?}", &updated_at_cursor); std::thread::sleep(Duration::from_millis(job_state.sleep_config.between_query_wait_millis));