From 1ae8d8d61527eeff8ec93e3e631b26fdb58b9f00 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 21 May 2023 18:16:44 -0700 Subject: [PATCH] Fail jobs with elapsed keepalive --- Cargo.lock | 1 + .../list_available_generic_inference_jobs.rs | 15 ++++++- crates/service/job/inference_job/Cargo.toml | 1 + .../src/job/job_loop/main_loop.rs | 43 ++++++++++++++----- .../src/job/job_loop/process_single_job.rs | 40 ++++++++++++++++- .../job/job_loop/process_single_job_error.rs | 2 + 6 files changed, 87 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0eb381123a..eadc68930a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3673,6 +3673,7 @@ dependencies = [ "mysql_queries", "newrelic-telemetry", "r2d2_redis", + "redis_common", "serde", "serde_derive", "serde_json", diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/job/list_available_generic_inference_jobs.rs b/crates/lib/mysql_queries/src/queries/generic_inference/job/list_available_generic_inference_jobs.rs index 84d86754a6..741cd5b95d 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/job/list_available_generic_inference_jobs.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/job/list_available_generic_inference_jobs.rs @@ -43,7 +43,10 @@ pub struct AvailableInferenceJob { // Job information pub status: JobStatus, pub attempt_count: u16, + pub priority_level: u16, + pub is_keepalive_required: bool, + pub is_from_premium_user: bool, pub is_from_api_user: bool, pub is_for_twitch: bool, @@ -116,6 +119,7 @@ pub async fn list_available_generic_inference_jobs( status: JobStatus::from_str(&record.status)?, attempt_count: record.attempt_count, priority_level: record.priority_level, + is_keepalive_required: i8_to_bool(record.is_keepalive_required), is_from_premium_user: i8_to_bool(record.is_from_premium_user), is_from_api_user: i8_to_bool(record.is_from_api_user), is_for_twitch: i8_to_bool(record.is_for_twitch), @@ -167,9 +171,11 @@ SELECT creator_set_visibility, status, - attempt_count, + priority_level, + is_keepalive_required, + is_from_premium_user, is_from_api_user, is_for_twitch, @@ -249,9 +255,11 @@ SELECT creator_set_visibility, status, - attempt_count, + priority_level, + is_keepalive_required, + is_from_premium_user, is_from_api_user, is_for_twitch, @@ -333,7 +341,10 @@ struct AvailableInferenceJobRawInternal { //pub status: JobStatus, pub status: String, pub attempt_count: u16, + pub priority_level: u16, + pub is_keepalive_required: i8, + pub is_from_premium_user: i8, pub is_from_api_user: i8, pub is_for_twitch: i8, diff --git a/crates/service/job/inference_job/Cargo.toml b/crates/service/job/inference_job/Cargo.toml index e2a056c79d..609b79f8a5 100644 --- a/crates/service/job/inference_job/Cargo.toml +++ b/crates/service/job/inference_job/Cargo.toml @@ -30,6 +30,7 @@ media = { path = "../../../lib/files/media" } memory_caching = { path = "../../../lib/caching/memory_caching" } mimetypes = { path = "../../../lib/files/mimetypes" } mysql_queries = { path = "../../../lib/mysql_queries" } +redis_common = { path = "../../../lib/redis_common" } subprocess_common = { path = "../../../lib/subprocess_common" } tokens = { path = "../../../_types/tokens" } tts_common = { path = "../../../lib/tts_common" } diff --git a/crates/service/job/inference_job/src/job/job_loop/main_loop.rs b/crates/service/job/inference_job/src/job/job_loop/main_loop.rs index 42a19bec10..b1930dfc09 100644 --- a/crates/service/job/inference_job/src/job/job_loop/main_loop.rs +++ b/crates/service/job/inference_job/src/job/job_loop/main_loop.rs @@ -1,13 +1,14 @@ +use crate::job::job_loop::clear_full_filesystem::clear_full_filesystem; use crate::job::job_loop::process_single_job::process_single_job; +use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; use crate::job_dependencies::JobDependencies; use errors::AnyhowResult; use jobs_common::noop_logger::NoOpLogger; use log::{error, info, warn}; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::{AvailableInferenceJob, list_available_generic_inference_jobs, ListAvailableGenericInferenceJobArgs}; +use mysql_queries::queries::generic_inference::job::mark_generic_inference_job_completely_failed::mark_generic_inference_job_completely_failed; use mysql_queries::queries::generic_inference::job::mark_generic_inference_job_failure::mark_generic_inference_job_failure; use std::time::Duration; -use crate::job::job_loop::clear_full_filesystem::clear_full_filesystem; -use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; // Job runner timeouts (guards MySQL) const START_TIMEOUT_MILLIS : u64 = 500; @@ -86,23 +87,43 @@ async fn process_job_batch(job_dependencies: &JobDependencies, jobs: Vec { warn!("Failure to process job: {:?}", e); - let failure_reason = ""; + let (permanent_failure, failure_reason) = match e { + // Permanent failures + ProcessSingleJobError::KeepAliveElapsed => (true, "keepalive elapsed"), + ProcessSingleJobError::InvalidJob(_) => (true, "invalid job"), - let _r = mark_generic_inference_job_failure( - &job_dependencies.mysql_pool, - &job, - failure_reason, - failure_reason, - job_dependencies.job_max_attempts - ).await; + // Non-permanent failures + ProcessSingleJobError::FilesystemFull => (false, "worker filesystem full"), + ProcessSingleJobError::Other(_) => (false, ""), + }; + + if permanent_failure { + let _r = mark_generic_inference_job_completely_failed( + &job_dependencies.mysql_pool, + &job, + Some(failure_reason), + Some(failure_reason), + ).await; + } else { + let _r = mark_generic_inference_job_failure( + &job_dependencies.mysql_pool, + &job, + failure_reason, + failure_reason, + job_dependencies.job_max_attempts + ).await; + } match e { - ProcessSingleJobError::Other(_) => {} // No-op + // Post failure handling ProcessSingleJobError::FilesystemFull => { warn!("Clearing full filesystem..."); clear_full_filesystem(&job_dependencies.semi_persistent_cache)?; } + // No-op + ProcessSingleJobError::Other(_) => {} ProcessSingleJobError::InvalidJob(_) => {} + ProcessSingleJobError::KeepAliveElapsed => {} } } } diff --git a/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs b/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs index 1a549fe006..824c0e0677 100644 --- a/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs +++ b/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs @@ -9,10 +9,14 @@ use crate::job_dependencies::JobDependencies; use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory; use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; use errors::AnyhowResult; -use log::{info, warn}; +use log::{error, info, warn}; use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob; use mysql_queries::queries::generic_inference::job::mark_generic_inference_job_pending_and_grab_lock::mark_generic_inference_job_pending_and_grab_lock; use mysql_queries::queries::generic_inference::job::mark_generic_inference_job_successfully_done::mark_generic_inference_job_successfully_done; +use r2d2_redis::RedisConnectionManager; +use r2d2_redis::r2d2::PooledConnection; +use r2d2_redis::redis::Commands; +use redis_common::redis_keys::RedisKeys; use std::time::Instant; pub async fn process_single_job( @@ -20,8 +24,14 @@ pub async fn process_single_job( job: &AvailableInferenceJob, ) -> Result { + let mut maybe_redis = job_dependencies + .maybe_redis_pool + .as_ref() + .map(|redis| redis.get()) + .transpose() + .map_err(|err| ProcessSingleJobError::Other(anyhow!("redis pool error: {:?}", err)))?; + // TODO(bt, 2023-01-11): Restore an optional status logger - //let mut redis = job_dependencies.redis_pool.get()?; //let mut redis_logger = RedisJobStatusLogger::new_generic_download(&mut redis, job.download_job_token.as_str()); let mut force_execution = false; @@ -85,6 +95,32 @@ pub async fn process_single_job( let job_start_time = Instant::now(); + // ==================== HANDLE KEEPALIVE (OPTIONAL) ==================== // + + if job.is_keepalive_required { + match &mut maybe_redis { + None => { + warn!("Keepalive is required for this job, but we do not have Redis configured to check!") + } + Some(redis) => { + let keepalive_key = + RedisKeys::generic_inference_keepalive(job.inference_job_token.as_str()); + + let _ : Option = match redis.get(&keepalive_key) { + Ok(None) => { + warn!("Job keepalive elapsed: {:?}", job.inference_job_token); + return Err(ProcessSingleJobError::KeepAliveElapsed) + }, + Ok(Some(value)) => Some(value), + Err(e) => { + error!("redis keepalive key check error: {:?}", e); + None // Fail open + }, + }; + } + } + } + // ==================== SETUP TEMP DIRS ==================== // let temp_dir = format!("temp_{}", job.id.0); diff --git a/crates/service/job/inference_job/src/job/job_loop/process_single_job_error.rs b/crates/service/job/inference_job/src/job/job_loop/process_single_job_error.rs index c5ba834a2a..299c5e9599 100644 --- a/crates/service/job/inference_job/src/job/job_loop/process_single_job_error.rs +++ b/crates/service/job/inference_job/src/job/job_loop/process_single_job_error.rs @@ -7,6 +7,8 @@ pub enum ProcessSingleJobError { FilesystemFull, /// The job is invalid (bad state, etc.) InvalidJob(anyhow::Error), + /// The job's keepalive signal has elapsed, so the job needs to be killed. + KeepAliveElapsed, /// This is any other kind of error. /// It might be important, we just haven't special cased it yet. Other(anyhow::Error),