Fail jobs with elapsed keepalive

This commit is contained in:
Brandon Thomas committed 2023-05-21 18:16:44 -07:00
1 parent 63c0124267
commit 1ae8d8d615
6 files changed
+87 -15

No files matched your search

Generated
+1
View File
@@ -3673,6 +3673,7 @@ dependencies = [
"mysql_queries",
"newrelic-telemetry",
"r2d2_redis",
"redis_common",
"serde",
"serde_derive",
"serde_json",
@@ -43,7 +43,10 @@ pub struct AvailableInferenceJob {
// Job information
pub status: JobStatus,
pub attempt_count: u16,
pub priority_level: u16,
pub is_keepalive_required: bool,
pub is_from_premium_user: bool,
pub is_from_api_user: bool,
pub is_for_twitch: bool,
@@ -116,6 +119,7 @@ pub async fn list_available_generic_inference_jobs(
status: JobStatus::from_str(&record.status)?,
attempt_count: record.attempt_count,
priority_level: record.priority_level,
is_keepalive_required: i8_to_bool(record.is_keepalive_required),
is_from_premium_user: i8_to_bool(record.is_from_premium_user),
is_from_api_user: i8_to_bool(record.is_from_api_user),
is_for_twitch: i8_to_bool(record.is_for_twitch),
@@ -167,9 +171,11 @@ SELECT
creator_set_visibility,
status,
attempt_count,
priority_level,
is_keepalive_required,
is_from_premium_user,
is_from_api_user,
is_for_twitch,
@@ -249,9 +255,11 @@ SELECT
creator_set_visibility,
status,
attempt_count,
priority_level,
is_keepalive_required,
is_from_premium_user,
is_from_api_user,
is_for_twitch,
@@ -333,7 +341,10 @@ struct AvailableInferenceJobRawInternal {
//pub status: JobStatus,
pub status: String,
pub attempt_count: u16,
pub priority_level: u16,
pub is_keepalive_required: i8,
pub is_from_premium_user: i8,
pub is_from_api_user: i8,
pub is_for_twitch: i8,
@@ -30,6 +30,7 @@ media = { path = "../../../lib/files/media" }
memory_caching = { path = "../../../lib/caching/memory_caching" }
mimetypes = { path = "../../../lib/files/mimetypes" }
mysql_queries = { path = "../../../lib/mysql_queries" }
redis_common = { path = "../../../lib/redis_common" }
subprocess_common = { path = "../../../lib/subprocess_common" }
tokens = { path = "../../../_types/tokens" }
tts_common = { path = "../../../lib/tts_common" }
@@ -1,13 +1,14 @@
use crate::job::job_loop::clear_full_filesystem::clear_full_filesystem;
use crate::job::job_loop::process_single_job::process_single_job;
use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
use crate::job_dependencies::JobDependencies;
use errors::AnyhowResult;
use jobs_common::noop_logger::NoOpLogger;
use log::{error, info, warn};
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::{AvailableInferenceJob, list_available_generic_inference_jobs, ListAvailableGenericInferenceJobArgs};
use mysql_queries::queries::generic_inference::job::mark_generic_inference_job_completely_failed::mark_generic_inference_job_completely_failed;
use mysql_queries::queries::generic_inference::job::mark_generic_inference_job_failure::mark_generic_inference_job_failure;
use std::time::Duration;
use crate::job::job_loop::clear_full_filesystem::clear_full_filesystem;
use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
// Job runner timeouts (guards MySQL)
const START_TIMEOUT_MILLIS : u64 = 500;
@@ -86,23 +87,43 @@ async fn process_job_batch(job_dependencies: &JobDependencies, jobs: Vec<Availab
Err(e) => {
warn!("Failure to process job: {:?}", e);
let failure_reason = "";
let (permanent_failure, failure_reason) = match e {
// Permanent failures
ProcessSingleJobError::KeepAliveElapsed => (true, "keepalive elapsed"),
ProcessSingleJobError::InvalidJob(_) => (true, "invalid job"),
let _r = mark_generic_inference_job_failure(
&job_dependencies.mysql_pool,
&job,
failure_reason,
failure_reason,
job_dependencies.job_max_attempts
).await;
// Non-permanent failures
ProcessSingleJobError::FilesystemFull => (false, "worker filesystem full"),
ProcessSingleJobError::Other(_) => (false, ""),
};
if permanent_failure {
let _r = mark_generic_inference_job_completely_failed(
&job_dependencies.mysql_pool,
&job,
Some(failure_reason),
Some(failure_reason),
).await;
} else {
let _r = mark_generic_inference_job_failure(
&job_dependencies.mysql_pool,
&job,
failure_reason,
failure_reason,
job_dependencies.job_max_attempts
).await;
}
match e {
ProcessSingleJobError::Other(_) => {} // No-op
// Post failure handling
ProcessSingleJobError::FilesystemFull => {
warn!("Clearing full filesystem...");
clear_full_filesystem(&job_dependencies.semi_persistent_cache)?;
}
// No-op
ProcessSingleJobError::Other(_) => {}
ProcessSingleJobError::InvalidJob(_) => {}
ProcessSingleJobError::KeepAliveElapsed => {}
}
}
}
@@ -9,10 +9,14 @@ use crate::job_dependencies::JobDependencies;
use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory;
use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
use errors::AnyhowResult;
use log::{info, warn};
use log::{error, info, warn};
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
use mysql_queries::queries::generic_inference::job::mark_generic_inference_job_pending_and_grab_lock::mark_generic_inference_job_pending_and_grab_lock;
use mysql_queries::queries::generic_inference::job::mark_generic_inference_job_successfully_done::mark_generic_inference_job_successfully_done;
use r2d2_redis::RedisConnectionManager;
use r2d2_redis::r2d2::PooledConnection;
use r2d2_redis::redis::Commands;
use redis_common::redis_keys::RedisKeys;
use std::time::Instant;
pub async fn process_single_job(
@@ -20,8 +24,14 @@ pub async fn process_single_job(
job: &AvailableInferenceJob,
) -> Result<ProcessSingleJobSuccessCase, ProcessSingleJobError> {
let mut maybe_redis = job_dependencies
.maybe_redis_pool
.as_ref()
.map(|redis| redis.get())
.transpose()
.map_err(|err| ProcessSingleJobError::Other(anyhow!("redis pool error: {:?}", err)))?;
// TODO(bt, 2023-01-11): Restore an optional status logger
//let mut redis = job_dependencies.redis_pool.get()?;
//let mut redis_logger = RedisJobStatusLogger::new_generic_download(&mut redis, job.download_job_token.as_str());
let mut force_execution = false;
@@ -85,6 +95,32 @@ pub async fn process_single_job(
let job_start_time = Instant::now();
// ==================== HANDLE KEEPALIVE (OPTIONAL) ==================== //
if job.is_keepalive_required {
match &mut maybe_redis {
None => {
warn!("Keepalive is required for this job, but we do not have Redis configured to check!")
}
Some(redis) => {
let keepalive_key =
RedisKeys::generic_inference_keepalive(job.inference_job_token.as_str());
let _ : Option<String> = match redis.get(&keepalive_key) {
Ok(None) => {
warn!("Job keepalive elapsed: {:?}", job.inference_job_token);
return Err(ProcessSingleJobError::KeepAliveElapsed)
},
Ok(Some(value)) => Some(value),
Err(e) => {
error!("redis keepalive key check error: {:?}", e);
None // Fail open
},
};
}
}
}
// ==================== SETUP TEMP DIRS ==================== //
let temp_dir = format!("temp_{}", job.id.0);
@@ -7,6 +7,8 @@ pub enum ProcessSingleJobError {
FilesystemFull,
/// The job is invalid (bad state, etc.)
InvalidJob(anyhow::Error),
/// The job's keepalive signal has elapsed, so the job needs to be killed.
KeepAliveElapsed,
/// This is any other kind of error.
/// It might be important, we just haven't special cased it yet.
Other(anyhow::Error),