mirror of
https://github.com/storytold/artcraft.git
synced 2026-11-18 23:55:16 +00:00
Fail jobs with elapsed keepalive
This commit is contained in:
1 parent
63c0124267
commit
1ae8d8d615
6 files changed
+87
-15
No files matched your search
Generated
+1
@@ -3673,6 +3673,7 @@ dependencies = [
|
||||
"mysql_queries",
|
||||
"newrelic-telemetry",
|
||||
"r2d2_redis",
|
||||
"redis_common",
|
||||
"serde",
|
||||
"serde_derive",
|
||||
"serde_json",
|
||||
|
||||
+13
-2
@@ -43,7 +43,10 @@ pub struct AvailableInferenceJob {
|
||||
// Job information
|
||||
pub status: JobStatus,
|
||||
pub attempt_count: u16,
|
||||
|
||||
pub priority_level: u16,
|
||||
pub is_keepalive_required: bool,
|
||||
|
||||
pub is_from_premium_user: bool,
|
||||
pub is_from_api_user: bool,
|
||||
pub is_for_twitch: bool,
|
||||
@@ -116,6 +119,7 @@ pub async fn list_available_generic_inference_jobs(
|
||||
status: JobStatus::from_str(&record.status)?,
|
||||
attempt_count: record.attempt_count,
|
||||
priority_level: record.priority_level,
|
||||
is_keepalive_required: i8_to_bool(record.is_keepalive_required),
|
||||
is_from_premium_user: i8_to_bool(record.is_from_premium_user),
|
||||
is_from_api_user: i8_to_bool(record.is_from_api_user),
|
||||
is_for_twitch: i8_to_bool(record.is_for_twitch),
|
||||
@@ -167,9 +171,11 @@ SELECT
|
||||
creator_set_visibility,
|
||||
|
||||
status,
|
||||
|
||||
attempt_count,
|
||||
|
||||
priority_level,
|
||||
is_keepalive_required,
|
||||
|
||||
is_from_premium_user,
|
||||
is_from_api_user,
|
||||
is_for_twitch,
|
||||
@@ -249,9 +255,11 @@ SELECT
|
||||
creator_set_visibility,
|
||||
|
||||
status,
|
||||
|
||||
attempt_count,
|
||||
|
||||
priority_level,
|
||||
is_keepalive_required,
|
||||
|
||||
is_from_premium_user,
|
||||
is_from_api_user,
|
||||
is_for_twitch,
|
||||
@@ -333,7 +341,10 @@ struct AvailableInferenceJobRawInternal {
|
||||
//pub status: JobStatus,
|
||||
pub status: String,
|
||||
pub attempt_count: u16,
|
||||
|
||||
pub priority_level: u16,
|
||||
pub is_keepalive_required: i8,
|
||||
|
||||
pub is_from_premium_user: i8,
|
||||
pub is_from_api_user: i8,
|
||||
pub is_for_twitch: i8,
|
||||
|
||||
@@ -30,6 +30,7 @@ media = { path = "../../../lib/files/media" }
|
||||
memory_caching = { path = "../../../lib/caching/memory_caching" }
|
||||
mimetypes = { path = "../../../lib/files/mimetypes" }
|
||||
mysql_queries = { path = "../../../lib/mysql_queries" }
|
||||
redis_common = { path = "../../../lib/redis_common" }
|
||||
subprocess_common = { path = "../../../lib/subprocess_common" }
|
||||
tokens = { path = "../../../_types/tokens" }
|
||||
tts_common = { path = "../../../lib/tts_common" }
|
||||
|
||||
@@ -1,13 +1,14 @@
|
||||
use crate::job::job_loop::clear_full_filesystem::clear_full_filesystem;
|
||||
use crate::job::job_loop::process_single_job::process_single_job;
|
||||
use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
|
||||
use crate::job_dependencies::JobDependencies;
|
||||
use errors::AnyhowResult;
|
||||
use jobs_common::noop_logger::NoOpLogger;
|
||||
use log::{error, info, warn};
|
||||
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::{AvailableInferenceJob, list_available_generic_inference_jobs, ListAvailableGenericInferenceJobArgs};
|
||||
use mysql_queries::queries::generic_inference::job::mark_generic_inference_job_completely_failed::mark_generic_inference_job_completely_failed;
|
||||
use mysql_queries::queries::generic_inference::job::mark_generic_inference_job_failure::mark_generic_inference_job_failure;
|
||||
use std::time::Duration;
|
||||
use crate::job::job_loop::clear_full_filesystem::clear_full_filesystem;
|
||||
use crate::job::job_loop::process_single_job_error::ProcessSingleJobError;
|
||||
|
||||
// Job runner timeouts (guards MySQL)
|
||||
const START_TIMEOUT_MILLIS : u64 = 500;
|
||||
@@ -86,23 +87,43 @@ async fn process_job_batch(job_dependencies: &JobDependencies, jobs: Vec<Availab
|
||||
Err(e) => {
|
||||
warn!("Failure to process job: {:?}", e);
|
||||
|
||||
let failure_reason = "";
|
||||
let (permanent_failure, failure_reason) = match e {
|
||||
// Permanent failures
|
||||
ProcessSingleJobError::KeepAliveElapsed => (true, "keepalive elapsed"),
|
||||
ProcessSingleJobError::InvalidJob(_) => (true, "invalid job"),
|
||||
|
||||
let _r = mark_generic_inference_job_failure(
|
||||
&job_dependencies.mysql_pool,
|
||||
&job,
|
||||
failure_reason,
|
||||
failure_reason,
|
||||
job_dependencies.job_max_attempts
|
||||
).await;
|
||||
// Non-permanent failures
|
||||
ProcessSingleJobError::FilesystemFull => (false, "worker filesystem full"),
|
||||
ProcessSingleJobError::Other(_) => (false, ""),
|
||||
};
|
||||
|
||||
if permanent_failure {
|
||||
let _r = mark_generic_inference_job_completely_failed(
|
||||
&job_dependencies.mysql_pool,
|
||||
&job,
|
||||
Some(failure_reason),
|
||||
Some(failure_reason),
|
||||
).await;
|
||||
} else {
|
||||
let _r = mark_generic_inference_job_failure(
|
||||
&job_dependencies.mysql_pool,
|
||||
&job,
|
||||
failure_reason,
|
||||
failure_reason,
|
||||
job_dependencies.job_max_attempts
|
||||
).await;
|
||||
}
|
||||
|
||||
match e {
|
||||
ProcessSingleJobError::Other(_) => {} // No-op
|
||||
// Post failure handling
|
||||
ProcessSingleJobError::FilesystemFull => {
|
||||
warn!("Clearing full filesystem...");
|
||||
clear_full_filesystem(&job_dependencies.semi_persistent_cache)?;
|
||||
}
|
||||
// No-op
|
||||
ProcessSingleJobError::Other(_) => {}
|
||||
ProcessSingleJobError::InvalidJob(_) => {}
|
||||
ProcessSingleJobError::KeepAliveElapsed => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -9,10 +9,14 @@ use crate::job_dependencies::JobDependencies;
|
||||
use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory;
|
||||
use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType;
|
||||
use errors::AnyhowResult;
|
||||
use log::{info, warn};
|
||||
use log::{error, info, warn};
|
||||
use mysql_queries::queries::generic_inference::job::list_available_generic_inference_jobs::AvailableInferenceJob;
|
||||
use mysql_queries::queries::generic_inference::job::mark_generic_inference_job_pending_and_grab_lock::mark_generic_inference_job_pending_and_grab_lock;
|
||||
use mysql_queries::queries::generic_inference::job::mark_generic_inference_job_successfully_done::mark_generic_inference_job_successfully_done;
|
||||
use r2d2_redis::RedisConnectionManager;
|
||||
use r2d2_redis::r2d2::PooledConnection;
|
||||
use r2d2_redis::redis::Commands;
|
||||
use redis_common::redis_keys::RedisKeys;
|
||||
use std::time::Instant;
|
||||
|
||||
pub async fn process_single_job(
|
||||
@@ -20,8 +24,14 @@ pub async fn process_single_job(
|
||||
job: &AvailableInferenceJob,
|
||||
) -> Result<ProcessSingleJobSuccessCase, ProcessSingleJobError> {
|
||||
|
||||
let mut maybe_redis = job_dependencies
|
||||
.maybe_redis_pool
|
||||
.as_ref()
|
||||
.map(|redis| redis.get())
|
||||
.transpose()
|
||||
.map_err(|err| ProcessSingleJobError::Other(anyhow!("redis pool error: {:?}", err)))?;
|
||||
|
||||
// TODO(bt, 2023-01-11): Restore an optional status logger
|
||||
//let mut redis = job_dependencies.redis_pool.get()?;
|
||||
//let mut redis_logger = RedisJobStatusLogger::new_generic_download(&mut redis, job.download_job_token.as_str());
|
||||
|
||||
let mut force_execution = false;
|
||||
@@ -85,6 +95,32 @@ pub async fn process_single_job(
|
||||
|
||||
let job_start_time = Instant::now();
|
||||
|
||||
// ==================== HANDLE KEEPALIVE (OPTIONAL) ==================== //
|
||||
|
||||
if job.is_keepalive_required {
|
||||
match &mut maybe_redis {
|
||||
None => {
|
||||
warn!("Keepalive is required for this job, but we do not have Redis configured to check!")
|
||||
}
|
||||
Some(redis) => {
|
||||
let keepalive_key =
|
||||
RedisKeys::generic_inference_keepalive(job.inference_job_token.as_str());
|
||||
|
||||
let _ : Option<String> = match redis.get(&keepalive_key) {
|
||||
Ok(None) => {
|
||||
warn!("Job keepalive elapsed: {:?}", job.inference_job_token);
|
||||
return Err(ProcessSingleJobError::KeepAliveElapsed)
|
||||
},
|
||||
Ok(Some(value)) => Some(value),
|
||||
Err(e) => {
|
||||
error!("redis keepalive key check error: {:?}", e);
|
||||
None // Fail open
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ==================== SETUP TEMP DIRS ==================== //
|
||||
|
||||
let temp_dir = format!("temp_{}", job.id.0);
|
||||
|
||||
@@ -7,6 +7,8 @@ pub enum ProcessSingleJobError {
|
||||
FilesystemFull,
|
||||
/// The job is invalid (bad state, etc.)
|
||||
InvalidJob(anyhow::Error),
|
||||
/// The job's keepalive signal has elapsed, so the job needs to be killed.
|
||||
KeepAliveElapsed,
|
||||
/// This is any other kind of error.
|
||||
/// It might be important, we just haven't special cased it yet.
|
||||
Other(anyhow::Error),
|
||||
|
||||
Reference in new issue
Block a user