From 0f7ff50cf605d3740d8a6f0702dd69b681300f5a Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Sun, 21 May 2023 20:01:08 -0700 Subject: [PATCH] better error logging --- .../generic_inference_jobs.sql | 2 ++ ...generic_inference_job_completely_failed.rs | 18 ++++++++++-- .../job/mark_generic_inference_job_failure.rs | 16 ++++++++-- .../src/job/job_loop/main_loop.rs | 29 +++++++++++-------- 4 files changed, 48 insertions(+), 17 deletions(-) diff --git a/_sql/analytics/generic_inference/generic_inference_jobs.sql b/_sql/analytics/generic_inference/generic_inference_jobs.sql index 92df3eb39b..0c482013e8 100644 --- a/_sql/analytics/generic_inference/generic_inference_jobs.sql +++ b/_sql/analytics/generic_inference/generic_inference_jobs.sql @@ -24,7 +24,9 @@ from ( select jobs.created_at, jobs.id, + jobs.status, jobs.attempt_count as attempts, + jobs.internal_debugging_failure_reason as i_failure_reason, u.username, jobs.assigned_worker, jobs.assigned_cluster, diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/job/mark_generic_inference_job_completely_failed.rs b/crates/lib/mysql_queries/src/queries/generic_inference/job/mark_generic_inference_job_completely_failed.rs index 6e58a8debc..7bd1e80aa5 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/job/mark_generic_inference_job_completely_failed.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/job/mark_generic_inference_job_completely_failed.rs @@ -10,6 +10,20 @@ pub async fn mark_generic_inference_job_completely_failed( maybe_internal_debugging_failure_reason: Option<&str>, ) -> AnyhowResult<()> { + let mut maybe_public_failure_reason + = maybe_public_failure_reason.map(|reason| { + let mut reason = reason.trim().to_string(); + reason.truncate(512); // Max length of column is 512 + reason + }); + + let mut maybe_internal_debugging_failure_reason + = maybe_internal_debugging_failure_reason.map(|reason| { + let mut reason = reason.trim().to_string(); + reason.truncate(512); // Max length of column is 512 + reason + }); + let query_result = sqlx::query!( r#" UPDATE generic_inference_jobs @@ -20,8 +34,8 @@ SET retry_at = NULL WHERE id = ? "#, - maybe_public_failure_reason, - maybe_internal_debugging_failure_reason, + maybe_public_failure_reason.as_deref(), + maybe_internal_debugging_failure_reason.as_deref(), job.id.0, ) .execute(pool) diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/job/mark_generic_inference_job_failure.rs b/crates/lib/mysql_queries/src/queries/generic_inference/job/mark_generic_inference_job_failure.rs index b3647447af..587521cc60 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/job/mark_generic_inference_job_failure.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/job/mark_generic_inference_job_failure.rs @@ -8,7 +8,7 @@ use sqlx; pub async fn mark_generic_inference_job_failure( pool: &MySqlPool, job: &AvailableInferenceJob, - failure_reason: &str, + maybe_public_failure_reason: Option<&str>, internal_debugging_failure_reason: &str, max_attempts: u16 ) -> AnyhowResult<()> { @@ -16,6 +16,16 @@ pub async fn mark_generic_inference_job_failure( // statuses: "attempt_failed", "complete_failure", "dead" let mut next_status = "attempt_failed"; + let mut maybe_public_failure_reason = maybe_public_failure_reason.map(|reason| { + let mut reason = reason.trim().to_string(); + reason.truncate(512); // Max length of column is 512 + reason + }); + + // Max length of column is 512 + let mut internal_debugging_failure_reason = internal_debugging_failure_reason.trim().to_string(); + internal_debugging_failure_reason.truncate(512); + if job.attempt_count >= max_attempts { // NB: Job attempt count is incremented at start next_status = "dead"; @@ -32,8 +42,8 @@ SET WHERE id = ? "#, next_status, - failure_reason, - internal_debugging_failure_reason, + maybe_public_failure_reason.as_deref(), + &internal_debugging_failure_reason, job.id.0, ) .execute(pool) diff --git a/crates/service/job/inference_job/src/job/job_loop/main_loop.rs b/crates/service/job/inference_job/src/job/job_loop/main_loop.rs index b1930dfc09..bc01d8dbf2 100644 --- a/crates/service/job/inference_job/src/job/job_loop/main_loop.rs +++ b/crates/service/job/inference_job/src/job/job_loop/main_loop.rs @@ -87,29 +87,34 @@ async fn process_job_batch(job_dependencies: &JobDependencies, jobs: Vec { warn!("Failure to process job: {:?}", e); - let (permanent_failure, failure_reason) = match e { - // Permanent failures - ProcessSingleJobError::KeepAliveElapsed => (true, "keepalive elapsed"), - ProcessSingleJobError::InvalidJob(_) => (true, "invalid job"), + let (permanent_failure, internal_failure_reason, maybe_public_failure_reason) = + match e { + // Permanent failures + ProcessSingleJobError::KeepAliveElapsed => + (true, "keepalive elapsed".to_string(), Some("keepalive elapsed")), + ProcessSingleJobError::InvalidJob(ref err) => + (true, format!("InvalidJob: {:?}", err), Some("invalid job")), - // Non-permanent failures - ProcessSingleJobError::FilesystemFull => (false, "worker filesystem full"), - ProcessSingleJobError::Other(_) => (false, ""), - }; + // Non-permanent failures + ProcessSingleJobError::FilesystemFull => + (false, "worker filesystem full".to_string(), Some("worker filesystem full")), + ProcessSingleJobError::Other(ref err) => + (false, format!("OtherErr: {:?}", err), None), + }; if permanent_failure { let _r = mark_generic_inference_job_completely_failed( &job_dependencies.mysql_pool, &job, - Some(failure_reason), - Some(failure_reason), + maybe_public_failure_reason, + Some(&internal_failure_reason), ).await; } else { let _r = mark_generic_inference_job_failure( &job_dependencies.mysql_pool, &job, - failure_reason, - failure_reason, + maybe_public_failure_reason, + &internal_failure_reason, job_dependencies.job_max_attempts ).await; }