better error logging

_sql/analytics/generic_inference/generic_inference_jobs.sql

@@ -24,7 +24,9 @@ from (
          select
              jobs.created_at,
              jobs.id,
+             jobs.status,
              jobs.attempt_count as attempts,
+             jobs.internal_debugging_failure_reason as i_failure_reason,
              u.username,
              jobs.assigned_worker,
              jobs.assigned_cluster,

crates/lib/mysql_queries/src/queries/generic_inference/job/mark_generic_inference_job_completely_failed.rs

@@ -10,6 +10,20 @@ pub async fn mark_generic_inference_job_completely_failed(
   maybe_internal_debugging_failure_reason: Option<&str>,
 ) -> AnyhowResult<()>
 {
+  let mut maybe_public_failure_reason
+      = maybe_public_failure_reason.map(|reason| {
+        let mut reason = reason.trim().to_string();
+        reason.truncate(512); // Max length of column is 512
+        reason
+      });
+
+  let mut maybe_internal_debugging_failure_reason
+      = maybe_internal_debugging_failure_reason.map(|reason| {
+        let mut reason = reason.trim().to_string();
+        reason.truncate(512); // Max length of column is 512
+        reason
+      });
+
   let query_result = sqlx::query!(
         r#"
 UPDATE generic_inference_jobs
@@ -20,8 +34,8 @@ SET
   retry_at = NULL
 WHERE id = ?
         "#,
-        maybe_public_failure_reason,
-        maybe_internal_debugging_failure_reason,
+        maybe_public_failure_reason.as_deref(),
+        maybe_internal_debugging_failure_reason.as_deref(),
         job.id.0,
     )
       .execute(pool)

crates/lib/mysql_queries/src/queries/generic_inference/job/mark_generic_inference_job_failure.rs

@@ -8,7 +8,7 @@ use sqlx;
 pub async fn mark_generic_inference_job_failure(
   pool: &MySqlPool,
   job: &AvailableInferenceJob,
-  failure_reason: &str,
+  maybe_public_failure_reason: Option<&str>,
   internal_debugging_failure_reason: &str,
   max_attempts: u16
 ) -> AnyhowResult<()> {
@@ -16,6 +16,16 @@ pub async fn mark_generic_inference_job_failure(
   // statuses: "attempt_failed", "complete_failure", "dead"
   let mut next_status = "attempt_failed";
 
+  let mut maybe_public_failure_reason = maybe_public_failure_reason.map(|reason| {
+    let mut reason = reason.trim().to_string();
+    reason.truncate(512); // Max length of column is 512
+    reason
+  });
+
+  // Max length of column is 512
+  let mut internal_debugging_failure_reason = internal_debugging_failure_reason.trim().to_string();
+  internal_debugging_failure_reason.truncate(512);
+
   if job.attempt_count >= max_attempts {
     // NB: Job attempt count is incremented at start
     next_status = "dead";
@@ -32,8 +42,8 @@ SET
 WHERE id = ?
         "#,
         next_status,
-        failure_reason,
-        internal_debugging_failure_reason,
+        maybe_public_failure_reason.as_deref(),
+        &internal_debugging_failure_reason,
         job.id.0,
     )
       .execute(pool)

crates/service/job/inference_job/src/job/job_loop/main_loop.rs

@@ -87,29 +87,34 @@ async fn process_job_batch(job_dependencies: &JobDependencies, jobs: Vec<Availab
       Err(e) => {
         warn!("Failure to process job: {:?}", e);
 
-        let (permanent_failure, failure_reason) = match e {
-          // Permanent failures
-          ProcessSingleJobError::KeepAliveElapsed => (true, "keepalive elapsed"),
-          ProcessSingleJobError::InvalidJob(_) => (true, "invalid job"),
+        let (permanent_failure, internal_failure_reason, maybe_public_failure_reason) =
+            match e {
+              // Permanent failures
+              ProcessSingleJobError::KeepAliveElapsed =>
+                (true, "keepalive elapsed".to_string(), Some("keepalive elapsed")),
+              ProcessSingleJobError::InvalidJob(ref err) =>
+                (true, format!("InvalidJob: {:?}", err), Some("invalid job")),
 
-          // Non-permanent failures
-          ProcessSingleJobError::FilesystemFull => (false, "worker filesystem full"),
-          ProcessSingleJobError::Other(_) => (false, ""),
-        };
+              // Non-permanent failures
+              ProcessSingleJobError::FilesystemFull =>
+                (false, "worker filesystem full".to_string(), Some("worker filesystem full")),
+              ProcessSingleJobError::Other(ref err) =>
+                (false, format!("OtherErr: {:?}", err), None),
+            };
 
         if permanent_failure {
           let _r = mark_generic_inference_job_completely_failed(
             &job_dependencies.mysql_pool,
             &job,
-            Some(failure_reason),
-            Some(failure_reason),
+            maybe_public_failure_reason,
+            Some(&internal_failure_reason),
           ).await;
         } else {
           let _r = mark_generic_inference_job_failure(
             &job_dependencies.mysql_pool,
             &job,
-            failure_reason,
-            failure_reason,
+            maybe_public_failure_reason,
+            &internal_failure_reason,
             job_dependencies.job_max_attempts
           ).await;
         }