show retries, failures, etc. for TTS inference (other jobs todo)

vocodes/src/App.tsx

@@ -13,6 +13,7 @@ import { TtsInferenceJob, TtsInferenceJobStateResponsePayload } from './jobs/Tts
 import { W2lInferenceJob, W2lInferenceJobStateResponsePayload } from './jobs/W2lInferenceJobs';
 import { TtsModelUploadJob, TtsModelUploadJobStateResponsePayload } from './jobs/TtsModelUploadJobs';
 import { W2lTemplateUploadJob, W2lTemplateUploadJobStateResponsePayload } from './jobs/W2lTemplateUploadJobs';
+import { JobState, jobStateCanChange } from './jobs/JobStates';
 
 enum MigrationMode {
   NEW_VOCODES,
@@ -152,7 +153,7 @@ class App extends React.Component<Props, State> {
 
       this.state.ttsInferenceJobs.forEach(existingJob => {
         if (existingJob.jobToken !== jobResponse.state!.job_token ||
-            !jobResponse.state!.maybe_result_token) { // NB: Don't replace until we're done.
+            !jobStateCanChange(existingJob.jobState)) {
           updatedJobs.push(existingJob);
           return;
         }
@@ -313,13 +314,9 @@ class App extends React.Component<Props, State> {
 
   pollJobs = () => {
     this.state.ttsInferenceJobs.forEach(job => {
-      switch (job.status) {
-        case 'unknown':
-        case 'pending':
-          this.checkTtsJob(job.jobToken);
-          break;
-        default:
-          return;
+      if (jobStateCanChange(job.jobState)) {
+        console.log('state can change');
+        this.checkTtsJob(job.jobToken);
       }
     });
     this.state.w2lInferenceJobs.forEach(job => {

vocodes/src/jobs/JobStates.ts

@@ -0,0 +1,44 @@
+export enum JobState {
+  UNKNOWN, // Only on frontend.
+  PENDING,
+  STARTED,
+  COMPLETE_SUCCESS,
+  COMPLETE_FAILURE,
+  ATTEMPT_FAILED,
+  DEAD,
+}
+
+export function jobStateFromString(jobStateString: string) : JobState {
+  switch (jobStateString) {
+    case 'pending':
+      return JobState.PENDING;
+    case 'started':
+      return JobState.STARTED;
+    case 'complete_success':
+      return JobState.COMPLETE_SUCCESS;
+    case 'complete_failure':
+      return JobState.COMPLETE_FAILURE;
+    case 'attempt_failed':
+      return JobState.ATTEMPT_FAILED;
+    case 'dead':
+      return JobState.DEAD;
+  }
+
+  return JobState.UNKNOWN;
+}
+
+
+export function jobStateCanChange(jobState: JobState) : boolean {
+  switch (jobState) {
+    case JobState.UNKNOWN:
+    case JobState.PENDING:
+    case JobState.STARTED:
+    case JobState.ATTEMPT_FAILED:
+      return true;
+    case JobState.COMPLETE_SUCCESS:
+    case JobState.COMPLETE_FAILURE:
+    case JobState.DEAD:
+    default:
+      return false;
+  }
+}

vocodes/src/jobs/TtsInferenceJobs.ts

@@ -1,9 +1,11 @@
+import { JobState, jobStateFromString } from "./JobStates";
 
 
 export class TtsInferenceJob {
   jobToken: string;
   modelToken?: string;
-  status: string;
+  jobState: JobState;
+  attemptCount: number;
   title?: string;
   maybeResultToken: string | undefined | null;
   maybePublicBucketWavAudioPath: string | undefined | null;
@@ -11,12 +13,14 @@ export class TtsInferenceJob {
   constructor(
     jobToken: string, 
     status: string = 'unknown',
+    attemptCount: number = 0,
     modelToken: string | undefined = undefined,
     title: string | undefined = undefined,
     maybeResulToken: string | undefined | null = null,
     maybePublicBucketWavAudioPath: string | undefined | null = null,
   ) {
-    this.status = status;
+    this.jobState = jobStateFromString(status);
+    this.attemptCount = attemptCount;
     this.jobToken = jobToken;
     this.maybeResultToken = maybeResulToken;
     if (!!modelToken) {
@@ -32,12 +36,28 @@ export class TtsInferenceJob {
     return new TtsInferenceJob(
       response.job_token,
       response.status,
+      response.attempt_count || 0,
       response.model_token,
       response.title,
       response.maybe_result_token,
       response.maybe_public_bucket_wav_audio_path
     );
   }
+
+  public jobStateCanChange() : boolean {
+    switch (this.jobState) {
+      case JobState.UNKNOWN:
+      case JobState.PENDING:
+      case JobState.STARTED:
+      case JobState.ATTEMPT_FAILED:
+        return true;
+      case JobState.COMPLETE_SUCCESS:
+      case JobState.COMPLETE_FAILURE:
+      case JobState.DEAD:
+      default:
+        return false;
+    }
+  }
 }
 
 export interface TtsInferenceJobStateResponsePayload {
@@ -48,6 +68,7 @@ export interface TtsInferenceJobStateResponsePayload {
 export interface TtsInferenceJobState {
   job_token: string,
   status: string,
+  attempt_count: number | null,
   maybe_result_token: string | null,
   maybe_public_bucket_wav_audio_path: string | null,
   model_token: string,

vocodes/src/v2/common/SessionTtsInferenceResultsListFc.tsx

@@ -1,6 +1,7 @@
 import React from 'react';
 import { Link } from 'react-router-dom';
 import { TtsInferenceJob } from '../../App'
+import { JobState } from '../../jobs/JobStates';
 
 interface Props {
   ttsInferenceJobs: Array<TtsInferenceJob>,
@@ -11,10 +12,33 @@ function SessionTtsInferenceResultListFc(props: Props) {
   let results : Array<JSX.Element> = [];
 
   props.ttsInferenceJobs.forEach(job => {
+    console.log('state', job.jobState);
 
     if (!job.maybeResultToken) {
+      let stateDescription = "Pending...";
+
+      switch (job.jobState) {
+        case JobState.PENDING:
+        case JobState.UNKNOWN:
+          stateDescription = "Pending..."
+          break;
+        case JobState.STARTED:
+          stateDescription = "Started...";
+          break;
+        case JobState.ATTEMPT_FAILED:
+          stateDescription = `Failed ${job.attemptCount} attempt(s). Will retry...`;
+          break;
+        case JobState.COMPLETE_FAILURE:
+        case JobState.DEAD:
+          stateDescription = "Failed Permanently. Please tell us in Discord so we can fix. :(";
+          break;
+        case JobState.COMPLETE_SUCCESS:
+          stateDescription = "Success!"; // Not sure why we're here instead of other branch!
+          break;
+      }
+
       results.push(
-        <div key={job.jobToken}>Pending&#8230;</div>
+        <div key={job.jobToken}>{stateDescription}</div>
       );
     } else {