add tokens to upload jobs, reorganize schemas, codegen

migrations/2021-05-15-071545_asset_upload_jobs/up.sql

@@ -5,24 +5,15 @@
 CREATE TABLE tts_model_upload_jobs (
   id BIGINT(20) NOT NULL AUTO_INCREMENT,
 
+  -- Effective "primary key" (PUBLIC)
+  -- This is so the in-progress results can be looked up by the UI.
+  token VARCHAR(32) NOT NULL,
+
   -- Idempotency token
+  -- This is so the frontend client doesn't submit duplicate jobs.
   uuid_idempotency_token VARCHAR(36) NOT NULL,
 
-  -- Foreign key to user
-  creator_user_token VARCHAR(32) NOT NULL,
-
-  -- For abuse tracking.
-  -- Wide enough for IPv4/6
-  creator_ip_address VARCHAR(40) NOT NULL,
-
-  -- (THIS MIGHT NOT BE USED)
-  -- NB: DO NOT SORT!
-  -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE.
-  creator_set_visibility ENUM(
-      'public',
-      'hidden',
-      'private'
-  ) NOT NULL DEFAULT 'public',
+  -- ========== UPLOAD DETAILS ==========
 
   -- The name of the voice
   -- The "name" of the voice model, which might be complicated.
@@ -51,6 +42,26 @@ CREATE TABLE tts_model_upload_jobs (
       'web'
   ) NOT NULL DEFAULT 'web',
 
+  -- ========== CREATOR DETAILS AND PREFERENCES ==========
+
+  -- Foreign key to user
+  creator_user_token VARCHAR(32) NOT NULL,
+
+  -- For abuse tracking.
+  -- Wide enough for IPv4/6
+  creator_ip_address VARCHAR(40) NOT NULL,
+
+  -- (THIS MIGHT NOT BE USED)
+  -- NB: DO NOT SORT!
+  -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE.
+  creator_set_visibility ENUM(
+      'public',
+      'hidden',
+      'private'
+  ) NOT NULL DEFAULT 'public',
+
+  -- ========== JOB SYSTEM DETAILS ==========
+
   -- Jobs begin as "pending", then transition to other states.
   --
   --  * Pending = job is ready to go
@@ -88,6 +99,7 @@ CREATE TABLE tts_model_upload_jobs (
 
   -- INDICES --
   PRIMARY KEY (id),
+  UNIQUE KEY (token),
   UNIQUE KEY (uuid_idempotency_token),
   KEY fk_creator_user_token (creator_user_token),
   KEY fk_maybe_subject_token (maybe_subject_token),
@@ -101,24 +113,15 @@ CREATE TABLE tts_model_upload_jobs (
 CREATE TABLE w2l_template_upload_jobs (
   id BIGINT(20) NOT NULL AUTO_INCREMENT,
 
+  -- Effective "primary key" (PUBLIC)
+  -- This is so the in-progress results can be looked up by the UI.
+  token VARCHAR(32) NOT NULL,
+
   -- Idempotency token
+  -- This is so the frontend client doesn't submit duplicate jobs.
   uuid_idempotency_token VARCHAR(36) NOT NULL,
 
-  -- Foreign key to user
-  creator_user_token VARCHAR(32) NOT NULL,
-
-  -- For abuse tracking.
-  -- Wide enough for IPv4/6
-  creator_ip_address VARCHAR(40) NOT NULL,
-
-  -- (THIS MIGHT NOT BE USED)
-  -- NB: DO NOT SORT!
-  -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE.
-  creator_set_visibility ENUM(
-      'public',
-      'hidden',
-      'private'
-  ) NOT NULL DEFAULT 'public',
+  -- ========== UPLOAD DETAILS ==========
 
   -- The title of the template
   title VARCHAR(255) NOT NULL,
@@ -144,6 +147,26 @@ CREATE TABLE w2l_template_upload_jobs (
       'web'
   ) NOT NULL DEFAULT 'web',
 
+  -- ========== CREATOR DETAILS AND PREFERENCES ==========
+
+  -- Foreign key to user
+  creator_user_token VARCHAR(32) NOT NULL,
+
+  -- For abuse tracking.
+  -- Wide enough for IPv4/6
+  creator_ip_address VARCHAR(40) NOT NULL,
+
+  -- (THIS MIGHT NOT BE USED)
+  -- NB: DO NOT SORT!
+  -- THIS MUST MATCH THE RESPECTIVE JOBS TABLE.
+  creator_set_visibility ENUM(
+      'public',
+      'hidden',
+      'private'
+  ) NOT NULL DEFAULT 'public',
+
+  -- ========== JOB SYSTEM DETAILS ==========
+
   -- Jobs begin as "pending", then transition to other states.
   --
   --  * Pending = job is ready to go
@@ -181,6 +204,7 @@ CREATE TABLE w2l_template_upload_jobs (
 
   -- INDICES --
   PRIMARY KEY (id),
+  UNIQUE KEY (token),
   UNIQUE KEY (uuid_idempotency_token),
   KEY fk_creator_user_token (creator_user_token),
   KEY fk_maybe_subject_token (maybe_subject_token),

migrations/2021-05-15-212913_inference_jobs/up.sql

@@ -4,9 +4,11 @@ CREATE TABLE tts_inference_jobs (
   id BIGINT(20) NOT NULL AUTO_INCREMENT,
 
   -- Effective "primary key" (PUBLIC)
+  -- This is so the in-progress results can be looked up by the UI.
   token VARCHAR(32) NOT NULL,
 
   -- Idempotency token from client
+  -- This is so the frontend client doesn't submit duplicate jobs.
   uuid_idempotency_token VARCHAR(36) NOT NULL,
 
   -- ========== INFERENCE DETAILS ==========
@@ -40,11 +42,26 @@ CREATE TABLE tts_inference_jobs (
   -- ========== JOB SYSTEM DETAILS ==========
 
   -- Jobs begin as "pending", then transition to other states.
-  -- Pending -> Started -> Complete
-  --                    \-> Failed -> Started -> { Complete, Failed, Dead }
-  status ENUM('pending', 'started', 'complete', 'failed', 'dead') NOT NULL DEFAULT 'pending',
+  --
+  --  * Pending = job is ready to go
+  --  * Started = job is running
+  --  * Complete_Success = job is done (success)
+  --  * Complete_Failure = job is done (failure)
+  --  * Attempt_Failed = job failed but may retry.
+  --  * Dead = job failed permanently.
+  --
+  -- Pending -> Started -> Complete_Success
+  --                    |-> Complete_Failure
+  --                    \-> Attempt_Failed -> Started -> { Complete, Failed, Dead }
+  status ENUM(
+      'pending',
+      'started',
+      'complete_success',
+      'complete_failure',
+      'attempt_failed',
+      'dead') NOT NULL DEFAULT 'pending',
 
-  -- We can track this against a "max_attempt_count"
+    -- We can track this against a "max_attempt_count"
   attempt_count INT(3) NOT NULL DEFAULT 0,
 
   -- If there is a failure, tell the user why.
@@ -74,9 +91,11 @@ CREATE TABLE w2l_inference_jobs (
   id BIGINT(20) NOT NULL AUTO_INCREMENT,
 
   -- Effective "primary key" (PUBLIC)
+  -- This is so the in-progress results can be looked up by the UI.
   token VARCHAR(32) NOT NULL,
 
   -- Idempotency token from client
+  -- This is so the frontend client doesn't submit duplicate jobs.
   uuid_idempotency_token VARCHAR(36) NOT NULL,
 
   -- ========== INFERENCE DETAILS : FACE TEMPLATE ==========
@@ -107,7 +126,7 @@ CREATE TABLE w2l_inference_jobs (
 
   maybe_audio_mime_type VARCHAR(32) DEFAULT NULL,
 
-  -- ========== CREATOR DETAILS ==========
+  -- ========== CREATOR DETAILS AND PREFERENCES ==========
 
   -- Foreign key to user
   -- If no user is logged in, this is null.
@@ -133,9 +152,24 @@ CREATE TABLE w2l_inference_jobs (
   -- ========== JOB SYSTEM DETAILS ==========
 
   -- Jobs begin as "pending", then transition to other states.
-  -- Pending -> Started -> Complete
-  --                    \-> Failed -> Started -> { Complete, Failed, Dead }
-  status ENUM('pending', 'started', 'complete', 'failed', 'dead') NOT NULL DEFAULT 'pending',
+  --
+  --  * Pending = job is ready to go
+  --  * Started = job is running
+  --  * Complete_Success = job is done (success)
+  --  * Complete_Failure = job is done (failure)
+  --  * Attempt_Failed = job failed but may retry.
+  --  * Dead = job failed permanently.
+  --
+  -- Pending -> Started -> Complete_Success
+  --                    |-> Complete_Failure
+  --                    \-> Attempt_Failed -> Started -> { Complete, Failed, Dead }
+  status ENUM(
+      'pending',
+      'started',
+      'complete_success',
+      'complete_failure',
+      'attempt_failed',
+      'dead') NOT NULL DEFAULT 'pending',
 
   -- We can track this against a "max_attempt_count"
   attempt_count INT(3) NOT NULL DEFAULT 0,

src/http_server/endpoints/tts/enqueue_upload_tts_model.rs

@@ -19,6 +19,7 @@ use sqlx::error::DatabaseError;
 use sqlx::error::Error::Database;
 use sqlx::mysql::MySqlDatabaseError;
 use std::sync::Arc;
+use crate::util::random_prefix_crockford_token::random_prefix_crockford_token;
 
 const NEW_USER_ROLE: &'static str = "new-user";
 
@@ -37,6 +38,8 @@ pub struct UploadTtsModelRequest {
 #[derive(Serialize)]
 pub struct UploadTtsModelSuccessResponse {
   pub success: bool,
+  /// This is how frontend clients can request the job execution status.
+  pub job_token: String,
 }
 
 #[derive(Serialize)]
@@ -126,10 +129,19 @@ pub async fn upload_tts_model_handler(
   let maybe_subject_token : Option<String> = None;
   let maybe_actor_subject_token : Option<String> = None;
 
+  // This token is returned to the client.
+  let job_token = random_prefix_crockford_token("TTS_UP:", 32)
+    .map_err(|e| {
+      warn!("Error creating token");
+      UploadTtsModelError::ServerError
+    })?;
+
+
   let query_result = sqlx::query!(
         r#"
 INSERT INTO tts_model_upload_jobs
 SET
+  token = ?,
   uuid_idempotency_token = ?,
   creator_user_token = ?,
   creator_ip_address = ?,
@@ -142,6 +154,7 @@ SET
   download_url_type = ?,
   status = "pending"
         "#,
+        job_token.to_string(),
         uuid.to_string(),
         user_session.user_token.to_string(),
         ip_address.to_string(),
@@ -189,6 +202,7 @@ SET
 
   let response = UploadTtsModelSuccessResponse {
     success: true,
+    job_token: job_token.to_string(),
   };
 
   let body = serde_json::to_string(&response)

src/http_server/endpoints/w2l/enqueue_infer_w2l_with_uploads.rs

@@ -47,6 +47,7 @@ pub struct W2lTemplateExistenceRecord {
 #[derive(Serialize)]
 pub struct InferW2lWithUploadSuccessResponse {
   pub success: bool,
+  /// This is how frontend clients can request the job execution status.
   pub job_token: String,
 }
 

src/http_server/endpoints/w2l/enqueue_upload_w2l_template.rs

@@ -19,6 +19,7 @@ use sqlx::error::DatabaseError;
 use sqlx::error::Error::Database;
 use sqlx::mysql::MySqlDatabaseError;
 use std::sync::Arc;
+use crate::util::random_prefix_crockford_token::random_prefix_crockford_token;
 
 const NEW_USER_ROLE: &'static str = "new-user";
 
@@ -37,6 +38,8 @@ pub struct UploadW2lTemplateRequest {
 #[derive(Serialize)]
 pub struct UploadW2lTemplateSuccessResponse {
   pub success: bool,
+  /// This is how frontend clients can request the job execution status.
+  pub job_token: String,
 }
 
 #[derive(Serialize)]
@@ -126,10 +129,19 @@ pub async fn upload_w2l_template_handler(
   let maybe_subject_token : Option<String> = None;
   let maybe_actor_subject_token : Option<String> = None;
 
+  // This token is returned to the client.
+  let job_token = random_prefix_crockford_token("W2L_UP:", 32)
+    .map_err(|e| {
+      warn!("Error creating token");
+      UploadW2lTemplateError::ServerError
+    })?;
+
+
   let query_result = sqlx::query!(
         r#"
 INSERT INTO w2l_template_upload_jobs
 SET
+  token = ?,
   uuid_idempotency_token = ?,
   creator_user_token = ?,
   creator_ip_address = ?,
@@ -142,6 +154,7 @@ SET
   download_url_type = ?,
   status = "pending"
         "#,
+        job_token.to_string(),
         uuid.to_string(),
         user_session.user_token.to_string(),
         ip_address.to_string(),
@@ -189,6 +202,7 @@ SET
 
   let response = UploadW2lTemplateSuccessResponse {
     success: true,
+    job_token: job_token.to_string(),
   };
 
   let body = serde_json::to_string(&response)

src/job_queries/tts_download_job_queries.rs

@@ -13,6 +13,7 @@ use crate::util::random_prefix_crockford_token::random_prefix_crockford_token;
 #[derive(Debug)]
 pub struct TtsUploadJobRecord {
   pub id: i64,
+  pub token: String,
   pub uuid_idempotency_token: String,
   pub creator_user_token: String,
   pub creator_ip_address: String,

src/job_queries/w2l_download_job_queries.rs

@@ -13,6 +13,7 @@ use crate::util::random_prefix_crockford_token::random_prefix_crockford_token;
 #[derive(Debug)]
 pub struct W2lTemplateUploadJobRecord {
   pub id: i64,
+  pub token: String,
   pub uuid_idempotency_token: String,
   pub creator_user_token: String,
   pub creator_ip_address: String,

src/w2l_inference_job.rs

@@ -494,10 +494,11 @@ async fn process_job(inferencer: &Inferencer, job: &W2lInferenceJobRecord) -> An
     file_metadata.duration_millis.unwrap_or(0))
     .await?;
 
+  info!("Marking job complete...");
+  mark_w2l_inference_job_done(&inferencer.mysql_pool, job, true).await?;
+
   info!("Job {} complete success! Downloaded, ran inference, and uploaded. Saved model record: {}",
         job.id, id);
 
-  mark_w2l_inference_job_done(&inferencer.mysql_pool, job, true).await?;
-
   Ok(())
 }