diff --git a/.idea/dictionaries/bt.xml b/.idea/dictionaries/bt.xml index f13dace540..454e03ff9e 100644 --- a/.idea/dictionaries/bt.xml +++ b/.idea/dictionaries/bt.xml @@ -23,6 +23,7 @@ retryably scrollback softvc + sql's symphonia tacotron techcrunch diff --git a/crates/lib/mysql_queries/sqlx-data.json b/crates/lib/mysql_queries/sqlx-data.json index af68459741..e9d7e717b8 100644 --- a/crates/lib/mysql_queries/sqlx-data.json +++ b/crates/lib/mysql_queries/sqlx-data.json @@ -2722,6 +2722,16 @@ }, "query": "\nUPDATE user_sessions\nSET deleted_at = CURRENT_TIMESTAMP()\nWHERE\n token = ?\n AND deleted_at IS NULL\n " }, + "2177f29265040bda6bb4418b521679fcbc372a73620415f4812c45fbb03d5933": { + "describe": { + "columns": [], + "nullable": [], + "parameters": { + "Right": 4 + } + }, + "query": "\nUPDATE generic_download_jobs\nSET\n status = \"complete_failure\",\n on_success_downloaded_entity_token = ?,\n on_success_downloaded_entity_type = ?,\n failure_reason = NULL,\n success_execution_millis = ?,\n retry_at = NULL\nWHERE id = ?\n " + }, "218b3fe667b34bfa158bb7d4f600a3b91a22460adf566dcdf6ce540006e6c684": { "describe": { "columns": [], @@ -3632,6 +3642,16 @@ }, "query": "\nUPDATE\n ip_address_bans\nSET\n mod_user_token = ?,\n deleted_at = NULL\nWHERE\n ip_address = ?\nLIMIT 1\n " }, + "348d5965a6e4038cbe00acdf0ad1eb1d26dc85d70834c6c86f802daa5a2d7794": { + "describe": { + "columns": [], + "nullable": [], + "parameters": { + "Right": 4 + } + }, + "query": "\nUPDATE generic_inference_jobs\nSET\n status = \"complete_success\",\n on_success_result_entity_type = ?,\n on_success_result_entity_token = ?,\n failure_reason = NULL,\n internal_debugging_failure_reason = NULL,\n success_execution_millis = ?,\n retry_at = NULL,\n successfully_completed_at = NOW()\nWHERE id = ?\n " + }, "359be6ce263afd62fc748c14c06c62ded5ed59fc0a9d1186967028af400d7d61": { "describe": { "columns": [ @@ -3927,16 +3947,6 @@ }, "query": "\nUPDATE generic_inference_jobs\nSET\n status = ?,\n failure_reason = ?,\n internal_debugging_failure_reason = ?,\n retry_at = NOW() + interval 2 minute\nWHERE id = ?\n " }, - "3e0c1b88d7c4f1b4d754ee2c6e32b99e72ac810d0da2a7e57fbf2aa8e9a9939a": { - "describe": { - "columns": [], - "nullable": [], - "parameters": { - "Right": 3 - } - }, - "query": "\nUPDATE generic_inference_jobs\nSET\n status = \"complete_success\",\n on_success_result_entity_type = ?,\n on_success_result_entity_token = ?,\n failure_reason = NULL,\n internal_debugging_failure_reason = NULL,\n retry_at = NULL,\n successfully_completed_at = NOW()\nWHERE id = ?\n " - }, "40a3301e9a3201e74d49b6b339f67291e2ae22d315188401708609ac2d28a68c": { "describe": { "columns": [], @@ -9354,16 +9364,6 @@ }, "query": "\nSELECT *\nFROM tts_model_upload_jobs\nWHERE\n (\n status IN (\"pending\", \"attempt_failed\")\n )\n AND\n (\n retry_at IS NULL\n OR\n retry_at < CURRENT_TIMESTAMP\n )\n LIMIT ?\n " }, - "7b2cbcc19549a9fa4882c57a52ad39a3f7fd8f65e31f487553cea2158b4e7208": { - "describe": { - "columns": [], - "nullable": [], - "parameters": { - "Right": 3 - } - }, - "query": "\nUPDATE generic_download_jobs\nSET\n status = \"complete_success\",\n on_success_downloaded_entity_token = ?,\n on_success_downloaded_entity_type = ?,\n failure_reason = NULL,\n retry_at = NULL,\n successfully_completed_at = NOW()\nWHERE id = ?\n " - }, "7cd0ad5d7b9a2c393f799beadc1e0a6bc85d952e07a1b82a52115b99be40ff6d": { "describe": { "columns": [ @@ -10459,16 +10459,6 @@ }, "query": "\nINSERT INTO stripe_webhook_event_logs\nSET\n stripe_event_id = ?,\n stripe_event_type = ?,\n maybe_stripe_event_entity_id = ?,\n maybe_stripe_customer_id = ?,\n stripe_event_created_at = ?,\n stripe_is_production = ?,\n maybe_user_token = ?,\n action_was_taken = ?,\n should_ignore_retry = ?\n " }, - "8951eb0c858fc803312db719820e8a2ae1446a9c741dc8a6874b8ae213793cfa": { - "describe": { - "columns": [], - "nullable": [], - "parameters": { - "Right": 3 - } - }, - "query": "\nUPDATE generic_download_jobs\nSET\n status = \"complete_failure\",\n on_success_downloaded_entity_token = ?,\n on_success_downloaded_entity_type = ?,\n failure_reason = NULL,\n retry_at = NULL\nWHERE id = ?\n " - }, "89b66d91d1f2cd5c9871671b21a90e77d9041572a90cc6be6dd49845424677fa": { "describe": { "columns": [], @@ -11072,6 +11062,16 @@ }, "query": "\nSELECT\n vc.token as `token: tokens::voice_conversion::model::VoiceConversionModelToken`,\n vc.model_type as `model_type: enums::by_table::voice_conversion_models::voice_conversion_model_type::VoiceConversionModelType`,\n\n vc.title,\n vc.private_bucket_hash,\n\n vc.created_at,\n vc.updated_at,\n vc.user_deleted_at,\n vc.mod_deleted_at\n\nFROM voice_conversion_models as vc\n\nWHERE vc.token = ?\n " }, + "9602cbbd0eac176d596ee2b66695274bffbf12947848d3bc9d75e1a9553d8934": { + "describe": { + "columns": [], + "nullable": [], + "parameters": { + "Right": 4 + } + }, + "query": "\nUPDATE generic_download_jobs\nSET\n status = \"complete_success\",\n on_success_downloaded_entity_token = ?,\n on_success_downloaded_entity_type = ?,\n failure_reason = NULL,\n success_execution_millis = ?,\n retry_at = NULL,\n successfully_completed_at = NOW()\nWHERE id = ?\n " + }, "961f792c24bee6009e3e8a215561db932871a53c48839b8cb80292de00e7b2e1": { "describe": { "columns": [], diff --git a/crates/lib/mysql_queries/src/queries/generic_download/job/mark_generic_download_job_done.rs b/crates/lib/mysql_queries/src/queries/generic_download/job/mark_generic_download_job_done.rs index 3df2385632..c23504bc29 100644 --- a/crates/lib/mysql_queries/src/queries/generic_download/job/mark_generic_download_job_done.rs +++ b/crates/lib/mysql_queries/src/queries/generic_download/job/mark_generic_download_job_done.rs @@ -1,7 +1,8 @@ use anyhow::anyhow; -use errors::AnyhowResult; use crate::queries::generic_download::job::list_available_generic_download_jobs::AvailableDownloadJob; +use errors::AnyhowResult; use sqlx::MySqlPool; +use std::time::Duration; pub async fn mark_generic_download_job_done( pool: &MySqlPool, @@ -9,8 +10,13 @@ pub async fn mark_generic_download_job_done( success: bool, maybe_entity_token: Option<&str>, maybe_entity_type: Option<&str>, + job_duration: Duration, ) -> AnyhowResult<()> { + // NB: MySql's unsigned int (32 bits) can store integers up to 4,294,967,295. + // Given milliseconds, this is ~49.71 days, which should be plenty for us. + let truncated_execution_millis = job_duration.as_millis() as u32; + let query_result = if success { sqlx::query!( r#" @@ -20,12 +26,14 @@ SET on_success_downloaded_entity_token = ?, on_success_downloaded_entity_type = ?, failure_reason = NULL, + success_execution_millis = ?, retry_at = NULL, successfully_completed_at = NOW() WHERE id = ? "#, maybe_entity_token, maybe_entity_type, + truncated_execution_millis, job.id.0, ) .execute(pool) @@ -39,11 +47,13 @@ SET on_success_downloaded_entity_token = ?, on_success_downloaded_entity_type = ?, failure_reason = NULL, + success_execution_millis = ?, retry_at = NULL WHERE id = ? "#, maybe_entity_token, maybe_entity_type, + truncated_execution_millis, job.id.0, ) .execute(pool) diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/job/mark_generic_inference_job_successfully_done.rs b/crates/lib/mysql_queries/src/queries/generic_inference/job/mark_generic_inference_job_successfully_done.rs index e684c7136c..69f369e011 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/job/mark_generic_inference_job_successfully_done.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/job/mark_generic_inference_job_successfully_done.rs @@ -3,14 +3,20 @@ use crate::queries::generic_inference::job::list_available_generic_inference_job use enums::by_table::generic_inference_jobs::inference_result_type::InferenceResultType; use errors::AnyhowResult; use sqlx::MySqlPool; +use std::time::Duration; pub async fn mark_generic_inference_job_successfully_done( pool: &MySqlPool, job: &AvailableInferenceJob, maybe_entity_type: Option, maybe_entity_token: Option<&str>, + job_duration: Duration, ) -> AnyhowResult<()> { + // NB: MySql's unsigned int (32 bits) can store integers up to 4,294,967,295. + // Given milliseconds, this is ~49.71 days, which should be plenty for us. + let truncated_execution_millis = job_duration.as_millis() as u32; + let query_result = sqlx::query!( r#" UPDATE generic_inference_jobs @@ -20,12 +26,14 @@ SET on_success_result_entity_token = ?, failure_reason = NULL, internal_debugging_failure_reason = NULL, + success_execution_millis = ?, retry_at = NULL, successfully_completed_at = NOW() WHERE id = ? "#, maybe_entity_type, maybe_entity_token, + truncated_execution_millis, job.id.0, ) .execute(pool) diff --git a/crates/service/job/download_job/src/job_loop/process_single_job.rs b/crates/service/job/download_job/src/job_loop/process_single_job.rs index 464c494762..804394060c 100644 --- a/crates/service/job/download_job/src/job_loop/process_single_job.rs +++ b/crates/service/job/download_job/src/job_loop/process_single_job.rs @@ -1,3 +1,4 @@ +use std::time::Instant; use anyhow::anyhow; use config::is_bad_download_url::is_bad_download_url; use container_common::anyhow_result::AnyhowResult; @@ -30,6 +31,8 @@ pub async fn process_single_job(job_state: &JobState, job: &AvailableDownloadJob info!("Beginning work on {:?} = {} | {}", job.download_type, job.download_job_token, job.download_url); + let job_start_time = Instant::now(); + // ==================== SETUP TEMP DIRS ==================== // let temp_dir = format!("temp_{}", job.id.0); @@ -77,13 +80,19 @@ pub async fn process_single_job(job_state: &JobState, job: &AvailableDownloadJob // ===================================================== + let job_duration = Instant::now().duration_since(job_start_time); + + info!("Job took duration to complete: {:?}", &job_duration); + info!("Marking job complete..."); + mark_generic_download_job_done( &job_state.mysql_pool, job, true, entity_token.as_deref(), entity_type.as_deref(), + job_duration, ).await?; info!("Saved model record: {} - {}", job.id.0, &job.download_job_token); diff --git a/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs b/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs index d0451b7242..64b763637c 100644 --- a/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs +++ b/crates/service/job/inference_job/src/job/job_loop/process_single_job.rs @@ -1,3 +1,4 @@ +use std::time::Instant; use anyhow::anyhow; use crate::job::job_loop::process_single_job_error::ProcessSingleJobError; use crate::job::job_types::tts::process_single_tts_job::process_single_tts_job; @@ -33,6 +34,8 @@ pub async fn process_single_job(job_dependencies: &JobDependencies, job: &Availa info!("Beginning work on {:?} = {}", job.inference_category, job.inference_job_token); + let job_start_time = Instant::now(); + // ==================== SETUP TEMP DIRS ==================== // let temp_dir = format!("temp_{}", job.id.0); @@ -64,6 +67,10 @@ pub async fn process_single_job(job_dependencies: &JobDependencies, job: &Availa // ===================================================== + let job_duration = Instant::now().duration_since(job_start_time); + + info!("Job took duration to complete: {:?}", &job_duration); + info!("Marking job complete..."); mark_generic_inference_job_successfully_done( @@ -71,6 +78,7 @@ pub async fn process_single_job(job_dependencies: &JobDependencies, job: &Availa job, maybe_entity_type, maybe_entity_token, + job_duration, ).await .map_err(|err| ProcessSingleJobError::Other(anyhow!("database error: {:?}", err)))?;