diff --git a/.sqlx/query-7d5030b221e9b470ee7af4b73539fd2ef477e6f3c98c56422016449c6775222d.json b/.sqlx/query-e5121d132d93ec43e48197e00b19040a50ff2942224dc70b0faa0ec988aee7cf.json similarity index 90% rename from .sqlx/query-7d5030b221e9b470ee7af4b73539fd2ef477e6f3c98c56422016449c6775222d.json rename to .sqlx/query-e5121d132d93ec43e48197e00b19040a50ff2942224dc70b0faa0ec988aee7cf.json index 322567181b..fd60892c18 100644 --- a/.sqlx/query-7d5030b221e9b470ee7af4b73539fd2ef477e6f3c98c56422016449c6775222d.json +++ b/.sqlx/query-e5121d132d93ec43e48197e00b19040a50ff2942224dc70b0faa0ec988aee7cf.json @@ -1,6 +1,6 @@ { "db_name": "MySQL", - "query": "\nSELECT\n jobs.token as `job_token: tokens::tokens::generic_inference_jobs::InferenceJobToken`,\n\n jobs.status as `status: enums::common::job_status_plus::JobStatusPlus`,\n jobs.attempt_count,\n\n jobs.maybe_creator_user_token as `maybe_creator_user_token: tokens::tokens::users::UserToken`,\n jobs.maybe_creator_anonymous_visitor_token as `maybe_creator_anonymous_visitor_token: tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken`,\n jobs.creator_ip_address,\n\n jobs.inference_category as `inference_category: enums::by_table::generic_inference_jobs::inference_category::InferenceCategory`,\n jobs.maybe_model_type,\n jobs.maybe_model_token,\n jobs.maybe_raw_inference_text,\n jobs.maybe_inference_args,\n\n jobs.frontend_failure_category as `maybe_frontend_failure_category: enums::by_table::generic_inference_jobs::frontend_failure_category::FrontendFailureCategory`,\n\n jobs.on_success_result_entity_type as maybe_result_entity_type,\n jobs.on_success_result_entity_token as maybe_result_entity_token,\n\n model_weights.title as maybe_model_weights_title,\n tts_models.title as maybe_tts_model_title,\n voice_conversion_models.title as maybe_voice_conversion_model_title,\n\n tts_results.public_bucket_wav_audio_path as maybe_tts_public_bucket_path,\n voice_conversion_results.public_bucket_hash as maybe_voice_conversion_public_bucket_hash,\n\n media_files.public_bucket_directory_hash as maybe_media_file_public_bucket_directory_hash,\n media_files.maybe_public_bucket_prefix as maybe_media_file_public_bucket_prefix,\n media_files.maybe_public_bucket_extension as maybe_media_file_public_bucket_extension,\n\n jobs.assigned_worker as maybe_assigned_worker,\n jobs.assigned_cluster as maybe_assigned_cluster,\n\n jobs.is_keepalive_required,\n\n jobs.created_at,\n jobs.updated_at,\n\n jobs.first_started_at as maybe_first_started_at,\n jobs.successfully_completed_at as maybe_successfully_completed_at\n\nFROM generic_inference_jobs as jobs\n\nLEFT OUTER JOIN model_weights ON jobs.maybe_model_token = model_weights.token\nLEFT OUTER JOIN tts_models ON jobs.maybe_model_token = tts_models.token\nLEFT OUTER JOIN voice_conversion_models ON jobs.maybe_model_token = voice_conversion_models.token\n\nLEFT OUTER JOIN tts_results ON jobs.on_success_result_entity_token = tts_results.token\nLEFT OUTER JOIN voice_conversion_results ON jobs.on_success_result_entity_token = voice_conversion_results.token\nLEFT OUTER JOIN media_files ON jobs.on_success_result_entity_token = media_files.token\n\nWHERE jobs.token = ?\n ", + "query": "\nSELECT\n jobs.token as `job_token: tokens::tokens::generic_inference_jobs::InferenceJobToken`,\n\n jobs.status as `status: enums::common::job_status_plus::JobStatusPlus`,\n jobs.attempt_count,\n\n jobs.maybe_creator_user_token as `maybe_creator_user_token: tokens::tokens::users::UserToken`,\n jobs.maybe_creator_anonymous_visitor_token as `maybe_creator_anonymous_visitor_token: tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken`,\n jobs.creator_ip_address,\n\n jobs.inference_category as `inference_category: enums::by_table::generic_inference_jobs::inference_category::InferenceCategory`,\n jobs.maybe_model_type,\n jobs.maybe_model_token,\n jobs.maybe_raw_inference_text,\n jobs.maybe_inference_args,\n\n jobs.frontend_failure_category as `maybe_frontend_failure_category: enums::by_table::generic_inference_jobs::frontend_failure_category::FrontendFailureCategory`,\n\n jobs.on_success_result_entity_type as maybe_result_entity_type,\n jobs.on_success_result_entity_token as maybe_result_entity_token,\n\n model_weights.title as maybe_model_weights_title,\n tts_models.title as maybe_tts_model_title,\n voice_conversion_models.title as maybe_voice_conversion_model_title,\n\n tts_results.public_bucket_wav_audio_path as maybe_tts_public_bucket_path,\n voice_conversion_results.public_bucket_hash as maybe_voice_conversion_public_bucket_hash,\n\n media_files.public_bucket_directory_hash as maybe_media_file_public_bucket_directory_hash,\n media_files.maybe_public_bucket_prefix as maybe_media_file_public_bucket_prefix,\n media_files.maybe_public_bucket_extension as maybe_media_file_public_bucket_extension,\n\n jobs.assigned_worker as maybe_assigned_worker,\n jobs.assigned_cluster as maybe_assigned_cluster,\n\n jobs.is_keepalive_required,\n\n jobs.created_at,\n jobs.updated_at,\n\n jobs.first_started_at as maybe_first_started_at,\n jobs.successfully_completed_at as maybe_successfully_completed_at,\n\n NOW() as database_clock\n\nFROM generic_inference_jobs as jobs\n\nLEFT OUTER JOIN model_weights ON jobs.maybe_model_token = model_weights.token\nLEFT OUTER JOIN tts_models ON jobs.maybe_model_token = tts_models.token\nLEFT OUTER JOIN voice_conversion_models ON jobs.maybe_model_token = voice_conversion_models.token\n\nLEFT OUTER JOIN tts_results ON jobs.on_success_result_entity_token = tts_results.token\nLEFT OUTER JOIN voice_conversion_results ON jobs.on_success_result_entity_token = voice_conversion_results.token\nLEFT OUTER JOIN media_files ON jobs.on_success_result_entity_token = media_files.token\n\nWHERE jobs.token = ?\n ", "describe": { "columns": [ { @@ -292,6 +292,16 @@ "char_set": 63, "max_size": 19 } + }, + { + "ordinal": 29, + "name": "database_clock", + "type_info": { + "type": "Datetime", + "flags": "NOT_NULL | BINARY", + "char_set": 63, + "max_size": 19 + } } ], "parameters": { @@ -326,8 +336,9 @@ false, false, true, - true + true, + false ] }, - "hash": "7d5030b221e9b470ee7af4b73539fd2ef477e6f3c98c56422016449c6775222d" + "hash": "e5121d132d93ec43e48197e00b19040a50ff2942224dc70b0faa0ec988aee7cf" } diff --git a/crates/cli/production/elasticsearch-cli/src/plans/media_files/test_search_media_files_documents.rs b/crates/cli/production/elasticsearch-cli/src/plans/media_files/test_search_media_files_documents.rs index 3c11b92fbb..9315a97fbb 100644 --- a/crates/cli/production/elasticsearch-cli/src/plans/media_files/test_search_media_files_documents.rs +++ b/crates/cli/production/elasticsearch-cli/src/plans/media_files/test_search_media_files_documents.rs @@ -11,6 +11,8 @@ pub async fn test_search_media_files(client: &Elasticsearch) -> AnyhowResult<()> let results = search_media_files(SearchArgs { search_term: "cac", + is_featured: None, + maybe_creator_user_token: None, maybe_media_classes: Some(HashSet::from_iter(vec![ MediaFileClass::Dimensional, MediaFileClass::Image, diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/web/batch_get_inference_job_status.rs b/crates/lib/mysql_queries/src/queries/generic_inference/web/batch_get_inference_job_status.rs index 7a6443290b..f592bd9b4d 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/web/batch_get_inference_job_status.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/web/batch_get_inference_job_status.rs @@ -115,7 +115,9 @@ SELECT jobs.updated_at, jobs.first_started_at as maybe_first_started_at, - jobs.successfully_completed_at as maybe_successfully_completed_at + jobs.successfully_completed_at as maybe_successfully_completed_at, + + NOW() as database_clock FROM generic_inference_jobs as jobs @@ -241,6 +243,7 @@ fn raw_records_to_public_result(records: Vec) -> V is_keepalive_required: i8_to_bool(record.is_keepalive_required), created_at: record.created_at, updated_at: record.updated_at, + database_clock: record.database_clock, } }) .collect::>() @@ -288,6 +291,8 @@ struct RawGenericInferenceJobStatus { pub maybe_first_started_at: Option>, pub maybe_successfully_completed_at: Option>, + + pub database_clock: DateTime, } // NB(bt,2023-12-05): There's an issue with type hinting in the `as` clauses with QueryBuilder (or @@ -334,6 +339,7 @@ impl FromRow<'_, MySqlRow> for RawGenericInferenceJobStatus { maybe_successfully_completed_at: row.try_get("maybe_successfully_completed_at")?, created_at: row.try_get("created_at")?, updated_at: row.try_get("updated_at")?, + database_clock: row.try_get("database_clock")?, }) } } diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs b/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs index eed6ec4c2d..4f435e91cf 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/web/get_inference_job_status.rs @@ -1,5 +1,5 @@ use anyhow::anyhow; -use chrono::{DateTime, Utc}; +use chrono::{DateTime, NaiveDateTime, TimeZone, Utc}; use log::warn; use sqlx::{MySql, MySqlPool}; use sqlx::pool::PoolConnection; @@ -80,7 +80,9 @@ SELECT jobs.updated_at, jobs.first_started_at as maybe_first_started_at, - jobs.successfully_completed_at as maybe_successfully_completed_at + jobs.successfully_completed_at as maybe_successfully_completed_at, + + NOW() as database_clock FROM generic_inference_jobs as jobs @@ -223,6 +225,7 @@ fn raw_record_to_public_result(record: RawGenericInferenceJobStatus) -> GenericI is_keepalive_required: i8_to_bool(record.is_keepalive_required), created_at: record.created_at, updated_at: record.updated_at, + database_clock: DateTime::from_naive_utc_and_offset(record.database_clock, Utc), } } @@ -268,6 +271,10 @@ struct RawGenericInferenceJobStatus { pub maybe_first_started_at: Option>, pub maybe_successfully_completed_at: Option>, + + // NB: Typed query can't convert to Utc, so we use NaiveDateTime and do the type conversion ourselves. + // The database server *should* be reporting in UTC. + pub database_clock: NaiveDateTime, } #[cfg(test)] diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/web/job_status.rs b/crates/lib/mysql_queries/src/queries/generic_inference/web/job_status.rs index d81fe0f245..e87bb67648 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/web/job_status.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/web/job_status.rs @@ -32,6 +32,9 @@ pub struct GenericInferenceJobStatus { pub created_at: DateTime, pub updated_at: DateTime, + + /// This is MySQL's NOW(), which will let us do duration math without clock skew. + pub database_clock: DateTime, } /// NB: Serialize is for internal moderator-only endpoints diff --git a/crates/lib/mysql_queries/src/queries/generic_inference/web/list_session_jobs.rs b/crates/lib/mysql_queries/src/queries/generic_inference/web/list_session_jobs.rs index 7e6a11086f..93ee11027e 100644 --- a/crates/lib/mysql_queries/src/queries/generic_inference/web/list_session_jobs.rs +++ b/crates/lib/mysql_queries/src/queries/generic_inference/web/list_session_jobs.rs @@ -167,7 +167,9 @@ SELECT jobs.updated_at, jobs.first_started_at as maybe_first_started_at, - jobs.successfully_completed_at as maybe_successfully_completed_at + jobs.successfully_completed_at as maybe_successfully_completed_at, + + NOW() as database_clock FROM generic_inference_jobs as jobs @@ -293,6 +295,7 @@ fn raw_records_to_public_result(records: Vec) -> V is_keepalive_required: i8_to_bool(record.is_keepalive_required), created_at: record.created_at, updated_at: record.updated_at, + database_clock: record.database_clock, } }) .collect::>() @@ -340,6 +343,8 @@ struct RawGenericInferenceJobStatus { pub maybe_first_started_at: Option>, pub maybe_successfully_completed_at: Option>, + + pub database_clock: DateTime, } // NB(bt,2023-12-05): There's an issue with type hinting in the `as` clauses with QueryBuilder (or @@ -386,6 +391,7 @@ impl FromRow<'_, MySqlRow> for RawGenericInferenceJobStatus { maybe_successfully_completed_at: row.try_get("maybe_successfully_completed_at")?, created_at: row.try_get("created_at")?, updated_at: row.try_get("updated_at")?, + database_clock: row.try_get("database_clock")?, }) } } diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/utils/estimate_job_progress.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/utils/estimate_job_progress.rs index 1a0bfa698b..75dfec7a2e 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/utils/estimate_job_progress.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/inference_job/utils/estimate_job_progress.rs @@ -6,6 +6,7 @@ use enums::by_table::generic_inference_jobs::inference_category::InferenceCatego use enums::common::job_status_plus::JobStatusPlus; use mysql_queries::queries::generic_inference::web::job_status::GenericInferenceJobStatus; +// TODO: These numbers are made up. We should measure the average job durations. const COMFY_JOB_AVERAGE_SECONDS : u64 = 60 * 4; const LIPSYNC_JOB_AVERAGE_SECONDS : u64 = 60 * 3; const TTS_JOB_AVERAGE_SECONDS : u64 = 7; @@ -32,12 +33,10 @@ pub fn estimate_job_progress(job: &GenericInferenceJobStatus) -> u8 { // and perhaps the code should be updated to always write every job attempt's start time. let started_at= job.maybe_first_started_at.unwrap_or(job.created_at); - // TODO: To avoid clock skew, we should select the current time from the database in the - // same query. It should be as easy as selecting another column with `NOW()`. - let now = Utc::now(); + let now = job.database_clock; if started_at < now { - return 0; // Clock skew for a probably newish job. + return 0; // We shouldn't see clock skew unless we read from a DB replica. } let duration = now.signed_duration_since(started_at);