mirror of
https://github.com/storytold/artcraft.git
synced 2026-11-19 16:15:04 +00:00
report time from the DB so as not to introduce clock skew
This commit is contained in:
1 parent
3f59e7ea64
commit
cbe63ed71d
7 files changed
+45
-11
No files matched your search
+14
-3
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"db_name": "MySQL",
|
||||
"query": "\nSELECT\n jobs.token as `job_token: tokens::tokens::generic_inference_jobs::InferenceJobToken`,\n\n jobs.status as `status: enums::common::job_status_plus::JobStatusPlus`,\n jobs.attempt_count,\n\n jobs.maybe_creator_user_token as `maybe_creator_user_token: tokens::tokens::users::UserToken`,\n jobs.maybe_creator_anonymous_visitor_token as `maybe_creator_anonymous_visitor_token: tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken`,\n jobs.creator_ip_address,\n\n jobs.inference_category as `inference_category: enums::by_table::generic_inference_jobs::inference_category::InferenceCategory`,\n jobs.maybe_model_type,\n jobs.maybe_model_token,\n jobs.maybe_raw_inference_text,\n jobs.maybe_inference_args,\n\n jobs.frontend_failure_category as `maybe_frontend_failure_category: enums::by_table::generic_inference_jobs::frontend_failure_category::FrontendFailureCategory`,\n\n jobs.on_success_result_entity_type as maybe_result_entity_type,\n jobs.on_success_result_entity_token as maybe_result_entity_token,\n\n model_weights.title as maybe_model_weights_title,\n tts_models.title as maybe_tts_model_title,\n voice_conversion_models.title as maybe_voice_conversion_model_title,\n\n tts_results.public_bucket_wav_audio_path as maybe_tts_public_bucket_path,\n voice_conversion_results.public_bucket_hash as maybe_voice_conversion_public_bucket_hash,\n\n media_files.public_bucket_directory_hash as maybe_media_file_public_bucket_directory_hash,\n media_files.maybe_public_bucket_prefix as maybe_media_file_public_bucket_prefix,\n media_files.maybe_public_bucket_extension as maybe_media_file_public_bucket_extension,\n\n jobs.assigned_worker as maybe_assigned_worker,\n jobs.assigned_cluster as maybe_assigned_cluster,\n\n jobs.is_keepalive_required,\n\n jobs.created_at,\n jobs.updated_at,\n\n jobs.first_started_at as maybe_first_started_at,\n jobs.successfully_completed_at as maybe_successfully_completed_at\n\nFROM generic_inference_jobs as jobs\n\nLEFT OUTER JOIN model_weights ON jobs.maybe_model_token = model_weights.token\nLEFT OUTER JOIN tts_models ON jobs.maybe_model_token = tts_models.token\nLEFT OUTER JOIN voice_conversion_models ON jobs.maybe_model_token = voice_conversion_models.token\n\nLEFT OUTER JOIN tts_results ON jobs.on_success_result_entity_token = tts_results.token\nLEFT OUTER JOIN voice_conversion_results ON jobs.on_success_result_entity_token = voice_conversion_results.token\nLEFT OUTER JOIN media_files ON jobs.on_success_result_entity_token = media_files.token\n\nWHERE jobs.token = ?\n ",
|
||||
"query": "\nSELECT\n jobs.token as `job_token: tokens::tokens::generic_inference_jobs::InferenceJobToken`,\n\n jobs.status as `status: enums::common::job_status_plus::JobStatusPlus`,\n jobs.attempt_count,\n\n jobs.maybe_creator_user_token as `maybe_creator_user_token: tokens::tokens::users::UserToken`,\n jobs.maybe_creator_anonymous_visitor_token as `maybe_creator_anonymous_visitor_token: tokens::tokens::anonymous_visitor_tracking::AnonymousVisitorTrackingToken`,\n jobs.creator_ip_address,\n\n jobs.inference_category as `inference_category: enums::by_table::generic_inference_jobs::inference_category::InferenceCategory`,\n jobs.maybe_model_type,\n jobs.maybe_model_token,\n jobs.maybe_raw_inference_text,\n jobs.maybe_inference_args,\n\n jobs.frontend_failure_category as `maybe_frontend_failure_category: enums::by_table::generic_inference_jobs::frontend_failure_category::FrontendFailureCategory`,\n\n jobs.on_success_result_entity_type as maybe_result_entity_type,\n jobs.on_success_result_entity_token as maybe_result_entity_token,\n\n model_weights.title as maybe_model_weights_title,\n tts_models.title as maybe_tts_model_title,\n voice_conversion_models.title as maybe_voice_conversion_model_title,\n\n tts_results.public_bucket_wav_audio_path as maybe_tts_public_bucket_path,\n voice_conversion_results.public_bucket_hash as maybe_voice_conversion_public_bucket_hash,\n\n media_files.public_bucket_directory_hash as maybe_media_file_public_bucket_directory_hash,\n media_files.maybe_public_bucket_prefix as maybe_media_file_public_bucket_prefix,\n media_files.maybe_public_bucket_extension as maybe_media_file_public_bucket_extension,\n\n jobs.assigned_worker as maybe_assigned_worker,\n jobs.assigned_cluster as maybe_assigned_cluster,\n\n jobs.is_keepalive_required,\n\n jobs.created_at,\n jobs.updated_at,\n\n jobs.first_started_at as maybe_first_started_at,\n jobs.successfully_completed_at as maybe_successfully_completed_at,\n\n NOW() as database_clock\n\nFROM generic_inference_jobs as jobs\n\nLEFT OUTER JOIN model_weights ON jobs.maybe_model_token = model_weights.token\nLEFT OUTER JOIN tts_models ON jobs.maybe_model_token = tts_models.token\nLEFT OUTER JOIN voice_conversion_models ON jobs.maybe_model_token = voice_conversion_models.token\n\nLEFT OUTER JOIN tts_results ON jobs.on_success_result_entity_token = tts_results.token\nLEFT OUTER JOIN voice_conversion_results ON jobs.on_success_result_entity_token = voice_conversion_results.token\nLEFT OUTER JOIN media_files ON jobs.on_success_result_entity_token = media_files.token\n\nWHERE jobs.token = ?\n ",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
@@ -292,6 +292,16 @@
|
||||
"char_set": 63,
|
||||
"max_size": 19
|
||||
}
|
||||
},
|
||||
{
|
||||
"ordinal": 29,
|
||||
"name": "database_clock",
|
||||
"type_info": {
|
||||
"type": "Datetime",
|
||||
"flags": "NOT_NULL | BINARY",
|
||||
"char_set": 63,
|
||||
"max_size": 19
|
||||
}
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
@@ -326,8 +336,9 @@
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
true
|
||||
true,
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "7d5030b221e9b470ee7af4b73539fd2ef477e6f3c98c56422016449c6775222d"
|
||||
"hash": "e5121d132d93ec43e48197e00b19040a50ff2942224dc70b0faa0ec988aee7cf"
|
||||
}
|
||||
+2
@@ -11,6 +11,8 @@ pub async fn test_search_media_files(client: &Elasticsearch) -> AnyhowResult<()>
|
||||
|
||||
let results = search_media_files(SearchArgs {
|
||||
search_term: "cac",
|
||||
is_featured: None,
|
||||
maybe_creator_user_token: None,
|
||||
maybe_media_classes: Some(HashSet::from_iter(vec![
|
||||
MediaFileClass::Dimensional,
|
||||
MediaFileClass::Image,
|
||||
|
||||
+7
-1
@@ -115,7 +115,9 @@ SELECT
|
||||
jobs.updated_at,
|
||||
|
||||
jobs.first_started_at as maybe_first_started_at,
|
||||
jobs.successfully_completed_at as maybe_successfully_completed_at
|
||||
jobs.successfully_completed_at as maybe_successfully_completed_at,
|
||||
|
||||
NOW() as database_clock
|
||||
|
||||
FROM generic_inference_jobs as jobs
|
||||
|
||||
@@ -241,6 +243,7 @@ fn raw_records_to_public_result(records: Vec<RawGenericInferenceJobStatus>) -> V
|
||||
is_keepalive_required: i8_to_bool(record.is_keepalive_required),
|
||||
created_at: record.created_at,
|
||||
updated_at: record.updated_at,
|
||||
database_clock: record.database_clock,
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
@@ -288,6 +291,8 @@ struct RawGenericInferenceJobStatus {
|
||||
|
||||
pub maybe_first_started_at: Option<DateTime<Utc>>,
|
||||
pub maybe_successfully_completed_at: Option<DateTime<Utc>>,
|
||||
|
||||
pub database_clock: DateTime<Utc>,
|
||||
}
|
||||
|
||||
// NB(bt,2023-12-05): There's an issue with type hinting in the `as` clauses with QueryBuilder (or
|
||||
@@ -334,6 +339,7 @@ impl FromRow<'_, MySqlRow> for RawGenericInferenceJobStatus {
|
||||
maybe_successfully_completed_at: row.try_get("maybe_successfully_completed_at")?,
|
||||
created_at: row.try_get("created_at")?,
|
||||
updated_at: row.try_get("updated_at")?,
|
||||
database_clock: row.try_get("database_clock")?,
|
||||
})
|
||||
}
|
||||
}
|
||||
+9
-2
@@ -1,5 +1,5 @@
|
||||
use anyhow::anyhow;
|
||||
use chrono::{DateTime, Utc};
|
||||
use chrono::{DateTime, NaiveDateTime, TimeZone, Utc};
|
||||
use log::warn;
|
||||
use sqlx::{MySql, MySqlPool};
|
||||
use sqlx::pool::PoolConnection;
|
||||
@@ -80,7 +80,9 @@ SELECT
|
||||
jobs.updated_at,
|
||||
|
||||
jobs.first_started_at as maybe_first_started_at,
|
||||
jobs.successfully_completed_at as maybe_successfully_completed_at
|
||||
jobs.successfully_completed_at as maybe_successfully_completed_at,
|
||||
|
||||
NOW() as database_clock
|
||||
|
||||
FROM generic_inference_jobs as jobs
|
||||
|
||||
@@ -223,6 +225,7 @@ fn raw_record_to_public_result(record: RawGenericInferenceJobStatus) -> GenericI
|
||||
is_keepalive_required: i8_to_bool(record.is_keepalive_required),
|
||||
created_at: record.created_at,
|
||||
updated_at: record.updated_at,
|
||||
database_clock: DateTime::from_naive_utc_and_offset(record.database_clock, Utc),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -268,6 +271,10 @@ struct RawGenericInferenceJobStatus {
|
||||
|
||||
pub maybe_first_started_at: Option<DateTime<Utc>>,
|
||||
pub maybe_successfully_completed_at: Option<DateTime<Utc>>,
|
||||
|
||||
// NB: Typed query can't convert to Utc, so we use NaiveDateTime and do the type conversion ourselves.
|
||||
// The database server *should* be reporting in UTC.
|
||||
pub database_clock: NaiveDateTime,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -32,6 +32,9 @@ pub struct GenericInferenceJobStatus {
|
||||
|
||||
pub created_at: DateTime<Utc>,
|
||||
pub updated_at: DateTime<Utc>,
|
||||
|
||||
/// This is MySQL's NOW(), which will let us do duration math without clock skew.
|
||||
pub database_clock: DateTime<Utc>,
|
||||
}
|
||||
|
||||
/// NB: Serialize is for internal moderator-only endpoints
|
||||
|
||||
@@ -167,7 +167,9 @@ SELECT
|
||||
jobs.updated_at,
|
||||
|
||||
jobs.first_started_at as maybe_first_started_at,
|
||||
jobs.successfully_completed_at as maybe_successfully_completed_at
|
||||
jobs.successfully_completed_at as maybe_successfully_completed_at,
|
||||
|
||||
NOW() as database_clock
|
||||
|
||||
FROM generic_inference_jobs as jobs
|
||||
|
||||
@@ -293,6 +295,7 @@ fn raw_records_to_public_result(records: Vec<RawGenericInferenceJobStatus>) -> V
|
||||
is_keepalive_required: i8_to_bool(record.is_keepalive_required),
|
||||
created_at: record.created_at,
|
||||
updated_at: record.updated_at,
|
||||
database_clock: record.database_clock,
|
||||
}
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
@@ -340,6 +343,8 @@ struct RawGenericInferenceJobStatus {
|
||||
|
||||
pub maybe_first_started_at: Option<DateTime<Utc>>,
|
||||
pub maybe_successfully_completed_at: Option<DateTime<Utc>>,
|
||||
|
||||
pub database_clock: DateTime<Utc>,
|
||||
}
|
||||
|
||||
// NB(bt,2023-12-05): There's an issue with type hinting in the `as` clauses with QueryBuilder (or
|
||||
@@ -386,6 +391,7 @@ impl FromRow<'_, MySqlRow> for RawGenericInferenceJobStatus {
|
||||
maybe_successfully_completed_at: row.try_get("maybe_successfully_completed_at")?,
|
||||
created_at: row.try_get("created_at")?,
|
||||
updated_at: row.try_get("updated_at")?,
|
||||
database_clock: row.try_get("database_clock")?,
|
||||
})
|
||||
}
|
||||
}
|
||||
+3
-4
@@ -6,6 +6,7 @@ use enums::by_table::generic_inference_jobs::inference_category::InferenceCatego
|
||||
use enums::common::job_status_plus::JobStatusPlus;
|
||||
use mysql_queries::queries::generic_inference::web::job_status::GenericInferenceJobStatus;
|
||||
|
||||
// TODO: These numbers are made up. We should measure the average job durations.
|
||||
const COMFY_JOB_AVERAGE_SECONDS : u64 = 60 * 4;
|
||||
const LIPSYNC_JOB_AVERAGE_SECONDS : u64 = 60 * 3;
|
||||
const TTS_JOB_AVERAGE_SECONDS : u64 = 7;
|
||||
@@ -32,12 +33,10 @@ pub fn estimate_job_progress(job: &GenericInferenceJobStatus) -> u8 {
|
||||
// and perhaps the code should be updated to always write every job attempt's start time.
|
||||
let started_at= job.maybe_first_started_at.unwrap_or(job.created_at);
|
||||
|
||||
// TODO: To avoid clock skew, we should select the current time from the database in the
|
||||
// same query. It should be as easy as selecting another column with `NOW()`.
|
||||
let now = Utc::now();
|
||||
let now = job.database_clock;
|
||||
|
||||
if started_at < now {
|
||||
return 0; // Clock skew for a probably newish job.
|
||||
return 0; // We shouldn't see clock skew unless we read from a DB replica.
|
||||
}
|
||||
|
||||
let duration = now.signed_duration_since(started_at);
|
||||
|
||||
Reference in new issue
Block a user