From 7f856321a74347bc2829429b05cd51eecf8d3aba Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 1 Aug 2024 17:55:32 -0400 Subject: [PATCH] update es-update-job to include db read time --- _elasticsearch/index_definitions/media_files_v1.json | 2 ++ .../index_definitions/model_weights_v1.json | 2 ++ .../src/documents/media_file_document.rs | 3 +++ .../src/documents/model_weight_document.rs | 3 +++ ..._files_for_elastic_search_backfill_using_cursor.rs | 11 ++++++++++- ...h_get_model_weights_for_elastic_search_backfill.rs | 10 +++++++++- .../update_all_model_weights.rs | 2 ++ .../update_engine_media_files.rs | 2 ++ 8 files changed, 33 insertions(+), 2 deletions(-) diff --git a/_elasticsearch/index_definitions/media_files_v1.json b/_elasticsearch/index_definitions/media_files_v1.json index 4132100e90..ab33a178a0 100644 --- a/_elasticsearch/index_definitions/media_files_v1.json +++ b/_elasticsearch/index_definitions/media_files_v1.json @@ -39,6 +39,8 @@ "user_deleted_at" : { "type" : "date" }, "mod_deleted_at" : { "type" : "date" }, + "database_read_time" : { "type" : "date" }, + "is_featured" : { "type": "boolean", "index": true diff --git a/_elasticsearch/index_definitions/model_weights_v1.json b/_elasticsearch/index_definitions/model_weights_v1.json index f23a37aa95..1fd77f1964 100644 --- a/_elasticsearch/index_definitions/model_weights_v1.json +++ b/_elasticsearch/index_definitions/model_weights_v1.json @@ -37,6 +37,8 @@ "user_deleted_at" : { "type" : "date" }, "mod_deleted_at" : { "type" : "date" }, + "database_read_time" : { "type" : "date" }, + "is_deleted" : { "type": "boolean", "index": true diff --git a/crates/lib/elasticsearch_schema/src/documents/media_file_document.rs b/crates/lib/elasticsearch_schema/src/documents/media_file_document.rs index dbd3a20654..b0fd6be84a 100644 --- a/crates/lib/elasticsearch_schema/src/documents/media_file_document.rs +++ b/crates/lib/elasticsearch_schema/src/documents/media_file_document.rs @@ -64,6 +64,9 @@ pub struct MediaFileDocument { /// Whether the file is featured. pub is_featured: bool, + /// When we read the record from the database + pub database_read_time: DateTime, + /// Calculated as "either user or mod deleted" pub is_deleted: bool, } diff --git a/crates/lib/elasticsearch_schema/src/documents/model_weight_document.rs b/crates/lib/elasticsearch_schema/src/documents/model_weight_document.rs index 1e2bd588d6..3c1f2d17ad 100644 --- a/crates/lib/elasticsearch_schema/src/documents/model_weight_document.rs +++ b/crates/lib/elasticsearch_schema/src/documents/model_weight_document.rs @@ -63,6 +63,9 @@ pub struct ModelWeightDocument { pub user_deleted_at: Option>, pub mod_deleted_at: Option>, + /// When we read the record from the database + pub database_read_time: DateTime, + /// Calculated as "either user or mod deleted" pub is_deleted: bool, } diff --git a/crates/lib/mysql_queries/src/queries/media_files/list/list_media_files_for_elastic_search_backfill_using_cursor.rs b/crates/lib/mysql_queries/src/queries/media_files/list/list_media_files_for_elastic_search_backfill_using_cursor.rs index 17942b1ad1..a492445ec0 100644 --- a/crates/lib/mysql_queries/src/queries/media_files/list/list_media_files_for_elastic_search_backfill_using_cursor.rs +++ b/crates/lib/mysql_queries/src/queries/media_files/list/list_media_files_for_elastic_search_backfill_using_cursor.rs @@ -112,6 +112,8 @@ pub struct MediaFileForElasticsearchRecord { pub user_deleted_at: Option>, pub mod_deleted_at: Option>, + + pub database_read_time: DateTime, } pub struct ListArgs<'a> { @@ -197,6 +199,7 @@ pub async fn list_media_files_for_elastic_search_backfill_using_cursor( updated_at: record.updated_at, user_deleted_at: record.user_deleted_at, mod_deleted_at: record.mod_deleted_at, + database_read_time: record.database_read_time, } }) .collect::>()) @@ -277,7 +280,9 @@ SELECT m.created_at, m.updated_at, m.user_deleted_at, - m.mod_deleted_at + m.mod_deleted_at, + + NOW() as database_read_time FROM media_files as m @@ -434,6 +439,8 @@ struct RawRecord { pub user_deleted_at: Option>, pub mod_deleted_at: Option>, + + pub database_read_time: DateTime, } impl FromRow<'_, MySqlRow> for RawRecord { @@ -509,6 +516,8 @@ impl FromRow<'_, MySqlRow> for RawRecord { updated_at: row.try_get("updated_at")?, user_deleted_at: row.try_get("user_deleted_at")?, mod_deleted_at: row.try_get("mod_deleted_at")?, + + database_read_time: row.try_get("database_read_time")?, }) } } diff --git a/crates/lib/mysql_queries/src/queries/model_weights/batch_get/batch_get_model_weights_for_elastic_search_backfill.rs b/crates/lib/mysql_queries/src/queries/model_weights/batch_get/batch_get_model_weights_for_elastic_search_backfill.rs index b82c882e9c..499aac6a75 100644 --- a/crates/lib/mysql_queries/src/queries/model_weights/batch_get/batch_get_model_weights_for_elastic_search_backfill.rs +++ b/crates/lib/mysql_queries/src/queries/model_weights/batch_get/batch_get_model_weights_for_elastic_search_backfill.rs @@ -56,6 +56,8 @@ pub struct ModelWeightForElasticsearchRecord { pub user_deleted_at: Option>, pub mod_deleted_at: Option>, + + pub database_read_time: DateTime, } pub async fn batch_get_model_weights_for_elastic_search_backfill<'e, 'c, E>( @@ -111,6 +113,7 @@ pub async fn batch_get_model_weights_for_elastic_search_backfill<'e, 'c, E>( updated_at: model.updated_at, user_deleted_at: model.user_deleted_at, mod_deleted_at: model.mod_deleted_at, + database_read_time: model.database_read_time, } }) .collect::>()) @@ -162,7 +165,9 @@ SELECT w.created_at, w.updated_at, w.user_deleted_at, - w.mod_deleted_at + w.mod_deleted_at, + + NOW() as database_read_time FROM model_weights as w @@ -242,6 +247,8 @@ struct RawRecord { pub user_deleted_at: Option>, pub mod_deleted_at: Option>, + + pub database_read_time: DateTime, } // NB(bt,2023-12-05): There's an issue with type hinting in the `as` clauses with QueryBuilder (or @@ -296,6 +303,7 @@ impl FromRow<'_, MySqlRow> for RawRecord { updated_at: row.try_get("updated_at")?, user_deleted_at: row.try_get("user_deleted_at")?, mod_deleted_at: row.try_get("mod_deleted_at")?, + database_read_time: row.try_get("database_read_time")?, }) } } diff --git a/crates/service/job/es-update-job/src/tasks/update_all_model_weights/update_all_model_weights.rs b/crates/service/job/es-update-job/src/tasks/update_all_model_weights/update_all_model_weights.rs index aa4259924e..373e8be48e 100644 --- a/crates/service/job/es-update-job/src/tasks/update_all_model_weights/update_all_model_weights.rs +++ b/crates/service/job/es-update-job/src/tasks/update_all_model_weights/update_all_model_weights.rs @@ -151,6 +151,8 @@ async fn create_document_from_record(elasticsearch: &Elasticsearch, record: Mode user_deleted_at: record.user_deleted_at, mod_deleted_at: record.mod_deleted_at, + database_read_time: record.database_read_time, + is_deleted, }; diff --git a/crates/service/job/es-update-job/src/tasks/update_engine_media_files/update_engine_media_files.rs b/crates/service/job/es-update-job/src/tasks/update_engine_media_files/update_engine_media_files.rs index ff7ae88782..ee2a71e697 100644 --- a/crates/service/job/es-update-job/src/tasks/update_engine_media_files/update_engine_media_files.rs +++ b/crates/service/job/es-update-job/src/tasks/update_engine_media_files/update_engine_media_files.rs @@ -156,6 +156,8 @@ async fn create_document_from_record(elasticsearch: &Elasticsearch, record: Medi user_deleted_at: record.user_deleted_at, mod_deleted_at: record.mod_deleted_at, + database_read_time: record.database_read_time, + is_deleted, };