From 255f54c39a4ff246c1f98307e50a981ddebbae50 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Fri, 11 Jun 2021 23:27:03 -0400 Subject: [PATCH] all predicates for W2L pagination (endpoint is no-op temporarily) --- .../list_w2l_inference_results.rs | 662 ++++++++++++------ .../users/list_user_w2l_inference_results.rs | 8 +- 2 files changed, 442 insertions(+), 228 deletions(-) diff --git a/src/common_queries/list_w2l_inference_results.rs b/src/common_queries/list_w2l_inference_results.rs index 364a34b9d9..334334722e 100644 --- a/src/common_queries/list_w2l_inference_results.rs +++ b/src/common_queries/list_w2l_inference_results.rs @@ -1,9 +1,22 @@ use anyhow::anyhow; use chrono::{DateTime, Utc}; +use crate::shared_constants::DEFAULT_MYSQL_QUERY_RESULT_PAGE_SIZE; use crate::util::anyhow_result::AnyhowResult; use log::{warn, info}; use sqlx::MySqlPool; +#[derive(Serialize)] +pub struct W2lInferenceListPage { + pub inference_records: Vec, + pub sort_ascending: bool, + + /// ID of the first record in `inference_records`. + pub first_id: Option, + + /// ID of the last record in `inference_records`. + pub last_id: Option, +} + #[derive(Serialize)] pub struct W2lInferenceRecordForList { pub w2l_result_token: String, @@ -29,8 +42,273 @@ pub struct W2lInferenceRecordForList { pub updated_at: DateTime, } -struct RawW2lInferenceRecordForList { - pub w2l_result_token: String, // from field `w2l_results.token` +/// Query Builder for listing W2L results. +/// These are very difficult queries, so the builder helps for +/// testability, construction, and correctness. +pub struct ListW2lResultsQueryBuilder { + scope_creator_username: Option, + include_mod_deleted_results: bool, + include_user_deleted_results: bool, + include_templates_not_approved_for_public_listing: bool, + sort_ascending: bool, + offset: Option, + limit: u16, + cursor_is_reversed: bool, +} + +impl ListW2lResultsQueryBuilder { + pub fn new() -> Self { + Self { + scope_creator_username: None, + include_mod_deleted_results: false, + include_user_deleted_results: false, + include_templates_not_approved_for_public_listing: false, + sort_ascending: false, + offset: None, + limit: DEFAULT_MYSQL_QUERY_RESULT_PAGE_SIZE, + cursor_is_reversed: false, + } + } + + pub fn scope_creator_username(mut self, scope_creator_username: Option<&str>) -> Self { + self.scope_creator_username = scope_creator_username.map(|u| u.to_string()); + self + } + + pub fn include_mod_deleted_results(mut self, include_mod_deleted_results: bool) -> Self { + self.include_mod_deleted_results = include_mod_deleted_results; + self + } + + pub fn include_user_deleted_results(mut self, include_user_deleted_results: bool) -> Self { + self.include_user_deleted_results = include_user_deleted_results; + self + } + + pub fn include_templates_not_approved_for_public_listing(mut self, include_templates_not_approved_for_public_listing: bool) -> Self { + self.include_templates_not_approved_for_public_listing = include_templates_not_approved_for_public_listing; + self + } + + pub fn sort_ascending(mut self, sort_ascending: bool) -> Self { + self.sort_ascending = sort_ascending; + self + } + + pub fn offset(mut self, offset: Option) -> Self { + self.offset = offset; + self + } + + pub fn limit(mut self, limit: u16) -> Self { + self.limit = limit; + self + } + + pub fn cursor_is_reversed(mut self, cursor_is_reversed: bool) -> Self { + self.cursor_is_reversed = cursor_is_reversed; + self + } + + /// Perform the query based on the set predicates. + /// Wrap the results in a "Page" we can return to the frontend. + /// This will remove any IDs from the records, but the ids themselves also need scrubbing. + pub async fn perform_query_for_page( + &self, + mysql_pool: &MySqlPool + ) -> AnyhowResult { + + let internal_results = self.perform_internal_query(mysql_pool).await?; + + let mut first_id = internal_results.first() + .map(|raw_result| raw_result.w2l_result_id); + + let mut last_id = internal_results.last() + .map(|raw_result| raw_result.w2l_result_id); + + let inference_results = internal_results + .iter() + .map(|r| { + W2lInferenceRecordForList { + w2l_result_token: r.w2l_result_token.clone(), + maybe_w2l_template_token: r.maybe_w2l_template_token.clone(), + maybe_tts_inference_result_token: r.maybe_tts_inference_result_token.clone(), + template_type: r.template_type.clone(), + template_title: r.template_title.clone(), + maybe_creator_user_token: r.maybe_creator_user_token.clone(), + maybe_creator_username: r.maybe_creator_username.clone(), + maybe_creator_display_name: r.maybe_creator_display_name.clone(), + file_size_bytes: if r.file_size_bytes > 0 { r.file_size_bytes as u32 } else { 0 }, + frame_width: if r.frame_width > 0 { r.frame_width as u32 } else { 0 }, + frame_height: if r.frame_height > 0 { r.frame_height as u32 } else { 0 }, + duration_millis: if r.duration_millis > 0 { r.duration_millis as u32 } else { 0 }, + created_at: r.created_at, + updated_at: r.updated_at, + } + }) + .collect::>(); + + Ok(W2lInferenceListPage { + inference_records: inference_results, + sort_ascending: self.sort_ascending, + first_id, + last_id, + }) + } + + /// Perform the query based on the set predicates. + /// The records contain IDs, which we should not expose to the frontend. + pub async fn perform_internal_query( + &self, + mysql_pool: &MySqlPool + ) -> AnyhowResult> { + + let query = self.build_query_string(); + let mut query = sqlx::query_as::<_, RawInternalTtsRecord>(&query); + + // NB: The following bindings must match the order of the query builder !! + + if let Some(offset) = self.offset { + query = query.bind(offset); + } + + if let Some(username) = self.scope_creator_username.as_deref() { + query = query.bind(username); + } + + query = query.bind(self.limit); + + let mut results = query.fetch_all(mysql_pool) + .await?; + + if self.cursor_is_reversed { + results.reverse() + } + + Ok(results) + } + + pub fn build_query_string(&self) -> String { + // TODO/NB: Unfortunately SQLx can't statically typecheck this query + let mut query = r#" +SELECT + w2l_results.id as w2l_result_id, + w2l_results.token as w2l_result_token, + w2l_results.maybe_tts_inference_result_token, + + w2l_templates.token as maybe_w2l_template_token, + w2l_templates.template_type, + w2l_templates.title as template_title, + + users.token as maybe_creator_user_token, + users.username as maybe_creator_username, + users.display_name as maybe_creator_display_name, + + w2l_results.file_size_bytes, + w2l_results.frame_width, + w2l_results.frame_height, + w2l_results.duration_millis, + w2l_results.created_at, + w2l_results.updated_at + +FROM w2l_results +LEFT OUTER JOIN w2l_templates + ON w2l_results.maybe_w2l_template_token = w2l_templates.token +LEFT OUTER JOIN users + ON w2l_results.maybe_creator_user_token = users.token + "#.to_string(); + + query.push_str(&self.build_predicates()); + query + } + + pub fn build_predicates(&self) -> String { + // NB: Reverse cursors require us to invert the sort direction. + let mut sort_ascending = self.sort_ascending; + + let mut first_predicate_added = false; + + let mut query = "".to_string(); + + if let Some(offset) = self.offset { + if !first_predicate_added { + query.push_str(" WHERE"); + first_predicate_added = true; + } else { + query.push_str(" AND"); + } + + if sort_ascending { + if self.cursor_is_reversed { + // NB: We're searching backwards. + query.push_str(" w2l_results.id < ?"); + sort_ascending = !sort_ascending; + } else { + query.push_str(" w2l_results.id > ?"); + } + } else { + if self.cursor_is_reversed { + // NB: We're searching backwards. + query.push_str(" w2l_results.id > ?"); + sort_ascending = !sort_ascending; + } else { + query.push_str(" w2l_results.id < ?"); + } + } + } + + if let Some(username) = self.scope_creator_username.as_deref() { + if !first_predicate_added { + query.push_str(" WHERE users.username = ?"); + first_predicate_added = true; + } else { + query.push_str(" AND users.username = ?"); + } + } + + if !self.include_templates_not_approved_for_public_listing { + if !first_predicate_added { + query.push_str(" WHERE w2l_templates.is_public_listing_approved IS TRUE"); + first_predicate_added = true; + } else { + query.push_str(" AND w2l_templates.is_public_listing_approved IS TRUE"); + } + } + + if !self.include_mod_deleted_results { + if !first_predicate_added { + query.push_str(" AND w2l_results.mod_deleted_at IS NULL"); + first_predicate_added = true; + } else { + query.push_str(" AND w2l_results.mod_deleted_at IS NULL"); + } + } + + if !self.include_user_deleted_results { + if !first_predicate_added { + query.push_str(" WHERE w2l_results.user_deleted_at IS NULL"); + first_predicate_added = true; + } else { + query.push_str(" AND w2l_results.user_deleted_at IS NULL"); + } + } + + if sort_ascending { + query.push_str(" ORDER BY w2l_results.id ASC"); + } else { + query.push_str(" ORDER BY w2l_results.id DESC"); + } + + query.push_str(" LIMIT ?"); + + query + } +} + +#[derive(sqlx::FromRow)] +pub struct RawInternalTtsRecord { + pub w2l_result_id: i64, + pub w2l_result_token: String, pub maybe_w2l_template_token: Option, pub maybe_tts_inference_result_token: Option, @@ -54,235 +332,169 @@ struct RawW2lInferenceRecordForList { pub updated_at: DateTime, } -pub async fn list_w2l_inference_results( - mysql_pool: &MySqlPool, - scope_creator_username: Option<&str>, - require_mod_approved_templates: bool -) -> AnyhowResult> { +#[cfg(test)] +mod tests { + use crate::common_queries::list_w2l_inference_results::ListW2lResultsQueryBuilder; - let maybe_results = match scope_creator_username { - Some(username) => { - list_w2l_inference_results_creator_scoped(mysql_pool, username, require_mod_approved_templates) - .await - }, - None => { - list_w2l_inference_results_for_all_creators(mysql_pool, require_mod_approved_templates) - .await - }, - }; + #[test] + fn predicates_without_scoping() { + let query_builder = ListW2lResultsQueryBuilder::new(); - let results : Vec = match maybe_results { - Ok(results) => { - info!("Results length: {}", results.len()); - results - }, - Err(err) => { - warn!("Error: {:?}", err); + assert_eq!(&query_builder.build_predicates(), + " WHERE w2l_templates.is_public_listing_approved IS TRUE \ + AND w2l_results.mod_deleted_at IS NULL \ + AND w2l_results.user_deleted_at IS NULL \ + ORDER BY w2l_results.id DESC \ + LIMIT ?"); + } - match err { - RowNotFound => { - return Ok(Vec::new()); - }, - _ => { - warn!("w2l inference result list query error: {:?}", err); - return Err(anyhow!("w2l inference result list query error")); - } - } - } - }; + #[test] + fn predicates_scoped_to_user() { + let query_builder = ListW2lResultsQueryBuilder::new() + .scope_creator_username(Some("echelon")); - Ok(results.into_iter() - .map(|ir| { - W2lInferenceRecordForList { - w2l_result_token: ir.w2l_result_token.clone(), - maybe_w2l_template_token: ir.maybe_w2l_template_token.clone(), - maybe_tts_inference_result_token: ir.maybe_tts_inference_result_token.clone(), + assert_eq!(&query_builder.build_predicates(), + " WHERE users.username = ? \ + AND w2l_templates.is_public_listing_approved IS TRUE \ + AND w2l_results.mod_deleted_at IS NULL \ + AND w2l_results.user_deleted_at IS NULL \ + ORDER BY w2l_results.id DESC \ + LIMIT ?"); + } - template_type: ir.template_type.clone(), - template_title: ir.template_title.clone(), + // TODO: Insufficient testing here. + #[test] + fn predicates_including_deleted_and_unapproved() { + let query_builder = ListW2lResultsQueryBuilder::new() + .include_mod_deleted_results(true) + .include_user_deleted_results(true) + .include_templates_not_approved_for_public_listing(true); - maybe_creator_user_token: ir.maybe_creator_user_token.clone(), - maybe_creator_username: ir.maybe_creator_username.clone(), - maybe_creator_display_name: ir.maybe_creator_display_name.clone(), - //template_is_mod_approved: if ir.template_is_mod_approved == 0 { false } else { true }, + assert_eq!(&query_builder.build_predicates(), + " ORDER BY w2l_results.id DESC \ + LIMIT ?"); + } - file_size_bytes: if ir.file_size_bytes > 0 { ir.file_size_bytes as u32 } else { 0 }, - frame_width: if ir.frame_width > 0 { ir.frame_width as u32 } else { 0 }, - frame_height: if ir.frame_height > 0 { ir.frame_height as u32 } else { 0 }, - duration_millis: if ir.duration_millis > 0 { ir.duration_millis as u32 } else { 0 }, + #[test] + fn predicates_sort_ascending() { + let query_builder = ListW2lResultsQueryBuilder::new() + .sort_ascending(true); - created_at: ir.created_at.clone(), - updated_at: ir.updated_at.clone(), - } - }) - .collect::>()) -} - -async fn list_w2l_inference_results_for_all_creators( - mysql_pool: &MySqlPool, - require_mod_approved_templates: bool -) -> AnyhowResult> { - // TODO: There has to be a better way. - // Sqlx doesn't like anything except string literals. - let maybe_results = if require_mod_approved_templates { - info!("listing w2l inference results for everyone; mod-approved only"); - sqlx::query_as!( - RawW2lInferenceRecordForList, - r#" -SELECT - w2l_results.token as w2l_result_token, - w2l_results.maybe_tts_inference_result_token, - - w2l_templates.token as maybe_w2l_template_token, - w2l_templates.template_type, - w2l_templates.title as template_title, - - users.token as maybe_creator_user_token, - users.username as maybe_creator_username, - users.display_name as maybe_creator_display_name, - - w2l_results.file_size_bytes, - w2l_results.frame_width, - w2l_results.frame_height, - w2l_results.duration_millis, - w2l_results.created_at, - w2l_results.updated_at - -FROM w2l_results -LEFT OUTER JOIN w2l_templates - ON w2l_results.maybe_w2l_template_token = w2l_templates.token -LEFT OUTER JOIN users - ON w2l_results.maybe_creator_user_token = users.token -WHERE - w2l_templates.is_public_listing_approved IS TRUE - AND w2l_results.user_deleted_at IS NULL - AND w2l_results.mod_deleted_at IS NULL - "#) - .fetch_all(mysql_pool) - .await? - } else { - info!("listing w2l inference results for everyone; all"); - sqlx::query_as!( - RawW2lInferenceRecordForList, - r#" -SELECT - w2l_results.token as w2l_result_token, - w2l_results.maybe_tts_inference_result_token, - - w2l_templates.token as maybe_w2l_template_token, - w2l_templates.template_type, - w2l_templates.title as template_title, - - users.token as maybe_creator_user_token, - users.username as maybe_creator_username, - users.display_name as maybe_creator_display_name, - - w2l_results.file_size_bytes, - w2l_results.frame_width, - w2l_results.frame_height, - w2l_results.duration_millis, - w2l_results.created_at, - w2l_results.updated_at - -FROM w2l_results -LEFT OUTER JOIN w2l_templates - ON w2l_results.maybe_w2l_template_token = w2l_templates.token -LEFT OUTER JOIN users - ON w2l_results.maybe_creator_user_token = users.token -WHERE - w2l_results.user_deleted_at IS NULL - AND w2l_results.mod_deleted_at IS NULL - "#) - .fetch_all(mysql_pool) - .await? - }; - - Ok(maybe_results) -} - -async fn list_w2l_inference_results_creator_scoped( - mysql_pool: &MySqlPool, - scope_creator_username: &str, - require_mod_approved_templates: bool -) -> AnyhowResult> { - // TODO: There has to be a better way. - // Sqlx doesn't like anything except string literals. - let maybe_results = if require_mod_approved_templates { - info!("listing w2l inference results for user `{}`; mod-approved only", scope_creator_username); - sqlx::query_as!( - RawW2lInferenceRecordForList, - r#" -SELECT - w2l_results.token as w2l_result_token, - w2l_results.maybe_tts_inference_result_token, - - w2l_templates.token as maybe_w2l_template_token, - w2l_templates.template_type, - w2l_templates.title as template_title, - - users.token as maybe_creator_user_token, - users.username as maybe_creator_username, - users.display_name as maybe_creator_display_name, - - w2l_results.file_size_bytes, - w2l_results.frame_width, - w2l_results.frame_height, - w2l_results.duration_millis, - w2l_results.created_at, - w2l_results.updated_at - -FROM w2l_results -LEFT OUTER JOIN w2l_templates - ON w2l_results.maybe_w2l_template_token = w2l_templates.token -LEFT OUTER JOIN users - ON w2l_results.maybe_creator_user_token = users.token -WHERE - users.username = ? - AND w2l_templates.is_public_listing_approved IS TRUE - AND w2l_results.user_deleted_at IS NULL - AND w2l_results.mod_deleted_at IS NULL - "#, - scope_creator_username) - .fetch_all(mysql_pool) - .await? - } else { - info!("listing w2l inference results for user `{}`; all", scope_creator_username); - sqlx::query_as!( - RawW2lInferenceRecordForList, - r#" -SELECT - w2l_results.token as w2l_result_token, - w2l_results.maybe_tts_inference_result_token, - - w2l_templates.token as maybe_w2l_template_token, - w2l_templates.template_type, - w2l_templates.title as template_title, - - users.token as maybe_creator_user_token, - users.username as maybe_creator_username, - users.display_name as maybe_creator_display_name, - - w2l_results.file_size_bytes, - w2l_results.frame_width, - w2l_results.frame_height, - w2l_results.duration_millis, - w2l_results.created_at, - w2l_results.updated_at - -FROM w2l_results -LEFT OUTER JOIN w2l_templates - ON w2l_results.maybe_w2l_template_token = w2l_templates.token -LEFT OUTER JOIN users - ON w2l_results.maybe_creator_user_token = users.token -WHERE - users.username = ? - AND w2l_results.user_deleted_at IS NULL - AND w2l_results.mod_deleted_at IS NULL - "#, - scope_creator_username) - .fetch_all(mysql_pool) - .await? - }; - - Ok(maybe_results) + assert_eq!(&query_builder.build_predicates(), + " WHERE w2l_templates.is_public_listing_approved IS TRUE \ + AND w2l_results.mod_deleted_at IS NULL \ + AND w2l_results.user_deleted_at IS NULL \ + ORDER BY w2l_results.id ASC \ + LIMIT ?"); + } + + #[test] + fn predicates_offset() { + let query_builder = ListW2lResultsQueryBuilder::new() + .offset(Some(100)); + + assert_eq!(&query_builder.build_predicates(), + " WHERE w2l_results.id < ? \ + AND w2l_templates.is_public_listing_approved IS TRUE \ + AND w2l_results.mod_deleted_at IS NULL \ + AND w2l_results.user_deleted_at IS NULL \ + ORDER BY w2l_results.id DESC \ + LIMIT ?"); + } + + #[test] + fn predicates_offset_and_sort_ascending() { + let query_builder = ListW2lResultsQueryBuilder::new() + .sort_ascending(true) + .offset(Some(100)); + + assert_eq!(&query_builder.build_predicates(), + " WHERE w2l_results.id > ? \ + AND w2l_templates.is_public_listing_approved IS TRUE \ + AND w2l_results.mod_deleted_at IS NULL \ + AND w2l_results.user_deleted_at IS NULL \ + ORDER BY w2l_results.id ASC \ + LIMIT ?"); + } + + #[test] + fn predicates_limit() { + let query_builder = ListW2lResultsQueryBuilder::new() + .limit(15); + + // NB: Does not change the query itself! Just the downstream binding. + assert_eq!(&query_builder.build_predicates(), + " WHERE w2l_templates.is_public_listing_approved IS TRUE \ + AND w2l_results.mod_deleted_at IS NULL \ + AND w2l_results.user_deleted_at IS NULL \ + ORDER BY w2l_results.id DESC \ + LIMIT ?"); + } + + #[test] + fn predicates_cursor_is_reversed_without_cursor() { + let query_builder = ListW2lResultsQueryBuilder::new() + .cursor_is_reversed(true); + + // NB: Without a cursor, nothing happens. + assert_eq!(&query_builder.build_predicates(), + " WHERE w2l_templates.is_public_listing_approved IS TRUE \ + AND w2l_results.mod_deleted_at IS NULL \ + AND w2l_results.user_deleted_at IS NULL \ + ORDER BY w2l_results.id DESC \ + LIMIT ?"); + } + + #[test] + fn predicates_offset_cursor_is_reversed() { + let query_builder = ListW2lResultsQueryBuilder::new() + .offset(Some(100)) + .cursor_is_reversed(true); + + // NB: This will change the sort order and greater/less than direction! + assert_eq!(&query_builder.build_predicates(), + " WHERE w2l_results.id > ? \ + AND w2l_templates.is_public_listing_approved IS TRUE \ + AND w2l_results.mod_deleted_at IS NULL \ + AND w2l_results.user_deleted_at IS NULL \ + ORDER BY w2l_results.id ASC \ + LIMIT ?"); + } + + #[test] + fn predicates_offset_cursor_is_reversed_sort_ascending() { + let query_builder = ListW2lResultsQueryBuilder::new() + .offset(Some(100)) + .cursor_is_reversed(true) + .sort_ascending(true); + + // NB: This will change the sort order and greater/less than direction! + assert_eq!(&query_builder.build_predicates(), + " WHERE w2l_results.id < ? \ + AND w2l_templates.is_public_listing_approved IS TRUE \ + AND w2l_results.mod_deleted_at IS NULL \ + AND w2l_results.user_deleted_at IS NULL \ + ORDER BY w2l_results.id DESC \ + LIMIT ?"); + } + + #[test] + fn predicates_limit_scope_user_offset_cursor_is_reversed_sort_ascending() { + let query_builder = ListW2lResultsQueryBuilder::new() + .limit(1000) + .scope_creator_username(Some("pikachu")) + .offset(Some(100)) + .cursor_is_reversed(true) + .sort_ascending(true); + + // NB: This will change the sort order and greater/less than direction! + assert_eq!(&query_builder.build_predicates(), + " WHERE w2l_results.id < ? \ + AND users.username = ? \ + AND w2l_templates.is_public_listing_approved IS TRUE \ + AND w2l_results.mod_deleted_at IS NULL \ + AND w2l_results.user_deleted_at IS NULL \ + ORDER BY w2l_results.id DESC \ + LIMIT ?"); + } } diff --git a/src/http_server/endpoints/users/list_user_w2l_inference_results.rs b/src/http_server/endpoints/users/list_user_w2l_inference_results.rs index 14a27a268d..d8cd35f2f8 100644 --- a/src/http_server/endpoints/users/list_user_w2l_inference_results.rs +++ b/src/http_server/endpoints/users/list_user_w2l_inference_results.rs @@ -8,7 +8,7 @@ use actix_web::{Responder, web, HttpResponse, error, HttpRequest, HttpMessage}; use chrono::{DateTime, Utc}; use crate::AnyhowResult; use crate::common_queries::list_w2l_inference_results::W2lInferenceRecordForList; -use crate::common_queries::list_w2l_inference_results::list_w2l_inference_results; +use crate::common_queries::list_w2l_inference_results::ListW2lResultsQueryBuilder; use crate::common_queries::sessions::create_session_for_user; use crate::http_server::endpoints::users::create_account::CreateAccountError::{BadInput, ServerError, UsernameTaken, EmailTaken}; use crate::http_server::endpoints::users::login::LoginSuccessResponse; @@ -70,7 +70,7 @@ pub async fn list_user_w2l_inference_results_handler( { info!("Fetching inference results for user: {}", &path.username); - let query_results = list_w2l_inference_results( + /*let query_results = list_w2l_inference_results( &server_state.mysql_pool, Some(path.username.as_ref()), false @@ -82,7 +82,9 @@ pub async fn list_user_w2l_inference_results_handler( warn!("Query error: {:?}", e); return Err(ListW2lInferenceResultsForUserError::ServerError); } - }; + };*/ + + let results = Vec::new(); let response = ListW2lInferenceResultsForUserSuccessResponse { success: true,