From ecd1a1243890304d840d4f0a1c9aa4102d5d7f57 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Wed, 3 Jan 2024 04:06:54 -0500 Subject: [PATCH] batch get user ratings --- Cargo.lock | 102 ++++++++++ .../user_ratings/batch_get_user_rating.rs | 85 ++++++++ .../src/queries/users/user_ratings/mod.rs | 1 + crates/service/web/storyteller_web/Cargo.toml | 1 + .../web/storyteller_web/src/api_doc.rs | 3 + .../batch_get_user_rating_handler.rs | 191 ++++++++++++++++++ .../user_ratings/get_user_rating_handler.rs | 24 +-- .../http_server/endpoints/user_ratings/mod.rs | 1 + .../service/web/storyteller_web/src/routes.rs | 5 + 9 files changed, 401 insertions(+), 12 deletions(-) create mode 100644 crates/lib/mysql_queries/src/queries/users/user_ratings/batch_get_user_rating.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/user_ratings/batch_get_user_rating_handler.rs diff --git a/Cargo.lock b/Cargo.lock index 706c846123..5e29620c49 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -301,6 +301,55 @@ dependencies = [ "syn 2.0.39", ] +[[package]] +name = "actix-web-lab" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6ddd78b365b70614e27f49a128ab8edba32bbc8f9b2abb1fbe07f15dd48cfab" +dependencies = [ + "actix-http", + "actix-router", + "actix-service", + "actix-utils", + "actix-web", + "actix-web-lab-derive", + "ahash 0.8.6", + "arc-swap", + "async-trait", + "bytes 1.5.0", + "bytestring", + "csv", + "derive_more", + "futures-core", + "futures-util", + "http", + "impl-more", + "itertools 0.11.0", + "local-channel", + "mediatype", + "mime", + "once_cell", + "pin-project-lite", + "regex", + "serde", + "serde_html_form", + "serde_json", + "tokio", + "tokio-stream", + "tracing", +] + +[[package]] +name = "actix-web-lab-derive" +version = "0.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9aa0b287c8de4a76b691f29dbb5451e8dd5b79d777eaf87350c9b0cbfdb5e968" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.39", +] + [[package]] name = "actix_derive" version = "0.6.1" @@ -543,6 +592,12 @@ version = "1.0.75" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4668cab20f66d8d020e1fbc0ebe47217433c1b6c8f2040faf858554e394ace6" +[[package]] +name = "arc-swap" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" + [[package]] name = "archives" version = "0.0.1" @@ -1464,6 +1519,27 @@ dependencies = [ "syn 2.0.39", ] +[[package]] +name = "csv" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe" +dependencies = [ + "csv-core", + "itoa 1.0.9", + "ryu", + "serde", +] + +[[package]] +name = "csv-core" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" +dependencies = [ + "memchr", +] + [[package]] name = "darling" version = "0.13.4" @@ -2922,6 +2998,12 @@ dependencies = [ "testing", ] +[[package]] +name = "impl-more" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "206ca75c9c03ba3d4ace2460e57b189f39f43de612c2f85836e65c929701bb2d" + [[package]] name = "indexmap" version = "2.1.0" @@ -3460,6 +3542,12 @@ dependencies = [ "symphonia", ] +[[package]] +name = "mediatype" +version = "0.19.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf0bc9784973713e4a90d515a4302991ca125a7c4516951cb607f2298cb757e5" + [[package]] name = "memchr" version = "2.6.4" @@ -5273,6 +5361,19 @@ dependencies = [ "syn 2.0.39", ] +[[package]] +name = "serde_html_form" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224e6a14f315852940f3ec103125aa6482f0e224732ed91ed3330ed633077c34" +dependencies = [ + "form_urlencoded", + "indexmap", + "itoa 1.0.9", + "ryu", + "serde", +] + [[package]] name = "serde_json" version = "1.0.108" @@ -5893,6 +5994,7 @@ dependencies = [ "actix-service", "actix-utils", "actix-web", + "actix-web-lab", "actix_helpers", "anyhow", "async-stripe", diff --git a/crates/lib/mysql_queries/src/queries/users/user_ratings/batch_get_user_rating.rs b/crates/lib/mysql_queries/src/queries/users/user_ratings/batch_get_user_rating.rs new file mode 100644 index 0000000000..47ad57b168 --- /dev/null +++ b/crates/lib/mysql_queries/src/queries/users/user_ratings/batch_get_user_rating.rs @@ -0,0 +1,85 @@ +use std::collections::HashSet; +use anyhow::anyhow; +use log::error; +use sqlx::{MySql, QueryBuilder}; +use sqlx::pool::PoolConnection; + +use enums::by_table::user_ratings::entity_type::UserRatingEntityType; +use enums::by_table::user_ratings::rating_value::UserRatingValue; +use errors::AnyhowResult; +use tokens::tokens::users::UserToken; + +pub struct BatchUserRating { + pub entity_token: String, + pub entity_type: UserRatingEntityType, + pub rating_value: UserRatingValue, +} + +/// Look up the user rating +pub async fn batch_get_user_ratings( + user_token: &UserToken, + tokens: &HashSet, + mysql_connection: &mut PoolConnection, +) -> AnyhowResult> +{ + if tokens.len() == 0 { + return Ok(Vec::new()); + } + + let mut query_builder: QueryBuilder = QueryBuilder::new( + r#" +SELECT + entity_type, + entity_token, + rating_value + +FROM user_ratings + +WHERE user_token = + "#); + + query_builder.push_bind(user_token.as_str()); + + query_builder.push(" AND entity_token IN ( "); + + // NB: Syntax will be wrong if list has zero length + let mut separated = query_builder.separated(", "); + + for token in tokens.iter() { + separated.push_bind(token); + } + + separated.push_unseparated(") "); + + let query = query_builder.build_query_as::(); + + let maybe_results = query.fetch_all(&mut **mysql_connection).await; + + match maybe_results { + Ok(records) => Ok(records + .into_iter() + .map(|record| BatchUserRating { + // NB: Fail open; W2lTemplates are dead, so this is a good sentinel value + entity_type: UserRatingEntityType::from_str(&record.entity_type) + .unwrap_or(UserRatingEntityType::W2lTemplate), + entity_token: record.entity_token, + // NB: Fail open; "neutral" is a good default on failure. + rating_value: UserRatingValue::from_str(&record.rating_value) + .unwrap_or(UserRatingValue::Neutral), + }).collect::>()), + Err(err) => match err { + sqlx::Error::RowNotFound => Ok(Vec::new()), + _ => { + error!("error querying job record: {:?}", err); + Err(anyhow!("error querying job record: {:?}", err)) + } + } + } +} + +#[derive(sqlx::FromRow)] +struct RawRating { + entity_token: String, + entity_type: String, + rating_value: String, +} diff --git a/crates/lib/mysql_queries/src/queries/users/user_ratings/mod.rs b/crates/lib/mysql_queries/src/queries/users/user_ratings/mod.rs index 8d77e55649..aabf007efb 100644 --- a/crates/lib/mysql_queries/src/queries/users/user_ratings/mod.rs +++ b/crates/lib/mysql_queries/src/queries/users/user_ratings/mod.rs @@ -1,3 +1,4 @@ +pub mod batch_get_user_rating; pub mod get_total_user_rating_count_for_entity; pub mod get_user_rating; pub mod update_tts_model_ratings; diff --git a/crates/service/web/storyteller_web/Cargo.toml b/crates/service/web/storyteller_web/Cargo.toml index 3746e8dcf1..d8224d0269 100644 --- a/crates/service/web/storyteller_web/Cargo.toml +++ b/crates/service/web/storyteller_web/Cargo.toml @@ -134,6 +134,7 @@ actix-rt = "2.9.0" actix-service = "2.0.2" actix-utils = "3.0.1" actix-web = "4.4.0" +actix-web-lab = "0.20.1" jwt = "0.13.0" # NB: Hyper is just for HTTP client use in jobs. I'd use Actix client, but I need tokio runtime diff --git a/crates/service/web/storyteller_web/src/api_doc.rs b/crates/service/web/storyteller_web/src/api_doc.rs index 16937d47ea..2230ef9250 100644 --- a/crates/service/web/storyteller_web/src/api_doc.rs +++ b/crates/service/web/storyteller_web/src/api_doc.rs @@ -29,6 +29,7 @@ use crate::http_server::endpoints::user_bookmarks::create_user_bookmark_handler: use crate::http_server::endpoints::user_bookmarks::delete_user_bookmark_handler::*; use crate::http_server::endpoints::user_bookmarks::list_user_bookmarks_for_entity_handler::*; use crate::http_server::endpoints::user_bookmarks::list_user_bookmarks_for_user_handler::*; +use crate::http_server::endpoints::user_ratings::batch_get_user_rating_handler::*; use crate::http_server::endpoints::user_ratings::get_user_rating_handler::*; use crate::http_server::endpoints::user_ratings::set_user_rating_handler::*; use crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::*; @@ -52,6 +53,7 @@ use crate::http_server::web_utils::response_success_helpers::*; crate::http_server::endpoints::user_bookmarks::delete_user_bookmark_handler::delete_user_bookmark_handler, crate::http_server::endpoints::user_bookmarks::list_user_bookmarks_for_entity_handler::list_user_bookmarks_for_entity_handler, crate::http_server::endpoints::user_bookmarks::list_user_bookmarks_for_user_handler::list_user_bookmarks_for_user_handler, + crate::http_server::endpoints::user_ratings::batch_get_user_rating_handler::batch_get_user_rating_handler, crate::http_server::endpoints::user_ratings::get_user_rating_handler::get_user_rating_handler, crate::http_server::endpoints::user_ratings::set_user_rating_handler::set_user_rating_handler, crate::http_server::endpoints::voice_designer::inference::enqueue_tts_request::enqueue_tts_request, @@ -101,6 +103,7 @@ use crate::http_server::web_utils::response_success_helpers::*; ListAvailableWeightsQuery,ListAvailableWeightsSuccessResponse,ModelWeightForList, ListWeightsByUserError,ListWeightsByUserSuccessResponse,ListWeightsByUserPathInfo,Weight, WeightsData,MediaFileData, + BatchGetUserRatingQueryParams,BatchGetUserRatingResponse,BatchGetUserRatingError,RatingRow, ListFeaturedMediaFilesSuccessResponse,MediaFile, ListFeaturedMediaFilesError, ListMediaFilesSuccessResponse, ListMediaFilesError, MediaFileListItem, ListMediaFilesQueryParams, ListMediaFilesForUserSuccessResponse,ListMediaFilesForUserQueryParams,ListMediaFilesForUserError,ListMediaFilesForUserPathInfo, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/user_ratings/batch_get_user_rating_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/user_ratings/batch_get_user_rating_handler.rs new file mode 100644 index 0000000000..fdb3579f8c --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/user_ratings/batch_get_user_rating_handler.rs @@ -0,0 +1,191 @@ +use std::collections::{HashMap, HashSet}; +use std::sync::Arc; + +use actix_web::{HttpRequest, HttpResponse, ResponseError, web}; +use actix_web::http::StatusCode; +use actix_web_lab::extract::Query; +use log::error; +use utoipa::{IntoParams, ToSchema}; + +use enums::by_table::user_ratings::entity_type::UserRatingEntityType; +use enums::by_table::user_ratings::rating_value::UserRatingValue; +use http_server_common::response::serialize_as_json_error::serialize_as_json_error; +use mysql_queries::queries::users::user_ratings::batch_get_user_rating::{batch_get_user_ratings, BatchUserRating}; +use tokens::tokens::media_files::MediaFileToken; +use tokens::tokens::model_weights::ModelWeightToken; +use tokens::tokens::tts_models::TtsModelToken; + +use crate::server_state::ServerState; + +// =============== Request =============== + +#[derive(Deserialize, ToSchema, IntoParams)] +pub struct BatchGetUserRatingQueryParams { + /// A grab bag of tokens of various types. + /// Technically we should pair with token types, as that's the unique index. + /// But since each token family generally has its own prefix, this should be fine. + /// + /// NB: We're using actix_web_lab's Query, because the default actix_web Query doesn't support URL + /// decoding sequences yet. + /// See https://github.com/actix/actix-web/issues/1301 + /// + pub tokens: HashSet, +} + +// =============== Success Response =============== + +#[derive(Serialize, ToSchema)] +pub struct BatchGetUserRatingResponse { + pub success: bool, + + /// Ratings on each item passed to us. + pub ratings: Vec, +} + +#[derive(Serialize, ToSchema)] +pub struct RatingRow { + /// The passed token + pub entity_token: String, + /// The type of entity + pub entity_type: UserRatingEntityType, + /// The rating value. Unrated items will be "neutral". + pub rating_value: UserRatingValue, +} + +// =============== Error Response =============== + +#[derive(Debug, Serialize, ToSchema)] +pub enum BatchGetUserRatingError { + BadInput(String), + NotAuthorized, + ServerError, +} + +impl ResponseError for BatchGetUserRatingError { + fn status_code(&self) -> StatusCode { + match *self { + BatchGetUserRatingError::BadInput(_) => StatusCode::BAD_REQUEST, + BatchGetUserRatingError::NotAuthorized => StatusCode::UNAUTHORIZED, + BatchGetUserRatingError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + } + } + + fn error_response(&self) -> HttpResponse { + serialize_as_json_error(self) + } +} + +// NB: Not using DeriveMore since Clion doesn't understand it. +impl std::fmt::Display for BatchGetUserRatingError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self) + } +} + +// =============== Handler =============== + + +#[utoipa::path( + get, + path = "/v1/user_rating/batch", + params( + BatchGetUserRatingQueryParams, + ), + responses( + (status = 200, description = "List User Bookmarks", body = BatchGetUserRatingResponse), + (status = 400, description = "Bad input", body = BatchGetUserRatingError), + (status = 401, description = "Not authorized", body = BatchGetUserRatingError), + (status = 500, description = "Server error", body = BatchGetUserRatingError), + ), +)] +pub async fn batch_get_user_rating_handler( + http_request: HttpRequest, + query: Query, + server_state: web::Data>) -> Result +{ + let mut mysql_connection = server_state.mysql_pool.acquire() + .await + .map_err(|e| { + error!("Could not acquire DB pool: {:?}", e); + BatchGetUserRatingError::ServerError + })?; + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session_from_connection(&http_request, &mut mysql_connection) + .await + .map_err(|e| { + error!("Session checker error: {:?}", e); + BatchGetUserRatingError::ServerError + })?; + + let user_session = match maybe_user_session { + Some(session) => session, + None => { + let ratings = fill_in_missed_ratings(&query.tokens, Vec::new()); + + // NB: Just return "neutral" for everything. + return Ok(HttpResponse::Ok() + .content_type("application/json") + .json(BatchGetUserRatingResponse { + success: true, + ratings, + })); + } + }; + + batch_get_user_ratings( + &user_session.user_token_typed, + &query.tokens, + &mut mysql_connection + ).await + .map_err(|e| { + error!("Batch get user ratings DB error: {:?}", e); + BatchGetUserRatingError::ServerError + }) + .map(|ratings| { + HttpResponse::Ok() + .content_type("application/json") + .json(BatchGetUserRatingResponse { + success: true, + ratings: fill_in_missed_ratings(&query.tokens, ratings), + }) + }) +} + +fn fill_in_missed_ratings(request_tokens: &HashSet, db_response: Vec) -> Vec { + let mut outputs = HashMap::with_capacity(request_tokens.len()); + + for record in db_response.into_iter() { + outputs.insert(record.entity_token.clone(),RatingRow { + entity_token: record.entity_token, + entity_type: record.entity_type, + rating_value: record.rating_value, + }); + } + + for request_token in request_tokens.iter() { + if !outputs.contains_key(request_token) { + outputs.insert(request_token.clone(), RatingRow { + entity_token: request_token.clone(), + entity_type: { + if request_token.starts_with(MediaFileToken::token_prefix()) { + UserRatingEntityType::MediaFile + } else if request_token.starts_with(ModelWeightToken::token_prefix()) { + UserRatingEntityType::ModelWeight + } else if request_token.starts_with(TtsModelToken::token_prefix()) { + UserRatingEntityType::TtsModel + } else { + // NB: Fail open; W2lTemplates are dead, so this is a good sentinel value + UserRatingEntityType::W2lTemplate + } + }, + rating_value: UserRatingValue::Neutral, + }); + } + } + + outputs.into_iter() + .map(|(_key, value)| value) + .collect::>() +} \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/user_ratings/get_user_rating_handler.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/user_ratings/get_user_rating_handler.rs index ae4f90a99b..a581dc309b 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/user_ratings/get_user_rating_handler.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/user_ratings/get_user_rating_handler.rs @@ -74,18 +74,18 @@ impl std::fmt::Display for GetUserRatingError { #[utoipa::path( -get, -path = "/v1/user_rating/view/{entity_type}/{entity_token}", -params( -("entity_type", description = "The type of the entity being rated."), -("entity_token", description = "Entity token"), -), -responses( -(status = 200, description = "List User Bookmarks", body = GetUserRatingResponse), -(status = 400, description = "Bad input", body = GetUserRatingError), -(status = 401, description = "Not authorized", body = GetUserRatingError), -(status = 500, description = "Server error", body = GetUserRatingError), -), + get, + path = "/v1/user_rating/view/{entity_type}/{entity_token}", + params( + ("entity_type", description = "The type of the entity being rated."), + ("entity_token", description = "Entity token"), + ), + responses( + (status = 200, description = "List User Bookmarks", body = GetUserRatingResponse), + (status = 400, description = "Bad input", body = GetUserRatingError), + (status = 401, description = "Not authorized", body = GetUserRatingError), + (status = 500, description = "Server error", body = GetUserRatingError), + ), )] pub async fn get_user_rating_handler( http_request: HttpRequest, diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/user_ratings/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/user_ratings/mod.rs index 124e231877..74f82deae3 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/user_ratings/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/user_ratings/mod.rs @@ -1,2 +1,3 @@ +pub mod batch_get_user_rating_handler; pub mod get_user_rating_handler; pub mod set_user_rating_handler; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index 6e85b5e805..ef300bd97b 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -113,6 +113,7 @@ use crate::http_server::endpoints::user_bookmarks::delete_user_bookmark_handler: use crate::http_server::endpoints::user_bookmarks::list_user_bookmarks_for_entity_handler::list_user_bookmarks_for_entity_handler; use crate::http_server::endpoints::user_bookmarks::list_user_bookmarks_for_session_handler::list_user_bookmarks_for_session_handler; use crate::http_server::endpoints::user_bookmarks::list_user_bookmarks_for_user_handler::list_user_bookmarks_for_user_handler; +use crate::http_server::endpoints::user_ratings::batch_get_user_rating_handler::batch_get_user_rating_handler; use crate::http_server::endpoints::user_ratings::get_user_rating_handler::get_user_rating_handler; use crate::http_server::endpoints::user_ratings::set_user_rating_handler::set_user_rating_handler; use crate::http_server::endpoints::vocoders::get_vocoder::get_vocoder_handler; @@ -1111,6 +1112,10 @@ fn add_user_rating_routes (app: App) -> App >, { app.service(web::scope("/v1/user_rating") + .service(web::resource("/batch") + .route(web::get().to(batch_get_user_rating_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) .service(web::resource("/rate") .route(web::post().to(set_user_rating_handler)) .route(web::head().to(|| HttpResponse::Ok()))